def _process_result(self, connection, outFile, wu):
"""
Read the output file, add the values to the PixelResult row, and insert the filter,
parameter and histogram rows.
"""
if is_gzip(outFile):
f = gzip.open(outFile , "rb")
else:
f = open(outFile, "r")
self._area_id = None
self._pxresult_id = None
lineNo = 0
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = False
result_count = 0
map_pixel_results = {}
map_pixel_parameter = {}
map_pixel_histograms = {}
list_filters = []
list_pixel_parameters = []
list_pixel_histograms = []
start_time = None
try:
for line in f:
lineNo += 1
if line.startswith(" ####### "):
if self._pxresult_id is not None:
self._save_results(connection, map_pixel_results, list_filters, list_pixel_parameters, map_pixel_histograms)
#self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id)
map_pixel_results = {}
map_pixel_parameter = {}
map_pixel_histograms = {}
list_filters = []
list_pixel_parameters = []
start_time = time.time()
values = line.split()
pointName = values[1]
pxresult_id = pointName[3:].rstrip()
self._get_pixel_result(connection, pxresult_id)
if self._pxresult_id is not None:
map_pixel_results['workunit_id'] = wu.id
lineNo = 0
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = False
result_count += 1
elif self._pxresult_id is not None:
if lineNo == 2:
filterNames = line.split()
for filterName in filterNames:
if filterName != '#':
list_filters.append({'filter_name': filterName, 'pxresult_id': self._pxresult_id})
elif lineNo == 3:
index = 0
values = line.split()
for value in values:
filter = list_filters[index]
filter['observed_flux'] = float(value)
index += 1
elif lineNo == 4:
index = 0
values = line.split()
for value in values:
filter = list_filters[index]
filter['observational_uncertainty'] = float(value)
index += 1
elif lineNo == 9:
values = line.split()
map_pixel_results['i_sfh'] = float(values[0])
map_pixel_results['i_ir'] = float(values[1])
map_pixel_results['chi2'] = float(values[2])
map_pixel_results['redshift'] = float(values[3])
elif lineNo == 11:
values = line.split()
map_pixel_results['fmu_sfh'] = float(values[0])
map_pixel_results['fmu_ir'] = float(values[1])
map_pixel_results['mu'] = float(values[2])
map_pixel_results['tauv'] = float(values[3])
map_pixel_results['s_sfr'] = float(values[4])
map_pixel_results['m'] = float(values[5])
map_pixel_results['ldust'] = float(values[6])
map_pixel_results['t_w_bc'] = float(values[7])
map_pixel_results['t_c_ism'] = float(values[8])
map_pixel_results['xi_c_tot'] = float(values[9])
map_pixel_results['xi_pah_tot'] = float(values[10])
map_pixel_results['xi_mir_tot'] = float(values[11])
map_pixel_results['x_w_tot'] = float(values[12])
map_pixel_results['tvism'] = float(values[13])
map_pixel_results['mdust'] = float(values[14])
map_pixel_results['sfr'] = float(values[15])
elif lineNo == 13:
index = 0
values = line.split()
for value in values:
filter = list_filters[index]
filter['flux_bfm'] = float(value)
index += 1
elif lineNo > 13:
if line.startswith("# ..."):
parts = line.split('...')
parameterName = parts[1].strip()
parameter_name_id = self._map_parameter_name[parameterName]
map_pixel_parameter = {'parameter_name_id': parameter_name_id, 'pxresult_id': self._pxresult_id}
list_pixel_parameters.append(map_pixel_parameter)
list_pixel_histograms = []
map_pixel_histograms[parameter_name_id] = list_pixel_histograms
percentiles_next = False
histogram_next = True
skynet_next1 = False
skynet_next2 = False
elif line.startswith("#....percentiles of the PDF......") and len(map_pixel_parameter) > 0:
percentiles_next = True
histogram_next = False
skynet_next1 = False
skynet_next2 = False
elif line.startswith(" #...theSkyNet"):
percentiles_next = False
histogram_next = False
skynet_next1 = True
skynet_next2 = False
elif line.startswith("# theSkyNet2"):
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = True
elif percentiles_next:
values = line.split()
map_pixel_parameter['percentile2_5'] = float(values[0])
map_pixel_parameter['percentile16'] = float(values[1])
map_pixel_parameter['percentile50'] = float(values[2])
map_pixel_parameter['percentile84'] = float(values[3])
map_pixel_parameter['percentile97_5'] = float(values[4])
percentiles_next = False
elif histogram_next:
values = line.split()
hist_value = float(values[1])
if hist_value > MIN_HIST_VALUE and not math.isnan(hist_value):
list_pixel_histograms.append({'pxresult_id': self._pxresult_id, 'x_axis': float(values[0]), 'hist_value': hist_value})
elif skynet_next1:
values = line.split()
map_pixel_results['i_opt'] = float(values[0])
map_pixel_results['i_ir'] = float(values[1])
map_pixel_results['dmstar'] = float(values[2])
map_pixel_results['dfmu_aux'] = float(values[3])
map_pixel_results['dz'] = float(values[4])
skynet_next1 = False
elif skynet_next2:
# We have the highest bin probability values which require the parameter_id
values = line.split()
map_pixel_parameter['high_prob_bin'] = float(values[0])
map_pixel_parameter['first_prob_bin'] = float(values[1])
map_pixel_parameter['last_prob_bin'] = float(values[2])
map_pixel_parameter['bin_step'] = float(values[3])
skynet_next2 = False
except IOError:
self.logCritical('IOError after %d lines\n', lineNo)
finally:
f.close()
if self._pxresult_id is not None:
self._save_results(connection, map_pixel_results, list_filters, list_pixel_parameters, map_pixel_histograms)
#self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id)
return result_count
def _process_result(self, connection, out_file, wu):
"""
Read the output file, add the values to the PixelResult row, and insert the filter,
parameter and histogram rows.
"""
if is_gzip(out_file):
f = gzip.open(out_file, "rb")
else:
f = open(out_file, "r")
self._area_id = None
self._pxresult_id = None
line_number = 0
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = False
result_count = 0
map_pixel_results = {}
column_name = None
start_time = None
try:
for line in f:
line_number += 1
if line.startswith(" ####### "):
if self._pxresult_id is not None:
self._save_results(map_pixel_results)
# self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id)
map_pixel_results = {}
start_time = time.time()
values = line.split()
point_name = values[1]
pxresult_id = point_name[3:].rstrip()
self._get_pixel_result(connection, pxresult_id)
if self._pxresult_id is not None:
map_pixel_results['workunit_id'] = wu.id
line_number = 0
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = False
result_count += 1
elif self._pxresult_id is not None:
if line_number == 9:
# We can ignore these
pass
elif line_number == 11:
# We prefer the median values
pass
elif line_number > 13:
if line.startswith("# ..."):
parts = line.split('...')
parameter_name = parts[1].strip()
column_name = self._map_parameter_name[parameter_name]
percentiles_next = False
histogram_next = True
skynet_next1 = False
skynet_next2 = False
elif line.startswith("#....percentiles of the PDF......"):
percentiles_next = True
histogram_next = False
skynet_next1 = False
skynet_next2 = False
elif line.startswith(" #...theSkyNet"):
percentiles_next = False
histogram_next = False
skynet_next1 = True
skynet_next2 = False
elif line.startswith("# theSkyNet2"):
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = True
elif percentiles_next:
values = line.split()
map_pixel_results[column_name] = float(values[2])
percentiles_next = False
elif histogram_next:
pass
elif skynet_next1:
skynet_next1 = False
elif skynet_next2:
# We have the highest bin probability values which require the parameter_id
skynet_next2 = False
except IOError:
self.logCritical('IOError after %d lines\n', line_number)
finally:
f.close()
if self._pxresult_id is not None:
self._save_results(map_pixel_results)
# self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id)
return result_count
def _process_result(self, connection, outFile, wu):
"""
Read the output file, add the values to the PixelResult row, and insert the filter,
parameter and histogram rows.
"""
if is_gzip(outFile):
f = gzip.open(outFile, "rb")
else:
f = open(outFile, "r")
self._area_id = None
self._pxresult_id = None
lineNo = 0
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = False
result_count = 0
map_pixel_results = {}
map_pixel_parameter = {}
map_pixel_histograms = {}
list_filters = []
list_pixel_parameters = []
list_pixel_histograms = []
start_time = None
try:
for line in f:
lineNo += 1
if line.startswith(" ####### "):
if self._pxresult_id is not None:
self._save_results(connection, map_pixel_results,
list_filters, list_pixel_parameters,
map_pixel_histograms)
#self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id)
map_pixel_results = {}
map_pixel_parameter = {}
map_pixel_histograms = {}
list_filters = []
list_pixel_parameters = []
start_time = time.time()
values = line.split()
pointName = values[1]
pxresult_id = pointName[3:].rstrip()
self._get_pixel_result(connection, pxresult_id)
if self._pxresult_id is not None:
map_pixel_results['workunit_id'] = wu.id
lineNo = 0
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = False
result_count += 1
elif self._pxresult_id is not None:
if lineNo == 2:
filterNames = line.split()
for filterName in filterNames:
if filterName != '#':
list_filters.append({
'filter_name':
filterName,
'pxresult_id':
self._pxresult_id
})
elif lineNo == 3:
index = 0
values = line.split()
for value in values:
filter = list_filters[index]
filter['observed_flux'] = float(value)
index += 1
elif lineNo == 4:
index = 0
values = line.split()
for value in values:
filter = list_filters[index]
filter['observational_uncertainty'] = float(value)
index += 1
elif lineNo == 9:
values = line.split()
map_pixel_results['i_sfh'] = float(values[0])
map_pixel_results['i_ir'] = float(values[1])
map_pixel_results['chi2'] = float(values[2])
map_pixel_results['redshift'] = float(values[3])
elif lineNo == 11:
values = line.split()
map_pixel_results['fmu_sfh'] = float(values[0])
map_pixel_results['fmu_ir'] = float(values[1])
map_pixel_results['mu'] = float(values[2])
map_pixel_results['tauv'] = float(values[3])
map_pixel_results['s_sfr'] = float(values[4])
map_pixel_results['m'] = float(values[5])
map_pixel_results['ldust'] = float(values[6])
map_pixel_results['t_w_bc'] = float(values[7])
map_pixel_results['t_c_ism'] = float(values[8])
map_pixel_results['xi_c_tot'] = float(values[9])
map_pixel_results['xi_pah_tot'] = float(values[10])
map_pixel_results['xi_mir_tot'] = float(values[11])
map_pixel_results['x_w_tot'] = float(values[12])
map_pixel_results['tvism'] = float(values[13])
map_pixel_results['mdust'] = float(values[14])
map_pixel_results['sfr'] = float(values[15])
elif lineNo == 13:
index = 0
values = line.split()
for value in values:
filter = list_filters[index]
filter['flux_bfm'] = float(value)
index += 1
elif lineNo > 13:
if line.startswith("# ..."):
parts = line.split('...')
parameterName = parts[1].strip()
parameter_name_id = self._map_parameter_name[
parameterName]
map_pixel_parameter = {
'parameter_name_id': parameter_name_id,
'pxresult_id': self._pxresult_id
}
list_pixel_parameters.append(map_pixel_parameter)
list_pixel_histograms = []
map_pixel_histograms[
parameter_name_id] = list_pixel_histograms
percentiles_next = False
histogram_next = True
skynet_next1 = False
skynet_next2 = False
elif line.startswith(
"#....percentiles of the PDF......"
) and len(map_pixel_parameter) > 0:
percentiles_next = True
histogram_next = False
skynet_next1 = False
skynet_next2 = False
elif line.startswith(" #...theSkyNet"):
percentiles_next = False
histogram_next = False
skynet_next1 = True
skynet_next2 = False
elif line.startswith("# theSkyNet2"):
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = True
elif percentiles_next:
values = line.split()
map_pixel_parameter['percentile2_5'] = float(
values[0])
map_pixel_parameter['percentile16'] = float(
values[1])
map_pixel_parameter['percentile50'] = float(
values[2])
map_pixel_parameter['percentile84'] = float(
values[3])
map_pixel_parameter['percentile97_5'] = float(
values[4])
percentiles_next = False
elif histogram_next:
values = line.split()
hist_value = float(values[1])
if hist_value > MIN_HIST_VALUE and not math.isnan(
hist_value):
list_pixel_histograms.append({
'pxresult_id':
self._pxresult_id,
'x_axis':
float(values[0]),
'hist_value':
hist_value
})
elif skynet_next1:
values = line.split()
map_pixel_results['i_opt'] = float(values[0])
map_pixel_results['i_ir'] = float(values[1])
map_pixel_results['dmstar'] = float(values[2])
map_pixel_results['dfmu_aux'] = float(values[3])
map_pixel_results['dz'] = float(values[4])
skynet_next1 = False
elif skynet_next2:
# We have the highest bin probability values which require the parameter_id
values = line.split()
map_pixel_parameter['high_prob_bin'] = float(
values[0])
map_pixel_parameter['first_prob_bin'] = float(
values[1])
map_pixel_parameter['last_prob_bin'] = float(
values[2])
map_pixel_parameter['bin_step'] = float(values[3])
skynet_next2 = False
except IOError:
self.logCritical('IOError after %d lines\n', lineNo)
finally:
f.close()
if self._pxresult_id is not None:
self._save_results(connection, map_pixel_results, list_filters,
list_pixel_parameters, map_pixel_histograms)
#self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id)
return result_count
def _process_result(self, connection, out_file, wu):
"""
Read the output file, add the values to the PixelResult row, and insert the filter,
parameter and histogram rows.
"""
if is_gzip(out_file):
f = gzip.open(out_file, "rb")
else:
f = open(out_file, "r")
self._area_id = None
self._pxresult_id = None
lineNo = 0
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = False
result_count = 0
map_pixel_results = {}
column_name = None
start_time = None
try:
for line in f:
lineNo += 1
if line.startswith(" ####### "):
if self._pxresult_id is not None:
self._save_results(connection, map_pixel_results)
#self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id)
map_pixel_results = {}
start_time = time.time()
values = line.split()
pointName = values[1]
pxresult_id = pointName[3:].rstrip()
self._get_pixel_result(connection, pxresult_id)
if self._pxresult_id is not None:
map_pixel_results['workunit_id'] = wu.id
lineNo = 0
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = False
result_count += 1
elif self._pxresult_id is not None:
if lineNo == 9:
# We can ignore these
pass
elif lineNo == 11:
# We prefer the median values
pass
elif lineNo > 13:
if line.startswith("# ..."):
parts = line.split('...')
parameter_name = parts[1].strip()
column_name = self._map_parameter_name[parameter_name]
percentiles_next = False
histogram_next = True
skynet_next1 = False
skynet_next2 = False
elif line.startswith("#....percentiles of the PDF......"):
percentiles_next = True
histogram_next = False
skynet_next1 = False
skynet_next2 = False
elif line.startswith(" #...theSkyNet"):
percentiles_next = False
histogram_next = False
skynet_next1 = True
skynet_next2 = False
elif line.startswith("# theSkyNet2"):
percentiles_next = False
histogram_next = False
skynet_next1 = False
skynet_next2 = True
elif percentiles_next:
values = line.split()
map_pixel_results[column_name] = float(values[2])
percentiles_next = False
elif histogram_next:
pass
elif skynet_next1:
skynet_next1 = False
elif skynet_next2:
# We have the highest bin probability values which require the parameter_id
skynet_next2 = False
except IOError:
self.logCritical('IOError after %d lines\n', lineNo)
finally:
f.close()
if self._pxresult_id is not None:
self._save_results(connection, map_pixel_results)
#self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id)
return result_count
