def _process_result(self, connection, outFile, wu): """ Read the output file, add the values to the PixelResult row, and insert the filter, parameter and histogram rows. """ if is_gzip(outFile): f = gzip.open(outFile , "rb") else: f = open(outFile, "r") self._area_id = None self._pxresult_id = None lineNo = 0 percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = False result_count = 0 map_pixel_results = {} map_pixel_parameter = {} map_pixel_histograms = {} list_filters = [] list_pixel_parameters = [] list_pixel_histograms = [] start_time = None try: for line in f: lineNo += 1 if line.startswith(" ####### "): if self._pxresult_id is not None: self._save_results(connection, map_pixel_results, list_filters, list_pixel_parameters, map_pixel_histograms) #self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id) map_pixel_results = {} map_pixel_parameter = {} map_pixel_histograms = {} list_filters = [] list_pixel_parameters = [] start_time = time.time() values = line.split() pointName = values[1] pxresult_id = pointName[3:].rstrip() self._get_pixel_result(connection, pxresult_id) if self._pxresult_id is not None: map_pixel_results['workunit_id'] = wu.id lineNo = 0 percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = False result_count += 1 elif self._pxresult_id is not None: if lineNo == 2: filterNames = line.split() for filterName in filterNames: if filterName != '#': list_filters.append({'filter_name': filterName, 'pxresult_id': self._pxresult_id}) elif lineNo == 3: index = 0 values = line.split() for value in values: filter = list_filters[index] filter['observed_flux'] = float(value) index += 1 elif lineNo == 4: index = 0 values = line.split() for value in values: filter = list_filters[index] filter['observational_uncertainty'] = float(value) index += 1 elif lineNo == 9: values = line.split() map_pixel_results['i_sfh'] = float(values[0]) map_pixel_results['i_ir'] = float(values[1]) map_pixel_results['chi2'] = float(values[2]) map_pixel_results['redshift'] = float(values[3]) elif lineNo == 11: values = line.split() map_pixel_results['fmu_sfh'] = float(values[0]) map_pixel_results['fmu_ir'] = float(values[1]) map_pixel_results['mu'] = float(values[2]) map_pixel_results['tauv'] = float(values[3]) map_pixel_results['s_sfr'] = float(values[4]) map_pixel_results['m'] = float(values[5]) map_pixel_results['ldust'] = float(values[6]) map_pixel_results['t_w_bc'] = float(values[7]) map_pixel_results['t_c_ism'] = float(values[8]) map_pixel_results['xi_c_tot'] = float(values[9]) map_pixel_results['xi_pah_tot'] = float(values[10]) map_pixel_results['xi_mir_tot'] = float(values[11]) map_pixel_results['x_w_tot'] = float(values[12]) map_pixel_results['tvism'] = float(values[13]) map_pixel_results['mdust'] = float(values[14]) map_pixel_results['sfr'] = float(values[15]) elif lineNo == 13: index = 0 values = line.split() for value in values: filter = list_filters[index] filter['flux_bfm'] = float(value) index += 1 elif lineNo > 13: if line.startswith("# ..."): parts = line.split('...') parameterName = parts[1].strip() parameter_name_id = self._map_parameter_name[parameterName] map_pixel_parameter = {'parameter_name_id': parameter_name_id, 'pxresult_id': self._pxresult_id} list_pixel_parameters.append(map_pixel_parameter) list_pixel_histograms = [] map_pixel_histograms[parameter_name_id] = list_pixel_histograms percentiles_next = False histogram_next = True skynet_next1 = False skynet_next2 = False elif line.startswith("#....percentiles of the PDF......") and len(map_pixel_parameter) > 0: percentiles_next = True histogram_next = False skynet_next1 = False skynet_next2 = False elif line.startswith(" #...theSkyNet"): percentiles_next = False histogram_next = False skynet_next1 = True skynet_next2 = False elif line.startswith("# theSkyNet2"): percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = True elif percentiles_next: values = line.split() map_pixel_parameter['percentile2_5'] = float(values[0]) map_pixel_parameter['percentile16'] = float(values[1]) map_pixel_parameter['percentile50'] = float(values[2]) map_pixel_parameter['percentile84'] = float(values[3]) map_pixel_parameter['percentile97_5'] = float(values[4]) percentiles_next = False elif histogram_next: values = line.split() hist_value = float(values[1]) if hist_value > MIN_HIST_VALUE and not math.isnan(hist_value): list_pixel_histograms.append({'pxresult_id': self._pxresult_id, 'x_axis': float(values[0]), 'hist_value': hist_value}) elif skynet_next1: values = line.split() map_pixel_results['i_opt'] = float(values[0]) map_pixel_results['i_ir'] = float(values[1]) map_pixel_results['dmstar'] = float(values[2]) map_pixel_results['dfmu_aux'] = float(values[3]) map_pixel_results['dz'] = float(values[4]) skynet_next1 = False elif skynet_next2: # We have the highest bin probability values which require the parameter_id values = line.split() map_pixel_parameter['high_prob_bin'] = float(values[0]) map_pixel_parameter['first_prob_bin'] = float(values[1]) map_pixel_parameter['last_prob_bin'] = float(values[2]) map_pixel_parameter['bin_step'] = float(values[3]) skynet_next2 = False except IOError: self.logCritical('IOError after %d lines\n', lineNo) finally: f.close() if self._pxresult_id is not None: self._save_results(connection, map_pixel_results, list_filters, list_pixel_parameters, map_pixel_histograms) #self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id) return result_count
def _process_result(self, connection, out_file, wu): """ Read the output file, add the values to the PixelResult row, and insert the filter, parameter and histogram rows. """ if is_gzip(out_file): f = gzip.open(out_file, "rb") else: f = open(out_file, "r") self._area_id = None self._pxresult_id = None line_number = 0 percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = False result_count = 0 map_pixel_results = {} column_name = None start_time = None try: for line in f: line_number += 1 if line.startswith(" ####### "): if self._pxresult_id is not None: self._save_results(map_pixel_results) # self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id) map_pixel_results = {} start_time = time.time() values = line.split() point_name = values[1] pxresult_id = point_name[3:].rstrip() self._get_pixel_result(connection, pxresult_id) if self._pxresult_id is not None: map_pixel_results['workunit_id'] = wu.id line_number = 0 percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = False result_count += 1 elif self._pxresult_id is not None: if line_number == 9: # We can ignore these pass elif line_number == 11: # We prefer the median values pass elif line_number > 13: if line.startswith("# ..."): parts = line.split('...') parameter_name = parts[1].strip() column_name = self._map_parameter_name[parameter_name] percentiles_next = False histogram_next = True skynet_next1 = False skynet_next2 = False elif line.startswith("#....percentiles of the PDF......"): percentiles_next = True histogram_next = False skynet_next1 = False skynet_next2 = False elif line.startswith(" #...theSkyNet"): percentiles_next = False histogram_next = False skynet_next1 = True skynet_next2 = False elif line.startswith("# theSkyNet2"): percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = True elif percentiles_next: values = line.split() map_pixel_results[column_name] = float(values[2]) percentiles_next = False elif histogram_next: pass elif skynet_next1: skynet_next1 = False elif skynet_next2: # We have the highest bin probability values which require the parameter_id skynet_next2 = False except IOError: self.logCritical('IOError after %d lines\n', line_number) finally: f.close() if self._pxresult_id is not None: self._save_results(map_pixel_results) # self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id) return result_count
def _process_result(self, connection, outFile, wu): """ Read the output file, add the values to the PixelResult row, and insert the filter, parameter and histogram rows. """ if is_gzip(outFile): f = gzip.open(outFile, "rb") else: f = open(outFile, "r") self._area_id = None self._pxresult_id = None lineNo = 0 percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = False result_count = 0 map_pixel_results = {} map_pixel_parameter = {} map_pixel_histograms = {} list_filters = [] list_pixel_parameters = [] list_pixel_histograms = [] start_time = None try: for line in f: lineNo += 1 if line.startswith(" ####### "): if self._pxresult_id is not None: self._save_results(connection, map_pixel_results, list_filters, list_pixel_parameters, map_pixel_histograms) #self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id) map_pixel_results = {} map_pixel_parameter = {} map_pixel_histograms = {} list_filters = [] list_pixel_parameters = [] start_time = time.time() values = line.split() pointName = values[1] pxresult_id = pointName[3:].rstrip() self._get_pixel_result(connection, pxresult_id) if self._pxresult_id is not None: map_pixel_results['workunit_id'] = wu.id lineNo = 0 percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = False result_count += 1 elif self._pxresult_id is not None: if lineNo == 2: filterNames = line.split() for filterName in filterNames: if filterName != '#': list_filters.append({ 'filter_name': filterName, 'pxresult_id': self._pxresult_id }) elif lineNo == 3: index = 0 values = line.split() for value in values: filter = list_filters[index] filter['observed_flux'] = float(value) index += 1 elif lineNo == 4: index = 0 values = line.split() for value in values: filter = list_filters[index] filter['observational_uncertainty'] = float(value) index += 1 elif lineNo == 9: values = line.split() map_pixel_results['i_sfh'] = float(values[0]) map_pixel_results['i_ir'] = float(values[1]) map_pixel_results['chi2'] = float(values[2]) map_pixel_results['redshift'] = float(values[3]) elif lineNo == 11: values = line.split() map_pixel_results['fmu_sfh'] = float(values[0]) map_pixel_results['fmu_ir'] = float(values[1]) map_pixel_results['mu'] = float(values[2]) map_pixel_results['tauv'] = float(values[3]) map_pixel_results['s_sfr'] = float(values[4]) map_pixel_results['m'] = float(values[5]) map_pixel_results['ldust'] = float(values[6]) map_pixel_results['t_w_bc'] = float(values[7]) map_pixel_results['t_c_ism'] = float(values[8]) map_pixel_results['xi_c_tot'] = float(values[9]) map_pixel_results['xi_pah_tot'] = float(values[10]) map_pixel_results['xi_mir_tot'] = float(values[11]) map_pixel_results['x_w_tot'] = float(values[12]) map_pixel_results['tvism'] = float(values[13]) map_pixel_results['mdust'] = float(values[14]) map_pixel_results['sfr'] = float(values[15]) elif lineNo == 13: index = 0 values = line.split() for value in values: filter = list_filters[index] filter['flux_bfm'] = float(value) index += 1 elif lineNo > 13: if line.startswith("# ..."): parts = line.split('...') parameterName = parts[1].strip() parameter_name_id = self._map_parameter_name[ parameterName] map_pixel_parameter = { 'parameter_name_id': parameter_name_id, 'pxresult_id': self._pxresult_id } list_pixel_parameters.append(map_pixel_parameter) list_pixel_histograms = [] map_pixel_histograms[ parameter_name_id] = list_pixel_histograms percentiles_next = False histogram_next = True skynet_next1 = False skynet_next2 = False elif line.startswith( "#....percentiles of the PDF......" ) and len(map_pixel_parameter) > 0: percentiles_next = True histogram_next = False skynet_next1 = False skynet_next2 = False elif line.startswith(" #...theSkyNet"): percentiles_next = False histogram_next = False skynet_next1 = True skynet_next2 = False elif line.startswith("# theSkyNet2"): percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = True elif percentiles_next: values = line.split() map_pixel_parameter['percentile2_5'] = float( values[0]) map_pixel_parameter['percentile16'] = float( values[1]) map_pixel_parameter['percentile50'] = float( values[2]) map_pixel_parameter['percentile84'] = float( values[3]) map_pixel_parameter['percentile97_5'] = float( values[4]) percentiles_next = False elif histogram_next: values = line.split() hist_value = float(values[1]) if hist_value > MIN_HIST_VALUE and not math.isnan( hist_value): list_pixel_histograms.append({ 'pxresult_id': self._pxresult_id, 'x_axis': float(values[0]), 'hist_value': hist_value }) elif skynet_next1: values = line.split() map_pixel_results['i_opt'] = float(values[0]) map_pixel_results['i_ir'] = float(values[1]) map_pixel_results['dmstar'] = float(values[2]) map_pixel_results['dfmu_aux'] = float(values[3]) map_pixel_results['dz'] = float(values[4]) skynet_next1 = False elif skynet_next2: # We have the highest bin probability values which require the parameter_id values = line.split() map_pixel_parameter['high_prob_bin'] = float( values[0]) map_pixel_parameter['first_prob_bin'] = float( values[1]) map_pixel_parameter['last_prob_bin'] = float( values[2]) map_pixel_parameter['bin_step'] = float(values[3]) skynet_next2 = False except IOError: self.logCritical('IOError after %d lines\n', lineNo) finally: f.close() if self._pxresult_id is not None: self._save_results(connection, map_pixel_results, list_filters, list_pixel_parameters, map_pixel_histograms) #self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id) return result_count
def _process_result(self, connection, out_file, wu): """ Read the output file, add the values to the PixelResult row, and insert the filter, parameter and histogram rows. """ if is_gzip(out_file): f = gzip.open(out_file, "rb") else: f = open(out_file, "r") self._area_id = None self._pxresult_id = None lineNo = 0 percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = False result_count = 0 map_pixel_results = {} column_name = None start_time = None try: for line in f: lineNo += 1 if line.startswith(" ####### "): if self._pxresult_id is not None: self._save_results(connection, map_pixel_results) #self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id) map_pixel_results = {} start_time = time.time() values = line.split() pointName = values[1] pxresult_id = pointName[3:].rstrip() self._get_pixel_result(connection, pxresult_id) if self._pxresult_id is not None: map_pixel_results['workunit_id'] = wu.id lineNo = 0 percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = False result_count += 1 elif self._pxresult_id is not None: if lineNo == 9: # We can ignore these pass elif lineNo == 11: # We prefer the median values pass elif lineNo > 13: if line.startswith("# ..."): parts = line.split('...') parameter_name = parts[1].strip() column_name = self._map_parameter_name[parameter_name] percentiles_next = False histogram_next = True skynet_next1 = False skynet_next2 = False elif line.startswith("#....percentiles of the PDF......"): percentiles_next = True histogram_next = False skynet_next1 = False skynet_next2 = False elif line.startswith(" #...theSkyNet"): percentiles_next = False histogram_next = False skynet_next1 = True skynet_next2 = False elif line.startswith("# theSkyNet2"): percentiles_next = False histogram_next = False skynet_next1 = False skynet_next2 = True elif percentiles_next: values = line.split() map_pixel_results[column_name] = float(values[2]) percentiles_next = False elif histogram_next: pass elif skynet_next1: skynet_next1 = False elif skynet_next2: # We have the highest bin probability values which require the parameter_id skynet_next2 = False except IOError: self.logCritical('IOError after %d lines\n', lineNo) finally: f.close() if self._pxresult_id is not None: self._save_results(connection, map_pixel_results) #self.logDebug('%.3f seconds for %d\n', time.time() - start_time, self._pxresult_id) return result_count