def _create_astrotable(rows, column_names, column_dtypes, column_units): results = Table(data = rows , names = column_names, dtypes = column_dtypes) for i in _np.arange(len(column_units)): results.field(i).units = column_units[i] return results
def _parse_result(data, output='astropy.table'): """ Only one output type at the moment """ if output == 'astropy.table': # get each line (i.e. each molecule) rows = data.split('\n') # get the names of the columns column_names = rows[0] column_names = column_names.split(':') for i in np.arange(len(column_names)): column_names[i] = column_names[i].replace('<br>', ' ') column_names[i] = column_names[i].replace('<sub>', '_') column_names[i] = column_names[i].replace('<sup>', '^') column_names[i] = column_names[i].replace('</sup>', '') column_names[i] = column_names[i].replace('</sub>', '') column_names[i] = column_names[i].replace('μ', 'mu') column_names[i] = column_names[i].replace('sid[0] is null', '') rows = rows[1:-1] rows = [i.split(':') for i in rows] rows = np.array(rows) rows[rows == ''] = 'nan' column_dtypes = [ 'str', 'str', 'float', 'float', 'float', 'float', 'str', 'str', 'float', 'float', 'float', 'float', 'str', 'float', 'float', 'float', 'float', 'float', 'float', 'float', 'str' ] column_units = [ None, None, 'GHz?', 'GHz?', 'GHz?', 'GHz?', None, None, '?', 'Debye^2', '?', 'log10(Aij)', '?', 'cm^-1', 'K', 'cm^-1', 'K', None, None, None, None ] results = Table(data=rows, names=column_names, dtypes=column_dtypes) for i in np.arange(len(column_units)): results.field(i).units = column_units[i] return results
def _parse_result(data, output='astropy.table'): """ Only one output type at the moment """ if output == 'astropy.table': # get each line (i.e. each molecule) rows = data.split('\n') # get the names of the columns column_names = rows[0] column_names = column_names.split(':') for i in np.arange(len(column_names)): column_names[i] = column_names[i].replace('<br>', ' ') column_names[i] = column_names[i].replace('<sub>', '_') column_names[i] = column_names[i].replace('<sup>', '^') column_names[i] = column_names[i].replace('</sup>', '') column_names[i] = column_names[i].replace('</sub>', '') column_names[i] = column_names[i].replace('μ', 'mu') column_names[i] = column_names[i].replace('sid[0] is null', '') rows = rows[1:-1] rows = [i.split(':') for i in rows] rows = np.array(rows) rows[rows == ''] = 'nan' column_dtypes = ['str', 'str', 'float', 'float', 'float' , 'float' , 'str', 'str', 'float', 'float', 'float', 'float', 'str', 'float', 'float', 'float', 'float', 'float', 'float','float','str'] column_units = [None, None, 'GHz?', 'GHz?', 'GHz?', 'GHz?', None, None, '?', 'Debye^2', '?', 'log10(Aij)', '?', 'cm^-1', 'K', 'cm^-1', 'K', None, None, None, None] results = Table(data = rows , names = column_names, dtypes = column_dtypes) for i in np.arange(len(column_units)): results.field(i).units = column_units[i] return results
class SED(list): """Class to plot GeV -- TeV SEDs Internally the same units as in the Fermi catalog are used: - Energies in MeV - Flux densities in cm^-2 s^-2 MeV^-1 - Fluxes in cm^-2 s^-1 - Energy fluxes in erg cm^-2 s^-1""" """ def add_Fermi(self, name): try: self._fermi self.append(self._fermi.sed_component(name)) except component = catalog.sed_component(name) self.append(component) """ def add(self, names, catalogs): for name in names: for catalog in catalogs: try: component = catalog.sed_component(name) self.append(component) log.info('%s found in %s', name, catalog.table.table_name) except ValueError as e: log.warning(e) log.warning('%s not found in %s', name, catalog.table.table_name) pass def plot(self, filename='sed.png', xlim=(8e-2, 2e5), ylim=(1e-14, 1e-8)): import matplotlib.pyplot as plt plt.figure() plt.ylabel(r'E$^2$ dF/DE (erg cm$^{-2}$ s$^{-1}$)') plt.xlabel('Energy (GeV)') plt.loglog() log.info('Plotting {0} components in SED'.format(len(self))) for component in self: component.plot() plt.xlim(xlim) plt.ylim(ylim) plt.legend() log.info('Writing {0}'.format(filename)) plt.savefig(filename) def add_component(self, catalog_format, catalog_name, object_name, plot_pivot=False, **ecpl_params): """ Read necessary parameters from FITS file and plot butterfly Parameters: catalog_format = 'hess', 'fermi' catalog_name = FITS file name object_name = object name string in 'name' column Note: Since every catalog has columns with different names and units, a general SED plotting is not possible. Instead for each catalog type a handler function that deals converts to a standard format is called. @todo: Possibly pass plotting parameters along here by appending them to the ecpl_params dictionary -> I don't think this works at the moment!!!""" from atpy import Table # Get the catalog from file and initialize some things self.catalog_format = catalog_format self.catalog_name = catalog_name self.object_name = object_name self.catalog = Table(catalog_name).data # Build a dictionary of parameters needed for the plot self.ecpl_params = ecpl_params self.get_ecpl_params() # Plot curve self.plot_ecpl(plot_pivot=plot_pivot, **ecpl_params) # Plot points if present if self.plot_points is not None: # Get the values needed for plotting e = self.plot_points[0] f = self.plot_points[1] f_err = self.plot_points[2] e_err = self.plot_points[3] is_ul = self.plot_points[4] for ii in range(e.size): self.plot_point(e[ii], f[ii], f_err=f_err[ii], e_err=[[e_err[0][ii]], [e_err[1][ii]]], ul=is_ul[ii]) # Remove so that it doesn't get plotted again. self.plot_points = None def get_ecpl_params(self): """Build self.ecpl_params dictionary by parsing one of the supported catalogs""" if self.catalog_format == 'hess': self.get_ecpl_params_hess_cat() elif self.catalog_format == 'fermi': self.get_ecpl_params_fermi_cat() # Change numpy types to regular types # and replace nan values with 0 for key, value in self.ecpl_params.items(): if isinstance(value, np.float32): value = float(value) if isinstance(value, np.int16): value = int(value) def get_ecpl_params_fermi_cat(self): """ Build self.ecpl_params dictionary from Fermi catalog fields """ i = self.find_object_index('source_name') # Set all plot parameters: self.ecpl_params['e_pivot'] = self.catalog.field('Pivot_Energy')[i] self.ecpl_params['e_min'] = 1e2 self.ecpl_params['e_max'] = 1e5 self.ecpl_params['e_cut'] = 0.0 self.ecpl_params['e_cut_err'] = 0.0 self.ecpl_params['e_scale'] = 1 self.ecpl_params['norm'] = self.catalog.field('Flux_Density')[i] self.ecpl_params['norm_err'] = self.catalog.field( 'Unc_Flux_Density')[i] self.ecpl_params['norm_scale'] = 1 self.ecpl_params['index'] = self.catalog.field('Spectral_Index')[i] self.ecpl_params['index_err'] = self.catalog.field( 'Unc_Spectral_Index')[i] self.ecpl_params['color'] = 'green' self.ecpl_params['butterfly'] = True # Set flux point data self.plot_points = self.get_flux_points_fermi(i) # Add text label fmt = '%s\n%s, %s\n' + \ r'S = %3.1f, C = %3.1f, $\Gamma = %1.2f \pm %1.2f$' values = (self.object_name, self.catalog.field('class1')[i], self.catalog.field('assoc1')[i], self.catalog.field('signif_avg')[i], self.catalog.field('curvature_index')[i], self.catalog.field('spectral_index')[i], self.catalog.field('unc_spectral_index')[i]) self.ax.text(0.05, 0.95, fmt % values, horizontalalignment='left', verticalalignment='top', transform=self.ax.transAxes)
class SED(list): """Class to plot GeV -- TeV SEDs Internally the same units as in the Fermi catalog are used: - Energies in MeV - Flux densities in cm^-2 s^-2 MeV^-1 - Fluxes in cm^-2 s^-1 - Energy fluxes in erg cm^-2 s^-1""" """ def add_Fermi(self, name): try: self._fermi self.append(self._fermi.sed_component(name)) except component = catalog.sed_component(name) self.append(component) """ def add(self, names, catalogs): for name in names: for catalog in catalogs: try: component = catalog.sed_component(name) self.append(component) log.info('%s found in %s', name, catalog.table.table_name) except ValueError as e: log.warning(e) log.warning('%s not found in %s', name, catalog.table.table_name) pass def plot(self, filename='sed.png', xlim=(8e-2, 2e5), ylim=(1e-14, 1e-8)): import matplotlib.pyplot as plt plt.figure() plt.ylabel(r'E$^2$ dF/DE (erg cm$^{-2}$ s$^{-1}$)') plt.xlabel('Energy (GeV)') plt.loglog() log.info('Plotting {0} components in SED'.format(len(self))) for component in self: component.plot() plt.xlim(xlim) plt.ylim(ylim) plt.legend() log.info('Writing {0}'.format(filename)) plt.savefig(filename) def add_component(self, catalog_format, catalog_name, object_name, plot_pivot=False, **ecpl_params): """ Read necessary parameters from FITS file and plot butterfly Parameters: catalog_format = 'hess', 'fermi' catalog_name = FITS file name object_name = object name string in 'name' column Note: Since every catalog has columns with different names and units, a general SED plotting is not possible. Instead for each catalog type a handler function that deals converts to a standard format is called. @todo: Possibly pass plotting parameters along here by appending them to the ecpl_params dictionary -> I don't think this works at the moment!!!""" from atpy import Table # Get the catalog from file and initialize some things self.catalog_format = catalog_format self.catalog_name = catalog_name self.object_name = object_name self.catalog = Table(catalog_name).data # Build a dictionary of parameters needed for the plot self.ecpl_params = ecpl_params self.get_ecpl_params() # Plot curve self.plot_ecpl(plot_pivot=plot_pivot, **ecpl_params) # Plot points if present if self.plot_points is not None: # Get the values needed for plotting e = self.plot_points[0] f = self.plot_points[1] f_err = self.plot_points[2] e_err = self.plot_points[3] is_ul = self.plot_points[4] for ii in range(e.size): self.plot_point(e[ii], f[ii], f_err=f_err[ii], e_err=[[e_err[0][ii]], [e_err[1][ii]]], ul=is_ul[ii]) # Remove so that it doesn't get plotted again. self.plot_points = None def get_ecpl_params(self): """Build self.ecpl_params dictionary by parsing one of the supported catalogs""" if self.catalog_format == 'hess': self.get_ecpl_params_hess_cat() elif self.catalog_format == 'fermi': self.get_ecpl_params_fermi_cat() # Change numpy types to regular types # and replace nan values with 0 for key, value in self.ecpl_params.items(): if isinstance(value, np.float32): value = float(value) if isinstance(value, np.int16): value = int(value) def get_ecpl_params_fermi_cat(self): """ Build self.ecpl_params dictionary from Fermi catalog fields """ i = self.find_object_index('source_name') # Set all plot parameters: self.ecpl_params['e_pivot'] = self.catalog.field('Pivot_Energy')[i] self.ecpl_params['e_min'] = 1e2 self.ecpl_params['e_max'] = 1e5 self.ecpl_params['e_cut'] = 0.0 self.ecpl_params['e_cut_err'] = 0.0 self.ecpl_params['e_scale'] = 1 self.ecpl_params['norm'] = self.catalog.field('Flux_Density')[i] self.ecpl_params['norm_err'] = self.catalog.field('Unc_Flux_Density')[i] self.ecpl_params['norm_scale'] = 1 self.ecpl_params['index'] = self.catalog.field('Spectral_Index')[i] self.ecpl_params['index_err'] = self.catalog.field('Unc_Spectral_Index')[i] self.ecpl_params['color'] = 'green' self.ecpl_params['butterfly'] = True # Set flux point data self.plot_points = self.get_flux_points_fermi(i) # Add text label fmt = '%s\n%s, %s\n' + \ r'S = %3.1f, C = %3.1f, $\Gamma = %1.2f \pm %1.2f$' values = (self.object_name, self.catalog.field('class1')[i], self.catalog.field('assoc1')[i], self.catalog.field('signif_avg')[i], self.catalog.field('curvature_index')[i], self.catalog.field('spectral_index')[i], self.catalog.field('unc_spectral_index')[i] ) self.ax.text(0.05, 0.95, fmt % values, horizontalalignment='left', verticalalignment='top', transform=self.ax.transAxes)
def _parse_results(data, output='astropy.table'): """ Only one output type at the moment, the astropy.table table """ #TODO : what if results are empty if output == 'astropy.table': if not use_astropy: #~ print('Astropy not installed, try other output format') raise ( ImportError('Astropy not installed, try other output format')) # get each line (i.e. each molecule) rows = data.split('\n') # get the names of the columns column_names = rows[0] column_names = column_names.split(':') # clean them up a bit for i in _np.arange(len(column_names)): column_names[i] = column_names[i].replace('<br>', ' ') column_names[i] = column_names[i].replace('<sub>', '_') column_names[i] = column_names[i].replace('<sup>', '^') column_names[i] = column_names[i].replace('</sup>', '') column_names[i] = column_names[i].replace('</sub>', '') column_names[i] = column_names[i].replace('μ', 'mu') column_names[i] = column_names[i].replace('sid[0] is null', '') column_names[i] = column_names[i].replace('sid[0] is null', '') """ Column Names should now be: ['Species', 'NRAO Recommended', 'Chemical Name', 'Freq-GHz', 'Freq Err', 'Meas Freq-GHz', 'Meas Freq Err', 'Resolved QNs', 'Unresolved Quantum Numbers', 'CDMS/JPL Intensity', 'S_ijmu^2 (D^2)', 'S_ij', 'Log_10 (A_ij)', 'Lovas/AST Intensity', 'E_L (cm^-1)', 'E_L (K)', 'E_U (cm^-1)', 'E_U (K)', 'HFS int', 'Upper State Degeneracy', 'Molecule Tag', 'Quantum Number Code', 'Linelist'] """ rows = rows[1:-1] rows = [i.split(':') for i in rows] rows = _np.array(rows) rows[rows == ''] = -999999 #~ print column_names #~ return rows column_dtypes = [ 'str', # 'Species', 'str', # 'NRAO Recommended', 'str', # 'Chemical Name', 'float', # 'Freq-GHz', 'float', # 'Freq Err', 'float', # 'Meas Freq-GHz', 'float', # 'Meas Freq Err', 'str', # 'Resolved QNs', 'str', # 'Unresolved Quantum Numbers', 'float', # 'CDMS/JPL Intensity', 'float', # 'S_ijmu^2 (D^2)', 'float', # 'S_ij', 'float', # 'Log_10 (A_ij)', 'str', # 'Lovas/AST Intensity', 'float', # 'E_L (cm^-1)', 'float', # 'E_L (K)', 'float', # 'E_U (cm^-1)', 'float', # 'E_U (K)', 'float', # 'HFS int', 'float', # 'Upper State Degeneracy', 'int', # 'Molecule Tag', 'int', # 'Quantum Number Code', 'str' ] # 'Linelist'] funit = str(column_names[3][-3:]) column_units = [ None, # 'Species', None, # 'NRAO Recommended', None, # 'Chemical Name', funit, # 'Freq-GHz', funit, # 'Freq Err', funit, # 'Meas Freq-GHz', funit, # 'Meas Freq Err', None, # 'Resolved QNs', None, # 'Unresolved Quantum Numbers', '?', # 'CDMS/JPL Intensity', 'Debye^2', # 'S_ijmu^2 (D^2)', '?', # 'S_ij', 'log10(s^-1)', # 'Log_10 (A_ij)', '?', # 'Lovas/AST Intensity', 'cm^-1', # 'E_L (cm^-1)', 'K', # 'E_L (K)', 'cm^-1', # 'E_U (cm^-1)', 'K', # 'E_U (K)', '?', # 'HFS int', None, # 'Upper State Degeneracy', None, # 'Molecule Tag', None, # 'Quantum Number Code', None ] # 'Linelist'] column_names_original = column_names[:] #~ column_names = [i.lower() for i in column_names] #~ for i in _np.arange(len(column_names)): #~ column_names[i] = column_names[i].replace('nrao recommended', 'nrao_rec') #~ column_names[i] = column_names[i].replace('chemical name', 'name') #~ if 'meas freq err' in column_names[i]: #~ column_names[i] = 'mferr' #~ elif 'meas freq' in column_names[i]: #~ column_names[i] = 'mfreq' #~ elif 'freq err' in column_names[i]: #~ column_names[i] = 'ferr' #~ elif 'freq' in column_names[i]: #~ column_names[i] = 'freq' #~ column_names[i] = column_names[i].replace('resolved qns', 'resqn') #~ column_names[i] = column_names[i].replace('unresolved quantum numbers', 'resqn') column_names = [ 'species', 'nrao_rec', 'name', 'ofreq', 'oferr', 'mfreq', 'mferr', 'res_qn', 'uresqn', 'cdmsjplint', 'sijmu2', 'Sij', 'logaij', 'lovasastint', 'el_cm', 'el_k', 'eu_cm', 'eu_k', 'hfsint', 'gu', 'tag', 'qncode', 'list' ] results = Table(data=rows, names=column_names, dtypes=column_dtypes) for i in _np.arange(len(column_units)): results.field(i).units = column_units[i] return results else: print('Nothing else than astropy.table output is implemented atm') return results
def _parse_results(data, output='astropy.table'): """ Only one output type at the moment, the astropy.table table """ #TODO : what if results are empty if output == 'astropy.table': if not use_astropy: #~ print('Astropy not installed, try other output format') raise(ImportError('Astropy not installed, try other output format')) # get each line (i.e. each molecule) rows = data.split('\n') # get the names of the columns column_names = rows[0] column_names = column_names.split(':') # clean them up a bit for i in _np.arange(len(column_names)): column_names[i] = column_names[i].replace('<br>', ' ') column_names[i] = column_names[i].replace('<sub>', '_') column_names[i] = column_names[i].replace('<sup>', '^') column_names[i] = column_names[i].replace('</sup>', '') column_names[i] = column_names[i].replace('</sub>', '') column_names[i] = column_names[i].replace('μ', 'mu') column_names[i] = column_names[i].replace('sid[0] is null', '') column_names[i] = column_names[i].replace('sid[0] is null', '') """ Column Names should now be: ['Species', 'NRAO Recommended', 'Chemical Name', 'Freq-GHz', 'Freq Err', 'Meas Freq-GHz', 'Meas Freq Err', 'Resolved QNs', 'Unresolved Quantum Numbers', 'CDMS/JPL Intensity', 'S_ijmu^2 (D^2)', 'S_ij', 'Log_10 (A_ij)', 'Lovas/AST Intensity', 'E_L (cm^-1)', 'E_L (K)', 'E_U (cm^-1)', 'E_U (K)', 'HFS int', 'Upper State Degeneracy', 'Molecule Tag', 'Quantum Number Code', 'Linelist'] """ rows = rows[1:-1] rows = [i.split(':') for i in rows] rows = _np.array(rows) rows[rows == ''] = -999999 #~ print column_names #~ return rows column_dtypes = ['str', # 'Species', 'str', # 'NRAO Recommended', 'str', # 'Chemical Name', 'float', # 'Freq-GHz', 'float', # 'Freq Err', 'float', # 'Meas Freq-GHz', 'float', # 'Meas Freq Err', 'str', # 'Resolved QNs', 'str', # 'Unresolved Quantum Numbers', 'float', # 'CDMS/JPL Intensity', 'float', # 'S_ijmu^2 (D^2)', 'float', # 'S_ij', 'float', # 'Log_10 (A_ij)', 'str', # 'Lovas/AST Intensity', 'float', # 'E_L (cm^-1)', 'float', # 'E_L (K)', 'float', # 'E_U (cm^-1)', 'float', # 'E_U (K)', 'float', # 'HFS int', 'float', # 'Upper State Degeneracy', 'int', # 'Molecule Tag', 'int', # 'Quantum Number Code', 'str'] # 'Linelist'] funit = str(column_names[3][-3:]) column_units = [None, # 'Species', None, # 'NRAO Recommended', None, # 'Chemical Name', funit, # 'Freq-GHz', funit, # 'Freq Err', funit, # 'Meas Freq-GHz', funit, # 'Meas Freq Err', None, # 'Resolved QNs', None, # 'Unresolved Quantum Numbers', '?', # 'CDMS/JPL Intensity', 'Debye^2', # 'S_ijmu^2 (D^2)', '?', # 'S_ij', 'log10(s^-1)', # 'Log_10 (A_ij)', '?', # 'Lovas/AST Intensity', 'cm^-1', # 'E_L (cm^-1)', 'K', # 'E_L (K)', 'cm^-1', # 'E_U (cm^-1)', 'K', # 'E_U (K)', '?', # 'HFS int', None, # 'Upper State Degeneracy', None, # 'Molecule Tag', None, # 'Quantum Number Code', None] # 'Linelist'] column_names_original = column_names[:] #~ column_names = [i.lower() for i in column_names] #~ for i in _np.arange(len(column_names)): #~ column_names[i] = column_names[i].replace('nrao recommended', 'nrao_rec') #~ column_names[i] = column_names[i].replace('chemical name', 'name') #~ if 'meas freq err' in column_names[i]: #~ column_names[i] = 'mferr' #~ elif 'meas freq' in column_names[i]: #~ column_names[i] = 'mfreq' #~ elif 'freq err' in column_names[i]: #~ column_names[i] = 'ferr' #~ elif 'freq' in column_names[i]: #~ column_names[i] = 'freq' #~ column_names[i] = column_names[i].replace('resolved qns', 'resqn') #~ column_names[i] = column_names[i].replace('unresolved quantum numbers', 'resqn') column_names = ['species', 'nrao_rec', 'name', 'ofreq', 'oferr', 'mfreq', 'mferr', 'res_qn', 'uresqn', 'cdmsjplint', 'sijmu2', 'Sij', 'logaij', 'lovasastint', 'el_cm', 'el_k', 'eu_cm', 'eu_k', 'hfsint', 'gu', 'tag', 'qncode', 'list'] results = Table(data = rows , names = column_names, dtypes = column_dtypes) for i in _np.arange(len(column_units)): results.field(i).units = column_units[i] return results else: print('Nothing else than astropy.table output is implemented atm') return results
def main(): print 'loading full catalog...' step_1 = timer() fll_n = 'results/full_120_1.2_0.5_0.083_20-21_1.cat' full_cat = fits.open(fll_n) full_db = Table(full_cat[2].data) full_db = full_db.to_pandas() print 'loading merged catalog...' step_2 = timer() print 'elapsed time {}'.format(step_2 - step_1) mrgd_n = 'results/merged_120_1.2_0.5_0.083_20-21_1.cat' merged_cat = fits.open(mrgd_n) merged_db = Table(merged_cat[2].data) print 'getting "/h from miliarc/year' step_3 = timer() print 'elapsed time {}'.format(step_3 - step_2) pmalpha = Series(merged_db.field('PMALPHA_J2000') / 8.75e6) # 8.75e6 pmdelta = Series(merged_db.field('PMDELTA_J2000') / 8.75e6) pmealpha = Series(merged_db.field('PMALPHAERR_J2000') / 8.75e6) pmedelta = Series(merged_db.field('PMDELTAERR_J2000') / 8.75e6) print 'getting proper motions' step_4 = timer() print 'elapsed time {}'.format(step_4 - step_3) pm = Series(sqrt(array(pmalpha**2 + pmdelta**2), dtype=float)) pme = Series(sqrt(array(pmealpha**2 + pmedelta**2), dtype=float)) print 'creating new DataFrame' step_5 = timer() print 'elapsed time {}'.format(step_5 - step_4) method_j = [] method_a_p = Process(target=method_a, args=( full_db, pm, pme, pmalpha, pmdelta, pmealpha, pmedelta, )) method_j.append(method_a_p) method_a_p.start() method_b_p = Process(target=method_b, args=( full_db, pm, pme, pmalpha, pmdelta, pmealpha, pmedelta, )) method_j.append(method_b_p) method_b_p.start() active_method = list([job.is_alive() for job in method_j]) while True in active_method: active_method = list([job.is_alive() for job in method_j]) pass step_6 = timer() print 'elapsed time for a and b {}'.format(step_6 - step_5)