def __init__(self, filename): DatasetArrays.__init__(self, filename) self.filename = filename self.path = filename votable = astropy.io.votable.parse(self.filename) self.first_table = votable.get_first_table() self.description = self.first_table.description for field in self.first_table.fields: name = field.name data = self.first_table.array[name] type = self.first_table.array[name].dtype clean_name = _python_save_name(name, self.columns.keys()) if field.ucd: self.ucds[clean_name] = field.ucd if field.unit: unit = _try_unit(field.unit) if unit: self.units[clean_name] = unit if field.description: self.descriptions[clean_name] = field.description if type.kind in "fiubSU": # only store float and int and boolean self.add_column(clean_name, data) #self.first_table.array[name].data) if type.kind == "O": print("column %r is of unsupported object type , will try to convert it to string" % (name,)) try: data = data.astype("S") self.add_column(name, data) except Exception as e: print("Giving up column %s, error: %r" (name, e))
def __init__(self, filename): DatasetArrays.__init__(self, filename) self.filename = filename self.path = filename votable = astropy.io.votable.parse(self.filename) self.first_table = votable.get_first_table() self.description = self.first_table.description for field in self.first_table.fields: name = field.name data = self.first_table.array[name].data type = self.first_table.array[name].dtype clean_name = re.sub("[^a-zA-Z_0-9]", "_", name) if clean_name in string.digits: clean_name = "_" + clean_name self.ucds[clean_name] = field.ucd self.units[clean_name] = field.unit self.descriptions[clean_name] = field.description if type.kind in ["f", "i"]: # only store float and int masked_array = self.first_table.array[name] if type.kind in ["f"]: masked_array.data[masked_array.mask] = np.nan if type.kind in ["i"]: masked_array.data[masked_array.mask] = 0 self.add_column(clean_name, self.first_table.array[name].data)
def __init__(self, filename): DatasetArrays.__init__(self, filename) self.filename = filename self.path = filename votable = astropy.io.votable.parse(self.filename) self.first_table = votable.get_first_table() self.description = self.first_table.description for field in self.first_table.fields: name = field.name data = self.first_table.array[name] type = self.first_table.array[name].dtype clean_name = _python_save_name(name, self.columns.keys()) if field.ucd: self.ucds[clean_name] = field.ucd if field.unit: unit = _try_unit(field.unit) if unit: self.units[clean_name] = unit if field.description: self.descriptions[clean_name] = field.description if type.kind in "fiubSU": # only store float and int and boolean self.add_column(clean_name, data) #self.first_table.array[name].data) if type.kind == "O": print( "column %r is of unsupported object type , will try to convert it to string" % (name, )) try: data = data.astype("S") self.add_column(name, data) except Exception as e: print("Giving up column %s, error: %r" (name, e))
def __getitem__(self, slice): start, stop, step = slice.start, slice.stop, slice.step required_length = stop - start assert start >= self.offset chunk_data = self.left_over_chunk enough = False if chunk_data is None else len( chunk_data) >= required_length if chunk_data is not None: logger.debug("start %s offset %s chunk length %s", start, self.offset, len(chunk_data)) #assert len(chunk_data) == start - self.offset if enough: logger.debug( "we can skip the query, already have results from previous query" ) while not enough: adql_query = "SELECT {column_name} FROM {table_name} WHERE alpha >= {alpha_min} AND alpha < {alpha_max} ORDER BY alpha ASC"\ .format(column_name=self.column_name, table_name=self.tap_dataset.table_name, alpha_min=self.alpha_min, alpha_max=self.alpha_max) logger.debug("executing: %s" % adql_query) logger.debug("executing: %s" % adql_query.replace(" ", "+")) url = self.tap_dataset.tap_url + "/sync?REQUEST=doQuery&LANG=ADQL&MAXREC=10000000&FORMAT=votable&QUERY=" + adql_query.replace( " ", "+") import urllib2 response = urllib2.urlopen(url) with open(self.download_file, "w") as f: f.write(response.read()) votable = astropy.io.votable.parse(self.download_file) data = votable.get_first_table().array[self.column_name].data # TODO: respect masked array #table = astropy.table.Table.read(url, format="votable") #, show_progress=False) #data = table[self.column_name].data.data.data logger.debug("new chunk is of lenght %d", len(data)) self.rows_left -= len(data) logger.debug("rows left %d", self.rows_left) if chunk_data is None: chunk_data = data else: chunk_data = np.concatenate([chunk_data, data]) if len(chunk_data) >= required_length: enough = True logger.debug("total chunk is of lenght %d, enough: %s", len(chunk_data), enough) self.alpha_min += self.alpha_step self.alpha_max += self.alpha_step result, self.left_over_chunk = chunk_data[: required_length], chunk_data[ required_length:] #print(result) logger.debug("left over is of length %d", len(self.left_over_chunk)) return result #np.zeros(N, dtype=self.dtype)
def __getitem__(self, slice): start, stop, step = slice.start, slice.stop, slice.step required_length = stop - start assert start >= self.offset chunk_data = self.left_over_chunk enough = False if chunk_data is None else len(chunk_data) >= required_length if chunk_data is not None: logger.debug("start %s offset %s chunk length %s", start, self.offset, len(chunk_data)) #assert len(chunk_data) == start - self.offset if enough: logger.debug("we can skip the query, already have results from previous query") while not enough: adql_query = "SELECT {column_name} FROM {table_name} WHERE alpha >= {alpha_min} AND alpha < {alpha_max} ORDER BY alpha ASC"\ .format(column_name=self.column_name, table_name=self.tap_dataset.table_name, alpha_min=self.alpha_min, alpha_max=self.alpha_max) logger.debug("executing: %s" % adql_query) logger.debug("executing: %s" % adql_query.replace(" ", "+")) url = self.tap_dataset.tap_url + "/sync?REQUEST=doQuery&LANG=ADQL&MAXREC=10000000&FORMAT=votable&QUERY=" +adql_query.replace(" ", "+") import urllib2 response = urllib2.urlopen(url) with open(self.download_file, "w") as f: f.write(response.read()) votable = astropy.io.votable.parse(self.download_file) data = votable.get_first_table().array[self.column_name].data # TODO: respect masked array #table = astropy.table.Table.read(url, format="votable") #, show_progress=False) #data = table[self.column_name].data.data.data logger.debug("new chunk is of lenght %d", len(data)) self.rows_left -= len(data) logger.debug("rows left %d", self.rows_left) if chunk_data is None: chunk_data = data else: chunk_data = np.concatenate([chunk_data, data]) if len(chunk_data) >= required_length: enough = True logger.debug("total chunk is of lenght %d, enough: %s", len(chunk_data), enough) self.alpha_min += self.alpha_step self.alpha_max += self.alpha_step result, self.left_over_chunk = chunk_data[:required_length], chunk_data[required_length:] #print(result) logger.debug("left over is of length %d", len(self.left_over_chunk)) return result #np.zeros(N, dtype=self.dtype)
def __init__(self, filename, fs_options={}, fs=None): super().__init__(filename) self.ucds = {} self.units = {} self.filename = filename self.path = filename with vaex.file.open(filename, fs_options=fs_options, fs=fs) as f: votable = astropy.io.votable.parse(f) self.first_table = votable.get_first_table() self.description = self.first_table.description for field in self.first_table.fields: name = field.name data = self.first_table.array[name] type = self.first_table.array[name].dtype clean_name = name if field.ucd: self.ucds[clean_name] = field.ucd if field.unit: unit = _try_unit(field.unit) if unit: self.units[clean_name] = unit if field.description: self.descriptions[clean_name] = field.description if type.kind in "fiubSU": # only store float and int and boolean self.add_column(clean_name, data) #self.first_table.array[name].data) if type.kind == "O": print( "column %r is of unsupported object type , will try to convert it to string" % (name, )) try: data = data.astype("S") self.add_column(name, data) except Exception as e: print("Giving up column %s, error: %r" % (name, e)) #if type.kind in ["S"]: # self.add_column(clean_name, self.first_table.array[name].data) self._freeze()
def output_spectra_catalogue(spectra, output_folder): """ Output the list of spectrum stats to a VOTable file thor-hi-spectra.vot :param spectra: The list of Spectrum objects :return: None """ rows = len(spectra) ids = np.empty(rows, dtype=object) days = np.zeros(rows, dtype=int) fields = np.empty(rows, dtype=object) sources = np.empty(rows, dtype=object) longitudes = np.zeros(rows) latitudes = np.zeros(rows) eq_ras = np.zeros(rows) eq_decs = np.zeros(rows) max_flux = np.zeros(rows) max_opacity = np.zeros(rows) min_opacity = np.zeros(rows) max_velocity = np.zeros(rows) min_velocity = np.zeros(rows) rms_opacity = np.zeros(rows) opacity_range = np.zeros(rows) continuum_sd = np.zeros(rows) continuum_temp = np.zeros(rows) max_s_max_n = np.zeros(rows) max_em_std = np.zeros(rows) max_emission = np.zeros(rows) min_emission = np.zeros(rows) rating = np.empty(rows, dtype=object) used = np.empty(rows, dtype=bool) resolved = np.empty(rows, dtype=bool) duplicate = np.empty(rows, dtype=bool) has_emission = np.empty(rows, dtype=bool) filenames = np.empty(rows, dtype=object) local_paths = np.empty(rows, dtype=object) local_emission_paths = np.empty(rows, dtype=object) base_path = os.path.realpath('.') i = 0 for spectrum in spectra: ids[i] = spectrum.name days[i] = int(spectrum.day) fields[i] = spectrum.field_name sources[i] = spectrum.src_id longitudes[i] = spectrum.longitude latitudes[i] = spectrum.latitude eq_ras[i] = spectrum.ra eq_decs[i] = spectrum.dec max_flux[i] = np.max(spectrum.flux) min_opacity[i] = np.min(spectrum.opacities) max_opacity[i] = np.max(spectrum.opacities) rms_opacity[i] = np.sqrt(np.mean(np.square(spectrum.opacities))) min_velocity[i] = np.min(spectrum.velocity) max_velocity[i] = np.max(spectrum.velocity) max_em_std[i] = np.max(spectrum.em_std) if spectrum.has_emission: max_emission[i] = np.max(spectrum.em_temps) min_emission[i] = np.min(spectrum.em_temps) opacity_range[i] = spectrum.opacity_range max_s_max_n[i] = spectrum.max_s_max_n continuum_sd[i] = spectrum.continuum_sd continuum_temp[i] = spectrum.continuum_temp rating[i] = spectrum.rating src_parts = spectrum.src_id.split('-') resolved[ i] = False # is_resolved(spectrum.day, spectrum.field_name, src_parts[0], src_parts[1], spectrum.beam_area, # None) duplicate[i] = spectrum.duplicate has_emission[i] = spectrum.has_emission used[i] = not spectrum.low_sn prefix = 'spectra/' + spectrum.field_name + '/' + spectrum.field_name + "_src" + spectrum.src_id filenames[i] = prefix + "_plot.png" em_filename = prefix + "_emission.png" local_paths[i] = base_path + '/' + filenames[i] local_emission_paths[i] = base_path + '/' + em_filename i += 1 spectra_table = Table( [ ids, days, fields, sources, longitudes, latitudes, eq_ras, eq_decs, max_flux, min_opacity, max_opacity, rms_opacity, min_emission, max_emission, min_velocity, max_velocity, used, continuum_temp, opacity_range, max_s_max_n, continuum_sd, max_em_std, rating, resolved, duplicate, has_emission, filenames, local_paths, local_emission_paths ], names=[ 'Name', 'Day', 'Field', 'Source', 'Longitude', 'Latitude', 'RA', 'Dec', 'Max_Flux', 'Min_Opacity', 'Max_Opacity', 'RMS_Opacity', 'Min_Emission', 'Max_Emission', 'Min_Velocity', 'Max_Velocity', 'Used', 'Continuum_Temp', 'Opacity_Range', 'Max_S_Max_N', 'Continuum_SD', 'max_em_std', 'Rating', 'Resolved', 'Duplicate', 'Has_Emission', 'Filename', 'Local_Path', 'Local_Emission_Path' ], meta={ 'ID': 'thor_hi_spectra', 'name': 'THOR HI Spectra ' + str(datetime.date.today()) }) votable = from_table(spectra_table) table = votable.get_first_table() set_field_metadata(table.get_field_by_id('Min_Emission'), 'stat.min', 'K', 'Minimum average emission') set_field_metadata(table.get_field_by_id('Max_Emission'), 'stat.max', 'K', 'Maximum average emission') filename = output_folder + "/thor-hi-spectra.vot" writeto(votable, filename) print("Wrote out", i, "spectra to", filename) for grade in "ABCDEF": num_rating = len(np.where(rating == grade)[0]) print("%s: %3d" % (grade, num_rating)) print("Mean continuum sd %f" % np.mean(continuum_sd))