def __init__(self, filename):
DatasetArrays.__init__(self, filename)
self.filename = filename
self.path = filename
votable = astropy.io.votable.parse(self.filename)
self.first_table = votable.get_first_table()
self.description = self.first_table.description
for field in self.first_table.fields:
name = field.name
data = self.first_table.array[name]
type = self.first_table.array[name].dtype
clean_name = _python_save_name(name, self.columns.keys())
if field.ucd:
self.ucds[clean_name] = field.ucd
if field.unit:
unit = _try_unit(field.unit)
if unit:
self.units[clean_name] = unit
if field.description:
self.descriptions[clean_name] = field.description
if type.kind in "fiubSU": # only store float and int and boolean
self.add_column(clean_name, data) #self.first_table.array[name].data)
if type.kind == "O":
print("column %r is of unsupported object type , will try to convert it to string" % (name,))
try:
data = data.astype("S")
self.add_column(name, data)
except Exception as e:
print("Giving up column %s, error: %r" (name, e))
def __init__(self, filename):
DatasetArrays.__init__(self, filename)
self.filename = filename
self.path = filename
votable = astropy.io.votable.parse(self.filename)
self.first_table = votable.get_first_table()
self.description = self.first_table.description
for field in self.first_table.fields:
name = field.name
data = self.first_table.array[name].data
type = self.first_table.array[name].dtype
clean_name = re.sub("[^a-zA-Z_0-9]", "_", name)
if clean_name in string.digits:
clean_name = "_" + clean_name
self.ucds[clean_name] = field.ucd
self.units[clean_name] = field.unit
self.descriptions[clean_name] = field.description
if type.kind in ["f", "i"]: # only store float and int
masked_array = self.first_table.array[name]
if type.kind in ["f"]:
masked_array.data[masked_array.mask] = np.nan
if type.kind in ["i"]:
masked_array.data[masked_array.mask] = 0
self.add_column(clean_name, self.first_table.array[name].data)
def __init__(self, filename):
DatasetArrays.__init__(self, filename)
self.filename = filename
self.path = filename
votable = astropy.io.votable.parse(self.filename)
self.first_table = votable.get_first_table()
self.description = self.first_table.description
for field in self.first_table.fields:
name = field.name
data = self.first_table.array[name]
type = self.first_table.array[name].dtype
clean_name = _python_save_name(name, self.columns.keys())
if field.ucd:
self.ucds[clean_name] = field.ucd
if field.unit:
unit = _try_unit(field.unit)
if unit:
self.units[clean_name] = unit
if field.description:
self.descriptions[clean_name] = field.description
if type.kind in "fiubSU": # only store float and int and boolean
self.add_column(clean_name,
data) #self.first_table.array[name].data)
if type.kind == "O":
print(
"column %r is of unsupported object type , will try to convert it to string"
% (name, ))
try:
data = data.astype("S")
self.add_column(name, data)
except Exception as e:
print("Giving up column %s, error: %r" (name, e))
def __getitem__(self, slice):
start, stop, step = slice.start, slice.stop, slice.step
required_length = stop - start
assert start >= self.offset
chunk_data = self.left_over_chunk
enough = False if chunk_data is None else len(
chunk_data) >= required_length
if chunk_data is not None:
logger.debug("start %s offset %s chunk length %s", start,
self.offset, len(chunk_data))
#assert len(chunk_data) == start - self.offset
if enough:
logger.debug(
"we can skip the query, already have results from previous query"
)
while not enough:
adql_query = "SELECT {column_name} FROM {table_name} WHERE alpha >= {alpha_min} AND alpha < {alpha_max} ORDER BY alpha ASC"\
.format(column_name=self.column_name, table_name=self.tap_dataset.table_name, alpha_min=self.alpha_min, alpha_max=self.alpha_max)
logger.debug("executing: %s" % adql_query)
logger.debug("executing: %s" % adql_query.replace(" ", "+"))
url = self.tap_dataset.tap_url + "/sync?REQUEST=doQuery&LANG=ADQL&MAXREC=10000000&FORMAT=votable&QUERY=" + adql_query.replace(
" ", "+")
import urllib2
response = urllib2.urlopen(url)
with open(self.download_file, "w") as f:
f.write(response.read())
votable = astropy.io.votable.parse(self.download_file)
data = votable.get_first_table().array[self.column_name].data
# TODO: respect masked array
#table = astropy.table.Table.read(url, format="votable") #, show_progress=False)
#data = table[self.column_name].data.data.data
logger.debug("new chunk is of lenght %d", len(data))
self.rows_left -= len(data)
logger.debug("rows left %d", self.rows_left)
if chunk_data is None:
chunk_data = data
else:
chunk_data = np.concatenate([chunk_data, data])
if len(chunk_data) >= required_length:
enough = True
logger.debug("total chunk is of lenght %d, enough: %s",
len(chunk_data), enough)
self.alpha_min += self.alpha_step
self.alpha_max += self.alpha_step
result, self.left_over_chunk = chunk_data[:
required_length], chunk_data[
required_length:]
#print(result)
logger.debug("left over is of length %d",
len(self.left_over_chunk))
return result #np.zeros(N, dtype=self.dtype)
def __getitem__(self, slice):
start, stop, step = slice.start, slice.stop, slice.step
required_length = stop - start
assert start >= self.offset
chunk_data = self.left_over_chunk
enough = False if chunk_data is None else len(chunk_data) >= required_length
if chunk_data is not None:
logger.debug("start %s offset %s chunk length %s", start, self.offset, len(chunk_data))
#assert len(chunk_data) == start - self.offset
if enough:
logger.debug("we can skip the query, already have results from previous query")
while not enough:
adql_query = "SELECT {column_name} FROM {table_name} WHERE alpha >= {alpha_min} AND alpha < {alpha_max} ORDER BY alpha ASC"\
.format(column_name=self.column_name, table_name=self.tap_dataset.table_name, alpha_min=self.alpha_min, alpha_max=self.alpha_max)
logger.debug("executing: %s" % adql_query)
logger.debug("executing: %s" % adql_query.replace(" ", "+"))
url = self.tap_dataset.tap_url + "/sync?REQUEST=doQuery&LANG=ADQL&MAXREC=10000000&FORMAT=votable&QUERY=" +adql_query.replace(" ", "+")
import urllib2
response = urllib2.urlopen(url)
with open(self.download_file, "w") as f:
f.write(response.read())
votable = astropy.io.votable.parse(self.download_file)
data = votable.get_first_table().array[self.column_name].data
# TODO: respect masked array
#table = astropy.table.Table.read(url, format="votable") #, show_progress=False)
#data = table[self.column_name].data.data.data
logger.debug("new chunk is of lenght %d", len(data))
self.rows_left -= len(data)
logger.debug("rows left %d", self.rows_left)
if chunk_data is None:
chunk_data = data
else:
chunk_data = np.concatenate([chunk_data, data])
if len(chunk_data) >= required_length:
enough = True
logger.debug("total chunk is of lenght %d, enough: %s", len(chunk_data), enough)
self.alpha_min += self.alpha_step
self.alpha_max += self.alpha_step
result, self.left_over_chunk = chunk_data[:required_length], chunk_data[required_length:]
#print(result)
logger.debug("left over is of length %d", len(self.left_over_chunk))
return result #np.zeros(N, dtype=self.dtype)
def __init__(self, filename, fs_options={}, fs=None):
super().__init__(filename)
self.ucds = {}
self.units = {}
self.filename = filename
self.path = filename
with vaex.file.open(filename, fs_options=fs_options, fs=fs) as f:
votable = astropy.io.votable.parse(f)
self.first_table = votable.get_first_table()
self.description = self.first_table.description
for field in self.first_table.fields:
name = field.name
data = self.first_table.array[name]
type = self.first_table.array[name].dtype
clean_name = name
if field.ucd:
self.ucds[clean_name] = field.ucd
if field.unit:
unit = _try_unit(field.unit)
if unit:
self.units[clean_name] = unit
if field.description:
self.descriptions[clean_name] = field.description
if type.kind in "fiubSU": # only store float and int and boolean
self.add_column(clean_name,
data) #self.first_table.array[name].data)
if type.kind == "O":
print(
"column %r is of unsupported object type , will try to convert it to string"
% (name, ))
try:
data = data.astype("S")
self.add_column(name, data)
except Exception as e:
print("Giving up column %s, error: %r" % (name, e))
#if type.kind in ["S"]:
# self.add_column(clean_name, self.first_table.array[name].data)
self._freeze()
def output_spectra_catalogue(spectra, output_folder):
"""
Output the list of spectrum stats to a VOTable file thor-hi-spectra.vot
:param spectra: The list of Spectrum objects
:return: None
"""
rows = len(spectra)
ids = np.empty(rows, dtype=object)
days = np.zeros(rows, dtype=int)
fields = np.empty(rows, dtype=object)
sources = np.empty(rows, dtype=object)
longitudes = np.zeros(rows)
latitudes = np.zeros(rows)
eq_ras = np.zeros(rows)
eq_decs = np.zeros(rows)
max_flux = np.zeros(rows)
max_opacity = np.zeros(rows)
min_opacity = np.zeros(rows)
max_velocity = np.zeros(rows)
min_velocity = np.zeros(rows)
rms_opacity = np.zeros(rows)
opacity_range = np.zeros(rows)
continuum_sd = np.zeros(rows)
continuum_temp = np.zeros(rows)
max_s_max_n = np.zeros(rows)
max_em_std = np.zeros(rows)
max_emission = np.zeros(rows)
min_emission = np.zeros(rows)
rating = np.empty(rows, dtype=object)
used = np.empty(rows, dtype=bool)
resolved = np.empty(rows, dtype=bool)
duplicate = np.empty(rows, dtype=bool)
has_emission = np.empty(rows, dtype=bool)
filenames = np.empty(rows, dtype=object)
local_paths = np.empty(rows, dtype=object)
local_emission_paths = np.empty(rows, dtype=object)
base_path = os.path.realpath('.')
i = 0
for spectrum in spectra:
ids[i] = spectrum.name
days[i] = int(spectrum.day)
fields[i] = spectrum.field_name
sources[i] = spectrum.src_id
longitudes[i] = spectrum.longitude
latitudes[i] = spectrum.latitude
eq_ras[i] = spectrum.ra
eq_decs[i] = spectrum.dec
max_flux[i] = np.max(spectrum.flux)
min_opacity[i] = np.min(spectrum.opacities)
max_opacity[i] = np.max(spectrum.opacities)
rms_opacity[i] = np.sqrt(np.mean(np.square(spectrum.opacities)))
min_velocity[i] = np.min(spectrum.velocity)
max_velocity[i] = np.max(spectrum.velocity)
max_em_std[i] = np.max(spectrum.em_std)
if spectrum.has_emission:
max_emission[i] = np.max(spectrum.em_temps)
min_emission[i] = np.min(spectrum.em_temps)
opacity_range[i] = spectrum.opacity_range
max_s_max_n[i] = spectrum.max_s_max_n
continuum_sd[i] = spectrum.continuum_sd
continuum_temp[i] = spectrum.continuum_temp
rating[i] = spectrum.rating
src_parts = spectrum.src_id.split('-')
resolved[
i] = False # is_resolved(spectrum.day, spectrum.field_name, src_parts[0], src_parts[1], spectrum.beam_area,
# None)
duplicate[i] = spectrum.duplicate
has_emission[i] = spectrum.has_emission
used[i] = not spectrum.low_sn
prefix = 'spectra/' + spectrum.field_name + '/' + spectrum.field_name + "_src" + spectrum.src_id
filenames[i] = prefix + "_plot.png"
em_filename = prefix + "_emission.png"
local_paths[i] = base_path + '/' + filenames[i]
local_emission_paths[i] = base_path + '/' + em_filename
i += 1
spectra_table = Table(
[
ids, days, fields, sources, longitudes, latitudes, eq_ras, eq_decs,
max_flux, min_opacity, max_opacity, rms_opacity, min_emission,
max_emission, min_velocity, max_velocity, used, continuum_temp,
opacity_range, max_s_max_n, continuum_sd, max_em_std, rating,
resolved, duplicate, has_emission, filenames, local_paths,
local_emission_paths
],
names=[
'Name', 'Day', 'Field', 'Source', 'Longitude', 'Latitude', 'RA',
'Dec', 'Max_Flux', 'Min_Opacity', 'Max_Opacity', 'RMS_Opacity',
'Min_Emission', 'Max_Emission', 'Min_Velocity', 'Max_Velocity',
'Used', 'Continuum_Temp', 'Opacity_Range', 'Max_S_Max_N',
'Continuum_SD', 'max_em_std', 'Rating', 'Resolved', 'Duplicate',
'Has_Emission', 'Filename', 'Local_Path', 'Local_Emission_Path'
],
meta={
'ID': 'thor_hi_spectra',
'name': 'THOR HI Spectra ' + str(datetime.date.today())
})
votable = from_table(spectra_table)
table = votable.get_first_table()
set_field_metadata(table.get_field_by_id('Min_Emission'), 'stat.min', 'K',
'Minimum average emission')
set_field_metadata(table.get_field_by_id('Max_Emission'), 'stat.max', 'K',
'Maximum average emission')
filename = output_folder + "/thor-hi-spectra.vot"
writeto(votable, filename)
print("Wrote out", i, "spectra to", filename)
for grade in "ABCDEF":
num_rating = len(np.where(rating == grade)[0])
print("%s: %3d" % (grade, num_rating))
print("Mean continuum sd %f" % np.mean(continuum_sd))
