def _try_votable(self, table):
try:
from io import BytesIO as StringIO
except:
from StringIO import StringIO
if table.data is None:
return
vodata = table.data.tostring()
if vodata.startswith(b"<?xml"):
f = StringIO()
f.write(vodata)
votable = astropy.io.votable.parse(f)
first_table = votable.get_first_table()
used_names = []
for field in first_table.fields:
name = field.name.strip()
clean_name = _python_save_name(name, used=used_names)
used_names.append(name)
if field.ucd:
self.ucds[clean_name] = field.ucd
unit = _try_unit(field.unit)
if unit:
self.units[clean_name] = unit
if unit is None and field.unit:
print("unit error for: %r", field.unit)
self.descriptions[clean_name] = field.description
self.description = first_table.description
def _get_column_meta_data(self, table, column_name, column, i):
ucd_header_name = "TUCD%d" % (i+1)
if ucd_header_name in table.header:
self.ucds[column_name] = table.header[ucd_header_name]
if column.unit:
try:
unit = _try_unit(column.unit)
if unit:
self.units[column_name] = unit
except:
logger.exception("could not understand unit: %s" % column.unit)
else: # we may want to try ourselves
unit_header_name = "TUNIT%d" % (i+1)
if unit_header_name in table.header:
unit_str = table.header[unit_header_name]
unit = _try_unit(unit_str)
if unit:
self.units[column_name] = unit
def __init__(self, filename, fs_options={}, fs=None):
super().__init__(filename)
self.ucds = {}
self.units = {}
self.filename = filename
self.path = filename
with vaex.file.open(filename, fs_options=fs_options, fs=fs) as f:
votable = astropy.io.votable.parse(f)
self.first_table = votable.get_first_table()
self.description = self.first_table.description
for field in self.first_table.fields:
name = field.name
data = self.first_table.array[name]
type = self.first_table.array[name].dtype
clean_name = name
if field.ucd:
self.ucds[clean_name] = field.ucd
if field.unit:
unit = _try_unit(field.unit)
if unit:
self.units[clean_name] = unit
if field.description:
self.descriptions[clean_name] = field.description
if type.kind in "fiubSU": # only store float and int and boolean
self.add_column(clean_name,
data) #self.first_table.array[name].data)
if type.kind == "O":
print(
"column %r is of unsupported object type , will try to convert it to string"
% (name, ))
try:
data = data.astype("S")
self.add_column(name, data)
except Exception as e:
print("Giving up column %s, error: %r" % (name, e))
#if type.kind in ["S"]:
# self.add_column(clean_name, self.first_table.array[name].data)
self._freeze()
def __init__(self, filename=None, format=None, table=None, **kwargs):
self.ucds = {}
self.units = {}
columns = {}
if table is None:
self.filename = filename
self.format = format
self.read_table()
else:
self.description = table.meta.get("description")
self.table = table
for i in range(len(self.table.dtype)):
name = self.table.dtype.names[i]
column = self.table[name]
type = self.table.dtype[i]
if type.kind in "fiuSU": # only store float and int
masked_array = self.table[name].data
if "ucd" in column._meta:
self.ucds[name] = column._meta["ucd"]
if column.unit:
unit = _try_unit(column.unit)
if unit:
self.units[name] = unit
if column.description:
self.descriptions[name] = column.description
if hasattr(masked_array, "mask"):
if type.kind in ["f"]:
masked_array.data[masked_array.mask] = np.nan
if type.kind in ["i"]:
masked_array.data[masked_array.mask] = 0
columns[name] = self.table[name].data
if type.kind in ["SU"]:
columns[name] = self.table[name].data
super().__init__(columns)