def _try_votable(self, table): try: from io import BytesIO as StringIO except: from StringIO import StringIO if table.data is None: return vodata = table.data.tostring() if vodata.startswith(b"<?xml"): f = StringIO() f.write(vodata) votable = astropy.io.votable.parse(f) first_table = votable.get_first_table() used_names = [] for field in first_table.fields: name = field.name.strip() clean_name = _python_save_name(name, used=used_names) used_names.append(name) if field.ucd: self.ucds[clean_name] = field.ucd unit = _try_unit(field.unit) if unit: self.units[clean_name] = unit if unit is None and field.unit: print("unit error for: %r", field.unit) self.descriptions[clean_name] = field.description self.description = first_table.description
def _get_column_meta_data(self, table, column_name, column, i): ucd_header_name = "TUCD%d" % (i+1) if ucd_header_name in table.header: self.ucds[column_name] = table.header[ucd_header_name] if column.unit: try: unit = _try_unit(column.unit) if unit: self.units[column_name] = unit except: logger.exception("could not understand unit: %s" % column.unit) else: # we may want to try ourselves unit_header_name = "TUNIT%d" % (i+1) if unit_header_name in table.header: unit_str = table.header[unit_header_name] unit = _try_unit(unit_str) if unit: self.units[column_name] = unit
def __init__(self, filename, fs_options={}, fs=None): super().__init__(filename) self.ucds = {} self.units = {} self.filename = filename self.path = filename with vaex.file.open(filename, fs_options=fs_options, fs=fs) as f: votable = astropy.io.votable.parse(f) self.first_table = votable.get_first_table() self.description = self.first_table.description for field in self.first_table.fields: name = field.name data = self.first_table.array[name] type = self.first_table.array[name].dtype clean_name = name if field.ucd: self.ucds[clean_name] = field.ucd if field.unit: unit = _try_unit(field.unit) if unit: self.units[clean_name] = unit if field.description: self.descriptions[clean_name] = field.description if type.kind in "fiubSU": # only store float and int and boolean self.add_column(clean_name, data) #self.first_table.array[name].data) if type.kind == "O": print( "column %r is of unsupported object type , will try to convert it to string" % (name, )) try: data = data.astype("S") self.add_column(name, data) except Exception as e: print("Giving up column %s, error: %r" % (name, e)) #if type.kind in ["S"]: # self.add_column(clean_name, self.first_table.array[name].data) self._freeze()
def __init__(self, filename=None, format=None, table=None, **kwargs): self.ucds = {} self.units = {} columns = {} if table is None: self.filename = filename self.format = format self.read_table() else: self.description = table.meta.get("description") self.table = table for i in range(len(self.table.dtype)): name = self.table.dtype.names[i] column = self.table[name] type = self.table.dtype[i] if type.kind in "fiuSU": # only store float and int masked_array = self.table[name].data if "ucd" in column._meta: self.ucds[name] = column._meta["ucd"] if column.unit: unit = _try_unit(column.unit) if unit: self.units[name] = unit if column.description: self.descriptions[name] = column.description if hasattr(masked_array, "mask"): if type.kind in ["f"]: masked_array.data[masked_array.mask] = np.nan if type.kind in ["i"]: masked_array.data[masked_array.mask] = 0 columns[name] = self.table[name].data if type.kind in ["SU"]: columns[name] = self.table[name].data super().__init__(columns)