def write_meta(self):
"""ucds, descriptions and units are written as attributes in the hdf5 file, instead of a seperate file as
the default :func:`Dataset.write_meta`.
"""
with h5py.File(self.filename, "r+") as h5file_output:
h5table_root = h5file_output[self.h5table_root_name]
if self.description is not None:
h5table_root.attrs["description"] = self.description
h5columns = h5table_root if self._version == 1 else h5table_root['columns']
for column_name in self.columns.keys():
h5dataset = None
if column_name in h5columns:
h5dataset = h5columns[column_name]
else:
for group in h5columns.values():
if 'type' in group.attrs:
if group.attrs['type'] in ['csr_matrix']:
for name, column in group.items():
if name == column_name:
h5dataset = column
if h5dataset is None:
raise ValueError('column {} not found'.format(column_name))
for name, values in [("ucd", self.ucds), ("unit", self.units), ("description", self.descriptions)]:
if column_name in values:
value = ensure_string(values[column_name], cast=True)
h5dataset.attrs[name] = value
else:
if name in h5columns.attrs:
del h5dataset.attrs[name]
def write_meta(self):
"""ucds, descriptions and units are written as attributes in the hdf5 file, instead of a seperate file as
the default :func:`Dataset.write_meta`.
"""
with h5py.File(self.filename, "r+") as h5file_output:
h5table_root = h5file_output[self.h5table_root_name]
if self.description is not None:
h5table_root.attrs["description"] = self.description
h5columns = h5table_root if self._version == 1 else h5table_root[
'columns']
for column_name in self.columns.keys():
h5dataset = h5columns[column_name]
for name, values in [("ucd", self.ucds), ("unit", self.units),
("description", self.descriptions)]:
if column_name in values:
value = ensure_string(values[column_name], cast=True)
h5dataset.attrs[name] = value
else:
if name in h5columns.attrs:
del h5dataset.attrs[name]
def _load_columns(self, h5data, first=[]):
# print h5data
# make sure x y x etc are first
finished = set()
if "description" in h5data.attrs:
self.description = ensure_string(h5data.attrs["description"])
# hdf5, or h5py doesn't keep the order of columns, so manually track that, also enables reordering later
h5columns = h5data if self._version == 1 else h5data['columns']
if "column_order" in h5columns.attrs:
column_order = ensure_string(
h5columns.attrs["column_order"]).split(",")
else:
column_order = []
# for name in list(h5columns):
# if name not in column_order:
# column_order.append(name)
# for column_name in column_order:
# if column_name in h5columns and column_name not in finished:
for group_name in list(h5columns):
logger.debug('loading column: %s', group_name)
group = h5columns[group_name]
if 'type' in group.attrs:
if group.attrs['type'] in ['csr_matrix']:
from scipy.sparse import csc_matrix, csr_matrix
class csr_matrix_nocheck(csr_matrix):
def check_format(self, *args, **kwargs):
pass
data = self._map_hdf5_array(group['data'])
indptr = self._map_hdf5_array(group['indptr'])
indices = self._map_hdf5_array(group['indices'])
#column_names = ensure_string(group.attrs["column_names"]).split(",")
# make sure we keep the original order
groups = [(name, value) for name, value in group.items()
if isinstance(value, h5py.Group)]
column_names = [None] * len(groups)
for name, column in groups:
column_names[column.attrs['column_index']] = name
matrix = csr_matrix_nocheck(
(data, indices, indptr),
shape=(len(indptr) - 1, len(column_names)))
assert matrix.data is data
# assert matrix.indptr is indptr
assert matrix.indices is indices
self.add_columns(column_names, matrix)
else:
column_name = group_name
column = h5columns[column_name]
if "alias" in column.attrs:
column_name = column.attrs["alias"]
if "ucd" in column.attrs:
self.ucds[column_name] = ensure_string(column.attrs["ucd"])
if "description" in column.attrs:
self.descriptions[column_name] = ensure_string(
column.attrs["description"])
if "unit" in column.attrs:
try:
unitname = ensure_string(column.attrs["unit"])
if unitname and unitname != "None":
self.units[column_name] = _try_unit(unitname)
except:
logger.exception("error parsing unit: %s",
column.attrs["unit"])
if "units" in column.attrs: # Amuse case
unitname = ensure_string(column.attrs["units"])
logger.debug("amuse unit: %s", unitname)
if unitname == "(0.01 * system.get('S.I.').base('length'))":
self.units[column_name] = astropy.units.Unit("cm")
if unitname == "((0.01 * system.get('S.I.').base('length')) * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("cm/s")
if unitname == "(0.001 * system.get('S.I.').base('mass'))":
self.units[column_name] = astropy.units.Unit("gram")
if unitname == "system.get('S.I.').base('length')":
self.units[column_name] = astropy.units.Unit("m")
if unitname == "(system.get('S.I.').base('length') * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("m/s")
if unitname == "system.get('S.I.').base('mass')":
self.units[column_name] = astropy.units.Unit("kg")
data = column if self._version == 1 else column['data']
if hasattr(data, "dtype"):
if "dtype" in data.attrs and data.attrs["dtype"] == "str":
indices = self._map_hdf5_array(column['indices'])
bytes = self._map_hdf5_array(data)
if "null_bitmap" in column:
null_bitmap = self._map_hdf5_array(
column['null_bitmap'])
else:
null_bitmap = None
if isinstance(
indices,
np.ndarray): # this is a real mmappable file
self.add_column(
column_name,
vaex.arrow.convert.
arrow_string_array_from_buffers(
bytes, indices, null_bitmap))
else:
# if not a reall mmappable array, we fall back to this, maybe we can generalize this
self.add_column(
column_name,
ColumnStringArrow(indices,
bytes,
null_bitmap=null_bitmap))
else:
shape = data.shape
if True: # len(shape) == 1:
dtype = data.dtype
if "dtype" in data.attrs:
dtype = data.attrs["dtype"]
logger.debug("adding column %r with dtype %r",
column_name, dtype)
# self.addColumn(column_name, offset, len(data), dtype=dtype)
if self._version > 1 and 'mask' in column:
self.add_column(
column_name,
self._map_hdf5_array(data, column['mask']))
else:
self.add_column(column_name,
self._map_hdf5_array(data))
else:
transposed = shape[1] < shape[0]
self.addRank1(column_name,
offset,
shape[1],
length1=shape[0],
dtype=data.dtype,
stride=1,
stride1=1,
transposed=transposed)
all_columns = dict(**self._columns)
# in case the column_order refers to non-existing columns
column_order = [k for k in column_order if k in all_columns]
column_names = []
self._columns = {}
for name in column_order:
self._columns[name] = all_columns.pop(name)
# add the rest
for name, col in all_columns.items():
self._columns[name] = col
def _load_columns(self, h5data, first=[]):
# print h5data
# make sure x y x etc are first
finished = set()
if "description" in h5data.attrs:
self.description = ensure_string(h5data.attrs["description"])
# hdf5, or h5py doesn't keep the order of columns, so manually track that, also enables reordering later
h5columns = h5data if self._version == 1 else h5data['columns']
if "column_order" in h5columns.attrs:
column_order = ensure_string(
h5columns.attrs["column_order"]).split(",")
else:
column_order = []
for name in list(h5columns):
if name not in column_order:
column_order.append(name)
for column_name in column_order:
if column_name in h5columns and column_name not in finished:
column = h5columns[column_name]
if "ucd" in column.attrs:
self.ucds[column_name] = ensure_string(column.attrs["ucd"])
if "description" in column.attrs:
self.descriptions[column_name] = ensure_string(
column.attrs["description"])
if "unit" in column.attrs:
try:
unitname = ensure_string(column.attrs["unit"])
if unitname and unitname != "None":
self.units[column_name] = _try_unit(unitname)
except:
logger.exception("error parsing unit: %s",
column.attrs["unit"])
if "units" in column.attrs: # Amuse case
unitname = ensure_string(column.attrs["units"])
logger.debug("amuse unit: %s", unitname)
if unitname == "(0.01 * system.get('S.I.').base('length'))":
self.units[column_name] = astropy.units.Unit("cm")
if unitname == "((0.01 * system.get('S.I.').base('length')) * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("cm/s")
if unitname == "(0.001 * system.get('S.I.').base('mass'))":
self.units[column_name] = astropy.units.Unit("gram")
if unitname == "system.get('S.I.').base('length')":
self.units[column_name] = astropy.units.Unit("m")
if unitname == "(system.get('S.I.').base('length') * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("m/s")
if unitname == "system.get('S.I.').base('mass')":
self.units[column_name] = astropy.units.Unit("kg")
data = column if self._version == 1 else column['data']
if hasattr(data, "dtype"):
# print column, column.shape
offset = data.id.get_offset()
if offset is None:
raise Exception(
"columns doesn't really exist in hdf5 file")
shape = data.shape
if True: # len(shape) == 1:
dtype = data.dtype
if "dtype" in data.attrs:
dtype = data.attrs["dtype"]
logger.debug("adding column %r with dtype %r",
column_name, dtype)
self.addColumn(column_name,
offset,
len(data),
dtype=dtype)
if self._version > 1 and 'mask' in column:
mask = column['mask']
offset = mask.id.get_offset()
self.addColumn("temp_mask",
offset,
len(data),
dtype=mask.dtype)
mask_array = self.columns['temp_mask']
del self.columns['temp_mask']
self.column_names.remove('temp_mask')
ar = self.columns[column_name] = np.ma.array(
self.columns[column_name],
mask=mask_array,
shrink=False)
assert ar.mask is mask_array, "masked array was copied"
else:
# transposed = self._length is None or shape[0] == self._length
transposed = shape[1] < shape[0]
self.addRank1(column_name,
offset,
shape[1],
length1=shape[0],
dtype=data.dtype,
stride=1,
stride1=1,
transposed=transposed)
# if len(shape[0]) == self._length:
# self.addRank1(column_name, offset, shape[1], length1=shape[0], dtype=column.dtype, stride=1, stride1=1)
# self.addColumn(column_name+"_0", offset, shape[1], dtype=column.dtype)
# self.addColumn(column_name+"_last", offset+(shape[0]-1)*shape[1]*column.dtype.itemsize, shape[1], dtype=column.dtype)
# self.addRank1(name, offset+8*i, length=self.numberParticles+1, length1=self.numberTimes-1, dtype=np.float64, stride=stride, stride1=1, filename=filename_extra)
finished.add(column_name)
def _load_columns(self, h5data, first=[]):
# print h5data
# make sure x y x etc are first
finished = set()
if "description" in h5data.attrs:
self.description = ensure_string(h5data.attrs["description"])
# hdf5, or h5py doesn't keep the order of columns, so manually track that, also enables reordering later
h5columns = h5data if self._version == 1 else h5data['columns']
if "column_order" in h5columns.attrs:
column_order = ensure_string(h5columns.attrs["column_order"]).split(",")
else:
column_order = []
# for name in list(h5columns):
# if name not in column_order:
# column_order.append(name)
# for column_name in column_order:
# if column_name in h5columns and column_name not in finished:
for group_name in list(h5columns):
group = h5columns[group_name]
if 'type' in group.attrs:
if group.attrs['type'] in ['csr_matrix']:
from scipy.sparse import csc_matrix, csr_matrix
class csr_matrix_nocheck(csr_matrix):
def check_format(self, *args, **kwargs):
pass
data = self._map_hdf5_array(group['data'])
indptr = self._map_hdf5_array(group['indptr'])
indices = self._map_hdf5_array(group['indices'])
#column_names = ensure_string(group.attrs["column_names"]).split(",")
# make sure we keep the original order
groups = [(name, value) for name, value in group.items() if isinstance(value, h5py.Group)]
column_names = [None] * len(groups)
for name, column in groups:
column_names[column.attrs['column_index']] = name
matrix = csr_matrix_nocheck((data, indices, indptr), shape=(len(indptr)-1, len(column_names)))
assert matrix.data is data
# assert matrix.indptr is indptr
assert matrix.indices is indices
self.add_columns(column_names, matrix)
else:
column_name = group_name
column = h5columns[column_name]
if "ucd" in column.attrs:
self.ucds[column_name] = ensure_string(column.attrs["ucd"])
if "description" in column.attrs:
self.descriptions[column_name] = ensure_string(column.attrs["description"])
if "unit" in column.attrs:
try:
unitname = ensure_string(column.attrs["unit"])
if unitname and unitname != "None":
self.units[column_name] = _try_unit(unitname)
except:
logger.exception("error parsing unit: %s", column.attrs["unit"])
if "units" in column.attrs: # Amuse case
unitname = ensure_string(column.attrs["units"])
logger.debug("amuse unit: %s", unitname)
if unitname == "(0.01 * system.get('S.I.').base('length'))":
self.units[column_name] = astropy.units.Unit("cm")
if unitname == "((0.01 * system.get('S.I.').base('length')) * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("cm/s")
if unitname == "(0.001 * system.get('S.I.').base('mass'))":
self.units[column_name] = astropy.units.Unit("gram")
if unitname == "system.get('S.I.').base('length')":
self.units[column_name] = astropy.units.Unit("m")
if unitname == "(system.get('S.I.').base('length') * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("m/s")
if unitname == "system.get('S.I.').base('mass')":
self.units[column_name] = astropy.units.Unit("kg")
data = column if self._version == 1 else column['data']
if hasattr(data, "dtype"):
if "dtype" in data.attrs and data.attrs["dtype"] == "str":
indices = self._map_hdf5_array(column['indices'])
bytes = self._map_hdf5_array(data)
if "null_bitmap" in column:
null_bitmap = self._map_hdf5_array(column['null_bitmap'])
else:
null_bitmap = None
from vaex.column import ColumnStringArrow
self.add_column(column_name, ColumnStringArrow(indices, bytes, null_bitmap=null_bitmap))
else:
shape = data.shape
if True: # len(shape) == 1:
dtype = data.dtype
if "dtype" in data.attrs:
dtype = data.attrs["dtype"]
logger.debug("adding column %r with dtype %r", column_name, dtype)
# self.addColumn(column_name, offset, len(data), dtype=dtype)
if self._version > 1 and 'mask' in column:
self.add_column(column_name, self._map_hdf5_array(data, column['mask']))
else:
self.add_column(column_name, self._map_hdf5_array(data))
else:
transposed = shape[1] < shape[0]
self.addRank1(column_name, offset, shape[1], length1=shape[0], dtype=data.dtype, stride=1, stride1=1, transposed=transposed)
all_columns = dict(**self.columns)
# print(all_columns, column_order)
self.column_names = []
for name in column_order:
self.columns[name] = all_columns.pop(name)
self.column_names.append(name)
# add the rest
for name, col in all_columns.items():
self.columns[name] = col
self.column_names.append(name)
def _load_columns(self, h5data):
#print h5data
# make sure x y x etc are first
first = "x y z vx vy vz".split()
finished = set()
if "description" in h5data.attrs:
self.description = ensure_string(h5data.attrs["description"])
for column_name in first + list(h5data):
if column_name in h5data and column_name not in finished:
#print type(column_name)
column = h5data[column_name]
if "ucd" in column.attrs:
self.ucds[column_name] = ensure_string(column.attrs["ucd"])
if "description" in column.attrs:
self.descriptions[column_name] = ensure_string(column.attrs["description"])
if "unit" in column.attrs:
try:
unitname = ensure_string(column.attrs["unit"])
if unitname and unitname != "None":
self.units[column_name] = astropy.units.Unit(unitname)
except:
logger.exception("error parsing unit: %s", column.attrs["unit"])
if "units" in column.attrs: # Amuse case
unitname = ensure_string(column.attrs["units"])
logger.debug("amuse unit: %s", unitname)
if unitname == "(0.01 * system.get('S.I.').base('length'))":
self.units[column_name] = astropy.units.Unit("cm")
if unitname == "((0.01 * system.get('S.I.').base('length')) * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("cm/s")
if unitname == "(0.001 * system.get('S.I.').base('mass'))":
self.units[column_name] = astropy.units.Unit("gram")
if unitname == "system.get('S.I.').base('length')":
self.units[column_name] = astropy.units.Unit("m")
if unitname == "(system.get('S.I.').base('length') * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("m/s")
if unitname == "system.get('S.I.').base('mass')":
self.units[column_name] = astropy.units.Unit("kg")
if hasattr(column, "dtype"):
#print column, column.shape
offset = column.id.get_offset()
if offset is None:
raise Exception("columns doesn't really exist in hdf5 file")
shape = column.shape
if True: #len(shape) == 1:
dtype = column.dtype
if "dtype" in column.attrs:
dtype = column.attrs["dtype"]
logger.debug("adding column %r with dtype %r", column_name, dtype)
self.addColumn(column_name, offset, len(column), dtype=dtype)
else:
#transposed = self._length is None or shape[0] == self._length
transposed = shape[1] < shape[0]
self.addRank1(column_name, offset, shape[1], length1=shape[0], dtype=column.dtype, stride=1, stride1=1, transposed=transposed)
#if len(shape[0]) == self._length:
# self.addRank1(column_name, offset, shape[1], length1=shape[0], dtype=column.dtype, stride=1, stride1=1)
#self.addColumn(column_name+"_0", offset, shape[1], dtype=column.dtype)
#self.addColumn(column_name+"_last", offset+(shape[0]-1)*shape[1]*column.dtype.itemsize, shape[1], dtype=column.dtype)
#self.addRank1(name, offset+8*i, length=self.numberParticles+1, length1=self.numberTimes-1, dtype=np.float64, stride=stride, stride1=1, filename=filename_extra)
finished.add(column_name)
def _load_columns(self, h5data, first=[]):
# print h5data
# make sure x y x etc are first
finished = set()
if "description" in h5data.attrs:
self.description = ensure_string(h5data.attrs["description"])
# hdf5, or h5py doesn't keep the order of columns, so manually track that, also enables reordering later
h5columns = h5data if self._version == 1 else h5data['columns']
if "column_order" in h5columns.attrs:
column_order = ensure_string(
h5columns.attrs["column_order"]).split(",")
else:
column_order = []
# for name in list(h5columns):
# if name not in column_order:
# column_order.append(name)
# for column_name in column_order:
# if column_name in h5columns and column_name not in finished:
for group_name in list(h5columns):
logger.debug('loading column: %s', group_name)
group = h5columns[group_name]
if 'type' in group.attrs:
type = group.attrs['type']
if type in ['csr_matrix']:
from scipy.sparse import csc_matrix, csr_matrix
class csr_matrix_nocheck(csr_matrix):
def check_format(self, *args, **kwargs):
pass
data = self._map_hdf5_array(group['data'])
indptr = self._map_hdf5_array(group['indptr'])
indices = self._map_hdf5_array(group['indices'])
#column_names = ensure_string(group.attrs["column_names"]).split(",")
# make sure we keep the original order
groups = [(name, value) for name, value in group.items()
if isinstance(value, h5py.Group)]
column_names = [None] * len(groups)
for name, column in groups:
column_names[column.attrs['column_index']] = name
matrix = csr_matrix_nocheck(
(data, indices, indptr),
shape=(len(indptr) - 1, len(column_names)))
assert matrix.data is data
# assert matrix.indptr is indptr
assert matrix.indices is indices
self.add_columns(column_names, matrix)
if type == 'dictionary_encoded':
index = self._map_column(group['indices'], as_arrow=True)
values = self._map_column(group['dictionary'],
as_arrow=True)
if 'null_bitmap' in group['indices'] or 'mask' in group[
'indices']:
raise ValueError(
f'Did not expect null data in encoded column {group_name}'
)
if isinstance(values, vaex.column.Column):
encoded = vaex.column.ColumnArrowDictionaryEncoded(
index, values)
else:
encoded = pa.DictionaryArray.from_arrays(index, values)
self.add_column(group_name, encoded)
else:
raise TypeError(
f'Unexpected type {type!r} in {group_name}')
else:
column_name = group_name
column = h5columns[column_name]
if "alias" in column.attrs:
column_name = column.attrs["alias"]
if "ucd" in column.attrs:
self.ucds[column_name] = ensure_string(column.attrs["ucd"])
if "description" in column.attrs:
self.descriptions[column_name] = ensure_string(
column.attrs["description"])
if "unit" in column.attrs:
try:
unitname = ensure_string(column.attrs["unit"])
if unitname and unitname != "None":
self.units[column_name] = _try_unit(unitname)
except:
logger.exception("error parsing unit: %s",
column.attrs["unit"])
if "units" in column.attrs: # Amuse case
unitname = ensure_string(column.attrs["units"])
logger.debug("amuse unit: %s", unitname)
if unitname == "(0.01 * system.get('S.I.').base('length'))":
self.units[column_name] = astropy.units.Unit("cm")
if unitname == "((0.01 * system.get('S.I.').base('length')) * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("cm/s")
if unitname == "(0.001 * system.get('S.I.').base('mass'))":
self.units[column_name] = astropy.units.Unit("gram")
if unitname == "system.get('S.I.').base('length')":
self.units[column_name] = astropy.units.Unit("m")
if unitname == "(system.get('S.I.').base('length') * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("m/s")
if unitname == "system.get('S.I.').base('mass')":
self.units[column_name] = astropy.units.Unit("kg")
if self._version == 1:
column = self._map_hdf5_array(column)
self.add_column(column_name, column)
elif hasattr(column["data"], "dtype"):
column = self._map_column(column)
self.add_column(column_name, column)
dtype = vaex.dtype_of(column)
logger.debug("adding column %r with dtype %r", column_name,
dtype)
else:
raise TypeError(f'{group_name} is missing dtype')
all_columns = dict(**self._columns)
# in case the column_order refers to non-existing columns
column_order = [k for k in column_order if k in all_columns]
column_names = []
self._columns = {}
for name in column_order:
self._columns[name] = all_columns.pop(name)
# add the rest
for name, col in all_columns.items():
self._columns[name] = col
def _load_columns(self, h5data, first=[]):
# print h5data
# make sure x y x etc are first
finished = set()
if "description" in h5data.attrs:
self.description = ensure_string(h5data.attrs["description"])
# hdf5, or h5py doesn't keep the order of columns, so manually track that, also enables reordering later
h5columns = h5data if self._version == 1 else h5data['columns']
if "column_order" in h5columns.attrs:
column_order = ensure_string(
h5columns.attrs["column_order"]).split(",")
else:
column_order = []
# for name in list(h5columns):
# if name not in column_order:
# column_order.append(name)
# for column_name in column_order:
# if column_name in h5columns and column_name not in finished:
for group_name in list(h5columns):
group = h5columns[group_name]
if 'type' in group.attrs:
if group.attrs['type'] in ['csr_matrix']:
from scipy.sparse import csc_matrix, csr_matrix
class csr_matrix_nocheck(csr_matrix):
def check_format(self, *args, **kwargs):
pass
data = self._map_hdf5_array(group['data'])
indptr = self._map_hdf5_array(group['indptr'])
indices = self._map_hdf5_array(group['indices'])
#column_names = ensure_string(group.attrs["column_names"]).split(",")
# make sure we keep the original order
groups = [(name, value) for name, value in group.items()
if isinstance(value, h5py.Group)]
column_names = [None] * len(groups)
for name, column in groups:
column_names[column.attrs['column_index']] = name
matrix = csr_matrix_nocheck(
(data, indices, indptr),
shape=(len(indptr) - 1, len(column_names)))
assert matrix.data is data
# assert matrix.indptr is indptr
assert matrix.indices is indices
self.add_columns(column_names, matrix)
else:
column_name = group_name
column = h5columns[column_name]
if "ucd" in column.attrs:
self.ucds[column_name] = ensure_string(column.attrs["ucd"])
if "description" in column.attrs:
self.descriptions[column_name] = ensure_string(
column.attrs["description"])
if "unit" in column.attrs:
try:
unitname = ensure_string(column.attrs["unit"])
if unitname and unitname != "None":
self.units[column_name] = _try_unit(unitname)
except:
logger.exception("error parsing unit: %s",
column.attrs["unit"])
if "units" in column.attrs: # Amuse case
unitname = ensure_string(column.attrs["units"])
logger.debug("amuse unit: %s", unitname)
if unitname == "(0.01 * system.get('S.I.').base('length'))":
self.units[column_name] = astropy.units.Unit("cm")
if unitname == "((0.01 * system.get('S.I.').base('length')) * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("cm/s")
if unitname == "(0.001 * system.get('S.I.').base('mass'))":
self.units[column_name] = astropy.units.Unit("gram")
if unitname == "system.get('S.I.').base('length')":
self.units[column_name] = astropy.units.Unit("m")
if unitname == "(system.get('S.I.').base('length') * (system.get('S.I.').base('time')**-1))":
self.units[column_name] = astropy.units.Unit("m/s")
if unitname == "system.get('S.I.').base('mass')":
self.units[column_name] = astropy.units.Unit("kg")
data = column if self._version == 1 else column['data']
if hasattr(data, "dtype"):
#logger.debug("adding column %r with dtype %r", column_name, dtype)
# print column, column.shape
# offset = data.id.get_offset()
# if offset is None:
# raise Exception("columns doesn't really exist in hdf5 file")
shape = data.shape
if True: # len(shape) == 1:
dtype = data.dtype
if "dtype" in data.attrs:
dtype = data.attrs["dtype"]
logger.debug("adding column %r with dtype %r",
column_name, dtype)
# self.addColumn(column_name, offset, len(data), dtype=dtype)
if self._version > 1 and 'mask' in column:
self.add_column(
column_name,
self._map_hdf5_array(data, column['mask']))
else:
self.add_column(column_name,
self._map_hdf5_array(data))
# mask = column['mask']
# offset = mask.id.get_offset()
# self.addColumn("temp_mask", offset, len(data), dtype=mask.dtype)
# mask_array = self.columns['temp_mask']
# del self.columns['temp_mask']
# self.column_names.remove('temp_mask')
# ar = self.columns[column_name] = np.ma.array(self.columns[column_name], mask=mask_array, shrink=False)
# assert ar.mask is mask_array, "masked array was copied"
else:
# transposed = self._length is None or shape[0] == self._length
transposed = shape[1] < shape[0]
self.addRank1(column_name,
offset,
shape[1],
length1=shape[0],
dtype=data.dtype,
stride=1,
stride1=1,
transposed=transposed)
# if len(shape[0]) == self._length:
# self.addRank1(column_name, offset, shape[1], length1=shape[0], dtype=column.dtype, stride=1, stride1=1)
# self.addColumn(column_name+"_0", offset, shape[1], dtype=column.dtype)
# self.addColumn(column_name+"_last", offset+(shape[0]-1)*shape[1]*column.dtype.itemsize, shape[1], dtype=column.dtype)
# self.addRank1(name, offset+8*i, length=self.numberParticles+1, length1=self.numberTimes-1, dtype=np.float64, stride=stride, stride1=1, filename=filename_extra)
# finished.add(column_name)
all_columns = dict(**self.columns)
# print(all_columns, column_order)
self.column_names = []
for name in column_order:
self.columns[name] = all_columns.pop(name)
self.column_names.append(name)
# add the rest
for name, col in all_columns.items():
self.columns[name] = col
self.column_names.append(name)
