def load_scalar_dataset(h_node):
_, base_type, data = get_type_and_data(h_node)
if (base_type == b'int'):
data = int(data)
return (data)
def load_pickled_data(h_node):
py_type, data = get_type_and_data(h_node)
try:
import cPickle as pickle
except ModuleNotFoundError:
import pickle
return pickle.loads(data[0])
def load_list_dataset(h_node):
_, _, data = get_type_and_data(h_node)
str_type = h_node.attrs.get('str_type', None)
if str_type == b'str':
return (np.array(data, copy=False, dtype=str).tolist())
else:
return (data.tolist())
def load_astropy_time_dataset(h_node):
py_type, data = get_type_and_data(h_node)
if six.PY3:
fmt = h_node.attrs["format"][0].decode('ascii')
scale = h_node.attrs["scale"][0].decode('ascii')
else:
fmt = h_node.attrs["format"][0]
scale = h_node.attrs["scale"][0]
q = Time(data, format=fmt, scale=scale)
return q
def load_astropy_time_dataset(h_node):
py_type, data = get_type_and_data(h_node)
if six.PY3:
fmt = h_node.attrs["format"][0].decode('ascii')
scale = h_node.attrs["scale"][0].decode('ascii')
else:
fmt = h_node.attrs["format"][0]
scale = h_node.attrs["scale"][0]
q = Time(data, format=fmt, scale=scale)
return q
def load_ndarray_masked_dataset(h_node):
_, _, data = get_type_and_data(h_node)
dtype = h_node.attrs['np_dtype']
try:
mask_path = h_node.name + "_mask"
h_root = h_node.parent
mask = h_root.get(mask_path)[:]
except (ValueError, IndexError):
mask = h_root.get(mask_path)
data = np.ma.array(data, mask=mask, dtype=dtype)
return data
def load_list_dataset(h_node):
py_type, data = get_type_and_data(h_node)
py3_str_type = get_py3_string_type(h_node)
if py3_str_type == b"<class 'bytes'>":
# Yuck. Convert numpy._bytes -> str -> bytes
return [bytes(str(item, 'utf8'), 'utf8') for item in data]
if py3_str_type == b"<class 'str'>":
return [str(item, 'utf8') for item in data]
else:
return list(data)
def load_astropy_table(h_node):
py_type, _, data = get_type_and_data(h_node)
metadata = dict(h_node.attrs.items())
metadata.pop('type')
metadata.pop('base_type')
metadata.pop('colnames')
colnames = [cn.decode('ascii') for cn in h_node.attrs["colnames"]]
t = py_type(data, names=colnames, meta=metadata)
return t
def load_python_dtype_dataset(h_node):
py_type, data = get_type_and_data(h_node)
subtype = h_node.attrs["python_subdtype"]
type_dict = {
"<type 'int'>": int,
"<type 'float'>": float,
"<type 'long'>": long,
"<type 'bool'>": bool,
"<type 'complex'>": complex
}
tcast = type_dict.get(subtype)
return tcast(data)
def load_ndarray_masked_dataset(h_node):
py_type, data = get_type_and_data(h_node)
try:
mask_path = h_node.name + "_mask"
h_root = h_node.parent
mask = h_root.get(mask_path)[:]
except IndexError:
mask = h_root.get(mask_path)
except ValueError:
mask = h_root.get(mask_path)
data = np.ma.array(data, mask=mask)
return data
def load_python_dtype_dataset(h_node):
py_type, data = get_type_and_data(h_node)
subtype = h_node.attrs["python_subdtype"]
type_dict = {
"<type 'int'>": int,
"<type 'float'>": float,
"<type 'long'>": long,
"<type 'bool'>": bool,
"<type 'complex'>": complex
}
tcast = type_dict.get(subtype)
return tcast(data)
def load_python_dtype_dataset(h_node):
py_type, data = get_type_and_data(h_node)
subtype = h_node.attrs["python_subdtype"]
type_dict = {
b"<class 'int'>": int,
b"<class 'float'>": float,
b"<class 'bool'>": bool,
b"<class 'complex'>": complex
}
tcast = type_dict.get(subtype)
return tcast(data)
def load_astropy_table(h_node):
py_type, data = get_type_and_data(h_node)
metadata = dict(h_node.attrs.items())
metadata.pop('type')
metadata.pop('colnames')
if six.PY3:
colnames = [cn.decode('ascii') for cn in h_node.attrs["colnames"]]
else:
colnames = h_node.attrs["colnames"]
t = Table(data, names=colnames, meta=metadata)
return t
def load_astropy_table(h_node):
py_type, data = get_type_and_data(h_node)
metadata = dict(h_node.attrs.items())
metadata.pop('type')
metadata.pop('colnames')
if six.PY3:
colnames = [cn.decode('ascii') for cn in h_node.attrs["colnames"]]
else:
colnames = h_node.attrs["colnames"]
t = Table(data, names=colnames, meta=metadata)
return t
def load_astropy_constant_dataset(h_node):
py_type, data = get_type_and_data(h_node)
unit = h_node.attrs["unit"][0]
abbrev = h_node.attrs["abbrev"][0]
name = h_node.attrs["name"][0]
ref = h_node.attrs["reference"][0]
unc = h_node.attrs["uncertainty"][0]
system = None
if "system" in h_node.attrs.keys():
system = h_node.attrs["system"][0]
c = Constant(abbrev, name, data, unit, unc, ref, system)
return c
def load_astropy_constant_dataset(h_node):
py_type, data = get_type_and_data(h_node)
unit = h_node.attrs["unit"][0]
abbrev = h_node.attrs["abbrev"][0]
name = h_node.attrs["name"][0]
ref = h_node.attrs["reference"][0]
unc = h_node.attrs["uncertainty"][0]
system = None
if "system" in h_node.attrs.keys():
system = h_node.attrs["system"][0]
c = Constant(abbrev, name, data, unit, unc, ref, system)
return c
def load_sparse_matrix_data(h_node):
py_type, data = get_type_and_data(h_node)
h_root = h_node.parent
indices = h_root.get('indices')[:]
indptr = h_root.get('indptr')[:]
shape = h_root.get('shape')[:]
if py_type == b'csc_matrix_data':
smat = sparse.csc_matrix((data, indices, indptr), dtype=data.dtype, shape=shape)
elif py_type == b'csr_matrix_data':
smat = sparse.csr_matrix((data, indices, indptr), dtype=data.dtype, shape=shape)
elif py_type == b'bsr_matrix_data':
smat = sparse.bsr_matrix((data, indices, indptr), dtype=data.dtype, shape=shape)
return smat
def load_astropy_angle_dataset(h_node):
py_type, data = get_type_and_data(h_node)
unit = h_node.attrs["unit"][0]
q = Angle(data, unit)
return q
def load_unicode_dataset(h_node):
py_type, data = get_type_and_data(h_node)
return unicode(data[0])
def load_astropy_quantity_dataset(h_node):
py_type, data = get_type_and_data(h_node)
unit = h_node.attrs["unit"][0]
q = Quantity(data, unit)
return q
def load_astropy_quantity_dataset(h_node):
py_type, _, data = get_type_and_data(h_node)
unit = h_node.attrs["unit"]
q = py_type(data, unit, copy=False)
return q
def load_astropy_time_dataset(h_node):
py_type, _, data = get_type_and_data(h_node)
fmt = h_node.attrs["format"].decode('ascii')
scale = h_node.attrs["scale"].decode('ascii')
q = py_type(data, format=fmt, scale=scale)
return q
def load_ndarray_dataset(h_node):
py_type, data = get_type_and_data(h_node)
return np.array(data, copy=False)
def load_pickled_data(h_node):
_, _, data = get_type_and_data(h_node)
return pickle.loads(data)
def load_string_dataset(h_node):
py_type, data = get_type_and_data(h_node)
return str(data[0])
def load_astropy_skycoord_dataset(h_node):
py_type, _, data = get_type_and_data(h_node)
lon_unit = h_node.attrs["lon_unit"]
lat_unit = h_node.attrs["lat_unit"]
q = py_type(data[..., 0], data[..., 1], unit=(lon_unit, lat_unit))
return q
def load_tuple_dataset(h_node):
py_type, data = get_type_and_data(h_node)
return tuple(data)
def load_np_scalar_dataset(h_node):
_, _, data = get_type_and_data(h_node)
return data
def load_np_dtype_dataset(h_node):
_, _, data = get_type_and_data(h_node)
data = np.dtype(data)
return data
def load_astropy_skycoord_dataset(h_node):
py_type, data = get_type_and_data(h_node)
lon_unit = h_node.attrs["lon_unit"][0]
lat_unit = h_node.attrs["lat_unit"][0]
q = SkyCoord(data[:, 0], data[:, 1], unit=(lon_unit, lat_unit))
return q
def load_np_dtype_dataset(h_node):
py_type, data = get_type_and_data(h_node)
data = np.dtype(data[0])
return data
def load_astropy_quantity_dataset(h_node):
py_type, data = get_type_and_data(h_node)
unit = h_node.attrs["unit"][0]
q = Quantity(data, unit)
return q
def load_unicode_dataset(h_node):
py_type, data = get_type_and_data(h_node)
return unicode(data[0])
def load_string_dataset(h_node):
py_type, data = get_type_and_data(h_node)
return str(data[0])
def load_bytes_dataset(h_node):
py_type, data = get_type_and_data(h_node)
return bytes(data[0])
def load_set_dataset(h_node):
py_type, data = get_type_and_data(h_node)
return set(data)
def load_astropy_angle_dataset(h_node):
py_type, data = get_type_and_data(h_node)
unit = h_node.attrs["unit"][0]
q = Angle(data, unit)
return q
def load_np_scalar_dataset(h_node):
py_type, data = get_type_and_data(h_node)
subtype = h_node.attrs["np_dtype"]
data = np.array([data], dtype=subtype)[0]
return data
def load_astropy_skycoord_dataset(h_node):
py_type, data = get_type_and_data(h_node)
lon_unit = h_node.attrs["lon_unit"][0]
lat_unit = h_node.attrs["lat_unit"][0]
q = SkyCoord(data[:,0], data[:, 1], unit=(lon_unit, lat_unit))
return q
def load_ndarray_dataset(h_node):
_, _, data = get_type_and_data(h_node)
dtype = h_node.attrs['np_dtype']
return np.array(data, copy=False, dtype=dtype)
def load_pickled_data(h_node):
py_type, data = get_type_and_data(h_node)
import dill as pickle
return pickle.loads(data[0])
