def test_change_filename_or_overwrite_always_overwrite(self):
filename = "Test.h5"
test_file = os.path.join(self.config.out.FOLDER_TEMP, filename)
self.writer.write_dictionary({"a": [1, 2, 3]}, test_file)
assert os.path.exists(test_file)
change_filename_or_overwrite(test_file, True)
assert not os.path.exists(test_file)
def write_surface(self, surface, path):
"""
:param surface: Surface object to write in H5
:param path: H5 path to be written
"""
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
h5_file.create_dataset(SurfaceH5Field.VERTICES, data=surface.vertices)
h5_file.create_dataset(SurfaceH5Field.TRIANGLES,
data=surface.triangles)
h5_file.create_dataset(SurfaceH5Field.VERTEX_NORMALS,
data=surface.vertex_normals)
h5_file.attrs.create(self.H5_TYPE_ATTRIBUTE, "Surface")
h5_file.attrs.create("Surface_subtype", surface.surface_subtype)
h5_file.attrs.create("Number_of_triangles", surface.triangles.shape[0])
h5_file.attrs.create("Number_of_vertices", surface.vertices.shape[0])
h5_file.attrs.create(
"Voxel_to_ras_matrix",
str(surface.vox2ras.flatten().tolist())[1:-1].replace(",", ""))
self.logger.info("Surface has been written to file: %s" % path)
h5_file.close()
def write_sensors(self, sensors, path):
"""
:param sensors: Sensors object to write in H5
:param path: H5 path to be written
"""
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
h5_file.create_dataset(SensorsH5Field.LABELS, data=sensors.labels)
h5_file.create_dataset(SensorsH5Field.LOCATIONS,
data=sensors.locations)
h5_file.create_dataset(SensorsH5Field.NEEDLES, data=sensors.needles)
gain_dataset = h5_file.create_dataset(SensorsH5Field.GAIN_MATRIX,
data=sensors.gain_matrix)
gain_dataset.attrs.create("Max", str(sensors.gain_matrix.max()))
gain_dataset.attrs.create("Min", str(sensors.gain_matrix.min()))
h5_file.attrs.create(self.H5_TYPE_ATTRIBUTE, "Sensors")
h5_file.attrs.create("Number_of_sensors",
str(sensors.number_of_sensors))
h5_file.attrs.create("Sensors_subtype", sensors.s_type)
self.logger.info("Sensors have been written to file: %s" % path)
h5_file.close()
def write_dictionary(self, dictionary, path):
"""
:param dictionary: dictionary to write in H5
:param path: H5 path to be written
"""
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
for key, value in dictionary.iteritems():
try:
if isinstance(value, numpy.ndarray) and value.size > 0:
h5_file.create_dataset(key, data=value)
else:
if isinstance(value, list) and len(value) > 0:
h5_file.create_dataset(key, data=value)
else:
h5_file.attrs.create(key, value)
except:
self.logger.warning("Did not manage to write " + key +
" to h5 file " + path + " !")
h5_file.attrs.create(self.H5_TYPE_ATTRIBUTE, "HypothesisModel")
h5_file.attrs.create(self.H5_SUBTYPE_ATTRIBUTE,
dictionary.__class__.__name__)
h5_file.close()
def write_model_inversion_service(self, model_inversion_service, path):
"""
:param model_inversion_service: ModelInversionService object to write in H5
:param path: H5 path to be written
"""
if getattr(model_inversion_service, "signals_inds", None) is not None:
model_inversion_service.signals_inds = numpy.array(
model_inversion_service.signals_inds)
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
datasets_dict, metadata_dict = self._determine_datasets_and_attributes(
model_inversion_service)
for key, value in datasets_dict.iteritems():
h5_file.create_dataset(key, data=value)
h5_file.attrs.create(self.H5_TYPE_ATTRIBUTE, "HypothesisModel")
h5_file.attrs.create(self.H5_SUBTYPE_ATTRIBUTE,
model_inversion_service.__class__.__name__)
for key, value in metadata_dict.iteritems():
h5_file.attrs.create(key, value)
h5_file.close()
def write_hypothesis(self, hypothesis, path):
"""
:param hypothesis: DiseaseHypothesis object to write in H5
:param path: H5 path to be written
"""
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
h5_file.create_dataset("x0_values", data=hypothesis.x0_values)
h5_file.create_dataset("e_values", data=hypothesis.e_values)
h5_file.create_dataset("w_values", data=hypothesis.w_values)
h5_file.create_dataset("lsa_propagation_strengths",
data=hypothesis.lsa_propagation_strengths)
# TODO: change HypothesisModel to GenericModel here and inside Epi
h5_file.attrs.create(self.H5_TYPE_ATTRIBUTE, "HypothesisModel")
h5_file.attrs.create(self.H5_SUBTYPE_ATTRIBUTE,
hypothesis.__class__.__name__)
h5_file.attrs.create("number_of_regions", hypothesis.number_of_regions)
h5_file.attrs.create("type", hypothesis.type)
h5_file.attrs.create("x0_indices", hypothesis.x0_indices)
h5_file.attrs.create("e_indices", hypothesis.e_indices)
h5_file.attrs.create("w_indices", hypothesis.w_indices)
h5_file.attrs.create("lsa_propagation_indices",
hypothesis.lsa_propagation_indices)
h5_file.close()
def write_ts_epi(self, raw_ts, sampling_period, path, source_ts=None):
path = change_filename_or_overwrite(os.path.join(path))
if raw_ts is None or len(raw_ts.squeezed.shape) != 3:
raise_value_error(
"Invalid TS data 3D (time, regions, sv) expected", self.logger)
self.logger.info("Writing a TS at:\n" + path)
if source_ts is None:
source_ts = raw_ts.source
h5_file = h5py.File(path, 'a', libver='latest')
h5_file.create_dataset("/data", data=raw_ts.squeezed)
h5_file.create_dataset("/lfpdata", data=source_ts.squeezed)
write_metadata({KEY_TYPE: "TimeSeries"}, h5_file, KEY_DATE,
KEY_VERSION)
write_metadata(
{
KEY_MAX: raw_ts.squeezed.max(),
KEY_MIN: raw_ts.squeezed.min(),
KEY_STEPS: raw_ts.squeezed.shape[0],
KEY_CHANNELS: raw_ts.squeezed.shape[1],
KEY_SV: raw_ts.squeezed.shape[2],
KEY_SAMPLING: sampling_period,
KEY_START: raw_ts.time_start
}, h5_file, KEY_DATE, KEY_VERSION, "/data")
write_metadata(
{
KEY_MAX: source_ts.squeezed.max(),
KEY_MIN: source_ts.squeezed.min(),
KEY_STEPS: source_ts.squeezed.shape[0],
KEY_CHANNELS: source_ts.squeezed.shape[1],
KEY_SV: 1,
KEY_SAMPLING: sampling_period,
KEY_START: source_ts.time_start
}, h5_file, KEY_DATE, KEY_VERSION, "/lfpdata")
h5_file.close()
def write_object_to_file(self,
path,
object,
h5_type_attribute="HypothesisModel",
nr_regions=None):
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
h5_file, _ = self._prepare_object_for_group(h5_file, object,
h5_type_attribute,
nr_regions)
h5_file.close()
def write_ts(self, raw_data, sampling_period, path):
path = change_filename_or_overwrite(path)
self.logger.info("Writing a TS at:\n" + path)
h5_file = h5py.File(path, 'a', libver='latest')
write_metadata({KEY_TYPE: "TimeSeries"}, h5_file, KEY_DATE,
KEY_VERSION)
if isinstance(raw_data, dict):
for data in raw_data:
if len(raw_data[data].shape) == 2 and str(
raw_data[data].dtype)[0] == "f":
h5_file.create_dataset("/" + data, data=raw_data[data])
write_metadata(
{
KEY_MAX: raw_data[data].max(),
KEY_MIN: raw_data[data].min(),
KEY_STEPS: raw_data[data].shape[0],
KEY_CHANNELS: raw_data[data].shape[1],
KEY_SV: 1,
KEY_SAMPLING: sampling_period,
KEY_START: 0.0
}, h5_file, KEY_DATE, KEY_VERSION, "/" + data)
else:
raise_value_error(
"Invalid TS data. 2D (time, nodes) numpy.ndarray of floats expected"
)
elif isinstance(raw_data, numpy.ndarray):
if len(raw_data.shape) != 2 and str(raw_data.dtype)[0] != "f":
h5_file.create_dataset("/data", data=raw_data)
write_metadata(
{
KEY_MAX: raw_data.max(),
KEY_MIN: raw_data.min(),
KEY_STEPS: raw_data.shape[0],
KEY_CHANNELS: raw_data.shape[1],
KEY_SV: 1,
KEY_SAMPLING: sampling_period,
KEY_START: 0.0
}, h5_file, KEY_DATE, KEY_VERSION, "/data")
else:
raise_value_error(
"Invalid TS data. 2D (time, nodes) numpy.ndarray of floats expected"
)
else:
raise_value_error(
"Invalid TS data. Dictionary or 2D (time, nodes) numpy.ndarray of floats expected"
)
h5_file.close()
def write_ts_epi(self, raw_data, sampling_period, path, lfp_data=None):
path = change_filename_or_overwrite(os.path.join(path))
if raw_data is None or len(raw_data.shape) != 3:
raise_value_error(
"Invalid TS data 3D (time, regions, sv) expected", self.logger)
self.logger.info("Writing a TS at:\n" + path)
if type(lfp_data) == int:
lfp_data = raw_data[:, :, lfp_data[1]]
raw_data[:, :, lfp_data[1]] = []
elif isinstance(lfp_data, list):
lfp_data = raw_data[:, :, lfp_data[1]] - raw_data[:, :,
lfp_data[0]]
elif isinstance(lfp_data, numpy.ndarray):
lfp_data = lfp_data.reshape(
(lfp_data.shape[0], lfp_data.shape[1], 1))
else:
raise_value_error(
"Invalid lfp_data 3D (time, regions, sv) expected",
self.logger)
h5_file = h5py.File(path, 'a', libver='latest')
h5_file.create_dataset("/data", data=raw_data)
h5_file.create_dataset("/lfpdata", data=lfp_data)
write_metadata({KEY_TYPE: "TimeSeries"}, h5_file, KEY_DATE,
KEY_VERSION)
write_metadata(
{
KEY_MAX: raw_data.max(),
KEY_MIN: raw_data.min(),
KEY_STEPS: raw_data.shape[0],
KEY_CHANNELS: raw_data.shape[1],
KEY_SV: raw_data.shape[2],
KEY_SAMPLING: sampling_period,
KEY_START: 0.0
}, h5_file, KEY_DATE, KEY_VERSION, "/data")
write_metadata(
{
KEY_MAX: lfp_data.max(),
KEY_MIN: lfp_data.min(),
KEY_STEPS: lfp_data.shape[0],
KEY_CHANNELS: lfp_data.shape[1],
KEY_SV: 1,
KEY_SAMPLING: sampling_period,
KEY_START: 0.0
}, h5_file, KEY_DATE, KEY_VERSION, "/lfpdata")
h5_file.close()
def write_to_h5(self, path):
"""
Store H5Model object to a hdf5 file
"""
final_path = change_filename_or_overwrite(path)
# final_path = ensure_unique_file(folder_name, file_name)
logger.info("Writing %s at: %s" % (self, final_path))
h5_file = h5py.File(final_path, 'a', libver='latest')
for attribute, field in self.datasets_dict.iteritems():
h5_file.create_dataset(attribute, data=field)
for meta, val in self.metadata_dict.iteritems():
dataset_path, attribute_name = os.path.split(meta)
if dataset_path == "":
h5_file.attrs.create(attribute_name, val)
else:
try:
h5_file[dataset_path].attrs.create(attribute_name, val)
except:
print("WTF")
h5_file.close()
def write_simulation_settings(self, simulation_settings, path):
"""
:param simulation_settings: SimulationSettings object to write in H5
:param path: H5 path to be written
"""
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
datasets_dict, metadata_dict = self._determine_datasets_and_attributes(
simulation_settings)
for key, value in datasets_dict.iteritems():
h5_file.create_dataset(key, data=value)
h5_file.attrs.create(self.H5_TYPE_ATTRIBUTE, "HypothesisModel")
h5_file.attrs.create(self.H5_SUBTYPE_ATTRIBUTE,
simulation_settings.__class__.__name__)
for key, value in metadata_dict.iteritems():
h5_file.attrs.create(key, value)
h5_file.close()
def write_lsa_service(self, lsa_service, path):
"""
:param lsa_service: LSAService object to write in H5
:param path: H5 path to be written
"""
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
datasets_dict, metadata_dict = self._determine_datasets_and_attributes(
lsa_service)
for key, value in datasets_dict.iteritems():
h5_file.create_dataset(key, data=value)
h5_file.attrs.create(self.H5_TYPE_ATTRIBUTE, "HypothesisModel")
h5_file.attrs.create(self.H5_SUBTYPE_ATTRIBUTE,
lsa_service.__class__.__name__)
for key, value in metadata_dict.iteritems():
h5_file.attrs.create(key, value)
h5_file.close()
def write_model_configuration(self, model_configuration, path):
"""
:param model_configuration: ModelConfiguration object to write in H5
:param path: H5 path to be written
"""
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
datasets_dict, metadata_dict = self._determine_datasets_and_attributes(
model_configuration)
for key, value in datasets_dict.iteritems():
h5_file.create_dataset(key, data=value)
h5_file.attrs.create(self.H5_TYPE_ATTRIBUTE, "HypothesisModel")
h5_file.attrs.create(self.H5_SUBTYPE_ATTRIBUTE,
model_configuration.__class__.__name__)
for key, value in metadata_dict.iteritems():
h5_file.attrs.create(key, value)
h5_file.close()
def write_connectivity(self, connectivity, path):
"""
:param connectivity: Connectivity object to be written in H5
:param path: H5 path to be written
"""
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
h5_file.create_dataset(ConnectivityH5Field.WEIGHTS,
data=connectivity.weights)
h5_file.create_dataset(ConnectivityH5Field.TRACTS,
data=connectivity.tract_lengths)
h5_file.create_dataset(ConnectivityH5Field.CENTERS,
data=connectivity.centres)
h5_file.create_dataset(ConnectivityH5Field.REGION_LABELS,
data=connectivity.region_labels)
h5_file.create_dataset(ConnectivityH5Field.ORIENTATIONS,
data=connectivity.orientations)
h5_file.create_dataset(ConnectivityH5Field.HEMISPHERES,
data=connectivity.hemispheres)
h5_file.attrs.create(self.H5_TYPE_ATTRIBUTE, "Connectivity")
h5_file.attrs.create("Number_of_regions",
str(connectivity.number_of_regions))
if connectivity.normalized_weights.size > 0:
dataset = h5_file.create_dataset(
"normalized_weights/" + ConnectivityH5Field.WEIGHTS,
data=connectivity.normalized_weights)
dataset.attrs.create(
"Operations",
"Removing diagonal, normalizing with 95th percentile, and ceiling to it"
)
self.logger.info("Connectivity has been written to file: %s" % path)
h5_file.close()
def write_probabilistic_model(self, probabilistic_model, nr_regions, path):
"""
:param object:
:param path:H5 path to be written
"""
def _set_parameter_to_group(parent_group,
parameter,
nr_regions,
param_name=None):
if param_name is None:
this_param_group = parent_group.create_group(parameter.name)
else:
this_param_group = parent_group.create_group(param_name)
this_param_group, parameter_subgroups = \
self._prepare_object_for_group(this_param_group, parameter, nr_regions=nr_regions)
for param_subgroup_key in parameter_subgroups:
if param_subgroup_key.find("p_shape") >= 0:
this_param_group[param_subgroup_key] = numpy.array(
getattr(param_value, param_subgroup_key))
elif param_subgroup_key == "star":
this_param_group, parameter_subgroup = \
_set_parameter_to_group(this_param_group, parameter.star, nr_regions, "star")
else:
parameter_subgroup = param_group.create_group(
param_subgroup_key)
parameter_subgroup, _ = self._prepare_object_for_group(
parameter_subgroup,
getattr(param_value, param_subgroup_key), nr_regions)
return parent_group, this_param_group
h5_file = h5py.File(change_filename_or_overwrite(path),
'a',
libver='latest')
datasets_dict, metadata_dict, groups_keys = self._determine_datasets_and_attributes(
probabilistic_model, nr_regions)
h5_file.attrs.create(self.H5_TYPE_ATTRIBUTE, "HypothesisModel")
h5_file.attrs.create(self.H5_SUBTYPE_ATTRIBUTE,
probabilistic_model.__class__.__name__)
self._write_dicts_at_location(datasets_dict, metadata_dict, h5_file)
for group_key in groups_keys:
if group_key == "active_regions":
h5_file.create_dataset(group_key,
data=numpy.array(
probabilistic_model.active_regions))
elif group_key == "parameters":
group = h5_file.create_group(group_key)
group.attrs.create(
self.H5_SUBTYPE_ATTRIBUTE,
probabilistic_model.parameters.__class__.__name__)
for param_key, param_value in probabilistic_model.parameters.iteritems(
):
group, param_group = _set_parameter_to_group(
group, param_value, nr_regions, param_key)
else:
group = h5_file.create_group(group_key)
group.attrs.create(
self.H5_SUBTYPE_ATTRIBUTE,
getattr(probabilistic_model, group_key).__class__.__name__)
group, _ = self._prepare_object_for_group(
group, getattr(probabilistic_model, group_key), nr_regions)
h5_file.close()
