def prepare_metadata(algo_category, burst=None, current_ga=GenericAttributes()):
"""
Gather generic_metadata from submitted fields and current to be execute algorithm.
Will populate STATE, GROUP, etc in generic_metadata
"""
generic_metadata = GenericAttributes()
generic_metadata.state = algo_category.defaultdatastate
generic_metadata.parent_burst = burst
generic_metadata.fill_from(current_ga)
return generic_metadata
def __init__(self, path):
# type: (str) -> None
self.path = path
storage_path, file_name = os.path.split(path)
self.storage_manager = HDF5StorageManager(storage_path, file_name)
# would be nice to have an opened state for the chunked api instead of the close_file=False
# common scalar headers
self.gid = Uuid(HasTraits.gid, self)
self.written_by = Scalar(Attr(str), self, name='written_by')
self.create_date = Scalar(Attr(str), self, name='create_date')
# Generic attributes descriptors
self.generic_attributes = GenericAttributes()
self.invalid = Scalar(Attr(bool), self, name='invalid')
self.is_nan = Scalar(Attr(bool), self, name='is_nan')
self.subject = Scalar(Attr(str), self, name='subject')
self.state = Scalar(Attr(str), self, name='state')
self.type = Scalar(Attr(str), self, name='type')
self.user_tag_1 = Scalar(Attr(str), self, name='user_tag_1')
self.user_tag_2 = Scalar(Attr(str), self, name='user_tag_2')
self.user_tag_3 = Scalar(Attr(str), self, name='user_tag_3')
self.user_tag_4 = Scalar(Attr(str), self, name='user_tag_4')
self.user_tag_5 = Scalar(Attr(str), self, name='user_tag_5')
self.visible = Scalar(Attr(bool), self, name='visible')
self.metadata_cache = None
if not self.storage_manager.is_valid_hdf5_file():
self.written_by.store(self.__class__.__module__ + '.' +
self.__class__.__name__)
self.is_new_file = True
def store(self, view_model, fname=None):
# type: (ViewModel, str) -> str
"""
Completely store any ViewModel object to the directory specified by self.base_dir.
Works recursively for view models that are serialized in multiple files (eg. SimulatorAdapterModel)
"""
if fname is None:
h5_path = self.path_for_has_traits(type(view_model), view_model.gid)
else:
h5_path = os.path.join(self.base_dir, fname)
with ViewModelH5(h5_path, view_model) as h5_file:
h5_file.store(view_model)
h5_file.type.store(self.get_class_path(view_model))
h5_file.create_date.store(date2string(datetime.now()))
if hasattr(view_model, "generic_attributes"):
h5_file.store_generic_attributes(view_model.generic_attributes)
else:
# For HasTraits not inheriting from ViewModel (e.g. Linear)
h5_file.store_generic_attributes(GenericAttributes())
references = h5_file.gather_references()
for trait_attr, gid in references:
if not gid:
continue
model_attr = getattr(view_model, trait_attr.field_name)
if isinstance(gid, list):
for idx, sub_gid in enumerate(gid):
self.store(model_attr[idx])
else:
self.store(model_attr)
return h5_path
def store_view_model(view_model, base_dir):
# type: (ViewModel, str) -> str
"""
Completely store any ViewModel object to the directory specified by base_dir.
It works recursively because there are view models that are serialized in multiple files (eg. SimulatorAdapterModel)
"""
h5_path = path_for(base_dir, ViewModelH5, view_model.gid,
type(view_model).__name__)
with ViewModelH5(h5_path, view_model) as h5_file:
h5_file.store(view_model)
h5_file.type.store(get_full_class_name(type(view_model)))
h5_file.create_date.store(date2string(datetime.now()))
if hasattr(view_model, "generic_attributes"):
h5_file.store_generic_attributes(view_model.generic_attributes)
else:
# For HasTraits not inheriting from ViewModel (e.g. Linear)
h5_file.store_generic_attributes(GenericAttributes())
references = h5_file.gather_references()
for trait_attr, gid in references:
if not gid:
continue
model_attr = getattr(view_model, trait_attr.field_name)
if isinstance(gid, list):
for idx, sub_gid in enumerate(gid):
store_view_model(model_attr[idx], base_dir)
else:
store_view_model(model_attr, base_dir)
return h5_path
def __import_time_series_csv_datatype(self, hrf_folder, connectivity_gid,
patient, user_tag):
path = os.path.join(hrf_folder, self.TIME_SERIES_CSV_FILE)
with open(path) as csv_file:
csv_reader = csv.reader(
csv_file, delimiter=CSVDelimiterOptionsEnum.COMMA.value)
ts = list(csv_reader)
ts_data = np.array(ts, dtype=np.float64).reshape(
(len(ts), 1, len(ts[0]), 1))
ts_time = np.random.rand(ts_data.shape[0], )
project = dao.get_project_by_id(self.current_project_id)
ts_gid = uuid.uuid4()
h5_path = "TimeSeries_{}.h5".format(ts_gid.hex)
operation_folder = self.storage_interface.get_project_folder(
project.name, str(self.operation_id))
h5_path = os.path.join(operation_folder, h5_path)
conn = h5.load_from_gid(connectivity_gid)
ts = TimeSeriesRegion()
ts.data = ts_data
ts.time = ts_time
ts.gid = ts_gid
ts.connectivity = conn
generic_attributes = GenericAttributes()
generic_attributes.user_tag_1 = user_tag
generic_attributes.state = DEFAULTDATASTATE_RAW_DATA
with TimeSeriesRegionH5(h5_path) as ts_h5:
ts_h5.store(ts)
ts_h5.nr_dimensions.store(4)
ts_h5.subject.store(patient)
ts_h5.store_generic_attributes(generic_attributes)
ts_index = TimeSeriesIndex()
ts_index.gid = ts_gid.hex
ts_index.fk_from_operation = self.operation_id
ts_index.time_series_type = "TimeSeriesRegion"
ts_index.data_length_1d = ts_data.shape[0]
ts_index.data_length_2d = ts_data.shape[1]
ts_index.data_length_3d = ts_data.shape[2]
ts_index.data_length_4d = ts_data.shape[3]
ts_index.data_ndim = len(ts_data.shape)
ts_index.sample_period_unit = 'ms'
ts_index.sample_period = TimeSeries.sample_period.default
ts_index.sample_rate = 1024.0
ts_index.subject = patient
ts_index.state = DEFAULTDATASTATE_RAW_DATA
ts_index.labels_ordering = json.dumps(
list(TimeSeries.labels_ordering.default))
ts_index.labels_dimensions = json.dumps(
TimeSeries.labels_dimensions.default)
ts_index.visible = False # we don't want to show these TimeSeries because they are dummy
dao.store_entity(ts_index)
return ts_gid
def __init__(self):
self.generic_attributes = GenericAttributes()
self.generic_attributes.subject = DataTypeMetaData.DEFAULT_SUBJECT
self.storage_interface = StorageInterface()
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.submitted_form = None
self.log = get_logger(self.__class__.__module__)
def store_complete_to_dir(datatype,
base_dir,
generic_attributes=GenericAttributes()):
h5_class = REGISTRY.get_h5file_for_datatype(datatype.__class__)
storage_path = path_by_dir(base_dir, h5_class, datatype.gid)
index_inst = __store_complete(datatype, storage_path, h5_class,
generic_attributes)
return index_inst
def store_complete(datatype,
op_id,
project_name,
generic_attributes=GenericAttributes()):
h5_class = REGISTRY.get_h5file_for_datatype(datatype.__class__)
storage_path = path_for(op_id, h5_class, datatype.gid, project_name)
index_inst = __store_complete(datatype, storage_path, h5_class,
generic_attributes)
return index_inst
def prepare_data_for_burst_copy(self, burst_config_id, burst_name_format, project):
burst_config = self.load_burst_configuration(burst_config_id)
burst_config_copy = burst_config.clone()
count = dao.count_bursts_with_name(burst_config.name, burst_config.fk_project)
burst_config_copy.name = burst_name_format.format(burst_config.name, count + 1)
storage_path = self.storage_interface.get_project_folder(project.name, str(burst_config.fk_simulation))
simulator = h5.load_view_model(burst_config.simulator_gid, storage_path)
simulator.generic_attributes = GenericAttributes()
return simulator, burst_config_copy
def _prepare_metadata(algo_category,
submit_data,
operation_group=None,
burst=None,
current_ga=GenericAttributes()):
"""
Gather generic_metadata from submitted fields and current to be execute algorithm.
Will populate STATE, GROUP, etc in generic_metadata
"""
generic_metadata = GenericAttributes()
generic_metadata.state = algo_category.defaultdatastate
generic_metadata.parent_burst = burst
if DataTypeMetaData.KEY_OPERATION_TAG in submit_data:
generic_metadata.operation_tag = submit_data[
DataTypeMetaData.KEY_OPERATION_TAG]
if DataTypeMetaData.KEY_TAG_1 in submit_data:
generic_metadata.user_tag_1 = submit_data[
DataTypeMetaData.KEY_TAG_1]
if operation_group is not None:
generic_metadata.user_tag_3 = operation_group.name
generic_metadata.fill_from(current_ga)
return generic_metadata
def __init__(self):
# It will be populate with key from DataTypeMetaData
self.meta_data = {
DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT
}
self.generic_attributes = GenericAttributes()
self.generic_attributes.subject = DataTypeMetaData.DEFAULT_SUBJECT
self.file_handler = FilesHelper()
self.storage_path = '.'
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.log = get_logger(self.__class__.__module__)
self.submitted_form = None
def build(analyzed_entity_index, analyzed_entity, operation, datatype_group, metrics='{"v": 3}'):
measure = DatatypeMeasureIndex()
measure.metrics = metrics
measure.source = analyzed_entity_index
measure.fk_from_operation = operation.id
measure.fk_datatype_group = datatype_group.id
measure = dao.store_entity(measure)
dm = DatatypeMeasure(analyzed_datatype=analyzed_entity, metrics=json.loads(metrics))
dm_path = h5.path_for_stored_index(measure)
with DatatypeMeasureH5(dm_path) as dm_h5:
dm_h5.store(dm)
dm_h5.store_generic_attributes(GenericAttributes())
return measure
def store_complete(datatype, base_dir):
# type: (HasTraits, str) -> DataType
"""
Stores the given HasTraits instance in a h5 file, and fill the Index entity for later storage in DB
"""
index_class = REGISTRY.get_index_for_datatype(datatype.__class__)
index_inst = index_class()
index_inst.fill_from_has_traits(datatype)
h5_class = REGISTRY.get_h5file_for_datatype(datatype.__class__)
storage_path = path_for(base_dir, h5_class, datatype.gid)
with h5_class(storage_path) as f:
f.store(datatype)
# Store empty Generic Attributes, in case the file is saved no through ABCAdapter it can still be used
f.store_generic_attributes(GenericAttributes())
return index_inst
def launch(self, view_model):
# type: (FooDataImporterModel) -> TimeSeriesIndex
array_data = numpy.loadtxt(view_model.array_data)
ts = TimeSeries(data=array_data)
ts.configure()
ts_index = TimeSeriesIndex()
ts_index.fill_from_has_traits(ts)
ts_h5_path = h5.path_for(self.storage_path, TimeSeriesH5, ts_index.gid)
with TimeSeriesH5(ts_h5_path) as ts_h5:
ts_h5.store(ts, scalars_only=True)
ts_h5.store_generic_attributes(GenericAttributes())
ts_h5.write_data_slice(array_data)
return ts_index
def _store(self, file, view_model):
file.store(view_model)
file.type.store(self.get_class_path(view_model))
file.create_date.store(date2string(datetime.now()))
if hasattr(view_model, "generic_attributes"):
file.store_generic_attributes(view_model.generic_attributes)
else:
# For HasTraits not inheriting from ViewModel (e.g. Linear)
file.store_generic_attributes(GenericAttributes())
references = file.gather_references()
for trait_attr, gid in references:
if not gid:
continue
model_attr = getattr(view_model, trait_attr.field_name)
if isinstance(gid, list):
for idx, sub_gid in enumerate(gid):
self.store(model_attr[idx])
else:
self.store(model_attr)
def build(data=None, op=None):
ts = time_series_factory(data)
if op is None:
op = operation_factory()
ts_db = TimeSeriesIndex()
ts_db.fk_from_operation = op.id
ts_db.fill_from_has_traits(ts)
ts_h5_path = h5.path_for_stored_index(ts_db)
with TimeSeriesH5(ts_h5_path) as f:
f.store(ts)
f.sample_rate.store(ts.sample_rate)
f.nr_dimensions.store(ts.data.ndim)
f.store_generic_attributes(GenericAttributes())
f.store_references(ts)
ts_db = dao.store_entity(ts_db)
return ts_db
def store(self, datatype, fname=None):
# type: (HasTraits, str) -> None
h5file_cls = self.registry.get_h5file_for_datatype(type(datatype))
if fname is None:
path = self.path_for(h5file_cls, datatype.gid)
else:
path = os.path.join(self.base_dir, fname)
sub_dt_refs = []
with h5file_cls(path) as f:
f.store(datatype)
# Store empty Generic Attributes, so that TVBLoader.load_complete_by_function can be still used
f.store_generic_attributes(GenericAttributes())
if self.recursive:
sub_dt_refs = f.gather_references()
for traited_attr, sub_gid in sub_dt_refs:
subdt = getattr(datatype, traited_attr.field_name)
if subdt is not None: # Because a non required reference may be not populated
self.store(subdt)
def __import_pearson_coefficients_datatype(self, fc_folder, patient,
user_tag, ts_gid):
path = os.path.join(fc_folder, self.FC_MAT_FILE)
result = ABCUploader.read_matlab_data(path, self.FC_DATASET_NAME)
result = result.reshape((result.shape[0], result.shape[1], 1, 1))
project = dao.get_project_by_id(self.current_project_id)
user = dao.get_user_by_id(project.fk_admin)
pearson_gid = uuid.uuid4()
h5_path = "CorrelationCoefficients_{}.h5".format(pearson_gid.hex)
operation_folder = self.storage_interface.get_project_folder(
project.name, str(self.operation_id))
h5_path = os.path.join(operation_folder, h5_path)
generic_attributes = GenericAttributes()
generic_attributes.user_tag_1 = user_tag
generic_attributes.state = DEFAULTDATASTATE_RAW_DATA
with CorrelationCoefficientsH5(h5_path) as pearson_correlation_h5:
pearson_correlation_h5.array_data.store(result)
pearson_correlation_h5.gid.store(pearson_gid)
pearson_correlation_h5.source.store(ts_gid)
pearson_correlation_h5.labels_ordering.store(
CorrelationCoefficients.labels_ordering.default)
pearson_correlation_h5.subject.store(patient)
pearson_correlation_h5.store_generic_attributes(generic_attributes)
pearson_correlation_index = CorrelationCoefficientsIndex()
pearson_correlation_index.gid = pearson_gid.hex
pearson_correlation_index.fk_from_operation = self.operation_id
pearson_correlation_index.subject = patient
pearson_correlation_index.state = DEFAULTDATASTATE_RAW_DATA
pearson_correlation_index.ndim = 4
pearson_correlation_index.fk_source_gid = ts_gid.hex # we need a random gid here to store the index
pearson_correlation_index.has_valid_time_series = False
dao.store_entity(pearson_correlation_index)
def __init__(self, path):
# type: (str) -> None
self.path = path
self.storage_manager = StorageInterface.get_storage_manager(self.path)
# would be nice to have an opened state for the chunked api instead of the close_file=False
# common scalar headers
self.gid = Uuid(HasTraits.gid, self)
self.written_by = Scalar(Attr(str), self, name=self.KEY_WRITTEN_BY)
self.create_date = Scalar(Attr(str), self, name='create_date')
self.type = Scalar(Attr(str), self, name='type')
# Generic attributes descriptors
self.generic_attributes = GenericAttributes()
self.invalid = Scalar(Attr(bool), self, name='invalid')
self.is_nan = Scalar(Attr(bool), self, name='is_nan')
self.subject = Scalar(Attr(str), self, name='subject')
self.state = Scalar(Attr(str), self, name='state')
self.user_tag_1 = Scalar(Attr(str), self, name='user_tag_1')
self.user_tag_2 = Scalar(Attr(str), self, name='user_tag_2')
self.user_tag_3 = Scalar(Attr(str), self, name='user_tag_3')
self.user_tag_4 = Scalar(Attr(str), self, name='user_tag_4')
self.user_tag_5 = Scalar(Attr(str), self, name='user_tag_5')
self.operation_tag = Scalar(Attr(str, required=False),
self,
name='operation_tag')
self.parent_burst = Uuid(Attr(uuid.UUID, required=False),
self,
name='parent_burst')
self.visible = Scalar(Attr(bool), self, name='visible')
self.metadata_cache = None
# Keep a list with datasets for which we should write metadata before closing the file
self.expandable_datasets = []
if not self.storage_manager.is_valid_tvb_file():
self.written_by.store(self.get_class_path())
self.is_new_file = True
def __init__(self, **kwargs):
super(ViewModel, self).__init__(**kwargs)
self.create_date = datetime.now()
self.generic_attributes = GenericAttributes()
def store_complete(self, datatype, generic_attributes=GenericAttributes()):
return h5.store_complete_to_dir(datatype, self.storage_path,
generic_attributes)
def store_complete(self, datatype, generic_attributes=GenericAttributes()):
project = dao.get_project_by_id(self.current_project_id)
return h5.store_complete(datatype, self.operation_id, project.name, generic_attributes)