def build():
time_series_index = time_series_index_factory()
time_series = h5.load_from_index(time_series_index)
data = numpy.random.random((10, 10))
covariance = graph.Covariance(source=time_series, array_data=data)
op = operation_factory()
covariance_index = CovarianceIndex()
covariance_index.fk_from_operation = op.id
covariance_index.fill_from_has_traits(covariance)
covariance_h5_path = h5.path_for_stored_index(covariance_index)
with CovarianceH5(covariance_h5_path) as f:
f.store(covariance)
covariance_index = dao.store_entity(covariance_index)
return covariance_index
def load_datatype_from_file(self,
current_file,
op_id,
datatype_group=None,
current_project_id=None):
# type: (str, int, DataTypeGroup, int) -> HasTraitsIndex
"""
Creates an instance of datatype from storage / H5 file
:returns: DatatypeIndex
"""
self.logger.debug("Loading DataType from file: %s" % current_file)
h5_class = H5File.h5_class_from_file(current_file)
if h5_class is BurstConfigurationH5:
if current_project_id is None:
op_entity = dao.get_operationgroup_by_id(op_id)
current_project_id = op_entity.fk_launched_in
h5_file = BurstConfigurationH5(current_file)
burst = BurstConfiguration(current_project_id)
burst.fk_simulation = op_id
# burst.fk_operation_group = TODO
# burst.fk_metric_operation_group = TODO
h5_file.load_into(burst)
result = burst
else:
datatype, generic_attributes = h5.load_with_links(current_file)
index_class = h5.REGISTRY.get_index_for_datatype(
datatype.__class__)
datatype_index = index_class()
datatype_index.fill_from_has_traits(datatype)
datatype_index.fill_from_generic_attributes(generic_attributes)
# Add all the required attributes
if datatype_group is not None:
datatype_index.fk_datatype_group = datatype_group.id
datatype_index.fk_from_operation = op_id
associated_file = h5.path_for_stored_index(datatype_index)
if os.path.exists(associated_file):
datatype_index.disk_size = FilesHelper.compute_size_on_disk(
associated_file)
result = datatype_index
return result
def __copy_linked_datatype_before_delete(self, op, datatype, project, fk_to_project):
new_op = Operation(op.view_model_gid,
dao.get_system_user().id,
fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op.visible = datatype.parent_operation.visible
new_op = dao.store_entity(new_op)
to_project = self.find_project(fk_to_project)
to_project_path = self.storage_interface.get_project_folder(to_project.name)
full_path = h5.path_for_stored_index(datatype)
old_folder = self.storage_interface.get_project_folder(project.name, str(op.id))
file_paths = h5.gather_references_of_view_model(op.view_model_gid, old_folder, only_view_models=True)[0]
file_paths.append(full_path)
# The BurstConfiguration h5 file has to be moved only when we handle the time series which has the operation
# folder containing the file
if datatype.is_ts and datatype.fk_parent_burst is not None:
bc_path = h5.path_for(datatype.parent_operation.id, BurstConfigurationH5, datatype.fk_parent_burst,
project.name)
if os.path.exists(bc_path):
file_paths.append(bc_path)
bc = dao.get_burst_for_operation_id(op.id)
bc.fk_simulation = new_op.id
dao.store_entity(bc)
# Move all files to the new operation folder
self.storage_interface.move_datatype_with_sync(to_project, to_project_path, new_op.id, file_paths)
datatype.fk_from_operation = new_op.id
datatype.parent_operation = new_op
dao.store_entity(datatype)
return new_op
def store_datatype(self, datatype, current_file=None):
"""This method stores data type into DB"""
try:
self.logger.debug("Store datatype: %s with Gid: %s" % (datatype.__class__.__name__, datatype.gid))
# Now move storage file into correct folder if necessary
if current_file is not None:
final_path = h5.path_for_stored_index(datatype)
if final_path != current_file:
shutil.move(current_file, final_path)
return dao.store_entity(datatype)
except MissingDataSetException as e:
self.logger.exception(e)
error_msg = "Datatype %s has missing data and could not be imported properly." % (datatype,)
raise ImportException(error_msg)
except IntegrityError as excep:
self.logger.exception(excep)
error_msg = "Could not import data with gid: %s. There is already a one with " \
"the same name or gid." % datatype.gid
raise ImportException(error_msg)
def build(data=None, op=None):
ts = time_series_factory(data)
if op is None:
op = operation_factory()
ts_db = TimeSeriesIndex()
ts_db.fk_from_operation = op.id
ts_db.fill_from_has_traits(ts)
ts_h5_path = h5.path_for_stored_index(ts_db)
with TimeSeriesH5(ts_h5_path) as f:
f.store(ts)
f.sample_rate.store(ts.sample_rate)
f.nr_dimensions.store(ts.data.ndim)
f.store_generic_attributes(GenericAttributes())
f.store_references(ts)
ts_db = dao.store_entity(ts_db)
return ts_db
def build():
time_series_index = time_series_index_factory()
time_series = h5.load_from_index(time_series_index)
cross_coherence = spectral.CoherenceSpectrum(source=time_series,
nfft=4,
array_data=numpy.random.random((10, 10)),
frequency=numpy.random.random((10,)))
op = operation_factory()
cross_coherence_index = CoherenceSpectrumIndex()
cross_coherence_index.fk_from_operation = op.id
cross_coherence_index.fill_from_has_traits(cross_coherence)
cross_coherence_h5_path = h5.path_for_stored_index(cross_coherence_index)
with CoherenceSpectrumH5(cross_coherence_h5_path) as f:
f.store(cross_coherence)
cross_coherence_index = dao.store_entity(cross_coherence_index)
return cross_coherence_index
def build():
time_series_index = time_series_index_factory()
time_series = h5.load_from_index(time_series_index)
data = numpy.random.random((10, 10, 10, 10, 10))
cross_correlation = temporal_correlations.CrossCorrelation(
source=time_series, array_data=data)
op = operation_factory()
cross_correlation_index = CrossCorrelationIndex()
cross_correlation_index.fk_from_operation = op.id
cross_correlation_index.fill_from_has_traits(cross_correlation)
cross_correlation_h5_path = h5.path_for_stored_index(
cross_correlation_index)
with CrossCorrelationH5(cross_correlation_h5_path) as f:
f.store(cross_correlation)
cross_correlation_index = dao.store_entity(cross_correlation_index)
return cross_correlation_index
def prepare_datatypes_for_export(data):
"""
Method used for exporting data type groups. This method returns a list of all datatype indexes needed to be
exported and a dictionary where keys are operation folder names and the values are lists containing the paths
that belong to one particular operation folder.
"""
all_datatypes = ProjectService.get_all_datatypes_from_data(data)
first_datatype = all_datatypes[0]
# We are exporting a group of datatype measures so we need to find the group of time series
if hasattr(first_datatype, 'fk_source_gid'):
ts = h5.load_entity_by_gid(first_datatype.fk_source_gid)
dt_metric_group = dao.get_datatypegroup_by_op_group_id(
ts.parent_operation.fk_operation_group)
datatype_measure_list = ProjectService.get_all_datatypes_from_data(
dt_metric_group)
all_datatypes = datatype_measure_list + all_datatypes
else:
ts_group = dao.get_datatype_measure_group_from_ts_from_pse(
first_datatype.gid, DatatypeMeasureIndex)
time_series_list = ProjectService.get_all_datatypes_from_data(
ts_group)
all_datatypes = all_datatypes + time_series_list
if all_datatypes is None or len(all_datatypes) == 0:
raise ExportException(
"Could not export a data type group with no data!")
op_file_dict = dict()
for dt in all_datatypes:
h5_path = h5.path_for_stored_index(dt)
op_folder = os.path.dirname(h5_path)
op_file_dict[op_folder] = [h5_path]
op = dao.get_operation_by_id(dt.fk_from_operation)
vms = h5.gather_references_of_view_model(op.view_model_gid,
os.path.dirname(h5_path),
only_view_models=True)
op_file_dict[op_folder].extend(vms[0])
return all_datatypes, op_file_dict
def build():
time = numpy.linspace(0, 1000, 4000)
data = numpy.zeros((time.size, 1, 3, 1))
data[:, 0, 0, 0] = numpy.sin(2 * numpy.pi * time / 1000.0 * 40)
data[:, 0, 1, 0] = numpy.sin(2 * numpy.pi * time / 1000.0 * 200)
data[:, 0, 2, 0] = numpy.sin(2 * numpy.pi * time / 1000.0 * 100) + numpy.sin(2 * numpy.pi * time / 1000.0 * 300)
ts = TimeSeries(time=time, data=data, sample_period=1.0 / 4000)
op = operation_factory()
ts_db = TimeSeriesIndex()
ts_db.fk_from_operation = op.id
ts_db.fill_from_has_traits(ts)
ts_h5_path = h5.path_for_stored_index(ts_db)
with TimeSeriesH5(ts_h5_path) as f:
f.store(ts)
session.add(ts_db)
session.commit()
return ts_db
def load_datatype_from_file(self,
storage_folder,
file_name,
op_id,
datatype_group=None,
move=True,
final_storage=None):
"""
Creates an instance of datatype from storage / H5 file
:returns: DatatypeIndex
"""
self.logger.debug("Loading DataType from file: %s" % file_name)
datatype, generic_attributes = h5.load_with_references(
os.path.join(storage_folder, file_name))
index_class = h5.REGISTRY.get_index_for_datatype(datatype.__class__)
datatype_index = index_class()
datatype_index.fill_from_has_traits(datatype)
datatype_index.fill_from_generic_attributes(generic_attributes)
# Add all the required attributes
if datatype_group is not None:
datatype_index.fk_datatype_group = datatype_group.id
datatype_index.fk_from_operation = op_id
associated_file = h5.path_for_stored_index(datatype_index)
if os.path.exists(associated_file):
datatype_index.disk_size = FilesHelper.compute_size_on_disk(
associated_file)
# Now move storage file into correct folder if necessary
if move and final_storage is not None:
current_file = os.path.join(storage_folder, file_name)
h5_type = h5.REGISTRY.get_h5file_for_datatype(datatype.__class__)
final_path = h5.path_for(final_storage, h5_type, datatype.gid)
if final_path != current_file and move:
shutil.move(current_file, final_path)
return datatype_index
def export(self, data, project, public_key_path, password):
"""
Exports data type:
1. If data is a normal data type, simply exports storage file (HDF format)
2. If data is a DataTypeGroup creates a zip with all files for all data types
"""
download_file_name = self._get_export_file_name(data)
if DataTypeGroup.is_data_a_group(data):
_, op_file_dict = self.prepare_datatypes_for_export(data)
# Create ZIP archive
zip_file = self.storage_interface.export_datatypes_structure(
op_file_dict, data, download_file_name, public_key_path,
password)
return download_file_name, zip_file, True
else:
data_path = h5.path_for_stored_index(data)
data_file = self.storage_interface.export_datatypes(
[data_path], data, download_file_name, public_key_path,
password)
return None, data_file, True
def build(row1=None,
row2=None,
project=None,
operation=None,
subject=None,
state=None):
data_type = dummy_datatype_factory()
data_type.row1 = row1
data_type.row2 = row2
if operation is None:
operation = operation_factory(test_project=project)
data_type_index = DummyDataTypeIndex(subject=subject, state=state)
data_type_index.fk_from_operation = operation.id
data_type_index.fill_from_has_traits(data_type)
data_type_h5_path = h5.path_for_stored_index(data_type_index)
with DummyDataTypeH5(data_type_h5_path) as f:
f.store(data_type)
data_type_index = dao.store_entity(data_type_index)
return data_type_index
def search_and_export_ts(project_id, export_folder=os.path.join("~", "TVB")):
# This is the simplest filter you could write: filter and entity by Subject
filter_connectivity = FilterChain(
fields=[FilterChain.datatype + '.subject'],
operations=["=="],
values=[DataTypeMetaData.DEFAULT_SUBJECT])
connectivities = _retrieve_entities_by_filters(ConnectivityIndex,
project_id,
filter_connectivity)
# A more complex filter: by linked entity (connectivity), saompling, operation date:
filter_timeseries = FilterChain(
fields=[
FilterChain.datatype + '.fk_connectivity_gid',
FilterChain.datatype + '.sample_period',
FilterChain.operation + '.create_date'
],
operations=["==", ">=", "<="],
values=[connectivities[0].gid, 0,
datetime.now()])
# If you want to filter another type of TS, change the kind class bellow,
# instead of TimeSeriesRegion use TimeSeriesEEG, or TimeSeriesSurface, etc.
timeseries = _retrieve_entities_by_filters(TimeSeriesRegionIndex,
project_id, filter_timeseries)
for ts in timeseries:
print("=============================")
print(ts.summary_info)
storage_h5 = h5.path_for_stored_index(ts)
print(" Original file: " + str(storage_h5))
destination_folder = os.path.expanduser(export_folder)
shutil.copy2(storage_h5, destination_folder)
print("File {0} exported in {1}".format(storage_h5,
destination_folder))
def load_with_references(dt_gid):
# type: (uuid.UUID) -> HasTraits
dt_index = load_entity_by_gid(dt_gid)
h5_path = h5.path_for_stored_index(dt_index)
dt, _ = h5.load_with_references(h5_path)
return dt
def load_with_references(self, dt_gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
dt_index = self.load_entity_by_gid(dt_gid)
h5_path = h5.path_for_stored_index(dt_index)
dt, _ = h5.load_with_references(h5_path)
return dt
def load_datatype_from_file(self,
current_file,
op_id,
datatype_group=None,
current_project_id=None):
# type: (str, int, DataTypeGroup, int) -> HasTraitsIndex
"""
Creates an instance of datatype from storage / H5 file
:returns: DatatypeIndex
"""
self.logger.debug("Loading DataType from file: %s" % current_file)
h5_class = H5File.h5_class_from_file(current_file)
if h5_class is BurstConfigurationH5:
if current_project_id is None:
op_entity = dao.get_operationgroup_by_id(op_id)
current_project_id = op_entity.fk_launched_in
h5_file = BurstConfigurationH5(current_file)
burst = BurstConfiguration(current_project_id)
burst.fk_simulation = op_id
h5_file.load_into(burst)
result = burst
else:
datatype, generic_attributes = h5.load_with_links(current_file)
already_existing_datatype = h5.load_entity_by_gid(datatype.gid)
if datatype_group is not None and already_existing_datatype is not None:
raise DatatypeGroupImportException(
"The datatype group that you are trying to import"
" already exists!")
index_class = h5.REGISTRY.get_index_for_datatype(
datatype.__class__)
datatype_index = index_class()
datatype_index.fill_from_has_traits(datatype)
datatype_index.fill_from_generic_attributes(generic_attributes)
if datatype_group is not None and hasattr(datatype_index, 'fk_source_gid') and \
datatype_index.fk_source_gid is not None:
ts = h5.load_entity_by_gid(datatype_index.fk_source_gid)
if ts is None:
op = dao.get_operations_in_group(
datatype_group.fk_operation_group,
only_first_operation=True)
op.fk_operation_group = None
dao.store_entity(op)
dao.remove_entity(OperationGroup,
datatype_group.fk_operation_group)
dao.remove_entity(DataTypeGroup, datatype_group.id)
raise DatatypeGroupImportException(
"Please import the time series group before importing the"
" datatype measure group!")
# Add all the required attributes
if datatype_group:
datatype_index.fk_datatype_group = datatype_group.id
if len(datatype_group.subject) == 0:
datatype_group.subject = datatype_index.subject
dao.store_entity(datatype_group)
datatype_index.fk_from_operation = op_id
associated_file = h5.path_for_stored_index(datatype_index)
if os.path.exists(associated_file):
datatype_index.disk_size = StorageInterface.compute_size_on_disk(
associated_file)
result = datatype_index
return result
def _load_h5_of_gid(self, entity_gid):
entity_index = self.load_entity_by_gid(entity_gid)
entity_h5_class = h5.REGISTRY.get_h5file_for_index(type(entity_index))
entity_h5_path = h5.path_for_stored_index(entity_index)
return entity_h5_class, entity_h5_path
def launch(self, simulator_gid):
"""
Called from the GUI to launch a simulation.
*: string class name of chosen model, etc...
*_parameters: dictionary of parameters for chosen model, etc...
connectivity: tvb.datatypes.connectivity.Connectivity object.
surface: tvb.datatypes.surfaces.CorticalSurface: or None.
stimulus: tvb.datatypes.patters.* object
"""
result_h5 = dict()
result_indexes = dict()
start_time = self.algorithm.current_step * self.algorithm.integrator.dt
self.algorithm.configure(full_configure=False)
if self.branch_simulation_state_gid is not None:
simulation_state_index = dao.get_datatype_by_gid(
self.branch_simulation_state_gid.hex)
self.branch_simulation_state_path = h5.path_for_stored_index(
simulation_state_index)
with SimulationStateH5(self.branch_simulation_state_path
) as branch_simulation_state_h5:
branch_simulation_state_h5.load_into(self.algorithm)
region_map, region_volume_map = self._try_load_region_mapping()
for monitor in self.algorithm.monitors:
m_name = monitor.__class__.__name__
ts = monitor.create_time_series(self.algorithm.connectivity,
self.algorithm.surface, region_map,
region_volume_map)
self.log.debug("Monitor created the TS")
ts.start_time = start_time
ts_index_class = h5.REGISTRY.get_index_for_datatype(type(ts))
ts_index = ts_index_class()
ts_index.fill_from_has_traits(ts)
ts_index.data_ndim = 4
ts_index.state = 'INTERMEDIATE'
# state_variable_dimension_name = ts.labels_ordering[1]
# if ts_index.user_tag_1:
# ts_index.labels_dimensions[state_variable_dimension_name] = ts.user_tag_1.split(';')
# elif m_name in self.HAVE_STATE_VARIABLES:
# selected_vois = [self.algorithm.model.variables_of_interest[idx] for idx in monitor.voi]
# ts.labels_dimensions[state_variable_dimension_name] = selected_vois
ts_h5_class = h5.REGISTRY.get_h5file_for_datatype(type(ts))
ts_h5_path = h5.path_for(self.storage_path, ts_h5_class, ts.gid)
ts_h5 = ts_h5_class(ts_h5_path)
ts_h5.store(ts, scalars_only=True, store_references=False)
ts_h5.sample_rate.store(ts.sample_rate)
ts_h5.nr_dimensions.store(ts_index.data_ndim)
if self.algorithm.surface:
ts_index.surface_gid = self.algorithm.surface.region_mapping_data.surface.gid.hex
ts_h5.surface.store(self.algorithm.surface.gid)
else:
ts_index.connectivity_gid = self.algorithm.connectivity.gid.hex
ts_h5.connectivity.store(self.algorithm.connectivity.gid)
if region_map:
ts_index.region_mapping_gid = region_map.gid.hex
ts_h5.region_mapping.store(region_map.gid)
if region_volume_map:
ts_index.region_mapping_volume_gid = region_volume_map.gid.hex
ts_h5.region_mapping_volume.store(region_volume_map.gid)
result_indexes[m_name] = ts_index
result_h5[m_name] = ts_h5
# Run simulation
self.log.debug("Starting simulation...")
for result in self.algorithm(simulation_length=self.simulation_length):
for j, monitor in enumerate(self.algorithm.monitors):
if result[j] is not None:
m_name = monitor.__class__.__name__
ts_h5 = result_h5[m_name]
ts_h5.write_time_slice([result[j][0]])
ts_h5.write_data_slice([result[j][1]])
self.log.debug(
"Completed simulation, starting to store simulation state ")
# Populate H5 file for simulator state. This step could also be done while running sim, in background.
if not self._is_group_launch():
simulation_state_index = SimulationStateIndex()
simulation_state_path = h5.path_for(self.storage_path,
SimulationStateH5,
self.algorithm.gid)
with SimulationStateH5(
simulation_state_path) as simulation_state_h5:
simulation_state_h5.store(self.algorithm)
self._capture_operation_results([simulation_state_index])
self.log.debug("Simulation state persisted, returning results ")
for monitor in self.algorithm.monitors:
m_name = monitor.__class__.__name__
ts_shape = result_h5[m_name].read_data_shape()
result_indexes[m_name].fill_shape(ts_shape)
result_h5[m_name].close()
# self.log.info("%s: Adapter simulation finished!!" % str(self))
return list(result_indexes.values())
def load_with_references(self, dt_gid, dt_class=None):
# type: (typing.Union[uuid.UUID, str], typing.Type[HasTraits]) -> HasTraits
dt_index = ABCAdapter.load_entity_by_gid(dt_gid)
h5_path = h5.path_for_stored_index(dt_index)
dt, _ = h5.load_with_references(h5_path)
return dt
def _remove_project_node_files(self, project_id, gid, skip_validation=False):
"""
Delegate removal of a node in the structure of the project.
In case of a problem will THROW StructureException.
"""
try:
project = self.find_project(project_id)
datatype = dao.get_datatype_by_gid(gid)
links = dao.get_links_for_datatype(datatype.id)
op = dao.get_operation_by_id(datatype.fk_from_operation)
adapter = ABCAdapter.build_adapter(op.algorithm)
if links:
was_link = False
for link in links:
# This means it's only a link and we need to remove it
if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id:
dao.remove_entity(Links, link.id)
was_link = True
if not was_link:
# Create a clone of the operation
# There is no view_model so the view_model_gid is None
new_op = Operation(op.view_model_gid,
dao.get_system_user().id,
links[0].fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op = dao.store_entity(new_op)
to_project = self.find_project(links[0].fk_to_project)
to_project_path = self.structure_helper.get_project_folder(to_project)
encryption_handler.set_project_active(to_project)
encryption_handler.sync_folders(to_project_path)
to_project_name = to_project.name
full_path = h5.path_for_stored_index(datatype)
self.structure_helper.move_datatype(datatype, to_project_name, str(new_op.id), full_path)
# Move also the ViewModel H5
old_folder = self.structure_helper.get_project_folder(project, str(op.id))
view_model = adapter.load_view_model(op)
vm_full_path = h5.determine_filepath(op.view_model_gid, old_folder)
self.structure_helper.move_datatype(view_model, to_project_name, str(new_op.id), vm_full_path)
encryption_handler.sync_folders(to_project_path)
encryption_handler.set_project_inactive(to_project)
datatype.fk_from_operation = new_op.id
datatype.parent_operation = new_op
dao.store_entity(datatype)
dao.remove_entity(Links, links[0].id)
else:
specific_remover = get_remover(datatype.type)(datatype)
specific_remover.remove_datatype(skip_validation)
h5_path = h5.path_for_stored_index(datatype)
self.structure_helper.remove_datatype_file(h5_path)
encryption_handler.push_folder_to_sync(self.structure_helper.get_project_folder_from_h5(h5_path))
except RemoveDataTypeException:
self.logger.exception("Could not execute operation Node Remove!")
raise
except FileStructureException:
self.logger.exception("Remove operation failed")
raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.")
def _edit_data(self, datatype, new_data, from_group=False):
# type: (DataType, dict, bool) -> None
"""
Private method, used for editing a meta-data XML file and a DataType row
for a given custom DataType entity with new dictionary of data from UI.
"""
# 1. First update Operation fields:
# Update group field if possible
new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG]
empty_group_value = (new_group_name is None or new_group_name == "")
if from_group:
if empty_group_value:
raise StructureException("Empty group is not allowed!")
group = dao.get_generic_entity(OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if group and len(group) > 0 and new_group_name != group[0].name:
group = group[0]
exists_group = dao.get_generic_entity(OperationGroup, new_group_name, 'name')
if exists_group:
raise StructureException("Group '" + new_group_name + "' already exists.")
group.name = new_group_name
dao.store_entity(group)
else:
operation = dao.get_operation_by_id(datatype.fk_from_operation)
operation.user_group = new_group_name
dao.store_entity(operation)
op_folder = self.structure_helper.get_project_folder(operation.project, str(operation.id))
vm_gid = operation.view_model_gid
view_model_file = h5.determine_filepath(vm_gid, op_folder)
if view_model_file:
view_model_class = H5File.determine_type(view_model_file)
view_model = view_model_class()
with ViewModelH5(view_model_file, view_model) as f:
ga = f.load_generic_attributes()
ga.operation_tag = new_group_name
f.store_generic_attributes(ga, False)
else:
self.logger.warning("Could not find ViewModel H5 file for op: {}".format(operation))
# 2. Update GenericAttributes in the associated H5 files:
h5_path = h5.path_for_stored_index(datatype)
with H5File.from_file(h5_path) as f:
ga = f.load_generic_attributes()
ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT]
ga.state = new_data[DataTypeOverlayDetails.DATA_STATE]
ga.operation_tag = new_group_name
if DataTypeOverlayDetails.DATA_TAG_1 in new_data:
ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1]
if DataTypeOverlayDetails.DATA_TAG_2 in new_data:
ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2]
if DataTypeOverlayDetails.DATA_TAG_3 in new_data:
ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3]
if DataTypeOverlayDetails.DATA_TAG_4 in new_data:
ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4]
if DataTypeOverlayDetails.DATA_TAG_5 in new_data:
ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5]
f.store_generic_attributes(ga, False)
# 3. Update MetaData in DT Index DB as well.
datatype.fill_from_generic_attributes(ga)
dao.store_entity(datatype)
def launch(self, view_model):
# type: (TVBImporterModel) -> []
"""
Execute import operations: unpack ZIP, build and store generic DataType objects.
:raises LaunchException: when data_file is None, nonexistent, or invalid \
(e.g. incomplete meta-data, not in ZIP / HDF5 format etc. )
"""
if view_model.data_file is None:
raise LaunchException(
"Please select file which contains data to import")
service = ImportService()
if os.path.exists(view_model.data_file):
current_op = dao.get_operation_by_id(self.operation_id)
if zipfile.is_zipfile(view_model.data_file):
# Creates a new TMP folder where to extract data
tmp_folder = os.path.join(self.storage_path, "tmp_import")
self.storage_interface.unpack_zip(view_model.data_file,
tmp_folder)
is_group = False
current_op_id = current_op.id
for file in os.listdir(tmp_folder):
# In case we import a DatatypeGroup, we want the default import flow
if os.path.isdir(os.path.join(tmp_folder, file)):
current_op_id = None
is_group = True
break
try:
operations, all_dts, stored_dts_count = service.import_project_operations(
current_op.project, tmp_folder, is_group,
current_op_id)
self.nr_of_datatypes += stored_dts_count
if stored_dts_count == 0:
current_op.additional_info = 'All chosen datatypes already exist!'
dao.store_entity(current_op)
elif stored_dts_count < all_dts:
current_op.additional_info = 'Part of the chosen datatypes already exist!'
dao.store_entity(current_op)
except ImportException as excep:
self.log.exception(excep)
current_op.additional_info = excep.message
current_op.status = STATUS_ERROR
raise LaunchException("Invalid file received as input. " +
str(excep))
finally:
shutil.rmtree(tmp_folder)
else:
# upgrade file if necessary
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(view_model.data_file)
if self.storage_interface.get_storage_manager(
view_model.data_file).is_valid_tvb_file():
datatype = None
try:
datatype = service.load_datatype_from_file(
view_model.data_file, self.operation_id)
stored_new_dt = service.store_or_link_datatype(
datatype, view_model.data_file,
current_op.project.id)
if stored_new_dt == 0:
current_op.additional_info = 'The chosen datatype already exists!'
dao.store_entity(current_op)
self.nr_of_datatypes += stored_new_dt
except ImportException as excep:
self.log.exception(excep)
if datatype is not None:
target_path = h5.path_for_stored_index(datatype)
if os.path.exists(target_path):
os.remove(target_path)
raise LaunchException(
"Invalid file received as input. " + str(excep))
else:
raise LaunchException(
"Uploaded file: %s is neither in ZIP or HDF5 format" %
view_model.data_file)
else:
raise LaunchException("File: %s to import does not exists." %
view_model.data_file)
