def copy_dt_to_export_folder(self, data, data_export_folder):
data_path = h5.path_for_stored_index(data)
file_destination = os.path.join(data_export_folder,
os.path.basename(data_path))
if not os.path.exists(file_destination):
FilesHelper().copy_file(data_path, file_destination)
H5File.remove_metadata_param(file_destination, 'parent_burst')
return file_destination
def _edit_data(self, datatype, new_data, from_group=False):
# type: (DataType, dict, bool) -> None
"""
Private method, used for editing a meta-data XML file and a DataType row
for a given custom DataType entity with new dictionary of data from UI.
"""
# 1. First update Operation fields:
# Update group field if possible
new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG]
empty_group_value = (new_group_name is None or new_group_name == "")
if from_group:
if empty_group_value:
raise StructureException("Empty group is not allowed!")
group = dao.get_generic_entity(
OperationGroup,
new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if group and len(group) > 0 and new_group_name != group[0].name:
group = group[0]
exists_group = dao.get_generic_entity(OperationGroup,
new_group_name, 'name')
if exists_group:
raise StructureException("Group '" + new_group_name +
"' already exists.")
group.name = new_group_name
dao.store_entity(group)
else:
operation = dao.get_operation_by_id(datatype.fk_from_operation)
operation.user_group = new_group_name
dao.store_entity(operation)
# 2. Update GenericAttributes on DataType index and in the associated H5 files:
h5_path = h5.path_for_stored_index(datatype)
with H5File.from_file(h5_path) as f:
ga = f.load_generic_attributes()
ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT]
ga.state = new_data[DataTypeOverlayDetails.DATA_STATE]
if DataTypeOverlayDetails.DATA_TAG_1 in new_data:
ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1]
if DataTypeOverlayDetails.DATA_TAG_2 in new_data:
ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2]
if DataTypeOverlayDetails.DATA_TAG_3 in new_data:
ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3]
if DataTypeOverlayDetails.DATA_TAG_4 in new_data:
ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4]
if DataTypeOverlayDetails.DATA_TAG_5 in new_data:
ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5]
datatype.fill_from_generic_attributes(ga)
datatype = dao.store_entity(datatype)
# 3. Update MetaData in DT H5 as well.
with H5File.from_file(h5_path) as f:
f.store_generic_attributes(ga, False)
def _capture_operation_results(self, result):
"""
Update h5 files with generic attributes
"""
for file in os.listdir(self._get_output_path()):
path = os.path.join(self._get_output_path(), file)
if issubclass(H5File.h5_class_from_file(path), ViewModelH5):
continue
with H5File.from_file(path) as f:
f.store_generic_attributes(self.generic_attributes)
return "", 2
def gather_all_references_of_view_model(gid, base_dir, ref_files):
vm_path = determine_filepath(gid, base_dir)
ref_files.append(vm_path)
view_model_class = H5File.determine_type(vm_path)
view_model = view_model_class()
with ViewModelH5(vm_path, view_model) as vm_h5:
references = vm_h5.gather_references()
uuids = vm_h5.gather_references_by_uuid()
for _, gid in references:
if not gid:
continue
if isinstance(gid, (list, tuple)):
for list_gid in gid:
gather_all_references_of_view_model(
list_gid, base_dir, ref_files)
else:
gather_all_references_of_view_model(gid, base_dir, ref_files)
uuid_files = []
for _, gid in uuids:
if not gid:
continue
index = load_entity_by_gid(gid.hex)
h5_file = h5_file_for_index(index)
uuid_files.append(h5_file.path)
gather_all_references_by_index(h5_file, uuid_files)
ref_files.extend(uuid_files)
def load_view_model_from_file(filepath):
# type: (str) -> ViewModel
"""
Load a ViewModel object by reading the H5 file specified by filepath.
"""
base_dir = os.path.dirname(filepath)
view_model_class = H5File.determine_type(filepath)
view_model = view_model_class()
with ViewModelH5(filepath, view_model) as h5_file:
h5_file.load_into(view_model)
references = h5_file.gather_references()
view_model.create_date = string2date(h5_file.create_date.load())
view_model.generic_attributes = h5_file.load_generic_attributes()
for trait_attr, gid in references:
if not gid:
continue
if isinstance(gid, list):
loaded_ref = []
for idx, sub_gid in enumerate(gid):
ref = load_view_model(sub_gid, base_dir)
loaded_ref.append(ref)
else:
loaded_ref = load_view_model(gid, base_dir)
setattr(view_model, trait_attr.field_name, loaded_ref)
return view_model
def load_complete_by_function(self, file_path, load_ht_function):
# type: (str, callable) -> (HasTraits, GenericAttributes)
with H5File.from_file(file_path) as f:
try:
datatype_cls = self.registry.get_datatype_for_h5file(f)
except KeyError:
datatype_cls = f.determine_datatype_from_file()
datatype = datatype_cls()
f.load_into(datatype)
ga = f.load_generic_attributes()
sub_dt_refs = f.gather_references(datatype_cls)
for traited_attr, sub_gid in sub_dt_refs:
if sub_gid is None:
continue
is_monitor = False
if isinstance(sub_gid, list):
sub_gid = sub_gid[0]
is_monitor = True
ref_ht = load_ht_function(sub_gid, traited_attr)
if is_monitor:
ref_ht = [ref_ht]
setattr(datatype, traited_attr.field_name, ref_ht)
return datatype, ga
def load(self, gid=None, fname=None):
# type: (typing.Union[uuid.UUID, str], str) -> HasTraits
"""
Load from file a HasTraits entity. Either gid or fname should be given, or else an error is raised.
:param gid: optional entity GUID to search for it under self.base_dir
:param fname: optional file name to search for it under self.base_dir.
:return: HasTraits instance read from the given location
"""
if fname is None:
if gid is None:
raise ValueError("Neither gid nor filename is provided to load!")
fname = self.find_file_by_gid(gid)
sub_dt_refs = []
with H5File.from_file(fname) as f:
datatype_cls = self.registry.get_datatype_for_h5file(f)
datatype = datatype_cls()
f.load_into(datatype)
if self.recursive:
sub_dt_refs = f.gather_references()
for traited_attr, sub_gid in sub_dt_refs:
if sub_gid is not None:
subdt = self.load(sub_gid)
setattr(datatype, traited_attr.field_name, subdt)
return datatype
def load(self, gid=None, fname=None):
# type: (typing.Union[uuid.UUID, str], str) -> ViewModel
"""
Load a ViewModel object by reading the H5 file with the given GID, from the directory self.base_dir
"""
if fname is None:
if gid is None:
raise ValueError(
"Neither gid nor filename is provided to load!")
fname = self.find_file_by_gid(gid)
else:
fname = os.path.join(self.base_dir, fname)
view_model_class = H5File.determine_type(fname)
view_model = view_model_class()
has_traits_h5 = self.registry.get_h5file_for_datatype(
view_model.__class__)
if has_traits_h5 != H5File:
with has_traits_h5(fname) as file:
self._load(file, view_model)
else:
with ViewModelH5(fname, view_model) as h5_file:
self._load(h5_file, view_model)
return view_model
def gather_reference_files(self,
gid,
vm_ref_files,
dt_ref_files,
load_dts=None):
vm_path = self.find_file_by_gid(gid)
vm_ref_files.append(vm_path)
view_model_class = H5File.determine_type(vm_path)
view_model = view_model_class()
with ViewModelH5(vm_path, view_model) as vm_h5:
references = vm_h5.gather_references()
for _, gid in references:
if not gid:
continue
if isinstance(gid, (list, tuple)):
for list_gid in gid:
self.gather_reference_files(list_gid, vm_ref_files,
dt_ref_files, load_dts)
else:
self.gather_reference_files(gid, vm_ref_files,
dt_ref_files, load_dts)
if load_dts:
load_dts(vm_h5, dt_ref_files)
def _store_imported_datatypes_in_db(self, project, all_datatypes):
# type: (Project, dict) -> int
sorted_dts = sorted(
all_datatypes.items(),
key=lambda dt_item: dt_item[1].create_date or datetime.now())
count = 0
for dt_path, datatype in sorted_dts:
datatype_already_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_already_in_tvb:
self.store_datatype(datatype, dt_path)
count += 1
else:
AlgorithmService.create_link([datatype_already_in_tvb.id],
project.id)
file_path = h5.h5_file_for_index(datatype).path
h5_class = H5File.h5_class_from_file(file_path)
reference_list = h5_class(file_path).gather_references()
for _, reference_gid in reference_list:
if not reference_gid:
continue
ref_index = dao.get_datatype_by_gid(reference_gid.hex)
if ref_index is None:
os.remove(file_path)
dao.remove_entity(datatype.__class__, datatype.id)
raise MissingReferenceException(
'Imported file depends on datatypes that do not exist. Please upload '
'those first!')
return count
def copy_dt_to_export_folder(self, data, data_export_folder):
data_path = h5.path_for_stored_index(data)
with H5File.from_file(data_path) as f:
file_destination = os.path.join(data_export_folder,
os.path.basename(data_path))
if not os.path.exists(file_destination):
FilesHelper().copy_file(data_path, file_destination)
sub_dt_refs = f.gather_references()
for _, ref_gid in sub_dt_refs:
if ref_gid:
dt = load.load_entity_by_gid(ref_gid)
self.copy_dt_to_export_folder(dt, data_export_folder)
H5File.remove_metadata_param(file_destination, 'parent_burst')
def load(self, gid=None, fname=None):
# type: (typing.Union[uuid.UUID, str], str) -> ViewModel
"""
Load a ViewModel object by reading the H5 file with the given GID, from the directory self.base_dir
"""
if fname is None:
if gid is None:
raise ValueError("Neither gid nor filename is provided to load!")
fname = self.find_file_by_gid(gid)
else:
fname = os.path.join(self.base_dir, fname)
view_model_class = H5File.determine_type(fname)
view_model = view_model_class()
with ViewModelH5(fname, view_model) as h5_file:
h5_file.load_into(view_model)
references = h5_file.gather_references()
view_model.create_date = string2date(h5_file.create_date.load())
view_model.generic_attributes = h5_file.load_generic_attributes()
for trait_attr, gid in references:
if not gid:
continue
if isinstance(gid, list):
loaded_ref = []
for idx, sub_gid in enumerate(gid):
ref = self.load(sub_gid)
loaded_ref.append(ref)
else:
loaded_ref = self.load(gid)
setattr(view_model, trait_attr.field_name, loaded_ref)
return view_model
def set_visibility(dt):
""" set visibility flag, persist in db and h5"""
dt.visible = is_visible
dt = dao.store_entity(dt)
h5_path = h5.path_for_stored_index(dt)
with H5File.from_file(h5_path) as f:
f.visible.store(is_visible)
def export_simulator_configuration(self, burst_id):
burst = dao.get_burst_by_id(burst_id)
if burst is None:
raise InvalidExportDataException("Could not find burst with ID " +
str(burst_id))
op_folder = self.files_helper.get_project_folder(
burst.project, str(burst.fk_simulation))
tmp_export_folder = self._build_data_export_folder(burst)
tmp_sim_folder = os.path.join(tmp_export_folder,
self.EXPORTED_SIMULATION_NAME)
if not os.path.exists(tmp_sim_folder):
os.makedirs(tmp_sim_folder)
all_view_model_paths, all_datatype_paths = h5.gather_references_of_view_model(
burst.simulator_gid, op_folder)
burst_path = h5.determine_filepath(burst.gid, op_folder)
all_view_model_paths.append(burst_path)
for vm_path in all_view_model_paths:
dest = os.path.join(tmp_sim_folder, os.path.basename(vm_path))
self.files_helper.copy_file(vm_path, dest)
for dt_path in all_datatype_paths:
dest = os.path.join(tmp_sim_folder,
self.EXPORTED_SIMULATION_DTS_DIR,
os.path.basename(dt_path))
self.files_helper.copy_file(dt_path, dest)
main_vm_path = h5.determine_filepath(burst.simulator_gid,
tmp_sim_folder)
H5File.remove_metadata_param(main_vm_path, 'history_gid')
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, str(burst_id),
self.ZIP_FILE_EXTENSION)
result_path = os.path.join(tmp_export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
zip_file.write_folder(tmp_sim_folder)
self.files_helper.remove_folder(tmp_sim_folder)
return result_path
def load(self, source):
# type: (str) -> HasTraits
with H5File.from_file(source) as f:
datatype_cls = self.registry.get_datatype_for_h5file(type(f))
datatype = datatype_cls()
f.load_into(datatype)
return datatype
def update_written_by(folder):
for root, _, files in os.walk(folder):
for file_name in files:
if file_name.endswith(".h5"):
full_path = os.path.join(root, file_name)
with H5File(full_path) as f:
prev_h5_path = f.written_by.load()
new_h5_path = prev_h5_path.replace("tvb.core.entities.file.datatypes", "tvb.adapters.datatypes.h5")
f.written_by.store(new_h5_path)
def __gather_datatypes_for_copy(self, data, dt_path_list):
data_path = h5.path_for_stored_index(data)
dt_path_list.append(data_path)
with H5File.from_file(data_path) as f:
sub_dt_refs = f.gather_references()
for _, ref_gid in sub_dt_refs:
if ref_gid:
dt = load.load_entity_by_gid(ref_gid)
self.__gather_datatypes_for_copy(dt, dt_path_list)
def _capture_operation_results(self, result):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(
datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(
operation.fk_operation_group).id
burst_reference = None
if DataTypeMetaData.KEY_BURST in self.meta_data:
burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST]
count_stored = 0
group_type = None # In case of a group, the first not-none type is sufficient to memorize here
for res in result:
if res is None:
continue
res.subject = self.generic_attributes.subject
res.state = self.generic_attributes.state
res.fk_parent_burst = burst_reference
res.fk_from_operation = self.operation_id
res.framework_metadata = self.meta_data
res.user_tag_1 = self.generic_attributes.user_tag_1
res.user_tag_2 = self.generic_attributes.user_tag_2
res.fk_datatype_group = data_type_group_id
# Compute size-on disk, in case file-storage is used
associated_file = h5.path_for_stored_index(res)
if os.path.exists(associated_file):
res.disk_size = self.file_handler.compute_size_on_disk(
associated_file)
with H5File.from_file(associated_file) as f:
f.store_generic_attributes(self.generic_attributes)
dao.store_entity(res)
group_type = res.type
count_stored += 1
if count_stored > 0 and self._is_group_launch():
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(
operation.fk_operation_group)
operation_group.fill_operationgroup_name(group_type)
dao.store_entity(operation_group)
return 'Operation ' + str(
self.operation_id) + ' has finished.', count_stored
def _capture_operation_results(self, result):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(
datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(
operation.fk_operation_group).id
count_stored = 0
if result is None:
return "", count_stored
group_type = None # In case of a group, the first not-none type is sufficient to memorize here
for res in result:
if res is None:
continue
if not res.fixed_generic_attributes:
res.fill_from_generic_attributes(self.generic_attributes)
res.fk_from_operation = self.operation_id
res.fk_datatype_group = data_type_group_id
associated_file = h5.path_for_stored_index(res)
if os.path.exists(associated_file):
if not res.fixed_generic_attributes:
with H5File.from_file(associated_file) as f:
f.store_generic_attributes(self.generic_attributes)
# Compute size-on disk, in case file-storage is used
res.disk_size = self.storage_interface.compute_size_on_disk(
associated_file)
dao.store_entity(res)
res.after_store()
group_type = res.type
count_stored += 1
if count_stored > 0 and self._is_group_launch():
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(
operation.fk_operation_group)
operation_group.fill_operationgroup_name(group_type)
dao.store_entity(operation_group)
return 'Operation ' + str(
self.operation_id) + ' has finished.', count_stored
def load_from_reference(self, gid):
config_path = self.get_reference_path(gid)
config_h5 = H5File.from_file(config_path)
config_type = config_h5.type.load()
package, cls_name = config_type.rsplit('.', 1)
module = importlib.import_module(package)
config_class = getattr(module, cls_name)
config_instance = config_class()
config_h5.load_into(config_instance)
config_h5.close()
return config_instance
def check_import_references(file_path, datatype):
h5_class = H5File.h5_class_from_file(file_path)
reference_list = h5_class(file_path).gather_references()
for _, reference_gid in reference_list:
if not reference_gid:
continue
ref_index = load.load_entity_by_gid(reference_gid)
if ref_index is None:
os.remove(file_path)
dao.remove_entity(datatype.__class__, datatype.id)
raise MissingReferenceException(
'Imported file depends on datatypes that do not exist. Please upload '
'those first!')
def load_complete_by_function(self, file_path, load_ht_function):
# type: (str, callable) -> (HasTraits, GenericAttributes)
with H5File.from_file(file_path) as f:
datatype_cls = self.registry.get_datatype_for_h5file(type(f))
datatype = datatype_cls()
f.load_into(datatype)
ga = f.load_generic_attributes()
sub_dt_refs = f.gather_references()
for traited_attr, sub_gid in sub_dt_refs:
if sub_gid is None:
continue
ref_ht = load_ht_function(sub_gid, traited_attr)
setattr(datatype, traited_attr.field_name, ref_ht)
return datatype, ga
def load_with_references(self, file_path):
# type: (str) -> (HasTraits, GenericAttributes)
with H5File.from_file(file_path) as f:
datatype_cls = self.registry.get_datatype_for_h5file(type(f))
datatype = datatype_cls()
f.load_into(datatype)
ga = f.load_generic_attributes()
sub_dt_refs = f.gather_references()
for traited_attr, sub_gid in sub_dt_refs:
if sub_gid is None:
continue
ref_idx = dao.get_datatype_by_gid(sub_gid.hex, load_lazy=False)
ref_ht = self.load_from_index(ref_idx, traited_attr.field_type)
setattr(datatype, traited_attr.field_name, ref_ht)
return datatype, ga
def load_from_reference(self, gid):
dir_loader = h5.DirLoader(os.path.dirname(self.path), h5.REGISTRY)
config_filename = dir_loader.find_file_name(gid)
config_path = os.path.join(dir_loader.base_dir, config_filename)
config_h5 = H5File.from_file(config_path)
config_type = config_h5.type.load()
package, cls_name = config_type.rsplit('.', 1)
module = importlib.import_module(package)
config_class = getattr(module, cls_name)
config_instance = config_class()
config_h5.load_into(config_instance)
config_h5.close()
return config_instance
def load_datatype_from_file(self,
current_file,
op_id,
datatype_group=None,
current_project_id=None):
# type: (str, int, DataTypeGroup, int) -> HasTraitsIndex
"""
Creates an instance of datatype from storage / H5 file
:returns: DatatypeIndex
"""
self.logger.debug("Loading DataType from file: %s" % current_file)
h5_class = H5File.h5_class_from_file(current_file)
if h5_class is BurstConfigurationH5:
if current_project_id is None:
op_entity = dao.get_operationgroup_by_id(op_id)
current_project_id = op_entity.fk_launched_in
h5_file = BurstConfigurationH5(current_file)
burst = BurstConfiguration(current_project_id)
burst.fk_simulation = op_id
h5_file.load_into(burst)
result = burst
else:
datatype, generic_attributes = h5.load_with_links(current_file)
index_class = h5.REGISTRY.get_index_for_datatype(
datatype.__class__)
datatype_index = index_class()
datatype_index.fill_from_has_traits(datatype)
datatype_index.fill_from_generic_attributes(generic_attributes)
# Add all the required attributes
if datatype_group:
datatype_index.fk_datatype_group = datatype_group.id
if len(datatype_group.subject) == 0:
datatype_group.subject = datatype_index.subject
dao.store_entity(datatype_group)
datatype_index.fk_from_operation = op_id
associated_file = h5.path_for_stored_index(datatype_index)
if os.path.exists(associated_file):
datatype_index.disk_size = FilesHelper.compute_size_on_disk(
associated_file)
result = datatype_index
return result
def _add_links_for_datatype_references(datatype, fk_to_project, link_to_delete, existing_dt_links):
# If we found a datatype that has links, we need to link those as well to the linked project
# so they can be also copied
linked_datatype_paths = []
h5_file = h5.h5_file_for_index(datatype)
h5.gather_all_references_by_index(h5_file, linked_datatype_paths)
for h5_path in linked_datatype_paths:
if existing_dt_links is not None and h5_path in existing_dt_links:
continue
gid = H5File.get_metadata_param(h5_path, 'gid')
dt_index = h5.load_entity_by_gid(uuid.UUID(gid))
new_link = Links(dt_index.id, fk_to_project)
dao.store_entity(new_link)
dao.remove_entity(Links, link_to_delete)
return linked_datatype_paths
def load(self, gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
fname = self.find_file_name(gid)
sub_dt_refs = []
with H5File.from_file(os.path.join(self.base_dir, fname)) as f:
datatype_cls = self.registry.get_datatype_for_h5file(type(f))
datatype = datatype_cls()
f.load_into(datatype)
if self.recursive:
sub_dt_refs = f.gather_references()
for traited_attr, sub_gid in sub_dt_refs:
subdt = self.load(sub_gid)
setattr(datatype, traited_attr.field_name, subdt)
return datatype
def _edit_data(self, datatype, new_data, from_group=False):
# type: (DataType, dict, bool) -> None
"""
Private method, used for editing a meta-data XML file and a DataType row
for a given custom DataType entity with new dictionary of data from UI.
"""
# 1. First update Operation fields:
# Update group field if possible
new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG]
empty_group_value = (new_group_name is None or new_group_name == "")
if from_group:
if empty_group_value:
raise StructureException("Empty group is not allowed!")
group = dao.get_generic_entity(OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if group and len(group) > 0 and new_group_name != group[0].name:
group = group[0]
exists_group = dao.get_generic_entity(OperationGroup, new_group_name, 'name')
if exists_group:
raise StructureException("Group '" + new_group_name + "' already exists.")
group.name = new_group_name
dao.store_entity(group)
else:
operation = dao.get_operation_by_id(datatype.fk_from_operation)
operation.user_group = new_group_name
dao.store_entity(operation)
op_folder = self.structure_helper.get_project_folder(operation.project, str(operation.id))
vm_gid = operation.view_model_gid
view_model_file = h5.determine_filepath(vm_gid, op_folder)
if view_model_file:
view_model_class = H5File.determine_type(view_model_file)
view_model = view_model_class()
with ViewModelH5(view_model_file, view_model) as f:
ga = f.load_generic_attributes()
ga.operation_tag = new_group_name
f.store_generic_attributes(ga, False)
else:
self.logger.warning("Could not find ViewModel H5 file for op: {}".format(operation))
# 2. Update GenericAttributes in the associated H5 files:
h5_path = h5.path_for_stored_index(datatype)
with H5File.from_file(h5_path) as f:
ga = f.load_generic_attributes()
ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT]
ga.state = new_data[DataTypeOverlayDetails.DATA_STATE]
ga.operation_tag = new_group_name
if DataTypeOverlayDetails.DATA_TAG_1 in new_data:
ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1]
if DataTypeOverlayDetails.DATA_TAG_2 in new_data:
ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2]
if DataTypeOverlayDetails.DATA_TAG_3 in new_data:
ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3]
if DataTypeOverlayDetails.DATA_TAG_4 in new_data:
ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4]
if DataTypeOverlayDetails.DATA_TAG_5 in new_data:
ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5]
f.store_generic_attributes(ga, False)
# 3. Update MetaData in DT Index DB as well.
datatype.fill_from_generic_attributes(ga)
dao.store_entity(datatype)
def index_for_h5_file(source_path):
# type: (str) -> typing.Type[DataType]
""""""
h5_class = H5File.h5_class_from_file(source_path)
return REGISTRY.get_index_for_h5file(h5_class)
def _retrieve_operations_in_order(self, project, import_path):
# type: (Project, str) -> list[Operation2ImportData]
retrieved_operations = []
for root, _, files in os.walk(import_path):
if OPERATION_XML in files:
# Previous Operation format for uploading previous versions of projects
operation_file_path = os.path.join(root, OPERATION_XML)
operation, operation_xml_parameters = self.__build_operation_from_file(
project, operation_file_path)
operation.import_file = operation_file_path
self.logger.debug("Found operation in old XML format: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(
operation,
root,
info_from_xml=operation_xml_parameters))
else:
# We strive for the new format with ViewModelH5
main_view_model = None
dt_paths = []
all_view_model_files = []
for file in files:
if file.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
h5_file = os.path.join(root, file)
try:
h5_class = H5File.h5_class_from_file(h5_file)
if h5_class is ViewModelH5:
all_view_model_files.append(h5_file)
if not main_view_model:
view_model = h5.load_view_model_from_file(
h5_file)
if type(view_model
) in VIEW_MODEL2ADAPTER.keys():
main_view_model = view_model
else:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
dt_paths.append(h5_file)
except Exception:
self.logger.warning(
"Unreadable H5 file will be ignored: %s" %
h5_file)
if main_view_model is not None:
alg = VIEW_MODEL2ADAPTER[type(main_view_model)]
operation = Operation(main_view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
user_group=main_view_model.
generic_attributes.operation_tag,
start_date=datetime.now(),
completion_date=datetime.now())
operation.create_date = main_view_model.create_date
self.logger.debug(
"Found main ViewModel to create operation for it: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(operation, root, main_view_model,
dt_paths, all_view_model_files))
elif len(dt_paths) > 0:
alg = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE,
TVB_IMPORTER_CLASS)
default_adapter = ABCAdapter.build_adapter(alg)
view_model = default_adapter.get_view_model_class()()
view_model.data_file = dt_paths[0]
vm_path = h5.store_view_model(view_model, root)
all_view_model_files.append(vm_path)
operation = Operation(view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
start_date=datetime.now(),
completion_date=datetime.now())
self.logger.debug(
"Found no ViewModel in folder, so we default to " +
str(operation))
retrieved_operations.append(
Operation2ImportData(operation, root, view_model,
dt_paths, all_view_model_files,
True))
return sorted(retrieved_operations,
key=lambda op_data: op_data.order_field)
