def _edit_data(self, datatype, new_data, from_group=False):
# type: (DataType, dict, bool) -> None
"""
Private method, used for editing a meta-data XML file and a DataType row
for a given custom DataType entity with new dictionary of data from UI.
"""
# 1. First update Operation fields:
# Update group field if possible
new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG]
empty_group_value = (new_group_name is None or new_group_name == "")
if from_group:
if empty_group_value:
raise StructureException("Empty group is not allowed!")
group = dao.get_generic_entity(
OperationGroup,
new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if group and len(group) > 0 and new_group_name != group[0].name:
group = group[0]
exists_group = dao.get_generic_entity(OperationGroup,
new_group_name, 'name')
if exists_group:
raise StructureException("Group '" + new_group_name +
"' already exists.")
group.name = new_group_name
dao.store_entity(group)
else:
operation = dao.get_operation_by_id(datatype.fk_from_operation)
operation.user_group = new_group_name
dao.store_entity(operation)
# 2. Update GenericAttributes on DataType index and in the associated H5 files:
h5_path = h5.path_for_stored_index(datatype)
with H5File.from_file(h5_path) as f:
ga = f.load_generic_attributes()
ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT]
ga.state = new_data[DataTypeOverlayDetails.DATA_STATE]
if DataTypeOverlayDetails.DATA_TAG_1 in new_data:
ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1]
if DataTypeOverlayDetails.DATA_TAG_2 in new_data:
ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2]
if DataTypeOverlayDetails.DATA_TAG_3 in new_data:
ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3]
if DataTypeOverlayDetails.DATA_TAG_4 in new_data:
ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4]
if DataTypeOverlayDetails.DATA_TAG_5 in new_data:
ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5]
datatype.fill_from_generic_attributes(ga)
datatype = dao.store_entity(datatype)
# 3. Update MetaData in DT H5 as well.
with H5File.from_file(h5_path) as f:
f.store_generic_attributes(ga, False)
def load_complete_by_function(self, file_path, load_ht_function):
# type: (str, callable) -> (HasTraits, GenericAttributes)
with H5File.from_file(file_path) as f:
try:
datatype_cls = self.registry.get_datatype_for_h5file(f)
except KeyError:
datatype_cls = f.determine_datatype_from_file()
datatype = datatype_cls()
f.load_into(datatype)
ga = f.load_generic_attributes()
sub_dt_refs = f.gather_references(datatype_cls)
for traited_attr, sub_gid in sub_dt_refs:
if sub_gid is None:
continue
is_monitor = False
if isinstance(sub_gid, list):
sub_gid = sub_gid[0]
is_monitor = True
ref_ht = load_ht_function(sub_gid, traited_attr)
if is_monitor:
ref_ht = [ref_ht]
setattr(datatype, traited_attr.field_name, ref_ht)
return datatype, ga
def load(self, gid=None, fname=None):
# type: (typing.Union[uuid.UUID, str], str) -> HasTraits
"""
Load from file a HasTraits entity. Either gid or fname should be given, or else an error is raised.
:param gid: optional entity GUID to search for it under self.base_dir
:param fname: optional file name to search for it under self.base_dir.
:return: HasTraits instance read from the given location
"""
if fname is None:
if gid is None:
raise ValueError("Neither gid nor filename is provided to load!")
fname = self.find_file_by_gid(gid)
sub_dt_refs = []
with H5File.from_file(fname) as f:
datatype_cls = self.registry.get_datatype_for_h5file(f)
datatype = datatype_cls()
f.load_into(datatype)
if self.recursive:
sub_dt_refs = f.gather_references()
for traited_attr, sub_gid in sub_dt_refs:
if sub_gid is not None:
subdt = self.load(sub_gid)
setattr(datatype, traited_attr.field_name, subdt)
return datatype
def set_visibility(dt):
""" set visibility flag, persist in db and h5"""
dt.visible = is_visible
dt = dao.store_entity(dt)
h5_path = h5.path_for_stored_index(dt)
with H5File.from_file(h5_path) as f:
f.visible.store(is_visible)
def load(self, source):
# type: (str) -> HasTraits
with H5File.from_file(source) as f:
datatype_cls = self.registry.get_datatype_for_h5file(type(f))
datatype = datatype_cls()
f.load_into(datatype)
return datatype
def __gather_datatypes_for_copy(self, data, dt_path_list):
data_path = h5.path_for_stored_index(data)
dt_path_list.append(data_path)
with H5File.from_file(data_path) as f:
sub_dt_refs = f.gather_references()
for _, ref_gid in sub_dt_refs:
if ref_gid:
dt = load.load_entity_by_gid(ref_gid)
self.__gather_datatypes_for_copy(dt, dt_path_list)
def _capture_operation_results(self, result):
"""
Update h5 files with generic attributes
"""
for file in os.listdir(self._get_output_path()):
path = os.path.join(self._get_output_path(), file)
if issubclass(H5File.h5_class_from_file(path), ViewModelH5):
continue
with H5File.from_file(path) as f:
f.store_generic_attributes(self.generic_attributes)
return "", 2
def _capture_operation_results(self, result):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(
datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(
operation.fk_operation_group).id
burst_reference = None
if DataTypeMetaData.KEY_BURST in self.meta_data:
burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST]
count_stored = 0
group_type = None # In case of a group, the first not-none type is sufficient to memorize here
for res in result:
if res is None:
continue
res.subject = self.generic_attributes.subject
res.state = self.generic_attributes.state
res.fk_parent_burst = burst_reference
res.fk_from_operation = self.operation_id
res.framework_metadata = self.meta_data
res.user_tag_1 = self.generic_attributes.user_tag_1
res.user_tag_2 = self.generic_attributes.user_tag_2
res.fk_datatype_group = data_type_group_id
# Compute size-on disk, in case file-storage is used
associated_file = h5.path_for_stored_index(res)
if os.path.exists(associated_file):
res.disk_size = self.file_handler.compute_size_on_disk(
associated_file)
with H5File.from_file(associated_file) as f:
f.store_generic_attributes(self.generic_attributes)
dao.store_entity(res)
group_type = res.type
count_stored += 1
if count_stored > 0 and self._is_group_launch():
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(
operation.fk_operation_group)
operation_group.fill_operationgroup_name(group_type)
dao.store_entity(operation_group)
return 'Operation ' + str(
self.operation_id) + ' has finished.', count_stored
def _capture_operation_results(self, result):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(
datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(
operation.fk_operation_group).id
count_stored = 0
if result is None:
return "", count_stored
group_type = None # In case of a group, the first not-none type is sufficient to memorize here
for res in result:
if res is None:
continue
if not res.fixed_generic_attributes:
res.fill_from_generic_attributes(self.generic_attributes)
res.fk_from_operation = self.operation_id
res.fk_datatype_group = data_type_group_id
associated_file = h5.path_for_stored_index(res)
if os.path.exists(associated_file):
if not res.fixed_generic_attributes:
with H5File.from_file(associated_file) as f:
f.store_generic_attributes(self.generic_attributes)
# Compute size-on disk, in case file-storage is used
res.disk_size = self.storage_interface.compute_size_on_disk(
associated_file)
dao.store_entity(res)
res.after_store()
group_type = res.type
count_stored += 1
if count_stored > 0 and self._is_group_launch():
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(
operation.fk_operation_group)
operation_group.fill_operationgroup_name(group_type)
dao.store_entity(operation_group)
return 'Operation ' + str(
self.operation_id) + ' has finished.', count_stored
def load_from_reference(self, gid):
config_path = self.get_reference_path(gid)
config_h5 = H5File.from_file(config_path)
config_type = config_h5.type.load()
package, cls_name = config_type.rsplit('.', 1)
module = importlib.import_module(package)
config_class = getattr(module, cls_name)
config_instance = config_class()
config_h5.load_into(config_instance)
config_h5.close()
return config_instance
def load_complete_by_function(self, file_path, load_ht_function):
# type: (str, callable) -> (HasTraits, GenericAttributes)
with H5File.from_file(file_path) as f:
datatype_cls = self.registry.get_datatype_for_h5file(type(f))
datatype = datatype_cls()
f.load_into(datatype)
ga = f.load_generic_attributes()
sub_dt_refs = f.gather_references()
for traited_attr, sub_gid in sub_dt_refs:
if sub_gid is None:
continue
ref_ht = load_ht_function(sub_gid, traited_attr)
setattr(datatype, traited_attr.field_name, ref_ht)
return datatype, ga
def copy_dt_to_export_folder(self, data, data_export_folder):
data_path = h5.path_for_stored_index(data)
with H5File.from_file(data_path) as f:
file_destination = os.path.join(data_export_folder,
os.path.basename(data_path))
if not os.path.exists(file_destination):
FilesHelper().copy_file(data_path, file_destination)
sub_dt_refs = f.gather_references()
for _, ref_gid in sub_dt_refs:
if ref_gid:
dt = load.load_entity_by_gid(ref_gid)
self.copy_dt_to_export_folder(dt, data_export_folder)
H5File.remove_metadata_param(file_destination, 'parent_burst')
def load_from_reference(self, gid):
dir_loader = h5.DirLoader(os.path.dirname(self.path), h5.REGISTRY)
config_filename = dir_loader.find_file_name(gid)
config_path = os.path.join(dir_loader.base_dir, config_filename)
config_h5 = H5File.from_file(config_path)
config_type = config_h5.type.load()
package, cls_name = config_type.rsplit('.', 1)
module = importlib.import_module(package)
config_class = getattr(module, cls_name)
config_instance = config_class()
config_h5.load_into(config_instance)
config_h5.close()
return config_instance
def load_with_references(self, file_path):
# type: (str) -> (HasTraits, GenericAttributes)
with H5File.from_file(file_path) as f:
datatype_cls = self.registry.get_datatype_for_h5file(type(f))
datatype = datatype_cls()
f.load_into(datatype)
ga = f.load_generic_attributes()
sub_dt_refs = f.gather_references()
for traited_attr, sub_gid in sub_dt_refs:
if sub_gid is None:
continue
ref_idx = dao.get_datatype_by_gid(sub_gid.hex, load_lazy=False)
ref_ht = self.load_from_index(ref_idx, traited_attr.field_type)
setattr(datatype, traited_attr.field_name, ref_ht)
return datatype, ga
def load(self, gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
fname = self.find_file_name(gid)
sub_dt_refs = []
with H5File.from_file(os.path.join(self.base_dir, fname)) as f:
datatype_cls = self.registry.get_datatype_for_h5file(type(f))
datatype = datatype_cls()
f.load_into(datatype)
if self.recursive:
sub_dt_refs = f.gather_references()
for traited_attr, sub_gid in sub_dt_refs:
subdt = self.load(sub_gid)
setattr(datatype, traited_attr.field_name, subdt)
return datatype
def _edit_data(self, datatype, new_data, from_group=False):
# type: (DataType, dict, bool) -> None
"""
Private method, used for editing a meta-data XML file and a DataType row
for a given custom DataType entity with new dictionary of data from UI.
"""
# 1. First update Operation fields:
# Update group field if possible
new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG]
empty_group_value = (new_group_name is None or new_group_name == "")
if from_group:
if empty_group_value:
raise StructureException("Empty group is not allowed!")
group = dao.get_generic_entity(OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if group and len(group) > 0 and new_group_name != group[0].name:
group = group[0]
exists_group = dao.get_generic_entity(OperationGroup, new_group_name, 'name')
if exists_group:
raise StructureException("Group '" + new_group_name + "' already exists.")
group.name = new_group_name
dao.store_entity(group)
else:
operation = dao.get_operation_by_id(datatype.fk_from_operation)
operation.user_group = new_group_name
dao.store_entity(operation)
op_folder = self.structure_helper.get_project_folder(operation.project, str(operation.id))
vm_gid = operation.view_model_gid
view_model_file = h5.determine_filepath(vm_gid, op_folder)
if view_model_file:
view_model_class = H5File.determine_type(view_model_file)
view_model = view_model_class()
with ViewModelH5(view_model_file, view_model) as f:
ga = f.load_generic_attributes()
ga.operation_tag = new_group_name
f.store_generic_attributes(ga, False)
else:
self.logger.warning("Could not find ViewModel H5 file for op: {}".format(operation))
# 2. Update GenericAttributes in the associated H5 files:
h5_path = h5.path_for_stored_index(datatype)
with H5File.from_file(h5_path) as f:
ga = f.load_generic_attributes()
ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT]
ga.state = new_data[DataTypeOverlayDetails.DATA_STATE]
ga.operation_tag = new_group_name
if DataTypeOverlayDetails.DATA_TAG_1 in new_data:
ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1]
if DataTypeOverlayDetails.DATA_TAG_2 in new_data:
ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2]
if DataTypeOverlayDetails.DATA_TAG_3 in new_data:
ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3]
if DataTypeOverlayDetails.DATA_TAG_4 in new_data:
ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4]
if DataTypeOverlayDetails.DATA_TAG_5 in new_data:
ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5]
f.store_generic_attributes(ga, False)
# 3. Update MetaData in DT Index DB as well.
datatype.fill_from_generic_attributes(ga)
dao.store_entity(datatype)
