def _edit_data(self, datatype, new_data, from_group=False): # type: (DataType, dict, bool) -> None """ Private method, used for editing a meta-data XML file and a DataType row for a given custom DataType entity with new dictionary of data from UI. """ # 1. First update Operation fields: # Update group field if possible new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG] empty_group_value = (new_group_name is None or new_group_name == "") if from_group: if empty_group_value: raise StructureException("Empty group is not allowed!") group = dao.get_generic_entity( OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if group and len(group) > 0 and new_group_name != group[0].name: group = group[0] exists_group = dao.get_generic_entity(OperationGroup, new_group_name, 'name') if exists_group: raise StructureException("Group '" + new_group_name + "' already exists.") group.name = new_group_name dao.store_entity(group) else: operation = dao.get_operation_by_id(datatype.fk_from_operation) operation.user_group = new_group_name dao.store_entity(operation) # 2. Update GenericAttributes on DataType index and in the associated H5 files: h5_path = h5.path_for_stored_index(datatype) with H5File.from_file(h5_path) as f: ga = f.load_generic_attributes() ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT] ga.state = new_data[DataTypeOverlayDetails.DATA_STATE] if DataTypeOverlayDetails.DATA_TAG_1 in new_data: ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1] if DataTypeOverlayDetails.DATA_TAG_2 in new_data: ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2] if DataTypeOverlayDetails.DATA_TAG_3 in new_data: ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3] if DataTypeOverlayDetails.DATA_TAG_4 in new_data: ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4] if DataTypeOverlayDetails.DATA_TAG_5 in new_data: ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5] datatype.fill_from_generic_attributes(ga) datatype = dao.store_entity(datatype) # 3. Update MetaData in DT H5 as well. with H5File.from_file(h5_path) as f: f.store_generic_attributes(ga, False)
def load_complete_by_function(self, file_path, load_ht_function): # type: (str, callable) -> (HasTraits, GenericAttributes) with H5File.from_file(file_path) as f: try: datatype_cls = self.registry.get_datatype_for_h5file(f) except KeyError: datatype_cls = f.determine_datatype_from_file() datatype = datatype_cls() f.load_into(datatype) ga = f.load_generic_attributes() sub_dt_refs = f.gather_references(datatype_cls) for traited_attr, sub_gid in sub_dt_refs: if sub_gid is None: continue is_monitor = False if isinstance(sub_gid, list): sub_gid = sub_gid[0] is_monitor = True ref_ht = load_ht_function(sub_gid, traited_attr) if is_monitor: ref_ht = [ref_ht] setattr(datatype, traited_attr.field_name, ref_ht) return datatype, ga
def load(self, gid=None, fname=None): # type: (typing.Union[uuid.UUID, str], str) -> HasTraits """ Load from file a HasTraits entity. Either gid or fname should be given, or else an error is raised. :param gid: optional entity GUID to search for it under self.base_dir :param fname: optional file name to search for it under self.base_dir. :return: HasTraits instance read from the given location """ if fname is None: if gid is None: raise ValueError("Neither gid nor filename is provided to load!") fname = self.find_file_by_gid(gid) sub_dt_refs = [] with H5File.from_file(fname) as f: datatype_cls = self.registry.get_datatype_for_h5file(f) datatype = datatype_cls() f.load_into(datatype) if self.recursive: sub_dt_refs = f.gather_references() for traited_attr, sub_gid in sub_dt_refs: if sub_gid is not None: subdt = self.load(sub_gid) setattr(datatype, traited_attr.field_name, subdt) return datatype
def set_visibility(dt): """ set visibility flag, persist in db and h5""" dt.visible = is_visible dt = dao.store_entity(dt) h5_path = h5.path_for_stored_index(dt) with H5File.from_file(h5_path) as f: f.visible.store(is_visible)
def load(self, source): # type: (str) -> HasTraits with H5File.from_file(source) as f: datatype_cls = self.registry.get_datatype_for_h5file(type(f)) datatype = datatype_cls() f.load_into(datatype) return datatype
def __gather_datatypes_for_copy(self, data, dt_path_list): data_path = h5.path_for_stored_index(data) dt_path_list.append(data_path) with H5File.from_file(data_path) as f: sub_dt_refs = f.gather_references() for _, ref_gid in sub_dt_refs: if ref_gid: dt = load.load_entity_by_gid(ref_gid) self.__gather_datatypes_for_copy(dt, dt_path_list)
def _capture_operation_results(self, result): """ Update h5 files with generic attributes """ for file in os.listdir(self._get_output_path()): path = os.path.join(self._get_output_path(), file) if issubclass(H5File.h5_class_from_file(path), ViewModelH5): continue with H5File.from_file(path) as f: f.store_generic_attributes(self.generic_attributes) return "", 2
def _capture_operation_results(self, result): """ After an operation was finished, make sure the results are stored in DB storage and the correct meta-data,IDs are set. """ data_type_group_id = None operation = dao.get_operation_by_id(self.operation_id) if operation.user_group is None or len(operation.user_group) == 0: operation.user_group = date2string( datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT) operation = dao.store_entity(operation) if self._is_group_launch(): data_type_group_id = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group).id burst_reference = None if DataTypeMetaData.KEY_BURST in self.meta_data: burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST] count_stored = 0 group_type = None # In case of a group, the first not-none type is sufficient to memorize here for res in result: if res is None: continue res.subject = self.generic_attributes.subject res.state = self.generic_attributes.state res.fk_parent_burst = burst_reference res.fk_from_operation = self.operation_id res.framework_metadata = self.meta_data res.user_tag_1 = self.generic_attributes.user_tag_1 res.user_tag_2 = self.generic_attributes.user_tag_2 res.fk_datatype_group = data_type_group_id # Compute size-on disk, in case file-storage is used associated_file = h5.path_for_stored_index(res) if os.path.exists(associated_file): res.disk_size = self.file_handler.compute_size_on_disk( associated_file) with H5File.from_file(associated_file) as f: f.store_generic_attributes(self.generic_attributes) dao.store_entity(res) group_type = res.type count_stored += 1 if count_stored > 0 and self._is_group_launch(): # Update the operation group name operation_group = dao.get_operationgroup_by_id( operation.fk_operation_group) operation_group.fill_operationgroup_name(group_type) dao.store_entity(operation_group) return 'Operation ' + str( self.operation_id) + ' has finished.', count_stored
def _capture_operation_results(self, result): """ After an operation was finished, make sure the results are stored in DB storage and the correct meta-data,IDs are set. """ data_type_group_id = None operation = dao.get_operation_by_id(self.operation_id) if operation.user_group is None or len(operation.user_group) == 0: operation.user_group = date2string( datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT) operation = dao.store_entity(operation) if self._is_group_launch(): data_type_group_id = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group).id count_stored = 0 if result is None: return "", count_stored group_type = None # In case of a group, the first not-none type is sufficient to memorize here for res in result: if res is None: continue if not res.fixed_generic_attributes: res.fill_from_generic_attributes(self.generic_attributes) res.fk_from_operation = self.operation_id res.fk_datatype_group = data_type_group_id associated_file = h5.path_for_stored_index(res) if os.path.exists(associated_file): if not res.fixed_generic_attributes: with H5File.from_file(associated_file) as f: f.store_generic_attributes(self.generic_attributes) # Compute size-on disk, in case file-storage is used res.disk_size = self.storage_interface.compute_size_on_disk( associated_file) dao.store_entity(res) res.after_store() group_type = res.type count_stored += 1 if count_stored > 0 and self._is_group_launch(): # Update the operation group name operation_group = dao.get_operationgroup_by_id( operation.fk_operation_group) operation_group.fill_operationgroup_name(group_type) dao.store_entity(operation_group) return 'Operation ' + str( self.operation_id) + ' has finished.', count_stored
def load_from_reference(self, gid): config_path = self.get_reference_path(gid) config_h5 = H5File.from_file(config_path) config_type = config_h5.type.load() package, cls_name = config_type.rsplit('.', 1) module = importlib.import_module(package) config_class = getattr(module, cls_name) config_instance = config_class() config_h5.load_into(config_instance) config_h5.close() return config_instance
def load_complete_by_function(self, file_path, load_ht_function): # type: (str, callable) -> (HasTraits, GenericAttributes) with H5File.from_file(file_path) as f: datatype_cls = self.registry.get_datatype_for_h5file(type(f)) datatype = datatype_cls() f.load_into(datatype) ga = f.load_generic_attributes() sub_dt_refs = f.gather_references() for traited_attr, sub_gid in sub_dt_refs: if sub_gid is None: continue ref_ht = load_ht_function(sub_gid, traited_attr) setattr(datatype, traited_attr.field_name, ref_ht) return datatype, ga
def copy_dt_to_export_folder(self, data, data_export_folder): data_path = h5.path_for_stored_index(data) with H5File.from_file(data_path) as f: file_destination = os.path.join(data_export_folder, os.path.basename(data_path)) if not os.path.exists(file_destination): FilesHelper().copy_file(data_path, file_destination) sub_dt_refs = f.gather_references() for _, ref_gid in sub_dt_refs: if ref_gid: dt = load.load_entity_by_gid(ref_gid) self.copy_dt_to_export_folder(dt, data_export_folder) H5File.remove_metadata_param(file_destination, 'parent_burst')
def load_from_reference(self, gid): dir_loader = h5.DirLoader(os.path.dirname(self.path), h5.REGISTRY) config_filename = dir_loader.find_file_name(gid) config_path = os.path.join(dir_loader.base_dir, config_filename) config_h5 = H5File.from_file(config_path) config_type = config_h5.type.load() package, cls_name = config_type.rsplit('.', 1) module = importlib.import_module(package) config_class = getattr(module, cls_name) config_instance = config_class() config_h5.load_into(config_instance) config_h5.close() return config_instance
def load_with_references(self, file_path): # type: (str) -> (HasTraits, GenericAttributes) with H5File.from_file(file_path) as f: datatype_cls = self.registry.get_datatype_for_h5file(type(f)) datatype = datatype_cls() f.load_into(datatype) ga = f.load_generic_attributes() sub_dt_refs = f.gather_references() for traited_attr, sub_gid in sub_dt_refs: if sub_gid is None: continue ref_idx = dao.get_datatype_by_gid(sub_gid.hex, load_lazy=False) ref_ht = self.load_from_index(ref_idx, traited_attr.field_type) setattr(datatype, traited_attr.field_name, ref_ht) return datatype, ga
def load(self, gid): # type: (typing.Union[uuid.UUID, str]) -> HasTraits fname = self.find_file_name(gid) sub_dt_refs = [] with H5File.from_file(os.path.join(self.base_dir, fname)) as f: datatype_cls = self.registry.get_datatype_for_h5file(type(f)) datatype = datatype_cls() f.load_into(datatype) if self.recursive: sub_dt_refs = f.gather_references() for traited_attr, sub_gid in sub_dt_refs: subdt = self.load(sub_gid) setattr(datatype, traited_attr.field_name, subdt) return datatype
def _edit_data(self, datatype, new_data, from_group=False): # type: (DataType, dict, bool) -> None """ Private method, used for editing a meta-data XML file and a DataType row for a given custom DataType entity with new dictionary of data from UI. """ # 1. First update Operation fields: # Update group field if possible new_group_name = new_data[CommonDetails.CODE_OPERATION_TAG] empty_group_value = (new_group_name is None or new_group_name == "") if from_group: if empty_group_value: raise StructureException("Empty group is not allowed!") group = dao.get_generic_entity(OperationGroup, new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if group and len(group) > 0 and new_group_name != group[0].name: group = group[0] exists_group = dao.get_generic_entity(OperationGroup, new_group_name, 'name') if exists_group: raise StructureException("Group '" + new_group_name + "' already exists.") group.name = new_group_name dao.store_entity(group) else: operation = dao.get_operation_by_id(datatype.fk_from_operation) operation.user_group = new_group_name dao.store_entity(operation) op_folder = self.structure_helper.get_project_folder(operation.project, str(operation.id)) vm_gid = operation.view_model_gid view_model_file = h5.determine_filepath(vm_gid, op_folder) if view_model_file: view_model_class = H5File.determine_type(view_model_file) view_model = view_model_class() with ViewModelH5(view_model_file, view_model) as f: ga = f.load_generic_attributes() ga.operation_tag = new_group_name f.store_generic_attributes(ga, False) else: self.logger.warning("Could not find ViewModel H5 file for op: {}".format(operation)) # 2. Update GenericAttributes in the associated H5 files: h5_path = h5.path_for_stored_index(datatype) with H5File.from_file(h5_path) as f: ga = f.load_generic_attributes() ga.subject = new_data[DataTypeOverlayDetails.DATA_SUBJECT] ga.state = new_data[DataTypeOverlayDetails.DATA_STATE] ga.operation_tag = new_group_name if DataTypeOverlayDetails.DATA_TAG_1 in new_data: ga.user_tag_1 = new_data[DataTypeOverlayDetails.DATA_TAG_1] if DataTypeOverlayDetails.DATA_TAG_2 in new_data: ga.user_tag_2 = new_data[DataTypeOverlayDetails.DATA_TAG_2] if DataTypeOverlayDetails.DATA_TAG_3 in new_data: ga.user_tag_3 = new_data[DataTypeOverlayDetails.DATA_TAG_3] if DataTypeOverlayDetails.DATA_TAG_4 in new_data: ga.user_tag_4 = new_data[DataTypeOverlayDetails.DATA_TAG_4] if DataTypeOverlayDetails.DATA_TAG_5 in new_data: ga.user_tag_5 = new_data[DataTypeOverlayDetails.DATA_TAG_5] f.store_generic_attributes(ga, False) # 3. Update MetaData in DT Index DB as well. datatype.fill_from_generic_attributes(ga) dao.store_entity(datatype)