def set_value_from_file(self, file_name: str, file_format: str, file_encoding: str): """ 设置value为文件内容 . 根据文件格式, 可以加载文件内容 . 如果文件格式为二进制, 则value存储文件名 :param file_name: :param file_format: :param file_encoding: :return: """ if CUtils.equal_ignore_case(file_format, CResource.FileFormat_TXT): self._value = { CResource.Name_Text: CFile.file_2_str(file_name), CResource.Name_Type: CResource.DataValueType_SQL } elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_XML): self._value = { CResource.Name_Text: CXml.file_2_str(file_name), CResource.Name_Type: CResource.DataValueType_SQL } elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_Json): self._value = { CResource.Name_Text: CJson.file_2_str(file_name), CResource.Name_Type: CResource.DataValueType_SQL } else: self._value = { CResource.Name_Text: file_name, CResource.Name_Type: CResource.DataValueType_File }
def process_mission(self, dataset) -> str: """ :param dataset: :return: """ ds_ib_id = dataset.value_by_name(0, 'query_ib_id', '') ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ds_storage_title = dataset.value_by_name(0, 'query_storage_title', '') ds_storage_root_dir = dataset.value_by_name(0, 'query_rootpath', '') ds_ib_directory_name = dataset.value_by_name(0, 'query_ib_relation_dir', '') ds_ib_directory_id = dataset.value_by_name(0, 'query_ib_relation_dir_id', '') ds_ib_batch_no = dataset.value_by_name(0, 'query_ib_batchno', '') # 按需要再开启 # ds_ib_option = CUtils.any_2_str(dataset.value_by_name(0, 'query_ib_option', '')) if not CUtils.equal_ignore_case(ds_ib_directory_name, ''): CLogger().debug('正在入库的是存储[{0}]下的目录[{1}]'.format( ds_storage_title, CFile.join_file(ds_storage_root_dir, ds_ib_directory_name))) else: CLogger().debug('正在入库的是存储[{0}]下的目录[{1}]'.format( ds_storage_title, ds_storage_root_dir)) try: ds_ib_information_updated = False # 检查目录名格式并自动修正 if not CUtils.equal_ignore_case(ds_ib_directory_name, ''): ds_ib_directory = CFile.unify( CFile.add_prefix(ds_ib_directory_name)) if not CUtils.equal_ignore_case(ds_ib_directory, ds_ib_directory_name): ds_ib_directory_name = ds_ib_directory ds_ib_information_updated = True if CUtils.equal_ignore_case(ds_ib_batch_no, ''): ds_ib_batch_no = CFactory().give_me_db( self.get_mission_db_id()).seq_next_value( self.Seq_Type_Date_AutoInc) ds_ib_information_updated = True if CUtils.equal_ignore_case(ds_ib_directory_id, ''): ds_ib_directory_id = CUtils.one_id() ds_ib_information_updated = True if ds_ib_information_updated: self.correct_ib_information(ds_ib_id, ds_ib_directory_name, ds_ib_batch_no, ds_ib_directory_id) if not CUtils.equal_ignore_case(ds_ib_directory_name, ''): ib_full_directory = CFile.join_file(ds_storage_root_dir, ds_ib_directory_name) else: ib_full_directory = ds_storage_root_dir self.clear_anything_in_directory(ds_ib_id) metadata_rule_file_name = CFile.join_file( ib_full_directory, self.FileName_MetaData_Rule) metadata_rule_content = '' if CFile.file_or_path_exist(metadata_rule_file_name): try: metadata_rule_content = CXml.file_2_str( metadata_rule_file_name) CLogger().debug('在目录[{0}]下发现元数据规则文件, 它的内容为[{1}]'.format( ib_full_directory, metadata_rule_content)) except Exception as error: result = CResult.merge_result( self.Failure, '在目录[{0}]下发现元数据规则文件, 但它的格式不合法, 详细错误为: [{1}]'.format( ib_full_directory, error.__str__())) self.update_inbound_qi_result(ds_ib_id, result) return result path_obj = CDMPathInfo(self.FileType_Dir, ib_full_directory, ds_storage_id, ds_ib_directory_id, ds_storage_id, None, self.get_mission_db_id(), metadata_rule_content) if path_obj.white_black_valid(): path_obj.db_check_and_update(ds_ib_id) result = CResult.merge_result( self.Success, '目录[{0}]的入库质检任务创建成功, 系统正在质检, 请稍后...'.format( ib_full_directory)) else: result = CResult.merge_result( self.Failure, '目录[{0}]未通过黑白名单检验, 不允许入库! '.format(ib_full_directory)) self.update_inbound_qi_result(ds_ib_id, result) return result except Exception as error: result = CResult.merge_result( self.Failure, '目录[{0}]的入库质检任务创建过程出现错误, 详细错误为: [{1}]'.format( CFile.join_file(ds_storage_root_dir, ds_ib_directory_name), error.__str__())) self.update_inbound_qi_result(ds_ib_id, result) return result
def db_check_and_update_metadata_rule(self, metadata_rule_file_name): """ 检查并判断指定的元数据扫描规则文件是否与数据库中的记录相等 1. 如果和记录中的不同 删除当前目录下的所有子目录, 文件 和对象 更新记录中的规则 设置子目录扫描状态为正常 2. 如果和记录中的相同 返回 :param metadata_rule_file_name: :return: """ metadata_rule_content = '' if CFile.file_or_path_exist(metadata_rule_file_name): try: metadata_rule_content = CXml.file_2_str( metadata_rule_file_name) CLogger().debug('在目录[{0}]下发现元数据规则文件, 它的内容为[{1}]'.format( self.file_name_with_full_path, metadata_rule_content)) except: pass db_metadata_rule_content = self.ds_file_or_path.value_by_name( 0, 'dsdscanrule', '') CLogger().debug('目录[{0}]在库中登记的规则内容为[{1}]'.format( self.file_name_with_full_path, db_metadata_rule_content)) if CUtils.equal_ignore_case(metadata_rule_content, db_metadata_rule_content): CLogger().debug('目录[{0}]的规则内容与库中的相同'.format( self.file_name_with_full_path)) return else: CLogger().debug('目录[{0}]的规则内容与库中的不同, 将清理目录下的文件, 重新入库!'.format( self.file_name_with_full_path)) sql_update_path_scan_rule = ''' update dm2_storage_directory set dsd_directory_valid = -1, dsdscanrule = :dsdScanRule where dsdid = :dsdID ''' sql_clear_files_of_path = ''' delete from dm2_storage_file where dsfdirectoryid in ( select dsdid from dm2_storage_directory where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory) = 1 ) ''' sql_clear_subpath_of_path = ''' delete from dm2_storage_directory where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory) = 1 and dsdid <> :dsdID ''' engine = CFactory().give_me_db(self.db_server_id) session = engine.give_me_session() try: params = dict() params['dsdID'] = self.my_id if metadata_rule_content == '': params['dsdScanRule'] = None else: params['dsdScanRule'] = metadata_rule_content engine.session_execute(session, sql_update_path_scan_rule, params) params = dict() params['dsdID'] = self.my_id params['dsdStorageID'] = self.storage_id params['dsdSubDirectory'] = CFile.join_file( self.file_name_with_rel_path, '') engine.session_execute(session, sql_clear_files_of_path, params) engine.session_execute(session, sql_clear_subpath_of_path, params) engine.session_commit(session) except Exception as error: CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__())) engine.session_rollback(session) finally: engine.session_close(session)