def set_value_from_file(self, file_name: str, file_format: str,
file_encoding: str):
"""
设置value为文件内容
. 根据文件格式, 可以加载文件内容
. 如果文件格式为二进制, 则value存储文件名
:param file_name:
:param file_format:
:param file_encoding:
:return:
"""
if CUtils.equal_ignore_case(file_format, CResource.FileFormat_TXT):
self._value = {
CResource.Name_Text: CFile.file_2_str(file_name),
CResource.Name_Type: CResource.DataValueType_SQL
}
elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_XML):
self._value = {
CResource.Name_Text: CXml.file_2_str(file_name),
CResource.Name_Type: CResource.DataValueType_SQL
}
elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_Json):
self._value = {
CResource.Name_Text: CJson.file_2_str(file_name),
CResource.Name_Type: CResource.DataValueType_SQL
}
else:
self._value = {
CResource.Name_Text: file_name,
CResource.Name_Type: CResource.DataValueType_File
}
def process_mission(self, dataset) -> str:
"""
:param dataset:
:return:
"""
ds_ib_id = dataset.value_by_name(0, 'query_ib_id', '')
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ds_storage_title = dataset.value_by_name(0, 'query_storage_title', '')
ds_storage_root_dir = dataset.value_by_name(0, 'query_rootpath', '')
ds_ib_directory_name = dataset.value_by_name(0,
'query_ib_relation_dir',
'')
ds_ib_directory_id = dataset.value_by_name(0,
'query_ib_relation_dir_id',
'')
ds_ib_batch_no = dataset.value_by_name(0, 'query_ib_batchno', '')
# 按需要再开启
# ds_ib_option = CUtils.any_2_str(dataset.value_by_name(0, 'query_ib_option', ''))
if not CUtils.equal_ignore_case(ds_ib_directory_name, ''):
CLogger().debug('正在入库的是存储[{0}]下的目录[{1}]'.format(
ds_storage_title,
CFile.join_file(ds_storage_root_dir, ds_ib_directory_name)))
else:
CLogger().debug('正在入库的是存储[{0}]下的目录[{1}]'.format(
ds_storage_title, ds_storage_root_dir))
try:
ds_ib_information_updated = False
# 检查目录名格式并自动修正
if not CUtils.equal_ignore_case(ds_ib_directory_name, ''):
ds_ib_directory = CFile.unify(
CFile.add_prefix(ds_ib_directory_name))
if not CUtils.equal_ignore_case(ds_ib_directory,
ds_ib_directory_name):
ds_ib_directory_name = ds_ib_directory
ds_ib_information_updated = True
if CUtils.equal_ignore_case(ds_ib_batch_no, ''):
ds_ib_batch_no = CFactory().give_me_db(
self.get_mission_db_id()).seq_next_value(
self.Seq_Type_Date_AutoInc)
ds_ib_information_updated = True
if CUtils.equal_ignore_case(ds_ib_directory_id, ''):
ds_ib_directory_id = CUtils.one_id()
ds_ib_information_updated = True
if ds_ib_information_updated:
self.correct_ib_information(ds_ib_id, ds_ib_directory_name,
ds_ib_batch_no, ds_ib_directory_id)
if not CUtils.equal_ignore_case(ds_ib_directory_name, ''):
ib_full_directory = CFile.join_file(ds_storage_root_dir,
ds_ib_directory_name)
else:
ib_full_directory = ds_storage_root_dir
self.clear_anything_in_directory(ds_ib_id)
metadata_rule_file_name = CFile.join_file(
ib_full_directory, self.FileName_MetaData_Rule)
metadata_rule_content = ''
if CFile.file_or_path_exist(metadata_rule_file_name):
try:
metadata_rule_content = CXml.file_2_str(
metadata_rule_file_name)
CLogger().debug('在目录[{0}]下发现元数据规则文件, 它的内容为[{1}]'.format(
ib_full_directory, metadata_rule_content))
except Exception as error:
result = CResult.merge_result(
self.Failure,
'在目录[{0}]下发现元数据规则文件, 但它的格式不合法, 详细错误为: [{1}]'.format(
ib_full_directory, error.__str__()))
self.update_inbound_qi_result(ds_ib_id, result)
return result
path_obj = CDMPathInfo(self.FileType_Dir, ib_full_directory,
ds_storage_id,
ds_ib_directory_id, ds_storage_id, None,
self.get_mission_db_id(),
metadata_rule_content)
if path_obj.white_black_valid():
path_obj.db_check_and_update(ds_ib_id)
result = CResult.merge_result(
self.Success, '目录[{0}]的入库质检任务创建成功, 系统正在质检, 请稍后...'.format(
ib_full_directory))
else:
result = CResult.merge_result(
self.Failure,
'目录[{0}]未通过黑白名单检验, 不允许入库! '.format(ib_full_directory))
self.update_inbound_qi_result(ds_ib_id, result)
return result
except Exception as error:
result = CResult.merge_result(
self.Failure, '目录[{0}]的入库质检任务创建过程出现错误, 详细错误为: [{1}]'.format(
CFile.join_file(ds_storage_root_dir, ds_ib_directory_name),
error.__str__()))
self.update_inbound_qi_result(ds_ib_id, result)
return result
def db_check_and_update_metadata_rule(self, metadata_rule_file_name):
"""
检查并判断指定的元数据扫描规则文件是否与数据库中的记录相等
1. 如果和记录中的不同
删除当前目录下的所有子目录, 文件 和对象
更新记录中的规则
设置子目录扫描状态为正常
2. 如果和记录中的相同
返回
:param metadata_rule_file_name:
:return:
"""
metadata_rule_content = ''
if CFile.file_or_path_exist(metadata_rule_file_name):
try:
metadata_rule_content = CXml.file_2_str(
metadata_rule_file_name)
CLogger().debug('在目录[{0}]下发现元数据规则文件, 它的内容为[{1}]'.format(
self.file_name_with_full_path, metadata_rule_content))
except:
pass
db_metadata_rule_content = self.ds_file_or_path.value_by_name(
0, 'dsdscanrule', '')
CLogger().debug('目录[{0}]在库中登记的规则内容为[{1}]'.format(
self.file_name_with_full_path, db_metadata_rule_content))
if CUtils.equal_ignore_case(metadata_rule_content,
db_metadata_rule_content):
CLogger().debug('目录[{0}]的规则内容与库中的相同'.format(
self.file_name_with_full_path))
return
else:
CLogger().debug('目录[{0}]的规则内容与库中的不同, 将清理目录下的文件, 重新入库!'.format(
self.file_name_with_full_path))
sql_update_path_scan_rule = '''
update dm2_storage_directory
set dsd_directory_valid = -1, dsdscanrule = :dsdScanRule
where dsdid = :dsdID
'''
sql_clear_files_of_path = '''
delete from dm2_storage_file
where dsfdirectoryid in (
select dsdid
from dm2_storage_directory
where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory) = 1
)
'''
sql_clear_subpath_of_path = '''
delete from dm2_storage_directory
where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory) = 1
and dsdid <> :dsdID
'''
engine = CFactory().give_me_db(self.db_server_id)
session = engine.give_me_session()
try:
params = dict()
params['dsdID'] = self.my_id
if metadata_rule_content == '':
params['dsdScanRule'] = None
else:
params['dsdScanRule'] = metadata_rule_content
engine.session_execute(session, sql_update_path_scan_rule, params)
params = dict()
params['dsdID'] = self.my_id
params['dsdStorageID'] = self.storage_id
params['dsdSubDirectory'] = CFile.join_file(
self.file_name_with_rel_path, '')
engine.session_execute(session, sql_clear_files_of_path, params)
engine.session_execute(session, sql_clear_subpath_of_path, params)
engine.session_commit(session)
except Exception as error:
CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__()))
engine.session_rollback(session)
finally:
engine.session_close(session)
