def get_attr_must_have_list(template_id):
'''
获取attr_name_not_null_map
:param template_id:
:return:
'''
global _attr_must_have_map_dict
if _attr_must_have_map_dict.has_key(template_id):
return _attr_must_have_map_dict[template_id]
else:
map_ids = db.select_one(
'select attr_must_have from des_template where id = ?',
template_id)
if not map_ids:
log_util.error(
'rules.py::get_attr_name_map():: template table no template_id row::id = '
+ str(template_id))
_attr_must_have_map_dict[template_id] = []
return _attr_must_have_map_dict[template_id]
if not map_ids.attr_must_have:
log_util.error(
'rules.py::get_attr_name_map():: template table no template_id row::id = '
+ str(template_id))
_attr_must_have_map_dict[template_id] = []
return _attr_must_have_map_dict[template_id]
_attr_must_have_map_dict[template_id] = json.loads(
map_ids.attr_must_have)
return _attr_must_have_map_dict[template_id]
def do_process(self):
'''
处理函数,同步函数
:return:
'''
try:
return self.process()
except:
traceback.print_exc()
log_util.error(traceback.format_exc())
return False
def start(self):
'''
启动任务
:return:
'''
try:
db.update('update des_task set status=? where id=?', 1,
self.task_info.id)
result = self.process()
if result:
db.update('update des_task set status=? where id=?', 2,
self.task_info.id)
else:
db.update('update des_task set status=? where id=?', 3,
self.task_info.id)
except:
traceback.print_exc()
log_util.error(traceback.format_exc())
db.update('update des_task set status=? where id=?', 3,
self.task_info.id)
def get_schema_map(template_id):
'''
获取attr_name_map
:param template_id:
:return:
'''
global _schema_map_dict
if _schema_map_dict.has_key(template_id):
return _schema_map_dict[template_id]
else:
sql = 'select target_attr_name,source_attr_name from des_schema_map where template_id =?'
results = db.select(sql, template_id)
if not results:
log_util.error('rules.py::get_schema_map():: des_schema_map '
'table no data')
raise RulesException('des_schema_map no data ')
schema_map = {}
for result in results:
schema_map[
result.source_attr_name] = result.target_attr_name.strip()
_schema_map_dict[template_id] = schema_map
return _schema_map_dict[template_id]
def get_data_clean_rule(template_id):
'''
获取value_format_rules
:param template_id:
:return:
'''
global _data_clean_map_dict
if _data_clean_map_dict.has_key(template_id):
return _data_clean_map_dict[template_id]
else:
cursor = db.select_one(
'select data_clean_rules from des_template where id = ?',
template_id)
if not cursor:
log_util.error(
'rules.py::get_data_clean_rule():: template table no template_id row::id = '
+ str(template_id))
# raise RulesException('cursor None')
_data_clean_map_dict[template_id] = None
return _data_clean_map_dict[template_id]
if not cursor.data_clean_rules:
log_util.error(
'rules.py::get_data_clean_rule():: template table no template_id row::id = '
+ str(template_id))
# raise RulesException('cursor.data_clean_rules None')
_data_clean_map_dict[template_id] = None
return _data_clean_map_dict[template_id]
data_clean_rules = json.loads(cursor.data_clean_rules)
results = {}
for rule in data_clean_rules:
sql = 'select * from des_data_clean_rules where id in (' \
+ db.format_with_separator(rule['rules_ids']) + ')'
clean_rules = db.select(sql)
if not clean_rules:
log_util.error(
'rules.py::get_data_clean_rule():: data_clean_rules '
'table no id row::id = ' + rule['rules_ids'])
raise RulesException('data_clean_rules no ids ')
results[rule['path']] = []
for item in clean_rules:
if item['recognize_rule_type'] is 0:
sql = 'select * from des_func_sets where id = ' + item[
'recognize_rule']
func = db.select_one(sql)
item['recognize_rule'] = func.func_name
if item['clean_rule_type'] is 0:
sql = 'select * from des_func_sets where id = ' + item[
'clean_rule']
func = db.select_one(sql)
item['clean_rule'] = func.func_name
results[rule['path']].append(item)
_data_clean_map_dict[template_id] = results
return _data_clean_map_dict[template_id]
def get_synonym_replaced_rule(template_id):
'''
获取value_normalize_rules
:param template_id:
:return:
'''
global _synonym_replaced_map_dict
if _synonym_replaced_map_dict.has_key(template_id):
return _synonym_replaced_map_dict[template_id]
else:
cursor = db.select_one(
'select synonym_replace_rules from des_template where id = ?',
template_id)
if not cursor:
log_util.error(
'rules.py::get_synonym_replaced_rule():: template table no template_id row::id = '
+ str(template_id))
# raise RulesException('cursor None')
_synonym_replaced_map_dict[template_id] = None
return _synonym_replaced_map_dict[template_id]
if not cursor.synonym_replace_rules:
log_util.error(
'rules.py::get_synonym_replaced_rule():: template table no template_id row::id = '
+ str(template_id))
# raise RulesException('cursor.synonym_replace_rules None')
_synonym_replaced_map_dict[template_id] = None
return _synonym_replaced_map_dict[template_id]
synonym_replace_rules = json.loads(cursor.synonym_replace_rules)
results = {}
for rule in synonym_replace_rules:
result = {}
sql = "select target_word,synonym from des_synonym_dict where synonym_table_id = ?"
thesaurus = db.select(sql, rule['synonym_table_id'])
if not thesaurus:
log_util.error(
'rules.py::get_synonym_replaced_rule():: des_synonym_dict '
'table no row ')
raise RulesException('des_synonym_dict no row ')
for word_pair in thesaurus:
synonyms = json.loads(word_pair['synonym'])
for synonym in synonyms:
result[synonym] = word_pair['target_word']
results[rule['path']] = result
_synonym_replaced_map_dict[template_id] = results
return _synonym_replaced_map_dict[template_id]