def test_sys_date():
ft = "%Y%m%d"
date = time.strftime(ft, time.localtime(time.time()))
assert date == dateu.sys_date()
ft = "%Y-%m-%d"
date = time.strftime(ft, time.localtime(time.time()))
assert date == dateu.sys_date(ft)
def test_sys_time():
ft = "%H%M%S"
date = time.strftime(ft, time.localtime(time.time()))
assert date == dateu.sys_time()
ft = "%H:%M:%S"
date = time.strftime(ft, time.localtime(time.time()))
assert date == dateu.sys_time(ft)
def translate_any_2_anyone(article, target="en"):
"""
临时用的，将任意语言的文章翻译为英文
:param article: String.文章
:param target: String.可选值有en、zh等
"""
url = config.translate_url
article_t = ""
try:
article_detect = detect(article)
if article_detect == 'ja' or article_detect == 'zh-cn':
article_array = article.split("。")
for sentence in article_array:
data = {
"from": "auto",
"to": target,
"apikey": config.translate_user_key,
"src_text": sentence
}
cur_time = date_util.sys_date("%Y-%m-%d %H:%M:%S")
logger.info(f"-----------翻译句子开始----------- : {cur_time}")
res = http_post(data, url)
cur_time = date_util.sys_date("%Y-%m-%d %H:%M:%S")
logger.info(f"-----------翻译句子结束----------- : {cur_time}")
res_dict = json.loads(res)
if "tgt_text" in res_dict:
content = res_dict['tgt_text']
article_t += content + ". "
else:
article_array = article.split(".")
for sentence in article_array:
data = {
"from": "auto",
"to": target,
"apikey": config.translate_user_key,
"src_text": sentence
}
cur_time = date_util.sys_date("%Y-%m-%d %H:%M:%S")
logger.info(f"-----------翻译句子开始----------- : {cur_time}")
res = http_post(data, url)
cur_time = date_util.sys_date("%Y-%m-%d %H:%M:%S")
logger.info(f"-----------翻译句子结束----------- : {cur_time}")
res_dict = json.loads(res)
if "tgt_text" in res_dict:
content = res_dict['tgt_text']
article_t += content + ". "
return article_t
except HTTPError as e:
logger.error('翻译时发生的错误：', e)
return ''
def article_clean(article_id, content):
db = DatabaseWrapper()
try:
db.execute(
"update t_article_msg set is_clean='1',content_cleared=%s,"
"clean_finish_date=%s,clean_finish_time=%s "
"where article_id=%s",
(content, date_util.sys_date("%Y-%m-%d"), date_util.sys_time("%H:%M:%S"), article_id))
db.commit()
except Exception as e:
db.rollback()
raise RuntimeError(e)
finally:
db.close()
def article_zdxj(article_id):
db = DatabaseWrapper()
try:
url = config.coref_interface_uri
article = db.query(f"SELECT article_id,content FROM t_article_msg_en "
f"where is_zdxj='0' and article_id='{article_id}'")
for article_id, content in zip(article.article_id, article.content):
logger.info(f"ID = {article_id}")
# 调用指代消解接口
data = {'content': content}
result = http_post(data, url)
result = json.loads(result)
logger.info(f"ID = {article_id}")
if result["status"] == "success":
article_title = db.query(
f"SELECT title FROM t_article_msg where article_id='{article_id}'",
(), QueryResultType.PANDAS)
title = ""
if len(article_title.title) > 0:
title = translate_any_2_anyone(article_title.title[0],
"zh")
# 数据插入表
db.execute(
"INSERT INTO t_article_msg_zh(article_id, content,title) VALUES(%s,%s,%s)",
(article_id, result["coref"], title))
db.execute(
"update t_article_msg_en set is_zdxj='1',finish_date=%s,finish_time=%s where article_id=%s",
(date_util.sys_date("%Y-%m-%d"),
date_util.sys_time("%H:%M:%S"), article_id))
db.commit()
return "success"
else:
logger.info(f"error article_id:{article_id}")
return "error"
except Exception as e:
db.rollback()
raise RuntimeError(e)
finally:
db.close()
def article_en(article_id):
db = DatabaseWrapper()
try:
article = db.query(
f"select article_id,content_cleared from t_article_msg "
f"where is_translated = '0' and is_clean='1' and article_id='{article_id}' "
)
for article_id, content_cleared in zip(article.article_id,
article.content_cleared):
logger.info(f"ID = {article_id}")
article_detect = detect(content_cleared)
if article_detect != "en":
content = translate_any_2_anyone(content_cleared, "en")
else:
content = content_cleared
cur_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
logger.info(f"end---cur_time = {cur_time}")
if content != "":
content = str(content).replace("'", "\"")
db.execute(
"insert into t_article_msg_en(article_id,content) values(%s,%s)",
(article_id, content))
db.execute(
"update t_article_msg set is_translated='1',translated_finish_date=%s,translated_finish_time=%s "
"where article_id=%s",
(date_util.sys_date("%Y-%m-%d"),
date_util.sys_time("%H:%M:%S"), article_id))
db.commit()
return "success"
else:
logger.info(f"error:{article_id}")
return "error"
except Exception as e:
db.rollback()
raise RuntimeError(e)
finally:
db.close()
def insert_event_attribute(sentiment_analysis, event_date, event_local,
event_state, nentity_place, nentity_org,
nentity_person, nentity_misc, event_id):
"""
插入数据到“事件属性表”。
:param sentiment_analysis: string.情感分析
:param event_date: string.事件发生日期
:param event_local: string.事件发生地点
:param event_state: string.事件状态
:param nentity_place: string.命名实体-地点
:param nentity_org: string.命名实体-组织机构
:param nentity_person: string.命名实体-人
:param nentity_misc: string.命名实体-杂项
:param event_id: string.事件编号
:return 属性id。
"""
db = DatabaseWrapper()
try:
if not nentity_place:
nentity_place = ''
if not nentity_org:
nentity_org = ''
if not nentity_person:
nentity_person = ''
event_local = str(event_local).replace("'", "\"")
nentity_place = str(nentity_place).replace("'", "\"")
nentity_org = str(nentity_org).replace("'", "\"")
nentity_person = str(nentity_person).replace("'", "\"")
attritute_id = gener_id_by_uuid()
create_date = date_util.sys_datetime("%Y-%m-%d")
# cursor = connect.cursor()
db.execute(
"INSERT INTO ebm_eventsent_rel(relation_id, sentiment_analysis, event_date, event_local, "
"event_state, nentity_place, nentity_org, nentity_person, nentity_misc, "
"create_date, event_id) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(attritute_id, sentiment_analysis, event_date, event_local,
event_state, nentity_place, nentity_org, nentity_person,
nentity_misc, create_date, event_id))
db.commit()
except Exception as e:
db.rollback()
raise RuntimeError(e)
finally:
db.close()
return attritute_id
import feedwork.utils.DateHelper as date_util
from feedwork.utils import logger
from jdqd.a04.event_interface.services.common.http_util import http_post
from jdqd.a04.event_interface.config.project import Config
config = Config()
db = DatabaseWrapper()
try:
# 1cd63e541ba6fe91c1b0483516f7dff0
df = db.query(
"select article_id,content from t_article_msg_zh where is_relation='0' "
)
for aid, content in zip(df.article_id, df.content):
logger.info(f"ID = {aid}")
cur_time = date_util.sys_date("%Y-%m-%d %H:%M:%S")
logger.info(f"end---cur_time = {cur_time}")
data = {"content_id": aid, "content": content}
res = http_post(data, config.relextract_interface_uri)
response = json.loads(res)
if response["status"] == "success":
db2 = DatabaseWrapper()
db2.execute(
f"update t_article_msg_zh set is_relation='1',finish_date=%s,finish_time=%s "
f"where article_id=%s", (date_util.sys_date("%Y-%m-%d"),
date_util.sys_time("%H:%M:%S"), aid))
db2.commit()
db2.close()
else:
logger.error(f"error,article_id:{aid}")
except Exception as e: