def test_compute_vcf_no_session_fail(self):
"""HK vcf协同计算，未登录，失败"""
Log.debug('test_compute_vcf_no_session_fail start')
para_id = 22
data_id = 22002
res = self.comp_vcf.base_compute_vcf(para_id=para_id,
data_id=data_id,
cookies=None,
flag=False,
calType=self.calType,
maxFile=0)
self.assertTrue(res, msg='result check fail')
Log.debug('test_compute_vcf_no_session_fail end')
pass
def test_cancel_task_task_history_id_not_exist_fail(self):
"""HK taskId不存在t_task_history，取消任务失败"""
Log.debug('test_cancel_task_task_history_id_not_exist_fail start')
para_id = 21
data_id = 21005
res = self.cancel_task.base_cancel_task(para_id=para_id,
data_id=data_id,
cookies=self.admin_cookies,
flag=False,
task_modify=1,
reason_modify=0)
self.assertTrue(res, msg='result check fail')
Log.debug('test_cancel_task_task_history_id_not_exist_fail end')
pass
def test_compute_vcf_fileId_not_exist_fail(self):
"""HK vcf协同计算，fileid不存在，失败"""
Log.debug('test_compute_vcf_fileId_not_exist_fail start')
para_id = 22
data_id = 22004
res = self.comp_vcf.base_compute_vcf(para_id=para_id,
data_id=data_id,
cookies=self.admin_cookies,
flag=False,
calType=self.calType,
maxFile=1)
self.assertTrue(res, msg='result check fail')
Log.debug('test_compute_vcf_fileId_not_exist_fail end')
pass
def test_cancel_task_no_session_fail(self):
"""HK 未登录 取消任务失败"""
Log.debug('test_cancel_task_no_session_fail start')
para_id = 21
data_id = 21003
res = self.cancel_task.base_cancel_task(para_id=para_id,
data_id=data_id,
cookies=None,
flag=False,
task_modify=0,
reason_modify=0)
self.assertTrue(res, msg='result check fail')
Log.debug('test_cancel_task_no_session_fail end')
pass
def test_cancel_task_type_2_success(self):
"""HK 取消历史任务成功"""
Log.debug('test_cancel_task_type_2_success start')
para_id = 21
data_id = 21002
res = self.cancel_task.base_cancel_task(para_id=para_id,
data_id=data_id,
cookies=self.admin_cookies,
flag=True,
task_modify=0,
reason_modify=0)
self.assertTrue(res, msg='result check fail')
Log.debug('test_cancel_task_type_2_success end')
pass
def test_cancel_task_reason_too_long_fail(self):
"""HK reason为超长，取消任务失败"""
Log.debug('test_cancel_task_reason_too_long_fail start')
para_id = 21
data_id = 21007
res = self.cancel_task.base_cancel_task(para_id=para_id,
data_id=data_id,
cookies=self.admin_cookies,
flag=False,
task_modify=0,
reason_modify=1)
self.assertTrue(res, msg='result check fail')
Log.debug('test_cancel_task_reason_too_long_fail end')
pass
def test_compute_vcf_success(self):
"""HK vcf协同计算，成功"""
Log.debug('test_compute_vcf_success start')
para_id = 22
data_id = 22001
res = self.comp_vcf.base_compute_vcf(para_id=para_id,
data_id=data_id,
cookies=self.admin_cookies,
flag=True,
calType=self.calType,
maxFile=0)
self.assertTrue(res, msg='result check fail')
Log.debug('test_compute_vcf_success end')
pass
def modify_check(self, res):
""" 修改密码结果验证，检查请求返回值和数据库
1 成功 0 失败"""
sql_select = 'SELECT password FROM t_user WHERE loginName = \'Automation\''
code = '00000'
msg = "成功"
if res["code"] == code and res["message"] == msg \
and self.af == self.s.select_single(sql_select):
Log.debug('actual res check is 1')
return 1
else:
Log.debug('actual res check is 0')
return 0
def test_compute_mul_no_session_fail(self):
"""HK 未登录，批量计算失败"""
Log.debug('test_compute_mul_no_session_fail start')
para_id = 26
data_id = 26002
res = self.comp_vcf.base_compute_mul(para_id=para_id,
data_id=data_id,
cookies=None,
flag=False,
maxId=0,
maxFile=0)
self.assertTrue(res, msg='result check fail')
Log.debug('test_compute_mul_no_session_fail end')
pass
def test_compute_mul_calType_2_fail(self):
"""HK calType=2，批量计算失败"""
Log.debug('test_compute_mul_calType_2_fail start')
para_id = 26
data_id = 26004
res = self.comp_vcf.base_compute_mul(para_id=para_id,
data_id=data_id,
cookies=self.admin_cookies,
flag=False,
maxId=0,
maxFile=0)
self.assertTrue(res, msg='result check fail')
Log.debug('test_compute_mul_calType_2_fail end')
pass
def test_compute_mul_file_not_exist_fail(self):
"""HK fileid不存在，批量计算失败"""
Log.debug('test_compute_mul_file_not_exist_fail start')
para_id = 26
data_id = 26005
res = self.comp_vcf.base_compute_mul(para_id=para_id,
data_id=data_id,
cookies=self.admin_cookies,
flag=False,
maxId=0,
maxFile=1)
self.assertTrue(res, msg='result check fail')
Log.debug('test_compute_mul_file_not_exist_fail end')
pass
def test_compute_mul_success(self):
"""HK 批量计算成功，成功"""
Log.debug('test_compute_mul_success start')
para_id = 26
data_id = 26001
res = self.comp_vcf.base_compute_mul(para_id=para_id,
data_id=data_id,
cookies=self.admin_cookies,
flag=True,
maxId=0,
maxFile=0)
self.assertTrue(res, msg='result check fail')
Log.debug('test_compute_mul_success end')
pass
def test_compute_vcf_taskId_wrong_3_fail(self):
"""HK vcf协同计算，taskid为GWAS的任务，失败"""
Log.debug('test_compute_vcf_taskId_wrong_3_fail start')
para_id = 22
data_id = 22006
res = self.comp_vcf.base_compute_vcf(para_id=para_id,
data_id=data_id,
cookies=self.admin_cookies,
flag=False,
calType=3,
maxFile=0)
self.assertTrue(res, msg='result check fail')
Log.debug('test_compute_vcf_taskId_wrong_3_fail end')
pass
def after_update_check(self, res, status):
""" 修改用户信息结果验证，检查请求返回值和数据库
1 成功 0 失败"""
code = '00000'
msg = "成功"
data = self.s.select_dic_single(self.sql_select_update)
Log.info("修改用户信息后用户状态:name={}，status={}".format(
data['name'], data['status']))
# 检查数据库group
if data['group'] != self.node:
Log.error('group error ! only {} allowed,it\'s {}'.format(
self.node, data['group']))
return 0
if status in self.user_status:
if res["code"] == code and res["message"] == msg \
and status == data['status']:
Log.debug('status valid,actual res check is 1')
return 1
else:
Log.debug('status valid,actual res check is 0')
return 0
elif res["code"] == code and res["message"] == msg:
Log.debug('status invalid,actual res check is 1')
return 1
else:
Log.debug('status invalid,actual res check is 0')
return 0
def get_responses(self,Url,method,Request_Data,Request_Data_Type):
try:
res = None
if method == 'POST' or method == 'POST':
res = requests.post(Url,data=json.dumps(Request_Data),headers=Request_Data_Type)
if method == 'GET' or method == 'get':
res = requests.get(Url,data=json.dumps(Request_Data),headers=Request_Data_Type)
responese = res.json()
return responese
# code = responese.get("code")
# msg = responese.get("msg")
# content = {"code":code,"msg":msg}
# return content
except HTTPError as e:
Log.debug('响应失败',e)
def run(self, dao, req):
Log.info('twitter_friendsを全件取得。')
friends = self.dao.table('twitter_friends').find({})
Log.info('インフルエンサー毎に過去15日のツイートを集計する。')
for friend in friends:
cur_repo = self.dao.table('stock_report_pre').find({'tweets.user_id': friend['id_str']})
month_ago = DateTimeUtil.now() - timedelta(days=15)
user_tweets = []
for r in cur_repo:
u_tws = [t for t in r['tweets'] if t['user_id'] == friend['id_str'] and t['created_at'] >= month_ago]
if u_tws:
ccode_group_list = [t for t in user_tweets if t['ccode'] == r['ccode']]
ccode_group = {'name': '', 'last_update_date': month_ago, 'tweet': []}
if ccode_group_list:
ccode_group = ccode_group_list[0]
else:
ccode_group['ccode'] = r['ccode']
brand = self.dao.table('stock_brands').find_one({'ccode': r['ccode']})
if brand:
ccode_group['name'] = brand['name']
user_tweets.append(ccode_group)
ccode_group['last_update_date'] = max(
ccode_group['last_update_date'], max([t['created_at'] for t in u_tws]))
ccode_group['tweet'].extend(u_tws)
user_tweets = sorted(user_tweets, key=lambda x: x['last_update_date'], reverse=True)
ret = []
for r in user_tweets:
r['is_market_time'] = DateTimeUtil.is_market_time(r['last_update_date'])
str_day = r['last_update_date'].strftime('%Y/%m/%d')
dat = [d for d in ret if d['str_day'] == str_day]
if dat:
dat[0]['tweet'].append(r)
else:
buf = {'str_day': str_day, 'day': r['last_update_date'], 'tweet': [r]}
ret.append(buf)
friend['tweet_summary'] = ret
dao.table('twitter_friends').update_one({'id_str': friend['id_str']}, {'$set': friend})
Log.debug('登録: {}', friend['screen_name'])
Log.info('終了')
def new_check(res, flag):
"""response对象.json对象异常捕获判定文件下载是否成功"""
try:
json_res = res.json()
if not flag:
Log.error('请求返回有误：{}'.format(json_res))
Log.info('test success')
return flag
else:
Log.error('文件下载失败，请求异常{}'.format(json_res))
return not flag
except JSONDecodeError as je:
Log.info('-content type-:{}'.format(type(res.content)))
Log.debug('-contest:{}-'.format(res.content))
Log.info('-contest is None :{}'.format(res.content is None))
return flag
except Exception as e:
Log.error('未知异常:{}'.format(e))
return not flag
def get_report_search_results(self,
start_day: datetime,
end_day: datetime,
start_page=1,
page_count=None):
"""
EDINETの書類検索（全文検索）の結果一覧を取得する。
対象は「有価証券報告書」「文字列に『大株主の状況』を含む」「指定期間」
検索一覧行オブジェクトリストを返す。
start_page(1始まり)、page_countの指定がない場合は全ページ対象。
"""
rows, idx, page = [], 0, 1
while True:
url = EdinetUrl.search_url.value.format(start_day.year,
start_day.month,
start_day.day,
end_day.year,
end_day.month, end_day.day,
str(idx))
html = requests.get(url,
timeout=20,
headers=self.user_agent,
verify=False)
bs = BeautifulSoup(html.text, "lxml")
if start_page <= page:
table = bs.find_all('table', {'class': 'resultTable'})
row_datas = self.get_row_datas(table)
rows.extend(row_datas)
Log.debug('{}ページ読込。計{}件'.format(page, len(rows)))
if page_count and page_count == page + 1 - start_page:
break
if not self.has_next_page(bs):
break
idx = idx + 100
page = page + 1
return rows
def download_check(res, flag):
# code = 13333
Log.debug('test1:res是否有.json:{}'.format(res.json()))
if not flag:
try:
json_res = res.json()
Log.error('请求返回有误：{}'.format(json_res))
return 0
except Exception:
Log.error("返回错误")
return 1
# if not flag and eval(res.json()["code"]) == code:
# Log.info('请求返回错误:{}'.format(res.json()))
# return 0
elif res.content is not None and res.status_code == 200:
Log.info('文件下载接口ok')
return 1
return not flag
def gen_check(self, res):
code = '00000'
# taskId = res['data']
# Log.debug('taskId is {}'.format(taskId))
# [data_n, data_o] = self.get_data(taskId)
# Log.info('本端数据为： {}'.format(data_n))
# Log.info('对端数据为： {}'.format(data_o))
if res['code'] == code:
taskId = res['data']
Log.debug('taskId is {}'.format(taskId))
[data_n, data_o] = self.get_data(taskId)
Log.info('本端数据为： {}'.format(data_n))
Log.info('对端数据为： {}'.format(data_o))
if self.compare(data_n, data_o):
Log.info('compare result is True ,success !')
return 1
else:
Log.error('compare result is False ,fail !')
return 0
Log.error('请求返回有误！{}'.format(res))
return 0
def get_stock_details_async(self, ccodes):
def _cb(results, code):
return results
start = 0
end = min(len(ccodes), 200)
ret = []
while True:
current_list = ccodes[start:end]
Log.debug('kabtan request. len(ccodes)={}, ccodes[{}:{}]'.format(
len(ccodes), start, end))
result, err = AsyncWorker(self.get_stock_detail,
_cb).go(current_list)
if not result is None:
ret.extend(result)
Log.debug('len(ret)= {} '.format(len(ret)))
time.sleep(10)
start = end
end = min(len(ccodes), end + 200)
if end >= len(ccodes):
return ret
def get_error_result_check(self, taskId, res, flag):
"""1 成功 0 失败"""
code = '00000'
if flag and res['code'] != code:
Log.error('请求返回应该为成功，实际失败 {}'.format(res))
return 0
if not flag and res['code'] == code:
Log.error('异常用例请求返回应该为失败，实际成功 {}'.format(res))
return 1
if res["code"] != code:
Log.error('返回错误：{}'.format(res))
return 0
sql_res = 'select * from t_task_history where id = %d' % taskId
Log.debug('====查询sql：{}'.format(sql_res))
data_database = self.con_n.select_dic_single(sql_res)
Log.debug('=数据库查询结果={}={}='.format(data_database['errType'],
data_database['errRole']))
data_res = res['data']
Log.debug('*接口返回*{}*{}*'.format(data_res['errType'],
data_res['errRole']))
if data_database['errType'] == data_res['errType'] and data_database[
'errRole'] == data_res['errRole']:
Log.info('data equal, compare success')
return 1
Log.error('data different, compare fail')
return 0
def vcf_status_check(self, res,status):
code = '00000'
if res['code'] == code:
taskId = res['data']
Log.debug('taskId is {}'.format(taskId))
time.sleep(5)
if status == 1:
[data_n, data_o] = self.get_data(taskId)
else:
[data_n, data_o] = self.get_data_history(taskId)
Log.info('本端状态为： {}'.format(data_n['status']))
Log.info('对端状态为： {}'.format(data_o['status']))
Log.info('期望状态为：{}'.format(status))
if data_n['status'] == data_o['status'] and data_n['status'] == status:
Log.info('双端状态与期望状态一致，用例执行成功 !')
return status
else:
Log.error('双端状态与期望状态不一致，用例执行失败 !')
return 0
Log.error('请求返回有误！{}'.format(res))
return 0
def get_maf_result_check(self, taskId, res, flag):
"""1 成功 0 失败"""
code = '00000'
if flag and res['code'] != code:
Log.error('请求返回应该为成功，实际失败 {}'.format(res))
return 0
if not flag and res['code'] == code:
Log.error('异常用例请求返回应该为失败，实际成功 {}'.format(res))
return 1
if res["code"] != code:
Log.error('返回错误：{}'.format(res))
return 0
sql_res = 'select count(*) from t_task_result where taskId = %d and logicType=1' % taskId
data_database = self.con_n.select_single(sql_res)
Log.debug('=={}=='.format(data_database))
data_res = res['totalCount']
Log.debug('**{}**'.format(data_res))
if data_database == data_res:
Log.info('data equal, compare success')
return 1
Log.error('data different, compare fail')
return 0
def cancel_check(self, res, temp, flag):
code = '00000'
if flag and res['code'] != code:
raise AssertionError('请求返回应该为成功，实际失败 {}'.format(res))
if not flag and res['code'] == code:
raise AssertionError('异常用例请求返回应该为失败，实际成功 {}'.format(res))
if not res['code'] == code:
Log.error('请求返回有误！{}'.format(res))
return 0
else:
if temp['startTaskId'] is None:
status = 4
id = temp['id']
sql_n = 'select * from t_task_history where id = %d ' % id
sql_o = 'select * from t_task_history where startTaskId = %d ' % id
else:
status = 5
id = temp['startTaskId']
sql_n = 'select * from t_task_history where startTaskId = %d ' % id
sql_o = 'select * from t_task_history where id = %d ' % id
Log.debug('本端查询sql：{}'.format(sql_n))
Log.debug('对端查询sql：{}'.format(sql_o))
data_n = self.con_n.select_dic_single(sql=sql_n)
Log.info('本端数据为： {}'.format(data_n))
time.sleep(10)
try:
data_o = self.con_o.select_dic_single(sql=sql_o)
except IndexError as e:
raise AssertionError('数据库查询为空，exception is:{}'.format(e))
Log.info('对端数据为： {}'.format(data_o))
if self.compare(data_n, data_o, status):
Log.info('compare result is True ,success !')
return 1
else:
Log.error('compare result is False ,fail !')
return 0
def run(self, dao: Dao, req):
Log.info("Task001 invoked.")
mecab_parser = MeCabParser()
tweet_ids = req['param']
for id_str in tweet_ids:
Log.debug('caller={}, reqeust tweet_id={}', req['caller'], id_str)
t = self.find_tweet_by_id(id_str)
# つぶやきをクリーニング
self.tweet_text_cleaning(t)
# つぶやきを単語分割
mecab_dic = mecab_parser.parse(t.text)
# つぶやき内から銘柄取得
stocks_in_tweet = self.find_stock_code_in(mecab_dic)
for b in stocks_in_tweet.values():
Log.info('株 みっけ!! : {} {} user={}', b.ccode, b.name,
t.user_name)
Log.info('tweet={}', t.text)
stock_repos = self.model.find_stock_report_by_ccode(b.ccode)
if stock_repos:
tweets = list(stock_repos.tweets)
if len([
tw for tw in tweets if tw['id_str'] == t['id_str']
]) == 0:
Log.debug('カウントアップ!! : {} {}, user={}', b.ccode,
b.name, t.user_name)
tweets.append(t)
stock_repos.tweets = tweets
stock_repos.last_updated_at = t.created_at
stock_repos.last_update_user = t.user_id
self.upsert_stock_report(b, stock_repos)
else:
Log.debug('初登場!! : {} {}, user={}', b.ccode, b.name,
t.user_name)
data = {
'ccode': b.ccode,
'name': b.name,
'create_user': t.user_id,
'created_at': t.created_at,
'last_updated_at': t.created_at,
'last_update_user': t.user_id,
'tweets': [t]
}
self.upsert_stock_report(b, data)
Log.info('Task004に通知')
MqCaller('Task001').call('Task004', {
'stocks': stocks_in_tweet,
'tweet_id': t['id_str']
})
Log.info('Task003に通知')
MqCaller('Task001').call('Task003', None)
def check(self):
# 检查update用户是否存在，不存在插入一条update用户数据
if 0 == len(self.s.select(self.s_select_u)):
Log.debug('update用户不存在，t_user表插入update:{}'.format(self.s_insert_u))
self.s.updata(self.s_insert_u)
# 检查Automation用户存在,不存在新增数据
if 0 == len(self.s.select(self.s_select_a)):
Log.debug('Automation用户不存在，t_user表插入update:{}'.format(self.s_insert_a))
self.s.updata(self.s_insert_a)
# 检查Automation用户初始密码是否为123456，不是则更改
if self.bf != self.s.select_dic_single(self.s_select_a)['password']:
Log.debug('Automation用户起始密码不为123456，更改为123456:{}'.format(self.s_update_a))
self.s.updata(self.s_update_a)
def compare_gen(self, data_database, data_res):
Log.debug('=数据库原始数据={}===='.format(data_database))
Log.debug('*接口返回原始数据*{}**'.format(data_res))
calTime = data_res['calTime']
calTime = Base.str2sec(calTime)
loadTime = data_res['loadTime']
loadTime = Base.str2sec(loadTime)
# 数据库加载时长毫秒转换为s，不足1s按1s返回，超出1s向下取整
data_database['loadTime'] = int(data_database['loadTime'] / 1000)
data_database['loadTime'] = 1 if data_database[
'loadTime'] == 0 else data_database['loadTime']
Log.debug('=处理后数据库={}={}==='.format(data_database['loadTime'],
data_database['calTime']))
Log.debug('*处理后接口返回*{}*{}*'.format(loadTime, calTime))
if data_database['calTime'] == calTime and data_database[
'loadTime'] == loadTime:
return True
return False
def compare_vcf(self, data_database, data_res):
Log.debug('=={}===='.format(data_database))
Log.debug('**{}**'.format(data_res))
calTime = data_res[0]['calTime']
calTime = Base.str2sec(calTime)
loadTime = data_res[0]['loadTime']
loadTime = Base.str2sec(loadTime)
target = data_res[0]['target']
# 数据库加载时长毫秒转换为s，不足1s按1s返回，超出1s向下取整
data_database['loadTime'] = int(data_database['loadTime'] / 1000)
data_database['loadTime'] = 1 if data_database[
'loadTime'] == 0 else data_database['loadTime']
Log.debug('**{}-{}-{}**'.format(calTime, loadTime, target))
Log.debug('**{}-{}-{}**'.format(
data_database['calTime'],
math.ceil(data_database['loadTime'] / 1000),
data_database['target']))
if data_database['calTime'] == calTime and data_database['loadTime'] == loadTime \
and data_database['target'] == target:
return True
return False
def base_upload(self, para_id, data_id, cookies, isChange=0, noFile=0):
"""isChange:0 不变更文件名，1 变更文件名
noFile:0 上传文件，1 无文件
"""
# 获取请求url
url_upload = self.domain + Base.dh.get_path(para_id)
Log.info('upload request url : {}'.format(url_upload))
# 获取请求数据
data_source = self.dh.get_data(data_id)
req_para = Base.get_req_para(para_id=para_id, data_id=data_id)
# 是否含上传文件
Log.debug('文件绝对路径：{}'.format(self.gene_file(req_para['fileName'])))
Log.debug('********：{}'.format(
type(self.gene_file(req_para['fileName']))))
if not noFile:
f = open(self.gene_file(req_para['fileName']), 'rb')
Log.debug('*********{}****'.format(type(f)))
files = {'file': f}
Log.info('要上传的文件是 :{}'.format(f.name))
else:
files = {}
Log.info('上传文件为空')
# 是否更改文件名
if isChange:
req_para['fileName'] = self.change_name(req_para['fileName'])
data_source[0][5] = req_para['fileName']
# 接口数据类型转换
req_para['fileType'] = eval(req_para['fileType'])
req_para['fileSize'] = eval(req_para['fileSize'])
Log.info('upload request data is {}'.format(json.dumps(req_para)))
# 请求接口
res = requests.post(url=url_upload,
cookies=cookies,
data=req_para,
files=files).json()
Log.info('upload response data is {}'.format(res))
# 结果检查
actual = self.upload_check(res)
# 结果写入
DataHandle.set_data(data_source[0], actual)
self.dh.write_data(data_source)
# 结果检查
return self.dh.check_result(data_source)
