def task001_handler(event, context):
print(DateTimeUtil.str_now())
dao = Dao()
# 前回取得した最後のtweetidを取得
dat = dao.table('condition').get_item(Key={'key': 'Task001_last_tweet_id'})
last_tweet_id = dat['val']
print('last_tweet_id: ', last_tweet_id)
tw = TwitterInspector(Irebaburn)
timeline = tw.get_list_timeline_rotate(list_name='株',
count=1000,
last_data_id=last_tweet_id)
print(timeline)
if timeline:
ids = [t['id_str'] for t in timeline]
# 重複取得されたレコードがある場合、削除して入れなおすため、一旦消す。
dao.table('tweet').delete_batch_silent(ids)
# ids = []
# for tweet in timeline:
# # 重複取得されたレコードがある場合、削除して入れなおすため、一旦消す。
# dao.table('tweet').delete_item_silent({'id_str': tweet['id_str']})
#
# ids.append(tweet['id_str'])
# 追加
dao.table('tweet').insert(timeline)
last_tweet_id = timeline[0]['id_str'] # 降順の先頭(取得した最新)
print('last_tweet_id for update: ', last_tweet_id)
# last_tweet_id 更新
dao.table("condition").update_item(
Key={"key": dat['key']},
ExpressionAttributeValues={
':val': last_tweet_id,
':dt': DateTimeUtil.str_now()
},
UpdateExpression="set val = :val, update_time = :dt")
# Task002呼び出し
boto3.client("lambda").invoke(
FunctionName=
"arn:aws:lambda:ap-northeast-1:007575903924:function:Task002",
InvocationType="Event",
Payload=json.dumps({"ids": ids}))
return ''
def task_exp_handler(event, context):
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
print(DateTimeUtil.str_now())
dao = Dao()
dao_edi = dao.table('stock_edinet')
edis = dao_edi.full_scan()
print('len(edi) = {}'.format(len(edis)))
for edi in edis:
if 'holders' not in edi:
edi['holders'] = 'non'
if 'holder_rate' not in edi:
edi['holder_rate'] = 'non'
if 'outstanding_share' not in edi:
edi['outstanding_share'] = 'non'
cd = edi['ccode']
nm = edi['name']
hr = edi['holder_rate']
ot = edi['outstanding_share']
tot = get(hr, 'tot')
tabs = '{}\t' * 6
print(
tabs.format(cd, nm, get(hr, 'unit'), get(tot, 1),
get(ot, 'hutuu_hakkou'), get(ot, 'unit')))
return ''
def task004_handler(event, context):
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
print(DateTimeUtil.str_now())
lambda_cnt = 6
dao = Dao()
brands = dao.table('stock_brands').full_scan()
ccode_list = [r['ccode'] for r in brands]
ccode_list.append('998407') # 日経平均
ccode_list.append('USDJPY') # ドル円
q = len(ccode_list) // lambda_cnt
chank_ccode = [ccode_list[i:i + q] for i in range(0, len(ccode_list), q)]
if len(chank_ccode) > lambda_cnt:
mod_chank = chank_ccode.pop(len(chank_ccode) - 1)
chank_ccode[-1].extend(mod_chank)
for i, ccodes in enumerate(chank_ccode, start=1):
func_name = 'arn:aws:lambda:ap-northeast-1:007575903924:function:Price{0:03d}'.format(
i)
cds = '_'.join(ccodes)
print(func_name)
boto3.client("lambda").invoke(FunctionName=func_name,
InvocationType="Event",
Payload=json.dumps({"cds": cds}))
return 0
def web010_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
uid = event['uid']
print('uid = {}'.format(uid))
tfs = dao.table('twitter_friends_sum').find_by_key(uid)
ranks = sorted(tfs['rank'].items(),
key=lambda r: len(r[1]['ds']),
reverse=True)
ranks = ranks[:min(20, len(ranks))]
ret = []
for r in ranks:
rank = {
'cd': r[0],
'nm': r[1]['nm'],
'ct': len(r[1]['ds']),
'd': r[1]['ds'][0]['d']
}
ret.append(rank)
tweet = {}
if 'tid' in tfs:
tweet = dao.table('tweet').find_by_key(tfs['tid'])
del tweet['id_str']
del tweet['user_id']
del tweet['user_name']
del tweet['user_screen_name']
cond = dao.table('condition').find_by_key('Task002_tweet_report_update')
ob = {'d': cond['update_time'], 't': tweet, 'r': ret}
return response({"v": "1", "tfs": ob})
def task006_handler(event, context):
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
print(DateTimeUtil.str_now())
dao = Dao()
ident_list = dao.table('stock_identify').full_scan()
data = {}
for ident in ident_list:
cd = ident['ccode']
dat = {} if cd not in data else data[cd]
if 'ns' in dat:
dat['ns'] = dat['ns'].split('_')
dat['ns'].append(ident['nm'])
else:
dat['ns'] = [ident['nm']]
if ident['main'] == 'y':
dat['n'] = ident['nm']
dat['ns'] = '_'.join(dat['ns'])
data[cd] = dat
ret = []
for d in data.items():
d[1]['c'] = d[0]
ret.append(d[1])
s3 = boto3.resource('s3').Bucket("kabupac.com")
s3.put_object(Key="suggest/dat.json", Body=JSON.dumps(ret))
return ''
def notify_list_pages_update_to_condition(dao, val):
dao.table("condition").update_item(
Key={"key": "Task003_list_pages_update"},
ExpressionAttributeValues={
':val': val,
':dt': DateTimeUtil.str_now()
},
UpdateExpression="set val = :val, update_time = :dt"
)
def price006_handler(event, context):
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
print(DateTimeUtil.str_now())
logic = PriceLogic()
logic.now_price_update(event["cds"].split('_'))
logic.notify_now_price_update_to_condition()
return ''
def notify_now_price_update_to_condition(self):
self.dao.table("condition").update_item(
Key={"key": "Task004_now_price_update"},
ExpressionAttributeValues={
':val': '-',
':dt': DateTimeUtil.str_now()
},
UpdateExpression="set val = :val, update_time = :dt"
)
def web002_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
cd = event['cd']
print('cd={}'.format(cd))
brand = dao.table("stock_brands").find_by_key(cd)
thema = dao.table("stock_thema_ccode").find_by_key(cd)
if thema:
brand['thema'] = thema['nms']
return response({"v": "1", "brand": brand})
def web004_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
ids = event['ids']
print('ids = {}'.format(ids))
id_list = ids.split('_')
friends = dao.table("twitter_friends").find_batch(id_list)
return response({"v": "1", "friends": friends})
def web005_handler(event, context):
""" 現在価格API """
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
logic = PriceLogic()
uptime = logic.get_now_price_update_time()
cds = event['cds']
print('cds = {}'.format(cds))
cd_list = cds.split('_')
prices = dao.table("stock_price_now").find_batch(cd_list)
return response({ "v": "1", "prices": prices, "now_price_uptime": uptime })
def web008_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
prices = dao.table("stock_price_history").find_query({'cd': event['cd']})
if prices:
if 'lt' in event and event['lt']:
prices = [r for r in prices if r['d'] > event['lt']]
else:
s = 0 if len(prices) <= 30 else abs(len(prices) - 30)
prices = prices[s:]
else:
prices = []
return response({"v": "1", "ps": prices})
def web001_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
page = event['p'] if 'p' in event else "1"
print('page = {}'.format(page))
tbl, uptime = get_available_table(dao)
if tbl == 'err':
return response({"v": "1", "pages": [], "repos": [], "upt": "err"})
repo_list_page = dao.table(tbl).find_by_key(page)
ccodes = repo_list_page["ccodes"].split(',')
repo_list_page["ccodes"] = ccodes
repos = dao.table("stock_report").find_batch(ccodes)
for repo in repos:
repo['tweets'] = str(len(repo['tweets']))
return response({"v": "1", "pages": repo_list_page, "repos": repos, "upt": uptime })
def web007_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
cd = event['cd']
print('cd = {}'.format(cd))
edi_dat = dao.table("stock_edinet").find_by_key(cd)
r = {}
if edi_dat:
r = {
'ho': edi_dat['holders'] if 'holders' in edi_dat else {},
'ra': edi_dat['holder_rate'] if 'holder_rate' in edi_dat else {},
'os': edi_dat['outstanding_share']
if 'outstanding_share' in edi_dat else {},
}
return response({"v": "1", "edi": r})
def web003_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
cd = event['cd']
last_tweet_id = event['lt'] if 'lt' in event else None
direction = event['dr'] if 'dr' in event else None
print('cd = {}, last_tweet_id = {}, direction = {}'.format(cd, last_tweet_id, direction))
repo = dao.table("stock_report").find_by_key(cd)
if not repo:
return response({"v": "1", "tweets": []})
tweet_ids = repo['tweets']
end_id = tweet_ids[0]
tweets = dao.table("tweet").find_batch(tweet_ids)
if tweets:
tweets = sorted(tweets, key=lambda r: r['created_at'], reverse=True)
tweets = tweets[:30]
target_tweets = []
more_flg = False
for tw in tweets:
if direction == 'new':
if last_tweet_id == tw['id_str']:
break
target_tweets.append(tw)
else:
if last_tweet_id == tw['id_str']:
more_flg = True
continue
elif more_flg:
target_tweets.append(tw)
if target_tweets:
if target_tweets[-1]["id_str"] == end_id:
target_tweets[-1]["e"] = 1
return response({"v": "1", "tweets": target_tweets})
def web006_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
cd = event['cd']
last_id = event['lt'] if 'lt' in event else None
print('cd = {}, last_id = {}'.format(cd, last_id))
irs = dao.table("stock_ir").find_query({'cd': cd}, asc=False)
buf_irs = []
if last_id:
for ir in irs:
# stock_ir テーブルはソートキー「d」の降順。
if ir['tid'] == last_id:
break
buf_irs.append(ir)
else:
buf_irs = irs
return response({"v": "1", "irs": buf_irs})
def web011_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
key = event['k']
p = int(event['p']) - 1
print('key = {}'.format(key))
thema = dao.table('stock_thema_nm').find_by_key(key)
ccodes, repos = [], []
if thema:
ccodes = thema['ccodes'].split(',')
repos = dao.table('stock_report').find_batch(ccodes)
repos = repos[p * 10:min((p + 1) * 10, len(repos))]
for repo in repos:
repo['tweets'] = str(len(repo['tweets']))
return response({"v": "1", "cds": ccodes, "repos": repos})
def web009_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
cd = event['cd']
print('cd = {}'.format(cd))
high_low = dao.table("stock_brands_high_low").find_query({'ccode': cd},
asc=False)
r = {}
if high_low:
high = [r for r in high_low if r['mode'] == 'high']
low = [r for r in high_low if r['mode'] == 'low']
r = {'h': high, 'l': low}
rise_fall = dao.table("stock_brands_rise_fall").find_query({'ccode': cd},
asc=False)
if rise_fall:
rise = [r for r in rise_fall if r['mode'] == 'rise']
fall = [r for r in rise_fall if r['mode'] == 'fall']
r['r'] = rise
r['f'] = fall
return response({"v": "1", "hl": r})
def run(self, dao: Dao, h1):
edinet = Edinet()
start_day = DateTimeUtil.yesterday()
end_day = DateTimeUtil.now()
# start_day = datetime(2018, 5, 19)
# end_day = datetime(2018, 5, 20)
h1('EDINETの書類検索から指定期間の有価証券報告書の検索結果を取得する。')
search_rows = edinet.get_report_search_results(start_day, end_day)
search_rows.reverse()
# search_rows = [r for r in search_rows if 'スタートトゥデイ' in r['company_name']]
h1('検索結果から有価証券報告書を取得し、銘柄毎スクレイピング開始。')
dao_identify = dao.table('stock_identify')
dao_brand = dao.table('stock_brands')
not_brands = []
for search_row in search_rows:
data_html = edinet.get_report_html(search_row['syorui_kanri_no'])
shorui_mei = search_row['syorui_mei']
Log.info('html取得完了 : {} : {}'.format(search_row['company_name'],
shorui_mei))
if '大株主' not in data_html:
Log.warn('HTMLに大株主が無い。別資料かも。 company_name : {}'.format(
search_row['company_name']))
continue
table_list = edinet.report_html_to_split_table_list(
data_html, shorui_mei)
Log.info('HTML -> table分割 : {}'.format(search_row['company_name']))
dat = {}
for t in table_list:
if t['title'] == '発行済株式' and 'outstanding_share' not in dat:
dat['outstanding_share'] = edinet.outstanding_share(
t, search_row)
Log.debug('** 発行済株式 : {}'.format(
search_row['company_name']))
if t['title'] == '所有者別状況':
dat['holder_rate'] = edinet.holder_rate_status(
t, search_row)
Log.debug('** 所有者別状況 : {}'.format(
search_row['company_name']))
if t['title'] == '大株主の状況':
dat['holders'] = edinet.major_shareholders(t, search_row)
Log.debug('** 大株主の状況 : {}'.format(
search_row['company_name']))
if not dat:
continue
name = search_row['company_name'].replace('(株)',
'').replace(' ', '')
name = re.sub(r'^ ', '', name)
name = re.sub(r' $', '', name)
identify = dao_identify.find_query({'nm': name})
if not identify:
name_s = zenhan.z2h(name, mode=1)
identify = dao_identify.find_query({'nm': name_s})
if not identify:
name_s = name_s.replace(' ', '')
identify = dao_identify.find_query({'nm': name_s})
if identify and len(identify) == 1:
brand = dao_brand.find_by_key(identify[0]['ccode'])
Log.info('stock_brands証券コード取得。 ccode, name : {}, {}'.format(
brand['ccode'], name))
dat['ccode'] = brand['ccode']
dat['name'] = brand['name']
dat['date'] = DateTimeUtil.strf_ymd_st(
DateTimeUtil.date_from_japanese_era(
search_row['submition_day'], short=True))
if empty(dat, 'outstanding_share') or empty(
dat, 'holder_rate') or empty(dat, 'holders'):
edi = dao.table('stock_edinet').find_by_key(dat['ccode'])
if edi:
if empty(dat, 'outstanding_share') and not empty(
edi, 'outstanding_share'):
dat['outstanding_share'] = edi['outstanding_share']
if empty(dat, 'holder_rate') and not empty(
edi, 'holder_rate'):
dat['holder_rate'] = edi['holder_rate']
if empty(dat, 'holders') and not empty(edi, 'holders'):
dat['holders'] = edi['holders']
h1('stock_edinetに登録')
dao.table('stock_edinet').put_item(Item=dat)
else:
dao.table('err_value').put_item(
Item={
'key': 'Job013_edi_name',
'd': DateTimeUtil.str_now(),
'val': name
})
h1('終了')
def task002_handler(event, context):
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
print(DateTimeUtil.str_now(), 'Task002', event['ids'])
dao = Dao()
mecab_parser = MeCabParser()
tweet_ids = event['ids']
tbl_repo = dao.table("stock_report")
tbl_tweet = dao.table('tweet')
tbl_brands = dao.table('stock_brands')
tbl_price_now = dao.table('stock_price_now')
tbl_f_sum = dao.table('twitter_friends_sum')
is_stock_find = False
for id_str in tweet_ids:
t = tbl_tweet.find_by_key(id_str)
if t is None:
continue
t_sum = {}
tw_txt = t["text"]
if "retweet_text" in t:
tw_txt = t["retweet_text"]
# ツイートのクリーニング
clean_twtxt = Util.mask_twitter_name(Util.all_normalize(tw_txt))
# つぶやきを単語分割
mecab_dic = mecab_parser.parse(clean_twtxt)
# つぶやき内から銘柄取得
stocks_in_tweet, words = find_stock_code_in(mecab_dic, dao, tbl_brands)
if not is_stock_find:
is_stock_find = len(stocks_in_tweet) > 0
Log.info('len(stocks_in_tweet) = {}, is_stock_find = {}'.format(
len(stocks_in_tweet), is_stock_find))
for b in stocks_in_tweet.values():
tweet_in_ccode = b["ccode"]
Log.info('株 みっけ!! : {} {} user={}', tweet_in_ccode, b["nm"],
t["user_name"])
Log.info('tweet={}', clean_twtxt)
stock_repo = tbl_repo.find_by_key(tweet_in_ccode)
price = get_price(tweet_in_ccode, tbl_price_now)
if "prices" not in t:
t["prices"] = {}
if tweet_in_ccode in t["prices"]:
t["prices"][tweet_in_ccode].append(price)
else:
t["prices"][tweet_in_ccode] = [price]
if not t_sum:
t_sum = get_tweet_summary(t, tweet_in_ccode, tbl_f_sum)
if tweet_in_ccode not in t_sum['rank']:
t_sum['rank'][tweet_in_ccode] = {'ds': []}
t_sum['rank'][tweet_in_ccode]['nm'] = b['nm']
t_sum['rank'][tweet_in_ccode]['ds'].insert(0, {
'd': t["created_at"],
't': t['id_str']
})
if stock_repo:
tweets = stock_repo["tweets"]
# 価格をpriceに付けてしまったがTweetに付けるように変更したため、価格についたpriceがあれば削除する
if "prices" in stock_repo:
del stock_repo["prices"]
if len([tw for tw in tweets if tw == t['id_str']]) == 0:
Log.debug('カウントアップ!! : {} {}, user={}', b["ccode"],
b["nm"], t["user_name"])
tweets.append(t['id_str'])
stock_repo["tweets"] = tweets
stock_repo["ago_uptime"] = stock_repo["last_updated_at"]
stock_repo["last_updated_at"] = t["created_at"]
stock_repo["last_update_user"] = t["user_id"]
stock_repo["last_up_uname"] = t["user_name"]
retouch_stock_name(stock_repo, tbl_brands)
tbl_repo.put_item(Item=stock_repo)
else:
Log.debug('初登場!! : {} {}, user={}', b["ccode"], b["nm"],
t["user_name"])
data = {
'ccode': b["ccode"],
'name': b["nm"],
'create_user': t["user_id"],
'created_at': t["created_at"],
'last_updated_at': t["created_at"],
'last_update_user': t["user_id"],
'last_up_uname': t["user_name"],
'tweets': [t["id_str"]],
"prices": [price]
}
retouch_stock_name(data, tbl_brands)
tbl_repo.put_item(Item=data)
if words:
t['ws'] = words
tbl_tweet.put_item(Item=t)
if t_sum:
tbl_f_sum.put_item(Item=t_sum)
if is_stock_find:
# Task003呼び出し
Log.info('Task003 呼び出し')
boto3.client("lambda").invoke(
FunctionName=
"arn:aws:lambda:ap-northeast-1:007575903924:function:Task003",
InvocationType="Event",
Payload=json.dumps({}))
dao.table("condition").update_item(
Key={"key": "Task002_tweet_report_update"},
ExpressionAttributeValues={
':val': '-',
':dt': DateTimeUtil.str_now()
},
UpdateExpression="set val = :val, update_time = :dt")
return 0
def task005_handler(event, context):
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
print(DateTimeUtil.str_now())
dao = Dao()
mecab_parser = MeCabParser()
mecab_logic = MeCabLogic()
dao_ir = dao.table('stock_ir')
# 前回取得した最後のtweetidを取得
dat = dao.table('condition').get_item(Key={'key': 'Task005_last_tweet_id'})
last_tweet_id = dat['val']
print('last_tweet_id: ', last_tweet_id)
tw = TwitterInspector(Kabpackab)
timeline = tw.get_list_timeline_rotate(list_name='IR',
count=1000,
last_data_id=last_tweet_id)
print(timeline)
if timeline:
data_list = []
for t in timeline:
tw_txt = t["text"]
# ツイートのクリーニング
clean_twtxt = Util.mask_twitter_name(Util.all_normalize(tw_txt))
# つぶやきを単語分割
mecab_dic = mecab_parser.parse(clean_twtxt)
# つぶやき内から銘柄取得
stocks_in_tweet, _ = mecab_logic.find_stock_code_in(mecab_dic)
for b in stocks_in_tweet.values():
tweet_in_ccode = b["ccode"]
Log.info('IR みっけ!! : {} {} user={}', tweet_in_ccode, b["nm"],
t["user_name"])
data = {
'cd': b["ccode"],
'nm': b["nm"],
'u': t["user_id"],
'd': t["created_at"],
't': tw_txt,
'tid': t['id_str']
}
data_list.append(data)
dao_ir.insert(data_list)
last_tweet_id = timeline[0]['id_str'] # 降順の先頭(取得した最新)
print('last_tweet_id for update: ', last_tweet_id)
# last_tweet_id 更新
dao.table("condition").update_item(
Key={"key": dat['key']},
ExpressionAttributeValues={
':val': last_tweet_id,
':dt': DateTimeUtil.str_now()
},
UpdateExpression="set val = :val, update_time = :dt")
return ''