def mark_high_low(self, key, lst, mark):
ymd_3dayago = DateTimeUtil.strf_ymd(DateTimeUtil.prevday_weekday(3))
if len(lst) == 0:
mark[key] = '-'
else:
d = lst[0]['d'].split('_')[0]
mark[key] = '-' if d < ymd_3dayago else '*'
return mark
def task001_handler(event, context):
print(DateTimeUtil.str_now())
dao = Dao()
# 前回取得した最後のtweetidを取得
dat = dao.table('condition').get_item(Key={'key': 'Task001_last_tweet_id'})
last_tweet_id = dat['val']
print('last_tweet_id: ', last_tweet_id)
tw = TwitterInspector(Irebaburn)
timeline = tw.get_list_timeline_rotate(list_name='株',
count=1000,
last_data_id=last_tweet_id)
print(timeline)
if timeline:
ids = [t['id_str'] for t in timeline]
# 重複取得されたレコードがある場合、削除して入れなおすため、一旦消す。
dao.table('tweet').delete_batch_silent(ids)
# ids = []
# for tweet in timeline:
# # 重複取得されたレコードがある場合、削除して入れなおすため、一旦消す。
# dao.table('tweet').delete_item_silent({'id_str': tweet['id_str']})
#
# ids.append(tweet['id_str'])
# 追加
dao.table('tweet').insert(timeline)
last_tweet_id = timeline[0]['id_str'] # 降順の先頭(取得した最新)
print('last_tweet_id for update: ', last_tweet_id)
# last_tweet_id 更新
dao.table("condition").update_item(
Key={"key": dat['key']},
ExpressionAttributeValues={
':val': last_tweet_id,
':dt': DateTimeUtil.str_now()
},
UpdateExpression="set val = :val, update_time = :dt")
# Task002呼び出し
boto3.client("lambda").invoke(
FunctionName=
"arn:aws:lambda:ap-northeast-1:007575903924:function:Task002",
InvocationType="Event",
Payload=json.dumps({"ids": ids}))
return ''
def date_to_str(item):
if type(item) is dict:
for key in item:
buf = item[key]
if type(buf) in (dict, list):
Util.blank_to_none(buf)
if type(buf) is datetime:
item[key] = DateTimeUtil.strf_ymdhms(buf)
elif type(item) is list:
for i in range(len(item)):
row = item[i]
if type(row) in (dict, list):
Util.blank_to_none(row)
if type(row) is datetime:
item[i] = DateTimeUtil.strf_ymdhms(row)
def get_ranking(self,
mode,
start_page=1,
last_page=1,
period='d',
today=True,
func=None):
"""Yahooランキング取得"""
ret_list = []
p = start_page
update_date = None
while True:
url = "https://info.finance.yahoo.co.jp/ranking/?kd={}&mk=1&tm={}&vl=a&p={}".format(
mode, period, p)
html = requests.get(url, timeout=10)
bs = BeautifulSoup(html.text, "lxml")
if update_date is None:
update_date = self.get_update_date(bs)
if today and update_date < DateTimeUtil.today():
return []
trs = bs.find_all('tr', {'class': 'rankingTabledata'})
ret = func(trs, update_date)
ret_list.extend(ret)
if not self.has_next_page(bs) or last_page == p:
return ret_list
p = p + 1
def run(self, dao: Dao, h1):
h1('twitter_friends_sum取得')
tbl_tfs = dao.table('twitter_friends_sum')
tfs_list = tbl_tfs.full_scan()
d = DateTimeUtil.now() - timedelta(days=15)
two_week_ago = d.strftime('%Y/%m/%d_00:00:00')
h1('集計開始')
for tfs in tfs_list:
for cd in tfs['rank']:
ds = tfs['rank'][cd]['ds']
n_ds = []
for r in ds:
if r['d'] >= two_week_ago:
n_ds.append(r)
if n_ds:
tfs['rank'][cd]['ds'] = n_ds
else:
print("tfs['rank'][cd] = 空. cd={}".format(cd))
del tfs['rank'][cd]
if tfs['rank']:
tbl_tfs.put_item(Item=tfs)
else:
print('空。 uid={}'.format(tfs['uid']))
tbl_tfs.delete_item_silent({"S": tfs['uid']})
h1('終了')
def convert_date(self, str_date):
str_date = self.norm(str_date)
if '平成' in str_date:
jap = DateTimeUtil.date_from_japanese_era(str_date)
if jap:
return DateTimeUtil.strf_ymd_st(jap)
else:
return '-'
else:
m = re.search('(\d+)年(\d+)月(\d+)日', str_date)
if m:
return DateTimeUtil.strf_ymd_st(
datetime(int(m.group(1)), int(m.group(2)),
int(m.group(3))))
else:
return '-'
def web010_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
uid = event['uid']
print('uid = {}'.format(uid))
tfs = dao.table('twitter_friends_sum').find_by_key(uid)
ranks = sorted(tfs['rank'].items(),
key=lambda r: len(r[1]['ds']),
reverse=True)
ranks = ranks[:min(20, len(ranks))]
ret = []
for r in ranks:
rank = {
'cd': r[0],
'nm': r[1]['nm'],
'ct': len(r[1]['ds']),
'd': r[1]['ds'][0]['d']
}
ret.append(rank)
tweet = {}
if 'tid' in tfs:
tweet = dao.table('tweet').find_by_key(tfs['tid'])
del tweet['id_str']
del tweet['user_id']
del tweet['user_name']
del tweet['user_screen_name']
cond = dao.table('condition').find_by_key('Task002_tweet_report_update')
ob = {'d': cond['update_time'], 't': tweet, 'r': ret}
return response({"v": "1", "tfs": ob})
def task004_handler(event, context):
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
print(DateTimeUtil.str_now())
lambda_cnt = 6
dao = Dao()
brands = dao.table('stock_brands').full_scan()
ccode_list = [r['ccode'] for r in brands]
ccode_list.append('998407') # 日経平均
ccode_list.append('USDJPY') # ドル円
q = len(ccode_list) // lambda_cnt
chank_ccode = [ccode_list[i:i + q] for i in range(0, len(ccode_list), q)]
if len(chank_ccode) > lambda_cnt:
mod_chank = chank_ccode.pop(len(chank_ccode) - 1)
chank_ccode[-1].extend(mod_chank)
for i, ccodes in enumerate(chank_ccode, start=1):
func_name = 'arn:aws:lambda:ap-northeast-1:007575903924:function:Price{0:03d}'.format(
i)
cds = '_'.join(ccodes)
print(func_name)
boto3.client("lambda").invoke(FunctionName=func_name,
InvocationType="Event",
Payload=json.dumps({"cds": cds}))
return 0
def convert_history_yahoo_to_pacpac(self, yahoo_list, ccode):
pac_list = []
for yh in yahoo_list:
d = DateTimeUtil.strf_ymd_st(yh['date'])
# d: date, o: open, l: low, h: high, c: close, v: volume
# j: jika_sougaku, b: pbr, e: per, t: nen_taka, y: nen_yasu, k: kabusuu
if ccode == 'USDJPY':
pac_list.append({
"cd": ccode,
"d": d,
"o": re.sub('0000$', '', yh["open"]),
"l": re.sub('0000$', '', yh["low"]),
"h": re.sub('0000$', '', yh["high"]),
"c": re.sub('0000$', '', yh["close"]),
"v": "-"
})
else:
pac_list.append({
"cd": ccode,
"d": d,
"o": str(int(yh["open"])) if yh["open"] != '-' else '-',
"l": str(int(yh["low"])) if yh["low"] != '-' else '-',
"h": str(int(yh["high"])) if yh["high"] != '-' else '-',
"c": str(int(yh["close"])) if yh["close"] != '-' else '-',
"v": str(int(yh["volume"])) if yh["volume"] != '-' else '-',
})
return pac_list
def task_exp_handler(event, context):
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
print(DateTimeUtil.str_now())
dao = Dao()
dao_edi = dao.table('stock_edinet')
edis = dao_edi.full_scan()
print('len(edi) = {}'.format(len(edis)))
for edi in edis:
if 'holders' not in edi:
edi['holders'] = 'non'
if 'holder_rate' not in edi:
edi['holder_rate'] = 'non'
if 'outstanding_share' not in edi:
edi['outstanding_share'] = 'non'
cd = edi['ccode']
nm = edi['name']
hr = edi['holder_rate']
ot = edi['outstanding_share']
tot = get(hr, 'tot')
tabs = '{}\t' * 6
print(
tabs.format(cd, nm, get(hr, 'unit'), get(tot, 1),
get(ot, 'hutuu_hakkou'), get(ot, 'unit')))
return ''
def task006_handler(event, context):
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
print(DateTimeUtil.str_now())
dao = Dao()
ident_list = dao.table('stock_identify').full_scan()
data = {}
for ident in ident_list:
cd = ident['ccode']
dat = {} if cd not in data else data[cd]
if 'ns' in dat:
dat['ns'] = dat['ns'].split('_')
dat['ns'].append(ident['nm'])
else:
dat['ns'] = [ident['nm']]
if ident['main'] == 'y':
dat['n'] = ident['nm']
dat['ns'] = '_'.join(dat['ns'])
data[cd] = dat
ret = []
for d in data.items():
d[1]['c'] = d[0]
ret.append(d[1])
s3 = boto3.resource('s3').Bucket("kabupac.com")
s3.put_object(Key="suggest/dat.json", Body=JSON.dumps(ret))
return ''
def run(self, dao, req):
Log.info('twitter_friendsを全件取得。')
friends = self.dao.table('twitter_friends').find({})
Log.info('インフルエンサー毎に過去15日のツイートを集計する。')
for friend in friends:
cur_repo = self.dao.table('stock_report_pre').find({'tweets.user_id': friend['id_str']})
month_ago = DateTimeUtil.now() - timedelta(days=15)
user_tweets = []
for r in cur_repo:
u_tws = [t for t in r['tweets'] if t['user_id'] == friend['id_str'] and t['created_at'] >= month_ago]
if u_tws:
ccode_group_list = [t for t in user_tweets if t['ccode'] == r['ccode']]
ccode_group = {'name': '', 'last_update_date': month_ago, 'tweet': []}
if ccode_group_list:
ccode_group = ccode_group_list[0]
else:
ccode_group['ccode'] = r['ccode']
brand = self.dao.table('stock_brands').find_one({'ccode': r['ccode']})
if brand:
ccode_group['name'] = brand['name']
user_tweets.append(ccode_group)
ccode_group['last_update_date'] = max(
ccode_group['last_update_date'], max([t['created_at'] for t in u_tws]))
ccode_group['tweet'].extend(u_tws)
user_tweets = sorted(user_tweets, key=lambda x: x['last_update_date'], reverse=True)
ret = []
for r in user_tweets:
r['is_market_time'] = DateTimeUtil.is_market_time(r['last_update_date'])
str_day = r['last_update_date'].strftime('%Y/%m/%d')
dat = [d for d in ret if d['str_day'] == str_day]
if dat:
dat[0]['tweet'].append(r)
else:
buf = {'str_day': str_day, 'day': r['last_update_date'], 'tweet': [r]}
ret.append(buf)
friend['tweet_summary'] = ret
dao.table('twitter_friends').update_one({'id_str': friend['id_str']}, {'$set': friend})
Log.debug('登録: {}', friend['screen_name'])
Log.info('終了')
def price006_handler(event, context):
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
print(DateTimeUtil.str_now())
logic = PriceLogic()
logic.now_price_update(event["cds"].split('_'))
logic.notify_now_price_update_to_condition()
return ''
def notify_now_price_update_to_condition(self):
self.dao.table("condition").update_item(
Key={"key": "Task004_now_price_update"},
ExpressionAttributeValues={
':val': '-',
':dt': DateTimeUtil.str_now()
},
UpdateExpression="set val = :val, update_time = :dt"
)
def _daily_get(self, key):
if self.trigger.is_weekday():
tm = self.db.hget(key, 'ct')
if tm is not None:
save_day = datetime.strptime(tm.decode(), '%Y/%m/%d').date()
if save_day < DateTimeUtil.today().date():
self.db.delete(key)
return None
return self._get_val(key)
def notify_list_pages_update_to_condition(dao, val):
dao.table("condition").update_item(
Key={"key": "Task003_list_pages_update"},
ExpressionAttributeValues={
':val': val,
':dt': DateTimeUtil.str_now()
},
UpdateExpression="set val = :val, update_time = :dt"
)
def test_date_from_japanese_era1(self):
assert DateTimeUtil.date_from_japanese_era('平成30年2月22日') == datetime(
2018, 2, 22)
assert DateTimeUtil.date_from_japanese_era('平成11年12月31日') == datetime(
1999, 12, 31)
assert DateTimeUtil.date_from_japanese_era('平成12年1月1日') == datetime(
2000, 1, 1)
assert DateTimeUtil.date_from_japanese_era('昭和1年5月6日') == datetime(
1926, 5, 6)
assert DateTimeUtil.date_from_japanese_era('昭和64年1月1日') == datetime(
1989, 1, 1)
assert DateTimeUtil.date_from_japanese_era('平成1年1月1日') == datetime(
1989, 1, 1)
assert DateTimeUtil.date_from_japanese_era('平成1年1月日') is None
assert DateTimeUtil.date_from_japanese_era('平成1年月2日') is None
assert DateTimeUtil.date_from_japanese_era('2011年1月2日') is None
def web004_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
ids = event['ids']
print('ids = {}'.format(ids))
id_list = ids.split('_')
friends = dao.table("twitter_friends").find_batch(id_list)
return response({"v": "1", "friends": friends})
def web002_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
cd = event['cd']
print('cd={}'.format(cd))
brand = dao.table("stock_brands").find_by_key(cd)
thema = dao.table("stock_thema_ccode").find_by_key(cd)
if thema:
brand['thema'] = thema['nms']
return response({"v": "1", "brand": brand})
def to_simple_tweet_list(self, tweet_list):
timeline_list = []
for t in tweet_list:
dup = [x for x in timeline_list if x['id_str'] == t['id_str']]
if dup:
# 重複除去
continue
dat = {}
dat['id_str'] = t['id_str']
dat['text'] = t['full_text']
dat['created_at'] = DateTimeUtil.strf_ymdhms(
DateTimeUtil.date_from_utc(t['created_at']))
user = t.get('user')
dat['user_id'] = user['id_str']
dat['user_name'] = user['name']
dat['user_screen_name'] = user['screen_name']
retweet = t.get("retweeted_status")
if retweet:
dat['retweet_id'] = retweet['id_str']
dat['retweet_user_id'] = retweet['user']['id_str']
dat['retweet_user_name'] = retweet['user']['name']
dat['retweet_user_profile_image'] = retweet['user'][
'profile_image_url']
dat['retweet_text'] = retweet['full_text']
media = t.get('entities').get('media')
if media:
m_url_list = []
for m in media:
m_url = m.get('media_url_https')
if m_url:
m_url_list.append(m_url)
dat['media_url'] = m_url_list
timeline_list.append(dat)
return timeline_list
def web005_handler(event, context):
""" 現在価格API """
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
logic = PriceLogic()
uptime = logic.get_now_price_update_time()
cds = event['cds']
print('cds = {}'.format(cds))
cd_list = cds.split('_')
prices = dao.table("stock_price_now").find_batch(cd_list)
return response({ "v": "1", "prices": prices, "now_price_uptime": uptime })
def web008_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
prices = dao.table("stock_price_history").find_query({'cd': event['cd']})
if prices:
if 'lt' in event and event['lt']:
prices = [r for r in prices if r['d'] > event['lt']]
else:
s = 0 if len(prices) <= 30 else abs(len(prices) - 30)
prices = prices[s:]
else:
prices = []
return response({"v": "1", "ps": prices})
def run(self, dao: Dao, h1):
ys = YahooStock()
h1('値上がり率取得')
rise_list = ys.get_rise_fall_price_rate(mode='rise',
period='w',
today=False)
h1('値下がり率取得')
fall_list = ys.get_rise_fall_price_rate(mode='fall',
period='w',
today=False)
h1('stock_brands_high_lowコレクションに追加')
rise_list.extend(fall_list)
if rise_list:
for i in range(len(rise_list)):
dat = rise_list[i]
dat['date'] = DateTimeUtil.strf_ymdhms(dat['date'])
dao.table('stock_brands_rise_fall').insert(rise_list)
def web007_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
cd = event['cd']
print('cd = {}'.format(cd))
edi_dat = dao.table("stock_edinet").find_by_key(cd)
r = {}
if edi_dat:
r = {
'ho': edi_dat['holders'] if 'holders' in edi_dat else {},
'ra': edi_dat['holder_rate'] if 'holder_rate' in edi_dat else {},
'os': edi_dat['outstanding_share']
if 'outstanding_share' in edi_dat else {},
}
return response({"v": "1", "edi": r})
def web001_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
page = event['p'] if 'p' in event else "1"
print('page = {}'.format(page))
tbl, uptime = get_available_table(dao)
if tbl == 'err':
return response({"v": "1", "pages": [], "repos": [], "upt": "err"})
repo_list_page = dao.table(tbl).find_by_key(page)
ccodes = repo_list_page["ccodes"].split(',')
repo_list_page["ccodes"] = ccodes
repos = dao.table("stock_report").find_batch(ccodes)
for repo in repos:
repo['tweets'] = str(len(repo['tweets']))
return response({"v": "1", "pages": repo_list_page, "repos": repos, "upt": uptime })
def web003_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
cd = event['cd']
last_tweet_id = event['lt'] if 'lt' in event else None
direction = event['dr'] if 'dr' in event else None
print('cd = {}, last_tweet_id = {}, direction = {}'.format(cd, last_tweet_id, direction))
repo = dao.table("stock_report").find_by_key(cd)
if not repo:
return response({"v": "1", "tweets": []})
tweet_ids = repo['tweets']
end_id = tweet_ids[0]
tweets = dao.table("tweet").find_batch(tweet_ids)
if tweets:
tweets = sorted(tweets, key=lambda r: r['created_at'], reverse=True)
tweets = tweets[:30]
target_tweets = []
more_flg = False
for tw in tweets:
if direction == 'new':
if last_tweet_id == tw['id_str']:
break
target_tweets.append(tw)
else:
if last_tweet_id == tw['id_str']:
more_flg = True
continue
elif more_flg:
target_tweets.append(tw)
if target_tweets:
if target_tweets[-1]["id_str"] == end_id:
target_tweets[-1]["e"] = 1
return response({"v": "1", "tweets": target_tweets})
def run(self, dao: Dao, h1):
h1('銘柄一覧取得')
stock_brands = dao.table('stock_brands').full_scan()
h1('銘柄ごとに時系列データを取得してDB登録(並列処理)')
ys = YahooStock()
def j010_historical(param):
last_plus_one = param[1] + timedelta(days=1)
str_last_plus_one_date = last_plus_one.strftime('%Y-%m-%d')
prices = ys.get_historical_prices(ccode=param[0], str_start_date=str_last_plus_one_date)
if not prices:
return AsyncWorkerException(param[0])
return prices
# @tracelog
def c010_historical(results, param):
if results:
price_list = PriceLogic().convert_history_yahoo_to_pacpac(results, param[0])
dao.table('stock_price_history').insert(price_list)
ccode_list = [r['ccode'] for r in stock_brands]
ccode_list.append('998407') # 日経平均
ccode_list.append('USDJPY') # ドル円
param_list = []
db_history = dao.table('stock_price_history')
for ccode in ccode_list:
history = db_history.get_query_sorted_max({"cd": ccode}, 1)
if history and history[0]:
str_last_day = history[0]['d']
last_day = DateTimeUtil.str_to_date(str_last_day)
param_list.append((ccode, last_day))
result = AsyncWorker(j010_historical, c010_historical, 5).go(param_list)
h1('終了')
err_list = result[1]
if err_list:
print('[失敗銘柄] len(err) = {}, err_targets = {}'.format(len(err_list), err_list))
def web011_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
key = event['k']
p = int(event['p']) - 1
print('key = {}'.format(key))
thema = dao.table('stock_thema_nm').find_by_key(key)
ccodes, repos = [], []
if thema:
ccodes = thema['ccodes'].split(',')
repos = dao.table('stock_report').find_batch(ccodes)
repos = repos[p * 10:min((p + 1) * 10, len(repos))]
for repo in repos:
repo['tweets'] = str(len(repo['tweets']))
return response({"v": "1", "cds": ccodes, "repos": repos})
def web006_handler(event, context):
print(DateTimeUtil.str_now())
os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用
dao = Dao()
cd = event['cd']
last_id = event['lt'] if 'lt' in event else None
print('cd = {}, last_id = {}'.format(cd, last_id))
irs = dao.table("stock_ir").find_query({'cd': cd}, asc=False)
buf_irs = []
if last_id:
for ir in irs:
# stock_ir テーブルはソートキー「d」の降順。
if ir['tid'] == last_id:
break
buf_irs.append(ir)
else:
buf_irs = irs
return response({"v": "1", "irs": buf_irs})
def run(self, dao: Dao, h1):
h1('stock_report取得')
repos = dao.table('stock_report').full_scan()
d = DateTimeUtil.now() - timedelta(days=15)
two_week_ago = d.strftime('%Y/%m/%d_00:00:00')
friend_tweet = {}
ret_list = []
h1('ツイート集計開始')
for repo in repos:
cd = repo['ccode']
tw_ids = repo['tweets']
tweets = dao.table("tweet").find_batch(tw_ids)
tweets = [t for t in tweets if t['created_at'] > two_week_ago]
tweets = sorted(tweets,
key=lambda r: r['created_at'],
reverse=True)
for t in tweets:
u_id = t['user_id']
d = t['created_at']
ob = {'nm': repo['name'], 'ds': [{'d': d, 't': t['id_str']}]}
if u_id in friend_tweet:
if cd in friend_tweet[t['user_id']]:
friend_tweet[u_id][cd]['ds'].append({
'd': d,
't': t['id_str']
})
else:
friend_tweet[u_id][cd] = ob
else:
friend_tweet[t['user_id']] = {}
friend_tweet[t['user_id']][cd] = ob
for k in friend_tweet:
r = {'uid': k, 'rank': friend_tweet[k]}
ret_list.append(r)
dao.table('twitter_friends_sum').insert(ret_list)
print('hoge')