def task001_handler(event, context): print(DateTimeUtil.str_now()) dao = Dao() # 前回取得した最後のtweetidを取得 dat = dao.table('condition').get_item(Key={'key': 'Task001_last_tweet_id'}) last_tweet_id = dat['val'] print('last_tweet_id: ', last_tweet_id) tw = TwitterInspector(Irebaburn) timeline = tw.get_list_timeline_rotate(list_name='株', count=1000, last_data_id=last_tweet_id) print(timeline) if timeline: ids = [t['id_str'] for t in timeline] # 重複取得されたレコードがある場合、削除して入れなおすため、一旦消す。 dao.table('tweet').delete_batch_silent(ids) # ids = [] # for tweet in timeline: # # 重複取得されたレコードがある場合、削除して入れなおすため、一旦消す。 # dao.table('tweet').delete_item_silent({'id_str': tweet['id_str']}) # # ids.append(tweet['id_str']) # 追加 dao.table('tweet').insert(timeline) last_tweet_id = timeline[0]['id_str'] # 降順の先頭(取得した最新) print('last_tweet_id for update: ', last_tweet_id) # last_tweet_id 更新 dao.table("condition").update_item( Key={"key": dat['key']}, ExpressionAttributeValues={ ':val': last_tweet_id, ':dt': DateTimeUtil.str_now() }, UpdateExpression="set val = :val, update_time = :dt") # Task002呼び出し boto3.client("lambda").invoke( FunctionName= "arn:aws:lambda:ap-northeast-1:007575903924:function:Task002", InvocationType="Event", Payload=json.dumps({"ids": ids})) return ''
def task_exp_handler(event, context): os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 print(DateTimeUtil.str_now()) dao = Dao() dao_edi = dao.table('stock_edinet') edis = dao_edi.full_scan() print('len(edi) = {}'.format(len(edis))) for edi in edis: if 'holders' not in edi: edi['holders'] = 'non' if 'holder_rate' not in edi: edi['holder_rate'] = 'non' if 'outstanding_share' not in edi: edi['outstanding_share'] = 'non' cd = edi['ccode'] nm = edi['name'] hr = edi['holder_rate'] ot = edi['outstanding_share'] tot = get(hr, 'tot') tabs = '{}\t' * 6 print( tabs.format(cd, nm, get(hr, 'unit'), get(tot, 1), get(ot, 'hutuu_hakkou'), get(ot, 'unit'))) return ''
def task004_handler(event, context): os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 print(DateTimeUtil.str_now()) lambda_cnt = 6 dao = Dao() brands = dao.table('stock_brands').full_scan() ccode_list = [r['ccode'] for r in brands] ccode_list.append('998407') # 日経平均 ccode_list.append('USDJPY') # ドル円 q = len(ccode_list) // lambda_cnt chank_ccode = [ccode_list[i:i + q] for i in range(0, len(ccode_list), q)] if len(chank_ccode) > lambda_cnt: mod_chank = chank_ccode.pop(len(chank_ccode) - 1) chank_ccode[-1].extend(mod_chank) for i, ccodes in enumerate(chank_ccode, start=1): func_name = 'arn:aws:lambda:ap-northeast-1:007575903924:function:Price{0:03d}'.format( i) cds = '_'.join(ccodes) print(func_name) boto3.client("lambda").invoke(FunctionName=func_name, InvocationType="Event", Payload=json.dumps({"cds": cds})) return 0
def web010_handler(event, context): print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() uid = event['uid'] print('uid = {}'.format(uid)) tfs = dao.table('twitter_friends_sum').find_by_key(uid) ranks = sorted(tfs['rank'].items(), key=lambda r: len(r[1]['ds']), reverse=True) ranks = ranks[:min(20, len(ranks))] ret = [] for r in ranks: rank = { 'cd': r[0], 'nm': r[1]['nm'], 'ct': len(r[1]['ds']), 'd': r[1]['ds'][0]['d'] } ret.append(rank) tweet = {} if 'tid' in tfs: tweet = dao.table('tweet').find_by_key(tfs['tid']) del tweet['id_str'] del tweet['user_id'] del tweet['user_name'] del tweet['user_screen_name'] cond = dao.table('condition').find_by_key('Task002_tweet_report_update') ob = {'d': cond['update_time'], 't': tweet, 'r': ret} return response({"v": "1", "tfs": ob})
def task006_handler(event, context): os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 print(DateTimeUtil.str_now()) dao = Dao() ident_list = dao.table('stock_identify').full_scan() data = {} for ident in ident_list: cd = ident['ccode'] dat = {} if cd not in data else data[cd] if 'ns' in dat: dat['ns'] = dat['ns'].split('_') dat['ns'].append(ident['nm']) else: dat['ns'] = [ident['nm']] if ident['main'] == 'y': dat['n'] = ident['nm'] dat['ns'] = '_'.join(dat['ns']) data[cd] = dat ret = [] for d in data.items(): d[1]['c'] = d[0] ret.append(d[1]) s3 = boto3.resource('s3').Bucket("kabupac.com") s3.put_object(Key="suggest/dat.json", Body=JSON.dumps(ret)) return ''
def notify_list_pages_update_to_condition(dao, val): dao.table("condition").update_item( Key={"key": "Task003_list_pages_update"}, ExpressionAttributeValues={ ':val': val, ':dt': DateTimeUtil.str_now() }, UpdateExpression="set val = :val, update_time = :dt" )
def price006_handler(event, context): os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 print(DateTimeUtil.str_now()) logic = PriceLogic() logic.now_price_update(event["cds"].split('_')) logic.notify_now_price_update_to_condition() return ''
def notify_now_price_update_to_condition(self): self.dao.table("condition").update_item( Key={"key": "Task004_now_price_update"}, ExpressionAttributeValues={ ':val': '-', ':dt': DateTimeUtil.str_now() }, UpdateExpression="set val = :val, update_time = :dt" )
def web002_handler(event, context): print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() cd = event['cd'] print('cd={}'.format(cd)) brand = dao.table("stock_brands").find_by_key(cd) thema = dao.table("stock_thema_ccode").find_by_key(cd) if thema: brand['thema'] = thema['nms'] return response({"v": "1", "brand": brand})
def web004_handler(event, context): print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() ids = event['ids'] print('ids = {}'.format(ids)) id_list = ids.split('_') friends = dao.table("twitter_friends").find_batch(id_list) return response({"v": "1", "friends": friends})
def web005_handler(event, context): """ 現在価格API """ print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() logic = PriceLogic() uptime = logic.get_now_price_update_time() cds = event['cds'] print('cds = {}'.format(cds)) cd_list = cds.split('_') prices = dao.table("stock_price_now").find_batch(cd_list) return response({ "v": "1", "prices": prices, "now_price_uptime": uptime })
def web008_handler(event, context): print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() prices = dao.table("stock_price_history").find_query({'cd': event['cd']}) if prices: if 'lt' in event and event['lt']: prices = [r for r in prices if r['d'] > event['lt']] else: s = 0 if len(prices) <= 30 else abs(len(prices) - 30) prices = prices[s:] else: prices = [] return response({"v": "1", "ps": prices})
def web001_handler(event, context): print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() page = event['p'] if 'p' in event else "1" print('page = {}'.format(page)) tbl, uptime = get_available_table(dao) if tbl == 'err': return response({"v": "1", "pages": [], "repos": [], "upt": "err"}) repo_list_page = dao.table(tbl).find_by_key(page) ccodes = repo_list_page["ccodes"].split(',') repo_list_page["ccodes"] = ccodes repos = dao.table("stock_report").find_batch(ccodes) for repo in repos: repo['tweets'] = str(len(repo['tweets'])) return response({"v": "1", "pages": repo_list_page, "repos": repos, "upt": uptime })
def web007_handler(event, context): print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() cd = event['cd'] print('cd = {}'.format(cd)) edi_dat = dao.table("stock_edinet").find_by_key(cd) r = {} if edi_dat: r = { 'ho': edi_dat['holders'] if 'holders' in edi_dat else {}, 'ra': edi_dat['holder_rate'] if 'holder_rate' in edi_dat else {}, 'os': edi_dat['outstanding_share'] if 'outstanding_share' in edi_dat else {}, } return response({"v": "1", "edi": r})
def web003_handler(event, context): print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() cd = event['cd'] last_tweet_id = event['lt'] if 'lt' in event else None direction = event['dr'] if 'dr' in event else None print('cd = {}, last_tweet_id = {}, direction = {}'.format(cd, last_tweet_id, direction)) repo = dao.table("stock_report").find_by_key(cd) if not repo: return response({"v": "1", "tweets": []}) tweet_ids = repo['tweets'] end_id = tweet_ids[0] tweets = dao.table("tweet").find_batch(tweet_ids) if tweets: tweets = sorted(tweets, key=lambda r: r['created_at'], reverse=True) tweets = tweets[:30] target_tweets = [] more_flg = False for tw in tweets: if direction == 'new': if last_tweet_id == tw['id_str']: break target_tweets.append(tw) else: if last_tweet_id == tw['id_str']: more_flg = True continue elif more_flg: target_tweets.append(tw) if target_tweets: if target_tweets[-1]["id_str"] == end_id: target_tweets[-1]["e"] = 1 return response({"v": "1", "tweets": target_tweets})
def web006_handler(event, context): print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() cd = event['cd'] last_id = event['lt'] if 'lt' in event else None print('cd = {}, last_id = {}'.format(cd, last_id)) irs = dao.table("stock_ir").find_query({'cd': cd}, asc=False) buf_irs = [] if last_id: for ir in irs: # stock_ir テーブルはソートキー「d」の降順。 if ir['tid'] == last_id: break buf_irs.append(ir) else: buf_irs = irs return response({"v": "1", "irs": buf_irs})
def web011_handler(event, context): print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() key = event['k'] p = int(event['p']) - 1 print('key = {}'.format(key)) thema = dao.table('stock_thema_nm').find_by_key(key) ccodes, repos = [], [] if thema: ccodes = thema['ccodes'].split(',') repos = dao.table('stock_report').find_batch(ccodes) repos = repos[p * 10:min((p + 1) * 10, len(repos))] for repo in repos: repo['tweets'] = str(len(repo['tweets'])) return response({"v": "1", "cds": ccodes, "repos": repos})
def web009_handler(event, context): print(DateTimeUtil.str_now()) os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 dao = Dao() cd = event['cd'] print('cd = {}'.format(cd)) high_low = dao.table("stock_brands_high_low").find_query({'ccode': cd}, asc=False) r = {} if high_low: high = [r for r in high_low if r['mode'] == 'high'] low = [r for r in high_low if r['mode'] == 'low'] r = {'h': high, 'l': low} rise_fall = dao.table("stock_brands_rise_fall").find_query({'ccode': cd}, asc=False) if rise_fall: rise = [r for r in rise_fall if r['mode'] == 'rise'] fall = [r for r in rise_fall if r['mode'] == 'fall'] r['r'] = rise r['f'] = fall return response({"v": "1", "hl": r})
def run(self, dao: Dao, h1): edinet = Edinet() start_day = DateTimeUtil.yesterday() end_day = DateTimeUtil.now() # start_day = datetime(2018, 5, 19) # end_day = datetime(2018, 5, 20) h1('EDINETの書類検索から指定期間の有価証券報告書の検索結果を取得する。') search_rows = edinet.get_report_search_results(start_day, end_day) search_rows.reverse() # search_rows = [r for r in search_rows if 'スタートトゥデイ' in r['company_name']] h1('検索結果から有価証券報告書を取得し、銘柄毎スクレイピング開始。') dao_identify = dao.table('stock_identify') dao_brand = dao.table('stock_brands') not_brands = [] for search_row in search_rows: data_html = edinet.get_report_html(search_row['syorui_kanri_no']) shorui_mei = search_row['syorui_mei'] Log.info('html取得完了 : {} : {}'.format(search_row['company_name'], shorui_mei)) if '大株主' not in data_html: Log.warn('HTMLに大株主が無い。別資料かも。 company_name : {}'.format( search_row['company_name'])) continue table_list = edinet.report_html_to_split_table_list( data_html, shorui_mei) Log.info('HTML -> table分割 : {}'.format(search_row['company_name'])) dat = {} for t in table_list: if t['title'] == '発行済株式' and 'outstanding_share' not in dat: dat['outstanding_share'] = edinet.outstanding_share( t, search_row) Log.debug('** 発行済株式 : {}'.format( search_row['company_name'])) if t['title'] == '所有者別状況': dat['holder_rate'] = edinet.holder_rate_status( t, search_row) Log.debug('** 所有者別状況 : {}'.format( search_row['company_name'])) if t['title'] == '大株主の状況': dat['holders'] = edinet.major_shareholders(t, search_row) Log.debug('** 大株主の状況 : {}'.format( search_row['company_name'])) if not dat: continue name = search_row['company_name'].replace('(株)', '').replace(' ', '') name = re.sub(r'^ ', '', name) name = re.sub(r' $', '', name) identify = dao_identify.find_query({'nm': name}) if not identify: name_s = zenhan.z2h(name, mode=1) identify = dao_identify.find_query({'nm': name_s}) if not identify: name_s = name_s.replace(' ', '') identify = dao_identify.find_query({'nm': name_s}) if identify and len(identify) == 1: brand = dao_brand.find_by_key(identify[0]['ccode']) Log.info('stock_brands証券コード取得。 ccode, name : {}, {}'.format( brand['ccode'], name)) dat['ccode'] = brand['ccode'] dat['name'] = brand['name'] dat['date'] = DateTimeUtil.strf_ymd_st( DateTimeUtil.date_from_japanese_era( search_row['submition_day'], short=True)) if empty(dat, 'outstanding_share') or empty( dat, 'holder_rate') or empty(dat, 'holders'): edi = dao.table('stock_edinet').find_by_key(dat['ccode']) if edi: if empty(dat, 'outstanding_share') and not empty( edi, 'outstanding_share'): dat['outstanding_share'] = edi['outstanding_share'] if empty(dat, 'holder_rate') and not empty( edi, 'holder_rate'): dat['holder_rate'] = edi['holder_rate'] if empty(dat, 'holders') and not empty(edi, 'holders'): dat['holders'] = edi['holders'] h1('stock_edinetに登録') dao.table('stock_edinet').put_item(Item=dat) else: dao.table('err_value').put_item( Item={ 'key': 'Job013_edi_name', 'd': DateTimeUtil.str_now(), 'val': name }) h1('終了')
def task002_handler(event, context): os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 print(DateTimeUtil.str_now(), 'Task002', event['ids']) dao = Dao() mecab_parser = MeCabParser() tweet_ids = event['ids'] tbl_repo = dao.table("stock_report") tbl_tweet = dao.table('tweet') tbl_brands = dao.table('stock_brands') tbl_price_now = dao.table('stock_price_now') tbl_f_sum = dao.table('twitter_friends_sum') is_stock_find = False for id_str in tweet_ids: t = tbl_tweet.find_by_key(id_str) if t is None: continue t_sum = {} tw_txt = t["text"] if "retweet_text" in t: tw_txt = t["retweet_text"] # ツイートのクリーニング clean_twtxt = Util.mask_twitter_name(Util.all_normalize(tw_txt)) # つぶやきを単語分割 mecab_dic = mecab_parser.parse(clean_twtxt) # つぶやき内から銘柄取得 stocks_in_tweet, words = find_stock_code_in(mecab_dic, dao, tbl_brands) if not is_stock_find: is_stock_find = len(stocks_in_tweet) > 0 Log.info('len(stocks_in_tweet) = {}, is_stock_find = {}'.format( len(stocks_in_tweet), is_stock_find)) for b in stocks_in_tweet.values(): tweet_in_ccode = b["ccode"] Log.info('株 みっけ!! : {} {} user={}', tweet_in_ccode, b["nm"], t["user_name"]) Log.info('tweet={}', clean_twtxt) stock_repo = tbl_repo.find_by_key(tweet_in_ccode) price = get_price(tweet_in_ccode, tbl_price_now) if "prices" not in t: t["prices"] = {} if tweet_in_ccode in t["prices"]: t["prices"][tweet_in_ccode].append(price) else: t["prices"][tweet_in_ccode] = [price] if not t_sum: t_sum = get_tweet_summary(t, tweet_in_ccode, tbl_f_sum) if tweet_in_ccode not in t_sum['rank']: t_sum['rank'][tweet_in_ccode] = {'ds': []} t_sum['rank'][tweet_in_ccode]['nm'] = b['nm'] t_sum['rank'][tweet_in_ccode]['ds'].insert(0, { 'd': t["created_at"], 't': t['id_str'] }) if stock_repo: tweets = stock_repo["tweets"] # 価格をpriceに付けてしまったがTweetに付けるように変更したため、価格についたpriceがあれば削除する if "prices" in stock_repo: del stock_repo["prices"] if len([tw for tw in tweets if tw == t['id_str']]) == 0: Log.debug('カウントアップ!! : {} {}, user={}', b["ccode"], b["nm"], t["user_name"]) tweets.append(t['id_str']) stock_repo["tweets"] = tweets stock_repo["ago_uptime"] = stock_repo["last_updated_at"] stock_repo["last_updated_at"] = t["created_at"] stock_repo["last_update_user"] = t["user_id"] stock_repo["last_up_uname"] = t["user_name"] retouch_stock_name(stock_repo, tbl_brands) tbl_repo.put_item(Item=stock_repo) else: Log.debug('初登場!! : {} {}, user={}', b["ccode"], b["nm"], t["user_name"]) data = { 'ccode': b["ccode"], 'name': b["nm"], 'create_user': t["user_id"], 'created_at': t["created_at"], 'last_updated_at': t["created_at"], 'last_update_user': t["user_id"], 'last_up_uname': t["user_name"], 'tweets': [t["id_str"]], "prices": [price] } retouch_stock_name(data, tbl_brands) tbl_repo.put_item(Item=data) if words: t['ws'] = words tbl_tweet.put_item(Item=t) if t_sum: tbl_f_sum.put_item(Item=t_sum) if is_stock_find: # Task003呼び出し Log.info('Task003 呼び出し') boto3.client("lambda").invoke( FunctionName= "arn:aws:lambda:ap-northeast-1:007575903924:function:Task003", InvocationType="Event", Payload=json.dumps({})) dao.table("condition").update_item( Key={"key": "Task002_tweet_report_update"}, ExpressionAttributeValues={ ':val': '-', ':dt': DateTimeUtil.str_now() }, UpdateExpression="set val = :val, update_time = :dt") return 0
def task005_handler(event, context): os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 print(DateTimeUtil.str_now()) dao = Dao() mecab_parser = MeCabParser() mecab_logic = MeCabLogic() dao_ir = dao.table('stock_ir') # 前回取得した最後のtweetidを取得 dat = dao.table('condition').get_item(Key={'key': 'Task005_last_tweet_id'}) last_tweet_id = dat['val'] print('last_tweet_id: ', last_tweet_id) tw = TwitterInspector(Kabpackab) timeline = tw.get_list_timeline_rotate(list_name='IR', count=1000, last_data_id=last_tweet_id) print(timeline) if timeline: data_list = [] for t in timeline: tw_txt = t["text"] # ツイートのクリーニング clean_twtxt = Util.mask_twitter_name(Util.all_normalize(tw_txt)) # つぶやきを単語分割 mecab_dic = mecab_parser.parse(clean_twtxt) # つぶやき内から銘柄取得 stocks_in_tweet, _ = mecab_logic.find_stock_code_in(mecab_dic) for b in stocks_in_tweet.values(): tweet_in_ccode = b["ccode"] Log.info('IR みっけ!! : {} {} user={}', tweet_in_ccode, b["nm"], t["user_name"]) data = { 'cd': b["ccode"], 'nm': b["nm"], 'u': t["user_id"], 'd': t["created_at"], 't': tw_txt, 'tid': t['id_str'] } data_list.append(data) dao_ir.insert(data_list) last_tweet_id = timeline[0]['id_str'] # 降順の先頭(取得した最新) print('last_tweet_id for update: ', last_tweet_id) # last_tweet_id 更新 dao.table("condition").update_item( Key={"key": dat['key']}, ExpressionAttributeValues={ ':val': last_tweet_id, ':dt': DateTimeUtil.str_now() }, UpdateExpression="set val = :val, update_time = :dt") return ''