def task005_handler(event, context): os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 print(DateTimeUtil.str_now()) dao = Dao() mecab_parser = MeCabParser() mecab_logic = MeCabLogic() dao_ir = dao.table('stock_ir') # 前回取得した最後のtweetidを取得 dat = dao.table('condition').get_item(Key={'key': 'Task005_last_tweet_id'}) last_tweet_id = dat['val'] print('last_tweet_id: ', last_tweet_id) tw = TwitterInspector(Kabpackab) timeline = tw.get_list_timeline_rotate(list_name='IR', count=1000, last_data_id=last_tweet_id) print(timeline) if timeline: data_list = [] for t in timeline: tw_txt = t["text"] # ツイートのクリーニング clean_twtxt = Util.mask_twitter_name(Util.all_normalize(tw_txt)) # つぶやきを単語分割 mecab_dic = mecab_parser.parse(clean_twtxt) # つぶやき内から銘柄取得 stocks_in_tweet, _ = mecab_logic.find_stock_code_in(mecab_dic) for b in stocks_in_tweet.values(): tweet_in_ccode = b["ccode"] Log.info('IR みっけ!! : {} {} user={}', tweet_in_ccode, b["nm"], t["user_name"]) data = { 'cd': b["ccode"], 'nm': b["nm"], 'u': t["user_id"], 'd': t["created_at"], 't': tw_txt, 'tid': t['id_str'] } data_list.append(data) dao_ir.insert(data_list) last_tweet_id = timeline[0]['id_str'] # 降順の先頭(取得した最新) print('last_tweet_id for update: ', last_tweet_id) # last_tweet_id 更新 dao.table("condition").update_item( Key={"key": dat['key']}, ExpressionAttributeValues={ ':val': last_tweet_id, ':dt': DateTimeUtil.str_now() }, UpdateExpression="set val = :val, update_time = :dt") return ''
def task002_handler(event, context): os.environ["PRODUCTION_DAO"] = "True" # TODO 本番向けテスト用 print(DateTimeUtil.str_now(), 'Task002', event['ids']) dao = Dao() mecab_parser = MeCabParser() tweet_ids = event['ids'] tbl_repo = dao.table("stock_report") tbl_tweet = dao.table('tweet') tbl_brands = dao.table('stock_brands') tbl_price_now = dao.table('stock_price_now') tbl_f_sum = dao.table('twitter_friends_sum') is_stock_find = False for id_str in tweet_ids: t = tbl_tweet.find_by_key(id_str) if t is None: continue t_sum = {} tw_txt = t["text"] if "retweet_text" in t: tw_txt = t["retweet_text"] # ツイートのクリーニング clean_twtxt = Util.mask_twitter_name(Util.all_normalize(tw_txt)) # つぶやきを単語分割 mecab_dic = mecab_parser.parse(clean_twtxt) # つぶやき内から銘柄取得 stocks_in_tweet, words = find_stock_code_in(mecab_dic, dao, tbl_brands) if not is_stock_find: is_stock_find = len(stocks_in_tweet) > 0 Log.info('len(stocks_in_tweet) = {}, is_stock_find = {}'.format( len(stocks_in_tweet), is_stock_find)) for b in stocks_in_tweet.values(): tweet_in_ccode = b["ccode"] Log.info('株 みっけ!! : {} {} user={}', tweet_in_ccode, b["nm"], t["user_name"]) Log.info('tweet={}', clean_twtxt) stock_repo = tbl_repo.find_by_key(tweet_in_ccode) price = get_price(tweet_in_ccode, tbl_price_now) if "prices" not in t: t["prices"] = {} if tweet_in_ccode in t["prices"]: t["prices"][tweet_in_ccode].append(price) else: t["prices"][tweet_in_ccode] = [price] if not t_sum: t_sum = get_tweet_summary(t, tweet_in_ccode, tbl_f_sum) if tweet_in_ccode not in t_sum['rank']: t_sum['rank'][tweet_in_ccode] = {'ds': []} t_sum['rank'][tweet_in_ccode]['nm'] = b['nm'] t_sum['rank'][tweet_in_ccode]['ds'].insert(0, { 'd': t["created_at"], 't': t['id_str'] }) if stock_repo: tweets = stock_repo["tweets"] # 価格をpriceに付けてしまったがTweetに付けるように変更したため、価格についたpriceがあれば削除する if "prices" in stock_repo: del stock_repo["prices"] if len([tw for tw in tweets if tw == t['id_str']]) == 0: Log.debug('カウントアップ!! : {} {}, user={}', b["ccode"], b["nm"], t["user_name"]) tweets.append(t['id_str']) stock_repo["tweets"] = tweets stock_repo["ago_uptime"] = stock_repo["last_updated_at"] stock_repo["last_updated_at"] = t["created_at"] stock_repo["last_update_user"] = t["user_id"] stock_repo["last_up_uname"] = t["user_name"] retouch_stock_name(stock_repo, tbl_brands) tbl_repo.put_item(Item=stock_repo) else: Log.debug('初登場!! : {} {}, user={}', b["ccode"], b["nm"], t["user_name"]) data = { 'ccode': b["ccode"], 'name': b["nm"], 'create_user': t["user_id"], 'created_at': t["created_at"], 'last_updated_at': t["created_at"], 'last_update_user': t["user_id"], 'last_up_uname': t["user_name"], 'tweets': [t["id_str"]], "prices": [price] } retouch_stock_name(data, tbl_brands) tbl_repo.put_item(Item=data) if words: t['ws'] = words tbl_tweet.put_item(Item=t) if t_sum: tbl_f_sum.put_item(Item=t_sum) if is_stock_find: # Task003呼び出し Log.info('Task003 呼び出し') boto3.client("lambda").invoke( FunctionName= "arn:aws:lambda:ap-northeast-1:007575903924:function:Task003", InvocationType="Event", Payload=json.dumps({})) dao.table("condition").update_item( Key={"key": "Task002_tweet_report_update"}, ExpressionAttributeValues={ ':val': '-', ':dt': DateTimeUtil.str_now() }, UpdateExpression="set val = :val, update_time = :dt") return 0
def tweet_text_cleaning(self, t): t.text = Util.mask_twitter_name(Util.all_normalize(t.text))