def date_to_str(item): if type(item) is dict: for key in item: buf = item[key] if type(buf) in (dict, list): Util.blank_to_none(buf) if type(buf) is datetime: item[key] = DateTimeUtil.strf_ymdhms(buf) elif type(item) is list: for i in range(len(item)): row = item[i] if type(row) in (dict, list): Util.blank_to_none(row) if type(row) is datetime: item[i] = DateTimeUtil.strf_ymdhms(row)
def run(self, dao: Dao, h1): ys = YahooStock() h1('値上がり率取得') rise_list = ys.get_rise_fall_price_rate(mode='rise', period='w', today=False) h1('値下がり率取得') fall_list = ys.get_rise_fall_price_rate(mode='fall', period='w', today=False) h1('stock_brands_high_lowコレクションに追加') rise_list.extend(fall_list) if rise_list: for i in range(len(rise_list)): dat = rise_list[i] dat['date'] = DateTimeUtil.strf_ymdhms(dat['date']) dao.table('stock_brands_rise_fall').insert(rise_list)
def to_simple_tweet_list(self, tweet_list): timeline_list = [] for t in tweet_list: dup = [x for x in timeline_list if x['id_str'] == t['id_str']] if dup: # 重複除去 continue dat = {} dat['id_str'] = t['id_str'] dat['text'] = t['full_text'] dat['created_at'] = DateTimeUtil.strf_ymdhms( DateTimeUtil.date_from_utc(t['created_at'])) user = t.get('user') dat['user_id'] = user['id_str'] dat['user_name'] = user['name'] dat['user_screen_name'] = user['screen_name'] retweet = t.get("retweeted_status") if retweet: dat['retweet_id'] = retweet['id_str'] dat['retweet_user_id'] = retweet['user']['id_str'] dat['retweet_user_name'] = retweet['user']['name'] dat['retweet_user_profile_image'] = retweet['user'][ 'profile_image_url'] dat['retweet_text'] = retweet['full_text'] media = t.get('entities').get('media') if media: m_url_list = [] for m in media: m_url = m.get('media_url_https') if m_url: m_url_list.append(m_url) dat['media_url'] = m_url_list timeline_list.append(dat) return timeline_list
def tbl_price(): # os.environ["PRODUCTION_DAO"] = "True" mongo = OldDao(DB()) dynamo = Dao() ccodes = mongo._db.client['price_history'].collection_names() # d: date # o: open # l: low # h: high # c: close # v: volume # j: jika_sougaku # b: pbr # e: per # t: nen_taka # y: nen_yasu # k: kabusuu col = ['d', 'o', 'l', 'h', 'c', 'v', 'j', 'b', 'e', 't', 'y', 'k'] for ccode in ccodes: ccode_path = 'c:/temp/pac_price/{}'.format(ccode) if not os.path.exists(ccode_path): os.mkdir(ccode_path) cur_price = mongo.table(ccode, dbname='price_history').find({}, { '_id': 0 }).sort([('date', 1)]) last_path, last_month, csv_path, now_month, datas = None, 0, '', 0, [] for i, p in enumerate(cur_price): date = p['date'] str_ym = date.strftime('%Y%m') csv_path = '{}/{}'.format(ccode_path, str_ym) now_month = int(str_ym[-2:]) if last_path and last_path != csv_path and not os.path.exists( csv_path) and now_month in [4, 7, 10, 1]: df = pd.DataFrame(datas, columns=col) df.to_csv(last_path + '.csv', encoding="utf-8", line_terminator='\n', index=False) datas = [] d = DateTimeUtil.strf_ymdhms(date) j = p['jika_sougaku'] if 'jikasougaku' in p else '' b = p['pbr'] if 'pbr' in p else '' e = p['per'] if 'per' in p else '' t = p['nen_taka'] if 'nen_taka' in p else '' y = p['nen_yasu'] if 'nen_yasu' in p else '' k = p['hakkou_kabusuu'] if 'hakkou_kabusuu' in p else '' datas.append([ d, p['open'], p['low'], p['high'], p['close'], p['volume'], j, b, e, t, y, k ]) last_path = csv_path if now_month not in [3, 6, 9, 12]: now_month += (3 - now_month // 3) buf = csv_path[:-2] csv_path = buf + '{0:02d}.csv'.format(now_month) df = pd.DataFrame(datas, columns=col) df.to_csv(csv_path, encoding="utf-8", line_terminator='\n', index=False)
def date_to_string(self, data_list): for i in range(len(data_list)): dat = data_list[i] dat['date'] = DateTimeUtil.strf_ymdhms(dat['date'])