def date_to_str(item):
if type(item) is dict:
for key in item:
buf = item[key]
if type(buf) in (dict, list):
Util.blank_to_none(buf)
if type(buf) is datetime:
item[key] = DateTimeUtil.strf_ymdhms(buf)
elif type(item) is list:
for i in range(len(item)):
row = item[i]
if type(row) in (dict, list):
Util.blank_to_none(row)
if type(row) is datetime:
item[i] = DateTimeUtil.strf_ymdhms(row)
def run(self, dao: Dao, h1):
ys = YahooStock()
h1('値上がり率取得')
rise_list = ys.get_rise_fall_price_rate(mode='rise',
period='w',
today=False)
h1('値下がり率取得')
fall_list = ys.get_rise_fall_price_rate(mode='fall',
period='w',
today=False)
h1('stock_brands_high_lowコレクションに追加')
rise_list.extend(fall_list)
if rise_list:
for i in range(len(rise_list)):
dat = rise_list[i]
dat['date'] = DateTimeUtil.strf_ymdhms(dat['date'])
dao.table('stock_brands_rise_fall').insert(rise_list)
def to_simple_tweet_list(self, tweet_list):
timeline_list = []
for t in tweet_list:
dup = [x for x in timeline_list if x['id_str'] == t['id_str']]
if dup:
# 重複除去
continue
dat = {}
dat['id_str'] = t['id_str']
dat['text'] = t['full_text']
dat['created_at'] = DateTimeUtil.strf_ymdhms(
DateTimeUtil.date_from_utc(t['created_at']))
user = t.get('user')
dat['user_id'] = user['id_str']
dat['user_name'] = user['name']
dat['user_screen_name'] = user['screen_name']
retweet = t.get("retweeted_status")
if retweet:
dat['retweet_id'] = retweet['id_str']
dat['retweet_user_id'] = retweet['user']['id_str']
dat['retweet_user_name'] = retweet['user']['name']
dat['retweet_user_profile_image'] = retweet['user'][
'profile_image_url']
dat['retweet_text'] = retweet['full_text']
media = t.get('entities').get('media')
if media:
m_url_list = []
for m in media:
m_url = m.get('media_url_https')
if m_url:
m_url_list.append(m_url)
dat['media_url'] = m_url_list
timeline_list.append(dat)
return timeline_list
def tbl_price():
# os.environ["PRODUCTION_DAO"] = "True"
mongo = OldDao(DB())
dynamo = Dao()
ccodes = mongo._db.client['price_history'].collection_names()
# d: date
# o: open
# l: low
# h: high
# c: close
# v: volume
# j: jika_sougaku
# b: pbr
# e: per
# t: nen_taka
# y: nen_yasu
# k: kabusuu
col = ['d', 'o', 'l', 'h', 'c', 'v', 'j', 'b', 'e', 't', 'y', 'k']
for ccode in ccodes:
ccode_path = 'c:/temp/pac_price/{}'.format(ccode)
if not os.path.exists(ccode_path):
os.mkdir(ccode_path)
cur_price = mongo.table(ccode, dbname='price_history').find({}, {
'_id': 0
}).sort([('date', 1)])
last_path, last_month, csv_path, now_month, datas = None, 0, '', 0, []
for i, p in enumerate(cur_price):
date = p['date']
str_ym = date.strftime('%Y%m')
csv_path = '{}/{}'.format(ccode_path, str_ym)
now_month = int(str_ym[-2:])
if last_path and last_path != csv_path and not os.path.exists(
csv_path) and now_month in [4, 7, 10, 1]:
df = pd.DataFrame(datas, columns=col)
df.to_csv(last_path + '.csv',
encoding="utf-8",
line_terminator='\n',
index=False)
datas = []
d = DateTimeUtil.strf_ymdhms(date)
j = p['jika_sougaku'] if 'jikasougaku' in p else ''
b = p['pbr'] if 'pbr' in p else ''
e = p['per'] if 'per' in p else ''
t = p['nen_taka'] if 'nen_taka' in p else ''
y = p['nen_yasu'] if 'nen_yasu' in p else ''
k = p['hakkou_kabusuu'] if 'hakkou_kabusuu' in p else ''
datas.append([
d, p['open'], p['low'], p['high'], p['close'], p['volume'], j,
b, e, t, y, k
])
last_path = csv_path
if now_month not in [3, 6, 9, 12]:
now_month += (3 - now_month // 3)
buf = csv_path[:-2]
csv_path = buf + '{0:02d}.csv'.format(now_month)
df = pd.DataFrame(datas, columns=col)
df.to_csv(csv_path,
encoding="utf-8",
line_terminator='\n',
index=False)
def date_to_string(self, data_list):
for i in range(len(data_list)):
dat = data_list[i]
dat['date'] = DateTimeUtil.strf_ymdhms(dat['date'])