def convert_history_yahoo_to_pacpac(self, yahoo_list, ccode):
pac_list = []
for yh in yahoo_list:
d = DateTimeUtil.strf_ymd_st(yh['date'])
# d: date, o: open, l: low, h: high, c: close, v: volume
# j: jika_sougaku, b: pbr, e: per, t: nen_taka, y: nen_yasu, k: kabusuu
if ccode == 'USDJPY':
pac_list.append({
"cd": ccode,
"d": d,
"o": re.sub('0000$', '', yh["open"]),
"l": re.sub('0000$', '', yh["low"]),
"h": re.sub('0000$', '', yh["high"]),
"c": re.sub('0000$', '', yh["close"]),
"v": "-"
})
else:
pac_list.append({
"cd": ccode,
"d": d,
"o": str(int(yh["open"])) if yh["open"] != '-' else '-',
"l": str(int(yh["low"])) if yh["low"] != '-' else '-',
"h": str(int(yh["high"])) if yh["high"] != '-' else '-',
"c": str(int(yh["close"])) if yh["close"] != '-' else '-',
"v": str(int(yh["volume"])) if yh["volume"] != '-' else '-',
})
return pac_list
def convert_date(self, str_date):
str_date = self.norm(str_date)
if '平成' in str_date:
jap = DateTimeUtil.date_from_japanese_era(str_date)
if jap:
return DateTimeUtil.strf_ymd_st(jap)
else:
return '-'
else:
m = re.search('(\d+)年(\d+)月(\d+)日', str_date)
if m:
return DateTimeUtil.strf_ymd_st(
datetime(int(m.group(1)), int(m.group(2)),
int(m.group(3))))
else:
return '-'
def run(self, dao: Dao, h1):
dao_brands = dao.table("stock_brands")
dao_thema = self.dao.table('stock_thema_ccode')
dao_ph = self.dao.table('stock_price_history')
dao_repo = self.dao.table('stock_report')
h1('銘柄一覧取得')
brands = dao_brands.full_scan()
h1('Yahooから詳細情報取得')
ys = YahooStock()
ccodes = [b['ccode'] for b in brands]
ys_details = ys.get_stock_details_async(ccodes)
h1('Kabtanから詳細情報取得')
ccodes_chank = list(chunked(ccodes, 50))
kb = Kabtan()
kb_details = []
for i, ccodes in enumerate(ccodes_chank):
for j, cd in enumerate(ccodes):
# print('kabtan: cd={}'.format(cd))
kbs = kb.get_stock_detail(cd)
kb_details.append(kbs)
if j % 7 == 0:
time.sleep(random.randint(1,2))
print('kabtan詳細取得中 : {}'.format(i))
time.sleep(1)
Log.debug('brands={}, ys_details={}, kb_details={}'.format(len(ccodes), len(ys_details), len(kb_details)))
h1('stock_brandsに登録')
ys_err_lst = []
kb_err_lst = []
brands = brands
for i, brand in enumerate(brands):
ccode = brand['ccode']
ys_lst = [d for d in ys_details if ccode == d['ccode']]
kb_lst = [d for d in kb_details if ccode == d['ccode']]
ys_detail = []
kb_detail = {}
if not kb_lst:
kb_err_lst.append(ccode)
else:
kb_detail = kb_lst[0]
if not ys_lst:
ys_err_lst.append(ccode)
continue
else:
ys_detail = ys_lst[0]
# stock_brands_detail登録
# j: jika_sougaku, b: pbr, e: per, t: nen_taka, y: nen_yasu, k: kabusuu
nentaka = '-' if ys_detail['nen_taka'] == '-' else str(int(ys_detail['nen_taka']))
nenyasu = '-' if ys_detail['nen_yasu'] == '-' else str(int(ys_detail['nen_yasu']))
market = brand['market'] if 'market' in brand and brand['market'] else '-'
info = brand['info'] if 'info' in brand and brand['info'] else '-'
r = {'ccode': ccode,
'market': market,
'name': brand['name'],
'info': info,
'tn': emp_to_str(ys_detail['tan']),
'j': emp_to_str(ys_detail['jika_sougaku']),
'k': emp_to_str(ys_detail['hakkou_kabusuu']),
'e': emp_to_str(ys_detail['per']),
'b': emp_to_str(ys_detail['pbr']),
't': nentaka,
'y': nenyasu,
'kj': emp_to_str(ys_detail['keijiban'])}
r['url'] = kb_detail['url'] if 'url' in kb_detail and kb_detail['url'] != '' else '-'
r['cate'] = kb_detail['cate'] if 'cate' in kb_detail and kb_detail['cate'] != '' else '-'
dao_brands.put_item(Item=r)
repo = dao_repo.find_by_key(ccode)
if repo:
repo['kj'] = r['kj']
repo['url'] = r['url']
dao_repo.put_item(Item=repo)
print('登録 ccode={}, i={}'.format(ccode, i))
d = DateTimeUtil.strf_ymd_st(DateTimeUtil.today())
ph_list = dao_ph.find_query({ 'cd': ccode, 'd': d })
if ph_list and len(ph_list) == 1:
ph = ph_list[0]
ph['e'] = emp_to_str(ys_detail['per'])
ph['b'] = emp_to_str(ys_detail['pbr'])
ph['t'] = nentaka
ph['y'] = nenyasu
ph['j'] = emp_to_str(ys_detail['jika_sougaku'])
ph['k'] = emp_to_str(ys_detail['hakkou_kabusuu'])
dao_ph.put_item(Item=ph)
tm = dao_thema.find_by_key(ccode)
nms = emp_to_str(kb_detail['thema'])
if tm:
tm['nms'] = nms
else:
tm = {
'ccode': ccode,
'nms': nms
}
dao_thema.put_item(Item=tm)
h1('Job012をキック')
boto3.client('batch').submit_job(
jobName='Job012',
jobQueue="arn:aws:batch:ap-northeast-1:007575903924:job-queue/Job012_high_low_to_brand_and_repo",
jobDefinition="Job012_high_low_to_brand_and_repo:1"
)
def day_key_now(self):
return DateTimeUtil.strf_ymd_st(DateTimeUtil.now())
def run(self, dao: Dao, h1):
edinet = Edinet()
start_day = DateTimeUtil.yesterday()
end_day = DateTimeUtil.now()
# start_day = datetime(2018, 5, 19)
# end_day = datetime(2018, 5, 20)
h1('EDINETの書類検索から指定期間の有価証券報告書の検索結果を取得する。')
search_rows = edinet.get_report_search_results(start_day, end_day)
search_rows.reverse()
# search_rows = [r for r in search_rows if 'スタートトゥデイ' in r['company_name']]
h1('検索結果から有価証券報告書を取得し、銘柄毎スクレイピング開始。')
dao_identify = dao.table('stock_identify')
dao_brand = dao.table('stock_brands')
not_brands = []
for search_row in search_rows:
data_html = edinet.get_report_html(search_row['syorui_kanri_no'])
shorui_mei = search_row['syorui_mei']
Log.info('html取得完了 : {} : {}'.format(search_row['company_name'],
shorui_mei))
if '大株主' not in data_html:
Log.warn('HTMLに大株主が無い。別資料かも。 company_name : {}'.format(
search_row['company_name']))
continue
table_list = edinet.report_html_to_split_table_list(
data_html, shorui_mei)
Log.info('HTML -> table分割 : {}'.format(search_row['company_name']))
dat = {}
for t in table_list:
if t['title'] == '発行済株式' and 'outstanding_share' not in dat:
dat['outstanding_share'] = edinet.outstanding_share(
t, search_row)
Log.debug('** 発行済株式 : {}'.format(
search_row['company_name']))
if t['title'] == '所有者別状況':
dat['holder_rate'] = edinet.holder_rate_status(
t, search_row)
Log.debug('** 所有者別状況 : {}'.format(
search_row['company_name']))
if t['title'] == '大株主の状況':
dat['holders'] = edinet.major_shareholders(t, search_row)
Log.debug('** 大株主の状況 : {}'.format(
search_row['company_name']))
if not dat:
continue
name = search_row['company_name'].replace('(株)',
'').replace(' ', '')
name = re.sub(r'^ ', '', name)
name = re.sub(r' $', '', name)
identify = dao_identify.find_query({'nm': name})
if not identify:
name_s = zenhan.z2h(name, mode=1)
identify = dao_identify.find_query({'nm': name_s})
if not identify:
name_s = name_s.replace(' ', '')
identify = dao_identify.find_query({'nm': name_s})
if identify and len(identify) == 1:
brand = dao_brand.find_by_key(identify[0]['ccode'])
Log.info('stock_brands証券コード取得。 ccode, name : {}, {}'.format(
brand['ccode'], name))
dat['ccode'] = brand['ccode']
dat['name'] = brand['name']
dat['date'] = DateTimeUtil.strf_ymd_st(
DateTimeUtil.date_from_japanese_era(
search_row['submition_day'], short=True))
if empty(dat, 'outstanding_share') or empty(
dat, 'holder_rate') or empty(dat, 'holders'):
edi = dao.table('stock_edinet').find_by_key(dat['ccode'])
if edi:
if empty(dat, 'outstanding_share') and not empty(
edi, 'outstanding_share'):
dat['outstanding_share'] = edi['outstanding_share']
if empty(dat, 'holder_rate') and not empty(
edi, 'holder_rate'):
dat['holder_rate'] = edi['holder_rate']
if empty(dat, 'holders') and not empty(edi, 'holders'):
dat['holders'] = edi['holders']
h1('stock_edinetに登録')
dao.table('stock_edinet').put_item(Item=dat)
else:
dao.table('err_value').put_item(
Item={
'key': 'Job013_edi_name',
'd': DateTimeUtil.str_now(),
'val': name
})
h1('終了')