def test_date_from_japanese_era1(self): assert DateTimeUtil.date_from_japanese_era('平成30年2月22日') == datetime( 2018, 2, 22) assert DateTimeUtil.date_from_japanese_era('平成11年12月31日') == datetime( 1999, 12, 31) assert DateTimeUtil.date_from_japanese_era('平成12年1月1日') == datetime( 2000, 1, 1) assert DateTimeUtil.date_from_japanese_era('昭和1年5月6日') == datetime( 1926, 5, 6) assert DateTimeUtil.date_from_japanese_era('昭和64年1月1日') == datetime( 1989, 1, 1) assert DateTimeUtil.date_from_japanese_era('平成1年1月1日') == datetime( 1989, 1, 1) assert DateTimeUtil.date_from_japanese_era('平成1年1月日') is None assert DateTimeUtil.date_from_japanese_era('平成1年月2日') is None assert DateTimeUtil.date_from_japanese_era('2011年1月2日') is None
def convert_date(self, str_date): str_date = self.norm(str_date) if '平成' in str_date: jap = DateTimeUtil.date_from_japanese_era(str_date) if jap: return DateTimeUtil.strf_ymd_st(jap) else: return '-' else: m = re.search('(\d+)年(\d+)月(\d+)日', str_date) if m: return DateTimeUtil.strf_ymd_st( datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)))) else: return '-'
def run(self, dao: Dao, h1): edinet = Edinet() start_day = DateTimeUtil.yesterday() end_day = DateTimeUtil.now() # start_day = datetime(2018, 5, 19) # end_day = datetime(2018, 5, 20) h1('EDINETの書類検索から指定期間の有価証券報告書の検索結果を取得する。') search_rows = edinet.get_report_search_results(start_day, end_day) search_rows.reverse() # search_rows = [r for r in search_rows if 'スタートトゥデイ' in r['company_name']] h1('検索結果から有価証券報告書を取得し、銘柄毎スクレイピング開始。') dao_identify = dao.table('stock_identify') dao_brand = dao.table('stock_brands') not_brands = [] for search_row in search_rows: data_html = edinet.get_report_html(search_row['syorui_kanri_no']) shorui_mei = search_row['syorui_mei'] Log.info('html取得完了 : {} : {}'.format(search_row['company_name'], shorui_mei)) if '大株主' not in data_html: Log.warn('HTMLに大株主が無い。別資料かも。 company_name : {}'.format( search_row['company_name'])) continue table_list = edinet.report_html_to_split_table_list( data_html, shorui_mei) Log.info('HTML -> table分割 : {}'.format(search_row['company_name'])) dat = {} for t in table_list: if t['title'] == '発行済株式' and 'outstanding_share' not in dat: dat['outstanding_share'] = edinet.outstanding_share( t, search_row) Log.debug('** 発行済株式 : {}'.format( search_row['company_name'])) if t['title'] == '所有者別状況': dat['holder_rate'] = edinet.holder_rate_status( t, search_row) Log.debug('** 所有者別状況 : {}'.format( search_row['company_name'])) if t['title'] == '大株主の状況': dat['holders'] = edinet.major_shareholders(t, search_row) Log.debug('** 大株主の状況 : {}'.format( search_row['company_name'])) if not dat: continue name = search_row['company_name'].replace('(株)', '').replace(' ', '') name = re.sub(r'^ ', '', name) name = re.sub(r' $', '', name) identify = dao_identify.find_query({'nm': name}) if not identify: name_s = zenhan.z2h(name, mode=1) identify = dao_identify.find_query({'nm': name_s}) if not identify: name_s = name_s.replace(' ', '') identify = dao_identify.find_query({'nm': name_s}) if identify and len(identify) == 1: brand = dao_brand.find_by_key(identify[0]['ccode']) Log.info('stock_brands証券コード取得。 ccode, name : {}, {}'.format( brand['ccode'], name)) dat['ccode'] = brand['ccode'] dat['name'] = brand['name'] dat['date'] = DateTimeUtil.strf_ymd_st( DateTimeUtil.date_from_japanese_era( search_row['submition_day'], short=True)) if empty(dat, 'outstanding_share') or empty( dat, 'holder_rate') or empty(dat, 'holders'): edi = dao.table('stock_edinet').find_by_key(dat['ccode']) if edi: if empty(dat, 'outstanding_share') and not empty( edi, 'outstanding_share'): dat['outstanding_share'] = edi['outstanding_share'] if empty(dat, 'holder_rate') and not empty( edi, 'holder_rate'): dat['holder_rate'] = edi['holder_rate'] if empty(dat, 'holders') and not empty(edi, 'holders'): dat['holders'] = edi['holders'] h1('stock_edinetに登録') dao.table('stock_edinet').put_item(Item=dat) else: dao.table('err_value').put_item( Item={ 'key': 'Job013_edi_name', 'd': DateTimeUtil.str_now(), 'val': name }) h1('終了')