class IfrsOperatingRevenueAssembler(): def __init__(self): self.base_xpath = '//html/body' self.content_screener = ContentScreener() self.string_utils = StringUtils() def assemble(self, param): content, stock_symbol, date = param['content'], param['stock_symbol'], param['date'] self.content_screener.screen(param) content = self.string_utils.normalize_string(content) html_object = lxml.html.fromstring(content) relative_html_object = self.__traverse_to_relative_html_object(html_object) column_name_list = self.__assemble_column_name_list(relative_html_object) row_list = self.__assemble_row_list(relative_html_object) return OperatingRevenueDao(column_name_list, row_list, stock_symbol, date) def __traverse_to_relative_html_object(self, html_object): relative_html_object_list = html_object.xpath(self.base_xpath) assert len(relative_html_object_list) > 0, 'invalid base_xpath' return relative_html_object_list[0] def __assemble_column_name_list(self, relative_html_object): # traverse and sanity check tr_tags = relative_html_object.xpath('./table[@class="hasBorder"]/tr') assert len(tr_tags) > 0, 'invalid tr_tags' # traverse and sanity check th_texts = tr_tags[0].xpath('./th/text()') assert len(th_texts) == 2, 'invalid th_texts size, should be 2' # should be account account = th_texts[0] # traverse and sanity check table_tags = relative_html_object.xpath('./table[@class="noBorder"]') assert len(table_tags) > 0, 'invalid table_tags' td_tags = table_tags[2].xpath('./td') assert len(td_tags) > 0, 'invalid td_tags' # should be snapdate snapdate = self.string_utils.from_local_string_to_date(td_tags[1].text) return [account, snapdate] def __assemble_row_list(self, relative_html_object): # skip one row of column name list tr_tags = relative_html_object.xpath('./table[@class="hasBorder"]/tr')[1:] return [self.__assemble_row(tr_tag) for tr_tag in tr_tags] def __assemble_row(self, relative_html_object): # should be item th_texts = relative_html_object.xpath('./th/text()') assert len(th_texts) == 1, 'invalid th_texts size, should be 1' item = th_texts[0] # should be number (operating revenue) td_texts = relative_html_object.xpath('./td/text()') assert len(th_texts) == 1, 'invalid td_texts size, should be 1' number_string = td_texts[0] number = self.string_utils.normalize_number(number_string) return [item, number]
class LegacyCashFlowAssembler(): def __init__(self): self.base_xpath = '//html/body/table[@class="hasBorder"]/tr/td/pre' self.string_utils = StringUtils() def assemble(self, param): content, stock_symbol, date = param['content'], param['stock_symbol'], param['date'] content = self.string_utils.normalize_string(content) html_object = lxml.html.fromstring(content) relative_html_object = self.__traverse_to_relative_html_object(html_object) # Parse cash flow statement column_name_list, row_list = self.__assemble_summary(relative_html_object.text) return CashFlowDao(column_name_list, row_list, stock_symbol, date) def __traverse_to_relative_html_object(self, html_object): relative_html_object_list = html_object.xpath(self.base_xpath) assert len(relative_html_object_list) == 1, 'invalid base_xpath' return relative_html_object_list[0] def __assemble_summary(self, text): return AriesParser(text).parse()
class XbrlIncomeStatementAssembler(): def __init__(self): self.base_xpath = '//html/body[@id="content_d"]/center/table[@class="main_table hasBorder"]' self.string_utils = StringUtils() def assemble(self, param): content, stock_symbol, date = param['content'], param['stock_symbol'], param['date'] content = self.string_utils.normalize_string(content) html_object = lxml.html.fromstring(content) relative_html_object = self.__traverse_to_relative_html_object(html_object) column_name_list = self.__assemble_column_name_list(relative_html_object) row_list = self.__assemble_row_list(relative_html_object) return IncomeStatementDao(column_name_list, row_list, stock_symbol, date) def __traverse_to_relative_html_object(self, html_object): relative_html_object_list = html_object.xpath(self.base_xpath) assert len(relative_html_object_list) > 0, 'invalid base_xpath' return relative_html_object_list[0] def __assemble_column_name_list(self, relative_html_object): # traverse and sanity check tr_tags = relative_html_object.xpath('./tr[@class="tblHead"]') assert len(tr_tags) == 2, 'invalid tr_tags' # traverse and sanity check statement_th_texts = tr_tags[1].xpath('./th/text()') assert len(statement_th_texts) == 1, 'invalid statement_th_texts' assert unicode(statement_th_texts[0]) == u'綜合損益表', 'invalid statement_th_texts[0]' column_name_list = [] # should be account type column_th_texts = tr_tags[0].xpath('./th/text()') account_type = column_th_texts[0] # of unicode type column_name_list.append(account_type) # should be date interval for local_string in column_th_texts[1:]: # of (datetime.date, datetime.date) type snapshot_date = self.string_utils.from_local_string_to_date_period(local_string) column_name_list.append(snapshot_date) return column_name_list def __assemble_row_list(self, relative_html_object): # skip one row of statement name and one row of column name list tr_tags = relative_html_object.xpath('./tr')[2:] return [self.__assemble_row(tr_tag) for tr_tag in tr_tags] def __assemble_row(self, relative_html_object): row = [] td_texts = relative_html_object.xpath('./td/text()') # should be account type account_type = td_texts[0].strip() row.append(account_type) # should be number for number_string in td_texts[1:]: number = self.string_utils.normalize_number(number_string) row.append(number) return row
class TaurusParser(): def __init__(self): self.base_xpath = '//html/body/center' self.date_utils = DateUtils() self.string_utils = StringUtils() def parse(self, content): html_object = self.__get_html_object(content) relative_html_object = self.__traverse_to_relative_html_object(html_object) column_name_list = self.__parse_column_name_list(relative_html_object) row_list = self.__parse_row_list(relative_html_object) release_date = self.__parse_release_date(relative_html_object) return column_name_list, row_list, release_date def __get_html_object(self, content): content = self.string_utils.normalize_string(content) content = content.replace(u'<br>', u'') return lxml.html.fromstring(content) def __traverse_to_relative_html_object(self, html_object): relative_html_object_list = html_object.xpath(self.base_xpath) assert len(relative_html_object_list) > 0, 'invalid base_xpath' return relative_html_object_list[0] def __parse_column_name_list(self, relative_html_object): # traverse and sanity check table_tags = relative_html_object.xpath('./table') assert len(table_tags) > 1, 'invalid table_tags' # skip first table of description about IFRS inner_table_tags = table_tags[1].xpath('./tr/td/table/tr/td/table') assert len(inner_table_tags) > 0, 'invalid inner_table_tags' tr_tags = inner_table_tags[0].xpath('./tr') assert len(tr_tags) > 1, 'invalid tr_tags' th_texts = tr_tags[1].xpath('./th/text()') return th_texts def __parse_row_list(self, relative_html_object): # traverse and sanity check table_tags = relative_html_object.xpath('./table') assert len(table_tags) > 1, 'invalid table_tags' # skip first table of description about IFRS inner_table_tags = table_tags[1].xpath('./tr/td/table/tr/td/table') assert len(inner_table_tags) > 0, 'invalid inner_table_tags' all_tr_tags = [] # every inner_table represents an industry for inner_table_tag in inner_table_tags: tr_tags = inner_table_tag.xpath('./tr') assert len(tr_tags) > 2, 'invalid tr_tags' # first two rows are headers # last row is u'合計' all_tr_tags += tr_tags[2:-1] return [self.__parse_row(tr_tag) for tr_tag in all_tr_tags] def __parse_row(self, relative_html_object): td_texts = relative_html_object.xpath('./td/text()') # record contains extra entry about comment assert len(td_texts) == 11, 'invalid td_texts size, should be 11' items = td_texts[:2] numbers = [] # skip the last entry about comment for td_text in td_texts[2:-1]: number = self.string_utils.normalize_number(td_text) numbers.append(number) return items + numbers def __parse_release_date(self, relative_html_object): div_tags = relative_html_object.xpath('./div') assert len(div_tags) > 0, 'invalid div_tags' groups = self.string_utils.match(u'^出表日期:(.*)$', div_tags[-1].text.strip()) assert len(groups) > 0, 'could not match ^出表日期:(.*)$' release_date = self.string_utils.from_local_string_to_date(groups[0]) return release_date
class LegacyIncomeStatementAssembler(): def __init__(self): self.base_xpath = '//html/body/center/table' self.string_utils = StringUtils() def assemble(self, param): content, stock_symbol, date = param['content'], param['stock_symbol'], param['date'] content = self.string_utils.normalize_string(content) html_object = lxml.html.fromstring(content) relative_html_object = self.__traverse_to_relative_html_object(html_object) column_name_list = self.__assemble_column_name_list(relative_html_object) row_list = self.__assemble_row_list(relative_html_object) return IncomeStatementDao(column_name_list, row_list, stock_symbol, date) def __traverse_to_relative_html_object(self, html_object): relative_html_object_list = html_object.xpath(self.base_xpath) assert len(relative_html_object_list) == 1, 'invalid base_xpath' return relative_html_object_list[0] def __assemble_column_name_list(self, relative_html_object): # traverse and sanity check tr_tags = relative_html_object.xpath('./tr') assert len(tr_tags) > 0, 'invalid tr_tags' column_name_list = [] # should be account type column_th_texts = tr_tags[3].xpath('./th/b/text()') account_type = column_th_texts[0] # of unicode type column_name_list.append(account_type) for local_string in column_th_texts[1:]: # of datetime.date type snapshot_date = self.string_utils.from_local_string_to_date(local_string) column_name_list.append(snapshot_date) return column_name_list def __assemble_row_list(self, relative_html_object): # skip column part (5 rows) tr_tags = relative_html_object.xpath('./tr')[5:] return [self.__assemble_row(tr_tag) for tr_tag in tr_tags] def __assemble_row(self, relative_html_object): row = [] td_texts = relative_html_object.xpath('./td/text()') td_texts = self.__remove_empty_string_from_string_list(td_texts) # should be account type account_type = td_texts[0].strip() row.append(account_type) # should be number for number_string in td_texts[1:]: number = self.string_utils.normalize_number(number_string) row.append(number) return row def __remove_empty_string_from_string_list(self, string_list): return [string for string in string_list if string.strip()]
class DividendPolicyAssembler(): def __init__(self): self.base_xpath = '//html/body/div[@id="SysJustIFRAMEDIV"]/table/tr/td/table/tr/td/table' self.content_screener = ContentScreener() self.string_utils = StringUtils() def assemble(self, param): content, stock_symbol = param['content'], param['stock_symbol'] self.content_screener.screen(param) html_object = self.__get_html_object(content) relative_html_object = self.__traverse_to_relative_html_object(html_object) column_name_list = self.__assemble_column_name_list(relative_html_object) row_list = self.__assemble_row_list(relative_html_object) return DividendPolicyDao(column_name_list, row_list, stock_symbol) def __get_html_object(self, content): content = self.string_utils.normalize_string(content) content = content.replace(u'<br>', u'') return lxml.html.fromstring(content) def __traverse_to_relative_html_object(self, html_object): relative_html_object_list = html_object.xpath(self.base_xpath) assert len(relative_html_object_list) > 0, 'invalid base_xpath (table_tags)' return relative_html_object_list[0] def __assemble_column_name_list(self, relative_html_object): # traverse and sanity check tr_tags = relative_html_object.xpath('./tr') assert len(tr_tags) > 2, 'invalid tr_tags' # skip the first row of header td_texts = tr_tags[2].xpath('./td/text()') assert len(td_texts) == 7, 'invalid td_texts size, should be 7' return [text.strip() for text in td_texts] def __assemble_row_list(self, relative_html_object): # skip the first row of header # skip the second row of empty lines # skip the third row of column name list tr_tags = relative_html_object.xpath('./tr') assert len(tr_tags) > 2, 'invalid tr_tags' return [self.__assemble_row(tr_tag) for tr_tag in tr_tags[3:]] def __assemble_row(self, relative_html_object): # should be item td_texts = relative_html_object.xpath('./td/text()') assert len(td_texts) == 7, 'invalid td_texts size, should be 7' row = [] # should be stmt_date stmt_date = self.string_utils.from_local_string_to_date(td_texts[0]) row.append(stmt_date) # should be number for number_string in td_texts[1:-1]: number = self.string_utils.normalize_number(number_string) row.append(number) # should be number in percentage for number_string in td_texts[-1:]: number = self.string_utils.normalize_number(number_string) * 0.01 row.append(number) return row
class StockSymbolAssembler(): def __init__(self): self.base_xpath = '//html/body' self.string_utils = StringUtils() def assemble(self, param): content = self.string_utils.normalize_string(param['content']) html_object = lxml.html.fromstring(content) relative_html_object = self.__traverse_to_relative_html_object(html_object) release_date = self.__assemble_release_date(relative_html_object) column_name_list = self.__assemble_column_name_list(relative_html_object) row_list = self.__assemble_row_list(relative_html_object) return StockSymbolDao(column_name_list, row_list, release_date) def __traverse_to_relative_html_object(self, html_object): relative_html_object_list = html_object.xpath(self.base_xpath) assert len(relative_html_object_list) == 1, 'invalid base_xpath' return relative_html_object_list[0] def __assemble_release_date(self, relative_html_object): # try to get release date table_tags = relative_html_object.xpath('./table') assert len(table_tags) > 0, 'invalid table_tags' headline_tags = table_tags[0].xpath('./h2') assert len(headline_tags) > 0, 'invalid headline_tags' headline_texts = headline_tags[1].xpath('./strong/center') groups = self.string_utils.match(u'^最近更新日期:(.*)$', headline_texts[0].text.strip()) assert len(groups) > 0, 'could not match ^最近更新日期:(.*)$' release_date = self.string_utils.from_local_string_to_date(groups[0]) return release_date def __assemble_column_name_list(self, relative_html_object): # traverse and sanity check tr_tags = relative_html_object.xpath('./table[@class="h4"]/tr') assert len(tr_tags) > 0, 'invalid tr_tags' # traverse and sanity check original_column_name_list = tr_tags[0].xpath('./td/text()') # handle the first column name: '有價證券代號及名稱' combined_column_name = original_column_name_list[0].strip() assert combined_column_name == u'有價證券代號及名稱', 'should be 有價證券代號及名稱 in unicode' # the chinese character '及' means 'and' so we need to seperate this column name seperated_column_name_list = combined_column_name.split(u'及') assert len(seperated_column_name_list) == 2 column_name_list = seperated_column_name_list + original_column_name_list[1:] assert len(column_name_list) == 8, 'invalid column_name_list size, should be 8' return column_name_list def __assemble_row_list(self, relative_html_object): # skip one row of column name list tr_tags = relative_html_object.xpath('./table[@class="h4"]/tr')[1:] row_list = [] for tr_tag in tr_tags: row = self.__assemble_row(tr_tag) # if there is only one cell '股票' in row, skip it if row: row_list.append(row) return row_list def __assemble_row(self, relative_html_object): td_tags = relative_html_object.xpath('./td') # we could not handle empty string between td tag if we use xpath './td/text()' # so we need to check each td.text one by one. td_texts = self.__get_lxml_text_list(td_tags) # if there is only one cell '股票', return None if len(td_texts) == 1: return None # sanity check assert len(td_texts) == 7 # handle the first cell: '有價證券代號及名稱' # it should be seperated as stock symbol and stock name combined_cell = td_texts[0].strip() seperated_cell_list = combined_cell.split() assert len(seperated_cell_list) == 2 # convert to datetime.date type listing_date = self.string_utils.from_local_string_to_date(td_texts[2]) row = seperated_cell_list + [td_texts[1]] + [listing_date] + td_texts[3:] return row def __get_lxml_text_list(self, tag_list): text_list = [] for tag in tag_list: if tag.text is None: text_list.append('') else: text_list.append(tag.text) return text_list
class GeminiParser: def __init__(self): self.base_xpath = "//html/body/center" self.date_utils = DateUtils() self.string_utils = StringUtils() def parse(self, content): html_object = self.__get_html_object(content) relative_html_object = self.__traverse_to_relative_html_object(html_object) column_name_list = self.__parse_column_name_list(relative_html_object) row_list = self.__parse_row_list(relative_html_object) release_date = self.__parse_release_date(relative_html_object) return column_name_list, row_list, release_date def __get_html_object(self, content): content = self.string_utils.normalize_string(content) content = content.replace(u"<br>", u"") return lxml.html.fromstring(content) def __traverse_to_relative_html_object(self, html_object): relative_html_object_list = html_object.xpath(self.base_xpath) assert len(relative_html_object_list) > 0, "invalid base_xpath" return relative_html_object_list[0] def __parse_column_name_list(self, relative_html_object): # traverse and sanity check table_tags = relative_html_object.xpath("./table") assert len(table_tags) > 1, "invalid table_tags" # skip first table of description about IFRS inner_table_tags = table_tags[1].xpath("./tr/td/table/tr/td/table") assert len(inner_table_tags) > 0, "invalid inner_table_tags" tr_tags = inner_table_tags[0].xpath("./tr") assert len(tr_tags) > 1, "invalid tr_tags" th_texts = tr_tags[1].xpath("./th/text()") return th_texts def __parse_row_list(self, relative_html_object): # traverse and sanity check table_tags = relative_html_object.xpath("./table") assert len(table_tags) > 1, "invalid table_tags" # skip first table of description about IFRS inner_table_tags = table_tags[1].xpath("./tr/td/table/tr/td/table") assert len(inner_table_tags) > 0, "invalid inner_table_tags" all_tr_tags = [] # every inner_table represents an industry for inner_table_tag in inner_table_tags: tr_tags = inner_table_tag.xpath("./tr") assert len(tr_tags) > 2, "invalid tr_tags" # first two rows are headers # last row is u'合計' all_tr_tags += tr_tags[2:-1] return [self.__parse_row(tr_tag) for tr_tag in all_tr_tags] def __parse_row(self, relative_html_object): td_texts = relative_html_object.xpath("./td/text()") assert len(td_texts) == 10, "invalid td_texts size, should be 10" items = td_texts[:2] numbers = [] for td_text in td_texts[2:]: number = self.string_utils.normalize_number(td_text) numbers.append(number) return items + numbers def __parse_release_date(self, relative_html_object): div_tags = relative_html_object.xpath("./div") assert len(div_tags) > 0, "invalid div_tags" groups = self.string_utils.match(u"^出表日期:(.*)$", div_tags[-1].text.strip()) assert len(groups) > 0, "could not match ^出表日期:(.*)$" release_date = self.string_utils.from_local_string_to_date(groups[0]) return release_date