def scrape_bills_and_urls_from_table(table, response_url): def get_bill_category_or_none(caption): if caption == '閣法の一覧': return BillCategory.KAKUHOU elif caption == '衆法の一覧': return BillCategory.SHUHOU elif caption == '参法の一覧': return BillCategory.SANHOU else: return None bills, urls = [], [] caption = extract_text(table.xpath('./caption')).strip() maybe_bill_category = get_bill_category_or_none(caption) if not maybe_bill_category: return bills, urls bill_category = maybe_bill_category for row in table.xpath('./tr')[1:]: # skip header cells = row.xpath('./td') assert len(cells) == 6 # build Bill instance with necessary info try: diet_number = int(extract_text(cells[0])) submission_number = int(extract_text(cells[1])) bill_name = extract_text(cells[2]) except Exception as e: LOGGER.warning(f'failed to parse row:\n{row.get()}\n{e}') continue bill = build_bill(bill_category, diet_number, submission_number, bill_name) bills.append(bill) # build keika URL if exists maybe_keika_href = extract_full_href_or_none( cells[4], response_url) if maybe_keika_href: url = build_url(maybe_keika_href, UrlTitle.KEIKA, ShugiinSpider.domain) url.meta = {'bill_id': bill.id} urls.append(url) # build honbun URL if exists maybe_honbun_href = extract_full_href_or_none( cells[5], response_url) if maybe_honbun_href: url = build_url(maybe_honbun_href, UrlTitle.HONBUN, ShugiinSpider.domain) url.meta = {'bill_id': bill.id} urls.append(url) return bills, urls
def scrape_minutes_urls_from_response(response): urls = [] for li in response.xpath('//div[@id="mainlayout"]/li'): url = extract_full_href_or_none(li, response.url) if url: urls.append(url) return urls
def parse_minutes(self, response): # merge url if exists maybe_href = extract_full_href_or_none(response.xpath('//h4'), response.url) if not maybe_href: LOGGER.warning(f'failed to find url in {response.url}') return url = build_url(maybe_href, title=UrlTitle.GAIYOU_PDF, domain=self.domain) self.gql_client.merge(url) LOGGER.debug(f'merged {url.id}') # link to minutes title = extract_text(response.xpath('//title')) committee_name = response.meta['committee_name'] date_time = self.extract_datetime_from_title(title) minutes = build_minutes(committee_name, date_time) try: self.gql_client.get(minutes.id, ['id']) # minutes should already exist self.gql_client.link(url.id, minutes.id) except GraphQLException: LOGGER.warning( f'failed to find minutes ({committee_name}, {date_time})')
def scrape_bills_and_urls_from_table(self, table, bill_category, response_url): bills, urls = [], [] for row in table.xpath('./tr')[1:]: # skip header cells = row.xpath('./td') assert len(cells) == 5 # build Bill instance with necessary info try: diet_number = int(extract_text(cells[0])) submission_number = int(extract_text(cells[1])) bill_name = extract_text(cells[2]) except Exception as e: LOGGER.warning(f'failed to parse row:\n{row.get()}\n{e}') continue bill = build_bill(bill_category, diet_number, submission_number, bill_name) bills.append(bill) # build URL if exists maybe_meisai_href = extract_full_href_or_none( cells[2], response_url) if maybe_meisai_href: url = build_url(maybe_meisai_href, UrlTitle.GIAN_ZYOUHOU, self.domain) url.meta = {'bill_id': bill.id} urls.append(url) return bills, urls
def scrape_committees_from_table(table, root_url): committees = [] for row in table.xpath('./tr'): for cell in row.xpath('./td'): committee = Committee(None) committee.name = '衆議院' + extract_text(cell.xpath('./span/a')) committee.url = extract_full_href_or_none(cell, root_url) committees.append(committee) return committees
def scrape_members_and_urls(self, response): members, urls = [], [] table = response.xpath('//table[@summary="議員一覧(50音順)"]')[0] for row in table.xpath('./tr')[1:]: # skip header cells = row.xpath('./td') assert len(cells) == 6 name = ''.join(extract_text(cells[0]).strip().split()) tags = [ # store 会派 and 選挙区 as tags for now extract_text(cells[2]).strip(), extract_text(cells[3]).strip() ] member = build_member(name) member.tags = tags member.house = 'COUNCILORS' members.append(member) maybe_href = extract_full_href_or_none(cells[0], response.url) if maybe_href: url = build_url(maybe_href, UrlTitle.GIIN_ZYOUHOU, self.domain) url.meta = {'member_id': member.id} urls.append(url) return members, urls