def process_message(raw_status_report): message = ServiceMessage(raw_status_report) return message.extract_full_data()
in_soup = BeautifulSoup(response.text, "html.parser") else: raise Exception(METRO_ARCHIVES, response.status_code, response.text) ret_vals = [] links = in_soup.findAll('a') for html_link in links: if 'href' in html_link.attrs and html_link.attrs['href'].find(REPORT_LINK_SIGNATURE) > -1: ret_vals.append(DatedLink(pagelink=html_link.attrs['href'], datetext=html_link.text, report_base=REPORT_BASE)) return ret_vals if __name__ == "__main__": out_data = [] report_page_links = gather_dated_links() for link in report_page_links: print(link) for item in link.parse_page(): cline = ServiceMessage(item, parent=link.make_parent()) full_data = cline.extract_full_data() full_data['event_dtg'] = full_data['event_dtg'].isoformat() out_data.append(full_data) out_file = open('out.json', 'w') out_file.write(json.dumps(out_data)) out_file.close()