def parse(): # parse gathered data and save as csv logger.info("parse") scrapper = Scraper(Persistor) parser = Parser() raw_data = scrapper.scrape() data = [] for raw in raw_data: data = scrapper.append_data(data, raw) parsed_files = [parser.parse_object(file) for file in data] scrapper.save_csv(parsed_files)
def parse(): logger.info("parse") storage = Persistor(SCRAPPED_FILE) parser = Parser() raw_data = storage.read_raw_data() ind_start = raw_data.find('table class=\"wikitable sortable\"') raw_data = raw_data[ind_start:] ind_end = raw_data.find('</table>') raw_data = raw_data[:ind_end + len('</table>')] all_rows = re.findall('<tr[^^]*?</tr>', raw_data) parsed_files = [parser.parse_object(raw) for raw in all_rows] storage.save_csv(parsed_files, TABLE_FORMAT_FILE)