def process_gremium(engine, indexer, url, force=False): try: data = scrape_gremium(engine, url, force=force) data = fetch_row(engine, 'gremium', key=data['key']) gremium = load_gremium(engine, data) indexer.add(gremium) except Unmodified: pass
def process_ablauf(engine, indexer, url, force=False): try: data = scrape_ablauf(engine, url, force=force) clean_ablauf(engine, data) extend_positions(engine, url) match_beitraege(engine, url) data = fetch_row(engine, 'ablauf', source_url=url) load_ablauf(engine, indexer, data) except Unmodified: pass except NoContentException: pass
def process_person(engine, indexer, url, force=False): try: data = scrape_mdb(engine, url, force=force) make_fingerprint(engine, data) try: parse_angaben(engine, data) except: log.warn("Angaben parse did not work") pass data = fetch_row(engine, 'person', mdb_id=data['mdb_id']) person = load_person(engine, data) indexer.add(person) except Unmodified: pass