def process_gremium(engine, indexer, url, force=False):
try:
data = scrape_gremium(engine, url, force=force)
data = fetch_row(engine, 'gremium', key=data['key'])
gremium = load_gremium(engine, data)
indexer.add(gremium)
except Unmodified: pass
def process_gremium(engine, indexer, url, force=False):
try:
data = scrape_gremium(engine, url, force=force)
data = fetch_row(engine, 'gremium', key=data['key'])
gremium = load_gremium(engine, data)
indexer.add(gremium)
except Unmodified:
pass
def process_ablauf(engine, indexer, url, force=False):
try:
data = scrape_ablauf(engine, url, force=force)
clean_ablauf(engine, data)
extend_positions(engine, url)
match_beitraege(engine, url)
data = fetch_row(engine, 'ablauf', source_url=url)
load_ablauf(engine, indexer, data)
except Unmodified: pass
except NoContentException: pass
def process_ablauf(engine, indexer, url, force=False):
try:
data = scrape_ablauf(engine, url, force=force)
clean_ablauf(engine, data)
extend_positions(engine, url)
match_beitraege(engine, url)
data = fetch_row(engine, 'ablauf', source_url=url)
load_ablauf(engine, indexer, data)
except Unmodified:
pass
except NoContentException:
pass
def process_person(engine, indexer, url, force=False):
try:
data = scrape_mdb(engine, url, force=force)
make_fingerprint(engine, data)
try:
parse_angaben(engine, data)
except:
log.warn("Angaben parse did not work")
pass
data = fetch_row(engine, 'person', mdb_id=data['mdb_id'])
person = load_person(engine, data)
indexer.add(person)
except Unmodified:
pass
