def update():
engine = get_engine()
q = engine.query(QUERY).next()
if not len(q.keys()):
return
next = q.values().pop()
year, num = next.split('-')
years = range(int(year), datetime.now().year+1)
offsets = [int(num)] + ([0] * len(years))
print "CONTINUING", zip(years, offsets)
for year, offset in zip(years, offsets):
make_session()
all_entries(year, offset=offset)
for paths in traverse_local([year], offset):
parse_tender(engine, paths)
def parse_threaded(engine):
def fnc(p):
parse_tender(engine, p)
threaded(traverse_local(), fnc, num_threads=10)
def parse(engine):
for paths in traverse_local():
parse_tender(engine, paths)
