import utils ########################################################################### if __name__ == "__main__": db.init() progress = int(db.get_param('location_update_progress', 0)) db.cur.execute("SELECT id FROM location WHERE id > ? ORDER BY id", (progress,)) rows = db.cur.fetchall() tobedone = map(lambda x: x[0], rows) lp = LocationParser() for id in tobedone: print "\n\nupdate location id", id url = utils.get_location_url(id) html = utils.get_html(url, data=id) try: lp.parse(id, html) db.cache_del(url) except Exception as e: print " ", id, "failed, error:", e db.update_table('location', 'id', id, parsed='F') raise db.set_param('location_update_progress', id) db.close()
import utils ########################################################################### if __name__ == "__main__": db.init() progress = int(db.get_param('route_update_progress', 0)) db.cur.execute("SELECT id FROM route WHERE id > ? ORDER BY id", (progress,)) rows = db.cur.fetchall() tobedone = map(lambda x: x[0], rows) rp = RouteParser() for id in tobedone: print "\n\nupdate route id", id url = utils.get_route_url(id) html = utils.get_html(url, data=id) try: rp.parse(id, html) db.cache_del(url) except Exception as e: print " ", id, "failed, error:", e db.update_table('route', 'id', id, parsed='F') raise db.set_param('route_update_progress', id) db.close()
from stop_parser import StopParser import db import utils MAX_STOP_COUNT = 23611 ########################################################################### if __name__ == "__main__": db.init() progress = int(db.get_param('stop_progress', 0)) tobedone = xrange(progress+1, MAX_STOP_COUNT+1, 1) sp = StopParser() for id in tobedone: print "\n\nprocess stop id", id url = utils.get_stop_url(id) html = utils.get_html(url, data=id) try: sp.parse(id, html) db.cache_del(url) except Exception as e: print " ", id, "failed, error:", e db.update_table('stop', 'id', id, parsed='F') db.set_param('stop_progress', id) db.close()