#!/usr/bin/python
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from location_parser import LocationParser
import db
import utils
###########################################################################
if __name__ == "__main__":
db.init()
progress = int(db.get_param('location_update_progress', 0))
db.cur.execute("SELECT id FROM location WHERE id > ? ORDER BY id", (progress,))
rows = db.cur.fetchall()
tobedone = map(lambda x: x[0], rows)
lp = LocationParser()
for id in tobedone:
print "\n\nupdate location id", id
url = utils.get_location_url(id)
html = utils.get_html(url, data=id)
try:
lp.parse(id, html)
db.cache_del(url)
except Exception as e:
print " ", id, "failed, error:", e
db.update_table('location', 'id', id, parsed='F')
#!/usr/bin/python
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from route_parser import RouteParser
import db
import utils
###########################################################################
if __name__ == "__main__":
db.init()
progress = int(db.get_param('route_update_progress', 0))
db.cur.execute("SELECT id FROM route WHERE id > ? ORDER BY id", (progress,))
rows = db.cur.fetchall()
tobedone = map(lambda x: x[0], rows)
rp = RouteParser()
for id in tobedone:
print "\n\nupdate route id", id
url = utils.get_route_url(id)
html = utils.get_html(url, data=id)
try:
rp.parse(id, html)
db.cache_del(url)
except Exception as e:
print " ", id, "failed, error:", e
db.update_table('route', 'id', id, parsed='F')
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from stop_parser import StopParser
import db
import utils
MAX_STOP_COUNT = 23611
###########################################################################
if __name__ == "__main__":
db.init()
progress = int(db.get_param('stop_progress', 0))
tobedone = xrange(progress+1, MAX_STOP_COUNT+1, 1)
sp = StopParser()
for id in tobedone:
print "\n\nprocess stop id", id
url = utils.get_stop_url(id)
html = utils.get_html(url, data=id)
try:
sp.parse(id, html)
db.cache_del(url)
except Exception as e:
print " ", id, "failed, error:", e
db.update_table('stop', 'id', id, parsed='F')
db.set_param('stop_progress', id)
