def build_managed_osm(conn, body): try: print "build_managed_osm" req_name = "managed_%s" % get_req_name(body['id']) directory = init_directory(req_name) feeds = body['feeds'] download_managed_gtfs(directory, feeds) builder.generate_osm_extract(directory) key = Key(osm_bucket()) osm_key = "%s.osm" % body['id'] key.key = osm_key key.set_contents_from_filename(os.path.join(directory, 'extract.osm')) print 'uploaded osm' publisher = conn.Producer(routing_key="osm_extract_done", exchange=exchange) publisher.publish({'id': body['id'], 'osm_key': osm_key}) print 'published extract_osm_done' if 'trigger_rebuild' in body and body['trigger_rebuild'] is True: publisher = conn.Producer(routing_key="build_managed", exchange=exchange) publisher.publish({ 'id': body['id'], 'osm_key': osm_key, 'feeds': feeds, 'otp_version': body['otp_version'] }) except: now = datetime.now() errfile = "/var/otp/gb_err_%s_%s" % (body['id'], now.strftime("%F-%T")) traceback.print_exc(file=open(errfile, "a")) traceback.print_exc()
def build_managed_osm(conn, body): try: print "build_managed_osm" req_name = "managed_%s" % get_req_name(body['id']); directory = init_directory(req_name); feeds = body['feeds'] download_managed_gtfs(directory, feeds) builder.generate_osm_extract(directory) key = Key(osm_bucket()) osm_key = "%s.osm" % body['id'] key.key = osm_key key.set_contents_from_filename(os.path.join(directory, 'extract.osm')) print 'uploaded osm' publisher = conn.Producer(routing_key="osm_extract_done", exchange=exchange) publisher.publish({ 'id' : body['id'], 'osm_key' : osm_key }) print 'published extract_osm_done' if 'trigger_rebuild' in body and body['trigger_rebuild'] is True: publisher = conn.Producer(routing_key="build_managed", exchange=exchange) publisher.publish({ 'id' : body['id'], 'osm_key' : osm_key, 'feeds' : feeds, 'otp_version' : body['otp_version'] }) except: now = datetime.now() errfile = "/var/otp/gb_err_%s_%s" % (body['id'], now.strftime("%F-%T")) traceback.print_exc(file=open(errfile,"a")) traceback.print_exc()
def create_instance(conn, body): try: req_name = get_req_name(body['request_id']); directory = init_directory(req_name); # extract gtfs files os.makedirs(os.path.join(directory, 'gtfs')) files = body['files'] extra_props_list = body['extra_properties'] extra_props_dict = { } out = [] i = 0 for s3_id in files: if s3_id is None: continue print "id: " + s3_id bucket = gtfs_bucket() key = Key(bucket) key.key = s3_id basename = os.path.basename(s3_id) path = os.path.join(directory, 'gtfs', basename) key.get_contents_to_filename(path) extra_props_dict[basename] = extra_props_list[i] i += 1 # prepare and run graph builder builder.generate_osm_extract(directory) builder.generate_graph_config(directory, body['fare_factory'], extra_props_dict) gbresults = builder.run_graph_builder(directory) print "finished gb: %s" % gbresults['success'] msgparams = { } msgparams['request_id'] = body['request_id'] msgparams['success'] = gbresults['success'] bucket = graph_bucket() if gbresults['success']: key = Key(bucket) key.key = "uploads/%s/Graph_%s.obj" % (body['request_id'], datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")) graph_file = os.path.join(directory,'Graph.obj') key.set_contents_from_filename(graph_file) key.set_acl('public-read') msgparams['key'] = key.key subprocess.call(['rm', graph_file]) msgparams['otp_version'] = get_otp_version() # create data tarball and upload to s3 tarball = os.path.join('/mnt', ('%s.tar.gz' % req_name)) subprocess.call(['tar', 'czf', tarball, directory]) key = Key(bucket) data_key = "data/%s.tar.gz" % req_name key.key = data_key key.set_contents_from_filename(tarball) msgparams['data_key'] = data_key # publish graph_done message publisher = conn.Producer(routing_key="graph_done", exchange=exchange) publisher.publish(msgparams) print 'published graph_done' # write gb output to file to s3 write_output_to_s3("output/%s_output.txt" % req_name, gbresults['output']) except: now = datetime.now() errfile = "/var/otp/gb_err_%s_%s" % (body['request_id'], now.strftime("%F-%T")) traceback.print_exc(file=open(errfile,"a"))
def create_instance(conn, body): try: req_name = get_req_name(body['request_id']) directory = init_directory(req_name) # extract gtfs files os.makedirs(os.path.join(directory, 'gtfs')) files = body['files'] extra_props_list = body['extra_properties'] extra_props_dict = {} out = [] i = 0 for s3_id in files: if s3_id is None: continue print "id: " + s3_id bucket = gtfs_bucket() key = Key(bucket) key.key = s3_id basename = os.path.basename(s3_id) path = os.path.join(directory, 'gtfs', basename) key.get_contents_to_filename(path) extra_props_dict[basename] = extra_props_list[i] i += 1 # prepare and run graph builder builder.generate_osm_extract(directory) builder.generate_graph_config(directory, body['fare_factory'], extra_props_dict) gbresults = builder.run_graph_builder(directory) print "finished gb: %s" % gbresults['success'] msgparams = {} msgparams['request_id'] = body['request_id'] msgparams['success'] = gbresults['success'] bucket = graph_bucket() if gbresults['success']: key = Key(bucket) key.key = "uploads/%s/Graph_%s.obj" % ( body['request_id'], datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")) graph_file = os.path.join(directory, 'Graph.obj') key.set_contents_from_filename(graph_file) key.set_acl('public-read') msgparams['key'] = key.key subprocess.call(['rm', graph_file]) msgparams['otp_version'] = get_otp_version() # create data tarball and upload to s3 tarball = os.path.join('/mnt', ('%s.tar.gz' % req_name)) subprocess.call(['tar', 'czf', tarball, directory]) key = Key(bucket) data_key = "data/%s.tar.gz" % req_name key.key = data_key key.set_contents_from_filename(tarball) msgparams['data_key'] = data_key # publish graph_done message publisher = conn.Producer(routing_key="graph_done", exchange=exchange) publisher.publish(msgparams) print 'published graph_done' # write gb output to file to s3 write_output_to_s3("output/%s_output.txt" % req_name, gbresults['output']) except: now = datetime.now() errfile = "/var/otp/gb_err_%s_%s" % (body['request_id'], now.strftime("%F-%T")) traceback.print_exc(file=open(errfile, "a"))