def generate_streamed_json_response(): cartogram_gen_output = b'' current_loading_point = "null" # We have to format our JSON manually, since we're not sending a complete object. # On the client side, Oboe.js is intelligent enough to parse this and get the loading information and cartogram output yield '{"loading_progress_points":[' for source, line in cartwrap.generate_cartogram( cartogram_handler.gen_area_data(values), cartogram_handler.get_gen_file(), settings.CARTOGRAM_EXE): if source == "stdout": cartogram_gen_output += line else: s = re.search(r'max\. abs\. area error: (.+)', line.decode()) if s != None: current_loading_point = s.groups(1)[0] # We always include the loading progress, even if it hasn't changed. # This makes life easier on the client side yield '{{"loading_point": {}, "stderr_line": "{}"}},'.format( current_loading_point, line.decode()) # We create a fake last entry because you can't have dangling commas in JSON yield '{"loading_point":0, "stderr_line": ""}],"cartogram_data":' if cartogram_handler.expect_geojson_output(): # Just confirm that we've been given valid JSON. Calculate the extrema if necessary cartogram_json = json.loads(cartogram_gen_output.decode()) if "bbox" not in cartogram_json: cartogram_json[ "bbox"] = geojson_extrema.get_extrema_from_geojson( cartogram_json) else: cartogram_json = gen2dict.translate( io.StringIO(cartogram_gen_output.decode()), settings.CARTOGRAM_COLOR, cartogram_handler.remove_holes()) cartogram_json['unique_sharing_key'] = unique_sharing_key cartogram_json = json.dumps(cartogram_json) if settings.USE_DATABASE: cartogram_entry = CartogramEntry.query.filter_by( string_key=unique_sharing_key).first() if cartogram_entry != None: cartogram_entry.cartogram_data = cartogram_json db.session.commit() yield cartogram_json yield "}"
def cartogram(): if 'handler' not in request.form: return Response('{"error":"badrequest"}', status=400, content_type="application/json") if request.form['handler'] not in cartogram_handlers: return Response('{"error":"badhandler"}', status=404, content_type="application/json") handler = request.form['handler'] cartogram_handler = cartogram_handlers[handler] if 'values' not in request.form: return Response('{"error":"badrequest"}', status=400, content_type="application/json") values = request.form['values'].split(";") # The existing verificaiton code expects all floats. To avoid modifying it, we replace the string "NA" with the # number 0.0 for verification purposes only. values_to_verify = [] try: for i in range(len(values)): if values[i] == "NA": values_to_verify.append(0.0) else: values[i] = float(values[i]) values_to_verify.append(values[i]) except ValueError: return Response('{"error":"badvalues"}', status=400, content_type="application/json") if cartogram_handler.validate_values(values_to_verify) != True: return Response('{"error":"badvalues"}', status=400, content_type="application/json") unique_sharing_key = "" if 'unique_sharing_key' in request.form: unique_sharing_key = request.form['unique_sharing_key'] lambda_result = awslambda.generate_cartogram( cartogram_handler.gen_area_data(values), cartogram_handler.get_gen_file(), settings.CARTOGRAM_LAMBDA_URL, settings.CARTOGRAM_LAMDA_API_KEY, unique_sharing_key) cartogram_gen_output = lambda_result['stdout'] if cartogram_handler.expect_geojson_output(): # Just confirm that we've been given valid JSON. Calculate the extrema if necessary cartogram_json = json.loads(cartogram_gen_output) if "bbox" not in cartogram_json: cartogram_json["bbox"] = geojson_extrema.get_extrema_from_geojson( cartogram_json) else: cartogram_json = gen2dict.translate(io.StringIO(cartogram_gen_output), settings.CARTOGRAM_COLOR, cartogram_handler.remove_holes()) cartogram_json['unique_sharing_key'] = unique_sharing_key if settings.USE_DATABASE: cartogram_entry = CartogramEntry.query.filter_by( string_key=unique_sharing_key).first() if cartogram_entry != None: cartogram_entry.cartogram_data = json.dumps(cartogram_json) db.session.commit() return Response(json.dumps({'cartogram_data': cartogram_json}), content_type='application/json', status=200)
def data(map_name): def cleanup(): print() print( "I cannot continue. If you contact the developers for support, send the stacktrace below: " ) print() traceback.print_stack(file=sys.stdout) print() print("An error occurred. Cleaning up...") try: os.remove("static/cartdata/{}/population.json".format(map_name)) os.remove("static/cartdata/{}/griddocument.json".format(map_name)) os.remove("static/cartdata/{}/original.json".format(map_name)) except OSError: pass if not os.path.exists("handlers/{}.py".format(map_name)): print( "Error: It looks like a map with the name '{}' doesn't exist (I didn't find handlers/{}.py)." .format(map_name, map_name)) return if not os.path.exists("static/cartdata/{}".format(map_name)): print( "Error: It looks like a map with the name '{}' doesn't exist (I didn't find static/cartdata/{})." .format(map_name, map_name)) return if not os.path.exists("{}.svg".format(map_name)): print( "Error: It looks like {}.svg doesn't exist. I need this file to add color information to your new map." ) return if not os.path.exists("{}-landarea.csv".format(map_name)): print( "Error: It looks like {}.svg doesn't exist. I need this file to add land area information to your new map." ) return if not os.path.exists("{}-population.csv".format(map_name)): print( "Error: It looks like {}.svg doesn't exist. I need this file to add population information to your new map." ) return print() print("I will now parse {}.svg to learn each map region's default color.". format(map_name)) print() print("Reading {}.svg...".format(map_name)) try: new_colors, colors_by_name = svg2color.convert( "{}.svg".format(map_name), "static/cartdata/{}/colors.json".format(map_name)) print(repr(new_colors)) print(repr(colors_by_name)) except Exception as e: print(repr(e)) cleanup() return print() print("I will now parse {}.svg for label information.".format(map_name)) print() print("Reading static/cartdata/{}/labels.json...".format(map_name)) try: with open("static/cartdata/{}/labels.json".format(map_name), "r") as labels_json_file: try: labels_json = json.load(labels_json_file) labels_scale_x = labels_json['scale_x'] labels_scale_y = labels_json['scale_y'] except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Reading {}.svg...".format(map_name)) try: labels = svg2labels.convert("{}.svg".format(map_name), labels_scale_x, labels_scale_y) except Exception as e: print(repr(e)) cleanup() return print() print("I will now parse {}.svg for configuration information.".format( map_name)) print() print("Reading {}.svg...".format(map_name)) try: config = svg2config.convert("{}.svg".format(map_name)) except Exception as e: print(repr(e)) cleanup() return print() print( "I will now parse the land area and population information from each region from {0}-landarea.csv and {0}-population.csv" .format(map_name)) print() try: map_module = importlib.import_module('handlers.{}'.format(map_name)) map_handler = map_module.CartogramHandler() print("Reading {}-landarea.csv...".format(map_name)) with open("{}-landarea.csv".format(map_name), "r", newline='') as landarea_csv: try: landarea_cartogramui = map_handler.csv_to_area_string_and_colors( landarea_csv) except Exception as e: print(repr(e)) cleanup() return print("Reading {}-population.csv...".format(map_name)) with open("{}-population.csv".format(map_name), "r", newline='') as population_csv: try: population_cartogramui = map_handler.csv_to_area_string_and_colors( population_csv) except Exception as e: print(repr(e)) cleanup() return print("Reading {}-population.csv...".format(map_name)) regions_populations = {} with open("{}-population.csv".format(map_name), "r", newline='') as population_csv: try: reader = csv.reader(population_csv) header_row = True for row in reader: if header_row: header_row = False continue regions_populations[row[0]] = row[2] except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print() print( "I will now generate the conventional and population map data. This may take a moment." ) print() print("Generating population map...") try: areas = population_cartogramui[0].split(";") #areas = list(map(lambda area: float(area), areas)) gen_output_lines = [] for source, line in cartwrap.generate_cartogram( map_handler.gen_area_data(areas), map_handler.get_gen_file(), os.environ["CARTOGRAM_EXE"]): if source == "stdout": gen_output_lines.append(line.decode().strip()) else: print("Generating population map: {}".format( line.decode().strip())) gen_output = "\n".join(gen_output_lines) with open("{}-population.gen".format(map_name), "w") as population_gen_file: population_gen_file.write(gen_output) cartogram_json = json.loads(gen_output) # Calculate the bounding box if necessary if "bbox" not in cartogram_json: cartogram_json["bbox"] = geojson_extrema.get_extrema_from_geojson( cartogram_json) cartogram_json["tooltip"] = population_cartogramui[2] except Exception as e: traceback.print_exc() print(repr(e)) cleanup() return print() print("Generating conventional map...") try: with open(map_handler.get_gen_file(), "r") as map_gen_file: try: original_json = json.load(map_gen_file) original_tooltip = landarea_cartogramui[2] original_tooltip['unit'] = 'km sq.' original_json['tooltip'] = original_tooltip except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print() print("I will now generate the finalized data entry template.") print() print("Reading static/cartdata/{}/template.csv...".format(map_name)) final_template = [] try: with open("static/cartdata/{}/template.csv".format(map_name), "r", newline='') as template_csv_file: try: reader = csv.reader(template_csv_file) header_row = True for row in reader: if header_row: final_template.append(row) header_row = False continue final_template.append([ row[0], regions_populations[row[0]], row[2], colors_by_name[row[0]] ]) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/template.csv...".format(map_name)) try: with open("static/cartdata/{}/template.csv".format(map_name), "w", newline='') as template_csv_file: try: writer = csv.writer(template_csv_file) for row in final_template: writer.writerow(row) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Reading static/cartdata/{}/template.csv...".format(map_name)) try: with open("static/cartdata/{}/template.csv".format(map_name), "r", newline='') as template_csv_file: try: cartogramui = map_handler.csv_to_area_string_and_colors( template_csv_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/griddocument.json...".format(map_name)) try: with open("static/cartdata/{}/griddocument.json".format(map_name), "w") as griddocument_json_file: try: json.dump(cartogramui[3], griddocument_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print() print("I will now finish up writing the map data files.") print() print("Writing static/cartdata/{}/original.json...".format(map_name)) try: with open("static/cartdata/{}/original.json".format(map_name), "w") as original_json_file: try: json.dump(original_json, original_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/population.json...".format(map_name)) try: with open("static/cartdata/{}/population.json".format(map_name), "w") as original_json_file: try: json.dump(cartogram_json, original_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/labels.json...".format(map_name)) try: with open("static/cartdata/{}/labels.json".format(map_name), "w") as labels_json_file: try: json.dump(labels, labels_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/colors.json...".format(map_name)) try: with open("static/cartdata/{}/colors.json".format(map_name), "w") as colors_json_file: try: json.dump(new_colors, colors_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/config.json...".format(map_name)) try: with open("static/cartdata/{}/config.json".format(map_name), "w") as colors_json_file: try: json.dump(config, colors_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Generating map pack in static/cartdata/{}/mappack.json...".format( map_name)) try: mappackify.mappackify(map_name) except Exception as e: print(repr(e)) cleanup() return print() print("All done!") print()
def data(map_name): def cleanup(): print() print( "I cannot continue. If you contact the developers for support, send the stacktrace below: " ) print() traceback.print_stack(file=sys.stdout) print() print("An error occurred. Cleaning up...") try: os.remove("static/cartdata/{}/population.json".format(map_name)) os.remove("static/cartdata/{}/griddocument.json".format(map_name)) os.remove("static/cartdata/{}/original.json".format(map_name)) except OSError: pass if not os.path.exists("handlers/{}.py".format(map_name)): print( "Error: It looks like a map with the name '{}' doesn't exist (I didn't find handlers/{}.py)." .format(map_name, map_name)) return if not os.path.exists("static/cartdata/{}".format(map_name)): print( "Error: It looks like a map with the name '{}' doesn't exist (I didn't find static/cartdata/{})." .format(map_name, map_name)) return if not os.path.exists("{}.svg".format(map_name)): print( "Error: It looks like {}.svg doesn't exist. I need this file to add color information to your new map." ) return if not os.path.exists("{}-landarea.csv".format(map_name)): print( "Error: It looks like {}.svg doesn't exist. I need this file to add land area information to your new map." ) return if not os.path.exists("{}-population.csv".format(map_name)): print( "Error: It looks like {}.svg doesn't exist. I need this file to add population information to your new map." ) return print() print("I will now parse {}.svg to learn each map region's default color.". format(map_name)) print() print("Reading {}.svg...".format(map_name)) try: new_colors, colors_by_name = svg2color.convert( "{}.svg".format(map_name), "static/cartdata/{}/colors.json".format(map_name)) print(repr(new_colors)) print(repr(colors_by_name)) except Exception as e: print(repr(e)) cleanup() return print() print("I will now parse {}.svg for label information.".format(map_name)) print() print("Reading static/cartdata/{}/labels.json...".format(map_name)) try: with open("static/cartdata/{}/labels.json".format(map_name), "r") as labels_json_file: try: labels_json = json.load(labels_json_file) labels_scale_x = labels_json['scale_x'] labels_scale_y = labels_json['scale_y'] except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Reading {}.svg...".format(map_name)) try: labels = svg2labels.convert("{}.svg".format(map_name), labels_scale_x, labels_scale_y) except Exception as e: print(repr(e)) cleanup() return print() print("I will now parse {}.svg for configuration information.".format( map_name)) print() print("Reading {}.svg...".format(map_name)) try: config = svg2config.convert("{}.svg".format(map_name)) except Exception as e: print(repr(e)) cleanup() return print() print( "I will now parse the land area and population information from each region from {0}-landarea.csv and {0}-population.csv" .format(map_name)) print() try: map_module = importlib.import_module('handlers.{}'.format(map_name)) map_handler = map_module.CartogramHandler() print("Reading {}-landarea.csv...".format(map_name)) with open("{}-landarea.csv".format(map_name), "r", newline='') as landarea_csv: try: landarea_cartogramui = map_handler.csv_to_area_string_and_colors( landarea_csv) except Exception as e: print(repr(e)) cleanup() return print("Reading {}-population.csv...".format(map_name)) with open("{}-population.csv".format(map_name), "r", newline='') as population_csv: try: population_cartogramui = map_handler.csv_to_area_string_and_colors( population_csv) population_cartogramui[2]["unit"] = "people" except Exception as e: print(repr(e)) cleanup() return print("Reading {}-population.csv...".format(map_name)) regions_populations = {} with open("{}-population.csv".format(map_name), "r", newline='') as population_csv: try: reader = csv.reader(population_csv) header_row = True for row in reader: if header_row: header_row = False continue regions_populations[row[0]] = row[2] except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print() print( "I will now generate the conventional and population map data. This may take a moment." ) print() print("Generating population map...") print("Making request to AWS Lambda function at {}.".format( settings.CARTOGRAM_LAMBDA_URL)) try: areas = population_cartogramui[0].split(";") def serverless_generate(): redis_conn = redis.Redis(host=settings.CARTOGRAM_REDIS_HOST, port=settings.CARTOGRAM_REDIS_PORT, db=0) q = queue.Queue() unique_key = get_random_string(50) def downloader_worker(): lambda_result = awslambda.generate_cartogram( map_handler.gen_area_data(areas), map_handler.get_gen_file(), settings.CARTOGRAM_LAMBDA_URL, settings.CARTOGRAM_LAMDA_API_KEY, unique_key) cartogram_gen_output = lambda_result['stdout'] q.put(cartogram_gen_output) threading.Thread(target=downloader_worker(), daemon=True).start() current_stderr = "" current_progress_level = None gen_output = "" while True: current_progress = redis_conn.get( "cartprogress-{}".format(unique_key)) if current_progress == None: pass else: current_progress = json.loads(current_progress.decode()) if current_progress['progress'] != current_progress_level: to_print = current_progress['stderr'].replace( current_stderr, "") for line in to_print.split("\n"): print("Generating population map: {}".format(line), flush=True) current_stderr = current_progress['stderr'] current_progress_level = current_progress['progress'] try: gen_output = q.get(False, timeout=0.1) break except queue.Empty: pass current_progress = redis_conn.get( "cartprogress-{}".format(unique_key)) if current_progress == None: pass else: current_progress = json.loads(current_progress.decode()) if current_progress['progress'] != current_progress_level: to_print = current_progress['stderr'].replace( current_stderr, "") for line in to_print.split("\n"): print("Generating population map: {}".format(line), flush=True) return json.loads(gen_output) def self_generate(): gen_output_lines = [] for source, line in cartwrap.generate_cartogram( map_handler.gen_area_data(areas), map_handler.get_gen_file(), os.environ["CARTOGRAM_EXE"]): if source == "stdout": gen_output_lines.append(line.decode().strip()) else: print("Generating population map: {}".format( line.decode().strip())) gen_output = "\n".join(gen_output_lines) return json.loads(gen_output) cartogram_json = serverless_generate( ) if settings.CARTOGRAM_LOCAL_DOCKERIZED else self_generate() # Calculate the bounding box if necessary if "bbox" not in cartogram_json: cartogram_json["bbox"] = geojson_extrema.get_extrema_from_geojson( cartogram_json) cartogram_json["tooltip"] = population_cartogramui[2] except Exception as e: traceback.print_exc() print(repr(e)) cleanup() return print() print("Generating conventional map...") try: with open(map_handler.get_gen_file(), "r") as map_gen_file: try: original_json = json.load(map_gen_file) original_tooltip = landarea_cartogramui[2] original_tooltip['unit'] = 'km sq.' original_json['tooltip'] = original_tooltip except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print() print("I will now generate the finalized data entry template.") print() print("Reading static/cartdata/{}/template.csv...".format(map_name)) final_template = [] try: with open("static/cartdata/{}/template.csv".format(map_name), "r", newline='') as template_csv_file: try: reader = csv.reader(template_csv_file) header_row = True for row in reader: if header_row: final_template.append(row) header_row = False continue final_template.append([ row[0], regions_populations[row[0]], row[2], colors_by_name[row[0]] ]) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/template.csv...".format(map_name)) try: with open("static/cartdata/{}/template.csv".format(map_name), "w", newline='') as template_csv_file: try: writer = csv.writer(template_csv_file) for row in final_template: writer.writerow(row) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Reading static/cartdata/{}/template.csv...".format(map_name)) try: with open("static/cartdata/{}/template.csv".format(map_name), "r", newline='') as template_csv_file: try: cartogramui = map_handler.csv_to_area_string_and_colors( template_csv_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/griddocument.json...".format(map_name)) try: with open("static/cartdata/{}/griddocument.json".format(map_name), "w") as griddocument_json_file: try: json.dump(cartogramui[3], griddocument_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print() print("I will now finish up writing the map data files.") print() print("Writing static/cartdata/{}/original.json...".format(map_name)) try: with open("static/cartdata/{}/original.json".format(map_name), "w") as original_json_file: try: json.dump(original_json, original_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/population.json...".format(map_name)) try: with open("static/cartdata/{}/population.json".format(map_name), "w") as original_json_file: try: json.dump(cartogram_json, original_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/labels.json...".format(map_name)) try: with open("static/cartdata/{}/labels.json".format(map_name), "w") as labels_json_file: try: json.dump(labels, labels_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/colors.json...".format(map_name)) try: with open("static/cartdata/{}/colors.json".format(map_name), "w") as colors_json_file: try: json.dump(new_colors, colors_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/config.json...".format(map_name)) try: with open("static/cartdata/{}/config.json".format(map_name), "w") as colors_json_file: try: json.dump(config, colors_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Generating map pack in static/cartdata/{}/mappack.json...".format( map_name)) try: mappackify.mappackify(map_name) except Exception as e: print(repr(e)) cleanup() return print() print("All done!") print()
cartogram_output = "" for source, line in cartwrap.generate_cartogram(area_data, handler.get_gen_file(), settings.CARTOGRAM_EXE): if source == "stderr": print("Generating {}: {}".format(sysname, line.decode())) else: cartogram_output += line.decode() if handler.expect_geojson_output(): cartogram_json = json.loads(cartogram_output) if "bbox" not in cartogram_json: cartogram_json["bbox"] = geojson_extrema.get_extrema_from_geojson( cartogram_json) else: cartogram_json = gen2dict.translate(io.StringIO(cartogram_output), settings.CARTOGRAM_COLOR, handler.remove_holes()) cartogram_json['tooltip'] = tooltip with open("static/cartdata/{}/population.json".format(sysname), 'w') as population_json_file: json.dump(cartogram_json, population_json_file)