def generate_streamed_json_response(): cartogram_gen_output = b'' current_loading_point = "null" # We have to format our JSON manually, since we're not sending a complete object. # On the client side, Oboe.js is intelligent enough to parse this and get the loading information and cartogram output yield '{"loading_progress_points":[' for source, line in cartwrap.generate_cartogram( cartogram_handler.gen_area_data(values), cartogram_handler.get_gen_file(), settings.CARTOGRAM_EXE): if source == "stdout": cartogram_gen_output += line else: s = re.search(r'max\. abs\. area error: (.+)', line.decode()) if s != None: current_loading_point = s.groups(1)[0] # We always include the loading progress, even if it hasn't changed. # This makes life easier on the client side yield '{{"loading_point": {}, "stderr_line": "{}"}},'.format( current_loading_point, line.decode()) # We create a fake last entry because you can't have dangling commas in JSON yield '{"loading_point":0, "stderr_line": ""}],"cartogram_data":' if cartogram_handler.expect_geojson_output(): # Just confirm that we've been given valid JSON. Calculate the extrema if necessary cartogram_json = json.loads(cartogram_gen_output.decode()) if "bbox" not in cartogram_json: cartogram_json[ "bbox"] = geojson_extrema.get_extrema_from_geojson( cartogram_json) else: cartogram_json = gen2dict.translate( io.StringIO(cartogram_gen_output.decode()), settings.CARTOGRAM_COLOR, cartogram_handler.remove_holes()) cartogram_json['unique_sharing_key'] = unique_sharing_key cartogram_json = json.dumps(cartogram_json) if settings.USE_DATABASE: cartogram_entry = CartogramEntry.query.filter_by( string_key=unique_sharing_key).first() if cartogram_entry != None: cartogram_entry.cartogram_data = cartogram_json db.session.commit() yield cartogram_json yield "}"
def lambda_handler(event, context): session = FuturesSession() stdout = "" stderr = "" order = 0 map_data_filename = "conventional.json" params = json.loads(event['body']) # The C code deduces from the map data file extension whether we have GeoJSON or .gen world = False try: conventional_json = json.loads(params["gen_file"]) if "extent" in conventional_json.keys(): if conventional_json['extent'] == "world": world = True except json.JSONDecodeError: map_data_filename = "conventional.gen" with open("/tmp/{}".format(map_data_filename), "w") as conventional_map_file: conventional_map_file.write(params["gen_file"]) for source, line in cartwrap.generate_cartogram(params["area_data"], "/tmp/{}".format(map_data_filename), "{}/cartogram".format(os.environ['LAMBDA_TASK_ROOT']), world): if source == "stdout": stdout += line.decode() else: stderr_line = line.decode() stderr += line.decode() s = re.search(r'max\. abs\. area error: (.+)', line.decode()) if s != None: current_progress = float(s.groups(1)[0]) session.post(os.environ['CARTOGRAM_PROGRESS_URL'], json={ 'secret': os.environ['CARTOGRAM_PROGRESS_SECRET'], 'key': params['key'], 'progress': current_progress, 'stderr': stderr, 'order': order }) order += 1 return { "statusCode": 200, "body": json.dumps({"stderr": stderr, "stdout": stdout}) }
def self_generate(): gen_output_lines = [] for source, line in cartwrap.generate_cartogram( map_handler.gen_area_data(areas), map_handler.get_gen_file(), os.environ["CARTOGRAM_EXE"]): if source == "stdout": gen_output_lines.append(line.decode().strip()) else: print("Generating population map: {}".format( line.decode().strip())) gen_output = "\n".join(gen_output_lines) return json.loads(gen_output)
def data(map_name): def cleanup(): print() print( "I cannot continue. If you contact the developers for support, send the stacktrace below: " ) print() traceback.print_stack(file=sys.stdout) print() print("An error occurred. Cleaning up...") try: os.remove("static/cartdata/{}/population.json".format(map_name)) os.remove("static/cartdata/{}/griddocument.json".format(map_name)) os.remove("static/cartdata/{}/original.json".format(map_name)) except OSError: pass if not os.path.exists("handlers/{}.py".format(map_name)): print( "Error: It looks like a map with the name '{}' doesn't exist (I didn't find handlers/{}.py)." .format(map_name, map_name)) return if not os.path.exists("static/cartdata/{}".format(map_name)): print( "Error: It looks like a map with the name '{}' doesn't exist (I didn't find static/cartdata/{})." .format(map_name, map_name)) return if not os.path.exists("{}.svg".format(map_name)): print( "Error: It looks like {}.svg doesn't exist. I need this file to add color information to your new map." ) return if not os.path.exists("{}-landarea.csv".format(map_name)): print( "Error: It looks like {}.svg doesn't exist. I need this file to add land area information to your new map." ) return if not os.path.exists("{}-population.csv".format(map_name)): print( "Error: It looks like {}.svg doesn't exist. I need this file to add population information to your new map." ) return print() print("I will now parse {}.svg to learn each map region's default color.". format(map_name)) print() print("Reading {}.svg...".format(map_name)) try: new_colors, colors_by_name = svg2color.convert( "{}.svg".format(map_name), "static/cartdata/{}/colors.json".format(map_name)) print(repr(new_colors)) print(repr(colors_by_name)) except Exception as e: print(repr(e)) cleanup() return print() print("I will now parse {}.svg for label information.".format(map_name)) print() print("Reading static/cartdata/{}/labels.json...".format(map_name)) try: with open("static/cartdata/{}/labels.json".format(map_name), "r") as labels_json_file: try: labels_json = json.load(labels_json_file) labels_scale_x = labels_json['scale_x'] labels_scale_y = labels_json['scale_y'] except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Reading {}.svg...".format(map_name)) try: labels = svg2labels.convert("{}.svg".format(map_name), labels_scale_x, labels_scale_y) except Exception as e: print(repr(e)) cleanup() return print() print("I will now parse {}.svg for configuration information.".format( map_name)) print() print("Reading {}.svg...".format(map_name)) try: config = svg2config.convert("{}.svg".format(map_name)) except Exception as e: print(repr(e)) cleanup() return print() print( "I will now parse the land area and population information from each region from {0}-landarea.csv and {0}-population.csv" .format(map_name)) print() try: map_module = importlib.import_module('handlers.{}'.format(map_name)) map_handler = map_module.CartogramHandler() print("Reading {}-landarea.csv...".format(map_name)) with open("{}-landarea.csv".format(map_name), "r", newline='') as landarea_csv: try: landarea_cartogramui = map_handler.csv_to_area_string_and_colors( landarea_csv) except Exception as e: print(repr(e)) cleanup() return print("Reading {}-population.csv...".format(map_name)) with open("{}-population.csv".format(map_name), "r", newline='') as population_csv: try: population_cartogramui = map_handler.csv_to_area_string_and_colors( population_csv) except Exception as e: print(repr(e)) cleanup() return print("Reading {}-population.csv...".format(map_name)) regions_populations = {} with open("{}-population.csv".format(map_name), "r", newline='') as population_csv: try: reader = csv.reader(population_csv) header_row = True for row in reader: if header_row: header_row = False continue regions_populations[row[0]] = row[2] except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print() print( "I will now generate the conventional and population map data. This may take a moment." ) print() print("Generating population map...") try: areas = population_cartogramui[0].split(";") #areas = list(map(lambda area: float(area), areas)) gen_output_lines = [] for source, line in cartwrap.generate_cartogram( map_handler.gen_area_data(areas), map_handler.get_gen_file(), os.environ["CARTOGRAM_EXE"]): if source == "stdout": gen_output_lines.append(line.decode().strip()) else: print("Generating population map: {}".format( line.decode().strip())) gen_output = "\n".join(gen_output_lines) with open("{}-population.gen".format(map_name), "w") as population_gen_file: population_gen_file.write(gen_output) cartogram_json = json.loads(gen_output) # Calculate the bounding box if necessary if "bbox" not in cartogram_json: cartogram_json["bbox"] = geojson_extrema.get_extrema_from_geojson( cartogram_json) cartogram_json["tooltip"] = population_cartogramui[2] except Exception as e: traceback.print_exc() print(repr(e)) cleanup() return print() print("Generating conventional map...") try: with open(map_handler.get_gen_file(), "r") as map_gen_file: try: original_json = json.load(map_gen_file) original_tooltip = landarea_cartogramui[2] original_tooltip['unit'] = 'km sq.' original_json['tooltip'] = original_tooltip except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print() print("I will now generate the finalized data entry template.") print() print("Reading static/cartdata/{}/template.csv...".format(map_name)) final_template = [] try: with open("static/cartdata/{}/template.csv".format(map_name), "r", newline='') as template_csv_file: try: reader = csv.reader(template_csv_file) header_row = True for row in reader: if header_row: final_template.append(row) header_row = False continue final_template.append([ row[0], regions_populations[row[0]], row[2], colors_by_name[row[0]] ]) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/template.csv...".format(map_name)) try: with open("static/cartdata/{}/template.csv".format(map_name), "w", newline='') as template_csv_file: try: writer = csv.writer(template_csv_file) for row in final_template: writer.writerow(row) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Reading static/cartdata/{}/template.csv...".format(map_name)) try: with open("static/cartdata/{}/template.csv".format(map_name), "r", newline='') as template_csv_file: try: cartogramui = map_handler.csv_to_area_string_and_colors( template_csv_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/griddocument.json...".format(map_name)) try: with open("static/cartdata/{}/griddocument.json".format(map_name), "w") as griddocument_json_file: try: json.dump(cartogramui[3], griddocument_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print() print("I will now finish up writing the map data files.") print() print("Writing static/cartdata/{}/original.json...".format(map_name)) try: with open("static/cartdata/{}/original.json".format(map_name), "w") as original_json_file: try: json.dump(original_json, original_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/population.json...".format(map_name)) try: with open("static/cartdata/{}/population.json".format(map_name), "w") as original_json_file: try: json.dump(cartogram_json, original_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/labels.json...".format(map_name)) try: with open("static/cartdata/{}/labels.json".format(map_name), "w") as labels_json_file: try: json.dump(labels, labels_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/colors.json...".format(map_name)) try: with open("static/cartdata/{}/colors.json".format(map_name), "w") as colors_json_file: try: json.dump(new_colors, colors_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Writing static/cartdata/{}/config.json...".format(map_name)) try: with open("static/cartdata/{}/config.json".format(map_name), "w") as colors_json_file: try: json.dump(config, colors_json_file) except Exception as e: print(repr(e)) cleanup() return except Exception as e: print(repr(e)) cleanup() return print("Generating map pack in static/cartdata/{}/mappack.json...".format( map_name)) try: mappackify.mappackify(map_name) except Exception as e: print(repr(e)) cleanup() return print() print("All done!") print()
with open("static/cartdata/{}/template.csv".format(sysname), newline='') as template_csv: template_reader = csv.DictReader(template_csv) for row in template_reader: pop_values.append(row['Population']) area_data = handler.gen_area_data(pop_values) cartogram_output = "" for source, line in cartwrap.generate_cartogram(area_data, handler.get_gen_file(), settings.CARTOGRAM_EXE): if source == "stderr": print("Generating {}: {}".format(sysname, line.decode())) else: cartogram_output += line.decode() if handler.expect_geojson_output(): cartogram_json = json.loads(cartogram_output) if "bbox" not in cartogram_json: cartogram_json["bbox"] = geojson_extrema.get_extrema_from_geojson( cartogram_json)