def generate_streamed_json_response():
cartogram_gen_output = b''
current_loading_point = "null"
# We have to format our JSON manually, since we're not sending a complete object.
# On the client side, Oboe.js is intelligent enough to parse this and get the loading information and cartogram output
yield '{"loading_progress_points":['
for source, line in cartwrap.generate_cartogram(
cartogram_handler.gen_area_data(values),
cartogram_handler.get_gen_file(), settings.CARTOGRAM_EXE):
if source == "stdout":
cartogram_gen_output += line
else:
s = re.search(r'max\. abs\. area error: (.+)', line.decode())
if s != None:
current_loading_point = s.groups(1)[0]
# We always include the loading progress, even if it hasn't changed.
# This makes life easier on the client side
yield '{{"loading_point": {}, "stderr_line": "{}"}},'.format(
current_loading_point, line.decode())
# We create a fake last entry because you can't have dangling commas in JSON
yield '{"loading_point":0, "stderr_line": ""}],"cartogram_data":'
if cartogram_handler.expect_geojson_output():
# Just confirm that we've been given valid JSON. Calculate the extrema if necessary
cartogram_json = json.loads(cartogram_gen_output.decode())
if "bbox" not in cartogram_json:
cartogram_json[
"bbox"] = geojson_extrema.get_extrema_from_geojson(
cartogram_json)
else:
cartogram_json = gen2dict.translate(
io.StringIO(cartogram_gen_output.decode()),
settings.CARTOGRAM_COLOR, cartogram_handler.remove_holes())
cartogram_json['unique_sharing_key'] = unique_sharing_key
cartogram_json = json.dumps(cartogram_json)
if settings.USE_DATABASE:
cartogram_entry = CartogramEntry.query.filter_by(
string_key=unique_sharing_key).first()
if cartogram_entry != None:
cartogram_entry.cartogram_data = cartogram_json
db.session.commit()
yield cartogram_json
yield "}"
def lambda_handler(event, context):
session = FuturesSession()
stdout = ""
stderr = ""
order = 0
map_data_filename = "conventional.json"
params = json.loads(event['body'])
# The C code deduces from the map data file extension whether we have GeoJSON or .gen
world = False
try:
conventional_json = json.loads(params["gen_file"])
if "extent" in conventional_json.keys():
if conventional_json['extent'] == "world":
world = True
except json.JSONDecodeError:
map_data_filename = "conventional.gen"
with open("/tmp/{}".format(map_data_filename), "w") as conventional_map_file:
conventional_map_file.write(params["gen_file"])
for source, line in cartwrap.generate_cartogram(params["area_data"], "/tmp/{}".format(map_data_filename), "{}/cartogram".format(os.environ['LAMBDA_TASK_ROOT']), world):
if source == "stdout":
stdout += line.decode()
else:
stderr_line = line.decode()
stderr += line.decode()
s = re.search(r'max\. abs\. area error: (.+)', line.decode())
if s != None:
current_progress = float(s.groups(1)[0])
session.post(os.environ['CARTOGRAM_PROGRESS_URL'], json={
'secret': os.environ['CARTOGRAM_PROGRESS_SECRET'],
'key': params['key'],
'progress': current_progress,
'stderr': stderr,
'order': order
})
order += 1
return {
"statusCode": 200,
"body": json.dumps({"stderr": stderr, "stdout": stdout})
}
def self_generate():
gen_output_lines = []
for source, line in cartwrap.generate_cartogram(
map_handler.gen_area_data(areas),
map_handler.get_gen_file(), os.environ["CARTOGRAM_EXE"]):
if source == "stdout":
gen_output_lines.append(line.decode().strip())
else:
print("Generating population map: {}".format(
line.decode().strip()))
gen_output = "\n".join(gen_output_lines)
return json.loads(gen_output)
def data(map_name):
def cleanup():
print()
print(
"I cannot continue. If you contact the developers for support, send the stacktrace below: "
)
print()
traceback.print_stack(file=sys.stdout)
print()
print("An error occurred. Cleaning up...")
try:
os.remove("static/cartdata/{}/population.json".format(map_name))
os.remove("static/cartdata/{}/griddocument.json".format(map_name))
os.remove("static/cartdata/{}/original.json".format(map_name))
except OSError:
pass
if not os.path.exists("handlers/{}.py".format(map_name)):
print(
"Error: It looks like a map with the name '{}' doesn't exist (I didn't find handlers/{}.py)."
.format(map_name, map_name))
return
if not os.path.exists("static/cartdata/{}".format(map_name)):
print(
"Error: It looks like a map with the name '{}' doesn't exist (I didn't find static/cartdata/{})."
.format(map_name, map_name))
return
if not os.path.exists("{}.svg".format(map_name)):
print(
"Error: It looks like {}.svg doesn't exist. I need this file to add color information to your new map."
)
return
if not os.path.exists("{}-landarea.csv".format(map_name)):
print(
"Error: It looks like {}.svg doesn't exist. I need this file to add land area information to your new map."
)
return
if not os.path.exists("{}-population.csv".format(map_name)):
print(
"Error: It looks like {}.svg doesn't exist. I need this file to add population information to your new map."
)
return
print()
print("I will now parse {}.svg to learn each map region's default color.".
format(map_name))
print()
print("Reading {}.svg...".format(map_name))
try:
new_colors, colors_by_name = svg2color.convert(
"{}.svg".format(map_name),
"static/cartdata/{}/colors.json".format(map_name))
print(repr(new_colors))
print(repr(colors_by_name))
except Exception as e:
print(repr(e))
cleanup()
return
print()
print("I will now parse {}.svg for label information.".format(map_name))
print()
print("Reading static/cartdata/{}/labels.json...".format(map_name))
try:
with open("static/cartdata/{}/labels.json".format(map_name),
"r") as labels_json_file:
try:
labels_json = json.load(labels_json_file)
labels_scale_x = labels_json['scale_x']
labels_scale_y = labels_json['scale_y']
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print("Reading {}.svg...".format(map_name))
try:
labels = svg2labels.convert("{}.svg".format(map_name), labels_scale_x,
labels_scale_y)
except Exception as e:
print(repr(e))
cleanup()
return
print()
print("I will now parse {}.svg for configuration information.".format(
map_name))
print()
print("Reading {}.svg...".format(map_name))
try:
config = svg2config.convert("{}.svg".format(map_name))
except Exception as e:
print(repr(e))
cleanup()
return
print()
print(
"I will now parse the land area and population information from each region from {0}-landarea.csv and {0}-population.csv"
.format(map_name))
print()
try:
map_module = importlib.import_module('handlers.{}'.format(map_name))
map_handler = map_module.CartogramHandler()
print("Reading {}-landarea.csv...".format(map_name))
with open("{}-landarea.csv".format(map_name), "r",
newline='') as landarea_csv:
try:
landarea_cartogramui = map_handler.csv_to_area_string_and_colors(
landarea_csv)
except Exception as e:
print(repr(e))
cleanup()
return
print("Reading {}-population.csv...".format(map_name))
with open("{}-population.csv".format(map_name), "r",
newline='') as population_csv:
try:
population_cartogramui = map_handler.csv_to_area_string_and_colors(
population_csv)
except Exception as e:
print(repr(e))
cleanup()
return
print("Reading {}-population.csv...".format(map_name))
regions_populations = {}
with open("{}-population.csv".format(map_name), "r",
newline='') as population_csv:
try:
reader = csv.reader(population_csv)
header_row = True
for row in reader:
if header_row:
header_row = False
continue
regions_populations[row[0]] = row[2]
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print()
print(
"I will now generate the conventional and population map data. This may take a moment."
)
print()
print("Generating population map...")
try:
areas = population_cartogramui[0].split(";")
#areas = list(map(lambda area: float(area), areas))
gen_output_lines = []
for source, line in cartwrap.generate_cartogram(
map_handler.gen_area_data(areas), map_handler.get_gen_file(),
os.environ["CARTOGRAM_EXE"]):
if source == "stdout":
gen_output_lines.append(line.decode().strip())
else:
print("Generating population map: {}".format(
line.decode().strip()))
gen_output = "\n".join(gen_output_lines)
with open("{}-population.gen".format(map_name),
"w") as population_gen_file:
population_gen_file.write(gen_output)
cartogram_json = json.loads(gen_output)
# Calculate the bounding box if necessary
if "bbox" not in cartogram_json:
cartogram_json["bbox"] = geojson_extrema.get_extrema_from_geojson(
cartogram_json)
cartogram_json["tooltip"] = population_cartogramui[2]
except Exception as e:
traceback.print_exc()
print(repr(e))
cleanup()
return
print()
print("Generating conventional map...")
try:
with open(map_handler.get_gen_file(), "r") as map_gen_file:
try:
original_json = json.load(map_gen_file)
original_tooltip = landarea_cartogramui[2]
original_tooltip['unit'] = 'km sq.'
original_json['tooltip'] = original_tooltip
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print()
print("I will now generate the finalized data entry template.")
print()
print("Reading static/cartdata/{}/template.csv...".format(map_name))
final_template = []
try:
with open("static/cartdata/{}/template.csv".format(map_name),
"r",
newline='') as template_csv_file:
try:
reader = csv.reader(template_csv_file)
header_row = True
for row in reader:
if header_row:
final_template.append(row)
header_row = False
continue
final_template.append([
row[0], regions_populations[row[0]], row[2],
colors_by_name[row[0]]
])
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print("Writing static/cartdata/{}/template.csv...".format(map_name))
try:
with open("static/cartdata/{}/template.csv".format(map_name),
"w",
newline='') as template_csv_file:
try:
writer = csv.writer(template_csv_file)
for row in final_template:
writer.writerow(row)
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print("Reading static/cartdata/{}/template.csv...".format(map_name))
try:
with open("static/cartdata/{}/template.csv".format(map_name),
"r",
newline='') as template_csv_file:
try:
cartogramui = map_handler.csv_to_area_string_and_colors(
template_csv_file)
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print("Writing static/cartdata/{}/griddocument.json...".format(map_name))
try:
with open("static/cartdata/{}/griddocument.json".format(map_name),
"w") as griddocument_json_file:
try:
json.dump(cartogramui[3], griddocument_json_file)
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print()
print("I will now finish up writing the map data files.")
print()
print("Writing static/cartdata/{}/original.json...".format(map_name))
try:
with open("static/cartdata/{}/original.json".format(map_name),
"w") as original_json_file:
try:
json.dump(original_json, original_json_file)
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print("Writing static/cartdata/{}/population.json...".format(map_name))
try:
with open("static/cartdata/{}/population.json".format(map_name),
"w") as original_json_file:
try:
json.dump(cartogram_json, original_json_file)
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print("Writing static/cartdata/{}/labels.json...".format(map_name))
try:
with open("static/cartdata/{}/labels.json".format(map_name),
"w") as labels_json_file:
try:
json.dump(labels, labels_json_file)
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print("Writing static/cartdata/{}/colors.json...".format(map_name))
try:
with open("static/cartdata/{}/colors.json".format(map_name),
"w") as colors_json_file:
try:
json.dump(new_colors, colors_json_file)
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print("Writing static/cartdata/{}/config.json...".format(map_name))
try:
with open("static/cartdata/{}/config.json".format(map_name),
"w") as colors_json_file:
try:
json.dump(config, colors_json_file)
except Exception as e:
print(repr(e))
cleanup()
return
except Exception as e:
print(repr(e))
cleanup()
return
print("Generating map pack in static/cartdata/{}/mappack.json...".format(
map_name))
try:
mappackify.mappackify(map_name)
except Exception as e:
print(repr(e))
cleanup()
return
print()
print("All done!")
print()
with open("static/cartdata/{}/template.csv".format(sysname),
newline='') as template_csv:
template_reader = csv.DictReader(template_csv)
for row in template_reader:
pop_values.append(row['Population'])
area_data = handler.gen_area_data(pop_values)
cartogram_output = ""
for source, line in cartwrap.generate_cartogram(area_data,
handler.get_gen_file(),
settings.CARTOGRAM_EXE):
if source == "stderr":
print("Generating {}: {}".format(sysname, line.decode()))
else:
cartogram_output += line.decode()
if handler.expect_geojson_output():
cartogram_json = json.loads(cartogram_output)
if "bbox" not in cartogram_json:
cartogram_json["bbox"] = geojson_extrema.get_extrema_from_geojson(
cartogram_json)