def post(self): # make tiles in zip file and write
#print self.request.arguments() # should be the same as given to preflight
self.response.headers['X-Content-Type-Options'] = 'nosniff' # Prevent browsers from MIME-sniffing the content-type:
self.response.headers["X-Frame-Options"] = "SAMEORIGIN" # prevent clickjacking
self.response.out.write('<html><body>')
self.response.out.write('<h2>Processing finished:</h2>')
# debug: print/log all args and then values
args = {} # put arg name and value in a dict as key:value
for k in ("DEM_name", "trlat", "trlon", "bllat", "bllon", "printres", "ntilesx", "ntilesy", "tilewidth",
"basethick", "zscale", "fileformat"):
v = self.request.get(k) # key = name of arg
args[k] = v # value
if k not in ["DEM_name", "fileformat"]: args[k] = float(args[k]) # floatify non-string args
#print k, args[k]
self.response.out.write("%s = %s <br>" % (k, str(args[k])))
logging.info("%s = %s" % (k, str(args[k])))
args["CPU_cores_to_use"] = NUM_CORES
# name of zip file is time since 2000 in 0.01 seconds
myname = str(int((datetime.now()-datetime(2000,1,1)).total_seconds() * 1000))
fname = tmp_folder + os.sep + myname + ".zip" # create filename for zip and put in tmp folder
args["zip_file_name"] = fname
# if this number of cells to process is exceeded, use a temp file instead of
args["max_cells_for_memory_only"] = MAX_CELLS
try:
# create zip and write to tmp
total_unzipped_size = TouchTerrainEarthEngine.get_zipped_tiles(**args) # all args are in a dict
except Exception, e:
logging.error(e)
print(e)
self.response.out.write(e)
return
def main():
# write an example json file, in case it gets deleted ...
with open('example_config.json', 'w+') as fp:
json.dump(args, fp, indent=0,
sort_keys=True) # indent = 0: newline after each comma
print(
'Wrote example_config.json with default values, you can use it as a template but make sure to rename it!'
)
import sys, os
from os.path import abspath, dirname
# parse args
if len(sys.argv) > 1: # sys.argv are the CLI args
json_fname = sys.argv[1]
try:
fp = open(json_fname, "rU")
except Exception as e:
sys.exit("Error: can't find " + json_fname + ": " + str(e))
file_content = fp.read()
try:
json_args = json.loads(file_content)
except Exception as e:
sys.exit("Error: can't json parse " + json_fname + ": " + str(e))
print("reading", json_fname)
for k in list(args.keys()):
try:
args[k] = json_args[
k] # try to find a value for k in json config file
#print k, args[k]
except:
print(
"info:", k,
"has missing or invalid value, using defaults where possible"
) # no match? no problem, just keep the default value
#print "%s = %s" % (k, str(args[k]))
else:
# no JSON config file given, setting config values in code
# you can comment out lines for which you don't want to overwrite the default settings
overwrite_args = {
"DEM_name":
'USGS/NED', # DEM_name: name of DEM source used in Google Earth Engine
# for all valid sources, see DEM_sources in TouchTerrainEarthEngine.py
"trlat": 44.69741706507476, # lat/lon of top right corner
"trlon": -107.97962089843747,
"bllat": 44.50185267072875, # lat/lon of bottom left corner
"bllon": -108.25427910156247,
"importedDEM":
None, # if not None, the raster file to use as DEM instead of using GEE (null in JSON)
"printres":
0.5, # resolution (horizontal) of 3D printer (= size of one pixel) in mm
"ntilesx": 1, # number of tiles in x and y
"ntilesy": 1,
"tilewidth":
80, # width of each tile in mm (<- !!!!!), tile height is calculated
"basethick": 1, # thickness (in mm) of printed base
"zscale": 1.0, # elevation (vertical) scaling
"fileformat":
"STLb", # format of 3D model files: "obj" wavefront obj (ascii),"STLa" ascii STL or "STLb" binary STL
"tile_centered":
False, # True-> all tiles are centered around 0/0, False, all tiles "fit together"
"zip_file_name":
"terrain", # base name of zipfile, .zip will be added
"CPU_cores_to_use":
0, # 0 means all cores, None (null in JSON!) => don't use multiprocessing
"max_cells_for_memory_only": 1000 *
1000, # if raster is bigger, use temp_files instead of memory
"no_bottom": False, # omit bottom triangles?
#"rot_degs": 0, # rotate by degrees ccw # CH disabled for now
"bottom_image":
None, # 1 band greyscale image used for bottom relief
"ignore_leq":
None, # set values <= this to NaN, so they are ignored
"unprojected":
False, # don't project to UTM, only usefull when using GEE for DEM rasters
"only":
None, # list of tile index [x,y] with is the only tile to be processed. None means process all tiles (index is 1 based)
}
# overwrite config settings in args
for k in overwrite_args:
args[k] = overwrite_args[k]
# print out current args
print("\nUsing these config values:")
for k in sorted(args.keys()):
print("%s = %s" % (k, str(args[k])))
# No DEM file given, use Google Earth Engine
if args["importedDEM"] == None:
# initialize ee - needs a google earth engine account! See TouchTerrain_standalone_installation.pdf
try:
import ee
except Exception as e:
print("Google Earth Engine module not installed",
e,
file=sys.stderr)
# try both ways of authenticating
try:
ee.Initialize() # uses .config/earthengine/credentials
except Exception as e:
print("EE init() error (with .config/earthengine/credentials)",
e,
file=sys.stderr)
try:
# try authenticating with a .pem file
from common import config # sets location of .pem file, config.py must be in this folder
from oauth2client.service_account import ServiceAccountCredentials
from ee import oauth
credentials = ServiceAccountCredentials.from_p12_keyfile(
config.EE_ACCOUNT,
config.EE_PRIVATE_KEY_FILE,
scopes=oauth.SCOPES)
ee.Initialize(credentials, config.EE_URL)
except Exception as e:
print("EE init() error (with config.py and .pem file)",
e,
file=sys.stderr)
else:
args["importedDEM"] = abspath(args["importedDEM"])
# TODO: should change TouchTerrainEarthEngine.py to TouchTerrain.py as it now also deals with file DEMs
from common import TouchTerrainEarthEngine as TouchTerrain
totalsize, full_zip_file_name = TouchTerrain.get_zipped_tiles(
**args) # all args are in a dict
print("\nCreated zip file", full_zip_file_name, "%.2f" % totalsize, "Mb")
# Optional: unzip the zip file into the current folder
if 1: # set this to 0 if you don't want the zip file tp be unzippeed
#import os.path
#folder, file = os.path.splitext(full_zip_file_name) # tmp folder
folder = os.getcwd() + os.sep + args[
"zip_file_name"] # new stl folder in current folder
# unzip the zipfile into the folder it's already in
import zipfile
zip_ref = zipfile.ZipFile(full_zip_file_name, 'r')
zip_ref.extractall(folder)
zip_ref.close()
print("unzipped file inside", full_zip_file_name, "into", folder)
'''
def post(self): # make tiles in zip file and write
#print self.request.arguments() # should be the same as given to preflight
self.response.headers[
'X-Content-Type-Options'] = 'nosniff' # Prevent browsers from MIME-sniffing the content-type:
self.response.headers[
"X-Frame-Options"] = "SAMEORIGIN" # prevent clickjacking
self.response.out.write('<html><body>')
self.response.out.write('<h2>Processing finished:</h2>')
# debug: print/log all args and then values
args = {} # put arg name and value in a dict as key:value
for k in ("DEM_name", "trlat", "trlon", "bllat", "bllon", "printres",
"ntilesx", "ntilesy", "tilewidth", "basethick", "zscale",
"fileformat"):
v = self.request.get(k) # key = name of arg
args[k] = v # value
if k not in ["DEM_name", "fileformat"]:
args[k] = float(args[k]) # floatify non-string args
if k[:4] == "tile":
args[k] = args[
k] * 10 # multiply tilewidth/height by 10 to get from cm to mm
#print k, args[k]
self.response.out.write("%s = %s <br>" % (k, str(args[k])))
logging.info("%s = %s" % (k, str(args[k])))
# name of file is seconds since 2000
myname = str(
int((datetime.now() - datetime(2000, 1, 1)).total_seconds() *
1000))
# create zip and write to tmp
str_buf = TouchTerrainEarthEngine.get_zipped_tiles(
**args) # all args are in a dict
fname = myname + ".zip" # create filename for zipped
logging.info("About to write: " + tmp_folder + os.sep + fname)
fname = tmp_folder + os.sep + fname # put in tmp folder
if SERVER_TYPE == "GAE_devserver":
f = stubs.FakeFile(
fname, "wb+"
) # make a fake Fakefile instance (with devserver's write restriction disabled)
else:
f = open(fname, "wb+") # write to folder
f.write(str_buf)
f.close()
logging.info("finished writing %s" % (myname))
#str_buf = TouchTerrain.get_zipped_tiles(**args)
#str_buf = TouchTerrain.get_zipped_tiles("USGS/NED", ntilesx=2, ntilesy=2, **args)
#self.response.headers['Content-Type'] = 'text/zip'
#self.response.write(str_buf)
self.response.out.write(
'<br><form action="tmp/%s.zip" method="GET" enctype="multipart/form-data">'
% (myname))
self.response.out.write(
'<input type="submit" value="Download zip File " title=""> (will be deleted in 24 hrs)</form>'
)
#self.response.out.write('<form action="/" method="GET" enctype="multipart/form-data">')
#self.response.out.write('<input type="submit" value="Go back to selection map"> </form>')
self.response.out.write(
"<br>To return to the selection map, click the back button in your browser twice"
)
self.response.out.write(
"""<br>After downloading you can preview a STL/OBJ file at <a href="http://www.viewstl.com/" target="_blank"> www.viewstl.com ) </a> (limit: 35 Mb)"""
)
def preflight_generator():
# create html string
html = '<html>'
html += GA_script # <head> with script that inits GA with my tracking id and calls send pageview
# onload event will only be triggered once </body> is given
html += '''<body onerror="document.getElementById('error').innerHTML='Error (non-python), possibly the server timed out ...'"\n onload="document.getElementById('gif').style.display='none'; document.getElementById('working').innerHTML='Processing finished'">\n'''
html += '<h2 id="working" >Processing terrain data into 3D print file(s), please be patient.<br>\n'
html += 'Once the animation stops, you can preview and download your file.</h2>\n'
yield html # this effectively prints html into the browser but doesn't block, so we can keep going and append more html later ...
#
# print/log all args and their values
#
# put all agrs we got from the browser in a dict as key:value
args = request.form.to_dict()
# list of the subset of args needed for processing
key_list = ("DEM_name", "trlat", "trlon", "bllat", "bllon", "printres",
"ntilesx", "ntilesy", "tilewidth", "basethick", "zscale",
"fileformat")
for k in key_list:
# float-ify some ags
if k in [
"trlat", "trlon", "bllat", "bllon", "printres",
"tilewidth", "basethick", "zscale"
]:
args[k] = float(args[k])
# int-ify some args
if k in ["ntilesx", "ntilesy"]:
args[k] = int(args[k])
# decode any extra (manual) args and put them in the args dict as
# separate args as the are needed in that form for processing
# Note: the type of each arg is decided by json.loads(), so 1.0 will be a float, etc.
manual = args.get("manual", None)
extra_args = {}
if manual != None:
JSON_str = "{ " + manual + "}"
try:
extra_args = json.loads(JSON_str)
except Exception as e:
s = "JSON decode Error for manual: " + manual + " " + str(e)
logging.warning(s)
print(e)
yield "Warning: " + s + "<br>"
else:
for k in extra_args:
args[k] = extra_args[k] # append/overwrite
# TODO: validate
# log and show args in browser
html = '<br>'
for k in key_list:
html += "%s = %s <br>" % (k, str(args[k]))
logging.info("%s = %s" % (k, str(args[k])))
html += "<br>"
for k in extra_args:
html += "%s = %s <br>" % (k, str(args[k]))
logging.info("%s = %s" % (k, str(args[k])))
html += "<br>"
yield html
#
# bail out if the raster would be too large
#
width = args["tilewidth"]
bllon = args["bllon"]
trlon = args["trlon"]
bllat = args["bllat"]
trlat = args["trlat"]
dlon = 180 - abs(abs(bllon - trlon) - 180) # width in degrees
dlat = 180 - abs(abs(bllat - trlat) - 180) # height in degrees
center_lat = bllat + abs((bllat - trlat) / 2.0)
latitude_in_m, longitude_in_m = arcDegr_in_meter(center_lat)
num_total_tiles = args["ntilesx"] * args["ntilesy"]
pr = args["printres"]
# if we have "only" set, divide load by number of tiles
div_by = 1
if extra_args.get("only") != None:
div_by = float(num_total_tiles)
# for geotiffs only, set a much higher limit b/c we don't do any processing,
# just d/l the GEE geotiff and zip it
if args["fileformat"] == "GeoTiff":
global MAX_CELLS_PERMITED # thanks Nick!
MAX_CELLS_PERMITED *= 100
# pr <= 0 means: use source
if pr > 0: # print res given by user (width and height are in mm)
height = width * (dlat / float(dlon))
pix_per_tile = (width / float(pr)) * (height / float(pr))
tot_pix = int((pix_per_tile * num_total_tiles) /
div_by) # total pixels to print
print("total requested pixels to print",
tot_pix,
", max is",
MAX_CELLS_PERMITED,
file=sys.stderr)
else:
# estimates the total number of cells from area and arc sec resolution of source
# this is done for the entire area, so number of cell is irrelevant
DEM_name = args["DEM_name"]
cell_width_arcsecs = {
"""USGS/NED""": 1 / 9.0,
"""USGS/GMTED2010""": 7.5,
"""NOAA/NGDC/ETOPO1""": 30,
"""USGS/SRTMGL1_003""": 1
} # in arcseconds!
cwas = float(cell_width_arcsecs[DEM_name])
tot_pix = int(
(((dlon * 3600) / cwas) * ((dlat * 3600) / cwas)) / div_by)
print("total requested pixels to print at a source resolution of",
round(cwas, 2),
"arc secs is ",
tot_pix,
", max is",
MAX_CELLS_PERMITED,
file=sys.stderr)
if tot_pix > MAX_CELLS_PERMITED:
html = "Your requested job is too large! Please reduce the area (red box) or lower the print resolution<br>"
html += "<br>Current total number of Kilo pixels is " + str(
round(tot_pix / 1000.0, 2))
html += " but must be less than " + str(
round(MAX_CELLS_PERMITED / 1000.0, 2))
html += "<br><br>Hit Back on your browser to go back to the Main page and make adjustments ...\n"
html += '</body></html>'
yield html
return "bailing out!"
args["CPU_cores_to_use"] = NUM_CORES
# check if we have a valid temp folder
args["temp_folder"] = TMP_FOLDER
print("temp_folder is set to", args["temp_folder"], file=sys.stderr)
if not os.path.exists(args["temp_folder"]):
s = "temp folder " + args["temp_folder"] + " does not exist!"
print(s, file=sys.stderr)
logging.error(s)
html = '</body></html>Error:' + s
yield html
return "bailing out!" # Cannot continue without proper temp folder
# name of zip file is time since 2000 in 0.01 seconds
fname = str(
int((datetime.now() - datetime(2000, 1, 1)).total_seconds() *
1000))
args["zip_file_name"] = fname
# if this number of cells to process is exceeded, use a temp file instead of memory only
args["max_cells_for_memory_only"] = MAX_CELLS
# show snazzy animate gif - set to style="display: none to hide once
html = '<img src="static/processing.gif" id="gif" alt="processing animation" style="display: block;">\n'
# add an empty paragraph for error messages during processing that come from JS
html += '<p id="error"> </p>\n'
yield html
#
# Create zip and write to tmp
#
try:
totalsize, full_zip_file_name = TouchTerrainEarthEngine.get_zipped_tiles(
**args) # all args are in a dict
except Exception as e:
print("Error:", e, file=sys.stderr)
html = '</body></html>' + "Error:," + str(e)
yield html
return "bailing out!"
# if totalsize is negative, something went wrong, error message is in full_zip_file_name
if totalsize < 0:
print("Error:", full_zip_file_name, file=sys.stderr)
html = '</body></html>' + "Error:," + str(full_zip_file_name)
yield html
return "bailing out!"
else:
html = ""
# move zip from temp folder to static folder so flask can serve it (. is server root!)
zip_file = fname + ".zip"
try:
os.rename(full_zip_file_name,
os.path.join(DOWNLOADS_FOLDER, zip_file))
except Exception as e:
print("Error:", e, file=sys.stderr)
html = '</body></html>' + "Error:," + str(e)
yield html
return "bailing out!"
zip_url = url_for("download", filename=zip_file)
if args["fileformat"] in ("STLa", "STLb"):
html += '<br><form action="' + url_for(
"preview", zip_file=zip_file
) + '" method="GET" enctype="multipart/form-data">'
html += ' <input type="submit" value="Preview STL " '
html += ''' onclick="ga('send', 'event', 'Preview', 'Click', 'preview', '0')" '''
html += ' title=""> '
html += 'Note: This uses WebGL for in-browser 3D rendering and may take a while to load for large models.<br>\n'
html += 'You may not see anything for a while even after the progress bar is full!'
html += '</form>\n'
html += "Optional: tell us what you're using this model for<br>\n"
html += '''<textarea autofocus form="dl" id="comment" cols="100" maxlength=150 rows="2"></textarea><br>\n'''
html += '<br>\n<form id="dl" action="' + zip_url + '" method="GET" enctype="multipart/form-data">\n'
html += ' <input type="submit" value="Download zip File " \n'
#https://stackoverflow.com/questions/57499732/google-analytics-events-present-in-console-but-no-more-in-api-v4-results
html += ''' onclick=onclick_for_dl();\n'''
#html += ''' onclick="ga('send', 'event', 'Download', 'Click', 'from preview', '0');\n
#ga('send', 'event', 'Comment1', 'Click', document.getElementById('comment') , 1);"\n '''
#{
#'dimension1': document.getElementById('comment').value,
#'dimension2': 'Test for setting dimension2 from download button click'
#'dimension03': 'Test for setting dimension03 from download button click'
#},
#1);" \n
html += ' title="zip file contains a log file, the geotiff of the processed area and the 3D model file (stl/obj) for each tile">\n'
html += " Size: %.2f Mb (All files will be deleted in 6 hrs.)<br>\n" % totalsize
html += " <br>To return to the selection map, click the back button in your browser once.\n"
html += '</form>\n'
html += '</body></html>'
yield html
if len(sys.argv) > 1: # sys.argv are the CLI args
json_fname = sys.argv[1]
try:
fp = open(json_fname, "rU")
json_args = json.load(fp)
except:
sys.exit("Error: config file", json_fname, "not found")
print "reading", json_fname
for k in args.keys():
try:
args[k] = json_args[k] # try to find a value for k in json config file
except:
print "warning: ignored invalid option", k # no match, no problem, just keep the default value
print "%s = %s" % (k, str(args[k]))
else:
print "no config file given, using defaults:"
for k in args.keys():
print "%s = %s" % (k, str(args[k]))
fname = args["zip_file_name"] + "_" + time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime()) + ".zip"
del args["zip_file_name"] # otherwise get_zipped_tiles complains about this argument
str_buf = TouchTerrain.get_zipped_tiles(**args) # all args are in a dict
with open(fname, 'wb+') as fp:
fp.write(str_buf)
print "finished writing %s" % (fname)