def run_get_zipped_tiles(overwrite_args, testname):
'''utility to actually run a test with its (overwrite) args, which will override the official
default (initial) args.'''
# Use this to force import of local modules, required for debuging those imports
import sys
oldsp = sys.path
sys.path = ["."] + sys.path
from touchterrain.common import TouchTerrainEarthEngine as TouchTerrain
sys.path = oldsp
import ee
ee.Initialize()
# update default args with overwrite args
args = {**TouchTerrain.initial_args, **overwrite_args}
print(testname, "\nUsing these config values:")
for k in sorted(args.keys()):
print("%s = %s" % (k, str(args[k])))
totalsize, full_zip_file_name = TouchTerrain.get_zipped_tiles(**args)
#print("In tmp, created zip file", full_zip_file_name, "%.2f" % totalsize, "Mb")
from os import getcwd, sep, remove
folder = getcwd() + sep + "test" + sep + args["zip_file_name"] # unzip into his folder inside test
import zipfile
zip_ref = zipfile.ZipFile(full_zip_file_name, 'r')
zip_ref.extractall(folder)
zip_ref.close()
print("unzipped files into", folder)
remove(full_zip_file_name)
def run_get_zipped_tiles(args):
'''utility to actually run get_zipped_tiles()'''
import ee
from touchterrain.common import TouchTerrainEarthEngine as TouchTerrain
ee.Initialize()
print("\nUsing these config values:")
for k in sorted(args.keys()):
print("%s = %s" % (k, str(args[k])))
totalsize, full_zip_file_name = TouchTerrain.get_zipped_tiles(**args)
print("In tmp, created zip file", full_zip_file_name, "%.2f" % totalsize, "Mb")
from os import getcwd, sep
folder = getcwd() + sep + "test" + sep + args["zip_file_name"] # unzip into his folder inside test
import zipfile
zip_ref = zipfile.ZipFile(full_zip_file_name, 'r')
zip_ref.extractall(folder)
zip_ref.close()
print("unzipped STL file into", folder)
def main():
# Default parameters:
# The JSON file overwrites values for the following keys, which are used as
# args for get_zipped_tiles() and save them inside a zipped folder.
# Print each tile on a 3D printer (they are already scaled!)
args = {
"DEM_name": 'USGS/NED',# DEM_name: name of DEM source used in Google Earth Engine
# for all valid sources, see DEM_sources in TouchTerrainEarthEngine.py
"trlat": 44.69741706507476, # lat/lon of top right corner
"trlon": -107.97962089843747,
"bllat": 44.50185267072875, # lat/lon of bottom left corner
"bllon": -108.25427910156247,
"importedDEM": None, # if not None, the raster file to use as DEM instead of using GEE (null in JSON)
"printres": 0.5, # resolution (horizontal) of 3D printer (= size of one pixel) in mm
"ntilesx": 1, # number of tiles in x and y
"ntilesy": 1,
"tilewidth": 80, # width of each tile in mm (<- !!!!!), tile height is calculated
"basethick": 1, # thickness (in mm) of printed base
"zscale": 1.0, # elevation (vertical) scaling
"fileformat": "STLb", # format of 3D model files: "obj" wavefront obj (ascii),"STLa" ascii STL or "STLb" binary STL
"tile_centered": False, # True-> all tiles are centered around 0/0, False, all tiles "fit together"
"zip_file_name": "terrain", # base name of zipfile, .zip will be added
"CPU_cores_to_use" : 0, # 0 means all cores, None (null in JSON!) => don't use multiprocessing
"max_cells_for_memory_only" : 1000 * 1000, # if raster is bigger, use temp_files instead of memory
# these are the args that could be given "manually" via the web UI
"no_bottom": False, # omit bottom triangles?
#"rot_degs": 0, # rotate by degrees ccw # CH disabled for now
"bottom_image": None, # 1 band greyscale image used for bottom relief
"ignore_leq": None, # set values <= this to NaN, so they are ignored
"lower_leq": None, # e.g. [0.0, 2.0] values <= 0.0 will be lowered by 2mm in the final model
"unprojected": False, # don't project to UTM, only usefull when using GEE for DEM rasters
"only": None,# list of tile index [x,y] with is the only tile to be processed. None means process all tiles (index is 1 based)
}
# write an example json file, in case it gets deleted ...
with open('example_config.json', 'w+') as fp:
json.dump(args, fp, indent=0, sort_keys=True) # indent = 0: newline after each comma
print('Wrote example_config.json with default values, you can use it as a template but make sure to rename it!')
# parse args
if len(sys.argv) > 1: # sys.argv are the CLI args
json_fname = sys.argv[1]
try:
fp = open(json_fname, "rU")
except Exception as e:
sys.exit("Error: can't find " + json_fname + ": " + str(e))
file_content = fp.read()
try:
json_args = json.loads(file_content)
except Exception as e:
sys.exit("Error: can't json parse " + json_fname + ": " + str(e))
print("reading", json_fname)
for k in list(args.keys()):
try:
args[k] = json_args[k] # try to find a value for k in json config file
#print k, args[k]
except:
print("info:", k, "has missing or invalid value, using defaults where possible") # no match? no problem, just keep the default value
#print "%s = %s" % (k, str(args[k]))
else:
# no JSON config file given, setting config values in code
# you can comment out lines for which you don't want to overwrite the default settings
overwrite_args = {
"DEM_name": 'USGS/NED',# DEM_name: name of DEM source used in Google Earth Engine
# for all valid sources, see DEM_sources in TouchTerrainEarthEngine.py
"trlat": 44.69741706507476, # lat/lon of top right corner
"trlon": -107.97962089843747,
"bllat": 44.50185267072875, # lat/lon of bottom left corner
"bllon": -108.25427910156247,
"importedDEM": None, # if not None, the raster file to use as DEM instead of using GEE (null in JSON)
"printres": 0.5, # resolution (horizontal) of 3D printer (= size of one pixel) in mm
"ntilesx": 1, # number of tiles in x and y
"ntilesy": 1,
"tilewidth": 80, # width of each tile in mm (<- !!!!!), tile height is calculated
"basethick": 1, # thickness (in mm) of printed base
"zscale": 1.0, # elevation (vertical) scaling
"fileformat": "STLb", # format of 3D model files: "obj" wavefront obj (ascii),"STLa" ascii STL or "STLb" binary STL
"tile_centered": False, # True-> all tiles are centered around 0/0, False, all tiles "fit together"
"zip_file_name": "terrain", # base name of zipfile, .zip will be added
"CPU_cores_to_use" : 0, # 0 means all cores, None (null in JSON!) => don't use multiprocessing
"max_cells_for_memory_only" : 1000 * 1000, # if raster is bigger, use temp_files instead of memory
"no_bottom": False, # omit bottom triangles?
#"rot_degs": 0, # rotate by degrees ccw # CH disabled for now
"bottom_image": None, # 1 band greyscale image used for bottom relief
"ignore_leq": None, # set values <= this to NaN, so they are ignored
"lower_leq": None, # e.g. [0.0, 2.0] values <= 0.0 will be lowered by 2mm in the final model
"unprojected": False, # don't project to UTM, only usefull when using GEE for DEM rasters
"only": None,# list of tile index [x,y] with is the only tile to be processed. None means process all tiles (index is 1 based)
}
# overwrite config settings in args
for k in overwrite_args:
args[k] = overwrite_args[k]
# CH testing gpx here, so I can use the debugger
'''
args = {
"importedDEM": None,
"DEM_name": "USGS/NED", # DEM source
# area for gpx test
"bllat": 39.32205105794382, # bottom left corner lat
"bllon": -120.37497608519418, # bottom left corner long
"trlat": 39.45763749030933, # top right corner lat
"trlon": -120.2002248034559, # top right corner long
"tilewidth": 120, # width of each tile in mm,
"printres": 0.4, # resolution (horizontal) of 3D printer in mm
"ntilesx": 1, # number of tiles in x
"ntilesy": 1, # number of tiles in y
"basethick": 0.5, # thickness (in mm) of printed base
"zscale": 1.5, # elevation (vertical) scaling
"fileformat": "STLb", # format of 3D model file
"zip_file_name": "test_get_zipped_tiles_gpx", # base name of zipfile, .zip will be added
"importedGPX": # Plot GPX paths from these files onto the model.
["stuff/gpx-test/DLRTnML.gpx",
"stuff/gpx-test/DonnerToFrog.gpx",
"stuff/gpx-test/CinTwistToFrog.gpx",
"stuff/gpx-test/sagehen.gpx",
"stuff/gpx-test/dd-to-prosser.gpx",
"stuff/gpx-test/alder-creek-to-crabtree-canyon.gpx",
"stuff/gpx-test/ugly-pop-without-solvang.gpx",
"stuff/gpx-test/tomstrail.gpx"
],
"gpxPathHeight": 100, # Currently we plot the GPX path by simply adjusting the raster elevation at the specified lat/lon,
# therefore this is in meters. Negative numbers are ok and put a dent in the mdoel
"gpxPixelsBetweenPoints" : 20, # GPX Files haves a lot of points. A higher number will create more space between lines drawn
# on the model and can have the effect of making the paths look a bit cleaner
"gpxPathThickness" : 5, # Stack parallel lines on either side of primary line to create thickness.
# A setting of 1 probably looks the best
}
'''
# print out current args
print("\nUsing these config values:")
for k in sorted(args.keys()):
print("%s = %s" % (k, str(args[k])))
# No DEM file given, use Google Earth Engine
if args["importedDEM"] == None:
pass
''' #not needed anymore? EE init is now done on TouchTerrainEarthEngine import
# drawback: local geotiff users will get init warnings/fails#
# initialize ee - needs a google earth engine account! See TouchTerrain_standalone_installation.pdf
try:
import ee
except Exception as e:
print("Google Earth Engine module not installed", e, file=sys.stderr)
# try both ways of authenticating
try:
ee.Initialize() # uses .config/earthengine/credentials
except Exception as e:
print("EE init() error (with .config/earthengine/credentials)", e, file=sys.stderr)
try:
# try authenticating with a .pem file
from touchterrain.common import config # sets location of .pem file, config.py must be in this folder
from oauth2client.service_account import ServiceAccountCredentials
from ee import oauth
credentials = ServiceAccountCredentials.from_p12_keyfile(config.EE_ACCOUNT, config.EE_PRIVATE_KEY_FILE, scopes=oauth.SCOPES)
ee.Initialize(credentials, config.EE_URL)
except Exception as e:
print("EE init() error (with config.py and .pem file)", e, file=sys.stderr)
'''
else:
args["importedDEM"] = abspath(args["importedDEM"])
# Give all config values to get_zipped_tiles for processing:
totalsize, full_zip_file_name = TouchTerrain.get_zipped_tiles(**args) # all args are in a dict
print("\nCreated zip file", full_zip_file_name, "%.2f" % totalsize, "Mb")
# Optional: unzip the zip file into the current folder
if 1: # set this to 0 if you don't want the zip file to be unzippeed
#import os.path
#folder, file = os.path.splitext(full_zip_file_name) # tmp folder
folder = os.getcwd() + os.sep + args["zip_file_name"]# new stl folder in current folder
# unzip the zipfile into the folder it's already in
import zipfile
zip_ref = zipfile.ZipFile(full_zip_file_name, 'r')
zip_ref.extractall(folder)
zip_ref.close()
print("unzipped file inside", full_zip_file_name, "into", folder)
'''
def preflight_generator():
# header info is stringified query parameters (to encode the GUI parameters via GA)
query_list = list(request.form.items())
header = make_current_URL(query_list)[1:] # skip leading ?
# create html string
html = '<html>'
html += make_GA_script(header) # <head> with script that inits GA with my tracking id and calls send pageview
# onload event will only be triggered once </body> is given
html += '''<body onerror="document.getElementById('error').innerHTML='Error (non-python), possibly the server timed out ...'"\n onload="document.getElementById('gif').style.display='none'; document.getElementById('working').innerHTML='Processing finished'">\n'''
html += '<h2 id="working" >Processing terrain data into 3D print file(s), please be patient.<br>\n'
html += 'Once the animation stops, you can preview and download your file.</h2>\n'
yield html # this effectively prints html into the browser but doesn't block, so we can keep going and append more html later ...
#
# print/log all args and their values
#
# put all args we got from the browser in a dict as key:value
args = request.form.to_dict()
# list of the subset of args needed for processing
key_list = ("DEM_name", "trlat", "trlon", "bllat", "bllon", "printres",
"ntilesx", "ntilesy", "tilewidth", "basethick", "zscale", "fileformat")
for k in key_list:
# float-ify some args
if k in ["trlat", "trlon", "bllat", "bllon","printres", "tilewidth", "basethick", "zscale"]:
args[k] = float(args[k])
# int-ify some args
if k in ["ntilesx", "ntilesy"]:
args[k] = int(args[k])
# decode any extra (manual) args and put them in the args dict as
# separate args as the are needed in that form for processing
# Note: the type of each arg is decided by json.loads(), so 1.0 will be a float, etc.
manual = args.get("manual", None)
extra_args={}
if manual != None:
JSON_str = "{ " + manual + "}"
try:
extra_args = json.loads(JSON_str)
except Exception as e:
s = "JSON decode Error for manual: " + manual + " " + str(e)
logging.warning(s)
print(e)
yield "Warning: " + s + "<br>"
else:
for k in extra_args:
args[k] = extra_args[k] # append/overwrite
# TODO: validate
# log and show args in browser
html = '<br>'
for k in key_list:
if args[k] != None and args[k] != '':
html += "%s = %s <br>" % (k, str(args[k]))
logging.info("%s = %s" % (k, str(args[k])))
html += "<br>"
for k in extra_args:
if args[k] != None and args[k] != '':
html += "%s = %s <br>" % (k, str(args[k]))
logging.info("%s = %s" % (k, str(args[k])))
# see if we have a optional kml file in requests
geojson_polygon = None
if 'kml_file' in request.files:
kml_file = request.files['kml_file']
if kml_file.filename != '':
from geojson import Polygon
# process kml file
kml_stream = kml_file.read()
coords, msg = TouchTerrainEarthEngine.get_KML_poly_geometry(kml_stream)
if msg != None: # Either got a line instead of polygon or nothing good at all
if coords == None: # got nothing good
html += "Warning: " + kml_file.filename + " contained neither polygon nor line, falling back to area selection box.<br>"
else:
html += "Warning: Using line with " + str(len(coords)) + " points in " + kml_file.filename + " as no polygon was found.<br>"
geojson_polygon = Polygon([coords])
else: # got polygon
geojson_polygon = Polygon([coords]) # coords must be [0], [1] etc. would be holes
html += "Using polygon from kml file " + kml_file.filename + " with " + str(len(coords)) + " points.<br>"
html += "<br>"
yield html
#
# bail out if the raster would be too large
#
width = args["tilewidth"]
bllon = args["bllon"]
trlon = args["trlon"]
bllat = args["bllat"]
trlat = args["trlat"]
dlon = 180 - abs(abs(bllon - trlon) - 180) # width in degrees
dlat = 180 - abs(abs(bllat - trlat) - 180) # height in degrees
center_lat = bllat + abs((bllat - trlat) / 2.0)
latitude_in_m, longitude_in_m = arcDegr_in_meter(center_lat)
num_total_tiles = args["ntilesx"] * args["ntilesy"]
pr = args["printres"]
# if we have "only" set, divide load by number of tiles
div_by = 1
if extra_args.get("only") != None:
div_by = float(num_total_tiles)
# for geotiffs only, set a much higher limit b/c we don't do any processing,
# just d/l the GEE geotiff and zip it
if args["fileformat"] == "GeoTiff":
global MAX_CELLS_PERMITED # thanks Nick!
MAX_CELLS_PERMITED *= 100
# pr <= 0 means: use source resolution
if pr > 0: # print res given by user (width and height are in mm)
height = width * (dlat / float(dlon))
pix_per_tile = (width / float(pr)) * (height / float(pr))
tot_pix = int((pix_per_tile * num_total_tiles) / div_by) # total pixels to print
print("total requested pixels to print", tot_pix, ", max is", MAX_CELLS_PERMITED, file=sys.stderr)
else:
# estimates the total number of cells from area and arc sec resolution of source
# this is done for the entire area, so number of cell is irrelevant
DEM_name = args["DEM_name"]
cell_width_arcsecs = {"USGS/NED":1/9.0, "MERIT/DEM/v1_0_3":3,"USGS/GMTED2010":7.5, "CPOM/CryoSat2/ANTARCTICA_DEM":30,
"NOAA/NGDC/ETOPO1":60, "USGS/GTOPO30":30, "USGS/SRTMGL1_003":1,
"JAXA/ALOS/AW3D30/V2_2":1, "NRCan/CDEM": 0.75,} # in arcseconds!
cwas = float(cell_width_arcsecs[DEM_name])
tot_pix = int( ( ((dlon * 3600) / cwas) * ((dlat * 3600) / cwas) ) / div_by)
print("total requested pixels to print at a source resolution of", round(cwas,2), "arc secs is ", tot_pix, ", max is", MAX_CELLS_PERMITED, file=sys.stderr)
if tot_pix > MAX_CELLS_PERMITED:
html = "Your requested job is too large! Please reduce the area (red box) or lower the print resolution<br>"
html += "<br>Current total number of Kilo pixels is " + str(round(tot_pix / 1000.0, 2))
html += " but must be less than " + str(round(MAX_CELLS_PERMITED / 1000.0, 2))
html + "If you're trying to process multiple tiles: Consider using the only manual setting to instead print one tile at a time (https://chharding.github.io/TouchTerrain_for_CAGEO/)"
html += "<br><br>Hit Back on your browser to go back to the Main page and make adjustments ...\n"
html += '</body></html>'
yield html
return "bailing out!"
args["CPU_cores_to_use"] = NUM_CORES
# check if we have a valid temp folder
args["temp_folder"] = TMP_FOLDER
print("temp_folder is set to", args["temp_folder"], file=sys.stderr)
if not os.path.exists(args["temp_folder"]):
s = "temp folder " + args["temp_folder"] + " does not exist!"
print(s, file=sys.stderr)
logging.error(s)
html = '</body></html>Error:' + s
yield html
return "bailing out!"# Cannot continue without proper temp folder
# name of zip file is time since 2000 in 0.01 seconds
fname = str(int((datetime.now()-datetime(2000,1,1)).total_seconds() * 1000))
args["zip_file_name"] = fname
# if this number of cells to process is exceeded, use a temp file instead of memory only
args["max_cells_for_memory_only"] = MAX_CELLS
# set geojson_polygon as polygon arg (None by default)
args["polygon"] = geojson_polygon
# show snazzy animated gif - set to style="display: none to hide once processing is done
html = '<img src="static/processing.gif" id="gif" alt="processing animation" style="display: block;">\n'
# add an empty paragraph for error messages during processing that come from JS
html += '<p id="error"> </p>\n'
yield html
# Grab a 640 x 640 Terrain Google map of the area
map_img_filename = store_static_Google_map(bllon, trlon, bllat, trlat,
google_maps_key, args["temp_folder"], args["zip_file_name"])
if map_img_filename != None:
args["map_img_filename"] = map_img_filename
#
# Create zip and write to tmp
#
try:
totalsize, full_zip_file_name = TouchTerrainEarthEngine.get_zipped_tiles(**args) # all args are in a dict
except Exception as e:
print("Error:", e, file=sys.stderr)
html = '</body></html>' + "Error: " + str(e)
yield html
return "bailing out!"
# if totalsize is negative, something went wrong, error message is in full_zip_file_name
if totalsize < 0:
print("Error:", full_zip_file_name, file=sys.stderr)
html = '</body></html>' + "Error:," + str(full_zip_file_name)
yield html
return "bailing out!"
else:
html = ""
# move zip from temp folder to static folder so flask can serve it (. is server root!)
zip_file = fname + ".zip"
try:
os.rename(full_zip_file_name, os.path.join(DOWNLOADS_FOLDER, zip_file))
except Exception as e:
print("Error moving file from tmp to downloads:", e, file=sys.stderr)
html = '</body></html>' + "Error:," + str(e)
yield html
return "bailing out!"
zip_url = url_for("download", filename=zip_file)
if args["fileformat"] in ("STLa", "STLb"):
html += '<br><form action="' + url_for("preview", zip_file=zip_file) +'" method="GET" enctype="multipart/form-data">'
html += ' <input type="submit" value="Preview STL " '
html += ''' onclick="ga('send', 'event', 'Preview', 'Click', 'preview', '0')" '''
html += ' title=""> '
html += 'Note: This uses WebGL for in-browser 3D rendering and may take a while to load for large models.<br>\n'
html += 'You may not see anything for a while even after the progress bar is full!'
html += '</form>\n'
html += "Optional: tell us what you're using this model for<br>\n"
html += '''<textarea autofocus form="dl" id="comment" cols="100" maxlength=150 rows="2"></textarea><br>\n'''
html += '<br>\n<form id="dl" action="' + zip_url +'" method="GET" enctype="multipart/form-data">\n'
html += ' <input type="submit" value="Download zip File " \n'
#https://stackoverflow.com/questions/57499732/google-analytics-events-present-in-console-but-no-more-in-api-v4-results
html += ''' onclick=onclick_for_dl();\n'''
html += ' title="zip file contains a log file, the geotiff of the processed area and the 3D model file (stl/obj) for each tile">\n'
html += " Size: %.2f Mb (All files will be deleted in 6 hrs.)<br>\n" % totalsize
html += '</form>\n'
html += " <br>To return to the selection map, click on the back button in your browser once, or on the link below:<br>"
#html += "<br>Click on the URL below to return to the selection map:<br>"
# print out the query parameters (note hardcoded server name!)
html += '<a href = "'
query_list = list(request.form.items())
server = "https://touchterrain.geol.iastate.edu/"
#server = "https://touchterrain-beta.geol.iastate.edu/"
query_str = server + make_current_URL(query_list)
html += query_str + '">' + query_str + "</a><br>"
html += "<br>To have somebody else generate the same model, have them copy&paste this URL into a browser<br>"
html += '</body></html>'
yield html
def main():
# Default parameters:
# The JSON file overwrites values for the following keys, which are used as
# args for get_zipped_tiles() and save them inside a zipped folder.
# Print each tile on a 3D printer (they are already scaled!)
args = {
"DEM_name":
'USGS/3DEP/10m', # DEM_name: name of DEM source used in Google Earth Engine
# for all valid sources, see DEM_sources in TouchTerrainEarthEngine.py
"trlat": 44.69741706507476, # lat/lon of top right corner
"trlon": -107.97962089843747,
"bllat": 44.50185267072875, # lat/lon of bottom left corner
"bllon": -108.25427910156247,
"poly_file": None, # path to a local kml file
"polyURL":
None, # URL to a publicly readable(!) kml file on Google Drive
"importedDEM":
None, # if not None, the raster file to use as DEM instead of using GEE (null in JSON)
"printres":
0.5, # resolution (horizontal) of 3D printer (= size of one pixel) in mm
"ntilesx": 1, # number of tiles in x and y
"ntilesy": 1,
"tilewidth":
80, # width of each tile in mm (<- !!!!!), tile height is calculated
"basethick": 1, # thickness (in mm) of printed base
"zscale": 1.0, # elevation (vertical) scaling
"fileformat":
"STLb", # format of 3D model files: "obj" wavefront obj (ascii),"STLa" ascii STL or "STLb" binary STL
"tile_centered":
False, # True-> all tiles are centered around 0/0, False, all tiles "fit together"
"zip_file_name": "terrain", # base name of zipfile, .zip will be added
"CPU_cores_to_use":
0, # 0 means all cores, None (null in JSON!) => don't use multiprocessing
"max_cells_for_memory_only":
1000 * 1000, # if raster is bigger, use temp_files instead of memory
# these are the args that could be given "manually" via the web UI
"no_bottom": False, # omit bottom triangles?
#"rot_degs": 0, # rotate by degrees ccw # CH disabled for now
"bottom_image": None, # 1 band greyscale image used for bottom relief
"ignore_leq": None, # set values <= this to NaN, so they are ignored
"lower_leq":
None, # e.g. [0.0, 2.0] values <= 0.0 will be lowered by 2mm in the final model
"unprojected":
False, # don't project to UTM, only usefull when using GEE for DEM rasters
"only":
None, # list of tile index [x,y] with is the only tile to be processed. None means process all tiles (index is 1 based)
"importedGPX": None, # Plot GPX paths from files onto the model.
"gpxPathHeight":
100, # Currently we plot the GPX path by simply adjusting the raster elevation at the specified lat/lon,
# therefore this is in meters. Negative numbers are ok and put a dent in the mdoel
"gpxPixelsBetweenPoints":
20, # GPX Files haves a lot of points. A higher number will create more space between lines drawn
# on the model and can have the effect of making the paths look a bit cleaner
"gpxPathThickness":
5, # Stack parallel lines on either side of primary line to create thickness.
"smooth_borders": True, # smooth borders
"offset_masks_lower":
None, # e.g. [[filename, offset], [filename2, offset2],...] Masked regions (pixel values > 0) in the file will be lowered by offset(mm) * pixel value in the final model.
"fill_holes":
None, # e.g. [10, 7] Specify number of interations to find and neighbor threshold to fill holes. -1 iterations will continue iterations until no more holes are found. Defaults to 7 neighbors in a 3x3 footprint with elevation > 0 to fill a hole with the average of the footprint.
}
# write an example json file, in case it gets deleted ...
with open('example_config.json', 'w+') as fp:
json.dump(args, fp, indent=0,
sort_keys=True) # indent = 0: newline after each comma
print(
'Wrote example_config.json with default values, you can use it as a template but make sure to rename it!'
)
# parse args
if len(sys.argv) > 1: # sys.argv are the CLI args
json_fname = sys.argv[1]
try:
fp = open(json_fname, "r")
except Exception as e:
sys.exit("Error: can't find " + json_fname + ": " + str(e))
file_content = fp.read()
try:
json_args = json.loads(file_content)
except Exception as e:
sys.exit("Error: can't json parse " + json_fname + ": " + str(e))
print("reading", json_fname)
for k in list(args.keys()):
try:
args[k] = json_args[
k] # try to find a value for k in json config file
#print k, args[k]
except:
print(
"info:", k,
"has missing or invalid value, using defaults where possible"
) # no match? no problem, just keep the default value
#print "%s = %s" % (k, str(args[k]))
else:
# no JSON config file given, setting config values in code
# you can comment out lines for which you don't want to overwrite the default settings
overwrite_args = {
"DEM_name":
'USGS/3DEP/10m', # DEM_name: name of DEM source used in Google Earth Engine
# for all valid sources, see DEM_sources in TouchTerrainEarthEngine.py
"trlat": 44.69741706507476, # lat/lon of top right corner
"trlon": -107.97962089843747,
"bllat": 44.50185267072875, # lat/lon of bottom left corner
"bllon": -108.25427910156247,
"importedDEM":
None, # if not None, the raster file to use as DEM instead of using GEE (null in JSON)
"printres":
0.5, # resolution (horizontal) of 3D printer (= size of one pixel) in mm
"ntilesx": 1, # number of tiles in x and y
"ntilesy": 1,
"tilewidth":
80, # width of each tile in mm (<- !!!!!), tile height is calculated
"basethick": 1, # thickness (in mm) of printed base
"zscale": 1.0, # elevation (vertical) scaling
"fileformat":
"STLb", # format of 3D model files: "obj" wavefront obj (ascii),"STLa" ascii STL or "STLb" binary STL
"tile_centered":
False, # True-> all tiles are centered around 0/0, False, all tiles "fit together"
"zip_file_name":
"terrain", # base name of zipfile, .zip will be added
"CPU_cores_to_use":
0, # 0 means all cores, None (null in JSON!) => don't use multiprocessing
"max_cells_for_memory_only": 1000 *
1000, # if raster is bigger, use temp_files instead of memory
"no_bottom": False, # omit bottom triangles?
#"rot_degs": 0, # rotate by degrees ccw # CH disabled for now
"bottom_image":
None, # 1 band greyscale image used for bottom relief
"ignore_leq":
None, # set values <= this to NaN, so they are ignored
"lower_leq":
None, # e.g. [0.0, 2.0] values <= 0.0 will be lowered by 2mm in the final model
"unprojected":
False, # don't project to UTM, only usefull when using GEE for DEM rasters
"only":
None, # list of tile index [x,y] with is the only tile to be processed. None means process all tiles (index is 1 based)
"poly_file": None, #"idaho.kml", #TT_poly_test.kml",
"smooth_borders": True, # smooth borders
}
# overwrite config settings in args
for k in overwrite_args:
args[k] = overwrite_args[k]
# CH testing gpx here, so I can use the debugger
'''
args = {
"importedDEM": None,
"DEM_name": "USGS/3DEP/10m", # DEM source
# area for gpx test
"bllat": 39.32205105794382, # bottom left corner lat
"bllon": -120.37497608519418, # bottom left corner long
"trlat": 39.45763749030933, # top right corner lat
"trlon": -120.2002248034559, # top right corner long
"tilewidth": 120, # width of each tile in mm,
"printres": 0.4, # resolution (horizontal) of 3D printer in mm
"ntilesx": 1, # number of tiles in x
"ntilesy": 1, # number of tiles in y
"basethick": 0.5, # thickness (in mm) of printed base
"zscale": 1.5, # elevation (vertical) scaling
"fileformat": "STLb", # format of 3D model file
"zip_file_name": "test_get_zipped_tiles_gpx", # base name of zipfile, .zip will be added
"importedGPX": # Plot GPX paths from these files onto the model.
["stuff/gpx-test/DLRTnML.gpx",
"stuff/gpx-test/DonnerToFrog.gpx",
"stuff/gpx-test/CinTwistToFrog.gpx",
"stuff/gpx-test/sagehen.gpx",
"stuff/gpx-test/dd-to-prosser.gpx",
"stuff/gpx-test/alder-creek-to-crabtree-canyon.gpx",
"stuff/gpx-test/ugly-pop-without-solvang.gpx",
"stuff/gpx-test/tomstrail.gpx"
],
"gpxPathHeight": 100, # Currently we plot the GPX path by simply adjusting the raster elevation at the specified lat/lon,
# therefore this is in meters. Negative numbers are ok and put a dent in the mdoel
"gpxPixelsBetweenPoints" : 20, # GPX Files haves a lot of points. A higher number will create more space between lines drawn
# on the model and can have the effect of making the paths look a bit cleaner
"gpxPathThickness" : 5, # Stack parallel lines on either side of primary line to create thickness.
# A setting of 1 probably looks the best
"smooth_borders": True, # smooth borders
}
ml = convert_to_GeoJSON(args["importedGPX"])
'''
# print out current args
print("\nUsing these config values:")
for k in sorted(args.keys()):
print("%s = %s" % (k, str(args[k])))
# for local DEM, get the full path to it
if not args["importedDEM"] == None:
args["importedDEM"] = abspath(args["importedDEM"])
# get full path to offset mask TIFF
if not args["offset_masks_lower"] == None and len(
args["offset_masks_lower"]) > 0:
for offset_mask_pair in args["offset_masks_lower"]:
offset_mask_pair[0] = abspath(offset_mask_pair[0])
# Give all config values to get_zipped_tiles for processing:
totalsize, full_zip_file_name = TouchTerrain.get_zipped_tiles(
**args) # all args are in a dict
print("\nCreated zip file", full_zip_file_name, "%.2f" % totalsize, "Mb")
# Optional: unzip the zip file into the current folder
if 1: # set this to 0 if you don't want the zip file to be unzippeed
#import os.path
#folder, file = os.path.splitext(full_zip_file_name) # tmp folder
folder = os.getcwd() + os.sep + args[
"zip_file_name"] # new stl folder in current folder
# unzip the zipfile into the folder it's already in
import zipfile
zip_ref = zipfile.ZipFile(full_zip_file_name, 'r')
zip_ref.extractall(folder)
zip_ref.close()
print("unzipped file inside", full_zip_file_name, "into", folder)
'''
def main():
# write an example json file, in case it gets deleted ...
with open('example_config.json', 'w+') as fp:
json.dump(args, fp, indent=0,
sort_keys=True) # indent = 0: newline after each comma
print(
'Wrote example_config.json with default values, you can use it as a template but make sure to rename it!'
)
import sys, os
from os.path import abspath, dirname
# parse args
if len(sys.argv) > 1: # sys.argv are the CLI args
json_fname = sys.argv[1]
try:
fp = open(json_fname, "rU")
except Exception as e:
sys.exit("Error: can't find " + json_fname + ": " + str(e))
file_content = fp.read()
try:
json_args = json.loads(file_content)
except Exception as e:
sys.exit("Error: can't json parse " + json_fname + ": " + str(e))
print("reading", json_fname)
for k in list(args.keys()):
try:
args[k] = json_args[
k] # try to find a value for k in json config file
#print k, args[k]
except:
print(
"info:", k,
"has missing or invalid value, using defaults where possible"
) # no match? no problem, just keep the default value
#print "%s = %s" % (k, str(args[k]))
else:
# no JSON config file given, setting config values in code
# you can comment out lines for which you don't want to overwrite the default settings
overwrite_args = {
"DEM_name":
'USGS/NED', # DEM_name: name of DEM source used in Google Earth Engine
# for all valid sources, see DEM_sources in TouchTerrainEarthEngine.py
"trlat": 44.69741706507476, # lat/lon of top right corner
"trlon": -107.97962089843747,
"bllat": 44.50185267072875, # lat/lon of bottom left corner
"bllon": -108.25427910156247,
"importedDEM":
None, # if not None, the raster file to use as DEM instead of using GEE (null in JSON)
"printres":
0.5, # resolution (horizontal) of 3D printer (= size of one pixel) in mm
"ntilesx": 1, # number of tiles in x and y
"ntilesy": 1,
"tilewidth":
80, # width of each tile in mm (<- !!!!!), tile height is calculated
"basethick": 1, # thickness (in mm) of printed base
"zscale": 1.0, # elevation (vertical) scaling
"fileformat":
"STLb", # format of 3D model files: "obj" wavefront obj (ascii),"STLa" ascii STL or "STLb" binary STL
"tile_centered":
False, # True-> all tiles are centered around 0/0, False, all tiles "fit together"
"zip_file_name":
"terrain", # base name of zipfile, .zip will be added
"CPU_cores_to_use":
0, # 0 means all cores, None (null in JSON!) => don't use multiprocessing
"max_cells_for_memory_only": 1000 *
1000, # if raster is bigger, use temp_files instead of memory
"no_bottom": False, # omit bottom triangles?
#"rot_degs": 0, # rotate by degrees ccw # CH disabled for now
"bottom_image":
None, # 1 band greyscale image used for bottom relief
"ignore_leq":
None, # set values <= this to NaN, so they are ignored
"lower_leq":
None, # e.g. [0.0, 2.0] values <= 0.0 will be lowered by 2mm in the final model
"unprojected":
False, # don't project to UTM, only usefull when using GEE for DEM rasters
"only":
None, # list of tile index [x,y] with is the only tile to be processed. None means process all tiles (index is 1 based)
}
# overwrite config settings in args
for k in overwrite_args:
args[k] = overwrite_args[k]
# print out current args
print("\nUsing these config values:")
for k in sorted(args.keys()):
print("%s = %s" % (k, str(args[k])))
# No DEM file given, use Google Earth Engine
if args["importedDEM"] == None:
# initialize ee - needs a google earth engine account! See TouchTerrain_standalone_installation.pdf
try:
import ee
except Exception as e:
print("Google Earth Engine module not installed",
e,
file=sys.stderr)
# try both ways of authenticating
try:
ee.Initialize() # uses .config/earthengine/credentials
except Exception as e:
print("EE init() error (with .config/earthengine/credentials)",
e,
file=sys.stderr)
try:
# try authenticating with a .pem file
from touchterrain.common import config # sets location of .pem file, config.py must be in this folder
from oauth2client.service_account import ServiceAccountCredentials
from ee import oauth
credentials = ServiceAccountCredentials.from_p12_keyfile(
config.EE_ACCOUNT,
config.EE_PRIVATE_KEY_FILE,
scopes=oauth.SCOPES)
ee.Initialize(credentials, config.EE_URL)
except Exception as e:
print("EE init() error (with config.py and .pem file)",
e,
file=sys.stderr)
else:
args["importedDEM"] = abspath(args["importedDEM"])
# Give all config values to get_zipped_tiles for processing:
totalsize, full_zip_file_name = TouchTerrain.get_zipped_tiles(
**args) # all args are in a dict
print("\nCreated zip file", full_zip_file_name, "%.2f" % totalsize, "Mb")
# Optional: unzip the zip file into the current folder
if 1: # set this to 0 if you don't want the zip file to be unzippeed
#import os.path
#folder, file = os.path.splitext(full_zip_file_name) # tmp folder
folder = os.getcwd() + os.sep + args[
"zip_file_name"] # new stl folder in current folder
# unzip the zipfile into the folder it's already in
import zipfile
zip_ref = zipfile.ZipFile(full_zip_file_name, 'r')
zip_ref.extractall(folder)
zip_ref.close()
print("unzipped file inside", full_zip_file_name, "into", folder)
'''