forked from stuporglue/solar_scripts
-
Notifications
You must be signed in to change notification settings - Fork 0
/
makeDems.py
executable file
·173 lines (145 loc) · 6.53 KB
/
makeDems.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Continue fetching fishnet tiles that haven't been processed yet
# Fetch the list of lidar files for those tiles and process them
# DEM states
# -3 -- errors encountered
# -2 -- areas calculated to have no tiles within 100m
# -1 -- square has no intersecting lidar bboxes within 100m
# 0 -- available to process
# 1 -- reserved, but not complete
# 2 -- completed
import dbconn,sys,os,subprocess,re,tempfile,time
from config import *
buffersize = config.get('buffers','dem_selection_buffer')
basedir = config.get('paths','las_dir')
outputdir = config.get('paths','dem_output_dir')
# Radiate outwards from Blegen Hall
reserveQuery = """
UPDATE """ + config.get('postgres','schema') + "." + config.get('postgres','dem_fishnet_table') + """ dem
SET state=1
WHERE dem.id in (
SELECT id FROM """ + config.get('postgres','schema') + "." + config.get('postgres','dem_fishnet_table') + """ WHERE state=0
ORDER BY ST_Distance(the_geom,ST_SetSrid(ST_MakePoint(""" + config.get('processing','starting_x') + """,""" + config.get('processing','starting_y') + """),""" + config.get('projection','srid') + """))
LIMIT 1
)
RETURNING
id,
ST_XMin(the_geom) as xmin,
ST_YMin(the_geom) as ymin,
ST_XMax(the_geom) as xmax,
ST_YMax(the_geom) as ymax
"""
lidarlist = """
SELECT bbox.* FROM
""" + config.get('postgres','schema') + "." + config.get('postgres','dem_fishnet_table') + """ dem,
lidar_bbox bbox
WHERE dem.id=DEMID
AND
ST_Intersects(ST_Buffer(dem.the_geom,""" + str(buffersize) + """),bbox.the_geom)
"""
completeQuery = """
UPDATE """ + config.get('postgres','schema') + "." + config.get('postgres','dem_fishnet_table') + """ dem,
SET state=NEWSTATE
WHERE
dem.id=DEMID
"""
# demid is the database ID of the dem fishnet square we're working on
# lidarlist is a text file with a list of lidar files to use. This is needed because the command line gets too long for Powershell or blast2dem (not sure which)
# line is (xmin,ymin,xmax,ymax) for the output area
# buffersize is the buffer to apply for consideration
# outputdir is the directory where the files should be saved
def blast2dem(demid,lidarlist,line,buffersize,outputdir):
outputfile = outputdir + '\\' + '_'.join(line) + '.img'
cmd = ['blast2dem']
# Input tiles
cmd.append('-lof ' + lidarlist)
# Processing parameters
cmd.append('-merged')
cmd.append('-step 1')
# Spatial Filtering
# This defines the buffered area used for calcultions
cmd.append('-inside ' + str(int(line[0]) - buffersize) + ' ' + str(int(line[1]) - buffersize) + ' ' + str(int(line[2]) + buffersize) + ' ' + str(int(line[3]) + buffersize))
# This defines the output lower-left corner
cmd.append('-ll ' + line[0] + ' ' + line[1])
# This defines the output tile's height and width
cmd.append('-ncols ' + str(int(line[2]) - int(line[0])))
cmd.append('-nrows ' + str(int(line[3]) - int(line[1])))
# Data Filtering
cmd.append(config.get('blast2dem','additional_parameters'))
# Output parameters
cmd.append('-v')
cmd.append('-oimg')
filename = str(demid) + '.img'
cmd.append('-o ' + filename)
cmd.append('-odir ' + outputdir)
command = ' '.join(cmd)
# Check output
try:
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
output,error = process.communicate()
returncode = process.poll()
except:
e = sys.exc_info()[0]
sys.stdout.write("\t\t\t" + str(e))
print "\n\t\t" + command
if os.path.isfile(outputdir + "\\" + filename):
os.unlink(outputdir + "\\" + filename)
return False
# Print errors
if error != None:
sys.stdout.write("\t\t\t" + error)
print "\n\t\t" + command
if os.path.isfile(outputdir + "\\" + filename):
os.unlink(outputdir + "\\" + filename)
return False
if returncode != 0:
sys.stdout.write("\t\t\t" + output)
print "\n\t\t" + command
if os.path.isfile(outputdir + "\\" + filename):
os.unlink(outputdir + "\\" + filename)
return False
if not os.path.isfile(outputdir + "\\" + filename):
sys.stdout.write("\t\t\t" + output)
sys.stdout.write("\t\t\tExpected to find output file " + filename + ", but didn't")
print "\n\t\t" + command
return False
# Remove empty files. Will happen where fishnet is off the map
# 750703 -- 748kb files when they're solid black (also no results)
if re.match('.*bounding box. skipping.*',output,re.DOTALL) or int(os.stat(outputdir + "\\" + filename).st_size) == 750703:
sys.stdout.write("\t\t\tNo data found, not saving tile.")
os.unlink(outputdir + "\\" + filename)
return True
return True
res = dbconn.run_query(reserveQuery).fetchall()
count = 0
average = 0;
while len(res) > 0:
for row in res:
count += 1
sys.stdout.write("Running blast2dem for row " + str(row['id']) + "\t\t\t")
starttime = time.time()
# The long lists of files was making the command too long for PowerShell to handle
# so instead we write the list of file names to a temp file and delete the file
# when we're done
tmp = tempfile.NamedTemporaryFile(delete=False,dir=config.get('paths','temp_dir'))
lidares = dbconn.run_query(lidarlist.replace("DEMID",str(row['id']))).fetchall()
for lidar in lidares:
tmp.write(basedir + '\\' + lidar['lasfile'] + "\n")
tmp.close()
try:
blasted = blast2dem(demid=row['id'],lidarlist=tmp.name,line=[str(int(row['xmin'])),str(int(row['ymin'])),str(int(row['xmax'])),str(int(row['ymax']))],buffersize=buffersize,outputdir=outputdir)
except:
e = sys.exc_info()[0]
print "\t\t\t" + str(e)
blasted = False
stoptime = time.time()
average = (average * (count - 1) + stoptime - starttime) / count
if blasted:
print "DONE! (" + str((stoptime - starttime)) + " seconds, running avg:" + str(average) + ")"
os.unlink(tmp.name)
dbconn.run_query(completeQuery.replace("DEMID",str(row['id'])).replace('NEWSTATE','2'))
else:
print "Error! (" + str((stoptime - starttime)) + " seconds, running avg:" + str(average) + ")"
dbconn.run_query(completeQuery.replace("DEMID",str(row['id'])).replace('NEWSTATE','-3'))
res = dbconn.run_query(reserveQuery).fetchall()