def main(args):
# Unpack args
inputList = args['inputList']
outputGDB = args['outGDB']
# Read input list into list. Assume all inputs exist.
# If they do not, FC should throw RuntimeError
# Can add skip later if we want to avoid error due to shp not existing
with open (inputList, 'r') as l:
inFcs = [x.strip('\r\n') for x in l.readlines()]
print "Adding {} feature classes to {}\n".format(len(inFcs), outputGDB)
for inFc in inFcs:
#zs.updateOutputGdb(outGdb, f)
fc = FeatureClass(inFc)
fc.addToFeatureClass(outputGDB)
"""
def main(args):
# Unpack args
stackType = args['stackType']
zonalType = args['zonalType']
bname = '{}__{}__ZonalStats'.format(zonalType, stackType)
outGdb = os.path.join(outDir, '{}.gdb'.format(bname))
globDir = glob.glob(os.path.join(inDir, '{}*gpkg'.format(bname)))
print "\nCreating {} from {} input files...\n".format(outGdb, len(globDir))
for f in globDir:
fc = FeatureClass(f)
#zs.updateOutputGdb(outGdb, f)
fc.addToFeatureClass(outGdb)
# Lastly, move the csv to its final directory
mvCsv = os.path.join(inDir, '{}.csv'.format(bname))
os.system('mv {} {}'.format(mvCsv, outDir))
def main(args):
# Unpack args, check inputs and set up vars
inH5 = args['input']
outGdb = args['outGdb']
cont = args['continent'] # eu or na (default na)
if cont != 'na' and cont != 'eu': # Check that continent is na or eu
raise RuntimeError("Continent must be 'na' or 'eu'")
if not inH5.endswith('.h5'):
sys.exit('Input file must have an .h5 extension')
if outGdb is not None:
if not outGdb.endswith('.gdb') and not outGdb.endswith('.gpkg') \
and not outGdb.endswith('.shp'):
sys.exit('Output GDB must have an .gdb, .gpkg, or .shp extension')
bname = os.path.basename(inH5).strip('.h5')
# Set output dir variables and make dirs
baseOutdir = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/ATL08/{}'.format(
cont)
outCsvDir = os.path.join(baseOutdir, 'flight_csvs')
outShpDir = os.path.join(baseOutdir, 'flight_shps')
outLogDir = os.path.join(
baseOutdir,
'flight_logs') # 5/15 - log file for each h5 file bc par processing
for d in [outCsvDir, outShpDir, outLogDir]:
os.system('mkdir -p {}'.format(d))
# Log output:
logFile = os.path.join(outLogDir,
'ATL08-h5_to_shp__{}__Log.txt'.format(bname))
print "See {} for log".format(logFile)
so = se = open(logFile, 'a', 0) # open our log file
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w',
0) # re-open stdout without buffering
os.dup2(so.fileno(), sys.stdout.fileno()
) # redirect stdout and stderr to the log file opened above
os.dup2(se.fileno(), sys.stderr.fileno())
print "BEGIN: {}".format(time.strftime("%m-%d-%y %I:%M:%S"))
print "Continent: {}".format(cont)
print "h5 File: {}\n".format(inH5)
outCsv = os.path.join(outCsvDir, '{}.csv'.format(bname))
outShp = os.path.join(outShpDir, '{}.shp'.format(bname))
# Check if output shp already exists
if os.path.isfile(outShp):
print "\n Output {} already exists".format(outShp)
return None
# 1. H5 file --> outCsv (Paul's extract code):
extractCode = '/att/home/mwooten3/code/icesat2/extract_atl08.py'
cmd = 'python {} -i {} -o {}'.format(extractCode, inH5, outCsvDir)
os.system(cmd)
# Check if ICESAT2GRID (extract) failed
if not os.path.isfile(outCsv):
print "\n Output {} was not created".format(outCsv)
return None
# 2. Import csv into pandas df and extract lat/lon columns into arrays
pdf = pd.read_csv(outCsv)
latArr = np.asarray(pdf['lat'])
lonArr = np.asarray(pdf['lon'])
# calculategrounddirection() fails if there is only one footprint in csv.
# Skip if only one footprint:
if len(latArr) <= 1:
print "\n CSV {} has only one entry. Skipping".format(outCsv)
return None
# 3. Convert lat/lon lists to appropriate UTM zone
epsg = getUTM(np.min(lonArr), np.max(latArr), np.max(lonArr),
np.min(latArr))
utmLonList, utmLatList = latLonToUtmLists(lonArr, latArr, epsg)
# Add more information to attributes/pandas df
addAttributesToDf(pdf, utmLonList, utmLatList, epsg, bname)
# 4. Run Eric's functions to get polygon shp - 5/27 using 11m
createShapefiles(utmLonList, utmLatList, 11, 100, int(epsg), pdf, outShp)
# Get number of features from shp and add to csv
trackCsv = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/ATL08/ATL08_{}_v3__featureCount.csv'.format(
cont)
outFc = FeatureClass(outShp)
with open(trackCsv, 'a') as c:
c.write('{},{},{}\n'.format(inH5, platform.node(), outFc.nFeatures))
# If output is specified, update the output .gdb (or .gpkg?)
if outGdb is not None:
outFc.addToFeatureClass(outGdb)
print "\nEND: {}\n".format(time.strftime("%m-%d-%y %I:%M:%S"))
return outShp
def main(args):
# Unpack arguments
listRange = args['range']
runPar = args['parallel']
cont = args['continent'] # eu or na (default na)
# Set variables that should more or less stay the same (but depend on input)
fileList = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/ATL08/ls_ATL08_{}_v3.txt'.format(
cont)
flightShpDir = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/ATL08/{}/flight_shps'.format(
cont)
outGdb = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/ATL08/{}/ATL08_{}_v3__{}.gdb'.format(
cont, cont, platform.node())
print "\nBEGIN: {}".format(time.strftime("%m-%d-%y %I:%M:%S"))
with open(fileList, 'r') as l:
csvFiles = [x.strip('\r\n') for x in l.readlines()]
# Check inputs:
# 1. Check that continent is na or eu
if cont != 'na' and cont != 'eu':
raise RuntimeError("Continent must be 'na' or 'eu'")
# 2. Check that range is supplied correctly
try:
listRange = listRange.split('-')
S, E = listRange
except:
raise RuntimeError("Range must be supplied like: 1-20")
# Get list using range
csvFiles = csvFiles[int(S) - 1:int(E)]
# NOTE 1/15/21: List of .shp that correspond to the input files in range is
# only needed if running in parallel. I think the below block gets us what
# we need, but since I am not running parallel I haven't tested this
# Get list of output shapefiles using input .csv files
shps = [
os.path.join(flightShpDir,
os.path.basename(i).replace('.csv', '.shp'))
for i in csvFiles
]
""" Exploring possible duplicates issue
import collection
print [item for item, count in collections.Counter(h5Files).items() if count > 1]
sys.exit()
"""
# Run in parallel
if runPar:
# Prepare inputs for parallel call:
call = "lscpu | awk '/^Socket.s.:/ {sockets=$NF} END {print sockets}'"
ncpu = os.popen(call).read().strip()
ncpu = int(ncpu) - 1 # all CPUs minus 1
parList = ' '.join(csvFiles)
print "\nProcessing {} .csv files in parallel...\n".format(
len(csvFiles))
# Do not supply output GDB, do supply continent
parCall = '{} -i '.format(runScript) + '{1} -continent {2}'
cmd = "parallel --progress -j {} --delay 1 '{}' ::: {} ::: {}".format(
ncpu, parCall, parList, cont)
os.system(cmd)
# And update node-specific GDB
print "\n\nCreating {} with completed shapefiles ({})...".format(
outGdb, time.strftime("%m-%d-%y %I:%M:%S"))
for shp in shps:
if os.path.isfile(shp):
fc = FeatureClass(shp)
fc.addToFeatureClass(outGdb)
# Do not run in parallel
else:
c = 0
for csv in csvFiles:
c += 1
print "\nProcessing {} of {}...".format(c, len(csvFiles))
# Call script one at a time and supply node-specific output GDB and continent
cmd = 'python {} -i {} -gdb {} -continent {}'.format(
runScript, csv, outGdb, cont)
#print '', cmd
os.system(cmd)
#o = os.popen(call).read()
print "\nEND: {}\n".format(time.strftime("%m-%d-%y %I:%M:%S"))
def applyNoDataMask(self, mask, transEpsg = None, outShp = None):
# Expecting mask to be 0 and 1, with 1 where we want to remove data
# This is specific to 3DSI and therefore is not kept in FeatureClass
# if transformEpsg is supplied, convert points to correct SRS before running ZS
# if not supplied, will assume projection of mask and ZFC are the same
# Get name output shp:
if not outShp:
outShp = self.filePath.replace(self.extension, '__filtered-ND.shp')
drv = ogr.GetDriverByName("ESRI Shapefile")
ds = drv.Open(self.filePath, 1)
layer = ds.GetLayer()
# This will work even if not needed. If needed and not supplied, could fail
outSrs = osr.SpatialReference()
if transEpsg:
outSrs.ImportFromEPSG(int(transEpsg))
else:
outSrs.ImportFromEPSG(int(self.epsg())) # If transformation EPSG not supplied, keep coords as is
# 6/11 New filtering method - Add column to for rows we want to keep
if 'keep' not in self.fieldNames():
fldDef = ogr.FieldDefn('keep', ogr.OFTString)
layer.CreateField(fldDef)
# 10/28: If mask has coarse resolution, use allTouched = True
allTouched = False
if Raster(mask).resolution()[0] >= 30:
allTouched = True
# 6/11 - just count keep features, do no need FIDs
#keepFIDs = []
keepFeat = 0
for feature in layer:
# Get polygon geometry and transform to outSrs just in case
geom = feature.GetGeometryRef()
geom.TransformTo(outSrs)
# Then export to WKT for ZS
wktPoly = geom.ExportToIsoWkt()
# Get info from mask underneath feature
z = zonal_stats(wktPoly, mask, stats="mean", all_touched = allTouched)
out = z[0]['mean']
if out >= 0.99 or out == None: # If 99% of pixels or more are NoData, skip
feature.SetField('keep', 'no')
continue
# 6/11 - Else, set the new keep column to yes to filter later
feature.SetField('keep', 'yes')
layer.SetFeature(feature)
#keepFIDs.append(feature.GetFID())
keepFeat += 1
# 6/11 - No longer doing filtering this way
"""
#if len(keepFIDs) == 0: # If there are no points remaining, return None
#return None
if len(keepFIDs) == 1: # tuple(listWithOneItem) wont work in Set Filter
query = "FID = {}".format(keepFIDs[0])
else: # If we have more than 1 item, call getFidQuery
query = self.getFidQuery(keepFIDs)
"""
# 6/11 New filtering method
query = "keep = 'yes'"
layer.SetAttributeFilter(query)
dsOut = drv.CreateDataSource(outShp)
layerOutName = os.path.basename(outShp).replace('.shp', '')
layerOut = dsOut.CopyLayer(layer, layerOutName)
if not layerOut: # If CopyLayer failed for whatever reason
print("Could not remove NoData polygons")
return self.filePath
ds = layer = dsOut = layerOut = feature = None
# 10/28: Try to remove 'keep' field - 4/26/21 - comment out to keep consistent with NA outputs - 6/17 uncomment out again bc won't be consistent anyways
fc = FeatureClass(outShp)
fc.removeField('keep')
return outShp
def main(args):
startTime = datetime.datetime.now()
# Arguments: Input GLAS .csv; output directory; buffer distance (m)
inCsv = args['inputCsv']
outGdb = args['outputGdb']
bufferDist = args['bufferDistance']
overwrite = True # Set to False if we want to append to gdb/gpkg
node = platform.node()
"""
inCsv = '/att/gpfsfs/briskfs01/ppl/pmontesa/userfs02/data/glas/circ_boreal/gla01-boreal50up-fix2-data.csv'
outGdb = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/GLAS/GLAS_boreal.gdb'
bufferDist = 15
"""
# Get the header. Chaotic, but the column names for the fix2-data .csv are here
hdrFile = '/att/nobackup/mwooten3/3DSI/GLAS/GLAS_csv_columns.txt'
with open(hdrFile, 'r') as hf:
hdr = hf.readline().strip('\n').split(',')
# To save rows with bad lat/lon values to csv:
#badCoordCsv = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/GLAS/badCoords.csv'
# To save info about batches (already made header in file)
batchCsv = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/GLAS/batchTrack.csv'
# Hardcode the additional columns we want to add to .gdb
# Doing uniqueID separate because we want it to be first
extraColumns = ['shotYear', 'shotMonth', 'shotDay', 'shotJD', 'bufferSize']
# Get basename (aka eventual layer name) of output .gdb
ext = os.path.splitext(outGdb)[1]
bname = os.path.basename(outGdb).strip(ext)
# Get the output .gpkg because that's what we are writing to
if ext == '.gdb':
outGpkg = outGdb.replace('.gdb', '__{}.gpkg'.format(node))
convertToGdb = True
elif ext == '.gpkg':
outGpkg = outGdb.replace('.gpkg', '__{}.gpkg'.format(node))
convertToGdb = False
convertToGdb = False #TEMP - set to False until last batch
driver = ogr.GetDriverByName('GPKG')
# Create the output if overwrite is True or file does not exist
if overwrite or not os.path.isfile(outGpkg):
print "\nCreating {}...".format(outGpkg)
# Set up the output gdb
ds = driver.CreateDataSource(outGpkg)
srs = ogr.osr.SpatialReference()
srs.ImportFromEPSG(4326) # WGS84 Lat/Lon
# Create file with srs 4326; layerName = basename
layer = ds.CreateLayer(bname, srs, ogr.wkbPolygon)
# Create the fields in the output .gdb using hdr contents and addl columns
createFields(layer, hdr + extraColumns)
# Open the layer if not overwriting
else:
print "\nOpening and appending to {}...".format(outGpkg)
ds = driver.Open(outGpkg)
layer = ds.GetLayer()
# 8/14 Now that we are hardcoding batches for crane nodes, get that info
cntStart = int(countDict[node].split(',')[0])
cntEnd = int(countDict[node].split(',')[1])
# Iterate through rows in .csv and write buffered polygons to output
totCnt = 0 # count all of the points, not just the unfiltered ones
badCnt = 0 # count of points with missing columns
fltrCnt = 0 # count of points that were filtered
cnt = 0 # count of points added to gdb
badLon = [] # list of longitudes that are < or > 180
#import pdb; pdb.set_trace()
with open(inCsv, 'r') as c:
data = csv.reader(c)
for row in data:
totCnt += 1
# split into batch depending on crane node
if totCnt < cntStart or totCnt >= cntEnd:
continue
# Print count at every half millionth point (total not just unfiltered)
if totCnt % 50000 == 0: # count ever 5,000 row now
print " Total count so far = {}".format(totCnt)
print " (Added count = {})".format(cnt)
#layer.SyncToDisk() # Hopefully save info from memory to gpkg - dont think this worked
# Sometimes a row has fewer columns than the header. Skip those
if len(row) != len(hdr):
badCnt += 1
continue
# Create dict that corresponds to the hdr with contents in row:
rowDict = OrderedDict() # keep column order
for c in hdr:
rowDict[c] = row[hdr.index(c)]
# Some rows have a too large/small longitude. Record info in list
if float(rowDict['lon']) > 180 or float(rowDict['lon']) < -180:
badLon.append('{}: {}'.format(rowDict['lon'], totCnt))
rowDict['totCnt'] = str(totCnt)
#rowDictToCsv(rowDict, badCoordCsv) # save to .csv
continue # and skip
# Check for lat too just in case
if float(rowDict['lat']) > 90 or float(rowDict['lat']) < -90:
badLon.append('{}: {}'.format(rowDict['lat'], totCnt))
rowDict['totCnt'] = str(totCnt)
#rowDictToCsv(rowDict, badCoordCsv)
continue # and skip
# Filter the row. Will return False if we want to keep the point
if filterRow(rowDict):
fltrCnt += 1
continue # if True, skip point
rowDict['uniqueID'] = cnt + 1 # Add unique ID to dict
rowDict['bufferSize'] = bufferDist
rowDict = addShotDate(rowDict)
# Now send the row to function to buffer and add to output
addBufferToOutput(rowDict, layer, bufferDist)
cnt += 1 # only add to count if point is added
nFeat1 = FeatureClass(outGpkg).nFeatures
# Try destroying the ds to free up memory and save:
ds.Destroy()
endTime1 = datetime.datetime.now()
elapsedTime1 = endTime1 - startTime
# Record info:
# node,start,end,totCnt,addCnt,nFeat,missingCols,badCoords,filtered, time (minutes)
rec = '{},{},{},{},{},{},{},{},{},{}\n'.format(
node, cntStart, cntEnd, totCnt, cnt, nFeat1, badCnt, len(badLon),
fltrCnt, *divmod(elapsedTime1.total_seconds(), 60))
with open(batchCsv, 'a') as bc:
bc.write(rec)
print "\nFinished creating .gpkg ({} features) in ~{} minutes". \
format(nFeat1, *divmod(elapsedTime1.total_seconds(), 60))
print "\nRow counts:"
print " Total rows = {}".format(totCnt)
print " Filtered rows = {}".format(fltrCnt)
print " Rows with missing columns = {}".format(badCnt)
print " Rows with longitude value too large = {}".format(len(badLon))
print " Added rows = {}".format(cnt)
if convertToGdb:
print "\nConverting output .gpkg to .gdb ({})".format(outGdb)
convertGpkgToGdb(outGpkg, outGdb)
nFeat2 = FeatureClass(outGdb).nFeatures
endTime2 = datetime.datetime.now()
elapsedTime2 = endTime2 - endTime1
print "\nFinished converting to .gdb ({} features) in ~{} minutes\n". \
format(nFeat2, *divmod(elapsedTime2.total_seconds(), 60))
return outGdb
return outGpkg
def main(args):
# Unpack and validate arguments
stackType, zonalType, stackRange, runPar = unpackValidateArgs(args)
# Get varsDict --> {inList; inZonal; outCsv}
varsDict = getVarsDict(stackType, zonalType)
# Get list of stacks to iterate
stackList = getStackList(varsDict['inList'], stackRange)
# Get node-specific output .gdb
outGdb = varsDict['outCsv'].replace('.csv',
'-{}.gpkg'.format(platform.node()))
if runPar: # If running in parallel
# Get list of output shp's that we are expecting based off stackList
shps = [
os.path.join(
mainDir, zonalType, stackType,
RasterStack(stack).stackName,
'{}__{}__zonalStats.shp'.format(zonalType,
RasterStack(stack).stackName))
for stack in stackList
]
# Prepare inputs for parallel call:
call = "lscpu | awk '/^Socket.s.:/ {sockets=$NF} END {print sockets}'"
ncpu = os.popen(call).read().strip()
ncpu = int(ncpu) - 1 # all CPUs minus 1
parList = ' '.join(stackList)
print("\nProcessing {} stack files in parallel...\n".format(
len(stackList)))
# Do not supply output GDB, just supply .csv
parCall = '{} -rs '.format(runScript) + '{1} -z {2} -o {3} -log'
cmd = "parallel --progress -j {} --delay 1 '{}' ::: {} ::: {} ::: {}". \
format(ncpu, parCall, parList, varsDict['inZonal'],
varsDict['outCsv'])
os.system(cmd)
# And update node-specific GDB if shp exists
print("\n\nCreating {} with completed shapefiles ({})...".format(
outGdb, time.strftime("%m-%d-%y %I:%M:%S")))
for shp in shps:
if os.path.isfile(shp):
fc = FeatureClass(shp)
if fc.nFeatures > 0: fc.addToFeatureClass(outGdb)
# Do not run in parallel
else:
# Iterate through stacks and call
print("\nProcessing {} stacks...".format(len(stackList)))
c = 0
for stack in stackList:
c += 1
print("\n{}/{}:".format(c, len(stackList)))
# Check stack's output csv's, and skip if it exists and overwrite is False
rs = RasterStack(stack)
check = os.path.join(
mainDir, zonalType, stackType, rs.stackName,
'{}__{}__zonalStats.csv'.format(zonalType, rs.stackName))
# not actually needed
#if region == 'EU':
#check = check.replace('{}__'.format(zonalType), '{}-EU__'.format(zonalType))
if not overwrite:
if os.path.isfile(check):
print("\nOutputs for {} already exist\n".format(
rs.stackName))
continue
# Not running in parallel, send the node-specific ouput .gdb and both should get written
cmd = 'python {} -rs {} -z {} -o {} -log'.format(runScript, stack, \
varsDict['inZonal'], outGdb)
print(cmd)
os.system(cmd)
def main(args):
# Unpack args, check inputs and set up vars
inCsv = args['input']
outGdb = args['outGdb']
cont = args['continent'] # eu or na (default na)
if cont != 'na' and cont != 'eu': # Check that continent is na or eu
raise RuntimeError("Continent must be 'na' or 'eu'")
if not inCsv.endswith('.csv'):
sys.exit('Input file must have an .csv extension')
if outGdb is not None:
if not outGdb.endswith('.gdb') and not outGdb.endswith('.gpkg') \
and not outGdb.endswith('.shp'):
sys.exit('Output GDB must have an .gdb, .gpkg, or .shp extension')
bname = os.path.basename(inCsv).replace('.csv', '')
# Set output dir variables and make dirs
baseOutdir = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/ATL08/{}'.format(cont)
#outCsvDir = os.path.join(baseOutdir, 'flight_csvs')
outShpDir = os.path.join(baseOutdir, 'flight_shps')
outLogDir = os.path.join(baseOutdir, 'flight_logs')
trackDir = os.path.join(baseOutdir, 'tracking_csvs')
for d in [outShpDir, outLogDir, trackDir]:
os.system('mkdir -p {}'.format(d))
# Tracking .csv:
trackCsv = os.path.join(trackDir, 'ATL08_{}_v3__featureCount-{}.csv'.format(cont, platform.node()))
# Log output:
logFile = os.path.join(outLogDir, 'ATL08-csv_to_shp__{}__Log.txt'.format(bname))
print "See {} for log".format(logFile)
so = se = open(logFile, 'a', 0) # open our log file
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) # re-open stdout without buffering
os.dup2(so.fileno(), sys.stdout.fileno()) # redirect stdout and stderr to the log file opened above
os.dup2(se.fileno(), sys.stderr.fileno())
print "BEGIN: {}".format(time.strftime("%m-%d-%y %I:%M:%S"))
print "Continent: {}".format(cont)
print "Node: {}".format(platform.node())
print ".csv File: {}".format(inCsv)
print ".gdb/.gpkg: {}\n".format(outGdb)
#outCsv = os.path.join(outCsvDir, '{}.csv'.format(bname))
outShp = os.path.join(outShpDir, '{}.shp'.format(bname))
# Check if output shp already exists
if os.path.isfile(outShp):
print "\n Output {} already exists".format(outShp)
return None
"""
# Skipping .h5 file portion now
# 1. H5 file --> outCsv (Paul's extract code):
extractCode = '/att/home/mwooten3/code/icesat2/extract_atl08.py'
cmd = 'python {} -i {} -o {}'.format(extractCode, inH5, outCsvDir)
os.system(cmd)
# Check if ICESAT2GRID (extract) failed
if not os.path.isfile(outCsv):
print "\n Output {} was not created".format(outCsv)
return None
"""
# 1. Import csv into pandas df and extract lat/lon columns into arrays
pdf = pd.read_csv(inCsv)
latArr = np.asarray(pdf['lat'])
lonArr = np.asarray(pdf['lon'])
nInputRows = len(pdf)
# calculategrounddirection() fails if there is only one footprint in csv.
# Skip if only one footprint:
if len(latArr) <= 1:
print "\n CSV {} has fewer than two rows. Skipping".format(inCsv)
return None
# 2. Convert lat/lon lists to appropriate UTM zone
epsg = getUTM(np.min(lonArr), np.max(latArr), np.max(lonArr), np.min(latArr))
utmLonList, utmLatList = latLonToUtmLists(lonArr, latArr, epsg)
# 3. Add more information to attributes/pandas df
addAttributesToDf(pdf, utmLonList, utmLatList, epsg, bname)
# 4. Remove NoData rows (h_can = 3.402823e+23)
pdf, nFiltered = filterRows(pdf)
# Cannot continue if at least two points do not exist
if len(pdf) <= 1:
print "\n CSV {} has fewer than two rows after filtering. Skipping".format(inCsv)
return None
# 5. Edit some columns
# 1/19: Added this to fix the columns that were encoded improperly and have the b'...' issue
pdf = fixColumns(pdf)
# 6. Run Eric's functions to get polygon shp - 5/27 using 11m
# xx and yy no longer match the filterd dataframe. Recreating these
# lists in the function now
createShapefiles(11, 100, int(epsg), pdf, outShp)
# 7. Track info: .csv file, node, number of input .csv features, number of filtered features, number of output .shp features
outFc = FeatureClass(outShp)
with open(trackCsv, 'a') as c:
c.write('{},{},{},{},{}\n'.format(inCsv, platform.node(), nInputRows, nFiltered, outFc.nFeatures))
# 8. If output is specified, update the output .gdb (or .gpkg?)
if outGdb is not None:
outFc.addToFeatureClass(outGdb)
outFc = None # Close feature class
print "\nEND: {}\n".format(time.strftime("%m-%d-%y %I:%M:%S"))
return outShp