def create_lasdataset(laz, las, lasd):
# unzip laz files and write to appropriate sub directories
# create lasdataset (ESRI format) using arcpy
print('Creating LAS dataset')
os.chdir(laz)
subprocess.call("T:/GIS/Projects/LiDAR/Project/tools/laszip *.laz")
print('Moving las files to las directory')
call = "move *.las C:\Projects\\3DBuilding\data\lidar\las"
subprocess.call(call)
print('Creating LAS Dataset')
arcpy.CreateLasDataset_management(las, lasd)
print('Finished!')
def pull_OpenTopo(lidar_path, OpenTopo):
lasd = os.path.join(lidar_path, "lidar.lasd")
lasfiles = []
#Define the root url for the Lidar website we're going to scrape from
#Open and read out Lidar Website Html
htmlfile_lev1 = u.urlopen(OpenTopo)
htmltext_lev1 = htmlfile_lev1.read()
#Split html string output from page to prepare for download URL extrapolation (via Regex)
htmltext_split = htmltext_lev1.split('<td class="colname">')
#Pull out and download URL's per lidar element (file)
num_files = 0
for item in htmltext_split:
regex_lid = '<a href="(.+?)">.*</a>'
pattern_lid = re.compile(regex_lid)
lidar_URL = re.findall(pattern_lid, item)
if lidar_URL and lidar_URL[0] != "../":
num_files += 1
arcpy.AddMessage(str(num_files) + " files to be downloaded.")
i = 0
for item in htmltext_split:
regex_lid = '<a href="(.+?)">.*</a>'
pattern_lid = re.compile(regex_lid)
lidar_URL = re.findall(pattern_lid, item)
#Take download URL and execute - note that the Lidar files will save in the same folder as this python script
if lidar_URL and lidar_URL[0] != "../":
newURL = OpenTopo + lidar_URL[0]
#print newURL
laz = lidar_URL[0][:-4] + ".laz"
las = lidar_URL[0][:-4] + ".las"
u.urlretrieve(newURL, laz)
# Unzip LAZ
args = "E:\\TFS_Fire\\fire-tools\\laszip-cli.exe -i " + laz + " -o " + las
subprocess.call(args)
i += 1
las = os.path.join(LAS_path, las)
lasfiles.extend([las])
arcpy.AddMessage(str(i) + " of " + str(num_files))
arcpy.CreateLasDataset_management(lasfiles, lasd)
def las_to_GandFrreturns(las_files, mask, name):
'''This function takes a list of las files, creates a las dataset. Then
it create a lasd for first and ground returns. After which it rasterizes
the lasd and masks them to a specified area.
las_files: a list of the paths for all the las files
mask:
'''
lasd = arcpy.CreateLasDataset_management(las_files, name)
g_lasd = arcpy.MakeLasDatasetLayer_management(lasd, "ground", 2)
g_ras = arcpy.LasDatasetToRaster_conversion(
g_lasd, "g_rast.img", "ELEVATION", "BINNING AVERAGE NATURAL_NEIGHBOR",
"FLOAT", "CELLSIZE", 0.5)
g_mask = arcpy.sa.ExtractByMask(g_ras, mask)
g_mask.save("g_mask" + name + ".img")
f_lasd = arcpy.MakeLasDatasetLayer_management(lasd, "first", '', 1)
f_ras = arcpy.LasDatasetToRaster_conversion(
f_lasd, "f_rast.img", "ELEVATION", "BINNING MAXIMUM NATURAL_NEIGHBOR",
"FLOAT", "CELLSIZE", 0.5)
f_mask = arcpy.sa.ExtractByMask(f_ras, mask, name)
f_mask.save("f_mask" + name + ".img")
return f_mask, g_mask
raise Exception(
'Riparian buffer width required if stream network is specified')
######################
progress = 0
support_folder = os.path.join(out_folder, 'support')
os.mkdir(out_folder)
os.mkdir(support_folder)
arcpy.AddMessage("Generating footprint and .lasd")
# make lasd
inlasd = os.path.join(support_folder, 'support.lasd')
arcpy.CreateLasDataset_management(inlas, inlasd)
# make footprint. if a clipping file was specified, the footprint will be of the extracted las
footprint_name = os.path.join(support_folder, 'las_footprint.shp')
files = [f for f in listdir(inlas) if isfile(join(inlas, f))]
spatial_ref = arcpy.Describe(os.path.join(inlas, files[0])).spatialReference
if files[0][-4:] == '.las':
file_type = '.las'
elif files[0][-5:] == '.zlas':
file_type = '.zlas'
else:
raise Exception('Input files must be las or zlas')
cell_edge_length = 1 / spatial_ref.metersPerUnit
output_gdb_path = "E:\\Caitlin\\ROSETTA\\ROSETTA Database\\ROSETTA.gdb\\"
# las_array is a list of lists, where each inner list represents one raster
# las_array = [["output filename", "input file path", "input file path", ...],
# ["output filename," "input file path", "input file path", ...]
# ... ... ]
las_array = [[
"AN03_F1004_1",
"X:\\trunk\\icepod\\antarctica\\20162017\\lidar\\vq580\\pointcloud\\F1004\\las\\full_swath\AN03_F1004_20161127_204658_LIDAR_VQ580_Record1.las"
]]
#######################################################
for las_list in las_array:
name = las_list[0]
path_list = las_list[1:]
arcpy.CreateLasDataset_management(path_list,
lasd_output_path + name + ".lasd",
"NO_RECURSION", "",
arcpy.SpatialReference(4326),
"COMPUTE_STATS", "RELATIVE_PATHS",
"NO_FILES")
arcpy.LasDatasetToRaster_conversion(lasd_output_path + name + ".lasd",
output_gdb_path + name, "ELEVATION",
'BINNING AVERAGE SIMPLE', 'FLOAT',
'CELLSIZE', 0.00001, 1)
#arcpy.RasterToOtherFormat_conversion(output_gdb_path + name, raster_output_path, {"TIFF"})
def processJob(ProjectJob,
project,
createQARasters=False,
createMissingRasters=True,
overrideBorderPath=None):
aaa = datetime.now()
a = aaa
lasd_boundary = None
ProjectFolder = ProjectFolders.getProjectFolderFromDBRow(
ProjectJob, project)
ProjectID = ProjectJob.getProjectID(project)
ProjectUID = ProjectJob.getUID(project)
target_path = ProjectFolder.derived.path
# Get the LAS QA Info to determine if it is classified or not
las_qainfo = getLasQAInfo(ProjectFolder)
if las_qainfo.num_las_files <= 0:
arcpy.AddError(
"Project with Job ID {} has no .las files in DELIVERED LAS_CLASSIFIED or LAS_UNCLASSIFIED folders, CANNOT CONTINUE."
.format(ProjectFolder.projectId))
else:
ProjectFolders.createAnalysisFolders(target_path,
las_qainfo.isClassified)
# Make the STAT folder if it doesn't already exist
stat_out_folder = ProjectFolder.derived.stats_path
if not os.path.exists(stat_out_folder):
os.makedirs(stat_out_folder)
arcpy.AddMessage(
"created Derived STAT folder '{}'".format(stat_out_folder))
else:
arcpy.AddMessage("STAT folder '{}'".format(stat_out_folder))
# Make the scratch file GDB for the project
if not os.path.exists(las_qainfo.filegdb_path):
arcpy.CreateFileGDB_management(target_path,
las_qainfo.filegdb_name)
Utility.addToolMessages()
else:
arcpy.AddMessage(
"Derived fGDB sand box already exists. Using '{}'".format(
las_qainfo.filegdb_path))
las_qainfo.lasd_spatial_ref = checkSpatialOnLas(
las_qainfo.las_directory, target_path, createQARasters,
las_qainfo.isClassified)
if las_qainfo.lasd_spatial_ref is None:
arcpy.AddError(
"ERROR: Neither spatial reference in PRJ or LAS files are valid CANNOT CONTINUE."
)
arcpy.AddError(
"ERROR: Please create a projection file (.prj) in the LAS folder using the '3D Analyst Tools/Conversion/From File/Point File Information' tool."
)
elif not las_qainfo.isValidSpatialReference():
las_qainfo.lasd_spatial_ref = None
arcpy.AddError(
"ERROR: Spatial Reference for the las files is not standard (see above)"
)
arcpy.AddError(
"ERROR: Please create a projection file (.prj) in the LAS folder using the '3D Analyst Tools/Conversion/From File/Point File Information' tool."
)
try:
arcpy.AddError("ERROR: '{}'".format(
Utility.getSpatialReferenceInfo(
las_qainfo.lasd_spatial_ref)))
except:
pass
elif las_qainfo.isUnknownSpatialReference():
las_qainfo.lasd_spatial_ref = None
arcpy.AddError(
"ERROR: Please provide a projection file (.prj) that provides a valid transformation in the LAS directory."
)
arcpy.AddError(
"ERROR: Please create a projection file (.prj) in the LAS folder using the '3D Analyst Tools/Conversion/From File/Point File Information' tool."
)
arcpy.AddError(
"ERROR: Spatial Reference for the las files is not standard")
try:
arcpy.AddError("ERROR: '{}'".format(
Utility.getSpatialReferenceInfo(
las_qainfo.lasd_spatial_ref)))
except:
pass
if las_qainfo.lasd_spatial_ref is None:
raise Exception(
"Error: Spatial Reference is invalid, unknown, or not specified."
)
else:
# prj_Count, prj_File = Utility.fileCounter(las_qainfo.las_directory, '.prj')
# if prj_Count > 0 and prj_File is not None and len(str(prj_File)) > 0:
# prj_spatial_ref = os.path.join(las_qainfo.las_directory, prj_File)
#
# if prj_Count > 0:
# las_qainfo.setProjectionFile(prj_File)
# las_spatial_ref = os.path.join(las_qainfo.las_directory, prj_File)
# arcpy.AddMessage("Found a projection file with the las files, OVERRIDE LAS SR (if set) '{}'".format(las_spatial_ref))
# arcpy.AddMessage(Utility.getSpatialReferenceInfo(las_qainfo.getSpatialReference()))
# else:
# arcpy.AddMessage("Using projection (coordinate system) from las files if available.")
fileList = getLasFileProcessList(las_qainfo.las_directory,
target_path, createQARasters,
las_qainfo.isClassified)
createLasStatistics(fileList, target_path,
las_qainfo.lasd_spatial_ref,
las_qainfo.isClassified, createQARasters,
createMissingRasters, overrideBorderPath)
# Create the project's las dataset. Don't do this before you validated that each .las file has a .lasx
if os.path.exists(las_qainfo.las_dataset_path):
arcpy.AddMessage("Using existing LAS Dataset {}".format(
las_qainfo.las_dataset_path))
# arcpy.AddMessage("Deleting existing LAS Dataset {}".format(las_qainfo.las_dataset_path))
# arcpy.Delete_management(las_qainfo.las_dataset_path)
else:
a = datetime.now()
# note: don't use method in A04_B because we don't want to compute statistics this time
arcpy.CreateLasDataset_management(
input=las_qainfo.las_directory,
out_las_dataset=las_qainfo.las_dataset_path,
folder_recursion="RECURSION",
in_surface_constraints="",
spatial_reference=las_qainfo.lasd_spatial_ref,
compute_stats="NO_COMPUTE_STATS",
relative_paths="RELATIVE_PATHS",
create_las_prj="FILES_MISSING_PROJECTION")
Utility.addToolMessages()
a = doTime(
a, "Created LAS Dataset '{}'".format(
las_qainfo.las_dataset_path))
desc = arcpy.Describe(las_qainfo.las_dataset_path)
# las_qainfo.lasd_spatial_ref = desc.SpatialReference
las_qainfo.LASDatasetPointCount = desc.pointCount
las_qainfo.LASDatasetFileCount = desc.fileCount
arcpy.AddMessage(
"LASDatasetPointCount {} and LASDatasetFileCount {}".format(
desc.pointCount, desc.fileCount))
lasd_boundary, las_footprint = A04_C_ConsolidateLASInfo.createRasterBoundaryAndFootprints(
las_qainfo.filegdb_path, target_path, ProjectID,
ProjectFolder.path, ProjectUID)
mxd = createMXD(las_qainfo, target_path, ProjectID)
# if createQARasters:
arcpy.AddMessage("Creating QA raster mosaics")
mosaics = A04_C_ConsolidateLASInfo.createQARasterMosaics(
las_qainfo.isClassified, las_qainfo.filegdb_path,
las_qainfo.lasd_spatial_ref, target_path, mxd, las_footprint,
lasd_boundary)
if mxd is not None:
a = datetime.now()
try:
mxd_path = mxd.filePath
for [md_path, md_name] in mosaics:
arcpy.AddMessage(
"Adding QA raster mosaic {} to mxd {}".format(
md_path, mxd_path))
try:
if not arcpy.Exists(md_path):
a = doTime(
a,
"\tMD doesn't exist {}. Can't add to MXD {}. Is it open?"
.format(md_path, mxd_path))
else:
df = mxd.activeDataFrame
if isLayerExist(mxd, df, md_name):
a = doTime(
a, "\t MD {} already exists in MXD {}".
format(md_name, mxd_path))
else:
if len(str(md_name)) > 0:
try:
lyr_md = arcpy.MakeMosaicLayer_management(
in_mosaic_dataset=md_path,
out_mosaic_layer=md_name
).getOutput(0)
df = mxd.activeDataFrame
arcpy.mapping.AddLayer(
df, lyr_md, 'BOTTOM')
# lyr_md.visible = False
mxd.save()
a = doTime(
a,
"\tAdded MD {} to MXD {} as {}"
.format(
md_name, mxd_path, lyr_md))
except:
a = doTime(
a,
"\tfailed to add MD {} to MXD {}. Is it open?"
.format(md_path, mxd_path))
except:
try:
a = doTime(
a,
"\tfailed to add MD to MXD {}. Is it open?"
.format(mxd_path))
except:
pass
mxd.save()
except:
try:
a = doTime(
a, "\tfailed to save MXD {}. Is it open?".format(
mxd_path))
except:
pass
bbb = datetime.now()
td = (bbb - aaa).total_seconds()
arcpy.AddMessage("Completed {} in {}".format(las_qainfo.las_dataset_path,
td))
return las_qainfo, lasd_boundary
def checkSpatialOnLas(start_dir, target_path, createQARasters, isClassified):
las_spatial_ref = None
prj_spatial_ref = None
las_f_path = getLasFileProcessList(start_dir,
target_path,
createQARasters,
isClassified,
returnFirst=True)
lasd_f_path = "{}d".format(las_f_path)
a = datetime.now()
deleteFileIfExists(lasd_f_path, True)
arcpy.AddMessage(
"{} Testing spatial reference on .las file: '{}' '{}'".format(
datetime.now(), las_f_path, lasd_f_path))
arcpy.CreateLasDataset_management(input=las_f_path,
spatial_reference=None,
out_las_dataset=lasd_f_path,
folder_recursion="NO_RECURSION",
in_surface_constraints="",
compute_stats="COMPUTE_STATS",
relative_paths="RELATIVE_PATHS",
create_las_prj="NO_FILES")
doTime(a, "\t{} Created LASD {}".format(datetime.now(), lasd_f_path))
desc = arcpy.Describe(lasd_f_path)
if desc is not None:
las_spatial_ref = desc.SpatialReference
if las_spatial_ref is not None:
try:
arcpy.AddMessage("\tFound spatial reference in LAS: {}".format(
las_spatial_ref.exportToString()))
except:
pass
prj_Count, prj_File = Utility.fileCounter(start_dir, '.prj')
arcpy.AddMessage("\tFound {} PRJ files, the first is: {}".format(
prj_Count, prj_File))
if prj_Count > 0 and prj_File is not None and len(str(prj_File)) > 0:
prj_Path = os.path.join(start_dir, prj_File)
arcpy.AddMessage(
"\tReading spatial reference from PRJ file: {}".format(prj_Path))
prj_spatial_ref = arcpy.SpatialReference(prj_Path)
arcpy.AddMessage("\tGot from PRJ file spatial reference: {}".format(
prj_spatial_ref.name))
if prj_spatial_ref is not None:
try:
arcpy.AddMessage("\tFound spatial reference in PRJ: {}".format(
prj_spatial_ref.exportToString()))
except:
pass
arcpy.AddMessage("Decoding LAS File Spatial Reference")
las_horz_cs_name, las_horz_cs_unit_name, las_horz_cs_factory_code, las_vert_cs_name, las_vert_cs_unit_name = Utility.getSRValues(
las_spatial_ref)
prj_horz_cs_name, prj_horz_cs_unit_name, prj_horz_cs_factory_code, prj_vert_cs_name, prj_vert_cs_unit_name = None, None, None, None, None
if prj_spatial_ref is not None:
arcpy.AddMessage("Decoding PRJ File Spatial Reference")
prj_horz_cs_name, prj_horz_cs_unit_name, prj_horz_cs_factory_code, prj_vert_cs_name, prj_vert_cs_unit_name = Utility.getSRValues(
prj_spatial_ref)
arcpy.AddMessage(
"LAS File Spatial Reference:\n\tH_Name: '{}'\n\tH_Unit: '{}'\n\tH_WKID: '{}'\n\tV_Name: '{}'\n\tV_Unit: '{}'"
.format(las_horz_cs_name, las_horz_cs_unit_name,
las_horz_cs_factory_code, las_vert_cs_name,
las_vert_cs_unit_name))
arcpy.AddMessage(
"PRJ File Spatial Reference:\n\tH_Name: '{}'\n\tH_Unit: '{}'\n\tH_WKID: '{}'\n\tV_Name: '{}'\n\tV_Unit: '{}'"
.format(prj_horz_cs_name, prj_horz_cs_unit_name,
prj_horz_cs_factory_code, prj_vert_cs_name,
prj_vert_cs_unit_name))
prj_horz_name_isValid = isSrValueValid(prj_horz_cs_name)
prj_vert_name_isValid = isSrValueValid(prj_vert_cs_name)
prj_horz_unit_isValid = isSrValueValid(prj_horz_cs_unit_name)
prj_vert_unit_isValid = isSrValueValid(prj_vert_cs_unit_name)
las_horz_name_isValid = isSrValueValid(las_horz_cs_name)
las_vert_name_isValid = isSrValueValid(las_vert_cs_name)
las_horz_unit_isValid = isSrValueValid(las_horz_cs_unit_name)
las_vert_unit_isValid = isSrValueValid(las_vert_cs_unit_name)
prj_isValid = prj_horz_name_isValid and prj_vert_name_isValid and prj_horz_unit_isValid and prj_vert_unit_isValid
las_isValid = las_horz_name_isValid and las_vert_name_isValid and las_horz_unit_isValid and las_vert_unit_isValid
sr_horz_name_isSame = prj_horz_name_isValid and las_horz_name_isValid and prj_horz_cs_name == las_horz_cs_name
sr_horz_unit_isSame = prj_horz_unit_isValid and las_horz_unit_isValid and prj_horz_cs_unit_name == las_horz_cs_unit_name
sr_vert_name_isSame = prj_vert_name_isValid and las_vert_name_isValid and prj_vert_cs_name == las_vert_cs_name
sr_vert_unit_isSame = prj_vert_unit_isValid and las_vert_unit_isValid and prj_vert_cs_unit_name == las_vert_cs_unit_name
sr_horz_isSame = sr_horz_name_isSame and sr_horz_unit_isSame
sr_vert_isSame = sr_vert_name_isSame and sr_vert_unit_isSame
sr_isSame = sr_horz_isSame and sr_vert_isSame
if prj_isValid or las_isValid:
if sr_horz_isSame:
arcpy.AddMessage(
" The LAS and PRJ horizontal spatial references MATCH".
format(Utility.getSpatialReferenceInfo(prj_spatial_ref)))
else:
arcpy.AddWarning(
"WARNING: The LAS and PRJ horizontal spatial references DO NOT MATCH."
)
if sr_vert_isSame:
arcpy.AddMessage(
" The LAS and PRJ vertical spatial references MATCH".
format(Utility.getSpatialReferenceInfo(prj_spatial_ref)))
else:
arcpy.AddWarning(
"WARNING: The LAS and PRJ vertical spatial references DO NOT MATCH."
)
if sr_isSame:
arcpy.AddMessage(
" The LAS and PRJ spatial references MATCH".format(
Utility.getSpatialReferenceInfo(prj_spatial_ref)))
else:
arcpy.AddWarning(
"WARNING: The LAS and PRJ spatial references DO NOT MATCH.")
result = None
if prj_isValid:
arcpy.AddMessage(
" Found a valid spatial reference in a PRJ file. Using this as the spatial reference: {}"
.format(Utility.getSpatialReferenceInfo(prj_spatial_ref)))
result = os.path.join(start_dir, prj_File)
elif las_isValid:
arcpy.AddMessage(
" Found a valid spatial reference in a LAS file. Using this as the spatial reference: {}"
.format(Utility.getSpatialReferenceInfo(las_spatial_ref)))
result = las_spatial_ref
return result
#DEM by tile
# get file names
#os.listdir(DOWNLOADS_DIR)
arcpy.env.workspace = DOWNLOADS_DIR
lasFiles = [
f for f in os.listdir(DOWNLOADS_DIR) if f.endswith(('.LAS', '.las'))
]
#lasFiles = lasFiles[0:10]
#f = lasFiles[0]
for f in lasFiles:
print("Processing %s..." % f)
ff = os.path.splitext(f)[0]
lasD = os.path.join(outLAS, 'tmp.lasd') #; print lasD
print(" Creating %s..." % lasD)
arcpy.CreateLasDataset_management(os.path.join(DOWNLOADS_DIR, f), lasD,
recursion, surfCons, sr)
print(" %s created!" % lasD)
#Create DEM
outDEMras = os.path.join(outDEM, ff + '_DEM.tif')
if not os.path.exists(outDEMras):
print("Begin %s processing..." % (ff + '_DEM.tif'))
lasDEM = arcpy.CreateUniqueName(ff + '_DEM')
returnValue = '' #[2,9]#'ANY'#['LAST', 'SINGLE']
class_code = [2, 9]
print(" Creating LAS layer...")
arcpy.MakeLasDatasetLayer_management(lasD, lasDEM, class_code)
print(" Creating surface...")
arcpy.conversion.LasDatasetToRaster(
lasDEM, outDEMras, 'ELEVATION', 'BINNING MINIMUM NATURAL_NEIGHBOR',
'FLOAT', 'CELLSIZE', cellSize, zFactor)
"", "*****", "SAVE_USERNAME", "", "",
"TRANSACTIONAL", "", "")
# Process: Create Table
arcpy.CreateTable_management(FIA_Data_sde, "FIA DATA", "", "")
# Process: Pivot Table
arcpy.PivotTable_management(FIA_DATA, "", "", "", FIADATA_PivotTable)
# Process: Add Join
arcpy.AddJoin_management("", "", FIADATA_PivotTable, "", "KEEP_ALL")
# Process: Create Feature Dataset
arcpy.CreateFeatureDataset_management("", "", "")
# Process: Exploratory Regression
arcpy.ExploratoryRegression_stats("", "", "", "", Output_Report_File,
Output_Results_Table, "5", "1", "0.5",
"0.05", "7.5", "0.1", "0.1")
# Process: Create LAS Dataset
arcpy.CreateLasDataset_management("", Output_LAS_Dataset, "NO_RECURSION", "",
"", "NO_COMPUTE_STATS", "ABSOLUTE_PATHS")
# Process: LAS Point Statistics as Raster
arcpy.LasPointStatsAsRaster_management(Output_LAS_Dataset, Output_Raster,
"PULSE_COUNT", "CELLSIZE", "10")
# Process: Extract by Points
arcpy.gp.ExtractByPoints_sa(Output_Raster, "", output, "INSIDE")