def mask_clouds(pix60in, bitmapout, identifier):
start_time = time.time()
polygonout_name_full = identifier + "_cloud_polygons_full.shp"
polygonout_full = os.path.join(maskdir,polygonout_name_full)
polygonout_name = identifier + "_cloud_polygons.shp"
polygonout = os.path.join(maskdir,polygonout_name)
id_string = "Cloud mask bitmap for file %s" % identifier
pcimod(file=pix60in, # Input 60m resolution atmospheric bands pix file
pciop='ADD', # Modification mode "Add"
pcival=[0, 0, 1, 0]) # Task - add one 16U channels
print "Classifying clouds from SWIR Cirrus band in file %s..." % identifier
kclus(file=pix60in, # Run classification atmospheric bands
dbic=[3], # Use Layer 3 (SWIR Cirrus)
dboc=[4], # Output to blank layer
numclus=[2], # Two clusters - clouds, not clouds
seedfile='',
maxiter=[20],
movethrs=[],
siggen="YES",
backval=[],
nsam=[])
print "Cloud classification complete."
print "Converting to polygon shapefile..."
ras2poly(fili=pix60in, # Use scratch PIX file
dbic=[4], # Use classification channel
filo=polygonout_full, # Polygon SHP output location
smoothv="YES", # Smooth boundaries
dbsd=id_string, # Layer description string
ftype="SHP", # Shapefile format
foptions="")
print "Shapefile conversion complete."
print "Converting polygons to bitmap layer..."
workspace = os.path.join(workingdir, "sable.gdb") # Define GDB workspace
polygonoutf_lyr = identifier + "_polygon_lyr" # Define feature layer name for polygon output
arcpy.env.workspace = workspace # Set default workspace
arcpy.MakeFeatureLayer_management(polygonout_full, polygonoutf_lyr) # Make polygon feature layer
arcpy.FeatureClassToFeatureClass_conversion(in_features=polygonoutf_lyr, # Extract only cloud polygons
out_path=maskdir, # Output location
out_name=polygonout_name, # Output filename
where_clause='"Area" > 1000000000') # Anything <1B SM not clouds
poly2bit(fili=polygonout, # Convert polygons to bitmap layer
dbvs=[1], # Input vector layer
filo=bitmapout, # Output file
dbsd=id_string, # Layer description
pixres=[10,10], # 10m resolution
ftype="PIX") # Pix format
pfull_dbf = polygonout_full[:-3] + "dbf"
pfull_prj = polygonout_full[:-3] + "prj"
pfull_pox = polygonout_full + ".pox"
pfull_shx = polygonout_full[:-3] + "shx"
os.remove(polygonout_full) # Clean up intermediate polygon file
os.remove(pfull_dbf)
os.remove(pfull_prj)
os.remove(pfull_pox)
os.remove(pfull_shx)
completion_time = time.time() - start_time # Calculate time to complete
print "Bitmap conversion completed in %i seconds. Output to \n\t%s" % (completion_time, bitmapout)
return bitmapout
def addChannels(fileInput):
try:
channelOperationType = 'ADD' # Channel operation type
channelsToADD = [0,0,3,0] #three channels of 16 bit each
pcimod(fileInput, channelOperationType, channelsToADD) #Add channels
except PCIException, e:
print e
def removePreviousResultChannels(fileInput,defNumChannels):
try:
with ds.open_dataset(fileInput, ds.eAM_READ) as pix:
# Get channel count
numberOfChannels = pix.chan_count
channelOperationType = 'DEL'
while numberOfChannels > defNumChannels: #While the number of channels in the image are greater than the default number of channels in Landsat 8
channelsToDelete = [numberOfChannels]
pcimod(fileInput,channelOperationType,channelsToDelete) #remove
numberOfChannels = numberOfChannels - 1
except Exception as e:
print e
print "Previous channels deleted"
def make_pca(merged_input, pca_out, identifier):
start_time = time.time()
print "Starting Principal Component Analysis for file %s" % identifier
pca_rep = os.path.join(pcadir, "PCA_" + identifier + "_report.txt")
fexport(fili=merged_input,
filo=pca_out,
dbic=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
ftype="PIX")
try:
Report.clear() # Clear report file
enableDefaultReport(pca_rep) # Change output folder location
pcimod(file=pca_out,
pciop="ADD",
pcival=[0, 0, 3]) # Add 3 16 bit unsigned channels
pca(file=pca_out,
dbic=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], # Use first ten bands
eign=[1, 2, 3], # Output first three eigenchannels
dboc=[11, 12, 13], # Output to 3 new channels
rtype="LONG") # Output extended report format
except PCIException, e:
print e
print("... Part 2: Clipping Scene Completed")
#-----------------------------------------------
# Part 3 : Add Raster Channels and Perform PCA
#-----------------------------------------------
print(" ...Part 3 : Adding Raster Channels and Performing PCA")
# find clipped scene and place in a list
input_pix = (makelist(rootdir = clip_scene, suffix = '.pix'))
# add raster channels to clipped scene
for image in input_pix:
try:
file = image
pciop = "ADD"
pcival = [6] # add 6 8-bit channels -->must match raw imagery
pcimod(file, pciop, pcival)
except PCIException as e:
print (e)
except Exception as e:
print (e)
print(" ...raster channels added ")
# report_dir has already been defined at the start
# rep_file = report_dir + "/" + "report.txt"
try:
Report.clear()
enableDefaultReport(rep_file)
pca(file= clip_scene + "/" + "scene_clipped.pix",
def land_cover(pixin, vout, rout, identifier, clouds):
start_time = time.time() # Start timer
print "Generating land cover classification..."
pct_string = "PCT generated using RGB channels from file %s" % identifier
id_string = "Classification from file %s." % identifier
landscr = os.path.join(landcoverdir, identifier + "_landcover.pix") # Scratch pix file
rgb8bit = os.path.join(landcoverdir, identifier + "_rgb8bit.pix") # Rescaled 8-bit pix file
fexport(fili=pixin, # Input with PCA
filo=landscr, # Output scratch file
dbiw=[],
dbic=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], # Use all channels
dbib=[2],
dbvs=[],
dblut=[],
dbpct=[],
ftype="PIX", # PIX filetype
foptions="")
pcimod(file=landscr, # Output scratch PIX file
pciop='ADD', # Modification mode "Add"
pcival=[0, 2, 0, 0]) # Task - add two 16U channels
if clouds:
kclus(file=landscr, # Run classification on scratch file
dbic=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], # Use all image layers
dboc=[14], # Output to blank layer
mask=[2], # Use not-cloud mask
numclus=[24], # 24 clusters (not all will be used, but
seedfile='', # this avoids cluster confusion)
maxiter=[20],
movethrs=[0.01],
siggen="YES", # Save signature layers
backval=[],
nsam=[])
else:
kclus(file=landscr, # Run classification on scratch file
dbic=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], # Use all image layers
dboc=[14], # Output to blank layer
numclus=[24], # 24 clusters (not all will be used, but
seedfile='', # this avoids cluster confusion)
maxiter=[20],
movethrs=[0.01],
siggen="YES", # Save signature layers
backval=[],
nsam=[])
print "Land cover classification for file %s completed." % identifier
print "Creating a colour table for classification result..."
scale(fili=landscr, # Rescale layers to 8-bit for use with pctmake
filo=rgb8bit,
dbic=[1, 2, 3, 14], # Rescale RGB and classification layer
dboc=[],
sfunct="LIN",
datatype="8U", # Scale to 8-bit unsigned
ftype="PIX") # PIX format
stretch(file=rgb8bit, # Creat lookup tables for histogram enhancement
dbic=[1], # Stretch band 1
dblut=[],
dbsn="LinLUT",
dbsd="Linear Stretch",
expo=[0.5]) # Linear stretch
stretch(file=rgb8bit,
dbic=[2], # Stretch band 2
dblut=[],
dbsn="LinLUT",
dbsd="Linear Stretch",
expo=[0.5]) # Linear stretch
stretch(file=rgb8bit,
dbic=[3], # Stretch band 3
dblut=[],
dbsn="LinLUT",
dbsd="Linear Stretch",
expo=[0.5]) # Linear stretch
pctmake(file=rgb8bit, # Make Colour table from rescaled RGB
dbic=[3, 2, 1], # RGB layers
dblut=[4, 3, 2], # Apply LUT stretch enhancement
dbtc=[4], # Classification layer
dbpct=[], # Make new PCT
mask=[],
dbsn="TC_PCT", # PCT name
dbsd=pct_string) # PCT description
print "Colour table generated from RGB layers and applied to %s classification result." % identifier
print "Converting Raster PCT to ArcMap Colour Layer..."
pct_txt = os.path.join(landcoverdir, identifier + "_pct.txt")
pctwrit(file=rgb8bit, # Export constructed PCT to text file
dbpct=[5], # PCT channel
pctform="ATT", # Write in attribute format
tfile=pct_txt)
clrfile = os.path.join(landcoverdir,identifier + "_landcover.clr")
if os.path.isfile(clrfile): # Using append mode, so make sure file is deleted
os.remove(clrfile)
pct = open(pct_txt, "r") # Open exported PCT text file in read mode
clr = open(clrfile, "w") # Open new CLR file in write mode
for line in pct:
if line[0] != "!" and line[3] == " ": # If line[3] not blank, line is unused range
a1 = line[8] # and if line starts with "!" it is the file header
a2 = line[9] # Number range is 0-255 so three possible digits
a3 = line[10] # Assign as int() to ignore blank digit spaces
att = int(a1 + a2 + a3) # Read number representing attribute
r1 = line[15]
r2 = line[16]
r3 = line[17]
red = int(r1 + r2 + r3) # Read number representing red value
g1 = line[20]
g2 = line[21]
g3 = line[22]
green = int(g1 + g2 + g3) # Read number representing green value
b1 = line[25]
b2 = line[26]
b3 = line[27]
blue = int(b1 + b2 + b3) # Read number representing blue value
clr.write("%s %s %s %s\n" % (att, red, green, blue)) # Write attribute and RGB value to CLR file
clr.close()
print "Raster PCT converted to ArcMap colour layer."
print "Exporting classified raster to tif..."
fexport(fili=rgb8bit, # Export raster
filo=rout, # Raster output location
dbic=[4], # Classification channel
dbpct=[5], # Colour table channel (2,3,4 are LUT)
ftype="TIF", # TIF format
foptions="")
print "Classification exported to %s." % rout
# ------------------------------------------------------------------------------------------------------------------- #
# TODO this section is included for reference, although not functional currently. Future exploration of automatic
# TODO application of symbology would be desirable.
# print "Applying colour map to layer file..."
# workspace = os.path.join(workingdir, "sable.gdb") # Define GDB workspace
# ras_lyr = identifier + "_raster_lyr" # Define feature layer name for polygon output
# lyr_out = rout[:-4] + ".lyr"
# arcpy.env.workspace = workspace # Set default workspace
# arcpy.env.overwriteOutput = True
# arcpy.env.outputCoordinateSystem = "PROJCS['WGS_1984_UTM_Zone_20N',GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',\
# SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],\
# PROJECTION['Transverse_Mercator'],PARAMETER['False_Easting',500000.0],PARAMETER['False_Northing',0.0],PARAMETER\
# ['Central_Meridian',-63.0],PARAMETER['Scale_Factor',0.9996],PARAMETER['Latitude_Of_Origin',0.0],UNIT['Meter',1.0]]"
# arcpy.BuildRasterAttributeTable_management(in_raster=rout)
# arcpy.MakeRasterLayer_management(in_raster=rout,
# out_rasterlayer=ras_lyr)
# arcpy.SaveToLayerFile_management(in_layer=ras_lyr,
# out_layer=lyr_out,
# is_relative_path="RELATIVE")
# arcpy.mapping.UpdateLayer(d)
# print "Exported layer file to %s" % lyr_out
# print "Applying colour map to layer file..."
# arcpy.AddColormap_management(in_raster=lyr_out,
# input_CLR_file=clrfile)
# print "Colour map applied to ArcMap layer file %s." % lyr_out
# ------------------------------------------------------------------------------------------------------------------- #
print "Exporting classification to shapefile..."
ras2poly(fili=rgb8bit, # Export to vector
dbic=[4], # Use classification channel
filo=vout, # Vector output location
smoothv="NO", # Don't smooth boundaries
dbsd=id_string, # Layer description string
ftype="SHP", # Shapefile format
foptions="")
print "Vector export complete. Wrote to %s." % vout
os.remove(landscr) # Delete intermediate PIX files
os.remove(rgb8bit)
completion_time = time.time() - start_time # Calculate time to complete
print "Land cover classification process completed for image %s in %i seconds." % (identifier, completion_time)
def coastline(pixin, polygonout, lineout, lineout_smooth, identifier, clouds):
start_time = time.time()
print "Generating coastline classification..."
id_string = "Coastline from file %s." % identifier
coastscr = os.path.join(coastdir, identifier + "_coastline.pix")
fexport(fili=pixin, # Input with PCA
filo=coastscr, # Output scratch file
dbiw=[],
dbic=[11, 12, 13], # PCA channels
dbib=[2],
dbvs=[],
dblut=[],
dbpct=[],
ftype="PIX", # PIX filetype
foptions="")
pcimod(file=coastscr, # Output scratch PIX file
pciop='ADD', # Modification mode "Add"
pcival=[0, 2, 0, 0]) # Task - add two 16U channels
if clouds:
kclus(file=coastscr, # Run classification on scratch file
dbic=[1, 2, 3], # Use three PCA layers
dboc=[4],
mask=[2], # Use the not-cloud mask
numclus=[2], # Two clusters - land, ocean
seedfile='',
maxiter=[20],
movethrs=[],
siggen="YES",
backval=[],
nsam=[])
else:
kclus(file=coastscr, # Run classification on scratch file
dbic=[1, 2, 3], # Use three PCA layers
dboc=[4], # Output to blank layer
numclus=[2], # Two clusters - land, ocean
seedfile='',
maxiter=[20],
movethrs=[],
siggen="YES",
backval=[],
nsam=[])
ras2poly(fili=coastscr, # Use scratch PIX file
dbic=[4], # Use classification channel
filo=polygonout, # Polygon SHP output location
smoothv="YES", # Smooth boundaries
dbsd=id_string, # Layer description string
ftype="SHP", # Shapefile format
foptions="")
selpoints = os.path.join(workingdir,"selection_points","selection_polygons.shp")
workspace = os.path.join(workingdir,"sable.gdb") # Define GDB workspace
polygonout_lyr = identifier + "_polygon_lyr" # Define feature layer name for polygon output
arcpy.env.workspace = workspace # Set default workspace
arcpy.env.overwriteOutput = True
arcpy.env.outputCoordinateSystem = "PROJCS['WGS_1984_UTM_Zone_20N',GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',\
SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],\
PROJECTION['Transverse_Mercator'],PARAMETER['False_Easting',500000.0],PARAMETER['False_Northing',0.0],PARAMETER\
['Central_Meridian',-63.0],PARAMETER['Scale_Factor',0.9996],PARAMETER['Latitude_Of_Origin',0.0],UNIT['Meter',1.0]]"
arcpy.MakeFeatureLayer_management(polygonout, polygonout_lyr) # Make polygon feature layer
arcpy.AddField_management(in_table=polygonout, # Add new dissolve field
field_name="DISV",
field_type="SHORT",
field_precision=1,
field_scale=0,
field_is_nullable="NULLABLE",
field_is_required="NON_REQUIRED")
cb = "def calDisv(area):\\n if area > 1000000000:\\n return 0\\n elif area < 1000000000:\\n return 1"
arcpy.CalculateField_management(in_table=polygonout, # Calculate dissolve field
field="DISV",
expression="calDisv(!Area!)", # =0 if Ocean, =1 otherwise
expression_type="PYTHON",
code_block=cb)
polygons_dissolved = identifier + "_poly_dissolved"
arcpy.Dissolve_management(in_features=polygonout, # Dissolve island polygons
out_feature_class=polygons_dissolved,
dissolve_field="DISV",
multi_part="SINGLE_PART",
unsplit_lines="DISSOLVE_LINES")
polygondisv_lyr = identifier + "_polygon_disv_lyr"
arcpy.MakeFeatureLayer_management(polygons_dissolved, polygondisv_lyr) # Make polygon feature layer
arcpy.SelectLayerByAttribute_management(in_layer_or_view=polygondisv_lyr,
selection_type="NEW_SELECTION",
where_clause="DISV=1")
island_poly_dissolved = identifier + "island_poly_dissolved"
arcpy.CopyFeatures_management(in_features=polygondisv_lyr,
out_feature_class=island_poly_dissolved)
island_poly_disv_lyr = identifier + "_isl_poly_disv_lyr"
arcpy.MakeFeatureLayer_management(island_poly_dissolved, island_poly_disv_lyr) # Make polygon feature layer
# Select island polygons that contain selection circle polygons (small, basically points).
# These points are placed where the island is likely to exist.
# Remove selection of features larger than 1B sq. m - this feature most likely represents ocean.
try: # Does feature layer 'selpoints_lyr' exist yet?
arcpy.SelectLayerByLocation_management(island_poly_disv_lyr,'CONTAINS','selpoints_lyr','','NEW_SELECTION')
except: # Didn't work, so create 'selpoints_lyr' first
arcpy.MakeFeatureLayer_management(selpoints, 'selpoints_lyr')
arcpy.SelectLayerByLocation_management(island_poly_disv_lyr, 'CONTAINS', 'selpoints_lyr', '', 'NEW_SELECTION')
# Convert polygon features to polyline
arcpy.PolygonToLine_management(island_poly_disv_lyr,lineout,'IDENTIFY_NEIGHBORS')
# Smooth line features to fix zig-zag from raster cells
arcpy.cartography.SmoothLine(lineout,lineout_smooth,"PAEK",50,"")
os.remove(coastscr)
completion_time = time.time() - start_time # Calculate time to complete
print "Coastline vector completed in %i seconds. Written to file %s." % (completion_time, lineout_smooth)
return lineout_smooth