def runMTCDmosaicPre(id, s3_location):
gbdx = Interface()
if (id == -2):
# do the mosaicing of 1 and 2
images1 = getS3location(gbdx, s3_location + "/1")
images2 = getS3location(gbdx, s3_location + "/2")
tsk = gbdx.Task('mtcdvrt', images1=images1, images2=images2, id="pre")
elif (id == -3):
# do the mosaicing of 1, 2 snd 3
images1 = getS3location(gbdx, s3_location + "/1")
images2 = getS3location(gbdx, s3_location + "/2")
images3 = getS3location(gbdx, s3_location + "/3")
tsk = gbdx.Task('mtcdvrt',
images1=images1,
images2=images2,
images3=images3,
id="pre")
else:
print("Wrong id in runMTCDmosaicPre", id)
return 1
workflow = gbdx.Workflow([tsk])
workflow.savedata(tsk.outputs.data, location=s3_location + "/image2image/")
workflow.execute()
print('MTCD mosaic start', id, images1, workflow.id)
waitForWorkflow(workflow)
print('MTCD mosaic done ' + images1 + ' ' + images2 + ' ' + str(id) + ' ' +
str(workflow.status) + ' at ' + str(datetime.now()))
if (not workflow.status["event"] == "succeeded"):
print('MTCD mosaic failed')
return 1
return 0
def aopImage(catalog_id,
s3_location,
local_dir=None,
panPixelSize=0.5,
clip=None):
gbdx = Interface()
isWV1 = catalog_id.startswith('102')
if (isWV1):
print(
"WARNING is a WV1 image and MS or Pansharpened image can't be ordered"
)
isSWIR = catalog_id.startswith('104A')
if (isSWIR):
print("ERROR SWIR image can't be orthorectified")
return
order_id = order(gbdx, catalog_id)
data = gbdx.ordering.status(order_id)[0]['location']
gdalwarpOptions = " -r near --config GDAL_CACHEMAX 4000 -wm 4000 -co TILED=TRUE -co COMPRESS=PACKBITS -co BIGTIFF=YES "
aoptask = gbdx.Task("AOP_Strip_Processor",
data=data,
enable_acomp=True,
enable_pansharpen=False,
enable_dra=False,
ortho_epsg='UTM',
bands='MS',
ortho_pixel_size=str(4 * panPixelSize),
ortho_dem_specifier=dem)
if (clip is not None):
clipTask = gbdx.Task("gdalcrop",
image=aoptask.outputs.data,
crop=clip + ' -tr ' + str(4 * panPixelSize) +
' ' + str(4 * panPixelSize) + gdalwarpOptions,
ship='false')
workflow = gbdx.Workflow([aoptask, clipTask])
workflow.savedata(clipTask.outputs.cropped,
location=s3_location + 'MS')
else:
workflow = gbdx.Workflow([aoptask])
workflow.savedata(aoptask.outputs.data, location=s3_location + 'MS')
workflow.execute()
print('AOP is processing image ' + catalog_id + ' MS workflow id is ' +
workflow.id)
waitForWorkflow(workflow)
print('MS image ' + catalog_id + ' ' + str(workflow.status) + ' at ' +
str(datetime.now()))
if local_dir == '':
return
if (local_dir is not None):
print('Downloading AOP images')
if not os.path.exists(local_dir):
os.makedirs(local_dir)
gbdx.s3.download(location=s3_location, local_dir=local_dir)
print('Image downloaded' + catalog_id + ' at ' + str(datetime.now()))
return
def changePrep(catalog_id, s3_location, clip=None):
size = '2'
gbdx = Interface()
isWV1 = catalog_id.startswith('102')
if (isWV1 and (ms or pansharpen or pansharpenship)):
print("ERROR Image is a WV1 image.")
return
isSWIR = catalog_id.startswith('104A')
if (isSWIR):
print("ERROR Image is a WV1 image.")
return
order_id = order(gbdx, catalog_id)
data = gbdx.ordering.status(order_id)[0]['location']
gdalwarpOptions = " -r near --config GDAL_CACHEMAX 4000 -wm 4000 -co TILED=TRUE -co COMPRESS=PACKBITS -co BIGTIFF=YES "
gdalwarpOptions = " -r near -co TILED=TRUE -co COMPRESS=PACKBITS -co BIGTIFF=YES "
aoptask = gbdx.Task("AOP_Strip_Processor",
data=data,
enable_acomp=True,
enable_pansharpen=False,
enable_dra=False,
ortho_epsg='UTM',
bands='MS',
ortho_pixel_size=size)
#aoptask = gbdx.Task("AOP_Strip_Processor", data=data, enable_acomp=False, enable_pansharpen=False, enable_dra=False, ortho_epsg='UTM', bands='MS', ortho_pixel_size='16')
topoTask = gbdx.Task("topo-correction", image=aoptask.outputs.data)
topoTask.impersonation_allowed = True
cloudTask = gbdx.Task("CloudPuncher",
image=topoTask.outputs.data,
maskOnly='false')
#cloudTask = gbdx.Task("CloudPuncher", image = aoptask.outputs.data, maskOnly = 'false')
if (clip is not None):
#####################################################################################################################################################################################
clipTask = gbdx.Task('gdalcrop',
image=cloudTask.outputs.mask,
crop=clip + ' -tr ' + size + ' ' + size +
gdalwarpOptions,
ship='false')
workflow = gbdx.Workflow([aoptask, topoTask, cloudTask, clipTask])
workflow.savedata(clipTask.outputs.cropped, location=s3_location)
else:
workflow = gbdx.Workflow([aoptask, topoTask, cloudTask])
workflow.savedata(cloudTask.outputs.mask, location=s3_location)
workflow.execute()
print('AOP is processing image ' + catalog_id + ' MS workflow id is ' +
workflow.id)
waitForWorkflow(workflow)
print('MS image ' + catalog_id + ' ' + str(workflow.status) +
' wfl id ' + workflow.id + ' at ' + str(datetime.now()))
if (not workflow.status["event"] == "succeeded"):
print("workflow.status", workflow.status,
workflow.status["event"] == "succeeded")
return 1
return 0
def image2image(ref_image_s3_dir,
image_s3_dir,
output_s3,
source_filename=None,
reference_filename=None,
clip=None,
pixelSize=2,
fileName=None):
gbdx = Interface()
full_ref_image_s3_dir = getS3location(gbdx, ref_image_s3_dir)
full_image_s3_dir = getS3location(gbdx, image_s3_dir)
task = gbdx.Task("image2image",
reference_directory=full_ref_image_s3_dir,
source_directory=full_image_s3_dir,
source_filename=source_filename,
reference_filename=reference_filename)
task.timeout = 36000
if (clip is not None):
clipTask1 = gbdx.Task("gdalcrop",
image=task.outputs.out,
crop=clip + ' -tr ' + str(pixelSize) + ' ' +
str(pixelSize),
ship='False',
updateName='False',
fileName=fileName)
workflow = gbdx.Workflow([task, clipTask1])
workflow.savedata(clipTask1.outputs.cropped, location=output_s3)
else:
workflow = gbdx.Workflow([task])
workflow.savedata(task.outputs.out, location=output_s3)
workflow.execute()
print('image2Image ' + ref_image_s3_dir + ' ' + image_s3_dir + ' ' +
output_s3 + ' ' + fileName + ' workflow ' + str(workflow.id) +
' started at ' + str(datetime.now()))
waitForWorkflow(workflow)
print('image2Image ' + str(workflow.id) + ' ' + ref_image_s3_dir + ' ' +
image_s3_dir + ' ' + output_s3 + ' ' + str(workflow.status) +
' at ' + str(datetime.now()))
if (not workflow.status["event"] == "succeeded"):
return 1
return 0
def runMTCDmosaic(id, s3_location):
gbdx = Interface()
images = getS3location(gbdx, s3_location + "/" + str(id))
tsk = gbdx.Task('mtcdvrt', images1=images, id=str(id))
workflow = gbdx.Workflow([tsk])
workflow.savedata(tsk.outputs.data, location=s3_location + "/image2image/")
workflow.execute()
print('MTCD mosaic start', id, images, workflow.id)
waitForWorkflow(workflow)
print('MTCD mosaic done ' + images + ' ' + str(id) + ' ' +
str(workflow.status) + ' at ' + str(datetime.now()))
if (not workflow.status["event"] == "succeeded"):
print('MTCD mosaic failed')
return 1
return 0
def runMTCD(s3_locationPre, s3_locationPost, out_s3_location):
gbdx = Interface()
preimage = getS3location(gbdx, s3_locationPre)
postimage = getS3location(gbdx, s3_locationPost)
tsk = gbdx.Task('mtcd', preimage=preimage, postimage=postimage)
workflow = gbdx.Workflow([tsk])
workflow.savedata(tsk.outputs.data, location=out_s3_location)
workflow.execute()
print('MTCD image pair start ' + s3_locationPre + ' ' + s3_locationPost +
' ' + out_s3_location + ' ' + str(workflow.id) + ' at ' +
str(datetime.now()))
waitForWorkflow(workflow)
print('MTCD image pair done ' + s3_locationPre + ' ' + s3_locationPost +
' ' + out_s3_location + ' ' + str(workflow.status) + ' at ' +
str(datetime.now()))
if (not workflow.status["event"] == "succeeded"):
return 1
return 0
from gbdxtools import Interface
gbdx = Interface()
s3_flood_extents = 's3://egolden/floodwatch/intersect_inputs/flood_shp_one/'
s3_footprints = 's3://egolden/floodwatch/intersect_inputs/footprints_shp/'
intersect_task = gbdx.Task('classify_flooded_footprints',
footprints_shp=s3_footprints,
flood_shp=s3_flood_extents)
workflow = gbdx.Workflow([intersect_task])
workflow.savedata(intersect_task.outputs.footprints_flood_results, location='egolden/intersect_results')
workflow.execute()
print(workflow.id)
from gbdxtools import Interface
gbdx = Interface()
QB = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/QB"
WV1 = "s3://receiving-dgcs-tdgplatform-com/054876516120_01_003"
WV2 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV2"
WV3 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV3"
GE = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/GE/055217125010_01"
#aoptask = gbdx.Task('AOP_Strip_Processor', data=QB, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
#aoptask = gbdx.Task('AOP_Strip_Processor', data=WV1, bands='PAN', enable_acomp=False, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
#aoptask = gbdx.Task('AOP_Strip_Processor', data=WV2, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
#aoptask = gbdx.Task('AOP_Strip_Processor', data=WV3, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
#aoptask = gbdx.Task('AOP_Strip_Processor', data=GE, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
isodata = gbdx.Task("ENVI_ISODATAClassification")
isodata.inputs.input_raster = GE
isodata.inputs.file_types = "tif"
sieve = gbdx.Task("ENVI_ClassificationSieving")
sieve.inputs.input_raster = isodata.outputs.output_raster_uri.value
sieve.inputs.file_types = "hdr"
clump = gbdx.Task("ENVI_ClassificationClumping")
clump.inputs.input_raster = sieve.outputs.output_raster_uri.value
clump.inputs.file_types = "hdr"
workflow = gbdx.Workflow([isodata, sieve, clump])
'''
workflow.savedata(
clump.outputs.output_raster_uri,
# To order the image with DG factory catalog id 10400100143FC900:
order_id = gbdx.ordering.order('10400100143FC900')
print order_id
# The order_id is unique to your image order and can be used to track the progress of your order. The ordered image sits in a directory on S3. The output of the following describes where:
status = gbdx.ordering.status(order_id)
# result:
# [{u'acquisition_id': u'10400100143FC900',
# u'state': u'delivered',
# u'location': u's3://receiving-dgcs-tdgplatform-com/055546367010_01_003'}]
# test a quick workflow on the item
data = str(status[0]['location'])
aoptask = gbdx.Task("AOP_Strip_Processor",
data=data,
enable_acomp=True,
enable_pansharpen=True)
workflow = gbdx.Workflow([aoptask])
data = "s3://receiving-dgcs-tdgplatform-com/054813633050_01_003" # WV02 Image over San Francisco
aoptask = gbdx.Task("AOP_Strip_Processor", data=data)
s3task = gbdx.Task("StageDataToS3")
s3task.inputs.data = aoptask.outputs.data.value
s3task.inputs.destination = "s3://gbd-customer-data"
workflow = gbdx.Workflow([aoptask, s3task])
workflow.execute()
# workflow.execute()
from gbdxtools import Interface
gbdx = Interface()
QB = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/QB"
WV1 = "s3://receiving-dgcs-tdgplatform-com/054876516120_01_003"
WV2 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV2"
WV3 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV3"
GE = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/GE/055217125010_01"
aop2envi = gbdx.Task("AOP_ENVI_HDR")
aop2envi.inputs.image = GE
envi_ndvi = gbdx.Task("ENVI_SpectralIndices")
envi_ndvi.inputs.input_raster = aop2envi.outputs.output_data.value
envi_ndvi.inputs.file_types = "hdr"
# Specify a string/list of indicies to run on the input_raster variable. The order of indicies wi
envi_ndvi.inputs.index = '["Normalized Difference Vegetation Index", "Simple Ratio"]'
workflow = gbdx.Workflow([aop2envi, envi_ndvi])
'''
workflow.savedata(
envi_ndvi.outputs.output_raster_uri,
location='Benchmark/spectralindices/QB'
)
workflow.savedata(
envi_ndvi.outputs.output_raster_uri,
location='Benchmark/spectralindices/WV2'
)
workflow.savedata(
from gbdxtools import Interface
gbdx = Interface()
QB = "s3://receiving-dgcs-tdgplatform-com/054876960040_01_003"
WV1 = "s3://receiving-dgcs-tdgplatform-com/054876516120_01_003"
WV2 = "s3://receiving-dgcs-tdgplatform-com/054876618060_01_003"
WV3 = "s3://receiving-dgcs-tdgplatform-com/055605759010_01_003"
GE = "s3://receiving-dgcs-tdgplatform-com/055217125010_01_003"
#aoptask = gbdx.Task("AOP_Strip_Processor", data=GE, enable_acomp=True, enable_pansharpen=False, enable_dra=False, bands='MS')
aoptask2 = gbdx.Task('AOP_Strip_Processor', data=WV1, bands='PAN', enable_acomp=False, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
# Capture AOP task outputs
#orthoed_output = aoptask.get_output('data')
aop2envi = gbdx.Task("Build_ENVI_HDR")
aop2envi.inputs.image = aoptask2.outputs.data.value
#hdr file used to compute spectral index
#envi_ndvi = gbdx.Task("ENVI_SpectralIndex")
#envi_ndvi.inputs.input_raster = aop2envi.outputs.output_data.value
#envi_ndvi.inputs.file_types = "hdr"
#envi_ndvi.inputs.index = "Normalized Difference Vegetation Index"
#spectral index file used in color slice classification task
envi_color = gbdx.Task('ENVI_ColorSliceClassification')
envi_color.inputs.input_raster = aoptask2.outputs.data.value
from gbdxtools import Interface
import boto3
gbdx = Interface()
sts = boto3.client('sts')
creds = sts.get_session_token()['Credentials']
ingest = gbdx.Task(
'ingest-s3-data',
data=
's3://viper-projects/open_data/intermediate/PreprocessImage/1030010069650A00/059441867010_01/',
aws_access_key_id=creds['AccessKeyId'],
aws_secret_access_key=creds['SecretAccessKey'],
aws_session_token=creds['SessionToken'])
cog = gbdx.Task(
'CloudOptimizedGeoTIFF:2.0.1',
data=ingest.outputs.data.value,
)
save = gbdx.Task(
'SaveToS3',
data=cog.outputs.data.value,
destination='s3://jduckworth/cog-tests/1030010069650A00',
access_key_id=creds['AccessKeyId'],
secret_key=creds['SecretAccessKey'],
session_token=creds['SessionToken'],
)
workflow = gbdx.Workflow([ingest, cog, save])
# First we'll run atmospheric compensation on Landsat8 data
from gbdxtools import Interface
gbdx = Interface()
acomp = gbdx.Task('AComp',
data='s3://landsat-pds/L8/033/032/LC80330322015035LGN00')
# Now we'll save the result to our own S3 bucket. First we need to generate temporary AWS credentials
# (this assumes you have an AWS account and your IAM credentials are appropriately accessible via boto)
import boto3
client = boto3.client('sts')
response = client.get_session_token(DurationSeconds=86400)
access_key_id = response['Credentials']['AccessKeyId']
secret_key = response['Credentials']['SecretAccessKey']
session_token = response['Credentials']['SessionToken']
# Save the data to your s3 bucket using the SaveToS3 task:
savetask = gbdx.Task('SaveToS3')
savetask.inputs.data = acomp.outputs.data.value
savetask.inputs.destination = "s3://your-bucket/your-path/"
savetask.inputs.access_key_id = access_key_id
savetask.inputs.secret_key = secret_key
savetask.inputs.session_token = session_token
workflow = gbdx.Workflow([acomp, savetask])
workflow.execute()
from gbdxtools import Interface
gbdx = Interface()
QB = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/QB"
WV1 = "s3://receiving-dgcs-tdgplatform-com/054876516120_01_003"
WV2 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV2"
WV3 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV3"
GE = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/GE/055217125010_01"
envi_task = gbdx.Task("ENVI_RXAnomalyDetection")
envi_task.inputs.file_types = 'til'
envi_task.inputs.kernel_size = '3'
envi_task.inputs.input_raster = WV3
workflow = gbdx.Workflow([envi_task])
workflow.savedata(envi_task.outputs.task_meta_data,
location='envi_task_output')
workflow.savedata(envi_task.outputs.output_raster_uri,
location='envi_task_output')
'''
workflow.savedata(
envi_task.outputs.task_meta_data,
location='Benchmark/RX/QB'
)
workflow.savedata(
envi_task.outputs.task_meta_data,
location='Benchmark/RX/WV1'
)
)
workflow.execute()
status = workflow.status["state"]
wf_id = workflow.id
from gbdxtools import Interface
gbdx = Interface()
'''
from gbdxtools import Interface
gbdx = Interface()
image = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/RPCOrtho/image1"
dem = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/RPCOrtho/DEM/image1"
envi_RPCO = gbdx.Task("ENVI_RPCOrthorectification")
envi_RPCO.inputs.input_raster_metadata = '{"sensor type": "IKONOS"}'
envi_RPCO.inputs.input_raster_band_grouping = 'multispectral'
envi_RPCO.inputs.input_raster = image
envi_RPCO.inputs.dem_raster = dem
workflow = gbdx.Workflow([envi_RPCO])
workflow.savedata(
envi_RPCO.outputs.output_raster_uri,
location='Benchmark/ENVI_RPCO/results'
)
workflow.execute()
from gbdxtools.simple_answerfactory import Recipe, RecipeParameter, Project, RecipeConfig
from gbdxtools import Interface
gbdx = Interface()
## the workflow that must be defined in order to specify a recipe
aop = gbdx.Task('AOP_Strip_Processor')
aop.inputs.ortho_interpolation_type = 'Bilinear'
aop.inputs.ortho_pixel_size = 'auto'
aop.inputs.bands = 'PAN+MS'
aop.inputs.ortho_epsg = 'UTM'
aop.inputs.enable_acomp = 'true'
aop.inputs.enable_pansharpen = 'true'
aop.inputs.enable_dra = 'true'
aop.inputs.ortho_pixel_size = '0.5'
# Answerfactory will automatically prepend an auto-ordering task and replace
# {raster_path} with the actual s3 path to the raster data
aop.inputs.data = '{raster_path}'
# remove xml files (causes a bug in skynet)
xmlfix = gbdx.Task('gdal-cli-multiplex')
xmlfix.inputs.data = aop.outputs.data.value
xmlfix.inputs.command = "find $indir/data/ -name *XML -type f -delete; mkdir -p $outdir; cp -R $indir/data/ $outdir/"
skynet = gbdx.Task('openskynet:0.0.10')
skynet.inputs.data = xmlfix.outputs.data.value
# AnswerFactory auto populates {model_location_s3} with the s3 location of the model referred to in
# the recipe property 'model_type'. This model must be previously registered with the model catalog service.
# AF searches the model catalog for the closest model with the specified type to the input acquisition
skynet.inputs.model = '{model_location_s3}'
skynet.inputs.log_level = 'trace'
#############################################################################################
from gbdxtools import Interface
gbdx = Interface()
# get output dir from arg on the command line
out_data_loc = "dloomis/lulc_wf_test"
# set the input data location. This could also be pulled from a catalog API response using a catalog_id
data = "s3://receiving-dgcs-tdgplatform-com/055186940010_01_003/"
# build the task used in the workflow
aoptask = gbdx.Task("AOP_Strip_Processor",
data=data,
enable_acomp=True,
enable_pansharpen=False,
enable_dra=False,
bands='MS')
pp_task = gbdx.Task(
"ProtogenPrep", raster=aoptask.outputs.data.value
) # ProtogenPrep task is used to get AOP output into proper format for protogen task
prot_lulc = gbdx.Task("protogenV2LULC", raster=pp_task.outputs.data.value)
# build the workflow ( AOP -> ProtogenPrep -> protogenV2LULC )
workflow = gbdx.Workflow([aoptask, pp_task, prot_lulc])
workflow.savedata(prot_lulc.outputs.data.value, location=out_data_loc)
# optional: print workflow tasks, to check the json
print
print(aoptask.generate_task_workflow_json())
print
print(pp_task.generate_task_workflow_json())
from gbdxtools import Interface
import os
from gbdxrun.local_task import LocalTask
from gbdxrun.local_workflow import LocalWorkflow
HOST = os.environ.get('GBDXTOOLS_HOST', None)
CONFIG = os.environ.get('GBDXTOOLS_PROFILE', None)
config_kwargs = {}
if HOST:
config_kwargs['host'] = HOST
elif CONFIG:
config_kwargs['config_file'] = CONFIG
gbdx = Interface(**config_kwargs)
gbdx.Task = LocalTask
gbdx.Workflow = LocalWorkflow
class LocalWorkflowError(Exception):
pass
# Run atmospheric compensation on Landsat8 data
from gbdxtools import Interface
gbdx = Interface()
acomp = gbdx.Task('AComp_New', data='s3://landsat-pds/L8/033/032/LC80330322015035LGN00')
workflow = gbdx.Workflow([acomp])
workflow.savedata(acomp.outputs.data, location='acomp_output_folder')
workflow.execute()
from gbdxtools import Interface
gbdx = Interface()
test = "s3://receiving-dgcs-tdgplatform-com/055777190010_01_003"
QB = "s3://receiving-dgcs-tdgplatform-com/054876960040_01_003"
WV1 = "s3://receiving-dgcs-tdgplatform-com/054876516120_01_003"
WV2 = "s3://receiving-dgcs-tdgplatform-com/054876618060_01_003"
WV3 = "s3://receiving-dgcs-tdgplatform-com/055605759010_01_003"
GE = "s3://receiving-dgcs-tdgplatform-com/055217125010_01_003"
#aoptask1 = gbdx.Task('AOP_Strip_Processor', data=test, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
aoptask2 = gbdx.Task('AOP_Strip_Processor',
data=WV1,
bands='PAN',
enable_acomp=False,
enable_pansharpen=False,
enable_dra=False) # creates acomp'd multispectral image
#aoptask3 = gbdx.Task('AOP_Strip_Processor', data=WV2, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
#aoptask4 = gbdx.Task('AOP_Strip_Processor', data=WV3, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
#aoptask5 = gbdx.Task('AOP_Strip_Processor', data=GE, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
envitask = gbdx.Task("ENVI_ISODATAClassification")
envitask.inputs.file_types = 'tif'
envitask.inputs.input_raster = aoptask1.outputs.data.value
envitask.outputs.output_raster = "ENVI"
shptask = gbdx.Task("ENVI_ClassificationToShapefile")
shptask.inputs.input_raster = envitask.outputs.output_raster_uri.value
shptask.inputs.file_types = "hdr"
workflow = gbdx.Workflow([aoptask1, envitas, shptask])
from gbdxtools import Interface
gbdx = Interface()
QB = "s3://receiving-dgcs-tdgplatform-com/054876960040_01_003"
WV1 = "s3://receiving-dgcs-tdgplatform-com/054876516120_01_003"
WV2 = "s3://receiving-dgcs-tdgplatform-com/054876618060_01_003"
WV3 = "s3://receiving-dgcs-tdgplatform-com/055605759010_01_003"
GE = "s3://receiving-dgcs-tdgplatform-com/055217125010_01_003"
aoptask1 = gbdx.Task('AOP_Strip_Processor', data=QB, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
#aoptask2 = gbdx.Task('AOP_Strip_Processor', data=WV1, bands='PAN', enable_acomp=False, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
#aoptask3 = gbdx.Task('AOP_Strip_Processor', data=WV2, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
#aoptask4 = gbdx.Task('AOP_Strip_Processor', data=WV3, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
#aoptask5 = gbdx.Task('AOP_Strip_Processor', data=GE, bands='MS', enable_acomp=True, enable_pansharpen=False, enable_dra=False) # creates acomp'd multispectral image
workflow = gbdx.Workflow([aoptask1])
workflow.savedata(
aoptask1.outputs.data,
location='Benchmark/QB'
)
'''
workflow.savedata(
aoptask2.outputs.data,
location='Benchmark/WV1'
)
workflow.savedata(
aoptask3.outputs.data,
from gbdxtools import Interface
gbdx = Interface()
QB = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/QB"
WV1 = "s3://receiving-dgcs-tdgplatform-com/054876516120_01_003"
WV2 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV2"
WV3 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV3"
GE = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/GE/055217125010_01"
# Capture AOP task outputs
#orthoed_output = aoptask.get_output('data')
task = gbdx.Task("ENVI_ImageThresholdToROI")
task.inputs.input_raster=WV1
task.inputs.file_types = "tif"
task.inputs.roi_name = "[\"Water\"]"
task.inputs.roi_color = "[[0,255,0]"
task.inputs.threshold = "[[138]"
task.inputs.output_roi_uri_filename = "roi.xml"
workflow = gbdx.Workflow([ task ])
'''
workflow.savedata(
task.outputs.output_roi_uri,
location='Benchmark/ImgToROI/QB'
)
'''
workflow.savedata(
task.outputs.output_roi_uri,
location='Benchmark/ImgToROI/WV1'
# and a few extra lines than it needs in the displayed example.
# This code removes the additional tabs and lines for the code that will be displayed in the markdown.
i['%s_text' % j] = i['%s_text' % j].replace(
'\n ', '\n')
i['%s_text' % j] = i['%s_text' % j].replace('\n\n', '\n')
# If the test failed, don't load the code into the docs. Load the default failed message.
else:
i['%s_text' % j] = fail_string
# For each task that is known to exist on GBDx already, hit the API and get the task details
for i in list_of_tasks:
if i['knowntask'] is True:
# retrieve task info and store it in the task data object
task = gbdx.Task(i['name'])
i['description'] = task.definition['description']
i['input_ports'] = task.input_ports
i['output_ports'] = task.output_ports
list_of_steps = {'create_md': True, 'run_tests': True}
# Now that we have all of the information we need, we can write it to the markdown file for each task in the list.
for i in list_of_tasks:
# We don't create markdowns for tasks that are not on GBDx
if i['knowntask'] is True:
# If the markdown file already exists, read it in
if i['markdown'] is True:
with open(i['markdown_file_name']) as f:
s = f.read()
# If not, we start with the template
from gbdxtools import Interface
gbdx = Interface(
) # Instantiate gbdxtools, which logs you in using your .config file
s3path1 = 's3://receiving-dgcs-tdgplatform-com/055364007010_01_003'
aoptask1 = gbdx.Task("AOP_Strip_Processor",
data=s3path1,
enable_acomp=True,
enable_pansharpen=False,
enable_dra=False,
bands='MS')
s3task1 = gbdx.Task("StageDataToS3")
s3task1.inputs.data = aoptask1.outputs.data.value
s3task1.inputs.destination = "s3://change_detection/test_job/Steps/acomp_fastortho_step-pre_image_task/Output"
s3path2 = 's3://receiving-dgcs-tdgplatform-com/055364005010_01_003'
aoptask2 = gbdx.Task("AOP_Strip_Processor",
data=s3path2,
enable_acomp=True,
enable_pansharpen=False,
enable_dra=False,
bands='MS')
s3task2 = gbdx.Task("StageDataToS3")
s3task2.inputs.data = aoptask1.outputs.data.value
s3task2.inputs.destination = "s3://change_detection/test_job/Steps/acomp_fastortho_step-post_image_task/Output"
cdtask = gbdx.Task("change_detection")
cdtask.inputs.pre_image = aoptask1.outputs.data.value
cdtask.inputs.post_image = aoptask2.outputs.data.value
def processChangeSet(idsets,
s3_location,
clip=None,
deleteIntermediateFromS3=False):
start_time = time.time()
ids = []
if (len(idsets) < 4 or len(idsets) > 6):
print("FATAL number of idsets has to be 4, 5, or 6. it is " +
str(len(idsets)))
return
for idset in idsets:
for catid in idset:
ids.append(catid)
print("AOP, Topocorrection, Cloud punching")
threads = []
id = 0
for idset in idsets:
id += 1
for catID in idset:
if (catID is not None and len(catID) > 3):
if catID.find(' ') > 0:
catID = catID[0:catID.index(' ')]
thread = startChangePrepThread(catID,
s3_location + "/" + str(id),
clip)
threads.append(thread)
time.sleep(2)
nThreads = len(threads)
i = 0
for thread in threads:
ret = thread.join()
i += 1
print(str(i) + " out of " + str(nThreads) + " threads done")
if (ret != 0):
print("ChangePrep error", ret)
return 1
print('AOP/Topo/CloudDet done. Elapsed time: {} min'.format(
round((time.time() - start_time) / 60)))
print("mosaicking")
id = 0
threads = []
if (len(idsets) == 4):
thread = startMTCDmosaicThread(-2, s3_location)
else:
thread = startMTCDmosaicThread(-3, s3_location)
threads.append(thread)
time.sleep(2)
for idset in idsets:
id += 1
thread = startMTCDmosaicThread(id, s3_location)
threads.append(thread)
time.sleep(2)
nThreads = len(threads)
i = 0
for thread in threads:
ret = thread.join()
i += 1
print(
str(i) + " out of " + str(nThreads) + " threads done. Return ",
ret)
if (ret != 0):
print("MTCDmosaic error", ret)
return 1
print('mtcdvrt done. Elapsed time: {} min'.format(
round((time.time() - start_time) / 60)))
threads = []
id = 0
for idset in idsets:
id += 1
if (deleteIntermediateFromS3):
deleteFromS3(s3_location + "/" + str(id) + "/")
fileName = str(id) + "_warped.tif"
thread = startimage2imageThread(
s3_location + "/image2image/pre.tif",
s3_location + "/image2image/" + str(id) + ".tif",
s3_location + "/image2imageFinal/" + str(id),
clip=clip,
fileName=fileName)
threads.append(thread)
time.sleep(2)
print("Running image2image alignment")
nThreads = len(threads)
i = 0
for thread in threads:
ret = thread.join()
i += 1
print(
str(i) + " out of " + str(nThreads) + " threads done. Return ",
ret)
if (ret != 0):
print("image2image error", ret)
return 1
print('Image2image done. Elapsed time: {} min'.format(
round((time.time() - start_time) / 60)))
gbdx = Interface()
watertsk = gbdx.Task('kk-watermask')
watertsk.inputs.image = getS3location(gbdx,
s3_location + "/image2imageFinal/1")
waterwfl = gbdx.Workflow([watertsk])
waterwfl.savedata(watertsk.outputs.mask,
location=s3_location + "/waterMask")
waterwfl.execute()
print('Water mask task start ' + str(waterwfl.id) + ' at ' +
str(datetime.now()))
nImages = len(idsets)
threads = []
for i in range(1, nImages + 1):
for j in range(i + 1, nImages + 1):
threads.append(
startMTCDThread(
s3_location + "/image2imageFinal/" + str(i) + "/" +
str(i) + "_warped.tif",
s3_location + "/image2imageFinal/" + str(j) + "/" +
str(j) + "_warped.tif", s3_location + "/changePairs/"))
nThreads = len(threads)
i = 0
for thread in threads:
ret = thread.join()
i += 1
print(
str(i) + " out of " + str(nThreads) + " threads done. Return ",
ret)
if (ret != 0):
print("MTCDThread error", ret)
return 1
print('Image pair change done. Elapsed time: {} min'.format(
round((time.time() - start_time) / 60)))
if (deleteIntermediateFromS3):
deleteFromS3(s3_location + "/image2imageFinal/")
deleteFromS3(s3_location + "/image2image/")
waitForWorkflow(waterwfl)
print('Water mask task done ' + str(waterwfl.status) + ' at ' +
str(datetime.now()))
gbdx = Interface()
changeImages = getS3location(gbdx, s3_location + "/changePairs/")
maskFolder = getS3location(gbdx, s3_location + "/waterMask")
tsk = gbdx.Task('mtcd2', image=changeImages, mask=maskFolder)
wfl = gbdx.Workflow([tsk])
wfl.savedata(tsk.outputs.data, location=s3_location + "/change/")
wfl.execute()
print('MTCD time filter start ' + str(wfl.id) + ' at ' +
str(datetime.now()))
waitForWorkflow(wfl)
print('MTCD time filter done ' + str(wfl.status) + ' at ' +
str(datetime.now()))
if (deleteIntermediateFromS3):
deleteFromS3(s3_location + "/changePairs/")
deleteFromS3(s3_location + "/waterMask/")
print('MTCD change ALL processes done. Elapsed time: {} min'.format(
round((time.time() - start_time) / 60)))
print("All done " + wfl.status['event'])
return 0
# Input data
from gbdxtools import Interface
gbdx = Interface()
WV2 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV2/"
WV3 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV3/"
prep2 = gbdx.Task('ProtogenPrep')
# ProtogenPrep will move the aoptask output to root where prototask can find it
prep2.inputs.raster = WV2
prep3 = gbdx.Task('ProtogenPrep')
# ProtogenPrep will move the aoptask output to root where prototask can find it
prep3.inputs.raster = WV3
'''
protoLULC = gbdx.Task("protogenV2LULC", raster=gluetask2.outputs.data.value)
protoPAN = gbdx.Task("protogenV2PANTEX10", raster=gluetask2.outputs.data.value)
protoRAC = gbdx.Task("protogenV2RAC", raster=gluetask2.outputs.data.value)
protoRAS = gbdx.Task("protogenV2RAS", raster=gluetask2.outputs.data.value)
protoRAV = gbdx.Task("protogenV2RAV", raster=gluetask2.outputs.data.value)
protoRAW = gbdx.Task("protogenV2RAW", raster=gluetask2.outputs.data.value)
'''
protoLULC3 = gbdx.Task("protogenV2LULC", raster=prep3.outputs.data)
protoPAN3 = gbdx.Task("protogenV2PANTEX10", raster=prep3.outputs.data)
protoRAC3 = gbdx.Task("protogenV2RAC", raster=prep3.outputs.data)
protoRAS3 = gbdx.Task("protogenV2RAS", raster=prep3.outputs.data)
protoRAV3 = gbdx.Task("protogenV2RAV", raster=prep3.outputs.data)
protoRAW3 = gbdx.Task("protogenV2RAW", raster=prep3.outputs.data)
workflow = gbdx.Workflow(
from gbdxtools import Interface
gbdx = Interface()
QB = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/QB"
WV1 = "s3://receiving-dgcs-tdgplatform-com/054876516120_01_003"
WV2 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV2"
WV3 = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/WV3"
GE = "s3://gbd-customer-data/7d8cfdb6-13ee-4a2a-bf7e-0aff4795d927/Benchmark/GE/055217125010_01"
aop2envi = gbdx.Task("AOP_ENVI_HDR")
#aop2envi.inputs.image = QB
#aop2envi.inputs.image = WV2
#aop2envi.inputs.image = WV3
aop2envi.inputs.image = GE
envi_query = gbdx.Task("ENVI_QuerySpectralIndices")
envi_query.inputs.input_raster = aop2envi.outputs.output_data.value
envi_query.inputs.file_types = "hdr"
workflow = gbdx.Workflow([aop2envi, envi_query])
'''
workflow.savedata(
envi_query.outputs.task_meta_data,
location='Benchmark/QSI/QB'
)
workflow.savedata(
envi_query.outputs.task_meta_data,
location='Benchmark/QSI/WV2'
from gbdxtools import Interface
"""
Example using multiple inputs with 1 submission
"""
gbdx = Interface()
# note there are 2 inputs
data = [
"s3://receiving-dgcs-tdgplatform-com/054813633050_01_003",
"http://test-tdgplatform-com/data/QB02/LV1B/053702625010_01_004/053702625010_01/053702625010_01_P013_MUL"
]
aoptask = gbdx.Task("AOP_Strip_Processor",
data=data,
enable_acomp=True,
enable_pansharpen=True)
workflow = gbdx.Workflow([aoptask])
workflow.savedata(aoptask.outputs.data, location='some_folder')
batch_workflow_id = workflow.execute()
scene_id = '10300100612DE400' # big strip! cloudy
scene_id = '103001002B3BEF00' # january, 2014 (lakewood)
scene_id1 = '103001001AAB3D00' #8-5-2012 big strip
order_id1 = gbdx.ordering.order(scene_id1)
print(order_id)
# The order_id is unique to your image order and can be used to track the progress of your order. The ordered image sits in a directory on S3. The output of the following describes where:
status = gbdx.ordering.status(order_id)
# Make sure DRA is disabled if you are processing both the PAN+MS files
#Edit the following line(s) to reflect specific folder(s) for the output file (example location provided)
# data = str(status[0]['location'])
data = str(gbdx.ordering.status(order_id)[0]['location'])
aoptask = gbdx.Task("AOP_Strip_Processor",
data=data,
enable_acomp=True,
bands="MS",
enable_pansharpen=False,
enable_dra=False)
# Capture AOP task outputs
log = aoptask.get_output('log')
orthoed_output = aoptask.get_output('data')
# Stage AOP output for the Protogen Task using the Protogen Prep Task
pp_task = gbdx.Task("ProtogenPrep", raster=aoptask.outputs.data.value)
# Setup ProtogenV2LULC Task
prot_lulc = gbdx.Task("protogenV2LULC", raster=pp_task.outputs.data.value)
# Run Combined Workflow
workflow = gbdx.Workflow([aoptask, pp_task, prot_lulc])
#### RUNNING WORKFLOWS
workflowList = [
] # We will keep track of our workflows by putting workflow IDs in a list as they are created
for catID in catalog_ids: # Loop through all of the catIDs we found earlier to find their s3 location. This will be the input for the first task.
s3path = gbdx.catalog.get_data_location(catalog_id=catID)
print s3path
if s3path != None:
s3path = '/'.join(s3path.split('/')[:-1]) + '/'
print s3path
aoptask = gbdx.Task("AOP_Strip_Processor",
data=s3path,
enable_acomp=True,
enable_pansharpen=False,
enable_dra=False,
bands='MS')
s3task = gbdx.Task("StageDataToS3")
s3task.inputs.data = aoptask.outputs.data.value
s3task.inputs.destination = "s3://molly-g/seattleAOP/"
pp_task = gbdx.Task(
"ProtogenPrep", raster=aoptask.outputs.data.value
) # ProtogenPrep task is used to get AOP output into proper format for protogen task
prot_ubfp = gbdx.Task("protogenV2UBFP",
raster=pp_task.outputs.data.value)
workflow = gbdx.Workflow([pp_task, aoptask, prot_ubfp, s3task
]) # Cool tip: these can be in any order!
workflow.savedata(
prot_ubfp.outputs.data.value, location="/molly-g/seattle"