def main(*args, **kwargs):
# initialize debug logger instance -------------------------------------
debug_log_output_directory = os.path.join(sys.path[0], "TRMM_logs")
etl_debug_logger = ETLDebugLogger(debug_log_output_directory, "TRMM", {
"debug_log_archive_days":7
})
update_debug_log = etl_debug_logger.updateDebugLog # retrieve a reference to the debug logger function
# color map that is applied to each 3-hour raster as well as each N-day cumulative raster (except the 30 day)
color_map = "PATH TO RASTER COLOR MAP\\ReferenceNode\\MapServices\\trmm_3hour.clr"
# output location for the raster catalog and each N-day cumulative raster, creates the FileGeoDatabase if it does not exist
trmm_fgdb = FileGeoDatabase("PATH TO FGDB \\ReferenceNode\\FileGeodatabases\\", "TRMM.gdb")
# spatial projection to apply to all rasters
spatial_projection = "GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]];-400 -400 1000000000;-100000 10000;-100000 10000;8.98315284119521E-09;0.001;0.001;IsHighPrecision"
# where the output Raster Catalog is saved
output_basepath = trmm_fgdb.fullpath
# get a reference to raster catalog, create one if it does not exist
raster_catalog = getRasterCatalog(output_basepath, spatial_projection)
# used to reference all start datetimes so that they are in sync with each other
start_datetime = datetime.utcnow()
end_datetime = start_datetime - timedelta(days=raster_catalog.options['archive_days'])
# execute the main ETL operation
is_successful_new_run = executeETL(raster_catalog, spatial_projection, start_datetime, end_datetime, color_map, update_debug_log)
if is_successful_new_run:
# refresh all services to update the data
trmm_agsm = ArcGISServiceManager({
'debug_logger':update_debug_log,
'server_name':'localhost',
'server_port':'6080',
'username':'******',
'password':'******',
'service_dir':'ReferenceNode',
'services':['TRMM.MapServer', 'TRMM_1DAY.MapServer', 'TRMM_7DAY.MapServer', 'TRMM_30DAY.MapServer']
})
update_debug_log("stopping services...")
trmm_agsm.stopServices()
createTRMMComposities(raster_catalog, output_basepath, start_datetime, color_map)
update_debug_log("starting services...")
trmm_agsm.startServices()
# delete outdated debug logs
etl_debug_logger.deleteOutdatedDebugLogs()
def main(*args, **kwargs):
debug_log_output_directory = os.path.join(sys.path[0], "Fire_logs")
etl_debug_logger = ETLDebugLogger(debug_log_output_directory, "Fire", {
"debug_log_archive_days":7
})
update_debug_log = etl_debug_logger.updateDebugLog # retrieve a reference to the debug logger function
# create the FileGeoDatabase if it does not already exist
fire_fgdb = FileGeoDatabase("PATH TO GEODATABASE ON DISK\\FileGeodatabases\\", "Fire.gdb", {
"compact_interval_days":7
})
feature_class_name = "global_fire"
# create the main fire feature class if it does not already exist
feature_class = createFeatureClass(fire_fgdb.fullpath, feature_class_name)
# execute the main ETL operation
is_successful_new_run = executeETL(feature_class, update_debug_log)
if is_successful_new_run:
fire_agsm = ArcGISServiceManager({
'debug_logger':update_debug_log,
'server_name':'localhost',
'server_port':'6080',
'username':'******',
'password':'******',
'service_dir':'ReferenceNode',
'services':['MODIS_Fire.MapServer', 'MODIS_Fire_1DAY.MapServer']
})
fire_agsm.refreshServices()
# this is ran due to the Fire.gdb having frequent updates and deletions
fire_fgdb.compactFileGeoDatabase()
# delete outdated debug logs
etl_debug_logger.deleteOutdatedDebugLogs()
def executeETL(raster_catalog):
# initialize utility objects -------------------------------------
debug_log_output_directory = os.path.join(sys.path[0], "MODIS_NDVI_logs")
etl_debug_logger = ETLDebugLogger(debug_log_output_directory, "MODIS_NDVI",
{"debug_log_archive_days": 7})
update_debug_log = etl_debug_logger.updateDebugLog # retrieve a reference to the debug logger function
etl_exception_manager = ETLExceptionManager(
sys.path[0], "MODIS_NDVI_exception_reports",
{"create_immediate_exception_reports": True})
# initialize core ETL objects -------------------------------------
start_datetime = datetime.utcnow()
end_datetime = start_datetime - timedelta(
days=raster_catalog.options['archive_days'])
image_extn = "tif" # target extension of MODIS images to downlaod
modis_extract_validator = MODISExtractValidator({
"raster_catalog":
raster_catalog,
"raster_name_field":
"Name",
"start_datetime":
start_datetime,
"end_datetime":
end_datetime,
'debug_logger':
update_debug_log
})
modis_extractor = MODISExtractor({
"image_content_types": [
'image/tiff'
], # checks the header content type for this value before downloading the images
"text_content_types": [
'text/html', 'text/plain'
], # checks the header content type for any of these values before downloading the meta-data
"subset": ['Bhutan', 'Nepal'],
"satellite": ['terra', 'aqua'],
"size": ['2km', '1km', '500m', '250m'],
"extn":
image_extn,
"subtype": [
'ndvi'
], # list only has one item since it is the category of the raster catalog the ETL is updating
"start_datetime":
start_datetime,
"end_datetime":
end_datetime,
'debug_logger':
update_debug_log
})
modis_meta_data_transformer = MODISMetaDataTransformer(
{'debug_logger': update_debug_log})
modis_loader = MODISLoader({
"raster_catalog": raster_catalog,
"CopyRaster_management_config": {
'config_keyword': '#',
'background_value': '#',
'nodata_value': '#',
'onebit_to_eightbit': 'NONE',
'colormap_to_RGB': 'NONE',
'pixel_type': '8_BIT_UNSIGNED'
},
'debug_logger': update_debug_log
})
etl_controller = ETLController(sys.path[0], "MODIS_NDVI",
{"remove_etl_workspace_on_finish": True})
modis_etl_delegate = MODISETLDelegate({
#MAY OR MAY NOT NEED TO UPDATE THIS URL
"url":
'http://rapidfire.sci.gsfc.nasa.gov/subsets/?subset=', # base URL for all images
"extn":
image_extn,
"meta_extn":
"txt",
"all_or_none_for_success":
False,
'debug_logger':
update_debug_log,
'exception_handler':
etl_exception_manager.handleException
})
# set ETLDelegate object properties-------------------------------------
modis_etl_delegate.setExtractValidator(modis_extract_validator)
modis_etl_delegate.setExtractor(modis_extractor)
modis_etl_delegate.setTransformer(modis_meta_data_transformer)
modis_etl_delegate.setLoader(modis_loader)
modis_etl_delegate.setETLController(etl_controller)
# execute the ETL operation -------------------------------------
successful_new_run = modis_etl_delegate.startETLProcess()
# perform post-ETL operations -------------------------------------
raster_catalog.deleteOutdatedRows()
etl_debug_logger.deleteOutdatedDebugLogs()
etl_exception_manager.finalizeExceptionXMLLog()
return successful_new_run
def executeETL(raster_catalog):
# initialize utility objects -------------------------------------
debug_log_output_directory = os.path.join(sys.path[0], "MODIS_721_logs")
etl_debug_logger = ETLDebugLogger(debug_log_output_directory, "MODIS_721", {
"debug_log_archive_days":7
})
update_debug_log = etl_debug_logger.updateDebugLog # retrieve a reference to the debug logger function
etl_exception_manager = ETLExceptionManager(sys.path[0], "MODIS_721_exception_reports", {
"create_immediate_exception_reports":True
})
# initialize core ETL objects -------------------------------------
start_datetime = datetime.utcnow()
end_datetime = start_datetime - timedelta(days=raster_catalog.options['archive_days'])
image_extn = "tif" # target extension for MODIS images to downlaod
modis_extract_validator = MODISExtractValidator({
"raster_catalog":raster_catalog,
"raster_name_field":"Name",
"start_datetime":start_datetime,
"end_datetime":end_datetime,
'debug_logger':update_debug_log
})
modis_extractor = MODISExtractor({
"image_content_types":['image/tiff'], # checks the header content type for this value before downloading the images
"text_content_types":['text/html', 'text/plain'], # checks the header content type for any of these values before downloading the meta-data
"subset":['Bhutan', 'Nepal'],
"satellite":['terra','aqua'],
"size":['2km','1km','500m','250m'],
"extn":image_extn,
"subtype":['721'], # list only has one item since it is the category of the raster catalog the ETL is updating
"start_datetime":start_datetime,
"end_datetime":end_datetime,
'debug_logger':update_debug_log
})
modis_meta_data_transformer = MODISMetaDataTransformer({
'debug_logger':update_debug_log
})
modis_loader = MODISLoader({
"raster_catalog":raster_catalog,
"CopyRaster_management_config":{
'config_keyword':'#',
'background_value':'#',
'nodata_value':'#',
'onebit_to_eightbit':'NONE',
'colormap_to_RGB':'NONE',
'pixel_type':'8_BIT_UNSIGNED'
},
'debug_logger':update_debug_log
})
etl_controller = ETLController(sys.path[0], "MODIS_721", {
"remove_etl_workspace_on_finish":True
})
modis_etl_delegate = MODISETLDelegate({
#URL MAY OR MAY NOT NEED TO BE UPDATED
"url":'http://rapidfire.sci.gsfc.nasa.gov/subsets/?subset=', # base URL for all images
"extn":image_extn,
"meta_extn":"txt",
"all_or_none_for_success":False,
'debug_logger':update_debug_log,
'exception_handler':etl_exception_manager.handleException
})
# set ETLDelegate object properties-------------------------------------
modis_etl_delegate.setExtractValidator(modis_extract_validator)
modis_etl_delegate.setExtractor(modis_extractor)
modis_etl_delegate.setTransformer(modis_meta_data_transformer)
modis_etl_delegate.setLoader(modis_loader)
modis_etl_delegate.setETLController(etl_controller)
# execute the ETL operation -------------------------------------
successful_new_run = modis_etl_delegate.startETLProcess()
# perform post-ETL operations -------------------------------------
raster_catalog.deleteOutdatedRows()
etl_debug_logger.deleteOutdatedDebugLogs()
etl_exception_manager.finalizeExceptionXMLLog()
return successful_new_run
def createTRMMComposities(raster_catalog, output_basepath, start_datetime, color_map):
# initialize utility objects -------------------------------------
debug_log_output_directory = os.path.join(sys.path[0], "TRMM_custom_raster_logs")
custom_raster_debug_logger = ETLDebugLogger(debug_log_output_directory, "custom_raster", {
'debug_log_archive_days':7
})
exception_manager = ExceptionManager(sys.path[0], "Raster_Exception_Reports", {
"create_immediate_exception_reports":True
})
custom_raster_debug_logger_ref = custom_raster_debug_logger.updateDebugLog
exception_handler_ref = exception_manager.handleException
# initialize request config objects -------------------------------------
factory_specifications = {
"AddColormap_management_config": { # optional, comment out/delete entire key if no color map is needed
"input_CLR_file":color_map
},
"CopyRaster_management_config":{
'config_keyword':'',
'background_value':'',
'nodata_value':'',
'onebit_to_eightbit':'',
'colormap_to_RGB':'',
'pixel_type':'16_BIT_UNSIGNED'
}
}
input_raster_catalog_options = {
'raster_catalog_fullpath':raster_catalog.fullpath,
"raster_name_field":'Name',
"datetime_field":raster_catalog.options['datetime_field'],
'datetime_sql_cast':raster_catalog.options['datetime_sql_cast'],
'datetime_field_format':raster_catalog.options['datetime_field_format'],
'start_datetime':start_datetime
}
# TRMM1Day config --------------------------------------------------------------------------------
factory_specifications_1day = deepcopy(factory_specifications)
factory_specifications_1day['output_raster_fullpath'] = os.path.join(output_basepath, "TRMM1Day")
factory_specifications_1day['AddColormap_management_config']['input_CLR_file'] = "PATH TO COLORMAP\\ReferenceNode\\MapServices\\trmm_1day.clr"
input_raster_catalog_options_1day = deepcopy(input_raster_catalog_options)
input_raster_catalog_options_1day['end_datetime'] = start_datetime - timedelta(days=1)
trmm_1day = TRMMCustomRasterRequest({
'factory_specifications':factory_specifications_1day,
'input_raster_catalog_options':input_raster_catalog_options_1day,
'debug_logger':custom_raster_debug_logger_ref,
'exception_handler':exception_handler_ref
})
# TRMM7Day config --------------------------------------------------------------------------------
factory_specifications_7day = deepcopy(factory_specifications)
factory_specifications_7day['output_raster_fullpath'] = os.path.join(output_basepath, "TRMM7Day")
factory_specifications_7day['AddColormap_management_config']['input_CLR_file'] = "PATH TO COLOR MAP\\ReferenceNode\\MapServices\\trmm_7day.clr"
input_raster_catalog_options_7day = deepcopy(input_raster_catalog_options)
input_raster_catalog_options_7day['end_datetime'] = start_datetime - timedelta(days=7)
trmm_7day = TRMMCustomRasterRequest({
'factory_specifications':factory_specifications_7day,
'input_raster_catalog_options':input_raster_catalog_options_7day,
'debug_logger':custom_raster_debug_logger_ref,
'exception_handler':exception_handler_ref
})
# TRMM30Day config --------------------------------------------------------------------------------
factory_specifications_30day = deepcopy(factory_specifications)
factory_specifications_30day['output_raster_fullpath'] = os.path.join(output_basepath, "TRMM30Day")
factory_specifications_30day['AddColormap_management_config']['input_CLR_file'] = "PATH TO COLOR MAP\\ReferenceNode\\MapServices\\TRMM_30Day.clr"
input_raster_catalog_options_30day = deepcopy(input_raster_catalog_options)
input_raster_catalog_options_30day['end_datetime'] = start_datetime - timedelta(days=30)
trmm_30day = TRMMCustomRasterRequest({
'factory_specifications':factory_specifications_30day,
'input_raster_catalog_options':input_raster_catalog_options_30day,
'debug_logger':custom_raster_debug_logger_ref,
'exception_handler':exception_handler_ref
})
# initialize object responsible for creating the TRMM composities
trmm_custom_raster_factory = TRMMCustomRasterCreator({
'workspace_fullpath':os.path.join(sys.path[0], "TRMMCustomRasters"),
'remove_all_rasters_on_finish':False,
'archive_options': {
'raster_name_prefix':"t_", # identify rasters to delete by this prefix
'local_raster_archive_days':30, # only keep rasters local within this many days
'raster_name_datetime_format':"t_%Y%m%d%H" # format of rasters to create a datetime object
},
'debug_logger':custom_raster_debug_logger_ref,
'exception_handler':exception_handler_ref
})
trmm_custom_raster_factory.addCustomRasterReuests([trmm_1day, trmm_7day, trmm_30day])
trmm_custom_raster_factory.createCustomRasters() # start the composite creation process
custom_raster_debug_logger.deleteOutdatedDebugLogs()
exception_manager.finalizeExceptionXMLLog()
def executeETL(feature_class):
etl_exception_manager = ETLExceptionManager(sys.path[0], "Fire_exception_reports", {
"create_immediate_exception_reports":True,
"delete_immediate_exception_reports_on_finish":True
})
debug_log_output_directory = "D:\\Logs\\ETL_Logs\\FIRE"
etl_debug_logger = ETLDebugLogger(debug_log_output_directory, "Fire", {
"debug_log_archive_days":7
})
update_debug_log = etl_debug_logger.updateDebugLog # retrieve a reference to the debug logger function
# KS Note 2014-07-23 Need global access to the Logger
global g_UpdateLog
g_UpdateLog = update_debug_log
start_datetime = datetime.datetime.utcnow()
end_datetime = start_datetime - timedelta(days=5) # this is how many days worth of CSVs it will retrieve from the ten-day rolling FTP
extract_validator = FireExtractValidator({
"feature_class":feature_class,
"satellite_ftp_directory_name_field":"SHORTNAME",
"ftp_file_name_field":"ftp_file_name", # field used to determine duplicates and current feature class membership
"start_datetime":start_datetime,
"end_datetime":end_datetime,
"debug_logger":update_debug_log
})
pkl_file = open('config.pkl', 'r')
myConfig = pickle.load(pkl_file) #store the data from config.pkl file
pkl_file.close()
extractor = FireExtractor({
"target_file_extn":"txt", # extension of fire CSV
"ftp_options": {
"ftp_host":myConfig['ftp_host'],
"ftp_user":myConfig['ftp_user'],
"ftp_pswrd":myConfig['ftp_pswrd'],
},
"debug_logger":update_debug_log
})
meta_data_transformer = FireMetaDataTransformer({"debug_logger":update_debug_log})
# projection used to create an XY event layer in arcpy.MakeXYEventLayer_management
spatial_projection = "GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]];-400 -400 1000000000;-100000 10000;-100000 10000;8.98315284119521E-09;0.001;0.001;IsHighPrecision"
transformer = FireTransformer({
'fire_fields':['LATITUDE','LONGITUDE','BRIGHTNESS','SCAN','TRACK','DATE','TIME','SATELLITE','CONFIDENCE','VERSION','BRIGHTT31','FRP'],
'admin_join_table_fullpath': os.path.join(sys.path[0], 'GAUL_LEVELS.gdb', 'g2006_2'),
'admin_fields':['ADM0_NAME', 'ADM1_NAME', 'ADM2_NAME'],
"CreateFeatureclass_management_config":{
'geometry_type':"POINT"
},
"MakeXYEventLayer_management_config":{
'in_x_field':'LONGITUDE',
'in_y_field':'LATITUDE',
'spatial_reference':spatial_projection
},
"debug_logger":update_debug_log
}, decoratee=meta_data_transformer) # NOTE: the transformer is decorating the meta_data_transformer to chain along transform() calls
loader = FireLoader({
"datetime_sql_cast":"date", # this is important when you change the underlying SQL database. PostGIS vs SQL for example.
"feature_class":feature_class,
"Append_management_config":{
'schema_type':"NO_TEST"
},
'satellite_mapping':{ # this is used to determine which satellite the granule came from in order to update the appropriate rows
"MYD14T":"A", # aqua
"MOD14T":"T" # terra
},
"debug_logger":update_debug_log
})
etl_controller = ETLController("Z:\\ETLscratch\\FIRE", "Fire_ETL", {
"remove_etl_workspace_on_finish":True
})
fire_etl_delegate = FireETLDelegate({
"ftp_dirs":['/allData/1/MOD14T/Recent/', '/allData/1/MYD14T/Recent/'],# iterate through both aqua and terra FTP directories
"ftp_file_meta_extn":"met", # extension of fire CSVs meta-data
"all_or_none_for_success":False,
"debug_logger":update_debug_log,
"exception_handler":etl_exception_manager.handleException
})
# set ETLDelegate object properties-------------------------------------
fire_etl_delegate.setExtractValidator(extract_validator)
fire_etl_delegate.setExtractor(extractor)
fire_etl_delegate.setTransformer(transformer)
fire_etl_delegate.setLoader(loader)
fire_etl_delegate.setETLController(etl_controller)
# execute the ETL operation -------------------------------------
successful_new_run = fire_etl_delegate.startETLProcess()
# perform post-ETL operations -------------------------------------
feature_class.deleteOutdatedRows()
etl_debug_logger.deleteOutdatedDebugLogs()
etl_exception_manager.finalizeExceptionXMLLog()
return successful_new_run
def main(*args, **kwargs):
# initialize debug logger instance -------------------------------------
debug_log_output_directory = os.path.join(sys.path[0], "TRMM_logs")
etl_debug_logger = ETLDebugLogger(debug_log_output_directory, "TRMM",
{"debug_log_archive_days": 7})
update_debug_log = etl_debug_logger.updateDebugLog # retrieve a reference to the debug logger function
# color map that is applied to each 3-hour raster as well as each N-day cumulative raster (except the 30 day)
color_map = "PATH TO RASTER COLOR MAP\\ReferenceNode\\MapServices\\trmm_3hour.clr"
# output location for the raster catalog and each N-day cumulative raster, creates the FileGeoDatabase if it does not exist
trmm_fgdb = FileGeoDatabase(
"PATH TO FGDB \\ReferenceNode\\FileGeodatabases\\", "TRMM.gdb")
# spatial projection to apply to all rasters
spatial_projection = "GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]];-400 -400 1000000000;-100000 10000;-100000 10000;8.98315284119521E-09;0.001;0.001;IsHighPrecision"
# where the output Raster Catalog is saved
output_basepath = trmm_fgdb.fullpath
# get a reference to raster catalog, create one if it does not exist
raster_catalog = getRasterCatalog(output_basepath, spatial_projection)
# used to reference all start datetimes so that they are in sync with each other
start_datetime = datetime.utcnow()
end_datetime = start_datetime - timedelta(
days=raster_catalog.options['archive_days'])
# execute the main ETL operation
is_successful_new_run = executeETL(raster_catalog, spatial_projection,
start_datetime, end_datetime, color_map,
update_debug_log)
if is_successful_new_run:
# refresh all services to update the data
trmm_agsm = ArcGISServiceManager({
'debug_logger':
update_debug_log,
'server_name':
'localhost',
'server_port':
'6080',
'username':
'******',
'password':
'******',
'service_dir':
'ReferenceNode',
'services': [
'TRMM.MapServer', 'TRMM_1DAY.MapServer', 'TRMM_7DAY.MapServer',
'TRMM_30DAY.MapServer'
]
})
update_debug_log("stopping services...")
trmm_agsm.stopServices()
createTRMMComposities(raster_catalog, output_basepath, start_datetime,
color_map)
update_debug_log("starting services...")
trmm_agsm.startServices()
# delete outdated debug logs
etl_debug_logger.deleteOutdatedDebugLogs()
def createTRMMComposities(raster_catalog, output_basepath, start_datetime,
color_map):
# initialize utility objects -------------------------------------
debug_log_output_directory = os.path.join(sys.path[0],
"TRMM_custom_raster_logs")
custom_raster_debug_logger = ETLDebugLogger(debug_log_output_directory,
"custom_raster",
{'debug_log_archive_days': 7})
exception_manager = ExceptionManager(
sys.path[0], "Raster_Exception_Reports",
{"create_immediate_exception_reports": True})
custom_raster_debug_logger_ref = custom_raster_debug_logger.updateDebugLog
exception_handler_ref = exception_manager.handleException
# initialize request config objects -------------------------------------
factory_specifications = {
"AddColormap_management_config":
{ # optional, comment out/delete entire key if no color map is needed
"input_CLR_file": color_map
},
"CopyRaster_management_config": {
'config_keyword': '',
'background_value': '',
'nodata_value': '',
'onebit_to_eightbit': '',
'colormap_to_RGB': '',
'pixel_type': '16_BIT_UNSIGNED'
}
}
input_raster_catalog_options = {
'raster_catalog_fullpath': raster_catalog.fullpath,
"raster_name_field": 'Name',
"datetime_field": raster_catalog.options['datetime_field'],
'datetime_sql_cast': raster_catalog.options['datetime_sql_cast'],
'datetime_field_format':
raster_catalog.options['datetime_field_format'],
'start_datetime': start_datetime
}
# TRMM1Day config --------------------------------------------------------------------------------
factory_specifications_1day = deepcopy(factory_specifications)
factory_specifications_1day['output_raster_fullpath'] = os.path.join(
output_basepath, "TRMM1Day")
factory_specifications_1day['AddColormap_management_config'][
'input_CLR_file'] = "PATH TO COLORMAP\\ReferenceNode\\MapServices\\trmm_1day.clr"
input_raster_catalog_options_1day = deepcopy(input_raster_catalog_options)
input_raster_catalog_options_1day[
'end_datetime'] = start_datetime - timedelta(days=1)
trmm_1day = TRMMCustomRasterRequest({
'factory_specifications':
factory_specifications_1day,
'input_raster_catalog_options':
input_raster_catalog_options_1day,
'debug_logger':
custom_raster_debug_logger_ref,
'exception_handler':
exception_handler_ref
})
# TRMM7Day config --------------------------------------------------------------------------------
factory_specifications_7day = deepcopy(factory_specifications)
factory_specifications_7day['output_raster_fullpath'] = os.path.join(
output_basepath, "TRMM7Day")
factory_specifications_7day['AddColormap_management_config'][
'input_CLR_file'] = "PATH TO COLOR MAP\\ReferenceNode\\MapServices\\trmm_7day.clr"
input_raster_catalog_options_7day = deepcopy(input_raster_catalog_options)
input_raster_catalog_options_7day[
'end_datetime'] = start_datetime - timedelta(days=7)
trmm_7day = TRMMCustomRasterRequest({
'factory_specifications':
factory_specifications_7day,
'input_raster_catalog_options':
input_raster_catalog_options_7day,
'debug_logger':
custom_raster_debug_logger_ref,
'exception_handler':
exception_handler_ref
})
# TRMM30Day config --------------------------------------------------------------------------------
factory_specifications_30day = deepcopy(factory_specifications)
factory_specifications_30day['output_raster_fullpath'] = os.path.join(
output_basepath, "TRMM30Day")
factory_specifications_30day['AddColormap_management_config'][
'input_CLR_file'] = "PATH TO COLOR MAP\\ReferenceNode\\MapServices\\TRMM_30Day.clr"
input_raster_catalog_options_30day = deepcopy(input_raster_catalog_options)
input_raster_catalog_options_30day[
'end_datetime'] = start_datetime - timedelta(days=30)
trmm_30day = TRMMCustomRasterRequest({
'factory_specifications':
factory_specifications_30day,
'input_raster_catalog_options':
input_raster_catalog_options_30day,
'debug_logger':
custom_raster_debug_logger_ref,
'exception_handler':
exception_handler_ref
})
# initialize object responsible for creating the TRMM composities
trmm_custom_raster_factory = TRMMCustomRasterCreator({
'workspace_fullpath':
os.path.join(sys.path[0], "TRMMCustomRasters"),
'remove_all_rasters_on_finish':
False,
'archive_options': {
'raster_name_prefix':
"t_", # identify rasters to delete by this prefix
'local_raster_archive_days':
30, # only keep rasters local within this many days
'raster_name_datetime_format':
"t_%Y%m%d%H" # format of rasters to create a datetime object
},
'debug_logger':
custom_raster_debug_logger_ref,
'exception_handler':
exception_handler_ref
})
trmm_custom_raster_factory.addCustomRasterReuests(
[trmm_1day, trmm_7day, trmm_30day])
trmm_custom_raster_factory.createCustomRasters(
) # start the composite creation process
custom_raster_debug_logger.deleteOutdatedDebugLogs()
exception_manager.finalizeExceptionXMLLog()
def executeETL(raster_catalog, land_cover_year_to_process):
# initialize utility objects -------------------------------------
debug_log_output_directory = os.path.join(sys.path[0], "Land_logs")
etl_debug_logger = ETLDebugLogger(debug_log_output_directory, "Land", {
"debug_log_archive_days":7
})
update_debug_log = etl_debug_logger.updateDebugLog # retrieve a reference to the debug logger function
etl_exception_manager = ETLExceptionManager(sys.path[0], "Land_exception_reports", {
"create_immediate_exception_reports":True
})
# initialize core ETL objects -------------------------------------
arcpy_land_extract_validator = LandExtractValidator({
"raster_catalog":raster_catalog,
"ftp_file_name_field":"DistributedFileName",
"debug_logger":update_debug_log
})
arcpy_land_extractor = LandExtractor({
"target_file_extn":'hdf',
"ftp_options": {
#THE SOURCE OF THE HDFs COLLECTED MAY NEED TO BE UPDATED
"ftp_host":'e4ftl01.cr.usgs.gov',
"ftp_user":'******',
"ftp_pswrd":'anonymous'
},
"debug_logger":update_debug_log
})
scratch_fgdb_fullpath = FileGeoDatabase(sys.path[0], 'scratch.gdb').fullpath
arcpy_land_transformer = LandTransformer({
"output_file_geodatabase":scratch_fgdb_fullpath,
"debug_logger":update_debug_log
})
land_meta_data_transformer = LandMetaDataTransformer(
{"debug_logger":update_debug_log},
decoratee=arcpy_land_transformer
)
arcpy_land_loader = LandLoader({
"raster_catalog":raster_catalog,
"CopyRaster_management_config":{
'config_keyword':'',
'background_value':'',
'nodata_value':'',
'onebit_to_eightbit':'',
'colormap_to_RGB':'',
'pixel_type':''
},
"debug_logger":update_debug_log
})
etl_controller = ETLController(sys.path[0], "LandCover_ETL", {
"remove_etl_workspace_on_finish":False
})
land_etl_delegate = LandETLDelegate({
"ftp_dirs":['/MOTA/MCD12Q1.005/'+land_cover_year_to_process+'.01.01/'],
"ftp_file_meta_extn":'xml',
"all_or_none_for_success":True,
"debug_logger":update_debug_log,
'exception_handler':etl_exception_manager.handleException
})
# set ETLDelegate object properties-------------------------------------
land_etl_delegate.setExtractValidator(arcpy_land_extract_validator)
land_etl_delegate.setExtractor(arcpy_land_extractor)
land_etl_delegate.setTransformer(land_meta_data_transformer)
land_etl_delegate.setLoader(arcpy_land_loader)
land_etl_delegate.setETLController(etl_controller)
# execute the ETL operation -------------------------------------
successful_new_run = land_etl_delegate.startETLProcess()
# perform post-ETL operations -------------------------------------
etl_exception_manager.finalizeExceptionXMLLog()
etl_debug_logger.deleteOutdatedDebugLogs()
return successful_new_run
def createLandCoverDataset(land_cover_year_to_process, land_cover_basepath, raster_catalog_fullpath):
# create debug logger instance -------------------------------------
debug_log_output_directory = os.path.join(sys.path[0], "Land_Raster_debug_logs")
land_raster_debug_logger = ETLDebugLogger(debug_log_output_directory, "LandRaster", {
"debug_log_archive_days":7
})
# land cover raster data set config -------------------------------------
output_raster_dataset = str("LandCover_Type1_" + land_cover_year_to_process)
output_dataset_fullpath = os.path.join(land_cover_basepath, output_raster_dataset)
land_raster_dataset_options = {
"land_cover_type":"LC1",
"land_cover_type_field":"land_cover_type",
"RasterCatalogToRasterDataset_management_config": {
"mosaic_type":'',
"colormap":"",
"order_by_field":'',
"ascending":'',
"Pixel_type":'',
"ColorBalancing":'',
"matchingMethod":'',
"ReferenceRaster":'',
"OID":''
},
# land cover classification config -------------------------------------
"land_cover_description_field":"igbp_class",
"land_type_value_field":"Value",
"land_type_description_dict": {
0:"Water",
1:"Evergreen Needleleaf forest",
2:"Evergreen Broadleaf forest",
3:"Deciduous Needleleaf forest",
4:"Deciduous Broadleaf forest",
5:"Mixed forest",
6:"Closed shrublands",
7:"Open shrublands",
8:"Woody savannas",
9:"Savannas",
10:"Grasslands",
11:"Permanent wetlands",
12:"Croplands",
13:"Urban and built-up",
14:"Cropland/Natural vegetation mosaic",
15:"Snow and ice",
16:"Barren or sparsely vegetated"
},
# re-project raster dataset config -------------------------------------
"gp_env_extent":arcpy.Extent(-20037507.2295943, -19971868.8804086, 20037507.2295943, 19971868.8804086),
"out_coor_system" :'PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0],AUTHORITY["EPSG",3857]]',
"reprojected_raster_dataset":os.path.join(land_cover_basepath, output_raster_dataset+"_WM"),
"debug_logger":land_raster_debug_logger.updateDebugLog
}
# create a LandCoverRasterDataset instance, set debug logger and start the creation process -------------------------------------
land_raster_dataset = LandCoverRasterDataset(output_dataset_fullpath, raster_catalog_fullpath, land_raster_dataset_options)
land_raster_dataset.createLandRasterDataset()