def main():
"""Execute the order disposition determination routine"""
cron_cfg = retrieve_cfg(CRON_CFG_FILENAME)
proc_cfg = retrieve_cfg(PROC_CFG_FILENAME)
# Configure and get the logger for this task
logger_filename = cron_cfg.get('logging', 'disposition_log_filename')
# Configure and get the logger for this task
logging.basicConfig(format=('%(asctime)s.%(msecs)03d %(process)d'
' %(levelname)-8s'
' %(filename)s:%(lineno)d:'
'%(funcName)s -- %(message)s'),
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
filename=logger_filename)
logger = logging.getLogger(LOGGER_NAME)
try:
determine_order_disposition(proc_cfg)
except Exception:
logger.exception('Processing failed')
sys.exit(1) # EXIT_FAILURE
sys.exit(0) # EXIT_SUCCESS
def main():
"""Some parameter and logging setup, then call the process routine
"""
# Create a command line argument parser
description = 'Main mapper for a request'
parser = ArgumentParser(description=description)
# Add our only options to determine if we are a developer or not
parser.add_argument('--developer',
action='store_true', dest='developer', default=False,
help='use a developer mode for sleeping')
# Parse the command line arguments
args = parser.parse_args()
proc_cfg = retrieve_cfg(PROC_CFG_FILENAME)
EspaLogging.configure_base_logger(filename=MAPPER_LOG_FILENAME)
# Initially set to the base logger
logger = EspaLogging.get_logger('base')
try:
# Joe-Developer doesn't want to wait so if set skip sleeping
developer_sleep_mode = args.developer
process(proc_cfg, developer_sleep_mode)
except Exception:
logger.exception('Processing failed stacktrace follows')
def main():
"""Execute the order disposition determination routine"""
description = 'Scene state disposition/advancement'
parser = ArgumentParser(description=description)
# Add our only options to determine if we are a developer or not
parser.add_argument('--user',
action='store', dest='username', default='ALL',
help='specific username to advance [ALL]')
# Parse the command line arguments
args = parser.parse_args()
username = args.username if args.username != 'ALL' else None
cron_cfg = retrieve_cfg(CRON_CFG_FILENAME)
proc_cfg = retrieve_cfg(PROC_CFG_FILENAME)
# Configure and get the logger for this task
logger_filename = cron_cfg.get('logging', 'disposition_log_filename')
# Configure and get the logger for this task
logging.basicConfig(format=('%(asctime)s.%(msecs)03d %(process)d'
' %(levelname)-8s'
' %(filename)s:%(lineno)d:'
'%(funcName)s -- %(message)s'),
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
filename=logger_filename)
logger = logging.getLogger(LOGGER_NAME)
try:
determine_order_disposition(proc_cfg, username)
except Exception:
logger.exception('Processing failed')
sys.exit(1) # EXIT_FAILURE
sys.exit(0) # EXIT_SUCCESS
def main ():
logger = logging.getLogger(__name__) # Get logger for the module.
# get the command line arguments
parser = OptionParser()
parser.add_option ('-s', '--start_year', type='int', dest='syear',
default=0, help='year for which to start pulling LAADS data')
parser.add_option ('-e', '--end_year', type='int', dest='eyear',
default=0, help='last year for which to pull LAADS data')
parser.add_option ('--today', dest='today', default=False,
action='store_true',
help='process LAADS data up through the most recent year and DOY')
msg = ('process or reprocess all LAADS data from today back to {}'
.format(START_YEAR))
parser.add_option ('--quarterly', dest='quarterly', default=False,
action='store_true', help=msg)
(options, args) = parser.parse_args()
syear = options.syear # starting year
eyear = options.eyear # ending year
today = options.today # process most recent year of data
quarterly = options.quarterly # process today back to START_YEAR
# check the arguments
if (today == False) and (quarterly == False) and \
(syear == 0 or eyear == 0):
msg = ('Invalid command line argument combination. Type --help '
'for more information.')
logger.error(msg)
return ERROR
# determine the auxiliary directory to store the data
auxdir = os.environ.get('L8_AUX_DIR')
if auxdir is None:
msg = 'L8_AUX_DIR environment variable not set... exiting'
logger.error(msg)
return ERROR
# Get the application token for the LAADS https interface. for ESPA
# systems, pull the token from the config file.
if TOKEN is None:
# ESPA Processing Environment
# Read ~/.usgs/espa/processing.conf to get the URL for the ESPA API.
# Connect to the ESPA API and get the application token for downloading
# the LAADS data from the internal database.
PROC_CFG_FILENAME = 'processing.conf'
proc_cfg = retrieve_cfg(PROC_CFG_FILENAME)
rpcurl = proc_cfg.get('processing', 'espa_api')
server = api_connect(rpcurl)
if server:
token = server.get_configuration('aux.downloads.laads.token')
else:
# Non-ESPA processing. TOKEN needs to be defined at the top of this
# script.
token = TOKEN
if token is None:
logger.error('Application token is None. This needs to be a valid '
'token provided for accessing the LAADS data. '
'https://ladsweb.modaps.eosdis.nasa.gov/tools-and-services/data-download-scripts/')
return ERROR
# if processing today then process the current year. if the current
# DOY is within the first month, then process the previous year as well
# to make sure we have all the recently available data processed.
if today:
msg = 'Processing LAADS data up to the most recent year and DOY.'
logger.info(msg)
now = datetime.datetime.now()
day_of_year = now.timetuple().tm_yday
eyear = now.year
if day_of_year <= 31:
syear = eyear - 1
else:
syear = eyear
elif quarterly:
msg = 'Processing LAADS data back to {}'.format(START_YEAR)
logger.info(msg)
eyear = now.year
syear = START_YEAR
msg = 'Processing LAADS data for {} - {}'.format(syear, eyear)
logger.info(msg)
for yr in range(eyear, syear-1, -1):
msg = 'Processing year: {}'.format(yr)
logger.info(msg)
status = getLadsData(auxdir, yr, today, token)
if status == ERROR:
msg = ('Problems occurred while processing LAADS data for year {}'
.format(yr))
logger.error(msg)
return ERROR
msg = 'LAADS processing complete.'
logger.info(msg)
return SUCCESS
def main():
"""Configures an order from the command line input and calls the
processing code using the order
"""
args = parse_command_line()
proc_cfg = config.retrieve_cfg(PROC_CFG_FILENAME)
proc_cfg = override_config(args, proc_cfg)
# Configure the base logger for this request
EspaLogging.configure_base_logger(filename=cli_log_filename(args))
# Configure the processing logger for this request
EspaLogging.configure(settings.PROCESSING_LOGGER,
order=args.order_id,
product=args.product_id,
debug=args.debug)
# CLI will use the base logger
logger = EspaLogging.get_logger('base')
logger.info('*** Begin ESPA Processing on host [{}] ***'.format(
socket.gethostname()))
# Set to error condition
proc_status = False
try:
# Extra command line validation
if args.pixel_size is not None and args.pixel_size_units is None:
raise CliError('Must specify --pixel-size-units if specifying'
' --pixel-size')
export_environment_variables(proc_cfg)
template = load_template(filename=TEMPLATE_FILENAME)
order = update_template(args=args, template=template)
# Change to the processing directory
current_directory = os.getcwd()
os.chdir(proc_cfg.get('processing', 'espa_work_dir'))
try:
# All processors are implemented in the processor module
pp = processor.get_instance(proc_cfg, order)
(destination_product_file, destination_cksum_file) = pp.process()
# Set to success condition
proc_status = True
finally:
# Change back to the previous directory
os.chdir(current_directory)
except Exception:
logger.exception('*** Errors during processing ***')
sys.exit(1)
finally:
logger.info('*** ESPA Processing Terminated ***')
if not args.bridge_mode:
archive_log_files(args, proc_cfg, proc_status)
def main():
"""Execute the core processing routine"""
cron_cfg = retrieve_cfg(CRON_CFG_FILENAME)
proc_cfg = retrieve_cfg(PROC_CFG_FILENAME)
# Create a command line argument parser
description = ('Builds and kicks-off hadoop jobs for the espa processing'
' system (to process product requests)')
parser = ArgumentParser(description=description)
# Add parameters
valid_priorities = queue_keys(cron_cfg)
valid_product_types = ['landsat', 'modis', 'plot']
parser.add_argument('--priority',
action='store',
dest='priority',
required=True,
choices=valid_priorities,
help='only process requests with this priority:'
' one of [{0}]'.format(', '.join(valid_priorities)))
parser.add_argument('--product-types',
action='store',
dest='product_types',
required=True,
nargs='+',
metavar='PRODUCT_TYPE',
help=('only process requests for the specified'
' product type(s)'))
parser.add_argument('--limit',
action='store',
dest='limit',
required=False,
default='500',
help='specify the max number of requests to process')
parser.add_argument('--user',
action='store',
dest='user',
required=False,
default=None,
help='only process requests for the specified user')
# Parse the command line arguments
args = parser.parse_args()
# Validate product_types
if ((set(['landsat', 'plot']) == set(args.product_types))
or (set(['modis', 'plot']) == set(args.product_types))
or (set(['landsat', 'modis', 'plot']) == set(args.product_types))):
print('Invalid --product-types: [plot] cannot be combined with any'
' other product types')
sys.exit(1) # EXIT_FAILURE
# Configure and get the logger for this task
logger_filename = cron_cfg.get('logging', 'log_filename')
if 'plot' in args.product_types:
logger_filename = cron_cfg.get('logging', 'plot_log_filename')
logger_format = ('%(asctime)s.%(msecs)03d %(process)d'
' %(levelname)-8s {0:>6}'
' %(filename)s:%(lineno)d:%(funcName)s'
' -- %(message)s'.format(args.priority.lower()))
# Setup the default logger format and level. Log to STDOUT.
logging.basicConfig(format=logger_format,
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
filename=logger_filename)
logger = logging.getLogger(LOGGER_NAME)
# Determine the appropriate priority value to use for the queue and request
queue_priority = args.priority.lower()
request_priority = queue_priority
if 'plot' in args.product_types:
if queue_priority == 'all':
# If all was specified default to high queue
queue_priority = 'high'
# The request priority is always None for plotting to get all of them
request_priority = None
else:
if request_priority == 'all':
# We need to use a value of None to get all of them
request_priority = None
# Setup and submit products to hadoop for processing
try:
process_requests(cron_cfg, proc_cfg, args, queue_priority,
request_priority)
except Exception:
logger.exception('Processing failed')
sys.exit(1) # EXIT_FAILURE
sys.exit(0) # EXIT_SUCCESS
def main():
"""Execute the core processing routine"""
cron_cfg = retrieve_cfg(CRON_CFG_FILENAME)
proc_cfg = retrieve_cfg(PROC_CFG_FILENAME)
# Create a command line argument parser
description = ('Builds and kicks-off hadoop jobs for the espa processing'
' system (to process product requests)')
parser = ArgumentParser(description=description)
# Add parameters
valid_priorities = queue_keys(cron_cfg)
valid_product_types = ['landsat', 'modis', 'plot']
parser.add_argument('--priority',
action='store', dest='priority', required=True,
choices=valid_priorities,
help='only process requests with this priority:'
' one of [{0}]'
.format(', '.join(valid_priorities)))
parser.add_argument('--product-types',
action='store', dest='product_types', required=True,
nargs='+', metavar='PRODUCT_TYPE',
help=('only process requests for the specified'
' product type(s)'))
parser.add_argument('--limit',
action='store', dest='limit', required=False,
default='500',
help='specify the max number of requests to process')
parser.add_argument('--user',
action='store', dest='user', required=False,
default=None,
help='only process requests for the specified user')
# Parse the command line arguments
args = parser.parse_args()
# Validate product_types
if ((set(['landsat', 'plot']) == set(args.product_types)) or
(set(['modis', 'plot']) == set(args.product_types)) or
(set(['landsat', 'modis', 'plot']) == set(args.product_types))):
print('Invalid --product-types: [plot] cannot be combined with any'
' other product types')
sys.exit(1) # EXIT_FAILURE
# Configure and get the logger for this task
logger_filename = cron_cfg.get('logging', 'log_filename')
if 'plot' in args.product_types:
logger_filename = cron_cfg.get('logging', 'plot_log_filename')
logger_format = ('%(asctime)s.%(msecs)03d %(process)d'
' %(levelname)-8s {0:>6}'
' %(filename)s:%(lineno)d:%(funcName)s'
' -- %(message)s'.format(args.priority.lower()))
# Setup the default logger format and level. Log to STDOUT.
logging.basicConfig(format=logger_format,
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
filename=logger_filename)
logger = logging.getLogger(LOGGER_NAME)
# Determine the appropriate priority value to use for the queue and request
queue_priority = args.priority.lower()
request_priority = queue_priority
if request_priority == 'all':
# We need to use a value of None to get all of them
request_priority = None
# Setup and submit products to hadoop for processing
try:
process_requests(cron_cfg, proc_cfg, args,
queue_priority, request_priority)
except Exception:
logger.exception('Processing failed')
sys.exit(1) # EXIT_FAILURE
sys.exit(0) # EXIT_SUCCESS
def main():
"""Configures an order from the command line input and calls the
processing code using the order
"""
args = parse_command_line()
proc_cfg = config.retrieve_cfg(PROC_CFG_FILENAME)
proc_cfg = override_config(args, proc_cfg)
# Configure the base logger for this request
EspaLogging.configure_base_logger(filename=cli_log_filename(args))
# Configure the processing logger for this request
EspaLogging.configure(settings.PROCESSING_LOGGER,
order=args.order_id,
product=args.product_id,
debug=args.debug)
# CLI will use the base logger
logger = EspaLogging.get_logger('base')
logger.info('*** Begin ESPA Processing on host [{}] ***'
.format(socket.gethostname()))
# Set to error condition
proc_status = False
try:
# Extra command line validation
if args.pixel_size is not None and args.pixel_size_units is None:
raise CliError('Must specify --pixel-size-units if specifying'
' --pixel-size')
export_environment_variables(proc_cfg)
template = load_template(filename=TEMPLATE_FILENAME)
order = update_template(args=args, template=template)
# Change to the processing directory
current_directory = os.getcwd()
os.chdir(proc_cfg.get('processing', 'espa_work_dir'))
try:
# All processors are implemented in the processor module
pp = processor.get_instance(proc_cfg, order)
(destination_product_file, destination_cksum_file) = pp.process()
# Set to success condition
proc_status = True
finally:
# Change back to the previous directory
os.chdir(current_directory)
except Exception:
logger.exception('*** Errors during processing ***')
sys.exit(1)
finally:
logger.info('*** ESPA Processing Terminated ***')
if not args.bridge_mode:
archive_log_files(args, proc_cfg, proc_status)
