def main(argv=None):
"""
The main method for the tool. This method is called from the command line,
processes the command line arguments and calls into the ImdTk library to do its work.
This main method takes no arguments so it can be called by setuptools.
"""
# the main method takes no arguments so it can be called by setuptools
if (argv is None): # if called by setuptools
argv = sys.argv[1:] # then fetch the arguments from the system
# setup command line argument parsing and add shared arguments
parser = argparse.ArgumentParser(
prog=TOOL_NAME,
formatter_class=argparse.RawTextHelpFormatter,
description=
'Add information about desired fields to a metadata structure and output it.'
)
cli_utils.add_shared_arguments(parser, TOOL_NAME)
cli_utils.add_input_file_argument(parser, TOOL_NAME)
cli_utils.add_fields_info_argument(parser, TOOL_NAME)
cli_utils.add_output_arguments(parser, TOOL_NAME)
# actually parse the arguments from the command line
args = vars(parser.parse_args(argv))
# if debugging, set verbose and echo input arguments
if (args.get('debug')):
args['verbose'] = True # if debug turn on verbose too
print("({}.main): ARGS={}".format(TOOL_NAME, args), file=sys.stderr)
# if input file path given, check the file path for validity
input_file = args.get('input_file')
cli_utils.check_input_file(
input_file, TOOL_NAME) # may system exit here and not return!
# if fields info file path given, check the file path for validity
fields_file = args.get('fields_file')
cli_utils.check_fields_file(fields_file,
TOOL_NAME) # may exit here and not return!
# add additional arguments to args
args['TOOL_NAME'] = TOOL_NAME
# call the task layer to process the given, validated input file
try:
task = FieldsInfoTask(args)
task.input_process_output()
except errors.ProcessingError as pe:
errMsg = "({}): ERROR: Processing Error ({}): {}".format(
TOOL_NAME, pe.error_code, pe.message)
print(errMsg, file=sys.stderr)
sys.exit(pe.error_code)
def test_add_fields_info_argument(self):
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
utils.add_fields_info_argument(parser, TOOL_NAME)
args = vars(parser.parse_args([]))
print(args)
assert 'fields_file' not in args # no default
args = vars(parser.parse_args(['-fi', 'fields_file.ini']))
print(args)
assert 'fields_file' in args
args = vars(
parser.parse_args(['--fields-info', '/fake/fields_file.ini']))
print(args)
assert 'fields_file' in args
def main(argv=None):
"""
The main method for the pipeline. This method is called from the command line,
processes the command line arguments and calls into the ImdTk library to do its work.
This main method takes no arguments so it can be called by setuptools.
"""
# the main method takes no arguments so it can be called by setuptools
if (argv is None): # if called by setuptools
argv = sys.argv[1:] # then fetch the arguments from the system
# setup command line argument parsing and add shared arguments
parser = argparse.ArgumentParser(
prog=TOOL_NAME,
formatter_class=argparse.RawTextHelpFormatter,
description=
'Pipeline to extract image metadata from iRods FITS files and store into a PostreSQL/JSON hybrid database.'
)
cli_utils.add_shared_arguments(parser, TOOL_NAME)
cli_utils.add_input_dir_argument(parser, TOOL_NAME)
cli_utils.add_hdu_argument(parser, TOOL_NAME)
cli_utils.add_ignore_list_argument(parser, TOOL_NAME)
cli_utils.add_aliases_argument(parser, TOOL_NAME)
cli_utils.add_fields_info_argument(parser, TOOL_NAME)
cli_utils.add_collection_argument(parser, TOOL_NAME)
cli_utils.add_report_format_argument(parser, TOOL_NAME)
cli_utils.add_output_arguments(parser, TOOL_NAME)
cli_utils.add_database_arguments(parser,
TOOL_NAME,
table_msg=DEFAULT_HYBRID_TABLE_NAME)
cli_utils.add_table_name_argument(parser, TOOL_NAME)
# actually parse the arguments from the command line
args = vars(parser.parse_args(argv))
# if debugging, set verbose and echo input arguments
if (args.get('debug')):
args['verbose'] = True # if debug turn on verbose too
print("({}.main): ARGS={}".format(TOOL_NAME, args), file=sys.stderr)
# add additional arguments to args
args['TOOL_NAME'] = TOOL_NAME
# get an instance of the iRods accessor class
firh = FitsIRodsHelper(args)
# instantiate the tasks which form the pipeline
irods_fits_image_mdTask = IRodsFitsImageMetadataTask(args, firh)
image_aliasesTask = ImageAliasesTask(args)
fields_infoTask = FieldsInfoTask(args)
irods_jwst_oc_calcTask = IRods_JWST_ObsCoreCalcTask(args, firh)
miss_reportTask = MissingFieldsTask(args)
jwst_pghyb_sinkTask = JWST_HybridPostgreSQLSink(args)
# get and check the required image directory path for validity
input_dir = args.get('input_dir')
if (not firh.collection_exists(input_dir)):
firh.cleanup() # cleanup resources opened here
cli_utils.irods_input_dir_exit(
TOOL_NAME, input_dir) # error exit out here: never returns
# make of list of absolute iRods file paths pointing to FITS files
ir_dir = firh.getc(input_dir, absolute=True)
irff_paths = [fpath for fpath in firh.gen_fits_file_paths(ir_dir)]
# call the pipeline on each FITS file in the input directory:
if (args.get('verbose')):
print("({}): Processing FITS files in '{}'.".format(
TOOL_NAME, input_dir),
file=sys.stderr)
proc_count = 0 # initialize count of processed files
for irff_path in irff_paths:
args[
'irods_fits_file'] = irff_path # reset the FITS file argument to next file
if (args.get('verbose')):
print("({}): Processing FITS file '{}'.".format(
TOOL_NAME, irff_path),
file=sys.stderr)
try:
jwst_pghyb_sinkTask.output_results( # sink: nothing returned
miss_reportTask.process( # report: passes data through
irods_jwst_oc_calcTask.process(
fields_infoTask.process(
image_aliasesTask.process(
irods_fits_image_mdTask.process(
None)))))) # metadata source
proc_count += 1 # increment count of processed files
except errors.UnsupportedType as ute:
errMsg = "({}): WARNING: Unsupported File Type ({}): {}".format(
TOOL_NAME, ute.error_code, ute.message)
print(errMsg, file=sys.stderr)
except errors.ProcessingError as pe:
errMsg = "({}): ERROR: Processing Error ({}): {}".format(
TOOL_NAME, pe.error_code, pe.message)
print(errMsg, file=sys.stderr)
# call cleanup method for tasks which opened resources
jwst_pghyb_sinkTask.cleanup()
irods_jwst_oc_calcTask.cleanup()
irods_fits_image_mdTask.cleanup()
firh.cleanup() # cleanup resources opened here
if (args.get('verbose')):
print("({}): Processed iRods {} FITS files.".format(
TOOL_NAME, proc_count),
file=sys.stderr)
def main(argv=None):
"""
The main method for the pipeline. This method is called from the command line,
processes the command line arguments and calls into the ImdTk library to do its work.
This main method takes no arguments so it can be called by setuptools.
"""
# the main method takes no arguments so it can be called by setuptools
if (argv is None): # if called by setuptools
argv = sys.argv[1:] # then fetch the arguments from the system
# setup command line argument parsing and add shared arguments
parser = argparse.ArgumentParser(
prog=TOOL_NAME,
formatter_class=argparse.RawTextHelpFormatter,
description=
'Pipeline to extract iRods FITS image metadata and store it into a PostreSQL/JSON hybrid database.'
)
cli_utils.add_shared_arguments(parser, TOOL_NAME)
cli_utils.add_irods_fits_file_argument(parser, TOOL_NAME)
cli_utils.add_hdu_argument(parser, TOOL_NAME)
cli_utils.add_ignore_list_argument(parser, TOOL_NAME)
cli_utils.add_aliases_argument(parser, TOOL_NAME)
cli_utils.add_fields_info_argument(parser, TOOL_NAME)
cli_utils.add_collection_argument(parser, TOOL_NAME)
cli_utils.add_report_format_argument(parser, TOOL_NAME)
cli_utils.add_output_arguments(parser, TOOL_NAME)
cli_utils.add_database_arguments(parser,
TOOL_NAME,
table_msg=DEFAULT_HYBRID_TABLE_NAME)
cli_utils.add_table_name_argument(parser, TOOL_NAME)
# actually parse the arguments from the command line
args = vars(parser.parse_args(argv))
# if debugging, set verbose and echo input arguments
if (args.get('debug')):
args['verbose'] = True # if debug turn on verbose too
print("({}.main): ARGS={}".format(TOOL_NAME, args), file=sys.stderr)
# add additional arguments to args
args['TOOL_NAME'] = TOOL_NAME
# get the iRods file path argument of the file to be opened
irff_path = args.get('irods_fits_file')
# the specified FITS file must have a valid FITS extension
cli_utils.check_irods_fits_file(
irff_path, TOOL_NAME) # may system exit here and not return!
# get an instance of the iRods accessor class
firh = FitsIRodsHelper(args)
# instantiate the tasks which form the pipeline
irods_fits_image_mdTask = IRodsFitsImageMetadataTask(args, firh)
image_aliasesTask = ImageAliasesTask(args)
fields_infoTask = FieldsInfoTask(args)
irods_jwst_oc_calcTask = IRods_JWST_ObsCoreCalcTask(args, firh)
miss_reportTask = MissingFieldsTask(args)
jwst_pghybrid_sinkTask = JWST_HybridPostgreSQLSink(args)
if (args.get('verbose')):
print("({}): Processing iRods FITS file '{}'.".format(
TOOL_NAME, irff_path),
file=sys.stderr)
# compose and call the pipeline tasks
try:
jwst_pghybrid_sinkTask.output_results( # sink: nothing returned
miss_reportTask.process( # report: passes data through
irods_jwst_oc_calcTask.process(
fields_infoTask.process(
image_aliasesTask.process(
irods_fits_image_mdTask.process(
None)))))) # metadata source
except errors.UnsupportedType as ute:
errMsg = "({}): WARNING: Unsupported File Type ({}): {}".format(
TOOL_NAME, ute.error_code, ute.message)
print(errMsg, file=sys.stderr)
sys.exit(ute.error_code)
except errors.ProcessingError as pe:
errMsg = "({}): ERROR: Processing Error ({}): {}".format(
TOOL_NAME, pe.error_code, pe.message)
print(errMsg, file=sys.stderr)
sys.exit(pe.error_code)
finally:
# call cleanup method for tasks which opened resources
jwst_pghybrid_sinkTask.cleanup()
irods_jwst_oc_calcTask.cleanup()
irods_fits_image_mdTask.cleanup()
firh.cleanup() # cleanup resources opened here
if (args.get('verbose')):
print("({}): Processed iRods FITS file '{}'.".format(
TOOL_NAME, irff_path),
file=sys.stderr)
def main(argv=None):
"""
The main method for the pipeline. This method is called from the command line,
processes the command line arguments and calls into the ImdTk library to do its work.
This main method takes no arguments so it can be called by setuptools.
"""
# the main method takes no arguments so it can be called by setuptools
if (argv is None): # if called by setuptools
argv = sys.argv[1:] # then fetch the arguments from the system
# setup command line argument parsing and add shared arguments
parser = argparse.ArgumentParser(
prog=TOOL_NAME,
formatter_class=argparse.RawTextHelpFormatter,
description=
'Pipeline to extract image metadata and store it into a PostreSQL database.'
)
cli_utils.add_shared_arguments(parser, TOOL_NAME)
cli_utils.add_input_dir_argument(parser, TOOL_NAME)
cli_utils.add_hdu_argument(parser, TOOL_NAME)
cli_utils.add_ignore_list_argument(parser, TOOL_NAME)
cli_utils.add_aliases_argument(parser, TOOL_NAME)
cli_utils.add_fields_info_argument(parser, TOOL_NAME)
cli_utils.add_collection_argument(parser, TOOL_NAME)
cli_utils.add_report_format_argument(parser, TOOL_NAME)
cli_utils.add_output_arguments(parser, TOOL_NAME)
cli_utils.add_database_arguments(parser, TOOL_NAME)
cli_utils.add_table_name_argument(parser, TOOL_NAME)
# actually parse the arguments from the command line
args = vars(parser.parse_args(argv))
# if debugging, set verbose and echo input arguments
if (args.get('debug')):
args['verbose'] = True # if debug turn on verbose too
print("({}.main): ARGS={}".format(TOOL_NAME, args), file=sys.stderr)
# check the required image directory path for validity
input_dir = args.get('input_dir')
cli_utils.check_input_dir(
input_dir, TOOL_NAME) # may system exit here and not return!
# add additional arguments to args
args['TOOL_NAME'] = TOOL_NAME
# instantiate the tasks which form the pipeline
fits_image_mdTask = FitsImageMetadataTask(args)
image_aliasesTask = ImageAliasesTask(args)
fields_infoTask = FieldsInfoTask(args)
jwst_oc_calcTask = JWST_ObsCoreCalcTask(args)
miss_reportTask = MissingFieldsTask(args)
jwst_pgsql_sinkTask = JWST_ObsCorePostgreSQLSink(args)
# call the pipeline on each FITS file in the input directory:
if (args.get('verbose')):
print("({}): Processing FITS files in '{}'.".format(
TOOL_NAME, input_dir),
file=sys.stderr)
proc_count = 0 # initialize count of processed files
for img_file in gen_fits_file_paths(input_dir):
args[
'fits_file'] = img_file # reset the FITS file argument to next file
if (args.get('verbose')):
print("({}): Processing FITS file '{}'.".format(
TOOL_NAME, img_file),
file=sys.stderr)
try:
jwst_pgsql_sinkTask.output_results( # sink: nothing returned
miss_reportTask.process( # report: passes data through
jwst_oc_calcTask.process(
fields_infoTask.process(
image_aliasesTask.process(
fits_image_mdTask.process(
None)))))) # metadata source
proc_count += 1 # increment count of processed files
except errors.UnsupportedType as ute:
errMsg = "({}): WARNING: Unsupported File Type ({}): {}".format(
TOOL_NAME, ute.error_code, ute.message)
print(errMsg, file=sys.stderr)
except errors.ProcessingError as pe:
errMsg = "({}): ERROR: Processing Error ({}): {}".format(
TOOL_NAME, pe.error_code, pe.message)
print(errMsg, file=sys.stderr)
if (args.get('verbose')):
print("({}): Processed {} FITS files.".format(TOOL_NAME, proc_count),
file=sys.stderr)