def output_results(self, metadata):
"""
Output a selected subset of the given metadata in CSV format.
"""
if (self._DEBUG):
print("({}.output_results): ARGS={}".format(
self.TOOL_NAME, self.args),
file=sys.stderr)
# select and/or filter the data for output
outdata, fieldnames = self.select_data_for_output(metadata)
genfile = self.args.get('gen_file_path')
outfile = self.args.get('output_file')
if (genfile): # if generating the output filename/path
file_info = md_utils.get_file_info(metadata)
fname = file_info.get('file_name') if file_info else "NO_FILENAME"
outfile = self.gen_output_file_path(fname, CSV_EXTENSION,
self.TOOL_NAME)
self.output_CSV(outdata, fieldnames, outfile)
elif (outfile is not None): # else if using the given filepath
self.output_CSV(outdata, fieldnames, outfile)
else: # else using standard output
self.output_CSV(outdata, fieldnames)
if (self._VERBOSE):
out_dest = outfile if (outfile) else STDOUT_NAME
print("({}): Results output to '{}'".format(
self.TOOL_NAME, out_dest),
file=sys.stderr)
def output_results (self, indata):
"""
Store the given data into the configured database OR just output SQL
to do so, depending on the 'output-only' flag.
"""
if (self._DEBUG):
print("({}.output_results): ARGS={}".format(self.TOOL_NAME, self.args), file=sys.stderr)
# load the database configuration from a given or default file path
dbconfig_file = self.args.get('dbconfig_file') or DEFAULT_DBCONFIG_FILEPATH
dbconfig = self.load_sql_db_config(dbconfig_file)
# check table name to see if it is still available in the database
catalog_table = self.args.get('catalog_table')
if (not self.table_exists(dbconfig, catalog_table)):
errMsg = "Catalog table to fill '{}' does not exist.".format(catalog_table)
raise errors.ProcessingError(errMsg)
# file information is needed by the SQL generation methods below
file_info = md_utils.get_file_info(indata)
# read the catalog table data from the input data structure
data = md_utils.get_data(indata)
# Decide whether we are creating a table in the DB or just outputting SQL statements.
sql_only = self.args.get('output_only')
if (sql_only): # if just outputting SQL
self.write_table(dbconfig, data, catalog_table, file_info)
else: # else creating the table in the database
self.fill_table(dbconfig, data, catalog_table)
def output_results(self, metadata):
""" Output the given metadata in the configured output format. """
genfile = self.args.get('gen_file_path')
outfile = self.args.get('output_file')
out_fmt = self.args.get('output_format') or DEFAULT_OUTPUT_FORMAT
if (out_fmt == 'json'):
if (genfile): # if generating the output filename/path
file_info = md_utils.get_file_info(metadata)
fname = file_info.get(
'file_name') if file_info else "NO_FILENAME"
outfile = self.gen_output_file_path(fname, out_fmt,
self.TOOL_NAME)
self.output_JSON(metadata, outfile)
elif (outfile is not None): # else if using the given filepath
self.output_JSON(metadata, outfile)
else: # else using standard output
self.output_JSON(metadata)
else:
errMsg = "({}.process): Invalid output format '{}'.".format(
self.TOOL_NAME, out_fmt)
raise errors.ProcessingError(errMsg)
if (self._VERBOSE):
out_dest = outfile if (outfile) else STDOUT_NAME
print("({}): Results output to '{}'".format(
self.TOOL_NAME, out_dest),
file=sys.stderr)
def calc_access_url(self, metadata, calculations):
""" Use the given metadata to create the URL to fetch the file from the server. """
file_info = md_utils.get_file_info(metadata)
file_path = file_info.get('file_path') if file_info else None
if (file_path is not None):
access_url = "{0}{1}".format(IMAGE_FETCH_PREFIX, file_path)
calculations['access_url'] = access_url
def calc_access_estsize(metadata, calculations):
"""
Calculate the file size and estimated size (in kB) of the FITS image file to be accessed.
"""
file_info = md_utils.get_file_info(metadata)
file_size = file_info.get('file_size') if file_info else 0
calculations['file_size'] = file_size
calculations['access_estsize'] = round(file_size / 1000)
def test_get_file_info(self):
data = utils.get_file_info(self.metadata)
print(data)
assert data is not None
assert len(data) > 1
assert 'file_name' in data
assert 'file_size' in data
assert 'equinox' not in data
assert 'COMMENT' not in data
def set_default_target_name(self, defaults, metadata, calculations):
"""
Use the given metadata to create a fallback/default target name.
This version is a crude heuristic based on early JWST filenames.
"""
file_info = md_utils.get_file_info(metadata)
filename = file_info.get('file_name') if file_info else None
if (filename is not None):
if (filename.lower().startswith("goods_s")):
calculations['target_name'] = "goods_south"
elif (filename.lower().startswith("goods_n")):
calculations['target_name'] = "goods_north"
else:
calculations['target_name'] = "UNKNOWN"
def output_results(self, metadata):
"""
Store the given data into the configured database OR just output SQL
to do so, depending on the 'output-only' flag.
"""
if (self._DEBUG):
print("({}.output_results): ARGS={}".format(
self.TOOL_NAME, self.args),
file=sys.stderr)
# get the iRods file path argument of the file to be annotated
imd_path = self.args.get(
'irods_md_file',
self.args.get('irods_fits_file')) # default is iRods input file
if ((imd_path is None) or (not imd_path.strip())):
errMsg = "A full iRods path to an annotatable iRods file must be specified."
raise errors.ProcessingError(errMsg)
# check the iRods metadata target file path for validity
try:
self.irods.getf(imd_path, absolute=True)
except (CollectionDoesNotExist, DataObjectDoesNotExist, NoResultFound):
errMsg = "Unable to find iRods file for metadata alteration at '{}'.".format(
imd_path)
raise errors.ProcessingError(errMsg)
# extract the data to be output from the given metadata
sink_data = self.get_data_for_output(metadata)
# When adding/replacing metadata, skip items in the skip list,
# otherwise do not skip any items: remove all user specified items
remove_only = self.args.get('remove_only') or False
if (not remove_only):
remove_entries(sink_data, ignore=self.skip_list)
# decide whether we are changing metadata in iRods or just outputting it
output_only = self.args.get('output_only')
if (not output_only): # if storing metadata to iRods
self.update_metadata(imd_path, sink_data, remove_only)
else: # else just outputting metadata
oodata = dict()
oodata['file_info'] = md_utils.get_file_info(metadata)
oodata['to_sink'] = sink_data
super().output_results(oodata)
def output_results(self, metadata):
""" Output the given metadata in the configured output format. """
genfile = self.args.get('gen_file_path')
outfile = self.args.get('output_file')
if (genfile): # if generating the output filename/path
file_info = md_utils.get_file_info(metadata)
fname = file_info.get('file_name') if file_info else "NO_FILENAME"
outfile = self.gen_output_file_path(fname, PICKLE_EXTENSION,
self.TOOL_NAME)
self.output_pickle(metadata, outfile)
elif (outfile is not None): # else if using the given filepath
self.output_pickle(metadata, outfile)
else: # else trying to use standard output
errMsg = "Pickle cannot be written to {}.".format(STDOUT_NAME)
raise errors.ProcessingError(errMsg)
if (self._VERBOSE):
print("({}): Pickled data output to '{}'".format(
self.TOOL_NAME, outfile),
file=sys.stderr)
def output_results(self, metadata):
"""
Store the given data into the configured database OR just output SQL
to do so, depending on the 'output-only' flag.
"""
if (self._DEBUG):
print("({}.output_results): ARGS={}".format(
self.TOOL_NAME, self.args),
file=sys.stderr)
# file information is needed by the SQL generation methods below
file_info = md_utils.get_file_info(metadata)
# select and/or filter the data for output
outdata = self.select_data_for_output(metadata)
# decide whether we are storing data in the DB or just outputting SQL statements
sql_only = self.args.get('output_only')
if (sql_only): # if just outputting SQL
self.write_results(outdata, file_info)
else: # else storing data in a database
self.store_results(outdata, file_info)
def copy_file_info(metadata, calculations):
""" Copy all file informations fields to the calculations structure. """
file_info = md_utils.get_file_info(metadata)
if (file_info):
calculations.update(file_info)