def select_data_for_output(self, metadata):
"""
Select a subset of data, from the given metadata, for output.
Returns a tuple of:
- a singleton list containing the the dictionary of selected data and
- a list of field names (keys in the returned dictionary) to be output
"""
# copy the metadata and remove the fields in the skip list
calculated = md_utils.get_calculated(metadata)
if (not calculated):
errMsg = "The 'calculated' data, required by this program, is missing from the input."
raise errors.ProcessingError(errMsg)
selected = calculated.copy()
remove_entries(selected, ignore=self.skip_list)
# list the fields in alphabetical order and, optionally, make one field primary
fieldnames = sorted(list(selected.keys()), key=str.lower)
if (self.FIRST_FIELD is not None):
fieldnames.remove(
self.FIRST_FIELD) # this must exist in fieldname list!
fieldnames.insert(0, self.FIRST_FIELD)
# return a tuple of the created data dictionary and list of fieldnames (keys)
return ([selected], fieldnames)
def select_data_for_output(self, metadata):
"""
Select a subset of data, from the given metadata, for output.
Returns a single dictionary of selected data.
"""
calculated = md_utils.get_calculated(metadata)
if (not calculated):
errMsg = "The 'calculated' data, required by this program, is missing from the input."
raise errors.ProcessingError(errMsg)
selected = calculated.copy()
remove_entries(selected, ignore=self.skip_list)
return selected # return selected and filtered dataset
def output_results(self, metadata):
"""
Store the given data into the configured database OR just output SQL
to do so, depending on the 'output-only' flag.
"""
if (self._DEBUG):
print("({}.output_results): ARGS={}".format(
self.TOOL_NAME, self.args),
file=sys.stderr)
# get the iRods file path argument of the file to be annotated
imd_path = self.args.get(
'irods_md_file',
self.args.get('irods_fits_file')) # default is iRods input file
if ((imd_path is None) or (not imd_path.strip())):
errMsg = "A full iRods path to an annotatable iRods file must be specified."
raise errors.ProcessingError(errMsg)
# check the iRods metadata target file path for validity
try:
self.irods.getf(imd_path, absolute=True)
except (CollectionDoesNotExist, DataObjectDoesNotExist, NoResultFound):
errMsg = "Unable to find iRods file for metadata alteration at '{}'.".format(
imd_path)
raise errors.ProcessingError(errMsg)
# extract the data to be output from the given metadata
sink_data = self.get_data_for_output(metadata)
# When adding/replacing metadata, skip items in the skip list,
# otherwise do not skip any items: remove all user specified items
remove_only = self.args.get('remove_only') or False
if (not remove_only):
remove_entries(sink_data, ignore=self.skip_list)
# decide whether we are changing metadata in iRods or just outputting it
output_only = self.args.get('output_only')
if (not output_only): # if storing metadata to iRods
self.update_metadata(imd_path, sink_data, remove_only)
else: # else just outputting metadata
oodata = dict()
oodata['file_info'] = md_utils.get_file_info(metadata)
oodata['to_sink'] = sink_data
super().output_results(oodata)
def test_remove_entries_nohist(self):
hdrs = {
'a': 1,
'b': 'bee',
'pi': 3.14159,
'AA': 'Milne',
'COMMENT': 'no comment',
'HISTORY': 'repeats',
'comment': 'A comment',
'history': 'again'
}
old_len = len(hdrs)
mutils.remove_entries(hdrs, ignore=['HISTORY', 'history'])
print(hdrs)
assert len(hdrs) == (old_len - 2)
assert 'HISTORY' not in hdrs
assert 'history' not in hdrs
assert 'COMMENT' in hdrs
assert 'comment' in hdrs
def test_remove_entries_default(self):
hdrs = {
'a': 1,
'b': 'bee',
'pi': 3.14159,
'AA': 'Milne',
'COMMENT': 'no comment',
'HISTORY': 'repeats',
'comment': 'A comment',
'history': 'again'
}
old_len = len(hdrs)
mutils.remove_entries(hdrs) # modifies by side-effect
print(hdrs)
assert len(hdrs) == old_len
assert 'HISTORY' in hdrs
assert 'COMMENT' in hdrs
assert 'history' in hdrs
assert 'comment' in hdrs
def test_remove_entries_all(self):
hdrs = {
'a': 1,
'b': 'bee',
'pi': 3.14159,
'AA': 'Milne',
'COMMENT': 'no comment',
'HISTORY': 'repeats',
'comment': 'A comment',
'history': 'again'
}
keys = list(
hdrs.keys()).copy() # prevent change-during-iteration error
mutils.remove_entries(hdrs, ignore=keys)
print(hdrs)
assert len(hdrs) == 0
assert 'HISTORY' not in hdrs
assert 'history' not in hdrs
assert 'COMMENT' not in hdrs
assert 'comment' not in hdrs
def test_remove_entries_ignore_nonexist(self):
hdrs = {
'a': 1,
'b': 'bee',
'pi': 3.14159,
'AA': 'Milne',
'COMMENT': 'no comment',
'HISTORY': 'repeats',
'comment': 'A comment',
'history': 'again'
}
old_len = len(hdrs)
mutils.remove_entries(hdrs,
ignore=['AAA',
'pikey']) # modifies by side-effect
print(hdrs)
assert len(hdrs) == old_len
assert 'AA' in hdrs
assert 'pi' in hdrs
assert 'AAA' not in hdrs
assert 'pikey' not in hdrs
