def select_data_for_output(self, metadata): """ Select a subset of data, from the given metadata, for output. Returns a tuple of: - a singleton list containing the the dictionary of selected data and - a list of field names (keys in the returned dictionary) to be output """ # copy the metadata and remove the fields in the skip list calculated = md_utils.get_calculated(metadata) if (not calculated): errMsg = "The 'calculated' data, required by this program, is missing from the input." raise errors.ProcessingError(errMsg) selected = calculated.copy() remove_entries(selected, ignore=self.skip_list) # list the fields in alphabetical order and, optionally, make one field primary fieldnames = sorted(list(selected.keys()), key=str.lower) if (self.FIRST_FIELD is not None): fieldnames.remove( self.FIRST_FIELD) # this must exist in fieldname list! fieldnames.insert(0, self.FIRST_FIELD) # return a tuple of the created data dictionary and list of fieldnames (keys) return ([selected], fieldnames)
def select_data_for_output(self, metadata): """ Select a subset of data, from the given metadata, for output. Returns a single dictionary of selected data. """ calculated = md_utils.get_calculated(metadata) if (not calculated): errMsg = "The 'calculated' data, required by this program, is missing from the input." raise errors.ProcessingError(errMsg) selected = calculated.copy() remove_entries(selected, ignore=self.skip_list) return selected # return selected and filtered dataset
def output_results(self, metadata): """ Store the given data into the configured database OR just output SQL to do so, depending on the 'output-only' flag. """ if (self._DEBUG): print("({}.output_results): ARGS={}".format( self.TOOL_NAME, self.args), file=sys.stderr) # get the iRods file path argument of the file to be annotated imd_path = self.args.get( 'irods_md_file', self.args.get('irods_fits_file')) # default is iRods input file if ((imd_path is None) or (not imd_path.strip())): errMsg = "A full iRods path to an annotatable iRods file must be specified." raise errors.ProcessingError(errMsg) # check the iRods metadata target file path for validity try: self.irods.getf(imd_path, absolute=True) except (CollectionDoesNotExist, DataObjectDoesNotExist, NoResultFound): errMsg = "Unable to find iRods file for metadata alteration at '{}'.".format( imd_path) raise errors.ProcessingError(errMsg) # extract the data to be output from the given metadata sink_data = self.get_data_for_output(metadata) # When adding/replacing metadata, skip items in the skip list, # otherwise do not skip any items: remove all user specified items remove_only = self.args.get('remove_only') or False if (not remove_only): remove_entries(sink_data, ignore=self.skip_list) # decide whether we are changing metadata in iRods or just outputting it output_only = self.args.get('output_only') if (not output_only): # if storing metadata to iRods self.update_metadata(imd_path, sink_data, remove_only) else: # else just outputting metadata oodata = dict() oodata['file_info'] = md_utils.get_file_info(metadata) oodata['to_sink'] = sink_data super().output_results(oodata)
def test_remove_entries_nohist(self): hdrs = { 'a': 1, 'b': 'bee', 'pi': 3.14159, 'AA': 'Milne', 'COMMENT': 'no comment', 'HISTORY': 'repeats', 'comment': 'A comment', 'history': 'again' } old_len = len(hdrs) mutils.remove_entries(hdrs, ignore=['HISTORY', 'history']) print(hdrs) assert len(hdrs) == (old_len - 2) assert 'HISTORY' not in hdrs assert 'history' not in hdrs assert 'COMMENT' in hdrs assert 'comment' in hdrs
def test_remove_entries_default(self): hdrs = { 'a': 1, 'b': 'bee', 'pi': 3.14159, 'AA': 'Milne', 'COMMENT': 'no comment', 'HISTORY': 'repeats', 'comment': 'A comment', 'history': 'again' } old_len = len(hdrs) mutils.remove_entries(hdrs) # modifies by side-effect print(hdrs) assert len(hdrs) == old_len assert 'HISTORY' in hdrs assert 'COMMENT' in hdrs assert 'history' in hdrs assert 'comment' in hdrs
def test_remove_entries_all(self): hdrs = { 'a': 1, 'b': 'bee', 'pi': 3.14159, 'AA': 'Milne', 'COMMENT': 'no comment', 'HISTORY': 'repeats', 'comment': 'A comment', 'history': 'again' } keys = list( hdrs.keys()).copy() # prevent change-during-iteration error mutils.remove_entries(hdrs, ignore=keys) print(hdrs) assert len(hdrs) == 0 assert 'HISTORY' not in hdrs assert 'history' not in hdrs assert 'COMMENT' not in hdrs assert 'comment' not in hdrs
def test_remove_entries_ignore_nonexist(self): hdrs = { 'a': 1, 'b': 'bee', 'pi': 3.14159, 'AA': 'Milne', 'COMMENT': 'no comment', 'HISTORY': 'repeats', 'comment': 'A comment', 'history': 'again' } old_len = len(hdrs) mutils.remove_entries(hdrs, ignore=['AAA', 'pikey']) # modifies by side-effect print(hdrs) assert len(hdrs) == old_len assert 'AA' in hdrs assert 'pi' in hdrs assert 'AAA' not in hdrs assert 'pikey' not in hdrs