def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True, ignore_keywords_overwrite=None): """ Compare ACSTOOLS output with "truth" using ``fitsdiff``. Parameters ---------- outputs : list of tuple A list of tuples, each containing filename (without path) of CALXXX output and truth, in that order. Example:: [('output1.fits', 'truth1.fits'), ('output2.fits', 'truth2.fits'), ...] atol, rtol : float Absolute and relative tolerance for data comparison. raise_error : bool Raise ``AssertionError`` if difference is found. ignore_keywords_overwrite : list of str or `None` If not `None`, these will overwrite ``self.ignore_keywords`` for the calling test. Returns ------- report : str Report from ``fitsdiff``. This is part of error message if ``raise_error=True``. """ all_okay = True creature_report = '' if ignore_keywords_overwrite is None: ignore_keywords = self.ignore_keywords else: ignore_keywords = ignore_keywords_overwrite for actual, desired in outputs: desired = get_bigdata('scsb-acstools', self.env, self.detector, 'truth', desired) fdiff = FITSDiff(actual, desired, rtol=rtol, atol=atol, ignore_keywords=ignore_keywords) creature_report += fdiff.report() if not fdiff.identical and all_okay: all_okay = False if not all_okay and raise_error: raise AssertionError(os.linesep + creature_report) return creature_report
def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True, ignore_keywords_overwrite=None): """ Compare ACSTOOLS output with "truth" using ``fitsdiff``. Parameters ---------- outputs : list of tuple A list of tuples, each containing filename (without path) of CALXXX output and truth, in that order. Example:: [('output1.fits', 'truth1.fits'), ('output2.fits', 'truth2.fits'), ...] atol, rtol : float Absolute and relative tolerance for data comparison. raise_error : bool Raise ``AssertionError`` if difference is found. ignore_keywords_overwrite : list of str or `None` If not `None`, these will overwrite ``self.ignore_keywords`` for the calling test. Returns ------- report : str Report from ``fitsdiff``. This is part of error message if ``raise_error=True``. """ all_okay = True creature_report = '' if ignore_keywords_overwrite is None: ignore_keywords = self.ignore_keywords else: ignore_keywords = ignore_keywords_overwrite for actual, desired in outputs: desired = get_bigdata('scsb-acstools', self.env, self.detector, 'truth', desired) fdiff = FITSDiff(actual, desired, rtol=rtol, atol=atol, ignore_keywords=ignore_keywords) creature_report += fdiff.report() if not fdiff.identical and all_okay: all_okay = False if not all_okay and raise_error: raise AssertionError(os.linesep + creature_report) return creature_report
def test_fitsdiff_openfile(tmpdir): """Make sure that failing FITSDiff doesn't leave open files.""" path1 = str(tmpdir.join("file1.fits")) path2 = str(tmpdir.join("file2.fits")) hdulist = HDUList([PrimaryHDU(), ImageHDU(data=np.zeros(5))]) hdulist.writeto(path1) hdulist[1].data[0] = 1 hdulist.writeto(path2) diff = FITSDiff(path1, path2) assert diff.identical, diff.report()
def difference(self): """Run fitsdiff on files named `old_file` and `new_file`. Returns: 0 no differences 1 some differences """ loc_old_file = self.locate_file(self.old_file) loc_new_file = self.locate_file(self.new_file) # Do the standard diff. fdiff = FITSDiff(loc_old_file, loc_new_file) # Do the diff by rows. rdiff = rowdiff.RowDiff(loc_old_file, loc_new_file) if not fdiff.identical: fdiff.report(fileobj=sys.stdout) print('\n', rdiff) return 0 if fdiff.identical else 1
def difference(self): """Run fitsdiff on files named `old_file` and `new_file`. Returns: 0 no differences 1 some differences """ loc_old_file = self.locate_file(self.old_file) loc_new_file = self.locate_file(self.new_file) # Do the standard diff. fdiff = FITSDiff(loc_old_file, loc_new_file) # Do the diff by rows. rdiff = rowdiff.RowDiff(loc_old_file, loc_new_file) if not fdiff.identical: fdiff.report(fileobj=sys.stdout) print('\n', rdiff) return 0 if fdiff.identical else 1
def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True, ignore_keywords_overwrite=None, verbose=True): """ Compare CALXXX output with "truth" using ``fitsdiff``. Parameters ---------- outputs : list of tuple A list of tuples, each containing filename (without path) of CALXXX output and truth, in that order. Example:: [('output1.fits', 'truth1.fits'), ('output2.fits', 'truth2.fits'), ...] atol, rtol : float Absolute and relative tolerance for data comparison. raise_error : bool Raise ``AssertionError`` if difference is found. ignore_keywords_overwrite : list of str or `None` If not `None`, these will overwrite ``self.ignore_keywords`` for the calling test. verbose : bool Print extra info to screen. Returns ------- report : str Report from ``fitsdiff``. This is part of error message if ``raise_error=True``. """ all_okay = True creature_report = '' updated_outputs = [] # To track outputs for Artifactory JSON schema if ignore_keywords_overwrite is None: ignore_keywords = self.ignore_keywords else: ignore_keywords = ignore_keywords_overwrite for actual, desired in outputs: desired = get_bigdata(self.env, self.instrument, self.detector, 'truth', desired) fdiff = FITSDiff(actual, desired, rtol=rtol, atol=atol, ignore_keywords=ignore_keywords) creature_report += fdiff.report() if not fdiff.identical: all_okay = False # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual, desired)) if not all_okay: if self.results_root is not None: # pragma: no cover schema_pattern, tree, testname = generate_upload_params( self.results_root, updated_outputs, verbose=verbose) generate_upload_schema(schema_pattern, tree, testname) if raise_error: raise AssertionError(os.linesep + creature_report) return creature_report
def compare_outputs(self, outputs, raise_error=True, delete_history=False): """ Compare output with "truth" using appropriate diff routine; namely, ``fitsdiff`` for FITS file comparisons ``unified_diff`` for ASCII products. Parameters ---------- outputs : list of tuple A list of tuples, each containing filename (without path) of CALXXX output and truth, in that order. raise_error : bool Raise ``AssertionError`` if difference is found. Returns ------- report : str Report from ``fitsdiff``. This is part of error message if ``raise_error=True``. """ all_okay = True creature_report = '' # Create instructions for uploading results to artifactory for use # as new comparison/truth files testpath, testname = os.path.split(os.path.abspath(os.curdir)) # organize results by day test was run...could replace with git-hash whoami = getpass.getuser() or 'nobody' dt = datetime.datetime.now().strftime("%d%b%YT") ttime = datetime.datetime.now().strftime("%H_%M_%S") user_tag = 'NOT_CI_{}_{}'.format(whoami, ttime) build_tag = os.environ.get('BUILD_TAG', user_tag) build_suffix = os.environ.get('BUILD_MATRIX_SUFFIX', 'standalone') testdir = "{}_{}_{}".format(testname, build_tag, build_suffix) tree = os.path.join(self.results_root, self.input_loc, dt, testdir) + os.sep updated_outputs = [] for actual, desired in outputs: # Get "truth" image s = self.get_data('truth', desired) if s is not None: desired = s if actual.endswith('fits'): # Working with FITS files... if delete_history is True: actual = fits.open(actual) desired = fits.open(desired) if 'HISTORY' in actual[0].header: del actual[0].header['HISTORY'] if 'HISTORY' in desired[0].header: del desired[0].header['HISTORY'] fdiff = FITSDiff(actual, desired, rtol=self.rtol, atol=self.atol, ignore_keywords=self.ignore_keywords) if delete_history is True: actual.close() desired.close() creature_report += fdiff.report() if not fdiff.identical: # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual, desired)) if not fdiff.identical and all_okay: all_okay = False else: # Try ASCII-based diff with open(actual) as afile: actual_lines = afile.readlines() with open(desired) as dfile: desired_lines = dfile.readlines() udiff = unified_diff(actual_lines, desired_lines, fromfile=actual, tofile=desired) old_stdout = sys.stdout udiffIO = StringIO() sys.stdout = udiffIO sys.stdout.writelines(udiff) sys.stdout = old_stdout udiff_report = udiffIO.getvalue() creature_report += udiff_report if len(udiff_report) > 2 and all_okay: all_okay = False if len(udiff_report) > 2: # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual, desired)) if not all_okay: # Write out JSON file to enable retention of different results new_truths = [os.path.abspath(i[1]) for i in updated_outputs] for files in updated_outputs: print("Renaming {} as new 'truth' file: {}".format( files[0], files[1])) shutil.move(files[0], files[1]) log_pattern = [ os.path.join(os.path.dirname(x), '*.log') for x in new_truths ] upload_results(pattern=new_truths + log_pattern, testname=testname, target=tree) if not all_okay and raise_error: raise AssertionError(os.linesep + creature_report) return creature_report
def compare_outputs(outputs, raise_error=True, ignore_keywords=[], ignore_hdus=[], ignore_fields=[], rtol=0.0, atol=0.0, input_path=[], docopy=True, results_root=None, verbose=True): """ Compare output with "truth" using appropriate diff routine; namely: * ``fitsdiff`` for FITS file comparisons. * ``unified_diff`` for ASCII products. Only after all elements of ``outputs`` have been processed will the method report any success or failure, with failure of any one comparison *not* preventing the rest of the comparisons to be performed. Parameters ---------- outputs : list of tuple or dict This list defines what outputs from running the test will be compared. Three distinct types of values as list elements are supported: * 2-tuple : ``(test output filename, truth filename)`` * 3-tuple : ``(test output filename, truth filename, HDU names)`` * dict : ``{'files': (output, truth), 'pars': {key: val}}`` If filename contains extension such as ``[hdrtab]``, it will be interpreted as specifying comparison of just that HDU. raise_error : bool Raise ``AssertionError`` if difference is found. ignore_keywords : list of str List of FITS header keywords to be ignored by ``FITSDiff`` and ``HDUDiff``. ignore_hdus : list of str List of FITS HDU names to ignore by ``FITSDiff``. This is only available for ``astropy>=3.1``. ignore_fields : list of str List FITS table column names to be ignored by ``FITSDiff`` and ``HDUDiff``. rtol, atol : float Relative and absolute tolerance to be used by ``FITSDiff`` and ``HDUDiff``. input_path : list or tuple A series of sub-directory names under :func:`get_bigdata_root` that leads to the path of the 'truth' files to be compared against. If not provided, it assumes that 'truth' is in the working directory. For example, with :func:`get_bigdata_root` pointing to ``/grp/test_data``, a file at:: /grp/test_data/pipeline/dev/ins/test_1/test_a.py would require ``input_path`` of:: ["pipeline", "dev", "ins", "test_1"] docopy : bool If `True`, 'truth' will be copied to output directory before comparison is done. results_root : str or `None` If not `None`, for every failed comparison, the test output is automatically renamed to the given 'truth' in the output directory and :func:`generate_upload_schema` will be called to generate a JSON scheme for Artifactory upload. If you do not need this functionality, use ``results_root=None``. verbose : bool Print extra info to screen. Returns ------- creature_report : str Report from FITS or ASCII comparator. This is part of error message if ``raise_error=True``. Examples -------- There are multiple use cases for this method, specifically related to how ``outputs`` are defined upon calling this method. The specification of the ``outputs`` can be any combination of the following patterns: 1. 2-tuple inputs:: outputs = [('file1.fits', 'file1_truth.fits')] This definition indicates that ``file1.fits`` should be compared as a whole with ``file1_truth.fits``. 2. 2-tuple inputs with extensions:: outputs = [('file1.fits[hdrtab]', 'file1_truth.fits[hdrtab]')] This definition indicates that only the HDRTAB extension from ``file1.fits`` will be compared to the HDRTAB extension from ``file1_truth.fits``. 3. 3-tuple inputs:: outputs = [('file1.fits', 'file1_truth.fits', ['primary', 'sci'])] This definition indicates that only the PRIMARY and SCI extensions should be compared between the two files. This creates a temporary ``HDUList`` object comprising only the given extensions for comparison. 4. Dictionary of inputs and parameters:: outputs = [{'files': ('file1.fits', 'file1_truth.fits'), 'pars': {'ignore_keywords': ['ROOTNAME']}}] This definition indicates that ROOTNAME will be ignored during the comparison between the files specified in ``'files'``. Any input parameter for ``FITSDiff`` or ``HDUDiff`` can be specified as part of the ``'pars'`` dictionary. In addition, the input files listed in ``'files'`` can also include an extension specification, such as ``[hdrtab]``, to limit the comparison to just that extension. This example from an actual test definition demonstrates how multiple input defintions can be used at the same time:: outputs = [ ('jw99999_nircam_f140m-maskbar_psfstack.fits', 'jw99999_nircam_f140m-maskbar_psfstack_ref.fits' ), ('jw9999947001_02102_00002_nrcb3_a3001_crfints.fits', 'jw9999947001_02102_00002_nrcb3_a3001_crfints_ref.fits' ), {'files': ('jw99999_nircam_f140m-maskbar_i2d.fits', 'jw99999_nircam_f140m-maskbar_i2d_ref.fits'), 'pars': {'ignore_hdus': ['HDRTAB']}, {'files': ('jw99999_nircam_f140m-maskbar_i2d.fits', 'jw99999_nircam_f140m-maskbar_i2d_ref.fits', ['primary','sci','dq']), 'pars': {'rtol': 0.000001} }, {'files': ('jw99999_nircam_f140m-maskbar_i2d.fits[hdrtab]', 'jw99999_nircam_f140m-maskbar_i2d_ref.fits[hdrtab]'), 'pars': {'ignore_keywords': ['NAXIS1', 'TFORM*'], 'ignore_fields': ['COL1', 'COL2']} }] .. note:: Each ``outputs`` entry in the list gets interpreted and processed separately. """ __tracebackhide__ = True default_kwargs = {'rtol': rtol, 'atol': atol, 'ignore_keywords': ignore_keywords, 'ignore_fields': ignore_fields, 'ignore_hdus': ignore_hdus} all_okay = True creature_report = '' updated_outputs = [] # To track outputs for Artifactory JSON schema for entry in outputs: diff_kwargs = copy.deepcopy(default_kwargs) extn_list = None num_entries = len(entry) if isinstance(entry, dict): entry_files = entry['files'] actual = entry_files[0] desired = entry_files[1] if len(entry_files) > 2: extn_list = entry_files[2] diff_kwargs.update(entry.get('pars', {})) elif num_entries == 2: actual, desired = entry elif num_entries == 3: actual, desired, extn_list = entry else: all_okay = False creature_report += '\nERROR: Cannot handle entry {}\n'.format( entry) continue # TODO: Use regex? if actual.endswith(']'): if extn_list is not None: all_okay = False creature_report += ( '\nERROR: Ambiguous extension requirements ' 'for {} ({})\n'.format(actual, extn_list)) continue actual_name, actual_extn = actual.split('[') actual_extn = actual_extn.replace(']', '') else: actual_name = actual actual_extn = None if desired.endswith(']'): if extn_list is not None: all_okay = False creature_report += ( '\nERROR: Ambiguous extension requirements ' 'for {} ({})\n'.format(desired, extn_list)) continue desired_name, desired_extn = desired.split('[') desired_extn = desired_extn.replace(']', '') else: desired_name = desired desired_extn = None actual = os.path.abspath(actual) # Get "truth" image try: os.makedirs('truth', exist_ok=True) os.chdir('truth') desired = get_bigdata(*input_path, desired_name, docopy=docopy) desired = os.path.abspath(desired) os.chdir('..') except BigdataError: all_okay = False creature_report += '\nERROR: Cannot find {} in {}\n'.format( desired_name, input_path) continue if desired_extn is not None: desired_name = desired desired = "{}[{}]".format(desired, desired_extn) if verbose: print("\nComparing:\n {}\n {}".format(actual, desired)) if actual.endswith('.fits') and desired.endswith('.fits'): # Build HDULists for comparison based on user-specified extensions if extn_list is not None: with fits.open(actual) as f_act: with fits.open(desired) as f_des: actual_hdu = fits.HDUList( [f_act[extn] for extn in extn_list]) actual_hdu.filename = lambda: os.path.basename(actual) desired_hdu = fits.HDUList( [f_des[extn] for extn in extn_list]) desired_hdu.filename = lambda: os.path.basename(desired) fdiff = FITSDiff(actual_hdu, desired_hdu, **diff_kwargs) creature_report += '\na: {}\nb: {}\n'.format( actual, desired) # diff report only gives hash # Working with FITS files... else: fdiff = FITSDiff(actual, desired, **diff_kwargs) creature_report += fdiff.report() if not fdiff.identical: all_okay = False # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual, desired)) elif actual_extn is not None or desired_extn is not None: if 'ignore_hdus' in diff_kwargs: # pragma: no cover diff_kwargs.pop('ignore_hdus') # Not applicable # Specific element of FITS file specified with fits.open(actual_name) as f_act: with fits.open(desired_name) as f_des: actual_hdu = f_act[actual_extn] desired_hdu = f_des[desired_extn] fdiff = HDUDiff(actual_hdu, desired_hdu, **diff_kwargs) creature_report += 'a: {}\nb: {}\n'.format(actual, desired) creature_report += fdiff.report() if not fdiff.identical: all_okay = False # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual_name, desired_name)) else: # ASCII-based diff with open(actual) as afile: actual_lines = afile.readlines() with open(desired) as dfile: desired_lines = dfile.readlines() udiff = unified_diff(actual_lines, desired_lines, fromfile=actual, tofile=desired) udiffIO = StringIO() udiffIO.writelines(udiff) udiff_report = udiffIO.getvalue() udiffIO.close() if len(udiff_report) == 0: creature_report += ('\na: {}\nb: {}\nNo differences ' 'found.\n'.format(actual, desired)) else: all_okay = False creature_report += udiff_report # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual, desired)) if not all_okay and results_root is not None: # pragma: no cover schema_pattern, tree, testname = generate_upload_params( results_root, updated_outputs, verbose=verbose) generate_upload_schema(schema_pattern, tree, testname) if not all_okay and raise_error: raise AssertionError(os.linesep + creature_report) return creature_report
caldbfile = find(fbase.replace("_marx", ""), CALDB) call("../utils/mksubpix.sl {0} {1}_new".format(caldbfile, f), shell=True) diff = FITSDiff(f, f + "_new", ignore_keywords=ignore_key) elif fbase.startswith("acisD1999-08-13contamN00"): caldbfile = find(fbase.replace("_marx", ""), CALDB) call("python ../utils/mkcontam.py {0} {1}_new".format(caldbfile, f), shell=True) diff = FITSDiff(f, f + "_new", ignore_keywords=ignore_key) else: print ("Don't know how to compare {0} - check manually".format(fbase)) if caldbfile is None: print ("No comparison for {0} found".format(f)) print ("Check CALDB manually!") else: if not diff.identical: print diff.report() else: print ("{0} is up to date.".format(f)) print ( """MARX uses some CALDB files directly, others are derived from CALDB (e.g. by reordering). This script has made new versions of those generated files, called *.fits_new If no differences were found, those file can be deleted. If any of these calibration files have changed, that probably also has an effect on /hrma/corr*.dat. See utils/Makefile for instructions on how to redo those. If any file name changes, edit marxcaldb.dat accordingly. """
def compare_outputs(self, outputs, raise_error=True): """ Compare output with "truth" using appropriate diff routine; namely, ``fitsdiff`` for FITS file comparisons ``unified_diff`` for ASCII products. Parameters ---------- outputs : list of tuple A list of tuples, each containing filename (without path) of CALXXX output and truth, in that order. raise_error : bool Raise ``AssertionError`` if difference is found. Returns ------- report : str Report from ``fitsdiff``. This is part of error message if ``raise_error=True``. """ all_okay = True creature_report = '' # Create instructions for uploading results to artifactory for use # as new comparison/truth files testpath, testname = os.path.split(os.path.abspath(os.curdir)) # organize results by day test was run...could replace with git-hash whoami = getpass.getuser() or 'nobody' dt = datetime.datetime.now().strftime("%d%b%YT") ttime = datetime.datetime.now().strftime("%H_%M_%S") user_tag = 'NOT_CI_{}_{}'.format(whoami, ttime) build_tag = os.environ.get('BUILD_TAG', user_tag) build_suffix = os.environ.get('BUILD_MATRIX_SUFFIX', 'standalone') testdir = "{}_{}_{}".format(testname, build_tag, build_suffix) tree = os.path.join(self.results_root, self.input_loc, dt, testdir) + os.sep updated_outputs = [] for actual, desired in outputs: # Get "truth" image s = self.get_data('truth', desired) if s is not None: desired = s if actual.endswith('fits'): # Working with FITS files... fdiff = FITSDiff(actual, desired, rtol=self.rtol, atol=self.atol, ignore_keywords=self.ignore_keywords) creature_report += fdiff.report() if not fdiff.identical: # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual, desired)) if not fdiff.identical and all_okay: all_okay = False else: # Try ASCII-based diff with open(actual) as afile: actual_lines = afile.readlines() with open(desired) as dfile: desired_lines = dfile.readlines() udiff = unified_diff(actual_lines, desired_lines, fromfile=actual, tofile=desired) old_stdout = sys.stdout udiffIO = StringIO() sys.stdout = udiffIO sys.stdout.writelines(udiff) sys.stdout = old_stdout udiff_report = udiffIO.getvalue() creature_report += udiff_report if len(udiff_report) > 2 and all_okay: all_okay = False if len(udiff_report) > 2: # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual, desired)) if not all_okay: # Write out JSON file to enable retention of different results new_truths = [os.path.abspath(i[1]) for i in updated_outputs] for files in updated_outputs: print("Renaming {} as new 'truth' file: {}".format( files[0], files[1])) shutil.move(files[0], files[1]) log_pattern = [os.path.join(os.path.dirname(x), '*.log') for x in new_truths] generate_upload_schema(pattern=new_truths + log_pattern, testname=testname, target= tree) if not all_okay and raise_error: raise AssertionError(os.linesep + creature_report) return creature_report
caldbfile = find(fbase.replace('_marx', ''), CALDB) call('../utils/mksubpix.sl {0} {1}_new'.format(caldbfile, f), shell = True) diff = FITSDiff(f, f + '_new', ignore_keywords=ignore_key) elif fbase.startswith('acisD1999-08-13contamN00'): caldbfile = find(fbase.replace('_marx', ''), CALDB) call('python ../utils/mkcontam.py {0} {1}_new'.format(caldbfile, f), shell = True) diff = FITSDiff(f, f + '_new', ignore_keywords=ignore_key) else: print("Don't know how to compare {0} - check manually".format(fbase)) if caldbfile is None: print('No comparison for {0} found'.format(f)) print('Check CALDB manually!') else: if not diff.identical: print diff.report() else: print('{0} is up to date.' .format(f)) print('''MARX uses some CALDB files directly, others are derived from CALDB (e.g. by reordering). This script has made new versions of those generated files, called *.fits_new If no differences were found, those file can be deleted. If any of these calibration files have changed, that probably also has an effect on /hrma/corr*.dat. See utils/Makefile for instructions on how to redo those. If any file name changes, edit marxcaldb.dat, Markefile.in, and marx.lis accordingly; then do git rm, git add ''')
def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True, ignore_keywords_overwrite=None, verbose=True): """ Compare CALXXX output with "truth" using ``fitsdiff``. Parameters ---------- outputs : list of tuple A list of tuples, each containing filename (without path) of CALXXX output and truth, in that order. Example:: [('output1.fits', 'truth1.fits'), ('output2.fits', 'truth2.fits'), ...] atol, rtol : float Absolute and relative tolerance for data comparison. raise_error : bool Raise ``AssertionError`` if difference is found. ignore_keywords_overwrite : list of str or `None` If not `None`, these will overwrite ``self.ignore_keywords`` for the calling test. verbose : bool Print extra info to screen. Returns ------- report : str Report from ``fitsdiff``. This is part of error message if ``raise_error=True``. """ all_okay = True creature_report = '' updated_outputs = [] # To track outputs for Artifactory JSON schema if ignore_keywords_overwrite is None: ignore_keywords = self.ignore_keywords else: ignore_keywords = ignore_keywords_overwrite for actual, desired in outputs: desired = get_bigdata(self.env, self.instrument, self.detector, 'truth', desired) fdiff = FITSDiff(actual, desired, rtol=rtol, atol=atol, ignore_keywords=ignore_keywords) creature_report += fdiff.report() if not fdiff.identical: all_okay = False # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual, desired)) if not all_okay: if self.results_root is not None: # pragma: no cover schema_pattern, tree, testname = generate_upload_params( self.results_root, updated_outputs, verbose=verbose) generate_upload_schema(schema_pattern, tree, testname) if raise_error: raise AssertionError(os.linesep + creature_report) return creature_report