def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True,
ignore_keywords_overwrite=None):
"""
Compare ACSTOOLS output with "truth" using ``fitsdiff``.
Parameters
----------
outputs : list of tuple
A list of tuples, each containing filename (without path)
of CALXXX output and truth, in that order. Example::
[('output1.fits', 'truth1.fits'),
('output2.fits', 'truth2.fits'),
...]
atol, rtol : float
Absolute and relative tolerance for data comparison.
raise_error : bool
Raise ``AssertionError`` if difference is found.
ignore_keywords_overwrite : list of str or `None`
If not `None`, these will overwrite
``self.ignore_keywords`` for the calling test.
Returns
-------
report : str
Report from ``fitsdiff``.
This is part of error message if ``raise_error=True``.
"""
all_okay = True
creature_report = ''
if ignore_keywords_overwrite is None:
ignore_keywords = self.ignore_keywords
else:
ignore_keywords = ignore_keywords_overwrite
for actual, desired in outputs:
desired = get_bigdata('scsb-acstools', self.env, self.detector,
'truth', desired)
fdiff = FITSDiff(actual, desired, rtol=rtol, atol=atol,
ignore_keywords=ignore_keywords)
creature_report += fdiff.report()
if not fdiff.identical and all_okay:
all_okay = False
if not all_okay and raise_error:
raise AssertionError(os.linesep + creature_report)
return creature_report
def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True,
ignore_keywords_overwrite=None):
"""
Compare ACSTOOLS output with "truth" using ``fitsdiff``.
Parameters
----------
outputs : list of tuple
A list of tuples, each containing filename (without path)
of CALXXX output and truth, in that order. Example::
[('output1.fits', 'truth1.fits'),
('output2.fits', 'truth2.fits'),
...]
atol, rtol : float
Absolute and relative tolerance for data comparison.
raise_error : bool
Raise ``AssertionError`` if difference is found.
ignore_keywords_overwrite : list of str or `None`
If not `None`, these will overwrite
``self.ignore_keywords`` for the calling test.
Returns
-------
report : str
Report from ``fitsdiff``.
This is part of error message if ``raise_error=True``.
"""
all_okay = True
creature_report = ''
if ignore_keywords_overwrite is None:
ignore_keywords = self.ignore_keywords
else:
ignore_keywords = ignore_keywords_overwrite
for actual, desired in outputs:
desired = get_bigdata('scsb-acstools', self.env, self.detector,
'truth', desired)
fdiff = FITSDiff(actual, desired, rtol=rtol, atol=atol,
ignore_keywords=ignore_keywords)
creature_report += fdiff.report()
if not fdiff.identical and all_okay:
all_okay = False
if not all_okay and raise_error:
raise AssertionError(os.linesep + creature_report)
return creature_report
def test_fitsdiff_openfile(tmpdir):
"""Make sure that failing FITSDiff doesn't leave open files."""
path1 = str(tmpdir.join("file1.fits"))
path2 = str(tmpdir.join("file2.fits"))
hdulist = HDUList([PrimaryHDU(), ImageHDU(data=np.zeros(5))])
hdulist.writeto(path1)
hdulist[1].data[0] = 1
hdulist.writeto(path2)
diff = FITSDiff(path1, path2)
assert diff.identical, diff.report()
def comparison(self, conf):
"""
this is the FITS comparison function
NB: the images are given by chronological order
:param conf: DirConfiguration
:return: list of strings
"""
fd1 = FITSDiff(conf["temp"] + self.image_1,
conf["temp"] + self.image_2,
numdiffs=self.num_differences,
tolerance=self.tolerance)
fd2 = FITSDiff(conf["temp"] + self.image_2,
conf["temp"] + self.image_3,
numdiffs=self.num_differences,
tolerance=self.tolerance)
fd3 = FITSDiff(conf["temp"] + self.image_1,
conf["temp"] + self.image_3,
numdiffs=self.num_differences,
tolerance=self.tolerance)
# eliminating first approximation interference -> no objects
if not fd1.identical and not fd2.identical:
if fd3.identical:
self.report = False
# storing the fit file
self.moveimage([self.image_1], conf)
return [conf["archive"] + self.image_1]
# anomaly detected -> repository notification
if not fd1.identical and not fd2.identical:
self.report = True
# storing the fit files
self.moveimage([self.image_1, self.image_2, self.image_3], conf)
return [
conf["archive"] + self.image_1, conf["archive"] + self.image_2,
conf["archive"] + self.image_3
]
def difference(self):
"""Run fitsdiff on files named `old_file` and `new_file`.
Returns:
0 no differences
1 some differences
"""
loc_old_file = self.locate_file(self.old_file)
loc_new_file = self.locate_file(self.new_file)
# Do the standard diff.
fdiff = FITSDiff(loc_old_file, loc_new_file)
# Do the diff by rows.
rdiff = rowdiff.RowDiff(loc_old_file, loc_new_file)
if not fdiff.identical:
fdiff.report(fileobj=sys.stdout)
print('\n', rdiff)
return 0 if fdiff.identical else 1
def difference(self):
"""Run fitsdiff on files named `old_file` and `new_file`.
Returns:
0 no differences
1 some differences
"""
loc_old_file = self.locate_file(self.old_file)
loc_new_file = self.locate_file(self.new_file)
# Do the standard diff.
fdiff = FITSDiff(loc_old_file, loc_new_file)
# Do the diff by rows.
rdiff = rowdiff.RowDiff(loc_old_file, loc_new_file)
if not fdiff.identical:
fdiff.report(fileobj=sys.stdout)
print('\n', rdiff)
return 0 if fdiff.identical else 1
def test_convert():
'''Convert tif to fits with and without statsfile'''
with tempfile.TemporaryDirectory() as tmpdirname:
# convert file without stats and add those in second step
tiff2fitsimg(tpath('wslit_-1.2deg.tif'), tmpdirname)
outfile = os.path.join(tmpdirname, 'wslit_-1.2deg_NoStats_img.fits')
assert os.path.isfile(outfile)
addstats2img(outfile, statfile=tpath(''), rename=False)
assert os.path.isfile(outfile)
# Use statsfile directly
outfile2 = os.path.join(tmpdirname, 'wslit_-1.2deg_img.fits')
assert not os.path.exists(outfile2)
out = tiff2fitsimg(tpath('wslit_-1.2deg.tif'), tmpdirname,
statfile=tpath(''))
assert os.path.isfile(outfile2)
assert out == outfile2
# Check result is the same
# DATE contains time of conversion and will be slightly different
diff = FITSDiff(outfile, outfile2, ignore_keywords=['DATE'])
assert diff.identical
def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True,
ignore_keywords_overwrite=None, verbose=True):
"""
Compare CALXXX output with "truth" using ``fitsdiff``.
Parameters
----------
outputs : list of tuple
A list of tuples, each containing filename (without path)
of CALXXX output and truth, in that order. Example::
[('output1.fits', 'truth1.fits'),
('output2.fits', 'truth2.fits'),
...]
atol, rtol : float
Absolute and relative tolerance for data comparison.
raise_error : bool
Raise ``AssertionError`` if difference is found.
ignore_keywords_overwrite : list of str or `None`
If not `None`, these will overwrite
``self.ignore_keywords`` for the calling test.
verbose : bool
Print extra info to screen.
Returns
-------
report : str
Report from ``fitsdiff``.
This is part of error message if ``raise_error=True``.
"""
all_okay = True
creature_report = ''
updated_outputs = [] # To track outputs for Artifactory JSON schema
if ignore_keywords_overwrite is None:
ignore_keywords = self.ignore_keywords
else:
ignore_keywords = ignore_keywords_overwrite
for actual, desired in outputs:
desired = get_bigdata(self.env, self.instrument, self.detector,
'truth', desired)
fdiff = FITSDiff(actual, desired, rtol=rtol, atol=atol,
ignore_keywords=ignore_keywords)
creature_report += fdiff.report()
if not fdiff.identical:
all_okay = False
# Only keep track of failed results which need to
# be used to replace the truth files (if OK).
updated_outputs.append((actual, desired))
if not all_okay:
if self.results_root is not None: # pragma: no cover
schema_pattern, tree, testname = generate_upload_params(
self.results_root, updated_outputs, verbose=verbose)
generate_upload_schema(schema_pattern, tree, testname)
if raise_error:
raise AssertionError(os.linesep + creature_report)
return creature_report
def compare_outputs(self, outputs, raise_error=True, delete_history=False):
"""
Compare output with "truth" using appropriate
diff routine; namely,
``fitsdiff`` for FITS file comparisons
``unified_diff`` for ASCII products.
Parameters
----------
outputs : list of tuple
A list of tuples, each containing filename (without path)
of CALXXX output and truth, in that order.
raise_error : bool
Raise ``AssertionError`` if difference is found.
Returns
-------
report : str
Report from ``fitsdiff``.
This is part of error message if ``raise_error=True``.
"""
all_okay = True
creature_report = ''
# Create instructions for uploading results to artifactory for use
# as new comparison/truth files
testpath, testname = os.path.split(os.path.abspath(os.curdir))
# organize results by day test was run...could replace with git-hash
whoami = getpass.getuser() or 'nobody'
dt = datetime.datetime.now().strftime("%d%b%YT")
ttime = datetime.datetime.now().strftime("%H_%M_%S")
user_tag = 'NOT_CI_{}_{}'.format(whoami, ttime)
build_tag = os.environ.get('BUILD_TAG', user_tag)
build_suffix = os.environ.get('BUILD_MATRIX_SUFFIX', 'standalone')
testdir = "{}_{}_{}".format(testname, build_tag, build_suffix)
tree = os.path.join(self.results_root, self.input_loc, dt,
testdir) + os.sep
updated_outputs = []
for actual, desired in outputs:
# Get "truth" image
s = self.get_data('truth', desired)
if s is not None:
desired = s
if actual.endswith('fits'):
# Working with FITS files...
if delete_history is True:
actual = fits.open(actual)
desired = fits.open(desired)
if 'HISTORY' in actual[0].header:
del actual[0].header['HISTORY']
if 'HISTORY' in desired[0].header:
del desired[0].header['HISTORY']
fdiff = FITSDiff(actual,
desired,
rtol=self.rtol,
atol=self.atol,
ignore_keywords=self.ignore_keywords)
if delete_history is True:
actual.close()
desired.close()
creature_report += fdiff.report()
if not fdiff.identical:
# Only keep track of failed results which need to
# be used to replace the truth files (if OK).
updated_outputs.append((actual, desired))
if not fdiff.identical and all_okay:
all_okay = False
else:
# Try ASCII-based diff
with open(actual) as afile:
actual_lines = afile.readlines()
with open(desired) as dfile:
desired_lines = dfile.readlines()
udiff = unified_diff(actual_lines,
desired_lines,
fromfile=actual,
tofile=desired)
old_stdout = sys.stdout
udiffIO = StringIO()
sys.stdout = udiffIO
sys.stdout.writelines(udiff)
sys.stdout = old_stdout
udiff_report = udiffIO.getvalue()
creature_report += udiff_report
if len(udiff_report) > 2 and all_okay:
all_okay = False
if len(udiff_report) > 2:
# Only keep track of failed results which need to
# be used to replace the truth files (if OK).
updated_outputs.append((actual, desired))
if not all_okay:
# Write out JSON file to enable retention of different results
new_truths = [os.path.abspath(i[1]) for i in updated_outputs]
for files in updated_outputs:
print("Renaming {} as new 'truth' file: {}".format(
files[0], files[1]))
shutil.move(files[0], files[1])
log_pattern = [
os.path.join(os.path.dirname(x), '*.log') for x in new_truths
]
upload_results(pattern=new_truths + log_pattern,
testname=testname,
target=tree)
if not all_okay and raise_error:
raise AssertionError(os.linesep + creature_report)
return creature_report
def compare_outputs(outputs, raise_error=True, ignore_keywords=[],
ignore_hdus=[], ignore_fields=[], rtol=0.0, atol=0.0,
input_path=[], docopy=True, results_root=None,
verbose=True):
"""
Compare output with "truth" using appropriate
diff routine; namely:
* ``fitsdiff`` for FITS file comparisons.
* ``unified_diff`` for ASCII products.
Only after all elements of ``outputs`` have been
processed will the method report any success or failure, with
failure of any one comparison *not* preventing the rest of the
comparisons to be performed.
Parameters
----------
outputs : list of tuple or dict
This list defines what outputs from running the test will be
compared. Three distinct types of values as list elements
are supported:
* 2-tuple : ``(test output filename, truth filename)``
* 3-tuple : ``(test output filename, truth filename, HDU names)``
* dict : ``{'files': (output, truth), 'pars': {key: val}}``
If filename contains extension such as ``[hdrtab]``,
it will be interpreted as specifying comparison of just that HDU.
raise_error : bool
Raise ``AssertionError`` if difference is found.
ignore_keywords : list of str
List of FITS header keywords to be ignored by
``FITSDiff`` and ``HDUDiff``.
ignore_hdus : list of str
List of FITS HDU names to ignore by ``FITSDiff``.
This is only available for ``astropy>=3.1``.
ignore_fields : list of str
List FITS table column names to be ignored by
``FITSDiff`` and ``HDUDiff``.
rtol, atol : float
Relative and absolute tolerance to be used by
``FITSDiff`` and ``HDUDiff``.
input_path : list or tuple
A series of sub-directory names under :func:`get_bigdata_root`
that leads to the path of the 'truth' files to be compared
against. If not provided, it assumes that 'truth' is in the
working directory. For example, with :func:`get_bigdata_root`
pointing to ``/grp/test_data``, a file at::
/grp/test_data/pipeline/dev/ins/test_1/test_a.py
would require ``input_path`` of::
["pipeline", "dev", "ins", "test_1"]
docopy : bool
If `True`, 'truth' will be copied to output directory before
comparison is done.
results_root : str or `None`
If not `None`, for every failed comparison, the test output
is automatically renamed to the given 'truth' in the output
directory and :func:`generate_upload_schema` will be called
to generate a JSON scheme for Artifactory upload.
If you do not need this functionality, use ``results_root=None``.
verbose : bool
Print extra info to screen.
Returns
-------
creature_report : str
Report from FITS or ASCII comparator.
This is part of error message if ``raise_error=True``.
Examples
--------
There are multiple use cases for this method, specifically
related to how ``outputs`` are defined upon calling this method.
The specification of the ``outputs`` can be any combination of the
following patterns:
1. 2-tuple inputs::
outputs = [('file1.fits', 'file1_truth.fits')]
This definition indicates that ``file1.fits`` should be compared
as a whole with ``file1_truth.fits``.
2. 2-tuple inputs with extensions::
outputs = [('file1.fits[hdrtab]', 'file1_truth.fits[hdrtab]')]
This definition indicates that only the HDRTAB extension from
``file1.fits`` will be compared to the HDRTAB extension from
``file1_truth.fits``.
3. 3-tuple inputs::
outputs = [('file1.fits', 'file1_truth.fits', ['primary', 'sci'])]
This definition indicates that only the PRIMARY and SCI extensions
should be compared between the two files. This creates a temporary
``HDUList`` object comprising only the given extensions for comparison.
4. Dictionary of inputs and parameters::
outputs = [{'files': ('file1.fits', 'file1_truth.fits'),
'pars': {'ignore_keywords': ['ROOTNAME']}}]
This definition indicates that ROOTNAME will be ignored during
the comparison between the files specified in ``'files'``.
Any input parameter for ``FITSDiff`` or ``HDUDiff`` can be specified
as part of the ``'pars'`` dictionary.
In addition, the input files listed in ``'files'`` can also include
an extension specification, such as ``[hdrtab]``, to limit the
comparison to just that extension.
This example from an actual test definition demonstrates
how multiple input defintions can be used at the same time::
outputs = [
('jw99999_nircam_f140m-maskbar_psfstack.fits',
'jw99999_nircam_f140m-maskbar_psfstack_ref.fits'
),
('jw9999947001_02102_00002_nrcb3_a3001_crfints.fits',
'jw9999947001_02102_00002_nrcb3_a3001_crfints_ref.fits'
),
{'files': ('jw99999_nircam_f140m-maskbar_i2d.fits',
'jw99999_nircam_f140m-maskbar_i2d_ref.fits'),
'pars': {'ignore_hdus': ['HDRTAB']},
{'files': ('jw99999_nircam_f140m-maskbar_i2d.fits',
'jw99999_nircam_f140m-maskbar_i2d_ref.fits',
['primary','sci','dq']),
'pars': {'rtol': 0.000001}
},
{'files': ('jw99999_nircam_f140m-maskbar_i2d.fits[hdrtab]',
'jw99999_nircam_f140m-maskbar_i2d_ref.fits[hdrtab]'),
'pars': {'ignore_keywords': ['NAXIS1', 'TFORM*'],
'ignore_fields': ['COL1', 'COL2']}
}]
.. note:: Each ``outputs`` entry in the list gets interpreted and processed
separately.
"""
__tracebackhide__ = True
default_kwargs = {'rtol': rtol, 'atol': atol,
'ignore_keywords': ignore_keywords,
'ignore_fields': ignore_fields,
'ignore_hdus': ignore_hdus}
all_okay = True
creature_report = ''
updated_outputs = [] # To track outputs for Artifactory JSON schema
for entry in outputs:
diff_kwargs = copy.deepcopy(default_kwargs)
extn_list = None
num_entries = len(entry)
if isinstance(entry, dict):
entry_files = entry['files']
actual = entry_files[0]
desired = entry_files[1]
if len(entry_files) > 2:
extn_list = entry_files[2]
diff_kwargs.update(entry.get('pars', {}))
elif num_entries == 2:
actual, desired = entry
elif num_entries == 3:
actual, desired, extn_list = entry
else:
all_okay = False
creature_report += '\nERROR: Cannot handle entry {}\n'.format(
entry)
continue
# TODO: Use regex?
if actual.endswith(']'):
if extn_list is not None:
all_okay = False
creature_report += (
'\nERROR: Ambiguous extension requirements '
'for {} ({})\n'.format(actual, extn_list))
continue
actual_name, actual_extn = actual.split('[')
actual_extn = actual_extn.replace(']', '')
else:
actual_name = actual
actual_extn = None
if desired.endswith(']'):
if extn_list is not None:
all_okay = False
creature_report += (
'\nERROR: Ambiguous extension requirements '
'for {} ({})\n'.format(desired, extn_list))
continue
desired_name, desired_extn = desired.split('[')
desired_extn = desired_extn.replace(']', '')
else:
desired_name = desired
desired_extn = None
actual = os.path.abspath(actual)
# Get "truth" image
try:
os.makedirs('truth', exist_ok=True)
os.chdir('truth')
desired = get_bigdata(*input_path, desired_name, docopy=docopy)
desired = os.path.abspath(desired)
os.chdir('..')
except BigdataError:
all_okay = False
creature_report += '\nERROR: Cannot find {} in {}\n'.format(
desired_name, input_path)
continue
if desired_extn is not None:
desired_name = desired
desired = "{}[{}]".format(desired, desired_extn)
if verbose:
print("\nComparing:\n {}\n {}".format(actual, desired))
if actual.endswith('.fits') and desired.endswith('.fits'):
# Build HDULists for comparison based on user-specified extensions
if extn_list is not None:
with fits.open(actual) as f_act:
with fits.open(desired) as f_des:
actual_hdu = fits.HDUList(
[f_act[extn] for extn in extn_list])
actual_hdu.filename = lambda: os.path.basename(actual)
desired_hdu = fits.HDUList(
[f_des[extn] for extn in extn_list])
desired_hdu.filename = lambda: os.path.basename(desired)
fdiff = FITSDiff(actual_hdu, desired_hdu,
**diff_kwargs)
creature_report += '\na: {}\nb: {}\n'.format(
actual, desired) # diff report only gives hash
# Working with FITS files...
else:
fdiff = FITSDiff(actual, desired, **diff_kwargs)
creature_report += fdiff.report()
if not fdiff.identical:
all_okay = False
# Only keep track of failed results which need to
# be used to replace the truth files (if OK).
updated_outputs.append((actual, desired))
elif actual_extn is not None or desired_extn is not None:
if 'ignore_hdus' in diff_kwargs: # pragma: no cover
diff_kwargs.pop('ignore_hdus') # Not applicable
# Specific element of FITS file specified
with fits.open(actual_name) as f_act:
with fits.open(desired_name) as f_des:
actual_hdu = f_act[actual_extn]
desired_hdu = f_des[desired_extn]
fdiff = HDUDiff(actual_hdu, desired_hdu, **diff_kwargs)
creature_report += 'a: {}\nb: {}\n'.format(actual, desired)
creature_report += fdiff.report()
if not fdiff.identical:
all_okay = False
# Only keep track of failed results which need to
# be used to replace the truth files (if OK).
updated_outputs.append((actual_name, desired_name))
else:
# ASCII-based diff
with open(actual) as afile:
actual_lines = afile.readlines()
with open(desired) as dfile:
desired_lines = dfile.readlines()
udiff = unified_diff(actual_lines, desired_lines,
fromfile=actual, tofile=desired)
udiffIO = StringIO()
udiffIO.writelines(udiff)
udiff_report = udiffIO.getvalue()
udiffIO.close()
if len(udiff_report) == 0:
creature_report += ('\na: {}\nb: {}\nNo differences '
'found.\n'.format(actual, desired))
else:
all_okay = False
creature_report += udiff_report
# Only keep track of failed results which need to
# be used to replace the truth files (if OK).
updated_outputs.append((actual, desired))
if not all_okay and results_root is not None: # pragma: no cover
schema_pattern, tree, testname = generate_upload_params(
results_root, updated_outputs, verbose=verbose)
generate_upload_schema(schema_pattern, tree, testname)
if not all_okay and raise_error:
raise AssertionError(os.linesep + creature_report)
return creature_report
def find(name, path):
for root, dirs, files in os.walk(path):
if os.path.basename(name) in files:
# Check if this is the last version number of a CALDB file.
marxver = int(name[-9:-5])
for f in files:
if (int(f[-9:-5]) > marxver) and (f[:-9] == os.path.basename(name)[:-9]):
print ("Check if {0} superceedes {1}".format(f, name))
return os.path.join(root, os.path.basename(name))
for f in marxcaldb:
diff = None
caldbfile = find(f, CALDB)
if caldbfile is not None:
diff = FITSDiff(f, caldbfile, ignore_keywords=ignore_key)
# special cases
else:
fbase = os.path.basename(f)
if os.path.basename(f) == "acisfef.fits":
caldbfile = find("acisD2000-01-29fef_phaN0005.fits", CALDB)
call("../caldb/fixfef.sl {0} {1}_new".format(caldbfile, f), shell=True)
diff = FITSDiff(f, f + "_new", ignore_keywords=ignore_key)
elif fbase.startswith("acisD1999-07-22subpixN00"):
caldbfile = find(fbase.replace("_marx", ""), CALDB)
call("../utils/mksubpix.sl {0} {1}_new".format(caldbfile, f), shell=True)
diff = FITSDiff(f, f + "_new", ignore_keywords=ignore_key)
elif fbase.startswith("acisD1999-08-13contamN00"):
caldbfile = find(fbase.replace("_marx", ""), CALDB)
call("python ../utils/mkcontam.py {0} {1}_new".format(caldbfile, f), shell=True)
diff = FITSDiff(f, f + "_new", ignore_keywords=ignore_key)
def compare_outputs(self, outputs, raise_error=True):
"""
Compare output with "truth" using appropriate
diff routine; namely,
``fitsdiff`` for FITS file comparisons
``unified_diff`` for ASCII products.
Parameters
----------
outputs : list of tuple
A list of tuples, each containing filename (without path)
of CALXXX output and truth, in that order.
raise_error : bool
Raise ``AssertionError`` if difference is found.
Returns
-------
report : str
Report from ``fitsdiff``.
This is part of error message if ``raise_error=True``.
"""
all_okay = True
creature_report = ''
# Create instructions for uploading results to artifactory for use
# as new comparison/truth files
testpath, testname = os.path.split(os.path.abspath(os.curdir))
# organize results by day test was run...could replace with git-hash
whoami = getpass.getuser() or 'nobody'
dt = datetime.datetime.now().strftime("%d%b%YT")
ttime = datetime.datetime.now().strftime("%H_%M_%S")
user_tag = 'NOT_CI_{}_{}'.format(whoami, ttime)
build_tag = os.environ.get('BUILD_TAG', user_tag)
build_suffix = os.environ.get('BUILD_MATRIX_SUFFIX', 'standalone')
testdir = "{}_{}_{}".format(testname, build_tag, build_suffix)
tree = os.path.join(self.results_root, self.input_loc,
dt, testdir) + os.sep
updated_outputs = []
for actual, desired in outputs:
# Get "truth" image
s = self.get_data('truth', desired)
if s is not None:
desired = s
if actual.endswith('fits'):
# Working with FITS files...
fdiff = FITSDiff(actual, desired, rtol=self.rtol, atol=self.atol,
ignore_keywords=self.ignore_keywords)
creature_report += fdiff.report()
if not fdiff.identical:
# Only keep track of failed results which need to
# be used to replace the truth files (if OK).
updated_outputs.append((actual, desired))
if not fdiff.identical and all_okay:
all_okay = False
else:
# Try ASCII-based diff
with open(actual) as afile:
actual_lines = afile.readlines()
with open(desired) as dfile:
desired_lines = dfile.readlines()
udiff = unified_diff(actual_lines, desired_lines,
fromfile=actual, tofile=desired)
old_stdout = sys.stdout
udiffIO = StringIO()
sys.stdout = udiffIO
sys.stdout.writelines(udiff)
sys.stdout = old_stdout
udiff_report = udiffIO.getvalue()
creature_report += udiff_report
if len(udiff_report) > 2 and all_okay:
all_okay = False
if len(udiff_report) > 2:
# Only keep track of failed results which need to
# be used to replace the truth files (if OK).
updated_outputs.append((actual, desired))
if not all_okay:
# Write out JSON file to enable retention of different results
new_truths = [os.path.abspath(i[1]) for i in updated_outputs]
for files in updated_outputs:
print("Renaming {} as new 'truth' file: {}".format(
files[0], files[1]))
shutil.move(files[0], files[1])
log_pattern = [os.path.join(os.path.dirname(x), '*.log') for x in new_truths]
generate_upload_schema(pattern=new_truths + log_pattern,
testname=testname,
target= tree)
if not all_okay and raise_error:
raise AssertionError(os.linesep + creature_report)
return creature_report
def find(name, path):
for root, dirs, files in os.walk(path):
if os.path.basename(name) in files:
# Check if this is the last version number of a CALDB file.
marxver = int(name[-9:-5])
for f in files:
if (int(f[-9:-5]) > marxver) and (f[:-9] == os.path.basename(name)[:-9]):
print('Check if {0} superceedes {1}'.format(f, name))
return os.path.join(root, os.path.basename(name))
for f in marxcaldb:
diff = None
caldbfile = find(f, CALDB)
if caldbfile is not None:
diff = FITSDiff(f, caldbfile, ignore_keywords=ignore_key)
# special cases
else:
fbase = os.path.basename(f)
if os.path.basename(f) == 'acisfef.fits':
caldbfile = find('acisD2000-01-29fef_phaN0005.fits', CALDB)
call('../caldb/fixfef.sl {0} {1}_new'.format(caldbfile, f), shell=True)
diff = FITSDiff(f, f + '_new', ignore_keywords=ignore_key)
elif fbase.startswith('acisD1999-07-22subpixN00'):
caldbfile = find(fbase.replace('_marx', ''), CALDB)
call('../utils/mksubpix.sl {0} {1}_new'.format(caldbfile, f), shell = True)
diff = FITSDiff(f, f + '_new', ignore_keywords=ignore_key)
elif fbase.startswith('acisD1999-08-13contamN00'):
caldbfile = find(fbase.replace('_marx', ''), CALDB)
call('python ../utils/mkcontam.py {0} {1}_new'.format(caldbfile, f), shell = True)
diff = FITSDiff(f, f + '_new', ignore_keywords=ignore_key)
def compare_outputs(self,
outputs,
atol=0,
rtol=1e-7,
raise_error=True,
ignore_keywords_overwrite=None,
verbose=True):
"""
Compare CALXXX output with "truth" using ``fitsdiff``.
Parameters
----------
outputs : list of tuple
A list of tuples, each containing filename (without path)
of CALXXX output and truth, in that order. Example::
[('output1.fits', 'truth1.fits'),
('output2.fits', 'truth2.fits'),
...]
atol, rtol : float
Absolute and relative tolerance for data comparison.
raise_error : bool
Raise ``AssertionError`` if difference is found.
ignore_keywords_overwrite : list of str or `None`
If not `None`, these will overwrite
``self.ignore_keywords`` for the calling test.
verbose : bool
Print extra info to screen.
Returns
-------
report : str
Report from ``fitsdiff``.
This is part of error message if ``raise_error=True``.
"""
all_okay = True
creature_report = ''
updated_outputs = [] # To track outputs for Artifactory JSON schema
if ignore_keywords_overwrite is None:
ignore_keywords = self.ignore_keywords
else:
ignore_keywords = ignore_keywords_overwrite
for actual, desired in outputs:
desired = get_bigdata(self.env, self.instrument, self.detector,
'truth', desired)
fdiff = FITSDiff(actual,
desired,
rtol=rtol,
atol=atol,
ignore_keywords=ignore_keywords)
creature_report += fdiff.report()
if not fdiff.identical:
all_okay = False
# Only keep track of failed results which need to
# be used to replace the truth files (if OK).
updated_outputs.append((actual, desired))
if not all_okay:
if self.results_root is not None: # pragma: no cover
schema_pattern, tree, testname = generate_upload_params(
self.results_root, updated_outputs, verbose=verbose)
generate_upload_schema(schema_pattern, tree, testname)
if raise_error:
raise AssertionError(os.linesep + creature_report)
return creature_report