def test_sextractor_md5sum(self):
# Testing that astromatic.sextractor_md5sum() works as expected means,
# by extension, checking that the four SExtractor configuration files
# (.sex, .param, .conv and .nnw) defined in the 'astromatic' module
# exist and are readable.
# If the SExtractor configuration files are not modified in between,
# two different executions of the function must yield the same hash.
checksum = astromatic.sextractor_md5sum()
identical = astromatic.sextractor_md5sum()
self.assertEqual(checksum, identical)
# The MD5 hash is that of the concatenation of the lines of the four
# configuration files and the overriding SExtractor options (i.e., the
# sequence of strings that are optionally passed to the method). If any
# of the files is modified, or if an overriding option is given, the
# MD5 hash returned by the function must be different.
#
# In order to test this we are not going to temporarily modify the
# SExtractor configuration files: although we could make a copy of
# them, something beyond our control (e.g., the SIGKILL signal or a
# power outage) could prevent the original file from being restored.
# Instead, what we will temporarily modify are the module-level
# variables, so that they refer to a modified copy of the files.
for variable in self.SEXTRACTOR_MODULE_VARS:
# The variable must exist and refer to an existing file
path = eval('astromatic.%s' % variable)
self.assertTrue(os.path.exists(path))
# Make a temporary copy of the configuration file and modify it,
# appending a comment to it. Then, mock the module-level variable
# so that it refers to the modified configuration file. Although
# such an irrelevant change does not alter how SExtractor works (as
# the settings are still the same), the MD5 hash must be different.
ext = os.path.splitext(path)[1]
copy_path = get_nonexistent_path(ext=ext)
try:
shutil.copy2(path, copy_path)
with open(copy_path, 'at') as fd:
fd.write("# useless comment\n")
with mock.patch.object(astromatic, variable, copy_path):
different = astromatic.sextractor_md5sum()
self.assertNotEqual(checksum, different)
finally:
os.unlink(copy_path)
# If overriding options are given, they are also used to compute the
# MD5 hash. Thus, the hash will be different even if the options have
# the same value as those defined in the configuration file (although
# that means that we are not actually overriding anything). Test this
# for all the options defined in the configuration file: in all cases
# the returned checksum must be different.
with open(astromatic.SEXTRACTOR_CONFIG) as fd:
for line in fd:
stripped = line.strip()
if stripped and stripped[0] != '#': # ignore comment lines
key, value = stripped.split()[:2]
options = {key: value}
different = astromatic.sextractor_md5sum(options)
self.assertNotEqual(checksum, different)
# Try also overriding the option with a different value
# (the result of incrementing it by one). Again, this must
# result in a different MD5 hash.
try:
options[key] = str(int(options[key]) + 1)
also_different = astromatic.sextractor_md5sum(options)
self.assertNotEqual(checksum, also_different)
self.assertNotEqual(different, also_different)
except ValueError:
pass # non-numeric option
# IOError is raised if any of the SExtractor configuration files is not
# readable or does not exist. To test that, mock again the module-level
# variables so that they refer to temporary copies of the configuration
# files, but which are unreadable by the user, first, and then removed.
for variable in self.SEXTRACTOR_MODULE_VARS:
path = eval('astromatic.%s' % variable)
ext = os.path.splitext(path)[1]
copy_path = get_nonexistent_path(ext=ext)
shutil.copy2(path, copy_path)
with mock.patch.object(astromatic, variable, copy_path):
# chmod u-r
mode = stat.S_IMODE(os.stat(copy_path)[stat.ST_MODE])
mode ^= stat.S_IRUSR
os.chmod(copy_path, mode)
args = IOError, astromatic.sextractor_md5sum
self.assertFalse(os.access(copy_path, os.R_OK))
self.assertRaises(*args)
os.unlink(copy_path)
self.assertFalse(os.path.exists(copy_path))
self.assertRaises(*args)
# TypeError raised if 'options' is not a dictionary
kwargs = dict(options=['DETECT_MINAREA', '5'])
with self.assertRaises(TypeError):
astromatic.sextractor_md5sum(**kwargs)
# ... or if any of its elements is not a string
kwargs['options'] = {'DETECT_MINAREA': 125}
with self.assertRaises(TypeError):
astromatic.sextractor_md5sum(**kwargs)
def test_sextractor_md5sum(self):
# Testing that astromatic.sextractor_md5sum() works as expected means,
# by extension, checking that the four SExtractor configuration files
# (.sex, .param, .conv and .nnw) defined in the 'astromatic' module
# exist and are readable.
# If the SExtractor configuration files are not modified in between,
# two different executions of the function must yield the same hash.
checksum = astromatic.sextractor_md5sum()
identical = astromatic.sextractor_md5sum()
self.assertEqual(checksum, identical)
# The MD5 hash is that of the concatenation of the lines of the four
# configuration files and the overriding SExtractor options (i.e., the
# sequence of strings that are optionally passed to the method). If any
# of the files is modified, or if an overriding option is given, the
# MD5 hash returned by the function must be different.
#
# In order to test this we are not going to temporarily modify the
# SExtractor configuration files: although we could make a copy of
# them, something beyond our control (e.g., the SIGKILL signal or a
# power outage) could prevent the original file from being restored.
# Instead, what we will temporarily modify are the module-level
# variables, so that they refer to a modified copy of the files.
for variable in self.SEXTRACTOR_MODULE_VARS:
# The variable must exist and refer to an existing file
path = eval('astromatic.%s' % variable)
self.assertTrue(os.path.exists(path))
# Make a temporary copy of the configuration file and modify it,
# appending a comment to it. Then, mock the module-level variable
# so that it refers to the modified configuration file. Although
# such an irrelevant change does not alter how SExtractor works (as
# the settings are still the same), the MD5 hash must be different.
ext = os.path.splitext(path)[1]
copy_path = get_nonexistent_path(ext = ext)
try:
shutil.copy2(path, copy_path)
with open(copy_path, 'at') as fd:
fd.write("# useless comment\n")
with mock.patch.object(astromatic, variable, copy_path):
different = astromatic.sextractor_md5sum()
self.assertNotEqual(checksum, different)
finally:
os.unlink(copy_path)
# If overriding options are given, they are also used to compute the
# MD5 hash. Thus, the hash will be different even if the options have
# the same value as those defined in the configuration file (although
# that means that we are not actually overriding anything). Test this
# for all the options defined in the configuration file: in all cases
# the returned checksum must be different.
with open(astromatic.SEXTRACTOR_CONFIG) as fd:
for line in fd:
stripped = line.strip()
if stripped and stripped[0] != '#': # ignore comment lines
key, value = stripped.split()[:2]
options = {key : value}
different = astromatic.sextractor_md5sum(options)
self.assertNotEqual(checksum, different)
# Try also overriding the option with a different value
# (the result of incrementing it by one). Again, this must
# result in a different MD5 hash.
try:
options[key] = str(int(options[key]) + 1)
also_different = astromatic.sextractor_md5sum(options)
self.assertNotEqual(checksum, also_different)
self.assertNotEqual(different, also_different)
except ValueError:
pass # non-numeric option
# IOError is raised if any of the SExtractor configuration files is not
# readable or does not exist. To test that, mock again the module-level
# variables so that they refer to temporary copies of the configuration
# files, but which are unreadable by the user, first, and then removed.
for variable in self.SEXTRACTOR_MODULE_VARS:
path = eval('astromatic.%s' % variable)
ext = os.path.splitext(path)[1]
copy_path = get_nonexistent_path(ext = ext)
shutil.copy2(path, copy_path)
with mock.patch.object(astromatic, variable, copy_path):
# chmod u-r
mode = stat.S_IMODE(os.stat(copy_path)[stat.ST_MODE])
mode ^= stat.S_IRUSR
os.chmod(copy_path, mode)
args = IOError, astromatic.sextractor_md5sum
self.assertFalse(os.access(copy_path, os.R_OK))
self.assertRaises(*args)
os.unlink(copy_path)
self.assertFalse(os.path.exists(copy_path))
self.assertRaises(*args)
# TypeError raised if 'options' is not a dictionary
kwargs = dict(options = ['DETECT_MINAREA', '5'])
with self.assertRaises(TypeError):
astromatic.sextractor_md5sum(**kwargs)
# ... or if any of its elements is not a string
kwargs['options'] = {'DETECT_MINAREA' : 125}
with self.assertRaises(TypeError):
astromatic.sextractor_md5sum(**kwargs)
def __init__(self, path, maximum, margin, coaddk = keywords.coaddk):
""" Instantiation method for the FITSeeingImage class.
The path to the SExtractor catalog is read from the FITS header: if the
keyword is not found, or if it is present but refers to a non-existent
file, SExtractor has to be executed again. Even if the catalog exists,
however, the MD5 of the SExtractor configuration files that were used
when the catalog was created must be the same.
The 'maximum' parameter determines the pixel value above which it is
considered saturated. This value depends not only on the CCD, but also
on the number of coadded images. If three images were coadded, for
example, the effective saturation level equals three times the value of
'saturation'. The number of effective coadds is read from the 'coaddk'
parameter. If the keyword is missing, a value of one (that is, that the
image consisted of a single exposure) is assumed.
The 'margin' argument gives the width, in pixels, of the areas adjacent
to the edges of the image that are to be ignored when detecting sources
on the reference image. Stars whose center is fewer than this number of
pixels from any border of the FITS image are not considered.
"""
super(FITSeeingImage, self).__init__(path)
self.margin = margin
msg = "%s: width of margin: %d pixels" % (self.path, self.margin)
logging.debug(msg)
# Compute the MD5 hash of the SExtractor configuration files and this
# saturation level, which overrides the definition of SATUR_LEVEL.
satur_level = self.saturation(maximum, coaddk = coaddk)
options = dict(SATUR_LEVEL = str(satur_level))
sex_md5sum = astromatic.sextractor_md5sum(options = options)
msg = "%s: SExtractor MD5 hash: %s" % (self.path, sex_md5sum)
logging.debug(msg)
try:
try:
self.catalog_path = self.read_keyword(keywords.sex_catalog)
except KeyError:
msg = "%s: keyword '%s' not found"
logging.debug(msg % (self.path, keywords.sex_catalog))
raise
# The path not only must be stored: the catalog must also exist
msg = "%s: on-disk catalog %s" % (self.path, self.catalog_path)
if not os.path.exists(self.catalog_path):
logging.debug(msg + " does not exist")
raise IOError
else:
logging.debug(msg)
try:
img_sex_md5sum = self.read_keyword(keywords.sex_md5sum)
msg = "%s: on-disk catalog SExtractor MD5 hash (%s): %s"
args = self.path, keywords.sex_md5sum, img_sex_md5sum
logging.debug(msg % args)
except KeyError:
msg = "%s: keyword '%s' not found"
logging.debug(msg % (self.path, keywords.sex_md5sum))
raise
if img_sex_md5sum != sex_md5sum:
msg = "%s: outdated catalog (SExtractor hashes do not match)"
logging.debug(msg % self.path)
try:
os.unlink(self.catalog_path)
msg = "%s: outdated catalog removed from disk" % self.path
logging.debug(msg)
except (IOError, OSError), e:
msg = "%s: could not remove outdated catalog (%s)"
logging.warning(msg % (self.path, e))
finally:
raise ValueError
def __init__(self, path, maximum, margin, coaddk=keywords.coaddk):
""" Instantiation method for the FITSeeingImage class.
The path to the SExtractor catalog is read from the FITS header: if the
keyword is not found, or if it is present but refers to a non-existent
file, SExtractor has to be executed again. Even if the catalog exists,
however, the MD5 of the SExtractor configuration files that were used
when the catalog was created must be the same.
The 'maximum' parameter determines the pixel value above which it is
considered saturated. This value depends not only on the CCD, but also
on the number of coadded images. If three images were coadded, for
example, the effective saturation level equals three times the value of
'saturation'. The number of effective coadds is read from the 'coaddk'
parameter. If the keyword is missing, a value of one (that is, that the
image consisted of a single exposure) is assumed.
The 'margin' argument gives the width, in pixels, of the areas adjacent
to the edges of the image that are to be ignored when detecting sources
on the reference image. Stars whose center is fewer than this number of
pixels from any border of the FITS image are not considered.
"""
super(FITSeeingImage, self).__init__(path)
self.margin = margin
msg = "%s: width of margin: %d pixels" % (self.path, self.margin)
logging.debug(msg)
# Compute the MD5 hash of the SExtractor configuration files and this
# saturation level, which overrides the definition of SATUR_LEVEL.
satur_level = self.saturation(maximum, coaddk=coaddk)
options = dict(SATUR_LEVEL=str(satur_level))
sex_md5sum = astromatic.sextractor_md5sum(options=options)
msg = "%s: SExtractor MD5 hash: %s" % (self.path, sex_md5sum)
logging.debug(msg)
try:
try:
self.catalog_path = self.read_keyword(keywords.sex_catalog)
except KeyError:
msg = "%s: keyword '%s' not found"
logging.debug(msg % (self.path, keywords.sex_catalog))
raise
# The path not only must be stored: the catalog must also exist
msg = "%s: on-disk catalog %s" % (self.path, self.catalog_path)
if not os.path.exists(self.catalog_path):
logging.debug(msg + " does not exist")
raise IOError
else:
logging.debug(msg)
try:
img_sex_md5sum = self.read_keyword(keywords.sex_md5sum)
msg = "%s: on-disk catalog SExtractor MD5 hash (%s): %s"
args = self.path, keywords.sex_md5sum, img_sex_md5sum
logging.debug(msg % args)
except KeyError:
msg = "%s: keyword '%s' not found"
logging.debug(msg % (self.path, keywords.sex_md5sum))
raise
if img_sex_md5sum != sex_md5sum:
msg = "%s: outdated catalog (SExtractor hashes do not match)"
logging.debug(msg % self.path)
try:
os.unlink(self.catalog_path)
msg = "%s: outdated catalog removed from disk" % self.path
logging.debug(msg)
except (IOError, OSError), e:
msg = "%s: could not remove outdated catalog (%s)"
logging.warning(msg % (self.path, e))
finally:
raise ValueError
