def main(arguments=None):
""" main() function, encapsulated in a method to allow for easy invokation.
This method follows Guido van Rossum's suggestions on how to write Python
main() functions in order to make them more flexible. By encapsulating the
main code of the script in a function and making it take an optional
argument the script can be called not only from other modules, but also
from the interactive Python prompt.
Guido van van Rossum - Python main() functions:
http://www.artima.com/weblogs/viewpost.jsp?thread=4829
Keyword arguments:
arguments - the list of command line arguments passed to the script.
"""
if arguments is None:
arguments = sys.argv[1:] # ignore argv[0], the script name
(options, args) = parser.parse_args(args=arguments)
# Print the help and abort the execution if there are fewer than three
# positional arguments left, as the user must specify at least two FITS
# images and the output mosaic into which they are assembled.
if len(args) < 3:
parser.print_help()
return 2 # used for command line syntax errors
else:
assert len(args) >= 3
input_paths = set(args[:-1])
output_path = args[-1]
# Refuse to overwrite the output FITS file unless explicitly instructed to
# do so. Note that, if the --overwritten option is given, we do not need to
# delete the existing file: it will be silently overwritten when the output
# of montage.mosaic() is shutil.move()'d to the output path.
if os.path.exists(output_path):
if not options.overwrite:
msg = "%sError. The output file '%s' already exists."
print msg % (style.prefix, output_path)
print style.error_exit_message
return 1
# Workaround for a bug in montage.mosaic() that raises an error ('mpirun
# has exited due to process rank [...] without calling "finalize"...') if
# mpi = True and background_match = True. Until this is fixed, we can only
# use one core if the --background-match option is given by the user.
if options.background_match and options.ncores > 1:
options.ncores = 1
for msg in (
"{0}Warning: --background-match is incompatible with --cores > 1.",
"{0}Setting the --cores option to a value of one.",
"{0}This is a workaround for a known bug in montage-wrapper:",
"{0}https://github.com/astropy/montage-wrapper/issues/18"):
print msg.format(style.prefix)
print
# Map each filter to a list of FITSImage objects
files = fitsimage.InputFITSFiles()
msg = "%sMaking sure the %d input paths are FITS images..."
print msg % (style.prefix, len(input_paths))
util.show_progress(0.0)
for index, path in enumerate(input_paths):
# fitsimage.FITSImage.__init__() raises fitsimage.NonStandardFITS if
# one of the paths is not a standard-conforming FITS file.
try:
img = fitsimage.FITSImage(path)
# If we do not need to know the photometric filter (because the
# --filter was not given) do not read it from the FITS header.
# Instead, use None. This means that 'files', a dictionary, will
# only have a key, None, mapping to all the input FITS images.
if options.filter:
pfilter = img.pfilter(options.filterk)
else:
pfilter = None
files[pfilter].append(img)
except fitsimage.NonStandardFITS:
print
msg = "'%s' is not a standard FITS file"
raise fitsimage.NonStandardFITS(msg % path)
percentage = (index + 1) / len(input_paths) * 100
util.show_progress(percentage)
print # progress bar doesn't include newline
# The --filter option allows the user to specify which FITS files, among
# all those received as input, must be combined: only those images taken
# in the options.filter photometric filter.
if options.filter:
msg = "%s%d different photometric filters were detected:"
print msg % (style.prefix, len(files.keys()))
for pfilter, images in sorted(files.iteritems()):
msg = "%s %s: %d files (%.2f %%)"
percentage = len(images) / len(files) * 100
print msg % (style.prefix, pfilter, len(images), percentage)
msg = "%sIgnoring images not taken in the '%s' photometric filter..."
print msg % (style.prefix, options.filter),
sys.stdout.flush()
discarded = 0
for pfilter, images in files.items():
if pfilter != options.filter:
discarded += len(images)
del files[pfilter]
if not files:
print
msg = "%sError. No image was taken in the '%s' filter."
print msg % (style.prefix, options.filter)
print style.error_exit_message
return 1
else:
print 'done.'
msg = "%s%d images taken in the '%s' filter, %d were discarded."
print msg % (style.prefix, len(files), options.filter, discarded)
# montage.mosaic() silently ignores those FITS images that have no WCS
# information in their headers, and also raises a rather cryptic exception
# (mMakeHdr: Invalid table file) if none of them has been astrometrically
# solved. Instead of ignoring some images without warning or showing a
# confusing error message that makes it almost impossible to understand
# what may be failing, use FITSImage.center_wcs() to make sure that all the
# images have WCS information, raising NoWCSInformationError otherwise.
for img in files:
# May raise NoWCSInformationError
img.center_wcs()
# montage.mosaic() requires as first argument the directory containing the
# input FITS images but, in order to maintain the same syntax across all
# LEMON commands, we receive them as command-line arguments. Thus, create a
# temporary directory and symlink from it the input images. Hard links are
# not an option because os.link() will raise "OSError: [Errno 18] Invalid
# cross-device link" if the temporary directory is created in a different
# partition.
pid = os.getpid()
suffix = "_LEMON_%d_mosaic" % pid
kwargs = dict(suffix=suffix + '_input')
input_dir = tempfile.mkdtemp(**kwargs)
atexit.register(util.clean_tmp_files, input_dir)
for img in files:
path = img.path
source = os.path.abspath(path)
basename = os.path.basename(path)
link_name = os.path.join(input_dir, basename)
os.symlink(source, link_name)
# The output of montage.mosaic() is another directory, to which several
# files are written, so we need the path to a second temporary directory.
# Delete it before calling mosaic(), as otherwise it will raise IOError
# ("Output directory already exists").
kwargs = dict(suffix=suffix + '_output')
output_dir = tempfile.mkdtemp(**kwargs)
atexit.register(util.clean_tmp_files, output_dir)
os.rmdir(output_dir)
kwargs = dict(
background_match=options.background_match,
combine=options.combine,
bitpix=-64,
)
if options.ncores > 1:
kwargs['mpi'] = True # use MPI whenever possible
kwargs['n_proc'] = options.ncores # number of MPI processes
montage.mosaic(input_dir, output_dir, **kwargs)
# montage.mosaic() writes several files to the output directory, but we are
# only interested in one of them: 'mosaic.fits', the mosaic FITS image.
MOSAIC_OUTPUT = 'mosaic.fits'
src = os.path.join(output_dir, MOSAIC_OUTPUT)
if options.reproject:
print "%sReproject mosaic to point North..." % style.prefix,
sys.stdout.flush()
kwargs = dict(north_aligned=True, silent_cleanup=True)
montage.reproject(src, output_path, **kwargs)
print 'done.'
else:
# No reprojection, move mosaic to the output path
shutil.move(src, output_path)
print "%sYou're done ^_^" % style.prefix
return 0
def main(arguments=None):
"""main() function, encapsulated in a method to allow for easy invokation.
This method follows Guido van Rossum's suggestions on how to write Python
main() functions in order to make them more flexible. By encapsulating the
main code of the script in a function and making it take an optional
argument the script can be called not only from other modules, but also
from the interactive Python prompt.
Guido van van Rossum - Python main() functions:
http://www.artima.com/weblogs/viewpost.jsp?thread=4829
Keyword arguments:
arguments - the list of command line arguments passed to the script.
"""
if arguments is None:
arguments = sys.argv[1:] # ignore argv[0], the script name
(options, args) = parser.parse_args(args=arguments)
# Adjust the logger level to WARNING, INFO or DEBUG, depending on the
# given number of -v options (none, one or two or more, respectively)
logging_level = logging.WARNING
if options.verbose == 1:
logging_level = logging.INFO
elif options.verbose >= 2:
logging_level = logging.DEBUG
logging.basicConfig(format=style.LOG_FORMAT, level=logging_level)
# Print the help and abort the execution if there are not two positional
# arguments left after parsing the options, as the user must specify at
# least one (only one?) input FITS file and the output JSON file.
if len(args) < 2:
parser.print_help()
return 2 # 2 is generally used for command line syntax errors
else:
sources_img_path = args[0]
input_paths = list(set(args[1:-1]))
output_json_path = args[-1]
# The execution of this module, especially when doing long-term monitoring
# of reasonably crowded fields, may easily take several *days*. The least
# we can do, in order to spare the end-user from insufferable grief because
# of the waste of billions of valuable CPU cycles, is to avoid to have the
# output file accidentally overwritten.
if os.path.exists(output_json_path):
if not options.overwrite:
msg = "%sError. The output file '%s' already exists."
print msg % (style.prefix, output_json_path)
print style.error_exit_message
return 1
msg = "%sExamining the headers of the %s FITS files given as input..."
print msg % (style.prefix, len(input_paths))
files = fitsimage.InputFITSFiles()
for index, img_path in enumerate(input_paths):
img = fitsimage.FITSImage(img_path)
pfilter = img.pfilter(options.filterk)
files[pfilter].append(img)
percentage = (index + 1) / len(input_paths) * 100
util.show_progress(percentage)
print # progress bar doesn't include newline
print style.prefix
# To begin with, we need to identify the most constant stars, something for
# which we have to do photometry on all the stars and for all the images of
# the campaign. But fret not, as this has to be done only this time: once
# we get the light curves of all the stars and for all the images, we will
# be able to determine which are the most constant among them and work
# always with this subset in order to determine which aperture and sky
# annulus are the optimal.
msg = "%sDoing initial photometry with FWHM-derived apertures..."
print msg % style.prefix
print style.prefix
# mkstemp() returns a tuple containing an OS-level handle to an open file
# and its absolute pathname. Thus, we need to close the file right after
# creating it, and tell the photometry module to overwrite (-w) it.
kwargs = dict(prefix="photometry_", suffix=".LEMONdB")
phot_db_handle, phot_db_path = tempfile.mkstemp(**kwargs)
atexit.register(util.clean_tmp_files, phot_db_path)
os.close(phot_db_handle)
basic_args = [sources_img_path] + input_paths + [phot_db_path, "--overwrite"]
phot_args = [
"--maximum",
options.maximum,
"--margin",
options.margin,
"--cores",
options.ncores,
"--min-sky",
options.min,
"--objectk",
options.objectk,
"--filterk",
options.filterk,
"--datek",
options.datek,
"--timek",
options.timek,
"--expk",
options.exptimek,
"--coaddk",
options.coaddk,
"--gaink",
options.gaink,
"--fwhmk",
options.fwhmk,
"--airmk",
options.airmassk,
]
# The --gain and --uik options default to None, so add them to the list of
# arguments only if they were given. Otherwise, (a) --gaink would be given
# a value of 'None', a string, that would result in an error when optparse
# attempted to convert it to float, and (b) --uik would understood 'None'
# as the name of the keyword storing the path to the uncalibrated image.
if options.gain:
phot_args += ["--gain", options.gain]
if options.uncimgk:
phot_args += ["--uncimgk", options.uncimgk]
# Pass as many '-v' options as we have received here
[phot_args.append("-v") for x in xrange(options.verbose)]
extra_args = [
"--aperture",
options.aperture,
"--annulus",
options.annulus,
"--dannulus",
options.dannulus,
]
# Non-zero return codes raise subprocess.CalledProcessError
args = basic_args + phot_args + extra_args
check_run(photometry.main, [str(a) for a in args])
# Now we need to compute the light curves and find those that are most
# constant. This, of course, has to be done for each filter, as a star
# identified as constant in Johnson I may be too faint in Johnson B, for
# example. In other words: we need to calculate the light curve of each
# star and for each filter, and then determine which are the
# options.nconstant stars with the lowest standard deviation.
print style.prefix
msg = "%sGenerating light curves for initial photometry."
print msg % style.prefix
print style.prefix
kwargs = dict(prefix="diffphot_", suffix=".LEMONdB")
diffphot_db_handle, diffphot_db_path = tempfile.mkstemp(**kwargs)
atexit.register(util.clean_tmp_files, diffphot_db_path)
os.close(diffphot_db_handle)
diff_args = [
phot_db_path,
"--output",
diffphot_db_path,
"--overwrite",
"--cores",
options.ncores,
"--minimum-images",
options.min_images,
"--stars",
options.nconstant,
"--minimum-stars",
options.min_cstars,
"--pct",
options.pct,
"--weights-threshold",
options.wminimum,
"--max-iters",
options.max_iters,
"--worst-fraction",
options.worst_fraction,
]
[diff_args.append("-v") for x in xrange(options.verbose)]
check_run(diffphot.main, [str(a) for a in diff_args])
print style.prefix
# Map each photometric filter to the path of the temporary file where the
# right ascension and declination of each constant star, one per line, will
# be saved. This file is from now on passed, along with the --coordinates
# option, to photometry.main(), so that photometry is not done on all the
# astronomical objects, but instead exclusively on these ones.
coordinates_files = {}
miner = mining.LEMONdBMiner(diffphot_db_path)
for pfilter in miner.pfilters:
# LEMONdBMiner.sort_by_curve() returns a list of two-element tuples,
# mapping the ID of each star to the standard deviation of its light
# curve in this photometric filter. The list is sorted in increasing
# order by the standard deviation. We are only interested in the first
# 'options.nconstant', needing at least 'options.pminimum'.
msg = "%sIdentifying the %d most constant stars for the %s filter..."
args = style.prefix, options.nconstant, pfilter
print msg % args,
sys.stdout.flush()
kwargs = dict(minimum=options.min_images)
stars_stdevs = miner.sort_by_curve_stdev(pfilter, **kwargs)
cstars = stars_stdevs[: options.nconstant]
if len(cstars) < options.pminimum:
msg = (
"fewer than %d stars identified as constant in the "
"initial photometry for the %s filter"
)
args = options.pminimum, pfilter
raise NotEnoughConstantStars(msg % args)
else:
print "done."
if len(cstars) < options.nconstant:
msg = "%sBut only %d stars were available. Using them all, anyway."
print msg % (style.prefix, len(cstars))
# Replacing whitespaces with underscores is easier than having to quote
# the path to the --coordinates file if the name of the filter contains
# them (otherwise, optparse would only see up to the first whitespace).
prefix = "%s_" % str(pfilter).replace(" ", "_")
kwargs = dict(prefix=prefix, suffix=".coordinates")
coords_fd, coordinates_files[pfilter] = tempfile.mkstemp(**kwargs)
atexit.register(util.clean_tmp_files, coordinates_files[pfilter])
# LEMONdBMiner.get_star() returns a five-element tuple with the x and y
# coordinates, right ascension, declination and instrumental magnitude
# of the astronomical object in the sources image.
for star_id, _ in cstars:
ra, dec = miner.get_star(star_id)[2:4]
os.write(coords_fd, "%.10f\t%.10f\n" % (ra, dec))
os.close(coords_fd)
msg = "%sStar coordinates for %s temporarily saved to %s"
print msg % (style.prefix, pfilter, coordinates_files[pfilter])
# The constant astronomical objects, the only ones to which we will pay
# attention from now on, have been identified. So far, so good. Now we
# generate the light curves of these objects for each candidate set of
# photometric parameters. We store the evaluated values in a dictionary in
# which each filter maps to a list of json_parse.CandidateAnnuli objects.
evaluated_annuli = collections.defaultdict(list)
for pfilter, coords_path in coordinates_files.iteritems():
print style.prefix
msg = "%sFinding the optimal photometric parameters for the %s filter."
print msg % (style.prefix, pfilter)
if len(files[pfilter]) < options.min_images:
msg = "fewer than %d images (--minimum-images option) for %s"
args = options.min_images, pfilter
raise NotEnoughConstantStars(msg % args)
# The median FWHM of the images is needed in order to calculate the
# range of apertures that we need to evaluate for this filter.
msg = "%sCalculating the median FWHM for this filter..."
print msg % style.prefix,
pfilter_fwhms = []
for img in files[pfilter]:
img_fwhm = photometry.get_fwhm(img, options)
logging.debug("%s: FWHM = %.3f" % (img.path, img_fwhm))
pfilter_fwhms.append(img_fwhm)
fwhm = numpy.median(pfilter_fwhms)
print " done."
# FWHM to range of pixels conversion
min_aperture = fwhm * options.lower
max_aperture = fwhm * options.upper
annulus = fwhm * options.sky
dannulus = fwhm * options.width
# The dimensions of the sky annulus remain fixed, while the
# aperture is in the range [lower * FWHM, upper FWHM], with
# increments of options.step pixels.
filter_apertures = numpy.arange(min_aperture, max_aperture, options.step)
assert filter_apertures[0] == min_aperture
msg = "%sFWHM (%s passband) = %.3f pixels, therefore:"
print msg % (style.prefix, pfilter, fwhm)
msg = "%sAperture radius, minimum = %.3f x %.2f = %.3f pixels "
print msg % (style.prefix, fwhm, options.lower, min_aperture)
msg = "%sAperture radius, maximum = %.3f x %.2f = %.3f pixels "
print msg % (style.prefix, fwhm, options.upper, max_aperture)
msg = "%sAperture radius, step = %.2f pixels, which means that:"
print msg % (style.prefix, options.step)
msg = "%sAperture radius, actual maximum = %.3f + %d x %.2f = %.3f pixels"
args = (
style.prefix,
min_aperture,
len(filter_apertures),
options.step,
max(filter_apertures),
)
print msg % args
msg = "%sSky annulus, inner radius = %.3f x %.2f = %.3f pixels"
print msg % (style.prefix, fwhm, options.sky, annulus)
msg = "%sSky annulus, width = %.3f x %.2f = %.3f pixels"
print msg % (style.prefix, fwhm, options.width, dannulus)
msg = "%s%d different apertures in the range [%.2f, %.2f] to be evaluated:"
args = (
style.prefix,
len(filter_apertures),
filter_apertures[0],
filter_apertures[-1],
)
print msg % args
# For each candidate aperture, and only with the images taken in
# this filter, do photometry on the constant stars and compute the
# median of the standard deviation of their light curves as a means
# of evaluating the suitability of this combination of parameters.
for index, aperture in enumerate(filter_apertures):
print style.prefix
kwargs = dict(prefix="photometry_", suffix=".LEMONdB")
fd, aper_phot_db_path = tempfile.mkstemp(**kwargs)
atexit.register(util.clean_tmp_files, aper_phot_db_path)
os.close(fd)
paths = [img.path for img in files[pfilter]]
basic_args = [sources_img_path] + paths + [aper_phot_db_path, "--overwrite"]
extra_args = [
"--filter",
str(pfilter),
"--coordinates",
coords_path,
"--aperture-pix",
aperture,
"--annulus-pix",
annulus,
"--dannulus-pix",
dannulus,
]
args = basic_args + phot_args + extra_args
check_run(photometry.main, [str(a) for a in args])
kwargs = dict(prefix="diffphot_", suffix=".LEMONdB")
fd, aper_diff_db_path = tempfile.mkstemp(**kwargs)
atexit.register(util.clean_tmp_files, aper_diff_db_path)
os.close(fd)
# Reuse the arguments used earlier for diffphot.main(). We only
# need to change the first argument (path to the input LEMONdB)
# and the third one (path to the output LEMONdB)
diff_args[0] = aper_phot_db_path
diff_args[2] = aper_diff_db_path
check_run(diffphot.main, [str(a) for a in diff_args])
miner = mining.LEMONdBMiner(aper_diff_db_path)
try:
kwargs = dict(minimum=options.min_images)
cstars = miner.sort_by_curve_stdev(pfilter, **kwargs)
except mining.NoStarsSelectedError:
# There are no light curves with at least options.min_images points.
# Therefore, much to our sorrow, we cannot evaluate this aperture.
msg = "%sNo constant stars for this aperture. Ignoring it..."
print msg % style.prefix
continue
# There must be at most 'nconstant' stars, but there may be fewer
# if this aperture causes one or more of the constant stars to be
# too faint (INDEF) in so many images as to prevent their lights
# curve from being computed.
assert len(cstars) <= options.nconstant
if len(cstars) < options.pminimum:
msg = (
"%sJust %d constant stars, fewer than the allowed "
"minimum of %d, had their light curves calculated "
"for this aperture. Ignoring it..."
)
args = style.prefix, len(cstars), options.pminimum
print style.prefix
continue
# 'cstars' contains two-element tuples: (ID, stdev)
stdevs_median = numpy.median([x[1] for x in cstars])
params = (aperture, annulus, dannulus, stdevs_median)
# NumPy floating-point data types are not JSON serializable
args = (float(x) for x in params)
candidate = json_parse.CandidateAnnuli(*args)
evaluated_annuli[pfilter].append(candidate)
msg = "%sAperture = %.3f, median stdev (%d stars) = %.4f"
args = style.prefix, aperture, len(cstars), stdevs_median
print msg % args
percentage = (index + 1) / len(filter_apertures) * 100
msg = "%s%s progress: %.2f %%"
args = style.prefix, pfilter, percentage
print msg % args
# Let the user know of the best 'annuli', that is, the one for
# which the standard deviation of the constant stars is minimal
kwargs = dict(key=operator.attrgetter("stdev"))
best_candidate = min(evaluated_annuli[pfilter], **kwargs)
msg = "%sBest aperture found at %.3f pixels with stdev = %.4f"
args = style.prefix, best_candidate.aperture, best_candidate.stdev
print msg % args
print style.prefix
msg = "%sSaving the evaluated apertures to the '%s' JSON file ..."
print msg % (style.prefix, output_json_path),
json_parse.CandidateAnnuli.dump(evaluated_annuli, output_json_path)
print " done."
print "%sYou're done ^_^" % style.prefix
return 0