def step1(expnum,
ccd,
prefix='',
version='p',
fwhm=4,
sex_thresh=1.3,
wave_thresh=2.7,
maxcount=30000):
"""run the actual step1jmp/matt codes.
expnum: the CFHT expousre to process
ccd: which ccd in the mosaic to process
fwhm: the image quality, FWHM, of the image. In pixels.
sex_thresh: the detection threhold to run sExtractor at
wave_thresh: the detection threshold for wavelet
maxcount: saturation level
"""
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
mopheader = storage.get_image(expnum, ccd, version=version,
ext='mopheader', prefix=prefix)
fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version)
basename = os.path.splitext(filename)[0]
outfile = util.exec_prog(['step1jmp',
'-f', basename,
'-t', str(wave_thresh),
'-w', str(fwhm),
'-m', str(maxcount)])
obj_uri = storage.get_uri(expnum,ccd,version=version,ext='obj.jmp',
prefix=prefix)
obj_filename = basename+".obj.jmp"
storage.copy(obj_filename,obj_uri)
## for step1matt we need the weight image
hdulist = fits.open(filename)
flat_name = hdulist[0].header.get('FLAT','06Bm02.flat.r.36.01.fits')
flat_name = flat_name[0:-5]
flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits',
subdir='calibrators', rescale=False)
if not os.access('weight.fits',os.R_OK):
os.symlink(flat_filename, 'weight.fits')
outfile = util.exec_prog(['step1matt',
'-f', basename,
'-t', str(sex_thresh),
'-w', str(fwhm),
'-m', str(maxcount)])
obj_uri = storage.get_uri(expnum,ccd,version=version,ext='obj.matt',
prefix=prefix)
obj_filename = basename+".obj.matt"
storage.copy(obj_filename,obj_uri)
return True
def plant(expnums, ccd, rmin, rmax, ang, width, version='s'):
'''run the plant script on this combination of exposures'''
ptf = open('proc-these-files','w')
ptf.write("# Files to be planted and search\n")
ptf.write("# image fwhm plant\n")
for expnum in expnums:
fwhm = storage.get_fwhm(expnum,ccd)
filename = storage.get_image(expnum, ccd=ccd, version=version)
ptf.write("%s %3.1f YES\n" % ( filename[0:-5],
fwhm ))
for ext in ['apcor',
'obj.jmp',
'trans.jmp',
'psf.fits',
'mopheader',
'phot',
'zeropoint.used']:
apcor = storage.get_image(expnum, ccd=ccd, version='s',
ext=ext)
ptf.close()
cmd_args = ['plant.csh',os.curdir,
str(rmin), str(rmax), str(ang), str(width)]
util.exec_prog(cmd_args)
if args.dryrun:
# Don't push back to VOSpace
return
uri = storage.get_uri('Object',ext='planted',version='',
subdir=str(
expnums[0])+"/ccd%s" % (str(ccd).zfill(2)))
storage.copy('Object.planted',uri)
uri = os.path.join(os.path.dirname(uri), 'shifts')
storage.copy('shifts', uri)
for expnum in expnums:
uri = storage.get_uri(expnum,
ccd=ccd,
version=version,
ext='fits', prefix='fk')
filename = os.path.basename(uri)
storage.copy(filename, uri)
for ext in ['mopheader',
'psf.fits',
'fwhm',
'apcor', 'zeropoint.used', 'trans.jmp']:
storage.delete(expnum, ccd, 's', ext, prefix='fk')
storage.vlink(expnum, ccd, 'p', ext,
expnum, ccd, 's', ext, l_prefix='fk')
return
def plant(expnums, ccd, rmin, rmax, ang, width, number=10, mmin=21.0, mmax=25.5, version='s', dry_run=False):
"""Plant artificial sources into the list of images provided.
:param expnums: list of MegaPrime exposure numbers to add artificial KBOs to
:param ccd: which ccd to work on.
:param rmin: The minimum rate of motion to add sources at (''/hour)
:param rmax: The maximum rate of motion to add sources at (''/hour)
:param ang: The mean angle of motion to add sources
:param width: The +/- range of angles of motion
:param version: Add sources to the 'o', 'p' or 's' images
:param dry_run: don't push results to VOSpace.
"""
# Construct a list of artificial KBOs with positions in the image
# and rates of motion within the bounds given by the caller.
filename = storage.get_image(expnums[0],
ccd=ccd,
version=version)
header = fits.open(filename)[0].header
bounds = util.get_pixel_bounds_from_datasec_keyword(header.get('DATASEC', '[33:2080,1:4612]'))
# generate a set of artificial KBOs to add to the image.
kbos = KBOGenerator.get_kbos(n=number,
rate=(rmin, rmax),
angle=(ang - width, ang + width),
mag=(mmin, mmax),
x=(bounds[0][0], bounds[0][1]),
y=(bounds[1][0], bounds[1][1]),
filename='Object.planted')
for expnum in expnums:
filename = storage.get_image(expnum, ccd, version)
psf = storage.get_file(expnum, ccd, version, ext='psf.fits')
plant_kbos(filename, psf, kbos, get_shifts(expnum, ccd, version), "fk")
if dry_run:
return
uri = storage.get_uri('Object', ext='planted', version='',
subdir=str(
expnums[0]) + "/ccd%s" % (str(ccd).zfill(2)))
storage.copy('Object.planted', uri)
for expnum in expnums:
uri = storage.get_uri(expnum,
ccd=ccd,
version=version,
ext='fits', prefix='fk')
filename = os.path.basename(uri)
storage.copy(filename, uri)
return
def mkpsf(expnum, ccd, version, dry_run=False, prefix=""):
"""Run the OSSOS jmpmakepsf script.
"""
## confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits'))
if not dry_run:
storage.mkdir(destdir)
## get image from the vospace storage area
logging.info("Getting file from VOSpace")
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
## launch the makepsf script
logging.info(util.exec_prog(['jmpmakepsf.csh',
'./',
filename,
'yes', 'yes']))
if dry_run:
return
## place the results into VOSpace
basename = os.path.splitext(filename)[0]
for ext in ('mopheader', 'psf.fits',
'zeropoint.used', 'apcor', 'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext)
source = basename + "." + str(ext)
storage.copy(source, dest)
return
def mkpsf(expnum, ccd, fversion):
"""Run the OSSOS makepsf script.
"""
## get image from the vospace storage area
filename = storage.get_image(expnum, ccd, version=fversion)
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
## launch the makepsf script
util.exec_prog(['jmpmakepsf.csh',
'./',
filename,
'no'])
## place the results into VOSpace
basename = os.path.splitext(filename)[0]
## confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, version=fversion,ext='fits'))
logging.info("Checking that destination directories exist")
storage.mkdir(destdir)
for ext in ('mopheader', 'psf.fits',
'zeropoint.used', 'apcor', 'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum, ccd, version=fversion, ext=ext)
source = basename + "." + ext
logging.info("Copying %s -> %s" % ( source, dest))
storage.remove(dest)
storage.copy(source, dest)
return
def mk_mopheader(expnum, ccd, version, dry_run=False, prefix=""):
"""Run the OSSOS mopheader script.
"""
## confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits'))
if not dry_run:
storage.mkdir(destdir)
## get image from the vospace storage area
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
logging.info("Running mopheader on %s %d" % (expnum, ccd))
## launch the mopheader script
## launch the makepsf script
expname = os.path.basename(filename).strip('.fits')
logging.info(util.exec_prog(['stepZjmp',
'-f',
expname]))
mopheader_filename = expname+".mopheader"
# mopheader_filename = mopheader.main(filename)
if dry_run:
return
destination = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='mopheader')
source = mopheader_filename
storage.copy(source, destination)
return
def scramble(expnums, ccd, version='p', dry_run=False):
"""run the plant script on this combination of exposures"""
mjds = []
fobjs = []
for expnum in expnums:
filename = storage.get_image(expnum, ccd=ccd, version=version)
fobjs.append(fits.open(filename))
# Pull out values to replace in headers.. must pull them
# as otherwise we get pointers...
mjds.append(fobjs[-1][0].header['MJD-OBS'])
order = [0, 2, 1]
for idx in range(len(fobjs)):
logging.info("Flipping %d to %d" % (fobjs[idx][0].header['EXPNUM'],
expnums[order[idx]]))
fobjs[idx][0].header['EXPNUM'] = expnums[order[idx]]
fobjs[idx][0].header['MJD-OBS'] = mjds[order[idx]]
uri = storage.get_uri(expnums[order[idx]],
ccd=ccd,
version='s',
ext='fits')
fname = os.path.basename(uri)
if os.access(fname, os.F_OK):
os.unlink(fname)
fobjs[idx].writeto(fname)
if dry_run:
continue
storage.copy(fname, uri)
return
def mkpsf(expnum, ccd):
"""Run the OSSOS makepsf script.
"""
## get image from the vospace storage area
filename = storage.get_image(expnum, ccd, version='p')
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
## launch the makepsf script
util.exec_prog(['jmpmakepsf.csh',
'./',
filename,
'no'])
## place the results into VOSpace
basename = os.path.splitext(filename)[0]
## confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, version='p',ext='fits'))
logging.info("Checking that destination direcoties exist")
storage.mkdir(destdir)
for ext in ('mopheader', 'psf.fits',
'zeropoint.used', 'apcor', 'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum, ccd, version='p', ext=ext)
source = basename + "." + ext
storage.copy(source, dest)
return
def scramble(expnums, ccd, version='p', dry_run=False):
"""run the plant script on this combination of exposures"""
mjds = []
fobjs = []
for expnum in expnums:
filename = storage.get_image(expnum, ccd=ccd, version=version)
fobjs.append(fits.open(filename))
# Pull out values to replace in headers.. must pull them
# as otherwise we get pointers...
mjds.append(fobjs[-1][0].header['MJD-OBS'])
order = [0, 2, 1]
for idx in range(len(fobjs)):
logging.info("Flipping %d to %d" % (fobjs[idx][0].header['EXPNUM'],
expnums[order[idx]]))
fobjs[idx][0].header['EXPNUM'] = expnums[order[idx]]
fobjs[idx][0].header['MJD-OBS'] = mjds[order[idx]]
uri = storage.get_uri(expnums[order[idx]],
ccd=ccd,
version='s',
ext='fits')
fname = os.path.basename(uri)
if os.access(fname, os.F_OK):
os.unlink(fname)
fobjs[idx].writeto(fname)
if dry_run:
continue
storage.copy(fname, uri)
return
def compute_trans(expnums, ccd, version, prefix=None, default="WCS"):
"""
Pull the astrometric header for each image, compute an x/y transform and compare to trans.jmp
this one overides trans.jmp if they are very different.
@param expnums:
@param ccd:
@param version:
@param prefix:
@return: None
"""
wcs_dict = {}
for expnum in expnums:
try:
# TODO This assumes that the image is already N/E flipped.
# If compute_trans is called after the image is retrieved from archive then we get the disk version.
filename = storage.get_image(expnum, ccd, version, prefix=prefix)
this_wcs = wcs.WCS(fits.open(filename)[0].header)
except Exception as err:
logging.warning("WCS Trans compute failed. {}".format(str(err)))
return
wcs_dict[expnum] = this_wcs
x0 = wcs_dict[expnums[0]].header['NAXIS1'] / 2.0
y0 = wcs_dict[expnums[0]].header['NAXIS2'] / 2.0
(ra0, dec0) = wcs_dict[expnums[0]].xy2sky(x0, y0)
result = ""
for expnum in expnums:
filename = storage.get_file(expnum,
ccd,
version,
ext='.trans.jmp',
prefix=prefix)
jmp_trans = file(filename, 'r').readline().split()
(x, y) = wcs_dict[expnum].sky2xy(ra0, dec0)
x1 = float(
jmp_trans[0]) + float(jmp_trans[1]) * x + float(jmp_trans[2]) * y
y1 = float(
jmp_trans[3]) + float(jmp_trans[4]) * x + float(jmp_trans[5]) * y
dr = math.sqrt((x1 - x0)**2 + (y1 - y0)**2)
if dr > 0.5:
result += "WARNING: WCS-JMP transforms mis-matched {} reverting to using {}.\n".format(
expnum, default)
if default == "WCS":
uri = storage.dbimages_uri(expnum,
ccd,
version,
ext='.trans.jmp',
prefix=prefix)
filename = os.path.basename(uri)
trans = file(filename, 'w')
trans.write("{:5.2f} 1. 0. {:5.2f} 0. 1.\n".format(
x0 - x, y0 - y))
trans.close()
else:
result += "WCS-JMP transforms match {}\n".format(expnum)
return result
def step3(expnums, ccd, version, rate_min,
rate_max, angle, width, field=None, prefix=None):
'''run the actual step2 on the given exp/ccd combo'''
jmp_args = ['step3jmp']
matt_args = ['step3matt']
idx = 0
cmd_args = []
for expnum in expnums:
idx += 1
for ext in ['unid.jmp', 'unid.matt',
'trans.jmp' ]:
filename = storage.get_image(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix
)
image = os.path.splitext(os.path.splitext(os.path.basename(filename))[0])[0]
cmd_args.append('-f%d' % ( idx))
cmd_args.append(image)
cmd_args.extend(['-rn', str(rate_min),
'-rx', str(rate_max),
'-a', str(angle),
'-w', str(width)])
jmp_args.extend(cmd_args)
matt_args.extend(cmd_args)
util.exec_prog(jmp_args)
util.exec_prog(matt_args)
if field is None:
field = str(expnums[0])
storage.mkdir(os.path.dirname(
storage.get_uri(field,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)))
for ext in ['moving.jmp', 'moving.matt']:
uri = storage.get_uri(field,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = '%s%d%s%s.%s' % ( prefix, expnums[0],
version,
str(ccd).zfill(2),
ext)
storage.copy(filename, uri)
return
def step2(expnums, ccd, version, prefix=None):
'''run the actual step2 on the given exp/ccd combo'''
jmp_args = ['step2jmp']
matt_args = ['step2matt_jmp']
idx = 0
for expnum in expnums:
jmp_args.append(
storage.get_image(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix
)[0:-8]
)
idx += 1
matt_args.append('-f%d' % ( idx))
matt_args.append(
storage.get_image(expnum,
ccd=ccd,
version=version,
ext='obj.matt',
prefix=prefix
)[0:-9]
)
util.exec_prog(jmp_args)
util.exec_prog(matt_args)
for expnum in expnums:
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
uri = storage.dbimages_uri(expnum,ccd=ccd,version=version,ext=ext, prefix=prefix)
filename = os.path.basename(uri)
storage.copy(filename, uri)
return
def plant(expnums, ccd, rmin, rmax, ang, width, version="s"):
"""run the plant script on this combination of exposures"""
ptf = open("proc-these-files", "w")
ptf.write("# Files to be planted and search\n")
ptf.write("# image fwhm plant\n")
for expnum in expnums:
fwhm = storage.get_fwhm(expnum, ccd, version=version)
filename = storage.get_image(expnum, ccd=ccd, version=version)
ptf.write("%s %3.1f YES\n" % (filename[0:-5], fwhm))
for ext in ["apcor", "obj.jmp", "trans.jmp", "psf.fits", "mopheader", "phot", "zeropoint.used"]:
apcor = storage.get_image(expnum, ccd=ccd, version=version, ext=ext)
ptf.close()
cmd_args = ["plant.csh", os.curdir, str(rmin), str(rmax), str(ang), str(width)]
util.exec_prog(cmd_args)
if args.dryrun:
# Don't push back to VOSpace
return
uri = storage.get_uri("Object", ext="planted", version="", subdir=str(expnums[0]) + "/ccd%s" % (str(ccd).zfill(2)))
storage.copy("Object.planted", uri)
uri = os.path.join(os.path.dirname(uri), "plant.shifts")
storage.copy("shifts", uri)
for expnum in expnums:
uri = storage.get_uri(expnum, ccd=ccd, version=version, ext="fits", prefix="fk")
filename = os.path.basename(uri)
storage.copy(filename, uri)
for ext in ["mopheader", "psf.fits", "fwhm", "apcor", "zeropoint.used", "trans.jmp"]:
storage.delete(expnum, ccd, "s", ext, prefix="fk")
storage.vlink(expnum, ccd, "s", ext, expnum, ccd, "s", ext, l_prefix="fk")
return
def _get_weight_map(filename, ccd):
# for step1matt we need the weight image
hdulist = fits.open(filename)
flat_name = hdulist[0].header.get('FLAT', 'weight.fits')
parts = os.path.splitext(flat_name)
if parts[1] == '.fz':
flat_name = os.path.splitext(parts[0])[0]
else:
flat_name = parts[0]
logging.info("Getting the Flat Field image.")
try:
flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators')
except Exception:
flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='old_calibrators')
if os.access('weight.fits', os.R_OK):
os.unlink('weight.fits')
if not os.access('weight.fits', os.R_OK):
os.symlink(flat_filename, 'weight.fits')
return flat_filename
def step2(expnums, ccd, version, prefix=None):
'''run the actual step2 on the given exp/ccd combo'''
jmp_args = ['step2jmp']
matt_args = ['step2matt_jmp']
idx = 0
for expnum in expnums:
jmp_args.append(
storage.get_image(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix)[0:-8])
idx += 1
matt_args.append('-f%d' % (idx))
matt_args.append(
storage.get_image(expnum,
ccd=ccd,
version=version,
ext='obj.matt',
prefix=prefix)[0:-9])
util.exec_prog(jmp_args)
util.exec_prog(matt_args)
for expnum in expnums:
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
uri = storage.dbimages_uri(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = os.path.basename(uri)
storage.copy(filename, uri)
return
def _get_weight_map(filename, ccd):
# for step1matt we need the weight image
hdulist = fits.open(filename)
flat_name = hdulist[0].header.get('FLAT', 'weight.fits')
parts = os.path.splitext(flat_name)
if parts[1] == '.fz':
flat_name = os.path.splitext(parts[0])[0]
else:
flat_name = parts[0]
logging.info("Getting the Flat Field image.")
try:
flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators')
except:
flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='old_calibrators')
if os.access('weight.fits', os.R_OK):
os.unlink('weight.fits')
if not os.access('weight.fits', os.R_OK):
os.symlink(flat_filename, 'weight.fits')
return flat_filename
def compute_trans(expnums, ccd, version, prefix=None, default="WCS"):
"""
Pull the astrometric header for each image, compute an x/y transform and compare to trans.jmp
this one overides trans.jmp if they are very different.
@param expnums:
@param ccd:
@param version:
@param prefix:
@return: None
"""
wcs_dict = {}
for expnum in expnums:
try:
# TODO This assumes that the image is already N/E flipped.
# If compute_trans is called after the image is retrieved from archive then we get the disk version.
filename = storage.get_image(expnum, ccd, version, prefix=prefix)
this_wcs = wcs.WCS(fits.open(filename)[0].header)
except Exception as err:
logging.warning("WCS Trans compute failed. {}".format(str(err)))
return
wcs_dict[expnum] = this_wcs
x0 = wcs_dict[expnums[0]].header['NAXIS1'] / 2.0
y0 = wcs_dict[expnums[0]].header['NAXIS2'] / 2.0
(ra0, dec0) = wcs_dict[expnums[0]].xy2sky(x0, y0)
result = ""
for expnum in expnums:
filename = storage.get_file(expnum, ccd, version, ext='.trans.jmp', prefix=prefix)
jmp_trans = file(filename, 'r').readline().split()
(x, y) = wcs_dict[expnum].sky2xy(ra0, dec0)
x1 = float(jmp_trans[0]) + float(jmp_trans[1]) * x + float(jmp_trans[2]) * y
y1 = float(jmp_trans[3]) + float(jmp_trans[4]) * x + float(jmp_trans[5]) * y
dr = math.sqrt((x1 - x0) ** 2 + (y1 - y0) ** 2)
if dr > 0.5:
result += "WARNING: WCS-JMP transforms mis-matched {} reverting to using {}.\n".format(expnum, default)
if default == "WCS":
uri = storage.dbimages_uri(expnum, ccd, version, ext='.trans.jmp', prefix=prefix)
filename = os.path.basename(uri)
trans = file(filename, 'w')
trans.write("{:5.2f} 1. 0. {:5.2f} 0. 1.\n".format(x0 - x, y0 - y))
trans.close()
else:
result += "WCS-JMP transforms match {}\n".format(expnum)
return result
def scramble(expnums, ccd, version='p'):
'''run the plant script on this combination of exposures'''
mjds = []
fobjs = []
for expnum in expnums:
filename = storage.get_image(expnum, ccd=ccd, version=version)
fobjs.append(fits.open(filename))
# Pull out values to replace in headers.. must pull them
# as otherwise we get pointers...
mjds.append(fobjs[-1][0].header['MJD-OBS'])
order = [1, 0, 2]
for idx in range(len(fobjs)):
logging.info("Flipping %d to %d" %
(fobjs[idx][0].header['EXPNUM'], expnums[order[idx]]))
fobjs[idx][0].header['EXPNUM'] = expnums[order[idx]]
fobjs[idx][0].header['MJD-OBS'] = mjds[order[idx]]
uri = storage.get_uri(expnums[order[idx]],
ccd=ccd,
version='s',
ext='fits')
fname = os.path.basename(uri)
if os.access(fname, os.F_OK):
os.unlink(fname)
fobjs[idx].writeto(fname)
storage.copy(fname, uri)
# now make a link between files that the plant system will need
for ext in [
'apcor', 'obj.jmp', 'mopheader', 'phot', 'psf.fits',
'trans.jmp', 'zeropoint.used', 'fwhm'
]:
storage.delete(expnums[order[idx]], ccd, 's', ext)
storage.vlink(expnums[idx], ccd, 'p', ext, expnums[order[idx]],
ccd, 's', ext)
return
def mkpsf(expnum, ccd, version, dry_run=False, prefix=""):
"""Run the OSSOS jmpmakepsf script.
"""
## confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum,
ccd,
prefix=prefix,
version=version,
ext='fits'))
if not dry_run:
storage.mkdir(destdir)
## get image from the vospace storage area
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
## launch the makepsf script
logging.info(util.exec_prog(['jmpmakepsf.csh', './', filename, 'no']))
if dry_run:
return
## place the results into VOSpace
basename = os.path.splitext(filename)[0]
for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm',
'phot'):
dest = storage.dbimages_uri(expnum,
ccd,
prefix=prefix,
version=version,
ext=ext)
source = basename + "." + str(ext)
storage.copy(source, dest)
return
def scramble(expnums, ccd, version='p'):
'''run the plant script on this combination of exposures'''
mjds = []
fobjs = []
for expnum in expnums:
filename = storage.get_image(expnum, ccd=ccd, version=version)
fobjs.append(fits.open(filename))
# Pull out values to replace in headers.. must pull them
# as otherwise we get pointers...
mjds.append(fobjs[-1][0].header['MJD-OBS'])
order = [1,0,2]
for idx in range(len(fobjs)):
logging.info("Flipping %d to %d" % ( fobjs[idx][0].header['EXPNUM'],
expnums[order[idx]]))
fobjs[idx][0].header['EXPNUM'] = expnums[order[idx]]
fobjs[idx][0].header['MJD-OBS'] = mjds[order[idx]]
uri = storage.get_uri(expnums[order[idx]],
ccd=ccd,
version='s',
ext='fits')
fname = os.path.basename(uri)
if os.access(fname, os.F_OK):
os.unlink(fname)
fobjs[idx].writeto(fname)
storage.copy(fname, uri)
# now make a link between files that the plant system will need
for ext in ['apcor', 'obj.jmp', 'mopheader', 'phot',
'psf.fits','trans.jmp', 'zeropoint.used', 'fwhm']:
if storage.exists(storage.get_uri(expnums[order[idx]], ccd, 's', ext)):
storage.delete(expnums[order[idx]], ccd, 's', ext)
storage.vlink(expnums[idx], ccd, 'p', ext,
expnums[order[idx]], ccd, 's', ext)
return
def plant(expnums,
ccd,
rmin,
rmax,
ang,
width,
number=10,
version='s',
dry_run=False):
"""Plant artificial sources into the list of images provided.
:param expnums: list of MegaPrime exposure numbers to add artificial KBOs to
:param ccd: which ccd to work on.
:param rmin: The minimum rate of motion to add sources at (''/hour)
:param rmax: The maximum rate of motion to add sources at (''/hour)
:param ang: The mean angle of motion to add sources
:param width: The +/- range of angles of motion
:param version: Add sources to the 'o', 'p' or 's' images
:param dry_run: don't push results to VOSpace.
"""
# Construct a list of artificial KBOs with positions in the image
# and rates of motion within the bounds given by the caller.
filename = storage.get_image(expnums[0], ccd=ccd, version=version)
header = fits.open(filename)[0].header
bounds = util.get_pixel_bounds_from_datasec_keyword(
header.get('DATASEC', '[33:2080,1:4612]'))
# generate a set of artifical KBOs to add to the image.
kbos = Table(names=('x', 'y', 'mag', 'sky_rate', 'angle', 'id'))
for kbo in KBOGenerator(n=number,
x=Range(bounds[0][0], bounds[0][1]),
y=Range(bounds[1][0], bounds[1][1]),
rate=Range(rmin, rmax),
angle=Range(ang - width, ang + width),
mag=Range(21.0, 25.0)):
kbos.add_row(kbo)
fd = open('Object.planted', 'w')
fd.write("# ")
kbos.write(fd, format='ascii.fixed_width', delimiter=None)
fd.close()
for expnum in expnums:
filename = storage.get_image(expnum, ccd, version)
psf = storage.get_file(expnum, ccd, version, ext='psf.fits')
plant_kbos(filename, psf, kbos, get_shifts(expnum, ccd, version), "fk")
if dry_run:
return
uri = storage.get_uri('Object',
ext='planted',
version='',
subdir=str(expnums[0]) + "/ccd%s" %
(str(ccd).zfill(2)))
storage.copy('Object.planted', uri)
for expnum in expnums:
uri = storage.get_uri(expnum,
ccd=ccd,
version=version,
ext='fits',
prefix='fk')
filename = os.path.basename(uri)
storage.copy(filename, uri)
return
def run(expnums,
ccd,
version,
prefix=None,
dry_run=False,
default="WCS",
force=False):
"""run the actual step2 on the given exp/ccd combo"""
jmp_trans = ['step2ajmp']
jmp_args = ['step2bjmp']
matt_args = ['step2matt_jmp']
if storage.get_status(task, prefix, expnums[0], version,
ccd) and not force:
logging.info("{} completed successfully for {}{}{}{:02d}".format(
task, prefix, expnums[0], version, ccd))
return
with storage.LoggingManager(task, prefix, expnums[0], ccd, version,
dry_run):
try:
for expnum in expnums:
if not storage.get_status(
dependency, prefix, expnum, version=version, ccd=ccd):
raise IOError(
35,
"Cannot start {} as {} not yet completed for {}{}{}{:02d}"
.format(task, dependency, prefix, expnum, version,
ccd))
message = storage.SUCCESS
idx = 0
logging.info("Retrieving catalog files to do matching.")
for expnum in expnums:
jmp_args.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix)[0:-8])
jmp_trans.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix)[0:-8])
idx += 1
matt_args.append('-f%d' % idx)
matt_args.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.matt',
prefix=prefix)[0:-9])
logging.info(
"Computing the catalog alignment using sources in catalogs.")
try:
logging.info(util.exec_prog(jmp_trans))
if default == "WCS":
logging.info("Comparing computed transform to WCS values")
logging.info(
compute_trans(expnums,
ccd,
version,
prefix,
default=default))
except Exception as ex:
logging.info("JMP Trans failed: {}".format(ex))
logging.info(
compute_trans(expnums, ccd, version, prefix,
default="WCS"))
logging.info("Using transform to match catalogs for three images.")
logging.info(util.exec_prog(jmp_args))
logging.info(util.exec_prog(matt_args))
# check that the shifts from step2 are rational by matching the bright star lists.
logging.info(
"Uisng checktrans to ensure that transforms were reasonable.")
check_args = ['checktrans']
if os.access('proc-these-files', os.R_OK):
os.unlink('proc-these-files')
ptf = open('proc-these-files', 'w')
ptf.write(
"# A dummy file that is created so checktrans could run.\n")
ptf.write("# Frame FWHM PSF?\n")
for expnum in expnums:
filename = os.path.splitext(
storage.get_image(expnum,
ccd,
version=version,
prefix=prefix))[0]
if not os.access(filename + ".bright.psf", os.R_OK):
os.link(filename + ".bright.jmp", filename + ".bright.psf")
if not os.access(filename + ".obj.psf", os.R_OK):
os.link(filename + ".obj.jmp", filename + ".obj.psf")
ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(
filename, _FWHM, "NO"))
ptf.close()
if os.access('BAD_TRANS', os.F_OK):
os.unlink('BAD_TRANS')
logging.info(util.exec_prog(check_args))
if os.access('BAD_TRANS', os.F_OK):
raise OSError(errno.EBADMSG, 'BAD_TRANS')
if os.access('proc-these-files', os.F_OK):
os.unlink('proc-these-files')
if dry_run:
return
for expnum in expnums:
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
uri = storage.dbimages_uri(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = os.path.basename(uri)
storage.copy(filename, uri)
except Exception as ex:
message = str(ex)
logging.error(message)
storage.set_status(task,
prefix,
expnums[0],
version,
ccd,
status=message)
return
def scramble(expnums, ccd, version='p', dry_run=False, force=False, prefix=''):
"""
run the plant script on this combination of exposures
@param expnums: list of exposure numbers to scramble the time on
@param ccd: which CCD in (assumes this is a CFHT MegaCam MEF)
@param version: should we scramble the 'p' or 'o' images?
@param dry_run: if dry run then don't save back to VOSpace.
@param force: if true then create scramble set, even if already exists.
@param prefix: a string that will be pre-pended to the EXPNUM to get the filename, sometimes 'fk'.
@return: None
"""
# Get a list of the MJD values and then write a re-ordering of those into files with 's'
# as their type instead of 'p' or 'o'
mjds = []
fobjs = []
message = storage.SUCCESS
if not (force or dry_run) and storage.get_status(
task, prefix, expnums[0], version='s', ccd=ccd):
logging.info("{} recorded as complete for {} ccd {}".format(
task, expnums, ccd))
return
with storage.LoggingManager(task, prefix, expnums[0], ccd, version):
try:
for expnum in expnums:
filename = storage.get_image(expnum, ccd=ccd, version=version)
fobjs.append(fits.open(filename))
# Pull out values to replace in headers.. must pull them
# as otherwise we get pointers...
mjds.append(fobjs[-1][0].header['MJD-OBS'])
order = [0, 2, 1]
for idx in range(len(fobjs)):
logging.info(
"Flipping %d to %d" %
(fobjs[idx][0].header['EXPNUM'], expnums[order[idx]]))
fobjs[idx][0].header['EXPNUM'] = expnums[order[idx]]
fobjs[idx][0].header['MJD-OBS'] = mjds[order[idx]]
uri = storage.get_uri(expnums[order[idx]],
ccd=ccd,
version='s',
ext='fits')
scramble_file_name = os.path.basename(uri)
if os.access(scramble_file_name, os.F_OK):
os.unlink(scramble_file_name)
fobjs[idx].writeto(scramble_file_name)
if not dry_run:
storage.copy(scramble_file_name, uri)
logging.info(message)
except Exception as ex:
message = str(ex)
logging.error(message)
if not dry_run:
storage.set_status(task,
prefix,
expnum,
version,
ccd,
status=message)
return
def run(expnum,
ccd,
prefix='',
version='p',
sex_thresh=_SEX_THRESHOLD,
wave_thresh=_WAVE_THRESHOLD,
maxcount=_MAX_COUNT,
dry_run=False,
force=True):
"""run the actual step1jmp/matt codes.
expnum: the CFHT expousre to process
ccd: which ccd in the mosaic to process
fwhm: the image quality, FWHM, of the image. In pixels.
sex_thresh: the detection threhold to run sExtractor at
wave_thresh: the detection threshold for wavelet
maxcount: saturation level
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version, ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(dependency, prefix, expnum, version, ccd):
raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format(
task, dependency, prefix, expnum, version, ccd))
logging.info("Retrieving imaging and input parameters from VOSpace")
storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader')
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version)
basename = os.path.splitext(filename)[0]
_get_weight_map(filename, ccd)
logging.info("Launching step1jmp")
logging.info(util.exec_prog(['step1jmp',
'-f', basename,
'-t', str(wave_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
logging.info(util.exec_prog(['step1matt',
'-f', basename,
'-t', str(sex_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
if os.access('weight.fits', os.R_OK):
os.unlink('weight.fits')
if not dry_run:
for ext in ['obj.jmp', 'obj.matt']:
obj_uri = storage.get_uri(expnum, ccd, version=version, ext=ext,
prefix=prefix)
obj_filename = basename + "." + ext
count = 0
with open(obj_filename, 'r'):
while True:
try:
count += 1
logging.info("Attempt {} to copy {} -> {}".format(count, obj_filename, obj_uri))
storage.copy(obj_filename, obj_uri)
break
except Exception as ex:
if count > 10:
raise ex
logging.info(message)
except Exception as ex:
message = str(ex)
logging.error(message)
if not dry_run:
storage.set_status(task, prefix, expnum, version, ccd, status=message)
def combine(expnum, ccd, prefix=None, type='p', field=None, measure3=MEASURE3 ):
if field is None:
field=str(expnum)
if prefix is not None and len(prefix) > 0:
field = "%s_%s" % ( prefix, field )
field += "_%s" % ( str(ccd))
for ext in ['moving.matt','moving.jmp']:
fname = storage.get_image(expnum,
ccd=ccd,
prefix=prefix,
version=type, ext=ext)
if prefix is not None and len(prefix) > 0:
planted = storage.get_image('Object',
subdir=str(expnum)+"/ccd%s" % (
str(ccd).zfill(2)),
version='',
ext='planted')
else:
prefix = ''
base_image = os.path.basename(
storage.get_uri(expnum,
ccd=ccd,
prefix=prefix,
version=type,
ext=None))
cmd_args = ['comb-list', prefix+str(expnum)+type+str(ccd).zfill(2)]
util.exec_prog(cmd_args)
ext_list = ['cands.comb']
if prefix is not None and len(prefix) > 0 :
ext_list.extend( [ 'jmp.missed', 'matt.missed',
'jmp.found', 'matt.found',
'comb.missed', 'comb.found' ] )
for ext in ext_list:
uri = storage.get_uri(expnum,
ccd=ccd,
prefix=prefix,
version=type,
ext=ext)
filename = os.path.basename(uri)
if not os.access(filename,os.R_OK):
logging.critical("No %s file" % (filename))
continue
vospace_name = "%s.%s" % ( field, ext )
logging.info("%s -> %s" % ( filename, os.path.join(measure3, vospace_name)))
storage.copy(filename, os.path.join(measure3, vospace_name))
base_name = prefix+str(expnum)+type+str(ccd).zfill(2)
cands_file = base_name+'.cands.comb'
if not os.access(cands_file,os.R_OK):
nocands_file = ( prefix+
str(expnum)+
type+
str(ccd).zfill(2)+
'.no_candidates' )
open(nocands_file, 'w').close()
vospace_name = "%s.no_candidates" % ( field )
storage.copy(nocands_file,os.path.join(measure3, vospace_name))
return storage.SUCCESS
# get the images we need to compute x/y ra/dec transforms
cands_file = mop_file.Parser().parse(cands_file)
for file_id in cands_file.header.file_ids:
rec_no=cands_file.header.file_ids.index(file_id)
storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no],
ccd=ccd,
version=type,
prefix=prefix,
ext='fits')
cmd_args = ['measure3', prefix+str(expnum)+type+str(ccd).zfill(2)]
logging.info("Running measure3")
util.exec_prog(cmd_args)
filename=base_name+".measure3.cands.astrom"
vospace_filename = "%s.measure3.cands.astrom" % ( field)
storage.copy(filename, os.path.join(measure3,vospace_filename))
return storage.SUCCESS
def main():
"""Do the script."""
parser = argparse.ArgumentParser(
description='replace image header')
parser.add_argument('--extname',
help='name of extension to in header')
parser.add_argument('expnum', type=str,
help='exposure to update')
parser.add_argument('-r', '--replace',
action='store_true',
help='store modified image back to VOSpace?')
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--force', action='store_true', help="Re-run even if previous success recorded")
parser.add_argument('--dbimages', help="VOSpace DATA storage area.", default="vos:OSSOS/dbimages")
args = parser.parse_args()
task = util.task()
dependency = 'preproc'
prefix = ""
storage.DBIMAGES = args.dbimages
level = logging.CRITICAL
message_format = "%(message)s"
if args.verbose:
level = logging.INFO
if args.debug:
level = logging.DEBUG
message_format = "%(module)s %(funcName)s %(lineno)s %(message)s"
logging.basicConfig(level=level, format=message_format)
storage.set_logger(task, prefix, args.expnum, None, None, False)
message = storage.SUCCESS
expnum = args.expnum
exit_status = 0
try:
# skip if already succeeded and not in force mode
if storage.get_status(task, prefix, expnum, "p", 36) and not args.force:
logging.info("Already updated, skipping")
sys.exit(0)
image_hdulist = storage.get_image(args.expnum, return_file=False)
ast_hdulist = storage.get_astheader(expnum, ccd=None)
run_update_header(image_hdulist, ast_hdulist)
image_filename = os.path.basename(storage.get_uri(expnum))
image_hdulist.writeto(image_filename)
if args.replace:
dest = storage.dbimages_uri(expnum)
storage.copy(image_filename, dest)
storage.set_status('update_header', "", expnum, 'p', 36, message)
except Exception as e:
message = str(e)
if args.replace:
storage.set_status(task, prefix, expnum, 'p', 36, message)
exit_status = message
logging.error(message)
return exit_status
def remeasure(mpc_in):
"""
re-measure the astrometry and photometry of the given mpc line
"""
# TODO Actually implement this.
if mpc_in.null_observation:
return mpc_in
mpc_obs = deepcopy(mpc_in)
logging.debug("rm start: {}".format(mpc_obs.to_string()))
if not isinstance(mpc_obs.comment, mpc.OSSOSComment):
return mpc_in
parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame)
if not parts:
return mpc_in
start_coordinate = mpc_in.coordinate
assert isinstance(start_coordinate, ICRSCoordinates)
try:
header = storage.get_astheader(parts.group('expnum'), int(parts.group('ccd')))
except IOError as ioerr:
logging.error(str(ioerr))
logging.error("Failed to get astrometric header for: {}".format(mpc_obs))
return mpc_in
this_wcs = wcs.WCS(header)
try:
x = float(mpc_obs.comment.x)
y = float(mpc_obs.comment.y)
if mpc_in.discovery:
logging.debug("Discovery astrometric lines are normally flipped relative to storage.")
x = header['NAXIS1'] - x + 1
y = header['NAXIS2'] - y + 1
except:
logging.warn("Failed to X/Y from comment line.")
return mpc_in
(ra, dec) = this_wcs.xy2sky(x, y)
mpc_obs.coordinate = (ra, dec)
logging.debug("rm updat: {}".format(mpc_obs.to_string()))
sep = start_coordinate.separation(mpc_obs.coordinate).degrees * 3600.0
if sep > TOLERANCE and mpc_in.discovery:
logging.warn("{} --> Using the unflipped X/Y for a discovery observation line.".format(sep))
logging.debug("{} {} {} {} {}".format(sep, x, y, mpc_obs.comment.x, mpc_obs.comment.y))
# Try un-flipping.
x = float(mpc_obs.comment.x)
y = float(mpc_obs.comment.y)
(ra, dec) = this_wcs.xy2sky(x, y)
#print "--> x/y coordinates ({},{}) and recomputing ra/dec ({},{})".format(x, y, ra, dec)
mpc_obs.coordinate = (ra, dec)
sep = start_coordinate.separation(mpc_obs.coordinate).degrees * 3600.0
logging.debug("remod: {}".format(mpc_obs.coordinate))
logging.debug("SEP: {}".format(sep))
logging.debug("rm flip: {}".format(mpc_obs.to_string()))
if sep > TOLERANCE:
## use the old header RA/DEC to predict the X/Y and then use that X/Y to get new RA/DEC
logging.debug("Ignoring recorded X/Y and using previous to RA/DEC and WCS to compute X/Y")
header2 = storage.get_image(parts.group('expnum'),
int(parts.group('ccd')),
return_file=False,
flip_image=False)[0].header
image_wcs = wcs.WCS(header2)
(x, y) = image_wcs.sky2xy(mpc_in.coordinate.ra.degrees, mpc_in.coordinate.dec.degrees)
(ra, dec) = this_wcs.xy2sky(x, y)
logging.debug("({},{}) --> ({},{})".format(mpc_obs.comment.x, mpc_obs.comment.y, x, y))
mpc_obs.coordinate = (ra, dec)
mpc_obs.comment.x = x
mpc_obs.comment.y = y
try:
merr = float(mpc_obs.comment.mag_uncertainty)
fwhm = float(storage.get_fwhm(parts.group('expnum'), int(parts.group('ccd')))) * header['PIXSCAL1']
centroid_err = merr * fwhm
logging.debug("Centroid uncertainty: {} {} => {}".format(merr, fwhm, centroid_err))
except Exception as err:
logging.error(str(err))
logging.error("Failed to compute centroid for observation:\n"
"{}\nUsing default of 0.2".format(mpc_obs.to_string()))
centroid_err = 0.2
mpc_obs.comment.astrometric_level = header.get('ASTLEVEL', "0")
try:
asterr = float(header['ASTERR'])
residuals = (asterr ** 2 + centroid_err ** 2) ** 0.5
logging.debug("Residuals: {} {} => {}".format(asterr, centroid_err, residuals))
except Exception as err:
logging.error(str(err))
logging.error("Failed while trying to compute plate uncertainty for\n{}".format(mpc_obs.to_stirng()))
logging.error('Using default of 0.25')
residuals = 0.25
mpc_obs.comment.plate_uncertainty = residuals
logging.debug("sending back: {}".format(mpc_obs.to_string()))
return mpc_obs
def step2(expnums, ccd, version, prefix=None, dry_run=False, default="WCS"):
"""run the actual step2 on the given exp/ccd combo"""
jmp_trans = ['step2ajmp']
jmp_args = ['step2bjmp']
matt_args = ['step2matt_jmp']
idx = 0
for expnum in expnums:
jmp_args.append(
storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]
)
jmp_trans.append(
storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]
)
idx += 1
matt_args.append('-f%d' % idx)
matt_args.append(
storage.get_file(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix)[0:-9]
)
logging.info(util.exec_prog(jmp_trans))
if default == "WCS":
logging.info(compute_trans(expnums, ccd, version, prefix, default=default))
logging.info(util.exec_prog(jmp_args))
logging.info(util.exec_prog(matt_args))
## check that the shifts from step2 are rational
check_args = ['checktrans']
if os.access('proc-these-files', os.R_OK):
os.unlink('proc-these-files')
ptf = open('proc-these-files', 'w')
ptf.write("# A dummy file that is created so checktrans could run.\n")
ptf.write("# Frame FWHM PSF?\n")
for expnum in expnums:
filename = os.path.splitext(storage.get_image(expnum, ccd, version=version, prefix=prefix))[0]
if not os.access(filename + ".bright.psf", os.R_OK):
os.link(filename + ".bright.jmp", filename + ".bright.psf")
if not os.access(filename + ".obj.psf", os.R_OK):
os.link(filename + ".obj.jmp", filename + ".obj.psf")
ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(filename,
_FWHM,
"NO"))
ptf.close()
if os.access('BAD_TRANS', os.F_OK):
os.unlink('BAD_TRANS')
logging.info(util.exec_prog(check_args))
if os.access('BAD_TRANS', os.F_OK):
raise OSError(errno.EBADMSG, 'BAD_TRANS')
if os.access('proc-these-files', os.F_OK):
os.unlink('proc-these-files')
if dry_run:
return
for expnum in expnums:
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
uri = storage.dbimages_uri(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix)
filename = os.path.basename(uri)
storage.copy(filename, uri)
return
def main():
"""Do the script."""
parser = argparse.ArgumentParser(description='replace image header')
parser.add_argument('--extname', help='name of extension to in header')
parser.add_argument('expnum', type=str, help='exposure to update')
parser.add_argument('-r',
'--replace',
action='store_true',
help='store modified image back to VOSpace?')
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--force',
action='store_true',
help="Re-run even if previous success recorded")
parser.add_argument('--dbimages',
help="VOSpace DATA storage area.",
default="vos:OSSOS/dbimages")
args = parser.parse_args()
task = util.task()
dependency = 'preproc'
prefix = ""
storage.DBIMAGES = args.dbimages
level = logging.CRITICAL
message_format = "%(message)s"
if args.verbose:
level = logging.INFO
if args.debug:
level = logging.DEBUG
message_format = "%(module)s %(funcName)s %(lineno)s %(message)s"
logging.basicConfig(level=level, format=message_format)
storage.set_logger(task, prefix, args.expnum, None, None, False)
message = storage.SUCCESS
expnum = args.expnum
exit_status = 0
try:
# skip if already succeeded and not in force mode
if storage.get_status(task, prefix, expnum, "p",
36) and not args.force:
logging.info("Already updated, skipping")
sys.exit(0)
image_hdulist = storage.get_image(args.expnum, return_file=False)
ast_hdulist = storage.get_astheader(expnum, ccd=None)
run_update_header(image_hdulist, ast_hdulist)
image_filename = os.path.basename(storage.get_uri(expnum))
image_hdulist.writeto(image_filename)
if args.replace:
dest = storage.dbimages_uri(expnum)
storage.copy(image_filename, dest)
storage.set_status('update_header', "", expnum, 'p', 36, message)
except Exception as e:
message = str(e)
if args.replace:
storage.set_status(task, prefix, expnum, 'p', 36, message)
exit_status = message
logging.error(message)
return exit_status
def run(expnum, ccd, version, dry_run=False, prefix="", force=False):
"""Run the OSSOS jmpmakepsf script.
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version=version,
ccd=ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(
task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(
dependency, prefix, expnum, version, ccd=ccd):
raise IOError("{} not yet run for {}".format(
dependency, expnum))
# confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum,
ccd,
prefix=prefix,
version=version,
ext='fits'))
if not dry_run:
storage.mkdir(destdir)
# get image from the vospace storage area
logging.info("Getting fits image from VOSpace")
filename = storage.get_image(expnum,
ccd,
version=version,
prefix=prefix)
# get mopheader from the vospace storage area
logging.info("Getting mopheader from VOSpace")
mopheader_filename = storage.get_file(expnum,
ccd,
version=version,
prefix=prefix,
ext='mopheader')
# run mkpsf process
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
logging.info(
util.exec_prog(
['jmpmakepsf.csh', './', filename, 'yes', 'yes']))
if dry_run:
return
# place the results into VOSpace
basename = os.path.splitext(filename)[0]
for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor',
'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum,
ccd,
prefix=prefix,
version=version,
ext=ext)
source = basename + "." + str(ext)
count = 0
with open(source, 'r'):
while True:
count += 1
try:
logging.info("Attempt {} to copy {} -> {}".format(
count, source, dest))
storage.copy(source, dest)
break
except Exception as ex:
if count > 10:
raise ex
# set some data parameters associated with the image, determined in this step.
storage.set_status('fwhm',
prefix,
expnum,
version=version,
ccd=ccd,
status=str(
storage.get_fwhm(expnum,
ccd=ccd,
prefix=prefix,
version=version)))
storage.set_status('zeropoint',
prefix,
expnum,
version=version,
ccd=ccd,
status=str(
storage.get_zeropoint(expnum,
ccd=ccd,
prefix=prefix,
version=version)))
logging.info(message)
except Exception as e:
message = str(e)
logging.error(message)
storage.set_status(task,
prefix,
expnum,
version,
ccd=ccd,
status=message)
return
def align(expnums, ccd, version='s', prefix='', dry_run=False, force=True):
"""Create a 'shifts' file that transforms the space/flux/time scale of all
images to the first image.
This function relies on the .fwhm, .trans.jmp, .phot and .zeropoint.used files for inputs.
The scaling we are computing here is for use in planting sources into the image at the same sky/flux locations
while accounting for motions of sources with time.
:param expnums: list of MegaPrime exposure numbers to add artificial KBOs to,
the first frame in the list is the reference.
:param ccd: which ccd to work on.
:param prefix: put this string in front of expnum when looking for exposure, normally '' or 'fk'
:param force: When true run task even if this task is recorded as having succeeded
:param version: Add sources to the 'o', 'p' or 's' images
:param dry_run: don't push results to VOSpace.
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnums[0], version,
ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(
task, prefix, expnums[0], version, ccd))
return
# Get the images and supporting files that we need from the VOSpace area
# get_image and get_file check if the image/file is already on disk.
# re-computed fluxes from the PSF stars and then recompute x/y/flux scaling.
# some dictionaries to hold the various scale
pos = {}
apcor = {}
mags = {}
zmag = {}
mjdates = {}
with storage.LoggingManager(task, prefix, expnums[0], ccd, version,
dry_run):
try:
for expnum in expnums:
filename = storage.get_image(expnum, ccd=ccd, version=version)
zmag[expnum] = storage.get_zeropoint(expnum,
ccd,
prefix=None,
version=version)
mjdates[expnum] = float(
fits.open(filename)[0].header.get('MJD-OBS'))
apcor[expnum] = [
float(x) for x in open(
storage.get_file(
expnum,
ccd=ccd,
version=version,
ext=storage.APCOR_EXT)).read().split()
]
keys = ['crval1', 'cd1_1', 'cd1_2', 'crval2', 'cd2_1', 'cd2_2']
# load the .trans.jmp values into a 'wcs' like dictionary.
# .trans.jmp maps current frame to reference frame in pixel coordinates.
# the reference frame of all the frames supplied must be the same.
shifts = dict(
list(
zip(keys, [
float(x) for x in open(
storage.get_file(
expnum,
ccd=ccd,
version=version,
ext='trans.jmp')).read().split()
])))
shifts['crpix1'] = 0.0
shifts['crpix2'] = 0.0
# now create a wcs object based on those transforms, this wcs links the current frame's
# pixel coordinates to the reference frame's pixel coordinates.
w = get_wcs(shifts)
# get the PHOT file that was produced by the mkpsf routine
logging.debug("Reading .phot file {}".format(expnum))
phot = ascii.read(storage.get_file(expnum,
ccd=ccd,
version=version,
ext='phot'),
format='daophot')
# compute the small-aperture magnitudes of the stars used in the PSF
logging.debug("Running phot on {}".format(filename))
mags[expnum] = daophot.phot(filename,
phot['XCENTER'],
phot['YCENTER'],
aperture=apcor[expnum][0],
sky=apcor[expnum][1] + 1,
swidth=apcor[expnum][0],
zmag=zmag[expnum])
# covert the x/y positions to positions in Frame 1 based on the trans.jmp values.
logging.debug(
"Doing the XY translation to refrence frame: {}".format(w))
(x, y) = w.wcs_pix2world(mags[expnum]["XCENTER"],
mags[expnum]["YCENTER"], 1)
pos[expnum] = numpy.transpose([x, y])
# match this exposures PSF stars position against those in the first image of the set.
logging.debug("Matching lists")
idx1, idx2 = util.match_lists(pos[expnums[0]], pos[expnum])
# compute the magnitdue offset between the current frame and the reference.
dmags = numpy.ma.array(
mags[expnums[0]]["MAG"] - apcor[expnums[0]][2] -
(mags[expnum]["MAG"][idx1] - apcor[expnum][2]),
mask=idx1.mask)
dmags.sort()
# logging.debug("Computed dmags between input and reference: {}".format(dmags))
error_count = 0
error_count += 1
logging.debug("{}".format(error_count))
# compute the median and determine if that shift is small compared to the scatter.
try:
midx = int(
numpy.sum(numpy.any([~dmags.mask], axis=0)) / 2.0)
dmag = float(dmags[midx])
logging.debug("Computed a mag delta of: {}".format(dmag))
except Exception as e:
logging.error(str(e))
logging.error(
"Failed to compute mag offset between plant and found using: {}"
.format(dmags))
dmag = 99.99
error_count += 1
logging.debug("{}".format(error_count))
try:
if math.fabs(dmag) > 3 * (dmags.std() + 0.01):
logging.warning(
"Magnitude shift {} between {} and {} is large: {}"
.format(dmag, expnums[0], expnum, shifts))
except Exception as e:
logging.error(str(e))
error_count += 1
logging.debug("{}".format(error_count))
shifts['dmag'] = dmag
shifts['emag'] = dmags.std()
shifts['nmag'] = len(dmags.mask) - dmags.mask.sum()
shifts['dmjd'] = mjdates[expnums[0]] - mjdates[expnum]
shift_file = os.path.basename(
storage.get_uri(expnum, ccd, version, '.shifts'))
error_count += 1
logging.debug("{}".format(error_count))
try:
fh = open(shift_file, 'w')
fh.write(
json.dumps(shifts,
sort_keys=True,
indent=4,
separators=(',', ': '),
cls=NpEncoder))
fh.write('\n')
fh.close()
except Exception as e:
logging.error(
"Creation of SHIFTS file failed while trying to write: {}"
.format(shifts))
raise e
error_count += 1
logging.debug("{}".format(error_count))
if not dry_run:
storage.copy(
shift_file,
storage.get_uri(expnum, ccd, version, '.shifts'))
logging.info(message)
except Exception as ex:
message = str(ex)
logging.error(message)
if not dry_run:
storage.set_status(task,
prefix,
expnum,
version,
ccd,
status=message)
def step2(expnums, ccd, version, prefix=None, dry_run=False, default="WCS"):
"""run the actual step2 on the given exp/ccd combo"""
jmp_trans = ['step2ajmp']
jmp_args = ['step2bjmp']
matt_args = ['step2matt_jmp']
idx = 0
for expnum in expnums:
jmp_args.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix)[0:-8])
jmp_trans.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix)[0:-8])
idx += 1
matt_args.append('-f%d' % idx)
matt_args.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.matt',
prefix=prefix)[0:-9])
logging.info(util.exec_prog(jmp_trans))
if default == "WCS":
logging.info(
compute_trans(expnums, ccd, version, prefix, default=default))
logging.info(util.exec_prog(jmp_args))
logging.info(util.exec_prog(matt_args))
## check that the shifts from step2 are rational
check_args = ['checktrans']
if os.access('proc-these-files', os.R_OK):
os.unlink('proc-these-files')
ptf = open('proc-these-files', 'w')
ptf.write("# A dummy file that is created so checktrans could run.\n")
ptf.write("# Frame FWHM PSF?\n")
for expnum in expnums:
filename = os.path.splitext(
storage.get_image(expnum, ccd, version=version, prefix=prefix))[0]
if not os.access(filename + ".bright.psf", os.R_OK):
os.link(filename + ".bright.jmp", filename + ".bright.psf")
if not os.access(filename + ".obj.psf", os.R_OK):
os.link(filename + ".obj.jmp", filename + ".obj.psf")
ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(filename, _FWHM, "NO"))
ptf.close()
if os.access('BAD_TRANS', os.F_OK):
os.unlink('BAD_TRANS')
logging.info(util.exec_prog(check_args))
if os.access('BAD_TRANS', os.F_OK):
raise ValueError(errno.EBADEXEC, 'BAD_TRANS')
if os.access('proc-these-files', os.F_OK):
os.unlink('proc-these-files')
if dry_run:
return
for expnum in expnums:
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
uri = storage.dbimages_uri(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = os.path.basename(uri)
storage.copy(filename, uri)
return
def run(expnum,
ccd,
prefix='',
version='p',
sex_thresh=_SEX_THRESHOLD,
wave_thresh=_WAVE_THRESHOLD,
maxcount=_MAX_COUNT,
dry_run=False,
force=True, ignore=False):
"""run the actual step1jmp/matt codes.
expnum: the CFHT expousre to process
ccd: which ccd in the mosaic to process
fwhm: the image quality, FWHM, of the image. In pixels.
sex_thresh: the detection threhold to run sExtractor at
wave_thresh: the detection threshold for wavelet
maxcount: saturation level
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version, ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(dependency, prefix, expnum, version, ccd) and not ignore:
raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format(
task, dependency, prefix, expnum, version, ccd))
logging.info("Retrieving imaging and input parameters from VOSpace")
storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader')
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version, default=3.5)
basename = os.path.splitext(filename)[0]
_get_weight_map(filename, ccd)
logging.info("Launching step1jmp")
logging.info(util.exec_prog(['step1jmp',
'-f', basename,
'-t', str(wave_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
logging.info(util.exec_prog(['step1matt',
'-f', basename,
'-t', str(sex_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
if os.access('weight.fits', os.R_OK):
os.unlink('weight.fits')
if not dry_run:
for ext in ['obj.jmp', 'obj.matt']:
obj_uri = storage.get_uri(expnum, ccd, version=version, ext=ext,
prefix=prefix)
obj_filename = basename + "." + ext
count = 0
with open(obj_filename, 'r'):
while True:
try:
count += 1
logging.info("Attempt {} to copy {} -> {}".format(count, obj_filename, obj_uri))
storage.copy(obj_filename, obj_uri)
break
except Exception as ex:
if count > 10:
raise ex
logging.info(message)
except Exception as ex:
message = str(ex)
logging.error(message)
if not dry_run:
storage.set_status(task, prefix, expnum, version, ccd, status=message)
def plant(expnums,
ccd,
rmin,
rmax,
ang,
width,
number=10,
mmin=21.0,
mmax=25.5,
version='s',
dry_run=False,
force=True):
"""Plant artificial sources into the list of images provided.
@param dry_run: don't push results to VOSpace.
@param width: The +/- range of angles of motion
@param ang: The mean angle of motion to add sources
@param rmax: The maximum rate of motion to add sources at (''/hour)
@param rmin: The minimum rate of motion to add sources at (''/hour)
@param expnums: list of MegaPrime exposure numbers to add artificial KBOs to
@param ccd: which ccd to work on.
@param mmax: Maximum magnitude to plant sources at
@param version: Add sources to the 'o', 'p' or 's' images
@param mmin: Minimum magnitude to plant sources at
@param number: number of sources to plant.
@param force: Run, even if we already succeeded at making a fk image.
"""
message = storage.SUCCESS
if storage.get_status(task, "", expnums[0], version, ccd) and not force:
logging.info("{} completed successfully for {}{}{}{:02d}".format(
task, "", expnums[0], version, ccd))
return
with storage.LoggingManager(task, "", expnums[0], ccd, version, dry_run):
try:
# Construct a list of artificial KBOs with positions in the image
# and rates of motion within the bounds given by the caller.
filename = storage.get_image(expnums[0], ccd=ccd, version=version)
header = fits.open(filename)[0].header
bounds = util.get_pixel_bounds_from_datasec_keyword(
header.get('DATASEC', '[33:2080,1:4612]'))
# generate a set of artificial KBOs to add to the image.
kbos = KBOGenerator.get_kbos(n=number,
rate=(rmin, rmax),
angle=(ang - width, ang + width),
mag=(mmin, mmax),
x=(bounds[0][0], bounds[0][1]),
y=(bounds[1][0], bounds[1][1]),
filename='Object.planted')
for expnum in expnums:
filename = storage.get_image(expnum, ccd, version)
psf = storage.get_file(expnum, ccd, version, ext='psf.fits')
plant_kbos(filename, psf, kbos,
get_shifts(expnum, ccd, version), "fk")
if dry_run:
return
uri = storage.get_uri('Object',
ext='planted',
version='',
subdir=f"{expnums[0]}/ccd{int(ccd):02d}")
storage.copy('Object.planted', uri)
for expnum in expnums:
uri = storage.get_uri(expnum,
ccd=ccd,
version=version,
ext='fits',
prefix='fk')
filename = os.path.basename(uri)
storage.copy(filename, uri)
except Exception as ex:
message = str(ex)
logging.error(message)
storage.set_status(task, "", expnums[0], version, ccd, status=message)
return
def step1(expnum,
ccd,
prefix='',
version='p',
sex_thresh=_SEX_THRESHOLD,
wave_thresh=_WAVE_THRESHOLD,
maxcount=_MAX_COUNT,
dry_run=False):
"""run the actual step1jmp/matt codes.
expnum: the CFHT expousre to process
ccd: which ccd in the mosaic to process
fwhm: the image quality, FWHM, of the image. In pixels.
sex_thresh: the detection threhold to run sExtractor at
wave_thresh: the detection threshold for wavelet
maxcount: saturation level
"""
storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader')
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version)
basename = os.path.splitext(filename)[0]
logging.info(util.exec_prog(['step1jmp',
'-f', basename,
'-t', str(wave_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.jmp',
prefix=prefix)
obj_filename = basename + ".obj.jmp"
if not dry_run:
storage.copy(obj_filename, obj_uri)
## for step1matt we need the weight image
hdulist = fits.open(filename)
flat_name = hdulist[0].header.get('FLAT', 'weight.fits')
parts = os.path.splitext(flat_name)
if parts[1] == '.fz':
flat_name = os.path.splitext(parts[0])[0]
else:
flat_name = parts[0]
try:
flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators')
except:
flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='old_calibrators')
if os.access('weight.fits', os.R_OK):
os.unlink('weight.fits')
if not os.access('weight.fits', os.R_OK):
os.symlink(flat_filename, 'weight.fits')
logging.info(util.exec_prog(['step1matt',
'-f', basename,
'-t', str(sex_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
if os.access('weight.fits', os.R_OK):
os.unlink('weight.fits')
obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.matt',
prefix=prefix)
obj_filename = basename + ".obj.matt"
if not dry_run:
storage.copy(obj_filename, obj_uri)
return True
def step3(expnums,
ccd,
version,
rate_min,
rate_max,
angle,
width,
field=None,
prefix=None):
'''run the actual step2 on the given exp/ccd combo'''
jmp_args = ['step3jmp']
matt_args = ['step3matt']
idx = 0
cmd_args = []
for expnum in expnums:
idx += 1
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
filename = storage.get_image(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
image = os.path.splitext(
os.path.splitext(os.path.basename(filename))[0])[0]
cmd_args.append('-f%d' % (idx))
cmd_args.append(image)
cmd_args.extend([
'-rn',
str(rate_min), '-rx',
str(rate_max), '-a',
str(angle), '-w',
str(width)
])
jmp_args.extend(cmd_args)
matt_args.extend(cmd_args)
util.exec_prog(jmp_args)
util.exec_prog(matt_args)
if field is None:
field = str(expnums[0])
storage.mkdir(
os.path.dirname(
storage.get_uri(field,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)))
for ext in ['moving.jmp', 'moving.matt']:
uri = storage.get_uri(field,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = '%s%d%s%s.%s' % (prefix, expnums[0], version,
str(ccd).zfill(2), ext)
storage.copy(filename, uri)
return
def align(expnums, ccd, version='s', dry_run=False):
"""Create a 'shifts' file that transforms the space/flux/time scale of all images to the first image.
This function relies on the .fwhm, .trans.jmp, .phot and .zeropoint.used files for inputs.
The scaling we are computing here is for use in planting sources into the image at the same sky/flux locations
while accounting for motions of sources with time.
:param expnums: list of MegaPrime exposure numbers to add artificial KBOs to,
the first frame in the list is the reference.
:param ccd: which ccd to work on.
:param version: Add sources to the 'o', 'p' or 's' images
:param dry_run: don't push results to VOSpace.
"""
# Get the images and supporting files that we need from the VOSpace area
# get_image and get_file check if the image/file is already on disk.
# re-computed fluxes from the PSF stars and then recompute x/y/flux scaling.
# some dictionaries to hold the various scale
pos = {}
apcor = {}
mags = {}
zmag = {}
mjdates = {}
for expnum in expnums:
filename = storage.get_image(expnum, ccd=ccd, version=version)
zmag[expnum] = storage.get_zeropoint(expnum,
ccd,
prefix=None,
version=version)
mjdates[expnum] = float(fits.open(filename)[0].header.get('MJD-OBS'))
apcor[expnum] = [
float(x) for x in open(
storage.get_file(
expnum, ccd=ccd, version=version,
ext=storage.APCOR_EXT)).read().split()
]
keys = ['crval1', 'cd1_1', 'cd1_2', 'crval2', 'cd2_1', 'cd2_2']
# load the .trans.jmp values into a 'wcs' like dictionary.
# .trans.jmp maps current frame to reference frame in pixel coordinates.
# the reference frame of all the frames supplied must be the same.
shifts = dict(
zip(keys, [
float(x) for x in open(
storage.get_file(
expnum, ccd=ccd, version=version,
ext='trans.jmp')).read().split()
]))
shifts['crpix1'] = 0.0
shifts['crpix2'] = 0.0
# now create a wcs object based on those transforms, this wcs links the current frame's
# pixel coordinates to the reference frame's pixel coordinates.
w = get_wcs(shifts)
# get the PHOT file that was produced by the mkpsf routine
phot = ascii.read(storage.get_file(expnum,
ccd=ccd,
version=version,
ext='phot'),
format='daophot')
# compute the small-aperture magnitudes of the stars used in the PSF
mags[expnum] = daophot.phot(filename,
phot['XCENTER'],
phot['YCENTER'],
aperture=apcor[expnum][0],
sky=apcor[expnum][1] + 1,
swidth=apcor[expnum][0],
zmag=zmag[expnum])
# covert the x/y positions to positions in Frame 1 based on the trans.jmp values.
(x, y) = w.wcs_pix2world(mags[expnum]["XCENTER"],
mags[expnum]["YCENTER"], 1)
pos[expnum] = numpy.transpose([x, y])
# match this exposures PSF stars position against those in the first image of the set.
idx1, idx2 = util.match_lists(pos[expnums[0]], pos[expnum])
# compute the magnitdue offset between the current frame and the reference.
dmags = numpy.ma.array(mags[expnums[0]]["MAG"] - apcor[expnums[0]][2] -
(mags[expnum]["MAG"][idx1] - apcor[expnum][2]),
mask=idx1.mask)
dmags.sort()
# compute the median and determine if that shift is small compared to the scatter.
dmag = dmags[int(len(dmags) / 2.0)]
if math.fabs(dmag) > 3 * (dmags.std() + 0.01):
logging.warn(
"Magnitude shift {} between {} and {} is large: {}".format(
dmag, expnums[0], expnum, shifts[expnum]))
shifts['dmag'] = dmag
shifts['emag'] = dmags.std()
shifts['nmag'] = len(dmags.mask) - dmags.mask.sum()
shifts['dmjd'] = mjdates[expnums[0]] - mjdates[expnum]
shift_file = os.path.basename(
storage.get_uri(expnum, ccd, version, '.shifts'))
fh = open(shift_file, 'w')
fh.write(
json.dumps(shifts,
sort_keys=True,
indent=4,
separators=(',', ': ')))
fh.write('\n')
fh.close()
if not dry_run:
storage.copy(
shift_file,
os.path.basename(
storage.get_uri(expnum, ccd, version, '.shifts')))
def step1(expnum,
ccd,
prefix='',
version='p',
fwhm=4,
sex_thresh=1.3,
wave_thresh=2.7,
maxcount=30000):
"""run the actual step1jmp/matt codes.
expnum: the CFHT expousre to process
ccd: which ccd in the mosaic to process
fwhm: the image quality, FWHM, of the image. In pixels.
sex_thresh: the detection threhold to run sExtractor at
wave_thresh: the detection threshold for wavelet
maxcount: saturation level
"""
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
mopheader = storage.get_image(expnum,
ccd,
version=version,
ext='mopheader',
prefix=prefix)
fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version)
basename = os.path.splitext(filename)[0]
outfile = util.exec_prog([
'step1jmp', '-f', basename, '-t',
str(wave_thresh), '-w',
str(fwhm), '-m',
str(maxcount)
])
obj_uri = storage.get_uri(expnum,
ccd,
version=version,
ext='obj.jmp',
prefix=prefix)
obj_filename = basename + ".obj.jmp"
storage.copy(obj_filename, obj_uri)
## for step1matt we need the weight image
flat_name = fits.open(filename)[0].header['FLAT']
flat_name = flat_name[0:-5]
flat_filename = storage.get_image(flat_name,
ccd,
version='',
ext='fits',
subdir='calibrators',
rescale=False)
if not os.access('weight.fits', os.R_OK):
os.symlink(flat_filename, 'weight.fits')
outfile = util.exec_prog([
'step1matt', '-f', basename, '-t',
str(sex_thresh), '-w',
str(fwhm), '-m',
str(maxcount)
])
obj_uri = storage.get_uri(expnum,
ccd,
version=version,
ext='obj.matt',
prefix=prefix)
obj_filename = basename + ".obj.matt"
storage.copy(obj_filename, obj_uri)
return True
def download_cutout(self, reading, focus=None, needs_apcor=False):
"""
Downloads a cutout of the FITS image for a given source reading.
Args:
source_reading: ossos.astrom.SourceReading
The reading which will be the focus of the downloaded image.
focus: tuple(int, int)
The x, y coordinates that should be the focus of the downloaded
image. These coordinates should be in terms of the
source_reading parameter's coordinate system.
Default value is None, in which case the source reading's x, y
position is used as the focus.
needs_apcor: bool
If True, the apcor file with data needed for photometry
calculations is downloaded in addition to the image.
Defaults to False.
Returns:
cutout: ossos.downloads.data.SourceCutout
"""
if focus is None:
focus = reading.source_point
assert isinstance(reading, SourceReading)
dx = dy = 2 * max(reading.dra, reading.ddec)
dx = max(reading.dx, dx)
dy = max(reading.dy, dy)
(NAXIS1, NAXIS2) = reading.get_original_image_size()
cutout_str, converter = self.cutout_calculator.build_cutout_str(
reading.get_extension(),
focus, (NAXIS1, NAXIS2),
dx=dx,
dy=dy,
inverted=reading.is_inverted)
image_uri = reading.get_image_uri()
cutout = re.findall(r'(\d+)', cutout_str)
y2 = int(cutout[-1])
y1 = int(cutout[-2])
logger.debug("Calculated cutout: %s for %s" % (cutout_str, image_uri))
hdulist = storage.get_image(expnum=reading.get_exposure_number(),
ccd=reading.get_ccd_num(),
cutout=cutout_str,
version=reading.get_observation().ftype,
prefix=reading.get_observation().fk,
return_file=False)
#hdulist = self.download_hdulist(image_uri, view="cutout",
# cutout=cutout_str)
# modify the DATASEC to account for possible flip/flop and changes in dimensions of the image.
DATASEC = hdulist[0].header.get('DATASEC', None)
if DATASEC is not None:
datasec = re.findall(r'(\d+)', DATASEC)
if y2 < y1:
x2 = int(NAXIS1) - int(datasec[0]) + 1
x1 = int(NAXIS1) - int(datasec[1]) + 1
y2 = int(NAXIS2) - int(datasec[2]) + 1
y1 = int(NAXIS2) - int(datasec[3]) + 1
logger.debug(
"Flip/Flopped DATASEC from {} to [{}:{}:{}:{}]".format(
DATASEC, x1, x2, y1, y2))
datasec = (x1, x2, y1, y2)
(x1, y1) = converter.convert((int(datasec[0]), int(datasec[2])))
x1 = max(1, x1)
y1 = max(1, y1)
(x2, y2) = converter.convert((int(datasec[1]), int(datasec[3])))
x2 = min(x2, int(hdulist[0].header['NAXIS1']))
y2 = min(y2, int(hdulist[0].header['NAXIS2']))
datasec = "[{}:{},{}:{}]".format(x1, x2, y1, y2)
logger.debug("Trimmed and offset DATASEC from {} to {}".format(
DATASEC, datasec))
hdulist[0].header['DATASEC'] = datasec
apcor = None
if needs_apcor:
try:
apcor = self.download_apcor(reading.get_apcor_uri())
except Exception as e:
logger.error(str(e))
apcor = None
zmag = None
try:
zmag = self.download_zmag(reading.get_zmag_uri())
except Exception as e:
logger.error(str(e))
pass
return SourceCutout(reading, hdulist, converter, apcor, zmag=zmag)
def combine(expnum, ccd, prefix=None, type='p'):
for ext in ['moving.matt', 'moving.jmp']:
fname = storage.get_image(expnum,
ccd=ccd,
prefix=prefix,
version=type,
ext=ext)
if prefix is not None and len(prefix) > 0:
planted = storage.get_image('Object',
subdir=str(expnum) + "/ccd%s" %
(str(ccd).zfill(2)),
version='',
ext='planted')
else:
prefix = ''
base_image = os.path.basename(
storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=type,
ext=None))
cmd_args = ['comb-list', prefix + str(expnum) + type + str(ccd).zfill(2)]
util.exec_prog(cmd_args)
for ext in [
'cands.comb', 'comb.found', 'comb.missed', 'jmp.found',
'jmp.missed', 'matt.found', 'matt.missed'
]:
uri = storage.get_uri(expnum,
ccd=ccd,
prefix=prefix,
version=type,
ext=ext)
filename = os.path.basename(uri)
if not os.access(filename, os.R_OK):
logging.critical("No %s file" % (filename))
continue
storage.copy(filename, uri)
base_name = prefix + str(expnum) + type + str(ccd).zfill(2)
cands_file = base_name + '.cands.comb'
if not os.access(cands_file, os.R_OK):
nocands_file = (prefix + str(expnum) + type + str(ccd).zfill(2) +
'.no_candidates')
open(nocands_file, 'w').close()
storage.copy(nocands_file, 'vos:OSSOS/measure3/' + nocands_file)
return storage.SUCCESS
cands_file = mop_file.Parser().parse(cands_file)
for file_id in cands_file.header.file_ids:
rec_no = cands_file.header.file_ids.index(file_id)
storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no],
ccd=ccd,
version=type,
prefix=prefix,
ext='fits')
cmd_args = ['measure3', prefix + str(expnum) + type + str(ccd).zfill(2)]
util.exec_prog(cmd_args)
filename = base_name + ".measure3.cands.astrom"
storage.copy(filename, 'vos:OSSOS/measure3/' + filename)
return storage.SUCCESS
def plant(expnums, ccd, rmin, rmax, ang, width, version='s'):
'''run the plant script on this combination of exposures'''
ptf = open('proc-these-files', 'w')
ptf.write("# Files to be planted and search\n")
ptf.write("# image fwhm plant\n")
for expnum in expnums:
fwhm = storage.get_fwhm(expnum, ccd)
filename = storage.get_image(expnum, ccd=ccd, version=version)
ptf.write("%s %3.1f YES\n" % (filename[0:-5], fwhm))
for ext in [
'apcor', 'obj.jmp', 'trans.jmp', 'psf.fits', 'mopheader',
'phot', 'zeropoint.used'
]:
apcor = storage.get_image(expnum, ccd=ccd, version='s', ext=ext)
ptf.close()
cmd_args = [
'plant.csh', os.curdir,
str(rmin),
str(rmax),
str(ang),
str(width)
]
util.exec_prog(cmd_args)
if args.dryrun:
# Don't push back to VOSpace
return
uri = storage.get_uri('Object',
ext='planted',
version='',
subdir=str(expnums[0]) + "/ccd%s" %
(str(ccd).zfill(2)))
storage.copy('Object.planted', uri)
uri = os.path.join(os.path.dirname(uri), 'shifts')
storage.copy('shifts', uri)
for expnum in expnums:
uri = storage.get_uri(expnum,
ccd=ccd,
version=version,
ext='fits',
prefix='fk')
filename = os.path.basename(uri)
storage.copy(filename, uri)
for ext in [
'mopheader', 'psf.fits', 'fwhm', 'apcor', 'zeropoint.used',
'trans.jmp'
]:
storage.delete(expnum, ccd, 's', ext, prefix='fk')
storage.vlink(expnum,
ccd,
'p',
ext,
expnum,
ccd,
's',
ext,
l_prefix='fk')
return
def run(expnum, ccd, version, dry_run=False, prefix="", force=False):
"""Run the OSSOS jmpmakepsf script.
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd):
raise IOError("{} not yet run for {}".format(dependency, expnum))
# confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits'))
if not dry_run:
storage.mkdir(destdir)
# get image from the vospace storage area
logging.info("Getting fits image from VOSpace")
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
# get mopheader from the vospace storage area
logging.info("Getting mopheader from VOSpace")
mopheader_filename = storage.get_file(expnum, ccd, version=version, prefix=prefix, ext='mopheader')
# run mkpsf process
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
logging.info(util.exec_prog(['jmpmakepsf.csh',
'./',
filename,
'yes', 'yes']))
if dry_run:
return
# place the results into VOSpace
basename = os.path.splitext(filename)[0]
for ext in ('mopheader', 'psf.fits',
'zeropoint.used', 'apcor', 'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext)
source = basename + "." + str(ext)
count = 0
with open(source, 'r'):
while True:
count += 1
try:
logging.info("Attempt {} to copy {} -> {}".format(count, source, dest))
storage.copy(source, dest)
break
except Exception as ex:
if count > 10:
raise ex
# set some data parameters associated with the image, determined in this step.
storage.set_status('fwhm', prefix, expnum, version=version, ccd=ccd, status=str(storage.get_fwhm(
expnum, ccd=ccd, prefix=prefix, version=version)))
storage.set_status('zeropoint', prefix, expnum, version=version, ccd=ccd,
status=str(storage.get_zeropoint(
expnum, ccd=ccd, prefix=prefix, version=version)))
logging.info(message)
except Exception as e:
message = str(e)
logging.error(message)
storage.set_status(task, prefix, expnum, version, ccd=ccd, status=message)
return
parser.add_argument('-r',
'--replace',
action='store_true',
help='store modified image back to VOSpace?')
parser.add_argument('-v', '--verbose', action='store_true')
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.INFO, format="%(message)s")
message = storage.SUCCESS
try:
image = (os.access(args.expnum, os.W_OK)
and args.expnum) or (storage.get_image(args.expnum))
header = (args.header is not None and
((os.access(args.header, os.W_OK) and args.header) or
(storage.get_image(args.header, ext='head')))) or (
storage.get_image(args.expnum, ext='head'))
logging.info("Swapping header for %s for contents in %s \n" %
(image, header))
run_update_header(image, header)
if args.replace:
expnum = args.expnum or fits.open(image)[0].header['EXPNUM']
dest = storage.dbimages_uri(expnum)
storage.copy(image, dest)
help='filename of replacement header')
parser.add_argument('-r','--replace',
action='store_true',
help='store modified image back to VOSpace?')
parser.add_argument('-v','--verbose', action='store_true')
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.INFO, format="%(message)s")
message = storage.SUCCESS
try:
image = (os.access(args.expnum,os.W_OK) and args.expnum ) or (
storage.get_image(args.expnum) )
header = (args.header is not None and ((
os.access(args.header, os.W_OK) and args.header ) or (
storage.get_image(args.header, ext='head')))) or (
storage.get_image(args.expnum, ext='head'))
logging.info(
"Swapping header for %s for contents in %s \n" % (
image, header) )
run_update_header(image, header)
if args.replace:
expnum = args.expnum or fits.open(image)[0].header['EXPNUM']
dest = storage.dbimages_uri(expnum)
def combine(expnum,
ccd,
prefix=None,
file_type='p',
field=None,
measure3=MEASURE3,
dry_run=False):
if field is None:
field = str(expnum)
if prefix is not None and len(prefix) > 0:
field = "%s_%s" % (prefix, field)
field += "_%s%s" % (str(file_type), str(ccd))
logging.info("Doing combine on field {}".format(field))
for ext in ['moving.matt', 'moving.jmp']:
storage.get_file(expnum,
ccd=ccd,
version=file_type,
ext=ext,
prefix=prefix)
if prefix is not None and len(prefix) > 0:
storage.get_file('Object',
version='',
ext='planted',
subdir=str(expnum) + "/ccd%s" % (str(ccd).zfill(2)))
else:
prefix = ''
cmd_args = [
'comb-list', prefix + str(expnum) + file_type + str(ccd).zfill(2)
]
logging.info(str(cmd_args))
logging.info(util.exec_prog(cmd_args))
ext_list = ['cands.comb']
if prefix is not None and len(prefix) > 0:
ext_list.extend([
'jmp.missed', 'matt.missed', 'jmp.found', 'matt.found',
'comb.missed', 'comb.found'
])
for ext in ext_list:
uri = storage.get_uri(expnum,
ccd=ccd,
prefix=prefix,
version=file_type,
ext=ext)
filename = os.path.basename(uri)
if not os.access(filename, os.R_OK):
logging.critical("No %s file" % filename)
continue
vospace_name = "%s.%s" % (field, ext)
if not dry_run:
logging.info("%s -> %s" %
(filename, os.path.join(measure3, vospace_name)))
storage.copy(filename, os.path.join(measure3, vospace_name))
base_name = prefix + str(expnum) + file_type + str(ccd).zfill(2)
cands_file = base_name + '.cands.comb'
if not os.access(cands_file, os.R_OK):
no_cands_file = (prefix + str(expnum) + file_type + str(ccd).zfill(2) +
'.no_candidates')
open(no_cands_file, 'w').close()
if not dry_run:
vospace_name = "%s.no_candidates" % field
storage.copy(no_cands_file, os.path.join(measure3, vospace_name))
return storage.SUCCESS
# get the images we need to compute x/y ra/dec transforms
cands_file = mop_file.Parser().parse(cands_file)
for file_id in cands_file.header.file_ids:
rec_no = cands_file.header.file_ids.index(file_id)
storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no],
ccd=ccd,
version=file_type,
ext='fits',
prefix=prefix)
cmd_args = [
'measure3', prefix + str(expnum) + file_type + str(ccd).zfill(2)
]
logging.info("Running measure3")
logging.info(util.exec_prog(cmd_args))
if not dry_run:
filename = base_name + ".measure3.cands.astrom"
vospace_filename = "%s.measure3.cands.astrom" % field
storage.copy(filename, os.path.join(measure3, vospace_filename))
return storage.SUCCESS
