def mkpsf(expnum, ccd, fversion):
"""Run the OSSOS makepsf script.
"""
## get image from the vospace storage area
filename = storage.get_image(expnum, ccd, version=fversion)
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
## launch the makepsf script
util.exec_prog(['jmpmakepsf.csh',
'./',
filename,
'no'])
## place the results into VOSpace
basename = os.path.splitext(filename)[0]
## confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, version=fversion,ext='fits'))
logging.info("Checking that destination directories exist")
storage.mkdir(destdir)
for ext in ('mopheader', 'psf.fits',
'zeropoint.used', 'apcor', 'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum, ccd, version=fversion, ext=ext)
source = basename + "." + ext
logging.info("Copying %s -> %s" % ( source, dest))
storage.remove(dest)
storage.copy(source, dest)
return
def mkpsf(expnum, ccd):
"""Run the OSSOS makepsf script.
"""
## get image from the vospace storage area
filename = storage.get_image(expnum, ccd, version='p')
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
## launch the makepsf script
util.exec_prog(['jmpmakepsf.csh',
'./',
filename,
'no'])
## place the results into VOSpace
basename = os.path.splitext(filename)[0]
## confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, version='p',ext='fits'))
logging.info("Checking that destination direcoties exist")
storage.mkdir(destdir)
for ext in ('mopheader', 'psf.fits',
'zeropoint.used', 'apcor', 'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum, ccd, version='p', ext=ext)
source = basename + "." + ext
storage.copy(source, dest)
return
def step1(expnum,
ccd,
prefix='',
version='p',
fwhm=4,
sex_thresh=1.3,
wave_thresh=2.7,
maxcount=30000):
"""run the actual step1jmp/matt codes.
expnum: the CFHT expousre to process
ccd: which ccd in the mosaic to process
fwhm: the image quality, FWHM, of the image. In pixels.
sex_thresh: the detection threhold to run sExtractor at
wave_thresh: the detection threshold for wavelet
maxcount: saturation level
"""
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
mopheader = storage.get_image(expnum, ccd, version=version,
ext='mopheader', prefix=prefix)
fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version)
basename = os.path.splitext(filename)[0]
outfile = util.exec_prog(['step1jmp',
'-f', basename,
'-t', str(wave_thresh),
'-w', str(fwhm),
'-m', str(maxcount)])
obj_uri = storage.get_uri(expnum,ccd,version=version,ext='obj.jmp',
prefix=prefix)
obj_filename = basename+".obj.jmp"
storage.copy(obj_filename,obj_uri)
## for step1matt we need the weight image
hdulist = fits.open(filename)
flat_name = hdulist[0].header.get('FLAT','06Bm02.flat.r.36.01.fits')
flat_name = flat_name[0:-5]
flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits',
subdir='calibrators', rescale=False)
if not os.access('weight.fits',os.R_OK):
os.symlink(flat_filename, 'weight.fits')
outfile = util.exec_prog(['step1matt',
'-f', basename,
'-t', str(sex_thresh),
'-w', str(fwhm),
'-m', str(maxcount)])
obj_uri = storage.get_uri(expnum,ccd,version=version,ext='obj.matt',
prefix=prefix)
obj_filename = basename+".obj.matt"
storage.copy(obj_filename,obj_uri)
return True
def plant(expnums, ccd, rmin, rmax, ang, width, version='s'):
'''run the plant script on this combination of exposures'''
ptf = open('proc-these-files','w')
ptf.write("# Files to be planted and search\n")
ptf.write("# image fwhm plant\n")
for expnum in expnums:
fwhm = storage.get_fwhm(expnum,ccd)
filename = storage.get_image(expnum, ccd=ccd, version=version)
ptf.write("%s %3.1f YES\n" % ( filename[0:-5],
fwhm ))
for ext in ['apcor',
'obj.jmp',
'trans.jmp',
'psf.fits',
'mopheader',
'phot',
'zeropoint.used']:
apcor = storage.get_image(expnum, ccd=ccd, version='s',
ext=ext)
ptf.close()
cmd_args = ['plant.csh',os.curdir,
str(rmin), str(rmax), str(ang), str(width)]
util.exec_prog(cmd_args)
if args.dryrun:
# Don't push back to VOSpace
return
uri = storage.get_uri('Object',ext='planted',version='',
subdir=str(
expnums[0])+"/ccd%s" % (str(ccd).zfill(2)))
storage.copy('Object.planted',uri)
uri = os.path.join(os.path.dirname(uri), 'shifts')
storage.copy('shifts', uri)
for expnum in expnums:
uri = storage.get_uri(expnum,
ccd=ccd,
version=version,
ext='fits', prefix='fk')
filename = os.path.basename(uri)
storage.copy(filename, uri)
for ext in ['mopheader',
'psf.fits',
'fwhm',
'apcor', 'zeropoint.used', 'trans.jmp']:
storage.delete(expnum, ccd, 's', ext, prefix='fk')
storage.vlink(expnum, ccd, 'p', ext,
expnum, ccd, 's', ext, l_prefix='fk')
return
def step3(expnums, ccd, version, rate_min,
rate_max, angle, width, field=None, prefix=None, dry_run=False,
maximum_flux_ratio=3, minimum_area=5, minimum_median_flux=1000.0):
"""run the actual step3 on the given exp/ccd combo"""
jmp_args = ['step3jmp']
matt_args = ['step3jjk']
idx = 0
cmd_args = []
for expnum in expnums:
idx += 1
for ext in ['unid.jmp', 'unid.matt']:
storage.get_file(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix)
image = os.path.splitext(os.path.basename(storage.get_uri(expnum, ccd, version=version, prefix=prefix)))[0]
cmd_args.append('-f%d' % idx)
cmd_args.append(image)
cmd_args.extend(['-rn', str(rate_min),
'-rx', str(rate_max),
'-a', str(angle),
'-w', str(width)])
jmp_args.extend(cmd_args)
# Add some extra arguemnents for the ISO search.
cmd_args.extend(['-fr', str(maximum_flux_ratio),
'-ma', str(minimum_area),
'-mf', str(minimum_median_flux)])
matt_args.extend(cmd_args)
logging.info(util.exec_prog(jmp_args))
logging.info(util.exec_prog(matt_args))
if dry_run:
return
if field is None:
field = str(expnums[0])
storage.mkdir(os.path.dirname(storage.get_uri(field,
ccd=ccd,
version=version,
prefix=prefix)))
for ext in ['moving.jmp', 'moving.matt']:
uri = storage.get_uri(field,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = '%s%d%s%s.%s' % (prefix, expnums[0],
version,
str(ccd).zfill(2),
ext)
storage.copy(filename, uri)
return
def step3(expnums, ccd, version, rate_min,
rate_max, angle, width, field=None, prefix=None):
'''run the actual step2 on the given exp/ccd combo'''
jmp_args = ['step3jmp']
matt_args = ['step3matt']
idx = 0
cmd_args = []
for expnum in expnums:
idx += 1
for ext in ['unid.jmp', 'unid.matt',
'trans.jmp' ]:
filename = storage.get_image(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix
)
image = os.path.splitext(os.path.splitext(os.path.basename(filename))[0])[0]
cmd_args.append('-f%d' % ( idx))
cmd_args.append(image)
cmd_args.extend(['-rn', str(rate_min),
'-rx', str(rate_max),
'-a', str(angle),
'-w', str(width)])
jmp_args.extend(cmd_args)
matt_args.extend(cmd_args)
util.exec_prog(jmp_args)
util.exec_prog(matt_args)
if field is None:
field = str(expnums[0])
storage.mkdir(os.path.dirname(
storage.get_uri(field,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)))
for ext in ['moving.jmp', 'moving.matt']:
uri = storage.get_uri(field,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = '%s%d%s%s.%s' % ( prefix, expnums[0],
version,
str(ccd).zfill(2),
ext)
storage.copy(filename, uri)
return
def mkpsf(expnum, ccd, version, dry_run=False, prefix=""):
"""Run the OSSOS jmpmakepsf script.
"""
## confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits'))
if not dry_run:
storage.mkdir(destdir)
## get image from the vospace storage area
logging.info("Getting file from VOSpace")
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
## launch the makepsf script
logging.info(util.exec_prog(['jmpmakepsf.csh',
'./',
filename,
'yes', 'yes']))
if dry_run:
return
## place the results into VOSpace
basename = os.path.splitext(filename)[0]
for ext in ('mopheader', 'psf.fits',
'zeropoint.used', 'apcor', 'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext)
source = basename + "." + str(ext)
storage.copy(source, dest)
return
def mk_mopheader(expnum, ccd, version, dry_run=False, prefix=""):
"""Run the OSSOS mopheader script.
"""
## confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits'))
if not dry_run:
storage.mkdir(destdir)
## get image from the vospace storage area
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
logging.info("Running mopheader on %s %d" % (expnum, ccd))
## launch the mopheader script
## launch the makepsf script
expname = os.path.basename(filename).strip('.fits')
logging.info(util.exec_prog(['stepZjmp',
'-f',
expname]))
mopheader_filename = expname+".mopheader"
# mopheader_filename = mopheader.main(filename)
if dry_run:
return
destination = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='mopheader')
source = mopheader_filename
storage.copy(source, destination)
return
def plant(expnums, ccd, rmin, rmax, ang, width, version="s"):
"""run the plant script on this combination of exposures"""
ptf = open("proc-these-files", "w")
ptf.write("# Files to be planted and search\n")
ptf.write("# image fwhm plant\n")
for expnum in expnums:
fwhm = storage.get_fwhm(expnum, ccd, version=version)
filename = storage.get_image(expnum, ccd=ccd, version=version)
ptf.write("%s %3.1f YES\n" % (filename[0:-5], fwhm))
for ext in ["apcor", "obj.jmp", "trans.jmp", "psf.fits", "mopheader", "phot", "zeropoint.used"]:
apcor = storage.get_image(expnum, ccd=ccd, version=version, ext=ext)
ptf.close()
cmd_args = ["plant.csh", os.curdir, str(rmin), str(rmax), str(ang), str(width)]
util.exec_prog(cmd_args)
if args.dryrun:
# Don't push back to VOSpace
return
uri = storage.get_uri("Object", ext="planted", version="", subdir=str(expnums[0]) + "/ccd%s" % (str(ccd).zfill(2)))
storage.copy("Object.planted", uri)
uri = os.path.join(os.path.dirname(uri), "plant.shifts")
storage.copy("shifts", uri)
for expnum in expnums:
uri = storage.get_uri(expnum, ccd=ccd, version=version, ext="fits", prefix="fk")
filename = os.path.basename(uri)
storage.copy(filename, uri)
for ext in ["mopheader", "psf.fits", "fwhm", "apcor", "zeropoint.used", "trans.jmp"]:
storage.delete(expnum, ccd, "s", ext, prefix="fk")
storage.vlink(expnum, ccd, "s", ext, expnum, ccd, "s", ext, l_prefix="fk")
return
def step2(expnums, ccd, version, prefix=None):
'''run the actual step2 on the given exp/ccd combo'''
jmp_args = ['step2jmp']
matt_args = ['step2matt_jmp']
idx = 0
for expnum in expnums:
jmp_args.append(
storage.get_image(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix
)[0:-8]
)
idx += 1
matt_args.append('-f%d' % ( idx))
matt_args.append(
storage.get_image(expnum,
ccd=ccd,
version=version,
ext='obj.matt',
prefix=prefix
)[0:-9]
)
util.exec_prog(jmp_args)
util.exec_prog(matt_args)
for expnum in expnums:
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
uri = storage.dbimages_uri(expnum,ccd=ccd,version=version,ext=ext, prefix=prefix)
filename = os.path.basename(uri)
storage.copy(filename, uri)
return
def step2(expnums, ccd, version, prefix=None):
'''run the actual step2 on the given exp/ccd combo'''
jmp_args = ['step2jmp']
matt_args = ['step2matt_jmp']
idx = 0
for expnum in expnums:
jmp_args.append(
storage.get_image(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix)[0:-8])
idx += 1
matt_args.append('-f%d' % (idx))
matt_args.append(
storage.get_image(expnum,
ccd=ccd,
version=version,
ext='obj.matt',
prefix=prefix)[0:-9])
util.exec_prog(jmp_args)
util.exec_prog(matt_args)
for expnum in expnums:
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
uri = storage.dbimages_uri(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = os.path.basename(uri)
storage.copy(filename, uri)
return
def mkpsf(expnum, ccd, version, dry_run=False, prefix=""):
"""Run the OSSOS jmpmakepsf script.
"""
## confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum,
ccd,
prefix=prefix,
version=version,
ext='fits'))
if not dry_run:
storage.mkdir(destdir)
## get image from the vospace storage area
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
## launch the makepsf script
logging.info(util.exec_prog(['jmpmakepsf.csh', './', filename, 'no']))
if dry_run:
return
## place the results into VOSpace
basename = os.path.splitext(filename)[0]
for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm',
'phot'):
dest = storage.dbimages_uri(expnum,
ccd,
prefix=prefix,
version=version,
ext=ext)
source = basename + "." + str(ext)
storage.copy(source, dest)
return
def run(expnum,
ccd,
prefix='',
version='p',
sex_thresh=_SEX_THRESHOLD,
wave_thresh=_WAVE_THRESHOLD,
maxcount=_MAX_COUNT,
dry_run=False,
force=True, ignore=False):
"""run the actual step1jmp/matt codes.
expnum: the CFHT expousre to process
ccd: which ccd in the mosaic to process
fwhm: the image quality, FWHM, of the image. In pixels.
sex_thresh: the detection threhold to run sExtractor at
wave_thresh: the detection threshold for wavelet
maxcount: saturation level
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version, ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(dependency, prefix, expnum, version, ccd) and not ignore:
raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format(
task, dependency, prefix, expnum, version, ccd))
logging.info("Retrieving imaging and input parameters from VOSpace")
storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader')
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version, default=3.5)
basename = os.path.splitext(filename)[0]
_get_weight_map(filename, ccd)
logging.info("Launching step1jmp")
logging.info(util.exec_prog(['step1jmp',
'-f', basename,
'-t', str(wave_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
logging.info(util.exec_prog(['step1matt',
'-f', basename,
'-t', str(sex_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
if os.access('weight.fits', os.R_OK):
os.unlink('weight.fits')
if not dry_run:
for ext in ['obj.jmp', 'obj.matt']:
obj_uri = storage.get_uri(expnum, ccd, version=version, ext=ext,
prefix=prefix)
obj_filename = basename + "." + ext
count = 0
with open(obj_filename, 'r'):
while True:
try:
count += 1
logging.info("Attempt {} to copy {} -> {}".format(count, obj_filename, obj_uri))
storage.copy(obj_filename, obj_uri)
break
except Exception as ex:
if count > 10:
raise ex
logging.info(message)
except Exception as ex:
message = str(ex)
logging.error(message)
if not dry_run:
storage.set_status(task, prefix, expnum, version, ccd, status=message)
def step1(expnum,
ccd,
prefix='',
version='p',
fwhm=4,
sex_thresh=1.3,
wave_thresh=2.7,
maxcount=30000):
"""run the actual step1jmp/matt codes.
expnum: the CFHT expousre to process
ccd: which ccd in the mosaic to process
fwhm: the image quality, FWHM, of the image. In pixels.
sex_thresh: the detection threhold to run sExtractor at
wave_thresh: the detection threshold for wavelet
maxcount: saturation level
"""
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
mopheader = storage.get_image(expnum,
ccd,
version=version,
ext='mopheader',
prefix=prefix)
fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version)
basename = os.path.splitext(filename)[0]
outfile = util.exec_prog([
'step1jmp', '-f', basename, '-t',
str(wave_thresh), '-w',
str(fwhm), '-m',
str(maxcount)
])
obj_uri = storage.get_uri(expnum,
ccd,
version=version,
ext='obj.jmp',
prefix=prefix)
obj_filename = basename + ".obj.jmp"
storage.copy(obj_filename, obj_uri)
## for step1matt we need the weight image
flat_name = fits.open(filename)[0].header['FLAT']
flat_name = flat_name[0:-5]
flat_filename = storage.get_image(flat_name,
ccd,
version='',
ext='fits',
subdir='calibrators',
rescale=False)
if not os.access('weight.fits', os.R_OK):
os.symlink(flat_filename, 'weight.fits')
outfile = util.exec_prog([
'step1matt', '-f', basename, '-t',
str(sex_thresh), '-w',
str(fwhm), '-m',
str(maxcount)
])
obj_uri = storage.get_uri(expnum,
ccd,
version=version,
ext='obj.matt',
prefix=prefix)
obj_filename = basename + ".obj.matt"
storage.copy(obj_filename, obj_uri)
return True
def run(expnum, ccd, prefix=None, version='p', field=None,
measure3_dir=storage.MEASURE3, dry_run=False, force=False):
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version, ccd) and not force:
logging.info(
"{} completed successfully for {} {} {} {}".format(task, prefix,
expnum, version,
ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(dependency, prefix, expnum, version, ccd):
raise IOError(35, "Cannot start {} as {} not yet completed "
"for {}{}{}{:02d}".format(task, dependency,
prefix, expnum,
version, ccd))
if field is None:
field = str(expnum)
if prefix is not None and len(prefix) > 0:
field = "%s_%s" % (prefix, field)
field += "_%s%s" % (str(version), str(ccd))
logging.info("Doing combine on field {}".format(field))
for ext in ['moving.matt', 'moving.jmp']:
storage.get_file(expnum, ccd=ccd, version=version, ext=ext,
prefix=prefix)
# Get the list of objects planted into the field if prefix='fk'
if prefix == 'fk':
storage.get_file('Object',
version='',
ext='planted',
subdir=str(expnum) + "/ccd%s" % (
str(ccd).zfill(2)))
else:
prefix = ''
cmd_args = ['comb-list',
prefix + str(expnum) + version + str(ccd).zfill(2)]
logging.info(str(cmd_args))
logging.info(util.exec_prog(cmd_args))
# things to copy back to VOSpace, if this is an 'fk' image
# then we have missed and found files too.
ext_list = ['cands.comb']
if prefix == 'fk':
ext_list.extend(['jmp.missed', 'matt.missed',
'jmp.found', 'matt.found',
'comb.missed', 'comb.found'])
for ext in ext_list:
uri = storage.get_uri(expnum,
ccd=ccd,
prefix=prefix,
version=version,
ext=ext)
filename = os.path.basename(uri)
if not os.access(filename, os.R_OK):
logging.critical("No %s file" % filename)
continue
vospace_name = "%s.%s" % (field, ext)
if not dry_run:
logging.info("%s -> %s" % (
filename, os.path.join(measure3_dir, vospace_name)))
storage.copy(filename, os.path.join(measure3_dir, vospace_name))
base_name = prefix + str(expnum) + version + str(ccd).zfill(2)
cands_file = base_name + '.cands.comb'
if not os.access(cands_file, os.R_OK):
no_cands_file = (prefix +
str(expnum) +
version +
str(ccd).zfill(2) +
'.no_candidates')
open(no_cands_file, 'w').close()
if not dry_run:
vospace_name = "%s.no_candidates" % field
storage.copy(no_cands_file,
os.path.join(measure3_dir, vospace_name))
else:
measure3.run(base_name, storage.DBIMAGES)
if not dry_run:
filename = base_name + ".measure3.cands.astrom"
vospace_filename = "%s.measure3.cands.astrom" % field
storage.copy(filename, os.path.join(measure3_dir, vospace_filename))
except Exception as ex:
message = str(ex)
logging.error(message)
if not dry_run:
storage.set_status(task, prefix, expnum, version, ccd,
status=message)
def step3(expnums,
ccd,
version,
rate_min,
rate_max,
angle,
width,
field=None,
prefix=None):
'''run the actual step2 on the given exp/ccd combo'''
jmp_args = ['step3jmp']
matt_args = ['step3matt']
idx = 0
cmd_args = []
for expnum in expnums:
idx += 1
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
filename = storage.get_image(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
image = os.path.splitext(
os.path.splitext(os.path.basename(filename))[0])[0]
cmd_args.append('-f%d' % (idx))
cmd_args.append(image)
cmd_args.extend([
'-rn',
str(rate_min), '-rx',
str(rate_max), '-a',
str(angle), '-w',
str(width)
])
jmp_args.extend(cmd_args)
matt_args.extend(cmd_args)
util.exec_prog(jmp_args)
util.exec_prog(matt_args)
if field is None:
field = str(expnums[0])
storage.mkdir(
os.path.dirname(
storage.get_uri(field,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)))
for ext in ['moving.jmp', 'moving.matt']:
uri = storage.get_uri(field,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = '%s%d%s%s.%s' % (prefix, expnums[0], version,
str(ccd).zfill(2), ext)
storage.copy(filename, uri)
return
def combine(expnum, ccd, prefix=None, type='p', field=None, measure3=MEASURE3 ):
if field is None:
field=str(expnum)
if prefix is not None and len(prefix) > 0:
field = "%s_%s" % ( prefix, field )
field += "_%s" % ( str(ccd))
for ext in ['moving.matt','moving.jmp']:
fname = storage.get_image(expnum,
ccd=ccd,
prefix=prefix,
version=type, ext=ext)
if prefix is not None and len(prefix) > 0:
planted = storage.get_image('Object',
subdir=str(expnum)+"/ccd%s" % (
str(ccd).zfill(2)),
version='',
ext='planted')
else:
prefix = ''
base_image = os.path.basename(
storage.get_uri(expnum,
ccd=ccd,
prefix=prefix,
version=type,
ext=None))
cmd_args = ['comb-list', prefix+str(expnum)+type+str(ccd).zfill(2)]
util.exec_prog(cmd_args)
ext_list = ['cands.comb']
if prefix is not None and len(prefix) > 0 :
ext_list.extend( [ 'jmp.missed', 'matt.missed',
'jmp.found', 'matt.found',
'comb.missed', 'comb.found' ] )
for ext in ext_list:
uri = storage.get_uri(expnum,
ccd=ccd,
prefix=prefix,
version=type,
ext=ext)
filename = os.path.basename(uri)
if not os.access(filename,os.R_OK):
logging.critical("No %s file" % (filename))
continue
vospace_name = "%s.%s" % ( field, ext )
logging.info("%s -> %s" % ( filename, os.path.join(measure3, vospace_name)))
storage.copy(filename, os.path.join(measure3, vospace_name))
base_name = prefix+str(expnum)+type+str(ccd).zfill(2)
cands_file = base_name+'.cands.comb'
if not os.access(cands_file,os.R_OK):
nocands_file = ( prefix+
str(expnum)+
type+
str(ccd).zfill(2)+
'.no_candidates' )
open(nocands_file, 'w').close()
vospace_name = "%s.no_candidates" % ( field )
storage.copy(nocands_file,os.path.join(measure3, vospace_name))
return storage.SUCCESS
# get the images we need to compute x/y ra/dec transforms
cands_file = mop_file.Parser().parse(cands_file)
for file_id in cands_file.header.file_ids:
rec_no=cands_file.header.file_ids.index(file_id)
storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no],
ccd=ccd,
version=type,
prefix=prefix,
ext='fits')
cmd_args = ['measure3', prefix+str(expnum)+type+str(ccd).zfill(2)]
logging.info("Running measure3")
util.exec_prog(cmd_args)
filename=base_name+".measure3.cands.astrom"
vospace_filename = "%s.measure3.cands.astrom" % ( field)
storage.copy(filename, os.path.join(measure3,vospace_filename))
return storage.SUCCESS
def plant(expnums, ccd, rmin, rmax, ang, width, version='s'):
'''run the plant script on this combination of exposures'''
ptf = open('proc-these-files', 'w')
ptf.write("# Files to be planted and search\n")
ptf.write("# image fwhm plant\n")
for expnum in expnums:
fwhm = storage.get_fwhm(expnum, ccd)
filename = storage.get_image(expnum, ccd=ccd, version=version)
ptf.write("%s %3.1f YES\n" % (filename[0:-5], fwhm))
for ext in [
'apcor', 'obj.jmp', 'trans.jmp', 'psf.fits', 'mopheader',
'phot', 'zeropoint.used'
]:
apcor = storage.get_image(expnum, ccd=ccd, version='s', ext=ext)
ptf.close()
cmd_args = [
'plant.csh', os.curdir,
str(rmin),
str(rmax),
str(ang),
str(width)
]
util.exec_prog(cmd_args)
if args.dryrun:
# Don't push back to VOSpace
return
uri = storage.get_uri('Object',
ext='planted',
version='',
subdir=str(expnums[0]) + "/ccd%s" %
(str(ccd).zfill(2)))
storage.copy('Object.planted', uri)
uri = os.path.join(os.path.dirname(uri), 'shifts')
storage.copy('shifts', uri)
for expnum in expnums:
uri = storage.get_uri(expnum,
ccd=ccd,
version=version,
ext='fits',
prefix='fk')
filename = os.path.basename(uri)
storage.copy(filename, uri)
for ext in [
'mopheader', 'psf.fits', 'fwhm', 'apcor', 'zeropoint.used',
'trans.jmp'
]:
storage.delete(expnum, ccd, 's', ext, prefix='fk')
storage.vlink(expnum,
ccd,
'p',
ext,
expnum,
ccd,
's',
ext,
l_prefix='fk')
return
def run(expnums,
ccd,
version,
prefix=None,
dry_run=False,
default="WCS",
force=False):
"""run the actual step2 on the given exp/ccd combo"""
jmp_trans = ['step2ajmp']
jmp_args = ['step2bjmp']
matt_args = ['step2matt_jmp']
if storage.get_status(task, prefix, expnums[0], version,
ccd) and not force:
logging.info("{} completed successfully for {}{}{}{:02d}".format(
task, prefix, expnums[0], version, ccd))
return
with storage.LoggingManager(task, prefix, expnums[0], ccd, version,
dry_run):
try:
for expnum in expnums:
if not storage.get_status(
dependency, prefix, expnum, version=version, ccd=ccd):
raise IOError(
35,
"Cannot start {} as {} not yet completed for {}{}{}{:02d}"
.format(task, dependency, prefix, expnum, version,
ccd))
message = storage.SUCCESS
idx = 0
logging.info("Retrieving catalog files to do matching.")
for expnum in expnums:
jmp_args.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix)[0:-8])
jmp_trans.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix)[0:-8])
idx += 1
matt_args.append('-f%d' % idx)
matt_args.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.matt',
prefix=prefix)[0:-9])
logging.info(
"Computing the catalog alignment using sources in catalogs.")
try:
logging.info(util.exec_prog(jmp_trans))
if default == "WCS":
logging.info("Comparing computed transform to WCS values")
logging.info(
compute_trans(expnums,
ccd,
version,
prefix,
default=default))
except Exception as ex:
logging.info("JMP Trans failed: {}".format(ex))
logging.info(
compute_trans(expnums, ccd, version, prefix,
default="WCS"))
logging.info("Using transform to match catalogs for three images.")
logging.info(util.exec_prog(jmp_args))
logging.info(util.exec_prog(matt_args))
# check that the shifts from step2 are rational by matching the bright star lists.
logging.info(
"Uisng checktrans to ensure that transforms were reasonable.")
check_args = ['checktrans']
if os.access('proc-these-files', os.R_OK):
os.unlink('proc-these-files')
ptf = open('proc-these-files', 'w')
ptf.write(
"# A dummy file that is created so checktrans could run.\n")
ptf.write("# Frame FWHM PSF?\n")
for expnum in expnums:
filename = os.path.splitext(
storage.get_image(expnum,
ccd,
version=version,
prefix=prefix))[0]
if not os.access(filename + ".bright.psf", os.R_OK):
os.link(filename + ".bright.jmp", filename + ".bright.psf")
if not os.access(filename + ".obj.psf", os.R_OK):
os.link(filename + ".obj.jmp", filename + ".obj.psf")
ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(
filename, _FWHM, "NO"))
ptf.close()
if os.access('BAD_TRANS', os.F_OK):
os.unlink('BAD_TRANS')
logging.info(util.exec_prog(check_args))
if os.access('BAD_TRANS', os.F_OK):
raise OSError(errno.EBADMSG, 'BAD_TRANS')
if os.access('proc-these-files', os.F_OK):
os.unlink('proc-these-files')
if dry_run:
return
for expnum in expnums:
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
uri = storage.dbimages_uri(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = os.path.basename(uri)
storage.copy(filename, uri)
except Exception as ex:
message = str(ex)
logging.error(message)
storage.set_status(task,
prefix,
expnums[0],
version,
ccd,
status=message)
return
def run(expnum,
ccd,
prefix='',
version='p',
sex_thresh=_SEX_THRESHOLD,
wave_thresh=_WAVE_THRESHOLD,
maxcount=_MAX_COUNT,
dry_run=False,
force=True):
"""run the actual step1jmp/matt codes.
expnum: the CFHT expousre to process
ccd: which ccd in the mosaic to process
fwhm: the image quality, FWHM, of the image. In pixels.
sex_thresh: the detection threhold to run sExtractor at
wave_thresh: the detection threshold for wavelet
maxcount: saturation level
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version, ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(dependency, prefix, expnum, version, ccd):
raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format(
task, dependency, prefix, expnum, version, ccd))
logging.info("Retrieving imaging and input parameters from VOSpace")
storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader')
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version)
basename = os.path.splitext(filename)[0]
_get_weight_map(filename, ccd)
logging.info("Launching step1jmp")
logging.info(util.exec_prog(['step1jmp',
'-f', basename,
'-t', str(wave_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
logging.info(util.exec_prog(['step1matt',
'-f', basename,
'-t', str(sex_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
if os.access('weight.fits', os.R_OK):
os.unlink('weight.fits')
if not dry_run:
for ext in ['obj.jmp', 'obj.matt']:
obj_uri = storage.get_uri(expnum, ccd, version=version, ext=ext,
prefix=prefix)
obj_filename = basename + "." + ext
count = 0
with open(obj_filename, 'r'):
while True:
try:
count += 1
logging.info("Attempt {} to copy {} -> {}".format(count, obj_filename, obj_uri))
storage.copy(obj_filename, obj_uri)
break
except Exception as ex:
if count > 10:
raise ex
logging.info(message)
except Exception as ex:
message = str(ex)
logging.error(message)
if not dry_run:
storage.set_status(task, prefix, expnum, version, ccd, status=message)
def step1(expnum,
ccd,
prefix='',
version='p',
sex_thresh=_SEX_THRESHOLD,
wave_thresh=_WAVE_THRESHOLD,
maxcount=_MAX_COUNT,
dry_run=False):
"""run the actual step1jmp/matt codes.
expnum: the CFHT expousre to process
ccd: which ccd in the mosaic to process
fwhm: the image quality, FWHM, of the image. In pixels.
sex_thresh: the detection threhold to run sExtractor at
wave_thresh: the detection threshold for wavelet
maxcount: saturation level
"""
storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader')
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version)
basename = os.path.splitext(filename)[0]
logging.info(util.exec_prog(['step1jmp',
'-f', basename,
'-t', str(wave_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.jmp',
prefix=prefix)
obj_filename = basename + ".obj.jmp"
if not dry_run:
storage.copy(obj_filename, obj_uri)
## for step1matt we need the weight image
hdulist = fits.open(filename)
flat_name = hdulist[0].header.get('FLAT', 'weight.fits')
parts = os.path.splitext(flat_name)
if parts[1] == '.fz':
flat_name = os.path.splitext(parts[0])[0]
else:
flat_name = parts[0]
try:
flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators')
except:
flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='old_calibrators')
if os.access('weight.fits', os.R_OK):
os.unlink('weight.fits')
if not os.access('weight.fits', os.R_OK):
os.symlink(flat_filename, 'weight.fits')
logging.info(util.exec_prog(['step1matt',
'-f', basename,
'-t', str(sex_thresh),
'-w', str(fwhm),
'-m', str(maxcount)]))
if os.access('weight.fits', os.R_OK):
os.unlink('weight.fits')
obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.matt',
prefix=prefix)
obj_filename = basename + ".obj.matt"
if not dry_run:
storage.copy(obj_filename, obj_uri)
return True
def run(expnum, ccd, version, dry_run=False, prefix="", force=False):
"""Run the OSSOS jmpmakepsf script.
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd):
raise IOError("{} not yet run for {}".format(dependency, expnum))
# confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits'))
if not dry_run:
storage.mkdir(destdir)
# get image from the vospace storage area
logging.info("Getting fits image from VOSpace")
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
# get mopheader from the vospace storage area
logging.info("Getting mopheader from VOSpace")
mopheader_filename = storage.get_file(expnum, ccd, version=version, prefix=prefix, ext='mopheader')
# run mkpsf process
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
logging.info(util.exec_prog(['jmpmakepsf.csh',
'./',
filename,
'yes', 'yes']))
if dry_run:
return
# place the results into VOSpace
basename = os.path.splitext(filename)[0]
for ext in ('mopheader', 'psf.fits',
'zeropoint.used', 'apcor', 'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext)
source = basename + "." + str(ext)
count = 0
with open(source, 'r'):
while True:
count += 1
try:
logging.info("Attempt {} to copy {} -> {}".format(count, source, dest))
storage.copy(source, dest)
break
except Exception as ex:
if count > 10:
raise ex
# set some data parameters associated with the image, determined in this step.
storage.set_status('fwhm', prefix, expnum, version=version, ccd=ccd, status=str(storage.get_fwhm(
expnum, ccd=ccd, prefix=prefix, version=version)))
storage.set_status('zeropoint', prefix, expnum, version=version, ccd=ccd,
status=str(storage.get_zeropoint(
expnum, ccd=ccd, prefix=prefix, version=version)))
logging.info(message)
except Exception as e:
message = str(e)
logging.error(message)
storage.set_status(task, prefix, expnum, version, ccd=ccd, status=message)
return
def step2(expnums, ccd, version, prefix=None, dry_run=False, default="WCS"):
"""run the actual step2 on the given exp/ccd combo"""
jmp_trans = ['step2ajmp']
jmp_args = ['step2bjmp']
matt_args = ['step2matt_jmp']
idx = 0
for expnum in expnums:
jmp_args.append(
storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]
)
jmp_trans.append(
storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]
)
idx += 1
matt_args.append('-f%d' % idx)
matt_args.append(
storage.get_file(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix)[0:-9]
)
logging.info(util.exec_prog(jmp_trans))
if default == "WCS":
logging.info(compute_trans(expnums, ccd, version, prefix, default=default))
logging.info(util.exec_prog(jmp_args))
logging.info(util.exec_prog(matt_args))
## check that the shifts from step2 are rational
check_args = ['checktrans']
if os.access('proc-these-files', os.R_OK):
os.unlink('proc-these-files')
ptf = open('proc-these-files', 'w')
ptf.write("# A dummy file that is created so checktrans could run.\n")
ptf.write("# Frame FWHM PSF?\n")
for expnum in expnums:
filename = os.path.splitext(storage.get_image(expnum, ccd, version=version, prefix=prefix))[0]
if not os.access(filename + ".bright.psf", os.R_OK):
os.link(filename + ".bright.jmp", filename + ".bright.psf")
if not os.access(filename + ".obj.psf", os.R_OK):
os.link(filename + ".obj.jmp", filename + ".obj.psf")
ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(filename,
_FWHM,
"NO"))
ptf.close()
if os.access('BAD_TRANS', os.F_OK):
os.unlink('BAD_TRANS')
logging.info(util.exec_prog(check_args))
if os.access('BAD_TRANS', os.F_OK):
raise OSError(errno.EBADMSG, 'BAD_TRANS')
if os.access('proc-these-files', os.F_OK):
os.unlink('proc-these-files')
if dry_run:
return
for expnum in expnums:
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
uri = storage.dbimages_uri(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix)
filename = os.path.basename(uri)
storage.copy(filename, uri)
return
def run(expnums,
ccd,
version,
rate_min,
rate_max,
angle,
width,
field=None,
prefix=None,
dry_run=False,
force=False):
"""run the actual step2 on the given exp/ccd combo"""
jmp_args = ['step3jmp']
matt_args = ['step3matt']
if storage.get_status(task, prefix, expnums[0], version=version,
ccd=ccd) and not force:
logging.info("{} completed successfully for {}{}{}{:02d}".format(
task, prefix, expnums[0], version, ccd))
return
with storage.LoggingManager(task, prefix, expnums[0], ccd, version,
dry_run):
try:
if not storage.get_status(
dependency, prefix, expnums[0], version=version, ccd=ccd):
raise IOError(
35, "Cannot start {} as {} not yet completed {}{}{}{:02d}".
format(task, dependency, prefix, expnums[0], version, ccd))
# Default message is success, message gets overwritten with failure messages.
message = storage.SUCCESS
idx = 0
cmd_args = []
for expnum in expnums:
idx += 1
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
storage.get_file(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
image = os.path.splitext(
os.path.basename(
storage.get_uri(expnum,
ccd,
version=version,
prefix=prefix)))[0]
cmd_args.append('-f%d' % idx)
cmd_args.append(image)
cmd_args.extend([
'-rn',
str(rate_min), '-rx',
str(rate_max), '-a',
str(angle), '-w',
str(width)
])
jmp_args.extend(cmd_args)
matt_args.extend(cmd_args)
logging.info(util.exec_prog(jmp_args))
logging.info(util.exec_prog(matt_args))
if dry_run:
return
if field is None:
field = str(expnums[0])
# Make sure a dbimages destination exists for this file.
storage.mkdir(
os.path.dirname(
storage.get_uri(field,
ccd=ccd,
version=version,
prefix=prefix)))
for ext in ['moving.jmp', 'moving.matt']:
uri = storage.get_uri(field,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = '%s%d%s%s.%s' % (prefix, expnums[0], version,
str(ccd).zfill(2), ext)
storage.copy(filename, uri)
except Exception as ex:
message = str(ex)
logging.error(message)
storage.set_status(task,
prefix,
expnums[0],
version=version,
ccd=ccd,
status=message)
return
def run(expnum, ccd, version, dry_run=False, prefix="", force=False):
"""Run the OSSOS jmpmakepsf script.
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version=version,
ccd=ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(
task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(
dependency, prefix, expnum, version, ccd=ccd):
raise IOError("{} not yet run for {}".format(
dependency, expnum))
# confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum,
ccd,
prefix=prefix,
version=version,
ext='fits'))
if not dry_run:
storage.mkdir(destdir)
# get image from the vospace storage area
logging.info("Getting fits image from VOSpace")
filename = storage.get_image(expnum,
ccd,
version=version,
prefix=prefix)
# get mopheader from the vospace storage area
logging.info("Getting mopheader from VOSpace")
mopheader_filename = storage.get_file(expnum,
ccd,
version=version,
prefix=prefix,
ext='mopheader')
# run mkpsf process
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
logging.info(
util.exec_prog(
['jmpmakepsf.csh', './', filename, 'yes', 'yes']))
if dry_run:
return
# place the results into VOSpace
basename = os.path.splitext(filename)[0]
for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor',
'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum,
ccd,
prefix=prefix,
version=version,
ext=ext)
source = basename + "." + str(ext)
count = 0
with open(source, 'r'):
while True:
count += 1
try:
logging.info("Attempt {} to copy {} -> {}".format(
count, source, dest))
storage.copy(source, dest)
break
except Exception as ex:
if count > 10:
raise ex
# set some data parameters associated with the image, determined in this step.
storage.set_status('fwhm',
prefix,
expnum,
version=version,
ccd=ccd,
status=str(
storage.get_fwhm(expnum,
ccd=ccd,
prefix=prefix,
version=version)))
storage.set_status('zeropoint',
prefix,
expnum,
version=version,
ccd=ccd,
status=str(
storage.get_zeropoint(expnum,
ccd=ccd,
prefix=prefix,
version=version)))
logging.info(message)
except Exception as e:
message = str(e)
logging.error(message)
storage.set_status(task,
prefix,
expnum,
version,
ccd=ccd,
status=message)
return
def combine(expnum, ccd, prefix=None, type='p'):
for ext in ['moving.matt', 'moving.jmp']:
fname = storage.get_image(expnum,
ccd=ccd,
prefix=prefix,
version=type,
ext=ext)
if prefix is not None and len(prefix) > 0:
planted = storage.get_image('Object',
subdir=str(expnum) + "/ccd%s" %
(str(ccd).zfill(2)),
version='',
ext='planted')
else:
prefix = ''
base_image = os.path.basename(
storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=type,
ext=None))
cmd_args = ['comb-list', prefix + str(expnum) + type + str(ccd).zfill(2)]
util.exec_prog(cmd_args)
for ext in [
'cands.comb', 'comb.found', 'comb.missed', 'jmp.found',
'jmp.missed', 'matt.found', 'matt.missed'
]:
uri = storage.get_uri(expnum,
ccd=ccd,
prefix=prefix,
version=type,
ext=ext)
filename = os.path.basename(uri)
if not os.access(filename, os.R_OK):
logging.critical("No %s file" % (filename))
continue
storage.copy(filename, uri)
base_name = prefix + str(expnum) + type + str(ccd).zfill(2)
cands_file = base_name + '.cands.comb'
if not os.access(cands_file, os.R_OK):
nocands_file = (prefix + str(expnum) + type + str(ccd).zfill(2) +
'.no_candidates')
open(nocands_file, 'w').close()
storage.copy(nocands_file, 'vos:OSSOS/measure3/' + nocands_file)
return storage.SUCCESS
cands_file = mop_file.Parser().parse(cands_file)
for file_id in cands_file.header.file_ids:
rec_no = cands_file.header.file_ids.index(file_id)
storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no],
ccd=ccd,
version=type,
prefix=prefix,
ext='fits')
cmd_args = ['measure3', prefix + str(expnum) + type + str(ccd).zfill(2)]
util.exec_prog(cmd_args)
filename = base_name + ".measure3.cands.astrom"
storage.copy(filename, 'vos:OSSOS/measure3/' + filename)
return storage.SUCCESS
def combine(expnum,
ccd,
prefix=None,
file_type='p',
field=None,
measure3=MEASURE3,
dry_run=False):
if field is None:
field = str(expnum)
if prefix is not None and len(prefix) > 0:
field = "%s_%s" % (prefix, field)
field += "_%s%s" % (str(file_type), str(ccd))
logging.info("Doing combine on field {}".format(field))
for ext in ['moving.matt', 'moving.jmp']:
storage.get_file(expnum,
ccd=ccd,
version=file_type,
ext=ext,
prefix=prefix)
if prefix is not None and len(prefix) > 0:
storage.get_file('Object',
version='',
ext='planted',
subdir=str(expnum) + "/ccd%s" % (str(ccd).zfill(2)))
else:
prefix = ''
cmd_args = [
'comb-list', prefix + str(expnum) + file_type + str(ccd).zfill(2)
]
logging.info(str(cmd_args))
logging.info(util.exec_prog(cmd_args))
ext_list = ['cands.comb']
if prefix is not None and len(prefix) > 0:
ext_list.extend([
'jmp.missed', 'matt.missed', 'jmp.found', 'matt.found',
'comb.missed', 'comb.found'
])
for ext in ext_list:
uri = storage.get_uri(expnum,
ccd=ccd,
prefix=prefix,
version=file_type,
ext=ext)
filename = os.path.basename(uri)
if not os.access(filename, os.R_OK):
logging.critical("No %s file" % filename)
continue
vospace_name = "%s.%s" % (field, ext)
if not dry_run:
logging.info("%s -> %s" %
(filename, os.path.join(measure3, vospace_name)))
storage.copy(filename, os.path.join(measure3, vospace_name))
base_name = prefix + str(expnum) + file_type + str(ccd).zfill(2)
cands_file = base_name + '.cands.comb'
if not os.access(cands_file, os.R_OK):
no_cands_file = (prefix + str(expnum) + file_type + str(ccd).zfill(2) +
'.no_candidates')
open(no_cands_file, 'w').close()
if not dry_run:
vospace_name = "%s.no_candidates" % field
storage.copy(no_cands_file, os.path.join(measure3, vospace_name))
return storage.SUCCESS
# get the images we need to compute x/y ra/dec transforms
cands_file = mop_file.Parser().parse(cands_file)
for file_id in cands_file.header.file_ids:
rec_no = cands_file.header.file_ids.index(file_id)
storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no],
ccd=ccd,
version=file_type,
ext='fits',
prefix=prefix)
cmd_args = [
'measure3', prefix + str(expnum) + file_type + str(ccd).zfill(2)
]
logging.info("Running measure3")
logging.info(util.exec_prog(cmd_args))
if not dry_run:
filename = base_name + ".measure3.cands.astrom"
vospace_filename = "%s.measure3.cands.astrom" % field
storage.copy(filename, os.path.join(measure3, vospace_filename))
return storage.SUCCESS
def step2(expnums, ccd, version, prefix=None, dry_run=False, default="WCS"):
"""run the actual step2 on the given exp/ccd combo"""
jmp_trans = ['step2ajmp']
jmp_args = ['step2bjmp']
matt_args = ['step2matt_jmp']
idx = 0
for expnum in expnums:
jmp_args.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix)[0:-8])
jmp_trans.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.jmp',
prefix=prefix)[0:-8])
idx += 1
matt_args.append('-f%d' % idx)
matt_args.append(
storage.get_file(expnum,
ccd=ccd,
version=version,
ext='obj.matt',
prefix=prefix)[0:-9])
logging.info(util.exec_prog(jmp_trans))
if default == "WCS":
logging.info(
compute_trans(expnums, ccd, version, prefix, default=default))
logging.info(util.exec_prog(jmp_args))
logging.info(util.exec_prog(matt_args))
## check that the shifts from step2 are rational
check_args = ['checktrans']
if os.access('proc-these-files', os.R_OK):
os.unlink('proc-these-files')
ptf = open('proc-these-files', 'w')
ptf.write("# A dummy file that is created so checktrans could run.\n")
ptf.write("# Frame FWHM PSF?\n")
for expnum in expnums:
filename = os.path.splitext(
storage.get_image(expnum, ccd, version=version, prefix=prefix))[0]
if not os.access(filename + ".bright.psf", os.R_OK):
os.link(filename + ".bright.jmp", filename + ".bright.psf")
if not os.access(filename + ".obj.psf", os.R_OK):
os.link(filename + ".obj.jmp", filename + ".obj.psf")
ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(filename, _FWHM, "NO"))
ptf.close()
if os.access('BAD_TRANS', os.F_OK):
os.unlink('BAD_TRANS')
logging.info(util.exec_prog(check_args))
if os.access('BAD_TRANS', os.F_OK):
raise ValueError(errno.EBADEXEC, 'BAD_TRANS')
if os.access('proc-these-files', os.F_OK):
os.unlink('proc-these-files')
if dry_run:
return
for expnum in expnums:
for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']:
uri = storage.dbimages_uri(expnum,
ccd=ccd,
version=version,
ext=ext,
prefix=prefix)
filename = os.path.basename(uri)
storage.copy(filename, uri)
return
