def mkpsf(expnum, ccd, fversion): """Run the OSSOS makepsf script. """ ## get image from the vospace storage area filename = storage.get_image(expnum, ccd, version=fversion) logging.info("Running mkpsf on %s %d" % (expnum, ccd)) ## launch the makepsf script util.exec_prog(['jmpmakepsf.csh', './', filename, 'no']) ## place the results into VOSpace basename = os.path.splitext(filename)[0] ## confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, version=fversion,ext='fits')) logging.info("Checking that destination directories exist") storage.mkdir(destdir) for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, version=fversion, ext=ext) source = basename + "." + ext logging.info("Copying %s -> %s" % ( source, dest)) storage.remove(dest) storage.copy(source, dest) return
def mkpsf(expnum, ccd): """Run the OSSOS makepsf script. """ ## get image from the vospace storage area filename = storage.get_image(expnum, ccd, version='p') logging.info("Running mkpsf on %s %d" % (expnum, ccd)) ## launch the makepsf script util.exec_prog(['jmpmakepsf.csh', './', filename, 'no']) ## place the results into VOSpace basename = os.path.splitext(filename)[0] ## confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, version='p',ext='fits')) logging.info("Checking that destination direcoties exist") storage.mkdir(destdir) for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, version='p', ext=ext) source = basename + "." + ext storage.copy(source, dest) return
def step1(expnum, ccd, prefix='', version='p', fwhm=4, sex_thresh=1.3, wave_thresh=2.7, maxcount=30000): """run the actual step1jmp/matt codes. expnum: the CFHT expousre to process ccd: which ccd in the mosaic to process fwhm: the image quality, FWHM, of the image. In pixels. sex_thresh: the detection threhold to run sExtractor at wave_thresh: the detection threshold for wavelet maxcount: saturation level """ filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) mopheader = storage.get_image(expnum, ccd, version=version, ext='mopheader', prefix=prefix) fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version) basename = os.path.splitext(filename)[0] outfile = util.exec_prog(['step1jmp', '-f', basename, '-t', str(wave_thresh), '-w', str(fwhm), '-m', str(maxcount)]) obj_uri = storage.get_uri(expnum,ccd,version=version,ext='obj.jmp', prefix=prefix) obj_filename = basename+".obj.jmp" storage.copy(obj_filename,obj_uri) ## for step1matt we need the weight image hdulist = fits.open(filename) flat_name = hdulist[0].header.get('FLAT','06Bm02.flat.r.36.01.fits') flat_name = flat_name[0:-5] flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators', rescale=False) if not os.access('weight.fits',os.R_OK): os.symlink(flat_filename, 'weight.fits') outfile = util.exec_prog(['step1matt', '-f', basename, '-t', str(sex_thresh), '-w', str(fwhm), '-m', str(maxcount)]) obj_uri = storage.get_uri(expnum,ccd,version=version,ext='obj.matt', prefix=prefix) obj_filename = basename+".obj.matt" storage.copy(obj_filename,obj_uri) return True
def plant(expnums, ccd, rmin, rmax, ang, width, version='s'): '''run the plant script on this combination of exposures''' ptf = open('proc-these-files','w') ptf.write("# Files to be planted and search\n") ptf.write("# image fwhm plant\n") for expnum in expnums: fwhm = storage.get_fwhm(expnum,ccd) filename = storage.get_image(expnum, ccd=ccd, version=version) ptf.write("%s %3.1f YES\n" % ( filename[0:-5], fwhm )) for ext in ['apcor', 'obj.jmp', 'trans.jmp', 'psf.fits', 'mopheader', 'phot', 'zeropoint.used']: apcor = storage.get_image(expnum, ccd=ccd, version='s', ext=ext) ptf.close() cmd_args = ['plant.csh',os.curdir, str(rmin), str(rmax), str(ang), str(width)] util.exec_prog(cmd_args) if args.dryrun: # Don't push back to VOSpace return uri = storage.get_uri('Object',ext='planted',version='', subdir=str( expnums[0])+"/ccd%s" % (str(ccd).zfill(2))) storage.copy('Object.planted',uri) uri = os.path.join(os.path.dirname(uri), 'shifts') storage.copy('shifts', uri) for expnum in expnums: uri = storage.get_uri(expnum, ccd=ccd, version=version, ext='fits', prefix='fk') filename = os.path.basename(uri) storage.copy(filename, uri) for ext in ['mopheader', 'psf.fits', 'fwhm', 'apcor', 'zeropoint.used', 'trans.jmp']: storage.delete(expnum, ccd, 's', ext, prefix='fk') storage.vlink(expnum, ccd, 'p', ext, expnum, ccd, 's', ext, l_prefix='fk') return
def step3(expnums, ccd, version, rate_min, rate_max, angle, width, field=None, prefix=None, dry_run=False, maximum_flux_ratio=3, minimum_area=5, minimum_median_flux=1000.0): """run the actual step3 on the given exp/ccd combo""" jmp_args = ['step3jmp'] matt_args = ['step3jjk'] idx = 0 cmd_args = [] for expnum in expnums: idx += 1 for ext in ['unid.jmp', 'unid.matt']: storage.get_file(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) image = os.path.splitext(os.path.basename(storage.get_uri(expnum, ccd, version=version, prefix=prefix)))[0] cmd_args.append('-f%d' % idx) cmd_args.append(image) cmd_args.extend(['-rn', str(rate_min), '-rx', str(rate_max), '-a', str(angle), '-w', str(width)]) jmp_args.extend(cmd_args) # Add some extra arguemnents for the ISO search. cmd_args.extend(['-fr', str(maximum_flux_ratio), '-ma', str(minimum_area), '-mf', str(minimum_median_flux)]) matt_args.extend(cmd_args) logging.info(util.exec_prog(jmp_args)) logging.info(util.exec_prog(matt_args)) if dry_run: return if field is None: field = str(expnums[0]) storage.mkdir(os.path.dirname(storage.get_uri(field, ccd=ccd, version=version, prefix=prefix))) for ext in ['moving.jmp', 'moving.matt']: uri = storage.get_uri(field, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = '%s%d%s%s.%s' % (prefix, expnums[0], version, str(ccd).zfill(2), ext) storage.copy(filename, uri) return
def step3(expnums, ccd, version, rate_min, rate_max, angle, width, field=None, prefix=None): '''run the actual step2 on the given exp/ccd combo''' jmp_args = ['step3jmp'] matt_args = ['step3matt'] idx = 0 cmd_args = [] for expnum in expnums: idx += 1 for ext in ['unid.jmp', 'unid.matt', 'trans.jmp' ]: filename = storage.get_image(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix ) image = os.path.splitext(os.path.splitext(os.path.basename(filename))[0])[0] cmd_args.append('-f%d' % ( idx)) cmd_args.append(image) cmd_args.extend(['-rn', str(rate_min), '-rx', str(rate_max), '-a', str(angle), '-w', str(width)]) jmp_args.extend(cmd_args) matt_args.extend(cmd_args) util.exec_prog(jmp_args) util.exec_prog(matt_args) if field is None: field = str(expnums[0]) storage.mkdir(os.path.dirname( storage.get_uri(field, ccd=ccd, version=version, ext=ext, prefix=prefix))) for ext in ['moving.jmp', 'moving.matt']: uri = storage.get_uri(field, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = '%s%d%s%s.%s' % ( prefix, expnums[0], version, str(ccd).zfill(2), ext) storage.copy(filename, uri) return
def mkpsf(expnum, ccd, version, dry_run=False, prefix=""): """Run the OSSOS jmpmakepsf script. """ ## confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits')) if not dry_run: storage.mkdir(destdir) ## get image from the vospace storage area logging.info("Getting file from VOSpace") filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) logging.info("Running mkpsf on %s %d" % (expnum, ccd)) ## launch the makepsf script logging.info(util.exec_prog(['jmpmakepsf.csh', './', filename, 'yes', 'yes'])) if dry_run: return ## place the results into VOSpace basename = os.path.splitext(filename)[0] for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext) source = basename + "." + str(ext) storage.copy(source, dest) return
def mk_mopheader(expnum, ccd, version, dry_run=False, prefix=""): """Run the OSSOS mopheader script. """ ## confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits')) if not dry_run: storage.mkdir(destdir) ## get image from the vospace storage area filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) logging.info("Running mopheader on %s %d" % (expnum, ccd)) ## launch the mopheader script ## launch the makepsf script expname = os.path.basename(filename).strip('.fits') logging.info(util.exec_prog(['stepZjmp', '-f', expname])) mopheader_filename = expname+".mopheader" # mopheader_filename = mopheader.main(filename) if dry_run: return destination = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='mopheader') source = mopheader_filename storage.copy(source, destination) return
def plant(expnums, ccd, rmin, rmax, ang, width, version="s"): """run the plant script on this combination of exposures""" ptf = open("proc-these-files", "w") ptf.write("# Files to be planted and search\n") ptf.write("# image fwhm plant\n") for expnum in expnums: fwhm = storage.get_fwhm(expnum, ccd, version=version) filename = storage.get_image(expnum, ccd=ccd, version=version) ptf.write("%s %3.1f YES\n" % (filename[0:-5], fwhm)) for ext in ["apcor", "obj.jmp", "trans.jmp", "psf.fits", "mopheader", "phot", "zeropoint.used"]: apcor = storage.get_image(expnum, ccd=ccd, version=version, ext=ext) ptf.close() cmd_args = ["plant.csh", os.curdir, str(rmin), str(rmax), str(ang), str(width)] util.exec_prog(cmd_args) if args.dryrun: # Don't push back to VOSpace return uri = storage.get_uri("Object", ext="planted", version="", subdir=str(expnums[0]) + "/ccd%s" % (str(ccd).zfill(2))) storage.copy("Object.planted", uri) uri = os.path.join(os.path.dirname(uri), "plant.shifts") storage.copy("shifts", uri) for expnum in expnums: uri = storage.get_uri(expnum, ccd=ccd, version=version, ext="fits", prefix="fk") filename = os.path.basename(uri) storage.copy(filename, uri) for ext in ["mopheader", "psf.fits", "fwhm", "apcor", "zeropoint.used", "trans.jmp"]: storage.delete(expnum, ccd, "s", ext, prefix="fk") storage.vlink(expnum, ccd, "s", ext, expnum, ccd, "s", ext, l_prefix="fk") return
def step2(expnums, ccd, version, prefix=None): '''run the actual step2 on the given exp/ccd combo''' jmp_args = ['step2jmp'] matt_args = ['step2matt_jmp'] idx = 0 for expnum in expnums: jmp_args.append( storage.get_image(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix )[0:-8] ) idx += 1 matt_args.append('-f%d' % ( idx)) matt_args.append( storage.get_image(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix )[0:-9] ) util.exec_prog(jmp_args) util.exec_prog(matt_args) for expnum in expnums: for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: uri = storage.dbimages_uri(expnum,ccd=ccd,version=version,ext=ext, prefix=prefix) filename = os.path.basename(uri) storage.copy(filename, uri) return
def step2(expnums, ccd, version, prefix=None): '''run the actual step2 on the given exp/ccd combo''' jmp_args = ['step2jmp'] matt_args = ['step2matt_jmp'] idx = 0 for expnum in expnums: jmp_args.append( storage.get_image(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]) idx += 1 matt_args.append('-f%d' % (idx)) matt_args.append( storage.get_image(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix)[0:-9]) util.exec_prog(jmp_args) util.exec_prog(matt_args) for expnum in expnums: for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: uri = storage.dbimages_uri(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = os.path.basename(uri) storage.copy(filename, uri) return
def mkpsf(expnum, ccd, version, dry_run=False, prefix=""): """Run the OSSOS jmpmakepsf script. """ ## confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits')) if not dry_run: storage.mkdir(destdir) ## get image from the vospace storage area filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) logging.info("Running mkpsf on %s %d" % (expnum, ccd)) ## launch the makepsf script logging.info(util.exec_prog(['jmpmakepsf.csh', './', filename, 'no'])) if dry_run: return ## place the results into VOSpace basename = os.path.splitext(filename)[0] for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext) source = basename + "." + str(ext) storage.copy(source, dest) return
def run(expnum, ccd, prefix='', version='p', sex_thresh=_SEX_THRESHOLD, wave_thresh=_WAVE_THRESHOLD, maxcount=_MAX_COUNT, dry_run=False, force=True, ignore=False): """run the actual step1jmp/matt codes. expnum: the CFHT expousre to process ccd: which ccd in the mosaic to process fwhm: the image quality, FWHM, of the image. In pixels. sex_thresh: the detection threhold to run sExtractor at wave_thresh: the detection threshold for wavelet maxcount: saturation level """ message = storage.SUCCESS if storage.get_status(task, prefix, expnum, version, ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run): try: if not storage.get_status(dependency, prefix, expnum, version, ccd) and not ignore: raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format( task, dependency, prefix, expnum, version, ccd)) logging.info("Retrieving imaging and input parameters from VOSpace") storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader') filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version, default=3.5) basename = os.path.splitext(filename)[0] _get_weight_map(filename, ccd) logging.info("Launching step1jmp") logging.info(util.exec_prog(['step1jmp', '-f', basename, '-t', str(wave_thresh), '-w', str(fwhm), '-m', str(maxcount)])) logging.info(util.exec_prog(['step1matt', '-f', basename, '-t', str(sex_thresh), '-w', str(fwhm), '-m', str(maxcount)])) if os.access('weight.fits', os.R_OK): os.unlink('weight.fits') if not dry_run: for ext in ['obj.jmp', 'obj.matt']: obj_uri = storage.get_uri(expnum, ccd, version=version, ext=ext, prefix=prefix) obj_filename = basename + "." + ext count = 0 with open(obj_filename, 'r'): while True: try: count += 1 logging.info("Attempt {} to copy {} -> {}".format(count, obj_filename, obj_uri)) storage.copy(obj_filename, obj_uri) break except Exception as ex: if count > 10: raise ex logging.info(message) except Exception as ex: message = str(ex) logging.error(message) if not dry_run: storage.set_status(task, prefix, expnum, version, ccd, status=message)
def step1(expnum, ccd, prefix='', version='p', fwhm=4, sex_thresh=1.3, wave_thresh=2.7, maxcount=30000): """run the actual step1jmp/matt codes. expnum: the CFHT expousre to process ccd: which ccd in the mosaic to process fwhm: the image quality, FWHM, of the image. In pixels. sex_thresh: the detection threhold to run sExtractor at wave_thresh: the detection threshold for wavelet maxcount: saturation level """ filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) mopheader = storage.get_image(expnum, ccd, version=version, ext='mopheader', prefix=prefix) fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version) basename = os.path.splitext(filename)[0] outfile = util.exec_prog([ 'step1jmp', '-f', basename, '-t', str(wave_thresh), '-w', str(fwhm), '-m', str(maxcount) ]) obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.jmp', prefix=prefix) obj_filename = basename + ".obj.jmp" storage.copy(obj_filename, obj_uri) ## for step1matt we need the weight image flat_name = fits.open(filename)[0].header['FLAT'] flat_name = flat_name[0:-5] flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators', rescale=False) if not os.access('weight.fits', os.R_OK): os.symlink(flat_filename, 'weight.fits') outfile = util.exec_prog([ 'step1matt', '-f', basename, '-t', str(sex_thresh), '-w', str(fwhm), '-m', str(maxcount) ]) obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.matt', prefix=prefix) obj_filename = basename + ".obj.matt" storage.copy(obj_filename, obj_uri) return True
def run(expnum, ccd, prefix=None, version='p', field=None, measure3_dir=storage.MEASURE3, dry_run=False, force=False): message = storage.SUCCESS if storage.get_status(task, prefix, expnum, version, ccd) and not force: logging.info( "{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run): try: if not storage.get_status(dependency, prefix, expnum, version, ccd): raise IOError(35, "Cannot start {} as {} not yet completed " "for {}{}{}{:02d}".format(task, dependency, prefix, expnum, version, ccd)) if field is None: field = str(expnum) if prefix is not None and len(prefix) > 0: field = "%s_%s" % (prefix, field) field += "_%s%s" % (str(version), str(ccd)) logging.info("Doing combine on field {}".format(field)) for ext in ['moving.matt', 'moving.jmp']: storage.get_file(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) # Get the list of objects planted into the field if prefix='fk' if prefix == 'fk': storage.get_file('Object', version='', ext='planted', subdir=str(expnum) + "/ccd%s" % ( str(ccd).zfill(2))) else: prefix = '' cmd_args = ['comb-list', prefix + str(expnum) + version + str(ccd).zfill(2)] logging.info(str(cmd_args)) logging.info(util.exec_prog(cmd_args)) # things to copy back to VOSpace, if this is an 'fk' image # then we have missed and found files too. ext_list = ['cands.comb'] if prefix == 'fk': ext_list.extend(['jmp.missed', 'matt.missed', 'jmp.found', 'matt.found', 'comb.missed', 'comb.found']) for ext in ext_list: uri = storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=version, ext=ext) filename = os.path.basename(uri) if not os.access(filename, os.R_OK): logging.critical("No %s file" % filename) continue vospace_name = "%s.%s" % (field, ext) if not dry_run: logging.info("%s -> %s" % ( filename, os.path.join(measure3_dir, vospace_name))) storage.copy(filename, os.path.join(measure3_dir, vospace_name)) base_name = prefix + str(expnum) + version + str(ccd).zfill(2) cands_file = base_name + '.cands.comb' if not os.access(cands_file, os.R_OK): no_cands_file = (prefix + str(expnum) + version + str(ccd).zfill(2) + '.no_candidates') open(no_cands_file, 'w').close() if not dry_run: vospace_name = "%s.no_candidates" % field storage.copy(no_cands_file, os.path.join(measure3_dir, vospace_name)) else: measure3.run(base_name, storage.DBIMAGES) if not dry_run: filename = base_name + ".measure3.cands.astrom" vospace_filename = "%s.measure3.cands.astrom" % field storage.copy(filename, os.path.join(measure3_dir, vospace_filename)) except Exception as ex: message = str(ex) logging.error(message) if not dry_run: storage.set_status(task, prefix, expnum, version, ccd, status=message)
def step3(expnums, ccd, version, rate_min, rate_max, angle, width, field=None, prefix=None): '''run the actual step2 on the given exp/ccd combo''' jmp_args = ['step3jmp'] matt_args = ['step3matt'] idx = 0 cmd_args = [] for expnum in expnums: idx += 1 for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: filename = storage.get_image(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) image = os.path.splitext( os.path.splitext(os.path.basename(filename))[0])[0] cmd_args.append('-f%d' % (idx)) cmd_args.append(image) cmd_args.extend([ '-rn', str(rate_min), '-rx', str(rate_max), '-a', str(angle), '-w', str(width) ]) jmp_args.extend(cmd_args) matt_args.extend(cmd_args) util.exec_prog(jmp_args) util.exec_prog(matt_args) if field is None: field = str(expnums[0]) storage.mkdir( os.path.dirname( storage.get_uri(field, ccd=ccd, version=version, ext=ext, prefix=prefix))) for ext in ['moving.jmp', 'moving.matt']: uri = storage.get_uri(field, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = '%s%d%s%s.%s' % (prefix, expnums[0], version, str(ccd).zfill(2), ext) storage.copy(filename, uri) return
def combine(expnum, ccd, prefix=None, type='p', field=None, measure3=MEASURE3 ): if field is None: field=str(expnum) if prefix is not None and len(prefix) > 0: field = "%s_%s" % ( prefix, field ) field += "_%s" % ( str(ccd)) for ext in ['moving.matt','moving.jmp']: fname = storage.get_image(expnum, ccd=ccd, prefix=prefix, version=type, ext=ext) if prefix is not None and len(prefix) > 0: planted = storage.get_image('Object', subdir=str(expnum)+"/ccd%s" % ( str(ccd).zfill(2)), version='', ext='planted') else: prefix = '' base_image = os.path.basename( storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=type, ext=None)) cmd_args = ['comb-list', prefix+str(expnum)+type+str(ccd).zfill(2)] util.exec_prog(cmd_args) ext_list = ['cands.comb'] if prefix is not None and len(prefix) > 0 : ext_list.extend( [ 'jmp.missed', 'matt.missed', 'jmp.found', 'matt.found', 'comb.missed', 'comb.found' ] ) for ext in ext_list: uri = storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=type, ext=ext) filename = os.path.basename(uri) if not os.access(filename,os.R_OK): logging.critical("No %s file" % (filename)) continue vospace_name = "%s.%s" % ( field, ext ) logging.info("%s -> %s" % ( filename, os.path.join(measure3, vospace_name))) storage.copy(filename, os.path.join(measure3, vospace_name)) base_name = prefix+str(expnum)+type+str(ccd).zfill(2) cands_file = base_name+'.cands.comb' if not os.access(cands_file,os.R_OK): nocands_file = ( prefix+ str(expnum)+ type+ str(ccd).zfill(2)+ '.no_candidates' ) open(nocands_file, 'w').close() vospace_name = "%s.no_candidates" % ( field ) storage.copy(nocands_file,os.path.join(measure3, vospace_name)) return storage.SUCCESS # get the images we need to compute x/y ra/dec transforms cands_file = mop_file.Parser().parse(cands_file) for file_id in cands_file.header.file_ids: rec_no=cands_file.header.file_ids.index(file_id) storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no], ccd=ccd, version=type, prefix=prefix, ext='fits') cmd_args = ['measure3', prefix+str(expnum)+type+str(ccd).zfill(2)] logging.info("Running measure3") util.exec_prog(cmd_args) filename=base_name+".measure3.cands.astrom" vospace_filename = "%s.measure3.cands.astrom" % ( field) storage.copy(filename, os.path.join(measure3,vospace_filename)) return storage.SUCCESS
def plant(expnums, ccd, rmin, rmax, ang, width, version='s'): '''run the plant script on this combination of exposures''' ptf = open('proc-these-files', 'w') ptf.write("# Files to be planted and search\n") ptf.write("# image fwhm plant\n") for expnum in expnums: fwhm = storage.get_fwhm(expnum, ccd) filename = storage.get_image(expnum, ccd=ccd, version=version) ptf.write("%s %3.1f YES\n" % (filename[0:-5], fwhm)) for ext in [ 'apcor', 'obj.jmp', 'trans.jmp', 'psf.fits', 'mopheader', 'phot', 'zeropoint.used' ]: apcor = storage.get_image(expnum, ccd=ccd, version='s', ext=ext) ptf.close() cmd_args = [ 'plant.csh', os.curdir, str(rmin), str(rmax), str(ang), str(width) ] util.exec_prog(cmd_args) if args.dryrun: # Don't push back to VOSpace return uri = storage.get_uri('Object', ext='planted', version='', subdir=str(expnums[0]) + "/ccd%s" % (str(ccd).zfill(2))) storage.copy('Object.planted', uri) uri = os.path.join(os.path.dirname(uri), 'shifts') storage.copy('shifts', uri) for expnum in expnums: uri = storage.get_uri(expnum, ccd=ccd, version=version, ext='fits', prefix='fk') filename = os.path.basename(uri) storage.copy(filename, uri) for ext in [ 'mopheader', 'psf.fits', 'fwhm', 'apcor', 'zeropoint.used', 'trans.jmp' ]: storage.delete(expnum, ccd, 's', ext, prefix='fk') storage.vlink(expnum, ccd, 'p', ext, expnum, ccd, 's', ext, l_prefix='fk') return
def run(expnums, ccd, version, prefix=None, dry_run=False, default="WCS", force=False): """run the actual step2 on the given exp/ccd combo""" jmp_trans = ['step2ajmp'] jmp_args = ['step2bjmp'] matt_args = ['step2matt_jmp'] if storage.get_status(task, prefix, expnums[0], version, ccd) and not force: logging.info("{} completed successfully for {}{}{}{:02d}".format( task, prefix, expnums[0], version, ccd)) return with storage.LoggingManager(task, prefix, expnums[0], ccd, version, dry_run): try: for expnum in expnums: if not storage.get_status( dependency, prefix, expnum, version=version, ccd=ccd): raise IOError( 35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}" .format(task, dependency, prefix, expnum, version, ccd)) message = storage.SUCCESS idx = 0 logging.info("Retrieving catalog files to do matching.") for expnum in expnums: jmp_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]) jmp_trans.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]) idx += 1 matt_args.append('-f%d' % idx) matt_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix)[0:-9]) logging.info( "Computing the catalog alignment using sources in catalogs.") try: logging.info(util.exec_prog(jmp_trans)) if default == "WCS": logging.info("Comparing computed transform to WCS values") logging.info( compute_trans(expnums, ccd, version, prefix, default=default)) except Exception as ex: logging.info("JMP Trans failed: {}".format(ex)) logging.info( compute_trans(expnums, ccd, version, prefix, default="WCS")) logging.info("Using transform to match catalogs for three images.") logging.info(util.exec_prog(jmp_args)) logging.info(util.exec_prog(matt_args)) # check that the shifts from step2 are rational by matching the bright star lists. logging.info( "Uisng checktrans to ensure that transforms were reasonable.") check_args = ['checktrans'] if os.access('proc-these-files', os.R_OK): os.unlink('proc-these-files') ptf = open('proc-these-files', 'w') ptf.write( "# A dummy file that is created so checktrans could run.\n") ptf.write("# Frame FWHM PSF?\n") for expnum in expnums: filename = os.path.splitext( storage.get_image(expnum, ccd, version=version, prefix=prefix))[0] if not os.access(filename + ".bright.psf", os.R_OK): os.link(filename + ".bright.jmp", filename + ".bright.psf") if not os.access(filename + ".obj.psf", os.R_OK): os.link(filename + ".obj.jmp", filename + ".obj.psf") ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format( filename, _FWHM, "NO")) ptf.close() if os.access('BAD_TRANS', os.F_OK): os.unlink('BAD_TRANS') logging.info(util.exec_prog(check_args)) if os.access('BAD_TRANS', os.F_OK): raise OSError(errno.EBADMSG, 'BAD_TRANS') if os.access('proc-these-files', os.F_OK): os.unlink('proc-these-files') if dry_run: return for expnum in expnums: for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: uri = storage.dbimages_uri(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = os.path.basename(uri) storage.copy(filename, uri) except Exception as ex: message = str(ex) logging.error(message) storage.set_status(task, prefix, expnums[0], version, ccd, status=message) return
def run(expnum, ccd, prefix='', version='p', sex_thresh=_SEX_THRESHOLD, wave_thresh=_WAVE_THRESHOLD, maxcount=_MAX_COUNT, dry_run=False, force=True): """run the actual step1jmp/matt codes. expnum: the CFHT expousre to process ccd: which ccd in the mosaic to process fwhm: the image quality, FWHM, of the image. In pixels. sex_thresh: the detection threhold to run sExtractor at wave_thresh: the detection threshold for wavelet maxcount: saturation level """ message = storage.SUCCESS if storage.get_status(task, prefix, expnum, version, ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run): try: if not storage.get_status(dependency, prefix, expnum, version, ccd): raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format( task, dependency, prefix, expnum, version, ccd)) logging.info("Retrieving imaging and input parameters from VOSpace") storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader') filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version) basename = os.path.splitext(filename)[0] _get_weight_map(filename, ccd) logging.info("Launching step1jmp") logging.info(util.exec_prog(['step1jmp', '-f', basename, '-t', str(wave_thresh), '-w', str(fwhm), '-m', str(maxcount)])) logging.info(util.exec_prog(['step1matt', '-f', basename, '-t', str(sex_thresh), '-w', str(fwhm), '-m', str(maxcount)])) if os.access('weight.fits', os.R_OK): os.unlink('weight.fits') if not dry_run: for ext in ['obj.jmp', 'obj.matt']: obj_uri = storage.get_uri(expnum, ccd, version=version, ext=ext, prefix=prefix) obj_filename = basename + "." + ext count = 0 with open(obj_filename, 'r'): while True: try: count += 1 logging.info("Attempt {} to copy {} -> {}".format(count, obj_filename, obj_uri)) storage.copy(obj_filename, obj_uri) break except Exception as ex: if count > 10: raise ex logging.info(message) except Exception as ex: message = str(ex) logging.error(message) if not dry_run: storage.set_status(task, prefix, expnum, version, ccd, status=message)
def step1(expnum, ccd, prefix='', version='p', sex_thresh=_SEX_THRESHOLD, wave_thresh=_WAVE_THRESHOLD, maxcount=_MAX_COUNT, dry_run=False): """run the actual step1jmp/matt codes. expnum: the CFHT expousre to process ccd: which ccd in the mosaic to process fwhm: the image quality, FWHM, of the image. In pixels. sex_thresh: the detection threhold to run sExtractor at wave_thresh: the detection threshold for wavelet maxcount: saturation level """ storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader') filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version) basename = os.path.splitext(filename)[0] logging.info(util.exec_prog(['step1jmp', '-f', basename, '-t', str(wave_thresh), '-w', str(fwhm), '-m', str(maxcount)])) obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.jmp', prefix=prefix) obj_filename = basename + ".obj.jmp" if not dry_run: storage.copy(obj_filename, obj_uri) ## for step1matt we need the weight image hdulist = fits.open(filename) flat_name = hdulist[0].header.get('FLAT', 'weight.fits') parts = os.path.splitext(flat_name) if parts[1] == '.fz': flat_name = os.path.splitext(parts[0])[0] else: flat_name = parts[0] try: flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators') except: flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='old_calibrators') if os.access('weight.fits', os.R_OK): os.unlink('weight.fits') if not os.access('weight.fits', os.R_OK): os.symlink(flat_filename, 'weight.fits') logging.info(util.exec_prog(['step1matt', '-f', basename, '-t', str(sex_thresh), '-w', str(fwhm), '-m', str(maxcount)])) if os.access('weight.fits', os.R_OK): os.unlink('weight.fits') obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.matt', prefix=prefix) obj_filename = basename + ".obj.matt" if not dry_run: storage.copy(obj_filename, obj_uri) return True
def run(expnum, ccd, version, dry_run=False, prefix="", force=False): """Run the OSSOS jmpmakepsf script. """ message = storage.SUCCESS if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run): try: if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd): raise IOError("{} not yet run for {}".format(dependency, expnum)) # confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits')) if not dry_run: storage.mkdir(destdir) # get image from the vospace storage area logging.info("Getting fits image from VOSpace") filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) # get mopheader from the vospace storage area logging.info("Getting mopheader from VOSpace") mopheader_filename = storage.get_file(expnum, ccd, version=version, prefix=prefix, ext='mopheader') # run mkpsf process logging.info("Running mkpsf on %s %d" % (expnum, ccd)) logging.info(util.exec_prog(['jmpmakepsf.csh', './', filename, 'yes', 'yes'])) if dry_run: return # place the results into VOSpace basename = os.path.splitext(filename)[0] for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext) source = basename + "." + str(ext) count = 0 with open(source, 'r'): while True: count += 1 try: logging.info("Attempt {} to copy {} -> {}".format(count, source, dest)) storage.copy(source, dest) break except Exception as ex: if count > 10: raise ex # set some data parameters associated with the image, determined in this step. storage.set_status('fwhm', prefix, expnum, version=version, ccd=ccd, status=str(storage.get_fwhm( expnum, ccd=ccd, prefix=prefix, version=version))) storage.set_status('zeropoint', prefix, expnum, version=version, ccd=ccd, status=str(storage.get_zeropoint( expnum, ccd=ccd, prefix=prefix, version=version))) logging.info(message) except Exception as e: message = str(e) logging.error(message) storage.set_status(task, prefix, expnum, version, ccd=ccd, status=message) return
def step2(expnums, ccd, version, prefix=None, dry_run=False, default="WCS"): """run the actual step2 on the given exp/ccd combo""" jmp_trans = ['step2ajmp'] jmp_args = ['step2bjmp'] matt_args = ['step2matt_jmp'] idx = 0 for expnum in expnums: jmp_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8] ) jmp_trans.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8] ) idx += 1 matt_args.append('-f%d' % idx) matt_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix)[0:-9] ) logging.info(util.exec_prog(jmp_trans)) if default == "WCS": logging.info(compute_trans(expnums, ccd, version, prefix, default=default)) logging.info(util.exec_prog(jmp_args)) logging.info(util.exec_prog(matt_args)) ## check that the shifts from step2 are rational check_args = ['checktrans'] if os.access('proc-these-files', os.R_OK): os.unlink('proc-these-files') ptf = open('proc-these-files', 'w') ptf.write("# A dummy file that is created so checktrans could run.\n") ptf.write("# Frame FWHM PSF?\n") for expnum in expnums: filename = os.path.splitext(storage.get_image(expnum, ccd, version=version, prefix=prefix))[0] if not os.access(filename + ".bright.psf", os.R_OK): os.link(filename + ".bright.jmp", filename + ".bright.psf") if not os.access(filename + ".obj.psf", os.R_OK): os.link(filename + ".obj.jmp", filename + ".obj.psf") ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(filename, _FWHM, "NO")) ptf.close() if os.access('BAD_TRANS', os.F_OK): os.unlink('BAD_TRANS') logging.info(util.exec_prog(check_args)) if os.access('BAD_TRANS', os.F_OK): raise OSError(errno.EBADMSG, 'BAD_TRANS') if os.access('proc-these-files', os.F_OK): os.unlink('proc-these-files') if dry_run: return for expnum in expnums: for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: uri = storage.dbimages_uri(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = os.path.basename(uri) storage.copy(filename, uri) return
def run(expnums, ccd, version, rate_min, rate_max, angle, width, field=None, prefix=None, dry_run=False, force=False): """run the actual step2 on the given exp/ccd combo""" jmp_args = ['step3jmp'] matt_args = ['step3matt'] if storage.get_status(task, prefix, expnums[0], version=version, ccd=ccd) and not force: logging.info("{} completed successfully for {}{}{}{:02d}".format( task, prefix, expnums[0], version, ccd)) return with storage.LoggingManager(task, prefix, expnums[0], ccd, version, dry_run): try: if not storage.get_status( dependency, prefix, expnums[0], version=version, ccd=ccd): raise IOError( 35, "Cannot start {} as {} not yet completed {}{}{}{:02d}". format(task, dependency, prefix, expnums[0], version, ccd)) # Default message is success, message gets overwritten with failure messages. message = storage.SUCCESS idx = 0 cmd_args = [] for expnum in expnums: idx += 1 for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: storage.get_file(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) image = os.path.splitext( os.path.basename( storage.get_uri(expnum, ccd, version=version, prefix=prefix)))[0] cmd_args.append('-f%d' % idx) cmd_args.append(image) cmd_args.extend([ '-rn', str(rate_min), '-rx', str(rate_max), '-a', str(angle), '-w', str(width) ]) jmp_args.extend(cmd_args) matt_args.extend(cmd_args) logging.info(util.exec_prog(jmp_args)) logging.info(util.exec_prog(matt_args)) if dry_run: return if field is None: field = str(expnums[0]) # Make sure a dbimages destination exists for this file. storage.mkdir( os.path.dirname( storage.get_uri(field, ccd=ccd, version=version, prefix=prefix))) for ext in ['moving.jmp', 'moving.matt']: uri = storage.get_uri(field, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = '%s%d%s%s.%s' % (prefix, expnums[0], version, str(ccd).zfill(2), ext) storage.copy(filename, uri) except Exception as ex: message = str(ex) logging.error(message) storage.set_status(task, prefix, expnums[0], version=version, ccd=ccd, status=message) return
def run(expnum, ccd, version, dry_run=False, prefix="", force=False): """Run the OSSOS jmpmakepsf script. """ message = storage.SUCCESS if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format( task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run): try: if not storage.get_status( dependency, prefix, expnum, version, ccd=ccd): raise IOError("{} not yet run for {}".format( dependency, expnum)) # confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits')) if not dry_run: storage.mkdir(destdir) # get image from the vospace storage area logging.info("Getting fits image from VOSpace") filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) # get mopheader from the vospace storage area logging.info("Getting mopheader from VOSpace") mopheader_filename = storage.get_file(expnum, ccd, version=version, prefix=prefix, ext='mopheader') # run mkpsf process logging.info("Running mkpsf on %s %d" % (expnum, ccd)) logging.info( util.exec_prog( ['jmpmakepsf.csh', './', filename, 'yes', 'yes'])) if dry_run: return # place the results into VOSpace basename = os.path.splitext(filename)[0] for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext) source = basename + "." + str(ext) count = 0 with open(source, 'r'): while True: count += 1 try: logging.info("Attempt {} to copy {} -> {}".format( count, source, dest)) storage.copy(source, dest) break except Exception as ex: if count > 10: raise ex # set some data parameters associated with the image, determined in this step. storage.set_status('fwhm', prefix, expnum, version=version, ccd=ccd, status=str( storage.get_fwhm(expnum, ccd=ccd, prefix=prefix, version=version))) storage.set_status('zeropoint', prefix, expnum, version=version, ccd=ccd, status=str( storage.get_zeropoint(expnum, ccd=ccd, prefix=prefix, version=version))) logging.info(message) except Exception as e: message = str(e) logging.error(message) storage.set_status(task, prefix, expnum, version, ccd=ccd, status=message) return
def combine(expnum, ccd, prefix=None, type='p'): for ext in ['moving.matt', 'moving.jmp']: fname = storage.get_image(expnum, ccd=ccd, prefix=prefix, version=type, ext=ext) if prefix is not None and len(prefix) > 0: planted = storage.get_image('Object', subdir=str(expnum) + "/ccd%s" % (str(ccd).zfill(2)), version='', ext='planted') else: prefix = '' base_image = os.path.basename( storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=type, ext=None)) cmd_args = ['comb-list', prefix + str(expnum) + type + str(ccd).zfill(2)] util.exec_prog(cmd_args) for ext in [ 'cands.comb', 'comb.found', 'comb.missed', 'jmp.found', 'jmp.missed', 'matt.found', 'matt.missed' ]: uri = storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=type, ext=ext) filename = os.path.basename(uri) if not os.access(filename, os.R_OK): logging.critical("No %s file" % (filename)) continue storage.copy(filename, uri) base_name = prefix + str(expnum) + type + str(ccd).zfill(2) cands_file = base_name + '.cands.comb' if not os.access(cands_file, os.R_OK): nocands_file = (prefix + str(expnum) + type + str(ccd).zfill(2) + '.no_candidates') open(nocands_file, 'w').close() storage.copy(nocands_file, 'vos:OSSOS/measure3/' + nocands_file) return storage.SUCCESS cands_file = mop_file.Parser().parse(cands_file) for file_id in cands_file.header.file_ids: rec_no = cands_file.header.file_ids.index(file_id) storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no], ccd=ccd, version=type, prefix=prefix, ext='fits') cmd_args = ['measure3', prefix + str(expnum) + type + str(ccd).zfill(2)] util.exec_prog(cmd_args) filename = base_name + ".measure3.cands.astrom" storage.copy(filename, 'vos:OSSOS/measure3/' + filename) return storage.SUCCESS
def combine(expnum, ccd, prefix=None, file_type='p', field=None, measure3=MEASURE3, dry_run=False): if field is None: field = str(expnum) if prefix is not None and len(prefix) > 0: field = "%s_%s" % (prefix, field) field += "_%s%s" % (str(file_type), str(ccd)) logging.info("Doing combine on field {}".format(field)) for ext in ['moving.matt', 'moving.jmp']: storage.get_file(expnum, ccd=ccd, version=file_type, ext=ext, prefix=prefix) if prefix is not None and len(prefix) > 0: storage.get_file('Object', version='', ext='planted', subdir=str(expnum) + "/ccd%s" % (str(ccd).zfill(2))) else: prefix = '' cmd_args = [ 'comb-list', prefix + str(expnum) + file_type + str(ccd).zfill(2) ] logging.info(str(cmd_args)) logging.info(util.exec_prog(cmd_args)) ext_list = ['cands.comb'] if prefix is not None and len(prefix) > 0: ext_list.extend([ 'jmp.missed', 'matt.missed', 'jmp.found', 'matt.found', 'comb.missed', 'comb.found' ]) for ext in ext_list: uri = storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=file_type, ext=ext) filename = os.path.basename(uri) if not os.access(filename, os.R_OK): logging.critical("No %s file" % filename) continue vospace_name = "%s.%s" % (field, ext) if not dry_run: logging.info("%s -> %s" % (filename, os.path.join(measure3, vospace_name))) storage.copy(filename, os.path.join(measure3, vospace_name)) base_name = prefix + str(expnum) + file_type + str(ccd).zfill(2) cands_file = base_name + '.cands.comb' if not os.access(cands_file, os.R_OK): no_cands_file = (prefix + str(expnum) + file_type + str(ccd).zfill(2) + '.no_candidates') open(no_cands_file, 'w').close() if not dry_run: vospace_name = "%s.no_candidates" % field storage.copy(no_cands_file, os.path.join(measure3, vospace_name)) return storage.SUCCESS # get the images we need to compute x/y ra/dec transforms cands_file = mop_file.Parser().parse(cands_file) for file_id in cands_file.header.file_ids: rec_no = cands_file.header.file_ids.index(file_id) storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no], ccd=ccd, version=file_type, ext='fits', prefix=prefix) cmd_args = [ 'measure3', prefix + str(expnum) + file_type + str(ccd).zfill(2) ] logging.info("Running measure3") logging.info(util.exec_prog(cmd_args)) if not dry_run: filename = base_name + ".measure3.cands.astrom" vospace_filename = "%s.measure3.cands.astrom" % field storage.copy(filename, os.path.join(measure3, vospace_filename)) return storage.SUCCESS
def step2(expnums, ccd, version, prefix=None, dry_run=False, default="WCS"): """run the actual step2 on the given exp/ccd combo""" jmp_trans = ['step2ajmp'] jmp_args = ['step2bjmp'] matt_args = ['step2matt_jmp'] idx = 0 for expnum in expnums: jmp_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]) jmp_trans.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]) idx += 1 matt_args.append('-f%d' % idx) matt_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix)[0:-9]) logging.info(util.exec_prog(jmp_trans)) if default == "WCS": logging.info( compute_trans(expnums, ccd, version, prefix, default=default)) logging.info(util.exec_prog(jmp_args)) logging.info(util.exec_prog(matt_args)) ## check that the shifts from step2 are rational check_args = ['checktrans'] if os.access('proc-these-files', os.R_OK): os.unlink('proc-these-files') ptf = open('proc-these-files', 'w') ptf.write("# A dummy file that is created so checktrans could run.\n") ptf.write("# Frame FWHM PSF?\n") for expnum in expnums: filename = os.path.splitext( storage.get_image(expnum, ccd, version=version, prefix=prefix))[0] if not os.access(filename + ".bright.psf", os.R_OK): os.link(filename + ".bright.jmp", filename + ".bright.psf") if not os.access(filename + ".obj.psf", os.R_OK): os.link(filename + ".obj.jmp", filename + ".obj.psf") ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(filename, _FWHM, "NO")) ptf.close() if os.access('BAD_TRANS', os.F_OK): os.unlink('BAD_TRANS') logging.info(util.exec_prog(check_args)) if os.access('BAD_TRANS', os.F_OK): raise ValueError(errno.EBADEXEC, 'BAD_TRANS') if os.access('proc-these-files', os.F_OK): os.unlink('proc-these-files') if dry_run: return for expnum in expnums: for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: uri = storage.dbimages_uri(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = os.path.basename(uri) storage.copy(filename, uri) return