def step1(expnum, ccd, prefix='', version='p', fwhm=4, sex_thresh=1.3, wave_thresh=2.7, maxcount=30000): """run the actual step1jmp/matt codes. expnum: the CFHT expousre to process ccd: which ccd in the mosaic to process fwhm: the image quality, FWHM, of the image. In pixels. sex_thresh: the detection threhold to run sExtractor at wave_thresh: the detection threshold for wavelet maxcount: saturation level """ filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) mopheader = storage.get_image(expnum, ccd, version=version, ext='mopheader', prefix=prefix) fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version) basename = os.path.splitext(filename)[0] outfile = util.exec_prog(['step1jmp', '-f', basename, '-t', str(wave_thresh), '-w', str(fwhm), '-m', str(maxcount)]) obj_uri = storage.get_uri(expnum,ccd,version=version,ext='obj.jmp', prefix=prefix) obj_filename = basename+".obj.jmp" storage.copy(obj_filename,obj_uri) ## for step1matt we need the weight image hdulist = fits.open(filename) flat_name = hdulist[0].header.get('FLAT','06Bm02.flat.r.36.01.fits') flat_name = flat_name[0:-5] flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators', rescale=False) if not os.access('weight.fits',os.R_OK): os.symlink(flat_filename, 'weight.fits') outfile = util.exec_prog(['step1matt', '-f', basename, '-t', str(sex_thresh), '-w', str(fwhm), '-m', str(maxcount)]) obj_uri = storage.get_uri(expnum,ccd,version=version,ext='obj.matt', prefix=prefix) obj_filename = basename+".obj.matt" storage.copy(obj_filename,obj_uri) return True
def plant(expnums, ccd, rmin, rmax, ang, width, version='s'): '''run the plant script on this combination of exposures''' ptf = open('proc-these-files','w') ptf.write("# Files to be planted and search\n") ptf.write("# image fwhm plant\n") for expnum in expnums: fwhm = storage.get_fwhm(expnum,ccd) filename = storage.get_image(expnum, ccd=ccd, version=version) ptf.write("%s %3.1f YES\n" % ( filename[0:-5], fwhm )) for ext in ['apcor', 'obj.jmp', 'trans.jmp', 'psf.fits', 'mopheader', 'phot', 'zeropoint.used']: apcor = storage.get_image(expnum, ccd=ccd, version='s', ext=ext) ptf.close() cmd_args = ['plant.csh',os.curdir, str(rmin), str(rmax), str(ang), str(width)] util.exec_prog(cmd_args) if args.dryrun: # Don't push back to VOSpace return uri = storage.get_uri('Object',ext='planted',version='', subdir=str( expnums[0])+"/ccd%s" % (str(ccd).zfill(2))) storage.copy('Object.planted',uri) uri = os.path.join(os.path.dirname(uri), 'shifts') storage.copy('shifts', uri) for expnum in expnums: uri = storage.get_uri(expnum, ccd=ccd, version=version, ext='fits', prefix='fk') filename = os.path.basename(uri) storage.copy(filename, uri) for ext in ['mopheader', 'psf.fits', 'fwhm', 'apcor', 'zeropoint.used', 'trans.jmp']: storage.delete(expnum, ccd, 's', ext, prefix='fk') storage.vlink(expnum, ccd, 'p', ext, expnum, ccd, 's', ext, l_prefix='fk') return
def plant(expnums, ccd, rmin, rmax, ang, width, number=10, mmin=21.0, mmax=25.5, version='s', dry_run=False): """Plant artificial sources into the list of images provided. :param expnums: list of MegaPrime exposure numbers to add artificial KBOs to :param ccd: which ccd to work on. :param rmin: The minimum rate of motion to add sources at (''/hour) :param rmax: The maximum rate of motion to add sources at (''/hour) :param ang: The mean angle of motion to add sources :param width: The +/- range of angles of motion :param version: Add sources to the 'o', 'p' or 's' images :param dry_run: don't push results to VOSpace. """ # Construct a list of artificial KBOs with positions in the image # and rates of motion within the bounds given by the caller. filename = storage.get_image(expnums[0], ccd=ccd, version=version) header = fits.open(filename)[0].header bounds = util.get_pixel_bounds_from_datasec_keyword(header.get('DATASEC', '[33:2080,1:4612]')) # generate a set of artificial KBOs to add to the image. kbos = KBOGenerator.get_kbos(n=number, rate=(rmin, rmax), angle=(ang - width, ang + width), mag=(mmin, mmax), x=(bounds[0][0], bounds[0][1]), y=(bounds[1][0], bounds[1][1]), filename='Object.planted') for expnum in expnums: filename = storage.get_image(expnum, ccd, version) psf = storage.get_file(expnum, ccd, version, ext='psf.fits') plant_kbos(filename, psf, kbos, get_shifts(expnum, ccd, version), "fk") if dry_run: return uri = storage.get_uri('Object', ext='planted', version='', subdir=str( expnums[0]) + "/ccd%s" % (str(ccd).zfill(2))) storage.copy('Object.planted', uri) for expnum in expnums: uri = storage.get_uri(expnum, ccd=ccd, version=version, ext='fits', prefix='fk') filename = os.path.basename(uri) storage.copy(filename, uri) return
def mkpsf(expnum, ccd, version, dry_run=False, prefix=""): """Run the OSSOS jmpmakepsf script. """ ## confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits')) if not dry_run: storage.mkdir(destdir) ## get image from the vospace storage area logging.info("Getting file from VOSpace") filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) logging.info("Running mkpsf on %s %d" % (expnum, ccd)) ## launch the makepsf script logging.info(util.exec_prog(['jmpmakepsf.csh', './', filename, 'yes', 'yes'])) if dry_run: return ## place the results into VOSpace basename = os.path.splitext(filename)[0] for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext) source = basename + "." + str(ext) storage.copy(source, dest) return
def mkpsf(expnum, ccd, fversion): """Run the OSSOS makepsf script. """ ## get image from the vospace storage area filename = storage.get_image(expnum, ccd, version=fversion) logging.info("Running mkpsf on %s %d" % (expnum, ccd)) ## launch the makepsf script util.exec_prog(['jmpmakepsf.csh', './', filename, 'no']) ## place the results into VOSpace basename = os.path.splitext(filename)[0] ## confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, version=fversion,ext='fits')) logging.info("Checking that destination directories exist") storage.mkdir(destdir) for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, version=fversion, ext=ext) source = basename + "." + ext logging.info("Copying %s -> %s" % ( source, dest)) storage.remove(dest) storage.copy(source, dest) return
def mk_mopheader(expnum, ccd, version, dry_run=False, prefix=""): """Run the OSSOS mopheader script. """ ## confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits')) if not dry_run: storage.mkdir(destdir) ## get image from the vospace storage area filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) logging.info("Running mopheader on %s %d" % (expnum, ccd)) ## launch the mopheader script ## launch the makepsf script expname = os.path.basename(filename).strip('.fits') logging.info(util.exec_prog(['stepZjmp', '-f', expname])) mopheader_filename = expname+".mopheader" # mopheader_filename = mopheader.main(filename) if dry_run: return destination = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='mopheader') source = mopheader_filename storage.copy(source, destination) return
def scramble(expnums, ccd, version='p', dry_run=False): """run the plant script on this combination of exposures""" mjds = [] fobjs = [] for expnum in expnums: filename = storage.get_image(expnum, ccd=ccd, version=version) fobjs.append(fits.open(filename)) # Pull out values to replace in headers.. must pull them # as otherwise we get pointers... mjds.append(fobjs[-1][0].header['MJD-OBS']) order = [0, 2, 1] for idx in range(len(fobjs)): logging.info("Flipping %d to %d" % (fobjs[idx][0].header['EXPNUM'], expnums[order[idx]])) fobjs[idx][0].header['EXPNUM'] = expnums[order[idx]] fobjs[idx][0].header['MJD-OBS'] = mjds[order[idx]] uri = storage.get_uri(expnums[order[idx]], ccd=ccd, version='s', ext='fits') fname = os.path.basename(uri) if os.access(fname, os.F_OK): os.unlink(fname) fobjs[idx].writeto(fname) if dry_run: continue storage.copy(fname, uri) return
def mkpsf(expnum, ccd): """Run the OSSOS makepsf script. """ ## get image from the vospace storage area filename = storage.get_image(expnum, ccd, version='p') logging.info("Running mkpsf on %s %d" % (expnum, ccd)) ## launch the makepsf script util.exec_prog(['jmpmakepsf.csh', './', filename, 'no']) ## place the results into VOSpace basename = os.path.splitext(filename)[0] ## confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, version='p',ext='fits')) logging.info("Checking that destination direcoties exist") storage.mkdir(destdir) for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, version='p', ext=ext) source = basename + "." + ext storage.copy(source, dest) return
def compute_trans(expnums, ccd, version, prefix=None, default="WCS"): """ Pull the astrometric header for each image, compute an x/y transform and compare to trans.jmp this one overides trans.jmp if they are very different. @param expnums: @param ccd: @param version: @param prefix: @return: None """ wcs_dict = {} for expnum in expnums: try: # TODO This assumes that the image is already N/E flipped. # If compute_trans is called after the image is retrieved from archive then we get the disk version. filename = storage.get_image(expnum, ccd, version, prefix=prefix) this_wcs = wcs.WCS(fits.open(filename)[0].header) except Exception as err: logging.warning("WCS Trans compute failed. {}".format(str(err))) return wcs_dict[expnum] = this_wcs x0 = wcs_dict[expnums[0]].header['NAXIS1'] / 2.0 y0 = wcs_dict[expnums[0]].header['NAXIS2'] / 2.0 (ra0, dec0) = wcs_dict[expnums[0]].xy2sky(x0, y0) result = "" for expnum in expnums: filename = storage.get_file(expnum, ccd, version, ext='.trans.jmp', prefix=prefix) jmp_trans = file(filename, 'r').readline().split() (x, y) = wcs_dict[expnum].sky2xy(ra0, dec0) x1 = float( jmp_trans[0]) + float(jmp_trans[1]) * x + float(jmp_trans[2]) * y y1 = float( jmp_trans[3]) + float(jmp_trans[4]) * x + float(jmp_trans[5]) * y dr = math.sqrt((x1 - x0)**2 + (y1 - y0)**2) if dr > 0.5: result += "WARNING: WCS-JMP transforms mis-matched {} reverting to using {}.\n".format( expnum, default) if default == "WCS": uri = storage.dbimages_uri(expnum, ccd, version, ext='.trans.jmp', prefix=prefix) filename = os.path.basename(uri) trans = file(filename, 'w') trans.write("{:5.2f} 1. 0. {:5.2f} 0. 1.\n".format( x0 - x, y0 - y)) trans.close() else: result += "WCS-JMP transforms match {}\n".format(expnum) return result
def step3(expnums, ccd, version, rate_min, rate_max, angle, width, field=None, prefix=None): '''run the actual step2 on the given exp/ccd combo''' jmp_args = ['step3jmp'] matt_args = ['step3matt'] idx = 0 cmd_args = [] for expnum in expnums: idx += 1 for ext in ['unid.jmp', 'unid.matt', 'trans.jmp' ]: filename = storage.get_image(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix ) image = os.path.splitext(os.path.splitext(os.path.basename(filename))[0])[0] cmd_args.append('-f%d' % ( idx)) cmd_args.append(image) cmd_args.extend(['-rn', str(rate_min), '-rx', str(rate_max), '-a', str(angle), '-w', str(width)]) jmp_args.extend(cmd_args) matt_args.extend(cmd_args) util.exec_prog(jmp_args) util.exec_prog(matt_args) if field is None: field = str(expnums[0]) storage.mkdir(os.path.dirname( storage.get_uri(field, ccd=ccd, version=version, ext=ext, prefix=prefix))) for ext in ['moving.jmp', 'moving.matt']: uri = storage.get_uri(field, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = '%s%d%s%s.%s' % ( prefix, expnums[0], version, str(ccd).zfill(2), ext) storage.copy(filename, uri) return
def step2(expnums, ccd, version, prefix=None): '''run the actual step2 on the given exp/ccd combo''' jmp_args = ['step2jmp'] matt_args = ['step2matt_jmp'] idx = 0 for expnum in expnums: jmp_args.append( storage.get_image(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix )[0:-8] ) idx += 1 matt_args.append('-f%d' % ( idx)) matt_args.append( storage.get_image(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix )[0:-9] ) util.exec_prog(jmp_args) util.exec_prog(matt_args) for expnum in expnums: for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: uri = storage.dbimages_uri(expnum,ccd=ccd,version=version,ext=ext, prefix=prefix) filename = os.path.basename(uri) storage.copy(filename, uri) return
def plant(expnums, ccd, rmin, rmax, ang, width, version="s"): """run the plant script on this combination of exposures""" ptf = open("proc-these-files", "w") ptf.write("# Files to be planted and search\n") ptf.write("# image fwhm plant\n") for expnum in expnums: fwhm = storage.get_fwhm(expnum, ccd, version=version) filename = storage.get_image(expnum, ccd=ccd, version=version) ptf.write("%s %3.1f YES\n" % (filename[0:-5], fwhm)) for ext in ["apcor", "obj.jmp", "trans.jmp", "psf.fits", "mopheader", "phot", "zeropoint.used"]: apcor = storage.get_image(expnum, ccd=ccd, version=version, ext=ext) ptf.close() cmd_args = ["plant.csh", os.curdir, str(rmin), str(rmax), str(ang), str(width)] util.exec_prog(cmd_args) if args.dryrun: # Don't push back to VOSpace return uri = storage.get_uri("Object", ext="planted", version="", subdir=str(expnums[0]) + "/ccd%s" % (str(ccd).zfill(2))) storage.copy("Object.planted", uri) uri = os.path.join(os.path.dirname(uri), "plant.shifts") storage.copy("shifts", uri) for expnum in expnums: uri = storage.get_uri(expnum, ccd=ccd, version=version, ext="fits", prefix="fk") filename = os.path.basename(uri) storage.copy(filename, uri) for ext in ["mopheader", "psf.fits", "fwhm", "apcor", "zeropoint.used", "trans.jmp"]: storage.delete(expnum, ccd, "s", ext, prefix="fk") storage.vlink(expnum, ccd, "s", ext, expnum, ccd, "s", ext, l_prefix="fk") return
def _get_weight_map(filename, ccd): # for step1matt we need the weight image hdulist = fits.open(filename) flat_name = hdulist[0].header.get('FLAT', 'weight.fits') parts = os.path.splitext(flat_name) if parts[1] == '.fz': flat_name = os.path.splitext(parts[0])[0] else: flat_name = parts[0] logging.info("Getting the Flat Field image.") try: flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators') except Exception: flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='old_calibrators') if os.access('weight.fits', os.R_OK): os.unlink('weight.fits') if not os.access('weight.fits', os.R_OK): os.symlink(flat_filename, 'weight.fits') return flat_filename
def step2(expnums, ccd, version, prefix=None): '''run the actual step2 on the given exp/ccd combo''' jmp_args = ['step2jmp'] matt_args = ['step2matt_jmp'] idx = 0 for expnum in expnums: jmp_args.append( storage.get_image(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]) idx += 1 matt_args.append('-f%d' % (idx)) matt_args.append( storage.get_image(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix)[0:-9]) util.exec_prog(jmp_args) util.exec_prog(matt_args) for expnum in expnums: for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: uri = storage.dbimages_uri(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = os.path.basename(uri) storage.copy(filename, uri) return
def _get_weight_map(filename, ccd): # for step1matt we need the weight image hdulist = fits.open(filename) flat_name = hdulist[0].header.get('FLAT', 'weight.fits') parts = os.path.splitext(flat_name) if parts[1] == '.fz': flat_name = os.path.splitext(parts[0])[0] else: flat_name = parts[0] logging.info("Getting the Flat Field image.") try: flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators') except: flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='old_calibrators') if os.access('weight.fits', os.R_OK): os.unlink('weight.fits') if not os.access('weight.fits', os.R_OK): os.symlink(flat_filename, 'weight.fits') return flat_filename
def compute_trans(expnums, ccd, version, prefix=None, default="WCS"): """ Pull the astrometric header for each image, compute an x/y transform and compare to trans.jmp this one overides trans.jmp if they are very different. @param expnums: @param ccd: @param version: @param prefix: @return: None """ wcs_dict = {} for expnum in expnums: try: # TODO This assumes that the image is already N/E flipped. # If compute_trans is called after the image is retrieved from archive then we get the disk version. filename = storage.get_image(expnum, ccd, version, prefix=prefix) this_wcs = wcs.WCS(fits.open(filename)[0].header) except Exception as err: logging.warning("WCS Trans compute failed. {}".format(str(err))) return wcs_dict[expnum] = this_wcs x0 = wcs_dict[expnums[0]].header['NAXIS1'] / 2.0 y0 = wcs_dict[expnums[0]].header['NAXIS2'] / 2.0 (ra0, dec0) = wcs_dict[expnums[0]].xy2sky(x0, y0) result = "" for expnum in expnums: filename = storage.get_file(expnum, ccd, version, ext='.trans.jmp', prefix=prefix) jmp_trans = file(filename, 'r').readline().split() (x, y) = wcs_dict[expnum].sky2xy(ra0, dec0) x1 = float(jmp_trans[0]) + float(jmp_trans[1]) * x + float(jmp_trans[2]) * y y1 = float(jmp_trans[3]) + float(jmp_trans[4]) * x + float(jmp_trans[5]) * y dr = math.sqrt((x1 - x0) ** 2 + (y1 - y0) ** 2) if dr > 0.5: result += "WARNING: WCS-JMP transforms mis-matched {} reverting to using {}.\n".format(expnum, default) if default == "WCS": uri = storage.dbimages_uri(expnum, ccd, version, ext='.trans.jmp', prefix=prefix) filename = os.path.basename(uri) trans = file(filename, 'w') trans.write("{:5.2f} 1. 0. {:5.2f} 0. 1.\n".format(x0 - x, y0 - y)) trans.close() else: result += "WCS-JMP transforms match {}\n".format(expnum) return result
def scramble(expnums, ccd, version='p'): '''run the plant script on this combination of exposures''' mjds = [] fobjs = [] for expnum in expnums: filename = storage.get_image(expnum, ccd=ccd, version=version) fobjs.append(fits.open(filename)) # Pull out values to replace in headers.. must pull them # as otherwise we get pointers... mjds.append(fobjs[-1][0].header['MJD-OBS']) order = [1, 0, 2] for idx in range(len(fobjs)): logging.info("Flipping %d to %d" % (fobjs[idx][0].header['EXPNUM'], expnums[order[idx]])) fobjs[idx][0].header['EXPNUM'] = expnums[order[idx]] fobjs[idx][0].header['MJD-OBS'] = mjds[order[idx]] uri = storage.get_uri(expnums[order[idx]], ccd=ccd, version='s', ext='fits') fname = os.path.basename(uri) if os.access(fname, os.F_OK): os.unlink(fname) fobjs[idx].writeto(fname) storage.copy(fname, uri) # now make a link between files that the plant system will need for ext in [ 'apcor', 'obj.jmp', 'mopheader', 'phot', 'psf.fits', 'trans.jmp', 'zeropoint.used', 'fwhm' ]: storage.delete(expnums[order[idx]], ccd, 's', ext) storage.vlink(expnums[idx], ccd, 'p', ext, expnums[order[idx]], ccd, 's', ext) return
def mkpsf(expnum, ccd, version, dry_run=False, prefix=""): """Run the OSSOS jmpmakepsf script. """ ## confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits')) if not dry_run: storage.mkdir(destdir) ## get image from the vospace storage area filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) logging.info("Running mkpsf on %s %d" % (expnum, ccd)) ## launch the makepsf script logging.info(util.exec_prog(['jmpmakepsf.csh', './', filename, 'no'])) if dry_run: return ## place the results into VOSpace basename = os.path.splitext(filename)[0] for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext) source = basename + "." + str(ext) storage.copy(source, dest) return
def scramble(expnums, ccd, version='p'): '''run the plant script on this combination of exposures''' mjds = [] fobjs = [] for expnum in expnums: filename = storage.get_image(expnum, ccd=ccd, version=version) fobjs.append(fits.open(filename)) # Pull out values to replace in headers.. must pull them # as otherwise we get pointers... mjds.append(fobjs[-1][0].header['MJD-OBS']) order = [1,0,2] for idx in range(len(fobjs)): logging.info("Flipping %d to %d" % ( fobjs[idx][0].header['EXPNUM'], expnums[order[idx]])) fobjs[idx][0].header['EXPNUM'] = expnums[order[idx]] fobjs[idx][0].header['MJD-OBS'] = mjds[order[idx]] uri = storage.get_uri(expnums[order[idx]], ccd=ccd, version='s', ext='fits') fname = os.path.basename(uri) if os.access(fname, os.F_OK): os.unlink(fname) fobjs[idx].writeto(fname) storage.copy(fname, uri) # now make a link between files that the plant system will need for ext in ['apcor', 'obj.jmp', 'mopheader', 'phot', 'psf.fits','trans.jmp', 'zeropoint.used', 'fwhm']: if storage.exists(storage.get_uri(expnums[order[idx]], ccd, 's', ext)): storage.delete(expnums[order[idx]], ccd, 's', ext) storage.vlink(expnums[idx], ccd, 'p', ext, expnums[order[idx]], ccd, 's', ext) return
def plant(expnums, ccd, rmin, rmax, ang, width, number=10, version='s', dry_run=False): """Plant artificial sources into the list of images provided. :param expnums: list of MegaPrime exposure numbers to add artificial KBOs to :param ccd: which ccd to work on. :param rmin: The minimum rate of motion to add sources at (''/hour) :param rmax: The maximum rate of motion to add sources at (''/hour) :param ang: The mean angle of motion to add sources :param width: The +/- range of angles of motion :param version: Add sources to the 'o', 'p' or 's' images :param dry_run: don't push results to VOSpace. """ # Construct a list of artificial KBOs with positions in the image # and rates of motion within the bounds given by the caller. filename = storage.get_image(expnums[0], ccd=ccd, version=version) header = fits.open(filename)[0].header bounds = util.get_pixel_bounds_from_datasec_keyword( header.get('DATASEC', '[33:2080,1:4612]')) # generate a set of artifical KBOs to add to the image. kbos = Table(names=('x', 'y', 'mag', 'sky_rate', 'angle', 'id')) for kbo in KBOGenerator(n=number, x=Range(bounds[0][0], bounds[0][1]), y=Range(bounds[1][0], bounds[1][1]), rate=Range(rmin, rmax), angle=Range(ang - width, ang + width), mag=Range(21.0, 25.0)): kbos.add_row(kbo) fd = open('Object.planted', 'w') fd.write("# ") kbos.write(fd, format='ascii.fixed_width', delimiter=None) fd.close() for expnum in expnums: filename = storage.get_image(expnum, ccd, version) psf = storage.get_file(expnum, ccd, version, ext='psf.fits') plant_kbos(filename, psf, kbos, get_shifts(expnum, ccd, version), "fk") if dry_run: return uri = storage.get_uri('Object', ext='planted', version='', subdir=str(expnums[0]) + "/ccd%s" % (str(ccd).zfill(2))) storage.copy('Object.planted', uri) for expnum in expnums: uri = storage.get_uri(expnum, ccd=ccd, version=version, ext='fits', prefix='fk') filename = os.path.basename(uri) storage.copy(filename, uri) return
def run(expnums, ccd, version, prefix=None, dry_run=False, default="WCS", force=False): """run the actual step2 on the given exp/ccd combo""" jmp_trans = ['step2ajmp'] jmp_args = ['step2bjmp'] matt_args = ['step2matt_jmp'] if storage.get_status(task, prefix, expnums[0], version, ccd) and not force: logging.info("{} completed successfully for {}{}{}{:02d}".format( task, prefix, expnums[0], version, ccd)) return with storage.LoggingManager(task, prefix, expnums[0], ccd, version, dry_run): try: for expnum in expnums: if not storage.get_status( dependency, prefix, expnum, version=version, ccd=ccd): raise IOError( 35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}" .format(task, dependency, prefix, expnum, version, ccd)) message = storage.SUCCESS idx = 0 logging.info("Retrieving catalog files to do matching.") for expnum in expnums: jmp_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]) jmp_trans.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]) idx += 1 matt_args.append('-f%d' % idx) matt_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix)[0:-9]) logging.info( "Computing the catalog alignment using sources in catalogs.") try: logging.info(util.exec_prog(jmp_trans)) if default == "WCS": logging.info("Comparing computed transform to WCS values") logging.info( compute_trans(expnums, ccd, version, prefix, default=default)) except Exception as ex: logging.info("JMP Trans failed: {}".format(ex)) logging.info( compute_trans(expnums, ccd, version, prefix, default="WCS")) logging.info("Using transform to match catalogs for three images.") logging.info(util.exec_prog(jmp_args)) logging.info(util.exec_prog(matt_args)) # check that the shifts from step2 are rational by matching the bright star lists. logging.info( "Uisng checktrans to ensure that transforms were reasonable.") check_args = ['checktrans'] if os.access('proc-these-files', os.R_OK): os.unlink('proc-these-files') ptf = open('proc-these-files', 'w') ptf.write( "# A dummy file that is created so checktrans could run.\n") ptf.write("# Frame FWHM PSF?\n") for expnum in expnums: filename = os.path.splitext( storage.get_image(expnum, ccd, version=version, prefix=prefix))[0] if not os.access(filename + ".bright.psf", os.R_OK): os.link(filename + ".bright.jmp", filename + ".bright.psf") if not os.access(filename + ".obj.psf", os.R_OK): os.link(filename + ".obj.jmp", filename + ".obj.psf") ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format( filename, _FWHM, "NO")) ptf.close() if os.access('BAD_TRANS', os.F_OK): os.unlink('BAD_TRANS') logging.info(util.exec_prog(check_args)) if os.access('BAD_TRANS', os.F_OK): raise OSError(errno.EBADMSG, 'BAD_TRANS') if os.access('proc-these-files', os.F_OK): os.unlink('proc-these-files') if dry_run: return for expnum in expnums: for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: uri = storage.dbimages_uri(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = os.path.basename(uri) storage.copy(filename, uri) except Exception as ex: message = str(ex) logging.error(message) storage.set_status(task, prefix, expnums[0], version, ccd, status=message) return
def scramble(expnums, ccd, version='p', dry_run=False, force=False, prefix=''): """ run the plant script on this combination of exposures @param expnums: list of exposure numbers to scramble the time on @param ccd: which CCD in (assumes this is a CFHT MegaCam MEF) @param version: should we scramble the 'p' or 'o' images? @param dry_run: if dry run then don't save back to VOSpace. @param force: if true then create scramble set, even if already exists. @param prefix: a string that will be pre-pended to the EXPNUM to get the filename, sometimes 'fk'. @return: None """ # Get a list of the MJD values and then write a re-ordering of those into files with 's' # as their type instead of 'p' or 'o' mjds = [] fobjs = [] message = storage.SUCCESS if not (force or dry_run) and storage.get_status( task, prefix, expnums[0], version='s', ccd=ccd): logging.info("{} recorded as complete for {} ccd {}".format( task, expnums, ccd)) return with storage.LoggingManager(task, prefix, expnums[0], ccd, version): try: for expnum in expnums: filename = storage.get_image(expnum, ccd=ccd, version=version) fobjs.append(fits.open(filename)) # Pull out values to replace in headers.. must pull them # as otherwise we get pointers... mjds.append(fobjs[-1][0].header['MJD-OBS']) order = [0, 2, 1] for idx in range(len(fobjs)): logging.info( "Flipping %d to %d" % (fobjs[idx][0].header['EXPNUM'], expnums[order[idx]])) fobjs[idx][0].header['EXPNUM'] = expnums[order[idx]] fobjs[idx][0].header['MJD-OBS'] = mjds[order[idx]] uri = storage.get_uri(expnums[order[idx]], ccd=ccd, version='s', ext='fits') scramble_file_name = os.path.basename(uri) if os.access(scramble_file_name, os.F_OK): os.unlink(scramble_file_name) fobjs[idx].writeto(scramble_file_name) if not dry_run: storage.copy(scramble_file_name, uri) logging.info(message) except Exception as ex: message = str(ex) logging.error(message) if not dry_run: storage.set_status(task, prefix, expnum, version, ccd, status=message) return
def run(expnum, ccd, prefix='', version='p', sex_thresh=_SEX_THRESHOLD, wave_thresh=_WAVE_THRESHOLD, maxcount=_MAX_COUNT, dry_run=False, force=True): """run the actual step1jmp/matt codes. expnum: the CFHT expousre to process ccd: which ccd in the mosaic to process fwhm: the image quality, FWHM, of the image. In pixels. sex_thresh: the detection threhold to run sExtractor at wave_thresh: the detection threshold for wavelet maxcount: saturation level """ message = storage.SUCCESS if storage.get_status(task, prefix, expnum, version, ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run): try: if not storage.get_status(dependency, prefix, expnum, version, ccd): raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format( task, dependency, prefix, expnum, version, ccd)) logging.info("Retrieving imaging and input parameters from VOSpace") storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader') filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version) basename = os.path.splitext(filename)[0] _get_weight_map(filename, ccd) logging.info("Launching step1jmp") logging.info(util.exec_prog(['step1jmp', '-f', basename, '-t', str(wave_thresh), '-w', str(fwhm), '-m', str(maxcount)])) logging.info(util.exec_prog(['step1matt', '-f', basename, '-t', str(sex_thresh), '-w', str(fwhm), '-m', str(maxcount)])) if os.access('weight.fits', os.R_OK): os.unlink('weight.fits') if not dry_run: for ext in ['obj.jmp', 'obj.matt']: obj_uri = storage.get_uri(expnum, ccd, version=version, ext=ext, prefix=prefix) obj_filename = basename + "." + ext count = 0 with open(obj_filename, 'r'): while True: try: count += 1 logging.info("Attempt {} to copy {} -> {}".format(count, obj_filename, obj_uri)) storage.copy(obj_filename, obj_uri) break except Exception as ex: if count > 10: raise ex logging.info(message) except Exception as ex: message = str(ex) logging.error(message) if not dry_run: storage.set_status(task, prefix, expnum, version, ccd, status=message)
def combine(expnum, ccd, prefix=None, type='p', field=None, measure3=MEASURE3 ): if field is None: field=str(expnum) if prefix is not None and len(prefix) > 0: field = "%s_%s" % ( prefix, field ) field += "_%s" % ( str(ccd)) for ext in ['moving.matt','moving.jmp']: fname = storage.get_image(expnum, ccd=ccd, prefix=prefix, version=type, ext=ext) if prefix is not None and len(prefix) > 0: planted = storage.get_image('Object', subdir=str(expnum)+"/ccd%s" % ( str(ccd).zfill(2)), version='', ext='planted') else: prefix = '' base_image = os.path.basename( storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=type, ext=None)) cmd_args = ['comb-list', prefix+str(expnum)+type+str(ccd).zfill(2)] util.exec_prog(cmd_args) ext_list = ['cands.comb'] if prefix is not None and len(prefix) > 0 : ext_list.extend( [ 'jmp.missed', 'matt.missed', 'jmp.found', 'matt.found', 'comb.missed', 'comb.found' ] ) for ext in ext_list: uri = storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=type, ext=ext) filename = os.path.basename(uri) if not os.access(filename,os.R_OK): logging.critical("No %s file" % (filename)) continue vospace_name = "%s.%s" % ( field, ext ) logging.info("%s -> %s" % ( filename, os.path.join(measure3, vospace_name))) storage.copy(filename, os.path.join(measure3, vospace_name)) base_name = prefix+str(expnum)+type+str(ccd).zfill(2) cands_file = base_name+'.cands.comb' if not os.access(cands_file,os.R_OK): nocands_file = ( prefix+ str(expnum)+ type+ str(ccd).zfill(2)+ '.no_candidates' ) open(nocands_file, 'w').close() vospace_name = "%s.no_candidates" % ( field ) storage.copy(nocands_file,os.path.join(measure3, vospace_name)) return storage.SUCCESS # get the images we need to compute x/y ra/dec transforms cands_file = mop_file.Parser().parse(cands_file) for file_id in cands_file.header.file_ids: rec_no=cands_file.header.file_ids.index(file_id) storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no], ccd=ccd, version=type, prefix=prefix, ext='fits') cmd_args = ['measure3', prefix+str(expnum)+type+str(ccd).zfill(2)] logging.info("Running measure3") util.exec_prog(cmd_args) filename=base_name+".measure3.cands.astrom" vospace_filename = "%s.measure3.cands.astrom" % ( field) storage.copy(filename, os.path.join(measure3,vospace_filename)) return storage.SUCCESS
def main(): """Do the script.""" parser = argparse.ArgumentParser( description='replace image header') parser.add_argument('--extname', help='name of extension to in header') parser.add_argument('expnum', type=str, help='exposure to update') parser.add_argument('-r', '--replace', action='store_true', help='store modified image back to VOSpace?') parser.add_argument('-v', '--verbose', action='store_true') parser.add_argument('--debug', action='store_true') parser.add_argument('--force', action='store_true', help="Re-run even if previous success recorded") parser.add_argument('--dbimages', help="VOSpace DATA storage area.", default="vos:OSSOS/dbimages") args = parser.parse_args() task = util.task() dependency = 'preproc' prefix = "" storage.DBIMAGES = args.dbimages level = logging.CRITICAL message_format = "%(message)s" if args.verbose: level = logging.INFO if args.debug: level = logging.DEBUG message_format = "%(module)s %(funcName)s %(lineno)s %(message)s" logging.basicConfig(level=level, format=message_format) storage.set_logger(task, prefix, args.expnum, None, None, False) message = storage.SUCCESS expnum = args.expnum exit_status = 0 try: # skip if already succeeded and not in force mode if storage.get_status(task, prefix, expnum, "p", 36) and not args.force: logging.info("Already updated, skipping") sys.exit(0) image_hdulist = storage.get_image(args.expnum, return_file=False) ast_hdulist = storage.get_astheader(expnum, ccd=None) run_update_header(image_hdulist, ast_hdulist) image_filename = os.path.basename(storage.get_uri(expnum)) image_hdulist.writeto(image_filename) if args.replace: dest = storage.dbimages_uri(expnum) storage.copy(image_filename, dest) storage.set_status('update_header', "", expnum, 'p', 36, message) except Exception as e: message = str(e) if args.replace: storage.set_status(task, prefix, expnum, 'p', 36, message) exit_status = message logging.error(message) return exit_status
def remeasure(mpc_in): """ re-measure the astrometry and photometry of the given mpc line """ # TODO Actually implement this. if mpc_in.null_observation: return mpc_in mpc_obs = deepcopy(mpc_in) logging.debug("rm start: {}".format(mpc_obs.to_string())) if not isinstance(mpc_obs.comment, mpc.OSSOSComment): return mpc_in parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame) if not parts: return mpc_in start_coordinate = mpc_in.coordinate assert isinstance(start_coordinate, ICRSCoordinates) try: header = storage.get_astheader(parts.group('expnum'), int(parts.group('ccd'))) except IOError as ioerr: logging.error(str(ioerr)) logging.error("Failed to get astrometric header for: {}".format(mpc_obs)) return mpc_in this_wcs = wcs.WCS(header) try: x = float(mpc_obs.comment.x) y = float(mpc_obs.comment.y) if mpc_in.discovery: logging.debug("Discovery astrometric lines are normally flipped relative to storage.") x = header['NAXIS1'] - x + 1 y = header['NAXIS2'] - y + 1 except: logging.warn("Failed to X/Y from comment line.") return mpc_in (ra, dec) = this_wcs.xy2sky(x, y) mpc_obs.coordinate = (ra, dec) logging.debug("rm updat: {}".format(mpc_obs.to_string())) sep = start_coordinate.separation(mpc_obs.coordinate).degrees * 3600.0 if sep > TOLERANCE and mpc_in.discovery: logging.warn("{} --> Using the unflipped X/Y for a discovery observation line.".format(sep)) logging.debug("{} {} {} {} {}".format(sep, x, y, mpc_obs.comment.x, mpc_obs.comment.y)) # Try un-flipping. x = float(mpc_obs.comment.x) y = float(mpc_obs.comment.y) (ra, dec) = this_wcs.xy2sky(x, y) #print "--> x/y coordinates ({},{}) and recomputing ra/dec ({},{})".format(x, y, ra, dec) mpc_obs.coordinate = (ra, dec) sep = start_coordinate.separation(mpc_obs.coordinate).degrees * 3600.0 logging.debug("remod: {}".format(mpc_obs.coordinate)) logging.debug("SEP: {}".format(sep)) logging.debug("rm flip: {}".format(mpc_obs.to_string())) if sep > TOLERANCE: ## use the old header RA/DEC to predict the X/Y and then use that X/Y to get new RA/DEC logging.debug("Ignoring recorded X/Y and using previous to RA/DEC and WCS to compute X/Y") header2 = storage.get_image(parts.group('expnum'), int(parts.group('ccd')), return_file=False, flip_image=False)[0].header image_wcs = wcs.WCS(header2) (x, y) = image_wcs.sky2xy(mpc_in.coordinate.ra.degrees, mpc_in.coordinate.dec.degrees) (ra, dec) = this_wcs.xy2sky(x, y) logging.debug("({},{}) --> ({},{})".format(mpc_obs.comment.x, mpc_obs.comment.y, x, y)) mpc_obs.coordinate = (ra, dec) mpc_obs.comment.x = x mpc_obs.comment.y = y try: merr = float(mpc_obs.comment.mag_uncertainty) fwhm = float(storage.get_fwhm(parts.group('expnum'), int(parts.group('ccd')))) * header['PIXSCAL1'] centroid_err = merr * fwhm logging.debug("Centroid uncertainty: {} {} => {}".format(merr, fwhm, centroid_err)) except Exception as err: logging.error(str(err)) logging.error("Failed to compute centroid for observation:\n" "{}\nUsing default of 0.2".format(mpc_obs.to_string())) centroid_err = 0.2 mpc_obs.comment.astrometric_level = header.get('ASTLEVEL', "0") try: asterr = float(header['ASTERR']) residuals = (asterr ** 2 + centroid_err ** 2) ** 0.5 logging.debug("Residuals: {} {} => {}".format(asterr, centroid_err, residuals)) except Exception as err: logging.error(str(err)) logging.error("Failed while trying to compute plate uncertainty for\n{}".format(mpc_obs.to_stirng())) logging.error('Using default of 0.25') residuals = 0.25 mpc_obs.comment.plate_uncertainty = residuals logging.debug("sending back: {}".format(mpc_obs.to_string())) return mpc_obs
def step2(expnums, ccd, version, prefix=None, dry_run=False, default="WCS"): """run the actual step2 on the given exp/ccd combo""" jmp_trans = ['step2ajmp'] jmp_args = ['step2bjmp'] matt_args = ['step2matt_jmp'] idx = 0 for expnum in expnums: jmp_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8] ) jmp_trans.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8] ) idx += 1 matt_args.append('-f%d' % idx) matt_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix)[0:-9] ) logging.info(util.exec_prog(jmp_trans)) if default == "WCS": logging.info(compute_trans(expnums, ccd, version, prefix, default=default)) logging.info(util.exec_prog(jmp_args)) logging.info(util.exec_prog(matt_args)) ## check that the shifts from step2 are rational check_args = ['checktrans'] if os.access('proc-these-files', os.R_OK): os.unlink('proc-these-files') ptf = open('proc-these-files', 'w') ptf.write("# A dummy file that is created so checktrans could run.\n") ptf.write("# Frame FWHM PSF?\n") for expnum in expnums: filename = os.path.splitext(storage.get_image(expnum, ccd, version=version, prefix=prefix))[0] if not os.access(filename + ".bright.psf", os.R_OK): os.link(filename + ".bright.jmp", filename + ".bright.psf") if not os.access(filename + ".obj.psf", os.R_OK): os.link(filename + ".obj.jmp", filename + ".obj.psf") ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(filename, _FWHM, "NO")) ptf.close() if os.access('BAD_TRANS', os.F_OK): os.unlink('BAD_TRANS') logging.info(util.exec_prog(check_args)) if os.access('BAD_TRANS', os.F_OK): raise OSError(errno.EBADMSG, 'BAD_TRANS') if os.access('proc-these-files', os.F_OK): os.unlink('proc-these-files') if dry_run: return for expnum in expnums: for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: uri = storage.dbimages_uri(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = os.path.basename(uri) storage.copy(filename, uri) return
def main(): """Do the script.""" parser = argparse.ArgumentParser(description='replace image header') parser.add_argument('--extname', help='name of extension to in header') parser.add_argument('expnum', type=str, help='exposure to update') parser.add_argument('-r', '--replace', action='store_true', help='store modified image back to VOSpace?') parser.add_argument('-v', '--verbose', action='store_true') parser.add_argument('--debug', action='store_true') parser.add_argument('--force', action='store_true', help="Re-run even if previous success recorded") parser.add_argument('--dbimages', help="VOSpace DATA storage area.", default="vos:OSSOS/dbimages") args = parser.parse_args() task = util.task() dependency = 'preproc' prefix = "" storage.DBIMAGES = args.dbimages level = logging.CRITICAL message_format = "%(message)s" if args.verbose: level = logging.INFO if args.debug: level = logging.DEBUG message_format = "%(module)s %(funcName)s %(lineno)s %(message)s" logging.basicConfig(level=level, format=message_format) storage.set_logger(task, prefix, args.expnum, None, None, False) message = storage.SUCCESS expnum = args.expnum exit_status = 0 try: # skip if already succeeded and not in force mode if storage.get_status(task, prefix, expnum, "p", 36) and not args.force: logging.info("Already updated, skipping") sys.exit(0) image_hdulist = storage.get_image(args.expnum, return_file=False) ast_hdulist = storage.get_astheader(expnum, ccd=None) run_update_header(image_hdulist, ast_hdulist) image_filename = os.path.basename(storage.get_uri(expnum)) image_hdulist.writeto(image_filename) if args.replace: dest = storage.dbimages_uri(expnum) storage.copy(image_filename, dest) storage.set_status('update_header', "", expnum, 'p', 36, message) except Exception as e: message = str(e) if args.replace: storage.set_status(task, prefix, expnum, 'p', 36, message) exit_status = message logging.error(message) return exit_status
def run(expnum, ccd, version, dry_run=False, prefix="", force=False): """Run the OSSOS jmpmakepsf script. """ message = storage.SUCCESS if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format( task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run): try: if not storage.get_status( dependency, prefix, expnum, version, ccd=ccd): raise IOError("{} not yet run for {}".format( dependency, expnum)) # confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits')) if not dry_run: storage.mkdir(destdir) # get image from the vospace storage area logging.info("Getting fits image from VOSpace") filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) # get mopheader from the vospace storage area logging.info("Getting mopheader from VOSpace") mopheader_filename = storage.get_file(expnum, ccd, version=version, prefix=prefix, ext='mopheader') # run mkpsf process logging.info("Running mkpsf on %s %d" % (expnum, ccd)) logging.info( util.exec_prog( ['jmpmakepsf.csh', './', filename, 'yes', 'yes'])) if dry_run: return # place the results into VOSpace basename = os.path.splitext(filename)[0] for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext) source = basename + "." + str(ext) count = 0 with open(source, 'r'): while True: count += 1 try: logging.info("Attempt {} to copy {} -> {}".format( count, source, dest)) storage.copy(source, dest) break except Exception as ex: if count > 10: raise ex # set some data parameters associated with the image, determined in this step. storage.set_status('fwhm', prefix, expnum, version=version, ccd=ccd, status=str( storage.get_fwhm(expnum, ccd=ccd, prefix=prefix, version=version))) storage.set_status('zeropoint', prefix, expnum, version=version, ccd=ccd, status=str( storage.get_zeropoint(expnum, ccd=ccd, prefix=prefix, version=version))) logging.info(message) except Exception as e: message = str(e) logging.error(message) storage.set_status(task, prefix, expnum, version, ccd=ccd, status=message) return
def align(expnums, ccd, version='s', prefix='', dry_run=False, force=True): """Create a 'shifts' file that transforms the space/flux/time scale of all images to the first image. This function relies on the .fwhm, .trans.jmp, .phot and .zeropoint.used files for inputs. The scaling we are computing here is for use in planting sources into the image at the same sky/flux locations while accounting for motions of sources with time. :param expnums: list of MegaPrime exposure numbers to add artificial KBOs to, the first frame in the list is the reference. :param ccd: which ccd to work on. :param prefix: put this string in front of expnum when looking for exposure, normally '' or 'fk' :param force: When true run task even if this task is recorded as having succeeded :param version: Add sources to the 'o', 'p' or 's' images :param dry_run: don't push results to VOSpace. """ message = storage.SUCCESS if storage.get_status(task, prefix, expnums[0], version, ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format( task, prefix, expnums[0], version, ccd)) return # Get the images and supporting files that we need from the VOSpace area # get_image and get_file check if the image/file is already on disk. # re-computed fluxes from the PSF stars and then recompute x/y/flux scaling. # some dictionaries to hold the various scale pos = {} apcor = {} mags = {} zmag = {} mjdates = {} with storage.LoggingManager(task, prefix, expnums[0], ccd, version, dry_run): try: for expnum in expnums: filename = storage.get_image(expnum, ccd=ccd, version=version) zmag[expnum] = storage.get_zeropoint(expnum, ccd, prefix=None, version=version) mjdates[expnum] = float( fits.open(filename)[0].header.get('MJD-OBS')) apcor[expnum] = [ float(x) for x in open( storage.get_file( expnum, ccd=ccd, version=version, ext=storage.APCOR_EXT)).read().split() ] keys = ['crval1', 'cd1_1', 'cd1_2', 'crval2', 'cd2_1', 'cd2_2'] # load the .trans.jmp values into a 'wcs' like dictionary. # .trans.jmp maps current frame to reference frame in pixel coordinates. # the reference frame of all the frames supplied must be the same. shifts = dict( list( zip(keys, [ float(x) for x in open( storage.get_file( expnum, ccd=ccd, version=version, ext='trans.jmp')).read().split() ]))) shifts['crpix1'] = 0.0 shifts['crpix2'] = 0.0 # now create a wcs object based on those transforms, this wcs links the current frame's # pixel coordinates to the reference frame's pixel coordinates. w = get_wcs(shifts) # get the PHOT file that was produced by the mkpsf routine logging.debug("Reading .phot file {}".format(expnum)) phot = ascii.read(storage.get_file(expnum, ccd=ccd, version=version, ext='phot'), format='daophot') # compute the small-aperture magnitudes of the stars used in the PSF logging.debug("Running phot on {}".format(filename)) mags[expnum] = daophot.phot(filename, phot['XCENTER'], phot['YCENTER'], aperture=apcor[expnum][0], sky=apcor[expnum][1] + 1, swidth=apcor[expnum][0], zmag=zmag[expnum]) # covert the x/y positions to positions in Frame 1 based on the trans.jmp values. logging.debug( "Doing the XY translation to refrence frame: {}".format(w)) (x, y) = w.wcs_pix2world(mags[expnum]["XCENTER"], mags[expnum]["YCENTER"], 1) pos[expnum] = numpy.transpose([x, y]) # match this exposures PSF stars position against those in the first image of the set. logging.debug("Matching lists") idx1, idx2 = util.match_lists(pos[expnums[0]], pos[expnum]) # compute the magnitdue offset between the current frame and the reference. dmags = numpy.ma.array( mags[expnums[0]]["MAG"] - apcor[expnums[0]][2] - (mags[expnum]["MAG"][idx1] - apcor[expnum][2]), mask=idx1.mask) dmags.sort() # logging.debug("Computed dmags between input and reference: {}".format(dmags)) error_count = 0 error_count += 1 logging.debug("{}".format(error_count)) # compute the median and determine if that shift is small compared to the scatter. try: midx = int( numpy.sum(numpy.any([~dmags.mask], axis=0)) / 2.0) dmag = float(dmags[midx]) logging.debug("Computed a mag delta of: {}".format(dmag)) except Exception as e: logging.error(str(e)) logging.error( "Failed to compute mag offset between plant and found using: {}" .format(dmags)) dmag = 99.99 error_count += 1 logging.debug("{}".format(error_count)) try: if math.fabs(dmag) > 3 * (dmags.std() + 0.01): logging.warning( "Magnitude shift {} between {} and {} is large: {}" .format(dmag, expnums[0], expnum, shifts)) except Exception as e: logging.error(str(e)) error_count += 1 logging.debug("{}".format(error_count)) shifts['dmag'] = dmag shifts['emag'] = dmags.std() shifts['nmag'] = len(dmags.mask) - dmags.mask.sum() shifts['dmjd'] = mjdates[expnums[0]] - mjdates[expnum] shift_file = os.path.basename( storage.get_uri(expnum, ccd, version, '.shifts')) error_count += 1 logging.debug("{}".format(error_count)) try: fh = open(shift_file, 'w') fh.write( json.dumps(shifts, sort_keys=True, indent=4, separators=(',', ': '), cls=NpEncoder)) fh.write('\n') fh.close() except Exception as e: logging.error( "Creation of SHIFTS file failed while trying to write: {}" .format(shifts)) raise e error_count += 1 logging.debug("{}".format(error_count)) if not dry_run: storage.copy( shift_file, storage.get_uri(expnum, ccd, version, '.shifts')) logging.info(message) except Exception as ex: message = str(ex) logging.error(message) if not dry_run: storage.set_status(task, prefix, expnum, version, ccd, status=message)
def step2(expnums, ccd, version, prefix=None, dry_run=False, default="WCS"): """run the actual step2 on the given exp/ccd combo""" jmp_trans = ['step2ajmp'] jmp_args = ['step2bjmp'] matt_args = ['step2matt_jmp'] idx = 0 for expnum in expnums: jmp_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]) jmp_trans.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.jmp', prefix=prefix)[0:-8]) idx += 1 matt_args.append('-f%d' % idx) matt_args.append( storage.get_file(expnum, ccd=ccd, version=version, ext='obj.matt', prefix=prefix)[0:-9]) logging.info(util.exec_prog(jmp_trans)) if default == "WCS": logging.info( compute_trans(expnums, ccd, version, prefix, default=default)) logging.info(util.exec_prog(jmp_args)) logging.info(util.exec_prog(matt_args)) ## check that the shifts from step2 are rational check_args = ['checktrans'] if os.access('proc-these-files', os.R_OK): os.unlink('proc-these-files') ptf = open('proc-these-files', 'w') ptf.write("# A dummy file that is created so checktrans could run.\n") ptf.write("# Frame FWHM PSF?\n") for expnum in expnums: filename = os.path.splitext( storage.get_image(expnum, ccd, version=version, prefix=prefix))[0] if not os.access(filename + ".bright.psf", os.R_OK): os.link(filename + ".bright.jmp", filename + ".bright.psf") if not os.access(filename + ".obj.psf", os.R_OK): os.link(filename + ".obj.jmp", filename + ".obj.psf") ptf.write("{:>19s}{:>10.1f}{:>5s}\n".format(filename, _FWHM, "NO")) ptf.close() if os.access('BAD_TRANS', os.F_OK): os.unlink('BAD_TRANS') logging.info(util.exec_prog(check_args)) if os.access('BAD_TRANS', os.F_OK): raise ValueError(errno.EBADEXEC, 'BAD_TRANS') if os.access('proc-these-files', os.F_OK): os.unlink('proc-these-files') if dry_run: return for expnum in expnums: for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: uri = storage.dbimages_uri(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = os.path.basename(uri) storage.copy(filename, uri) return
def run(expnum, ccd, prefix='', version='p', sex_thresh=_SEX_THRESHOLD, wave_thresh=_WAVE_THRESHOLD, maxcount=_MAX_COUNT, dry_run=False, force=True, ignore=False): """run the actual step1jmp/matt codes. expnum: the CFHT expousre to process ccd: which ccd in the mosaic to process fwhm: the image quality, FWHM, of the image. In pixels. sex_thresh: the detection threhold to run sExtractor at wave_thresh: the detection threshold for wavelet maxcount: saturation level """ message = storage.SUCCESS if storage.get_status(task, prefix, expnum, version, ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run): try: if not storage.get_status(dependency, prefix, expnum, version, ccd) and not ignore: raise IOError(35, "Cannot start {} as {} not yet completed for {}{}{}{:02d}".format( task, dependency, prefix, expnum, version, ccd)) logging.info("Retrieving imaging and input parameters from VOSpace") storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader') filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version, default=3.5) basename = os.path.splitext(filename)[0] _get_weight_map(filename, ccd) logging.info("Launching step1jmp") logging.info(util.exec_prog(['step1jmp', '-f', basename, '-t', str(wave_thresh), '-w', str(fwhm), '-m', str(maxcount)])) logging.info(util.exec_prog(['step1matt', '-f', basename, '-t', str(sex_thresh), '-w', str(fwhm), '-m', str(maxcount)])) if os.access('weight.fits', os.R_OK): os.unlink('weight.fits') if not dry_run: for ext in ['obj.jmp', 'obj.matt']: obj_uri = storage.get_uri(expnum, ccd, version=version, ext=ext, prefix=prefix) obj_filename = basename + "." + ext count = 0 with open(obj_filename, 'r'): while True: try: count += 1 logging.info("Attempt {} to copy {} -> {}".format(count, obj_filename, obj_uri)) storage.copy(obj_filename, obj_uri) break except Exception as ex: if count > 10: raise ex logging.info(message) except Exception as ex: message = str(ex) logging.error(message) if not dry_run: storage.set_status(task, prefix, expnum, version, ccd, status=message)
def plant(expnums, ccd, rmin, rmax, ang, width, number=10, mmin=21.0, mmax=25.5, version='s', dry_run=False, force=True): """Plant artificial sources into the list of images provided. @param dry_run: don't push results to VOSpace. @param width: The +/- range of angles of motion @param ang: The mean angle of motion to add sources @param rmax: The maximum rate of motion to add sources at (''/hour) @param rmin: The minimum rate of motion to add sources at (''/hour) @param expnums: list of MegaPrime exposure numbers to add artificial KBOs to @param ccd: which ccd to work on. @param mmax: Maximum magnitude to plant sources at @param version: Add sources to the 'o', 'p' or 's' images @param mmin: Minimum magnitude to plant sources at @param number: number of sources to plant. @param force: Run, even if we already succeeded at making a fk image. """ message = storage.SUCCESS if storage.get_status(task, "", expnums[0], version, ccd) and not force: logging.info("{} completed successfully for {}{}{}{:02d}".format( task, "", expnums[0], version, ccd)) return with storage.LoggingManager(task, "", expnums[0], ccd, version, dry_run): try: # Construct a list of artificial KBOs with positions in the image # and rates of motion within the bounds given by the caller. filename = storage.get_image(expnums[0], ccd=ccd, version=version) header = fits.open(filename)[0].header bounds = util.get_pixel_bounds_from_datasec_keyword( header.get('DATASEC', '[33:2080,1:4612]')) # generate a set of artificial KBOs to add to the image. kbos = KBOGenerator.get_kbos(n=number, rate=(rmin, rmax), angle=(ang - width, ang + width), mag=(mmin, mmax), x=(bounds[0][0], bounds[0][1]), y=(bounds[1][0], bounds[1][1]), filename='Object.planted') for expnum in expnums: filename = storage.get_image(expnum, ccd, version) psf = storage.get_file(expnum, ccd, version, ext='psf.fits') plant_kbos(filename, psf, kbos, get_shifts(expnum, ccd, version), "fk") if dry_run: return uri = storage.get_uri('Object', ext='planted', version='', subdir=f"{expnums[0]}/ccd{int(ccd):02d}") storage.copy('Object.planted', uri) for expnum in expnums: uri = storage.get_uri(expnum, ccd=ccd, version=version, ext='fits', prefix='fk') filename = os.path.basename(uri) storage.copy(filename, uri) except Exception as ex: message = str(ex) logging.error(message) storage.set_status(task, "", expnums[0], version, ccd, status=message) return
def step1(expnum, ccd, prefix='', version='p', sex_thresh=_SEX_THRESHOLD, wave_thresh=_WAVE_THRESHOLD, maxcount=_MAX_COUNT, dry_run=False): """run the actual step1jmp/matt codes. expnum: the CFHT expousre to process ccd: which ccd in the mosaic to process fwhm: the image quality, FWHM, of the image. In pixels. sex_thresh: the detection threhold to run sExtractor at wave_thresh: the detection threshold for wavelet maxcount: saturation level """ storage.get_file(expnum, ccd, prefix=prefix, version=version, ext='mopheader') filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version) basename = os.path.splitext(filename)[0] logging.info(util.exec_prog(['step1jmp', '-f', basename, '-t', str(wave_thresh), '-w', str(fwhm), '-m', str(maxcount)])) obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.jmp', prefix=prefix) obj_filename = basename + ".obj.jmp" if not dry_run: storage.copy(obj_filename, obj_uri) ## for step1matt we need the weight image hdulist = fits.open(filename) flat_name = hdulist[0].header.get('FLAT', 'weight.fits') parts = os.path.splitext(flat_name) if parts[1] == '.fz': flat_name = os.path.splitext(parts[0])[0] else: flat_name = parts[0] try: flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators') except: flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='old_calibrators') if os.access('weight.fits', os.R_OK): os.unlink('weight.fits') if not os.access('weight.fits', os.R_OK): os.symlink(flat_filename, 'weight.fits') logging.info(util.exec_prog(['step1matt', '-f', basename, '-t', str(sex_thresh), '-w', str(fwhm), '-m', str(maxcount)])) if os.access('weight.fits', os.R_OK): os.unlink('weight.fits') obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.matt', prefix=prefix) obj_filename = basename + ".obj.matt" if not dry_run: storage.copy(obj_filename, obj_uri) return True
def step3(expnums, ccd, version, rate_min, rate_max, angle, width, field=None, prefix=None): '''run the actual step2 on the given exp/ccd combo''' jmp_args = ['step3jmp'] matt_args = ['step3matt'] idx = 0 cmd_args = [] for expnum in expnums: idx += 1 for ext in ['unid.jmp', 'unid.matt', 'trans.jmp']: filename = storage.get_image(expnum, ccd=ccd, version=version, ext=ext, prefix=prefix) image = os.path.splitext( os.path.splitext(os.path.basename(filename))[0])[0] cmd_args.append('-f%d' % (idx)) cmd_args.append(image) cmd_args.extend([ '-rn', str(rate_min), '-rx', str(rate_max), '-a', str(angle), '-w', str(width) ]) jmp_args.extend(cmd_args) matt_args.extend(cmd_args) util.exec_prog(jmp_args) util.exec_prog(matt_args) if field is None: field = str(expnums[0]) storage.mkdir( os.path.dirname( storage.get_uri(field, ccd=ccd, version=version, ext=ext, prefix=prefix))) for ext in ['moving.jmp', 'moving.matt']: uri = storage.get_uri(field, ccd=ccd, version=version, ext=ext, prefix=prefix) filename = '%s%d%s%s.%s' % (prefix, expnums[0], version, str(ccd).zfill(2), ext) storage.copy(filename, uri) return
def align(expnums, ccd, version='s', dry_run=False): """Create a 'shifts' file that transforms the space/flux/time scale of all images to the first image. This function relies on the .fwhm, .trans.jmp, .phot and .zeropoint.used files for inputs. The scaling we are computing here is for use in planting sources into the image at the same sky/flux locations while accounting for motions of sources with time. :param expnums: list of MegaPrime exposure numbers to add artificial KBOs to, the first frame in the list is the reference. :param ccd: which ccd to work on. :param version: Add sources to the 'o', 'p' or 's' images :param dry_run: don't push results to VOSpace. """ # Get the images and supporting files that we need from the VOSpace area # get_image and get_file check if the image/file is already on disk. # re-computed fluxes from the PSF stars and then recompute x/y/flux scaling. # some dictionaries to hold the various scale pos = {} apcor = {} mags = {} zmag = {} mjdates = {} for expnum in expnums: filename = storage.get_image(expnum, ccd=ccd, version=version) zmag[expnum] = storage.get_zeropoint(expnum, ccd, prefix=None, version=version) mjdates[expnum] = float(fits.open(filename)[0].header.get('MJD-OBS')) apcor[expnum] = [ float(x) for x in open( storage.get_file( expnum, ccd=ccd, version=version, ext=storage.APCOR_EXT)).read().split() ] keys = ['crval1', 'cd1_1', 'cd1_2', 'crval2', 'cd2_1', 'cd2_2'] # load the .trans.jmp values into a 'wcs' like dictionary. # .trans.jmp maps current frame to reference frame in pixel coordinates. # the reference frame of all the frames supplied must be the same. shifts = dict( zip(keys, [ float(x) for x in open( storage.get_file( expnum, ccd=ccd, version=version, ext='trans.jmp')).read().split() ])) shifts['crpix1'] = 0.0 shifts['crpix2'] = 0.0 # now create a wcs object based on those transforms, this wcs links the current frame's # pixel coordinates to the reference frame's pixel coordinates. w = get_wcs(shifts) # get the PHOT file that was produced by the mkpsf routine phot = ascii.read(storage.get_file(expnum, ccd=ccd, version=version, ext='phot'), format='daophot') # compute the small-aperture magnitudes of the stars used in the PSF mags[expnum] = daophot.phot(filename, phot['XCENTER'], phot['YCENTER'], aperture=apcor[expnum][0], sky=apcor[expnum][1] + 1, swidth=apcor[expnum][0], zmag=zmag[expnum]) # covert the x/y positions to positions in Frame 1 based on the trans.jmp values. (x, y) = w.wcs_pix2world(mags[expnum]["XCENTER"], mags[expnum]["YCENTER"], 1) pos[expnum] = numpy.transpose([x, y]) # match this exposures PSF stars position against those in the first image of the set. idx1, idx2 = util.match_lists(pos[expnums[0]], pos[expnum]) # compute the magnitdue offset between the current frame and the reference. dmags = numpy.ma.array(mags[expnums[0]]["MAG"] - apcor[expnums[0]][2] - (mags[expnum]["MAG"][idx1] - apcor[expnum][2]), mask=idx1.mask) dmags.sort() # compute the median and determine if that shift is small compared to the scatter. dmag = dmags[int(len(dmags) / 2.0)] if math.fabs(dmag) > 3 * (dmags.std() + 0.01): logging.warn( "Magnitude shift {} between {} and {} is large: {}".format( dmag, expnums[0], expnum, shifts[expnum])) shifts['dmag'] = dmag shifts['emag'] = dmags.std() shifts['nmag'] = len(dmags.mask) - dmags.mask.sum() shifts['dmjd'] = mjdates[expnums[0]] - mjdates[expnum] shift_file = os.path.basename( storage.get_uri(expnum, ccd, version, '.shifts')) fh = open(shift_file, 'w') fh.write( json.dumps(shifts, sort_keys=True, indent=4, separators=(',', ': '))) fh.write('\n') fh.close() if not dry_run: storage.copy( shift_file, os.path.basename( storage.get_uri(expnum, ccd, version, '.shifts')))
def step1(expnum, ccd, prefix='', version='p', fwhm=4, sex_thresh=1.3, wave_thresh=2.7, maxcount=30000): """run the actual step1jmp/matt codes. expnum: the CFHT expousre to process ccd: which ccd in the mosaic to process fwhm: the image quality, FWHM, of the image. In pixels. sex_thresh: the detection threhold to run sExtractor at wave_thresh: the detection threshold for wavelet maxcount: saturation level """ filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) mopheader = storage.get_image(expnum, ccd, version=version, ext='mopheader', prefix=prefix) fwhm = storage.get_fwhm(expnum, ccd, prefix=prefix, version=version) basename = os.path.splitext(filename)[0] outfile = util.exec_prog([ 'step1jmp', '-f', basename, '-t', str(wave_thresh), '-w', str(fwhm), '-m', str(maxcount) ]) obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.jmp', prefix=prefix) obj_filename = basename + ".obj.jmp" storage.copy(obj_filename, obj_uri) ## for step1matt we need the weight image flat_name = fits.open(filename)[0].header['FLAT'] flat_name = flat_name[0:-5] flat_filename = storage.get_image(flat_name, ccd, version='', ext='fits', subdir='calibrators', rescale=False) if not os.access('weight.fits', os.R_OK): os.symlink(flat_filename, 'weight.fits') outfile = util.exec_prog([ 'step1matt', '-f', basename, '-t', str(sex_thresh), '-w', str(fwhm), '-m', str(maxcount) ]) obj_uri = storage.get_uri(expnum, ccd, version=version, ext='obj.matt', prefix=prefix) obj_filename = basename + ".obj.matt" storage.copy(obj_filename, obj_uri) return True
def download_cutout(self, reading, focus=None, needs_apcor=False): """ Downloads a cutout of the FITS image for a given source reading. Args: source_reading: ossos.astrom.SourceReading The reading which will be the focus of the downloaded image. focus: tuple(int, int) The x, y coordinates that should be the focus of the downloaded image. These coordinates should be in terms of the source_reading parameter's coordinate system. Default value is None, in which case the source reading's x, y position is used as the focus. needs_apcor: bool If True, the apcor file with data needed for photometry calculations is downloaded in addition to the image. Defaults to False. Returns: cutout: ossos.downloads.data.SourceCutout """ if focus is None: focus = reading.source_point assert isinstance(reading, SourceReading) dx = dy = 2 * max(reading.dra, reading.ddec) dx = max(reading.dx, dx) dy = max(reading.dy, dy) (NAXIS1, NAXIS2) = reading.get_original_image_size() cutout_str, converter = self.cutout_calculator.build_cutout_str( reading.get_extension(), focus, (NAXIS1, NAXIS2), dx=dx, dy=dy, inverted=reading.is_inverted) image_uri = reading.get_image_uri() cutout = re.findall(r'(\d+)', cutout_str) y2 = int(cutout[-1]) y1 = int(cutout[-2]) logger.debug("Calculated cutout: %s for %s" % (cutout_str, image_uri)) hdulist = storage.get_image(expnum=reading.get_exposure_number(), ccd=reading.get_ccd_num(), cutout=cutout_str, version=reading.get_observation().ftype, prefix=reading.get_observation().fk, return_file=False) #hdulist = self.download_hdulist(image_uri, view="cutout", # cutout=cutout_str) # modify the DATASEC to account for possible flip/flop and changes in dimensions of the image. DATASEC = hdulist[0].header.get('DATASEC', None) if DATASEC is not None: datasec = re.findall(r'(\d+)', DATASEC) if y2 < y1: x2 = int(NAXIS1) - int(datasec[0]) + 1 x1 = int(NAXIS1) - int(datasec[1]) + 1 y2 = int(NAXIS2) - int(datasec[2]) + 1 y1 = int(NAXIS2) - int(datasec[3]) + 1 logger.debug( "Flip/Flopped DATASEC from {} to [{}:{}:{}:{}]".format( DATASEC, x1, x2, y1, y2)) datasec = (x1, x2, y1, y2) (x1, y1) = converter.convert((int(datasec[0]), int(datasec[2]))) x1 = max(1, x1) y1 = max(1, y1) (x2, y2) = converter.convert((int(datasec[1]), int(datasec[3]))) x2 = min(x2, int(hdulist[0].header['NAXIS1'])) y2 = min(y2, int(hdulist[0].header['NAXIS2'])) datasec = "[{}:{},{}:{}]".format(x1, x2, y1, y2) logger.debug("Trimmed and offset DATASEC from {} to {}".format( DATASEC, datasec)) hdulist[0].header['DATASEC'] = datasec apcor = None if needs_apcor: try: apcor = self.download_apcor(reading.get_apcor_uri()) except Exception as e: logger.error(str(e)) apcor = None zmag = None try: zmag = self.download_zmag(reading.get_zmag_uri()) except Exception as e: logger.error(str(e)) pass return SourceCutout(reading, hdulist, converter, apcor, zmag=zmag)
def combine(expnum, ccd, prefix=None, type='p'): for ext in ['moving.matt', 'moving.jmp']: fname = storage.get_image(expnum, ccd=ccd, prefix=prefix, version=type, ext=ext) if prefix is not None and len(prefix) > 0: planted = storage.get_image('Object', subdir=str(expnum) + "/ccd%s" % (str(ccd).zfill(2)), version='', ext='planted') else: prefix = '' base_image = os.path.basename( storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=type, ext=None)) cmd_args = ['comb-list', prefix + str(expnum) + type + str(ccd).zfill(2)] util.exec_prog(cmd_args) for ext in [ 'cands.comb', 'comb.found', 'comb.missed', 'jmp.found', 'jmp.missed', 'matt.found', 'matt.missed' ]: uri = storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=type, ext=ext) filename = os.path.basename(uri) if not os.access(filename, os.R_OK): logging.critical("No %s file" % (filename)) continue storage.copy(filename, uri) base_name = prefix + str(expnum) + type + str(ccd).zfill(2) cands_file = base_name + '.cands.comb' if not os.access(cands_file, os.R_OK): nocands_file = (prefix + str(expnum) + type + str(ccd).zfill(2) + '.no_candidates') open(nocands_file, 'w').close() storage.copy(nocands_file, 'vos:OSSOS/measure3/' + nocands_file) return storage.SUCCESS cands_file = mop_file.Parser().parse(cands_file) for file_id in cands_file.header.file_ids: rec_no = cands_file.header.file_ids.index(file_id) storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no], ccd=ccd, version=type, prefix=prefix, ext='fits') cmd_args = ['measure3', prefix + str(expnum) + type + str(ccd).zfill(2)] util.exec_prog(cmd_args) filename = base_name + ".measure3.cands.astrom" storage.copy(filename, 'vos:OSSOS/measure3/' + filename) return storage.SUCCESS
def plant(expnums, ccd, rmin, rmax, ang, width, version='s'): '''run the plant script on this combination of exposures''' ptf = open('proc-these-files', 'w') ptf.write("# Files to be planted and search\n") ptf.write("# image fwhm plant\n") for expnum in expnums: fwhm = storage.get_fwhm(expnum, ccd) filename = storage.get_image(expnum, ccd=ccd, version=version) ptf.write("%s %3.1f YES\n" % (filename[0:-5], fwhm)) for ext in [ 'apcor', 'obj.jmp', 'trans.jmp', 'psf.fits', 'mopheader', 'phot', 'zeropoint.used' ]: apcor = storage.get_image(expnum, ccd=ccd, version='s', ext=ext) ptf.close() cmd_args = [ 'plant.csh', os.curdir, str(rmin), str(rmax), str(ang), str(width) ] util.exec_prog(cmd_args) if args.dryrun: # Don't push back to VOSpace return uri = storage.get_uri('Object', ext='planted', version='', subdir=str(expnums[0]) + "/ccd%s" % (str(ccd).zfill(2))) storage.copy('Object.planted', uri) uri = os.path.join(os.path.dirname(uri), 'shifts') storage.copy('shifts', uri) for expnum in expnums: uri = storage.get_uri(expnum, ccd=ccd, version=version, ext='fits', prefix='fk') filename = os.path.basename(uri) storage.copy(filename, uri) for ext in [ 'mopheader', 'psf.fits', 'fwhm', 'apcor', 'zeropoint.used', 'trans.jmp' ]: storage.delete(expnum, ccd, 's', ext, prefix='fk') storage.vlink(expnum, ccd, 'p', ext, expnum, ccd, 's', ext, l_prefix='fk') return
def run(expnum, ccd, version, dry_run=False, prefix="", force=False): """Run the OSSOS jmpmakepsf script. """ message = storage.SUCCESS if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run): try: if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd): raise IOError("{} not yet run for {}".format(dependency, expnum)) # confirm destination directory exists. destdir = os.path.dirname( storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits')) if not dry_run: storage.mkdir(destdir) # get image from the vospace storage area logging.info("Getting fits image from VOSpace") filename = storage.get_image(expnum, ccd, version=version, prefix=prefix) # get mopheader from the vospace storage area logging.info("Getting mopheader from VOSpace") mopheader_filename = storage.get_file(expnum, ccd, version=version, prefix=prefix, ext='mopheader') # run mkpsf process logging.info("Running mkpsf on %s %d" % (expnum, ccd)) logging.info(util.exec_prog(['jmpmakepsf.csh', './', filename, 'yes', 'yes'])) if dry_run: return # place the results into VOSpace basename = os.path.splitext(filename)[0] for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor', 'fwhm', 'phot'): dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext) source = basename + "." + str(ext) count = 0 with open(source, 'r'): while True: count += 1 try: logging.info("Attempt {} to copy {} -> {}".format(count, source, dest)) storage.copy(source, dest) break except Exception as ex: if count > 10: raise ex # set some data parameters associated with the image, determined in this step. storage.set_status('fwhm', prefix, expnum, version=version, ccd=ccd, status=str(storage.get_fwhm( expnum, ccd=ccd, prefix=prefix, version=version))) storage.set_status('zeropoint', prefix, expnum, version=version, ccd=ccd, status=str(storage.get_zeropoint( expnum, ccd=ccd, prefix=prefix, version=version))) logging.info(message) except Exception as e: message = str(e) logging.error(message) storage.set_status(task, prefix, expnum, version, ccd=ccd, status=message) return
parser.add_argument('-r', '--replace', action='store_true', help='store modified image back to VOSpace?') parser.add_argument('-v', '--verbose', action='store_true') args = parser.parse_args() if args.verbose: logging.basicConfig(level=logging.INFO, format="%(message)s") message = storage.SUCCESS try: image = (os.access(args.expnum, os.W_OK) and args.expnum) or (storage.get_image(args.expnum)) header = (args.header is not None and ((os.access(args.header, os.W_OK) and args.header) or (storage.get_image(args.header, ext='head')))) or ( storage.get_image(args.expnum, ext='head')) logging.info("Swapping header for %s for contents in %s \n" % (image, header)) run_update_header(image, header) if args.replace: expnum = args.expnum or fits.open(image)[0].header['EXPNUM'] dest = storage.dbimages_uri(expnum) storage.copy(image, dest)
help='filename of replacement header') parser.add_argument('-r','--replace', action='store_true', help='store modified image back to VOSpace?') parser.add_argument('-v','--verbose', action='store_true') args = parser.parse_args() if args.verbose: logging.basicConfig(level=logging.INFO, format="%(message)s") message = storage.SUCCESS try: image = (os.access(args.expnum,os.W_OK) and args.expnum ) or ( storage.get_image(args.expnum) ) header = (args.header is not None and (( os.access(args.header, os.W_OK) and args.header ) or ( storage.get_image(args.header, ext='head')))) or ( storage.get_image(args.expnum, ext='head')) logging.info( "Swapping header for %s for contents in %s \n" % ( image, header) ) run_update_header(image, header) if args.replace: expnum = args.expnum or fits.open(image)[0].header['EXPNUM'] dest = storage.dbimages_uri(expnum)
def combine(expnum, ccd, prefix=None, file_type='p', field=None, measure3=MEASURE3, dry_run=False): if field is None: field = str(expnum) if prefix is not None and len(prefix) > 0: field = "%s_%s" % (prefix, field) field += "_%s%s" % (str(file_type), str(ccd)) logging.info("Doing combine on field {}".format(field)) for ext in ['moving.matt', 'moving.jmp']: storage.get_file(expnum, ccd=ccd, version=file_type, ext=ext, prefix=prefix) if prefix is not None and len(prefix) > 0: storage.get_file('Object', version='', ext='planted', subdir=str(expnum) + "/ccd%s" % (str(ccd).zfill(2))) else: prefix = '' cmd_args = [ 'comb-list', prefix + str(expnum) + file_type + str(ccd).zfill(2) ] logging.info(str(cmd_args)) logging.info(util.exec_prog(cmd_args)) ext_list = ['cands.comb'] if prefix is not None and len(prefix) > 0: ext_list.extend([ 'jmp.missed', 'matt.missed', 'jmp.found', 'matt.found', 'comb.missed', 'comb.found' ]) for ext in ext_list: uri = storage.get_uri(expnum, ccd=ccd, prefix=prefix, version=file_type, ext=ext) filename = os.path.basename(uri) if not os.access(filename, os.R_OK): logging.critical("No %s file" % filename) continue vospace_name = "%s.%s" % (field, ext) if not dry_run: logging.info("%s -> %s" % (filename, os.path.join(measure3, vospace_name))) storage.copy(filename, os.path.join(measure3, vospace_name)) base_name = prefix + str(expnum) + file_type + str(ccd).zfill(2) cands_file = base_name + '.cands.comb' if not os.access(cands_file, os.R_OK): no_cands_file = (prefix + str(expnum) + file_type + str(ccd).zfill(2) + '.no_candidates') open(no_cands_file, 'w').close() if not dry_run: vospace_name = "%s.no_candidates" % field storage.copy(no_cands_file, os.path.join(measure3, vospace_name)) return storage.SUCCESS # get the images we need to compute x/y ra/dec transforms cands_file = mop_file.Parser().parse(cands_file) for file_id in cands_file.header.file_ids: rec_no = cands_file.header.file_ids.index(file_id) storage.get_image(expnum=cands_file.header.keywords['EXPNUM'][rec_no], ccd=ccd, version=file_type, ext='fits', prefix=prefix) cmd_args = [ 'measure3', prefix + str(expnum) + file_type + str(ccd).zfill(2) ] logging.info("Running measure3") logging.info(util.exec_prog(cmd_args)) if not dry_run: filename = base_name + ".measure3.cands.astrom" vospace_filename = "%s.measure3.cands.astrom" % field storage.copy(filename, os.path.join(measure3, vospace_filename)) return storage.SUCCESS