def get_iq_and_zeropoint(image, header_extract):
# Options: it can be in the image tag, it can be in the file if that's been created.
fwhm = storage.get_tag(image, 'fwhm_22') # 22 is the standard chip: mid lower
if fwhm is None: # not in tag, maybe in the file: does that exist?
sys.stdout.write('...no fwhm vtag. Instead trying file. ')
try:
fwhm = storage.get_fwhm(image, ccd=22)
except: # no file yet then either
fwhm = None
sys.stdout.write('...fwhm not yet measured. ')
if fwhm is not None:
iq = float(fwhm) * 0.1850 # plate scale is 0.1850 arcsec/pixel
header_extract['iq_ossos'] = iq
else:
header_extract['iq_ossos'] = None # This will need to be updated in the future, then.
sys.stdout.write('Storing IQ: %s\n' % str(header_extract['iq_ossos']))
zeropt = storage.get_tag(image, 'zeropoint_22')
if zeropt is None:
sys.stdout.write('...no zeropoint vtag. Instead trying file. ')
try:
zeropt = storage.get_zeropoint(image, ccd=22)
except: # no file yet then either
zeropt = None
sys.stdout.write('...zeropoint not yet measured. ')
header_extract['zeropt'] = zeropt
sys.stdout.write('Storing zeropoint: %s\n' % str(header_extract['zeropt']))
return header_extract
def get_iq_and_zeropoint(image, header_extract):
# Options: it can be in the image tag, it can be in the file if that's been created.
fwhm = storage.get_tag(image,
'fwhm_22') # 22 is the standard chip: mid lower
if fwhm is None: # not in tag, maybe in the file: does that exist?
sys.stdout.write('...no fwhm vtag. Instead trying file. ')
try:
fwhm = storage.get_fwhm(image, ccd=22)
except: # no file yet then either
fwhm = None
sys.stdout.write('...fwhm not yet measured. ')
if fwhm is not None:
iq = float(fwhm) * 0.1850 # plate scale is 0.1850 arcsec/pixel
header_extract['iq_ossos'] = iq
else:
header_extract[
'iq_ossos'] = None # This will need to be updated in the future, then.
sys.stdout.write('Storing IQ: %s\n' % str(header_extract['iq_ossos']))
zeropt = storage.get_tag(image, 'zeropoint_22')
if zeropt is None:
sys.stdout.write('...no zeropoint vtag. Instead trying file. ')
try:
zeropt = storage.get_zeropoint(image, ccd=22)
except: # no file yet then either
zeropt = None
sys.stdout.write('...zeropoint not yet measured. ')
header_extract['zeropt'] = zeropt
sys.stdout.write('Storing zeropoint: %s\n' % str(header_extract['zeropt']))
return header_extract
def run(expnum, ccd, version, dry_run=False, prefix="", force=False):
"""Run the OSSOS jmpmakepsf script.
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version=version,
ccd=ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(
task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(
dependency, prefix, expnum, version, ccd=ccd):
raise IOError("{} not yet run for {}".format(
dependency, expnum))
# confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum,
ccd,
prefix=prefix,
version=version,
ext='fits'))
if not dry_run:
storage.mkdir(destdir)
# get image from the vospace storage area
logging.info("Getting fits image from VOSpace")
filename = storage.get_image(expnum,
ccd,
version=version,
prefix=prefix)
# get mopheader from the vospace storage area
logging.info("Getting mopheader from VOSpace")
mopheader_filename = storage.get_file(expnum,
ccd,
version=version,
prefix=prefix,
ext='mopheader')
# run mkpsf process
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
logging.info(
util.exec_prog(
['jmpmakepsf.csh', './', filename, 'yes', 'yes']))
if dry_run:
return
# place the results into VOSpace
basename = os.path.splitext(filename)[0]
for ext in ('mopheader', 'psf.fits', 'zeropoint.used', 'apcor',
'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum,
ccd,
prefix=prefix,
version=version,
ext=ext)
source = basename + "." + str(ext)
count = 0
with open(source, 'r'):
while True:
count += 1
try:
logging.info("Attempt {} to copy {} -> {}".format(
count, source, dest))
storage.copy(source, dest)
break
except Exception as ex:
if count > 10:
raise ex
# set some data parameters associated with the image, determined in this step.
storage.set_status('fwhm',
prefix,
expnum,
version=version,
ccd=ccd,
status=str(
storage.get_fwhm(expnum,
ccd=ccd,
prefix=prefix,
version=version)))
storage.set_status('zeropoint',
prefix,
expnum,
version=version,
ccd=ccd,
status=str(
storage.get_zeropoint(expnum,
ccd=ccd,
prefix=prefix,
version=version)))
logging.info(message)
except Exception as e:
message = str(e)
logging.error(message)
storage.set_status(task,
prefix,
expnum,
version,
ccd=ccd,
status=message)
return
@property
def uri(self):
return storage.get_uri(self.expnum, self.ccd, ext='apcor')
def get_sky(expnum, ccd):
uri = storage.get_uri(expnum, ccd, ext='phot')
fobj = StringIO(storage.open_vos_or_local(uri).read())
fobj.seek(0)
phot_table = ascii.read(fobj)
return phot_table['MSKY'].mean()
apcor = Apcor(expnum, ccd)
fwhm = storage.get_fwhm(expnum, ccd)
zp = storage.get_zeropoint(expnum, ccd)
sky = get_sky(expnum, ccd)
flux_24 = 10**(-((24.0 + apcor.ap_cor) - zp)/2.5)
flux_sky = math.pi * apcor.ap_in**2 * sky
snr = flux_24 / math.sqrt(flux_24 + flux_sky)
storage.set_tag(expnum, "snr_{:02d}".format(int(ccd)), "{:5.2f}".format(snr))
print expnum, ccd, sky, zp, fwhm, flux_sky, flux_24, snr
def main(task='mkpsf'):
parser = argparse.ArgumentParser(
description='Run makepsf chunk of the OSSOS pipeline')
parser.add_argument('--ccd',
'-c',
action='store',
type=int,
dest='ccd',
default=None,
help='which ccd to process, default is all')
parser.add_argument('--ignore-update-headers',
action='store_true',
dest='ignore_update_headers')
parser.add_argument("--dbimages",
action="store",
default="vos:OSSOS/dbimages",
help='vospace dbimages containerNode')
parser.add_argument("expnum",
type=int,
nargs='+',
help="expnum(s) to process")
parser.add_argument(
"--dry_run",
action="store_true",
help="DRY RUN, don't copy results to VOSpace, implies --force")
parser.add_argument("--fk", action="store_true", help="Run fk images")
parser.add_argument(
"--type",
"-t",
choices=['o', 'p', 's'],
help="which type of image: o-RAW, p-ELIXIR, s-SCRAMBLE",
default='p')
parser.add_argument("--verbose", "-v", action="store_true")
parser.add_argument("--force", default=False, action="store_true")
parser.add_argument("--debug", "-d", action="store_true")
args = parser.parse_args()
if args.dry_run:
args.force = True
if args.debug:
logging.basicConfig(level=logging.DEBUG)
elif args.verbose:
logging.basicConfig(level=logging.INFO)
prefix = (args.fk and 'fk') or ''
storage.DBIMAGES = args.dbimages
if args.ccd is None:
ccdlist = range(0, 36)
else:
ccdlist = [args.ccd]
exit_code = 0
for expnum in args.expnum:
for ccd in ccdlist:
storage.set_logger(
os.path.splitext(os.path.basename(sys.argv[0]))[0], prefix,
expnum, ccd, args.type, args.dry_run)
if storage.get_status(
expnum, ccd, prefix + task,
version=args.type) and not args.force:
logging.info(
"{} completed successfully for {} {} {} {}".format(
task, prefix, expnum, args.type, ccd))
continue
message = 'success'
try:
if not storage.get_status(
expnum, 36,
'update_header') and not args.ignore_update_headers:
raise IOError(
"update_header not yet run for {}".format(expnum))
mkpsf(expnum, ccd, args.type, args.dry_run, prefix=prefix)
if args.dry_run:
continue
storage.set_status(expnum,
ccd,
prefix + 'fwhm',
version=args.type,
status=str(
storage.get_fwhm(expnum,
ccd,
version=args.type)))
storage.set_status(expnum,
ccd,
prefix + 'zeropoint',
version=args.type,
status=str(
storage.get_zeropoint(
expnum, ccd, version=args.type)))
except CalledProcessError as cpe:
message = str(cpe.output)
exit_code = message
except Exception as e:
message = str(e)
logging.error(message)
if not args.dry_run:
storage.set_status(expnum,
ccd,
prefix + 'mkpsf',
version=args.type,
status=message)
return exit_code
@property
def ap_cor(self):
return float(self.apcor_array[2])
@property
def uri(self):
return storage.get_uri(self.expnum, self.ccd, ext='apcor')
def get_sky(expnum, ccd):
uri = storage.get_uri(expnum, ccd, ext='phot')
fobj = StringIO(storage.open_vos_or_local(uri).read())
fobj.seek(0)
phot_table = ascii.read(fobj)
return phot_table['MSKY'].mean()
apcor = Apcor(expnum, ccd)
fwhm = storage.get_fwhm(expnum, ccd)
zp = storage.get_zeropoint(expnum, ccd)
sky = get_sky(expnum, ccd)
flux_24 = 10**(-((24.0 + apcor.ap_cor) - zp) / 2.5)
flux_sky = math.pi * apcor.ap_in**2 * sky
snr = flux_24 / math.sqrt(flux_24 + flux_sky)
storage.set_tag(expnum, "snr_{:02d}".format(int(ccd)), "{:5.2f}".format(snr))
print expnum, ccd, sky, zp, fwhm, flux_sky, flux_24, snr
if storage.get_status(expnum, ccd, 'mkpsf', version=args.type) and not args.force:
logging.info("Already did %s %s, skipping" %( str(expnum),
str(ccd)))
continue
try:
message = 'success'
mkpsf(expnum, ccd, args.type)
storage.set_status(expnum,
ccd,
'fwhm',
version=args.type,
status=str(storage.get_fwhm(
expnum, ccd, version=args.type)))
storage.set_status(expnum,
ccd,
'zeropoint',
version=args.type,
status=str(storage.get_zeropoint(
expnum, ccd, version=args.type)))
except Exception as e:
message = str(e)
logging.error(message)
storage.set_status( expnum,
ccd,
'mkpsf',
version=args.type,
status=message)
def align(expnums, ccd, version='s', dry_run=False):
"""Create a 'shifts' file that transforms the space/flux/time scale of all images to the first image.
This function relies on the .fwhm, .trans.jmp, .phot and .zeropoint.used files for inputs.
The scaling we are computing here is for use in planting sources into the image at the same sky/flux locations
while accounting for motions of sources with time.
:param expnums: list of MegaPrime exposure numbers to add artificial KBOs to,
the first frame in the list is the reference.
:param ccd: which ccd to work on.
:param version: Add sources to the 'o', 'p' or 's' images
:param dry_run: don't push results to VOSpace.
"""
# Get the images and supporting files that we need from the VOSpace area
# get_image and get_file check if the image/file is already on disk.
# re-computed fluxes from the PSF stars and then recompute x/y/flux scaling.
# some dictionaries to hold the various scale
pos = {}
apcor = {}
mags = {}
zmag = {}
mjdates = {}
for expnum in expnums:
filename = storage.get_image(expnum, ccd=ccd, version=version)
zmag[expnum] = storage.get_zeropoint(expnum,
ccd,
prefix=None,
version=version)
mjdates[expnum] = float(fits.open(filename)[0].header.get('MJD-OBS'))
apcor[expnum] = [
float(x) for x in open(
storage.get_file(
expnum, ccd=ccd, version=version,
ext=storage.APCOR_EXT)).read().split()
]
keys = ['crval1', 'cd1_1', 'cd1_2', 'crval2', 'cd2_1', 'cd2_2']
# load the .trans.jmp values into a 'wcs' like dictionary.
# .trans.jmp maps current frame to reference frame in pixel coordinates.
# the reference frame of all the frames supplied must be the same.
shifts = dict(
zip(keys, [
float(x) for x in open(
storage.get_file(
expnum, ccd=ccd, version=version,
ext='trans.jmp')).read().split()
]))
shifts['crpix1'] = 0.0
shifts['crpix2'] = 0.0
# now create a wcs object based on those transforms, this wcs links the current frame's
# pixel coordinates to the reference frame's pixel coordinates.
w = get_wcs(shifts)
# get the PHOT file that was produced by the mkpsf routine
phot = ascii.read(storage.get_file(expnum,
ccd=ccd,
version=version,
ext='phot'),
format='daophot')
# compute the small-aperture magnitudes of the stars used in the PSF
mags[expnum] = daophot.phot(filename,
phot['XCENTER'],
phot['YCENTER'],
aperture=apcor[expnum][0],
sky=apcor[expnum][1] + 1,
swidth=apcor[expnum][0],
zmag=zmag[expnum])
# covert the x/y positions to positions in Frame 1 based on the trans.jmp values.
(x, y) = w.wcs_pix2world(mags[expnum]["XCENTER"],
mags[expnum]["YCENTER"], 1)
pos[expnum] = numpy.transpose([x, y])
# match this exposures PSF stars position against those in the first image of the set.
idx1, idx2 = util.match_lists(pos[expnums[0]], pos[expnum])
# compute the magnitdue offset between the current frame and the reference.
dmags = numpy.ma.array(mags[expnums[0]]["MAG"] - apcor[expnums[0]][2] -
(mags[expnum]["MAG"][idx1] - apcor[expnum][2]),
mask=idx1.mask)
dmags.sort()
# compute the median and determine if that shift is small compared to the scatter.
dmag = dmags[int(len(dmags) / 2.0)]
if math.fabs(dmag) > 3 * (dmags.std() + 0.01):
logging.warn(
"Magnitude shift {} between {} and {} is large: {}".format(
dmag, expnums[0], expnum, shifts[expnum]))
shifts['dmag'] = dmag
shifts['emag'] = dmags.std()
shifts['nmag'] = len(dmags.mask) - dmags.mask.sum()
shifts['dmjd'] = mjdates[expnums[0]] - mjdates[expnum]
shift_file = os.path.basename(
storage.get_uri(expnum, ccd, version, '.shifts'))
fh = open(shift_file, 'w')
fh.write(
json.dumps(shifts,
sort_keys=True,
indent=4,
separators=(',', ': ')))
fh.write('\n')
fh.close()
if not dry_run:
storage.copy(
shift_file,
os.path.basename(
storage.get_uri(expnum, ccd, version, '.shifts')))
def align(expnums, ccd, version='s', prefix='', dry_run=False, force=True):
"""Create a 'shifts' file that transforms the space/flux/time scale of all
images to the first image.
This function relies on the .fwhm, .trans.jmp, .phot and .zeropoint.used files for inputs.
The scaling we are computing here is for use in planting sources into the image at the same sky/flux locations
while accounting for motions of sources with time.
:param expnums: list of MegaPrime exposure numbers to add artificial KBOs to,
the first frame in the list is the reference.
:param ccd: which ccd to work on.
:param prefix: put this string in front of expnum when looking for exposure, normally '' or 'fk'
:param force: When true run task even if this task is recorded as having succeeded
:param version: Add sources to the 'o', 'p' or 's' images
:param dry_run: don't push results to VOSpace.
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnums[0], version,
ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(
task, prefix, expnums[0], version, ccd))
return
# Get the images and supporting files that we need from the VOSpace area
# get_image and get_file check if the image/file is already on disk.
# re-computed fluxes from the PSF stars and then recompute x/y/flux scaling.
# some dictionaries to hold the various scale
pos = {}
apcor = {}
mags = {}
zmag = {}
mjdates = {}
with storage.LoggingManager(task, prefix, expnums[0], ccd, version,
dry_run):
try:
for expnum in expnums:
filename = storage.get_image(expnum, ccd=ccd, version=version)
zmag[expnum] = storage.get_zeropoint(expnum,
ccd,
prefix=None,
version=version)
mjdates[expnum] = float(
fits.open(filename)[0].header.get('MJD-OBS'))
apcor[expnum] = [
float(x) for x in open(
storage.get_file(
expnum,
ccd=ccd,
version=version,
ext=storage.APCOR_EXT)).read().split()
]
keys = ['crval1', 'cd1_1', 'cd1_2', 'crval2', 'cd2_1', 'cd2_2']
# load the .trans.jmp values into a 'wcs' like dictionary.
# .trans.jmp maps current frame to reference frame in pixel coordinates.
# the reference frame of all the frames supplied must be the same.
shifts = dict(
list(
zip(keys, [
float(x) for x in open(
storage.get_file(
expnum,
ccd=ccd,
version=version,
ext='trans.jmp')).read().split()
])))
shifts['crpix1'] = 0.0
shifts['crpix2'] = 0.0
# now create a wcs object based on those transforms, this wcs links the current frame's
# pixel coordinates to the reference frame's pixel coordinates.
w = get_wcs(shifts)
# get the PHOT file that was produced by the mkpsf routine
logging.debug("Reading .phot file {}".format(expnum))
phot = ascii.read(storage.get_file(expnum,
ccd=ccd,
version=version,
ext='phot'),
format='daophot')
# compute the small-aperture magnitudes of the stars used in the PSF
logging.debug("Running phot on {}".format(filename))
mags[expnum] = daophot.phot(filename,
phot['XCENTER'],
phot['YCENTER'],
aperture=apcor[expnum][0],
sky=apcor[expnum][1] + 1,
swidth=apcor[expnum][0],
zmag=zmag[expnum])
# covert the x/y positions to positions in Frame 1 based on the trans.jmp values.
logging.debug(
"Doing the XY translation to refrence frame: {}".format(w))
(x, y) = w.wcs_pix2world(mags[expnum]["XCENTER"],
mags[expnum]["YCENTER"], 1)
pos[expnum] = numpy.transpose([x, y])
# match this exposures PSF stars position against those in the first image of the set.
logging.debug("Matching lists")
idx1, idx2 = util.match_lists(pos[expnums[0]], pos[expnum])
# compute the magnitdue offset between the current frame and the reference.
dmags = numpy.ma.array(
mags[expnums[0]]["MAG"] - apcor[expnums[0]][2] -
(mags[expnum]["MAG"][idx1] - apcor[expnum][2]),
mask=idx1.mask)
dmags.sort()
# logging.debug("Computed dmags between input and reference: {}".format(dmags))
error_count = 0
error_count += 1
logging.debug("{}".format(error_count))
# compute the median and determine if that shift is small compared to the scatter.
try:
midx = int(
numpy.sum(numpy.any([~dmags.mask], axis=0)) / 2.0)
dmag = float(dmags[midx])
logging.debug("Computed a mag delta of: {}".format(dmag))
except Exception as e:
logging.error(str(e))
logging.error(
"Failed to compute mag offset between plant and found using: {}"
.format(dmags))
dmag = 99.99
error_count += 1
logging.debug("{}".format(error_count))
try:
if math.fabs(dmag) > 3 * (dmags.std() + 0.01):
logging.warning(
"Magnitude shift {} between {} and {} is large: {}"
.format(dmag, expnums[0], expnum, shifts))
except Exception as e:
logging.error(str(e))
error_count += 1
logging.debug("{}".format(error_count))
shifts['dmag'] = dmag
shifts['emag'] = dmags.std()
shifts['nmag'] = len(dmags.mask) - dmags.mask.sum()
shifts['dmjd'] = mjdates[expnums[0]] - mjdates[expnum]
shift_file = os.path.basename(
storage.get_uri(expnum, ccd, version, '.shifts'))
error_count += 1
logging.debug("{}".format(error_count))
try:
fh = open(shift_file, 'w')
fh.write(
json.dumps(shifts,
sort_keys=True,
indent=4,
separators=(',', ': '),
cls=NpEncoder))
fh.write('\n')
fh.close()
except Exception as e:
logging.error(
"Creation of SHIFTS file failed while trying to write: {}"
.format(shifts))
raise e
error_count += 1
logging.debug("{}".format(error_count))
if not dry_run:
storage.copy(
shift_file,
storage.get_uri(expnum, ccd, version, '.shifts'))
logging.info(message)
except Exception as ex:
message = str(ex)
logging.error(message)
if not dry_run:
storage.set_status(task,
prefix,
expnum,
version,
ccd,
status=message)
def recompute_mag(mpc_in):
"""
Get the mag of the object given the mpc.Observation
"""
# TODO this really shouldn't need to build a 'reading' to get the cutout...
from ossos.downloads.cutouts import downloader
dlm = downloader.ImageCutoutDownloader()
mpc_obs = deepcopy(mpc_in)
assert isinstance(mpc_obs, mpc.Observation)
if mpc_obs.null_observation:
return mpc_obs
parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame)
if parts is None:
return mpc_obs
expnum = parts.group('expnum')
ccd = parts.group('ccd')
file_type = parts.group('type')
observation = astrom.Observation(expnum, file_type, ccd)
assert isinstance(observation, astrom.Observation)
ast_header = storage._get_sghead(int(expnum))[int(ccd)+1]
filter_value = None
for keyword in ['FILTER', 'FILT1 NAME']:
filter_value = ast_header.get(keyword, None)
if filter_value is not None:
if filter_value.startswith('gri'):
filter_value = 'w'
else:
filter_value = filter_value[0]
break
# The ZP for the current astrometric lines is the pipeline one. The new ZP is in the astheader file.
new_zp = ast_header.get('PHOTZP')
# The .zeropoint.used value is likely the one used for the original photometry.
old_zp = storage.get_zeropoint(int(expnum), int(ccd))
reading = astrom.SourceReading(float(mpc_obs.comment.x), float(mpc_obs.comment.y), float(mpc_obs.comment.x),
float(mpc_obs.comment.y), mpc_obs.coordinate.ra.degree,
mpc_obs.coordinate.dec.degree, float(mpc_obs.comment.x), float(mpc_obs.comment.y),
observation, ssos=True, from_input_file=True, null_observation=False,
discovery=mpc_obs.discovery)
cutout = dlm.download_cutout(reading, needs_apcor=True)
cutout._zmag = new_zp
# try:
# if mpc_in.discovery and ccdnum < 18 or ccdnum in [36, 37]:
# logging.warn("This is a discovery line so flipping/flopping the x/y position "
# "recorded in comment as that is likely taken from a flip/flopped image, and we now work"
# "in unflip/flop frame" )
# # Get the full header of the image so we can flip/flop the coordinates
# header = storage._get_sghead(parts.group('expnum'))[ccdnum + 1]
# # Get the X/Y position of the source from the MPC line comment.
# x = mpc_obs.comment.x
# y = mpc_obs.comment.y
# # Do the coordinate flip/flop
# x = header['NAXIS1'] - x + 1
# y = header['NAXIS2'] - y + 1
# # Now x/y are in the full frame of the MEF image which is stored in cutout.
# # so we transform to the pixel coordinates of the cutout.
# (x_p, y_p, hdu_list_idx) = cutout.get_pixel_coordinates((x, y), ccdnum)
# # Now turn those in RA/DEC coordinates.
# mpc_obs.coordinate = cutout.pix2world(x_p, y_p, hdu_list_idx)
# if mpc_in.coordinate.separation(mpc_obs.coordinate) > TOLERANCE:
# logger.warning("Flipping coordinates don't match RA/DEC. reverting to oringal values.")
# # Try reverting back to the orignal X/Y pixel locations from the input astrometry line.
# (x_p, y_p, hdu_list_idx) = cutout.get_pixel_coordinates((mpc_obs.comment.x, mpc_in.comment.y), ccdnum)
# mpc_obs.coordinate = cutout.pix2world(x_p, y_p, hdu_list_idx)
# if mpc_in.coordinate.separation(mpc_obs.coordinate) > TOLERANCE:
# # Something is not good.
# raise ValueError("Can't determine x/y to RA/DEC mapping on discovery images.")
# else:
# logger.warn("Updating X/Y to use the not flipped coordinates frame.")
# # OK, we have the correct pixel coordinates for the cutout we are looking, store the orignal observerd coord
# cutout.update_pixel_location((x_p, y_p), hdu_list_idx)
# except Exception as ex:
# logging.warn(ex)
# logging.warn("Failed to get the X Y coordinate from comment line.")
# return mpc_in
#
if math.fabs(cutout.zmag - old_zp) > 0.3:
logging.warn("Large change in zeropoint detected: {} -> {}".format(old_zp, cutout.zmag))
try:
PHOT = cutout.get_observed_magnitude()
x = PHOT['XCENTER']
y = PHOT['YCENTER']
mag = PHOT['MAG']
merr = PHOT['MERR']
cutout.update_pixel_location((x, y), hdu_index=cutout.extno)
x, y = cutout.observed_source_point
except Exception as ex:
logging.error("ERROR: {}".format(str(ex)))
logging.warn("Failed to do photometry.")
return mpc_obs
if mpc_obs.comment.mag is not None and math.fabs(mpc_obs.comment.mag - mag) > 3.5 * mpc_obs.comment.mag_uncertainty:
logging.warn("recomputed magnitude shift large: {} --> {}".format(mpc_obs.mag, mag[0]))
if math.sqrt((x.value - mpc_obs.comment.x) ** 2 + (y.value - mpc_obs.comment.y) ** 2) > 1.9:
logging.warn("Centroid shifted ({},{}) -> ({},{})".format(mpc_obs.comment.x,
mpc_obs.comment.y,
x.value,
y.value))
# Don't use the new X/Y for Hand measured entries.
if str(mpc_obs.note1) != "H":
mpc_obs.comment.x = x.value
mpc_obs.comment.y = y.value
mpc_obs._band = filter_value
mpc_obs.comment.mag = mag
mpc_obs.comment.mag_uncertainty = merr
logging.warn("Previous mag: {} {}".format( mpc_in.mag,mpc_in.comment.photometry_note))
if mpc_obs.mag is not None or mpc_in.comment.photometry_note[0] == "Z":
mpc_obs.mag = mag
logging.warn("Resulting mag: {} {}".format( mpc_obs.mag,mpc_obs.comment.photometry_note))
return mpc_obs
for expnum in args.expnum:
for ccd in ccdlist:
if storage.get_status(expnum, ccd, 'mkpsf') and not args.force:
logging.info("Already did %s %s, skipping" %( str(expnum),
str(ccd)))
continue
try:
message = 'success'
mkpsf(expnum, ccd)
storage.set_status(expnum,
ccd,
'fwhm',
str(storage.get_fwhm(
expnum, ccd)))
storage.set_status(expnum,
ccd,
'zeropoint',
str(storage.get_zeropoint(
expnum, ccd)))
except Exception as e:
message = str(e)
logging.error(message)
storage.set_status( expnum,
ccd,
'mkpsf',
message)
def recompute_mag(mpc_in):
"""
Get the mag of the object given the mpc.Observation
"""
# TODO this really shouldn't need to build a 'reading' to get the cutout...
from ossos.downloads.cutouts import ImageCutoutDownloader
mpc_obs = deepcopy(mpc_in)
assert isinstance(mpc_obs, mpc.Observation)
if mpc_obs.null_observation:
return mpc_obs
parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame)
if parts is None:
return mpc_obs
expnum = parts.group('expnum')
ccd = parts.group('ccd')
file_type = parts.group('type')
observation = astrom.Observation(expnum, file_type, ccd)
assert isinstance(observation, astrom.Observation)
ast_header = storage.get_astheader(int(expnum), int(ccd))
# The ZP for the current astrometric lines is the pipeline one. The new ZP is in the astheader file.
new_zp = ast_header.get('PHOTZP')
# The ZP in the image header is likely the one used for the original photometry.
old_zp = storage.get_zeropoint(int(expnum), int(ccd))
reading = astrom.SourceReading(float(mpc_obs.comment.x), float(mpc_obs.comment.y), float(mpc_obs.comment.x),
float(mpc_obs.comment.y), mpc_obs.coordinate.ra.degrees,
mpc_obs.coordinate.dec.degrees, float(mpc_obs.comment.x),
float(mpc_obs.comment.y),
observation, ssos=True, from_input_file=True, null_observation=False,
discovery=mpc_obs.discovery)
# reading.is_inverted = reading.compute_inverted()
reading._header = storage.get_mopheader(expnum, ccd)
reading._header['MAXCOUNT'] = 30000.0
image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100)
cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True)
cutout.zmag = new_zp
if math.fabs(new_zp - old_zp) > 0.3:
logging.warning("Large change in zeropoint detected: {} -> {}".format(old_zp, new_zp))
try:
(x, y, mag, merr) = cutout.get_observed_magnitude(zmag=new_zp)
(x, y) = cutout.get_observed_coordinates((x, y))
except:
logging.warn("Failed to do photometry.")
return mpc_obs
if mpc_obs.comment.mag is not None and math.fabs(mpc_obs.comment.mag - mag) > 3.5 * mpc_obs.comment.mag_uncertainty:
logging.warn("recomputed magnitude shift large: {} --> {}".format(mpc_obs.mag, mag))
if math.sqrt((x - mpc_obs.comment.x) ** 2 + (y - mpc_obs.comment.y) ** 2) > 1.0:
logging.warn("Centroid shifted ({},{}) -> ({},{})".format(mpc_obs.comment.x,
mpc_obs.comment.y,
x,
y))
# Don't use the new X/Y for Hand measured entries.
if str(mpc_obs.note1) != "H":
mpc_obs.comment.x = x
mpc_obs.comment.y = y
mpc_obs.comment.mag = mag
mpc_obs.comment.mag_uncertainty = merr
if mpc_obs.mag is not None:
mpc_obs.mag = mag
return mpc_obs
def main(task='mkpsf'):
parser = argparse.ArgumentParser(
description='Run makepsf chunk of the OSSOS pipeline')
parser.add_argument('--ccd', '-c',
action='store',
type=int,
dest='ccd',
default=None,
help='which ccd to process, default is all')
parser.add_argument("--dbimages",
action="store",
default="vos:OSSOS/dbimages",
help='vospace dbimages containerNode')
parser.add_argument("expnum",
type=int,
nargs='+',
help="expnum(s) to process")
parser.add_argument("--dry-run",
action="store_true",
help="DRY RUN, don't copy results to VOSpace, implies --force")
parser.add_argument("--fk", action="store_true", help="Run fk images")
parser.add_argument("--type", "-t", choices=['o', 'p', 's'],
help="which type of image: o-RAW, p-ELIXIR, s-SCRAMBLE", default='p')
parser.add_argument("--verbose", "-v",
action="store_true")
parser.add_argument("--force", default=False,
action="store_true")
parser.add_argument("--debug", "-d",
action="store_true")
args = parser.parse_args()
util.set_logger(args)
prefix = (args.fk and 'fk') or ''
task = util.task()
dependency = 'mk_mopheader'
storage.DBIMAGES = args.dbimages
exit_code = 0
for expnum in args.expnum:
if args.ccd is None:
if int(expnum) < 1785619:
# Last exposures with 36 CCD Megaprime
ccdlist = range(0,36)
else:
# First exposrues with 40 CCD Megaprime
ccdlist = range(0, 40)
else:
ccdlist = [args.ccd]
for ccd in ccdlist:
if storage.get_status(task, prefix, expnum, version=args.type, ccd=ccd) and not args.force:
logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, args.type, ccd))
continue
storage.set_logger(task,
prefix, expnum, ccd, args.type, args.dry_run)
message = 'success'
try:
if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd):
raise IOError("{} not yet run for {}".format(dependency, expnum))
mkpsf(expnum, ccd, args.type, args.dry_run, prefix=prefix)
if args.dry_run:
continue
storage.set_status('fwhm', prefix, expnum, version=args.type, ccd=ccd, status=str(storage.get_fwhm(
expnum, ccd, version=args.type)))
storage.set_status('zeropoint', prefix, expnum, version=args.type, ccd=ccd,
status=str(storage.get_zeropoint(
expnum, ccd, version=args.type)))
except Exception as e:
message = str(e)
logging.error(message)
if not args.dry_run:
storage.set_status(task, prefix, expnum, version=args.type, ccd=ccd, status=message)
return exit_code
def run(expnum, ccd, version, dry_run=False, prefix="", force=False):
"""Run the OSSOS jmpmakepsf script.
"""
message = storage.SUCCESS
if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task, prefix, expnum, ccd, version, dry_run):
try:
if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd):
raise IOError("{} not yet run for {}".format(dependency, expnum))
# confirm destination directory exists.
destdir = os.path.dirname(
storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext='fits'))
if not dry_run:
storage.mkdir(destdir)
# get image from the vospace storage area
logging.info("Getting fits image from VOSpace")
filename = storage.get_image(expnum, ccd, version=version, prefix=prefix)
# get mopheader from the vospace storage area
logging.info("Getting mopheader from VOSpace")
mopheader_filename = storage.get_file(expnum, ccd, version=version, prefix=prefix, ext='mopheader')
# run mkpsf process
logging.info("Running mkpsf on %s %d" % (expnum, ccd))
logging.info(util.exec_prog(['jmpmakepsf.csh',
'./',
filename,
'yes', 'yes']))
if dry_run:
return
# place the results into VOSpace
basename = os.path.splitext(filename)[0]
for ext in ('mopheader', 'psf.fits',
'zeropoint.used', 'apcor', 'fwhm', 'phot'):
dest = storage.dbimages_uri(expnum, ccd, prefix=prefix, version=version, ext=ext)
source = basename + "." + str(ext)
count = 0
with open(source, 'r'):
while True:
count += 1
try:
logging.info("Attempt {} to copy {} -> {}".format(count, source, dest))
storage.copy(source, dest)
break
except Exception as ex:
if count > 10:
raise ex
# set some data parameters associated with the image, determined in this step.
storage.set_status('fwhm', prefix, expnum, version=version, ccd=ccd, status=str(storage.get_fwhm(
expnum, ccd=ccd, prefix=prefix, version=version)))
storage.set_status('zeropoint', prefix, expnum, version=version, ccd=ccd,
status=str(storage.get_zeropoint(
expnum, ccd=ccd, prefix=prefix, version=version)))
logging.info(message)
except Exception as e:
message = str(e)
logging.error(message)
storage.set_status(task, prefix, expnum, version, ccd=ccd, status=message)
return