def get_header(image):
retval = None
header_text = storage.get_astheader(image, 0) # zeroth ccd: first see if the preproc'd version is there
if header_text is None:
header_text = storage.get_astheader(image, 0, version='o') # 'o' for an object acquisition
if header_text is not None: # checked twice, it must be there now if it's there at all...
retval = wanted_keys(header_text)
return retval, header_text
def get_header(image):
retval = None
header_text = storage.get_astheader(image, 0) # zeroth ccd: first see if the preproc'd version is there
if header_text is None:
header_text = storage.get_astheader(image, 0, version='o') # 'o' for an object acquisition
if header_text is not None: # checked twice, it must be there now if it's there at all...
retval = wanted_keys(header_text)
return retval, header_text
def get_coord_offset(self, expnum, ccd, X, Y, ref_expnum, ref_ccd):
# determine offsets to reference reference frame using wcs
astheader = get_astheader(expnum, ccd)
ref_astheader = get_astheader(ref_expnum, ref_ccd)
ref_pvwcs = wcs.WCS(ref_astheader)
pvwcs = wcs.WCS(astheader)
(ra, dec) = pvwcs.xy2sky(X, Y)
(x0, y0) = ref_pvwcs.sky2xy(ra, dec)
logger.debug("{}p{} {},{} -> {}p{} {},{}".format(expnum, ccd,
X, Y,
ref_expnum, ref_ccd,
x0, y0))
return (x0, y0)
def get_coord_offset(self, expnum, ccd, X, Y, ref_expnum, ref_ccd):
# determine offsets to reference reference frame using wcs
astheader = get_astheader(expnum, ccd)
ref_astheader = get_astheader(ref_expnum, ref_ccd)
ref_pvwcs = wcs.WCS(ref_astheader)
pvwcs = wcs.WCS(astheader)
(ra,dec) = pvwcs.xy2sky(X, Y)
(x0, y0) = ref_pvwcs.sky2xy(ra,dec)
logger.debug("{}p{} {},{} -> {}p{} {},{}".format(expnum,ccd,
X,Y,
ref_expnum, ref_ccd,
x0,y0))
return (x0, y0)
def ossos_pointings(self):
"""
plot an OSSOS observation on the OSSOS plot.
"""
match = re.match('(\d+)\D(\d+)', self.expnum.get())
if match is not None:
expnum = int(match.group(1))
ccd = int(match.group(2))
x = 2112 / 2.0
y = 4644 / 2.0
else:
expnum = int(str(self.expnum.get()))
ccd = 22
x = 1000
y = 4644 - 15 / 0.185
header = None
try:
header = storage.get_astheader(expnum, ccd=ccd)
except:
if header is None:
print "Didn't get a header... "
return
ossos_wcs = wcs.WCS(header)
(ra, dec) = ossos_wcs.xy2sky(x, y)
class MyEvent(object):
def __init__(self, x, y):
self.x = x
self.y = y
(x, y) = self.p2s((math.radians(ra), math.radians(dec)))
event = MyEvent(x, y)
self.create_pointing(event, label_text=header['OBJECT'] + ' ccd{}'.format(ccd))
def main(expnum, ccd):
header = storage.get_astheader(expnum, ccd)
datasec = storage.datasec_to_list(header.get('DATASEC', '[80:2080,30,4160]'))
try:
fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd))
except:
fwhm = 'unknown'
for keyword in del_keyword_list:
try:
del(header[keyword])
except:
pass
header['FWHM'] = (fwhm, 'FWHM in pixels')
header['EXTNAME'] = 'header'
primary_hdu = fits.PrimaryHDU(header=header)
hdu_list = fits.HDUList([primary_hdu, ])
for ext in ['jmp', 'matt']:
extension = 'obj.'+ext
name = "{}p{:02d}.{}".format(expnum, ccd, extension)
try:
os.unlink(name)
os.unlink(name+".fits")
except:
pass
logging.info("Retrieving {}".format(name))
obj_file = mop_file.Parser(expnum, ccd, extension)
obj_file.parse()
t = numpy.all([datasec[0] < obj_file.data['X'], obj_file.data['X'] < datasec[1],
datasec[2] < obj_file.data['Y'], obj_file.data['Y'] < datasec[3]], axis=0)
logging.info("Source remaining after datasec cut: {} of {}".format(len(obj_file.data[t]['X']), len(t)))
table_hdu = fits.table_to_hdu(obj_file.data[t])
table_hdu.header['CATALOG'] = name
table_hdu.header['EXTNAME'] = ext
hdu_list.append(table_hdu)
del table_hdu
del obj_file
os.unlink(name)
name = "{}p{:02d}.{}".format(expnum, ccd, 'obj.fits')
if os.access(name, os.F_OK):
os.unlink(name)
hdu_list.writeto(name)
uri = storage.dbimages_uri(expnum, ccd, 'p', ext=".obj.fits")
logging.info(name+" -> "+uri)
count = 0
while True:
print("Copy attempt {}".format(count))
try:
storage.copy(name, uri)
os.unlink(name)
break
except Exception as ex:
if count > 10:
raise ex
count += 1
def is_inverted(self):
"""
Returns:
inverted: bool
True if the stored image is inverted.
"""
astheader = storage.get_astheader(self.obs.expnum, self.obs.ccdnum, version=self.obs.ftype)
pvwcs = wcs.WCS(astheader)
(x,y) = pvwcs.sky2xy(self.ra, self.dec)
logger.debug("is_inverted: X,Y {},{} -> wcs X,Y {},{}".format(self.x, self.y, x, y))
dr2 = ((x-self.x)**2 + (y-self.y)**2)
logger.debug("inverted is {}".format(dr2>2))
return dr2 > 2
def compute_inverted(self):
"""
Returns:
inverted: bool
True if the stored image is inverted.
"""
astheader = storage.get_astheader(self.obs.expnum,
self.obs.ccdnum,
version=self.obs.ftype)
pvwcs = wcs.WCS(astheader)
(x, y) = pvwcs.sky2xy(self.ra, self.dec)
logger.debug("is_inverted: X,Y {},{} -> wcs X,Y {},{}".format(
self.x, self.y, x, y))
dr2 = ((x - self.x)**2 + (y - self.y)**2)
logger.debug("inverted is {}".format(dr2 > 2))
return dr2 > 2
def remeasure(mpc_obs):
"""
re-measure the astrometry and photometry of the given mpc line
"""
# TODO Actually implement this.
if mpc_obs.null_observation:
return mpc_obs
assert isinstance(mpc_obs.comment, mpc.MPCComment)
parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)',mpc_obs.comment.frame)
header = storage.get_astheader(parts.group('expnum'),int(parts.group('ccd')))
this_wcs = wcs.WCS(header)
x = float(mpc_obs.comment.X)
y = float(mpc_obs.comment.Y)
(ra,dec) = this_wcs.xy2sky(x,y)
mpc_obs.coordinate = (ra,dec)
recompute_mag(mpc_obs)
return mpc_obs
def ossos_pointings(self):
"""
plot an OSSOS observation on the OSSOS plot.
"""
match = re.match('(\d+)\D(\d+)', self.expnum.get())
if match is not None:
expnum = int(match.group(1))
ccd = int(match.group(2))
x = 2112 / 2.0
y = 4644 / 2.0
else:
expnum = int(str(self.expnum.get()))
ccd = 22
x = 1000
y = 4644 - 15 / 0.185
header = None
try:
header = storage.get_astheader(expnum, ccd=ccd)
except:
if header is None:
print "Didn't get a header... "
return
ossos_wcs = wcs.WCS(header)
(ra, dec) = ossos_wcs.xy2sky(x, y)
class MyEvent(object):
def __init__(self, x, y):
self.x = x
self.y = y
(x, y) = self.p2s((math.radians(ra), math.radians(dec)))
event = MyEvent(x, y)
self.create_pointing(event,
label_text=header['OBJECT'] +
' ccd{}'.format(ccd))
def run(expnum, ccd, version='p', prefix='', dry_run=False, force=False):
message = 'success'
if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task=task, prefix=prefix, expnum=expnum, ccd=ccd, version=version, dry_run=dry_run):
try:
if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd):
raise IOError("{} not yet run for {}".format(dependency, expnum))
header = storage.get_astheader(expnum, ccd)
datasec = storage.datasec_to_list(header.get('DATASEC', '[80:2080,30,4160]'))
try:
fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd))
except:
fwhm = 'unknown'
for keyword in del_keyword_list:
try:
del(header[keyword])
except:
pass
header['FWHM'] = (fwhm, 'FWHM in pixels')
header['EXTNAME'] = 'header'
primary_hdu = fits.PrimaryHDU(header=header)
hdu_list = fits.HDUList([primary_hdu, ])
for ext in ['jmp', 'matt']:
extension = 'obj.'+ext
name = "{}p{:02d}.{}".format(expnum, ccd, extension)
try:
os.unlink(name)
os.unlink(name+".fits")
except:
pass
logging.info("Retrieving {}".format(name))
obj_file = mop_file.Parser(expnum, ccd, extension)
obj_file.parse()
t = numpy.all([datasec[0] < obj_file.data['X'], obj_file.data['X'] < datasec[1],
datasec[2] < obj_file.data['Y'], obj_file.data['Y'] < datasec[3]], axis=0)
logging.info("Source remaining after datasec cut: {} of {}".format(len(obj_file.data[t]['X']), len(t)))
table_hdu = fits.table_to_hdu(obj_file.data[t])
table_hdu.header['CATALOG'] = name
table_hdu.header['EXTNAME'] = ext
hdu_list.append(table_hdu)
del table_hdu
del obj_file
os.unlink(name)
name = "{}p{:02d}.{}".format(expnum, ccd, 'obj.fits')
if os.access(name, os.F_OK):
os.unlink(name)
hdu_list.writeto(name)
uri = storage.dbimages_uri(expnum, ccd, 'p', ext=".obj.fits")
logging.info(name+" -> "+uri)
count = 0
with open(name):
while True:
count += 1
logging.info("Copy attempt {}".format(count))
try:
storage.copy(name, uri)
os.unlink(name)
break
except Exception as ex:
if count > 10:
raise ex
logging.info(message)
except Exception as e:
message = str(e)
logging.error(message)
if not dry_run:
storage.set_status(task, prefix, expnum, version=version, ccd=ccd, status=message)
def run(expnum, ccd, version='p', prefix='', dry_run=False, force=False):
message = 'success'
if storage.get_status(task, prefix, expnum, version=version,
ccd=ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(
task, prefix, expnum, version, ccd))
return
with storage.LoggingManager(task=task,
prefix=prefix,
expnum=expnum,
ccd=ccd,
version=version,
dry_run=dry_run):
try:
if not storage.get_status(dependency, prefix, expnum, "p",
ccd=ccd):
raise IOError("{} not yet run for {}".format(
dependency, expnum))
header = storage.get_astheader(expnum, ccd)
datasec = storage.datasec_to_list(
header.get('DATASEC', '[80:2080,30,4160]'))
try:
fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd))
except:
fwhm = 'unknown'
for keyword in del_keyword_list:
try:
del (header[keyword])
except:
pass
header['FWHM'] = (fwhm, 'FWHM in pixels')
header['EXTNAME'] = 'header'
primary_hdu = fits.PrimaryHDU(header=header)
hdu_list = fits.HDUList([
primary_hdu,
])
for ext in ['jmp', 'matt']:
extension = 'obj.' + ext
name = "{}p{:02d}.{}".format(expnum, ccd, extension)
try:
os.unlink(name)
os.unlink(name + ".fits")
except:
pass
logging.info("Retrieving {}".format(name))
obj_file = mop_file.Parser(expnum, ccd, extension)
obj_file.parse()
t = numpy.all([
datasec[0] < obj_file.data['X'],
obj_file.data['X'] < datasec[1],
datasec[2] < obj_file.data['Y'],
obj_file.data['Y'] < datasec[3]
],
axis=0)
logging.info(
"Source remaining after datasec cut: {} of {}".format(
len(obj_file.data[t]['X']), len(t)))
table_hdu = fits.table_to_hdu(obj_file.data[t])
table_hdu.header['CATALOG'] = name
table_hdu.header['EXTNAME'] = ext
hdu_list.append(table_hdu)
del table_hdu
del obj_file
os.unlink(name)
name = "{}p{:02d}.{}".format(expnum, ccd, 'obj.fits')
if os.access(name, os.F_OK):
os.unlink(name)
hdu_list.writeto(name)
uri = storage.dbimages_uri(expnum, ccd, 'p', ext=".obj.fits")
logging.info(name + " -> " + uri)
count = 0
with open(name):
while True:
count += 1
logging.info("Copy attempt {}".format(count))
try:
storage.copy(name, uri)
os.unlink(name)
break
except Exception as ex:
if count > 10:
raise ex
logging.info(message)
except Exception as e:
message = str(e)
logging.error(message)
if not dry_run:
storage.set_status(task,
prefix,
expnum,
version=version,
ccd=ccd,
status=message)
def caom2(mpc_filename, search_date="2014 07 24.0"):
"""
builds a TSV file in the format of SSOIS by querying for possilbe observations in CADC/CAOM2.
This is a fall back program, should only be useful when SSOIS is behind.
"""
columns = ('Image',
'Ext',
'X',
'Y',
'MJD',
'Filter',
'Exptime',
'Object_RA',
'Object_Dec',
'Image_target',
'Telescope/Instrument',
'MetaData',
'Datalink')
ephem_table = Table(names=columns,
dtypes=('S10', 'i4', 'f8', 'f8',
'f8', 'S10', 'f8', 'f8', 'f8', 'S20', 'S20', 'S20', 'S50'))
ephem_table.pprint()
o = orbfit.Orbfit(mpc.MPCReader(mpc_filename).mpc_observations)
o.predict(search_date)
fields = storage.cone_search(o.coordinate.ra.degrees, o.coordinate.dec.degrees, dra=0.3, ddec=0.3,
calibration_level=1)
mjdates = numpy.unique(fields['mjdate'])
collectionIDs = []
for mjdate in mjdates:
jd = 2400000.5 + mjdate
o.predict(jd)
for field in storage.cone_search(o.coordinate.ra.degrees, o.coordinate.dec.degrees,
dra=30./3600.0, ddec=30./3600.0,
mjdate=mjdate, calibration_level=1):
collectionIDs.append(field['collectionID'])
expnums = numpy.unique(numpy.array(collectionIDs))
for expnum in expnums:
header = storage.get_astheader(expnum, 22)
o.predict(header['MJDATE']+2400000.5)
print o.time.iso, o.coordinate.ra.degrees, o.coordinate.dec.degrees
for ccd in range(36):
header = storage.get_astheader(expnum, ccd)
w = wcs.WCS(header)
(x, y) = w.sky2xy(o.coordinate.ra.degrees, o.coordinate.dec.degrees)
print ccd, x, y
if 0 < x < header['NAXIS1'] and 0 < y < header['NAXIS2']:
ephem_table.add_row([expnum, ccd+1, x, y,
header['MJDATE'], header['FILTER'], header['EXPTIME'],
o.coordinate.ra.degrees, o.coordinate.dec.degrees,
header['OBJECT'],
'CFHT/MegaCam',
None,
"http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/data/pub/CFHT/{}p[{}]".format(expnum, ccd)])
break
ephem_table.pprint()
ephem_table.write('backdoor.tsv', format='ascii', delimiter='\t')
def recompute_mag(mpc_in):
"""
Get the mag of the object given the mpc.Observation
"""
# TODO this really shouldn't need to build a 'reading' to get the cutout...
from ossos.downloads.cutouts import ImageCutoutDownloader
mpc_obs = deepcopy(mpc_in)
assert isinstance(mpc_obs, mpc.Observation)
if mpc_obs.null_observation:
return mpc_obs
parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame)
if parts is None:
return mpc_obs
expnum = parts.group('expnum')
ccd = parts.group('ccd')
file_type = parts.group('type')
observation = astrom.Observation(expnum, file_type, ccd)
assert isinstance(observation, astrom.Observation)
ast_header = storage.get_astheader(int(expnum), int(ccd))
# The ZP for the current astrometric lines is the pipeline one. The new ZP is in the astheader file.
new_zp = ast_header.get('PHOTZP')
# The ZP in the image header is likely the one used for the original photometry.
old_zp = storage.get_zeropoint(int(expnum), int(ccd))
reading = astrom.SourceReading(float(mpc_obs.comment.x), float(mpc_obs.comment.y), float(mpc_obs.comment.x),
float(mpc_obs.comment.y), mpc_obs.coordinate.ra.degrees,
mpc_obs.coordinate.dec.degrees, float(mpc_obs.comment.x),
float(mpc_obs.comment.y),
observation, ssos=True, from_input_file=True, null_observation=False,
discovery=mpc_obs.discovery)
# reading.is_inverted = reading.compute_inverted()
reading._header = storage.get_mopheader(expnum, ccd)
reading._header['MAXCOUNT'] = 30000.0
image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100)
cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True)
cutout.zmag = new_zp
if math.fabs(new_zp - old_zp) > 0.3:
logging.warning("Large change in zeropoint detected: {} -> {}".format(old_zp, new_zp))
try:
(x, y, mag, merr) = cutout.get_observed_magnitude(zmag=new_zp)
(x, y) = cutout.get_observed_coordinates((x, y))
except:
logging.warn("Failed to do photometry.")
return mpc_obs
if mpc_obs.comment.mag is not None and math.fabs(mpc_obs.comment.mag - mag) > 3.5 * mpc_obs.comment.mag_uncertainty:
logging.warn("recomputed magnitude shift large: {} --> {}".format(mpc_obs.mag, mag))
if math.sqrt((x - mpc_obs.comment.x) ** 2 + (y - mpc_obs.comment.y) ** 2) > 1.0:
logging.warn("Centroid shifted ({},{}) -> ({},{})".format(mpc_obs.comment.x,
mpc_obs.comment.y,
x,
y))
# Don't use the new X/Y for Hand measured entries.
if str(mpc_obs.note1) != "H":
mpc_obs.comment.x = x
mpc_obs.comment.y = y
mpc_obs.comment.mag = mag
mpc_obs.comment.mag_uncertainty = merr
if mpc_obs.mag is not None:
mpc_obs.mag = mag
return mpc_obs
else:
block_files = files[block]
# not all provisional names have the full Oyearsemesterblock pattern. Because counter.
reals_files = [
c for c in block_files
if (c.__contains__(provisional_name)
or c.__contains__(provisional_name[-3:]))
]
if len(reals_files) > 0: # ensure it's present in this directory
chip = reals_files[0].split('.')[0]
cand_filename = '{}.measure3.cands.astrom'.format(chip)
# now who set the tag?
uri = path + cand_filename
discoverer = storage.vospace.getNode(
uri, force=True).props[storage.OSSOS_TAG_URI_BASE + '#done']
outfile.write('{:<12} {:<12}'.format(discoverer, chip))
# map the exposure number of the triplet to the header value for the Observer
# FIXME: doesn't work for 14BH because the block is not called that in the path. Will need to overwrite
triplet_path = '{}/{}_{}_discovery_expnums.txt'.format(
storage.TRIPLETS, block[-1:], block[2:5])
triplets = storage.open_vos_or_local(triplet_path).read().split(
'\n')
triplet = [n for n in triplets
if n.endswith(chip.split('_')[0])][0]
header = storage.get_astheader(triplet.split()[0],
chip.split('_')[1].strip('p'))
outfile.write('{:<20} {:<20}\n'.format(header['QOBSERVE'],
header['QCOORD']))
lines = open(sys.argv[1]).readlines()
except:
lines = sys.argv[1:]
for line in lines:
expnum = int(line.strip())
if expnum < 1785619:
# Last exposures with 36 CCD Megaprime
ccdlist = list(range(0, 36))
else:
# First exposrues with 40 CCD Megaprime
ccdlist = list(range(0, 40))
for ccd in ccdlist:
try:
try:
header = storage.get_astheader(expnum, ccd)
datasec = storage.datasec_to_list(
header.get('DATASEC', '[80:2080,30,4160]'))
except Exception as ex:
print(ex)
continue
try:
fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd))
except:
fwhm = 'unknown'
for keyword in del_keyword_list:
try:
del (header[keyword])
except:
pass
header['FWHM'] = (fwhm, 'FWHM in pixels')
def remeasure(mpc_in, recentroided=False):
"""
re-measure the astrometry and photometry of the given mpc line
"""
if mpc_in.null_observation:
return mpc_in
mpc_obs = deepcopy(mpc_in)
logging.debug("rm start: {}".format(mpc_obs.to_string()))
if not isinstance(mpc_obs.comment, mpc.OSSOSComment):
logging.error( "Failed to convert comment line")
return mpc_in
parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame)
if not parts:
logging.error( "Failed to parse expnum from frame info in comment line")
return mpc_in
start_coordinate = mpc_in.coordinate
#assert isinstance(start_coordinate, ICRSCoordinates)
try:
header = storage._get_sghead(parts.group('expnum'))[int(parts.group('ccd')) + 1]
except IOError as ioerr:
logging.error(str(ioerr))
logging.error("Failed to get astrometric header for: {}".format(mpc_obs))
return mpc_in
this_wcs = wcs.WCS(header)
try:
#(x , y ) = this_wcs.sky2xy(mpc_obs.coordinate.ra.degree, mpc_obs.coordinate.dec.degree, usepv=True)
x = mpc_obs.comment.x
y = mpc_obs.comment.y
(ra, dec) = this_wcs.xy2sky(x, y, usepv=True)
mpc_obs.coordinate = (ra, dec)
sep = mpc_in.coordinate.separation(mpc_obs.coordinate)
if sep > TOLERANCE*20 and mpc_in.discovery and (int(parts.group('ccd')) < 18 or int(parts.group('ccd')) in [36, 37]):
logging.warn("Large ({}) offset using X/Y in comment line to compute RA/DEC".format(sep))
logging.warn("This is a discovery line so flipping/flopping the x/y position recorded in comment as that "
"may be taken from a flip/flopped image.")
x = header['NAXIS1'] - x + 1
y = header['NAXIS2'] - y + 1
sep = mpc_in.coordinate.separation(mpc_obs.coordinate)
(ra, dec) = this_wcs.xy2sky(x, y, usepv=True)
# print "--> x/y coordinates ({},{}) and recomputing ra/dec ({},{})".format(x, y, ra, dec)
mpc_obs.coordinate = (ra, dec)
if sep > TOLERANCE*20 and mpc_in.discovery:
logging.warn(
"Large ({}) offset after flipping, so reverting.".format(sep))
# Try un-flipping.
x = float(mpc_in.comment.x)
y = float(mpc_in.comment.y)
(ra, dec) = this_wcs.xy2sky(x, y, usepv=True)
# print "--> x/y coordinates ({},{}) and recomputing ra/dec ({},{})".format(x, y, ra, dec)
mpc_obs.coordinate = (ra, dec)
except:
logging.warn("Failed to get the X Y coordinate from comment line.")
return mpc_in
# (ra, dec) = this_wcs.xy2sky(x, y, usepv=True)
# mpc_obs.coordinate = (ra, dec)
# sep = mpc_in.coordinate.separation(mpc_obs.coordinate)
if sep > TOLERANCE and not recentroided:
# use the old header RA/DEC to predict the X/Y and then use that X/Y to get new RA/DEC
logging.warn("sep: {} --> large offset when using comment line X/Y to compute RA/DEC")
logging.warn("Using RA/DEC and original WCS to compute X/Y and replacing X/Y in comment.".format(sep))
header2 = storage.get_astheader(parts.group('expnum'), int(parts.group('ccd')))
image_wcs = wcs.WCS(header2)
(x, y) = image_wcs.sky2xy(mpc_in.coordinate.ra.degree, mpc_in.coordinate.dec.degree, usepv=False)
(ra, dec) = this_wcs.xy2sky(x, y, usepv=True)
logging.warn("Coordinate changed: ({:5.2f},{:5.2f}) --> ({:5.2f},{:5.2f})".format(mpc_obs.comment.x,
mpc_obs.comment.y,
x, y))
mpc_obs.coordinate = (ra, dec)
mpc_obs.comment.x = x
mpc_obs.comment.y = y
try:
merr = float(mpc_obs.comment.mag_uncertainty)
fwhm = float(storage.get_fwhm(parts.group('expnum'), int(parts.group('ccd')))) * header['PIXSCAL1']
centroid_err = merr * fwhm
logging.debug("Centroid uncertainty: {} {} => {}".format(merr, fwhm, centroid_err))
except Exception as err:
logging.error(str(err))
logging.error("Failed to compute centroid_err for observation:\n"
"{}\nUsing default of 0.2".format(mpc_obs.to_string()))
centroid_err = 0.2
mpc_obs.comment.astrometric_level = header.get('ASTLEVEL', "0")
try:
asterr = float(header['ASTERR'])
residuals = (asterr ** 2 + centroid_err ** 2) ** 0.5
logging.debug("Residuals: {} {} => {}".format(asterr, centroid_err, residuals))
except Exception as err:
logging.error(str(err))
logging.error("Failed while trying to compute plate uncertainty for\n{}".format(mpc_obs.to_stirng()))
logging.error('Using default of 0.25')
residuals = 0.25
mpc_obs.comment.plate_uncertainty = residuals
logging.debug("sending back: {}".format(mpc_obs.to_string()))
return mpc_obs
if block in ['2013A-E', '2013A-O']:
# 13AE and 13AO ended up particularly weird. Fortunately, have a local list
with open('/Users/michele/Desktop/{}_mpclist.txt'.format(block), 'r') as infile:
block_files = infile.readlines()
else:
print 'starting VOSpace read' # getting file listings from big listings on VOSpace is slooooowwww...
block_files = filter(lambda name: not name.startswith('fk'), storage.listdir(path))
files[block] = block_files
else:
block_files = files[block]
# not all provisional names have the full Oyearsemesterblock pattern. Because counter.
reals_files = [c for c in block_files if
(c.__contains__(provisional_name) or c.__contains__(provisional_name[-3:]))]
if len(reals_files) > 0: # ensure it's present in this directory
chip = reals_files[0].split('.')[0]
cand_filename = '{}.measure3.cands.astrom'.format(chip)
# now who set the tag?
uri = path + cand_filename
discoverer = storage.vospace.getNode(uri, force=True).props[storage.OSSOS_TAG_URI_BASE + '#done']
outfile.write('{:<12} {:<12}'.format(discoverer, chip))
# map the exposure number of the triplet to the header value for the Observer
# FIXME: doesn't work for 14BH because the block is not called that in the path. Will need to overwrite
triplet_path = '{}/{}_{}_discovery_expnums.txt'.format(storage.TRIPLETS, block[-1:], block[2:5])
triplets = storage.open_vos_or_local(triplet_path).read().split('\n')
triplet = [n for n in triplets if n.endswith(chip.split('_')[0])][0]
header = storage.get_astheader(triplet.split()[0], chip.split('_')[1].strip('p'))
outfile.write('{:<20} {:<20}\n'.format(header['QOBSERVE'], header['QCOORD']))
def main():
"""Do the script."""
parser = argparse.ArgumentParser(
description='replace image header')
parser.add_argument('--extname',
help='name of extension to in header')
parser.add_argument('expnum', type=str,
help='exposure to update')
parser.add_argument('-r', '--replace',
action='store_true',
help='store modified image back to VOSpace?')
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--force', action='store_true', help="Re-run even if previous success recorded")
parser.add_argument('--dbimages', help="VOSpace DATA storage area.", default="vos:OSSOS/dbimages")
args = parser.parse_args()
task = util.task()
dependency = 'preproc'
prefix = ""
storage.DBIMAGES = args.dbimages
level = logging.CRITICAL
message_format = "%(message)s"
if args.verbose:
level = logging.INFO
if args.debug:
level = logging.DEBUG
message_format = "%(module)s %(funcName)s %(lineno)s %(message)s"
logging.basicConfig(level=level, format=message_format)
storage.set_logger(task, prefix, args.expnum, None, None, False)
message = storage.SUCCESS
expnum = args.expnum
exit_status = 0
try:
# skip if already succeeded and not in force mode
if storage.get_status(task, prefix, expnum, "p", 36) and not args.force:
logging.info("Already updated, skipping")
sys.exit(0)
image_hdulist = storage.get_image(args.expnum, return_file=False)
ast_hdulist = storage.get_astheader(expnum, ccd=None)
run_update_header(image_hdulist, ast_hdulist)
image_filename = os.path.basename(storage.get_uri(expnum))
image_hdulist.writeto(image_filename)
if args.replace:
dest = storage.dbimages_uri(expnum)
storage.copy(image_filename, dest)
storage.set_status('update_header', "", expnum, 'p', 36, message)
except Exception as e:
message = str(e)
if args.replace:
storage.set_status(task, prefix, expnum, 'p', 36, message)
exit_status = message
logging.error(message)
return exit_status
def remeasure(mpc_in):
"""
re-measure the astrometry and photometry of the given mpc line
"""
# TODO Actually implement this.
if mpc_in.null_observation:
return mpc_in
mpc_obs = deepcopy(mpc_in)
logging.debug("rm start: {}".format(mpc_obs.to_string()))
if not isinstance(mpc_obs.comment, mpc.OSSOSComment):
return mpc_in
parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame)
if not parts:
return mpc_in
start_coordinate = mpc_in.coordinate
assert isinstance(start_coordinate, ICRSCoordinates)
try:
header = storage.get_astheader(parts.group('expnum'), int(parts.group('ccd')))
except IOError as ioerr:
logging.error(str(ioerr))
logging.error("Failed to get astrometric header for: {}".format(mpc_obs))
return mpc_in
this_wcs = wcs.WCS(header)
try:
x = float(mpc_obs.comment.x)
y = float(mpc_obs.comment.y)
if mpc_in.discovery:
logging.debug("Discovery astrometric lines are normally flipped relative to storage.")
x = header['NAXIS1'] - x + 1
y = header['NAXIS2'] - y + 1
except:
logging.warn("Failed to X/Y from comment line.")
return mpc_in
(ra, dec) = this_wcs.xy2sky(x, y)
mpc_obs.coordinate = (ra, dec)
logging.debug("rm updat: {}".format(mpc_obs.to_string()))
sep = start_coordinate.separation(mpc_obs.coordinate).degrees * 3600.0
if sep > TOLERANCE and mpc_in.discovery:
logging.warn("{} --> Using the unflipped X/Y for a discovery observation line.".format(sep))
logging.debug("{} {} {} {} {}".format(sep, x, y, mpc_obs.comment.x, mpc_obs.comment.y))
# Try un-flipping.
x = float(mpc_obs.comment.x)
y = float(mpc_obs.comment.y)
(ra, dec) = this_wcs.xy2sky(x, y)
#print "--> x/y coordinates ({},{}) and recomputing ra/dec ({},{})".format(x, y, ra, dec)
mpc_obs.coordinate = (ra, dec)
sep = start_coordinate.separation(mpc_obs.coordinate).degrees * 3600.0
logging.debug("remod: {}".format(mpc_obs.coordinate))
logging.debug("SEP: {}".format(sep))
logging.debug("rm flip: {}".format(mpc_obs.to_string()))
if sep > TOLERANCE:
## use the old header RA/DEC to predict the X/Y and then use that X/Y to get new RA/DEC
logging.debug("Ignoring recorded X/Y and using previous to RA/DEC and WCS to compute X/Y")
header2 = storage.get_image(parts.group('expnum'),
int(parts.group('ccd')),
return_file=False,
flip_image=False)[0].header
image_wcs = wcs.WCS(header2)
(x, y) = image_wcs.sky2xy(mpc_in.coordinate.ra.degrees, mpc_in.coordinate.dec.degrees)
(ra, dec) = this_wcs.xy2sky(x, y)
logging.debug("({},{}) --> ({},{})".format(mpc_obs.comment.x, mpc_obs.comment.y, x, y))
mpc_obs.coordinate = (ra, dec)
mpc_obs.comment.x = x
mpc_obs.comment.y = y
try:
merr = float(mpc_obs.comment.mag_uncertainty)
fwhm = float(storage.get_fwhm(parts.group('expnum'), int(parts.group('ccd')))) * header['PIXSCAL1']
centroid_err = merr * fwhm
logging.debug("Centroid uncertainty: {} {} => {}".format(merr, fwhm, centroid_err))
except Exception as err:
logging.error(str(err))
logging.error("Failed to compute centroid for observation:\n"
"{}\nUsing default of 0.2".format(mpc_obs.to_string()))
centroid_err = 0.2
mpc_obs.comment.astrometric_level = header.get('ASTLEVEL', "0")
try:
asterr = float(header['ASTERR'])
residuals = (asterr ** 2 + centroid_err ** 2) ** 0.5
logging.debug("Residuals: {} {} => {}".format(asterr, centroid_err, residuals))
except Exception as err:
logging.error(str(err))
logging.error("Failed while trying to compute plate uncertainty for\n{}".format(mpc_obs.to_stirng()))
logging.error('Using default of 0.25')
residuals = 0.25
mpc_obs.comment.plate_uncertainty = residuals
logging.debug("sending back: {}".format(mpc_obs.to_string()))
return mpc_obs
def main():
"""Do the script."""
parser = argparse.ArgumentParser(description='replace image header')
parser.add_argument('--extname', help='name of extension to in header')
parser.add_argument('expnum', type=str, help='exposure to update')
parser.add_argument('-r',
'--replace',
action='store_true',
help='store modified image back to VOSpace?')
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--force',
action='store_true',
help="Re-run even if previous success recorded")
parser.add_argument('--dbimages',
help="VOSpace DATA storage area.",
default="vos:OSSOS/dbimages")
args = parser.parse_args()
task = util.task()
dependency = 'preproc'
prefix = ""
storage.DBIMAGES = args.dbimages
level = logging.CRITICAL
message_format = "%(message)s"
if args.verbose:
level = logging.INFO
if args.debug:
level = logging.DEBUG
message_format = "%(module)s %(funcName)s %(lineno)s %(message)s"
logging.basicConfig(level=level, format=message_format)
storage.set_logger(task, prefix, args.expnum, None, None, False)
message = storage.SUCCESS
expnum = args.expnum
exit_status = 0
try:
# skip if already succeeded and not in force mode
if storage.get_status(task, prefix, expnum, "p",
36) and not args.force:
logging.info("Already updated, skipping")
sys.exit(0)
image_hdulist = storage.get_image(args.expnum, return_file=False)
ast_hdulist = storage.get_astheader(expnum, ccd=None)
run_update_header(image_hdulist, ast_hdulist)
image_filename = os.path.basename(storage.get_uri(expnum))
image_hdulist.writeto(image_filename)
if args.replace:
dest = storage.dbimages_uri(expnum)
storage.copy(image_filename, dest)
storage.set_status('update_header', "", expnum, 'p', 36, message)
except Exception as e:
message = str(e)
if args.replace:
storage.set_status(task, prefix, expnum, 'p', 36, message)
exit_status = message
logging.error(message)
return exit_status
def main(expnum, ccd):
header = storage.get_astheader(expnum, ccd)
datasec = storage.datasec_to_list(
header.get('DATASEC', '[80:2080,30,4160]'))
try:
fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd))
except:
fwhm = 'unknown'
for keyword in del_keyword_list:
try:
del (header[keyword])
except:
pass
header['FWHM'] = (fwhm, 'FWHM in pixels')
header['EXTNAME'] = 'header'
primary_hdu = fits.PrimaryHDU(header=header)
hdu_list = fits.HDUList([
primary_hdu,
])
for ext in ['jmp', 'matt']:
extension = 'obj.' + ext
name = "{}p{:02d}.{}".format(expnum, ccd, extension)
try:
os.unlink(name)
os.unlink(name + ".fits")
except:
pass
logging.info("Retrieving {}".format(name))
obj_file = mop_file.Parser(expnum, ccd, extension)
obj_file.parse()
t = numpy.all([
datasec[0] < obj_file.data['X'], obj_file.data['X'] < datasec[1],
datasec[2] < obj_file.data['Y'], obj_file.data['Y'] < datasec[3]
],
axis=0)
logging.info("Source remaining after datasec cut: {} of {}".format(
len(obj_file.data[t]['X']), len(t)))
table_hdu = fits.table_to_hdu(obj_file.data[t])
table_hdu.header['CATALOG'] = name
table_hdu.header['EXTNAME'] = ext
hdu_list.append(table_hdu)
del table_hdu
del obj_file
os.unlink(name)
name = "{}p{:02d}.{}".format(expnum, ccd, 'obj.fits')
if os.access(name, os.F_OK):
os.unlink(name)
hdu_list.writeto(name)
uri = storage.dbimages_uri(expnum, ccd, 'p', ext=".obj.fits")
logging.info(name + " -> " + uri)
count = 0
while True:
print(("Copy attempt {}".format(count)))
try:
storage.copy(name, uri)
os.unlink(name)
break
except Exception as ex:
if count > 10:
raise ex
count += 1
def ephem_search(mpc_filename, search_date="2014 07 24.0"):
"""
builds a TSV file in the format of SSOIS by querying for possible observations in CADC/CAOM2.
This is a fall back program, should only be useful when SSOIS is behind.
"""
columns = ('Image', 'Ext', 'X', 'Y', 'MJD', 'Filter', 'Exptime',
'Object_RA', 'Object_Dec', 'Image_target',
'Telescope/Instrument', 'MetaData', 'Datalink')
ephem_table = Table(names=columns,
dtypes=('S10', 'i4', 'f8', 'f8', 'f8', 'S10', 'f8',
'f8', 'f8', 'S20', 'S20', 'S20', 'S50'))
ephem_table.pprint()
o = orbfit.Orbfit(mpc.MPCReader(mpc_filename).mpc_observations)
o.predict(search_date)
fields = storage.cone_search(o.coordinate.ra.degrees,
o.coordinate.dec.degrees,
dra=0.3,
ddec=0.3,
calibration_level=1)
mjdates = numpy.unique(fields['mjdate'])
collectionIDs = []
for mjdate in mjdates:
jd = 2400000.5 + mjdate
o.predict(jd)
for field in storage.cone_search(o.coordinate.ra.degrees,
o.coordinate.dec.degrees,
dra=30. / 3600.0,
ddec=30. / 3600.0,
mjdate=mjdate,
calibration_level=1):
collectionIDs.append(field['collectionID'])
expnums = numpy.unique(numpy.array(collectionIDs))
for expnum in expnums:
header = storage.get_astheader(expnum, 22)
o.predict(header['MJDATE'] + 2400000.5)
print(o.time.iso, o.coordinate.ra.degrees, o.coordinate.dec.degrees)
for ccd in range(36):
header = storage.get_astheader(expnum, ccd)
w = wcs.WCS(header)
(x, y) = w.sky2xy(o.coordinate.ra.degrees,
o.coordinate.dec.degrees)
print(ccd, x, y)
if 0 < x < header['NAXIS1'] and 0 < y < header['NAXIS2']:
ephem_table.add_row([
expnum, ccd + 1, x, y, header['MJDATE'], header['FILTER'],
header['EXPTIME'], o.coordinate.ra.degrees,
o.coordinate.dec.degrees, header['OBJECT'], 'CFHT/MegaCam',
None,
"https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/data/pub/CFHT/{}p[{}]"
.format(expnum, ccd)
])
break
ephem_table.pprint()
ephem_table.write('backdoor.tsv', format='ascii', delimiter='\t')
lines = open(sys.argv[1]).readlines()
except:
lines = sys.argv[1:]
for line in lines:
expnum = int(line.strip())
if expnum < 1785619:
# Last exposures with 36 CCD Megaprime
ccdlist = range(0, 36)
else:
# First exposrues with 40 CCD Megaprime
ccdlist = range(0, 40)
for ccd in ccdlist:
try:
try:
header = storage.get_astheader(expnum, ccd)
datasec = storage.datasec_to_list(header.get('DATASEC', '[80:2080,30,4160]'))
except Exception as ex:
print(ex)
continue
try:
fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd))
except:
fwhm = 'unknown'
for keyword in del_keyword_list:
try:
del(header[keyword])
except:
pass
header['FWHM'] = (fwhm, 'FWHM in pixels')
header['EXTNAME'] = 'header'
