def get_header(image): retval = None header_text = storage.get_astheader(image, 0) # zeroth ccd: first see if the preproc'd version is there if header_text is None: header_text = storage.get_astheader(image, 0, version='o') # 'o' for an object acquisition if header_text is not None: # checked twice, it must be there now if it's there at all... retval = wanted_keys(header_text) return retval, header_text
def get_coord_offset(self, expnum, ccd, X, Y, ref_expnum, ref_ccd): # determine offsets to reference reference frame using wcs astheader = get_astheader(expnum, ccd) ref_astheader = get_astheader(ref_expnum, ref_ccd) ref_pvwcs = wcs.WCS(ref_astheader) pvwcs = wcs.WCS(astheader) (ra, dec) = pvwcs.xy2sky(X, Y) (x0, y0) = ref_pvwcs.sky2xy(ra, dec) logger.debug("{}p{} {},{} -> {}p{} {},{}".format(expnum, ccd, X, Y, ref_expnum, ref_ccd, x0, y0)) return (x0, y0)
def get_coord_offset(self, expnum, ccd, X, Y, ref_expnum, ref_ccd): # determine offsets to reference reference frame using wcs astheader = get_astheader(expnum, ccd) ref_astheader = get_astheader(ref_expnum, ref_ccd) ref_pvwcs = wcs.WCS(ref_astheader) pvwcs = wcs.WCS(astheader) (ra,dec) = pvwcs.xy2sky(X, Y) (x0, y0) = ref_pvwcs.sky2xy(ra,dec) logger.debug("{}p{} {},{} -> {}p{} {},{}".format(expnum,ccd, X,Y, ref_expnum, ref_ccd, x0,y0)) return (x0, y0)
def ossos_pointings(self): """ plot an OSSOS observation on the OSSOS plot. """ match = re.match('(\d+)\D(\d+)', self.expnum.get()) if match is not None: expnum = int(match.group(1)) ccd = int(match.group(2)) x = 2112 / 2.0 y = 4644 / 2.0 else: expnum = int(str(self.expnum.get())) ccd = 22 x = 1000 y = 4644 - 15 / 0.185 header = None try: header = storage.get_astheader(expnum, ccd=ccd) except: if header is None: print "Didn't get a header... " return ossos_wcs = wcs.WCS(header) (ra, dec) = ossos_wcs.xy2sky(x, y) class MyEvent(object): def __init__(self, x, y): self.x = x self.y = y (x, y) = self.p2s((math.radians(ra), math.radians(dec))) event = MyEvent(x, y) self.create_pointing(event, label_text=header['OBJECT'] + ' ccd{}'.format(ccd))
def main(expnum, ccd): header = storage.get_astheader(expnum, ccd) datasec = storage.datasec_to_list(header.get('DATASEC', '[80:2080,30,4160]')) try: fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd)) except: fwhm = 'unknown' for keyword in del_keyword_list: try: del(header[keyword]) except: pass header['FWHM'] = (fwhm, 'FWHM in pixels') header['EXTNAME'] = 'header' primary_hdu = fits.PrimaryHDU(header=header) hdu_list = fits.HDUList([primary_hdu, ]) for ext in ['jmp', 'matt']: extension = 'obj.'+ext name = "{}p{:02d}.{}".format(expnum, ccd, extension) try: os.unlink(name) os.unlink(name+".fits") except: pass logging.info("Retrieving {}".format(name)) obj_file = mop_file.Parser(expnum, ccd, extension) obj_file.parse() t = numpy.all([datasec[0] < obj_file.data['X'], obj_file.data['X'] < datasec[1], datasec[2] < obj_file.data['Y'], obj_file.data['Y'] < datasec[3]], axis=0) logging.info("Source remaining after datasec cut: {} of {}".format(len(obj_file.data[t]['X']), len(t))) table_hdu = fits.table_to_hdu(obj_file.data[t]) table_hdu.header['CATALOG'] = name table_hdu.header['EXTNAME'] = ext hdu_list.append(table_hdu) del table_hdu del obj_file os.unlink(name) name = "{}p{:02d}.{}".format(expnum, ccd, 'obj.fits') if os.access(name, os.F_OK): os.unlink(name) hdu_list.writeto(name) uri = storage.dbimages_uri(expnum, ccd, 'p', ext=".obj.fits") logging.info(name+" -> "+uri) count = 0 while True: print("Copy attempt {}".format(count)) try: storage.copy(name, uri) os.unlink(name) break except Exception as ex: if count > 10: raise ex count += 1
def is_inverted(self): """ Returns: inverted: bool True if the stored image is inverted. """ astheader = storage.get_astheader(self.obs.expnum, self.obs.ccdnum, version=self.obs.ftype) pvwcs = wcs.WCS(astheader) (x,y) = pvwcs.sky2xy(self.ra, self.dec) logger.debug("is_inverted: X,Y {},{} -> wcs X,Y {},{}".format(self.x, self.y, x, y)) dr2 = ((x-self.x)**2 + (y-self.y)**2) logger.debug("inverted is {}".format(dr2>2)) return dr2 > 2
def compute_inverted(self): """ Returns: inverted: bool True if the stored image is inverted. """ astheader = storage.get_astheader(self.obs.expnum, self.obs.ccdnum, version=self.obs.ftype) pvwcs = wcs.WCS(astheader) (x, y) = pvwcs.sky2xy(self.ra, self.dec) logger.debug("is_inverted: X,Y {},{} -> wcs X,Y {},{}".format( self.x, self.y, x, y)) dr2 = ((x - self.x)**2 + (y - self.y)**2) logger.debug("inverted is {}".format(dr2 > 2)) return dr2 > 2
def remeasure(mpc_obs): """ re-measure the astrometry and photometry of the given mpc line """ # TODO Actually implement this. if mpc_obs.null_observation: return mpc_obs assert isinstance(mpc_obs.comment, mpc.MPCComment) parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)',mpc_obs.comment.frame) header = storage.get_astheader(parts.group('expnum'),int(parts.group('ccd'))) this_wcs = wcs.WCS(header) x = float(mpc_obs.comment.X) y = float(mpc_obs.comment.Y) (ra,dec) = this_wcs.xy2sky(x,y) mpc_obs.coordinate = (ra,dec) recompute_mag(mpc_obs) return mpc_obs
def run(expnum, ccd, version='p', prefix='', dry_run=False, force=False): message = 'success' if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format(task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task=task, prefix=prefix, expnum=expnum, ccd=ccd, version=version, dry_run=dry_run): try: if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd): raise IOError("{} not yet run for {}".format(dependency, expnum)) header = storage.get_astheader(expnum, ccd) datasec = storage.datasec_to_list(header.get('DATASEC', '[80:2080,30,4160]')) try: fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd)) except: fwhm = 'unknown' for keyword in del_keyword_list: try: del(header[keyword]) except: pass header['FWHM'] = (fwhm, 'FWHM in pixels') header['EXTNAME'] = 'header' primary_hdu = fits.PrimaryHDU(header=header) hdu_list = fits.HDUList([primary_hdu, ]) for ext in ['jmp', 'matt']: extension = 'obj.'+ext name = "{}p{:02d}.{}".format(expnum, ccd, extension) try: os.unlink(name) os.unlink(name+".fits") except: pass logging.info("Retrieving {}".format(name)) obj_file = mop_file.Parser(expnum, ccd, extension) obj_file.parse() t = numpy.all([datasec[0] < obj_file.data['X'], obj_file.data['X'] < datasec[1], datasec[2] < obj_file.data['Y'], obj_file.data['Y'] < datasec[3]], axis=0) logging.info("Source remaining after datasec cut: {} of {}".format(len(obj_file.data[t]['X']), len(t))) table_hdu = fits.table_to_hdu(obj_file.data[t]) table_hdu.header['CATALOG'] = name table_hdu.header['EXTNAME'] = ext hdu_list.append(table_hdu) del table_hdu del obj_file os.unlink(name) name = "{}p{:02d}.{}".format(expnum, ccd, 'obj.fits') if os.access(name, os.F_OK): os.unlink(name) hdu_list.writeto(name) uri = storage.dbimages_uri(expnum, ccd, 'p', ext=".obj.fits") logging.info(name+" -> "+uri) count = 0 with open(name): while True: count += 1 logging.info("Copy attempt {}".format(count)) try: storage.copy(name, uri) os.unlink(name) break except Exception as ex: if count > 10: raise ex logging.info(message) except Exception as e: message = str(e) logging.error(message) if not dry_run: storage.set_status(task, prefix, expnum, version=version, ccd=ccd, status=message)
def run(expnum, ccd, version='p', prefix='', dry_run=False, force=False): message = 'success' if storage.get_status(task, prefix, expnum, version=version, ccd=ccd) and not force: logging.info("{} completed successfully for {} {} {} {}".format( task, prefix, expnum, version, ccd)) return with storage.LoggingManager(task=task, prefix=prefix, expnum=expnum, ccd=ccd, version=version, dry_run=dry_run): try: if not storage.get_status(dependency, prefix, expnum, "p", ccd=ccd): raise IOError("{} not yet run for {}".format( dependency, expnum)) header = storage.get_astheader(expnum, ccd) datasec = storage.datasec_to_list( header.get('DATASEC', '[80:2080,30,4160]')) try: fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd)) except: fwhm = 'unknown' for keyword in del_keyword_list: try: del (header[keyword]) except: pass header['FWHM'] = (fwhm, 'FWHM in pixels') header['EXTNAME'] = 'header' primary_hdu = fits.PrimaryHDU(header=header) hdu_list = fits.HDUList([ primary_hdu, ]) for ext in ['jmp', 'matt']: extension = 'obj.' + ext name = "{}p{:02d}.{}".format(expnum, ccd, extension) try: os.unlink(name) os.unlink(name + ".fits") except: pass logging.info("Retrieving {}".format(name)) obj_file = mop_file.Parser(expnum, ccd, extension) obj_file.parse() t = numpy.all([ datasec[0] < obj_file.data['X'], obj_file.data['X'] < datasec[1], datasec[2] < obj_file.data['Y'], obj_file.data['Y'] < datasec[3] ], axis=0) logging.info( "Source remaining after datasec cut: {} of {}".format( len(obj_file.data[t]['X']), len(t))) table_hdu = fits.table_to_hdu(obj_file.data[t]) table_hdu.header['CATALOG'] = name table_hdu.header['EXTNAME'] = ext hdu_list.append(table_hdu) del table_hdu del obj_file os.unlink(name) name = "{}p{:02d}.{}".format(expnum, ccd, 'obj.fits') if os.access(name, os.F_OK): os.unlink(name) hdu_list.writeto(name) uri = storage.dbimages_uri(expnum, ccd, 'p', ext=".obj.fits") logging.info(name + " -> " + uri) count = 0 with open(name): while True: count += 1 logging.info("Copy attempt {}".format(count)) try: storage.copy(name, uri) os.unlink(name) break except Exception as ex: if count > 10: raise ex logging.info(message) except Exception as e: message = str(e) logging.error(message) if not dry_run: storage.set_status(task, prefix, expnum, version=version, ccd=ccd, status=message)
def caom2(mpc_filename, search_date="2014 07 24.0"): """ builds a TSV file in the format of SSOIS by querying for possilbe observations in CADC/CAOM2. This is a fall back program, should only be useful when SSOIS is behind. """ columns = ('Image', 'Ext', 'X', 'Y', 'MJD', 'Filter', 'Exptime', 'Object_RA', 'Object_Dec', 'Image_target', 'Telescope/Instrument', 'MetaData', 'Datalink') ephem_table = Table(names=columns, dtypes=('S10', 'i4', 'f8', 'f8', 'f8', 'S10', 'f8', 'f8', 'f8', 'S20', 'S20', 'S20', 'S50')) ephem_table.pprint() o = orbfit.Orbfit(mpc.MPCReader(mpc_filename).mpc_observations) o.predict(search_date) fields = storage.cone_search(o.coordinate.ra.degrees, o.coordinate.dec.degrees, dra=0.3, ddec=0.3, calibration_level=1) mjdates = numpy.unique(fields['mjdate']) collectionIDs = [] for mjdate in mjdates: jd = 2400000.5 + mjdate o.predict(jd) for field in storage.cone_search(o.coordinate.ra.degrees, o.coordinate.dec.degrees, dra=30./3600.0, ddec=30./3600.0, mjdate=mjdate, calibration_level=1): collectionIDs.append(field['collectionID']) expnums = numpy.unique(numpy.array(collectionIDs)) for expnum in expnums: header = storage.get_astheader(expnum, 22) o.predict(header['MJDATE']+2400000.5) print o.time.iso, o.coordinate.ra.degrees, o.coordinate.dec.degrees for ccd in range(36): header = storage.get_astheader(expnum, ccd) w = wcs.WCS(header) (x, y) = w.sky2xy(o.coordinate.ra.degrees, o.coordinate.dec.degrees) print ccd, x, y if 0 < x < header['NAXIS1'] and 0 < y < header['NAXIS2']: ephem_table.add_row([expnum, ccd+1, x, y, header['MJDATE'], header['FILTER'], header['EXPTIME'], o.coordinate.ra.degrees, o.coordinate.dec.degrees, header['OBJECT'], 'CFHT/MegaCam', None, "http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/data/pub/CFHT/{}p[{}]".format(expnum, ccd)]) break ephem_table.pprint() ephem_table.write('backdoor.tsv', format='ascii', delimiter='\t')
def recompute_mag(mpc_in): """ Get the mag of the object given the mpc.Observation """ # TODO this really shouldn't need to build a 'reading' to get the cutout... from ossos.downloads.cutouts import ImageCutoutDownloader mpc_obs = deepcopy(mpc_in) assert isinstance(mpc_obs, mpc.Observation) if mpc_obs.null_observation: return mpc_obs parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame) if parts is None: return mpc_obs expnum = parts.group('expnum') ccd = parts.group('ccd') file_type = parts.group('type') observation = astrom.Observation(expnum, file_type, ccd) assert isinstance(observation, astrom.Observation) ast_header = storage.get_astheader(int(expnum), int(ccd)) # The ZP for the current astrometric lines is the pipeline one. The new ZP is in the astheader file. new_zp = ast_header.get('PHOTZP') # The ZP in the image header is likely the one used for the original photometry. old_zp = storage.get_zeropoint(int(expnum), int(ccd)) reading = astrom.SourceReading(float(mpc_obs.comment.x), float(mpc_obs.comment.y), float(mpc_obs.comment.x), float(mpc_obs.comment.y), mpc_obs.coordinate.ra.degrees, mpc_obs.coordinate.dec.degrees, float(mpc_obs.comment.x), float(mpc_obs.comment.y), observation, ssos=True, from_input_file=True, null_observation=False, discovery=mpc_obs.discovery) # reading.is_inverted = reading.compute_inverted() reading._header = storage.get_mopheader(expnum, ccd) reading._header['MAXCOUNT'] = 30000.0 image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100) cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True) cutout.zmag = new_zp if math.fabs(new_zp - old_zp) > 0.3: logging.warning("Large change in zeropoint detected: {} -> {}".format(old_zp, new_zp)) try: (x, y, mag, merr) = cutout.get_observed_magnitude(zmag=new_zp) (x, y) = cutout.get_observed_coordinates((x, y)) except: logging.warn("Failed to do photometry.") return mpc_obs if mpc_obs.comment.mag is not None and math.fabs(mpc_obs.comment.mag - mag) > 3.5 * mpc_obs.comment.mag_uncertainty: logging.warn("recomputed magnitude shift large: {} --> {}".format(mpc_obs.mag, mag)) if math.sqrt((x - mpc_obs.comment.x) ** 2 + (y - mpc_obs.comment.y) ** 2) > 1.0: logging.warn("Centroid shifted ({},{}) -> ({},{})".format(mpc_obs.comment.x, mpc_obs.comment.y, x, y)) # Don't use the new X/Y for Hand measured entries. if str(mpc_obs.note1) != "H": mpc_obs.comment.x = x mpc_obs.comment.y = y mpc_obs.comment.mag = mag mpc_obs.comment.mag_uncertainty = merr if mpc_obs.mag is not None: mpc_obs.mag = mag return mpc_obs
else: block_files = files[block] # not all provisional names have the full Oyearsemesterblock pattern. Because counter. reals_files = [ c for c in block_files if (c.__contains__(provisional_name) or c.__contains__(provisional_name[-3:])) ] if len(reals_files) > 0: # ensure it's present in this directory chip = reals_files[0].split('.')[0] cand_filename = '{}.measure3.cands.astrom'.format(chip) # now who set the tag? uri = path + cand_filename discoverer = storage.vospace.getNode( uri, force=True).props[storage.OSSOS_TAG_URI_BASE + '#done'] outfile.write('{:<12} {:<12}'.format(discoverer, chip)) # map the exposure number of the triplet to the header value for the Observer # FIXME: doesn't work for 14BH because the block is not called that in the path. Will need to overwrite triplet_path = '{}/{}_{}_discovery_expnums.txt'.format( storage.TRIPLETS, block[-1:], block[2:5]) triplets = storage.open_vos_or_local(triplet_path).read().split( '\n') triplet = [n for n in triplets if n.endswith(chip.split('_')[0])][0] header = storage.get_astheader(triplet.split()[0], chip.split('_')[1].strip('p')) outfile.write('{:<20} {:<20}\n'.format(header['QOBSERVE'], header['QCOORD']))
lines = open(sys.argv[1]).readlines() except: lines = sys.argv[1:] for line in lines: expnum = int(line.strip()) if expnum < 1785619: # Last exposures with 36 CCD Megaprime ccdlist = list(range(0, 36)) else: # First exposrues with 40 CCD Megaprime ccdlist = list(range(0, 40)) for ccd in ccdlist: try: try: header = storage.get_astheader(expnum, ccd) datasec = storage.datasec_to_list( header.get('DATASEC', '[80:2080,30,4160]')) except Exception as ex: print(ex) continue try: fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd)) except: fwhm = 'unknown' for keyword in del_keyword_list: try: del (header[keyword]) except: pass header['FWHM'] = (fwhm, 'FWHM in pixels')
def remeasure(mpc_in, recentroided=False): """ re-measure the astrometry and photometry of the given mpc line """ if mpc_in.null_observation: return mpc_in mpc_obs = deepcopy(mpc_in) logging.debug("rm start: {}".format(mpc_obs.to_string())) if not isinstance(mpc_obs.comment, mpc.OSSOSComment): logging.error( "Failed to convert comment line") return mpc_in parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame) if not parts: logging.error( "Failed to parse expnum from frame info in comment line") return mpc_in start_coordinate = mpc_in.coordinate #assert isinstance(start_coordinate, ICRSCoordinates) try: header = storage._get_sghead(parts.group('expnum'))[int(parts.group('ccd')) + 1] except IOError as ioerr: logging.error(str(ioerr)) logging.error("Failed to get astrometric header for: {}".format(mpc_obs)) return mpc_in this_wcs = wcs.WCS(header) try: #(x , y ) = this_wcs.sky2xy(mpc_obs.coordinate.ra.degree, mpc_obs.coordinate.dec.degree, usepv=True) x = mpc_obs.comment.x y = mpc_obs.comment.y (ra, dec) = this_wcs.xy2sky(x, y, usepv=True) mpc_obs.coordinate = (ra, dec) sep = mpc_in.coordinate.separation(mpc_obs.coordinate) if sep > TOLERANCE*20 and mpc_in.discovery and (int(parts.group('ccd')) < 18 or int(parts.group('ccd')) in [36, 37]): logging.warn("Large ({}) offset using X/Y in comment line to compute RA/DEC".format(sep)) logging.warn("This is a discovery line so flipping/flopping the x/y position recorded in comment as that " "may be taken from a flip/flopped image.") x = header['NAXIS1'] - x + 1 y = header['NAXIS2'] - y + 1 sep = mpc_in.coordinate.separation(mpc_obs.coordinate) (ra, dec) = this_wcs.xy2sky(x, y, usepv=True) # print "--> x/y coordinates ({},{}) and recomputing ra/dec ({},{})".format(x, y, ra, dec) mpc_obs.coordinate = (ra, dec) if sep > TOLERANCE*20 and mpc_in.discovery: logging.warn( "Large ({}) offset after flipping, so reverting.".format(sep)) # Try un-flipping. x = float(mpc_in.comment.x) y = float(mpc_in.comment.y) (ra, dec) = this_wcs.xy2sky(x, y, usepv=True) # print "--> x/y coordinates ({},{}) and recomputing ra/dec ({},{})".format(x, y, ra, dec) mpc_obs.coordinate = (ra, dec) except: logging.warn("Failed to get the X Y coordinate from comment line.") return mpc_in # (ra, dec) = this_wcs.xy2sky(x, y, usepv=True) # mpc_obs.coordinate = (ra, dec) # sep = mpc_in.coordinate.separation(mpc_obs.coordinate) if sep > TOLERANCE and not recentroided: # use the old header RA/DEC to predict the X/Y and then use that X/Y to get new RA/DEC logging.warn("sep: {} --> large offset when using comment line X/Y to compute RA/DEC") logging.warn("Using RA/DEC and original WCS to compute X/Y and replacing X/Y in comment.".format(sep)) header2 = storage.get_astheader(parts.group('expnum'), int(parts.group('ccd'))) image_wcs = wcs.WCS(header2) (x, y) = image_wcs.sky2xy(mpc_in.coordinate.ra.degree, mpc_in.coordinate.dec.degree, usepv=False) (ra, dec) = this_wcs.xy2sky(x, y, usepv=True) logging.warn("Coordinate changed: ({:5.2f},{:5.2f}) --> ({:5.2f},{:5.2f})".format(mpc_obs.comment.x, mpc_obs.comment.y, x, y)) mpc_obs.coordinate = (ra, dec) mpc_obs.comment.x = x mpc_obs.comment.y = y try: merr = float(mpc_obs.comment.mag_uncertainty) fwhm = float(storage.get_fwhm(parts.group('expnum'), int(parts.group('ccd')))) * header['PIXSCAL1'] centroid_err = merr * fwhm logging.debug("Centroid uncertainty: {} {} => {}".format(merr, fwhm, centroid_err)) except Exception as err: logging.error(str(err)) logging.error("Failed to compute centroid_err for observation:\n" "{}\nUsing default of 0.2".format(mpc_obs.to_string())) centroid_err = 0.2 mpc_obs.comment.astrometric_level = header.get('ASTLEVEL', "0") try: asterr = float(header['ASTERR']) residuals = (asterr ** 2 + centroid_err ** 2) ** 0.5 logging.debug("Residuals: {} {} => {}".format(asterr, centroid_err, residuals)) except Exception as err: logging.error(str(err)) logging.error("Failed while trying to compute plate uncertainty for\n{}".format(mpc_obs.to_stirng())) logging.error('Using default of 0.25') residuals = 0.25 mpc_obs.comment.plate_uncertainty = residuals logging.debug("sending back: {}".format(mpc_obs.to_string())) return mpc_obs
if block in ['2013A-E', '2013A-O']: # 13AE and 13AO ended up particularly weird. Fortunately, have a local list with open('/Users/michele/Desktop/{}_mpclist.txt'.format(block), 'r') as infile: block_files = infile.readlines() else: print 'starting VOSpace read' # getting file listings from big listings on VOSpace is slooooowwww... block_files = filter(lambda name: not name.startswith('fk'), storage.listdir(path)) files[block] = block_files else: block_files = files[block] # not all provisional names have the full Oyearsemesterblock pattern. Because counter. reals_files = [c for c in block_files if (c.__contains__(provisional_name) or c.__contains__(provisional_name[-3:]))] if len(reals_files) > 0: # ensure it's present in this directory chip = reals_files[0].split('.')[0] cand_filename = '{}.measure3.cands.astrom'.format(chip) # now who set the tag? uri = path + cand_filename discoverer = storage.vospace.getNode(uri, force=True).props[storage.OSSOS_TAG_URI_BASE + '#done'] outfile.write('{:<12} {:<12}'.format(discoverer, chip)) # map the exposure number of the triplet to the header value for the Observer # FIXME: doesn't work for 14BH because the block is not called that in the path. Will need to overwrite triplet_path = '{}/{}_{}_discovery_expnums.txt'.format(storage.TRIPLETS, block[-1:], block[2:5]) triplets = storage.open_vos_or_local(triplet_path).read().split('\n') triplet = [n for n in triplets if n.endswith(chip.split('_')[0])][0] header = storage.get_astheader(triplet.split()[0], chip.split('_')[1].strip('p')) outfile.write('{:<20} {:<20}\n'.format(header['QOBSERVE'], header['QCOORD']))
def main(): """Do the script.""" parser = argparse.ArgumentParser( description='replace image header') parser.add_argument('--extname', help='name of extension to in header') parser.add_argument('expnum', type=str, help='exposure to update') parser.add_argument('-r', '--replace', action='store_true', help='store modified image back to VOSpace?') parser.add_argument('-v', '--verbose', action='store_true') parser.add_argument('--debug', action='store_true') parser.add_argument('--force', action='store_true', help="Re-run even if previous success recorded") parser.add_argument('--dbimages', help="VOSpace DATA storage area.", default="vos:OSSOS/dbimages") args = parser.parse_args() task = util.task() dependency = 'preproc' prefix = "" storage.DBIMAGES = args.dbimages level = logging.CRITICAL message_format = "%(message)s" if args.verbose: level = logging.INFO if args.debug: level = logging.DEBUG message_format = "%(module)s %(funcName)s %(lineno)s %(message)s" logging.basicConfig(level=level, format=message_format) storage.set_logger(task, prefix, args.expnum, None, None, False) message = storage.SUCCESS expnum = args.expnum exit_status = 0 try: # skip if already succeeded and not in force mode if storage.get_status(task, prefix, expnum, "p", 36) and not args.force: logging.info("Already updated, skipping") sys.exit(0) image_hdulist = storage.get_image(args.expnum, return_file=False) ast_hdulist = storage.get_astheader(expnum, ccd=None) run_update_header(image_hdulist, ast_hdulist) image_filename = os.path.basename(storage.get_uri(expnum)) image_hdulist.writeto(image_filename) if args.replace: dest = storage.dbimages_uri(expnum) storage.copy(image_filename, dest) storage.set_status('update_header', "", expnum, 'p', 36, message) except Exception as e: message = str(e) if args.replace: storage.set_status(task, prefix, expnum, 'p', 36, message) exit_status = message logging.error(message) return exit_status
def remeasure(mpc_in): """ re-measure the astrometry and photometry of the given mpc line """ # TODO Actually implement this. if mpc_in.null_observation: return mpc_in mpc_obs = deepcopy(mpc_in) logging.debug("rm start: {}".format(mpc_obs.to_string())) if not isinstance(mpc_obs.comment, mpc.OSSOSComment): return mpc_in parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame) if not parts: return mpc_in start_coordinate = mpc_in.coordinate assert isinstance(start_coordinate, ICRSCoordinates) try: header = storage.get_astheader(parts.group('expnum'), int(parts.group('ccd'))) except IOError as ioerr: logging.error(str(ioerr)) logging.error("Failed to get astrometric header for: {}".format(mpc_obs)) return mpc_in this_wcs = wcs.WCS(header) try: x = float(mpc_obs.comment.x) y = float(mpc_obs.comment.y) if mpc_in.discovery: logging.debug("Discovery astrometric lines are normally flipped relative to storage.") x = header['NAXIS1'] - x + 1 y = header['NAXIS2'] - y + 1 except: logging.warn("Failed to X/Y from comment line.") return mpc_in (ra, dec) = this_wcs.xy2sky(x, y) mpc_obs.coordinate = (ra, dec) logging.debug("rm updat: {}".format(mpc_obs.to_string())) sep = start_coordinate.separation(mpc_obs.coordinate).degrees * 3600.0 if sep > TOLERANCE and mpc_in.discovery: logging.warn("{} --> Using the unflipped X/Y for a discovery observation line.".format(sep)) logging.debug("{} {} {} {} {}".format(sep, x, y, mpc_obs.comment.x, mpc_obs.comment.y)) # Try un-flipping. x = float(mpc_obs.comment.x) y = float(mpc_obs.comment.y) (ra, dec) = this_wcs.xy2sky(x, y) #print "--> x/y coordinates ({},{}) and recomputing ra/dec ({},{})".format(x, y, ra, dec) mpc_obs.coordinate = (ra, dec) sep = start_coordinate.separation(mpc_obs.coordinate).degrees * 3600.0 logging.debug("remod: {}".format(mpc_obs.coordinate)) logging.debug("SEP: {}".format(sep)) logging.debug("rm flip: {}".format(mpc_obs.to_string())) if sep > TOLERANCE: ## use the old header RA/DEC to predict the X/Y and then use that X/Y to get new RA/DEC logging.debug("Ignoring recorded X/Y and using previous to RA/DEC and WCS to compute X/Y") header2 = storage.get_image(parts.group('expnum'), int(parts.group('ccd')), return_file=False, flip_image=False)[0].header image_wcs = wcs.WCS(header2) (x, y) = image_wcs.sky2xy(mpc_in.coordinate.ra.degrees, mpc_in.coordinate.dec.degrees) (ra, dec) = this_wcs.xy2sky(x, y) logging.debug("({},{}) --> ({},{})".format(mpc_obs.comment.x, mpc_obs.comment.y, x, y)) mpc_obs.coordinate = (ra, dec) mpc_obs.comment.x = x mpc_obs.comment.y = y try: merr = float(mpc_obs.comment.mag_uncertainty) fwhm = float(storage.get_fwhm(parts.group('expnum'), int(parts.group('ccd')))) * header['PIXSCAL1'] centroid_err = merr * fwhm logging.debug("Centroid uncertainty: {} {} => {}".format(merr, fwhm, centroid_err)) except Exception as err: logging.error(str(err)) logging.error("Failed to compute centroid for observation:\n" "{}\nUsing default of 0.2".format(mpc_obs.to_string())) centroid_err = 0.2 mpc_obs.comment.astrometric_level = header.get('ASTLEVEL', "0") try: asterr = float(header['ASTERR']) residuals = (asterr ** 2 + centroid_err ** 2) ** 0.5 logging.debug("Residuals: {} {} => {}".format(asterr, centroid_err, residuals)) except Exception as err: logging.error(str(err)) logging.error("Failed while trying to compute plate uncertainty for\n{}".format(mpc_obs.to_stirng())) logging.error('Using default of 0.25') residuals = 0.25 mpc_obs.comment.plate_uncertainty = residuals logging.debug("sending back: {}".format(mpc_obs.to_string())) return mpc_obs
def main(): """Do the script.""" parser = argparse.ArgumentParser(description='replace image header') parser.add_argument('--extname', help='name of extension to in header') parser.add_argument('expnum', type=str, help='exposure to update') parser.add_argument('-r', '--replace', action='store_true', help='store modified image back to VOSpace?') parser.add_argument('-v', '--verbose', action='store_true') parser.add_argument('--debug', action='store_true') parser.add_argument('--force', action='store_true', help="Re-run even if previous success recorded") parser.add_argument('--dbimages', help="VOSpace DATA storage area.", default="vos:OSSOS/dbimages") args = parser.parse_args() task = util.task() dependency = 'preproc' prefix = "" storage.DBIMAGES = args.dbimages level = logging.CRITICAL message_format = "%(message)s" if args.verbose: level = logging.INFO if args.debug: level = logging.DEBUG message_format = "%(module)s %(funcName)s %(lineno)s %(message)s" logging.basicConfig(level=level, format=message_format) storage.set_logger(task, prefix, args.expnum, None, None, False) message = storage.SUCCESS expnum = args.expnum exit_status = 0 try: # skip if already succeeded and not in force mode if storage.get_status(task, prefix, expnum, "p", 36) and not args.force: logging.info("Already updated, skipping") sys.exit(0) image_hdulist = storage.get_image(args.expnum, return_file=False) ast_hdulist = storage.get_astheader(expnum, ccd=None) run_update_header(image_hdulist, ast_hdulist) image_filename = os.path.basename(storage.get_uri(expnum)) image_hdulist.writeto(image_filename) if args.replace: dest = storage.dbimages_uri(expnum) storage.copy(image_filename, dest) storage.set_status('update_header', "", expnum, 'p', 36, message) except Exception as e: message = str(e) if args.replace: storage.set_status(task, prefix, expnum, 'p', 36, message) exit_status = message logging.error(message) return exit_status
def main(expnum, ccd): header = storage.get_astheader(expnum, ccd) datasec = storage.datasec_to_list( header.get('DATASEC', '[80:2080,30,4160]')) try: fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd)) except: fwhm = 'unknown' for keyword in del_keyword_list: try: del (header[keyword]) except: pass header['FWHM'] = (fwhm, 'FWHM in pixels') header['EXTNAME'] = 'header' primary_hdu = fits.PrimaryHDU(header=header) hdu_list = fits.HDUList([ primary_hdu, ]) for ext in ['jmp', 'matt']: extension = 'obj.' + ext name = "{}p{:02d}.{}".format(expnum, ccd, extension) try: os.unlink(name) os.unlink(name + ".fits") except: pass logging.info("Retrieving {}".format(name)) obj_file = mop_file.Parser(expnum, ccd, extension) obj_file.parse() t = numpy.all([ datasec[0] < obj_file.data['X'], obj_file.data['X'] < datasec[1], datasec[2] < obj_file.data['Y'], obj_file.data['Y'] < datasec[3] ], axis=0) logging.info("Source remaining after datasec cut: {} of {}".format( len(obj_file.data[t]['X']), len(t))) table_hdu = fits.table_to_hdu(obj_file.data[t]) table_hdu.header['CATALOG'] = name table_hdu.header['EXTNAME'] = ext hdu_list.append(table_hdu) del table_hdu del obj_file os.unlink(name) name = "{}p{:02d}.{}".format(expnum, ccd, 'obj.fits') if os.access(name, os.F_OK): os.unlink(name) hdu_list.writeto(name) uri = storage.dbimages_uri(expnum, ccd, 'p', ext=".obj.fits") logging.info(name + " -> " + uri) count = 0 while True: print(("Copy attempt {}".format(count))) try: storage.copy(name, uri) os.unlink(name) break except Exception as ex: if count > 10: raise ex count += 1
def ephem_search(mpc_filename, search_date="2014 07 24.0"): """ builds a TSV file in the format of SSOIS by querying for possible observations in CADC/CAOM2. This is a fall back program, should only be useful when SSOIS is behind. """ columns = ('Image', 'Ext', 'X', 'Y', 'MJD', 'Filter', 'Exptime', 'Object_RA', 'Object_Dec', 'Image_target', 'Telescope/Instrument', 'MetaData', 'Datalink') ephem_table = Table(names=columns, dtypes=('S10', 'i4', 'f8', 'f8', 'f8', 'S10', 'f8', 'f8', 'f8', 'S20', 'S20', 'S20', 'S50')) ephem_table.pprint() o = orbfit.Orbfit(mpc.MPCReader(mpc_filename).mpc_observations) o.predict(search_date) fields = storage.cone_search(o.coordinate.ra.degrees, o.coordinate.dec.degrees, dra=0.3, ddec=0.3, calibration_level=1) mjdates = numpy.unique(fields['mjdate']) collectionIDs = [] for mjdate in mjdates: jd = 2400000.5 + mjdate o.predict(jd) for field in storage.cone_search(o.coordinate.ra.degrees, o.coordinate.dec.degrees, dra=30. / 3600.0, ddec=30. / 3600.0, mjdate=mjdate, calibration_level=1): collectionIDs.append(field['collectionID']) expnums = numpy.unique(numpy.array(collectionIDs)) for expnum in expnums: header = storage.get_astheader(expnum, 22) o.predict(header['MJDATE'] + 2400000.5) print(o.time.iso, o.coordinate.ra.degrees, o.coordinate.dec.degrees) for ccd in range(36): header = storage.get_astheader(expnum, ccd) w = wcs.WCS(header) (x, y) = w.sky2xy(o.coordinate.ra.degrees, o.coordinate.dec.degrees) print(ccd, x, y) if 0 < x < header['NAXIS1'] and 0 < y < header['NAXIS2']: ephem_table.add_row([ expnum, ccd + 1, x, y, header['MJDATE'], header['FILTER'], header['EXPTIME'], o.coordinate.ra.degrees, o.coordinate.dec.degrees, header['OBJECT'], 'CFHT/MegaCam', None, "https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/data/pub/CFHT/{}p[{}]" .format(expnum, ccd) ]) break ephem_table.pprint() ephem_table.write('backdoor.tsv', format='ascii', delimiter='\t')
lines = open(sys.argv[1]).readlines() except: lines = sys.argv[1:] for line in lines: expnum = int(line.strip()) if expnum < 1785619: # Last exposures with 36 CCD Megaprime ccdlist = range(0, 36) else: # First exposrues with 40 CCD Megaprime ccdlist = range(0, 40) for ccd in ccdlist: try: try: header = storage.get_astheader(expnum, ccd) datasec = storage.datasec_to_list(header.get('DATASEC', '[80:2080,30,4160]')) except Exception as ex: print(ex) continue try: fwhm = "{:5.2f}".format(storage.get_fwhm(expnum, ccd)) except: fwhm = 'unknown' for keyword in del_keyword_list: try: del(header[keyword]) except: pass header['FWHM'] = (fwhm, 'FWHM in pixels') header['EXTNAME'] = 'header'