def healpix_for_radec(self, ra, dec): ''' Returns the healpix number for a given single (scalar) RA,Dec. ''' from astrometry.util.util import radecdegtohealpix, healpix_xy_to_ring hpxy = radecdegtohealpix(ra, dec, self.nside) ipring = healpix_xy_to_ring(hpxy, self.nside) return ipring
def healpix_for_radec(self, ra, dec): ''' Returns the healpix number for a given single (scalar) RA,Dec. ''' from astrometry.util.util import radecdegtohealpix, healpix_xy_to_ring, healpix_xy_to_nested hpxy = radecdegtohealpix(ra, dec, self.nside) if self.indexing == 'ring': hp = healpix_xy_to_ring(hpxy, self.nside) elif self.indexing == 'nested': hp = healpix_xy_to_nested(hpxy, self.nside) else: # assume XY! hp = hpxy return hp
def get_cat(self,ra,dec): """Read the healpixed PS1 catalogs given input ra,dec coordinates.""" from astrometry.util.fits import fits_table, merge_tables from astrometry.util.util import radecdegtohealpix, healpix_xy_to_ring ipring = np.empty(len(ra)).astype(int) for iobj, (ra1, dec1) in enumerate(zip(ra,dec)): hpxy = radecdegtohealpix(ra1,dec1,self.nside) ipring[iobj] = healpix_xy_to_ring(hpxy,self.nside) pix = np.unique(ipring) cat = list() for ipix in pix: fname = os.path.join(self.ps1dir,'ps1-'+'{:05d}'.format(ipix)+'.fits') print('Reading {}'.format(fname)) cat.append(fits_table(fname)) cat = merge_tables(cat) return cat
def dojob(job, userimage, log=None, solve_command=None, solve_locally=None, tempfiles=None): print('dojob: tempdir:', tempfile.gettempdir()) jobdir = job.make_dir() #print('Created job dir', jobdir) #log = create_job_logger(job) #jobdir = job.get_dir() if log is None: log = create_job_logger(job) log.msg('Starting Job processing for', job) job.set_start_time() job.save() #os.chdir(dirnm) - not thread safe (working directory is global)! log.msg('Creating directory', jobdir) axyfn = 'job.axy' axypath = os.path.join(jobdir, axyfn) sub = userimage.submission log.msg('submission id', sub.id) df = userimage.image.disk_file img = userimage.image # Build command-line arguments for the augment-xylist program, which # detects sources in the image and adds processing arguments to the header # to produce a "job.axy" file. slo, shi = sub.get_scale_bounds() # Note, this must match Job.get_wcs_file(). wcsfile = 'wcs.fits' corrfile = 'corr.fits' axyflags = [] axyargs = { '--out': axypath, '--scale-low': slo, '--scale-high': shi, '--scale-units': sub.scale_units, '--wcs': wcsfile, '--corr': corrfile, '--rdls': 'rdls.fits', '--pixel-error': sub.positional_error, '--ra': sub.center_ra, '--dec': sub.center_dec, '--radius': sub.radius, '--downsample': sub.downsample_factor, # tuning-up maybe fixed; if not, turn it off with: #'--odds-to-tune': 1e9, # Other things we might want include... # --invert # -g / --guess-scale: try to guess the image scale from the FITS headers # --crpix-x <pix>: set the WCS reference point to the given position # --crpix-y <pix>: set the WCS reference point to the given position # -w / --width <pixels>: specify the field width # -e / --height <pixels>: specify the field height # -X / --x-column <column-name>: the FITS column name # -Y / --y-column <column-name> } if hasattr(img, 'sourcelist'): # image is a source list; use --xylist axyargs['--xylist'] = img.sourcelist.get_fits_path(tempfiles=tempfiles) w, h = img.width, img.height if sub.image_width: w = sub.image_width if sub.image_height: h = sub.image_height axyargs['--width'] = w axyargs['--height'] = h else: axyargs['--image'] = df.get_path() # UGLY if sub.parity == 0: axyargs['--parity'] = 'pos' elif sub.parity == 1: axyargs['--parity'] = 'neg' if sub.tweak_order == 0: axyflags.append('--no-tweak') else: axyargs['--tweak-order'] = '%i' % sub.tweak_order if sub.use_sextractor: axyflags.append('--use-source-extractor') if sub.crpix_center: axyflags.append('--crpix-center') if sub.invert: axyflags.append('--invert') cmd = 'augment-xylist ' for (k, v) in list(axyargs.items()): if v: cmd += k + ' ' + str(v) + ' ' for k in axyflags: cmd += k + ' ' log.msg('running: ' + cmd) (rtn, out, err) = run_command(cmd) if rtn: log.msg('out: ' + out) log.msg('err: ' + err) logmsg('augment-xylist failed: rtn val', rtn, 'err', err) raise Exception log.msg('created axy file', axypath) # shell into compute server... logfn = job.get_log_file() # the "tar" commands both use "-C" to chdir, and the ssh command # and redirect uses absolute paths. if solve_locally is not None: cmd = (('cd %(jobdir)s && %(solvecmd)s %(jobid)s %(axyfile)s >> ' + '%(logfile)s') % dict(jobid='job-%s-%i' % (settings.sitename, job.id), solvecmd=solve_locally, axyfile=axyfn, jobdir=jobdir, logfile=logfn)) log.msg('command:', cmd) w = os.system(cmd) if not os.WIFEXITED(w): log.msg('Solver failed (sent signal?)') logmsg('Call to solver failed for job', job.id) raise Exception rtn = os.WEXITSTATUS(w) if rtn: log.msg('Solver failed with return value %i' % rtn) logmsg('Call to solver failed for job', job.id, 'with return val', rtn) raise Exception log.msg('Solver completed successfully.') else: if solve_command is None: solve_command = 'ssh -x -T %(sshconfig)s' cmd = (( '(echo %(jobid)s; ' 'tar cf - --ignore-failed-read -C %(jobdir)s %(axyfile)s) | ' + solve_command + ' 2>>%(logfile)s | ' 'tar xf - --atime-preserve -m --exclude=%(axyfile)s -C %(jobdir)s ' '>>%(logfile)s 2>&1') % dict(jobid='job-%s-%i' % (settings.sitename, job.id), axyfile=axyfn, jobdir=jobdir, sshconfig=settings.ssh_solver_config, logfile=logfn)) log.msg('command:', cmd) w = os.system(cmd) if not os.WIFEXITED(w): log.msg('Solver failed (sent signal?)') logmsg('Call to solver failed for job', job.id) raise Exception rtn = os.WEXITSTATUS(w) if rtn: log.msg('Solver failed with return value %i' % rtn) logmsg('Call to solver failed for job', job.id, 'with return val', rtn) raise Exception log.msg('Solver completed successfully.') # Solved? wcsfn = os.path.join(jobdir, wcsfile) log.msg('Checking for WCS file', wcsfn) if os.path.exists(wcsfn): log.msg('WCS file exists') # Parse the wcs.fits file wcs = Tan(wcsfn, 0) # Convert to database model... tan = TanWCS(crval1=wcs.crval[0], crval2=wcs.crval[1], crpix1=wcs.crpix[0], crpix2=wcs.crpix[1], cd11=wcs.cd[0], cd12=wcs.cd[1], cd21=wcs.cd[2], cd22=wcs.cd[3], imagew=img.width, imageh=img.height) tan.save() log.msg('Created TanWCS:', tan) # Find field's healpix nside and index ra, dec, radius = tan.get_center_radecradius() nside = anutil.healpix_nside_for_side_length_arcmin(radius * 60) nside = int(2**round(math.log(nside, 2))) nside = max(1, nside) healpix = anutil.radecdegtohealpix(ra, dec, nside) try: sky_location, created = SkyLocation.objects.get_or_create( nside=nside, healpix=healpix) except MultipleObjectsReturned: log.msg('Multiple SkyLocations for nside %i, healpix %i' % (nside, healpix)) # arbitrarily take the first one. sky_location = SkyLocation.objects.filter(nside=nside, healpix=healpix)[0] log.msg('SkyLocation:', sky_location) # Find bounds for the Calibration object. r0, r1, d0, d1 = wcs.radec_bounds() # Find cartesian coordinates ra *= math.pi / 180 dec *= math.pi / 180 tempr = math.cos(dec) x = tempr * math.cos(ra) y = tempr * math.sin(ra) z = math.sin(dec) r = radius / 180 * math.pi calib = Calibration(raw_tan=tan, ramin=r0, ramax=r1, decmin=d0, decmax=d1, x=x, y=y, z=z, r=r, sky_location=sky_location) calib.save() log.msg('Created Calibration', calib) job.calibration = calib job.save() # save calib before adding machine tags job.status = 'S' job.user_image.add_machine_tags(job) job.user_image.add_sky_objects(job) else: job.status = 'F' job.set_end_time() job.save() log.msg('Finished job', job.id) logmsg('Finished job', job.id) return job.id
def dojob(job, userimage, log=None): jobdir = job.make_dir() #print 'Created job dir', jobdir #log = create_job_logger(job) #jobdir = job.get_dir() if log is None: log = create_job_logger(job) log.msg('Starting Job processing for', job) job.set_start_time() job.save() #os.chdir(dirnm) - not thread safe (working directory is global)! log.msg('Creating directory', jobdir) axyfn = 'job.axy' axypath = os.path.join(jobdir, axyfn) sub = userimage.submission log.msg('submission id', sub.id) df = userimage.image.disk_file img = userimage.image # Build command-line arguments for the augment-xylist program, which # detects sources in the image and adds processing arguments to the header # to produce a "job.axy" file. slo,shi = sub.get_scale_bounds() # Note, this must match Job.get_wcs_file(). wcsfile = 'wcs.fits' axyflags = [] axyargs = { '--out': axypath, '--scale-low': slo, '--scale-high': shi, '--scale-units': sub.scale_units, '--wcs': wcsfile, '--rdls': 'rdls.fits', '--pixel-error': sub.positional_error, '--ra': sub.center_ra, '--dec': sub.center_dec, '--radius': sub.radius, '--downsample': sub.downsample_factor, # tuning-up maybe fixed; if not, turn it off with: #'--odds-to-tune': 1e9, # Other things we might want include... # --invert # -g / --guess-scale: try to guess the image scale from the FITS headers # --crpix-x <pix>: set the WCS reference point to the given position # --crpix-y <pix>: set the WCS reference point to the given position # -w / --width <pixels>: specify the field width # -e / --height <pixels>: specify the field height # -X / --x-column <column-name>: the FITS column name # -Y / --y-column <column-name> } if hasattr(img,'sourcelist'): # image is a source list; use --xylist axyargs['--xylist'] = img.sourcelist.get_fits_path() w,h = img.width, img.height if sub.image_width: w = sub.image_width if sub.image_height: h = sub.image_height axyargs['--width' ] = w axyargs['--height'] = h else: axyargs['--image'] = df.get_path() # UGLY if sub.parity == 0: axyargs['--parity'] = 'pos' elif sub.parity == 1: axyargs['--parity'] = 'neg' if sub.tweak_order == 0: axyflags.append('--no-tweak') else: axyargs['--tweak-order'] = '%i' % sub.tweak_order if sub.use_sextractor: axyflags.append('--use-sextractor') if sub.crpix_center: axyflags.append('--crpix-center') if sub.invert: axyflags.append('--invert') cmd = 'augment-xylist ' for (k,v) in axyargs.items(): if v: cmd += k + ' ' + str(v) + ' ' for k in axyflags: cmd += k + ' ' log.msg('running: ' + cmd) (rtn, out, err) = run_command(cmd) if rtn: log.msg('out: ' + out) log.msg('err: ' + err) logmsg('augment-xylist failed: rtn val', rtn, 'err', err) raise Exception log.msg('created axy file', axypath) # shell into compute server... logfn = job.get_log_file() # the "tar" commands both use "-C" to chdir, and the ssh command # and redirect uses absolute paths. cmd = ('(echo %(jobid)s; ' 'tar cf - --ignore-failed-read -C %(jobdir)s %(axyfile)s) | ' 'ssh -x -T %(sshconfig)s 2>>%(logfile)s | ' 'tar xf - --atime-preserve -m --exclude=%(axyfile)s -C %(jobdir)s ' '>>%(logfile)s 2>&1' % dict(jobid='job-%s-%i' % (settings.sitename, job.id), axyfile=axyfn, jobdir=jobdir, sshconfig=settings.ssh_solver_config, logfile=logfn)) log.msg('command:', cmd) w = os.system(cmd) if not os.WIFEXITED(w): log.msg('Solver failed (sent signal?)') logmsg('Call to solver failed for job', job.id) raise Exception rtn = os.WEXITSTATUS(w) if rtn: log.msg('Solver failed with return value %i' % rtn) logmsg('Call to solver failed for job', job.id, 'with return val', rtn) raise Exception log.msg('Solver completed successfully.') # Solved? wcsfn = os.path.join(jobdir, wcsfile) log.msg('Checking for WCS file', wcsfn) if os.path.exists(wcsfn): log.msg('WCS file exists') # Parse the wcs.fits file wcs = Tan(wcsfn, 0) # Convert to database model... tan = TanWCS(crval1=wcs.crval[0], crval2=wcs.crval[1], crpix1=wcs.crpix[0], crpix2=wcs.crpix[1], cd11=wcs.cd[0], cd12=wcs.cd[1], cd21=wcs.cd[2], cd22=wcs.cd[3], imagew=img.width, imageh=img.height) tan.save() log.msg('Created TanWCS:', tan) # Find field's healpix nside and index ra, dec, radius = tan.get_center_radecradius() nside = anutil.healpix_nside_for_side_length_arcmin(radius*60) nside = int(2**round(math.log(nside, 2))) healpix = anutil.radecdegtohealpix(ra, dec, nside) sky_location, created = SkyLocation.objects.get_or_create(nside=nside, healpix=healpix) log.msg('SkyLocation:', sky_location) # Find bounds for the Calibration object. r0,r1,d0,d1 = wcs.radec_bounds() # Find cartesian coordinates ra *= math.pi/180 dec *= math.pi/180 tempr = math.cos(dec) x = tempr*math.cos(ra) y = tempr*math.sin(ra) z = math.sin(dec) r = radius/180*math.pi calib = Calibration(raw_tan=tan, ramin=r0, ramax=r1, decmin=d0, decmax=d1, x=x,y=y,z=z,r=r, sky_location=sky_location) calib.save() log.msg('Created Calibration', calib) job.calibration = calib job.save() # save calib before adding machine tags job.status = 'S' job.user_image.add_machine_tags(job) job.user_image.add_sky_objects(job) else: job.status = 'F' job.set_end_time() job.save() log.msg('Finished job', job.id) logmsg('Finished job',job.id) return job.id
wcs = Tan(wcsfn, 0) model_solve_text = ('<TanWCS: CRVAL (%f, %f)' % (wcs.crval[0], wcs.crval[1]) + ' CRPIX (%f, %f)' % (wcs.crpix[0], wcs.crpix[1]) + ' CD (%f, %f; %f %f)' % tuple(wcs.cd) + ' Image size (%d, %d)>' % (image_width, image_height)) logger.info('Created Tan-WCS solution!: {}'.format(model_solve_text)) # Find field's healpix nside and index ra, dec = wcs.radec_center() radius = wcs.pixel_scale() * math.hypot(wcs.imagew, wcs.imageh) / 2.0 / 3600.0 nside = anutil.healpix_nside_for_side_length_arcmin(radius * 60) nside = int(2**round(math.log(nside, 2))) nside = max(1, nside) healpix = anutil.radecdegtohealpix(ra, dec, nside) # Find bounds for the Calibration object. r0, r1, d0, d1 = wcs.radec_bounds() # Find cartesian coordinates ra *= math.pi / 180 dec *= math.pi / 180 tempr = math.cos(dec) x = tempr * math.cos(ra) y = tempr * math.sin(ra) z = math.sin(dec) r = radius / 180 * math.pi output = """ Solution: ra: {ra}
def main(fn, mp): basefn = os.path.basename(fn) tiles = fits_table(fn) #I = np.flatnonzero(tiles.in_imaging) #tiles.cut(I) ### Split the tiles into nearby chunks of work for multi-processing. from astrometry.util.util import radecdegtohealpix nside = 8 Nhp = 12 * nside**2 Ihps = [[] for i in range(Nhp)] for i, (r, d) in enumerate(zip(tiles.ra, tiles.dec)): hp = radecdegtohealpix(r, d, nside) assert (hp >= 0) Ihps[hp].append(i) args = [] tiles.index = np.arange(len(tiles)) for hp, I in enumerate(Ihps): if len(I) == 0: continue args.append((tiles[np.array(I)], 'HP %i' % hp)) print(len(args), 'big healpixes are populated') R = mp.map(run_tiles, args) tiles_ann = merge_tables(R) print(len(tiles), 'tiles') print(len(tiles_ann), 'annotated') tiles_ann.about() # unpermute I = np.zeros(len(tiles_ann), np.int32) I[tiles_ann.index] = np.arange(len(tiles_ann)) tiles_ann.cut(I) tiles.add_columns_from(tiles_ann) tiles.delete_column('index') outfn = basefn.replace('.fits', '-brightest.fits') tiles.writeto(outfn) # Nudge (only inside imaging footprint) from functools import reduce brightest = reduce(np.minimum, [ tiles.brightest_guide_0_expanded[:, 0], tiles.brightest_guide_2_expanded[:, 0], tiles.brightest_guide_3_expanded[:, 0], tiles.brightest_guide_5_expanded[:, 0], tiles.brightest_guide_7_expanded[:, 0], tiles.brightest_guide_8_expanded[:, 0], tiles.brightest_focus_1_expanded[:, 0], tiles.brightest_focus_4_expanded[:, 0], tiles.brightest_focus_6_expanded[:, 0], tiles.brightest_focus_9_expanded[:, 0], ]) if 'in_imaging' in tiles.get_columns(): I = np.flatnonzero((brightest < 6.) * tiles.in_imaging) else: I = np.flatnonzero((brightest < 6.)) print(len(I), 'tiles with G<6') tiles.nudge_ra = np.zeros(len(tiles), np.float32) tiles.nudge_dec = np.zeros(len(tiles), np.float32) tiles.nudge_brightest = np.zeros(len(tiles), np.float32) tiles.index = np.arange(len(tiles)) for nudge in range(1, 20): if len(I) == 0: break print('Nudging', len(I), 'by', nudge) ddec = 10. / 3600. dra = ddec / np.cos(np.deg2rad(tiles.dec[I])) # copy tiles nudgetiles = tiles[np.repeat(I, 4)] nudgetiles.nudge_ra[0::4] = +nudge * dra nudgetiles.nudge_ra[1::4] = -nudge * dra nudgetiles.nudge_dec[2::4] = +nudge * ddec nudgetiles.nudge_dec[3::4] = -nudge * ddec nudgetiles.ra += nudgetiles.nudge_ra nudgetiles.dec += nudgetiles.nudge_dec #ann = run_tiles((nudgetiles, 'nudge%i' % nudge)) # split into subsets args = [] isplit = np.linspace(0, len(nudgetiles), 33, dtype=int) for i0, i1 in zip(isplit, isplit[1:]): if i0 == i1: continue args.append((nudgetiles[i0:i1], 'nudge%i+%i' % (nudge, i0))) A = mp.map(run_tiles, args) ann = merge_tables(A) ann.brightest = reduce(np.minimum, [ ann.brightest_guide_0_expanded[:, 0], ann.brightest_guide_2_expanded[:, 0], ann.brightest_guide_3_expanded[:, 0], ann.brightest_guide_5_expanded[:, 0], ann.brightest_guide_7_expanded[:, 0], ann.brightest_guide_8_expanded[:, 0], ann.brightest_focus_1_expanded[:, 0], ann.brightest_focus_4_expanded[:, 0], ann.brightest_focus_6_expanded[:, 0], ann.brightest_focus_9_expanded[:, 0], ]) ok = (ann.brightest > 6) found = np.zeros(len(I), bool) for idir, dirok in enumerate([ok[0::4], ok[1::4], ok[2::4], ok[3::4]]): J = np.flatnonzero(np.logical_not(found) * dirok) print('Nudge dir', idir, ':', len(J), 'are okay') found[J] = True tiles.nudge_ra[I[J]] = nudgetiles.nudge_ra[J * 4 + idir] tiles.nudge_dec[I[J]] = nudgetiles.nudge_dec[J * 4 + idir] tiles.nudge_brightest[I[J]] = ann.brightest[J * 4 + idir] I = I[np.flatnonzero(found == False)] outfn = basefn.replace('.fits', '-brightest-nudged.fits') tiles.writeto(outfn)