def healpix_for_radec(self, ra, dec):
'''
Returns the healpix number for a given single (scalar) RA,Dec.
'''
from astrometry.util.util import radecdegtohealpix, healpix_xy_to_ring
hpxy = radecdegtohealpix(ra, dec, self.nside)
ipring = healpix_xy_to_ring(hpxy, self.nside)
return ipring
def healpix_for_radec(self, ra, dec):
'''
Returns the healpix number for a given single (scalar) RA,Dec.
'''
from astrometry.util.util import radecdegtohealpix, healpix_xy_to_ring, healpix_xy_to_nested
hpxy = radecdegtohealpix(ra, dec, self.nside)
if self.indexing == 'ring':
hp = healpix_xy_to_ring(hpxy, self.nside)
elif self.indexing == 'nested':
hp = healpix_xy_to_nested(hpxy, self.nside)
else:
# assume XY!
hp = hpxy
return hp
def get_cat(self,ra,dec):
"""Read the healpixed PS1 catalogs given input ra,dec coordinates."""
from astrometry.util.fits import fits_table, merge_tables
from astrometry.util.util import radecdegtohealpix, healpix_xy_to_ring
ipring = np.empty(len(ra)).astype(int)
for iobj, (ra1, dec1) in enumerate(zip(ra,dec)):
hpxy = radecdegtohealpix(ra1,dec1,self.nside)
ipring[iobj] = healpix_xy_to_ring(hpxy,self.nside)
pix = np.unique(ipring)
cat = list()
for ipix in pix:
fname = os.path.join(self.ps1dir,'ps1-'+'{:05d}'.format(ipix)+'.fits')
print('Reading {}'.format(fname))
cat.append(fits_table(fname))
cat = merge_tables(cat)
return cat
def get_cat(self,ra,dec):
"""Read the healpixed PS1 catalogs given input ra,dec coordinates."""
from astrometry.util.fits import fits_table, merge_tables
from astrometry.util.util import radecdegtohealpix, healpix_xy_to_ring
ipring = np.empty(len(ra)).astype(int)
for iobj, (ra1, dec1) in enumerate(zip(ra,dec)):
hpxy = radecdegtohealpix(ra1,dec1,self.nside)
ipring[iobj] = healpix_xy_to_ring(hpxy,self.nside)
pix = np.unique(ipring)
cat = list()
for ipix in pix:
fname = os.path.join(self.ps1dir,'ps1-'+'{:05d}'.format(ipix)+'.fits')
print('Reading {}'.format(fname))
cat.append(fits_table(fname))
cat = merge_tables(cat)
return cat
def dojob(job,
userimage,
log=None,
solve_command=None,
solve_locally=None,
tempfiles=None):
print('dojob: tempdir:', tempfile.gettempdir())
jobdir = job.make_dir()
#print('Created job dir', jobdir)
#log = create_job_logger(job)
#jobdir = job.get_dir()
if log is None:
log = create_job_logger(job)
log.msg('Starting Job processing for', job)
job.set_start_time()
job.save()
#os.chdir(dirnm) - not thread safe (working directory is global)!
log.msg('Creating directory', jobdir)
axyfn = 'job.axy'
axypath = os.path.join(jobdir, axyfn)
sub = userimage.submission
log.msg('submission id', sub.id)
df = userimage.image.disk_file
img = userimage.image
# Build command-line arguments for the augment-xylist program, which
# detects sources in the image and adds processing arguments to the header
# to produce a "job.axy" file.
slo, shi = sub.get_scale_bounds()
# Note, this must match Job.get_wcs_file().
wcsfile = 'wcs.fits'
corrfile = 'corr.fits'
axyflags = []
axyargs = {
'--out': axypath,
'--scale-low': slo,
'--scale-high': shi,
'--scale-units': sub.scale_units,
'--wcs': wcsfile,
'--corr': corrfile,
'--rdls': 'rdls.fits',
'--pixel-error': sub.positional_error,
'--ra': sub.center_ra,
'--dec': sub.center_dec,
'--radius': sub.radius,
'--downsample': sub.downsample_factor,
# tuning-up maybe fixed; if not, turn it off with:
#'--odds-to-tune': 1e9,
# Other things we might want include...
# --invert
# -g / --guess-scale: try to guess the image scale from the FITS headers
# --crpix-x <pix>: set the WCS reference point to the given position
# --crpix-y <pix>: set the WCS reference point to the given position
# -w / --width <pixels>: specify the field width
# -e / --height <pixels>: specify the field height
# -X / --x-column <column-name>: the FITS column name
# -Y / --y-column <column-name>
}
if hasattr(img, 'sourcelist'):
# image is a source list; use --xylist
axyargs['--xylist'] = img.sourcelist.get_fits_path(tempfiles=tempfiles)
w, h = img.width, img.height
if sub.image_width:
w = sub.image_width
if sub.image_height:
h = sub.image_height
axyargs['--width'] = w
axyargs['--height'] = h
else:
axyargs['--image'] = df.get_path()
# UGLY
if sub.parity == 0:
axyargs['--parity'] = 'pos'
elif sub.parity == 1:
axyargs['--parity'] = 'neg'
if sub.tweak_order == 0:
axyflags.append('--no-tweak')
else:
axyargs['--tweak-order'] = '%i' % sub.tweak_order
if sub.use_sextractor:
axyflags.append('--use-source-extractor')
if sub.crpix_center:
axyflags.append('--crpix-center')
if sub.invert:
axyflags.append('--invert')
cmd = 'augment-xylist '
for (k, v) in list(axyargs.items()):
if v:
cmd += k + ' ' + str(v) + ' '
for k in axyflags:
cmd += k + ' '
log.msg('running: ' + cmd)
(rtn, out, err) = run_command(cmd)
if rtn:
log.msg('out: ' + out)
log.msg('err: ' + err)
logmsg('augment-xylist failed: rtn val', rtn, 'err', err)
raise Exception
log.msg('created axy file', axypath)
# shell into compute server...
logfn = job.get_log_file()
# the "tar" commands both use "-C" to chdir, and the ssh command
# and redirect uses absolute paths.
if solve_locally is not None:
cmd = (('cd %(jobdir)s && %(solvecmd)s %(jobid)s %(axyfile)s >> ' +
'%(logfile)s') % dict(jobid='job-%s-%i' %
(settings.sitename, job.id),
solvecmd=solve_locally,
axyfile=axyfn,
jobdir=jobdir,
logfile=logfn))
log.msg('command:', cmd)
w = os.system(cmd)
if not os.WIFEXITED(w):
log.msg('Solver failed (sent signal?)')
logmsg('Call to solver failed for job', job.id)
raise Exception
rtn = os.WEXITSTATUS(w)
if rtn:
log.msg('Solver failed with return value %i' % rtn)
logmsg('Call to solver failed for job', job.id, 'with return val',
rtn)
raise Exception
log.msg('Solver completed successfully.')
else:
if solve_command is None:
solve_command = 'ssh -x -T %(sshconfig)s'
cmd = ((
'(echo %(jobid)s; '
'tar cf - --ignore-failed-read -C %(jobdir)s %(axyfile)s) | ' +
solve_command + ' 2>>%(logfile)s | '
'tar xf - --atime-preserve -m --exclude=%(axyfile)s -C %(jobdir)s '
'>>%(logfile)s 2>&1') % dict(jobid='job-%s-%i' %
(settings.sitename, job.id),
axyfile=axyfn,
jobdir=jobdir,
sshconfig=settings.ssh_solver_config,
logfile=logfn))
log.msg('command:', cmd)
w = os.system(cmd)
if not os.WIFEXITED(w):
log.msg('Solver failed (sent signal?)')
logmsg('Call to solver failed for job', job.id)
raise Exception
rtn = os.WEXITSTATUS(w)
if rtn:
log.msg('Solver failed with return value %i' % rtn)
logmsg('Call to solver failed for job', job.id, 'with return val',
rtn)
raise Exception
log.msg('Solver completed successfully.')
# Solved?
wcsfn = os.path.join(jobdir, wcsfile)
log.msg('Checking for WCS file', wcsfn)
if os.path.exists(wcsfn):
log.msg('WCS file exists')
# Parse the wcs.fits file
wcs = Tan(wcsfn, 0)
# Convert to database model...
tan = TanWCS(crval1=wcs.crval[0],
crval2=wcs.crval[1],
crpix1=wcs.crpix[0],
crpix2=wcs.crpix[1],
cd11=wcs.cd[0],
cd12=wcs.cd[1],
cd21=wcs.cd[2],
cd22=wcs.cd[3],
imagew=img.width,
imageh=img.height)
tan.save()
log.msg('Created TanWCS:', tan)
# Find field's healpix nside and index
ra, dec, radius = tan.get_center_radecradius()
nside = anutil.healpix_nside_for_side_length_arcmin(radius * 60)
nside = int(2**round(math.log(nside, 2)))
nside = max(1, nside)
healpix = anutil.radecdegtohealpix(ra, dec, nside)
try:
sky_location, created = SkyLocation.objects.get_or_create(
nside=nside, healpix=healpix)
except MultipleObjectsReturned:
log.msg('Multiple SkyLocations for nside %i, healpix %i' %
(nside, healpix))
# arbitrarily take the first one.
sky_location = SkyLocation.objects.filter(nside=nside,
healpix=healpix)[0]
log.msg('SkyLocation:', sky_location)
# Find bounds for the Calibration object.
r0, r1, d0, d1 = wcs.radec_bounds()
# Find cartesian coordinates
ra *= math.pi / 180
dec *= math.pi / 180
tempr = math.cos(dec)
x = tempr * math.cos(ra)
y = tempr * math.sin(ra)
z = math.sin(dec)
r = radius / 180 * math.pi
calib = Calibration(raw_tan=tan,
ramin=r0,
ramax=r1,
decmin=d0,
decmax=d1,
x=x,
y=y,
z=z,
r=r,
sky_location=sky_location)
calib.save()
log.msg('Created Calibration', calib)
job.calibration = calib
job.save() # save calib before adding machine tags
job.status = 'S'
job.user_image.add_machine_tags(job)
job.user_image.add_sky_objects(job)
else:
job.status = 'F'
job.set_end_time()
job.save()
log.msg('Finished job', job.id)
logmsg('Finished job', job.id)
return job.id
def dojob(job, userimage, log=None):
jobdir = job.make_dir()
#print 'Created job dir', jobdir
#log = create_job_logger(job)
#jobdir = job.get_dir()
if log is None:
log = create_job_logger(job)
log.msg('Starting Job processing for', job)
job.set_start_time()
job.save()
#os.chdir(dirnm) - not thread safe (working directory is global)!
log.msg('Creating directory', jobdir)
axyfn = 'job.axy'
axypath = os.path.join(jobdir, axyfn)
sub = userimage.submission
log.msg('submission id', sub.id)
df = userimage.image.disk_file
img = userimage.image
# Build command-line arguments for the augment-xylist program, which
# detects sources in the image and adds processing arguments to the header
# to produce a "job.axy" file.
slo,shi = sub.get_scale_bounds()
# Note, this must match Job.get_wcs_file().
wcsfile = 'wcs.fits'
axyflags = []
axyargs = {
'--out': axypath,
'--scale-low': slo,
'--scale-high': shi,
'--scale-units': sub.scale_units,
'--wcs': wcsfile,
'--rdls': 'rdls.fits',
'--pixel-error': sub.positional_error,
'--ra': sub.center_ra,
'--dec': sub.center_dec,
'--radius': sub.radius,
'--downsample': sub.downsample_factor,
# tuning-up maybe fixed; if not, turn it off with:
#'--odds-to-tune': 1e9,
# Other things we might want include...
# --invert
# -g / --guess-scale: try to guess the image scale from the FITS headers
# --crpix-x <pix>: set the WCS reference point to the given position
# --crpix-y <pix>: set the WCS reference point to the given position
# -w / --width <pixels>: specify the field width
# -e / --height <pixels>: specify the field height
# -X / --x-column <column-name>: the FITS column name
# -Y / --y-column <column-name>
}
if hasattr(img,'sourcelist'):
# image is a source list; use --xylist
axyargs['--xylist'] = img.sourcelist.get_fits_path()
w,h = img.width, img.height
if sub.image_width:
w = sub.image_width
if sub.image_height:
h = sub.image_height
axyargs['--width' ] = w
axyargs['--height'] = h
else:
axyargs['--image'] = df.get_path()
# UGLY
if sub.parity == 0:
axyargs['--parity'] = 'pos'
elif sub.parity == 1:
axyargs['--parity'] = 'neg'
if sub.tweak_order == 0:
axyflags.append('--no-tweak')
else:
axyargs['--tweak-order'] = '%i' % sub.tweak_order
if sub.use_sextractor:
axyflags.append('--use-sextractor')
if sub.crpix_center:
axyflags.append('--crpix-center')
if sub.invert:
axyflags.append('--invert')
cmd = 'augment-xylist '
for (k,v) in axyargs.items():
if v:
cmd += k + ' ' + str(v) + ' '
for k in axyflags:
cmd += k + ' '
log.msg('running: ' + cmd)
(rtn, out, err) = run_command(cmd)
if rtn:
log.msg('out: ' + out)
log.msg('err: ' + err)
logmsg('augment-xylist failed: rtn val', rtn, 'err', err)
raise Exception
log.msg('created axy file', axypath)
# shell into compute server...
logfn = job.get_log_file()
# the "tar" commands both use "-C" to chdir, and the ssh command
# and redirect uses absolute paths.
cmd = ('(echo %(jobid)s; '
'tar cf - --ignore-failed-read -C %(jobdir)s %(axyfile)s) | '
'ssh -x -T %(sshconfig)s 2>>%(logfile)s | '
'tar xf - --atime-preserve -m --exclude=%(axyfile)s -C %(jobdir)s '
'>>%(logfile)s 2>&1' %
dict(jobid='job-%s-%i' % (settings.sitename, job.id),
axyfile=axyfn, jobdir=jobdir,
sshconfig=settings.ssh_solver_config,
logfile=logfn))
log.msg('command:', cmd)
w = os.system(cmd)
if not os.WIFEXITED(w):
log.msg('Solver failed (sent signal?)')
logmsg('Call to solver failed for job', job.id)
raise Exception
rtn = os.WEXITSTATUS(w)
if rtn:
log.msg('Solver failed with return value %i' % rtn)
logmsg('Call to solver failed for job', job.id, 'with return val', rtn)
raise Exception
log.msg('Solver completed successfully.')
# Solved?
wcsfn = os.path.join(jobdir, wcsfile)
log.msg('Checking for WCS file', wcsfn)
if os.path.exists(wcsfn):
log.msg('WCS file exists')
# Parse the wcs.fits file
wcs = Tan(wcsfn, 0)
# Convert to database model...
tan = TanWCS(crval1=wcs.crval[0], crval2=wcs.crval[1],
crpix1=wcs.crpix[0], crpix2=wcs.crpix[1],
cd11=wcs.cd[0], cd12=wcs.cd[1],
cd21=wcs.cd[2], cd22=wcs.cd[3],
imagew=img.width, imageh=img.height)
tan.save()
log.msg('Created TanWCS:', tan)
# Find field's healpix nside and index
ra, dec, radius = tan.get_center_radecradius()
nside = anutil.healpix_nside_for_side_length_arcmin(radius*60)
nside = int(2**round(math.log(nside, 2)))
healpix = anutil.radecdegtohealpix(ra, dec, nside)
sky_location, created = SkyLocation.objects.get_or_create(nside=nside, healpix=healpix)
log.msg('SkyLocation:', sky_location)
# Find bounds for the Calibration object.
r0,r1,d0,d1 = wcs.radec_bounds()
# Find cartesian coordinates
ra *= math.pi/180
dec *= math.pi/180
tempr = math.cos(dec)
x = tempr*math.cos(ra)
y = tempr*math.sin(ra)
z = math.sin(dec)
r = radius/180*math.pi
calib = Calibration(raw_tan=tan, ramin=r0, ramax=r1, decmin=d0, decmax=d1,
x=x,y=y,z=z,r=r,
sky_location=sky_location)
calib.save()
log.msg('Created Calibration', calib)
job.calibration = calib
job.save() # save calib before adding machine tags
job.status = 'S'
job.user_image.add_machine_tags(job)
job.user_image.add_sky_objects(job)
else:
job.status = 'F'
job.set_end_time()
job.save()
log.msg('Finished job', job.id)
logmsg('Finished job',job.id)
return job.id
wcs = Tan(wcsfn, 0)
model_solve_text = ('<TanWCS: CRVAL (%f, %f)' % (wcs.crval[0], wcs.crval[1]) +
' CRPIX (%f, %f)' % (wcs.crpix[0], wcs.crpix[1]) +
' CD (%f, %f; %f %f)' % tuple(wcs.cd) +
' Image size (%d, %d)>' % (image_width, image_height))
logger.info('Created Tan-WCS solution!: {}'.format(model_solve_text))
# Find field's healpix nside and index
ra, dec = wcs.radec_center()
radius = wcs.pixel_scale() * math.hypot(wcs.imagew, wcs.imageh) / 2.0 / 3600.0
nside = anutil.healpix_nside_for_side_length_arcmin(radius * 60)
nside = int(2**round(math.log(nside, 2)))
nside = max(1, nside)
healpix = anutil.radecdegtohealpix(ra, dec, nside)
# Find bounds for the Calibration object.
r0, r1, d0, d1 = wcs.radec_bounds()
# Find cartesian coordinates
ra *= math.pi / 180
dec *= math.pi / 180
tempr = math.cos(dec)
x = tempr * math.cos(ra)
y = tempr * math.sin(ra)
z = math.sin(dec)
r = radius / 180 * math.pi
output = """
Solution:
ra: {ra}
def main(fn, mp):
basefn = os.path.basename(fn)
tiles = fits_table(fn)
#I = np.flatnonzero(tiles.in_imaging)
#tiles.cut(I)
### Split the tiles into nearby chunks of work for multi-processing.
from astrometry.util.util import radecdegtohealpix
nside = 8
Nhp = 12 * nside**2
Ihps = [[] for i in range(Nhp)]
for i, (r, d) in enumerate(zip(tiles.ra, tiles.dec)):
hp = radecdegtohealpix(r, d, nside)
assert (hp >= 0)
Ihps[hp].append(i)
args = []
tiles.index = np.arange(len(tiles))
for hp, I in enumerate(Ihps):
if len(I) == 0:
continue
args.append((tiles[np.array(I)], 'HP %i' % hp))
print(len(args), 'big healpixes are populated')
R = mp.map(run_tiles, args)
tiles_ann = merge_tables(R)
print(len(tiles), 'tiles')
print(len(tiles_ann), 'annotated')
tiles_ann.about()
# unpermute
I = np.zeros(len(tiles_ann), np.int32)
I[tiles_ann.index] = np.arange(len(tiles_ann))
tiles_ann.cut(I)
tiles.add_columns_from(tiles_ann)
tiles.delete_column('index')
outfn = basefn.replace('.fits', '-brightest.fits')
tiles.writeto(outfn)
# Nudge (only inside imaging footprint)
from functools import reduce
brightest = reduce(np.minimum, [
tiles.brightest_guide_0_expanded[:, 0],
tiles.brightest_guide_2_expanded[:, 0],
tiles.brightest_guide_3_expanded[:, 0],
tiles.brightest_guide_5_expanded[:, 0],
tiles.brightest_guide_7_expanded[:, 0],
tiles.brightest_guide_8_expanded[:, 0],
tiles.brightest_focus_1_expanded[:, 0],
tiles.brightest_focus_4_expanded[:, 0],
tiles.brightest_focus_6_expanded[:, 0],
tiles.brightest_focus_9_expanded[:, 0],
])
if 'in_imaging' in tiles.get_columns():
I = np.flatnonzero((brightest < 6.) * tiles.in_imaging)
else:
I = np.flatnonzero((brightest < 6.))
print(len(I), 'tiles with G<6')
tiles.nudge_ra = np.zeros(len(tiles), np.float32)
tiles.nudge_dec = np.zeros(len(tiles), np.float32)
tiles.nudge_brightest = np.zeros(len(tiles), np.float32)
tiles.index = np.arange(len(tiles))
for nudge in range(1, 20):
if len(I) == 0:
break
print('Nudging', len(I), 'by', nudge)
ddec = 10. / 3600.
dra = ddec / np.cos(np.deg2rad(tiles.dec[I]))
# copy tiles
nudgetiles = tiles[np.repeat(I, 4)]
nudgetiles.nudge_ra[0::4] = +nudge * dra
nudgetiles.nudge_ra[1::4] = -nudge * dra
nudgetiles.nudge_dec[2::4] = +nudge * ddec
nudgetiles.nudge_dec[3::4] = -nudge * ddec
nudgetiles.ra += nudgetiles.nudge_ra
nudgetiles.dec += nudgetiles.nudge_dec
#ann = run_tiles((nudgetiles, 'nudge%i' % nudge))
# split into subsets
args = []
isplit = np.linspace(0, len(nudgetiles), 33, dtype=int)
for i0, i1 in zip(isplit, isplit[1:]):
if i0 == i1:
continue
args.append((nudgetiles[i0:i1], 'nudge%i+%i' % (nudge, i0)))
A = mp.map(run_tiles, args)
ann = merge_tables(A)
ann.brightest = reduce(np.minimum, [
ann.brightest_guide_0_expanded[:, 0],
ann.brightest_guide_2_expanded[:, 0],
ann.brightest_guide_3_expanded[:, 0],
ann.brightest_guide_5_expanded[:, 0],
ann.brightest_guide_7_expanded[:, 0],
ann.brightest_guide_8_expanded[:, 0],
ann.brightest_focus_1_expanded[:, 0],
ann.brightest_focus_4_expanded[:, 0],
ann.brightest_focus_6_expanded[:, 0],
ann.brightest_focus_9_expanded[:, 0],
])
ok = (ann.brightest > 6)
found = np.zeros(len(I), bool)
for idir, dirok in enumerate([ok[0::4], ok[1::4], ok[2::4], ok[3::4]]):
J = np.flatnonzero(np.logical_not(found) * dirok)
print('Nudge dir', idir, ':', len(J), 'are okay')
found[J] = True
tiles.nudge_ra[I[J]] = nudgetiles.nudge_ra[J * 4 + idir]
tiles.nudge_dec[I[J]] = nudgetiles.nudge_dec[J * 4 + idir]
tiles.nudge_brightest[I[J]] = ann.brightest[J * 4 + idir]
I = I[np.flatnonzero(found == False)]
outfn = basefn.replace('.fits', '-brightest-nudged.fits')
tiles.writeto(outfn)
