def dojob(job,
userimage,
log=None,
solve_command=None,
solve_locally=None,
tempfiles=None):
print('dojob: tempdir:', tempfile.gettempdir())
jobdir = job.make_dir()
#print('Created job dir', jobdir)
#log = create_job_logger(job)
#jobdir = job.get_dir()
if log is None:
log = create_job_logger(job)
log.msg('Starting Job processing for', job)
job.set_start_time()
job.save()
#os.chdir(dirnm) - not thread safe (working directory is global)!
log.msg('Creating directory', jobdir)
axyfn = 'job.axy'
axypath = os.path.join(jobdir, axyfn)
sub = userimage.submission
log.msg('submission id', sub.id)
df = userimage.image.disk_file
img = userimage.image
# Build command-line arguments for the augment-xylist program, which
# detects sources in the image and adds processing arguments to the header
# to produce a "job.axy" file.
slo, shi = sub.get_scale_bounds()
# Note, this must match Job.get_wcs_file().
wcsfile = 'wcs.fits'
corrfile = 'corr.fits'
axyflags = []
axyargs = {
'--out': axypath,
'--scale-low': slo,
'--scale-high': shi,
'--scale-units': sub.scale_units,
'--wcs': wcsfile,
'--corr': corrfile,
'--rdls': 'rdls.fits',
'--pixel-error': sub.positional_error,
'--ra': sub.center_ra,
'--dec': sub.center_dec,
'--radius': sub.radius,
'--downsample': sub.downsample_factor,
# tuning-up maybe fixed; if not, turn it off with:
#'--odds-to-tune': 1e9,
# Other things we might want include...
# --invert
# -g / --guess-scale: try to guess the image scale from the FITS headers
# --crpix-x <pix>: set the WCS reference point to the given position
# --crpix-y <pix>: set the WCS reference point to the given position
# -w / --width <pixels>: specify the field width
# -e / --height <pixels>: specify the field height
# -X / --x-column <column-name>: the FITS column name
# -Y / --y-column <column-name>
}
if hasattr(img, 'sourcelist'):
# image is a source list; use --xylist
axyargs['--xylist'] = img.sourcelist.get_fits_path(tempfiles=tempfiles)
w, h = img.width, img.height
if sub.image_width:
w = sub.image_width
if sub.image_height:
h = sub.image_height
axyargs['--width'] = w
axyargs['--height'] = h
else:
axyargs['--image'] = df.get_path()
# UGLY
if sub.parity == 0:
axyargs['--parity'] = 'pos'
elif sub.parity == 1:
axyargs['--parity'] = 'neg'
if sub.tweak_order == 0:
axyflags.append('--no-tweak')
else:
axyargs['--tweak-order'] = '%i' % sub.tweak_order
if sub.use_sextractor:
axyflags.append('--use-source-extractor')
if sub.crpix_center:
axyflags.append('--crpix-center')
if sub.invert:
axyflags.append('--invert')
cmd = 'augment-xylist '
for (k, v) in list(axyargs.items()):
if v:
cmd += k + ' ' + str(v) + ' '
for k in axyflags:
cmd += k + ' '
log.msg('running: ' + cmd)
(rtn, out, err) = run_command(cmd)
if rtn:
log.msg('out: ' + out)
log.msg('err: ' + err)
logmsg('augment-xylist failed: rtn val', rtn, 'err', err)
raise Exception
log.msg('created axy file', axypath)
# shell into compute server...
logfn = job.get_log_file()
# the "tar" commands both use "-C" to chdir, and the ssh command
# and redirect uses absolute paths.
if solve_locally is not None:
cmd = (('cd %(jobdir)s && %(solvecmd)s %(jobid)s %(axyfile)s >> ' +
'%(logfile)s') % dict(jobid='job-%s-%i' %
(settings.sitename, job.id),
solvecmd=solve_locally,
axyfile=axyfn,
jobdir=jobdir,
logfile=logfn))
log.msg('command:', cmd)
w = os.system(cmd)
if not os.WIFEXITED(w):
log.msg('Solver failed (sent signal?)')
logmsg('Call to solver failed for job', job.id)
raise Exception
rtn = os.WEXITSTATUS(w)
if rtn:
log.msg('Solver failed with return value %i' % rtn)
logmsg('Call to solver failed for job', job.id, 'with return val',
rtn)
raise Exception
log.msg('Solver completed successfully.')
else:
if solve_command is None:
solve_command = 'ssh -x -T %(sshconfig)s'
cmd = ((
'(echo %(jobid)s; '
'tar cf - --ignore-failed-read -C %(jobdir)s %(axyfile)s) | ' +
solve_command + ' 2>>%(logfile)s | '
'tar xf - --atime-preserve -m --exclude=%(axyfile)s -C %(jobdir)s '
'>>%(logfile)s 2>&1') % dict(jobid='job-%s-%i' %
(settings.sitename, job.id),
axyfile=axyfn,
jobdir=jobdir,
sshconfig=settings.ssh_solver_config,
logfile=logfn))
log.msg('command:', cmd)
w = os.system(cmd)
if not os.WIFEXITED(w):
log.msg('Solver failed (sent signal?)')
logmsg('Call to solver failed for job', job.id)
raise Exception
rtn = os.WEXITSTATUS(w)
if rtn:
log.msg('Solver failed with return value %i' % rtn)
logmsg('Call to solver failed for job', job.id, 'with return val',
rtn)
raise Exception
log.msg('Solver completed successfully.')
# Solved?
wcsfn = os.path.join(jobdir, wcsfile)
log.msg('Checking for WCS file', wcsfn)
if os.path.exists(wcsfn):
log.msg('WCS file exists')
# Parse the wcs.fits file
wcs = Tan(wcsfn, 0)
# Convert to database model...
tan = TanWCS(crval1=wcs.crval[0],
crval2=wcs.crval[1],
crpix1=wcs.crpix[0],
crpix2=wcs.crpix[1],
cd11=wcs.cd[0],
cd12=wcs.cd[1],
cd21=wcs.cd[2],
cd22=wcs.cd[3],
imagew=img.width,
imageh=img.height)
tan.save()
log.msg('Created TanWCS:', tan)
# Find field's healpix nside and index
ra, dec, radius = tan.get_center_radecradius()
nside = anutil.healpix_nside_for_side_length_arcmin(radius * 60)
nside = int(2**round(math.log(nside, 2)))
nside = max(1, nside)
healpix = anutil.radecdegtohealpix(ra, dec, nside)
try:
sky_location, created = SkyLocation.objects.get_or_create(
nside=nside, healpix=healpix)
except MultipleObjectsReturned:
log.msg('Multiple SkyLocations for nside %i, healpix %i' %
(nside, healpix))
# arbitrarily take the first one.
sky_location = SkyLocation.objects.filter(nside=nside,
healpix=healpix)[0]
log.msg('SkyLocation:', sky_location)
# Find bounds for the Calibration object.
r0, r1, d0, d1 = wcs.radec_bounds()
# Find cartesian coordinates
ra *= math.pi / 180
dec *= math.pi / 180
tempr = math.cos(dec)
x = tempr * math.cos(ra)
y = tempr * math.sin(ra)
z = math.sin(dec)
r = radius / 180 * math.pi
calib = Calibration(raw_tan=tan,
ramin=r0,
ramax=r1,
decmin=d0,
decmax=d1,
x=x,
y=y,
z=z,
r=r,
sky_location=sky_location)
calib.save()
log.msg('Created Calibration', calib)
job.calibration = calib
job.save() # save calib before adding machine tags
job.status = 'S'
job.user_image.add_machine_tags(job)
job.user_image.add_sky_objects(job)
else:
job.status = 'F'
job.set_end_time()
job.save()
log.msg('Finished job', job.id)
logmsg('Finished job', job.id)
return job.id
def plot_sdss_image(wcsfn, plotfn, image_scale=1.0, debug_ps=None):
from astrometry.util import util as anutil
from astrometry.blind import plotstuff as ps
# Parse the wcs.fits file
wcs = anutil.Tan(wcsfn, 0)
# grab SDSS tiles with about the same resolution as this image.
pixscale = wcs.pixel_scale()
pixscale = pixscale / image_scale
logmsg('Original image scale is', wcs.pixel_scale(), 'arcsec/pix; scaled',
image_scale, '->', pixscale)
# size of SDSS image tiles to request, in pixels
sdsssize = 512
scale = sdsssize * pixscale / 60.
# healpix-vs-north-up rotation
nside = anutil.healpix_nside_for_side_length_arcmin(scale / math.sqrt(2.))
nside = 2**int(math.ceil(math.log(nside) / math.log(2.)))
logmsg('Next power-of-2 nside:', nside)
ra, dec = wcs.radec_center()
logmsg('Image center is RA,Dec', ra, dec)
dirnm = os.path.join(settings.SDSS_TILE_DIR, 'nside%i' % nside)
if not os.path.exists(dirnm):
os.makedirs(dirnm)
#hp = anutil.radecdegtohealpix(ra, dec, nside)
#logmsg('Healpix of center:', hp)
radius = wcs.radius()
hps = anutil.healpix_rangesearch_radec(ra, dec, radius, nside)
logmsg('Healpixes in range:', len(hps), ': ', hps)
scale = math.sqrt(2.) * anutil.healpix_side_length_arcmin(
nside) * 60. / float(sdsssize)
logmsg('Grabbing SDSS tile with scale', scale, 'arcsec/pix')
size = [int(image_scale * wcs.imagew), int(image_scale * wcs.imageh)]
plot = ps.Plotstuff(outformat='png', wcsfn=wcsfn, size=size)
plot.scale_wcs(image_scale)
img = plot.image
img.format = ps.PLOTSTUFF_FORMAT_JPG
img.resample = 1
for hp in hps:
fn = os.path.join(dirnm, '%i.jpg' % hp)
logmsg('Checking for filename', fn)
if not os.path.exists(fn):
ra, dec = anutil.healpix_to_radecdeg(hp, nside, 0.5, 0.5)
logmsg('Healpix center is RA,Dec', ra, dec)
url = (
'http://skyservice.pha.jhu.edu/DR8/ImgCutout/getjpeg.aspx?' +
'ra=%f&dec=%f&scale=%f&opt=&width=%i&height=%i' %
(ra, dec, scale, sdsssize, sdsssize))
urlretrieve(url, fn) #nosec
logmsg('Wrote', fn)
swcsfn = os.path.join(dirnm, '%i.wcs' % hp)
logmsg('Checking for WCS', swcsfn)
if not os.path.exists(swcsfn):
# Create WCS header
cd = scale / 3600.
swcs = anutil.Tan(ra, dec, sdsssize / 2 + 0.5, sdsssize / 2 + 0.5,
-cd, 0, 0, -cd, sdsssize, sdsssize)
swcs.write_to(swcsfn)
logmsg('Wrote WCS to', swcsfn)
img.set_wcs_file(swcsfn, 0)
img.set_file(fn)
plot.plot('image')
if debug_ps is not None:
fn = debug_ps.getnext()
plot.write(fn)
print('Wrote', fn)
if debug_ps is not None:
out = plot.outline
plot.color = 'white'
plot.alpha = 0.25
for hp in hps:
swcsfn = os.path.join(dirnm, '%i.wcs' % hp)
ps.plot_outline_set_wcs_file(out, swcsfn, 0)
plot.plot('outline')
plot.write(fn)
print('Wrote', fn)
plot.write(plotfn)
def dojob(job, userimage, log=None):
jobdir = job.make_dir()
#print 'Created job dir', jobdir
#log = create_job_logger(job)
#jobdir = job.get_dir()
if log is None:
log = create_job_logger(job)
log.msg('Starting Job processing for', job)
job.set_start_time()
job.save()
#os.chdir(dirnm) - not thread safe (working directory is global)!
log.msg('Creating directory', jobdir)
axyfn = 'job.axy'
axypath = os.path.join(jobdir, axyfn)
sub = userimage.submission
log.msg('submission id', sub.id)
df = userimage.image.disk_file
img = userimage.image
# Build command-line arguments for the augment-xylist program, which
# detects sources in the image and adds processing arguments to the header
# to produce a "job.axy" file.
slo,shi = sub.get_scale_bounds()
# Note, this must match Job.get_wcs_file().
wcsfile = 'wcs.fits'
axyflags = []
axyargs = {
'--out': axypath,
'--scale-low': slo,
'--scale-high': shi,
'--scale-units': sub.scale_units,
'--wcs': wcsfile,
'--rdls': 'rdls.fits',
'--pixel-error': sub.positional_error,
'--ra': sub.center_ra,
'--dec': sub.center_dec,
'--radius': sub.radius,
'--downsample': sub.downsample_factor,
# tuning-up maybe fixed; if not, turn it off with:
#'--odds-to-tune': 1e9,
# Other things we might want include...
# --invert
# -g / --guess-scale: try to guess the image scale from the FITS headers
# --crpix-x <pix>: set the WCS reference point to the given position
# --crpix-y <pix>: set the WCS reference point to the given position
# -w / --width <pixels>: specify the field width
# -e / --height <pixels>: specify the field height
# -X / --x-column <column-name>: the FITS column name
# -Y / --y-column <column-name>
}
if hasattr(img,'sourcelist'):
# image is a source list; use --xylist
axyargs['--xylist'] = img.sourcelist.get_fits_path()
w,h = img.width, img.height
if sub.image_width:
w = sub.image_width
if sub.image_height:
h = sub.image_height
axyargs['--width' ] = w
axyargs['--height'] = h
else:
axyargs['--image'] = df.get_path()
# UGLY
if sub.parity == 0:
axyargs['--parity'] = 'pos'
elif sub.parity == 1:
axyargs['--parity'] = 'neg'
if sub.tweak_order == 0:
axyflags.append('--no-tweak')
else:
axyargs['--tweak-order'] = '%i' % sub.tweak_order
if sub.use_sextractor:
axyflags.append('--use-sextractor')
if sub.crpix_center:
axyflags.append('--crpix-center')
if sub.invert:
axyflags.append('--invert')
cmd = 'augment-xylist '
for (k,v) in axyargs.items():
if v:
cmd += k + ' ' + str(v) + ' '
for k in axyflags:
cmd += k + ' '
log.msg('running: ' + cmd)
(rtn, out, err) = run_command(cmd)
if rtn:
log.msg('out: ' + out)
log.msg('err: ' + err)
logmsg('augment-xylist failed: rtn val', rtn, 'err', err)
raise Exception
log.msg('created axy file', axypath)
# shell into compute server...
logfn = job.get_log_file()
# the "tar" commands both use "-C" to chdir, and the ssh command
# and redirect uses absolute paths.
cmd = ('(echo %(jobid)s; '
'tar cf - --ignore-failed-read -C %(jobdir)s %(axyfile)s) | '
'ssh -x -T %(sshconfig)s 2>>%(logfile)s | '
'tar xf - --atime-preserve -m --exclude=%(axyfile)s -C %(jobdir)s '
'>>%(logfile)s 2>&1' %
dict(jobid='job-%s-%i' % (settings.sitename, job.id),
axyfile=axyfn, jobdir=jobdir,
sshconfig=settings.ssh_solver_config,
logfile=logfn))
log.msg('command:', cmd)
w = os.system(cmd)
if not os.WIFEXITED(w):
log.msg('Solver failed (sent signal?)')
logmsg('Call to solver failed for job', job.id)
raise Exception
rtn = os.WEXITSTATUS(w)
if rtn:
log.msg('Solver failed with return value %i' % rtn)
logmsg('Call to solver failed for job', job.id, 'with return val', rtn)
raise Exception
log.msg('Solver completed successfully.')
# Solved?
wcsfn = os.path.join(jobdir, wcsfile)
log.msg('Checking for WCS file', wcsfn)
if os.path.exists(wcsfn):
log.msg('WCS file exists')
# Parse the wcs.fits file
wcs = Tan(wcsfn, 0)
# Convert to database model...
tan = TanWCS(crval1=wcs.crval[0], crval2=wcs.crval[1],
crpix1=wcs.crpix[0], crpix2=wcs.crpix[1],
cd11=wcs.cd[0], cd12=wcs.cd[1],
cd21=wcs.cd[2], cd22=wcs.cd[3],
imagew=img.width, imageh=img.height)
tan.save()
log.msg('Created TanWCS:', tan)
# Find field's healpix nside and index
ra, dec, radius = tan.get_center_radecradius()
nside = anutil.healpix_nside_for_side_length_arcmin(radius*60)
nside = int(2**round(math.log(nside, 2)))
healpix = anutil.radecdegtohealpix(ra, dec, nside)
sky_location, created = SkyLocation.objects.get_or_create(nside=nside, healpix=healpix)
log.msg('SkyLocation:', sky_location)
# Find bounds for the Calibration object.
r0,r1,d0,d1 = wcs.radec_bounds()
# Find cartesian coordinates
ra *= math.pi/180
dec *= math.pi/180
tempr = math.cos(dec)
x = tempr*math.cos(ra)
y = tempr*math.sin(ra)
z = math.sin(dec)
r = radius/180*math.pi
calib = Calibration(raw_tan=tan, ramin=r0, ramax=r1, decmin=d0, decmax=d1,
x=x,y=y,z=z,r=r,
sky_location=sky_location)
calib.save()
log.msg('Created Calibration', calib)
job.calibration = calib
job.save() # save calib before adding machine tags
job.status = 'S'
job.user_image.add_machine_tags(job)
job.user_image.add_sky_objects(job)
else:
job.status = 'F'
job.set_end_time()
job.save()
log.msg('Finished job', job.id)
logmsg('Finished job',job.id)
return job.id
def plot_sdss_image(wcsfn, plotfn, image_scale=1.0, debug_ps=None):
from astrometry.util import util as anutil
from astrometry.blind import plotstuff as ps
# Parse the wcs.fits file
wcs = anutil.Tan(wcsfn, 0)
# grab SDSS tiles with about the same resolution as this image.
pixscale = wcs.pixel_scale()
pixscale = pixscale / image_scale
logmsg('Original image scale is', wcs.pixel_scale(), 'arcsec/pix; scaled', image_scale, '->', pixscale)
# size of SDSS image tiles to request, in pixels
sdsssize = 512
scale = sdsssize * pixscale / 60.
# healpix-vs-north-up rotation
nside = anutil.healpix_nside_for_side_length_arcmin(scale / math.sqrt(2.))
nside = 2 ** int(math.ceil(math.log(nside)/math.log(2.)))
logmsg('Next power-of-2 nside:', nside)
ra,dec = wcs.radec_center()
logmsg('Image center is RA,Dec', ra, dec)
dirnm = os.path.join(settings.SDSS_TILE_DIR, 'nside%i'%nside)
if not os.path.exists(dirnm):
os.makedirs(dirnm)
#hp = anutil.radecdegtohealpix(ra, dec, nside)
#logmsg('Healpix of center:', hp)
radius = wcs.radius()
hps = anutil.healpix_rangesearch_radec(ra, dec, radius, nside)
logmsg('Healpixes in range:', len(hps), ': ', hps)
scale = math.sqrt(2.) * anutil.healpix_side_length_arcmin(nside) * 60. / float(sdsssize)
logmsg('Grabbing SDSS tile with scale', scale, 'arcsec/pix')
size = [int(image_scale*wcs.imagew),int(image_scale*wcs.imageh)]
plot = ps.Plotstuff(outformat='png', wcsfn=wcsfn, size=size)
plot.scale_wcs(image_scale)
img = plot.image
img.format = ps.PLOTSTUFF_FORMAT_JPG
img.resample = 1
for hp in hps:
fn = os.path.join(dirnm, '%i.jpg'%hp)
logmsg('Checking for filename', fn)
if not os.path.exists(fn):
ra,dec = anutil.healpix_to_radecdeg(hp, nside, 0.5, 0.5)
logmsg('Healpix center is RA,Dec', ra, dec)
url = ('http://skyservice.pha.jhu.edu/DR8/ImgCutout/getjpeg.aspx?' +
'ra=%f&dec=%f&scale=%f&opt=&width=%i&height=%i' %
(ra, dec, scale, sdsssize, sdsssize))
urllib.urlretrieve(url, fn)
logmsg('Wrote', fn)
swcsfn = os.path.join(dirnm, '%i.wcs'%hp)
logmsg('Checking for WCS', swcsfn)
if not os.path.exists(swcsfn):
# Create WCS header
cd = scale / 3600.
swcs = anutil.Tan(ra, dec, sdsssize/2 + 0.5, sdsssize/2 + 0.5,
-cd, 0, 0, -cd, sdsssize, sdsssize)
swcs.write_to(swcsfn)
logmsg('Wrote WCS to', swcsfn)
img.set_wcs_file(swcsfn, 0)
img.set_file(fn)
plot.plot('image')
if debug_ps is not None:
fn = debug_ps.getnext()
plot.write(fn)
print 'Wrote', fn
if debug_ps is not None:
out = plot.outline
plot.color = 'white'
plot.alpha = 0.25
for hp in hps:
swcsfn = os.path.join(dirnm, '%i.wcs'%hp)
ps.plot_outline_set_wcs_file(out, swcsfn, 0)
plot.plot('outline')
plot.write(fn)
print 'Wrote', fn
plot.write(plotfn)
astrometry_engine_end - astrometry_engine_start))
# Parse the wcs.fits file
wcs = Tan(wcsfn, 0)
model_solve_text = ('<TanWCS: CRVAL (%f, %f)' % (wcs.crval[0], wcs.crval[1]) +
' CRPIX (%f, %f)' % (wcs.crpix[0], wcs.crpix[1]) +
' CD (%f, %f; %f %f)' % tuple(wcs.cd) +
' Image size (%d, %d)>' % (image_width, image_height))
logger.info('Created Tan-WCS solution!: {}'.format(model_solve_text))
# Find field's healpix nside and index
ra, dec = wcs.radec_center()
radius = wcs.pixel_scale() * math.hypot(wcs.imagew, wcs.imageh) / 2.0 / 3600.0
nside = anutil.healpix_nside_for_side_length_arcmin(radius * 60)
nside = int(2**round(math.log(nside, 2)))
nside = max(1, nside)
healpix = anutil.radecdegtohealpix(ra, dec, nside)
# Find bounds for the Calibration object.
r0, r1, d0, d1 = wcs.radec_bounds()
# Find cartesian coordinates
ra *= math.pi / 180
dec *= math.pi / 180
tempr = math.cos(dec)
x = tempr * math.cos(ra)
y = tempr * math.sin(ra)
z = math.sin(dec)
r = radius / 180 * math.pi
