def dojob(job, userimage, log=None, solve_command=None, solve_locally=None, tempfiles=None): print('dojob: tempdir:', tempfile.gettempdir()) jobdir = job.make_dir() #print('Created job dir', jobdir) #log = create_job_logger(job) #jobdir = job.get_dir() if log is None: log = create_job_logger(job) log.msg('Starting Job processing for', job) job.set_start_time() job.save() #os.chdir(dirnm) - not thread safe (working directory is global)! log.msg('Creating directory', jobdir) axyfn = 'job.axy' axypath = os.path.join(jobdir, axyfn) sub = userimage.submission log.msg('submission id', sub.id) df = userimage.image.disk_file img = userimage.image # Build command-line arguments for the augment-xylist program, which # detects sources in the image and adds processing arguments to the header # to produce a "job.axy" file. slo, shi = sub.get_scale_bounds() # Note, this must match Job.get_wcs_file(). wcsfile = 'wcs.fits' corrfile = 'corr.fits' axyflags = [] axyargs = { '--out': axypath, '--scale-low': slo, '--scale-high': shi, '--scale-units': sub.scale_units, '--wcs': wcsfile, '--corr': corrfile, '--rdls': 'rdls.fits', '--pixel-error': sub.positional_error, '--ra': sub.center_ra, '--dec': sub.center_dec, '--radius': sub.radius, '--downsample': sub.downsample_factor, # tuning-up maybe fixed; if not, turn it off with: #'--odds-to-tune': 1e9, # Other things we might want include... # --invert # -g / --guess-scale: try to guess the image scale from the FITS headers # --crpix-x <pix>: set the WCS reference point to the given position # --crpix-y <pix>: set the WCS reference point to the given position # -w / --width <pixels>: specify the field width # -e / --height <pixels>: specify the field height # -X / --x-column <column-name>: the FITS column name # -Y / --y-column <column-name> } if hasattr(img, 'sourcelist'): # image is a source list; use --xylist axyargs['--xylist'] = img.sourcelist.get_fits_path(tempfiles=tempfiles) w, h = img.width, img.height if sub.image_width: w = sub.image_width if sub.image_height: h = sub.image_height axyargs['--width'] = w axyargs['--height'] = h else: axyargs['--image'] = df.get_path() # UGLY if sub.parity == 0: axyargs['--parity'] = 'pos' elif sub.parity == 1: axyargs['--parity'] = 'neg' if sub.tweak_order == 0: axyflags.append('--no-tweak') else: axyargs['--tweak-order'] = '%i' % sub.tweak_order if sub.use_sextractor: axyflags.append('--use-source-extractor') if sub.crpix_center: axyflags.append('--crpix-center') if sub.invert: axyflags.append('--invert') cmd = 'augment-xylist ' for (k, v) in list(axyargs.items()): if v: cmd += k + ' ' + str(v) + ' ' for k in axyflags: cmd += k + ' ' log.msg('running: ' + cmd) (rtn, out, err) = run_command(cmd) if rtn: log.msg('out: ' + out) log.msg('err: ' + err) logmsg('augment-xylist failed: rtn val', rtn, 'err', err) raise Exception log.msg('created axy file', axypath) # shell into compute server... logfn = job.get_log_file() # the "tar" commands both use "-C" to chdir, and the ssh command # and redirect uses absolute paths. if solve_locally is not None: cmd = (('cd %(jobdir)s && %(solvecmd)s %(jobid)s %(axyfile)s >> ' + '%(logfile)s') % dict(jobid='job-%s-%i' % (settings.sitename, job.id), solvecmd=solve_locally, axyfile=axyfn, jobdir=jobdir, logfile=logfn)) log.msg('command:', cmd) w = os.system(cmd) if not os.WIFEXITED(w): log.msg('Solver failed (sent signal?)') logmsg('Call to solver failed for job', job.id) raise Exception rtn = os.WEXITSTATUS(w) if rtn: log.msg('Solver failed with return value %i' % rtn) logmsg('Call to solver failed for job', job.id, 'with return val', rtn) raise Exception log.msg('Solver completed successfully.') else: if solve_command is None: solve_command = 'ssh -x -T %(sshconfig)s' cmd = (( '(echo %(jobid)s; ' 'tar cf - --ignore-failed-read -C %(jobdir)s %(axyfile)s) | ' + solve_command + ' 2>>%(logfile)s | ' 'tar xf - --atime-preserve -m --exclude=%(axyfile)s -C %(jobdir)s ' '>>%(logfile)s 2>&1') % dict(jobid='job-%s-%i' % (settings.sitename, job.id), axyfile=axyfn, jobdir=jobdir, sshconfig=settings.ssh_solver_config, logfile=logfn)) log.msg('command:', cmd) w = os.system(cmd) if not os.WIFEXITED(w): log.msg('Solver failed (sent signal?)') logmsg('Call to solver failed for job', job.id) raise Exception rtn = os.WEXITSTATUS(w) if rtn: log.msg('Solver failed with return value %i' % rtn) logmsg('Call to solver failed for job', job.id, 'with return val', rtn) raise Exception log.msg('Solver completed successfully.') # Solved? wcsfn = os.path.join(jobdir, wcsfile) log.msg('Checking for WCS file', wcsfn) if os.path.exists(wcsfn): log.msg('WCS file exists') # Parse the wcs.fits file wcs = Tan(wcsfn, 0) # Convert to database model... tan = TanWCS(crval1=wcs.crval[0], crval2=wcs.crval[1], crpix1=wcs.crpix[0], crpix2=wcs.crpix[1], cd11=wcs.cd[0], cd12=wcs.cd[1], cd21=wcs.cd[2], cd22=wcs.cd[3], imagew=img.width, imageh=img.height) tan.save() log.msg('Created TanWCS:', tan) # Find field's healpix nside and index ra, dec, radius = tan.get_center_radecradius() nside = anutil.healpix_nside_for_side_length_arcmin(radius * 60) nside = int(2**round(math.log(nside, 2))) nside = max(1, nside) healpix = anutil.radecdegtohealpix(ra, dec, nside) try: sky_location, created = SkyLocation.objects.get_or_create( nside=nside, healpix=healpix) except MultipleObjectsReturned: log.msg('Multiple SkyLocations for nside %i, healpix %i' % (nside, healpix)) # arbitrarily take the first one. sky_location = SkyLocation.objects.filter(nside=nside, healpix=healpix)[0] log.msg('SkyLocation:', sky_location) # Find bounds for the Calibration object. r0, r1, d0, d1 = wcs.radec_bounds() # Find cartesian coordinates ra *= math.pi / 180 dec *= math.pi / 180 tempr = math.cos(dec) x = tempr * math.cos(ra) y = tempr * math.sin(ra) z = math.sin(dec) r = radius / 180 * math.pi calib = Calibration(raw_tan=tan, ramin=r0, ramax=r1, decmin=d0, decmax=d1, x=x, y=y, z=z, r=r, sky_location=sky_location) calib.save() log.msg('Created Calibration', calib) job.calibration = calib job.save() # save calib before adding machine tags job.status = 'S' job.user_image.add_machine_tags(job) job.user_image.add_sky_objects(job) else: job.status = 'F' job.set_end_time() job.save() log.msg('Finished job', job.id) logmsg('Finished job', job.id) return job.id
def plot_sdss_image(wcsfn, plotfn, image_scale=1.0, debug_ps=None): from astrometry.util import util as anutil from astrometry.blind import plotstuff as ps # Parse the wcs.fits file wcs = anutil.Tan(wcsfn, 0) # grab SDSS tiles with about the same resolution as this image. pixscale = wcs.pixel_scale() pixscale = pixscale / image_scale logmsg('Original image scale is', wcs.pixel_scale(), 'arcsec/pix; scaled', image_scale, '->', pixscale) # size of SDSS image tiles to request, in pixels sdsssize = 512 scale = sdsssize * pixscale / 60. # healpix-vs-north-up rotation nside = anutil.healpix_nside_for_side_length_arcmin(scale / math.sqrt(2.)) nside = 2**int(math.ceil(math.log(nside) / math.log(2.))) logmsg('Next power-of-2 nside:', nside) ra, dec = wcs.radec_center() logmsg('Image center is RA,Dec', ra, dec) dirnm = os.path.join(settings.SDSS_TILE_DIR, 'nside%i' % nside) if not os.path.exists(dirnm): os.makedirs(dirnm) #hp = anutil.radecdegtohealpix(ra, dec, nside) #logmsg('Healpix of center:', hp) radius = wcs.radius() hps = anutil.healpix_rangesearch_radec(ra, dec, radius, nside) logmsg('Healpixes in range:', len(hps), ': ', hps) scale = math.sqrt(2.) * anutil.healpix_side_length_arcmin( nside) * 60. / float(sdsssize) logmsg('Grabbing SDSS tile with scale', scale, 'arcsec/pix') size = [int(image_scale * wcs.imagew), int(image_scale * wcs.imageh)] plot = ps.Plotstuff(outformat='png', wcsfn=wcsfn, size=size) plot.scale_wcs(image_scale) img = plot.image img.format = ps.PLOTSTUFF_FORMAT_JPG img.resample = 1 for hp in hps: fn = os.path.join(dirnm, '%i.jpg' % hp) logmsg('Checking for filename', fn) if not os.path.exists(fn): ra, dec = anutil.healpix_to_radecdeg(hp, nside, 0.5, 0.5) logmsg('Healpix center is RA,Dec', ra, dec) url = ( 'http://skyservice.pha.jhu.edu/DR8/ImgCutout/getjpeg.aspx?' + 'ra=%f&dec=%f&scale=%f&opt=&width=%i&height=%i' % (ra, dec, scale, sdsssize, sdsssize)) urlretrieve(url, fn) #nosec logmsg('Wrote', fn) swcsfn = os.path.join(dirnm, '%i.wcs' % hp) logmsg('Checking for WCS', swcsfn) if not os.path.exists(swcsfn): # Create WCS header cd = scale / 3600. swcs = anutil.Tan(ra, dec, sdsssize / 2 + 0.5, sdsssize / 2 + 0.5, -cd, 0, 0, -cd, sdsssize, sdsssize) swcs.write_to(swcsfn) logmsg('Wrote WCS to', swcsfn) img.set_wcs_file(swcsfn, 0) img.set_file(fn) plot.plot('image') if debug_ps is not None: fn = debug_ps.getnext() plot.write(fn) print('Wrote', fn) if debug_ps is not None: out = plot.outline plot.color = 'white' plot.alpha = 0.25 for hp in hps: swcsfn = os.path.join(dirnm, '%i.wcs' % hp) ps.plot_outline_set_wcs_file(out, swcsfn, 0) plot.plot('outline') plot.write(fn) print('Wrote', fn) plot.write(plotfn)
def dojob(job, userimage, log=None): jobdir = job.make_dir() #print 'Created job dir', jobdir #log = create_job_logger(job) #jobdir = job.get_dir() if log is None: log = create_job_logger(job) log.msg('Starting Job processing for', job) job.set_start_time() job.save() #os.chdir(dirnm) - not thread safe (working directory is global)! log.msg('Creating directory', jobdir) axyfn = 'job.axy' axypath = os.path.join(jobdir, axyfn) sub = userimage.submission log.msg('submission id', sub.id) df = userimage.image.disk_file img = userimage.image # Build command-line arguments for the augment-xylist program, which # detects sources in the image and adds processing arguments to the header # to produce a "job.axy" file. slo,shi = sub.get_scale_bounds() # Note, this must match Job.get_wcs_file(). wcsfile = 'wcs.fits' axyflags = [] axyargs = { '--out': axypath, '--scale-low': slo, '--scale-high': shi, '--scale-units': sub.scale_units, '--wcs': wcsfile, '--rdls': 'rdls.fits', '--pixel-error': sub.positional_error, '--ra': sub.center_ra, '--dec': sub.center_dec, '--radius': sub.radius, '--downsample': sub.downsample_factor, # tuning-up maybe fixed; if not, turn it off with: #'--odds-to-tune': 1e9, # Other things we might want include... # --invert # -g / --guess-scale: try to guess the image scale from the FITS headers # --crpix-x <pix>: set the WCS reference point to the given position # --crpix-y <pix>: set the WCS reference point to the given position # -w / --width <pixels>: specify the field width # -e / --height <pixels>: specify the field height # -X / --x-column <column-name>: the FITS column name # -Y / --y-column <column-name> } if hasattr(img,'sourcelist'): # image is a source list; use --xylist axyargs['--xylist'] = img.sourcelist.get_fits_path() w,h = img.width, img.height if sub.image_width: w = sub.image_width if sub.image_height: h = sub.image_height axyargs['--width' ] = w axyargs['--height'] = h else: axyargs['--image'] = df.get_path() # UGLY if sub.parity == 0: axyargs['--parity'] = 'pos' elif sub.parity == 1: axyargs['--parity'] = 'neg' if sub.tweak_order == 0: axyflags.append('--no-tweak') else: axyargs['--tweak-order'] = '%i' % sub.tweak_order if sub.use_sextractor: axyflags.append('--use-sextractor') if sub.crpix_center: axyflags.append('--crpix-center') if sub.invert: axyflags.append('--invert') cmd = 'augment-xylist ' for (k,v) in axyargs.items(): if v: cmd += k + ' ' + str(v) + ' ' for k in axyflags: cmd += k + ' ' log.msg('running: ' + cmd) (rtn, out, err) = run_command(cmd) if rtn: log.msg('out: ' + out) log.msg('err: ' + err) logmsg('augment-xylist failed: rtn val', rtn, 'err', err) raise Exception log.msg('created axy file', axypath) # shell into compute server... logfn = job.get_log_file() # the "tar" commands both use "-C" to chdir, and the ssh command # and redirect uses absolute paths. cmd = ('(echo %(jobid)s; ' 'tar cf - --ignore-failed-read -C %(jobdir)s %(axyfile)s) | ' 'ssh -x -T %(sshconfig)s 2>>%(logfile)s | ' 'tar xf - --atime-preserve -m --exclude=%(axyfile)s -C %(jobdir)s ' '>>%(logfile)s 2>&1' % dict(jobid='job-%s-%i' % (settings.sitename, job.id), axyfile=axyfn, jobdir=jobdir, sshconfig=settings.ssh_solver_config, logfile=logfn)) log.msg('command:', cmd) w = os.system(cmd) if not os.WIFEXITED(w): log.msg('Solver failed (sent signal?)') logmsg('Call to solver failed for job', job.id) raise Exception rtn = os.WEXITSTATUS(w) if rtn: log.msg('Solver failed with return value %i' % rtn) logmsg('Call to solver failed for job', job.id, 'with return val', rtn) raise Exception log.msg('Solver completed successfully.') # Solved? wcsfn = os.path.join(jobdir, wcsfile) log.msg('Checking for WCS file', wcsfn) if os.path.exists(wcsfn): log.msg('WCS file exists') # Parse the wcs.fits file wcs = Tan(wcsfn, 0) # Convert to database model... tan = TanWCS(crval1=wcs.crval[0], crval2=wcs.crval[1], crpix1=wcs.crpix[0], crpix2=wcs.crpix[1], cd11=wcs.cd[0], cd12=wcs.cd[1], cd21=wcs.cd[2], cd22=wcs.cd[3], imagew=img.width, imageh=img.height) tan.save() log.msg('Created TanWCS:', tan) # Find field's healpix nside and index ra, dec, radius = tan.get_center_radecradius() nside = anutil.healpix_nside_for_side_length_arcmin(radius*60) nside = int(2**round(math.log(nside, 2))) healpix = anutil.radecdegtohealpix(ra, dec, nside) sky_location, created = SkyLocation.objects.get_or_create(nside=nside, healpix=healpix) log.msg('SkyLocation:', sky_location) # Find bounds for the Calibration object. r0,r1,d0,d1 = wcs.radec_bounds() # Find cartesian coordinates ra *= math.pi/180 dec *= math.pi/180 tempr = math.cos(dec) x = tempr*math.cos(ra) y = tempr*math.sin(ra) z = math.sin(dec) r = radius/180*math.pi calib = Calibration(raw_tan=tan, ramin=r0, ramax=r1, decmin=d0, decmax=d1, x=x,y=y,z=z,r=r, sky_location=sky_location) calib.save() log.msg('Created Calibration', calib) job.calibration = calib job.save() # save calib before adding machine tags job.status = 'S' job.user_image.add_machine_tags(job) job.user_image.add_sky_objects(job) else: job.status = 'F' job.set_end_time() job.save() log.msg('Finished job', job.id) logmsg('Finished job',job.id) return job.id
def plot_sdss_image(wcsfn, plotfn, image_scale=1.0, debug_ps=None): from astrometry.util import util as anutil from astrometry.blind import plotstuff as ps # Parse the wcs.fits file wcs = anutil.Tan(wcsfn, 0) # grab SDSS tiles with about the same resolution as this image. pixscale = wcs.pixel_scale() pixscale = pixscale / image_scale logmsg('Original image scale is', wcs.pixel_scale(), 'arcsec/pix; scaled', image_scale, '->', pixscale) # size of SDSS image tiles to request, in pixels sdsssize = 512 scale = sdsssize * pixscale / 60. # healpix-vs-north-up rotation nside = anutil.healpix_nside_for_side_length_arcmin(scale / math.sqrt(2.)) nside = 2 ** int(math.ceil(math.log(nside)/math.log(2.))) logmsg('Next power-of-2 nside:', nside) ra,dec = wcs.radec_center() logmsg('Image center is RA,Dec', ra, dec) dirnm = os.path.join(settings.SDSS_TILE_DIR, 'nside%i'%nside) if not os.path.exists(dirnm): os.makedirs(dirnm) #hp = anutil.radecdegtohealpix(ra, dec, nside) #logmsg('Healpix of center:', hp) radius = wcs.radius() hps = anutil.healpix_rangesearch_radec(ra, dec, radius, nside) logmsg('Healpixes in range:', len(hps), ': ', hps) scale = math.sqrt(2.) * anutil.healpix_side_length_arcmin(nside) * 60. / float(sdsssize) logmsg('Grabbing SDSS tile with scale', scale, 'arcsec/pix') size = [int(image_scale*wcs.imagew),int(image_scale*wcs.imageh)] plot = ps.Plotstuff(outformat='png', wcsfn=wcsfn, size=size) plot.scale_wcs(image_scale) img = plot.image img.format = ps.PLOTSTUFF_FORMAT_JPG img.resample = 1 for hp in hps: fn = os.path.join(dirnm, '%i.jpg'%hp) logmsg('Checking for filename', fn) if not os.path.exists(fn): ra,dec = anutil.healpix_to_radecdeg(hp, nside, 0.5, 0.5) logmsg('Healpix center is RA,Dec', ra, dec) url = ('http://skyservice.pha.jhu.edu/DR8/ImgCutout/getjpeg.aspx?' + 'ra=%f&dec=%f&scale=%f&opt=&width=%i&height=%i' % (ra, dec, scale, sdsssize, sdsssize)) urllib.urlretrieve(url, fn) logmsg('Wrote', fn) swcsfn = os.path.join(dirnm, '%i.wcs'%hp) logmsg('Checking for WCS', swcsfn) if not os.path.exists(swcsfn): # Create WCS header cd = scale / 3600. swcs = anutil.Tan(ra, dec, sdsssize/2 + 0.5, sdsssize/2 + 0.5, -cd, 0, 0, -cd, sdsssize, sdsssize) swcs.write_to(swcsfn) logmsg('Wrote WCS to', swcsfn) img.set_wcs_file(swcsfn, 0) img.set_file(fn) plot.plot('image') if debug_ps is not None: fn = debug_ps.getnext() plot.write(fn) print 'Wrote', fn if debug_ps is not None: out = plot.outline plot.color = 'white' plot.alpha = 0.25 for hp in hps: swcsfn = os.path.join(dirnm, '%i.wcs'%hp) ps.plot_outline_set_wcs_file(out, swcsfn, 0) plot.plot('outline') plot.write(fn) print 'Wrote', fn plot.write(plotfn)
astrometry_engine_end - astrometry_engine_start)) # Parse the wcs.fits file wcs = Tan(wcsfn, 0) model_solve_text = ('<TanWCS: CRVAL (%f, %f)' % (wcs.crval[0], wcs.crval[1]) + ' CRPIX (%f, %f)' % (wcs.crpix[0], wcs.crpix[1]) + ' CD (%f, %f; %f %f)' % tuple(wcs.cd) + ' Image size (%d, %d)>' % (image_width, image_height)) logger.info('Created Tan-WCS solution!: {}'.format(model_solve_text)) # Find field's healpix nside and index ra, dec = wcs.radec_center() radius = wcs.pixel_scale() * math.hypot(wcs.imagew, wcs.imageh) / 2.0 / 3600.0 nside = anutil.healpix_nside_for_side_length_arcmin(radius * 60) nside = int(2**round(math.log(nside, 2))) nside = max(1, nside) healpix = anutil.radecdegtohealpix(ra, dec, nside) # Find bounds for the Calibration object. r0, r1, d0, d1 = wcs.radec_bounds() # Find cartesian coordinates ra *= math.pi / 180 dec *= math.pi / 180 tempr = math.cos(dec) x = tempr * math.cos(ra) y = tempr * math.sin(ra) z = math.sin(dec) r = radius / 180 * math.pi