def run_pnmfile(fn): """ run the pnmfile command to get image dimenensions""" # bandit warns about security implications. We provide our own filename # and the sysadmin has to make sure the correct pnmfile is found in the PATH # TODO: document this for sysadmins out = check_output(['pnmfile', fn]).decode().strip() #nosec logdebug('pnmfile output: ' + out) pat = re.compile( r'P(?P<pnmtype>[BGP])M .*, (?P<width>\d*) by (?P<height>\d*)( *maxval (?P<maxval>\d*))?' ) match = pat.search(out) if not match: logmsg('pnmfile: No regex match.') return None w = int(match.group('width')) h = int(match.group('height')) pnmtype = match.group('pnmtype') mv = match.group('maxval') if mv is None: maxval = 1 else: maxval = int(mv) logdebug('Type %s, w %i, h %i, maxval %i' % (pnmtype, w, h, maxval)) return (w, h, pnmtype, maxval)
def is_tarball(fn): logmsg('is_tarball: %s' % fn) types = filetype_short(fn) logmsg('filetypes:', types) for t in types: if t.startswith('POSIX tar archive'): return True return False
def create_job_logger(job): ''' Create a MyLogger object that writes to a log file within a Job directory. ''' logmsg("getlogger") logger = logging.getLogger('job.%i' % job.id) logger.setLevel(logging.DEBUG) fh = logging.FileHandler(job.get_log_file2()) fh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(message)s') fh.setFormatter(formatter) logger.addHandler(fh) return MyLogger(logger)
def try_dojob(job, userimage, solve_command, solve_locally): print('try_dojob', job, '(sub', job.user_image.submission.id, ')') tempfiles = [] rtn = None try: rtn = dojob(job, userimage, solve_command=solve_command, solve_locally=solve_locally, tempfiles=tempfiles) print('try_dojob', job, 'completed:', rtn) except OSError as e: print('OSError processing job', job) print(e) import errno # Too many open files print('e.errno:', e.errno) print('errno.EMFILE:', errno.EMFILE) if e.errno == errno.EMFILE: print('Too many open files -- exiting!') #sys.exit(-1) ## Want to stop the main process, not just this pool worker process raise e except IOError as e: import errno print('Caught IOError') print('Errno:', e.errno) if e.errno == errno.EMFILE: print('Too many open files!') ##sys.exit(-1) raise e except: print('Caught exception while processing Job', job) traceback.print_exc(None, sys.stdout) # FIXME -- job.set_status()... job.set_end_time() job.status = 'F' job.save() log = create_job_logger(job) log.msg('Caught exception while processing Job', job.id) log.msg(traceback.format_exc(None)) for fn in tempfiles: if os.path.exists(fn): try: os.remove(fn) except OSError as e: logmsg('Failed to delete temp file', fn, ':', e) return rtn
def try_dosub(sub, max_retries): subid = sub.id tempfiles = [] tempdirs = [] rtn = None try: rtn = dosub(sub, tempfiles=tempfiles, tempdirs=tempdirs) except DatabaseError as e: print('Caught DatabaseError while processing Submission', sub) traceback.print_exc(None, sys.stdout) # Try... django.db.connection.close() sub = Submission.objects.get(id=subid) if (sub.processing_retries < max_retries): print('Retrying processing Submission %s' % str(sub)) sub.processing_retries += 1 sub.save() rtn = try_dosub(sub, max_retries, tempfiles=tempfiles, tempdirs=tempdirs) else: print('Submission retry limit reached') sub.set_error_message( 'Caught exception while processing Submission: ' + str(sub) + '\n' + traceback.format_exc(None)) sub.set_processing_finished() sub.save() rtn = 'exception' except: print('Caught exception while processing Submission', sub) traceback.print_exc(None, sys.stdout) sub.set_error_message( 'Caught exception while processing Submission: ' + str(sub) + '\n' + traceback.format_exc(None)) sub.set_processing_finished() sub.save() logmsg('Caught exception while processing Submission ' + str(sub)) logmsg(' ' + traceback.format_exc(None)) rtn = 'exception' for dirnm in tempdirs: if os.path.exists(dirnm): try: shutil.rmtree(dirnm) except OSError as e: logmsg('Failed to delete temp dir', dirnm, ':', e) for fn in tempfiles: if os.path.exists(fn): try: os.remove(fn) except OSError as e: logmsg('Failed to delete temp file', fn, ':', e) return rtn
def create_image(df, tempfiles=None): img = None try: img = Image(disk_file=df) # FIXME -- move this code to Image? # Convert file to pnm to find its size. pnmfn = img.get_pnm_path(tempfiles=tempfiles) x = run_pnmfile(pnmfn) if x is None: raise RuntimeError('Could not find image file size') (w, h, pnmtype, maxval) = x logdebug('Type %s, w %i, h %i' % (pnmtype, w, h)) img.width = w img.height = h img.save() except: logmsg('file is not an image file: ' + traceback.format_exc()) img = None return img
def index(req): ps = ProcessSubmissions.objects.all().order_by('-watchdog') logmsg('ProcessSubmissions:', ps) return render(req, 'admin.html', {'procsubs': ps,})
def procsub(req, psid=None): ps = get_object_or_404(ProcessSubmissions, pk=psid) logmsg('ProcessSubmission:', ps) logmsg('jobs:', ps.jobs.all()) for j in ps.jobs.all(): logmsg(' ', j) logmsg(' ', j.job) logmsg(' ', j.job.user_image) logmsg(' ', j.job.user_image.submission) now = datetime.now() now = now.replace(microsecond=0) now = now.isoformat() return render(req, 'procsub.html', 'procsub': ps, 'now': now,})
def get_tarball_files(fn, tempdirs=None): # create temp dir to extract tarfile. tempdir = tempfile.mkdtemp() if tempdirs is not None: tempdirs.append(tempdir) cmd = 'tar xvf %s -C %s' % (fn, tempdir) print('Extracting tarball: %s' % cmd) (rtn, out, err) = run_command(cmd) if rtn: print('Failed to un-tar file:\n' + err) #bailout(submission, 'failed to extract tar file') print('failed to extract tar file') return None fns = out.strip('\n').split('\n') validpaths = [] for fn in fns: path = os.path.join(tempdir, fn) logmsg('Path "%s"' % path) if not os.path.exists(path): logmsg('Path "%s" does not exist.' % path) continue if os.path.islink(path): logmsg('Path "%s" is a symlink.' % path) continue if os.path.isfile(path): validpaths.append(path) else: logmsg('Path "%s" is not a file.' % path) if len(validpaths) == 0: #userlog('Tar file contains no regular files.') #bailout(submission, "tar file contains no regular files.") #return -1 logmsg('No real files in tar file') return None logmsg('Got %i paths.' % len(validpaths)) return validpaths
def create_source_list(df, tempfiles=None): img = None fits = None source_type = None path = df.get_path() print('path:', path, type(path)) try: # see if disk file is a fits list fits = fits_table(str(df.get_path())) source_type = 'fits' except: logmsg('file is not a fits table') # otherwise, check to see if it is a text list try: fitsfn = get_temp_file(tempfiles=tempfiles) text_file = open(str(df.get_path())) text = text_file.read() text_file.close() # add x y header # potential hack, assumes it doesn't exist... text = "# x y\n" + text text_table = text_table_fields("", text=text) text_table.write_to(fitsfn) logmsg("Creating fits table from text list") fits = fits_table(fitsfn) source_type = 'text' except Exception as e: #logmsg('Traceback:\n' + traceback.format_exc()) logmsg('fitsfn: %s' % fitsfn) logmsg(e) logmsg('file is not a text list') if fits: try: img = SourceList(disk_file=df, source_type=source_type) # w = fits.x.max()-fits.x.min() # h = fits.y.max()-fits.y.min() # w = int(w) # h = int(h) w = int(math.ceil(fits.x.max())) h = int(math.ceil(fits.y.max())) logmsg('w %i, h %i' % (w, h)) if w < 1 or h < 1: raise RuntimeError( 'Source list must contain POSITIVE x,y coordinates') img.width = w img.height = h img.save() except Exception as e: logmsg(e) img = None raise e return img
def dosub(sub, tempfiles=None, tempdirs=None): print('dosub: tempdir:', tempfile.gettempdir()) sub.set_processing_started() sub.save() print('Submission disk file:', sub.disk_file) if sub.disk_file is None: logmsg('Sub %i: retrieving URL' % (sub.id), sub.url) (fn, headers) = urllib.request.urlretrieve(sub.url) logmsg('Sub %i: wrote URL to file' % (sub.id), fn) df = DiskFile.from_file(fn, Image.ORIG_COLLECTION) logmsg('Created DiskFile', df) # Try to split the URL into a filename component and save it p = urlparse(sub.url) p = p.path if p: s = p.split('/') origname = s[-1] sub.original_filename = origname df.save() sub.disk_file = df sub.save() logmsg('Saved DiskFile', df) else: logmsg('uploaded disk file for this submission is ' + str(sub.disk_file)) df = sub.disk_file fn = df.get_path() logmsg('DiskFile path ' + fn) original_filename = sub.original_filename # check if file is a gzipped file try: with gzip.open(fn) as gzip_file: tempfn = get_temp_file(tempfiles=tempfiles) with open(tempfn, 'wb') as f: # should fail on the following line if not a gzip file f.write(gzip_file.read()) df = DiskFile.from_file(tempfn, 'uploaded-gunzip') i = original_filename.find('.gz') if i != -1: original_filename = original_filename[:i] logmsg('extracted gzip file %s' % original_filename) #fn = tempfn fn = df.get_path() except: # not a gzip file pass is_tar = False try: is_tar = tarfile.is_tarfile(fn) except: pass if is_tar: logmsg('File %s: tarball' % fn) tar = tarfile.open(fn) dirnm = tempfile.mkdtemp() if tempdirs is not None: tempdirs.append(dirnm) for tarinfo in tar.getmembers(): if tarinfo.isfile(): logmsg('extracting file %s' % tarinfo.name) tar.extract(tarinfo, dirnm) tempfn = os.path.join(dirnm, tarinfo.name) df = DiskFile.from_file(tempfn, 'uploaded-untar') # create Image object img = get_or_create_image(df, tempfiles=tempfiles) # create UserImage object. if img: create_user_image(sub, img, tarinfo.name) tar.close() shutil.rmtree(dirnm, ignore_errors=True) else: # assume file is single image logmsg('File %s: single file' % fn) # create Image object img = get_or_create_image(df, tempfiles=tempfiles) logdebug('File %s: created Image %s' % (fn, str(img))) # create UserImage object. if img: uimg = create_user_image(sub, img, original_filename) logmsg('File %s: Image id %i, UserImage id %i' % (fn, img.id, uimg.id)) sub.set_processing_finished() sub.save() return sub.id
def dojob(job, userimage, log=None, solve_command=None, solve_locally=None, tempfiles=None): print('dojob: tempdir:', tempfile.gettempdir()) jobdir = job.make_dir() #print('Created job dir', jobdir) #log = create_job_logger(job) #jobdir = job.get_dir() if log is None: log = create_job_logger(job) log.msg('Starting Job processing for', job) job.set_start_time() job.save() #os.chdir(dirnm) - not thread safe (working directory is global)! log.msg('Creating directory', jobdir) axyfn = 'job.axy' axypath = os.path.join(jobdir, axyfn) sub = userimage.submission log.msg('submission id', sub.id) df = userimage.image.disk_file img = userimage.image # Build command-line arguments for the augment-xylist program, which # detects sources in the image and adds processing arguments to the header # to produce a "job.axy" file. slo, shi = sub.get_scale_bounds() # Note, this must match Job.get_wcs_file(). wcsfile = 'wcs.fits' corrfile = 'corr.fits' axyflags = [] axyargs = { '--out': axypath, '--scale-low': slo, '--scale-high': shi, '--scale-units': sub.scale_units, '--wcs': wcsfile, '--corr': corrfile, '--rdls': 'rdls.fits', '--pixel-error': sub.positional_error, '--ra': sub.center_ra, '--dec': sub.center_dec, '--radius': sub.radius, '--downsample': sub.downsample_factor, # tuning-up maybe fixed; if not, turn it off with: #'--odds-to-tune': 1e9, # Other things we might want include... # --invert # -g / --guess-scale: try to guess the image scale from the FITS headers # --crpix-x <pix>: set the WCS reference point to the given position # --crpix-y <pix>: set the WCS reference point to the given position # -w / --width <pixels>: specify the field width # -e / --height <pixels>: specify the field height # -X / --x-column <column-name>: the FITS column name # -Y / --y-column <column-name> } if hasattr(img, 'sourcelist'): # image is a source list; use --xylist axyargs['--xylist'] = img.sourcelist.get_fits_path(tempfiles=tempfiles) w, h = img.width, img.height if sub.image_width: w = sub.image_width if sub.image_height: h = sub.image_height axyargs['--width'] = w axyargs['--height'] = h else: axyargs['--image'] = df.get_path() # UGLY if sub.parity == 0: axyargs['--parity'] = 'pos' elif sub.parity == 1: axyargs['--parity'] = 'neg' if sub.tweak_order == 0: axyflags.append('--no-tweak') else: axyargs['--tweak-order'] = '%i' % sub.tweak_order if sub.use_sextractor: axyflags.append('--use-source-extractor') if sub.crpix_center: axyflags.append('--crpix-center') if sub.invert: axyflags.append('--invert') cmd = 'augment-xylist ' for (k, v) in list(axyargs.items()): if v: cmd += k + ' ' + str(v) + ' ' for k in axyflags: cmd += k + ' ' log.msg('running: ' + cmd) (rtn, out, err) = run_command(cmd) if rtn: log.msg('out: ' + out) log.msg('err: ' + err) logmsg('augment-xylist failed: rtn val', rtn, 'err', err) raise Exception log.msg('created axy file', axypath) # shell into compute server... logfn = job.get_log_file() # the "tar" commands both use "-C" to chdir, and the ssh command # and redirect uses absolute paths. if solve_locally is not None: cmd = (('cd %(jobdir)s && %(solvecmd)s %(jobid)s %(axyfile)s >> ' + '%(logfile)s') % dict(jobid='job-%s-%i' % (settings.sitename, job.id), solvecmd=solve_locally, axyfile=axyfn, jobdir=jobdir, logfile=logfn)) log.msg('command:', cmd) w = os.system(cmd) if not os.WIFEXITED(w): log.msg('Solver failed (sent signal?)') logmsg('Call to solver failed for job', job.id) raise Exception rtn = os.WEXITSTATUS(w) if rtn: log.msg('Solver failed with return value %i' % rtn) logmsg('Call to solver failed for job', job.id, 'with return val', rtn) raise Exception log.msg('Solver completed successfully.') else: if solve_command is None: solve_command = 'ssh -x -T %(sshconfig)s' cmd = (( '(echo %(jobid)s; ' 'tar cf - --ignore-failed-read -C %(jobdir)s %(axyfile)s) | ' + solve_command + ' 2>>%(logfile)s | ' 'tar xf - --atime-preserve -m --exclude=%(axyfile)s -C %(jobdir)s ' '>>%(logfile)s 2>&1') % dict(jobid='job-%s-%i' % (settings.sitename, job.id), axyfile=axyfn, jobdir=jobdir, sshconfig=settings.ssh_solver_config, logfile=logfn)) log.msg('command:', cmd) w = os.system(cmd) if not os.WIFEXITED(w): log.msg('Solver failed (sent signal?)') logmsg('Call to solver failed for job', job.id) raise Exception rtn = os.WEXITSTATUS(w) if rtn: log.msg('Solver failed with return value %i' % rtn) logmsg('Call to solver failed for job', job.id, 'with return val', rtn) raise Exception log.msg('Solver completed successfully.') # Solved? wcsfn = os.path.join(jobdir, wcsfile) log.msg('Checking for WCS file', wcsfn) if os.path.exists(wcsfn): log.msg('WCS file exists') # Parse the wcs.fits file wcs = Tan(wcsfn, 0) # Convert to database model... tan = TanWCS(crval1=wcs.crval[0], crval2=wcs.crval[1], crpix1=wcs.crpix[0], crpix2=wcs.crpix[1], cd11=wcs.cd[0], cd12=wcs.cd[1], cd21=wcs.cd[2], cd22=wcs.cd[3], imagew=img.width, imageh=img.height) tan.save() log.msg('Created TanWCS:', tan) # Find field's healpix nside and index ra, dec, radius = tan.get_center_radecradius() nside = anutil.healpix_nside_for_side_length_arcmin(radius * 60) nside = int(2**round(math.log(nside, 2))) nside = max(1, nside) healpix = anutil.radecdegtohealpix(ra, dec, nside) try: sky_location, created = SkyLocation.objects.get_or_create( nside=nside, healpix=healpix) except MultipleObjectsReturned: log.msg('Multiple SkyLocations for nside %i, healpix %i' % (nside, healpix)) # arbitrarily take the first one. sky_location = SkyLocation.objects.filter(nside=nside, healpix=healpix)[0] log.msg('SkyLocation:', sky_location) # Find bounds for the Calibration object. r0, r1, d0, d1 = wcs.radec_bounds() # Find cartesian coordinates ra *= math.pi / 180 dec *= math.pi / 180 tempr = math.cos(dec) x = tempr * math.cos(ra) y = tempr * math.sin(ra) z = math.sin(dec) r = radius / 180 * math.pi calib = Calibration(raw_tan=tan, ramin=r0, ramax=r1, decmin=d0, decmax=d1, x=x, y=y, z=z, r=r, sky_location=sky_location) calib.save() log.msg('Created Calibration', calib) job.calibration = calib job.save() # save calib before adding machine tags job.status = 'S' job.user_image.add_machine_tags(job) job.user_image.add_sky_objects(job) else: job.status = 'F' job.set_end_time() job.save() log.msg('Finished job', job.id) logmsg('Finished job', job.id) return job.id