def bounce_try_dojob(X):
jobid, solve_command, solve_locally = X
try:
from process_submissions import try_dojob
print('Trying Job ID', jobid)
job = Job.objects.filter(id=jobid)[0]
print('Found Job', job)
r = try_dojob(job, job.user_image, solve_command, solve_locally)
print('Job result for', job, ':', r)
return r
except:
import traceback
traceback.print_exc()
def bounce_try_dojob(jobid):
print('Trying Job ID', jobid)
job = Job.objects.filter(id=jobid)[0]
print('Found Job', job)
return try_dojob(job, job.user_image)
if opt.job:
job = Job.objects.all().get(id=opt.job)
print('Job', job)
print(job.get_dir())
print('Status:', job.status)
print('Error message:', job.error_message)
ui = job.user_image
print('UserImage:', ui)
print('User', ui.user)
im = ui.image
print('Image', im)
sub = ui.submission
print('Submission', sub)
print(sub.disk_file.get_path())
if opt.rerun:
from astrometry.net.process_submissions import try_dojob
print('Re-trying job', job.id)
try_dojob(job, ui, opt.solve_command, opt.solve_locally)
if opt.uimage:
ui = UserImage.objects.all().get(id=opt.uimage)
print('UserImage', ui)
if opt.chown:
user = User.objects.all().get(id=opt.chown)
print('User:'******'chowning', ui, 'to', user)
ui.user = user
ui.save()
def main():
import optparse
parser = optparse.OptionParser('%(prog)')
parser.add_option('-s',
'--sub',
type=int,
dest='sub',
help='Submission ID')
parser.add_option('-j', '--job', type=int, dest='job', help='Job ID')
parser.add_option('-u',
'--userimage',
type=int,
dest='uimage',
help='UserImage ID')
parser.add_option('-e',
'--email',
help='Find user id with given email address')
parser.add_option('-U',
'--userid',
help='Find user with given numerical ID',
type=int)
parser.add_option('-i', '--image', type=int, dest='image', help='Image ID')
parser.add_option('-d',
'--disk-file',
type=str,
dest='df',
help='DiskFile id')
parser.add_option('-r',
'--rerun',
dest='rerun',
action='store_true',
help='Re-run this submission/job?')
parser.add_option(
'--threads',
type=int,
help=
'Re-run failed jobs within this process using N threads; else submit to process_submissions process.'
)
parser.add_option(
'--chown',
dest='chown',
type=int,
default=0,
help='Change owner of userimage or submission by user id #')
parser.add_option(
'--solve-command',
help='Command to run instead of ssh to actually solve image')
parser.add_option(
'--solve-locally',
help='Command to run astrometry-engine on this machine, not via ssh')
parser.add_option('--ssh',
action='store_true',
default=False,
help='Find submissions whose jobs have ssh errors')
parser.add_option('--minsub',
type='int',
default=0,
help='Minimum submission id to look at')
parser.add_option('--empty',
action='store_true',
default=False,
help='Find submissions whose jobs have no log files')
parser.add_option('--delete',
action='store_true',
default=False,
help='Delete everything associated with the given image')
parser.add_option('--delextra',
action='store_true',
default=False,
help='Delete extraneous duplicate jobs?')
parser.add_option('--hide',
action='store_true',
default=False,
help='For a UserImage, set publicly_visible=False')
opt, args = parser.parse_args()
if opt.email:
users = User.objects.filter(email__contains=opt.email)
print('Users with email containing "%s":' % opt.email)
for u in users:
print(u.id, u, u.email)
sys.exit(0)
if opt.userid:
users = User.objects.filter(id=opt.userid)
print('Users with ID', opt.userid)
for u in users:
print(u.id, u, u.email)
print(u.profile)
sys.exit(0)
if not (opt.sub or opt.job or opt.uimage or opt.image or opt.ssh
or opt.empty or opt.df):
print(
'Must specify one of --sub, --job, or --userimage or --image (or --ssh or --empty)'
)
parser.print_help()
sys.exit(-1)
if opt.threads is not None:
mp = multiproc(opt.threads)
else:
mp = None
if opt.ssh or opt.empty or opt.delextra:
subs = Submission.objects.all()
if opt.minsub:
subs = subs.filter(id__gt=opt.minsub)
subs = subs.order_by('-id')
failedsubs = []
failedjobs = []
for sub in subs:
print('Checking submission', sub.id)
allfailed = True
# last failed Job
failedjob = None
uis = sub.user_images.all()
for ui in uis:
jobs = ui.jobs.all()
for job in jobs:
print(' job', job.id)
if job.status == 'S':
print(' -> succeeded')
allfailed = False
break
print(' error msg', job.error_message)
logfn = job.get_log_file()
if not os.path.exists(logfn):
failedjob = job
continue
if opt.ssh:
log = read_file(logfn)
# 'Connection refused'
# 'Connection timed out'
if not 'ssh: connect to host astro.cs.toronto.edu port 22:' in log:
allfailed = False
break
print('SSH failed')
failedjob = job
if opt.empty:
# log file found
allfailed = False
break
if opt.delextra:
print('Delextra:', len(jobs), 'jobs', len(uis),
'uis; failedjob:', failedjob)
if len(jobs) > 1 and failedjob is not None:
print('Delextra: delete', failedjob)
if not allfailed:
continue
print('All jobs failed for sub', sub.id) #, 'via ssh failure')
#failedsubs.append(sub)
failedjobs.append(failedjob)
print('Found total of', len(failedsubs), 'failed Submissions and',
len(failedjobs), 'failed Jobs')
if opt.rerun:
from process_submissions import try_dosub, try_dojob
if opt.threads is not None:
args = []
for j in failedjobs:
if j is None:
continue
args.append((j.id, opt.solve_command, opt.solve_locally))
mp.map(bounce_try_dojob, args)
else:
for job in failedjobs:
if job is None:
continue
print('Re-trying job', job.id)
try_dojob(job, job.user_image, opt.solve_command,
opt.solve_locally)
# FIXME -- failed subs??
#
# else:
# for sub in failedsubs:
# print('Re-trying sub', sub.id)
# try_dosub(sub, 1)
if opt.df:
dfs = DiskFile.objects.filter(file_hash=opt.df)
print('Found', dfs.count(), 'matching DiskFiles')
for df in dfs:
print(' ', df)
print(' ', df.get_path())
ims = Image.objects.filter(disk_file=df)
print(' Used by', ims.count(), 'Image objects')
for im in ims:
print(' ', im, im.id)
print(' size %i x %i' % (im.width, im.height))
print(' thumb Image', im.thumbnail,
(im.thumbnail and im.thumbnail.id or ''))
print(' display Image', im.display_image,
(im.display_image and im.display_image.id or ''))
ims = Image.objects.filter(disk_file=df,
display_image__isnull=False,
thumbnail__isnull=False)
print(' found', ims.count(),
'Images for this df w/ display and thumbnails')
if opt.sub:
sub = Submission.objects.all().get(id=opt.sub)
print('Submission', sub)
if sub.disk_file is None:
print(' no disk file')
else:
print('Path', sub.disk_file.get_path())
print('Is fits image:', sub.disk_file.is_fits_image())
print('Is fits image:', sub.disk_file.file_type)
print('User:'******'UserImages:', len(uis))
for ui in uis:
print(' ', ui)
print(' user', ui.user)
print(' with Jobs:', len(ui.jobs.all()))
for j in ui.jobs.all():
print(' ', j)
if opt.chown:
newuser = User.objects.all().get(id=opt.chown)
for ui in uis:
ui.user = newuser
ui.save()
if opt.rerun:
from process_submissions import try_dosub, dosub
print('Re-trying sub', sub.id)
#try_dosub(sub, 1)
dosub(sub)
if opt.delete:
print('Deleting submission', sub)
sub.delete()
if opt.job:
job = Job.objects.all().get(id=opt.job)
print('Job', job)
print(job.get_dir())
print('Status:', job.status)
print('Error message:', job.error_message)
ui = job.user_image
print('UserImage:', ui)
print('User', ui.user)
im = ui.image
print('Image', im)
sub = ui.submission
print('Submission', sub)
print(sub.disk_file.get_path())
if opt.rerun:
from astrometry.net.process_submissions import try_dojob
print('Re-trying job', job.id)
try_dojob(job, ui, opt.solve_command, opt.solve_locally)
if opt.delete:
print('Deleting job', job)
job.delete()
if opt.uimage:
ui = UserImage.objects.all().get(id=opt.uimage)
print('UserImage', ui)
im = ui.image
print('Image', im)
sub = ui.submission
print('User', ui.user)
print('Submission', sub)
print(sub.disk_file.get_path())
if opt.chown:
user = User.objects.all().get(id=opt.chown)
print('User:'******'chowning', ui, 'to', user)
ui.user = user
ui.save()
if opt.delete:
print('Deleting ui', ui)
ui.delete()
if opt.hide:
print('Hiding ui', ui)
ui.hide()
if opt.image:
im = Image.objects.all().get(id=opt.image)
# thumbnail
# display_image
print('Image:', im, im.id)
#uis = im.userimage_set()
uis = UserImage.objects.all().filter(image=im.id)
print('UserImages:', uis)
print('Thumbnail:', im.thumbnail)
print('Display:', im.display_image)
if opt.delete:
print('Deleting...')
im.delete()
if im.thumbnail:
im.thumbnail.delete()
if im.display_image:
im.display_image.delete()
def bounce_try_dojob(jobid):
print('Trying Job ID', jobid)
job = Job.objects.filter(id=jobid)[0]
print('Found Job', job)
return try_dojob(job, job.user_image)
if opt.job:
job = Job.objects.all().get(id=opt.job)
print('Job', job)
print(job.get_dir())
print('Status:', job.status)
print('Error message:', job.error_message)
ui = job.user_image
print('UserImage:', ui)
print('User', ui.user)
im = ui.image
print('Image', im)
sub = ui.submission
print('Submission', sub)
print(sub.disk_file.get_path())
if opt.rerun:
from astrometry.net.process_submissions import try_dojob
print('Re-trying job', job.id)
try_dojob(job, ui, opt.solve_command, opt.solve_locally)
if opt.uimage:
ui = UserImage.objects.all().get(id=opt.uimage)
print('UserImage', ui)
if opt.chown:
user = User.objects.all().get(id=opt.chown)
print('User:'******'chowning', ui, 'to', user)
ui.user = user
ui.save()
if opt.job: job = Job.objects.all().get(id=opt.job) print 'Job', job print job.get_dir() ui = job.user_image print 'UserImage:', ui print 'User', ui.user im = ui.image print 'Image', im sub = ui.submission print 'Submission', sub print sub.disk_file.get_path() if opt.rerun: from process_submissions import try_dojob print 'Re-trying job', job.id try_dojob(job, ui) if opt.uimage: ui = UserImage.objects.all().get(id=opt.uimage) print 'UserImage', ui if opt.chown: user = User.objects.all().get(id=opt.chown) print 'User:'******'chowning', ui, 'to', user ui.user = user ui.save()
try_dosub(sub, 1)
if opt.job:
job = Job.objects.all().get(id=opt.job)
print 'Job', job
print job.get_dir()
ui = job.user_image
print 'UserImage:', ui
print 'User', ui.user
im = ui.image
print 'Image', im
sub = ui.submission
print 'Submission', sub
print sub.disk_file.get_path()
if opt.rerun:
from process_submissions import try_dojob
print 'Re-trying job', job.id
try_dojob(job, ui)
if opt.uimage:
ui = UserImage.objects.all().get(id=opt.uimage)
print 'UserImage', ui
if opt.chown:
user = User.objects.all().get(id=opt.chown)
print 'User:'******'chowning', ui, 'to', user
ui.user = user
ui.save()