def retry_job(request, msgid):
""" Request retry of an ImageJob
:param msgid: ImageJob ID
"""
oldjob = ImageJob.objects.get(image_id__exact=msgid)
imgjob = ImageJob()
imgjob.image_id = "%s-%s" % ( request.user.id,
time.strftime('%Y%m%d-%H%M%S') )
imgjob.user = request.user
imgjob.image_type = oldjob.image_type
imgjob.overlay = oldjob.overlay
imgjob.tokenmap = oldjob.tokenmap
imgjob.arch = oldjob.arch
imgjob.extra_repos = oldjob.extra_repos
imgjob.kickstart = oldjob.kickstart
imgjob.name = oldjob.name
imgjob.queue = oldjob.queue
imgjob.save()
messages.add_message(request, messages.INFO, "Image resubmitted with new id %s." % imgjob.image_id)
return HttpResponseRedirect(reverse('img-app-queue'))
def submit(request):
"""
GET: returns an unbound ImageJobForm
POST: process a user submitted UploadFileForm
"""
postProcessFormset = formset_factory(PostProcessForm, formset=BasePostProcessFormset, extra=PostProcess.objects.filter(active=True).count())
if request.method == 'GET':
jobform = ImageJobForm(initial = {'devicegroup':settings.DEVICEGROUP,
'email':request.user.email}
)
reposformset = extraReposFormset()
tokensformset = extraTokensFormset()
ppformset = postProcessFormset()
return render_to_response('app/upload.html',
{'jobform' : jobform, 'reposformset' : reposformset, 'tokensformset' : tokensformset,
'ppformset' : ppformset}, context_instance=RequestContext(request)
)
if request.method == 'POST':
jobform = ImageJobForm(request.POST, request.FILES)
reposformset = extraReposFormset(request.POST)
tokensformset = extraTokensFormset(request.POST)
ppformset = postProcessFormset(request.POST)
if not jobform.is_valid() or not reposformset.is_valid() or not ppformset.is_valid():
return render_to_response('app/upload.html',
{'jobform': jobform, 'reposformset' : reposformset, 'tokensformset' : tokensformset,
'ppformset' : ppformset},
context_instance=RequestContext(request)
)
jobdata = jobform.cleaned_data
reposdata = reposformset.cleaned_data
tokensdata = tokensformset.cleaned_data[0]
imgjob = ImageJob()
ks = ""
ksname = ""
ks_type = ""
if 'template' in jobdata and jobdata['template']:
ksname = jobdata['template']
filename = os.path.join(settings.TEMPLATESDIR, ksname)
with open(filename, mode='r') as ffd:
ks = ffd.readlines()
elif 'ksfile' in jobdata and jobdata['ksfile']:
ksname = jobdata['ksfile'].name
ks = jobdata['ksfile'].readlines()
imgjob.kickstart = "".join(ks)
for line in ks:
if re.match(r'^#.*?KickstartType:.+$', line):
ks_type = line.split(":", 1)[1].strip()
break
if ksname.endswith('.ks'):
ksname = ksname[0:-3]
extra_repos = set()
for repo in reposdata:
if repo.get('obs', None):
reponame = repo['repo']
if not reponame.startswith('/'):
reponame = "/%s" % reponame
repo_url = repo['obs'] + repo['project'].replace(':', ':/') + reponame
extra_repos.add(repo_url)
overlay = set([ x for x in jobdata['overlay'].split(',') if x.strip()])
if 'features' in jobdata:
for feat in jobdata['features']:
print feat
repos_type = 'repositories-%s' % ks_type
if not ks_type or not repos_type in feat:
repos_type = 'repositories'
extra_repos.update(feat.get(repos_type, set()))
overlay.update(feat.get('pattern', ''))
overlay.update(feat.get('packages', set()))
tokenmap = {}
for token in Token.objects.all():
if token.name in tokensdata:
tokenvalue = tokensdata[token.name]
else:
tokenvalue = token.default
if token.name == "RNDFLAVOUR":
rndpattern = ":/%s" % tokenvalue
if tokenvalue == "devel":
rndpattern = ""
tokenmap["RNDPATTERN"] = rndpattern
if token.name == "RELEASE":
if tokenvalue == "":
tokenmap["RELEASEPATTERN"] = ""
else:
tokenmap["RELEASEPATTERN"] = ":/%s" % tokenvalue
tokenmap[token.name] = tokenvalue
archtoken = jobdata['architecture']
if archtoken == "i686":
archtoken = "i586"
tokenmap["ARCH"] = archtoken
devicemodeltoken = jobdata.get('devicemodel')
if devicemodeltoken:
tokenmap['DEVICEMODEL'] = devicemodeltoken
devicevarianttoken = jobdata.get('devicevariant')
if devicevarianttoken:
tokenmap['DEVICEVARIANT'] = devicevarianttoken
tokens_list = []
extra_repos_tmp = []
for token, tokenvalue in tokenmap.items():
ksname = ksname.replace("@%s@" % token, tokenvalue)
tokens_list.append("%s:%s" % (token, tokenvalue))
for repo in extra_repos:
extra_repos_tmp.append(repo.replace("@%s@" % token, tokenvalue))
extra_repos = extra_repos_tmp[:]
extra_repos_tmp = []
imgjob.name = ksname
imgjob.arch = jobdata['architecture']
imgjob.tokenmap = ",".join(tokens_list)
imgjob.image_type = jobdata['imagetype']
imgjob.user = request.user
imgjob.extra_repos = ",".join(extra_repos)
imgjob.overlay = ",".join(overlay)
imgjob.queue = Queue.objects.get(name="web")
all_pps = PostProcess.objects.filter(active=True)
post_processes = set()
post_processes_args = {}
for ppform in ppformset:
ppdata = ppform.cleaned_data
for pp in all_pps:
if ppdata.get(pp.name, pp.default):
post_processes.add(pp.id)
pparg = ppdata.get(pp.argname, False)
if pparg:
try:
post_processes_args[pp.argname] = json.loads(pparg)
except ValueError:
post_processes_args[pp.argname] = pparg
imgjob.pp_args = json.dumps(post_processes_args)
saved = False
while not saved:
try:
imgjob.image_id = "%s-%s" % ( request.user.id,
time.strftime('%Y%m%d-%H%M%S') )
imgjob.save()
saved = True
except IntegrityError, exc:
print exc
print "couldn't save %s, retrying" % imgjob.image_id
time.sleep(1)
print "saved %s" % imgjob.image_id
imgjob.post_processes.add(*list(post_processes))
if jobdata["pinned"]:
imgjob.tags.add("pinned")
if jobdata["tags"]:
tags = [tag.replace(" ","_") for tag in jobdata["tags"].split(",")]
imgjob.tags.add(*tags)
return HttpResponseRedirect(reverse('img-app-queue'))
def submit(request):
"""
GET: returns an unbound UploadFileForm
POST: process a user submitted UploadFileForm
"""
postProcessFormset = formset_factory(
PostProcessForm,
formset=BasePostProcessFormset,
extra=PostProcess.objects.filter(active=True).count())
if request.method == 'GET':
jobform = ImageJobForm(initial={
'devicegroup': settings.DEVICEGROUP,
'email': request.user.email
})
reposformset = extraReposFormset()
tokensformset = extraTokensFormset()
ppformset = postProcessFormset()
return render_to_response('app/upload.html', {
'jobform': jobform,
'reposformset': reposformset,
'tokensformset': tokensformset,
'ppformset': ppformset
},
context_instance=RequestContext(request))
if request.method == 'POST':
jobform = ImageJobForm(request.POST, request.FILES)
reposformset = extraReposFormset(request.POST)
tokensformset = extraTokensFormset(request.POST)
ppformset = postProcessFormset(request.POST)
if not jobform.is_valid() or not reposformset.is_valid(
) or not ppformset.is_valid():
return render_to_response('app/upload.html', {
'jobform': jobform,
'reposformset': reposformset,
'tokensformset': tokensformset,
'ppformset': ppformset
},
context_instance=RequestContext(request))
jobdata = jobform.cleaned_data
reposdata = reposformset.cleaned_data
tokensdata = tokensformset.cleaned_data[0]
imgjob = ImageJob()
ks = ""
ksname = ""
ks_type = ""
if 'template' in jobdata and jobdata['template']:
ksname = jobdata['template']
filename = os.path.join(settings.TEMPLATESDIR, ksname)
with open(filename, mode='r') as ffd:
ks = ffd.readlines()
elif 'ksfile' in jobdata and jobdata['ksfile']:
ksname = jobdata['ksfile'].name
ks = jobdata['ksfile'].readlines()
imgjob.kickstart = "\n".join(ks)
for line in ks:
if re.match(r'^#.*?KickstartType:.+$', line):
ks_type = line.split(":", 1)[1].strip()
break
if ksname.endswith('.ks'):
ksname = ksname[0:-3]
extra_repos = set()
for repo in reposdata:
if repo.get('obs', None):
reponame = repo['repo']
if not reponame.startswith('/'):
reponame = "/%s" % reponame
repo_url = repo['obs'] + repo['project'].replace(
':', ':/') + reponame
extra_repos.add(repo_url)
overlay = set([x for x in jobdata['overlay'].split(',') if x.strip()])
if 'features' in jobdata:
for feat in jobdata['features']:
print feat
repos_type = 'repositories-%s' % ks_type
if not ks_type or not repos_type in feat:
repos_type = 'repositories'
extra_repos.update(feat.get(repos_type, set()))
overlay.update(feat.get('pattern', ''))
overlay.update(feat.get('packages', set()))
tokenmap = {}
for token in Token.objects.all():
if token.name in tokensdata:
tokenvalue = tokensdata[token.name]
else:
tokenvalue = token.default
if token.name == "RNDFLAVOUR":
rndpattern = ":/%s" % tokenvalue
if tokenvalue == "devel":
rndpattern = ""
tokenmap["RNDPATTERN"] = rndpattern
if token.name == "RELEASE":
if tokenvalue == "":
tokenmap["RELEASEPATTERN"] = ""
else:
tokenmap["RELEASEPATTERN"] = ":/%s" % tokenvalue
if " " in tokenvalue:
tokenvalue = '"%s"' % tokenvalue
tokenmap[token.name] = tokenvalue
archtoken = jobdata['architecture']
if archtoken == "i686":
archtoken = "i586"
tokenmap["ARCH"] = archtoken
tokens_list = []
extra_repos_tmp = []
for token, tokenvalue in tokenmap.items():
ksname = ksname.replace("@%s@" % token, tokenvalue)
tokens_list.append("%s:%s" % (token, tokenvalue))
for repo in extra_repos:
extra_repos_tmp.append(repo.replace("@%s@" % token,
tokenvalue))
extra_repos = extra_repos_tmp[:]
extra_repos_tmp = []
imgjob.name = ksname
imgjob.arch = jobdata['architecture']
imgjob.tokenmap = ",".join(tokens_list)
imgjob.image_type = jobdata['imagetype']
imgjob.user = request.user
imgjob.extra_repos = ",".join(extra_repos)
imgjob.overlay = ",".join(overlay)
imgjob.queue = Queue.objects.get(name="web")
all_pps = PostProcess.objects.filter(active=True)
post_processes = set()
post_processes_args = {}
for ppform in ppformset:
ppdata = ppform.cleaned_data
for pp in all_pps:
if ppdata.get(pp.name, pp.default):
post_processes.add(pp.id)
pparg = ppdata.get(pp.argname, False)
if pparg:
try:
post_processes_args[pp.argname] = json.loads(pparg)
except ValueError:
post_processes_args[pp.argname] = pparg
imgjob.pp_args = json.dumps(post_processes_args)
saved = False
while not saved:
try:
imgjob.image_id = "%s-%s" % (request.user.id,
time.strftime('%Y%m%d-%H%M%S'))
imgjob.save()
saved = True
except IntegrityError, exc:
print exc
print "couldn't save %s, retrying" % imgjob.image_id
time.sleep(1)
print "saved %s" % imgjob.image_id
imgjob.post_processes.add(*list(post_processes))
if jobdata["pinned"]:
imgjob.tags.add("pinned")
if jobdata["tags"]:
tags = [
tag.replace(" ", "_") for tag in jobdata["tags"].split(",")
]
imgjob.tags.add(*tags)
return HttpResponseRedirect(reverse('img-app-queue'))
def retry_job(request, msgid):
""" Request retry of an ImageJob
:param msgid: ImageJob ID
"""
oldjob = ImageJob.objects.get(image_id__exact=msgid)
imgjob = ImageJob()
imgjob.image_id = "%s-%s" % (request.user.id,
time.strftime('%Y%m%d-%H%M%S'))
imgjob.user = request.user
#imgjob.email = oldjob.email
imgjob.image_type = oldjob.image_type
imgjob.overlay = oldjob.overlay
imgjob.tokenmap = oldjob.tokenmap
imgjob.arch = oldjob.arch
#imgjob.devicegroup = oldjob.devicegroup
#imgjob.test_image = oldjob.test_image
#imgjob.notify = oldjob.notify
imgjob.extra_repos = oldjob.extra_repos
imgjob.kickstart = oldjob.kickstart
imgjob.name = oldjob.name
imgjob.queue = oldjob.queue
imgjob.save()
messages.add_message(request, messages.INFO,
"Image resubmitted with new id %s." % imgjob.image_id)
return HttpResponseRedirect(reverse('img-app-queue'))
def submit(request):
"""
GET: returns an unbound UploadFileForm
POST: process a user submitted UploadFileForm
"""
if request.method == 'GET':
form = UploadFileForm(initial = {'devicegroup':settings.DEVICEGROUP,
'email':request.user.email}
)
formset = extraReposFormset()
formset2 = extraTokensFormset()
return render_to_response('app/upload.html',
{'form' : form, 'formset' : formset, 'formset2' : formset2},
context_instance=RequestContext(request)
)
if request.method == 'POST':
form = UploadFileForm(request.POST, request.FILES)
formset = extraReposFormset(request.POST)
formset2 = extraTokensFormset(request.POST)
if not form.is_valid() or not formset.is_valid():
return render_to_response('app/upload.html',
{'form': form, 'formset' : formset, 'formset2' : formset2},
context_instance=RequestContext(request)
)
data = form.cleaned_data
data2 = formset.cleaned_data
data3 = formset2.cleaned_data[0]
imgjob = ImageJob()
ksname = ""
if 'template' in data and data['template']:
ksname = data['template']
filename = os.path.join(settings.TEMPLATESDIR, ksname)
with open(filename, mode='r') as ffd:
imgjob.kickstart = ffd.read()
elif 'ksfile' in data and data['ksfile']:
ksname = data['ksfile'].name
imgjob.kickstart = data['ksfile'].read()
if ksname.endswith('.ks'):
ksname = ksname[0:-3]
extra_repos = set()
for repo in data2:
if repo['obs']:
repo_url = repo['obs'] + repo['project'].replace(':', ':/') + repo['repo']
extra_repos.add(repo_url)
overlay = set([ x for x in data['overlay'].split(',') if x.strip()])
if 'features' in data:
for feat in data['features']:
print feat
extra_repos.update(feat.get('repos', set()))
overlay.update(feat.get('pattern', set()))
tokenmap = {}
for token in Token.objects.all():
if token.name in data3:
tokenvalue = data3[token.name]
else:
tokenvalue = token.default
if token.name == "RNDFLAVOUR":
rndpattern = ":/%s" % tokenvalue
if tokenvalue == "devel":
rndpattern = ""
tokenmap["RNDPATTERN"] = rndpattern
tokenmap[token.name] = tokenvalue
archtoken = data['architecture']
if archtoken == "i686":
archtoken = "i586"
tokenmap["ARCH"] = archtoken
tokens_list = []
extra_repos_tmp = []
for token, tokenvalue in tokenmap.items():
ksname = ksname.replace("@%s@" % token, tokenvalue)
tokens_list.append("%s:%s" % (token, tokenvalue))
for repo in extra_repos:
extra_repos_tmp.append(repo.replace("@%s@" % token, tokenvalue))
extra_repos = extra_repos_tmp[:]
extra_repos_tmp = []
imgjob.name = ksname
imgjob.arch = data['architecture']
imgjob.tokenmap = ",".join(tokens_list)
imgjob.image_id = "%s-%s" % ( request.user.id,
time.strftime('%Y%m%d-%H%M%S') )
imgjob.email = data['email']
imgjob.image_type = data['imagetype']
imgjob.user = request.user
if "test_image" in data.keys():
imgjob.devicegroup = data['devicegroup']
imgjob.test_image = data['test_image']
if "notify_image" in data.keys():
imgjob.notify = data["notify_image"]
imgjob.extra_repos = ",".join(extra_repos)
imgjob.overlay = ",".join(overlay)
imgjob.queue = Queue.objects.get(name="web")
imgjob.save()
if data["pinned"]:
imgjob.tags.add("pinned")
if data["tags"]:
tags = [tag.replace(" ","_") for tag in data["tags"].split(",")]
imgjob.tags.add(*tags)
return HttpResponseRedirect(reverse('img-app-queue'))
def handle_wi(self, wid):
wid.result = False
f = wid.fields
p = wid.params
if not f.msg:
f.msg = []
if (not f.ev or not f.ev.id or not f.image.kickstart
or not f.image.image_type or not f.image.arch or not f.image.name):
f.__error__ = "One of the mandatory fields: ev.id,"\
" image.kickstart, image_type, image.arch,"\
" or image.name doesn't exist."
f.msg.append(f.__error__)
raise RuntimeError("Missing mandatory field")
qname = "requests"
if f.image.queue and not f.image.queue == "web":
qname = f.image.queue
queue = Queue.objects.get(name=qname)
image_args = { "queue" : queue, "user" : self.user,
"image_type" : f.image.image_type,
"arch" : f.image.arch,
"kickstart" : f.image.kickstart,
"name" : f.image.name
}
if f.image.extra_opts:
image_args["extra_opts"] = f.image.extra_opts
if f.image.tokenmap:
image_args["tokenmap"] = f.image.tokenmap
job = None
if wid.params.action == "get_or_create":
self.log.info("get_or_create")
jobs = ImageJob.objects.filter(**image_args).filter(status__startswith="DONE")
self.log.info(jobs.count())
self.log.info("- %(queue)s - %(image_type)s - %(arch)s - %(name)s - %(user)s - kickstart:\n%(kickstart)s\n\n" % (image_args))
if wid.params.max_age:
ts = datetime.datetime.now() - datetime.timedelta(days=int(wid.params.max_age))
self.log.info(ts)
jobs = jobs.exclude(done__lte = ts)
self.log.info(jobs.count())
if jobs.count():
job = jobs[0]
f.image.image_url = job.image_url
f.image.files_url = job.files_url
f.image.logfile_url = job.logfile_url
if not job:
self.log.info("New job")
job = ImageJob(**image_args)
saved = False
while not saved:
try:
job.image_id = "%s-%s" % ( str(f.ev.id),
time.strftime('%Y%m%d-%H%M%S') )
job.save()
saved = True
f.image.image_url = ""
except IntegrityError, exc:
print exc
print "couldn't save %s, retrying" % job.image_id
time.sleep(1)
print "saved %s" % job.image_id
def handle_wi(self, wid):
wid.result = False
f = wid.fields
p = wid.params
if not f.msg:
f.msg = []
if (not f.ev or not f.ev.id or not f.image.kickstart
or not f.image.image_type or not f.image.arch or not f.image.name):
f.__error__ = "One of the mandatory fields: ev.id,"\
" image.kickstart, image_type, image.arch,"\
" or image.name doesn't exist."
f.msg.append(f.__error__)
raise RuntimeError("Missing mandatory field")
qname = "requests"
if f.image.queue and not f.image.queue == "web":
qname = f.image.queue
queue = Queue.objects.get(name=qname)
image_args = { "queue" : queue, "user" : self.user,
"image_type" : f.image.image_type,
"arch" : f.image.arch,
"kickstart" : f.image.kickstart,
"name" : f.image.name
}
if f.emails:
image_args["email"] = ",".join(f.emails)
else:
image_args["email"] = self.user.email
if f.image.devicegroup:
image_args["devicegroup"] = f.image.devicegroup
if f.image.extra_opts:
image_args["extra_opts"] = f.image.extra_opts
if f.image.test_options:
image_args["test_options"] = f.image.test_options
if f.image.tokenmap:
image_args["tokenmap"] = f.image.tokenmap
job = None
if wid.params.action == "get_or_create":
self.log.info("get_or_create")
jobs = ImageJob.objects.filter(**image_args).filter(status__startswith="DONE")
if wid.params.max_age:
ts = datetime.datetime.now() - datetime.timedelta(days=int(wid.params.max_age))
self.log.info(ts)
jobs = jobs.exclude(done__lte = ts)
self.log.info(jobs.count())
if jobs.count():
job = jobs[0]
f.image.image_url = job.image_url
f.image.files_url = job.files_url
f.image.logfile_url = job.logfile_url
if not job:
self.log.info("New job")
job = ImageJob(**image_args)
job.image_id = "%s-%s" % ( str(f.ev.id),
time.strftime('%Y%m%d-%H%M%S') )
job.save()
f.image.image_url = ""
if not f.image.prefix:
f.image.prefix = "%s/%s" % (job.queue.name,
job.user.username)
f.image.image_id = job.image_id
self.log.info("Requested image %s" % f.image.image_id)
wid.result = True