def makegridpacks(args):
if args.debug: logging.getLogger().setLevel(logging.DEBUG)
from helperstuff.jobsubmission import condorsetup
from helperstuff.utilities import cmsswversion, scramarch
condorsetup(args.condorjobid, args.condorjobflavor, args.condorjobtime)
if args.disable_duplicate_check: disableduplicatecheck()
with ApprovalQueue() as approvalqueue, BadRequestQueue(
) as badrequestqueue, CloneQueue() as clonequeue:
startprinting = False
for sample in allsamples(filter=args.filter):
if args.suppressfinished(sample) and sample.finished: continue
if args.hide_wrong_cmssw and (sample.cmsswversion, sample.scramarch
) != (cmsswversion, scramarch):
continue
if args.start_from(sample): startprinting = True
if not startprinting: continue
print sample, sample.makegridpack(
approvalqueue,
badrequestqueue,
clonequeue,
setneedsupdate=args.setneedsupdate)
sys.stdout.flush()
def requesturl(filter=lambda sample: True, status=("defined",), onlymysamples=False):
prepids = [sample.prepid for sample in allsamples(
filter=lambda sample:
sample.prepid is not None
and filter(sample)
and (status==allstatuses or sample.status in status),
onlymysamples=onlymysamples,
)]
if not prepids:
print "no prepids!"
return
return requesturl_prepids(prepids)
def maketicket(block,
chain,
tags,
filter=lambda sample: True,
modifyticket=None,
notes=None,
dryrun=False,
status=("defined", ),
onlymysamples=False):
prepids = [
sample.prepid for sample in allsamples(
filter=lambda sample: not sample.finished and sample.prepid is
not None and filter(sample) and sample.status in status,
onlymysamples=onlymysamples,
)
]
if not prepids:
print "no prepids!"
return
prepids.sort()
firstpart = {prepid.rsplit("-", 1)[0] for prepid in prepids}
if len(firstpart) > 1:
raise ValueError(
"The prepids have to be all the same except for the final number, but there are multiple different ones:\n"
+ ", ".join(firstpart))
firstpart = firstpart.pop()
if modifyticket is not None:
ticket = restful().get("mccms", modifyticket)
else:
ticket = {
"prepid": firstpart.split("-")[0],
"pwg": firstpart.split("-")[0],
}
requests = []
currentrequestrange = []
for prepid in prepids:
if len(currentrequestrange) == 2:
if int(prepid.split("-")[-1]) == int(
currentrequestrange[1].split("-")[-1]) + 1:
currentrequestrange[1] = prepid
else:
currentrequestrange = []
if len(currentrequestrange) == 1:
if int(prepid.split("-")[-1]) == int(
currentrequestrange[0].split("-")[-1]) + 1:
currentrequestrange.append(prepid)
else:
currentrequestrange = []
if not currentrequestrange: #not elif!!
currentrequestrange.append(prepid)
requests.append(currentrequestrange)
assert prepid == currentrequestrange[-1]
assert currentrequestrange is requests[
-1] #is rather than ==, so that we can modify it in place
ticket.update({
"block": block,
"chains": [chain],
"repetitions": 1,
"requests": requests,
"tags": tags,
})
if notes is not None: ticket["notes"] = notes
pprint.pprint(ticket)
if dryrun: return
answer = (restful().updateA if modifyticket else restful().putA)('mccms',
ticket)
print answer
if not answer['results']:
raise RuntimeError(answer)
def makegridpacks(args):
with ApprovalQueue() as approvalqueue, BadRequestQueue() as badrequestqueue, CloneQueue() as clonequeue:
for sample in allsamples(filter=args.filter):
if args.suppressfinished(sample) and sample.finished: continue
print sample, sample.makegridpack(approvalqueue, badrequestqueue, clonequeue, setneedsupdate=args.setneedsupdate)
sys.stdout.flush()
def maketicket(block, chain, tags, year, filter=lambda sample: True, modifytickets=[], notes=None, dryrun=False, status=("defined",), onlymysamples=False):
prepids = [sample.prepid for sample in allsamples(
filter=lambda sample:
not sample.finished
and sample.prepid is not None
and sample.year == year
and filter(sample)
and sample.status in status,
onlymysamples=onlymysamples,
)]
if not prepids:
print "no prepids!"
return
prepids.sort()
firsttwoparts = {prepid.rsplit("-", 1)[0] for prepid in prepids}
if len(firsttwoparts) > 1:
raise ValueError("The prepids have to be all the same except for the final number, but there are multiple different ones:\n"+", ".join(firsttwoparts))
firsttwoparts = firsttwoparts.pop()
firstpart, middlepart = firsttwoparts.split("-")
chainfirstpart = chain.split("_")[1]
if middlepart != chainfirstpart:
raise ValueError("Chain first part "+chainfirstpart+" is not the same as the request prepids' middle part "+middlepart)
ntickets = int(math.ceil(len(prepids) / 40.))
ineachticket = int(math.ceil(1.*len(prepids) / ntickets))
if len(modifytickets) > ntickets:
raise ValueError("Too many tickets to modify, there are only enough requests for "+str(ntickets))
tickets = []
for i in xrange(ntickets):
if i < len(modifytickets):
tickets.append(McM().get("mccms", modifytickets[i]))
else:
tickets.append({
"prepid": firstpart,
"pwg": firstpart,
})
ticketrequests = []
requests = []
currentrequestrange = []
i = 0
for prepid in prepids:
if i == 0:
requests = []
currentrequestrange = []
ticketrequests.append(requests)
if len(currentrequestrange) == 2:
if int(prepid.split("-")[-1]) == int(currentrequestrange[1].split("-")[-1])+1:
currentrequestrange[1] = prepid
else:
currentrequestrange = []
if len(currentrequestrange) == 1:
if int(prepid.split("-")[-1]) == int(currentrequestrange[0].split("-")[-1])+1:
currentrequestrange.append(prepid)
else:
currentrequestrange = []
if not currentrequestrange: #not elif!!
currentrequestrange.append(prepid)
requests.append(currentrequestrange)
assert prepid == currentrequestrange[-1]
assert currentrequestrange is requests[-1] #is rather than ==, so that we can modify it in place
i += 1
if i == ineachticket:
i = 0
for ticket, requests in itertools.izip(tickets, ticketrequests):
assert all(1 <= len(lst) <= 2 for lst in requests)
requests = [lst if len(lst) == 2 else lst[0] for lst in requests]
ticket.update({
"block": block,
"chains": [chain],
"repetitions": 1,
"requests": requests,
"tags": tags,
})
if notes is not None: ticket["notes"] = notes
pprint.pprint(ticket)
if dryrun: return
for ticket in tickets:
answer = (McM().update if modifytickets else McM().put)('mccms', ticket)
print answer
if not answer['results']:
raise RuntimeError(answer)