def alloc(requests, logstream = None, nonseparable = False, saveinteresting = None, backtracklim = 100000000, verbose = True, sample = random.sample):
"""
Takes an iterable over requests, which are iterables of candidate node ids,
and returns a specific node id for each request (if successful).
If it cannot, it will raise an ResourceAllocationError.
"""
# First, materialize the request iterable
requests = map(set,requests)
# Classify all candidates
universe = reduce(operator.or_, requests, set())
partition = setclusters.disjoint_partition(*requests)
# Classify requests
c_reqlist = classify.classify(requests, partition)
c_req = dict(
(c,len(r))
for c,r in c_reqlist.iteritems()
)
# Classify universe
c_uni = map(len, partition)
# Perform invariant sanity checks
if multicardinal(c_req) > sum(c_uni):
raise ResourceAllocationError, "Insufficient resources to grant request"
for c,nreq in c_req.iteritems():
if nreq > len(avail(c, partition)):
raise ResourceAllocationError, "Insufficient resources to grant request, empty categories %s" % (
filter(lambda i : classify.classContains(c,i), xrange(len(c))),
)
# Test for separability
if nonseparable:
components = clusters = []
else:
components = [
classify.classMembers(c, partition)
for c in c_req
]
clusters = setclusters.disjoint_sets(*components)
if len(clusters) > 1:
if verbose:
_log(logstream, "\nDetected %d clusters", len(clusters))
# Requests are separable
# Solve each part separately, then rejoin them
# Build a class for each cluster
clustermaps = []
compmap = dict([(pid,idx) for idx,pid in enumerate(map(id,components))])
for cluster in clusters:
cluster_class = classify.getClass(
reduce(operator.or_, cluster, set()),
partition )
clustermaps.append(cluster_class)
# Build a plan: assign a cluster to each request
plan = []
for cluster_class in clustermaps:
plan_reqs = []
for c, c_requests in c_reqlist.iteritems():
if classify.isSubclass(cluster_class, c):
plan_reqs.extend(c_requests)
plan.append(plan_reqs)
# Execute the plan
partial_results = []
for i,plan_req in enumerate(plan):
if verbose:
_log(logstream, "Solving cluster %d/%d", i+1, len(plan))
partial_results.append(alloc(plan_req,
logstream,
nonseparable = True,
saveinteresting = saveinteresting,
backtracklim = backtracklim,
verbose = verbose))
# Join results
if verbose:
_log(logstream, "Joining partial results")
reqmap = dict([(pid,idx) for idx,pid in enumerate(map(id,requests))])
joint = [None] * len(requests)
for partial_result, partial_requests in zip(partial_results, plan):
for assignment, partial_request in zip(partial_result, partial_requests):
joint[reqmap[id(partial_request)]] = assignment
return joint
else:
# Non-separable request, solve
#_log(logstream, "Non-separable request")
# Solve
partial = collections.defaultdict(list)
Pavail = list(c_uni)
Gavail = dict([
(c, len(avail(c, partition)))
for c in c_req
])
req = dict(c_req)
# build a cardinality map
cardinality = dict([
(c, [classify.classCardinality(c,partition), -nreq])
for c,nreq in req.iteritems()
])
classContains = classify.classContains
isSubclass = classify.isSubclass
stats = [
0, # ops
0, # back tracks
0, # skipped branches
]
def recursive_alloc():
# Successful termination condition: all requests satisfied
if not req:
return True
# Try in cardinality order
if quickstage:
order = heapq.nsmallest(2, req, key=Gavail.__getitem__)
else:
order = sorted(req, key=Gavail.__getitem__)
# Do backtracking on those whose cardinality leaves a choice
# Force a pick when it does not
if order and (Gavail[order[0]] <= 1
or classify.classCardinality(order[0]) <= 1):
order = order[:1]
for c in order:
nreq = req[c]
#carditem = cardinality[c]
for i,bit in enumerate(c):
if bit == "1" and Pavail[i]:
stats[0] += 1 # ops+1
subreq = min(Pavail[i], nreq)
# branch sanity check
skip = False
for c2,navail in Gavail.iteritems():
if c2 != c and classContains(c2, i) and (navail - subreq) < req.get(c2,0):
# Fail branch, don't even try
skip = True
break
if skip:
stats[2] += 1 # skipped branches + 1
continue
# forward track
partial[c].append((i,subreq))
Pavail[i] -= subreq
#carditem[1] -= subreq
for c2 in Gavail:
if classContains(c2, i):
Gavail[c2] -= subreq
if subreq < nreq:
req[c] -= subreq
else:
del req[c]
# Try to solve recursively
success = recursive_alloc()
if success:
return success
# Back track
del partial[c][-1]
Pavail[i] += subreq
#carditem[1] += subreq
for c2 in Gavail:
if classContains(c2, i):
Gavail[c2] += subreq
if subreq < nreq:
req[c] += subreq
else:
req[c] = subreq
stats[1] += 1 # backtracks + 1
if (logstream or (saveinteresting is not None)) and (stats[1] & 0xffff) == 0:
_log(logstream, "%r\n%r\n... stats: ops=%d, backtracks=%d, skipped=%d", Gavail, req,
*stats)
if stats[1] == 0x1400000:
# Interesting case, log it out
_log(logstream, "... interesting case: %r", requests)
if saveinteresting is not None:
saveinteresting.append(requests)
if stats[1] > backtracklim:
break
# We tried and tried... and failed
return False
# First try quickly (assign most selective first exclusively)
quickstage = True
success = recursive_alloc()
if not success:
# If it fails, retry exhaustively (try all assignment orders)
quickstage = False
success = recursive_alloc()
if verbose or (not success or stats[1] or stats[2]):
_log(logstream, "%s with stats: ops=%d, backtracks=%d, skipped=%d",
("Succeeded" if success else "Failed"),
*stats)
if not success:
raise ResourceAllocationError, "Insufficient resources to grant request"
# Perform actual assignment
Pavail = map(set, partition)
solution = {}
for c, partial_assignments in partial.iteritems():
psol = set()
for i, nreq in partial_assignments:
part = Pavail[i]
if len(part) < nreq:
raise AssertionError, "Cannot allocate resources for supposedly valid solution!"
assigned = set(sample(part, nreq))
psol |= assigned
part -= assigned
solution[c] = psol
# Format solution for the caller (return a node id for each request)
reqmap = {}
for c,reqs in c_reqlist.iteritems():
for req in reqs:
reqmap[id(req)] = c
req_solution = []
for req in requests:
c = reqmap[id(req)]
req_solution.append(solution[c].pop())
return req_solution
def recursive_alloc():
# Successful termination condition: all requests satisfied
if not req:
return True
# Try in cardinality order
if quickstage:
order = heapq.nsmallest(2, req, key=Gavail.__getitem__)
else:
order = sorted(req, key=Gavail.__getitem__)
# Do backtracking on those whose cardinality leaves a choice
# Force a pick when it does not
if order and (Gavail[order[0]] <= 1
or classify.classCardinality(order[0]) <= 1):
order = order[:1]
for c in order:
nreq = req[c]
#carditem = cardinality[c]
for i,bit in enumerate(c):
if bit == "1" and Pavail[i]:
stats[0] += 1 # ops+1
subreq = min(Pavail[i], nreq)
# branch sanity check
skip = False
for c2,navail in Gavail.iteritems():
if c2 != c and classContains(c2, i) and (navail - subreq) < req.get(c2,0):
# Fail branch, don't even try
skip = True
break
if skip:
stats[2] += 1 # skipped branches + 1
continue
# forward track
partial[c].append((i,subreq))
Pavail[i] -= subreq
#carditem[1] -= subreq
for c2 in Gavail:
if classContains(c2, i):
Gavail[c2] -= subreq
if subreq < nreq:
req[c] -= subreq
else:
del req[c]
# Try to solve recursively
success = recursive_alloc()
if success:
return success
# Back track
del partial[c][-1]
Pavail[i] += subreq
#carditem[1] += subreq
for c2 in Gavail:
if classContains(c2, i):
Gavail[c2] += subreq
if subreq < nreq:
req[c] += subreq
else:
req[c] = subreq
stats[1] += 1 # backtracks + 1
if (logstream or (saveinteresting is not None)) and (stats[1] & 0xffff) == 0:
_log(logstream, "%r\n%r\n... stats: ops=%d, backtracks=%d, skipped=%d", Gavail, req,
*stats)
if stats[1] == 0x1400000:
# Interesting case, log it out
_log(logstream, "... interesting case: %r", requests)
if saveinteresting is not None:
saveinteresting.append(requests)
if stats[1] > backtracklim:
break
# We tried and tried... and failed
return False
