def alloc(requests, logstream = None, nonseparable = False, saveinteresting = None, backtracklim = 100000000, verbose = True, sample = random.sample): """ Takes an iterable over requests, which are iterables of candidate node ids, and returns a specific node id for each request (if successful). If it cannot, it will raise an ResourceAllocationError. """ # First, materialize the request iterable requests = map(set,requests) # Classify all candidates universe = reduce(operator.or_, requests, set()) partition = setclusters.disjoint_partition(*requests) # Classify requests c_reqlist = classify.classify(requests, partition) c_req = dict( (c,len(r)) for c,r in c_reqlist.iteritems() ) # Classify universe c_uni = map(len, partition) # Perform invariant sanity checks if multicardinal(c_req) > sum(c_uni): raise ResourceAllocationError, "Insufficient resources to grant request" for c,nreq in c_req.iteritems(): if nreq > len(avail(c, partition)): raise ResourceAllocationError, "Insufficient resources to grant request, empty categories %s" % ( filter(lambda i : classify.classContains(c,i), xrange(len(c))), ) # Test for separability if nonseparable: components = clusters = [] else: components = [ classify.classMembers(c, partition) for c in c_req ] clusters = setclusters.disjoint_sets(*components) if len(clusters) > 1: if verbose: _log(logstream, "\nDetected %d clusters", len(clusters)) # Requests are separable # Solve each part separately, then rejoin them # Build a class for each cluster clustermaps = [] compmap = dict([(pid,idx) for idx,pid in enumerate(map(id,components))]) for cluster in clusters: cluster_class = classify.getClass( reduce(operator.or_, cluster, set()), partition ) clustermaps.append(cluster_class) # Build a plan: assign a cluster to each request plan = [] for cluster_class in clustermaps: plan_reqs = [] for c, c_requests in c_reqlist.iteritems(): if classify.isSubclass(cluster_class, c): plan_reqs.extend(c_requests) plan.append(plan_reqs) # Execute the plan partial_results = [] for i,plan_req in enumerate(plan): if verbose: _log(logstream, "Solving cluster %d/%d", i+1, len(plan)) partial_results.append(alloc(plan_req, logstream, nonseparable = True, saveinteresting = saveinteresting, backtracklim = backtracklim, verbose = verbose)) # Join results if verbose: _log(logstream, "Joining partial results") reqmap = dict([(pid,idx) for idx,pid in enumerate(map(id,requests))]) joint = [None] * len(requests) for partial_result, partial_requests in zip(partial_results, plan): for assignment, partial_request in zip(partial_result, partial_requests): joint[reqmap[id(partial_request)]] = assignment return joint else: # Non-separable request, solve #_log(logstream, "Non-separable request") # Solve partial = collections.defaultdict(list) Pavail = list(c_uni) Gavail = dict([ (c, len(avail(c, partition))) for c in c_req ]) req = dict(c_req) # build a cardinality map cardinality = dict([ (c, [classify.classCardinality(c,partition), -nreq]) for c,nreq in req.iteritems() ]) classContains = classify.classContains isSubclass = classify.isSubclass stats = [ 0, # ops 0, # back tracks 0, # skipped branches ] def recursive_alloc(): # Successful termination condition: all requests satisfied if not req: return True # Try in cardinality order if quickstage: order = heapq.nsmallest(2, req, key=Gavail.__getitem__) else: order = sorted(req, key=Gavail.__getitem__) # Do backtracking on those whose cardinality leaves a choice # Force a pick when it does not if order and (Gavail[order[0]] <= 1 or classify.classCardinality(order[0]) <= 1): order = order[:1] for c in order: nreq = req[c] #carditem = cardinality[c] for i,bit in enumerate(c): if bit == "1" and Pavail[i]: stats[0] += 1 # ops+1 subreq = min(Pavail[i], nreq) # branch sanity check skip = False for c2,navail in Gavail.iteritems(): if c2 != c and classContains(c2, i) and (navail - subreq) < req.get(c2,0): # Fail branch, don't even try skip = True break if skip: stats[2] += 1 # skipped branches + 1 continue # forward track partial[c].append((i,subreq)) Pavail[i] -= subreq #carditem[1] -= subreq for c2 in Gavail: if classContains(c2, i): Gavail[c2] -= subreq if subreq < nreq: req[c] -= subreq else: del req[c] # Try to solve recursively success = recursive_alloc() if success: return success # Back track del partial[c][-1] Pavail[i] += subreq #carditem[1] += subreq for c2 in Gavail: if classContains(c2, i): Gavail[c2] += subreq if subreq < nreq: req[c] += subreq else: req[c] = subreq stats[1] += 1 # backtracks + 1 if (logstream or (saveinteresting is not None)) and (stats[1] & 0xffff) == 0: _log(logstream, "%r\n%r\n... stats: ops=%d, backtracks=%d, skipped=%d", Gavail, req, *stats) if stats[1] == 0x1400000: # Interesting case, log it out _log(logstream, "... interesting case: %r", requests) if saveinteresting is not None: saveinteresting.append(requests) if stats[1] > backtracklim: break # We tried and tried... and failed return False # First try quickly (assign most selective first exclusively) quickstage = True success = recursive_alloc() if not success: # If it fails, retry exhaustively (try all assignment orders) quickstage = False success = recursive_alloc() if verbose or (not success or stats[1] or stats[2]): _log(logstream, "%s with stats: ops=%d, backtracks=%d, skipped=%d", ("Succeeded" if success else "Failed"), *stats) if not success: raise ResourceAllocationError, "Insufficient resources to grant request" # Perform actual assignment Pavail = map(set, partition) solution = {} for c, partial_assignments in partial.iteritems(): psol = set() for i, nreq in partial_assignments: part = Pavail[i] if len(part) < nreq: raise AssertionError, "Cannot allocate resources for supposedly valid solution!" assigned = set(sample(part, nreq)) psol |= assigned part -= assigned solution[c] = psol # Format solution for the caller (return a node id for each request) reqmap = {} for c,reqs in c_reqlist.iteritems(): for req in reqs: reqmap[id(req)] = c req_solution = [] for req in requests: c = reqmap[id(req)] req_solution.append(solution[c].pop()) return req_solution
def recursive_alloc(): # Successful termination condition: all requests satisfied if not req: return True # Try in cardinality order if quickstage: order = heapq.nsmallest(2, req, key=Gavail.__getitem__) else: order = sorted(req, key=Gavail.__getitem__) # Do backtracking on those whose cardinality leaves a choice # Force a pick when it does not if order and (Gavail[order[0]] <= 1 or classify.classCardinality(order[0]) <= 1): order = order[:1] for c in order: nreq = req[c] #carditem = cardinality[c] for i,bit in enumerate(c): if bit == "1" and Pavail[i]: stats[0] += 1 # ops+1 subreq = min(Pavail[i], nreq) # branch sanity check skip = False for c2,navail in Gavail.iteritems(): if c2 != c and classContains(c2, i) and (navail - subreq) < req.get(c2,0): # Fail branch, don't even try skip = True break if skip: stats[2] += 1 # skipped branches + 1 continue # forward track partial[c].append((i,subreq)) Pavail[i] -= subreq #carditem[1] -= subreq for c2 in Gavail: if classContains(c2, i): Gavail[c2] -= subreq if subreq < nreq: req[c] -= subreq else: del req[c] # Try to solve recursively success = recursive_alloc() if success: return success # Back track del partial[c][-1] Pavail[i] += subreq #carditem[1] += subreq for c2 in Gavail: if classContains(c2, i): Gavail[c2] += subreq if subreq < nreq: req[c] += subreq else: req[c] = subreq stats[1] += 1 # backtracks + 1 if (logstream or (saveinteresting is not None)) and (stats[1] & 0xffff) == 0: _log(logstream, "%r\n%r\n... stats: ops=%d, backtracks=%d, skipped=%d", Gavail, req, *stats) if stats[1] == 0x1400000: # Interesting case, log it out _log(logstream, "... interesting case: %r", requests) if saveinteresting is not None: saveinteresting.append(requests) if stats[1] > backtracklim: break # We tried and tried... and failed return False