def test_connection_pooling(self): with patch('swift.common.memcached.socket') as mock_module: # patch socket, stub socket.socket, mock sock mock_sock = mock_module.socket.return_value # track clients waiting for connections connected = [] connections = Queue() errors = [] def wait_connect(addr): connected.append(addr) sleep(0.1) # yield val = connections.get() if val is not None: errors.append(val) mock_sock.connect = wait_connect memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'], connect_timeout=10) # sanity self.assertEquals(1, len(memcache_client._client_cache)) for server, pool in memcache_client._client_cache.items(): self.assertEqual(2, pool.max_size) # make 10 requests "at the same time" p = GreenPool() for i in range(10): p.spawn(memcache_client.set, 'key', 'value') for i in range(3): sleep(0.1) self.assertEqual(2, len(connected)) # give out a connection connections.put(None) # at this point, only one connection should have actually been # created, the other is in the creation step, and the rest of the # clients are not attempting to connect. we let this play out a # bit to verify. for i in range(3): sleep(0.1) self.assertEqual(2, len(connected)) # finish up, this allows the final connection to be created, so # that all the other clients can use the two existing connections # and no others will be created. connections.put(None) connections.put('nono') self.assertEqual(2, len(connected)) p.waitall() self.assertEqual(2, len(connected)) self.assertEqual(0, len(errors), "A client was allowed a third connection") connections.get_nowait() self.assertTrue(connections.empty())
def main(): args = options() global ACCOUNT, PROXY, QUEUE, NS, VERBOSE, TIMEOUT global COUNTERS, ELECTIONS ACCOUNT = args.account NS = args.namespace VERBOSE = args.verbose TIMEOUT = args.timeout PROXY = ObjectStorageApi(NS) ELECTIONS = AtomicInteger() num_worker_threads = int(args.max_worker) print("Using %d workers" % num_worker_threads) total_objects = {'size': 0, 'files': 0, 'elapsed': 0} total_containers = {'size': 0, 'files': 0, 'elapsed': 0} for path in args.path: path = path.rstrip('/') if '/' in path: bucket, path = path.split('/', 1) else: bucket = path path = "" containers = [] QUEUE = Queue() pool = eventlet.GreenPool(num_worker_threads) for i in range(num_worker_threads): pool.spawn(worker_objects) COUNTERS = AtomicInteger() _bucket = container_hierarchy(bucket, path) # we don't use placeholders, we use prefix path as prefix for entry in full_list(prefix=container_hierarchy(bucket, path)): name, _files, _size, _ = entry if name != _bucket and not name.startswith(_bucket + '%2F'): continue if _files: QUEUE.put(name) containers.append(name) # we have to wait all objects print("Waiting flush of objects") report = args.report while not QUEUE.empty(): ts = time.time() while time.time() - ts < report and not QUEUE.empty(): time.sleep(1) diff = time.time() - ts val = COUNTERS.reset() elections = ELECTIONS.reset() print("Objects: %5.2f / Size: %5.2f" % (val[0] / diff, val[1] / diff), "Elections failed: %5.2f/s total: %d" % (elections[0] / diff, ELECTIONS.total()[0]), " " * 20, end='\r') sys.stdout.flush() print("Waiting end of workers") QUEUE.join() val = COUNTERS.total() total_objects['files'] += val[0] total_objects['size'] += val[1] total_objects['elapsed'] += COUNTERS.time() COUNTERS = AtomicInteger() QUEUE = Queue() for i in range(num_worker_threads): pool.spawn(worker_container) print("We have to delete", len(containers), "containers") for container in containers: QUEUE.put(container) while not QUEUE.empty(): ts = time.time() while time.time() - ts < report and not QUEUE.empty(): time.sleep(1) diff = time.time() - ts val = COUNTERS.reset() elections = ELECTIONS.reset() print("Containers: %5.2f" % (val[0] / diff), "Elections failed: %5.2f/s total: %d" % (elections[0] / diff, ELECTIONS.total()[0]), " " * 20, end='\r') sys.stdout.flush() QUEUE.join() val = COUNTERS.total() total_containers['files'] += val[0] total_containers['size'] += val[1] total_containers['elapsed'] += COUNTERS.time() print(""" Objects: - ran during {o[elapsed]:5.2f} - {o[files]} objects removed (size {size}) - {o_file_avg:5.2f} objects/s ({o_size_avg} avg. size/s) """.format(o=total_objects, size=show(total_objects['size'], True), o_file_avg=total_objects['files'] / total_objects['elapsed'], o_size_avg=show(total_objects['size'] / total_objects['elapsed'], True))) print(""" Containers: - ran during {o[elapsed]:5.2f} - {o[files]} containers - {o_file_avg:5.2f} containers/s """.format(o=total_containers, o_file_avg=total_containers['files'] / total_containers['elapsed'])) print("Elections failed: %d" % ELECTIONS.total()[0])
class Interpreter(object): ''' The class repsonsible for keeping track of the execution of the statemachine. ''' def __init__(self): self.running = True self.configuration = OrderedSet() self.internalQueue = Queue() self.externalQueue = Queue() self.statesToInvoke = OrderedSet() self.historyValue = {} self.dm = None self.invokeId = None self.parentId = None self.logger = None def interpret(self, document, invokeId=None): '''Initializes the interpreter given an SCXMLDocument instance''' self.doc = document self.invokeId = invokeId transition = Transition(document.rootState) transition.target = document.rootState.initial transition.exe = document.rootState.initial.exe self.executeTransitionContent([transition]) self.enterStates([transition]) def mainEventLoop(self): while self.running: enabledTransitions = None stable = False # now take any newly enabled null transitions and any transitions triggered by internal events while self.running and not stable: enabledTransitions = self.selectEventlessTransitions() if not enabledTransitions: if self.internalQueue.empty(): stable = True else: internalEvent = self.internalQueue.get() # this call returns immediately if no event is available self.logger.info("internal event found: %s", internalEvent.name) self.dm["__event"] = internalEvent enabledTransitions = self.selectTransitions(internalEvent) if enabledTransitions: self.microstep(enabledTransitions) # eventlet.greenthread.sleep() eventlet.greenthread.sleep() for state in self.statesToInvoke: for inv in state.invoke: inv.invoke(inv) self.statesToInvoke.clear() if not self.internalQueue.empty(): continue externalEvent = self.externalQueue.get() # this call blocks until an event is available if externalEvent.name == "cancel.invoke.%s" % self.dm.sessionid: continue self.logger.info("external event found: %s", externalEvent.name) self.dm["__event"] = externalEvent for state in self.configuration: for inv in state.invoke: if inv.invokeid == externalEvent.invokeid: # event is the result of an <invoke> in this state self.applyFinalize(inv, externalEvent) if inv.autoforward: inv.send(externalEvent) enabledTransitions = self.selectTransitions(externalEvent) if enabledTransitions: self.microstep(enabledTransitions) # if we get here, we have reached a top-level final state or some external entity has set running to False self.exitInterpreter() def exitInterpreter(self): statesToExit = sorted(self.configuration, key=exitOrder) for s in statesToExit: for content in s.onexit: self.executeContent(content) for inv in s.invoke: self.cancelInvoke(inv) self.configuration.delete(s) if isFinalState(s) and isScxmlState(s.parent): if self.invokeId and self.parentId and self.parentId in self.dm.sessions: self.send(["done", "invoke", self.invokeId], s.donedata(), self.invokeId, self.dm.sessions[self.parentId].interpreter.externalQueue) self.logger.info("Exiting interpreter") dispatcher.send("signal_exit", self, final=s.id) return dispatcher.send("signal_exit", self, final=None) def selectEventlessTransitions(self): enabledTransitions = OrderedSet() atomicStates = filter(isAtomicState, self.configuration) atomicStates = sorted(atomicStates, key=documentOrder) for state in atomicStates: done = False for s in [state] + getProperAncestors(state, None): if done: break for t in s.transition: if not t.event and self.conditionMatch(t): enabledTransitions.add(t) done = True break filteredTransitions = self.filterPreempted(enabledTransitions) return filteredTransitions def selectTransitions(self, event): enabledTransitions = OrderedSet() atomicStates = filter(isAtomicState, self.configuration) atomicStates = sorted(atomicStates, key=documentOrder) for state in atomicStates: done = False for s in [state] + getProperAncestors(state, None): if done: break for t in s.transition: if t.event and nameMatch(t.event, event.name.split(".")) and self.conditionMatch(t): enabledTransitions.add(t) done = True break filteredTransitions = self.filterPreempted(enabledTransitions) return filteredTransitions def preemptsTransition(self, t, t2): if self.isType1(t): return False elif self.isType2(t) and self.isType3(t2): return True elif self.isType3(t): return True return False def getCommonParallel(self, states): ancestors = set(getProperAncestors(states[0], None)) for s in states[1:]: ancestors = ancestors.intersection(getProperAncestors(s, None)) if ancestors: return sorted(ancestors, key=exitOrder)[0] def isType1(self, t): return not t.target def isType2(self, t): source = t.source if t.type == "internal" else t.source.parent p = self.getCommonParallel([source] + self.getTargetStates(t.target)) return not isScxmlState(p) def isType3(self, t): return not self.isType2(t) and not self.isType1(t) def filterPreempted(self, enabledTransitions): filteredTransitions = [] for t in enabledTransitions: # does any t2 in filteredTransitions preempt t? if not, add t to filteredTransitions if not any(map(lambda t2: self.preemptsTransition(t2, t), filteredTransitions)): filteredTransitions.append(t) return OrderedSet(filteredTransitions) def microstep(self, enabledTransitions): self.exitStates(enabledTransitions) self.executeTransitionContent(enabledTransitions) self.enterStates(enabledTransitions) self.logger.info("new config: {" + ", ".join([s.id for s in self.configuration if s.id != "__main__"]) + "}") def exitStates(self, enabledTransitions): statesToExit = OrderedSet() for t in enabledTransitions: if t.target: tstates = self.getTargetStates(t.target) if t.type == "internal" and isCompoundState(t.source) and all(map(lambda s: isDescendant(s,t.source), tstates)): ancestor = t.source else: ancestor = self.findLCA([t.source] + tstates) for s in self.configuration: if isDescendant(s,ancestor): statesToExit.add(s) for s in statesToExit: self.statesToInvoke.delete(s) statesToExit.sort(key=exitOrder) for s in statesToExit: for h in s.history: if h.type == "deep": f = lambda s0: isAtomicState(s0) and isDescendant(s0,s) else: f = lambda s0: s0.parent == s self.historyValue[h.id] = filter(f,self.configuration) #+ s.parent for s in statesToExit: for content in s.onexit: self.executeContent(content) for inv in s.invoke: self.cancelInvoke(inv) self.configuration.delete(s) def cancelInvoke(self, inv): inv.cancel() def executeTransitionContent(self, enabledTransitions): for t in enabledTransitions: self.executeContent(t) def enterStates(self, enabledTransitions): statesToEnter = OrderedSet() statesForDefaultEntry = OrderedSet() for t in enabledTransitions: if t.target: tstates = self.getTargetStates(t.target) if t.type == "internal" and isCompoundState(t.source) and all(map(lambda s: isDescendant(s,t.source), tstates)): ancestor = t.source else: ancestor = self.findLCA([t.source] + tstates) for s in tstates: self.addStatesToEnter(s,statesToEnter,statesForDefaultEntry) for s in tstates: for anc in getProperAncestors(s,ancestor): statesToEnter.add(anc) if isParallelState(anc): for child in getChildStates(anc): if not any(map(lambda s: isDescendant(s,child), statesToEnter)): self.addStatesToEnter(child, statesToEnter,statesForDefaultEntry) statesToEnter.sort(key=enterOrder) for s in statesToEnter: self.statesToInvoke.add(s) self.configuration.add(s) if self.doc.binding == "late" and s.isFirstEntry: s.initDatamodel() s.isFirstEntry = False for content in s.onentry: self.executeContent(content) if s in statesForDefaultEntry: self.executeContent(s.initial) if isFinalState(s): parent = s.parent grandparent = parent.parent self.internalQueue.put(Event(["done", "state", parent.id], s.donedata())) if isParallelState(grandparent): if all(map(self.isInFinalState, getChildStates(grandparent))): self.internalQueue.put(Event(["done", "state", grandparent.id])) for s in self.configuration: if isFinalState(s) and isScxmlState(s.parent): self.running = False; def addStatesToEnter(self, state,statesToEnter,statesForDefaultEntry): if isHistoryState(state): if state.id in self.historyValue: for s in self.historyValue[state.id]: self.addStatesToEnter(s, statesToEnter, statesForDefaultEntry) for anc in getProperAncestors(s,state): statesToEnter.add(anc) else: for t in state.transition: for s in self.getTargetStates(t.target): self.addStatesToEnter(s, statesToEnter, statesForDefaultEntry) else: statesToEnter.add(state) if isCompoundState(state): statesForDefaultEntry.add(state) for s in self.getTargetStates(state.initial): self.addStatesToEnter(s, statesToEnter, statesForDefaultEntry) elif isParallelState(state): for s in getChildStates(state): self.addStatesToEnter(s,statesToEnter,statesForDefaultEntry) def isInFinalState(self, s): if isCompoundState(s): return any(map(lambda s: isFinalState(s) and s in self.configuration, getChildStates(s))) elif isParallelState(s): return all(map(self.isInFinalState, getChildStates(s))) else: return False def findLCA(self, stateList): for anc in filter(isCompoundState, getProperAncestors(stateList[0], None)): # for anc in getProperAncestors(stateList[0], None): if all(map(lambda(s): isDescendant(s,anc), stateList[1:])): return anc
class Interpreter(object): ''' The class repsonsible for keeping track of the execution of the statemachine. ''' def __init__(self): self.running = True self.exited = False self.cancelled = False self.configuration = OrderedSet() self.internalQueue = Queue() self.externalQueue = Queue() self.statesToInvoke = OrderedSet() self.historyValue = {} self.dm = None self.invokeId = None self.parentId = None self.logger = None def interpret(self, document, invokeId=None): '''Initializes the interpreter given an SCXMLDocument instance''' self.doc = document self.invokeId = invokeId transition = Transition(document.rootState) transition.target = document.rootState.initial transition.exe = document.rootState.initial.exe self.executeTransitionContent([transition]) self.enterStates([transition]) def mainEventLoop(self): while self.running: enabledTransitions = None stable = False # now take any newly enabled null transitions and any transitions triggered by internal events while self.running and not stable: enabledTransitions = self.selectEventlessTransitions() if not enabledTransitions: if self.internalQueue.empty(): stable = True else: internalEvent = self.internalQueue.get() # this call returns immediately if no event is available self.logger.info("internal event found: %s", internalEvent.name) self.dm["__event"] = internalEvent enabledTransitions = self.selectTransitions(internalEvent) if enabledTransitions: self.microstep(enabledTransitions) # eventlet.greenthread.sleep() eventlet.greenthread.sleep() for state in self.statesToInvoke: for inv in state.invoke: inv.invoke(inv) self.statesToInvoke.clear() if not self.internalQueue.empty(): continue externalEvent = self.externalQueue.get() # this call blocks until an event is available # if externalEvent.name == "cancel.invoke.%s" % self.dm.sessionid: # continue # our parent session also might cancel us. The mechanism for this is platform specific, if isCancelEvent(externalEvent): self.running = False continue self.logger.info("external event found: %s", externalEvent.name) self.dm["__event"] = externalEvent for state in self.configuration: for inv in state.invoke: if inv.invokeid == externalEvent.invokeid: # event is the result of an <invoke> in this state self.applyFinalize(inv, externalEvent) if inv.autoforward: inv.send(externalEvent) enabledTransitions = self.selectTransitions(externalEvent) if enabledTransitions: self.microstep(enabledTransitions) # if we get here, we have reached a top-level final state or some external entity has set running to False self.exitInterpreter() def exitInterpreter(self): statesToExit = sorted(self.configuration, key=exitOrder) for s in statesToExit: for content in s.onexit: self.executeContent(content) for inv in s.invoke: self.cancelInvoke(inv) self.configuration.delete(s) if isFinalState(s) and isScxmlState(s.parent): if self.invokeId and self.parentId and self.parentId in self.dm.sessions: self.send(["done", "invoke", self.invokeId], s.donedata(), self.invokeId, self.dm.sessions[self.parentId].interpreter.externalQueue) self.logger.info("Exiting interpreter") dispatcher.send("signal_exit", self, final=s.id) self.exited = True return self.exited = True dispatcher.send("signal_exit", self, final=None) def selectEventlessTransitions(self): enabledTransitions = OrderedSet() atomicStates = filter(isAtomicState, self.configuration) atomicStates = sorted(atomicStates, key=documentOrder) for state in atomicStates: done = False for s in [state] + getProperAncestors(state, None): if done: break for t in s.transition: if not t.event and self.conditionMatch(t): enabledTransitions.add(t) done = True break filteredTransitions = self.filterPreempted(enabledTransitions) return filteredTransitions def selectTransitions(self, event): enabledTransitions = OrderedSet() atomicStates = filter(isAtomicState, self.configuration) atomicStates = sorted(atomicStates, key=documentOrder) for state in atomicStates: done = False for s in [state] + getProperAncestors(state, None): if done: break for t in s.transition: if t.event and nameMatch(t.event, event.name.split(".")) and self.conditionMatch(t): enabledTransitions.add(t) done = True break filteredTransitions = self.filterPreempted(enabledTransitions) return filteredTransitions def preemptsTransition(self, t, t2): if self.isType1(t): return False elif self.isType2(t) and self.isType3(t2): return True elif self.isType3(t): return True return False def findLCPA(self, states): ''' Gets the least common parallel ancestor of states. Just like findLCA but only for parallel states. ''' for anc in filter(isParallelState, getProperAncestors(states[0], None)): if all(map(lambda(s): isDescendant(s,anc), states[1:])): return anc