def test_connection_pooling(self):
with patch('swift.common.memcached.socket') as mock_module:
# patch socket, stub socket.socket, mock sock
mock_sock = mock_module.socket.return_value
# track clients waiting for connections
connected = []
connections = Queue()
errors = []
def wait_connect(addr):
connected.append(addr)
sleep(0.1) # yield
val = connections.get()
if val is not None:
errors.append(val)
mock_sock.connect = wait_connect
memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'],
connect_timeout=10)
# sanity
self.assertEquals(1, len(memcache_client._client_cache))
for server, pool in memcache_client._client_cache.items():
self.assertEqual(2, pool.max_size)
# make 10 requests "at the same time"
p = GreenPool()
for i in range(10):
p.spawn(memcache_client.set, 'key', 'value')
for i in range(3):
sleep(0.1)
self.assertEqual(2, len(connected))
# give out a connection
connections.put(None)
# at this point, only one connection should have actually been
# created, the other is in the creation step, and the rest of the
# clients are not attempting to connect. we let this play out a
# bit to verify.
for i in range(3):
sleep(0.1)
self.assertEqual(2, len(connected))
# finish up, this allows the final connection to be created, so
# that all the other clients can use the two existing connections
# and no others will be created.
connections.put(None)
connections.put('nono')
self.assertEqual(2, len(connected))
p.waitall()
self.assertEqual(2, len(connected))
self.assertEqual(0, len(errors),
"A client was allowed a third connection")
connections.get_nowait()
self.assertTrue(connections.empty())
def test_connection_pooling(self):
with patch('swift.common.memcached.socket') as mock_module:
# patch socket, stub socket.socket, mock sock
mock_sock = mock_module.socket.return_value
# track clients waiting for connections
connected = []
connections = Queue()
errors = []
def wait_connect(addr):
connected.append(addr)
sleep(0.1) # yield
val = connections.get()
if val is not None:
errors.append(val)
mock_sock.connect = wait_connect
memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'],
connect_timeout=10)
# sanity
self.assertEquals(1, len(memcache_client._client_cache))
for server, pool in memcache_client._client_cache.items():
self.assertEqual(2, pool.max_size)
# make 10 requests "at the same time"
p = GreenPool()
for i in range(10):
p.spawn(memcache_client.set, 'key', 'value')
for i in range(3):
sleep(0.1)
self.assertEqual(2, len(connected))
# give out a connection
connections.put(None)
# at this point, only one connection should have actually been
# created, the other is in the creation step, and the rest of the
# clients are not attempting to connect. we let this play out a
# bit to verify.
for i in range(3):
sleep(0.1)
self.assertEqual(2, len(connected))
# finish up, this allows the final connection to be created, so
# that all the other clients can use the two existing connections
# and no others will be created.
connections.put(None)
connections.put('nono')
self.assertEqual(2, len(connected))
p.waitall()
self.assertEqual(2, len(connected))
self.assertEqual(0, len(errors),
"A client was allowed a third connection")
connections.get_nowait()
self.assertTrue(connections.empty())
def main():
args = options()
global ACCOUNT, PROXY, QUEUE, NS, VERBOSE, TIMEOUT
global COUNTERS, ELECTIONS
ACCOUNT = args.account
NS = args.namespace
VERBOSE = args.verbose
TIMEOUT = args.timeout
PROXY = ObjectStorageApi(NS)
ELECTIONS = AtomicInteger()
num_worker_threads = int(args.max_worker)
print("Using %d workers" % num_worker_threads)
total_objects = {'size': 0, 'files': 0, 'elapsed': 0}
total_containers = {'size': 0, 'files': 0, 'elapsed': 0}
for path in args.path:
path = path.rstrip('/')
if '/' in path:
bucket, path = path.split('/', 1)
else:
bucket = path
path = ""
containers = []
QUEUE = Queue()
pool = eventlet.GreenPool(num_worker_threads)
for i in range(num_worker_threads):
pool.spawn(worker_objects)
COUNTERS = AtomicInteger()
_bucket = container_hierarchy(bucket, path)
# we don't use placeholders, we use prefix path as prefix
for entry in full_list(prefix=container_hierarchy(bucket, path)):
name, _files, _size, _ = entry
if name != _bucket and not name.startswith(_bucket + '%2F'):
continue
if _files:
QUEUE.put(name)
containers.append(name)
# we have to wait all objects
print("Waiting flush of objects")
report = args.report
while not QUEUE.empty():
ts = time.time()
while time.time() - ts < report and not QUEUE.empty():
time.sleep(1)
diff = time.time() - ts
val = COUNTERS.reset()
elections = ELECTIONS.reset()
print("Objects: %5.2f / Size: %5.2f" %
(val[0] / diff, val[1] / diff),
"Elections failed: %5.2f/s total: %d" %
(elections[0] / diff, ELECTIONS.total()[0]),
" " * 20,
end='\r')
sys.stdout.flush()
print("Waiting end of workers")
QUEUE.join()
val = COUNTERS.total()
total_objects['files'] += val[0]
total_objects['size'] += val[1]
total_objects['elapsed'] += COUNTERS.time()
COUNTERS = AtomicInteger()
QUEUE = Queue()
for i in range(num_worker_threads):
pool.spawn(worker_container)
print("We have to delete", len(containers), "containers")
for container in containers:
QUEUE.put(container)
while not QUEUE.empty():
ts = time.time()
while time.time() - ts < report and not QUEUE.empty():
time.sleep(1)
diff = time.time() - ts
val = COUNTERS.reset()
elections = ELECTIONS.reset()
print("Containers: %5.2f" % (val[0] / diff),
"Elections failed: %5.2f/s total: %d" %
(elections[0] / diff, ELECTIONS.total()[0]),
" " * 20,
end='\r')
sys.stdout.flush()
QUEUE.join()
val = COUNTERS.total()
total_containers['files'] += val[0]
total_containers['size'] += val[1]
total_containers['elapsed'] += COUNTERS.time()
print("""
Objects:
- ran during {o[elapsed]:5.2f}
- {o[files]} objects removed (size {size})
- {o_file_avg:5.2f} objects/s ({o_size_avg} avg. size/s)
""".format(o=total_objects,
size=show(total_objects['size'], True),
o_file_avg=total_objects['files'] / total_objects['elapsed'],
o_size_avg=show(total_objects['size'] / total_objects['elapsed'],
True)))
print("""
Containers:
- ran during {o[elapsed]:5.2f}
- {o[files]} containers
- {o_file_avg:5.2f} containers/s
""".format(o=total_containers,
o_file_avg=total_containers['files'] / total_containers['elapsed']))
print("Elections failed: %d" % ELECTIONS.total()[0])
class Interpreter(object):
'''
The class repsonsible for keeping track of the execution of the
statemachine.
'''
def __init__(self):
self.running = True
self.configuration = OrderedSet()
self.internalQueue = Queue()
self.externalQueue = Queue()
self.statesToInvoke = OrderedSet()
self.historyValue = {}
self.dm = None
self.invokeId = None
self.parentId = None
self.logger = None
def interpret(self, document, invokeId=None):
'''Initializes the interpreter given an SCXMLDocument instance'''
self.doc = document
self.invokeId = invokeId
transition = Transition(document.rootState)
transition.target = document.rootState.initial
transition.exe = document.rootState.initial.exe
self.executeTransitionContent([transition])
self.enterStates([transition])
def mainEventLoop(self):
while self.running:
enabledTransitions = None
stable = False
# now take any newly enabled null transitions and any transitions triggered by internal events
while self.running and not stable:
enabledTransitions = self.selectEventlessTransitions()
if not enabledTransitions:
if self.internalQueue.empty():
stable = True
else:
internalEvent = self.internalQueue.get() # this call returns immediately if no event is available
self.logger.info("internal event found: %s", internalEvent.name)
self.dm["__event"] = internalEvent
enabledTransitions = self.selectTransitions(internalEvent)
if enabledTransitions:
self.microstep(enabledTransitions)
# eventlet.greenthread.sleep()
eventlet.greenthread.sleep()
for state in self.statesToInvoke:
for inv in state.invoke:
inv.invoke(inv)
self.statesToInvoke.clear()
if not self.internalQueue.empty():
continue
externalEvent = self.externalQueue.get() # this call blocks until an event is available
if externalEvent.name == "cancel.invoke.%s" % self.dm.sessionid:
continue
self.logger.info("external event found: %s", externalEvent.name)
self.dm["__event"] = externalEvent
for state in self.configuration:
for inv in state.invoke:
if inv.invokeid == externalEvent.invokeid: # event is the result of an <invoke> in this state
self.applyFinalize(inv, externalEvent)
if inv.autoforward:
inv.send(externalEvent)
enabledTransitions = self.selectTransitions(externalEvent)
if enabledTransitions:
self.microstep(enabledTransitions)
# if we get here, we have reached a top-level final state or some external entity has set running to False
self.exitInterpreter()
def exitInterpreter(self):
statesToExit = sorted(self.configuration, key=exitOrder)
for s in statesToExit:
for content in s.onexit:
self.executeContent(content)
for inv in s.invoke:
self.cancelInvoke(inv)
self.configuration.delete(s)
if isFinalState(s) and isScxmlState(s.parent):
if self.invokeId and self.parentId and self.parentId in self.dm.sessions:
self.send(["done", "invoke", self.invokeId], s.donedata(), self.invokeId, self.dm.sessions[self.parentId].interpreter.externalQueue)
self.logger.info("Exiting interpreter")
dispatcher.send("signal_exit", self, final=s.id)
return
dispatcher.send("signal_exit", self, final=None)
def selectEventlessTransitions(self):
enabledTransitions = OrderedSet()
atomicStates = filter(isAtomicState, self.configuration)
atomicStates = sorted(atomicStates, key=documentOrder)
for state in atomicStates:
done = False
for s in [state] + getProperAncestors(state, None):
if done: break
for t in s.transition:
if not t.event and self.conditionMatch(t):
enabledTransitions.add(t)
done = True
break
filteredTransitions = self.filterPreempted(enabledTransitions)
return filteredTransitions
def selectTransitions(self, event):
enabledTransitions = OrderedSet()
atomicStates = filter(isAtomicState, self.configuration)
atomicStates = sorted(atomicStates, key=documentOrder)
for state in atomicStates:
done = False
for s in [state] + getProperAncestors(state, None):
if done: break
for t in s.transition:
if t.event and nameMatch(t.event, event.name.split(".")) and self.conditionMatch(t):
enabledTransitions.add(t)
done = True
break
filteredTransitions = self.filterPreempted(enabledTransitions)
return filteredTransitions
def preemptsTransition(self, t, t2):
if self.isType1(t): return False
elif self.isType2(t) and self.isType3(t2): return True
elif self.isType3(t): return True
return False
def getCommonParallel(self, states):
ancestors = set(getProperAncestors(states[0], None))
for s in states[1:]:
ancestors = ancestors.intersection(getProperAncestors(s, None))
if ancestors:
return sorted(ancestors, key=exitOrder)[0]
def isType1(self, t):
return not t.target
def isType2(self, t):
source = t.source if t.type == "internal" else t.source.parent
p = self.getCommonParallel([source] + self.getTargetStates(t.target))
return not isScxmlState(p)
def isType3(self, t):
return not self.isType2(t) and not self.isType1(t)
def filterPreempted(self, enabledTransitions):
filteredTransitions = []
for t in enabledTransitions:
# does any t2 in filteredTransitions preempt t? if not, add t to filteredTransitions
if not any(map(lambda t2: self.preemptsTransition(t2, t), filteredTransitions)):
filteredTransitions.append(t)
return OrderedSet(filteredTransitions)
def microstep(self, enabledTransitions):
self.exitStates(enabledTransitions)
self.executeTransitionContent(enabledTransitions)
self.enterStates(enabledTransitions)
self.logger.info("new config: {" + ", ".join([s.id for s in self.configuration if s.id != "__main__"]) + "}")
def exitStates(self, enabledTransitions):
statesToExit = OrderedSet()
for t in enabledTransitions:
if t.target:
tstates = self.getTargetStates(t.target)
if t.type == "internal" and isCompoundState(t.source) and all(map(lambda s: isDescendant(s,t.source), tstates)):
ancestor = t.source
else:
ancestor = self.findLCA([t.source] + tstates)
for s in self.configuration:
if isDescendant(s,ancestor):
statesToExit.add(s)
for s in statesToExit:
self.statesToInvoke.delete(s)
statesToExit.sort(key=exitOrder)
for s in statesToExit:
for h in s.history:
if h.type == "deep":
f = lambda s0: isAtomicState(s0) and isDescendant(s0,s)
else:
f = lambda s0: s0.parent == s
self.historyValue[h.id] = filter(f,self.configuration) #+ s.parent
for s in statesToExit:
for content in s.onexit:
self.executeContent(content)
for inv in s.invoke:
self.cancelInvoke(inv)
self.configuration.delete(s)
def cancelInvoke(self, inv):
inv.cancel()
def executeTransitionContent(self, enabledTransitions):
for t in enabledTransitions:
self.executeContent(t)
def enterStates(self, enabledTransitions):
statesToEnter = OrderedSet()
statesForDefaultEntry = OrderedSet()
for t in enabledTransitions:
if t.target:
tstates = self.getTargetStates(t.target)
if t.type == "internal" and isCompoundState(t.source) and all(map(lambda s: isDescendant(s,t.source), tstates)):
ancestor = t.source
else:
ancestor = self.findLCA([t.source] + tstates)
for s in tstates:
self.addStatesToEnter(s,statesToEnter,statesForDefaultEntry)
for s in tstates:
for anc in getProperAncestors(s,ancestor):
statesToEnter.add(anc)
if isParallelState(anc):
for child in getChildStates(anc):
if not any(map(lambda s: isDescendant(s,child), statesToEnter)):
self.addStatesToEnter(child, statesToEnter,statesForDefaultEntry)
statesToEnter.sort(key=enterOrder)
for s in statesToEnter:
self.statesToInvoke.add(s)
self.configuration.add(s)
if self.doc.binding == "late" and s.isFirstEntry:
s.initDatamodel()
s.isFirstEntry = False
for content in s.onentry:
self.executeContent(content)
if s in statesForDefaultEntry:
self.executeContent(s.initial)
if isFinalState(s):
parent = s.parent
grandparent = parent.parent
self.internalQueue.put(Event(["done", "state", parent.id], s.donedata()))
if isParallelState(grandparent):
if all(map(self.isInFinalState, getChildStates(grandparent))):
self.internalQueue.put(Event(["done", "state", grandparent.id]))
for s in self.configuration:
if isFinalState(s) and isScxmlState(s.parent):
self.running = False;
def addStatesToEnter(self, state,statesToEnter,statesForDefaultEntry):
if isHistoryState(state):
if state.id in self.historyValue:
for s in self.historyValue[state.id]:
self.addStatesToEnter(s, statesToEnter, statesForDefaultEntry)
for anc in getProperAncestors(s,state):
statesToEnter.add(anc)
else:
for t in state.transition:
for s in self.getTargetStates(t.target):
self.addStatesToEnter(s, statesToEnter, statesForDefaultEntry)
else:
statesToEnter.add(state)
if isCompoundState(state):
statesForDefaultEntry.add(state)
for s in self.getTargetStates(state.initial):
self.addStatesToEnter(s, statesToEnter, statesForDefaultEntry)
elif isParallelState(state):
for s in getChildStates(state):
self.addStatesToEnter(s,statesToEnter,statesForDefaultEntry)
def isInFinalState(self, s):
if isCompoundState(s):
return any(map(lambda s: isFinalState(s) and s in self.configuration, getChildStates(s)))
elif isParallelState(s):
return all(map(self.isInFinalState, getChildStates(s)))
else:
return False
def findLCA(self, stateList):
for anc in filter(isCompoundState, getProperAncestors(stateList[0], None)):
# for anc in getProperAncestors(stateList[0], None):
if all(map(lambda(s): isDescendant(s,anc), stateList[1:])):
return anc
class Interpreter(object):
'''
The class repsonsible for keeping track of the execution of the
statemachine.
'''
def __init__(self):
self.running = True
self.exited = False
self.cancelled = False
self.configuration = OrderedSet()
self.internalQueue = Queue()
self.externalQueue = Queue()
self.statesToInvoke = OrderedSet()
self.historyValue = {}
self.dm = None
self.invokeId = None
self.parentId = None
self.logger = None
def interpret(self, document, invokeId=None):
'''Initializes the interpreter given an SCXMLDocument instance'''
self.doc = document
self.invokeId = invokeId
transition = Transition(document.rootState)
transition.target = document.rootState.initial
transition.exe = document.rootState.initial.exe
self.executeTransitionContent([transition])
self.enterStates([transition])
def mainEventLoop(self):
while self.running:
enabledTransitions = None
stable = False
# now take any newly enabled null transitions and any transitions triggered by internal events
while self.running and not stable:
enabledTransitions = self.selectEventlessTransitions()
if not enabledTransitions:
if self.internalQueue.empty():
stable = True
else:
internalEvent = self.internalQueue.get() # this call returns immediately if no event is available
self.logger.info("internal event found: %s", internalEvent.name)
self.dm["__event"] = internalEvent
enabledTransitions = self.selectTransitions(internalEvent)
if enabledTransitions:
self.microstep(enabledTransitions)
# eventlet.greenthread.sleep()
eventlet.greenthread.sleep()
for state in self.statesToInvoke:
for inv in state.invoke:
inv.invoke(inv)
self.statesToInvoke.clear()
if not self.internalQueue.empty():
continue
externalEvent = self.externalQueue.get() # this call blocks until an event is available
# if externalEvent.name == "cancel.invoke.%s" % self.dm.sessionid:
# continue
# our parent session also might cancel us. The mechanism for this is platform specific,
if isCancelEvent(externalEvent):
self.running = False
continue
self.logger.info("external event found: %s", externalEvent.name)
self.dm["__event"] = externalEvent
for state in self.configuration:
for inv in state.invoke:
if inv.invokeid == externalEvent.invokeid: # event is the result of an <invoke> in this state
self.applyFinalize(inv, externalEvent)
if inv.autoforward:
inv.send(externalEvent)
enabledTransitions = self.selectTransitions(externalEvent)
if enabledTransitions:
self.microstep(enabledTransitions)
# if we get here, we have reached a top-level final state or some external entity has set running to False
self.exitInterpreter()
def exitInterpreter(self):
statesToExit = sorted(self.configuration, key=exitOrder)
for s in statesToExit:
for content in s.onexit:
self.executeContent(content)
for inv in s.invoke:
self.cancelInvoke(inv)
self.configuration.delete(s)
if isFinalState(s) and isScxmlState(s.parent):
if self.invokeId and self.parentId and self.parentId in self.dm.sessions:
self.send(["done", "invoke", self.invokeId], s.donedata(), self.invokeId, self.dm.sessions[self.parentId].interpreter.externalQueue)
self.logger.info("Exiting interpreter")
dispatcher.send("signal_exit", self, final=s.id)
self.exited = True
return
self.exited = True
dispatcher.send("signal_exit", self, final=None)
def selectEventlessTransitions(self):
enabledTransitions = OrderedSet()
atomicStates = filter(isAtomicState, self.configuration)
atomicStates = sorted(atomicStates, key=documentOrder)
for state in atomicStates:
done = False
for s in [state] + getProperAncestors(state, None):
if done: break
for t in s.transition:
if not t.event and self.conditionMatch(t):
enabledTransitions.add(t)
done = True
break
filteredTransitions = self.filterPreempted(enabledTransitions)
return filteredTransitions
def selectTransitions(self, event):
enabledTransitions = OrderedSet()
atomicStates = filter(isAtomicState, self.configuration)
atomicStates = sorted(atomicStates, key=documentOrder)
for state in atomicStates:
done = False
for s in [state] + getProperAncestors(state, None):
if done: break
for t in s.transition:
if t.event and nameMatch(t.event, event.name.split(".")) and self.conditionMatch(t):
enabledTransitions.add(t)
done = True
break
filteredTransitions = self.filterPreempted(enabledTransitions)
return filteredTransitions
def preemptsTransition(self, t, t2):
if self.isType1(t): return False
elif self.isType2(t) and self.isType3(t2): return True
elif self.isType3(t): return True
return False
def findLCPA(self, states):
'''
Gets the least common parallel ancestor of states.
Just like findLCA but only for parallel states.
'''
for anc in filter(isParallelState, getProperAncestors(states[0], None)):
if all(map(lambda(s): isDescendant(s,anc), states[1:])):
return anc