def disconnectAll(self):
reactor.runUntilCurrent()
def _disconnectAll():
for subscriber in self.connections_ready:
yield subscriber.transport.loseConnection()
task.cooperate(_disconnectAll())
def main():
print "Starting up..."
state = SharedState()
bot = BotFactory(config, state)
reactor.connectSSL(config.get('server'), config.get('port'), bot, ssl.ClientContextFactory())
cooperate(SlackBot(state, config).listen_to_slack())
reactor.run()
def setup(self):
sb = self.blackboard.bot_standing_on_block(self.blackboard.bot_object)
while sb is None:
yield utils.reactor_break()
sb = self.blackboard.bot_standing_on_block(self.blackboard.bot_object)
else:
if self.travel_multiple_goals is not None:
d = cooperate(AStarMultiCoords(dimension=self.blackboard.dimension,
start_coords=sb.coords,
goal_coords=self.travel_coords,
multiple_goals=self.travel_multiple_goals)).whenDone()
elif self.travel_coords is not None:
d = cooperate(AStarCoords(dimension=self.blackboard.dimension,
start_coords=sb.coords,
goal_coords=self.travel_coords)).whenDone()
else:
d = cooperate(AStarBBCol(dimension=self.blackboard.dimension,
start_coords=sb.coords,
bb=self.travel_bb)).whenDone()
d.addErrback(logbot.exit_on_error)
astar = yield d
if astar.path is not None:
current_start = self.blackboard.bot_standing_on_block(self.blackboard.bot_object)
if sb == current_start:
self.path = astar.path
if self.shorten_path_by > 0:
self.path = self.path[self.shorten_path_by:]
self.start_coords = current_start.coords
def update_chunks(self):
x, y, z = self.location.pos.to_block()
x, chaff, z, chaff = split_coords(x, z)
new = set((i + x, j + z) for i, j in circle)
old = set(self.chunks.iterkeys())
added = new - old
discarded = old - new
# Perhaps some explanation is in order.
# The cooperate() function iterates over the iterable it is fed,
# without tying up the reactor, by yielding after each iteration. The
# inner part of the generator expression generates all of the chunks
# around the currently needed chunk, and it sorts them by distance to
# the current chunk. The end result is that we load chunks one-by-one,
# nearest to furthest, without stalling other clients.
if self.chunk_tasks:
for task in self.chunk_tasks:
try:
task.stop()
except (TaskDone, TaskFailed):
pass
self.chunk_tasks = [
cooperate(
self.enable_chunk(i, j) for i, j in
sorted(added, key=lambda t: (t[0] - x)**2 + (t[1] - z)**2)
),
cooperate(self.disable_chunk(i, j) for i, j in discarded)
]
def pushInfoToSubscribers(self):
# Generate the json doc
vars = {}
for subscriber in self.connections_ready:
subscriber_vars = subscriber.vars.copy()
subscriber_vars['port'] = subscriber.id + 12000
subscriber_vars['host'] = subscriber.transport.getPeer().host
vars[subscriber.id] = subscriber_vars
json_vars = json.dumps(vars)
del vars
msg("Pushing a %d bytes long json doc." % len(json_vars))
# Send the json doc to the subscribers
task.cooperate(self._sendLineToAllGenerator(json_vars))
def _disconnect(self, receipt, failure, timeout):
if failure:
self._disconnectReason = failure
self.log.info('Disconnecting ...%s' % ('' if (not failure) else ('[reason=%s]' % failure)))
protocol = self._protocol
try:
# notify that we are ready to disconnect after outstanding messages are ack'ed
if self._messages:
self.log.info('Waiting for outstanding message handlers to finish ... [timeout=%s]' % timeout)
try:
yield task.cooperate(iter([wait(handler, timeout, StompCancelledError('Going down to disconnect now')) for handler in self._messages.values()])).whenDone()
except StompCancelledError as e:
self._disconnectReason = StompCancelledError('Handlers did not finish in time.')
else:
self.log.info('All handlers complete. Resuming disconnect ...')
if self.session.state == self.session.CONNECTED:
frame = self.session.disconnect(receipt)
try:
self.sendFrame(frame)
except Exception as e:
self._disconnectReason = StompConnectionError('Could not send %s. [%s]' % (frame.info(), e))
try:
yield self._waitForReceipt(receipt)
except StompCancelledError:
self._disconnectReason = StompCancelledError('Receipt for disconnect command did not arrive on time.')
protocol.loseConnection()
except Exception as e:
self._disconnectReason = e
def connectionMade(self):
print("Connected protocol")
self.queue = Queue.Queue()
#self.transport.registerProducer(QueueProducer(self.transport), False)
self.sendLine("3,0,0")
gen = DoodleQueueWorkload(self.transport.reactor, self)
self.task = cooperate(gen)
def main(reactor, *argv):
parameters = ConnectionParameters.fromCommandLine(reactor, argv)
endpoint = parameters.endpointForCommand(b"/bin/cat")
done = []
factory = Factory()
factory.protocol = Protocol
d = endpoint.connect(factory)
def gotConnection(proto):
conn = proto.transport.conn
for i in range(50):
factory = Factory()
factory.protocol = PrinterProtocol
factory.done = Deferred()
done.append(factory.done)
e = SSHCommandClientEndpoint.existingConnection(
conn, b"/bin/echo %d" % (i,))
yield e.connect(factory)
d.addCallback(gotConnection)
d.addCallback(lambda work: cooperate(work).whenDone())
d.addCallback(lambda ignored: gatherResults(done))
return d
def startStreaming(self):
"""
This should be called by the consumer when the producer is registered.
Start streaming data to the consumer.
"""
self._coopTask = cooperate(self._pull())
def startProducing(self, consumer):
""" Must NOT call registerProducer on the consumer """
self._consumer = consumer
self._iterable = json.JSONEncoder().iterencode(self.body)
self._task = cooperate(self._produce())
d = self._task.whenDone()
return d
def searchIndices(indices, query, period, lastId=None, reverse=False, fields=None, limit=100):
"""
Search the specified indices for events matching the specified query.
:param indices: A list of indices to search.
:type indices: A list of objects implementing :class:`terane.bier.index.IIndex`
:param query: The programmatic query to use for searching the indices.
:type query: An object implementing :class:`terane.bier.searching.IQuery`
:param period: The period within which the search is constrained.
:type period: :class:`terane.bier.searching.Period`
:param lastId: The real key to start iterating from.
:type lastId: :class:`terane.bier.evid.EVID`
:param reverse: If True, then reverse the order of events.
:type reverse: bool
:param fields: If not None, then only return the specified fields of each event.
:type fields: list or None
:param limit: Only returned the specified number of events.
:type limit: int
:returns: A CooperativeTask which contains a Deferred and manages the search task.
:rtype: :class:`twisted.internet.task.CooperativeTask`
"""
start = time.time()
# determine the evids to use as start and end keys
if reverse == False:
startId, endId = period.getRange()
else:
endId, startId = period.getRange()
if lastId != None:
if not lastId in period:
raise SearcherError("lastId %s is not within period" % lastId)
startId = lastId
# search each index separately, then merge the results
try:
searchers = []
postingLists = []
for index in indices:
if not IIndex.providedBy(index):
raise TypeError("index does not implement IIndex")
# we create a copy of the original query, which can possibly be optimized
# with index-specific knowledge.
_query = copy.deepcopy(query)
try:
_query = _query.optimizeMatcher(index)
except NotImplementedError, e:
raise SearcherError(str(e))
logger.debug("optimized query for index '%s': %s" % (index.name, str(_query)))
# if the query optimized out entirely, then skip to the next index
if _query == None:
continue
# get the posting list to iterate through
searcher = index.newSearcher()
if not ISearcher.providedBy(searcher):
raise TypeError("searcher does not implement ISearcher")
postingList = _query.iterMatches(searcher, startId, endId)
if not IPostingList.providedBy(postingList):
raise TypeError("posting list does not implement IPostingList")
searchers.append(searcher)
postingLists.append(postingList)
# return a cooperative task
return cooperate(ResultSet(searchers, postingLists, start, reverse, fields, limit))
def startProducing(self, consumer):
self._consumer = consumer
self._iterable = SmapEncoder().iterencode(self._value)
self._task = cooperate(self._produce())
d = self._task.whenDone()
d.addBoth(self._unregister)
return d
def send_initial_chunk_and_location(self):
bigx, smallx, bigz, smallz = split_coords(self.location.x,
self.location.z)
# Spawn the 25 chunks in a square around the spawn, *before* spawning
# the player. Otherwise, there's a funky Beta 1.2 bug which causes the
# player to not be able to move.
d = cooperate(
self.enable_chunk(i, j)
for i, j in product(
xrange(bigx - 3, bigx + 3),
xrange(bigz - 3, bigz + 3)
)
).whenDone()
# Don't dare send more chunks beyond the initial one until we've
# spawned.
d.addCallback(lambda none: self.update_location())
d.addCallback(lambda none: self.position_changed())
# Send the MOTD.
if self.motd:
packet = make_packet("chat",
message=self.motd.replace("<tagline>", get_motd()))
d.addCallback(lambda none: self.transport.write(packet))
# Finally, start the secondary chunk loop.
d.addCallback(lambda none: self.update_chunks())
def _driveWorker(self, worker, result, testCases, cooperate):
"""
Drive a L{LocalWorkerAMP} instance, iterating the tests and calling
C{run} for every one of them.
@param worker: The L{LocalWorkerAMP} to drive.
@param result: The global L{DistReporter} instance.
@param testCases: The global list of tests to iterate.
@param cooperate: The cooperate function to use, to be customized in
tests.
@type cooperate: C{function}
@return: A C{Deferred} firing when all the tests are finished.
"""
def resultErrback(error, case):
result.original.addFailure(case, error)
return error
def task(case):
d = worker.run(case, result)
d.addErrback(resultErrback, case)
return d
return cooperate(task(case) for case in testCases).whenDone()
def test_onDisconnect_waitForOutstandingMessagesToFinish(self):
config = self.getConfig(StompSpec.VERSION_1_0)
client = async.Stomp(config)
client.add(ReceiptListener(1.0))
# connect
yield client.connect(host=VIRTUALHOST)
yield task.cooperate(
iter([client.send(self.queue, self.frame, receipt="message-%d" % j) for j in range(self.numMsgs)])
).whenDone()
client.subscribe(
self.queue,
{StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL},
listener=SubscriptionListener(self._frameHandler),
)
# wait for disconnect
yield client.disconnected
# reconnect and subscribe again to make sure that all messages in the queue were ack'ed
yield client.connect(host=VIRTUALHOST)
self.timeExpired = False
self.timeoutDelayedCall = reactor.callLater(1, self._timesUp, client) # @UndefinedVariable
client.subscribe(
self.queue,
{StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL},
listener=SubscriptionListener(self._eatOneFrameAndDisconnect),
)
# wait for disconnect
yield client.disconnected
# time should have expired if there were no messages left in the queue
self.assertTrue(self.timeExpired)
def wrapper(n):
def stop_task(unused_deferred):
try: t.stop()
except task.TaskFailed: pass
d = defer.Deferred(canceller=stop_task)
t = task.cooperate(cooperator(func(n), n+1,
yield_interval, d.callback))
return d
def compute(self, node, recheck=False):
if self.incomplete_nodes:
self.incomplete_nodes[node] = recheck
else:
self.incomplete_nodes[node] = recheck
cootask = cooperate(self.do_incomplete_nodes())
d = cootask.whenDone()
d.addErrback(logbot.exit_on_error)
def main(res):
"""
Go main Go!
"""
log.msg("main() fired!")
jobs = DeferredQueue()
ingest = MyProductIngestor()
ingest.jobs = jobs
ldmbridge.LDMProductFactory(ingest)
for _ in range(6):
cooperate(worker(jobs))
lc = LoopingCall(save_current)
lc.start(373, now=False)
lc2 = LoopingCall(service_guard, jobs)
lc2.start(61, now=False)
def begin(self, consumer):
self._consumer=consumer
self._iterable=JSONEncoder().iterencode(self._value)
self._consumer.registerProducer(self, True)
self._task=cooperate(self._produce())
defer=self._task.whenDone()
defer.addBoth(self._unregister)
return defer
def _receivedEvent(self, event):
# run the fields through the filter chain, then reschedule the signal
self._input.getContract().validateEventAfter(event, self.parent._fieldstore)
task = cooperate(EventProcessor(event, self._filters, self.parent._fieldstore))
d = task.whenDone()
d.addCallbacks(self._processedEvent, lambda failure: failure)
d.addErrback(self._errorProcessingEvent)
self._scheduleReceivedEvent()
def main():
logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s")
logging.info('Starting up')
#sensor_pollers, buffer_unloaders, sqlite_unloaders are worker lists
sensor_pollers, buffer_unloaders, sqlite_unloaders = [], [], []
upload_tasks = defer.DeferredQueue()
#Setup monitoring of fake sensors
#Create sensor instance in sensor list
fake_sensors = [fake_sensor()]
#Iterate over sensors
for source in fake_sensors:
name = 'temp%d' % (1,)
#Create a datapointBuffer which will call source_method (with optional source_method_args) to retrieve a datapoint)
dp_buffer = sensorup.datapointBuffer(source_method=source.get_temp_channel,
source_method_args=0,
datastream_id=name)
#Add this new datasource (datapointBuffer instance) to worker lists
#sensorPollAndBuffer: poll for new data and store in an in-memory list
sensor_pollers.append(task.LoopingCall(sensorup.sensorPollAndBuffer, dp_buffer))
#bufferUnloadAndUpload: Flushes in-memory list of datapoints, and attempts uploading to Cosm. Failures go into sqlite.
buffer_unloaders.append(task.LoopingCall(sensorup.bufferUnloadAndUpload, dp_buffer, upload_tasks))
#Collect datapoint every 1s
for poller in sensor_pollers:
poller.start(1.0)
#Upload every 30s
for poller in buffer_unloaders:
poller.start(30.0, False)
#interval for sqlite_unloaders should be sufficiently large as to prevent double uploading
# sqlite_unloader interval > buffer_unloaders interval
# sqlite_unloader interval > client.request_timeout * sensor count
#Flush old datapoints from the database every 86400s
sqlite_purger = task.LoopingCall(sensorup.sqlite_purge)
sqlite_purger.start(86400)
#Only one concurrent upload task
task.cooperate(sensorup.worker(upload_tasks))
reactor.run()
def beginProducing(self, consumer):
""" See: twisted.internet.interfaces.IConsumer """
self._consumer = consumer
self._iterable = json.JSONEncoder().iterencode(self.value)
self._consumer.registerProducer(self, True)
self._task = cooperate(self._produce())
d = self._task.whenDone()
d.addBoth(self._unregister)
return d
def update_chunks(self):
# Don't send chunks unless we're located.
if self.state != STATE_LOCATED:
return
x, y, z = self.location.pos.to_block()
x, chaff, z, chaff = split_coords(x, z)
# These numbers come from a couple spots, including minecraftwiki, but
# I verified them experimentally using torches and pillars to mark
# distances on each setting. ~ C.
distances = {
"tiny": 2,
"short": 4,
"far": 16,
}
radius = distances.get(self.settings.distance, 8)
new = set(circling(x, z, radius))
old = set(self.chunks.iterkeys())
added = new - old
discarded = old - new
# Perhaps some explanation is in order.
# The cooperate() function iterates over the iterable it is fed,
# without tying up the reactor, by yielding after each iteration. The
# inner part of the generator expression generates all of the chunks
# around the currently needed chunk, and it sorts them by distance to
# the current chunk. The end result is that we load chunks one-by-one,
# nearest to furthest, without stalling other clients.
if self.chunk_tasks:
for task in self.chunk_tasks:
try:
task.stop()
except (TaskDone, TaskFailed):
pass
to_enable = sorted_by_distance(added, x, z)
self.chunk_tasks = [
cooperate(self.enable_chunk(i, j) for i, j in to_enable),
cooperate(self.disable_chunk(i, j) for i, j in discarded),
]
def _method_wrapper(self, request, serializer, handler, method, logger):
self.request = request
self.serializer = serializer
self.handler = handler
self.method = method
self.logger = logger
self.request.registerProducer(self, True)
self._task = cooperate(self._call_method())
d = self._task.whenDone()
d.addBoth(self._unregister)
return d
def uploadEvents(numEvents, agent, uri, cal):
def worker():
for i in range(numEvents):
event = makeEvent(i, 1, 0)
yield agent.request(
'PUT',
'%s%s%d.ics' % (uri, cal, i),
Headers({"content-type": ["text/calendar"]}),
StringProducer(event))
worker = worker()
return gatherResults([
cooperate(worker).whenDone() for _ignore_i in range(3)])
def run(service, config):
log.debug("Starting run")
servers = yield service.listSharesToUpdate()
work = (
service.updateOne(*r).addErrback(log_err, log, "failed to update")
for r in servers
)
numworkers = int(config.get("workers", 10)) or 1
workers = [task.cooperate(work).whenDone()
for i in xrange(numworkers)]
dl = yield defer.DeferredList(workers)
log.debug("Done")
def perform_task(self):
log.debug("Starting run")
service = self.parent.updater
servers = yield service.listSharesToUpdate()
work = (
service.updateOne(*r).addErrback(log_err, log, "failed to update")
for r in servers
)
numworkers = self.workers
workers = [task.cooperate(work).whenDone()
for i in xrange(numworkers)]
dl = yield defer.DeferredList(workers)
log.debug("Done")
def test_cooperate(self):
"""
L{twisted.internet.task.cooperate} ought to run the generator that it is
"""
d = defer.Deferred()
def doit():
yield 1
yield 2
yield 3
d.callback("yay")
it = doit()
theTask = task.cooperate(it)
self.assertIn(theTask, task._theCooperator._tasks)
return d
def created(filesystem):
# Save it for later.
self.filesystem = filesystem
# Take a couple snapshots now that there is a filesystem.
return cooperate(
zfs_command(
reactor, [
b"snapshot",
u"{}@{}".format(filesystem.name, name).encode("ascii"),
]
)
for name in expected_names
).whenDone()
def coopReduce(func, iterable, initializer=None):
it = iter(iterable)
x = initializer or next(it)
def cooperator(func, it, x):
for y in it:
x = func(x, y)
yield
returnValue(x)
task = cooperate(cooperator(func, it, x))
result = yield task.whenDone()
returnValue(result)
def test_cooperate(self):
"""
L{twisted.internet.task.cooperate} ought to run the generator that it is
"""
d = defer.Deferred()
def doit():
yield 1
yield 2
yield 3
d.callback("yay")
it = doit()
theTask = task.cooperate(it)
self.assertIn(theTask, task._theCooperator._tasks)
return d
def collect_samples(ignored):
collecting = Deferred()
task = cooperate(
sample(operation, metric, i).addCallback(samples.append)
for i in range(num_samples))
# If the scenario collapses, stop sampling
def stop_sampling_on_scenario_collapse(failure):
task.stop()
collecting.errback(failure)
scenario.maintained().addErrback(stop_sampling_on_scenario_collapse)
# Leaving the errback unhandled makes tests fail
task.whenDone().addCallbacks(
lambda ignored: collecting.callback(samples), lambda ignored: None)
return collecting
def send_initial_chunk_and_location(self):
bigx, smallx, bigz, smallz = split_coords(self.location.x,
self.location.z)
# Spawn the 25 chunks in a square around the spawn, *before* spawning
# the player. Otherwise, there's a funky Beta 1.2 bug which causes the
# player to not be able to move.
d = cooperate(
self.enable_chunk(i, j)
for i, j in product(xrange(bigx - 3, bigx +
3), xrange(bigz - 3, bigz +
3))).whenDone()
# Don't dare send more chunks beyond the initial one until we've
# spawned.
d.addCallback(lambda none: self.update_location())
d.addCallback(lambda none: self.position_changed())
d.addCallback(lambda none: self.update_chunks())
def main(reactor):
"""
Create and verify one S4 subscription.
"""
root_url = argv[1]
emails = argv[2:]
tahoe_env = mkdtemp(prefix=u"tahoe-lafs-env")
task = cooperate(
subscribe(
reactor,
root_url,
email,
).addCallback(
check_subscription,
tahoe_env,
) for email in emails)
return task.whenDone()
def critical():
self._datastore.flush(key)
snap = tables.open_file(snapfile, "w")
raw = snap.create_vlarray(
snap.root,
"raw",
atom=tables.UInt8Atom(shape=()),
filters=tables.Filters(complevel=0))
snap.create_group(snap.root, "tables")
for segment in segments:
yield cooperate(generate(raw, key, segment)).whenDone()
yield details.progress({
"capture_key": key,
"status": True
})
raw.flush()
snap.flush()
snap.close()
def run(self):
"""
Perform the migration.
"""
if self._running:
return
self._running = True
def _done(ign):
self._running = False
migrations = chain(
self.store.query(PendingMigration,
PendingMigration.parent == self),
iter(self._nextObject, None))
it = (m.attemptMigration() for m in migrations)
tasks = [cooperate(it) for _ in xrange(self.concurrency)]
d = gatherResults([task.whenDone() for task in tasks])
d.addCallback(_done)
return d
def test_onDisconnect_waitForOutstandingMessagesToFinish(self):
config = self.getConfig(StompSpec.VERSION_1_0)
client = async.Stomp(config)
client.add(ReceiptListener(1.0))
# connect
client = yield client.connect(host=VIRTUALHOST)
yield task.cooperate(iter([client.send(self.queue, self.frame, receipt='message-%d' % j) for j in xrange(self.numMsgs)])).whenDone()
client.subscribe(self.queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}, listener=SubscriptionListener(self._frameHandler))
# wait for disconnect
yield client.disconnected
# reconnect and subscribe again to make sure that all messages in the queue were ack'ed
client = yield client.connect(host=VIRTUALHOST)
self.timeExpired = False
self.timeoutDelayedCall = reactor.callLater(1, self._timesUp, client) # @UndefinedVariable
client.subscribe(self.queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}, listener=SubscriptionListener(self._eatOneFrameAndDisconnect))
# wait for disconnect
yield client.disconnected
# time should have expired if there were no messages left in the queue
self.assertTrue(self.timeExpired)
def start(self):
iterator = self._gen_data()
self.task = task.cooperate(iterator)
self.request.registerProducer(self, streaming=True)
def iterate():
return cooperate(generate([key, ref])).whenDone()
def create_cooperate(iterObj):
return cooperate(TxnProcessor.iterator(iterObj))
def cb_start_task(self, writer):
self._task = task.cooperate(self._writeFlusher(writer))
def attachTarget(self, target):
self.target = target
task.cooperate(self._gen_data())
def startProducing(self, consumer):
self._consumer = consumer
self._task = cooperate(self._produce())
d = self._task.whenDone()
d.addBoth(self._unregister)
return d
def __init__(self, iterator):
self.task = cooperate(iterator)
def go():
task = cooperate(work(job) for job in f)
d = task.whenDone()
d.addErrback(log.err, "Problem coordinating benchmarks")
d.addCallback(lambda ignored: reactor.stop())
def _waitForMessages(self, timeout):
return task.cooperate(
handler.wait(
timeout, StompCancelledError(
'Handlers did not finish in time.'))
for handler in self._messages.values()).whenDone()
