def publish_astral(self, now=None):
"""I am called every minute."""
def _logFailure(failure):
self.log.debug("reported {message}",
message=failure.getErrorMessage())
return failure
def _logAll(*args):
self.log.debug("all publishing complete args={args!r}", args=args)
deferreds = []
now = self._now(now)
moon_msg = self.moon_message(now)
if moon_msg:
deferreds.append(self._create_msg('moon', moon_msg))
solar_msg = self.sun_message(now)
if solar_msg:
deferreds.append(self._create_msg('solar', solar_msg))
if deferreds:
dl = DeferredList(deferreds)
dl.addCallbacks(_logAll, _logFailure)
return dl
else:
return None
def exchange((xi, rhoi1, rhoi2)):
# Send share to all receivers.
ds = runtime.broadcast(receivers, receivers,
str((str(xi), str(rhoi1), str(rhoi2))))
dls = DeferredList(ds)
dls.addCallbacks(check, runtime.error_handler)
return dls
def enqueue(self):
now = int(time.time())
# Compare the heap min timestamp with now().
# If it's time for the item to be queued, pop it, update the
# timestamp and add it back to the heap for the next go round.
queue_items = []
if self.amqp_queue_size < 100000:
queue_items_a = queue_items.append
LOGGER.debug("%s:%s" % (self.heap[0][0], now))
while self.heap[0][0] < now and len(queue_items) < 1000:
job = heappop(self.heap)
uuid = UUID(bytes=job[1][0])
if not uuid.hex in self.unscheduled_items:
queue_items_a(job[1][0])
new_job = (now + job[1][1], job[1])
heappush(self.heap, new_job)
else:
self.unscheduled_items.remove(uuid.hex)
else:
LOGGER.critical('AMQP queue is at or beyond max limit (%d/100000)'
% self.amqp_queue_size)
# add items to amqp
if queue_items:
LOGGER.info('Found %d new uuids, adding them to the queue'
% len(queue_items))
msgs = [Content(uuid) for uuid in queue_items]
deferreds = [self.chan.basic_publish(
exchange=self.amqp_exchange, content=msg) for msg in msgs]
d = DeferredList(deferreds, consumeErrors=True)
d.addCallbacks(self._addToQueueComplete, self._addToQueueErr)
else:
self.enqueueCallLater = reactor.callLater(1, self.enqueue)
def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
"""Return a Deferred with the output of all successful calls to the given
callbacks
返回带有所有成功调用给定回调的输出的Deferred
"""
dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks]
d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
return d
def _newConnections(self, conn_str):
""" Make a pool of new connections. """
# lock with the semaphore before calling this
logging.debug("IndxConnectionPool _newConnections")
return_d = Deferred()
self.connections[conn_str] = DBConnectionPool(conn_str)
try:
d_list = []
for i in range(MIN_CONNS):
connection_d = self._newConnection(conn_str)
d_list.append(connection_d)
dl = DeferredList(d_list)
dl.addCallbacks(return_d.callback, return_d.errback)
except Exception as e:
logging.error("IndxConnectionPool error in _newConnections: {0}".format(e))
return_d.errback(Failure(e))
return return_d
def _newConnections(self, conn_str):
""" Make a pool of new connections. """
# lock with the semaphore before calling this
logging.debug("IndxConnectionPool _newConnections")
return_d = Deferred()
self.connections[conn_str] = DBConnectionPool(conn_str)
try:
d_list = []
for i in range(MIN_CONNS):
connection_d = self._newConnection(conn_str)
d_list.append(connection_d)
dl = DeferredList(d_list)
dl.addCallbacks(return_d.callback, return_d.errback)
except Exception as e:
logging.error(
"IndxConnectionPool error in _newConnections: {0}".format(e))
return_d.errback(Failure(e))
return return_d
def _process_incoming_email(self, otherself, filepath, mask):
"""
Callback that processes incoming email
@param otherself: Watch object for the current callback from
inotify
@type otherself: twisted.internet.inotify._Watch
@param filepath: Path of the file that changed
@type filepath: twisted.python.filepath.FilePath
@param mask: identifier for the type of change that triggered
this callback
@type mask: int
"""
if os.path.split(filepath.dirname())[-1] == "new":
log.msg("Processing new mail at %s" % (filepath.path,))
with filepath.open("r") as f:
mail_data = f.read()
mail = message_from_string(mail_data)
owner = mail["To"]
if owner is None: # default to Delivered-To
owner = mail["Delivered-To"]
if owner is None:
log.err("Malformed mail, neither To: nor "
"Delivered-To: field")
log.msg("Mail owner: %s" % (owner,))
log.msg("%s received a new mail" % (owner,))
dpubk = self._users_cdb.getPubKey(owner)
duuid = self._users_cdb.queryByAddress(owner)
d = DeferredList([dpubk, duuid])
d.addCallbacks(self._gather_uuid_pubkey, log.err)
d.addCallbacks(self._encrypt_message, log.err,
(owner, mail_data))
d.addCallbacks(self._export_message, log.err)
d.addCallbacks(self._conditional_remove, log.err,
(filepath,))
d.addErrback(log.err)
def test_minimal(self):
embedded_components, client_components = [], [
Case2_Backend, Case2_Frontend
]
## create a WAMP router factory
##
router_factory = RouterFactory()
## create a WAMP router session factory
##
session_factory = RouterSessionFactory(router_factory)
## .. and create and add an WAMP application session to
## run next to the router
##
config = types.ComponentConfig(realm=self.realm,
extra={'caselog': 'case1.log'})
try:
log = io.open('caselog.log', 'w')
except Exception as e:
print(e)
return
# log = io.open(config.extra['caselog'], 'w')
config.log = log
config.dlog = []
config.components = []
config.all_done = []
for C in embedded_components:
one_done = Deferred()
config.all_done.append(one_done)
c = C(config, one_done)
config.components.append(c)
session_factory.add(c)
if self.transport == "websocket":
## create a WAMP-over-WebSocket transport server factory
##
transport_factory = WampWebSocketServerFactory(
session_factory, debug_wamp=self.debug)
transport_factory.setProtocolOptions(failByDrop=False,
openHandshakeTimeout=0,
closeHandshakeTimeout=0)
elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']:
## create a WAMP-over-RawSocket transport server factory
##
if self.transport == 'rawsocket-msgpack':
serializer = MsgPackSerializer()
elif self.transport == 'rawsocket-json':
serializer = JsonSerializer()
else:
raise Exception("should not arrive here")
transport_factory = WampRawSocketServerFactory(
session_factory, serializer, debug=self.debug)
else:
raise Exception("should not arrive here")
## start the server from an endpoint
##
from twisted.internet import reactor
server = serverFromString(reactor, self.server)
d = server.listen(transport_factory)
def onlisten(port):
config.port = port
d.addCallback(onlisten)
clients = []
clients_d = []
for C in client_components:
## create a WAMP application session factory
##
session_factory = ApplicationSessionFactory(config)
one_done = Deferred()
config.all_done.append(one_done)
def make_make(Klass, done):
def make(config):
c = Klass(config, done)
config.components.append(c)
return c
return make
## .. and set the session class on the factory
##
session_factory.session = make_make(C, one_done)
if self.transport == "websocket":
serializers = [JsonSerializer()]
## create a WAMP-over-WebSocket transport client factory
##
transport_factory = WampWebSocketClientFactory(
session_factory,
serializers=serializers,
url=self.url,
debug_wamp=self.debug)
if True:
def maker(Klass):
class TestClientProtocol(
WampWebSocketClientProtocol):
def onOpen(self):
self.txcnt = 0
self.rxcnt = 0
WampWebSocketClientProtocol.onOpen(self)
def sendMessage(self, bytes, isBinary):
self.txcnt += 1
print("> : {:>3} : {:<20} : {}".format(
self.txcnt, Klass.__name__, bytes))
WampWebSocketClientProtocol.sendMessage(
self, bytes, isBinary)
def onMessage(self, bytes, isBinary):
self.rxcnt += 1
print("< : {:>3} : {:<20} : {}".format(
self.rxcnt, Klass.__name__, bytes))
WampWebSocketClientProtocol.onMessage(
self, bytes, isBinary)
return TestClientProtocol
transport_factory.protocol = maker(C)
else:
transport_factory.protocol = WampWebSocketClientProtocol
transport_factory.setProtocolOptions(
failByDrop=False,
openHandshakeTimeout=0,
closeHandshakeTimeout=0)
elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']:
## create a WAMP-over-RawSocket transport client factory
##
if self.transport == 'rawsocket-msgpack':
serializer = MsgPackSerializer()
elif self.transport == 'rawsocket-json':
serializer = JsonSerializer()
else:
raise Exception("should not arrive here")
transport_factory = WampRawSocketClientFactory(
session_factory, serializer, debug=self.debug)
## start the client from an endpoint
##
cl = clientFromString(reactor, self.client)
clients_d.append(cl.connect(transport_factory))
clients.append(cl)
config.connected_clients = None
def client_connected(res):
config.connected_clients = [
proto for success, proto in res if success
]
DeferredList(clients_d).addCallback(client_connected)
d = DeferredList(config.all_done, consumeErrors=True)
#d = config.components[1]._done
def done(res):
log.flush()
log.close()
if config.port:
config.port.stopListening()
if config.connected_clients:
for proto in config.connected_clients:
proto.transport.abortConnection()
print("Log length: {}".format(len(config.dlog)))
print(config.dlog)
#from twisted.internet import reactor
#reactor.callLater(1, reactor.stop)
def error(err):
print(err)
d.addCallbacks(done, error)
# d2 = Deferred()
return d
def test_minimal(self):
embedded_components, client_components = [], [Case2_Backend, Case2_Frontend]
## create a WAMP router factory
##
router_factory = RouterFactory()
## create a WAMP router session factory
##
session_factory = RouterSessionFactory(router_factory)
## .. and create and add an WAMP application session to
## run next to the router
##
config = types.ComponentConfig(realm = self.realm,
extra = {
'caselog': 'case1.log'
}
)
try:
log = io.open('caselog.log', 'w')
except Exception as e:
print(e)
return
# log = io.open(config.extra['caselog'], 'w')
config.log = log
config.dlog = []
config.components = []
config.all_done = []
for C in embedded_components:
one_done = Deferred()
config.all_done.append(one_done)
c = C(config, one_done)
config.components.append(c)
session_factory.add(c)
if self.transport == "websocket":
## create a WAMP-over-WebSocket transport server factory
##
transport_factory = WampWebSocketServerFactory(session_factory, debug_wamp = self.debug)
transport_factory.setProtocolOptions(failByDrop = False, openHandshakeTimeout = 0, closeHandshakeTimeout = 0)
elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']:
## create a WAMP-over-RawSocket transport server factory
##
if self.transport == 'rawsocket-msgpack':
serializer = MsgPackSerializer()
elif self.transport == 'rawsocket-json':
serializer = JsonSerializer()
else:
raise Exception("should not arrive here")
transport_factory = WampRawSocketServerFactory(session_factory, serializer, debug = self.debug)
else:
raise Exception("should not arrive here")
## start the server from an endpoint
##
from twisted.internet import reactor
server = serverFromString(reactor, self.server)
d = server.listen(transport_factory)
def onlisten(port):
config.port = port
d.addCallback(onlisten)
clients = []
clients_d = []
for C in client_components:
## create a WAMP application session factory
##
session_factory = ApplicationSessionFactory(config)
one_done = Deferred()
config.all_done.append(one_done)
def make_make(Klass, done):
def make(config):
c = Klass(config, done)
config.components.append(c)
return c
return make
## .. and set the session class on the factory
##
session_factory.session = make_make(C, one_done)
if self.transport == "websocket":
serializers = [JsonSerializer()]
## create a WAMP-over-WebSocket transport client factory
##
transport_factory = WampWebSocketClientFactory(session_factory, serializers = serializers, url = self.url, debug_wamp = self.debug)
if True:
def maker(Klass):
class TestClientProtocol(WampWebSocketClientProtocol):
def onOpen(self):
self.txcnt = 0
self.rxcnt = 0
WampWebSocketClientProtocol.onOpen(self)
def sendMessage(self, payload, isBinary):
self.txcnt += 1
print("> : {0:>3} : {1:<20} : {3}".format(self.txcnt, Klass.__name__, payload))
WampWebSocketClientProtocol.sendMessage(self, payload, isBinary)
def onMessage(self, payload, isBinary):
self.rxcnt += 1
print("< : {0:>3} : {1:<20} : {2}".format(self.rxcnt, Klass.__name__, payload))
WampWebSocketClientProtocol.onMessage(self, payload, isBinary)
return TestClientProtocol
transport_factory.protocol = maker(C)
else:
transport_factory.protocol = WampWebSocketClientProtocol
transport_factory.setProtocolOptions(failByDrop = False, openHandshakeTimeout = 0, closeHandshakeTimeout = 0)
elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']:
## create a WAMP-over-RawSocket transport client factory
##
if self.transport == 'rawsocket-msgpack':
serializer = MsgPackSerializer()
elif self.transport == 'rawsocket-json':
serializer = JsonSerializer()
else:
raise Exception("should not arrive here")
transport_factory = WampRawSocketClientFactory(session_factory, serializer, debug = self.debug)
## start the client from an endpoint
##
cl = clientFromString(reactor, self.client)
clients_d.append(cl.connect(transport_factory))
clients.append(cl)
config.connected_clients = None
def client_connected(res):
config.connected_clients = [proto for success, proto in res if success]
DeferredList(clients_d).addCallback(client_connected)
d = DeferredList(config.all_done, consumeErrors = True)
#d = config.components[1]._done
def done(_):
log.flush()
log.close()
if config.port:
config.port.stopListening()
if config.connected_clients:
for proto in config.connected_clients:
proto.transport.abortConnection()
print("Log length: {0}".format(len(config.dlog)))
print(config.dlog)
#from twisted.internet import reactor
#reactor.callLater(1, reactor.stop)
def error(err):
print(err)
d.addCallbacks(done, error)
# d2 = Deferred()
return d
df = []
for i in range(options.workers):
args = [options.wsuri,
str(options.threads),
str(options.conns),
str(options.lowmark),
str(options.highmark),
options.resultfile % i]
## run wsperf executable
d = getProcessOutput(options.wsperf, args, os.environ)
## accumulate any output
df.append(d)
d = DeferredList(df, consumeErrors = True)
def onok(res):
if not options.skipanalyze:
analyze.printResults(resultfiles)
reactor.stop()
def onerr(err):
print err
reactor.stop()
d.addCallbacks(onok, onerr)
reactor.run()
"""
entry = make_entry(logid, ts, app, level, msg, src_names, dst_names)
l = []
for f in self.filters:
try:
r = f(entry)
if isinstance(r, Deferred):
l.append(r)
except Exception, e:
self.report([(False, e)])
dl = DeferredList(l, fireOnOneCallback=False,
fireOnOneErrback=False,
consumeErrors=True)
dl.addCallback(self.report) # DeferredList never calls an errback
dl.addCallbacks(defer.callback, defer.errback)
#process_ = lambda logid, ts, app, level, msg, src_names, dst_names:\
# process(logid, ts, app, level, msg, src_names, dst_names, d)
#
#self.uel.get_log_entry(id, process_)
#return d
def report(self, results):
for result in results:
if not result[0]:
# Unfortunatively, we can't print out the exact
# Failure (and included stack trace) due to a bug
# in Twisted mail library. It'll result in an
# exception within the library:
#
for i in range(options.workers):
args = [
options.wsuri,
str(options.threads),
str(options.conns),
str(options.lowmark),
str(options.highmark), options.resultfile % i
]
## run wsperf executable
d = getProcessOutput(options.wsperf, args, os.environ)
## accumulate any output
df.append(d)
d = DeferredList(df, consumeErrors=True)
def onok(res):
if not options.skipanalyze:
analyze.printResults(resultfiles)
reactor.stop()
def onerr(err):
print err
reactor.stop()
d.addCallbacks(onok, onerr)
reactor.run()
def exchange((xi, rhoi1, rhoi2)):
# Send share to all receivers.
ds = runtime.broadcast(receivers, receivers, str((str(xi), str(rhoi1), str(rhoi2))))
dls = DeferredList(ds)
dls.addCallbacks(check, runtime.error_handler)
return dls
class asyncScanner:
def __init__(self, ips, ports, max_threads):
self.ips = ips
self.ports = ports
self.max_threads = max_threads
self.scan_complete = False
# this is a semaphore to limit the number of concurrent requests.
#self.semaphore = DeferredSemaphore(self.max_threads)
self.targets_list = Targets(ips, ports)
self.ok = []
self.error = []
self.count = 0
for ip in self.ips:
# create target
for port in ports:
self.count += 1
self.scan()
def CheckPort(self, host, port, max_threads, timeout=10):
d=Deferred()
factory = ScanClientFactory(d, host, port)
reactor.connectTCP(host, port, factory, timeout)
return d
def scan(self):
self.scan_list = list()
for ip in self.ips:
for port in self.ports:
# make a deferred job to check the host
d = self.CheckPort(ip, port, self.max_threads)
d.addCallbacks(self.scan_success, self.scan_failure)
d.addBoth(self.scan_completed)
self.scan_list.append(d)
self.dl = DeferredList(self.scan_list)
return self.dl.addCallbacks(self.jobSuccess, self.jobError)
def get_results(self):
if self.scan_complete:
return self.ok
else:
return None
### Count number of success
def scan_success(self, result):
self.ok.append(result)
### Count number of errors
def scan_failure(self, reason):
self.error.append(reason)
# When a scan completes, check if it's the last one... to stop the reactor
def scan_completed(self, data):
if len(self.ok) + len(self.error) == self.count:
reactor.stop()
def jobSuccess(self, x):
self.scan_complete = True
return True
def jobError(self, reason):
return False
#print 'Scan Error'
entry = make_entry(logid, ts, app, level, msg, src_names, dst_names)
l = []
for f in self.filters:
try:
r = f(entry)
if isinstance(r, Deferred):
l.append(r)
except Exception, e:
self.report([(False, e)])
dl = DeferredList(l,
fireOnOneCallback=False,
fireOnOneErrback=False,
consumeErrors=True)
dl.addCallback(self.report) # DeferredList never calls an errback
dl.addCallbacks(defer.callback, defer.errback)
#process_ = lambda logid, ts, app, level, msg, src_names, dst_names:\
# process(logid, ts, app, level, msg, src_names, dst_names, d)
#
#self.uel.get_log_entry(id, process_)
#return d
def report(self, results):
for result in results:
if not result[0]:
# Unfortunatively, we can't print out the exact
# Failure (and included stack trace) due to a bug
# in Twisted mail library. It'll result in an
# exception within the library:
#