def __init__(self, id, implementation=None, interface=None, *args, **kwargs):
"""
Creates a new service handler for a generic module.
@param id: The instance ID of this module on the system.
@param implementation: The interface specific implementation of this
module. If the value is not provided, the class
name will be used as default.
@param interface: The interface of this module. If the value is not
provided, the module name of the last implemented
interface is used.
"""
self.id = id
if not implementation:
implementation = utils.get_implementation_from_instance(self)
log.debug("Implementation guessed to be '{0}'".format(implementation))
self.implementation = implementation
if not interface:
interface = utils.get_interface_from_instance(self)
log.debug("Interface guessed to be '{0}'".format(interface))
self.interface = interface
super(Module, self).__init__(*args, **kwargs)
def destroy(ref):
log.debug("Server for {0} garbage collected, removing " \
"subscriptions".format(processor_name))
try:
yield defer.DeferredList([channel.basic_cancel(t) for t in tags])
except Exception as e:
pass
def declare(self):
"""
Declares the queue on the AMQP message broker using the given channel
and saving its instance for further use.
If the queue is an anonymous queue, then the name property will be
populated with the server-generated one.
@precondition: C{channel} shall already be open.
@return: A deferred which will be fired upon queue declaration.
"""
yield self.declaring_lock.acquire()
try:
if self.declared:
return
if self.name:
log.debug("Declaring named queue '{0}'".format(self.name))
else:
log.debug("Declaring anonymous queue")
queue = yield self.channel.queue_declare(queue=self.name, **self.extra)
self.name = queue.queue
self.declared = True
defer.returnValue(None)
finally:
self.declaring_lock.release()
def destroy(ref):
log.debug("Client for {0} garbage collected, removing " \
"subscriptions".format(service_name))
try:
yield channel.basic_cancel(consumer_tag)
except Exception as e:
pass
def start(self, *args, **kwargs):
log.debug("Starting server for compound task '{0}'".format(self.id))
address = amqp.models.Address(routing_key='*')
yield amqp.build_server(address, TaskListener, self, self.queues)
address = amqp.models.Address(routing_key='tasks.{0}.updates'.format(self.id))
self._task_client = yield amqp.build_client(address, TaskListener, distribution='tasks')
r = yield defer.maybeDeferred(self.run, *args, **kwargs)
self.notify()
defer.returnValue(r)
def add(self, task):
assert task.parent == self.id
yield self.add_lock.acquire()
try:
if task.id not in self.children:
log.debug("New child task received ({0})".format(task.id))
address = amqp.models.Address(routing_key='tasks.{0}.commands'.format(task.id))
self.clients[task.id] = yield amqp.build_client(address, TaskServer, distribution='tasks')
self.children[task.id] = ITask(task)
self.tasks = self.children.values()
finally:
self.add_lock.release()
def declare(self):
"""
Declares the exchange on the AMQP message broker using the given
channel and saving its instance for further use.
@precondition: C{channel} shall already be open.
@return: A deferred which will be fired upon exchange declaration.
"""
if self.declared:
return
log.debug("Declaring {0} exchange with name '{1}'".format(
self.type, self.name))
yield self.channel.exchange_declare(exchange=self.name,
type=self.type)
self.declared = True
def startService(self):
log.info("Starting {0} service".format(self.name))
c = protocol.ClientCreator(reactor, self.protocol, self.delegate,
self.vhost, self.specs)
self.client = yield c.connectTCP(self.host, self.port)
log.debug("Connection OK")
yield self.client.authenticate(self.user, self.password)
log.debug("Authentication OK")
channel = yield self.client.channel(self.channel)
yield channel.channel_open()
self.client_factory = SMACClientFactory(self.client, channel)
self.server_factory = SMACServerFactory(self.client, channel)
super(AMQService, self).startService()
def bind(self, exchange, routing_key=None):
"""
Binds the queue instance with the given exchange and using the
supplied C{routing_key} (if present).
@precondition: This queue instance shall already be declared (and the
channel attribute set).
@return: A deferred which will be fired upon queue binding.
"""
yield self.declare()
yield exchange.declare()
if routing_key is None:
routing_key = self.name
log.debug("Binding '{0}' to '{1}' with routing key '{2}'".format(
self.name, exchange.name, routing_key))
yield self.channel.queue_bind(queue=self.name, exchange=exchange.name,
routing_key=routing_key)
def finish(self, checksum):
# Wait for the transfer to complete
yield self.deferred_completed
# @todo: Cleanup queues and make sure the tempfile is removed from the
# system
# Close the file and move it to the final destination
fh, temppath = self.tempfile
# Remove references to allow garbage collecting the file handler
del self.tempfile
# Close the file handler
fh.close()
# Move the file to the final destination in another thread to avoid to
# block the reactor if the move operation acts on different disks
yield threads.deferToThread(os.rename, temppath, self.path)
self.duration = time.time() - self.start_time
error = self.checksum.hexdigest() != checksum
if error:
log.error("Transfer with key '{0}' failed".format(self.transfer_key))
log.debug(" - Checksums: source {0}".format(checksum))
log.debug(" receiv {0}".format(self.checksum.hexdigest()))
else:
log.info("Transfer with key '{0}' successfully completed".format(self.transfer_key))
log.debug(" - Checksum: {0}".format(checksum))
log.debug(" - Duration: {0:.1f} s".format(self.duration))
log.debug(" - Average speed: {0}/s".format(sizeof_fmt(self.size / self.duration)))
if error:
raise InvalidChecksum(self.transfer_key)
def __init__(self, transfer_key, path, size, parent='', hashmethod=hashlib.md5):
super(FileReceiver, self).__init__(parent=parent)
self.transfer_key = transfer_key
self.path = path
fd, temp = tempfile.mkstemp()
self.tempfile = os.fdopen(fd, 'w+b', self.bufsize), temp
self.start_time = time.time()
self.size = size
self.remaining_size = size
self.received = 0
self.checksum = hashmethod()
self.deferred_completed = defer.Deferred()
log.info("Initializing transfer with key '{0}'".format(transfer_key))
log.debug(" - Temporary file: {0}".format(self.tempfile[1]))
log.debug(" - Transfer size: {0}".format(sizeof_fmt(size)))
log.debug(" - Final path: {0}".format(path))
log.debug(" - Hash method: {0}".format(hashmethod.__name__))
self.start()
def build_server(self, delegate, processor, handler, address, queues=None, standalone=True):
processor_name = processor.__name__
log.debug("Creating new server for {0} with ID {1}".format(
processor_name, address.instance))
address = IAddress(address)
if not queues:
queues = topology.queues
if isinstance(self.channel, int):
channel = yield self.client.channel(self.channel)
yield channel.channel_open()
else:
# Assume it's already open!
channel = self.channel
deferreds = []
# Declare all exchanges
exchanges = {}
for k, e in topology.exchanges.iteritems():
e = Exchange(channel, **e)
e.format_name(**dict(address))
e.declare()
exchanges[k] = e
self.responses = Exchange(channel, **topology.exchanges['responses'])
# Declare all queues
qs = []
for q in queues:
q = q.copy()
bindings = q.pop('bindings')
q = Queue(channel, **q)
q.format_name(**dict(address))
q.declare()
deferreds += [q.bind(exchanges[e], k.format(**dict(address))) for e, k in bindings]
qs.append(q)
# Wait for declarations and bindings
yield defer.DeferredList(deferreds)
log.debug("All queues and needed exchanges declared and bound, start listening")
tags = []
for queue in qs:
tag = yield queue.consume()
tags.append(tag)
@defer.inlineCallbacks
def destroy(ref):
log.debug("Server for {0} garbage collected, removing " \
"subscriptions".format(processor_name))
try:
yield defer.DeferredList([channel.basic_cancel(t) for t in tags])
except Exception as e:
pass
if not standalone:
handler = weakref.proxy(handler, destroy)
processor = processor.Processor(handler)
for tag in tags:
queue = yield self.client.queue(tag)
self.get_next_message(channel, queue, processor, delegate)
def handle_closed_queue(self, failure):
log.debug("Queue closed")
def build_client(self, address, service=None, distribution=None, cache=True):
yield self.client_lock.acquire()
try:
address = IAddress(address)
if not service:
service = utils.get_module_from_address(address)
service_name = service.__name__ + address.routing_key
distribution = distribution or address.distribution
if not distribution:
raise ValueError("The distribution mode was not defined and " \
"could not be inferred from the address.")
key = (service, address.routing_key, distribution)
try:
client = self.clients[key]
except KeyError:
log.debug("Creating new client for {0} with routing key {1} and distribution {2}".format(
service.__name__, address.routing_key, distribution))
if isinstance(self.channel, int):
channel = yield self.client.channel(self.channel)
yield channel.channel_open()
else:
# Assume it's already open!
channel = self.channel
response_exchange = Exchange(channel, **topology.exchanges['responses'])
response_queue = Queue(channel, exclusive=True, auto_delete=True)
yield response_queue.declare()
yield response_queue.bind(response_exchange)
consumer_tag = yield response_queue.consume()
service_exchange = Exchange(channel, **topology.exchanges[distribution])
service_exchange.format_name(**dict(address))
yield service_exchange.declare()
amqp_transport = TwistedAMQPTransport(channel, str(service_exchange),
address.routing_key, service_name,
str(response_queue), self.reply_to)
client = service.Client(amqp_transport, self.oprot_factory)
client.address = address
client.factory = self
if cache:
weak_client = client
self.clients[key] = client
else:
@defer.inlineCallbacks
def destroy(ref):
log.debug("Client for {0} garbage collected, removing " \
"subscriptions".format(service_name))
try:
yield channel.basic_cancel(consumer_tag)
except Exception as e:
pass
weak_client = weakref.proxy(client, destroy)
queue = yield self.client.queue(consumer_tag)
self.get_next_message(channel, queue, weak_client)
queue = yield self.client.get_return_queue(service_name)
self.get_next_unroutable_message(channel, queue, weak_client)
else:
log.debug("Using cached client for {0} with routing key {1} and distribution {2}".format(
service.__name__, address.routing_key, distribution))
finally:
self.client_lock.release()
defer.returnValue(client)
