def launch_spider(module_name, urls, **kwargs):
# Create a dummy message for the spider; this is so that the TaskContext can properly report
# when everything is finished
context_id = kwargs.get('context_id', None)
if context_id:
spider_message = dq.Message(queue_name='dummy',
actor_name='dummy',
args=tuple(),
kwargs={},
options={})
context = TaskContext(id=context_id)
messages_key = context._key_for('messages')
sent_key = context._key_for('sent')
pipe = _redis.from_url(cfg.Config.REDIS_URL).pipeline()
pipe.execute_command('ZADD', messages_key, 'NX', time.time(),
json.dumps(spider_message.asdict()))
pipe.execute_command('ZADD', sent_key, 'NX', time.time(),
spider_message.message_id)
pipe.execute()
kwargs.update(spider_message_id=spider_message.message_id)
proc = mp.Process(target=_launch_spider,
args=(module_name, urls),
kwargs=kwargs)
proc.start()
return proc.pid
def fixture_current_message() -> dramatiq.MessageProxy:
message = dramatiq.Message(queue_name='dummy-queue-name',
actor_name='dummy-actor-name',
args=tuple(),
kwargs=dict(),
options=dict())
proxy = dramatiq.MessageProxy(message)
tft.artemis.context.CURRENT_MESSAGE.set(proxy)
return proxy
def get_result(self, message, *, block=False, timeout=None):
try:
return super().get_result(message, block=block, timeout=timeout)
except TypeError:
# TypeError will occur when this message is partial
if message.options.get('pipe_source') is None:
raise
# attempt to build the full key from the pipeline source results
source_message = dramatiq.Message(**message.options['pipe_source'])
source_result = self.get_result(source_message,
block=block,
timeout=timeout)
full_args = tuple(it.chain(message.args, [source_result]))
full_message = message.copy(args=full_args)
return self.get_result(full_message, block=block, timeout=timeout)
def _enqueue_correlated(ctx: _Context):
"""Enqueues a correlated deploy for further processing by orchestrator.
"""
dramatiq.get_broker().enqueue(
dramatiq.Message(
queue_name="orchestration.engine.step",
actor_name="on_step_deploy_finalized",
args=([
encoder.encode(ctx.deploy_execution_ctx),
encoder.encode(ctx.node_id), ctx.block_hash, ctx.deploy_hash
]),
kwargs=dict(),
options=dict(),
))
def send_signalbus_message(self): # pragma: no cover
model = type(self)
if model.queue_name is None:
assert not hasattr(model, 'actor_name'), \
'SignalModel.actor_name is set, but SignalModel.queue_name is not'
actor_name = self.event_name
routing_key = f'events.{actor_name}'
else:
actor_name = model.actor_name
routing_key = model.queue_name
data = model.__marshmallow_schema__.dump(self)
message = dramatiq.Message(
queue_name=model.queue_name,
actor_name=actor_name,
args=(),
kwargs=data,
options={},
)
protocol_broker.publish_message(message, exchange=MAIN_EXCHANGE_NAME, routing_key=routing_key)
def send_signalbus_message(self):
model = type(self)
if model.queue_name is None:
assert not hasattr(model, 'actor_name'), \
'SignalModel.queue_name is not set, but SignalModel.actor_model is set'
exchange_name = current_app.config['RABBITMQ_EVENT_EXCHANGE']
actor_prefix = f'on_{exchange_name}_' if exchange_name else 'on_'
actor_name = actor_prefix + model.__tablename__
else:
exchange_name = ''
actor_name = model.actor_name
data = model.__marshmallow_schema__.dump(self)
message = dramatiq.Message(
queue_name=model.queue_name,
actor_name=actor_name,
args=(),
kwargs=data,
options={},
)
tasks.broker.publish_message(message, exchange=exchange_name)
def test_abortable_configs(
stub_event_backend: EventBackend,
middleware_abortable: bool,
actor_abortable: Optional[bool],
message_abortable: Optional[bool],
is_abortable: bool,
) -> None:
abortable = Abortable(backend=stub_event_backend, abortable=middleware_abortable)
message = dramatiq.Message(
queue_name="some-queue",
actor_name="some-actor",
args=(),
kwargs={},
options={"abortable": message_abortable},
)
@dramatiq.actor(abortable=actor_abortable)
def actor() -> None:
pass
assert abortable.is_abortable(actor, message) == is_abortable
def messages(self):
"""The messages that this context will execute."""
key = self._key_for('messages')
data = self.redis.zrange(key, 0, -1)
return [dramatiq.Message(**json.loads(msg)) for msg in data]