def backup_and_extract_content(sender, instance, **kwargs):
"""Faz backup e extrai conteúdo de um arquivo após sua criação."""
if bool(os.getenv("ENABLE_SIGNAL_FOR_FILE_TASKS", False)):
from .tasks import backup_file, content_from_file
if instance.s3_url is None or instance.content is None:
pipeline([
backup_file.message(instance.pk),
content_from_file.message_with_options(pipe_ignore=True,
args=(instance.pk, )),
]).run()
def save_file(url, content_type, object_id, checksum=None):
file_, created = File.objects.get_or_create(
url=url,
content_type=content_type,
object_id=object_id,
checksum=checksum,
)
if file_.s3_url is None or file_.content is None:
pipeline([
backup_file.message(file_.pk),
content_from_file.message_with_options(pipe_ignore=True,
args=(file_.pk, )),
]).run()
def find_something_in_something_and_make_some_improvements():
pipe = dramatiq.pipeline([
prepare_big_random_list.message(),
find_smallest_in_big_array.message(),
make_smallest_a_bit_bigger.message()
]).run()
return pipe.get_result(block=True, timeout=200000)
def handle(self, *args, **options):
if options.get("date"):
# converte para datetime para verificar se o formato está correto
target_date = datetime.strptime(options.get("date"), "%Y-%m-%d").date()
else:
# ontem
target_date = date.today() - timedelta(days=1)
pipeline(
[
get_city_council_updates.message(target_date.strftime("%Y-%m-%d")),
distribute_city_council_objects_to_sync.message(),
]
).run()
self.stdout.write(
f"Syncronização com a Câmara iniciada (data alvo: {target_date})."
)
def scrape_contracts(limit=None, date_of_grant_start=None, date_of_grant_end=None):
offset = 0
total_records = 0
default_limit = 1000
while offset <= total_records:
real_limit = limit or default_limit
contracts = get_contracts(
offset,
real_limit,
date_of_grant_start=date_of_grant_start,
date_of_grant_end=date_of_grant_end,
)
if not total_records:
total_records = limit if limit else contracts["recordsFiltered"]
for contract in contracts["data"]:
dramatiq.pipeline(
[expand_contract.message(contract), update_contract.message()]
).run()
offset += real_limit
def test_pipelines_flatten_child_pipelines(stub_broker):
# Given an actor that adds two numbers together
@dramatiq.actor
def add(x, y):
return x + y
# When I pipe a message intended for that actor and another pipeline together
pipe = pipeline([add.message(1, 2), add.message(3) | add.message(4), add.message(5)])
# Then the inner pipeline should be flattened into the outer pipeline
assert len(pipe) == 4
assert pipe.messages[0].args == (1, 2)
assert pipe.messages[1].args == (3,)
assert pipe.messages[2].args == (4,)
assert pipe.messages[3].args == (5,)
def run_scheduler():
scheduler = BlockingScheduler()
scheduler.add_job(
pipeline([
dispatch_taking_screenshots.message(),
write_games_screenshots_json_report.message(),
]).run,
IntervalTrigger(seconds=settings.TASK_INTERVAL_SCREENSHOT),
name="dispatch_taking_screenshots",
)
scheduler.add_job(
delete_expired_screenshots_cache.send,
IntervalTrigger(seconds=settings.TASK_INTERVAL_DELETE_CACHE),
name="delete_expired_screenshots_cache",
)
try:
logger.info("Start scheduler")
scheduler.start()
except KeyboardInterrupt:
logger.info("Shutdown scheduler")
scheduler.shutdown()
#!/usr/bin/env python
# vim: set fileencoding=utf-8
import time
import dramatiq
from dramatiq.brokers.redis import RedisBroker
from test_worker_10 import test_worker_A, test_worker_B, test_worker_C, setup_broker_and_backend
setup_broker_and_backend()
for parameter in "XYZ":
p = dramatiq.pipeline([
test_worker_A.message(parameter),
test_worker_B.message(),
test_worker_C.message(),
test_worker_A.message(),
test_worker_B.message(),
test_worker_C.message()
]).run()
print(p.get_result(block=True, timeout=5000))
#!/usr/bin/env python
# vim: set fileencoding=utf-8
import time
import dramatiq
from dramatiq.brokers.redis import RedisBroker
from test_worker_9 import test_worker_A, test_worker_B, test_worker_C, setup_broker_and_backend
setup_broker_and_backend()
p = dramatiq.pipeline([
test_worker_A.message("!"),
test_worker_B.message(),
test_worker_C.message()
]).run()
print(p.get_result(block=True, timeout=5000))
def run(self, *, delay=None, dispatch_window=None):
"""Run the actors in this group over a dispatch window.
Parameters:
delay(int): The minimum amount of time, in milliseconds,
each message in the group should be delayed by.
"""
if self.completion_callbacks:
from dramatiq.middleware.group_callbacks import GROUP_CALLBACK_BARRIER_TTL, GroupCallbacks
rate_limiter_backend = None
for middleware in self.broker.middleware:
if isinstance(middleware, GroupCallbacks):
rate_limiter_backend = middleware.rate_limiter_backend
break
else:
raise RuntimeError(
"GroupCallbacks middleware not found! Did you forget "
"to set it up? It is required if you want to use "
"group callbacks.")
# Generate a new completion uuid on every run so that if a
# group is re-run, the barriers are all separate.
# Re-using a barrier's name is an unsafe operation.
completion_uuid = str(uuid4())
completion_barrier = Barrier(rate_limiter_backend,
completion_uuid,
ttl=GROUP_CALLBACK_BARRIER_TTL)
completion_barrier.create(len(self.children))
children = []
for child in self.children:
if isinstance(child, group):
raise NotImplementedError
elif isinstance(child, pipeline):
pipeline_children = child.messages[:]
pipeline_children[-1] = pipeline_children[-1].copy(
options={
"group_completion_uuid":
completion_uuid,
"group_completion_callbacks":
self.completion_callbacks,
})
children.append(
pipeline(pipeline_children, broker=child.broker))
else:
children.append(
child.copy(
options={
"group_completion_uuid":
completion_uuid,
"group_completion_callbacks":
self.completion_callbacks,
}))
else:
children = self.children
for child in children:
if isinstance(child, (group, pipeline)):
child.run(delay=delay)
else:
delay = random.randint(
0, dispatch_window) if dispatch_window else delay
self.broker.enqueue(child, delay=delay)
return self