def clean_states(**kwargs):
backend = get_broker().get_state_backend()
if not isinstance(backend, PostgresBackend):
return {
"error": "deleting states is only supported by the PostgresBackend"
}, 400
get_broker().get_state_backend().clean(**kwargs)
return {"result": "ok"}
def clean_states():
backend = get_broker().get_state_backend()
if not isinstance(backend, PostgresBackend):
return {
"error": "deleting states is only supported by the PostgresBackend"
}, 400
states_kwargs = DeleteSchema().load(request.json or {})
get_broker().get_state_backend().clean(**states_kwargs)
return {"result": "ok"}
def test_broker_uses_rabbitmq_if_not_set():
# Given that no global broker is set
remoulade.broker.global_broker = None
# If I try to get the global broker
# I expect a ValueError to be raised
with pytest.raises(ValueError) as e:
remoulade.get_broker()
assert str(e.value) == 'Broker not found, are you sure you called set_broker(broker) ?'
def get_groups():
backend = get_broker().get_state_backend()
groups_by_id: DefaultDict[str, List[Dict]] = defaultdict(list)
states = (state for state in backend.get_states(get_groups=True))
for state in states:
groups_by_id[state.group_id].append(
state.as_dict(exclude_keys=("args", "kwargs", "options")))
groups: Iterable[GroupMessagesT] = ({
"group_id": group_id,
"messages": messages
} for group_id, messages in groups_by_id.items())
sorted_groups: List[GroupMessagesT] = sorted(
groups,
key=lambda x: x["messages"][0].get("enqueued_datetime"
) or datetime.datetime.min,
reverse=True)
if request.json is None:
return {"data": sorted_groups, "count": len(sorted_groups)}
states_kwargs = PageSchema().load(request.json)
return {
"data":
sorted_groups[states_kwargs["offset"]:states_kwargs["size"] +
states_kwargs["offset"]],
"count":
len(sorted_groups),
}
def get_groups():
args = PageSchema().load(request.args.to_dict())
backend = remoulade.get_broker().get_state_backend()
groups = defaultdict(list)
states = (state for state in backend.get_states() if state.group_id)
if args.get("search_value"):
keys = ["message_id", "name", "actor_name", "group_id"]
value = args["search_value"].lower()
states = [
state for state in states if dict_has(state.as_dict(), keys, value)
] # type: ignore
for state in states:
groups[state.group_id].append(
state.as_dict(exclude_keys=("args", "kwargs", "options")))
groups = sorted( # type: ignore
({
"group_id": group_id,
"messages": messages
} for group_id, messages in groups.items()),
key=lambda x: x["messages"][0].get("enqueued_datetime") or datetime.
datetime.min,
reverse=True,
)
return {
"data": groups[args["offset"]:args["size"] + args["offset"]],
"count": len(groups)
}
def get_states(**kwargs):
backend = get_broker().get_state_backend()
data = [state.as_dict() for state in backend.get_states(**kwargs)]
StatesResponseSchema().load({
"data": data,
"count": backend.get_states_count(**kwargs)
})
return {"data": data, "count": backend.get_states_count(**kwargs)}
def add():
broker = remoulade.get_broker()
@remoulade.actor()
def add(x, y):
return x + y
broker.declare_actor(add)
return add
def mul():
broker = remoulade.get_broker()
@remoulade.actor()
def mul(x, y):
return x * y
broker.declare_actor(mul)
return mul
def main():
args = parse_arguments()
for path in args.path:
sys.path.insert(0, path)
for module in args.modules:
importlib.import_module(module)
if args.actor_name not in get_broker().actors:
print("{} is not an available actor".format(args.actor_name))
return
_args = args.args or []
kwargs = args.kwargs or {}
print(get_broker().actors[args.actor_name](*_args, **kwargs))
def do_work():
broker = remoulade.get_broker()
@remoulade.actor
def do_work():
return 1
broker.declare_actor(do_work)
return do_work
def get_states():
name_state = request.args.get("name")
backend = remoulade.get_broker().get_state_backend()
data = [s.asdict() for s in backend.get_states()]
if name_state is not None:
name_state = name_state.lower().capitalize()
try:
state = StateNamesEnum(name_state)
data = [s for s in data if s["name"] == state.value]
except ValueError:
raise BadRequest("{} state is not defined.".format(name_state))
return {"result": data}
def requeue_message(message_id):
broker = get_broker()
backend = broker.get_state_backend()
state = backend.get_state(message_id)
actor = broker.get_actor(state.actor_name)
payload = {"args": state.args, "kwargs": state.kwargs}
pipe_target = state.options.get("pipe_target")
if pipe_target is None:
actor.send_with_options(**payload, **state.options)
return {"result": "ok"}
else:
return {"error": "requeue message in a pipeline not supported"}, 400
def worker_process(args, worker_id, logging_fd):
try:
# Re-seed the random number generator from urandom on
# supported platforms. This should make it so that worker
# processes don't all follow the same sequence.
random.seed()
logging_pipe = os.fdopen(logging_fd, "w")
logger = setup_worker_logging(args, worker_id, logging_pipe)
for module in args.modules:
importlib.import_module(module)
broker = get_broker()
broker.emit_after("process_boot")
worker = Worker(broker,
queues=args.queues,
worker_threads=args.threads,
prefetch_multiplier=args.prefetch_multiplier)
worker.start()
except ImportError:
logger.exception("Failed to import module.")
return os._exit(RET_IMPORT)
def termhandler(signum, frame):
nonlocal running # type: ignore
if running:
logger.info("Stopping worker process...")
running = False
else:
logger.warning("Killing worker process...")
return os._exit(RET_KILLED)
logger.info("Worker process is ready for action.")
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, termhandler)
signal.signal(signal.SIGHUP, termhandler)
running = True
ret_code = RET_OK
while running:
if worker.consumer_stopped:
running = False
ret_code = RET_CONNECT
else:
time.sleep(1)
worker.stop()
broker.emit_before("process_stop")
broker.close()
logging_pipe.close()
return ret_code
def main():
args = parse_arguments()
for path in args.path:
sys.path.insert(0, path)
for module in args.modules:
importlib.import_module(module)
print("List of available actors:")
for actor in get_broker().actors:
print(actor)
def start_worker(args, logger):
try:
for module in args.modules:
importlib.import_module(module)
broker = get_broker()
broker.emit_after("process_boot")
worker = Worker(
broker, queues=args.queues, worker_threads=args.threads, prefetch_multiplier=args.prefetch_multiplier
)
worker.start()
except ImportError:
logger.exception("Failed to import module.")
return os._exit(RET_IMPORT)
def termhandler(signum, frame):
nonlocal running # type: ignore
if running:
logger.info("Stopping worker...")
running = False
else:
logger.warning("Killing worker...")
return os._exit(RET_KILLED)
logger.info("Worker is ready for action.")
signal.signal(signal.SIGINT, termhandler)
signal.signal(signal.SIGTERM, termhandler)
signal.signal(signal.SIGHUP, termhandler)
running = True
ret_code = RET_OK
while running:
if worker.consumer_stopped:
running = False
ret_code = RET_CONNECT
if worker.worker_stopped:
running = False
ret_code = RET_OK
logger.info("Worker thread is not running anymore, stopping Worker.")
else:
time.sleep(1)
worker.stop()
broker.emit_before("process_stop")
broker.close()
return ret_code
def cancel_message(message_id):
broker = get_broker()
cancel_backend = broker.get_cancel_backend()
try:
# If a single message in a composition is canceled, we cancel the whole composition
state_backend = broker.get_state_backend()
states_count = state_backend.get_states_count(
selected_composition_ids=[message_id])
state = state_backend.get_state(message_id)
if states_count == 0 and state is None:
raise ValidationError(
f"This message id or composition id {message_id} does not exist."
)
if state and state.composition_id:
message_id = state.composition_id
except NoStateBackend:
pass
cancel_backend.cancel([message_id])
return {"result": "ok"}
def get_states():
args = PageSchema().load(request.args.to_dict())
backend = remoulade.get_broker().get_state_backend()
data = [
s.as_dict(exclude_keys=("args", "kwargs", "options"))
for s in backend.get_states()
]
if args.get("search_value"):
keys = ["message_id", "name", "actor_name", "args", "kwargs"]
value = args["search_value"].lower()
data = [item for item in data if dict_has(item, keys, value)]
if args.get("sort_column"):
reverse = args.get("sort_direction") == "desc"
sort_column = args["sort_column"]
data = sort_dicts(data, sort_column, reverse)
return {
"data": data[args["offset"]:args["size"] + args["offset"]],
"count": len(data)
}
def get_state(message_id):
backend = get_broker().get_state_backend()
data = backend.get_state(message_id)
if data is None:
raise NotFound("message_id = {} does not exist".format(message_id))
return data.as_dict(encode_args=True)
def validate_actor_name(self, data, **kwargs):
actor_name = data.get("actor_name")
if actor_name not in remoulade.get_broker().actors.keys():
raise ValidationError(f"No actor named {actor_name} exists")
def get_states():
states_kwargs = PageSchema().load(request.json or {})
backend = get_broker().get_state_backend()
data = [state.as_dict() for state in backend.get_states(**states_kwargs)]
return {"data": data, "count": len(data)}
def get_options():
broker = get_broker()
return {"options": list(broker.actor_options)}
def get_actors():
return {
"result": [actor.as_dict() for actor in get_broker().actors.values()]
}
def postgres_state_middleware(postgres_state_backend):
broker = remoulade.get_broker()
middleware = MessageState(backend=postgres_state_backend)
broker.add_middleware(middleware)
return middleware
def get_state(message_id):
backend = remoulade.get_broker().get_state_backend()
data = backend.get_state(message_id)
if data is None:
raise NotFound("message_id = {} does not exist".format(message_id))
return data.asdict()
def get_state(message_id):
backend = get_broker().get_state_backend()
data = backend.get_state(message_id)
if data is None:
raise NotFound(f"message_id = {message_id} does not exist")
return data.as_dict()
def cancel_message(message_id):
backend = remoulade.get_broker().get_cancel_backend()
backend.cancel([message_id])
return {"result": True}
def cancel_message(message_id):
backend = get_broker().get_cancel_backend()
backend.cancel([message_id])
return {"result": "ok"}
def enqueue_message():
payload = MessageSchema().load(request.json)
actor = get_broker().get_actor(payload.pop("actor_name"))
options = payload.pop("options") or {}
actor.send_with_options(**payload, **options)
return {"result": "ok"}
def check_postgres_begin():
client = remoulade.get_broker().get_state_backend().client
if not check_postgres(client):
pytest.skip("Postgres Database is not in the proper state. Database initialisation is probably incorrect.")
def result_middleware(result_backend):
broker = remoulade.get_broker()
middleware = Results(backend=result_backend)
broker.add_middleware(middleware)
return middleware