def insert_system_data() -> SystemData:
system = SystemData(uuid_str(), utc(), 1)
log.info(f"Create new system data entry: {system}")
db.insert_document("system_data", {
"_key": "system",
**to_js(system)
},
overwrite=True)
return system
async def parse_event() -> Job:
event, command = re.split("\\s*:\\s*", stripped, 1)
command = strip_quotes(command, "'")
await self.cli.evaluate_cli_command(command,
ctx,
replace_place_holder=False)
uid = uuid_str(f"{command}{event}")[0:8]
return Job(uid, ExecuteCommand(command), timeout,
EventTrigger(event), None, ctx.env, mutable)
async def create_work(self, request: Request) -> StreamResponse:
attrs = {k: v for k, v in request.query.items() if k != "task"}
future = asyncio.get_event_loop().create_future()
task = WorkerTask(uuid_str(), "test", attrs, {
"some": "data",
"foo": "bla"
}, future, timedelta(seconds=3))
await self.worker_task_queue.add_task(task)
await future
return web.HTTPOk()
async def parse_with_cron() -> Job:
parts = re.split("\\s+", stripped, 5)
if len(parts) != 6:
raise ValueError(f"Invalid job {stripped}")
wait: Optional[Tuple[EventTrigger, timedelta]] = None
trigger = TimeTrigger(" ".join(parts[0:5]))
command = strip_quotes(parts[5], "'")
# check if we also need to wait for an event: name_of_event : command
if self.event_re.match(command):
event, command = re.split("\\s*:\\s*", command, 1)
command = strip_quotes(command, "'")
wait = EventTrigger(event), wait_timeout
await self.cli.evaluate_cli_command(command,
ctx,
replace_place_holder=False)
uid = uuid_str(f"{command}{trigger}{wait}")[0:8]
return Job(uid, ExecuteCommand(command), timeout, trigger, wait,
ctx.env, mutable)
async def acknowledge_config_change(self, cfg_id: str, config: Json) -> None:
"""
In case an external entity should acknowledge this config change.
This method either return, which signals success or throws an exception.
"""
future = asyncio.get_event_loop().create_future()
task = WorkerTask(
TaskId(uuid_str()),
WorkerTaskName.validate_config,
{"config_id": cfg_id},
{"task": WorkerTaskName.validate_config, "config": config},
future,
timedelta(seconds=30),
)
# add task to queue - do not retry
await self.task_queue.add_task(task)
# In case the config is not valid or no worker is available
# this future will throw an exception.
# Do not handle it here and let the error bubble up.
await future
async def handle_work_tasks(self, request: Request) -> WebSocketResponse:
worker_id = WorkerId(uuid_str())
task_param = request.query.get("task")
if not task_param:
raise AttributeError(
"A worker needs to define at least one task that it can perform"
)
attrs = {
k: re.split("\\s*,\\s*", v)
for k, v in request.query.items() if k != "task"
}
task_descriptions = [
WorkerTaskDescription(name, attrs)
for name in re.split("\\s*,\\s*", task_param)
]
async def handle_message(msg: str) -> None:
tr = from_js(json.loads(msg), WorkerTaskResult)
if tr.result == "error":
error = tr.error if tr.error else "worker signalled error without detailed error message"
await self.worker_task_queue.error_task(
worker_id, tr.task_id, error)
elif tr.result == "done":
await self.worker_task_queue.acknowledge_task(
worker_id, tr.task_id, tr.data)
else:
log.info(f"Do not understand this message: {msg}")
def task_json(task: WorkerTask) -> str:
return to_js_str(task.to_json())
return await accept_websocket(
request,
handle_incoming=handle_message,
outgoing_context=partial(self.worker_task_queue.attach, worker_id,
task_descriptions),
websocket_handler=self.websocket_handler,
outgoing_fn=task_json,
)
async def merge_graph_process(
db: GraphDB,
event_sender: AnalyticsEventSender,
args: Namespace,
content: AsyncGenerator[Union[bytes, Json], None],
max_wait: timedelta,
maybe_batch: Optional[str],
) -> GraphUpdate:
change_id = maybe_batch if maybe_batch else uuid_str()
write = Queue() # type: ignore
read = Queue() # type: ignore
updater = DbUpdaterProcess(
write, read,
args) # the process reads from our write queue and vice versa
stale = timedelta(seconds=5).total_seconds(
) # consider dead communication after this amount of time
deadline = utc() + max_wait
dead_adjusted = False
async def send_to_child(pa: ProcessAction) -> bool:
alive = updater.is_alive()
if alive:
await run_async(write.put, pa, True, stale)
return alive
def read_results() -> Task: # type: ignore # pypy
async def read_forever() -> GraphUpdate:
nonlocal deadline
nonlocal dead_adjusted
while utc() < deadline:
# After exit of updater: adjust the deadline once
if not updater.is_alive() and not dead_adjusted:
log.debug("Import process done or dead. Adjust deadline.")
deadline = utc() + timedelta(seconds=30)
dead_adjusted = True
try:
action = await run_async(read.get, True, stale)
if isinstance(action, EmitAnalyticsEvent):
await event_sender.capture(action.event)
elif isinstance(action, Result):
return action.get_value()
except Empty:
# empty is fine
pass
raise ImportAborted(
f"Import process died. (ExitCode: {updater.exitcode})")
return asyncio.create_task(read_forever())
task: Optional[Task] = None # type: ignore # pypy
result: Optional[GraphUpdate] = None
try:
reset_process_start_method(
) # other libraries might have tampered the value in the mean time
updater.start()
task = read_results() # concurrently read result queue
chunked: Stream = stream.chunks(content, BatchSize)
async with chunked.stream() as streamer: # pylint: disable=no-member
async for lines in streamer:
if not await send_to_child(ReadElement(lines)):
# in case the child is dead, we should stop
break
await send_to_child(
MergeGraph(db.name, change_id, maybe_batch is not None))
result = cast(GraphUpdate, await task) # wait for final result
return result
finally:
if task is not None and not task.done():
task.cancel()
if not result:
# make sure the change is aborted in case of transaction
log.info(f"Abort update manually: {change_id}")
await db.abort_update(change_id)
await send_to_child(PoisonPill())
await run_async(updater.join, stale)
if updater.is_alive():
log.warning(
f"Process is still alive after poison pill. Terminate process {updater.pid}"
)
with suppress(Exception):
updater.terminate()
await asyncio.sleep(3)
if updater.is_alive():
log.warning(
f"Process is still alive after terminate. Kill process {updater.pid}"
)
with suppress(Exception):
updater.kill()
await asyncio.sleep(3)
if not updater.is_alive():
with suppress(Exception):
updater.close()
async def handle_work_tasks(self,
request: Request) -> web.WebSocketResponse:
ws = web.WebSocketResponse()
await ws.prepare(request)
worker_id = uuid_str()
task_param = request.query.get("task")
if not task_param:
raise AttributeError(
"A worker needs to define at least one task that it can perform"
)
attrs = {
k: re.split("\\s*,\\s*", v)
for k, v in request.query.items() if k != "task"
}
task_descriptions = [
WorkerTaskDescription(name, attrs)
for name in re.split("\\s*,\\s*", task_param)
]
async def receive() -> None:
try:
async for msg in ws:
if isinstance(
msg,
WSMessage) and msg.type == WSMsgType.TEXT and len(
msg.data.strip()) > 0:
log.info(
f"Incoming message: type={msg.type} data={msg.data} extra={msg.extra}"
)
tr = from_js(json.loads(msg.data), WorkerTaskResult)
if tr.result == "error":
error = tr.error if tr.error else "worker signalled error without detailed error message"
await self.worker_task_queue.error_task(
worker_id, tr.task_id, error)
elif tr.result == "done":
await self.worker_task_queue.acknowledge_task(
worker_id, tr.task_id, tr.data)
else:
log.info(
f"Do not understand this message: {msg.data}")
except Exception as ex:
# do not allow any exception - it will destroy the async fiber and cleanup
log.info(f"Receive: worker:{worker_id}: {ex}. Hang up.")
finally:
await self.clean_ws_handler(worker_id)
async def send() -> None:
try:
async with self.worker_task_queue.attach(
worker_id, task_descriptions) as tasks:
while True:
task = await tasks.get()
await ws.send_str(to_js_str(task.to_json()) + "\n")
except Exception as ex:
# do not allow any exception - it will destroy the async fiber and cleanup
log.info(f"Send: worker:{worker_id}: {ex}. Hang up.")
finally:
await self.clean_ws_handler(worker_id)
to_wait = asyncio.gather(asyncio.create_task(receive()),
asyncio.create_task(send()))
self.websocket_handler[worker_id] = (to_wait, ws)
await to_wait
return ws
async def listen_to_events(
self,
request: Request,
listener_id: str,
event_types: List[str],
initial_messages: Optional[Sequence[Message]] = None,
) -> web.WebSocketResponse:
ws = web.WebSocketResponse()
await ws.prepare(request)
wsid = uuid_str()
async def receive() -> None:
try:
async for msg in ws:
if isinstance(
msg,
WSMessage) and msg.type == WSMsgType.TEXT and len(
msg.data.strip()) > 0:
log.info(
f"Incoming message: type={msg.type} data={msg.data} extra={msg.extra}"
)
js = json.loads(msg.data)
if "data" in js:
js["data"]["subscriber_id"] = listener_id
message: Message = from_js(js, Message)
if isinstance(message, Action):
raise AttributeError(
"Actors should not emit action messages. ")
elif isinstance(message, ActionDone):
await self.workflow_handler.handle_action_done(
message)
elif isinstance(message, ActionError):
await self.workflow_handler.handle_action_error(
message)
else:
await self.message_bus.emit(message)
except Exception as ex:
# do not allow any exception - it will destroy the async fiber and cleanup
log.info(
f"Receive: message listener {listener_id}: {ex}. Hang up.")
finally:
await self.clean_ws_handler(wsid)
async def send() -> None:
try:
async with self.message_bus.subscribe(listener_id,
event_types) as events:
while True:
event = await events.get()
await ws.send_str(to_js_str(event) + "\n")
except Exception as ex:
# do not allow any exception - it will destroy the async fiber and cleanup
log.info(
f"Send: message listener {listener_id}: {ex}. Hang up.")
finally:
await self.clean_ws_handler(wsid)
if initial_messages:
for msg in initial_messages:
await ws.send_str(to_js_str(msg) + "\n")
to_wait = asyncio.gather(asyncio.create_task(receive()),
asyncio.create_task(send()))
self.websocket_handler[wsid] = (to_wait, ws)
await to_wait
return ws
import subprocess
from pathlib import Path
import shlex
except ImportError:
print(
f"Can't import one or more modules. Is resoto dev environment activated?"
)
print(
f"Hint: see https://resoto.com/docs/contributing/components for more info."
)
exit(1)
JsonElement = Union[str, int, float, bool, None, Mapping[str, Any],
Sequence[Any]]
run_id = uuid_str()
resoto_assets_path = f"{str(Path.home())}/.resoto/cache/aws_icon_assets/"
class ResourceKind(Enum):
INSTANCE = 1
VOLUME = 2
IMAGE = 3
FIREWALL = 4
K8S_CLUSER = 5
NETWORK = 6
LOAD_BALANCER = 7
CLOUD = 8
kind_colors = {
def test_uuid() -> None:
assert uuid_str("foo") == uuid_str("foo")
assert uuid_str("foo") != uuid_str("bla")
assert uuid_str() != uuid_str()