def insert_system_data() -> SystemData: system = SystemData(uuid_str(), utc(), 1) log.info(f"Create new system data entry: {system}") db.insert_document("system_data", { "_key": "system", **to_js(system) }, overwrite=True) return system
async def parse_event() -> Job: event, command = re.split("\\s*:\\s*", stripped, 1) command = strip_quotes(command, "'") await self.cli.evaluate_cli_command(command, ctx, replace_place_holder=False) uid = uuid_str(f"{command}{event}")[0:8] return Job(uid, ExecuteCommand(command), timeout, EventTrigger(event), None, ctx.env, mutable)
async def create_work(self, request: Request) -> StreamResponse: attrs = {k: v for k, v in request.query.items() if k != "task"} future = asyncio.get_event_loop().create_future() task = WorkerTask(uuid_str(), "test", attrs, { "some": "data", "foo": "bla" }, future, timedelta(seconds=3)) await self.worker_task_queue.add_task(task) await future return web.HTTPOk()
async def parse_with_cron() -> Job: parts = re.split("\\s+", stripped, 5) if len(parts) != 6: raise ValueError(f"Invalid job {stripped}") wait: Optional[Tuple[EventTrigger, timedelta]] = None trigger = TimeTrigger(" ".join(parts[0:5])) command = strip_quotes(parts[5], "'") # check if we also need to wait for an event: name_of_event : command if self.event_re.match(command): event, command = re.split("\\s*:\\s*", command, 1) command = strip_quotes(command, "'") wait = EventTrigger(event), wait_timeout await self.cli.evaluate_cli_command(command, ctx, replace_place_holder=False) uid = uuid_str(f"{command}{trigger}{wait}")[0:8] return Job(uid, ExecuteCommand(command), timeout, trigger, wait, ctx.env, mutable)
async def acknowledge_config_change(self, cfg_id: str, config: Json) -> None: """ In case an external entity should acknowledge this config change. This method either return, which signals success or throws an exception. """ future = asyncio.get_event_loop().create_future() task = WorkerTask( TaskId(uuid_str()), WorkerTaskName.validate_config, {"config_id": cfg_id}, {"task": WorkerTaskName.validate_config, "config": config}, future, timedelta(seconds=30), ) # add task to queue - do not retry await self.task_queue.add_task(task) # In case the config is not valid or no worker is available # this future will throw an exception. # Do not handle it here and let the error bubble up. await future
async def handle_work_tasks(self, request: Request) -> WebSocketResponse: worker_id = WorkerId(uuid_str()) task_param = request.query.get("task") if not task_param: raise AttributeError( "A worker needs to define at least one task that it can perform" ) attrs = { k: re.split("\\s*,\\s*", v) for k, v in request.query.items() if k != "task" } task_descriptions = [ WorkerTaskDescription(name, attrs) for name in re.split("\\s*,\\s*", task_param) ] async def handle_message(msg: str) -> None: tr = from_js(json.loads(msg), WorkerTaskResult) if tr.result == "error": error = tr.error if tr.error else "worker signalled error without detailed error message" await self.worker_task_queue.error_task( worker_id, tr.task_id, error) elif tr.result == "done": await self.worker_task_queue.acknowledge_task( worker_id, tr.task_id, tr.data) else: log.info(f"Do not understand this message: {msg}") def task_json(task: WorkerTask) -> str: return to_js_str(task.to_json()) return await accept_websocket( request, handle_incoming=handle_message, outgoing_context=partial(self.worker_task_queue.attach, worker_id, task_descriptions), websocket_handler=self.websocket_handler, outgoing_fn=task_json, )
async def merge_graph_process( db: GraphDB, event_sender: AnalyticsEventSender, args: Namespace, content: AsyncGenerator[Union[bytes, Json], None], max_wait: timedelta, maybe_batch: Optional[str], ) -> GraphUpdate: change_id = maybe_batch if maybe_batch else uuid_str() write = Queue() # type: ignore read = Queue() # type: ignore updater = DbUpdaterProcess( write, read, args) # the process reads from our write queue and vice versa stale = timedelta(seconds=5).total_seconds( ) # consider dead communication after this amount of time deadline = utc() + max_wait dead_adjusted = False async def send_to_child(pa: ProcessAction) -> bool: alive = updater.is_alive() if alive: await run_async(write.put, pa, True, stale) return alive def read_results() -> Task: # type: ignore # pypy async def read_forever() -> GraphUpdate: nonlocal deadline nonlocal dead_adjusted while utc() < deadline: # After exit of updater: adjust the deadline once if not updater.is_alive() and not dead_adjusted: log.debug("Import process done or dead. Adjust deadline.") deadline = utc() + timedelta(seconds=30) dead_adjusted = True try: action = await run_async(read.get, True, stale) if isinstance(action, EmitAnalyticsEvent): await event_sender.capture(action.event) elif isinstance(action, Result): return action.get_value() except Empty: # empty is fine pass raise ImportAborted( f"Import process died. (ExitCode: {updater.exitcode})") return asyncio.create_task(read_forever()) task: Optional[Task] = None # type: ignore # pypy result: Optional[GraphUpdate] = None try: reset_process_start_method( ) # other libraries might have tampered the value in the mean time updater.start() task = read_results() # concurrently read result queue chunked: Stream = stream.chunks(content, BatchSize) async with chunked.stream() as streamer: # pylint: disable=no-member async for lines in streamer: if not await send_to_child(ReadElement(lines)): # in case the child is dead, we should stop break await send_to_child( MergeGraph(db.name, change_id, maybe_batch is not None)) result = cast(GraphUpdate, await task) # wait for final result return result finally: if task is not None and not task.done(): task.cancel() if not result: # make sure the change is aborted in case of transaction log.info(f"Abort update manually: {change_id}") await db.abort_update(change_id) await send_to_child(PoisonPill()) await run_async(updater.join, stale) if updater.is_alive(): log.warning( f"Process is still alive after poison pill. Terminate process {updater.pid}" ) with suppress(Exception): updater.terminate() await asyncio.sleep(3) if updater.is_alive(): log.warning( f"Process is still alive after terminate. Kill process {updater.pid}" ) with suppress(Exception): updater.kill() await asyncio.sleep(3) if not updater.is_alive(): with suppress(Exception): updater.close()
async def handle_work_tasks(self, request: Request) -> web.WebSocketResponse: ws = web.WebSocketResponse() await ws.prepare(request) worker_id = uuid_str() task_param = request.query.get("task") if not task_param: raise AttributeError( "A worker needs to define at least one task that it can perform" ) attrs = { k: re.split("\\s*,\\s*", v) for k, v in request.query.items() if k != "task" } task_descriptions = [ WorkerTaskDescription(name, attrs) for name in re.split("\\s*,\\s*", task_param) ] async def receive() -> None: try: async for msg in ws: if isinstance( msg, WSMessage) and msg.type == WSMsgType.TEXT and len( msg.data.strip()) > 0: log.info( f"Incoming message: type={msg.type} data={msg.data} extra={msg.extra}" ) tr = from_js(json.loads(msg.data), WorkerTaskResult) if tr.result == "error": error = tr.error if tr.error else "worker signalled error without detailed error message" await self.worker_task_queue.error_task( worker_id, tr.task_id, error) elif tr.result == "done": await self.worker_task_queue.acknowledge_task( worker_id, tr.task_id, tr.data) else: log.info( f"Do not understand this message: {msg.data}") except Exception as ex: # do not allow any exception - it will destroy the async fiber and cleanup log.info(f"Receive: worker:{worker_id}: {ex}. Hang up.") finally: await self.clean_ws_handler(worker_id) async def send() -> None: try: async with self.worker_task_queue.attach( worker_id, task_descriptions) as tasks: while True: task = await tasks.get() await ws.send_str(to_js_str(task.to_json()) + "\n") except Exception as ex: # do not allow any exception - it will destroy the async fiber and cleanup log.info(f"Send: worker:{worker_id}: {ex}. Hang up.") finally: await self.clean_ws_handler(worker_id) to_wait = asyncio.gather(asyncio.create_task(receive()), asyncio.create_task(send())) self.websocket_handler[worker_id] = (to_wait, ws) await to_wait return ws
async def listen_to_events( self, request: Request, listener_id: str, event_types: List[str], initial_messages: Optional[Sequence[Message]] = None, ) -> web.WebSocketResponse: ws = web.WebSocketResponse() await ws.prepare(request) wsid = uuid_str() async def receive() -> None: try: async for msg in ws: if isinstance( msg, WSMessage) and msg.type == WSMsgType.TEXT and len( msg.data.strip()) > 0: log.info( f"Incoming message: type={msg.type} data={msg.data} extra={msg.extra}" ) js = json.loads(msg.data) if "data" in js: js["data"]["subscriber_id"] = listener_id message: Message = from_js(js, Message) if isinstance(message, Action): raise AttributeError( "Actors should not emit action messages. ") elif isinstance(message, ActionDone): await self.workflow_handler.handle_action_done( message) elif isinstance(message, ActionError): await self.workflow_handler.handle_action_error( message) else: await self.message_bus.emit(message) except Exception as ex: # do not allow any exception - it will destroy the async fiber and cleanup log.info( f"Receive: message listener {listener_id}: {ex}. Hang up.") finally: await self.clean_ws_handler(wsid) async def send() -> None: try: async with self.message_bus.subscribe(listener_id, event_types) as events: while True: event = await events.get() await ws.send_str(to_js_str(event) + "\n") except Exception as ex: # do not allow any exception - it will destroy the async fiber and cleanup log.info( f"Send: message listener {listener_id}: {ex}. Hang up.") finally: await self.clean_ws_handler(wsid) if initial_messages: for msg in initial_messages: await ws.send_str(to_js_str(msg) + "\n") to_wait = asyncio.gather(asyncio.create_task(receive()), asyncio.create_task(send())) self.websocket_handler[wsid] = (to_wait, ws) await to_wait return ws
import subprocess from pathlib import Path import shlex except ImportError: print( f"Can't import one or more modules. Is resoto dev environment activated?" ) print( f"Hint: see https://resoto.com/docs/contributing/components for more info." ) exit(1) JsonElement = Union[str, int, float, bool, None, Mapping[str, Any], Sequence[Any]] run_id = uuid_str() resoto_assets_path = f"{str(Path.home())}/.resoto/cache/aws_icon_assets/" class ResourceKind(Enum): INSTANCE = 1 VOLUME = 2 IMAGE = 3 FIREWALL = 4 K8S_CLUSER = 5 NETWORK = 6 LOAD_BALANCER = 7 CLOUD = 8 kind_colors = {
def test_uuid() -> None: assert uuid_str("foo") == uuid_str("foo") assert uuid_str("foo") != uuid_str("bla") assert uuid_str() != uuid_str()