def create_new_job(payload, python_main=MAIN, drop_collections=True): configure_loguru_stdout("DEBUG") connect_to_mongodb(host=config.MONGODB_HOST, db=config.MONGODB_DB) logger.info("Connected to mongodb") time.sleep(2) if drop_collections: logger.info("Dropping Job and Worker collections...") Job.drop_collection() Worker.drop_collection() logger.debug("Creating new job...") payload = dict(payload) payload.update({ "message_broker": config.TRANSPORT, }) job = Job( type="test", payload=payload, runnable={ "image": "akadaner/test-worker", "python_main": python_main, }, running_timeout=60, ) job.save()
def test_sample(): from utils_ak.loguru import configure_loguru_stdout configure_loguru_stdout("INFO") boiling_plan_df = read_boiling_plan( config.abs_path("app/data/inputs/ricotta/sample_boiling_plan.xlsx")) print(make_schedule(boiling_plan_df))
def run(): configure_loguru_stdout("DEBUG") run_listener_async("monitor_out", message_broker=config.TRANSPORT) time.sleep(1) multiprocessing.Process(target=run_monitor).start() time.sleep(3) multiprocessing.Process(target=run_worker).start()
def _test_microservice_worker(worker_cls, payload, run_listener=True): configure_loguru_stdout("DEBUG") if run_listener: run_listener_async("monitor_in", message_broker=payload["message_broker"]) time.sleep(2) run_worker(worker_cls, {"worker_id": "worker_id", "payload": payload}) logger.info("Finished")
def test(): configure_loguru_stdout("DEBUG") @log_function() def f(*args, **kwargs): logger.info("Inside the function") return 1 f("a", b=1)
def run_worker(): configure_loguru_stdout("DEBUG") logger.info("Running SampleWorker instance...") worker = SampleWorker( "WorkerId", { "type": "batch", "message_broker": config.TRANSPORT }, ) worker.run()
def test_kubernetes_controller(): import anyconfig import time from utils_ak.loguru import logger, configure_loguru_stdout configure_loguru_stdout("DEBUG") deployment = anyconfig.load("../../example/deployment.yml") ctrl = KubernetesController() ctrl.start(deployment) time.sleep(10) ctrl.log(deployment["id"]) ctrl.stop(deployment["id"])
def run_job_orchestrator(payload=None): configure_loguru_stdout("DEBUG") connect_to_mongodb(host=config.MONGODB_HOST, db=config.MONGODB_DB) logger.info("Connected to mongodb") controller = ProcessController() run_listener_async("job_orchestrator", message_broker=config.TRANSPORT) logger.info('Running job orchestrator...') job_orchestrator = JobOrchestrator(controller, config.TRANSPORT) multiprocessing.Process(target=run_monitor).start() if payload: multiprocessing.Process(target=create_new_job, args=(payload, )).start() job_orchestrator.run()
def test_dummy_push(): from utils_ak.loguru import configure_loguru_stdout configure_loguru_stdout() def brute_validator(parent, block): for c in parent.children[:-1]: validate_disjoint_by_axis(c, block, axis=parent.props["axis"]) logger.debug("Dummy push test") root = ParallelepipedBlock("root", n_dims=1, x=[2], axis=0) a = ParallelepipedBlock("a", n_dims=1, size=[4], axis=0) b = ParallelepipedBlock("b", n_dims=1, size=[3], axis=0) dummy_push(root, a, brute_validator) dummy_push(root, b, brute_validator, start_from=0) logger.debug("Root", root=root)
def _test_controller(controller_cls): configure_loguru_stdout("DEBUG") deployment_fn = os.path.join( os.path.dirname(os.path.dirname(__file__)), "examples/hello_world/deployment.yml", ) deployment = read_deployment(deployment_fn) ctrl = controller_cls() logger.info("Starting") ctrl.start(deployment) time.sleep(3) logger.info("Logs") ctrl.log(deployment["id"]) logger.info("Stopping") ctrl.stop(deployment["id"])
def test1(): configure_loguru_stdout("DEBUG") boiling_plan_df = read_boiling_plan( r"C:\Users\Mi\Desktop\master\code\git\2020.10-umalat\umalat\app\data\inputs\2021-02-19 План по варкам.xlsx" ) boiling_plan_df = boiling_plan_df[boiling_plan_df["boiling"].apply( lambda b: b.line.name == LineName.SALT)] for _, grp in boiling_plan_df.groupby("group_id"): grp["packing_speed"] = grp["sku"].apply(lambda sku: sku.packing_speed) print(grp[["sku_name", "kg"]]) boilings = make_boilings_parallel_dynamic(grp) for boiling in boilings: print(boiling) mp = boiling["melting_and_packing"] mp.props.update(x=(0, 0))
def run_pong(): configure_loguru_stdout() Pong(message_broker=( "zmq", { "endpoints": { "ping": { "type": "sub", "endpoint": endpoint("localhost", 6554) }, "pong": { "type": "sub", "endpoint": endpoint("localhost", 6555) }, } }, )).run()
def _test_microservice_worker_deployment( payload, python_main, controller, run_listener=True, ): configure_loguru_stdout("DEBUG") deployment = create_deployment("worker", "<deployment_id>", payload, python_main=python_main) if run_listener: run_listener_async("monitor_in", message_broker=payload["message_broker"]) controller.stop(deployment["id"]) controller.start(deployment) time.sleep(5) controller.stop(deployment["id"])
def run_ping(): configure_loguru_stdout("TRACE") ping = Ping(message_broker=("kafka", {}), ) ping.run()
from utils_ak.zmq import endpoint from utils_ak.simple_microservice import SimpleMicroservice, run_listener_async from utils_ak.loguru import configure_loguru_stdout class Publisher(SimpleMicroservice): def __init__(self, *args, **kwargs): super().__init__("Publisher", *args, **kwargs) self.add_timer(self.timer_function, 2) def timer_function(self): self.publish("collection", "") if __name__ == "__main__": configure_loguru_stdout() run_listener_async( "collection", message_broker=( "zmq", { "endpoints": { "collection": { "type": "sub", "endpoint": endpoint("localhost", 6554), } } }, ), ) Publisher(message_broker=(
def run_listener(collection, topic="", *args, **kwargs): """ Run listener process for current collection. """ configure_loguru_stdout("DEBUG") Listener(collection, topic, *args, **kwargs).run()
def run_monitor(): configure_loguru_stdout("DEBUG") logger.info("Running monitor...") monitor = Monitor(config.TRANSPORT) monitor.microservice.run()
def run_pong(): configure_loguru_stdout("TRACE") Pong(message_broker=("kafka", {}), ).run()