def serialize_fx_inputs(*args, **kwargs): """Pack and serialize inputs """ fx_serializer = FuncXSerializer() ser_args = fx_serializer.serialize(args) ser_kwargs = fx_serializer.serialize(kwargs) payload = fx_serializer.pack_buffers([ser_args, ser_kwargs]) return payload
def dont_run_yet(endpoint_id=None, tasks=10, duration=1, hostname=None): tasks_rq = EndpointQueue(f"task_{endpoint_id}", hostname) fxs = FuncXSerializer() ser_code = fxs.serialize(slow_double) fn_code = fxs.pack_buffers([ser_code]) tasks_rq.connect() start = time.time() task_ids = {} for i in range(tasks): task_id = str(uuid.uuid4()) ser_args = fxs.serialize([i]) ser_kwargs = fxs.serialize({"duration": duration}) input_data = fxs.pack_buffers([ser_args, ser_kwargs]) payload = fn_code + input_data container_id = "RAW" task = Task(tasks_rq.redis_client, task_id, container_id, serializer="", payload=payload) tasks_rq.enqueue(task) task_ids[i] = task_id d1 = time.time() - start print(f"Time to launch {tasks} tasks: {d1:8.3f} s") print(f"Launched {tasks} tasks") for i in range(tasks): task_id = task_ids[i] task = Task.from_id(tasks_rq.redis_client, task_id) # TODO: wait for task result... time.sleep(2) print(f"Result: {task.result}") # res = results_rq.get('result', timeout=300) # print("Result : ", res) delta = time.time() - start print(f"Time to complete {tasks} tasks: {delta:8.3f} s") print(f"Throughput : {tasks / delta:8.3f} Tasks/s") return delta
def test(endpoint_id=None, tasks=10, duration=1, hostname=None, port=None): tasks_rq = RedisQueue(f'task_{endpoint_id}', hostname) results_rq = RedisQueue('results', hostname) fxs = FuncXSerializer() ser_code = fxs.serialize(slow_double) fn_code = fxs.pack_buffers([ser_code]) tasks_rq.connect() results_rq.connect() while True: try: _ = results_rq.get(timeout=1) except Exception: print("No more results left") break start = time.time() for i in range(tasks): ser_args = fxs.serialize([i]) ser_kwargs = fxs.serialize({'duration': duration}) input_data = fxs.pack_buffers([ser_args, ser_kwargs]) payload = fn_code + input_data container_id = "odd" if i % 2 else "even" tasks_rq.put(f"0{i};{container_id}", payload) d1 = time.time() - start print("Time to launch {} tasks: {:8.3f} s".format(tasks, d1)) print(f"Launched {tasks} tasks") for i in range(tasks): _ = results_rq.get(timeout=300) # print("Result : ", res) delta = time.time() - start print("Time to complete {} tasks: {:8.3f} s".format(tasks, delta)) print("Throughput : {:8.3f} Tasks/s".format(tasks / delta)) return delta