def serialize_fx_inputs(*args, **kwargs):
"""Pack and serialize inputs
"""
fx_serializer = FuncXSerializer()
ser_args = fx_serializer.serialize(args)
ser_kwargs = fx_serializer.serialize(kwargs)
payload = fx_serializer.pack_buffers([ser_args, ser_kwargs])
return payload
def dont_run_yet(endpoint_id=None, tasks=10, duration=1, hostname=None):
tasks_rq = EndpointQueue(f"task_{endpoint_id}", hostname)
fxs = FuncXSerializer()
ser_code = fxs.serialize(slow_double)
fn_code = fxs.pack_buffers([ser_code])
tasks_rq.connect()
start = time.time()
task_ids = {}
for i in range(tasks):
task_id = str(uuid.uuid4())
ser_args = fxs.serialize([i])
ser_kwargs = fxs.serialize({"duration": duration})
input_data = fxs.pack_buffers([ser_args, ser_kwargs])
payload = fn_code + input_data
container_id = "RAW"
task = Task(tasks_rq.redis_client,
task_id,
container_id,
serializer="",
payload=payload)
tasks_rq.enqueue(task)
task_ids[i] = task_id
d1 = time.time() - start
print(f"Time to launch {tasks} tasks: {d1:8.3f} s")
print(f"Launched {tasks} tasks")
for i in range(tasks):
task_id = task_ids[i]
task = Task.from_id(tasks_rq.redis_client, task_id)
# TODO: wait for task result...
time.sleep(2)
print(f"Result: {task.result}")
# res = results_rq.get('result', timeout=300)
# print("Result : ", res)
delta = time.time() - start
print(f"Time to complete {tasks} tasks: {delta:8.3f} s")
print(f"Throughput : {tasks / delta:8.3f} Tasks/s")
return delta
def test(endpoint_id=None, tasks=10, duration=1, hostname=None, port=None):
tasks_rq = RedisQueue(f'task_{endpoint_id}', hostname)
results_rq = RedisQueue('results', hostname)
fxs = FuncXSerializer()
ser_code = fxs.serialize(slow_double)
fn_code = fxs.pack_buffers([ser_code])
tasks_rq.connect()
results_rq.connect()
while True:
try:
_ = results_rq.get(timeout=1)
except Exception:
print("No more results left")
break
start = time.time()
for i in range(tasks):
ser_args = fxs.serialize([i])
ser_kwargs = fxs.serialize({'duration': duration})
input_data = fxs.pack_buffers([ser_args, ser_kwargs])
payload = fn_code + input_data
container_id = "odd" if i % 2 else "even"
tasks_rq.put(f"0{i};{container_id}", payload)
d1 = time.time() - start
print("Time to launch {} tasks: {:8.3f} s".format(tasks, d1))
print(f"Launched {tasks} tasks")
for i in range(tasks):
_ = results_rq.get(timeout=300)
# print("Result : ", res)
delta = time.time() - start
print("Time to complete {} tasks: {:8.3f} s".format(tasks, delta))
print("Throughput : {:8.3f} Tasks/s".format(tasks / delta))
return delta
