def multi_pi(ctx, number_times=6):
backoff = lambda x: min(max(1, x // 2), number_times)
output_file = "/usr/local/code/faasm/wasm/omp/multi_pi/bench.csv"
# * 2 in this experiment since starting from `- iterations` instead of 0
sizes = {
"Tiny": 200000,
# "Small": 10000000,
"Big": 200000000,
"Huge": 10000000000,
}
modes = {
"Local": 0,
"Distributed": -1,
}
threads = list(range(2, 41, 2))
r = redis.Redis(host="koala10")
times_key = "multi_pi_times"
r.delete(times_key)
num_times = 0
for mode in modes.values():
for iter_size in sizes.values():
for num_threads in threads:
for _ in range(backoff(num_threads)):
cmd = f"{num_threads} {iter_size} {mode}"
print(f"running omp/multi_pi -- {cmd}")
invoke_impl("omp", "multi_pi", knative=True, cmdline=cmd)
# allow for async flag in invoke too
while r.llen(times_key) == num_times:
print("Waiting for function to finish")
time.sleep(1.5)
num_times += 1
times = list(map(int, r.lrange(times_key, 0, num_times)))
assert len(times) == num_times
idx = 0
with open(output_file, "w") as csv:
csv.write("iterations,numThreads,type,milliseconds\n")
for mode in modes.keys():
for iter_name in sizes.keys():
for num_threads in threads:
for _ in range(backoff(num_threads)):
result = (
f"{iter_name},{num_threads},{mode},{times[idx]}\n"
)
idx += 1
csv.write(result)
def invoke(
ctx,
user,
func,
host=None,
port=None,
input=None,
py=False,
asynch=False,
knative=True,
poll=False,
cmdline=None,
debug=False,
):
"""
Invoke a function
"""
res = invoke_impl(
user,
func,
host=host,
port=port,
input=input,
py=py,
asynch=asynch,
knative=knative,
poll=poll,
cmdline=cmdline,
debug=debug,
)
if asynch:
print("Call ID: " + str(res))
with open(LAST_CALL_ID_FILE, "w") as fh:
fh.write(str(res))
def invoke(ctx, func, native=False, iface=None, np=8):
"""
Invoke one of the ParRes Kernels functions
"""
if func not in PRK_CMDLINE:
print("Invalid PRK function {}".format(func))
return 1
cmdline = PRK_CMDLINE[func]
if func == "random" and not is_power_of_two(np):
print("Must have a power of two number of processes for random")
exit(1)
elif func == "sparse" and not (SPARSE_GRID_SIZE % np == 0):
print(
"To run sparse, grid size must be a multiple of --np (currently grid_size={} and np={})"
.format(SPARSE_GRID_SIZE, np))
exit(1)
if native:
executable = PRK_NATIVE_EXECUTABLES[func]
cmd_out = mpi_run(executable, iface=iface, cmdline=cmdline, np=np)
cmd_out = cmd_out.decode()
print(cmd_out)
else:
host, port = get_invoke_host_port()
cmd_out = invoke_impl(FAASM_USER,
func,
cmdline=cmdline,
host=host,
port=port,
mpi_world_size=np)
_parse_prk_out(func, cmd_out)
def invoke(ctx, user, func,
host=None,
port=None,
input=None,
py=False,
asynch=False,
knative=True,
native=False,
ibm=False,
poll=False,
cmdline=None,
debug=False,
):
"""
Invoke a function
"""
invoke_impl(user, func, host=host, port=port, input=input, py=py, asynch=asynch,
knative=knative, native=native, ibm=ibm, poll=poll, cmdline=cmdline, debug=debug)
def execute_benchmark(self, native):
success, output = invoke_impl(
self.user,
self.func,
poll=True,
poll_interval_ms=self.poll_interval,
knative=True,
input=self.input_data,
native=native,
py=self.is_python,
)
return success, output
def invoke(
ctx,
user,
func,
input=None,
py=False,
asynch=False,
knative=True,
poll=False,
cmdline=None,
mpi_world_size=None,
debug=False,
sgx=False,
graph=False,
):
"""
Invoke a function
"""
res = invoke_impl(
user,
func,
input=input,
py=py,
asynch=asynch,
knative=knative,
poll=poll,
cmdline=cmdline,
mpi_world_size=mpi_world_size,
debug=debug,
sgx=sgx,
graph=graph,
)
if asynch:
print("Call ID: " + str(res))
with open(LAST_CALL_ID_FILE, "w") as fh:
fh.write(str(res))
def multi_cr(ctx, debug=False, num_times=200, num_threads=1):
output_file = f"/usr/local/code/faasm/wasm/omp/multi_cr/bench_all.csv"
modes = {
"native": 0,
"wasm": -1,
}
func = "multi_cr"
threads = [1] + list(range(2, 25, 2))
r = redis.Redis(host="localhost")
rkey = f"{func}_fork_times"
rkey1 = f"{func}_local_fork_times"
r.delete(rkey)
# rindex = 0
with open(output_file, "w") as csv:
csv.write("numThreads,type,microseconds\n")
for num_threads in threads:
cmd = f"{num_threads} {num_times} 0"
print(f"NATIVE: running omp/multi_cr-- {cmd}")
# t_native = run(["/usr/local/code/faasm/ninja-build/bin/multi_cr", f"{num_threads}", "1", "0"], stdout=subprocess.PIPE).stdout.decode('utf-8')
t_native = run(
[
"/usr/local/code/faasm/ninja-build/bin/multi_cr",
f"{num_threads}",
f"{num_times}",
"0",
],
stdout=subprocess.PIPE,
).stdout.decode("utf-8")
csv.write(t_native)
# WasmMP
cmd = f"{num_threads} {num_times} 0"
print(f"WASM running omp/multi_cr-- {cmd}")
invoke_impl("omp", "multi_cr_local", knative=True, cmdline=cmd)
if r.llen(rkey1) != num_times:
print("Failed to run wasm")
exit(1)
t_wasm = list(map(int, r.lrange(rkey1, 0, num_times)))
for t in t_wasm:
wasm_line = f"{num_threads},WasmMP,{t}\n"
csv.write(wasm_line)
r.flushall()
# faasmp
cmd = f"{num_threads} {num_times} -1"
print(f"FAASM running omp/multi_cr-- {cmd}")
invoke_impl("omp", "multi_cr", knative=True, cmdline=cmd)
if r.llen(rkey) != num_times:
print("Failed to run faasm")
exit(1)
t_wasm = list(map(int, r.lrange(rkey, 0, num_times)))
r.delete(rkey)
for t in t_wasm:
wasm_line = f"{num_threads},FaasMP,{t}\n"
csv.write(wasm_line)
r.flushall()
def mapping(ctx, download=False):
"""
Run genomics mapping using Faasm
"""
read_idxs, _ = get_reads_from_dir()
start_time = time()
# Iterate through and make the calls to the worker
call_ids = list()
for read_idx in read_idxs:
print("Mapping read chunk {}".format(read_idx))
call_id = invoke_impl(
"gene",
"mapper",
input="{}".format(read_idx),
asynch=True,
poll=False,
)
call_ids.append(call_id)
# Poll for completion of each read
completed_read_idxs = list()
host, port = get_invoke_host_port()
print("Polling workers...")
while len(completed_read_idxs) < len(read_idxs):
for i, read_idx in enumerate(read_idxs):
sleep(1)
# See whether this call is still running
call_id = call_ids[i]
result, output = status_call_impl("gene",
"mapper",
call_id,
host,
port,
quiet=True)
if result == STATUS_RUNNING:
continue
# Check for success or failure
if result == STATUS_SUCCESS:
print("Read chunk {} completed.".format(read_idx))
# Download the results of this read
if download:
print("Downloading output for read chunk {}.".format(
read_idx))
state_key = "output_read_{}".format(read_idx)
if not exists(GENOMICS_OUTPUT_DIR):
makedirs(GENOMICS_OUTPUT_DIR)
output_file = join(GENOMICS_OUTPUT_DIR, state_key)
host, port = get_upload_host_port(None, None)
download_binary_state("gene",
state_key,
output_file,
host=host,
port=port)
elif result == STATUS_FAILED:
print("Read chunk {} failed: {}", read_idx, output)
# Check if we're done
completed_read_idxs.append(read_idx)
for call_id in call_ids:
exec_graph(
ctx,
call_id=call_id,
host=host,
headless=True,
output_file="/tmp/exec_graph_{}.png".format(call_id),
)
print("-----------------------------------------")
print("FAASM MAPPING COMPLETE")
print("Time: {:.2f}s".format(time() - start_time))
print("-----------------------------------------")
input=None,
py=False,
async=False,
knative=True,
native=False,
ibm=False,
poll=False,
cmdline=None,
debug=False,
):
"""
Invoke a function
"""
invoke_impl(user,
func,
host=host,
port=port,
input=input,
py=py,
async=async,
knative=knative,
native=native,
ibm=ibm,
poll=poll,
cmdline=cmdline,
debug=debug)
@task
def status(ctx, call_id, host=None, port=None):
"""
Get the status of an async function call