def mapping(ctx):
"""
Run genomics mapping
"""
read_idxs, _ = get_reads_from_dir()
# Iterate through and make the calls to the worker
call_ids = list()
for read_idx in read_idxs:
print("Invoking worker for read chunk {}".format(read_idx))
call_id = invoke_impl("gene",
"mapper",
input="{}".format(read_idx),
async=True,
poll=False)
call_ids.append(call_id)
# Poll for completion of each read
completed_read_idxs = list()
host, port = get_invoke_host_port()
print("Polling workers...")
while len(completed_read_idxs) < len(read_idxs):
for i, read_idx in enumerate(read_idxs):
sleep(0.5)
# See whether this call is still running
call_id = call_ids[i]
result, output = status_call_impl(call_id, host, port)
if result == STATUS_RUNNING:
continue
# Check for success or failure
if result == STATUS_SUCCESS:
# Download the results of this read
print("Read chunk {} completed. Downloading output".format(
read_idx))
state_key = "output_read_{}".format(read_idx)
output_dir = join(FAASM_DATA_DIR, "genomics_output")
if not exists(output_dir):
makedirs(output_dir)
output_file = join(output_dir, state_key)
host, port = get_upload_host_port(None, None)
download_binary_state("gene",
state_key,
output_file,
host=host,
port=port)
elif result == STATUS_FAILED:
print("Read chunk {} failed: {}", read_idx, output)
# Check if we're done
completed_read_idxs.append(read_idx)
print("All read chunks finished")
def tf_state(ctx, host=None, knative=True):
"""
Upload TF experiment state
"""
data_dir = join(FUNC_DIR, "tf", "data")
model_file = "mobilenet_v1_1.0_224.tflite"
host, _ = get_upload_host_port(host, None)
_do_upload(data_dir, model_file, "tf", host, key="mobilenet_v1")
def upload_all(ctx,
host=None,
port=None,
py=False,
prebuilt=False,
local_copy=False):
host, port = get_upload_host_port(host, port)
_do_upload_all(host=host,
port=port,
py=py,
prebuilt=prebuilt,
local_copy=local_copy)
def upload(ctx,
user,
func,
host=None,
s3=False,
ibm=False,
py=False,
ts=False,
prebuilt=False,
file=None):
host, port = get_upload_host_port(host, None)
if py:
func_file = join(PROJ_ROOT, "func", user, "{}.py".format(func))
url = "http://{}:{}/p/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
elif ts:
func_file = join(PROJ_ROOT, "typescript", "build",
"{}.wasm".format(func))
url = "http://{}:{}/f/ts/{}".format(host, port, func)
curl_file(url, func_file)
else:
if file:
func_file = file
else:
base_dir = WASM_DIR if prebuilt else FUNC_BUILD_DIR
if prebuilt:
func_file = join(base_dir, user, func, "function.wasm")
else:
func_file = join(base_dir, user, "{}.wasm".format(func))
if s3:
print("Uploading {}/{} to S3".format(user, func))
s3_key = _get_s3_key(user, func)
upload_file_to_s3(func_file, RUNTIME_S3_BUCKET, s3_key)
if ibm:
print("Uploading {}/{} to IBM cloud storage".format(user, func))
ibm_key = _get_s3_key(user, func)
upload_file_to_ibm(func_file, RUNTIME_S3_BUCKET, ibm_key)
else:
url = "http://{}:{}/f/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
def upload_genomics(ctx, host="localhost", port=8002):
# When uploading genomics, we are uploading the mapper entrypoint as a normal
# function, but the worker functions are all from the same source file
# Upload the entrypoint function
upload(ctx, "gene", "mapper")
# Upload the worker functions (one for each index chunk)
host, port = get_upload_host_port(host, port)
for i in INDEX_CHUNKS:
func_name = "mapper_index{}".format(i)
print("Uploading function gene/{} to {}:{}".format(
func_name, host, port))
file_path = join(PROJ_ROOT,
"third-party/gem3-mapper/wasm_bin/gem-mapper")
url = "http://{}:{}/f/gene/{}".format(host, port, func_name)
curl_file(url, file_path)
def tf_upload_data(ctx, host=None, local_copy=False):
host, port = get_upload_host_port(host, None)
source_data = join(FUNC_DIR, "tf", "data")
dest_root = join(FAASM_SHARED_STORAGE_ROOT, "tfdata")
if local_copy and not exists(dest_root):
makedirs(dest_root)
for root, dirs, files in walk(source_data):
for filename in files:
file_path = join(source_data, filename)
if local_copy:
dest_file = join(dest_root, filename)
call("cp {} {}".format(file_path, dest_file), shell=True)
else:
shared_path = "tfdata/{}".format(filename)
upload_shared_file(host, file_path, shared_path)
def _upload_function(user, func, port=None, host=None, s3=False, ibm=False, py=False, ts=False, file=None,
local_copy=False):
host, port = get_upload_host_port(host, port)
if py and local_copy:
storage_dir = join(FAASM_SHARED_STORAGE_ROOT, "pyfuncs", user, func)
shared_dir = join(FAASM_SHARED_ROOT, "pyfuncs", user, func)
if exists(shared_dir):
rmtree(shared_dir)
if not exists(storage_dir):
makedirs(storage_dir)
src_file = join(FUNC_DIR, user, "{}.py".format(func))
dest_file = join(storage_dir, "function.py")
copy(src_file, dest_file)
elif py:
func_file = join(PROJ_ROOT, "func", user, "{}.py".format(func))
url = "http://{}:{}/p/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
elif ts:
func_file = join(PROJ_ROOT, "typescript", "build", "{}.wasm".format(func))
url = "http://{}:{}/f/ts/{}".format(host, port, func)
curl_file(url, func_file)
else:
if file:
func_file = file
else:
func_file = join(WASM_DIR, user, func, "function.wasm")
if s3:
print("Uploading {}/{} to S3".format(user, func))
s3_key = _get_s3_key(user, func)
upload_file_to_s3(func_file, RUNTIME_S3_BUCKET, s3_key)
if ibm:
print("Uploading {}/{} to IBM cloud storage".format(user, func))
ibm_key = _get_s3_key(user, func)
upload_file_to_ibm(func_file, RUNTIME_S3_BUCKET, ibm_key)
else:
url = "http://{}:{}/f/{}/{}".format(host, port, user, func)
curl_file(url, func_file)