def mapping(ctx): """ Run genomics mapping """ read_idxs, _ = get_reads_from_dir() # Iterate through and make the calls to the worker call_ids = list() for read_idx in read_idxs: print("Invoking worker for read chunk {}".format(read_idx)) call_id = invoke_impl("gene", "mapper", input="{}".format(read_idx), async=True, poll=False) call_ids.append(call_id) # Poll for completion of each read completed_read_idxs = list() host, port = get_invoke_host_port() print("Polling workers...") while len(completed_read_idxs) < len(read_idxs): for i, read_idx in enumerate(read_idxs): sleep(0.5) # See whether this call is still running call_id = call_ids[i] result, output = status_call_impl(call_id, host, port) if result == STATUS_RUNNING: continue # Check for success or failure if result == STATUS_SUCCESS: # Download the results of this read print("Read chunk {} completed. Downloading output".format( read_idx)) state_key = "output_read_{}".format(read_idx) output_dir = join(FAASM_DATA_DIR, "genomics_output") if not exists(output_dir): makedirs(output_dir) output_file = join(output_dir, state_key) host, port = get_upload_host_port(None, None) download_binary_state("gene", state_key, output_file, host=host, port=port) elif result == STATUS_FAILED: print("Read chunk {} failed: {}", read_idx, output) # Check if we're done completed_read_idxs.append(read_idx) print("All read chunks finished")
def tf_state(ctx, host=None, knative=True): """ Upload TF experiment state """ data_dir = join(FUNC_DIR, "tf", "data") model_file = "mobilenet_v1_1.0_224.tflite" host, _ = get_upload_host_port(host, None) _do_upload(data_dir, model_file, "tf", host, key="mobilenet_v1")
def upload_all(ctx, host=None, port=None, py=False, prebuilt=False, local_copy=False): host, port = get_upload_host_port(host, port) _do_upload_all(host=host, port=port, py=py, prebuilt=prebuilt, local_copy=local_copy)
def upload(ctx, user, func, host=None, s3=False, ibm=False, py=False, ts=False, prebuilt=False, file=None): host, port = get_upload_host_port(host, None) if py: func_file = join(PROJ_ROOT, "func", user, "{}.py".format(func)) url = "http://{}:{}/p/{}/{}".format(host, port, user, func) curl_file(url, func_file) elif ts: func_file = join(PROJ_ROOT, "typescript", "build", "{}.wasm".format(func)) url = "http://{}:{}/f/ts/{}".format(host, port, func) curl_file(url, func_file) else: if file: func_file = file else: base_dir = WASM_DIR if prebuilt else FUNC_BUILD_DIR if prebuilt: func_file = join(base_dir, user, func, "function.wasm") else: func_file = join(base_dir, user, "{}.wasm".format(func)) if s3: print("Uploading {}/{} to S3".format(user, func)) s3_key = _get_s3_key(user, func) upload_file_to_s3(func_file, RUNTIME_S3_BUCKET, s3_key) if ibm: print("Uploading {}/{} to IBM cloud storage".format(user, func)) ibm_key = _get_s3_key(user, func) upload_file_to_ibm(func_file, RUNTIME_S3_BUCKET, ibm_key) else: url = "http://{}:{}/f/{}/{}".format(host, port, user, func) curl_file(url, func_file)
def upload_genomics(ctx, host="localhost", port=8002): # When uploading genomics, we are uploading the mapper entrypoint as a normal # function, but the worker functions are all from the same source file # Upload the entrypoint function upload(ctx, "gene", "mapper") # Upload the worker functions (one for each index chunk) host, port = get_upload_host_port(host, port) for i in INDEX_CHUNKS: func_name = "mapper_index{}".format(i) print("Uploading function gene/{} to {}:{}".format( func_name, host, port)) file_path = join(PROJ_ROOT, "third-party/gem3-mapper/wasm_bin/gem-mapper") url = "http://{}:{}/f/gene/{}".format(host, port, func_name) curl_file(url, file_path)
def tf_upload_data(ctx, host=None, local_copy=False): host, port = get_upload_host_port(host, None) source_data = join(FUNC_DIR, "tf", "data") dest_root = join(FAASM_SHARED_STORAGE_ROOT, "tfdata") if local_copy and not exists(dest_root): makedirs(dest_root) for root, dirs, files in walk(source_data): for filename in files: file_path = join(source_data, filename) if local_copy: dest_file = join(dest_root, filename) call("cp {} {}".format(file_path, dest_file), shell=True) else: shared_path = "tfdata/{}".format(filename) upload_shared_file(host, file_path, shared_path)
def _upload_function(user, func, port=None, host=None, s3=False, ibm=False, py=False, ts=False, file=None, local_copy=False): host, port = get_upload_host_port(host, port) if py and local_copy: storage_dir = join(FAASM_SHARED_STORAGE_ROOT, "pyfuncs", user, func) shared_dir = join(FAASM_SHARED_ROOT, "pyfuncs", user, func) if exists(shared_dir): rmtree(shared_dir) if not exists(storage_dir): makedirs(storage_dir) src_file = join(FUNC_DIR, user, "{}.py".format(func)) dest_file = join(storage_dir, "function.py") copy(src_file, dest_file) elif py: func_file = join(PROJ_ROOT, "func", user, "{}.py".format(func)) url = "http://{}:{}/p/{}/{}".format(host, port, user, func) curl_file(url, func_file) elif ts: func_file = join(PROJ_ROOT, "typescript", "build", "{}.wasm".format(func)) url = "http://{}:{}/f/ts/{}".format(host, port, func) curl_file(url, func_file) else: if file: func_file = file else: func_file = join(WASM_DIR, user, func, "function.wasm") if s3: print("Uploading {}/{} to S3".format(user, func)) s3_key = _get_s3_key(user, func) upload_file_to_s3(func_file, RUNTIME_S3_BUCKET, s3_key) if ibm: print("Uploading {}/{} to IBM cloud storage".format(user, func)) ibm_key = _get_s3_key(user, func) upload_file_to_ibm(func_file, RUNTIME_S3_BUCKET, ibm_key) else: url = "http://{}:{}/f/{}/{}".format(host, port, user, func) curl_file(url, func_file)