def upload_binary_state(user, key, binary_file, host=None, s3_bucket=None):
assert host or s3_bucket, "Must provide either host or S3 bucket"
print("Uploading binary file at {} for user {}".format(binary_file, user))
if s3_bucket:
s3_key = "{}/{}".format(user, key)
print("Uploading matrix binary to S3 {} -> {}/{}".format(key, s3_bucket, s3_key))
upload_file_to_s3(binary_file, s3_bucket, s3_key)
else:
url = "http://{}:8002/s/{}/{}".format(host, user, key)
curl_file(url, binary_file)
def upload_genomics(ctx, host="localhost", port=8002):
# When uploading genomics, we are uploading the mapper entrypoint as a normal
# function, but the worker functions are all from the same source file
# Upload the entrypoint function
upload(ctx, "gene", "mapper")
# Upload the worker functions (one for each index chunk)
host, port = get_upload_host_port(host, port)
for i in INDEX_CHUNKS:
func_name = "mapper_index{}".format(i)
print("Uploading function gene/{} to {}:{}".format(
func_name, host, port))
file_path = join(PROJ_ROOT,
"third-party/gem3-mapper/wasm_bin/gem-mapper")
url = "http://{}:{}/f/gene/{}".format(host, port, func_name)
curl_file(url, file_path)
def upload(ctx, user, func, host=None,
s3=False, ibm=False, subdir=None,
py=False, ts=False, prebuilt=False):
host, port = _get_host_port(host, None)
if py:
func_file = join(PROJ_ROOT, "func", user, "{}.py".format(func))
url = "http://{}:{}/p/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
elif ts:
func_file = join(PROJ_ROOT, "typescript", "build", "{}.wasm".format(func))
url = "http://{}:{}/f/ts/{}".format(host, port, func)
curl_file(url, func_file)
else:
base_dir = WASM_DIR if prebuilt else FUNC_BUILD_DIR
if subdir:
func_file = join(base_dir, user, subdir, "{}.wasm".format(func))
elif prebuilt:
func_file = join(base_dir, user, func, "function.wasm")
else:
func_file = join(base_dir, user, "{}.wasm".format(func))
if s3:
print("Uploading {}/{} to S3".format(user, func))
s3_key = _get_s3_key(user, func)
upload_file_to_s3(func_file, RUNTIME_S3_BUCKET, s3_key)
if ibm:
print("Uploading {}/{} to IBM cloud storage".format(user, func))
ibm_key = _get_s3_key(user, func)
upload_file_to_ibm(func_file, RUNTIME_S3_BUCKET, ibm_key)
else:
url = "http://{}:{}/f/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
def _upload_function(user, func, port=None, host=None, s3=False, ibm=False, py=False, ts=False, file=None,
local_copy=False):
host, port = get_upload_host_port(host, port)
if py and local_copy:
storage_dir = join(FAASM_SHARED_STORAGE_ROOT, "pyfuncs", user, func)
shared_dir = join(FAASM_SHARED_ROOT, "pyfuncs", user, func)
if exists(shared_dir):
rmtree(shared_dir)
if not exists(storage_dir):
makedirs(storage_dir)
src_file = join(FUNC_DIR, user, "{}.py".format(func))
dest_file = join(storage_dir, "function.py")
copy(src_file, dest_file)
elif py:
func_file = join(PROJ_ROOT, "func", user, "{}.py".format(func))
url = "http://{}:{}/p/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
elif ts:
func_file = join(PROJ_ROOT, "typescript", "build", "{}.wasm".format(func))
url = "http://{}:{}/f/ts/{}".format(host, port, func)
curl_file(url, func_file)
else:
if file:
func_file = file
else:
func_file = join(WASM_DIR, user, func, "function.wasm")
if s3:
print("Uploading {}/{} to S3".format(user, func))
s3_key = _get_s3_key(user, func)
upload_file_to_s3(func_file, RUNTIME_S3_BUCKET, s3_key)
if ibm:
print("Uploading {}/{} to IBM cloud storage".format(user, func))
ibm_key = _get_s3_key(user, func)
upload_file_to_ibm(func_file, RUNTIME_S3_BUCKET, ibm_key)
else:
url = "http://{}:{}/f/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
def upload_shared_file(host, local_path, shared_path):
url = "http://{}:8002/file/".format(host)
curl_file(url, local_path, headers={
"FilePath": shared_path,
})
def upload_genomics(ctx, host="localhost"):
func_path = join(PROJ_ROOT, "third-party/gem3-mapper/wasm_bin/gem-mapper")
url = "http://{}:8002/f/gene/mapper".format(host)
curl_file(url, func_path)