def upload(ctx, user, func, host=None,
s3=False, ibm=False, subdir=None,
py=False, ts=False, prebuilt=False):
host, port = _get_host_port(host, None)
if py:
func_file = join(PROJ_ROOT, "func", user, "{}.py".format(func))
url = "http://{}:{}/p/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
elif ts:
func_file = join(PROJ_ROOT, "typescript", "build", "{}.wasm".format(func))
url = "http://{}:{}/f/ts/{}".format(host, port, func)
curl_file(url, func_file)
else:
base_dir = WASM_DIR if prebuilt else FUNC_BUILD_DIR
if subdir:
func_file = join(base_dir, user, subdir, "{}.wasm".format(func))
elif prebuilt:
func_file = join(base_dir, user, func, "function.wasm")
else:
func_file = join(base_dir, user, "{}.wasm".format(func))
if s3:
print("Uploading {}/{} to S3".format(user, func))
s3_key = _get_s3_key(user, func)
upload_file_to_s3(func_file, RUNTIME_S3_BUCKET, s3_key)
if ibm:
print("Uploading {}/{} to IBM cloud storage".format(user, func))
ibm_key = _get_s3_key(user, func)
upload_file_to_ibm(func_file, RUNTIME_S3_BUCKET, ibm_key)
else:
url = "http://{}:{}/f/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
def upload_binary_state(user, key, binary_file, host=None, s3_bucket=None):
assert host or s3_bucket, "Must provide either host or S3 bucket"
print("Uploading binary file at {} for user {}".format(binary_file, user))
if s3_bucket:
s3_key = "{}/{}".format(user, key)
print("Uploading matrix binary to S3 {} -> {}/{}".format(key, s3_bucket, s3_key))
upload_file_to_s3(binary_file, s3_bucket, s3_key)
else:
url = "http://{}:8002/s/{}/{}".format(host, user, key)
curl_file(url, binary_file)
def _do_s3_upload(tar_path, tar_dir, tar_name):
# Compress
print("Creating archive of data {}".format(tar_path))
check_output("tar -cf {} {}".format(tar_path, tar_dir),
shell=True,
cwd=FAASM_DATA_DIR)
# Upload
print("Uploading archive to S3")
upload_file_to_s3(tar_path, DATA_S3_BUCKET, tar_name)
# Remove old tar
print("Removing archive")
check_output("rm {}".format(tar_path), shell=True)
def backup_sysroot(ctx):
print("Creating archive of Faasm sysroot")
check_output("tar -cf {} llvm-sysroot".format(SYSROOT_TAR_NAME),
shell=True,
cwd=FAASM_LOCAL_DIR)
# Upload
print("Uploading archive to S3")
upload_file_to_s3(SYSROOT_TAR_PATH,
MISC_S3_BUCKET,
SYSROOT_TAR_NAME,
public=True)
# Remove old tar
print("Removing archive")
remove(SYSROOT_TAR_PATH)
def backup_sysroot(ctx):
tar_name = _get_sysroot_tar_name()
tar_path = _get_sysroot_tar_path()
print("Creating archive of Faasm sysroot")
check_output("tar -cf {} llvm-sysroot".format(tar_name),
shell=True,
cwd=FAASM_LOCAL_DIR)
# Upload
print("Uploading archive to S3")
upload_file_to_s3(tar_path, MISC_S3_BUCKET, tar_name, public=True)
# Remove old tar
print("Removing archive")
remove(tar_path)
def backup_toolchain(ctx):
backup_sysroot(ctx)
print("Creating archive of Faasm toolchain")
check_output("tar -cf {} install".format(TOOLCHAIN_TAR_NAME),
shell=True,
cwd=TOOLCHAIN_ROOT)
# Upload
print("Uploading archive to S3")
upload_file_to_s3(TOOLCHAIN_TAR_PATH,
MISC_S3_BUCKET,
TOOLCHAIN_TAR_NAME,
public=True)
# Remove old tar
print("Removing archive")
remove(TOOLCHAIN_TAR_PATH)
def backup_runtime_root(ctx):
# Nuke the existing runtime root
if exists(FAASM_RUNTIME_ROOT):
print("Moving existing runtime root to {}".format(BACKUP_LOCATION))
if exists(BACKUP_LOCATION):
rmtree(BACKUP_LOCATION)
call("mv {} {}".format(FAASM_RUNTIME_ROOT, BACKUP_LOCATION),
shell=True)
print("Creating new runtime root dir")
makedirs(FAASM_RUNTIME_ROOT)
# Run the Ansible set-up script
ret = call("ansible-playbook runtime_fs.yml", cwd=ANSIBLE_ROOT, shell=True)
if ret != 0:
print("Running ansible script failed")
return 1
# Set up the Python runtime
set_up_python_runtime(ctx)
# Set up tensorflow data
tf_upload_data(ctx, local_copy=True)
# Create a tmp directory
makedirs(join(FAASM_RUNTIME_ROOT, "tmp"))
# Compress
print("Creating archive of Faasm runtime root")
check_output("tar -cf {} runtime_root".format(RUNTIME_TAR_PATH),
shell=True,
cwd=FAASM_LOCAL_DIR)
# Upload
print("Uploading archive to S3")
upload_file_to_s3(RUNTIME_TAR_PATH,
MISC_S3_BUCKET,
RUNTIME_TAR_NAME,
public=True)
# Remove old tar
print("Removing archive")
remove(RUNTIME_TAR_PATH)
def _upload_function(user, func, port=None, host=None, s3=False, ibm=False, py=False, ts=False, file=None,
local_copy=False):
host, port = get_upload_host_port(host, port)
if py and local_copy:
storage_dir = join(FAASM_SHARED_STORAGE_ROOT, "pyfuncs", user, func)
shared_dir = join(FAASM_SHARED_ROOT, "pyfuncs", user, func)
if exists(shared_dir):
rmtree(shared_dir)
if not exists(storage_dir):
makedirs(storage_dir)
src_file = join(FUNC_DIR, user, "{}.py".format(func))
dest_file = join(storage_dir, "function.py")
copy(src_file, dest_file)
elif py:
func_file = join(PROJ_ROOT, "func", user, "{}.py".format(func))
url = "http://{}:{}/p/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
elif ts:
func_file = join(PROJ_ROOT, "typescript", "build", "{}.wasm".format(func))
url = "http://{}:{}/f/ts/{}".format(host, port, func)
curl_file(url, func_file)
else:
if file:
func_file = file
else:
func_file = join(WASM_DIR, user, func, "function.wasm")
if s3:
print("Uploading {}/{} to S3".format(user, func))
s3_key = _get_s3_key(user, func)
upload_file_to_s3(func_file, RUNTIME_S3_BUCKET, s3_key)
if ibm:
print("Uploading {}/{} to IBM cloud storage".format(user, func))
ibm_key = _get_s3_key(user, func)
upload_file_to_ibm(func_file, RUNTIME_S3_BUCKET, ibm_key)
else:
url = "http://{}:{}/f/{}/{}".format(host, port, user, func)
curl_file(url, func_file)
def _do_upload_all(host=None,
port=None,
upload_s3=False,
py=False,
prebuilt=False,
local_copy=False):
to_upload = []
if py:
dir_to_walk = FUNC_DIR
else:
dir_to_walk = WASM_DIR if prebuilt else FUNC_BUILD_DIR
extension = ".py" if py else ".wasm"
url_part = "p" if py else "f"
if upload_s3 and py:
raise RuntimeError("Not yet implemented python and S3 upload")
if local_copy and not py:
raise RuntimeError("Not yet implemented local copy for non-python")
else:
storage_dir = join(FAASM_SHARED_STORAGE_ROOT, "pyfuncs")
if not exists(storage_dir):
makedirs(storage_dir)
# Walk the function directory tree
for root, dirs, files in os.walk(dir_to_walk):
# Strip original dir from root
rel_path = root.replace(dir_to_walk, "")
rel_path = rel_path.strip("/")
path_parts = rel_path.split("/")
path_parts = [p for p in path_parts if p]
if not path_parts:
continue
if path_parts[0] not in DIRS_TO_INCLUDE:
continue
user = path_parts[0]
for f in files:
if f.endswith(extension):
if prebuilt:
func = path_parts[1]
else:
func = f.replace(extension, "")
func_file = join(root, f)
if upload_s3:
print("Uploading {}/{} to S3".format(user, func))
s3_key = _get_s3_key(user, func)
upload_file_to_s3(func_file, RUNTIME_S3_BUCKET, s3_key)
elif local_copy:
# Copy files directly into place
func_storage_dir = join(storage_dir, user, func)
if not exists(func_storage_dir):
makedirs(func_storage_dir)
dest_file = join(func_storage_dir, "function.py")
call("cp {} {}".format(func_file, dest_file), shell=True)
else:
print("Uploading {}/{} to host {}".format(
user, func, host))
url = "http://{}:{}/{}/{}/{}".format(
host, port, url_part, user, func)
to_upload.append((url, func_file))
# Drop out if already done local copy
if local_copy:
return
# Pool of uploaders
p = multiprocessing.Pool(multiprocessing.cpu_count() - 1)
p.starmap(curl_file, to_upload)
def _upload_lambda_to_s3(s3_key, zip_file_path):
print("Uploading lambda {} to S3".format(s3_key))
upload_file_to_s3(zip_file_path, RUNTIME_S3_BUCKET, s3_key)
