def _do_reuters_upload(host=None, s3_bucket=None, knative=False):
assert host or s3_bucket or knative, "Must give a host, S3 bucket or knative"
user = "******"
host = get_kubernetes_upload_host(knative, host)
# Upload the matrix data
upload_sparse_matrix(user,
"inputs",
_REUTERS_DATA_DIR,
host=host,
s3_bucket=s3_bucket)
# Upload the categories data
cat_path = join(_REUTERS_DATA_DIR, "outputs")
upload_binary_state(user,
"outputs",
cat_path,
host=host,
s3_bucket=s3_bucket)
# Upload the feature counts
counts_path = join(_REUTERS_DATA_DIR, "feature_counts")
upload_binary_state(user,
"feature_counts",
counts_path,
host=host,
s3_bucket=s3_bucket)
def matrix_state_upload(ctx, mat_size, n_splits, host=None, knative=True):
user = "******"
host = get_kubernetes_upload_host(knative, host)
data_dir = get_matrix_dir(mat_size, n_splits)
file_names = listdir(data_dir)
if len(file_names) == 0:
print("Invalid matrix data: {} {}".format(mat_size, n_splits))
exit(1)
# Multiple uploaders as there may be lots of files
args = [(data_dir, f, user, host) for f in file_names]
p = multiprocessing.Pool(multiprocessing.cpu_count() - 1)
p.starmap(_do_upload, args)
def tf_state_upload(ctx, host=None, knative=True):
data_dir = join(FUNC_DIR, "tf", "data")
model_file = "mobilenet_v1_1.0_224.tflite"
host = get_kubernetes_upload_host(knative, host)
_do_upload(data_dir, model_file, "tf", host, key="mobilenet_v1")