def _do_reuters_upload(host=None, s3_bucket=None, knative=False): assert host or s3_bucket or knative, "Must give a host, S3 bucket or knative" user = "******" host = get_kubernetes_upload_host(knative, host) # Upload the matrix data upload_sparse_matrix(user, "inputs", _REUTERS_DATA_DIR, host=host, s3_bucket=s3_bucket) # Upload the categories data cat_path = join(_REUTERS_DATA_DIR, "outputs") upload_binary_state(user, "outputs", cat_path, host=host, s3_bucket=s3_bucket) # Upload the feature counts counts_path = join(_REUTERS_DATA_DIR, "feature_counts") upload_binary_state(user, "feature_counts", counts_path, host=host, s3_bucket=s3_bucket)
def matrix_state_upload(ctx, mat_size, n_splits, host=None, knative=True): user = "******" host = get_kubernetes_upload_host(knative, host) data_dir = get_matrix_dir(mat_size, n_splits) file_names = listdir(data_dir) if len(file_names) == 0: print("Invalid matrix data: {} {}".format(mat_size, n_splits)) exit(1) # Multiple uploaders as there may be lots of files args = [(data_dir, f, user, host) for f in file_names] p = multiprocessing.Pool(multiprocessing.cpu_count() - 1) p.starmap(_do_upload, args)
def tf_state_upload(ctx, host=None, knative=True): data_dir = join(FUNC_DIR, "tf", "data") model_file = "mobilenet_v1_1.0_224.tflite" host = get_kubernetes_upload_host(knative, host) _do_upload(data_dir, model_file, "tf", host, key="mobilenet_v1")