def remote_init_train(args): input, state, cache = readin_remote(args) n_features = cache["n_features"] use_CV = cache["use_CV"] if not list(list_recursive(cache, "i_lambda")): cache["i_lambda"] = 0 lambdas = set_lambdas(cache["lambdas"], input, cache) cache["lambdas"] = lambdas cache["n_lambdas"] = len(lambdas) # init i_fold, i_lambda if use_CV: if not list(list_recursive(cache, "i_fold")): cache["i_fold"] = 0 if not list(list_recursive(cache, "MSEs_CV")): cache["MSEs_CV"] = [] cache["convergeds_CV"] = [] cache["n_iters_CV"] = [] else: if not list(list_recursive(cache, "ws")): cache["ws_for_scaled_data"] = [] cache["ws"] = [] cache["intercepts"] = [] cache["R2s"] = [] cache["MSEs"] = [] cache["w_ranks"] = [] cache["convergeds"] = [] cache["n_iters"] = [] # init i_iter = 0 i_feature = 0 w = np.zeros(n_features).tolist() jj = pick_jth_feature(cache["selection"], n_features, i_feature) dw = 0.0 # dicts output = {"msg": "to_train", "w": w, "jj": jj} if use_CV: output["i_fold"] = cache["i_fold"] cache["i_iter"] = i_iter cache["i_feature"] = i_feature cache["w"] = w cache["jj"] = jj cache["dw"] = dw result_dict = {"output": output, "cache": cache} return json.dumps(result_dict)
def local_train(args): input, state, cache, cache_dir, id, owner = readin(args) if cache["train_split"] > 0: # has train data w = np.array(input["w"], dtype="float64") jj = input["jj"] tmp = list(list_recursive(input, "i_fold")) if tmp: # CV ongoing i_fold = tmp[0] with open( os.path.join(cache_dir, "X_fold_" + str(i_fold) + ".npy"), "rb") as fp: X = np.load(fp) with open( os.path.join(cache_dir, "y_fold_" + str(i_fold) + ".npy"), "rb") as fp: y = np.load(fp) else: # non-CV with open(os.path.join(cache_dir, "X_train.npy"), "rb") as fp: X = np.load(fp) with open(os.path.join(cache_dir, "y_train.npy"), "rb") as fp: y = np.load(fp) c_jj = np.dot(X[:, jj], (y - np.matmul(X, w) + w[jj] * X[:, jj])) else: c_jj = 0.0 output = {"msg": "to_agg_train", "c_jj_local": float(c_jj)} result_dict = {"output": output} return json.dumps(result_dict)
def local_test(args): input, state, cache, cache_dir, id, owner = readin(args) w = np.array(input["w"], dtype="float64") intercept = input["intercept"] output = {} tmp = list(list_recursive(input, "i_fold")) if tmp: # CV ongoing if cache["train_split"] > 0: # has train data i_fold = tmp[0] with open( os.path.join(cache_dir, "X_valid_" + str(i_fold) + ".npy"), "rb") as fp: X_test = np.load(fp) with open( os.path.join(cache_dir, "y_valid_" + str(i_fold) + ".npy"), "rb") as fp: y_test = np.load(fp) y_pred = predict(X_test, w, intercept) se = squared_error(y_test, y_pred) # for MSE output["se_local"] = se else: output["se_local"] = 0.0 else: # non-CV if cache["train_split"] < 1: # has test data with open(os.path.join(cache_dir, "X_test.npy"), "rb") as fp: X_test = np.load(fp) with open(os.path.join(cache_dir, "y_test.npy"), "rb") as fp: y_test = np.load(fp) y_pred = predict(X_test, w, intercept) se = squared_error(y_test, y_pred) se_denominator = squared_error( y_test, input["mean_y_test"]) # for R2 score output["se_local"] = float(se) output["se_denominator_local"] = float(se_denominator) else: output["se_local"] = 0.0 output["se_denominator_local"] = 0.0 if id == owner: output["msg"] = "to_agg_test" result_dict = {"output": output} return json.dumps(result_dict)
"acc_test_owner": float(acc_test_owner), "f1_test_owner": float(f1_test_owner), "rocauc_test_owner": float(rocauc_test_owner), "cm_test_locals": cm_test_locals, "cm_test_locals_normalized": cm_test_locals_normalized, "acc_test_locals": acc_test_locals, "f1_test_locals": f1_test_locals, "rocauc_test_locals": rocauc_test_locals, "n_samples_owner_train": n_samples_owner, "n_samples_owner_test": y_test.shape[0], "phase": "local_1", } else: output_dict = {"phase": "local_1"} result_dict = {"output": output_dict} return json.dumps(result_dict) if __name__ == "__main__": parsed_args = json.loads(sys.stdin.read()) phase_key = list(list_recursive(parsed_args, "phase")) if not phase_key: result_dict = local_0(parsed_args) sys.stdout.write(result_dict) elif "remote_0" in phase_key: result_dict = local_1(parsed_args) sys.stdout.write(result_dict) else: raise Exception("Error occurred at Local")
output["se_local"] = float(se) output["se_denominator_local"] = float(se_denominator) else: output["se_local"] = 0.0 output["se_denominator_local"] = 0.0 if id == owner: output["msg"] = "to_agg_test" result_dict = {"output": output} return json.dumps(result_dict) if __name__ == "__main__": parsed_args = json.loads(sys.stdin.read()) msg = list(list_recursive(parsed_args, "msg")) if not msg: # iter 0 result_dict = local_stats_sum(parsed_args) sys.stdout.write(result_dict) elif "to_sqsum" in msg: result_dict = local_stats_sqsum(parsed_args) sys.stdout.write(result_dict) elif "to_preprocess" in msg: result_dict = local_preprocess(parsed_args) sys.stdout.write(result_dict) elif "to_train" in msg: result_dict = local_train(parsed_args) sys.stdout.write(result_dict) elif "to_test" in msg: result_dict = local_test(parsed_args)