def remote_init_train(args):
input, state, cache = readin_remote(args)
n_features = cache["n_features"]
use_CV = cache["use_CV"]
if not list(list_recursive(cache, "i_lambda")):
cache["i_lambda"] = 0
lambdas = set_lambdas(cache["lambdas"], input, cache)
cache["lambdas"] = lambdas
cache["n_lambdas"] = len(lambdas)
# init i_fold, i_lambda
if use_CV:
if not list(list_recursive(cache, "i_fold")):
cache["i_fold"] = 0
if not list(list_recursive(cache, "MSEs_CV")):
cache["MSEs_CV"] = []
cache["convergeds_CV"] = []
cache["n_iters_CV"] = []
else:
if not list(list_recursive(cache, "ws")):
cache["ws_for_scaled_data"] = []
cache["ws"] = []
cache["intercepts"] = []
cache["R2s"] = []
cache["MSEs"] = []
cache["w_ranks"] = []
cache["convergeds"] = []
cache["n_iters"] = []
# init
i_iter = 0
i_feature = 0
w = np.zeros(n_features).tolist()
jj = pick_jth_feature(cache["selection"], n_features, i_feature)
dw = 0.0
# dicts
output = {"msg": "to_train", "w": w, "jj": jj}
if use_CV:
output["i_fold"] = cache["i_fold"]
cache["i_iter"] = i_iter
cache["i_feature"] = i_feature
cache["w"] = w
cache["jj"] = jj
cache["dw"] = dw
result_dict = {"output": output, "cache": cache}
return json.dumps(result_dict)
def local_train(args):
input, state, cache, cache_dir, id, owner = readin(args)
if cache["train_split"] > 0: # has train data
w = np.array(input["w"], dtype="float64")
jj = input["jj"]
tmp = list(list_recursive(input, "i_fold"))
if tmp: # CV ongoing
i_fold = tmp[0]
with open(
os.path.join(cache_dir, "X_fold_" + str(i_fold) + ".npy"),
"rb") as fp:
X = np.load(fp)
with open(
os.path.join(cache_dir, "y_fold_" + str(i_fold) + ".npy"),
"rb") as fp:
y = np.load(fp)
else: # non-CV
with open(os.path.join(cache_dir, "X_train.npy"), "rb") as fp:
X = np.load(fp)
with open(os.path.join(cache_dir, "y_train.npy"), "rb") as fp:
y = np.load(fp)
c_jj = np.dot(X[:, jj], (y - np.matmul(X, w) + w[jj] * X[:, jj]))
else:
c_jj = 0.0
output = {"msg": "to_agg_train", "c_jj_local": float(c_jj)}
result_dict = {"output": output}
return json.dumps(result_dict)
def local_test(args):
input, state, cache, cache_dir, id, owner = readin(args)
w = np.array(input["w"], dtype="float64")
intercept = input["intercept"]
output = {}
tmp = list(list_recursive(input, "i_fold"))
if tmp: # CV ongoing
if cache["train_split"] > 0: # has train data
i_fold = tmp[0]
with open(
os.path.join(cache_dir, "X_valid_" + str(i_fold) + ".npy"),
"rb") as fp:
X_test = np.load(fp)
with open(
os.path.join(cache_dir, "y_valid_" + str(i_fold) + ".npy"),
"rb") as fp:
y_test = np.load(fp)
y_pred = predict(X_test, w, intercept)
se = squared_error(y_test, y_pred) # for MSE
output["se_local"] = se
else:
output["se_local"] = 0.0
else: # non-CV
if cache["train_split"] < 1: # has test data
with open(os.path.join(cache_dir, "X_test.npy"), "rb") as fp:
X_test = np.load(fp)
with open(os.path.join(cache_dir, "y_test.npy"), "rb") as fp:
y_test = np.load(fp)
y_pred = predict(X_test, w, intercept)
se = squared_error(y_test, y_pred)
se_denominator = squared_error(
y_test, input["mean_y_test"]) # for R2 score
output["se_local"] = float(se)
output["se_denominator_local"] = float(se_denominator)
else:
output["se_local"] = 0.0
output["se_denominator_local"] = 0.0
if id == owner:
output["msg"] = "to_agg_test"
result_dict = {"output": output}
return json.dumps(result_dict)
"acc_test_owner": float(acc_test_owner),
"f1_test_owner": float(f1_test_owner),
"rocauc_test_owner": float(rocauc_test_owner),
"cm_test_locals": cm_test_locals,
"cm_test_locals_normalized": cm_test_locals_normalized,
"acc_test_locals": acc_test_locals,
"f1_test_locals": f1_test_locals,
"rocauc_test_locals": rocauc_test_locals,
"n_samples_owner_train": n_samples_owner,
"n_samples_owner_test": y_test.shape[0],
"phase": "local_1",
}
else:
output_dict = {"phase": "local_1"}
result_dict = {"output": output_dict}
return json.dumps(result_dict)
if __name__ == "__main__":
parsed_args = json.loads(sys.stdin.read())
phase_key = list(list_recursive(parsed_args, "phase"))
if not phase_key:
result_dict = local_0(parsed_args)
sys.stdout.write(result_dict)
elif "remote_0" in phase_key:
result_dict = local_1(parsed_args)
sys.stdout.write(result_dict)
else:
raise Exception("Error occurred at Local")
output["se_local"] = float(se)
output["se_denominator_local"] = float(se_denominator)
else:
output["se_local"] = 0.0
output["se_denominator_local"] = 0.0
if id == owner:
output["msg"] = "to_agg_test"
result_dict = {"output": output}
return json.dumps(result_dict)
if __name__ == "__main__":
parsed_args = json.loads(sys.stdin.read())
msg = list(list_recursive(parsed_args, "msg"))
if not msg: # iter 0
result_dict = local_stats_sum(parsed_args)
sys.stdout.write(result_dict)
elif "to_sqsum" in msg:
result_dict = local_stats_sqsum(parsed_args)
sys.stdout.write(result_dict)
elif "to_preprocess" in msg:
result_dict = local_preprocess(parsed_args)
sys.stdout.write(result_dict)
elif "to_train" in msg:
result_dict = local_train(parsed_args)
sys.stdout.write(result_dict)
elif "to_test" in msg:
result_dict = local_test(parsed_args)