def local_2(args):
cache_ = args["cache"]
state_ = args["state"]
input_dir = state_["baseDirectory"]
cache_dir = state_["cacheDirectory"]
local_sharedRows = args["cache"]["shared_rows"]
local_n = args["cache"]["local_n"]
local_Y = np.load(os.path.join(cache_dir, cache_["local_Y"]))
local_dY = np.load(os.path.join(cache_dir, args["cache"]["local_dY"]))
local_IY = np.load(os.path.join(cache_dir, args["cache"]["local_IY"]))
local_P = np.load(os.path.join(cache_dir, args["cache"]["local_P"]))
local_gains = np.load(os.path.join(cache_dir,
args["cache"]["local_gains"]))
compAvgError1 = args["input"]["compAvgError"]
iter = args["input"]["number_of_iterations"]
# Made changes here. Instead of extract shared_Y I am extracting it from cache
if (iter > 0):
shared_Y = np.load(os.path.join(args['state']['baseDirectory'],
args['input']['shared_Y']),
allow_pickle=True)
local_Y[:local_sharedRows, :] = shared_Y
#It should be the average one
#raise Exception()
C = compAvgError1['error']
demeanAvg = (np.mean(local_Y, 0))
demeanAvg[0] = compAvgError1['avgX']
demeanAvg[1] = compAvgError1['avgY']
local_Y = demeanL(local_Y, demeanAvg)
local_Y, dY, local_IY, gains, n, sharedRows, P, C = master_child(
local_Y, local_dY, local_IY, local_gains, local_n, local_sharedRows,
local_P, iter, C)
local_Y[local_sharedRows:, :] = local_Y[local_sharedRows:, :] + local_IY[
local_sharedRows:, :]
local_Shared_Y = local_Y[:local_sharedRows, :]
local_Shared_IY = local_IY[:local_sharedRows, :]
meanValue = (np.mean(local_Y, 0))
# save files to transfer
np.save(
os.path.join(args['state']['transferDirectory'], 'local_Shared_Y.npy'),
local_Shared_Y)
np.save(
os.path.join(args['state']['transferDirectory'],
'local_Shared_IY.npy'), local_Shared_IY)
# ------ Need to change here. Instead of sending all local_Y, I have to send mean of local_Y and local site data length
np.save(os.path.join(args['state']['transferDirectory'], 'local_Y.npy'),
local_Y[local_sharedRows:, :])
# save file to local cache directory
np.save(os.path.join(args['state']['cacheDirectory'], 'local_Y.npy'),
local_Y)
np.save(os.path.join(args['state']['cacheDirectory'], 'local_dY.npy'),
local_dY)
np.save(os.path.join(args['state']['cacheDirectory'], 'local_IY.npy'),
local_IY)
np.save(os.path.join(args['state']['cacheDirectory'], 'local_P.npy'), P)
np.save(os.path.join(args['state']['cacheDirectory'], 'local_gains.npy'),
local_gains)
#np.save(os.path.join(args['state']['cacheDirectory'], 'local_shared_Y.npy'), local_Shared_Y)
if (iter > 10):
with open(
os.path.join(args["state"]["baseDirectory"],
'local_site_data_label.txt')) as fh2:
local_Y_labels = np.loadtxt(fh2.readlines())
np.save(
os.path.join(args['state']['transferDirectory'],
'local_Y_labels.npy'), local_Y_labels)
local_site_value = local_Y[local_sharedRows:, :]
local_Y_final_emdedding = np.zeros((local_site_value.shape[0], 3))
local_Y_final_emdedding[:, 0] = local_site_value[:, 0]
local_Y_final_emdedding[:, 1] = local_site_value[:, 1]
local_Y_final_emdedding[:, 2] = local_Y_labels
np.save(
os.path.join(args['state']['transferDirectory'],
'local_Y_final_emdedding.npy'),
local_Y_final_emdedding)
np.save(
os.path.join(args['state']['cacheDirectory'],
'local_Y_final_emdedding.npy'),
local_Y_final_emdedding)
#raise Exception('I am at local 2 function at iteration 14')
computation_output = {
"output": {
"MeanX": meanValue[0],
"MeanY": meanValue[1],
"error": C,
"local_Shared_iY": 'local_Shared_IY.npy',
"local_Y_final_emdedding": 'local_Y_final_emdedding.npy',
"local_Shared_Y": 'local_Shared_Y.npy',
"local_Y": 'local_Y.npy',
"local_Y_labels": 'local_Y_labels.npy',
"computation_phase": "local_2"
},
"cache": {
"local_Y": 'local_Y.npy',
"local_Y_final_emdedding": 'local_Y_final_emdedding.npy',
"local_dY": 'local_dY.npy',
"local_iY": 'local_IY.npy',
"local_P": 'local_P.npy',
"local_n": local_n,
"local_gains": 'local_gains.npy',
"shared_rows": sharedRows
#"local_shared_Y": 'local_shared_Y.npy'
}
}
else:
with open(
os.path.join(args["state"]["baseDirectory"],
'local_site_data_label.txt')) as fh2:
local_Y_labels = np.loadtxt(fh2.readlines())
local_Y_labels = local_Y_labels.astype(int)
np.save(
os.path.join(args['state']['transferDirectory'],
'local_Y_labels.npy'), local_Y_labels)
computation_output = {
"output": {
"MeanX": meanValue[0],
"MeanY": meanValue[1],
"error": C,
"local_Shared_iY": 'local_Shared_IY.npy',
"local_Shared_Y": 'local_Shared_Y.npy',
"local_Y": 'local_Y.npy',
"local_Y_labels": 'local_Y_labels.npy',
"computation_phase": "local_2"
},
"cache": {
"local_Y": 'local_Y.npy',
"local_dY": 'local_dY.npy',
"local_iY": 'local_IY.npy',
"local_P": 'local_P.npy',
"local_n": local_n,
"local_gains": 'local_gains.npy',
"shared_rows": sharedRows
#"local_shared_Y": 'local_shared_Y.npy'
}
}
return json.dumps(computation_output)
def local_site1(args, compAvgError, computation_phase):
''' It will load local data and download remote data and place it on top. Then it will run tsne on combined data(shared + local) and return low dimensional shared Y and IY
args (dictionary): {
"shared_X" (str): file path to remote site data,
"shared_Label" (str): file path to remote site labels
"no_dims" (int): Final plotting dimensions,
"initial_dims" (int): number of dimensions that PCA should produce
"perplexity" (int): initial guess for nearest neighbor
"shared_Y" (str): the low-dimensional remote site data
}
Returns:
computation_phase(local): It will return only low dimensional shared data from local site
computation_phase(final): It will return only low dimensional local site data
computation_phase(computation): It will return only low dimensional shared data Y and corresponding IY
'''
C= 0
if computation_phase is 'local':
shared_X = np.loadtxt(args["shared_X"])
shared_Y = np.loadtxt(args["shared_Y"])
no_dims = args["no_dims"]
initial_dims = args["initial_dims"]
perplexity = args["perplexity"]
local_site1.sharedRows, local_site1.sharedColumns = shared_X.shape
parser = argparse.ArgumentParser(
description='''read in coinstac args for local computation''')
parser.add_argument('--run', type=json.loads, help='grab coinstac args')
localSite1_Data = ''' {
"site1_Data": "Site_1_Mnist_X.txt",
"site1_Label": "Site_1_Label.txt"
} '''
site1args = parser.parse_args(['--run', localSite1_Data])
Site1Data = np.loadtxt(site1args.run["site1_Data"])
# create combinded list by local and remote data
combined_X = np.concatenate((shared_X, Site1Data), axis=0)
combined_X = normalize_columns(combined_X)
# create low dimensional position
combined_Y = np.random.randn(combined_X.shape[0], no_dims)
combined_Y[:shared_Y.shape[0], :] = shared_Y
local_site1.Y, local_site1.dY, local_site1.iY, local_site1.gains, local_site1.P, local_site1.n = tsne(
combined_X,
combined_Y,
local_site1.sharedRows,
no_dims=no_dims,
initial_dims=initial_dims,
perplexity=perplexity,
computation_phase=computation_phase)
# save local site sharedIY data into file
with open("site1SharedY.txt", "w") as f1:
for i in range(0, local_site1.sharedRows):
f1.write(str(local_site1.Y[i][0]) + '\t')
f1.write(str(local_site1.Y[i][1]) + '\n')
# pass data to remote in json format
localJsonY = ''' {"localSite1SharedY": "site1SharedY.txt"} '''
sharedY = parser.parse_args(['--run', localJsonY])
return (sharedY.run)
if computation_phase is 'computation':
parser = argparse.ArgumentParser(description='''read in coinstac args for local computation''')
parser.add_argument('--run', type=json.loads, help='grab coinstac args')
shared_Y = np.loadtxt(args["shared_Y"])
local_site1.Y[:local_site1.sharedRows, :] = shared_Y;
compAvgError1 = parser.parse_args(['--run', compAvgError])
C = compAvgError1.run['output']['error']
demeanAvg = (np.mean(local_site1.Y, 0));
demeanAvg[0]= compAvgError1.run['output']['avgX']
demeanAvg[1] = compAvgError1.run['output']['avgY']
local_site1.Y = demeanL(local_site1.Y, demeanAvg)
local_site1.Y, iY, local_site1.Q, C, local_site1.P = master_child(local_site1.Y, local_site1.dY, local_site1.iY, local_site1.gains, local_site1.n, local_site1.sharedRows, local_site1.P, iter, C)
local_site1.Y[local_site1.sharedRows:, :] = updateL(local_site1.Y[local_site1.sharedRows:, :], local_site1.iY[local_site1.sharedRows:, :])
# save local site sharedY data into file
with open("site1SharedY.txt", "w") as f1:
for i in range(0,local_site1.sharedRows):
f1.write(str(local_site1.Y[i][0]) + '\t')
f1.write(str(local_site1.Y[i][1]) + '\n')
# pass data to remote in json format
localJson = ''' {"localSite1SharedY": "site1SharedY.txt"} '''
sharedY = parser.parse_args(['--run', localJson])
# save local site sharedIY data into file
with open("site1SharedIY.txt", "w") as f1:
for i in range(0, local_site1.sharedRows):
f1.write(str(local_site1.iY[i][0]) + '\t')
f1.write(str(local_site1.iY[i][1]) + '\n')
# pass data to remote in json format
localJsonIY = ''' {"localSite1SharedIY": "site1SharedIY.txt"} '''
sharedIY = parser.parse_args(['--run', localJsonIY])
meanValue = (np.mean(local_site1.Y, 0));
comp = {'output': {'MeanX' : meanValue[0], 'MeanY' : meanValue[1], 'error': C}}
return (sharedY.run, sharedIY.run, json.dumps(comp, sort_keys=True,indent=4,separators=(',' ,':')))
if computation_phase is 'final':
'''It will add only local site data in the dictionary'''
parser = argparse.ArgumentParser(description='''read in coinstac args for local computation''')
parser.add_argument('--run', type=json.loads, help='grab coinstac args')
with open("local_site1Y.txt", "w") as f1:
for i in range(local_site1.sharedRows, len(local_site1.Y)):
f1.write(str(local_site1.Y[i][0]) + '\t')
f1.write(str(local_site1.Y[i][1]) + '\n')
localJsonY = ''' {"localSite1": "local_site1Y.txt"} '''
sharedY = parser.parse_args(['--run', localJsonY])
return(sharedY.run)
def local_2(args):
# corresponds to computation
#raise Exception('test test test')
local_sharedRows = args["cache"]["shared_rows"]
shared_Y = np.array(args["cache"]["shared_y"])
compAvgError1 = args["input"]["compAvgError"]
local_Y = np.array(args["cache"]["local_Y"])
local_dY = np.array(args["cache"]["local_dY"])
local_IY = np.array(args["cache"]["local_iY"])
local_P = np.array(args["cache"]["local_P"])
local_n = args["cache"]["local_n"]
local_gains = np.array(args["cache"]["local_gains"])
shared_Y = np.array(args["input"]["shared_Y"])
iter = args["input"]["number_of_iterations"]
a,b = shared_Y.shape
c,d = local_Y.shape
raise Exception( a, b,c,d,local_sharedRows)
#It should be the average one
local_Y[:local_sharedRows, :] = shared_Y
C = compAvgError1['error']
demeanAvg = (np.mean(local_Y, 0))
demeanAvg[0] = compAvgError1['avgX']
demeanAvg[1] = compAvgError1['avgY']
local_Y = demeanL(local_Y, demeanAvg)
local_Y, dY, local_IY, gains, n, sharedRows, P, C = master_child( local_Y, local_dY, local_IY, local_gains, local_n, local_sharedRows,local_P, iter, C)
local_Y[local_sharedRows:, :] = local_Y[local_sharedRows:, :] + local_IY[local_sharedRows:, :]
local_Shared_Y = local_Y[:local_sharedRows, :]
local_Shared_IY = local_IY[:local_sharedRows, :]
meanValue = (np.mean(local_Y, 0))
#local_Y_labels = np.loadtxt('test/input/simulatorRun/site1_y.txt') ## there is problem here
local_Y_labels = np.loadtxt('test/input/simulatorRun/test_high_dimensional_site_1_mnist_label.txt') ## there is problem here
#local_Y_labels = np.loadtxt('test/input/simulatorRun/site1_high_dimensional_label.txt') ## there is problem here
if iter > 2:
#local_Y_labels = np.loadtxt('test/input/simulatorRun/site1_y.txt') ## there is problem here
#local_Y_labels = np.loadtxt('test/input/simulatorRun/site1_high_dimensional_label.txt') ## there is problem here
local_Y_labels = np.loadtxt('test/input/simulatorRun/test_high_dimensional_site_1_mnist_label.txt') ## there is problem here
computation_output = {
"output": {
"MeanX": meanValue[0],
"MeanY": meanValue[1],
"error": C,
"local_Shared_iY": local_Shared_IY.tolist(),
"local_Shared_Y": local_Shared_Y.tolist(),
"local_Y": local_Y[local_sharedRows:, :].tolist(),
"local_Y_labels": local_Y_labels.tolist(),
"computation_phase": "local_2"
},
"cache": {
"local_Y": local_Y.tolist(),
"local_dY": local_dY.tolist(),
"local_iY": local_IY.tolist(),
"local_P": P.tolist(),
"local_n": n,
"local_gains": local_gains.tolist(),
"shared_rows": sharedRows,
"shared_y": local_Shared_Y.tolist()
}
}
## there is problem here
else:
computation_output = {
"output": {
"MeanX": meanValue[0],
"MeanY": meanValue[1],
"error": C,
"local_Shared_iY": local_Shared_IY.tolist(),
"local_Shared_Y": local_Shared_Y.tolist(),
"local_Y": local_Y[local_sharedRows:, :].tolist(),
"local_Y_labels": local_Y_labels.tolist(),
"computation_phase": "local_2"
},
"cache": {
"local_Y": local_Y.tolist(),
"local_dY": local_dY.tolist(),
"local_iY": local_IY.tolist(),
"local_P": P.tolist(),
"local_n": n,
"local_gains": local_gains.tolist(),
"shared_rows": sharedRows,
"shared_y": local_Shared_Y.tolist()
}
}
return json.dumps(computation_output)