def upload_parameters_ori(self):
self.lock.acquire()
model = self.tensorgraph.get_parameters()
text = comp.preprocess(model)
self.sw.accumulate('preprocess')
self.status['GlobalStep'] = self.ps.upload(self.id, text)
self.lock.release()
def __init__(self, ps_id, predict_service=True):
self.ip = cluster_spec['ps'][ps_id]['IP']
self.port = cluster_spec['ps'][ps_id]['Port']
self.service_list = list()
if predict_service:
self.service_list.append(self.start_predict_service)
self.service_list.append(self.start_store_service)
# training model
config = gpu_configure()
with CNN(config) as cnn_graph:
compressed_parameters = comp.preprocess(cnn_graph.get_parameters())
check_size(compressed_parameters)
self.model = compressed_parameters
# message queue used between store service and predict service
self.mes_queue = Queue.Queue(maxsize=0)
# handle for store service
handler = Dispatcher(self.model, self.mes_queue)
self.server = init_server(self.ip, self.port, handler)
# handle for predict service
self.predictor = fprd.Predictor(
[len(cluster_spec['cn']) * 4, 10,
len(cluster_spec['cn'])], cluster_spec, handler, self.mes_queue)
super(ParameterServer, self).__init__()
def __init__(self, ps_id):
self.ip = cluster_spec['ps'][ps_id]['IP']
self.port = cluster_spec['ps'][ps_id]['Port']
self.service_list = list()
self.service_list.append(self.start_store_service)
# training model
config = gpu_configure()
self.cnn_graph = CNN(config)
compressed_parameters = comp.preprocess(
self.cnn_graph.get_parameters())
check_size(compressed_parameters)
# handle for store service
handler = Dispatcher(self.cnn_graph)
self.server = init_server(self.ip, self.port, handler)
super(ParameterServer, self).__init__()
def download(self):
self.lock.acquire()
model = self.cnn_graph.get_parameters()
text = comp.preprocess(model)
self.lock.release()
return text
def upload_parameters_opt(self):
model = self.tensorgraph.get_parameters()
text = comp.preprocess(model)
self.sw.accumulate('preprocess')
self.status['GlobalStep'] = self.ps.getGlobalStatus() + 1
self.ps.non_blocking_upload(self.id, text)
def upload_gradients(self, grads):
self.sw.reset()
text = comp.preprocess(grads)
self.sw.accumulate('preprocess')
self.ps.upload(self.id, text)
self.sw.accumulate('upload_gradients')
import thriftpy
import time
test_thrift = thriftpy.load("benchmarkmessage.thrift",
module_name="test_thrift")
from thriftpy.rpc import make_client
client = make_client(test_thrift.Helloworld, '127.0.0.1', 6000)
import compression as comp
import sys
sys.path.append('../../src')
from CIFAR10_CNN import CIFAR10_CNN as CNN
cnn_graph = CNN()
text = comp.preprocess(cnn_graph.get_parameters())
print len(text)
ts = time.time()
ret = client.echo(text)
te = time.time()
print "networking elapsed time : %fs" % (te - ts)
model = comp.deprocess(ret, cnn_graph.get_configure())
tdataset, tlabels, _, _, _, _ = open_cifar10_dataset(0, 100)
gradients = cnn_graph.get_gradients(tdataset, tlabels)
for i in range(10):
print "[Learning]epoch %d......" % i
cnn_graph.optimize(tdataset, tlabels)
parameters = cnn_graph.get_parameters()
print "type of gradients : ", gradients[0].dtype
print "shape of gradients",
for layer in gradients:
print layer.shape,
print ""
tStart = time.time()
comp.preprocess(gradients)
tEnd = time.time()
print "compress grad time : %fs" % (tEnd - tStart)
print "type of parameters : ", parameters[0].dtype
print "shape of parameters",
for layer in parameters:
print layer.shape,
print ""
tStart = time.time()
comp.preprocess(parameters)
tEnd = time.time()
print "compress para time : %fs" % (tEnd - tStart)
flat_grad = comp.to_one_dim(gradients)
import matplotlib.pyplot as plt