# -*- coding: utf-8 -*-
from socket import *
import struct
import thrift.Thrift as Thrift
import thrift.protocol.TBinaryProtocol as TBinaryProtocol
import thrift.protocol.TCompactProtocol as TCompactProtocol
import thrift.protocol.TJSONProtocol as TJSONProtocol
import thrift.transport.TTransport as TTransport
import thrift.msg_def.ttypes as ffrpc_msg_def
from gen_py.echo import ttypes
g_WriteTMemoryBuffer = TTransport.TMemoryBuffer()
g_WriteTBinaryProtocol = TBinaryProtocol.TBinaryProtocol(g_WriteTMemoryBuffer)
g_ReadTMemoryBuffer = TTransport.TMemoryBuffer()
g_ReadTBinaryProtocol = TBinaryProtocol.TBinaryProtocol(g_ReadTMemoryBuffer)
def encode_msg(msg, g_protocol = 0):
#debub print('encode_msg begin', msg)
if hasattr(msg, 'thrift_spec'):
g_WriteTMemoryBuffer.cstringio_buf.truncate()
g_WriteTMemoryBuffer.cstringio_buf.seek(0)
if g_protocol == 1:
proto = TJSONProtocol.TJSONProtocol(g_WriteTMemoryBuffer)
proto.writeMessageBegin(msg.__class__.__name__, 0, 0);
msg.write(proto)
proto.writeMessageEnd();
else:
g_WriteTBinaryProtocol.writeMessageBegin(msg.__class__.__name__, 0, 0);
msg.write(g_WriteTBinaryProtocol)
import sys
import time
sys.path.append('./gen-py')
from rpc import NewsServlet
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer
socket = TSocket.TSocket('localhost', 12345)
transport = TTransport.TBufferedTransport(socket) #缓冲
protocol = TBinaryProtocol.TBinaryProtocol(transport) #协议
client = NewsServlet.Client(protocol)
transport.open()
result = client.get_news_detail()
print result
def __init__(self, url, port):
self.transport = TSocket.TSocket(url, port)
self.transport = TTransport.TFramedTransport(self.transport)
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = ServerSvc.Client(self.protocol)
def lmem_loopback_dfe(size, in_a, in_b):
"""LMemLoopback DFE implementation."""
try:
start_time = time.time()
# Make socket
socket = TSocket.TSocket('localhost', 9090)
# _buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(socket)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = LMemLoopbackService.Client(protocol)
print ('Creating a client:\t\t\t\t%.5lfs' %
(time.time() - start_time))
# Connect!
start_time = time.time()
transport.open()
print ('Opening connection:\t\t\t\t%.5lfs' %
(time.time() - start_time))
size_bytes = size * 4
# Initialize maxfile
start_time = time.time()
max_file = client.LMemLoopback_init()
print ('Initializing maxfile:\t\t\t\t%.5lfs' %
(time.time() - start_time))
# Load DFE
start_time = time.time()
max_engine = client.max_load(max_file, '*')
print ('Loading DFE:\t\t\t\t\t%.5lfs' %
(time.time() - start_time))
# Allocate and send input streams to server
start_time = time.time()
address_in_a = client.malloc_int32_t(size)
client.send_data_int32_t(address_in_a, in_a)
address_in_b = client.malloc_int32_t(size)
client.send_data_int32_t(address_in_b, in_b)
print ('Sending input data:\t\t\t\t%.5lfs' %
(time.time() - start_time))
# Allocate memory for output stream on server
start_time = time.time()
address_out_data = client.malloc_int32_t(size)
print ('Allocating memory for output stream on server:\t%.5lfs'%
(time.time() - start_time))
# Write to LMem
start_time = time.time()
actions_write_lmem = (
LMemLoopback_writeLMem_actions_t_struct(0, size_bytes,
address_in_a))
address_actions_write_lmem = (
client.send_LMemLoopback_writeLMem_actions_t(actions_write_lmem))
client.LMemLoopback_writeLMem_run(max_engine,
address_actions_write_lmem)
actions_write_lmem = (
LMemLoopback_writeLMem_actions_t_struct(size_bytes, size_bytes,
address_in_b))
address_actions_write_lmem = (
client.send_LMemLoopback_writeLMem_actions_t(actions_write_lmem))
client.LMemLoopback_writeLMem_run(max_engine,
address_actions_write_lmem)
print ('Writing to LMem:\t\t\t\t%.5lfs'%
(time.time() - start_time))
# Action default
start_time = time.time()
actions = LMemLoopback_actions_t_struct(size)
address_actions = client.send_LMemLoopback_actions_t(actions)
client.LMemLoopback_run(max_engine, address_actions)
print ('LMemLoopback time:\t\t\t\t%.5lfs'%
(time.time() - start_time))
# Read from LMem
start_time = time.time()
actions_read_lmem = (
LMemLoopback_readLMem_actions_t_struct(2 * size_bytes, size_bytes,
address_out_data))
address_actions_read_lmem = (
client.send_LMemLoopback_readLMem_actions_t(actions_read_lmem))
client.LMemLoopback_readLMem_run(max_engine, address_actions_read_lmem)
print ('Reading from LMem:\t\t\t\t%.5lfs'%
(time.time() - start_time))
# Unload DFE
start_time = time.time()
client.max_unload(max_engine)
print ('Unloading DFE:\t\t\t\t\t%.5lfs' %
(time.time() - start_time))
# Get output stream from server
start_time = time.time()
out_data = client.receive_data_int32_t(address_out_data, size)
print ('Getting output stream:\t(size = %d bit)\t%.5lfs' %
((size * 32), (time.time() - start_time)))
# Free allocated memory for streams on server
start_time = time.time()
client.free(address_in_a)
client.free(address_in_b)
client.free(address_out_data)
client.free(address_actions_write_lmem)
client.free(address_actions)
client.free(address_actions_read_lmem)
print ('Freeing allocated memory for streams on server:\t%.5lfs' %
(time.time() - start_time))
# Free allocated maxfile data
start_time = time.time()
client.LMemLoopback_free()
print ('Freeing allocated maxfile data:\t\t\t%.5lfs' %
(time.time() - start_time))
# Close!
start_time = time.time()
transport.close()
print ('Closing connection:\t\t\t\t%.5lfs' %
(time.time() - start_time))
except Thrift.TException, thrift_exceptiion:
print '%s' % (thrift_exceptiion.message)
sys.exit(-1)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TCompactProtocol
from accumulo import AccumuloProxy
from accumulo.ttypes import *
transport = TSocket.TSocket('localhost', 42424)
transport = TTransport.TFramedTransport(transport)
protocol = TCompactProtocol.TCompactProtocol(transport)
client = AccumuloProxy.Client(protocol)
transport.open()
login = client.login('root', {'password': '******'})
print client.listTables(login)
testtable = "pythontest"
if not client.tableExists(login, testtable):
client.createTable(login, testtable, True, TimeType.MILLIS)
row1 = {
'a': [
ColumnUpdate('a', 'a', value='value1'),
def _serialize(self, obj): trans = TTransport.TMemoryBuffer() prot = self.protocol_factory.getProtocol(trans) obj.write(prot) return trans.getvalue()
def get_client_transport(self, service):
host, port = service.get_host_port()
transport = TTransport.TFramedTransport(TSocket.TSocket(host, port))
protocol = TBinaryProtocol.TBinaryProtocol(transport)
transport.open()
return LucidaService.Client(protocol), transport
def main():
transport = TSocket.TSocket('localhost', 9081)
transport = TTransport.TBufferedTransport(transport)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
device = Device.Client(TMultiplexedProtocol(protocol, "Device"))
fridge = Fridge.Client(TMultiplexedProtocol(protocol, "Fridge"))
camera = Camera.Client(TMultiplexedProtocol(protocol, "Camera"))
sensor = Sensor.Client(TMultiplexedProtocol(protocol, "TemperatureSensor"))
try:
transport.open()
while True:
print("Write 1. to list all devices ")
print("Write 2. to operate on fridges ")
print("Write 3. to operate on cameras ")
print("Write 4. to operate sensors ")
action = input()
if action == '1':
for dev in device.getEveryDevice():
print(dev)
elif action == '2':
print("Write down in format operationNumber/arg1_arg2...")
print("1.start freezing/id")
print("2.stop freezing/id")
print("3.change temp/id_value")
print("4.get temp and is freezing/id")
print("5.turn on/id")
print("6.turn off/id")
print("7.get state/id")
target = input()
command = target.split("/")
operation = command[0]
args = command[1]
if operation == '1':
print(fridge.startFreezing(args))
elif operation == '2':
print(fridge.stopFreezing(args))
elif operation == '3':
arg_split = args.split("_")
dev_id = arg_split[0]
value = arg_split[1]
print(fridge.changeTemperatureTo(dev_id, int(value)))
elif operation == '4':
print(fridge.getTemperatureAndIsFreezing(args))
elif operation == '5':
print(fridge.turnOn(args))
elif operation == '6':
print(fridge.turnOff(args))
elif operation == '7':
print(fridge.getState(args))
else:
pass
elif action == '3':
print("Write down in format operationNumber/arg1_arg2...")
print(
"1.move to position and zoom/id_verticalAngle_horizontalAngle_zoom"
)
print("2.turn on/id")
print("3.turn off/id")
print("4.get state/id")
print("5.to go to menu")
target = input()
command = target.split("/")
operation = command[0]
args = command[1]
if operation == '1':
argSplit = args.split("_")
position = AngularPosition(int(argSplit[1]),
int(argSplit[2]))
print(
camera.moveToPositionAndZoom(argSplit[0], position,
int(argSplit[3])))
elif operation == '2':
print(camera.turnOn(args))
elif operation == '3':
print(camera.turnOff(args))
elif operation == '4':
print(camera.getState(args))
else:
pass
elif action == '4':
print("Write down in format operationNumber/arg1_arg2...")
print("1.change alarm temperature value/id_alarmValue")
print("2.get temperature/id")
print("3.turn on/id")
print("4.turn off/id")
print("5.get state/id")
print("6.to go to menu")
target = input()
command = target.split("/")
operation = command[0]
args = command[1]
if operation == '1':
argSplit = args.split("_")
sensor.changeAlarmTemperatureValue(argSplit[0],
int(argSplit[1]))
elif operation == '2':
print(sensor.getTemperature(args))
elif operation == '3':
print(sensor.turnOn(args))
elif operation == '4':
print(sensor.turnOff(args))
elif operation == '5':
print(sensor.getState(args))
else:
pass
else:
pass
except InvalidArguments as iarg:
print(iarg)
main()
except IndexError as ierr:
print("WRONG PARSED COMMAND - TOO LESS INDICES")
transport.close()
main()
except ValueError as verr:
print("WRONG PARSED COMMAND - WRONG VALUE")
transport.close()
main()
except TTransportException as tterr:
print("Connection closed - Trying to reconnect")
transport.close()
main()
except ConnectionResetError as conerr:
print("Connection not established - waiting for another reconnect")
time.sleep(10)
transport.close()
main()
transport.close()
def handler(event, context):
start_time = time.time()
bucket = event['bucket_name']
worker_index = event['rank']
num_workers = event['num_workers']
key = event['file'].split(",")
num_classes = event['num_classes']
num_features = event['num_features']
pos_tag = event['pos_tag']
num_epochs = event['num_epochs']
learning_rate = event['learning_rate']
batch_size = event['batch_size']
host = event['host']
port = event['port']
print('bucket = {}'.format(bucket))
print('number of workers = {}'.format(num_workers))
print('worker index = {}'.format(worker_index))
print("file = {}".format(key))
print('number of workers = {}'.format(num_workers))
print('worker index = {}'.format(worker_index))
print('num epochs = {}'.format(num_epochs))
print('num classes = {}'.format(num_classes))
print('num features = {}'.format(num_features))
print('positive tag = {}'.format(pos_tag))
print('learning rate = {}'.format(learning_rate))
print("batch_size = {}".format(batch_size))
print("host = {}".format(host))
print("port = {}".format(port))
# Set thrift connection
# Make socket
transport = TSocket.TSocket(host, port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
t_client = ParameterServer.Client(protocol)
# Connect!
transport.open()
# test thrift connection
ps_client.ping(t_client)
print("create and ping thrift server >>> HOST = {}, PORT = {}".format(
host, port))
# read file from s3
file = get_object(bucket, key[0]).read().decode('utf-8').split("\n")
dataset = DenseLibsvmDataset(file, num_features, pos_tag)
if len(key) > 1:
for more_key in key[1:]:
file = get_object(bucket,
more_key).read().decode('utf-8').split("\n")
dataset.add_more(file)
print("read data cost {} s".format(time.time() - start_time))
parse_start = time.time()
total_count = dataset.__len__()
pos_count = 0
for i in range(total_count):
if dataset.__getitem__(i)[1] == 1:
pos_count += 1
print("{} positive observations out of {}".format(pos_count, total_count))
print("parse data cost {} s".format(time.time() - parse_start))
preprocess_start = time.time()
# Creating data indices for training and validation splits:
dataset_size = len(dataset)
indices = list(range(dataset_size))
split = int(np.floor(VALIDATION_RATIO * dataset_size))
if SHUFFLE_DATASET:
np.random.seed(RANDOM_SEED)
np.random.shuffle(indices)
train_indices, val_indices = indices[split:], indices[:split]
# Creating PT data samplers and loaders:
train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(val_indices)
train_loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
sampler=train_sampler)
validation_loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
sampler=valid_sampler)
print("preprocess data cost {} s, dataset size = {}".format(
time.time() - preprocess_start, dataset_size))
model = LogisticRegression(NUM_FEATURES, NUM_CLASSES)
# Loss and Optimizer
# Softmax is internally computed.
# Set parameters to be updated.
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE)
# register model
model_name = "w.b"
weight_shape = model.linear.weight.data.numpy().shape
weight_length = weight_shape[0] * weight_shape[1]
bias_shape = model.linear.bias.data.numpy().shape
bias_length = bias_shape[0]
model_length = weight_length + bias_length
ps_client.register_model(t_client, worker_index, model_name, model_length,
num_workers)
ps_client.exist_model(t_client, model_name)
print("register and check model >>> name = {}, length = {}".format(
model_name, model_length))
# Training the Model
train_start = time.time()
iter_counter = 0
for epoch in range(NUM_EPOCHS):
epoch_start = time.time()
for batch_index, (items, labels) in enumerate(train_loader):
print("------worker {} epoch {} batch {}------".format(
worker_index, epoch, batch_index))
batch_start = time.time()
# pull latest model
ps_client.can_pull(t_client, model_name, iter_counter,
worker_index)
latest_model = ps_client.pull_model(t_client, model_name,
iter_counter, worker_index)
model.linear.weight = Parameter(
torch.from_numpy(
np.asarray(latest_model[:weight_length],
dtype=np.double).reshape(weight_shape)))
model.linear.bias = Parameter(
torch.from_numpy(
np.asarray(latest_model[weight_length:],
dtype=np.double).reshape(bias_shape[0])))
items = Variable(items.view(-1, NUM_FEATURES))
labels = Variable(labels)
# Forward + Backward + Optimize
optimizer.zero_grad()
outputs = model(items.double())
loss = criterion(outputs, labels)
loss.backward()
# flatten and concat gradients of weight and bias
w_b_grad = np.concatenate(
(model.linear.weight.grad.data.numpy().flatten(),
model.linear.bias.grad.data.numpy().flatten()))
cal_time = time.time() - batch_start
# push gradient to PS
sync_start = time.time()
ps_client.can_push(t_client, model_name, iter_counter,
worker_index)
ps_client.push_grad(t_client, model_name, w_b_grad, LEARNING_RATE,
iter_counter, worker_index)
ps_client.can_pull(t_client, model_name, iter_counter + 1,
worker_index) # sync all workers
sync_time = time.time() - sync_start
print(
'Epoch: [%d/%d], Step: [%d/%d] >>> Time: %.4f, Loss: %.4f, epoch cost %.4f, '
'batch cost %.4f s: cal cost %.4f s and communication cost %.4f s'
% (epoch + 1, NUM_EPOCHS, batch_index + 1,
len(train_indices) / BATCH_SIZE, time.time() - train_start,
loss.data, time.time() - epoch_start,
time.time() - batch_start, cal_time, sync_time))
iter_counter += 1
# Test the Model
correct = 0
total = 0
test_loss = 0
for items, labels in validation_loader:
items = Variable(items.view(-1, NUM_FEATURES))
labels = Variable(labels)
outputs = model(items)
test_loss += criterion(outputs, labels).data
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum()
print(
'Time = %.4f, accuracy of the model on the %d test samples: %d %%, loss = %f'
% (time.time() - train_start, len(val_indices),
100 * correct / total, test_loss))
end_time = time.time()
print("Elapsed time = {} s".format(end_time - start_time))
def main():
p = sys.argv[1]
# Make socket
transport = TSocket.TSocket('localhost', p)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = Graph.Client(protocol)
# Connect!
transport.open()
client.ping()
print(client.add_upd_vertex(name_hash("Alice"), pessoa, "Alice", 0))
print(client.add_upd_vertex(name_hash("Bob"), pessoa, "Bob", 0))
print(client.add_upd_vertex(name_hash("Carla"), pessoa, "Carla", 0))
print(client.add_upd_vertex(name_hash("Dan"), pessoa, "Dan", 0))
print(client.add_upd_vertex(name_hash("Eve"), pessoa, "Eve", 0))
print(
client.add_upd_vertex(name_hash("Blade Runner"), filme, "Blade Runner",
0))
print(
client.add_upd_vertex(name_hash("Terminator 2"), filme, "Terminator 2",
0))
print(
client.add_upd_vertex(name_hash("The Matrix"), filme, "The Matrix", 0))
print(
client.add_upd_vertex(name_hash("Ex Machina"), filme, "Ex Machina", 0))
print(client.add_upd_vertex(name_hash("Her"), filme, "Her", 0))
print(
client.add_upd_vertex(name_hash("Harrison Ford"), cast,
"Harrison Ford", 0))
print(
client.add_upd_vertex(name_hash("Arnold Schwarzenegger"), cast,
"Arnold Schwarzenegger", 0))
print(
client.add_upd_vertex(name_hash("Keanu Reeves"), cast, "Keanu Reeves",
0))
print(
client.add_upd_vertex(name_hash("Domhnall Gleeson"), cast,
"Domhnall Gleeson", 0))
print(
client.add_upd_vertex(name_hash("Joaquin Phoenix"), cast,
"Joaquin Phoenix", 0))
print(client.add_upd_vertex(name_hash("oldschool"), grupo, "oldschool", 0))
print(client.add_upd_vertex(name_hash("scifi"), grupo, "scifi", 0))
print(
client.add_upd_edge(name_hash("Blade Runner"),
name_hash("Harrison Ford"), cst, True))
print(
client.add_upd_edge(name_hash("Terminator 2"),
name_hash("Arnold Schwarzenegger"), cst, True))
print(
client.add_upd_edge(name_hash("The Matrix"), name_hash("Keanu Reeves"),
cst, True))
print(
client.add_upd_edge(name_hash("Ex Machina"),
name_hash("Domhnall Gleeson"), cst, True))
print(
client.add_upd_edge(name_hash("Her"), name_hash("Joaquin Phoenix"),
cst, True))
print(
client.add_upd_edge(name_hash("Alice"), name_hash("Terminator 2"),
91.0, True))
print(
client.add_upd_edge(name_hash("Alice"), name_hash("Ex Machina"), 79.0,
True))
print(
client.add_upd_edge(name_hash("Bob"), name_hash("The Matrix"), 90.0,
True))
print(client.add_upd_edge(name_hash("Carla"), name_hash("Her"), 75.0,
True))
print(
client.add_upd_edge(name_hash("Dan"), name_hash("Blade Runner"), 75.0,
True))
print(
client.add_upd_edge(name_hash("Dan"), name_hash("Terminator 2"), 80.0,
True))
print(
client.add_upd_edge(name_hash("Dan"), name_hash("The Matrix"), 79.0,
True))
print(
client.add_upd_edge(name_hash("Eve"), name_hash("Ex Machina"), 60.0,
True))
print(client.add_upd_edge(name_hash("scifi"), name_hash("Alice"), 0,
False))
print(client.add_upd_edge(name_hash("scifi"), name_hash("Bob"), 0, False))
print(client.add_upd_edge(name_hash("scifi"), name_hash("Carla"), 0,
False))
print(client.add_upd_edge(name_hash("scifi"), name_hash("Dan"), 0, False))
print(
client.add_upd_edge(name_hash("oldschool"), name_hash("Alice"), 0,
False))
print(
client.add_upd_edge(name_hash("oldschool"), name_hash("Dan"), 0,
False))
# Close!
transport.close()
def handler(argv):
from archived.pytorch_model import MobileNet
start_time = time.time()
worker_index = argv[1]
num_worker = argv[2]
print('number of workers = {}'.format(num_worker))
print('worker index = {}'.format(worker_index))
# Set thrift connection
# Make socket
transport = TSocket.TSocket(constants.HOST, constants.PORT)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
t_client = ParameterServer.Client(protocol)
# Connect!
transport.open()
# test thrift connection
ps_client.ping(t_client)
print("create and ping thrift server >>> HOST = {}, PORT = {}".format(
constants.HOST, constants.PORT))
preprocess_start = time.time()
batch_size = 200
train_path = "../../dataset/training.pt"
test_path = "../../dataset/test.pt"
trainset = torch.load(train_path)
testset = torch.load(test_path)
train_loader = torch.utils.data.DataLoader(trainset,
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(testset,
batch_size=batch_size,
shuffle=False)
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck')
device = 'cpu'
print("preprocess data cost {} s".format(time.time() - preprocess_start))
model = MobileNet()
model = model.to(device)
# Loss and Optimizer
# Softmax is internally computed.
# Set parameters to be updated.
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(),
lr=learning_rate,
momentum=0.9,
weight_decay=5e-4)
# register model
model_name = "mobilenet"
parameter_shape = []
parameter_length = []
model_length = 0
for param in model.parameters():
tmp_shape = 1
parameter_shape.append(param.data.numpy().shape)
for w in param.data.numpy().shape:
tmp_shape *= w
parameter_length.append(tmp_shape)
model_length += tmp_shape
print("model_length = {}".format(model_length))
model_length = 830
"""
ps_client.register_model(t_client, worker_index, model_name, model_length, num_worker)
#ps_client.exist_model(t_client, model_name)
print("register and check model >>> name = {}, length = {}".format(model_name, model_length))
"""
# Training the Model
train_start = time.time()
iter_counter = 0
for epoch in range(num_epochs):
epoch_start = time.time()
for batch_index, (inputs, targets) in enumerate(train_loader):
print("------worker {} epoch {} batch {}------".format(
worker_index, epoch, batch_index))
batch_start = time.time()
# pull latest model
ps_client.can_pull(t_client, model_name, iter_counter,
worker_index)
latest_model = ps_client.pull_model(t_client, model_name,
iter_counter, worker_index)
pos = 0
for layer_index, param in enumerate(model.parameters()):
param.data = Variable(
torch.from_numpy(
np.asarray(latest_model[pos:pos +
parameter_length[layer_index]],
dtype=np.float32).reshape(
parameter_shape[layer_index])))
pos += parameter_length[layer_index]
# Forward + Backward + Optimize
inputs, targets = inputs.to(device), targets.to(device)
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs, targets)
loss.backward()
# flatten and concat gradients of weight and bias
param_grad = np.ones((1)) * (-1)
for param in model.parameters():
print("shape of layer = {}".format(
param.data.numpy().flatten().shape))
param_grad = np.concatenate(
(param_grad, param.data.numpy().flatten()))
param_grad = np.delete(param_grad, 0)
print("model_length = {}".format(param_grad.shape))
cal_time = time.time() - batch_start
# push gradient to PS
sync_start = time.time()
ps_client.can_push(t_client, model_name, iter_counter,
worker_index)
ps_client.push_grad(t_client, model_name, param_grad,
learning_rate, iter_counter, worker_index)
ps_client.can_pull(t_client, model_name, iter_counter + 1,
worker_index) # sync all workers
sync_time = time.time() - sync_start
sync_time = time.time()
print(
'Epoch: [%d/%d], Step: [%d/%d] >>> Time: %.4f, Loss: %.4f, epoch cost %.4f, '
'batch cost %.4f s: cal cost %.4f s and communication cost %.4f s'
% (epoch + 1, num_epochs, batch_index + 1,
len(train_loader) / batch_size, time.time() - train_start,
loss.data, time.time() - epoch_start,
time.time() - batch_start, cal_time, sync_time))
iter_counter += 1
test(epoch, model, test_loader, criterion, device)
from thrift.transport import TSocket, TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer
from thrift.server import TNonblockingServer
from genpy.calc.ICalc import Processor
import time
class CalcHandler(object):
def add(self, num1, num2):
print "starting...\n"
time.sleep(20)
return num1 + num2
if __name__ == '__main__':
HANDLER = CalcHandler()
PROCESSOR = Processor(HANDLER)
TRANSPORT = TSocket.TServerSocket('localhost', 9090)
TFACTORY = TTransport.TBufferedTransportFactory()
PFACTORY = TBinaryProtocol.TBinaryProtocolFactory()
SERVER = TServer.TThreadPoolServer(PROCESSOR, TRANSPORT, TFACTORY,
PFACTORY)
print 'starting server...'
SERVER.serve()
print 'Done.'
def __init__(self, port):
self.transport = TSocket.TSocket('localhost', port)
self.transport = TTransport.TBufferedTransport(self.transport)
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = Client(protocol)
def main(cfg, reqhandle, resphandle):
if cfg.unix:
if cfg.addr == "":
sys.exit("invalid unix domain socket: {}".format(cfg.addr))
socket = TSocket.TSocket(unix_socket=cfg.addr)
else:
try:
(host, port) = cfg.addr.rsplit(":", 1)
if host == "":
host = "localhost"
socket = TSocket.TSocket(host=host, port=int(port))
except ValueError:
sys.exit("invalid address: {}".format(cfg.addr))
transport = TRecordingTransport(socket, reqhandle, resphandle)
if cfg.transport == "framed":
transport = TTransport.TFramedTransport(transport)
elif cfg.transport == "unframed":
transport = TTransport.TBufferedTransport(transport)
elif cfg.transport == "header":
transport = THeaderTransport.THeaderTransport(
transport,
client_type=THeaderTransport.CLIENT_TYPE.HEADER,
)
if cfg.headers is not None:
pairs = cfg.headers.split(",")
for p in pairs:
key, value = p.split("=")
transport.set_header(key, value)
if cfg.protocol == "binary":
transport.set_protocol_id(THeaderTransport.T_BINARY_PROTOCOL)
elif cfg.protocol == "compact":
transport.set_protocol_id(THeaderTransport.T_COMPACT_PROTOCOL)
else:
sys.exit(
"header transport cannot be used with protocol {0}".format(
cfg.protocol))
else:
sys.exit("unknown transport {0}".format(cfg.transport))
transport.open()
if cfg.protocol == "binary":
protocol = TBinaryProtocol.TBinaryProtocol(transport)
elif cfg.protocol == "compact":
protocol = TCompactProtocol.TCompactProtocol(transport)
elif cfg.protocol == "json":
protocol = TJSONProtocol.TJSONProtocol(transport)
elif cfg.protocol == "finagle":
protocol = TFinagleProtocol(transport, client_id="thrift-playground")
else:
sys.exit("unknown protocol {0}".format(cfg.protocol))
if cfg.service is not None:
protocol = TMultiplexedProtocol.TMultiplexedProtocol(
protocol, cfg.service)
client = Example.Client(protocol)
try:
if cfg.method == "ping":
client.ping()
print("client: pinged")
elif cfg.method == "poke":
client.poke()
print("client: poked")
elif cfg.method == "add":
if len(cfg.params) != 2:
sys.exit("add takes 2 arguments, got: {0}".format(cfg.params))
a = int(cfg.params[0])
b = int(cfg.params[1])
v = client.add(a, b)
print("client: added {0} + {1} = {2}".format(a, b, v))
elif cfg.method == "execute":
param = Param(return_fields=cfg.params,
the_works=TheWorks(
field_1=True,
field_2=0x7f,
field_3=0x7fff,
field_4=0x7fffffff,
field_5=0x7fffffffffffffff,
field_6=-1.5,
field_7=u"string is UTF-8: \U0001f60e",
field_8=b"binary is bytes: \x80\x7f\x00\x01",
field_9={
1: "one",
2: "two",
3: "three"
},
field_10=[1, 2, 4, 8],
field_11=set(["a", "b", "c"]),
field_12=False,
))
try:
result = client.execute(param)
print("client: executed {0}: {1}".format(param, result))
except AppException as e:
print("client: execute failed with IDL Exception: {0}".format(
e.why))
else:
sys.exit("unknown method {0}".format(cfg.method))
except Thrift.TApplicationException as e:
print("client exception: {0}: {1}".format(e.type, e.message))
if cfg.request is None:
#req = "".join(["%02X " % ord(x) for x in reqhandle.getvalue()]).strip()
req = "".join(["%02X " % x for x in reqhandle.getvalue()]).strip()
print("request: {}".format(req))
if cfg.response is None:
resp = "".join(["%02X " % x for x in resphandle.getvalue()]).strip()
print("response: {}".format(resp))
transport.close()
def addNode(self, host, port):
new_node_key = hashlib.sha256(host + ":" + str(port)).hexdigest()
new_node = NodeID()
new_node.ip = host
new_node.port = port
new_node.id = new_node_key
## Make its fingertable (current node to find all answers)
new_fingertable = []
for i in range(256):
finger_key = \
hex((int(new_node_key, 16) + (2**i)) % (2**256)).strip("0x").strip('L')
succ = self.findSucc(finger_key)
if(self.contains(new_node_key, finger_key, succ.id)):
new_fingertable.append(new_node)
else:
new_fingertable.append(self.findSucc(finger_key))
## Launch new server
Popen(["python2", "server.py"] + [str(port)])
sleep(3)
## Update other nodes fingertables
transport = TSocket.TSocket(host, port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = FileStore.Client(protocol)
transport.open()
client.setFingertable(new_fingertable)
transport.close()
new_node_succ = self.findSucc(new_node_key)
new_node_pred = self.findPred(new_node_key)
succ_files = []
if(new_node_succ.id == self.myNode.id):
succ_files = self.getFiles()
else:
## Update other nodes fingertables
transport = TSocket.TSocket(new_node_succ.ip, new_node_succ.port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = FileStore.Client(protocol)
transport.open()
succ_files = client.getFiles()
transport.close()
# New node must take ownership of some files from successor,
# while successor must relinquish ownership of some
for file_obj in succ_files:
if(self.contains(file_obj.meta.contentHash,\
new_node_pred.id,\
new_node_key)):
## File belongs to new node
if(new_node_succ.id == self.myNode.id):
self.removeFile(file_obj.meta.contentHash)
else:
## Update other nodes fingertables
transport = TSocket.TSocket(new_node_succ.ip, new_node_succ.port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = FileStore.Client(protocol)
transport.open()
client.removeFile(file_obj.meta.contentHash)
## Update other nodes fingertables
transport = TSocket.TSocket(host, port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = FileStore.Client(protocol)
transport.open()
client.addFile(file_obj)
# Apache Thrift Hello World Python Server
import sys
sys.path.append("gen-py")
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer
from hello import helloSvc
class HelloHandler(helloSvc.Iface):
def getMessage(self, name):
print("[Server] Handling client request: " + name)
return "Hello " + name
handler = HelloHandler()
proc = helloSvc.Processor(handler)
trans_svr = TSocket.TServerSocket(port=9090)
trans_fac = TTransport.TBufferedTransportFactory()
proto_fac = TBinaryProtocol.TBinaryProtocolFactory()
server = TServer.TSimpleServer(proc, trans_svr, trans_fac, proto_fac)
print("[Server] Started")
server.serve()
def wrappee(*args, **kwargs):
logging.debug('in decorator before wrapee with flag ' +
str(decorator_self._thrift_class.__name__))
dec_name = str(
decorator_self._thrift_class.__name__).split(".")[-1]
print(dec_name)
assert "use_rpc" in kwargs.keys() and "server" in kwargs.keys(), \
"Wrapped function must be subclass of RPC."
if "use_rpc" in kwargs.keys() and "server" in kwargs.keys() \
and kwargs["use_rpc"] and kwargs["server"]:
from thrift.protocol import TBinaryProtocol
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.server import TServer
handler = original_clazz(*args, **kwargs)
processor = decorator_self._thrift_class.Processor(handler)
transport = TSocket.TServerSocket(port=decorator_self._port)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
self.__inst = handler
server = TServer.TThreadPoolServer(processor, transport,
tfactory, pfactory)
server.setNumThreads(cpu_count())
logging.info("Serving: " + dec_name)
server.serve()
logging.info('Done: ' + dec_name)
elif "use_rpc" in kwargs.keys() and "server" in kwargs.keys() \
and kwargs["use_rpc"] and not kwargs["server"]:
from thrift.protocol import TBinaryProtocol
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.server import TServer
# Make socket
self._transport = TSocket.TSocket(host=dec_name,
port=decorator_self._port)
# Buffering is critical. Raw sockets are very slow
self._transport = TTransport.TBufferedTransport(
self._transport)
# Connect!
self._transport.open()
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(self._transport)
# Create a client to use the protocol encoder
client = decorator_self._thrift_class.Client(protocol)
logging.info("Client (" + dec_name +
") connected to server: " +
str(self._transport.isOpen()))
return client
else:
return original_clazz(*args, **kwargs)
def __init__(self):
transport = TSocket.TSocket(config.IP, config.PORT)
self.transport = TTransport.TBufferedTransport(transport)
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = ClassificationService.Client(protocol)
self.transport.open()
def _deserialize(self, objtype, data): prot = self.protocol_factory.getProtocol(TTransport.TMemoryBuffer(data)) ret = objtype() ret.read(prot) return ret
def connectionLost(self, reason=connectionDone):
for k, v in self.client._reqs.iteritems():
tex = TTransport.TTransportException(
type=TTransport.TTransportException.END_OF_FILE,
message='Connection closed')
v.errback(tex)
def _buildSerializationContext(self, bytes=None):
self.transport = TTransport.TMemoryBuffer(bytes)
protocol = SelfDescribingBinaryProtocol.SelfDescribingBinaryProtocol(
self.transport)
return ThriftSerializationContext.ThriftSerializationContext(
self, protocol)
def run(module_name="Judicator",
etcd_conf_path="config/etcd.json",
mongodb_conf_path="config/mongodb.json",
main_conf_path="config/main.json"):
"""
Load config and run judicator main program
:param module_name: Name of the caller module
:param etcd_conf_path: Path to etcd config file
:param mongodb_conf_path: Path to mongodb config file
:param main_conf_path: Path to main config file
:return: None
"""
global working
# Load configuration
with open(main_conf_path, "r") as f:
config = json.load(f)
# Generate logger
if "log" in config:
logger = get_logger("main", config["log"]["info"],
config["log"]["error"])
else:
logger = get_logger("main", None, None)
logger.info("%s main program started." % module_name)
# Generate proxy for etcd and mongodb
with open(etcd_conf_path, "r") as f:
local_etcd = generate_local_etcd_proxy(json.load(f)["etcd"], logger)
with open(mongodb_conf_path, "r") as f:
local_mongodb = generate_local_mongodb_proxy(
json.load(f)["mongodb"], local_etcd, logger)
# Get a connection to both task and executor collection in mongodb
mongodb_task = local_mongodb.client[config["task"]["database"]][
config["task"]["collection"]]
mongodb_executor = local_mongodb.client[config["executor"]["database"]][
config["executor"]["collection"]]
# Create and start the register thread
register_thread = threading.Thread(target=register,
args=(config, local_etcd, local_mongodb,
logger))
register_thread.setDaemon(True)
register_thread.start()
# Create and start the
lead_thread = threading.Thread(target=lead,
args=(config, local_etcd, local_mongodb,
mongodb_task, mongodb_executor,
logger))
lead_thread.setDaemon(True)
lead_thread.start()
# Start the rpc server and serve until terminated
server = TServer.TThreadedServer(
Judicator.Processor(RPCService(logger, mongodb_task,
mongodb_executor)),
TSocket.TServerSocket(config["listen"]["address"],
int(config["listen"]["port"])),
TTransport.TBufferedTransportFactory(),
TBinaryProtocol.TBinaryProtocolFactory())
try:
logger.info("Starting rpc server.")
server.serve()
except KeyboardInterrupt:
logger.info(
"Received SIGINT. Cancelling judicator registration on etcd.",
exc_info=True)
# Wait for the register thread to delete registration and then stop
working = False
register_thread.join()
except:
logger.error("Accidentally terminated.", exc_info=True)
working = False
logger.info("%s main program exiting." % module_name)
return
def __init__(self, host='localhost', port=9090):
transport = TTransport.TBufferedTransport(TSocket.TSocket(host, port))
protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.client = Hbase.Client(protocol)
transport.open()
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from hbase import Hbase
from hbase.ttypes import TScan
if __name__ == '__main__':
#
host = 'bigdata-training01.hpsk.com'
port = 9090
# Client
transport = TTransport.TBufferedTransport(TSocket.TSocket(host, port))
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = Hbase.Client(protocol)
transport.open()
# 获取所有表
table_names = client.getTableNames()
for table_name in table_names:
print table_name
"""
=====================================================================
"""
# 获取数据
scan = TScan()
scanner = client.scannerOpenWithScan('wc_count', scan, None)
import datetime
import time
from datetime import date
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer
from thrift.transport import TSocket, TTransport
from thriftDemo.service.hello import HelloService
class HelloServiceHandler:
def ping(self, msg):
result = str(datetime.time) + msg
print("receive %s" % msg)
return result
if __name__ == '__main__':
handler = HelloServiceHandler()
processor = HelloService.Processor(handler)
transport = TSocket.TServerSocket("localhost", 12580)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = TServer.TSimpleServer(processor, transport, tfactory, pfactory)
print("thrift server start !")
server.serve()
def correlate_dfe(data, size_timeseries, num_timeseries, correlations):
"""Calculates correlations on DFE."""
start_time = time.time()
# Make socket
socket = TSocket.TSocket('localhost', 9090)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(socket)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = correlationService.Client(protocol)
current_time = time.time() - start_time
print 'Creating a client:\t\t\t\t%.5lfs' % current_time
try:
# Connect!
start_time = time.time()
transport.open()
current_time = time.time() - start_time
print 'Opening connection:\t\t\t\t%.5lfs' % current_time
# Initialize maxfile
start_time = time.time()
max_file = client.correlation_init()
current_time = time.time() - start_time
print 'Initializing maxfile:\t\t\t\t%.5lfs' % current_time
# Load DFE
start_time = time.time()
engine = client.max_load(max_file, "*")
current_time = time.time() - start_time
print 'Loading DFE:\t\t\t\t\t%.5lfs' % current_time
num_timesteps = size_timeseries
window_size = size_timeseries
num_bursts = calc_num_bursts(num_timeseries)
# Get loop length
start_time = time.time()
loop_length = client.correlation_get_CorrelationKernel_loopLength()
current_time = time.time() - start_time
print 'Getting Correlation Kernel loopLength:\t\t%.5lfs' % current_time
# Prepare data for DFE
start_time = time.time()
burst_size = 384 / 2 # for anything other than ISCA this should be 384
in_mem_load = [0] * (num_bursts * burst_size)
precalculations, data_pairs = prepare_data_for_dfe(
data, size_timeseries, num_timeseries)
current_time = time.time() - start_time
print 'Data reordering time:\t\t\t\t%.5lfs' % current_time
# Allocate and send input streams to server
start_time = time.time()
loop_length_size = 1
address_loop_length = client.malloc_int32_t(loop_length_size)
client.send_data_int32_t(address_loop_length, [loop_length])
loop_length_time = time.time() - start_time
print('\tSending LoopLength:\t\t(size = %d bit)\t\t%.5lfs' %
(32 * loop_length_size, loop_length_time))
start_time = time.time()
in_mem_load_size = num_bursts * burst_size
address_in_mem_load = client.malloc_int32_t(in_mem_load_size)
client.send_data_int32_t(address_in_mem_load, in_mem_load)
in_mem_load_time = time.time() - start_time
print('\tSending InMemLoad:\t\t(size = %d bit)\t%.5lfs' %
(32 * in_mem_load_size, in_mem_load_time))
start_time = time.time()
precalculations_size = 2 * num_timeseries * num_timesteps
address_precalculations = client.malloc_double(precalculations_size)
client.send_data_double(address_precalculations, precalculations)
precalculations_time = time.time() - start_time
print('\tSending Precalculations:\t(size = %d bit)\t%.5lfs' %
(64 * precalculations_size, precalculations_time))
start_time = time.time()
data_pairs_size = 2 * num_timeseries * num_timesteps
address_data_pairs = client.malloc_double(data_pairs_size)
client.send_data_double(address_data_pairs, data_pairs)
data_pairs_time = time.time() - start_time
print('\tSending DataPairs:\t\t(size = %d bit)\t%.5lfs' %
(64 * data_pairs_size, data_pairs_time))
current_time = (loop_length_time + in_mem_load_time +
precalculations_time + data_pairs_time)
speed = ((32 * loop_length_size + 32 * in_mem_load_size +
64 * precalculations_size + 64 * data_pairs_size) /
(current_time * 1000000))
print(
'Sending input streams to server total time:\t%.5lfs' %
current_time + '\t(average speed = %.5lfMb/s)' % (speed))
# Allocate memory for output stream on server
start_time = time.time()
out_correlation_size = (num_timesteps * CORRELATION_NUM_TOP_SCORES *
CORRELATION_NUM_PIPES * loop_length +
num_bursts * 24
) # for anything other than ISCA
# 48 should be instead of 24
address_out_correlation = client.malloc_double(out_correlation_size)
out_indices_size = (2 * num_timesteps * loop_length *
CORRELATION_NUM_TOP_SCORES * CORRELATION_NUM_PIPES)
address_out_indices = client.malloc_int32_t(out_indices_size)
current_time = time.time() - start_time
print('Allocating memory for output stream on server:\t%.5lfs' %
current_time)
# Initialize LMem
start_time = time.time()
actions_load_lmem = correlation_loadLMem_actions_t_struct(
num_bursts, address_loop_length, address_in_mem_load)
address_actions_load_lmem = client.send_correlation_loadLMem_actions_t(
actions_load_lmem)
client.correlation_loadLMem_run(engine, address_actions_load_lmem)
current_time = time.time() - start_time
print 'LMem initialization:\t\t\t\t%.5lfs' % current_time
# Executing correlation action
start_time = time.time()
actions = correlation_actions_t_struct(
num_bursts, # scalar input
num_timesteps, # scalar input
num_timeseries, # scalar input
1, # scalar input
float(window_size), # scalar input
address_precalculations, # streaming reordered input
address_data_pairs, # streaming reordered input
address_out_correlation, # streaming unordered output
address_out_indices) # streaming unordered output
address_actions = client.send_correlation_actions_t(actions)
client.correlation_run(engine, address_actions)
current_time = time.time() - start_time
print 'Correlation time:\t\t\t\t%.5lfs' % current_time
# Unload DFE
start_time = time.time()
client.max_unload(engine)
current_time = time.time() - start_time
print 'Unloading DFE:\t\t\t\t\t%.5lfs' % current_time
# Get output stream from server
start_time = time.time()
out_correlation = client.receive_data_double(address_out_correlation,
out_correlation_size)
out_correlation_time = time.time() - start_time
print('\tGet output stream Correlation:\t(size = %d bit)\t%.5lfs' %
(64 * out_correlation_size, out_correlation_time))
start_time = time.time()
_ = client.receive_data_int32_t(address_out_indices, out_indices_size)
out_indices_time = time.time() - start_time
print('\tGet output stream outIndices:\t(size = %d bit)\t%.5lfs' %
(32 * out_indices_size, out_indices_time))
start_time = time.time()
loop_length_size = 1
loop_length = client.receive_data_int32_t(address_loop_length,
loop_length_size)
loop_length_time = time.time() - start_time
print('\tGet output stream loopLength:\t(size = %d bit)\t\t%.5lfs' %
(32 * loop_length_size, loop_length_time))
current_time = (out_indices_time + out_correlation_time +
loop_length_time)
speed = ((32 * loop_length_size + 32 * out_indices_time +
64 * out_correlation_size) / (current_time * 1000000))
print(
'Getting output stream from server total time:\t%.5lfs' %
current_time + '\t(average speed = %.5lfMb/s)' % (speed))
# Free allocated memory for streams on server
start_time = time.time()
client.free(address_loop_length)
client.free(address_in_mem_load)
client.free(address_precalculations)
client.free(address_data_pairs)
client.free(address_out_correlation)
client.free(address_out_indices)
client.free(address_actions)
client.free(address_actions_load_lmem)
current_time = time.time() - start_time
print('Freeing allocated memory for streams on server:\t%.5lfs' %
current_time)
# Free allocated maxfile data
start_time = time.time()
client.correlation_free()
current_time = time.time() - start_time
print 'Freeing allocated maxfile data:\t\t\t%.5lfs' % current_time
# Close!
start_time = time.time()
transport.close()
current_time = time.time() - start_time
print 'Closing connection:\t\t\t\t%.5lfs' % current_time
# Store data
start_time = time.time()
position = 0
index = 0
start = ((num_timesteps - 1) * loop_length[0] *
CORRELATION_NUM_TOP_SCORES * CORRELATION_NUM_PIPES)
for i in range(num_timeseries):
for j in range(i):
correlations[index + j] = out_correlation[start + position + j]
index += i
position += ((i / 12) + 1) * 12
current_time = time.time() - start_time
print 'Storing time:\t\t\t\t\t%.5lfs' % current_time
except Thrift.TException, thrift_exception:
print '%s' % (thrift_exception.message)
sys.exit(-1)
parser.add_argument("--fps", type=int, default=0, help="target fps for playback. Default 0: get from bvh file.")
parser.add_argument("filename", type=str, help="BVH motion file to play.")
args = parser.parse_args()
file_in = args.filename
if not os.path.exists(file_in):
print("Error: file {} not found.".format(file_in))
sys.exit(1)
fps = args.fps
### Network Thrift Server ###
thrift_host = '127.0.0.1'
out_port = args.port
handler = LiveSkeletonHandler()
processor = LiveSkeletonService.Processor(handler)
transport = TSocket.TServerSocket(thrift_host, out_port) # Pass host in case we're not connected to a network.
tfactory = TTransport.TFramedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = TServer.TThreadedServer(processor, transport, tfactory, pfactory, daemon=True)
# Start a thread with the server -- that thread will then start one
# more thread for each request
server_thread = threading.Thread(target=server.serve)
# Exit the server thread when the main thread terminates
server_thread.name = 'LiveSkeletonServerThread'
server_thread.daemon = True
server_thread.start()
print("Server loop running on port {} in thread: {}".format(out_port, server_thread.name))
sys.stdout.flush()
### Simulation ###
skeleton_simulator = LiveSkeletonSimulation(bvh_file=file_in, target_fps=fps)
def __thrift_to_json(x):
trans = TTransport.TMemoryBuffer()
proto = TSimpleJSONProtocol.TSimpleJSONProtocol(trans)
x.write(proto)
return json.loads(trans.getvalue())
from helloworld import HelloWorld
from helloworld.ttypes import *
from helloworld.constants import *
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
try:
# Make socket
transport = TSocket.TSocket('127.0.0.1', 30303)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = HelloWorld.Client(protocol)
# Connect!
transport.open()
client.ping()
print("ping()")
msg = client.sayHello()
print(msg)
def createPackage(args, static_input):
'''
Creates a Storm submission package for the given dispel4py graph.
:param module_name: name of the graph module that creates a graph
:param attr: name of the graph attribute within the module - if None the first WorkflowGraph is used
:param res: resource directory - if None the default is "resources"
:rtype: name of the temporary directory that contains the submission package
'''
module_name = args.module
attr = args.attr
res = args.resources
if res is None: res = 'resources'
graph = loadGraph(module_name, attr)
# we don't want any nested subgraphs
graph.flatten()
# create a temporary directory
tmpdir = tempfile.mkdtemp()
resources_dir = tmpdir + '/' + res
# copy dependencies of PEs in the graph to resources in temp directory
shutil.copytree(res, resources_dir)
dispel4py_dir = resources_dir + '/dispel4py'
_mkdir_ifnotexists(dispel4py_dir)
_mkdir_ifnotexists(dispel4py_dir + '/storm')
shutil.copy('dispel4py/__init__.py', dispel4py_dir)
shutil.copy('dispel4py/core.py', dispel4py_dir)
shutil.copy('dispel4py/base.py', dispel4py_dir)
shutil.copy('dispel4py/__init__.py', dispel4py_dir + '/storm/')
shutil.copy('dispel4py/storm/utils.py', dispel4py_dir + '/storm/')
# copy client and dependencies for storm submission to the temp directory
dispel4py_dir = tmpdir + '/dispel4py'
_mkdir_ifnotexists(dispel4py_dir)
_mkdir_ifnotexists(dispel4py_dir + '/storm')
shutil.copy('dispel4py/__init__.py', dispel4py_dir)
shutil.copy('dispel4py/__init__.py', dispel4py_dir + '/storm/')
shutil.copy('dispel4py/storm/client.py', dispel4py_dir + '/storm/')
shutil.copy('dispel4py/storm/storm_submission_client.py', tmpdir)
shutil.copy('java/src/dispel4py/storm/ThriftSubmit.java',
tmpdir + '/dispel4py/storm/')
sources = []
for node in graph.graph.nodes():
pe = node.getContainedObject()
is_source = True
for edge in graph.graph.edges(node, data=True):
if pe == edge[2]['DIRECTION'][1]:
is_source = False
break
if is_source:
sources.append(pe)
print "Sources: %s" % [pe.id for pe in sources]
for pe in sources:
pe._static_input = static_input
# create the storm topology
topology = buildTopology(graph)
# cache PE dependencies imported from the registry to resources_dir
registry.createResources(resources_dir, registry.currentRegistry())
# write thrift representation of the topology to a file
transportOut = TTransport.TMemoryBuffer()
protocolOut = TBinaryProtocol.TBinaryProtocol(transportOut)
topology.write(protocolOut)
bytes = transportOut.getvalue()
with open(tmpdir + '/' + TOPOLOGY_THRIFT_FILE, "w") as thrift_file:
thrift_file.write(bytes)
return tmpdir
