# -*- coding: utf-8 -*- from socket import * import struct import thrift.Thrift as Thrift import thrift.protocol.TBinaryProtocol as TBinaryProtocol import thrift.protocol.TCompactProtocol as TCompactProtocol import thrift.protocol.TJSONProtocol as TJSONProtocol import thrift.transport.TTransport as TTransport import thrift.msg_def.ttypes as ffrpc_msg_def from gen_py.echo import ttypes g_WriteTMemoryBuffer = TTransport.TMemoryBuffer() g_WriteTBinaryProtocol = TBinaryProtocol.TBinaryProtocol(g_WriteTMemoryBuffer) g_ReadTMemoryBuffer = TTransport.TMemoryBuffer() g_ReadTBinaryProtocol = TBinaryProtocol.TBinaryProtocol(g_ReadTMemoryBuffer) def encode_msg(msg, g_protocol = 0): #debub print('encode_msg begin', msg) if hasattr(msg, 'thrift_spec'): g_WriteTMemoryBuffer.cstringio_buf.truncate() g_WriteTMemoryBuffer.cstringio_buf.seek(0) if g_protocol == 1: proto = TJSONProtocol.TJSONProtocol(g_WriteTMemoryBuffer) proto.writeMessageBegin(msg.__class__.__name__, 0, 0); msg.write(proto) proto.writeMessageEnd(); else: g_WriteTBinaryProtocol.writeMessageBegin(msg.__class__.__name__, 0, 0); msg.write(g_WriteTBinaryProtocol)
import sys import time sys.path.append('./gen-py') from rpc import NewsServlet from thrift.transport import TSocket from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol from thrift.server import TServer socket = TSocket.TSocket('localhost', 12345) transport = TTransport.TBufferedTransport(socket) #缓冲 protocol = TBinaryProtocol.TBinaryProtocol(transport) #协议 client = NewsServlet.Client(protocol) transport.open() result = client.get_news_detail() print result
def __init__(self, url, port): self.transport = TSocket.TSocket(url, port) self.transport = TTransport.TFramedTransport(self.transport) self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport) self.client = ServerSvc.Client(self.protocol)
def lmem_loopback_dfe(size, in_a, in_b): """LMemLoopback DFE implementation.""" try: start_time = time.time() # Make socket socket = TSocket.TSocket('localhost', 9090) # _buffering is critical. Raw sockets are very slow transport = TTransport.TBufferedTransport(socket) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder client = LMemLoopbackService.Client(protocol) print ('Creating a client:\t\t\t\t%.5lfs' % (time.time() - start_time)) # Connect! start_time = time.time() transport.open() print ('Opening connection:\t\t\t\t%.5lfs' % (time.time() - start_time)) size_bytes = size * 4 # Initialize maxfile start_time = time.time() max_file = client.LMemLoopback_init() print ('Initializing maxfile:\t\t\t\t%.5lfs' % (time.time() - start_time)) # Load DFE start_time = time.time() max_engine = client.max_load(max_file, '*') print ('Loading DFE:\t\t\t\t\t%.5lfs' % (time.time() - start_time)) # Allocate and send input streams to server start_time = time.time() address_in_a = client.malloc_int32_t(size) client.send_data_int32_t(address_in_a, in_a) address_in_b = client.malloc_int32_t(size) client.send_data_int32_t(address_in_b, in_b) print ('Sending input data:\t\t\t\t%.5lfs' % (time.time() - start_time)) # Allocate memory for output stream on server start_time = time.time() address_out_data = client.malloc_int32_t(size) print ('Allocating memory for output stream on server:\t%.5lfs'% (time.time() - start_time)) # Write to LMem start_time = time.time() actions_write_lmem = ( LMemLoopback_writeLMem_actions_t_struct(0, size_bytes, address_in_a)) address_actions_write_lmem = ( client.send_LMemLoopback_writeLMem_actions_t(actions_write_lmem)) client.LMemLoopback_writeLMem_run(max_engine, address_actions_write_lmem) actions_write_lmem = ( LMemLoopback_writeLMem_actions_t_struct(size_bytes, size_bytes, address_in_b)) address_actions_write_lmem = ( client.send_LMemLoopback_writeLMem_actions_t(actions_write_lmem)) client.LMemLoopback_writeLMem_run(max_engine, address_actions_write_lmem) print ('Writing to LMem:\t\t\t\t%.5lfs'% (time.time() - start_time)) # Action default start_time = time.time() actions = LMemLoopback_actions_t_struct(size) address_actions = client.send_LMemLoopback_actions_t(actions) client.LMemLoopback_run(max_engine, address_actions) print ('LMemLoopback time:\t\t\t\t%.5lfs'% (time.time() - start_time)) # Read from LMem start_time = time.time() actions_read_lmem = ( LMemLoopback_readLMem_actions_t_struct(2 * size_bytes, size_bytes, address_out_data)) address_actions_read_lmem = ( client.send_LMemLoopback_readLMem_actions_t(actions_read_lmem)) client.LMemLoopback_readLMem_run(max_engine, address_actions_read_lmem) print ('Reading from LMem:\t\t\t\t%.5lfs'% (time.time() - start_time)) # Unload DFE start_time = time.time() client.max_unload(max_engine) print ('Unloading DFE:\t\t\t\t\t%.5lfs' % (time.time() - start_time)) # Get output stream from server start_time = time.time() out_data = client.receive_data_int32_t(address_out_data, size) print ('Getting output stream:\t(size = %d bit)\t%.5lfs' % ((size * 32), (time.time() - start_time))) # Free allocated memory for streams on server start_time = time.time() client.free(address_in_a) client.free(address_in_b) client.free(address_out_data) client.free(address_actions_write_lmem) client.free(address_actions) client.free(address_actions_read_lmem) print ('Freeing allocated memory for streams on server:\t%.5lfs' % (time.time() - start_time)) # Free allocated maxfile data start_time = time.time() client.LMemLoopback_free() print ('Freeing allocated maxfile data:\t\t\t%.5lfs' % (time.time() - start_time)) # Close! start_time = time.time() transport.close() print ('Closing connection:\t\t\t\t%.5lfs' % (time.time() - start_time)) except Thrift.TException, thrift_exceptiion: print '%s' % (thrift_exceptiion.message) sys.exit(-1)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys from thrift import Thrift from thrift.transport import TSocket from thrift.transport import TTransport from thrift.protocol import TCompactProtocol from accumulo import AccumuloProxy from accumulo.ttypes import * transport = TSocket.TSocket('localhost', 42424) transport = TTransport.TFramedTransport(transport) protocol = TCompactProtocol.TCompactProtocol(transport) client = AccumuloProxy.Client(protocol) transport.open() login = client.login('root', {'password': '******'}) print client.listTables(login) testtable = "pythontest" if not client.tableExists(login, testtable): client.createTable(login, testtable, True, TimeType.MILLIS) row1 = { 'a': [ ColumnUpdate('a', 'a', value='value1'),
def _serialize(self, obj): trans = TTransport.TMemoryBuffer() prot = self.protocol_factory.getProtocol(trans) obj.write(prot) return trans.getvalue()
def get_client_transport(self, service): host, port = service.get_host_port() transport = TTransport.TFramedTransport(TSocket.TSocket(host, port)) protocol = TBinaryProtocol.TBinaryProtocol(transport) transport.open() return LucidaService.Client(protocol), transport
def main(): transport = TSocket.TSocket('localhost', 9081) transport = TTransport.TBufferedTransport(transport) protocol = TBinaryProtocol.TBinaryProtocol(transport) device = Device.Client(TMultiplexedProtocol(protocol, "Device")) fridge = Fridge.Client(TMultiplexedProtocol(protocol, "Fridge")) camera = Camera.Client(TMultiplexedProtocol(protocol, "Camera")) sensor = Sensor.Client(TMultiplexedProtocol(protocol, "TemperatureSensor")) try: transport.open() while True: print("Write 1. to list all devices ") print("Write 2. to operate on fridges ") print("Write 3. to operate on cameras ") print("Write 4. to operate sensors ") action = input() if action == '1': for dev in device.getEveryDevice(): print(dev) elif action == '2': print("Write down in format operationNumber/arg1_arg2...") print("1.start freezing/id") print("2.stop freezing/id") print("3.change temp/id_value") print("4.get temp and is freezing/id") print("5.turn on/id") print("6.turn off/id") print("7.get state/id") target = input() command = target.split("/") operation = command[0] args = command[1] if operation == '1': print(fridge.startFreezing(args)) elif operation == '2': print(fridge.stopFreezing(args)) elif operation == '3': arg_split = args.split("_") dev_id = arg_split[0] value = arg_split[1] print(fridge.changeTemperatureTo(dev_id, int(value))) elif operation == '4': print(fridge.getTemperatureAndIsFreezing(args)) elif operation == '5': print(fridge.turnOn(args)) elif operation == '6': print(fridge.turnOff(args)) elif operation == '7': print(fridge.getState(args)) else: pass elif action == '3': print("Write down in format operationNumber/arg1_arg2...") print( "1.move to position and zoom/id_verticalAngle_horizontalAngle_zoom" ) print("2.turn on/id") print("3.turn off/id") print("4.get state/id") print("5.to go to menu") target = input() command = target.split("/") operation = command[0] args = command[1] if operation == '1': argSplit = args.split("_") position = AngularPosition(int(argSplit[1]), int(argSplit[2])) print( camera.moveToPositionAndZoom(argSplit[0], position, int(argSplit[3]))) elif operation == '2': print(camera.turnOn(args)) elif operation == '3': print(camera.turnOff(args)) elif operation == '4': print(camera.getState(args)) else: pass elif action == '4': print("Write down in format operationNumber/arg1_arg2...") print("1.change alarm temperature value/id_alarmValue") print("2.get temperature/id") print("3.turn on/id") print("4.turn off/id") print("5.get state/id") print("6.to go to menu") target = input() command = target.split("/") operation = command[0] args = command[1] if operation == '1': argSplit = args.split("_") sensor.changeAlarmTemperatureValue(argSplit[0], int(argSplit[1])) elif operation == '2': print(sensor.getTemperature(args)) elif operation == '3': print(sensor.turnOn(args)) elif operation == '4': print(sensor.turnOff(args)) elif operation == '5': print(sensor.getState(args)) else: pass else: pass except InvalidArguments as iarg: print(iarg) main() except IndexError as ierr: print("WRONG PARSED COMMAND - TOO LESS INDICES") transport.close() main() except ValueError as verr: print("WRONG PARSED COMMAND - WRONG VALUE") transport.close() main() except TTransportException as tterr: print("Connection closed - Trying to reconnect") transport.close() main() except ConnectionResetError as conerr: print("Connection not established - waiting for another reconnect") time.sleep(10) transport.close() main() transport.close()
def handler(event, context): start_time = time.time() bucket = event['bucket_name'] worker_index = event['rank'] num_workers = event['num_workers'] key = event['file'].split(",") num_classes = event['num_classes'] num_features = event['num_features'] pos_tag = event['pos_tag'] num_epochs = event['num_epochs'] learning_rate = event['learning_rate'] batch_size = event['batch_size'] host = event['host'] port = event['port'] print('bucket = {}'.format(bucket)) print('number of workers = {}'.format(num_workers)) print('worker index = {}'.format(worker_index)) print("file = {}".format(key)) print('number of workers = {}'.format(num_workers)) print('worker index = {}'.format(worker_index)) print('num epochs = {}'.format(num_epochs)) print('num classes = {}'.format(num_classes)) print('num features = {}'.format(num_features)) print('positive tag = {}'.format(pos_tag)) print('learning rate = {}'.format(learning_rate)) print("batch_size = {}".format(batch_size)) print("host = {}".format(host)) print("port = {}".format(port)) # Set thrift connection # Make socket transport = TSocket.TSocket(host, port) # Buffering is critical. Raw sockets are very slow transport = TTransport.TBufferedTransport(transport) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder t_client = ParameterServer.Client(protocol) # Connect! transport.open() # test thrift connection ps_client.ping(t_client) print("create and ping thrift server >>> HOST = {}, PORT = {}".format( host, port)) # read file from s3 file = get_object(bucket, key[0]).read().decode('utf-8').split("\n") dataset = DenseLibsvmDataset(file, num_features, pos_tag) if len(key) > 1: for more_key in key[1:]: file = get_object(bucket, more_key).read().decode('utf-8').split("\n") dataset.add_more(file) print("read data cost {} s".format(time.time() - start_time)) parse_start = time.time() total_count = dataset.__len__() pos_count = 0 for i in range(total_count): if dataset.__getitem__(i)[1] == 1: pos_count += 1 print("{} positive observations out of {}".format(pos_count, total_count)) print("parse data cost {} s".format(time.time() - parse_start)) preprocess_start = time.time() # Creating data indices for training and validation splits: dataset_size = len(dataset) indices = list(range(dataset_size)) split = int(np.floor(VALIDATION_RATIO * dataset_size)) if SHUFFLE_DATASET: np.random.seed(RANDOM_SEED) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] # Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler) validation_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler) print("preprocess data cost {} s, dataset size = {}".format( time.time() - preprocess_start, dataset_size)) model = LogisticRegression(NUM_FEATURES, NUM_CLASSES) # Loss and Optimizer # Softmax is internally computed. # Set parameters to be updated. criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE) # register model model_name = "w.b" weight_shape = model.linear.weight.data.numpy().shape weight_length = weight_shape[0] * weight_shape[1] bias_shape = model.linear.bias.data.numpy().shape bias_length = bias_shape[0] model_length = weight_length + bias_length ps_client.register_model(t_client, worker_index, model_name, model_length, num_workers) ps_client.exist_model(t_client, model_name) print("register and check model >>> name = {}, length = {}".format( model_name, model_length)) # Training the Model train_start = time.time() iter_counter = 0 for epoch in range(NUM_EPOCHS): epoch_start = time.time() for batch_index, (items, labels) in enumerate(train_loader): print("------worker {} epoch {} batch {}------".format( worker_index, epoch, batch_index)) batch_start = time.time() # pull latest model ps_client.can_pull(t_client, model_name, iter_counter, worker_index) latest_model = ps_client.pull_model(t_client, model_name, iter_counter, worker_index) model.linear.weight = Parameter( torch.from_numpy( np.asarray(latest_model[:weight_length], dtype=np.double).reshape(weight_shape))) model.linear.bias = Parameter( torch.from_numpy( np.asarray(latest_model[weight_length:], dtype=np.double).reshape(bias_shape[0]))) items = Variable(items.view(-1, NUM_FEATURES)) labels = Variable(labels) # Forward + Backward + Optimize optimizer.zero_grad() outputs = model(items.double()) loss = criterion(outputs, labels) loss.backward() # flatten and concat gradients of weight and bias w_b_grad = np.concatenate( (model.linear.weight.grad.data.numpy().flatten(), model.linear.bias.grad.data.numpy().flatten())) cal_time = time.time() - batch_start # push gradient to PS sync_start = time.time() ps_client.can_push(t_client, model_name, iter_counter, worker_index) ps_client.push_grad(t_client, model_name, w_b_grad, LEARNING_RATE, iter_counter, worker_index) ps_client.can_pull(t_client, model_name, iter_counter + 1, worker_index) # sync all workers sync_time = time.time() - sync_start print( 'Epoch: [%d/%d], Step: [%d/%d] >>> Time: %.4f, Loss: %.4f, epoch cost %.4f, ' 'batch cost %.4f s: cal cost %.4f s and communication cost %.4f s' % (epoch + 1, NUM_EPOCHS, batch_index + 1, len(train_indices) / BATCH_SIZE, time.time() - train_start, loss.data, time.time() - epoch_start, time.time() - batch_start, cal_time, sync_time)) iter_counter += 1 # Test the Model correct = 0 total = 0 test_loss = 0 for items, labels in validation_loader: items = Variable(items.view(-1, NUM_FEATURES)) labels = Variable(labels) outputs = model(items) test_loss += criterion(outputs, labels).data _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).sum() print( 'Time = %.4f, accuracy of the model on the %d test samples: %d %%, loss = %f' % (time.time() - train_start, len(val_indices), 100 * correct / total, test_loss)) end_time = time.time() print("Elapsed time = {} s".format(end_time - start_time))
def main(): p = sys.argv[1] # Make socket transport = TSocket.TSocket('localhost', p) # Buffering is critical. Raw sockets are very slow transport = TTransport.TBufferedTransport(transport) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder client = Graph.Client(protocol) # Connect! transport.open() client.ping() print(client.add_upd_vertex(name_hash("Alice"), pessoa, "Alice", 0)) print(client.add_upd_vertex(name_hash("Bob"), pessoa, "Bob", 0)) print(client.add_upd_vertex(name_hash("Carla"), pessoa, "Carla", 0)) print(client.add_upd_vertex(name_hash("Dan"), pessoa, "Dan", 0)) print(client.add_upd_vertex(name_hash("Eve"), pessoa, "Eve", 0)) print( client.add_upd_vertex(name_hash("Blade Runner"), filme, "Blade Runner", 0)) print( client.add_upd_vertex(name_hash("Terminator 2"), filme, "Terminator 2", 0)) print( client.add_upd_vertex(name_hash("The Matrix"), filme, "The Matrix", 0)) print( client.add_upd_vertex(name_hash("Ex Machina"), filme, "Ex Machina", 0)) print(client.add_upd_vertex(name_hash("Her"), filme, "Her", 0)) print( client.add_upd_vertex(name_hash("Harrison Ford"), cast, "Harrison Ford", 0)) print( client.add_upd_vertex(name_hash("Arnold Schwarzenegger"), cast, "Arnold Schwarzenegger", 0)) print( client.add_upd_vertex(name_hash("Keanu Reeves"), cast, "Keanu Reeves", 0)) print( client.add_upd_vertex(name_hash("Domhnall Gleeson"), cast, "Domhnall Gleeson", 0)) print( client.add_upd_vertex(name_hash("Joaquin Phoenix"), cast, "Joaquin Phoenix", 0)) print(client.add_upd_vertex(name_hash("oldschool"), grupo, "oldschool", 0)) print(client.add_upd_vertex(name_hash("scifi"), grupo, "scifi", 0)) print( client.add_upd_edge(name_hash("Blade Runner"), name_hash("Harrison Ford"), cst, True)) print( client.add_upd_edge(name_hash("Terminator 2"), name_hash("Arnold Schwarzenegger"), cst, True)) print( client.add_upd_edge(name_hash("The Matrix"), name_hash("Keanu Reeves"), cst, True)) print( client.add_upd_edge(name_hash("Ex Machina"), name_hash("Domhnall Gleeson"), cst, True)) print( client.add_upd_edge(name_hash("Her"), name_hash("Joaquin Phoenix"), cst, True)) print( client.add_upd_edge(name_hash("Alice"), name_hash("Terminator 2"), 91.0, True)) print( client.add_upd_edge(name_hash("Alice"), name_hash("Ex Machina"), 79.0, True)) print( client.add_upd_edge(name_hash("Bob"), name_hash("The Matrix"), 90.0, True)) print(client.add_upd_edge(name_hash("Carla"), name_hash("Her"), 75.0, True)) print( client.add_upd_edge(name_hash("Dan"), name_hash("Blade Runner"), 75.0, True)) print( client.add_upd_edge(name_hash("Dan"), name_hash("Terminator 2"), 80.0, True)) print( client.add_upd_edge(name_hash("Dan"), name_hash("The Matrix"), 79.0, True)) print( client.add_upd_edge(name_hash("Eve"), name_hash("Ex Machina"), 60.0, True)) print(client.add_upd_edge(name_hash("scifi"), name_hash("Alice"), 0, False)) print(client.add_upd_edge(name_hash("scifi"), name_hash("Bob"), 0, False)) print(client.add_upd_edge(name_hash("scifi"), name_hash("Carla"), 0, False)) print(client.add_upd_edge(name_hash("scifi"), name_hash("Dan"), 0, False)) print( client.add_upd_edge(name_hash("oldschool"), name_hash("Alice"), 0, False)) print( client.add_upd_edge(name_hash("oldschool"), name_hash("Dan"), 0, False)) # Close! transport.close()
def handler(argv): from archived.pytorch_model import MobileNet start_time = time.time() worker_index = argv[1] num_worker = argv[2] print('number of workers = {}'.format(num_worker)) print('worker index = {}'.format(worker_index)) # Set thrift connection # Make socket transport = TSocket.TSocket(constants.HOST, constants.PORT) # Buffering is critical. Raw sockets are very slow transport = TTransport.TBufferedTransport(transport) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder t_client = ParameterServer.Client(protocol) # Connect! transport.open() # test thrift connection ps_client.ping(t_client) print("create and ping thrift server >>> HOST = {}, PORT = {}".format( constants.HOST, constants.PORT)) preprocess_start = time.time() batch_size = 200 train_path = "../../dataset/training.pt" test_path = "../../dataset/test.pt" trainset = torch.load(train_path) testset = torch.load(test_path) train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False) classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') device = 'cpu' print("preprocess data cost {} s".format(time.time() - preprocess_start)) model = MobileNet() model = model.to(device) # Loss and Optimizer # Softmax is internally computed. # Set parameters to be updated. criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, weight_decay=5e-4) # register model model_name = "mobilenet" parameter_shape = [] parameter_length = [] model_length = 0 for param in model.parameters(): tmp_shape = 1 parameter_shape.append(param.data.numpy().shape) for w in param.data.numpy().shape: tmp_shape *= w parameter_length.append(tmp_shape) model_length += tmp_shape print("model_length = {}".format(model_length)) model_length = 830 """ ps_client.register_model(t_client, worker_index, model_name, model_length, num_worker) #ps_client.exist_model(t_client, model_name) print("register and check model >>> name = {}, length = {}".format(model_name, model_length)) """ # Training the Model train_start = time.time() iter_counter = 0 for epoch in range(num_epochs): epoch_start = time.time() for batch_index, (inputs, targets) in enumerate(train_loader): print("------worker {} epoch {} batch {}------".format( worker_index, epoch, batch_index)) batch_start = time.time() # pull latest model ps_client.can_pull(t_client, model_name, iter_counter, worker_index) latest_model = ps_client.pull_model(t_client, model_name, iter_counter, worker_index) pos = 0 for layer_index, param in enumerate(model.parameters()): param.data = Variable( torch.from_numpy( np.asarray(latest_model[pos:pos + parameter_length[layer_index]], dtype=np.float32).reshape( parameter_shape[layer_index]))) pos += parameter_length[layer_index] # Forward + Backward + Optimize inputs, targets = inputs.to(device), targets.to(device) optimizer.zero_grad() outputs = model(inputs) loss = criterion(outputs, targets) loss.backward() # flatten and concat gradients of weight and bias param_grad = np.ones((1)) * (-1) for param in model.parameters(): print("shape of layer = {}".format( param.data.numpy().flatten().shape)) param_grad = np.concatenate( (param_grad, param.data.numpy().flatten())) param_grad = np.delete(param_grad, 0) print("model_length = {}".format(param_grad.shape)) cal_time = time.time() - batch_start # push gradient to PS sync_start = time.time() ps_client.can_push(t_client, model_name, iter_counter, worker_index) ps_client.push_grad(t_client, model_name, param_grad, learning_rate, iter_counter, worker_index) ps_client.can_pull(t_client, model_name, iter_counter + 1, worker_index) # sync all workers sync_time = time.time() - sync_start sync_time = time.time() print( 'Epoch: [%d/%d], Step: [%d/%d] >>> Time: %.4f, Loss: %.4f, epoch cost %.4f, ' 'batch cost %.4f s: cal cost %.4f s and communication cost %.4f s' % (epoch + 1, num_epochs, batch_index + 1, len(train_loader) / batch_size, time.time() - train_start, loss.data, time.time() - epoch_start, time.time() - batch_start, cal_time, sync_time)) iter_counter += 1 test(epoch, model, test_loader, criterion, device)
from thrift.transport import TSocket, TTransport from thrift.protocol import TBinaryProtocol from thrift.server import TServer from thrift.server import TNonblockingServer from genpy.calc.ICalc import Processor import time class CalcHandler(object): def add(self, num1, num2): print "starting...\n" time.sleep(20) return num1 + num2 if __name__ == '__main__': HANDLER = CalcHandler() PROCESSOR = Processor(HANDLER) TRANSPORT = TSocket.TServerSocket('localhost', 9090) TFACTORY = TTransport.TBufferedTransportFactory() PFACTORY = TBinaryProtocol.TBinaryProtocolFactory() SERVER = TServer.TThreadPoolServer(PROCESSOR, TRANSPORT, TFACTORY, PFACTORY) print 'starting server...' SERVER.serve() print 'Done.'
def __init__(self, port): self.transport = TSocket.TSocket('localhost', port) self.transport = TTransport.TBufferedTransport(self.transport) protocol = TBinaryProtocol.TBinaryProtocol(self.transport) self.client = Client(protocol)
def main(cfg, reqhandle, resphandle): if cfg.unix: if cfg.addr == "": sys.exit("invalid unix domain socket: {}".format(cfg.addr)) socket = TSocket.TSocket(unix_socket=cfg.addr) else: try: (host, port) = cfg.addr.rsplit(":", 1) if host == "": host = "localhost" socket = TSocket.TSocket(host=host, port=int(port)) except ValueError: sys.exit("invalid address: {}".format(cfg.addr)) transport = TRecordingTransport(socket, reqhandle, resphandle) if cfg.transport == "framed": transport = TTransport.TFramedTransport(transport) elif cfg.transport == "unframed": transport = TTransport.TBufferedTransport(transport) elif cfg.transport == "header": transport = THeaderTransport.THeaderTransport( transport, client_type=THeaderTransport.CLIENT_TYPE.HEADER, ) if cfg.headers is not None: pairs = cfg.headers.split(",") for p in pairs: key, value = p.split("=") transport.set_header(key, value) if cfg.protocol == "binary": transport.set_protocol_id(THeaderTransport.T_BINARY_PROTOCOL) elif cfg.protocol == "compact": transport.set_protocol_id(THeaderTransport.T_COMPACT_PROTOCOL) else: sys.exit( "header transport cannot be used with protocol {0}".format( cfg.protocol)) else: sys.exit("unknown transport {0}".format(cfg.transport)) transport.open() if cfg.protocol == "binary": protocol = TBinaryProtocol.TBinaryProtocol(transport) elif cfg.protocol == "compact": protocol = TCompactProtocol.TCompactProtocol(transport) elif cfg.protocol == "json": protocol = TJSONProtocol.TJSONProtocol(transport) elif cfg.protocol == "finagle": protocol = TFinagleProtocol(transport, client_id="thrift-playground") else: sys.exit("unknown protocol {0}".format(cfg.protocol)) if cfg.service is not None: protocol = TMultiplexedProtocol.TMultiplexedProtocol( protocol, cfg.service) client = Example.Client(protocol) try: if cfg.method == "ping": client.ping() print("client: pinged") elif cfg.method == "poke": client.poke() print("client: poked") elif cfg.method == "add": if len(cfg.params) != 2: sys.exit("add takes 2 arguments, got: {0}".format(cfg.params)) a = int(cfg.params[0]) b = int(cfg.params[1]) v = client.add(a, b) print("client: added {0} + {1} = {2}".format(a, b, v)) elif cfg.method == "execute": param = Param(return_fields=cfg.params, the_works=TheWorks( field_1=True, field_2=0x7f, field_3=0x7fff, field_4=0x7fffffff, field_5=0x7fffffffffffffff, field_6=-1.5, field_7=u"string is UTF-8: \U0001f60e", field_8=b"binary is bytes: \x80\x7f\x00\x01", field_9={ 1: "one", 2: "two", 3: "three" }, field_10=[1, 2, 4, 8], field_11=set(["a", "b", "c"]), field_12=False, )) try: result = client.execute(param) print("client: executed {0}: {1}".format(param, result)) except AppException as e: print("client: execute failed with IDL Exception: {0}".format( e.why)) else: sys.exit("unknown method {0}".format(cfg.method)) except Thrift.TApplicationException as e: print("client exception: {0}: {1}".format(e.type, e.message)) if cfg.request is None: #req = "".join(["%02X " % ord(x) for x in reqhandle.getvalue()]).strip() req = "".join(["%02X " % x for x in reqhandle.getvalue()]).strip() print("request: {}".format(req)) if cfg.response is None: resp = "".join(["%02X " % x for x in resphandle.getvalue()]).strip() print("response: {}".format(resp)) transport.close()
def addNode(self, host, port): new_node_key = hashlib.sha256(host + ":" + str(port)).hexdigest() new_node = NodeID() new_node.ip = host new_node.port = port new_node.id = new_node_key ## Make its fingertable (current node to find all answers) new_fingertable = [] for i in range(256): finger_key = \ hex((int(new_node_key, 16) + (2**i)) % (2**256)).strip("0x").strip('L') succ = self.findSucc(finger_key) if(self.contains(new_node_key, finger_key, succ.id)): new_fingertable.append(new_node) else: new_fingertable.append(self.findSucc(finger_key)) ## Launch new server Popen(["python2", "server.py"] + [str(port)]) sleep(3) ## Update other nodes fingertables transport = TSocket.TSocket(host, port) # Buffering is critical. Raw sockets are very slow transport = TTransport.TBufferedTransport(transport) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder client = FileStore.Client(protocol) transport.open() client.setFingertable(new_fingertable) transport.close() new_node_succ = self.findSucc(new_node_key) new_node_pred = self.findPred(new_node_key) succ_files = [] if(new_node_succ.id == self.myNode.id): succ_files = self.getFiles() else: ## Update other nodes fingertables transport = TSocket.TSocket(new_node_succ.ip, new_node_succ.port) # Buffering is critical. Raw sockets are very slow transport = TTransport.TBufferedTransport(transport) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder client = FileStore.Client(protocol) transport.open() succ_files = client.getFiles() transport.close() # New node must take ownership of some files from successor, # while successor must relinquish ownership of some for file_obj in succ_files: if(self.contains(file_obj.meta.contentHash,\ new_node_pred.id,\ new_node_key)): ## File belongs to new node if(new_node_succ.id == self.myNode.id): self.removeFile(file_obj.meta.contentHash) else: ## Update other nodes fingertables transport = TSocket.TSocket(new_node_succ.ip, new_node_succ.port) # Buffering is critical. Raw sockets are very slow transport = TTransport.TBufferedTransport(transport) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder client = FileStore.Client(protocol) transport.open() client.removeFile(file_obj.meta.contentHash) ## Update other nodes fingertables transport = TSocket.TSocket(host, port) # Buffering is critical. Raw sockets are very slow transport = TTransport.TBufferedTransport(transport) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder client = FileStore.Client(protocol) transport.open() client.addFile(file_obj)
# Apache Thrift Hello World Python Server import sys sys.path.append("gen-py") from thrift.transport import TSocket from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol from thrift.server import TServer from hello import helloSvc class HelloHandler(helloSvc.Iface): def getMessage(self, name): print("[Server] Handling client request: " + name) return "Hello " + name handler = HelloHandler() proc = helloSvc.Processor(handler) trans_svr = TSocket.TServerSocket(port=9090) trans_fac = TTransport.TBufferedTransportFactory() proto_fac = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TSimpleServer(proc, trans_svr, trans_fac, proto_fac) print("[Server] Started") server.serve()
def wrappee(*args, **kwargs): logging.debug('in decorator before wrapee with flag ' + str(decorator_self._thrift_class.__name__)) dec_name = str( decorator_self._thrift_class.__name__).split(".")[-1] print(dec_name) assert "use_rpc" in kwargs.keys() and "server" in kwargs.keys(), \ "Wrapped function must be subclass of RPC." if "use_rpc" in kwargs.keys() and "server" in kwargs.keys() \ and kwargs["use_rpc"] and kwargs["server"]: from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket from thrift.transport import TTransport from thrift.server import TServer handler = original_clazz(*args, **kwargs) processor = decorator_self._thrift_class.Processor(handler) transport = TSocket.TServerSocket(port=decorator_self._port) tfactory = TTransport.TBufferedTransportFactory() pfactory = TBinaryProtocol.TBinaryProtocolFactory() self.__inst = handler server = TServer.TThreadPoolServer(processor, transport, tfactory, pfactory) server.setNumThreads(cpu_count()) logging.info("Serving: " + dec_name) server.serve() logging.info('Done: ' + dec_name) elif "use_rpc" in kwargs.keys() and "server" in kwargs.keys() \ and kwargs["use_rpc"] and not kwargs["server"]: from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket from thrift.transport import TTransport from thrift.server import TServer # Make socket self._transport = TSocket.TSocket(host=dec_name, port=decorator_self._port) # Buffering is critical. Raw sockets are very slow self._transport = TTransport.TBufferedTransport( self._transport) # Connect! self._transport.open() # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(self._transport) # Create a client to use the protocol encoder client = decorator_self._thrift_class.Client(protocol) logging.info("Client (" + dec_name + ") connected to server: " + str(self._transport.isOpen())) return client else: return original_clazz(*args, **kwargs)
def __init__(self): transport = TSocket.TSocket(config.IP, config.PORT) self.transport = TTransport.TBufferedTransport(transport) protocol = TBinaryProtocol.TBinaryProtocol(self.transport) self.client = ClassificationService.Client(protocol) self.transport.open()
def _deserialize(self, objtype, data): prot = self.protocol_factory.getProtocol(TTransport.TMemoryBuffer(data)) ret = objtype() ret.read(prot) return ret
def connectionLost(self, reason=connectionDone): for k, v in self.client._reqs.iteritems(): tex = TTransport.TTransportException( type=TTransport.TTransportException.END_OF_FILE, message='Connection closed') v.errback(tex)
def _buildSerializationContext(self, bytes=None): self.transport = TTransport.TMemoryBuffer(bytes) protocol = SelfDescribingBinaryProtocol.SelfDescribingBinaryProtocol( self.transport) return ThriftSerializationContext.ThriftSerializationContext( self, protocol)
def run(module_name="Judicator", etcd_conf_path="config/etcd.json", mongodb_conf_path="config/mongodb.json", main_conf_path="config/main.json"): """ Load config and run judicator main program :param module_name: Name of the caller module :param etcd_conf_path: Path to etcd config file :param mongodb_conf_path: Path to mongodb config file :param main_conf_path: Path to main config file :return: None """ global working # Load configuration with open(main_conf_path, "r") as f: config = json.load(f) # Generate logger if "log" in config: logger = get_logger("main", config["log"]["info"], config["log"]["error"]) else: logger = get_logger("main", None, None) logger.info("%s main program started." % module_name) # Generate proxy for etcd and mongodb with open(etcd_conf_path, "r") as f: local_etcd = generate_local_etcd_proxy(json.load(f)["etcd"], logger) with open(mongodb_conf_path, "r") as f: local_mongodb = generate_local_mongodb_proxy( json.load(f)["mongodb"], local_etcd, logger) # Get a connection to both task and executor collection in mongodb mongodb_task = local_mongodb.client[config["task"]["database"]][ config["task"]["collection"]] mongodb_executor = local_mongodb.client[config["executor"]["database"]][ config["executor"]["collection"]] # Create and start the register thread register_thread = threading.Thread(target=register, args=(config, local_etcd, local_mongodb, logger)) register_thread.setDaemon(True) register_thread.start() # Create and start the lead_thread = threading.Thread(target=lead, args=(config, local_etcd, local_mongodb, mongodb_task, mongodb_executor, logger)) lead_thread.setDaemon(True) lead_thread.start() # Start the rpc server and serve until terminated server = TServer.TThreadedServer( Judicator.Processor(RPCService(logger, mongodb_task, mongodb_executor)), TSocket.TServerSocket(config["listen"]["address"], int(config["listen"]["port"])), TTransport.TBufferedTransportFactory(), TBinaryProtocol.TBinaryProtocolFactory()) try: logger.info("Starting rpc server.") server.serve() except KeyboardInterrupt: logger.info( "Received SIGINT. Cancelling judicator registration on etcd.", exc_info=True) # Wait for the register thread to delete registration and then stop working = False register_thread.join() except: logger.error("Accidentally terminated.", exc_info=True) working = False logger.info("%s main program exiting." % module_name) return
def __init__(self, host='localhost', port=9090): transport = TTransport.TBufferedTransport(TSocket.TSocket(host, port)) protocol = TBinaryProtocol.TBinaryProtocol(transport) self.client = Hbase.Client(protocol) transport.open()
from thrift.transport import TSocket from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol from hbase import Hbase from hbase.ttypes import TScan if __name__ == '__main__': # host = 'bigdata-training01.hpsk.com' port = 9090 # Client transport = TTransport.TBufferedTransport(TSocket.TSocket(host, port)) protocol = TBinaryProtocol.TBinaryProtocol(transport) client = Hbase.Client(protocol) transport.open() # 获取所有表 table_names = client.getTableNames() for table_name in table_names: print table_name """ ===================================================================== """ # 获取数据 scan = TScan() scanner = client.scannerOpenWithScan('wc_count', scan, None)
import datetime import time from datetime import date from thrift.protocol import TBinaryProtocol from thrift.server import TServer from thrift.transport import TSocket, TTransport from thriftDemo.service.hello import HelloService class HelloServiceHandler: def ping(self, msg): result = str(datetime.time) + msg print("receive %s" % msg) return result if __name__ == '__main__': handler = HelloServiceHandler() processor = HelloService.Processor(handler) transport = TSocket.TServerSocket("localhost", 12580) tfactory = TTransport.TBufferedTransportFactory() pfactory = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) print("thrift server start !") server.serve()
def correlate_dfe(data, size_timeseries, num_timeseries, correlations): """Calculates correlations on DFE.""" start_time = time.time() # Make socket socket = TSocket.TSocket('localhost', 9090) # Buffering is critical. Raw sockets are very slow transport = TTransport.TBufferedTransport(socket) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder client = correlationService.Client(protocol) current_time = time.time() - start_time print 'Creating a client:\t\t\t\t%.5lfs' % current_time try: # Connect! start_time = time.time() transport.open() current_time = time.time() - start_time print 'Opening connection:\t\t\t\t%.5lfs' % current_time # Initialize maxfile start_time = time.time() max_file = client.correlation_init() current_time = time.time() - start_time print 'Initializing maxfile:\t\t\t\t%.5lfs' % current_time # Load DFE start_time = time.time() engine = client.max_load(max_file, "*") current_time = time.time() - start_time print 'Loading DFE:\t\t\t\t\t%.5lfs' % current_time num_timesteps = size_timeseries window_size = size_timeseries num_bursts = calc_num_bursts(num_timeseries) # Get loop length start_time = time.time() loop_length = client.correlation_get_CorrelationKernel_loopLength() current_time = time.time() - start_time print 'Getting Correlation Kernel loopLength:\t\t%.5lfs' % current_time # Prepare data for DFE start_time = time.time() burst_size = 384 / 2 # for anything other than ISCA this should be 384 in_mem_load = [0] * (num_bursts * burst_size) precalculations, data_pairs = prepare_data_for_dfe( data, size_timeseries, num_timeseries) current_time = time.time() - start_time print 'Data reordering time:\t\t\t\t%.5lfs' % current_time # Allocate and send input streams to server start_time = time.time() loop_length_size = 1 address_loop_length = client.malloc_int32_t(loop_length_size) client.send_data_int32_t(address_loop_length, [loop_length]) loop_length_time = time.time() - start_time print('\tSending LoopLength:\t\t(size = %d bit)\t\t%.5lfs' % (32 * loop_length_size, loop_length_time)) start_time = time.time() in_mem_load_size = num_bursts * burst_size address_in_mem_load = client.malloc_int32_t(in_mem_load_size) client.send_data_int32_t(address_in_mem_load, in_mem_load) in_mem_load_time = time.time() - start_time print('\tSending InMemLoad:\t\t(size = %d bit)\t%.5lfs' % (32 * in_mem_load_size, in_mem_load_time)) start_time = time.time() precalculations_size = 2 * num_timeseries * num_timesteps address_precalculations = client.malloc_double(precalculations_size) client.send_data_double(address_precalculations, precalculations) precalculations_time = time.time() - start_time print('\tSending Precalculations:\t(size = %d bit)\t%.5lfs' % (64 * precalculations_size, precalculations_time)) start_time = time.time() data_pairs_size = 2 * num_timeseries * num_timesteps address_data_pairs = client.malloc_double(data_pairs_size) client.send_data_double(address_data_pairs, data_pairs) data_pairs_time = time.time() - start_time print('\tSending DataPairs:\t\t(size = %d bit)\t%.5lfs' % (64 * data_pairs_size, data_pairs_time)) current_time = (loop_length_time + in_mem_load_time + precalculations_time + data_pairs_time) speed = ((32 * loop_length_size + 32 * in_mem_load_size + 64 * precalculations_size + 64 * data_pairs_size) / (current_time * 1000000)) print( 'Sending input streams to server total time:\t%.5lfs' % current_time + '\t(average speed = %.5lfMb/s)' % (speed)) # Allocate memory for output stream on server start_time = time.time() out_correlation_size = (num_timesteps * CORRELATION_NUM_TOP_SCORES * CORRELATION_NUM_PIPES * loop_length + num_bursts * 24 ) # for anything other than ISCA # 48 should be instead of 24 address_out_correlation = client.malloc_double(out_correlation_size) out_indices_size = (2 * num_timesteps * loop_length * CORRELATION_NUM_TOP_SCORES * CORRELATION_NUM_PIPES) address_out_indices = client.malloc_int32_t(out_indices_size) current_time = time.time() - start_time print('Allocating memory for output stream on server:\t%.5lfs' % current_time) # Initialize LMem start_time = time.time() actions_load_lmem = correlation_loadLMem_actions_t_struct( num_bursts, address_loop_length, address_in_mem_load) address_actions_load_lmem = client.send_correlation_loadLMem_actions_t( actions_load_lmem) client.correlation_loadLMem_run(engine, address_actions_load_lmem) current_time = time.time() - start_time print 'LMem initialization:\t\t\t\t%.5lfs' % current_time # Executing correlation action start_time = time.time() actions = correlation_actions_t_struct( num_bursts, # scalar input num_timesteps, # scalar input num_timeseries, # scalar input 1, # scalar input float(window_size), # scalar input address_precalculations, # streaming reordered input address_data_pairs, # streaming reordered input address_out_correlation, # streaming unordered output address_out_indices) # streaming unordered output address_actions = client.send_correlation_actions_t(actions) client.correlation_run(engine, address_actions) current_time = time.time() - start_time print 'Correlation time:\t\t\t\t%.5lfs' % current_time # Unload DFE start_time = time.time() client.max_unload(engine) current_time = time.time() - start_time print 'Unloading DFE:\t\t\t\t\t%.5lfs' % current_time # Get output stream from server start_time = time.time() out_correlation = client.receive_data_double(address_out_correlation, out_correlation_size) out_correlation_time = time.time() - start_time print('\tGet output stream Correlation:\t(size = %d bit)\t%.5lfs' % (64 * out_correlation_size, out_correlation_time)) start_time = time.time() _ = client.receive_data_int32_t(address_out_indices, out_indices_size) out_indices_time = time.time() - start_time print('\tGet output stream outIndices:\t(size = %d bit)\t%.5lfs' % (32 * out_indices_size, out_indices_time)) start_time = time.time() loop_length_size = 1 loop_length = client.receive_data_int32_t(address_loop_length, loop_length_size) loop_length_time = time.time() - start_time print('\tGet output stream loopLength:\t(size = %d bit)\t\t%.5lfs' % (32 * loop_length_size, loop_length_time)) current_time = (out_indices_time + out_correlation_time + loop_length_time) speed = ((32 * loop_length_size + 32 * out_indices_time + 64 * out_correlation_size) / (current_time * 1000000)) print( 'Getting output stream from server total time:\t%.5lfs' % current_time + '\t(average speed = %.5lfMb/s)' % (speed)) # Free allocated memory for streams on server start_time = time.time() client.free(address_loop_length) client.free(address_in_mem_load) client.free(address_precalculations) client.free(address_data_pairs) client.free(address_out_correlation) client.free(address_out_indices) client.free(address_actions) client.free(address_actions_load_lmem) current_time = time.time() - start_time print('Freeing allocated memory for streams on server:\t%.5lfs' % current_time) # Free allocated maxfile data start_time = time.time() client.correlation_free() current_time = time.time() - start_time print 'Freeing allocated maxfile data:\t\t\t%.5lfs' % current_time # Close! start_time = time.time() transport.close() current_time = time.time() - start_time print 'Closing connection:\t\t\t\t%.5lfs' % current_time # Store data start_time = time.time() position = 0 index = 0 start = ((num_timesteps - 1) * loop_length[0] * CORRELATION_NUM_TOP_SCORES * CORRELATION_NUM_PIPES) for i in range(num_timeseries): for j in range(i): correlations[index + j] = out_correlation[start + position + j] index += i position += ((i / 12) + 1) * 12 current_time = time.time() - start_time print 'Storing time:\t\t\t\t\t%.5lfs' % current_time except Thrift.TException, thrift_exception: print '%s' % (thrift_exception.message) sys.exit(-1)
parser.add_argument("--fps", type=int, default=0, help="target fps for playback. Default 0: get from bvh file.") parser.add_argument("filename", type=str, help="BVH motion file to play.") args = parser.parse_args() file_in = args.filename if not os.path.exists(file_in): print("Error: file {} not found.".format(file_in)) sys.exit(1) fps = args.fps ### Network Thrift Server ### thrift_host = '127.0.0.1' out_port = args.port handler = LiveSkeletonHandler() processor = LiveSkeletonService.Processor(handler) transport = TSocket.TServerSocket(thrift_host, out_port) # Pass host in case we're not connected to a network. tfactory = TTransport.TFramedTransportFactory() pfactory = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TThreadedServer(processor, transport, tfactory, pfactory, daemon=True) # Start a thread with the server -- that thread will then start one # more thread for each request server_thread = threading.Thread(target=server.serve) # Exit the server thread when the main thread terminates server_thread.name = 'LiveSkeletonServerThread' server_thread.daemon = True server_thread.start() print("Server loop running on port {} in thread: {}".format(out_port, server_thread.name)) sys.stdout.flush() ### Simulation ### skeleton_simulator = LiveSkeletonSimulation(bvh_file=file_in, target_fps=fps)
def __thrift_to_json(x): trans = TTransport.TMemoryBuffer() proto = TSimpleJSONProtocol.TSimpleJSONProtocol(trans) x.write(proto) return json.loads(trans.getvalue())
from helloworld import HelloWorld from helloworld.ttypes import * from helloworld.constants import * from thrift import Thrift from thrift.transport import TSocket from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol try: # Make socket transport = TSocket.TSocket('127.0.0.1', 30303) # Buffering is critical. Raw sockets are very slow transport = TTransport.TBufferedTransport(transport) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder client = HelloWorld.Client(protocol) # Connect! transport.open() client.ping() print("ping()") msg = client.sayHello() print(msg)
def createPackage(args, static_input): ''' Creates a Storm submission package for the given dispel4py graph. :param module_name: name of the graph module that creates a graph :param attr: name of the graph attribute within the module - if None the first WorkflowGraph is used :param res: resource directory - if None the default is "resources" :rtype: name of the temporary directory that contains the submission package ''' module_name = args.module attr = args.attr res = args.resources if res is None: res = 'resources' graph = loadGraph(module_name, attr) # we don't want any nested subgraphs graph.flatten() # create a temporary directory tmpdir = tempfile.mkdtemp() resources_dir = tmpdir + '/' + res # copy dependencies of PEs in the graph to resources in temp directory shutil.copytree(res, resources_dir) dispel4py_dir = resources_dir + '/dispel4py' _mkdir_ifnotexists(dispel4py_dir) _mkdir_ifnotexists(dispel4py_dir + '/storm') shutil.copy('dispel4py/__init__.py', dispel4py_dir) shutil.copy('dispel4py/core.py', dispel4py_dir) shutil.copy('dispel4py/base.py', dispel4py_dir) shutil.copy('dispel4py/__init__.py', dispel4py_dir + '/storm/') shutil.copy('dispel4py/storm/utils.py', dispel4py_dir + '/storm/') # copy client and dependencies for storm submission to the temp directory dispel4py_dir = tmpdir + '/dispel4py' _mkdir_ifnotexists(dispel4py_dir) _mkdir_ifnotexists(dispel4py_dir + '/storm') shutil.copy('dispel4py/__init__.py', dispel4py_dir) shutil.copy('dispel4py/__init__.py', dispel4py_dir + '/storm/') shutil.copy('dispel4py/storm/client.py', dispel4py_dir + '/storm/') shutil.copy('dispel4py/storm/storm_submission_client.py', tmpdir) shutil.copy('java/src/dispel4py/storm/ThriftSubmit.java', tmpdir + '/dispel4py/storm/') sources = [] for node in graph.graph.nodes(): pe = node.getContainedObject() is_source = True for edge in graph.graph.edges(node, data=True): if pe == edge[2]['DIRECTION'][1]: is_source = False break if is_source: sources.append(pe) print "Sources: %s" % [pe.id for pe in sources] for pe in sources: pe._static_input = static_input # create the storm topology topology = buildTopology(graph) # cache PE dependencies imported from the registry to resources_dir registry.createResources(resources_dir, registry.currentRegistry()) # write thrift representation of the topology to a file transportOut = TTransport.TMemoryBuffer() protocolOut = TBinaryProtocol.TBinaryProtocol(transportOut) topology.write(protocolOut) bytes = transportOut.getvalue() with open(tmpdir + '/' + TOPOLOGY_THRIFT_FILE, "w") as thrift_file: thrift_file.write(bytes) return tmpdir