def run_worker_process(self): setup_logger(self.quiet, log) if self.registry_type == "UDP": if self.registry_host is None: self.registry_host = "255.255.255.255" self.registrar = UDPRegistryClient(ip=self.registry_host, port=self.registry_port) else: if self.registry_host is None: raise ValueError( "With TCP registry, you must specify --registry-host") self.registrar = TCPRegistryClient(ip=self.registry_host, port=self.registry_port) t = ThreadPoolServer(ServiceEnvelope.SERVICE_CLASS, hostname=self.host, port=self.port, reuse_addr=True, ipv6=self.ipv6, registrar=self.registrar, auto_register=self.auto_register, nbThreads=self.max_client_count, requestBatchSize=self.max_requests_per_client, protocol_config={ 'exposed_prefix': '', 'log_exceptions': True }) t.start()
def _remote(): """ Code to execute on forked process. """ service = _create_srpo_service(obj, name, registry_path=registry_path) # set new process group protocol = dict(allow_all_attrs=True) kwargs = dict( hostname="localhost", nbThreads=server_threads, protocol_config=protocol, port=port, ) server = ThreadPoolServer(service(), **kwargs) sql_kwargs = dict( filename=server_registry.filename, tablename=server_registry.tablename, flag="c", ) # register new server registery = SqliteDict(**sql_kwargs) registery[name] = (server.host, server.port, os.getpid()) registery.commit() # get a new new view of registry, make sure name is there assert name in SqliteDict(**sql_kwargs) service._server = server server.start()
def run_training(head_host, head_port, debug_out=None): """Main worker training routine (creates the Seq2SeqTrainingService and connects it to the head. @param head_host: hostname of the head @param head_port: head port number @param debug_out: path to the debugging output file (debug output discarded if None) """ # setup debugging output, if applicable if debug_out is not None: set_debug_stream(file_stream(debug_out, mode='w')) # start the server (in the background) log_info('Creating training server...') server = ThreadPoolServer(service=Seq2SeqTrainingService, nbThreads=1) server_thread = Thread(target=server.start) server_thread.start() my_host = socket.getfqdn() log_info('Worker server created at %s:%d. Connecting to head at %s:%d...' % (my_host, server.port, head_host, head_port)) # notify main about this server conn = connect(head_host, head_port, config={'allow_pickle': True}) conn.root.register_worker(my_host, server.port) conn.close() log_info('Worker is registered with the head.') # now serve until we're killed (the server thread will continue to run) server_thread.join()
def _init_server(self): """Initializes a server that registers new workers.""" registrar_class, ranker_dump_path = get_worker_registrar_for(self) n_tries = 0 self.server = None last_error = None while self.server is None and n_tries < 10: try: n_tries += 1 self.server = ThreadPoolServer(service=registrar_class, nbThreads=1, port=self.port) except socket.error as e: log_warn('Port %d in use, trying to use a higher port...' % self.port) self.port += 1 last_error = e if self.server is None: if last_error is not None: raise last_error raise Exception('Could not initialize server') self.services = set() self.free_services = deque() self.pending_requests = set() self.jobs = [] self.server_thread = Thread(target=self.server.start) self.server_thread.setDaemon(True) self.server_thread.start() self.ranker_dump_path = ranker_dump_path
def main(): x = [ ThreadPoolServer(raftnode.RaftNode('config.txt', 0), port=5001), ThreadPoolServer(raftnode.RaftNode('config.txt', 1), port=5002), ThreadPoolServer(raftnode.RaftNode('config.txt', 2), port=5003), ThreadPoolServer(raftnode.RaftNode('config.txt', 3), port=5004), ThreadPoolServer(raftnode.RaftNode('config.txt', 4), port=5005), ] server_list = [] for server in x: thread1 = threading.Thread(target=server.start) server_list.append(thread1) thread1.start() # test 1 while True: y = int(input("Leader ID: ")) print(str(rpyc.connect('localhost', 5000 + y).root.is_leader()))
def run_server(): print("Started.") serverLog = logging.getLogger("Main.RPyCServer") server = ThreadPoolServer(service=DbInterfaceServer, port=12345, hostname='localhost', logger=serverLog, nbThreads=6) server.start()
def __init__(self): self.axon_port = conf.AXON_PORT self.service = AxonService() self.protocol_config = self.service.RPYC_PROTOCOL_CONFIG self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.WARN) self.axon_service = ThreadPoolServer( self.service, port=self.axon_port, reuse_addr=True, protocol_config=self.protocol_config, logger=self.logger, nbThreads=50)
def run(self): self.authenticator = SSLAuthenticator( self.config.get("pupyd", "keyfile").replace("\\", os.sep).replace("/", os.sep), self.config.get("pupyd", "certfile").replace("\\", os.sep).replace("/", os.sep), ciphers="SHA256+AES256:SHA1+AES256:@STRENGTH") self.server = ThreadPoolServer(PupyService.PupyService, port=self.port, hostname=self.address, authenticator=self.authenticator) self.server.start()
def main(): # arguments parser = argparse.ArgumentParser() parser.add_argument('--pid', action='store', dest='pid', default='/var/tmp/harvester_rpc.pid', help='pid filename') parser.add_argument('--port', dest='port', type=int, default=18861, help='the TCP port to bind to') parser.add_argument('--backlog', dest='backlog', type=int, default=10, help='backlog for the port') parser.add_argument('--stdout', action='store', dest='stdout', default='/var/tmp/harvester_rpc.out', help='stdout filename') parser.add_argument('--stderr', action='store', dest='stderr', default='/var/tmp/harvester_rpc.err', help='stderr filename') options = parser.parse_args() # logger _logger = logging.getLogger('rpc_bot') setupLogger(_logger, pid=os.getpid()) # make daemon context outfile = open(options.stdout, 'a+') errfile = open(options.stderr, 'a+') dc = daemon.DaemonContext(pidfile=daemon.pidfile.PIDLockFile(options.pid), stdout=outfile, stderr=errfile) # run thread server with dc: from rpyc.utils.server import ThreadPoolServer t = ThreadPoolServer(RpcBot, port=options.port, backlog=options.backlog, logger=_logger, protocol_config={"allow_all_attrs": True}) t.start() # finalize outfile.close() errfile.close()
class MetadataStore(rpyc.Service): """ Initialize the class using the config file provided and also initialize any datastructures you may need. """ def __init__(self, config): pass ''' ModifyFile(f,v,hl): Modifies file f so that it now contains the contents refered to by the hashlist hl. The version provided, v, must be exactly one larger than the current version that the MetadataStore maintains. As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call ''' def exposed_modify_file(self, filename, version, hashlist): ''' DeleteFile(f,v): Deletes file f. Like ModifyFile(), the provided version number v must be one bigger than the most up-date-date version. As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call ''' def exposed_delete_file(self, filename, version): pass ''' (v,hl) = ReadFile(f): Reads the file with filename f, returning the most up-to-date version number v, and the corresponding hashlist hl. If the file does not exist, v will be 0. As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call ''' def exposed_read_file(self, filename): pass if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer server = ThreadPoolServer(MetadataStore(sys.argv[1]), port = 6000) server.start()
def start_server(service, thread=True, host='0.0.0.0', port=18861): from rpyc.utils.server import ThreadedServer, ThreadPoolServer server = ThreadPoolServer( service, hostname=host, port=port, reuse_addr=True, auto_register=True, ) if thread: import threading server_thread = threading.Thread(target=server.start) server_thread.daemon = True server_thread.start() return (host, port) else: server.start() return (host, port)
def run_worker(head_host, head_port, debug_out=None): # setup debugging output, if applicable if debug_out is not None: set_debug_stream(file_stream(debug_out, mode='w')) # start the server (in the background) log_info('Creating worker server...') server = ThreadPoolServer(service=RankerTrainingService, nbThreads=1) server_thread = Thread(target=server.start) server_thread.start() my_host = socket.getfqdn() log_info('Worker server created at %s:%d. Connecting to head at %s:%d...' % (my_host, server.port, head_host, head_port)) # notify main about this server conn = connect(head_host, head_port, config={'allow_pickle': True}) conn.root.register_worker(my_host, server.port) conn.close() log_info('Worker is registered with the head.') # now serve until we're killed (the server thread will continue to run) server_thread.join()
def start_service(service_class, port=None): """ Start an rpyc service given by the provided class. Port can be overridden. :param rpyc.Service service_class: a child class of :class:`rpyc.Service`. :param int port: the port the service should listen for requests on. If it isn't provided by the caller, we we get a value from :class:`combtest.config` :return: a handle to the resulting :class:`ThreadedServer`. """ # rpyc uses logging, and we want to dump its logging somehow on errors _ensure_stderr_handler() if port is None: port = get_service_port() else: set_service_port(port) from rpyc.utils.server import ThreadPoolServer t = ThreadPoolServer(service_class, port=port) t.start() return t
self.term = term self.hasVoted = True return True elif self.term == term and not self.hasVoted: self.hasVoted = True return True else: return False def exposed_who_is_leader(self): return self.leader def exposed_is_leader(self): return self.state == 'leader' if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer if len(sys.argv) != 3: print("usage: python %s <config.json> <id>" % sys.argv[0]) else: config = sys.argv[1] sid = int(sys.argv[2]) with open(config) as f: config = json.load(f) server_port = config['metastores'][sid * 2 + 1] server = ThreadPoolServer(MetadataStore(config, sid), port = server_port) server.start()
self.hashlist[h] = block # print(self.hashlist) """ b = get_block(h) : Retrieves a block indexed by hash value h As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call """ def exposed_get_block(self, h): return self.hashlist[h] """ True/False = has_block(h) : Signals whether block indexed by h exists in the BlockStore service As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call """ def exposed_has_block(self, h): return bool(self.hashlist.get(h)) if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer port = int(sys.argv[1]) server = ThreadPoolServer(BlockStore(), port=port) server.start()
self.active_leader = leaderId # notify of valid leader return (True, self.term) # run the program if __name__ == '__main__': # ensure the number of args supplied is correct if (len(sys.argv) != 4): print("ERROR: Invalid number of params - Required : 4 - Given : " + str(len(sys.argv))) print("\tFormat: python3 raftnode.py config_file, node_id, port") exit(-1) # create an instance of our class node = RaftNode(config=sys.argv[1], node_id=int(sys.argv[2]), port=int(sys.argv[3])) print('Attempting to run server') # start a thread to run the synchronization of our server t = threading.Thread(target=run_asynch, args=(node, )) t.start() # start the threadpoolserver which waits to accept incoming rpc logging.debug('Server running') server = ThreadPoolServer(node, port=int(sys.argv[3])) server.start()
#self.logfile.seek(0) # self.logfile = open('./tmp/log' + str(self.curNode) + '.txt', 'w') # self.logfile.write('Term: {0}\nVotedFor: {1}'.format(self.currentTerm, self.votedFor)) # self.logfile.flush() threading.Thread(target = self.updateLogFile) return True else: return False elif term < self.currentTerm: return False ''' x = is_leader(): returns True or False, depending on whether this node is a leader As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE ''' def exposed_is_leader(self): return self.currentLeader == self.curNode if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer nodeNum = sys.argv[2] port = sys.argv[3] server = ThreadPoolServer(RaftNode(sys.argv[1], nodeNum, port), port = int(port)) server.start()
""" :param text: the target text :return: (prediction, a list of important words) """ try: assert type(text) == str, "not valid input" except: pass res = self.ip2.predict_topk(text, 5) # res['labels_prob'] = [0.03528977019242901 * 31] #acsending sort # res['topk_label_proba'] = [('SPORTS', 0.03528977019242901) * 5] #acsending sort # res['label__feat_coef'] = {'SPORTS': [('curry', 0.5), ('james', 0.3), ('soccer', 0.3)] * 31 } #order by label name return json.dumps(res) def exposed_test(self, t=None): if t is not None: return t else: return "good job" if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer server = ThreadPoolServer(PredictServer(ip.explain, ip2.explain), port=PORT) server.start()
if term == self.term and self.voteFor == name: return self.term, True if term > self.term: self.convert_to_follower(term, False) with self._lock: self.voteFor = name self.persist(self.to_string()) return self.term, True ''' x = is_leader(): returns True or False, depending on whether this node is a leader As per rpyc syntax, addi ng the prefix 'exposed_' will expose this method as an RPC call CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE ''' def exposed_is_leader(self): return self.state == 2 if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer server = ThreadPoolServer(RaftNode(sys.argv[1], sys.argv[2]), port = int(sys.argv[3])) server.start() # python raftnode.py config.txt 0 5001
self.votedFor=request_id self.hasvoted==True return 1 else: return 0 def exposed_rec_heartbeats(self, heartbeat_id): self.last_update = time.time() ''' x = is_leader(): returns True or False, depending on whether this node is a leader As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE ''' def exposed_is_leader(self): return False if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer print (sys.argv[3]) port=int(sys.argv[3]) server = ThreadPoolServer(RaftNode(sys.argv[1]), port=port ) # print ('i', server) server.start()
def setUp(self): self.server = ThreadPoolServer(SlaveService, port=18878, auto_register=False) self.server.logger.quiet = False self.server._start_in_thread()
nodeNumStr = nodeNumLine[2:] nodeNumStr = nodeNumStr.strip() nodeNum = int(nodeNumStr) else: print("invalid config file- bad initial node count line: %s" % nodeNumLine) raise Exception("bad config file") if currNodeIndex < nodeNum: nodeDescriptions = configFile.readlines() if len(nodeDescriptions) == nodeNum: currNodeLine = nodeDescriptions[currNodeIndex] nodeTerms = currNodeLine.split(":") nodePortStr = nodeTerms[2].strip() currNodePort = int(nodePortStr) else: print( "invalid config file- wrong number of lines of node descriptions %s" % nodeNumLine) raise Exception("bad config file") else: print( "unacceptably high index %d for node system which only has %d nodes" % (currNodeIndex, nodeNum)) raise Exception("bad node index") if currNodePort > 0: server = ThreadPoolServer(RaftNode(configFileName, currNodeIndex), port=currNodePort) server.start()
self.queryTimer.cancel() self.queryTimer = None if self.leaderHost == "": print("no server connection") return #print("connecting", self.leaderHost, self.leaderPort) queryStr = 'SELECT RoomID, Type, Floor from RoomInfo where Type = 4' args = (self.leaderHost, self.leaderPort, queryStr) t = threading.Thread(target=self.connectAndQuery, args=args) t.start() def insert(self): if self.bookTimer != None: self.bookTimer.cancel() self.bookTimer = None if self.leaderHost == "": print("no server connection") return #print("connecting", self.leaderHost, self.leaderPort) inserStr = f'INSERT INTO RoomInfo (RoomID,Type,Floor) VALUES ({self.roomStartId+1}, 4, 3)' self.roomStartId += 1 args = (self.leaderHost, self.leaderPort, inserStr) t = threading.Thread(target=self.connectAndInsert, args=args) t.start() if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer client = ThreadPoolServer(Client(), port=4999) client.start()
}, '/static': { 'tools.sessions.on': False, 'tools.auth_basic.on': False, 'tools.auth_digest.on': False, 'tools.auth.on': False, 'tools.staticdir.on': True, 'tools.staticdir.dir': TWISTER_PATH + '/server/static' } } while 1: # Diff RPyc port RPYC_PORT = SERVER_PORT + 10 try: RPYC_SERVER = ThreadPoolServer(CeRpycService, port=RPYC_PORT, protocol_config=CONFIG) RPYC_SERVER.logger.setLevel(30) except Exception: logCritical('Twister Server: Cannot launch the RPyc server on port `{}`!'.format(RPYC_PORT)) exit(1) # Project manager does everything PROJ = Project() PROJ.rsrv = RPYC_SERVER # CE is the XML-RPC interface CE = CeXmlRpc(PROJ) def close(): """ Close server. """ RPYC_SERVER.close() logDebug('---- RPYC CLOSED')
PRINT_VERB.append('xstats') PRINT_VERB.append('master') [PRINT_VERB.append(b) for b in BACKBONES] printtime(('Threaded heartbeat server listening on port %d\n' 'press Ctrl-C to stop\n') % RPC_PORT) FINISHED_EVENT.set() printer = Printer(goOnEvent = FINISHED_EVENT) printer.start() runner = Runner() runner.start() try: t = ThreadPoolServer(MasterService, port = RPC_PORT) t.start() except Exception, e: printtime('%s' % e) FINISHED_EVENT.clear() printtime('Master_Server killing all clients') for host in NODE_WATCHERS: NODE_WATCHERS[host].clearFinish() NODE_WATCHERS[host].clearGoOn() while len(NODE_WATCHERS): time.sleep(1) printtime('Exiting, please wait...') printer.join()
for nodeIdx in self.allNodesHost: if nodeIdx == self.curNodeIdx: continue try: conn = rpyc.connect(self.allNodesHost[nodeIdx], self.allNodesPort[nodeIdx]) result = conn.root.commitAsFollower() if result == False: print("Node", self.allNodesPort[nodeIdx], "commit failed") allCommit = False break except Exception: print("Node", self.allNodesPort[nodeIdx], "commit failed") allCommit = False break committedNodes.append(nodeIdx) if allCommit == False: self.rollbackAsLeader(committedNodes) return allCommit ''' 1.5 ends ''' if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer nodeIdx = sys.argv[1] host = sys.argv[2] port = sys.argv[3] server = ThreadPoolServer(Node(nodeIdx, host, port), port = int(port)) server.start()
if not res: res="no response from planemo test...." allres.append(res) else: with open(planemo_log, 'w') as replog: replog.write(res) res = self.run_cmd(f"tar -cvz -f {ptooldir}/{toolname}_tested.toolshed.gz --directory {pwork} {toolname}") allres.append(res) res = self.run_cmd(f"cp -r {ptooldir} /galaxy-central/tools/TFtools/") allres.append(res) for fname in os.listdir(prepdir): print('fname', fname, collection) if fname.endswith('.json'): # if this is included, the html disappears. Go figure. continue res = self.run_cmd(f"cp {prepdir}/{fname} {collection}/{fname}") self.run_rsync(f"{ptooldir}/{toolname}_tested.toolshed.gz", toolwork) self.run_rsync(f"{ptooldir}/{toolname}_tested.toolshed.gz", galtooldir) res = self.run_cmd(f"chown -R galaxy:galaxy {pwork} {pworkrep} {galtooldir} {collection}") allres.append(res) return '\n'.join([x for x in allres if len(x) > 0]) if __name__ == "__main__": logger = logging.getLogger() logging.basicConfig(level='INFO') t = ThreadPoolServer(planemo_run, port=9999, logger=logger, nbThreads=1) # single thread experiment to see if planemo/conda behave better. Many condas spoil the conga. t.start()
if conn.root.is_leader() != True: self.leaderHost = "" self.leaderPort = 0 self.leaderIdx = -1 except Exception: print("Leader", self.leaderPort, "connection failed.") del self.allNodesHost[self.leaderIdx] del self.allNodesPort[self.leaderIdx] for nodeIdx in self.allNodesHost: args = (self.leaderIdx, self.allNodesHost[nodeIdx], self.allNodesPort[nodeIdx]) t = threading.Thread(target=self.broadcastRemoveMember, args=args) t.start() self.leaderHost = "" self.leaderPort = 0 self.leaderIdx = -1 print(self.allNodesHost) print(self.allNodesPort) if self.checkLeaderTimer != None: self.checkLeaderTimer.cancel() self.checkLeaderTimer = None if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer port = sys.argv[1] server = ThreadPoolServer(Middleware(), port=int(port)) server.start()
else: return version, str(self._filename_to_v_bl[f][1]) else: # print('version:', 0) print('version:', 0) return 0, str([]) def eprint(*args, **kwargs): """ Use eprint to print debug messages to stderr Example: - self.eprint("This is a debug message") :param args: :param kwargs: :return: """ print(*args, file=sys.stderr, **kwargs) if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer # rpyc.core.vinegar._generic_exceptions_cache['ErrorResponse'] = ErrorResponse config_filename = sys.argv[1] with open(config_filename, 'r') as config_f: port = int(config_f.readlines()[1].split(':')[2].strip()) server = ThreadPoolServer(MetadataStore(sys.argv[1]), port=port) server.start()
''' def exposed_is_leader(self): return self.leader_id == self.id def read_config(self, file): config_dict = {} with open(file) as f: N = int(f.readline().split()[-1]) for i in range(N): line = f.readline() id, ip, port_num = line.split(":") id = id[4:] ip = ip.strip() port_num = int(port_num.strip()) config_dict[id] = (ip, port_num) return config_dict def exposed_say_hi(self): # flush_print(f"hello my name is {self.id}") return if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer print("flag 1", flush=True) server = ThreadPoolServer(RaftNode(argv.config, argv.node_id), port=int(argv.port)) server.start() print("flag 2", flush=True)