def run_worker_process(self):
setup_logger(self.quiet, log)
if self.registry_type == "UDP":
if self.registry_host is None:
self.registry_host = "255.255.255.255"
self.registrar = UDPRegistryClient(ip=self.registry_host,
port=self.registry_port)
else:
if self.registry_host is None:
raise ValueError(
"With TCP registry, you must specify --registry-host")
self.registrar = TCPRegistryClient(ip=self.registry_host,
port=self.registry_port)
t = ThreadPoolServer(ServiceEnvelope.SERVICE_CLASS,
hostname=self.host,
port=self.port,
reuse_addr=True,
ipv6=self.ipv6,
registrar=self.registrar,
auto_register=self.auto_register,
nbThreads=self.max_client_count,
requestBatchSize=self.max_requests_per_client,
protocol_config={
'exposed_prefix': '',
'log_exceptions': True
})
t.start()
def _remote():
""" Code to execute on forked process. """
service = _create_srpo_service(obj, name, registry_path=registry_path)
# set new process group
protocol = dict(allow_all_attrs=True)
kwargs = dict(
hostname="localhost",
nbThreads=server_threads,
protocol_config=protocol,
port=port,
)
server = ThreadPoolServer(service(), **kwargs)
sql_kwargs = dict(
filename=server_registry.filename,
tablename=server_registry.tablename,
flag="c",
)
# register new server
registery = SqliteDict(**sql_kwargs)
registery[name] = (server.host, server.port, os.getpid())
registery.commit()
# get a new new view of registry, make sure name is there
assert name in SqliteDict(**sql_kwargs)
service._server = server
server.start()
def run_training(head_host, head_port, debug_out=None):
"""Main worker training routine (creates the Seq2SeqTrainingService and connects it to the
head.
@param head_host: hostname of the head
@param head_port: head port number
@param debug_out: path to the debugging output file (debug output discarded if None)
"""
# setup debugging output, if applicable
if debug_out is not None:
set_debug_stream(file_stream(debug_out, mode='w'))
# start the server (in the background)
log_info('Creating training server...')
server = ThreadPoolServer(service=Seq2SeqTrainingService, nbThreads=1)
server_thread = Thread(target=server.start)
server_thread.start()
my_host = socket.getfqdn()
log_info('Worker server created at %s:%d. Connecting to head at %s:%d...' %
(my_host, server.port, head_host, head_port))
# notify main about this server
conn = connect(head_host, head_port, config={'allow_pickle': True})
conn.root.register_worker(my_host, server.port)
conn.close()
log_info('Worker is registered with the head.')
# now serve until we're killed (the server thread will continue to run)
server_thread.join()
def _init_server(self):
"""Initializes a server that registers new workers."""
registrar_class, ranker_dump_path = get_worker_registrar_for(self)
n_tries = 0
self.server = None
last_error = None
while self.server is None and n_tries < 10:
try:
n_tries += 1
self.server = ThreadPoolServer(service=registrar_class,
nbThreads=1,
port=self.port)
except socket.error as e:
log_warn('Port %d in use, trying to use a higher port...' %
self.port)
self.port += 1
last_error = e
if self.server is None:
if last_error is not None:
raise last_error
raise Exception('Could not initialize server')
self.services = set()
self.free_services = deque()
self.pending_requests = set()
self.jobs = []
self.server_thread = Thread(target=self.server.start)
self.server_thread.setDaemon(True)
self.server_thread.start()
self.ranker_dump_path = ranker_dump_path
def main():
x = [
ThreadPoolServer(raftnode.RaftNode('config.txt', 0), port=5001),
ThreadPoolServer(raftnode.RaftNode('config.txt', 1), port=5002),
ThreadPoolServer(raftnode.RaftNode('config.txt', 2), port=5003),
ThreadPoolServer(raftnode.RaftNode('config.txt', 3), port=5004),
ThreadPoolServer(raftnode.RaftNode('config.txt', 4), port=5005),
]
server_list = []
for server in x:
thread1 = threading.Thread(target=server.start)
server_list.append(thread1)
thread1.start()
# test 1
while True:
y = int(input("Leader ID: "))
print(str(rpyc.connect('localhost', 5000 + y).root.is_leader()))
def run_server():
print("Started.")
serverLog = logging.getLogger("Main.RPyCServer")
server = ThreadPoolServer(service=DbInterfaceServer,
port=12345,
hostname='localhost',
logger=serverLog,
nbThreads=6)
server.start()
def __init__(self):
self.axon_port = conf.AXON_PORT
self.service = AxonService()
self.protocol_config = self.service.RPYC_PROTOCOL_CONFIG
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.WARN)
self.axon_service = ThreadPoolServer(
self.service,
port=self.axon_port,
reuse_addr=True,
protocol_config=self.protocol_config,
logger=self.logger, nbThreads=50)
def run(self):
self.authenticator = SSLAuthenticator(
self.config.get("pupyd",
"keyfile").replace("\\",
os.sep).replace("/", os.sep),
self.config.get("pupyd",
"certfile").replace("\\",
os.sep).replace("/", os.sep),
ciphers="SHA256+AES256:SHA1+AES256:@STRENGTH")
self.server = ThreadPoolServer(PupyService.PupyService,
port=self.port,
hostname=self.address,
authenticator=self.authenticator)
self.server.start()
def main():
# arguments
parser = argparse.ArgumentParser()
parser.add_argument('--pid',
action='store',
dest='pid',
default='/var/tmp/harvester_rpc.pid',
help='pid filename')
parser.add_argument('--port',
dest='port',
type=int,
default=18861,
help='the TCP port to bind to')
parser.add_argument('--backlog',
dest='backlog',
type=int,
default=10,
help='backlog for the port')
parser.add_argument('--stdout',
action='store',
dest='stdout',
default='/var/tmp/harvester_rpc.out',
help='stdout filename')
parser.add_argument('--stderr',
action='store',
dest='stderr',
default='/var/tmp/harvester_rpc.err',
help='stderr filename')
options = parser.parse_args()
# logger
_logger = logging.getLogger('rpc_bot')
setupLogger(_logger, pid=os.getpid())
# make daemon context
outfile = open(options.stdout, 'a+')
errfile = open(options.stderr, 'a+')
dc = daemon.DaemonContext(pidfile=daemon.pidfile.PIDLockFile(options.pid),
stdout=outfile,
stderr=errfile)
# run thread server
with dc:
from rpyc.utils.server import ThreadPoolServer
t = ThreadPoolServer(RpcBot,
port=options.port,
backlog=options.backlog,
logger=_logger,
protocol_config={"allow_all_attrs": True})
t.start()
# finalize
outfile.close()
errfile.close()
class MetadataStore(rpyc.Service):
"""
Initialize the class using the config file provided and also initialize
any datastructures you may need.
"""
def __init__(self, config):
pass
'''
ModifyFile(f,v,hl): Modifies file f so that it now contains the
contents refered to by the hashlist hl. The version provided, v, must
be exactly one larger than the current version that the MetadataStore
maintains.
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
'''
def exposed_modify_file(self, filename, version, hashlist):
'''
DeleteFile(f,v): Deletes file f. Like ModifyFile(), the provided
version number v must be one bigger than the most up-date-date version.
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
'''
def exposed_delete_file(self, filename, version):
pass
'''
(v,hl) = ReadFile(f): Reads the file with filename f, returning the
most up-to-date version number v, and the corresponding hashlist hl. If
the file does not exist, v will be 0.
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
'''
def exposed_read_file(self, filename):
pass
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
server = ThreadPoolServer(MetadataStore(sys.argv[1]), port = 6000)
server.start()
def start_server(service, thread=True, host='0.0.0.0', port=18861):
from rpyc.utils.server import ThreadedServer, ThreadPoolServer
server = ThreadPoolServer(
service,
hostname=host,
port=port,
reuse_addr=True,
auto_register=True,
)
if thread:
import threading
server_thread = threading.Thread(target=server.start)
server_thread.daemon = True
server_thread.start()
return (host, port)
else:
server.start()
return (host, port)
def run_worker(head_host, head_port, debug_out=None):
# setup debugging output, if applicable
if debug_out is not None:
set_debug_stream(file_stream(debug_out, mode='w'))
# start the server (in the background)
log_info('Creating worker server...')
server = ThreadPoolServer(service=RankerTrainingService, nbThreads=1)
server_thread = Thread(target=server.start)
server_thread.start()
my_host = socket.getfqdn()
log_info('Worker server created at %s:%d. Connecting to head at %s:%d...' %
(my_host, server.port, head_host, head_port))
# notify main about this server
conn = connect(head_host, head_port, config={'allow_pickle': True})
conn.root.register_worker(my_host, server.port)
conn.close()
log_info('Worker is registered with the head.')
# now serve until we're killed (the server thread will continue to run)
server_thread.join()
def start_service(service_class, port=None):
"""
Start an rpyc service given by the provided class. Port can be overridden.
:param rpyc.Service service_class: a child class of :class:`rpyc.Service`.
:param int port: the port the service should listen for requests on. If it
isn't provided by the caller, we we get a value from
:class:`combtest.config`
:return: a handle to the resulting :class:`ThreadedServer`.
"""
# rpyc uses logging, and we want to dump its logging somehow on errors
_ensure_stderr_handler()
if port is None:
port = get_service_port()
else:
set_service_port(port)
from rpyc.utils.server import ThreadPoolServer
t = ThreadPoolServer(service_class, port=port)
t.start()
return t
self.term = term
self.hasVoted = True
return True
elif self.term == term and not self.hasVoted:
self.hasVoted = True
return True
else:
return False
def exposed_who_is_leader(self):
return self.leader
def exposed_is_leader(self):
return self.state == 'leader'
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
if len(sys.argv) != 3:
print("usage: python %s <config.json> <id>" % sys.argv[0])
else:
config = sys.argv[1]
sid = int(sys.argv[2])
with open(config) as f:
config = json.load(f)
server_port = config['metastores'][sid * 2 + 1]
server = ThreadPoolServer(MetadataStore(config, sid), port = server_port)
server.start()
self.hashlist[h] = block
# print(self.hashlist)
"""
b = get_block(h) : Retrieves a block indexed by hash value h
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
"""
def exposed_get_block(self, h):
return self.hashlist[h]
"""
True/False = has_block(h) : Signals whether block indexed by h exists
in the BlockStore service
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
"""
def exposed_has_block(self, h):
return bool(self.hashlist.get(h))
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
port = int(sys.argv[1])
server = ThreadPoolServer(BlockStore(), port=port)
server.start()
self.active_leader = leaderId
# notify of valid leader
return (True, self.term)
# run the program
if __name__ == '__main__':
# ensure the number of args supplied is correct
if (len(sys.argv) != 4):
print("ERROR: Invalid number of params - Required : 4 - Given : " +
str(len(sys.argv)))
print("\tFormat: python3 raftnode.py config_file, node_id, port")
exit(-1)
# create an instance of our class
node = RaftNode(config=sys.argv[1],
node_id=int(sys.argv[2]),
port=int(sys.argv[3]))
print('Attempting to run server')
# start a thread to run the synchronization of our server
t = threading.Thread(target=run_asynch, args=(node, ))
t.start()
# start the threadpoolserver which waits to accept incoming rpc
logging.debug('Server running')
server = ThreadPoolServer(node, port=int(sys.argv[3]))
server.start()
#self.logfile.seek(0)
# self.logfile = open('./tmp/log' + str(self.curNode) + '.txt', 'w')
# self.logfile.write('Term: {0}\nVotedFor: {1}'.format(self.currentTerm, self.votedFor))
# self.logfile.flush()
threading.Thread(target = self.updateLogFile)
return True
else:
return False
elif term < self.currentTerm:
return False
'''
x = is_leader(): returns True or False, depending on whether
this node is a leader
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE
'''
def exposed_is_leader(self):
return self.currentLeader == self.curNode
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
nodeNum = sys.argv[2]
port = sys.argv[3]
server = ThreadPoolServer(RaftNode(sys.argv[1], nodeNum, port), port = int(port))
server.start()
"""
:param text: the target text
:return: (prediction, a list of important words)
"""
try:
assert type(text) == str, "not valid input"
except:
pass
res = self.ip2.predict_topk(text, 5)
# res['labels_prob'] = [0.03528977019242901 * 31] #acsending sort
# res['topk_label_proba'] = [('SPORTS', 0.03528977019242901) * 5] #acsending sort
# res['label__feat_coef'] = {'SPORTS': [('curry', 0.5), ('james', 0.3), ('soccer', 0.3)] * 31 } #order by label name
return json.dumps(res)
def exposed_test(self, t=None):
if t is not None:
return t
else:
return "good job"
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
server = ThreadPoolServer(PredictServer(ip.explain, ip2.explain),
port=PORT)
server.start()
if term == self.term and self.voteFor == name:
return self.term, True
if term > self.term:
self.convert_to_follower(term, False)
with self._lock:
self.voteFor = name
self.persist(self.to_string())
return self.term, True
'''
x = is_leader(): returns True or False, depending on whether
this node is a leader
As per rpyc syntax, addi
ng the prefix 'exposed_' will expose this
method as an RPC call
CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE
'''
def exposed_is_leader(self):
return self.state == 2
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
server = ThreadPoolServer(RaftNode(sys.argv[1], sys.argv[2]), port = int(sys.argv[3]))
server.start()
# python raftnode.py config.txt 0 5001
self.votedFor=request_id
self.hasvoted==True
return 1
else:
return 0
def exposed_rec_heartbeats(self, heartbeat_id):
self.last_update = time.time()
'''
x = is_leader(): returns True or False, depending on whether
this node is a leader
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
CHANGE THIS METHOD TO RETURN THE APPROPRIATE RESPONSE
'''
def exposed_is_leader(self):
return False
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
print (sys.argv[3])
port=int(sys.argv[3])
server = ThreadPoolServer(RaftNode(sys.argv[1]), port=port )
# print ('i', server)
server.start()
def setUp(self):
self.server = ThreadPoolServer(SlaveService,
port=18878,
auto_register=False)
self.server.logger.quiet = False
self.server._start_in_thread()
nodeNumStr = nodeNumLine[2:]
nodeNumStr = nodeNumStr.strip()
nodeNum = int(nodeNumStr)
else:
print("invalid config file- bad initial node count line: %s" %
nodeNumLine)
raise Exception("bad config file")
if currNodeIndex < nodeNum:
nodeDescriptions = configFile.readlines()
if len(nodeDescriptions) == nodeNum:
currNodeLine = nodeDescriptions[currNodeIndex]
nodeTerms = currNodeLine.split(":")
nodePortStr = nodeTerms[2].strip()
currNodePort = int(nodePortStr)
else:
print(
"invalid config file- wrong number of lines of node descriptions %s"
% nodeNumLine)
raise Exception("bad config file")
else:
print(
"unacceptably high index %d for node system which only has %d nodes"
% (currNodeIndex, nodeNum))
raise Exception("bad node index")
if currNodePort > 0:
server = ThreadPoolServer(RaftNode(configFileName, currNodeIndex),
port=currNodePort)
server.start()
self.queryTimer.cancel()
self.queryTimer = None
if self.leaderHost == "":
print("no server connection")
return
#print("connecting", self.leaderHost, self.leaderPort)
queryStr = 'SELECT RoomID, Type, Floor from RoomInfo where Type = 4'
args = (self.leaderHost, self.leaderPort, queryStr)
t = threading.Thread(target=self.connectAndQuery, args=args)
t.start()
def insert(self):
if self.bookTimer != None:
self.bookTimer.cancel()
self.bookTimer = None
if self.leaderHost == "":
print("no server connection")
return
#print("connecting", self.leaderHost, self.leaderPort)
inserStr = f'INSERT INTO RoomInfo (RoomID,Type,Floor) VALUES ({self.roomStartId+1}, 4, 3)'
self.roomStartId += 1
args = (self.leaderHost, self.leaderPort, inserStr)
t = threading.Thread(target=self.connectAndInsert, args=args)
t.start()
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
client = ThreadPoolServer(Client(), port=4999)
client.start()
},
'/static': {
'tools.sessions.on': False,
'tools.auth_basic.on': False,
'tools.auth_digest.on': False,
'tools.auth.on': False,
'tools.staticdir.on': True,
'tools.staticdir.dir': TWISTER_PATH + '/server/static'
}
}
while 1:
# Diff RPyc port
RPYC_PORT = SERVER_PORT + 10
try:
RPYC_SERVER = ThreadPoolServer(CeRpycService, port=RPYC_PORT, protocol_config=CONFIG)
RPYC_SERVER.logger.setLevel(30)
except Exception:
logCritical('Twister Server: Cannot launch the RPyc server on port `{}`!'.format(RPYC_PORT))
exit(1)
# Project manager does everything
PROJ = Project()
PROJ.rsrv = RPYC_SERVER
# CE is the XML-RPC interface
CE = CeXmlRpc(PROJ)
def close():
""" Close server. """
RPYC_SERVER.close()
logDebug('---- RPYC CLOSED')
PRINT_VERB.append('xstats')
PRINT_VERB.append('master')
[PRINT_VERB.append(b) for b in BACKBONES]
printtime(('Threaded heartbeat server listening on port %d\n'
'press Ctrl-C to stop\n') % RPC_PORT)
FINISHED_EVENT.set()
printer = Printer(goOnEvent = FINISHED_EVENT)
printer.start()
runner = Runner()
runner.start()
try:
t = ThreadPoolServer(MasterService, port = RPC_PORT)
t.start()
except Exception, e:
printtime('%s' % e)
FINISHED_EVENT.clear()
printtime('Master_Server killing all clients')
for host in NODE_WATCHERS:
NODE_WATCHERS[host].clearFinish()
NODE_WATCHERS[host].clearGoOn()
while len(NODE_WATCHERS): time.sleep(1)
printtime('Exiting, please wait...')
printer.join()
for nodeIdx in self.allNodesHost:
if nodeIdx == self.curNodeIdx:
continue
try:
conn = rpyc.connect(self.allNodesHost[nodeIdx], self.allNodesPort[nodeIdx])
result = conn.root.commitAsFollower()
if result == False:
print("Node", self.allNodesPort[nodeIdx], "commit failed")
allCommit = False
break
except Exception:
print("Node", self.allNodesPort[nodeIdx], "commit failed")
allCommit = False
break
committedNodes.append(nodeIdx)
if allCommit == False:
self.rollbackAsLeader(committedNodes)
return allCommit
'''
1.5 ends
'''
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
nodeIdx = sys.argv[1]
host = sys.argv[2]
port = sys.argv[3]
server = ThreadPoolServer(Node(nodeIdx, host, port), port = int(port))
server.start()
if not res:
res="no response from planemo test...."
allres.append(res)
else:
with open(planemo_log, 'w') as replog:
replog.write(res)
res = self.run_cmd(f"tar -cvz -f {ptooldir}/{toolname}_tested.toolshed.gz --directory {pwork} {toolname}")
allres.append(res)
res = self.run_cmd(f"cp -r {ptooldir} /galaxy-central/tools/TFtools/")
allres.append(res)
for fname in os.listdir(prepdir):
print('fname', fname, collection)
if fname.endswith('.json'): # if this is included, the html disappears. Go figure.
continue
res = self.run_cmd(f"cp {prepdir}/{fname} {collection}/{fname}")
self.run_rsync(f"{ptooldir}/{toolname}_tested.toolshed.gz", toolwork)
self.run_rsync(f"{ptooldir}/{toolname}_tested.toolshed.gz", galtooldir)
res = self.run_cmd(f"chown -R galaxy:galaxy {pwork} {pworkrep} {galtooldir} {collection}")
allres.append(res)
return '\n'.join([x for x in allres if len(x) > 0])
if __name__ == "__main__":
logger = logging.getLogger()
logging.basicConfig(level='INFO')
t = ThreadPoolServer(planemo_run, port=9999, logger=logger, nbThreads=1)
# single thread experiment to see if planemo/conda behave better. Many condas spoil the conga.
t.start()
if conn.root.is_leader() != True:
self.leaderHost = ""
self.leaderPort = 0
self.leaderIdx = -1
except Exception:
print("Leader", self.leaderPort, "connection failed.")
del self.allNodesHost[self.leaderIdx]
del self.allNodesPort[self.leaderIdx]
for nodeIdx in self.allNodesHost:
args = (self.leaderIdx, self.allNodesHost[nodeIdx],
self.allNodesPort[nodeIdx])
t = threading.Thread(target=self.broadcastRemoveMember,
args=args)
t.start()
self.leaderHost = ""
self.leaderPort = 0
self.leaderIdx = -1
print(self.allNodesHost)
print(self.allNodesPort)
if self.checkLeaderTimer != None:
self.checkLeaderTimer.cancel()
self.checkLeaderTimer = None
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
port = sys.argv[1]
server = ThreadPoolServer(Middleware(), port=int(port))
server.start()
else:
return version, str(self._filename_to_v_bl[f][1])
else:
# print('version:', 0)
print('version:', 0)
return 0, str([])
def eprint(*args, **kwargs):
"""
Use eprint to print debug messages to stderr
Example:
- self.eprint("This is a debug message")
:param args:
:param kwargs:
:return:
"""
print(*args, file=sys.stderr, **kwargs)
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
# rpyc.core.vinegar._generic_exceptions_cache['ErrorResponse'] = ErrorResponse
config_filename = sys.argv[1]
with open(config_filename, 'r') as config_f:
port = int(config_f.readlines()[1].split(':')[2].strip())
server = ThreadPoolServer(MetadataStore(sys.argv[1]), port=port)
server.start()
'''
def exposed_is_leader(self):
return self.leader_id == self.id
def read_config(self, file):
config_dict = {}
with open(file) as f:
N = int(f.readline().split()[-1])
for i in range(N):
line = f.readline()
id, ip, port_num = line.split(":")
id = id[4:]
ip = ip.strip()
port_num = int(port_num.strip())
config_dict[id] = (ip, port_num)
return config_dict
def exposed_say_hi(self):
# flush_print(f"hello my name is {self.id}")
return
if __name__ == '__main__':
from rpyc.utils.server import ThreadPoolServer
print("flag 1", flush=True)
server = ThreadPoolServer(RaftNode(argv.config, argv.node_id),
port=int(argv.port))
server.start()
print("flag 2", flush=True)
