def main():
global LOG_STREAMS
op = ArgumentParser(description="PYME rule server for task distribution. This should run once per cluster.")
#NOTE - currently squatting on port 15346 for testing - TODO can we use an ephemeral port
op.add_argument('-p', '--port', dest='port', default=config.get('ruleserver-port', 15346), type=int,
help="port number to serve on (default: 15346, see also 'ruleserver-port' config entry)")
op.add_argument('-a','--advertisements', dest='advertisements', choices=['zeroconf', 'local'], default='zeroconf',
help='Optionally restrict advertisements to local machine')
args = op.parse_args()
serverPort = args.port
if args.advertisements == 'local':
#bind on localhost
bind_addr = '127.0.0.1'
else:
bind_addr = '' #bind all interfaces
#set up logging
data_root = config.get('dataserver-root')
if data_root:
distr_log_dir = '%s/LOGS' % data_root
try: # make sure the directory exists
os.makedirs(distr_log_dir) # exist_ok flag not present on py2
except OSError as e:
import errno
if e.errno != errno.EEXIST:
raise e
dist_log_err_file = os.path.join(distr_log_dir, 'distributor.log')
if os.path.exists(dist_log_err_file):
os.remove(dist_log_err_file)
dist_err_handler = logging.handlers.RotatingFileHandler(filename=dist_log_err_file, mode='w', maxBytes=1e6, backupCount=1)
#dist_err_handler.setFormatter(logging.Formatter('%(message)s'))
distLogErr = logging.getLogger('distributor')
distLogErr.setLevel(logging.DEBUG)
distLogErr.addHandler(dist_err_handler)
proc = ruleserver.ServerThread(serverPort, bind_addr=bind_addr, profile=False)
proc.start()
#proc = subprocess.Popen('python -m PYME.ParallelTasks.distributor 1234', shell=True)
if args.advertisements == 'zeroconf':
ns = pyme_zeroconf.getNS('_pyme-taskdist')
else:
#assume 'local'
ns = sqlite_ns.getNS('_pyme-taskdist')
time.sleep(0.5)
#get the actual adress (port) we bound to
sa = proc.distributor.socket.getsockname()
service_name = get_service_name('PYMERuleServer')
ns.register_service(service_name, proc.externalAddr, int(sa[1]))
try:
while proc.is_alive():
time.sleep(1)
finally:
logger.debug('trying to shut down server')
proc.shutdown()
ns.unregister(service_name)
def main():
global LOG_STREAMS
confFile = os.path.join(conf.user_config_dir, 'distributor.yaml')
with open(confFile) as f:
config = yaml.load(f)
serverAddr, serverPort = config['distributor']['http_endpoint'].split(':')
externalAddr = socket.gethostbyname(socket.gethostname())
#set up logging
#logfile_error = None
#logfile_debug = None
data_root = conf.get('dataserver-root')
if data_root:
#logfile_error = open('%s/LOGS/distributor_error.log' % data_root, 'w')
#logfile_debug = open('%s/LOGS/distributor_debug.log' % data_root, 'w')
distr_log_dir = '%s/LOGS' % data_root
dist_log_err_file = os.path.join(distr_log_dir,
'distributor_error.log')
if os.path.exists(dist_log_err_file):
os.remove(dist_log_err_file)
dist_err_handler = logging.handlers.RotatingFileHandler(
dist_log_err_file, 'w', maxBytes=1e6, backupCount=1)
dist_err_handler.setFormatter(logging.Formatter('%(message)s'))
distLogErr = logging.getLogger('dist_err')
distLogErr.addHandler(dist_err_handler)
distLogErr.setLevel(logging.DEBUG)
distLogErr.propagate = False
dist_log_dbg_file = os.path.join(distr_log_dir,
'distributor_debug.log')
if os.path.exists(dist_log_dbg_file):
os.remove(dist_log_dbg_file)
dist_dbg_handler = logging.handlers.RotatingFileHandler(
dist_log_dbg_file, 'w', maxBytes=1e6, backupCount=1)
dist_dbg_handler.setFormatter(logging.Formatter('%(message)s'))
distLogDbg = logging.getLogger('dist_debug')
distLogDbg.addHandler(dist_dbg_handler)
distLogDbg.setLevel(logging.DEBUG)
distLogDbg.propagate = False
if not (len(sys.argv) == 2 and sys.argv[1] == '-n'):
proc = subprocess.Popen('distributor -c %s' % confFile,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
proc = subprocess.Popen('python -m PYME.cluster.distributor 1234',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
t_log_stderr = threading.Thread(target=log_stream,
args=(proc.stderr, distLogErr))
t_log_stderr.setDaemon(False)
t_log_stderr.start()
t_log_stdout = threading.Thread(target=log_stream,
args=(proc.stdout, distLogDbg))
t_log_stdout.setDaemon(False)
t_log_stdout.start()
else:
if not (len(sys.argv) == 2 and sys.argv[1] == '-n'):
proc = subprocess.Popen('distributor -c %s' % confFile, shell=True)
else:
proc = subprocess.Popen('python -m PYME.cluster.distributor 1234',
shell=True)
ns = pyme_zeroconf.getNS('_pyme-taskdist')
service_name = get_service_name('PYMEDistributor')
ns.register_service(service_name, externalAddr, int(serverPort))
try:
while not proc.poll():
time.sleep(1)
# if logfile_error:
# #do crude log rotation
# if logfile_error.tell() > 1e6:
# logfile_error.seek(0)
#
# if logfile_debug.tell() > 1e6:
# logfile_debug.seek(0)
finally:
ns.unregister(service_name)
#try and shut down the distributor cleanly
proc.send_signal(1)
time.sleep(2)
proc.kill()
LOG_STREAMS = False
def main(protocol="HTTP/1.0"):
global GPU_STATS
"""Test the HTTP request handler class.
This runs an HTTP server on port 8000 (or the first command line
argument).
"""
from optparse import OptionParser
op = OptionParser(usage='usage: %s [options]' % sys.argv[0])
#NOTE - currently squatting on port 15348 for testing - TODO can we use an ephemeral port?
op.add_option(
'-p',
'--port',
dest='port',
default=config.get('dataserver-port', 15348),
help=
"port number to serve on (default: 15348, see also 'dataserver-port' config entry)"
)
op.add_option('-t',
'--test',
dest='test',
help="Set up for bandwidth test (don't save files)",
action="store_true",
default=False)
op.add_option('-v',
'--protocol',
dest='protocol',
help="HTTP protocol version",
default="1.1")
op.add_option('-l',
'--log-requests',
dest='log_requests',
help="Display http request info",
default=False,
action="store_true")
default_root = config.get('dataserver-root', os.curdir)
op.add_option(
'-r',
'--root',
dest='root',
help=
"Root directory of virtual filesystem (default %s, see also 'dataserver-root' config entry)"
% dataserver_root,
default=default_root)
op.add_option('-k',
'--profile',
dest='profile',
help="Enable profiling",
default=False,
action="store_true")
op.add_option('--thread-profile',
dest='thread_profile',
help="Enable thread profiling",
default=False,
action="store_true")
default_server_filter = config.get('dataserver-filter', compName)
op.add_option(
'-f',
'--server-filter',
dest='server_filter',
help='Add a serverfilter for distinguishing between different clusters',
default=default_server_filter)
op.add_option(
'--timeout-test',
dest='timeout_test',
help=
'deliberately make requests timeout for testing error handling in calling modules',
default=0)
op.add_option('-a',
'--advertisements',
dest='advertisements',
choices=['zeroconf', 'local'],
default='zeroconf',
help='Optionally restrict advertisements to local machine')
options, args = op.parse_args()
if options.profile:
from PYME.util import mProfile
mProfile.profileOn(['HTTPDataServer.py', 'clusterListing.py'])
profileOutDir = options.root + '/LOGS/%s/mProf' % compName
if options.thread_profile:
from PYME.util import fProfile
tp = fProfile.ThreadProfiler()
#tp.profile_on(subs=['PYME/', 'http/server', 'socketserver'],outfile=options.root + '/LOGS/%s/tProf/dataserver.txt' % compName)
tp.profile_on(subs=[
'PYME/',
],
outfile=options.root +
'/LOGS/%s/tProf/dataserver.txt' % compName)
# setup logging to file
log_dir = '%s/LOGS/%s' % (options.root, compName)
makedirs_safe(log_dir)
log_file = '%s/LOGS/%s/PYMEDataServer.log' % (options.root, compName)
fh = logging.handlers.RotatingFileHandler(filename=log_file,
mode='w',
maxBytes=1e6,
backupCount=1)
logger.addHandler(fh)
logger.info(
'========================================\nPYMEDataServer, running on python %s\n'
% sys.version)
#change to the dataserver root if given'
logger.info('Serving from directory: %s' % options.root)
os.chdir(options.root)
if options.advertisements == 'local':
# preference is to avoid zeroconf on clusterofone due to poor
# performance on crowded networks
if config.get('clusterIO-hybridns', True):
ns = sqlite_ns.getNS('_pyme-http')
else:
# if we aren't using the hybridns, we are using zeroconf in clusterIO
# TODO - warn that we might run into performance issues???
ns = pzc.getNS('_pyme-http')
server_address = ('127.0.0.1', int(options.port))
ip_addr = '127.0.0.1'
else:
#default
ns = pzc.getNS('_pyme-http')
server_address = ('', int(options.port))
try:
ip_addr = socket.gethostbyname(socket.gethostname())
except:
ip_addr = socket.gethostbyname(socket.gethostname() + '.local')
PYMEHTTPRequestHandler.protocol_version = 'HTTP/%s' % options.protocol
PYMEHTTPRequestHandler.bandwidthTesting = options.test
PYMEHTTPRequestHandler.timeoutTesting = options.timeout_test
PYMEHTTPRequestHandler.logrequests = options.log_requests
httpd = ThreadedHTTPServer(server_address, PYMEHTTPRequestHandler)
#httpd = http.server.HTTPServer(server_address, PYMEHTTPRequestHandler)
httpd.daemon_threads = True
#get the actual adress (port) we bound to
sa = httpd.socket.getsockname()
service_name = get_service_name('PYMEDataServer [%s]' %
options.server_filter)
ns.register_service(service_name, ip_addr, sa[1])
status['IPAddress'] = ip_addr
status['BindAddress'] = server_address
status['Port'] = sa[1]
status['Protocol'] = options.protocol
status['TestMode'] = options.test
status['ComputerName'] = GetComputerName()
if GPU_STATS:
try:
pynvml.nvmlInit()
except:
GPU_STATS = False
sp = statusPoller()
sp.start()
logger.info("Serving HTTP on %s port %d ..." % (ip_addr, sa[1]))
try:
httpd.serve_forever()
finally:
logger.info('Shutting down ...')
httpd.shutdown()
httpd.server_close()
ns.unregister(service_name)
if options.profile:
mProfile.report(display=False, profiledir=profileOutDir)
if options.thread_profile:
tp.profile_off()
sp.stop()
if GPU_STATS:
pynvml.nvmlShutdown()
try:
from pytest_cov.embed import cleanup
cleanup()
except:
pass
sys.exit()
def main():
op = ArgumentParser(
description=
"PYME node server for task distribution. This should run on every node of the cluster"
)
#NOTE - currently squatting on port 15347 for testing - TODO can we use an ephemeral port?
op.add_argument(
'-p',
'--port',
dest='port',
default=conf.get('nodeserver-port', 15347),
type=int,
help=
"port number to serve on (default: 15347, see also 'nodeserver-port' config entry)"
)
op.add_argument('-a',
'--advertisements',
dest='advertisements',
choices=['zeroconf', 'local'],
default='zeroconf',
help='Optionally restrict advertisements to local machine')
args = op.parse_args()
serverPort = args.port
externalAddr = socket.gethostbyname(socket.gethostname())
if args.advertisements == 'zeroconf':
ns = pyme_zeroconf.getNS('_pyme-taskdist')
else:
#assume local
ns = sqlite_ns.getNS('_pyme-taskdist')
externalAddr = '127.0.0.1' #bind to localhost
#TODO - move this into the nodeserver proper so that the ruleserver doesn't need to be up before we start
print(distribution.getDistributorInfo(ns).values())
distributors = [
u.lstrip('http://').rstrip('/')
for u in distribution.getDistributorInfo(ns).values()
]
#set up nodeserver logging
cluster_root = conf.get('dataserver-root')
if cluster_root:
nodeserver_log_dir = os.path.join(cluster_root, 'LOGS',
GetComputerName())
#remove old log files
try:
os.remove(os.path.join(nodeserver_log_dir, 'nodeserver.log'))
except OSError: # if we cant clear out old log files, we might not have a log directory set up
try:
if not os.path.exists(os.path.join(nodeserver_log_dir)):
os.makedirs(
os.path.join(nodeserver_log_dir)
) # NB - this will create all intermediate directories as well
except: # throw error because the RotatingFileHandler will fail to initialize
raise IOError('Unable to initialize log files at %s' %
nodeserver_log_dir)
try:
shutil.rmtree(os.path.join(nodeserver_log_dir, 'taskWorkerHTTP'))
except:
pass
nodeserver_log_handler = logging.handlers.RotatingFileHandler(
os.path.join(nodeserver_log_dir, 'nodeserver.log'),
'w',
maxBytes=1e6,
backupCount=0)
nodeserverLog = logging.getLogger('nodeserver')
nodeserverLog.setLevel(logging.DEBUG)
nodeserver_log_handler.setLevel(logging.DEBUG)
nodeserver_log_handler.setFormatter(formatter)
nodeserverLog.addHandler(nodeserver_log_handler)
nodeserverLog.addHandler(stream_handler)
#nodeserverLog.propagate=False
else:
nodeserver_log_dir = os.path.join(os.curdir, 'LOGS', GetComputerName())
nodeserverLog = logger
proc = rulenodeserver.ServerThread(distributors[0],
serverPort,
externalAddr=externalAddr,
profile=False)
proc.start()
# TODO - do we need this advertisement
#get the actual adress (port) we bound to
time.sleep(0.5)
sa = proc.nodeserver.socket.getsockname()
serverPort = int(sa[1])
service_name = get_service_name('PYMENodeServer')
ns.register_service(service_name, externalAddr, serverPort)
time.sleep(2)
nodeserverLog.debug('Launching worker processors')
numWorkers = conf.get('nodeserver-num_workers', cpu_count())
workerProcs = [
subprocess.Popen('"%s" -m PYME.cluster.taskWorkerHTTP -s %d' %
(sys.executable, serverPort),
shell=True,
stdin=subprocess.PIPE) for i in range(numWorkers - 1)
]
#last worker has profiling enabled
profiledir = os.path.join(nodeserver_log_dir, 'mProf')
workerProcs.append(
subprocess.Popen(
'"%s" -m PYME.cluster.taskWorkerHTTP -s % d -p --profile-dir="%s"'
% (sys.executable, serverPort, profiledir),
shell=True,
stdin=subprocess.PIPE))
try:
while proc.is_alive():
time.sleep(1)
finally:
logger.info('Shutting down workers')
try:
ns.unregister(service_name)
except:
pass
for p in workerProcs:
#ask the workers to quit (nicely)
try:
p.send_signal(1)
except:
pass
time.sleep(2)
for p in workerProcs:
#now kill them off
try:
p.kill()
except:
pass
logger.info('Shutting down nodeserver')
proc.shutdown()
proc.join()
logger.info('Workers and nodeserver are shut down')
sys.exit()
def main(protocol="HTTP/1.0"):
"""Test the HTTP request handler class.
This runs an HTTP server on port 8000 (or the first command line
argument).
"""
from optparse import OptionParser
op = OptionParser(usage='usage: %s [options] [filename]' % sys.argv[0])
op.add_option('-p',
'--port',
dest='port',
default=config.get('dataserver-port', 8080),
help="port number to serve on")
op.add_option('-t',
'--test',
dest='test',
help="Set up for bandwidth test (don't save files)",
action="store_true",
default=False)
op.add_option('-v',
'--protocol',
dest='protocol',
help="HTTP protocol version",
default="1.1")
op.add_option('-l',
'--log-requests',
dest='log_requests',
help="Display http request info",
default=False,
action="store_true")
op.add_option('-r',
'--root',
dest='root',
help="Root directory of virtual filesystem",
default=config.get('dataserver-root', os.curdir))
options, args = op.parse_args()
#change to the dataserver root if given
logger.info('Serving from directory: %s' % options.root)
os.chdir(options.root)
#PYMEHTTPRequestHandler.protocol_version = 'HTTP/%s' % options.protocol
server_config['bandwidthTesting'] = options.test
#PYMEHTTPRequestHandler.logrequests = options.log_requests
#httpd = ThreadedHTTPServer(server_address, PYMEHTTPRequestHandler)
ip_addr = socket.gethostbyname(socket.gethostname())
server_address = ('', int(options.port))
global_status['IPAddress'] = ip_addr
global_status['BindAddress'] = server_address
global_status['Port'] = int(options.port)
global_status['Protocol'] = options.protocol
global_status['TestMode'] = options.test
global_status['ComputerName'] = GetComputerName()
ns = pzc.getNS('_pyme-http')
service_name = get_service_name('PYMEDataServer')
ns.register_service(service_name, ip_addr, int(options.port))
print("Serving HTTP on %s port %d ..." % (ip_addr, options.port))
#wsgiref_server(options)
cherrypy_server(options)
def main():
global LOG_STREAMS
cluster_root = conf.get('dataserver-root', conf.user_config_dir)
confFile = os.path.join(conf.user_config_dir, 'nodeserver.yaml')
with open(confFile) as f:
config = yaml.load(f)
serverAddr, serverPort = config['nodeserver']['http_endpoint'].split(':')
externalAddr = socket.gethostbyname(socket.gethostname())
ns = pyme_zeroconf.getNS('_pyme-taskdist')
#
# #find distributor(s)
# distributors = []
# for name, info in ns.advertised_services.items():
# if name.startswith('PYMEDistributor'):
# distributors.append('%s:%d' % (socket.inet_ntoa(info.address), info.port))
distributors = [
u.lstrip('http://').rstrip('/')
for u in distribution.getDistributorInfo().values()
]
#modify the configuration to reflect the discovered distributor(s)
config['nodeserver']['distributors'] = distributors
#write a new config file for the nodeserver
with tempfile.NamedTemporaryFile(suffix='.yaml',
delete=False) as temp_conf_file:
temp_conf_file_name = temp_conf_file.name
temp_conf_file.write(yaml.dump(config))
logging.debug('Config file: ' + temp_conf_file_name)
#set up nodeserver logging
nodeserver_log_dir = os.path.join(cluster_root, 'LOGS', GetComputerName())
#remove old log files
try:
os.remove(os.path.join(nodeserver_log_dir, 'nodeserver.log'))
except OSError: # if we cant clear out old log files, we might not have a log directory set up
try:
os.makedirs(
os.path.join(nodeserver_log_dir)
) # NB - this will create all intermediate directories as well
except: # throw error because the RotatingFileHandler will fail to initialize
raise IOError('Unable to initialize log files at %s' %
nodeserver_log_dir)
pass
try:
shutil.rmtree(os.path.join(nodeserver_log_dir, 'taskWorkerHTTP'))
except:
pass
#nodeserverLog = open(os.path.join(nodeserver_log_dir, 'nodeserver.log'), 'w')
nodeserver_log_handler = logging.handlers.RotatingFileHandler(
os.path.join(nodeserver_log_dir, 'nodeserver.log'),
'w',
maxBytes=1e6,
backupCount=0)
nodeserver_log_handler.setFormatter(logging.Formatter('%(message)s'))
nodeserverLog = logging.getLogger('nodeserver')
nodeserverLog.addHandler(nodeserver_log_handler)
nodeserverLog.setLevel(logging.DEBUG)
nodeserverLog.propagate = False
if not (len(sys.argv) == 2 and sys.argv[1] == '-n'):
proc = subprocess.Popen('nodeserver -c %s' % temp_conf_file_name,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
proc = subprocess.Popen('python -m PYME.cluster.nodeserver %s %s' %
(distributors[0], serverPort),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
t_log_stderr = threading.Thread(target=log_stream,
args=(proc.stderr, nodeserverLog))
t_log_stderr.setDaemon(False)
t_log_stderr.start()
t_log_stdout = threading.Thread(target=log_stream,
args=(proc.stdout, nodeserverLog))
t_log_stdout.setDaemon(False)
t_log_stdout.start()
service_name = get_service_name('PYMENodeServer')
ns.register_service(service_name, externalAddr, int(serverPort))
time.sleep(2)
logging.debug('Launching worker processors')
numWorkers = config.get('numWorkers', cpu_count())
subprocess.Popen('"%s" -m PYME.cluster.PYMERuleNodeServer -a local -p 0' %
sys.executable,
shell=True)
workerProcs = [
subprocess.Popen('"%s" -m PYME.cluster.taskWorkerHTTP' %
sys.executable,
shell=True,
stdin=subprocess.PIPE) for i in range(numWorkers - 1)
]
#last worker has profiling enabled
profiledir = os.path.join(nodeserver_log_dir, 'mProf')
workerProcs.append(
subprocess.Popen('"%s" -m PYME.cluster.taskWorkerHTTP -p "%s"' %
(sys.executable, profiledir),
shell=True,
stdin=subprocess.PIPE))
try:
while not proc.poll():
time.sleep(1)
#try to keep log size under control by doing crude rotation
#if nodeserverLog.tell() > 1e6:
# nodeserverLog.seek(0)
except KeyboardInterrupt:
pass
finally:
LOG_STREAMS = False
logging.info('Shutting down workers')
try:
ns.unregister(service_name)
except:
pass
os.unlink(temp_conf_file_name)
for p in workerProcs:
#ask the workers to quit (nicely)
try:
p.send_signal(1)
except:
pass
time.sleep(2)
for p in workerProcs:
#now kill them off
try:
p.kill()
except:
pass
logging.info('Shutting down nodeserver')
try:
proc.kill()
except:
pass
logging.info('Workers and nodeserver are shut down')
sys.exit()