def _genURI(taskQueueName):
try:
from PYME.misc import pyme_zeroconf
ns = pyme_zeroconf.getNS()
return ns.resolve(taskQueueName)
except:
return 'PYRONAME://' + taskQueueName
def _getTaskQueueURI(n_retries=2):
"""Discover the distributors using zeroconf and choose one"""
ns = pzc.getNS('_pyme-taskdist')
queueURLs = {}
def _search():
for name, info in ns.get_advertised_services():
if name.startswith('PYMEDistributor'):
queueURLs[name] = 'http://%s:%d' % (socket.inet_ntoa(info.address), info.port)
_search()
while not queueURLs and (n_retries > 0):
logging.info('could not find a distributor, waiting 5s and trying again')
time.sleep(5)
n_retries -= 1
_search()
try:
#try to grab the distributor on the local computer
return queueURLs[compName]
except KeyError:
#if there is no local distributor, choose one at random
logging.info('no local distributor, choosing one at random')
return random.choice(queueURLs.values())
def __init__(self, driftTracker):
threading.Thread.__init__(self)
import socket
ip_addr = socket.gethostbyname(socket.gethostname())
compName = GetComputerName()
Pyro.core.initServer()
pname = "%s.DriftTracker" % compName
try:
from PYME.misc import pyme_zeroconf
ns = pyme_zeroconf.getNS()
except:
ns = Pyro.naming.NameServerLocator().getNS()
if not compName in [n[0] for n in ns.list('')]:
ns.createGroup(compName)
#get rid of any previous instance
try:
ns.unregister(pname)
except Pyro.errors.NamingError:
pass
self.daemon = Pyro.core.Daemon(host=ip_addr)
self.daemon.useNameServer(ns)
self.driftCorr = piezoOffsetProxy(driftTracker)
#pname = "%s.Piezo" % compName
uri = self.daemon.connect(self.driftCorr, pname)
def __init__(self, *args, **kwds):
# begin wxGlade: MyFrame.__init__
kwds["style"] = wx.DEFAULT_FRAME_STYLE
wx.Frame.__init__(self, *args, **kwds)
self.panel_1 = wx.Panel(self, -1)
self.sizer_4_staticbox = wx.StaticBox(self.panel_1, -1, "Queues")
self.sizer_5_staticbox = wx.StaticBox(self.panel_1, -1, "Workers")
#self.sizer_3_staticbox = wx.StaticBox(self.panel_1, -1, "General")
#self.label_1 = wx.StaticText(self.panel_1, -1, "label_1")
self.gQueues = wx.grid.Grid(self.panel_1, -1, size=(400, 150))
self.gWorkers = wx.grid.Grid(self.panel_1, -1, size=(400, 150))
self.__set_properties()
self.__do_layout()
# end wxGlade
try:
from PYME.misc import pyme_zeroconf
ns = pyme_zeroconf.getNS()
time.sleep(15)
print(ns.get_advertised_services())
URI = ns.resolve(taskQueueName)
except:
print('Could not resolve using zeroconf, trying pyro-ns')
URI = 'PYRONAME://' + taskQueueName
self.tq = Pyro.core.getProxyForURI(URI)
#self.tq = Pyro.core.getProxyForURI("PYRONAME://" + taskQueueName)
self.workerProc = {}
self.tLast = 0
self.gQueues.SetRowLabelSize(10)
self.gWorkers.SetRowLabelSize(0)
self.onTimer()
self.timer = wx.Timer(self, -1)
self.Bind(wx.EVT_TIMER, self.onTimer)
self.timer.Start(10000)
logmenu = wx.Menu()
menuerrLog = logmenu.Append(wx.ID_ANY, "Worker-0 Error Logs",
"Log info")
menuoutLog = logmenu.Append(wx.ID_ANY, "Worker-0 Output", "Log info")
menuServErrLog = logmenu.Append(wx.ID_ANY, "Sever Error Logs",
"Log info")
menuServOutLog = logmenu.Append(wx.ID_ANY, "Server Output", "Log info")
menuWorkerDialog = logmenu.Append(
wx.ID_ANY, "Load worker output using file dialog", "Log info")
# Creating the menubar.
menuBar = wx.MenuBar()
menuBar.Append(logmenu, "&Logs") # Adding the "logmenu" to the MenuBar
self.SetMenuBar(menuBar) # Adding the MenuBar to the Frame content.
self.Bind(wx.EVT_MENU, self.OnErrScrolledDialog, menuerrLog)
self.Bind(wx.EVT_MENU, self.OnOutScrolledDialog, menuoutLog)
self.Bind(wx.EVT_MENU, self.OnServerErrScrolledDialog, menuServErrLog)
self.Bind(wx.EVT_MENU, self.OnServerOutScrolledDialog, menuServOutLog)
self.Bind(wx.EVT_MENU, self.OnWorkerFileDialog, menuWorkerDialog)
def getClient(compName=GetComputerName()):
try:
from PYME.misc import pyme_zeroconf
ns = pyme_zeroconf.getNS()
URI = ns.resolve('%s.DriftTracker' % compName)
except:
URI = 'PYRONAME://%s.DriftTracker' % compName
return Pyro.core.getProxyForURI(URI)
def get_ns():
global _ns
with _ns_lock:
if _ns is None:
#stagger query times
time.sleep(3 * np.random.rand())
_ns = pzc.getNS('_pyme-http')
#wait for replies
time.sleep(5)
return _ns
def getDistributorInfo():
ns = pyme_zeroconf.getNS('_pyme-taskdist')
queueURLs = {}
for name, info in ns.get_advertised_services():
if name.startswith('PYMEDistributor') or name.startswith(
'PYMERuleServer'):
queueURLs[name] = 'http://%s:%d/' % (socket.inet_ntoa(
info.address), info.port)
return queueURLs
def getClient(compName = GetComputerName()):
#try:
from PYME.misc import pyme_zeroconf
ns = pyme_zeroconf.getNS()
time.sleep(3)
#print ns.list()
URI = ns.resolve('%s.Piezo' % compName)
#except:
# URI ='PYRONAME://%s.Piezo'%compName
#print URI
return Pyro.core.getProxyForURI(URI)
def checkServer(timeOut=TIMEOUTDEFAULT, showServices=False):
if servicesPresent(timeOut=timeOut, showServices=showServices):
logging.info('zeroconf services detected')
else:
logging.error('no zeroconf services detected - this should not happen')
ns = pzc.getNS()
adserv = get_advertised_services(ns)
if len(adserv) > 0:
logging.info(repr(adserv))
else:
logging.error(
'no advertised pyro services - apparently there is no server running on this network'
)
def analysisTasks(request):
try:
from PYME.misc import pyme_zeroconf
ns = pyme_zeroconf.getNS()
except:
ns = Pyro.naming.NameServerLocator().getNS()
serverNames = [n.split('.')[1] for n in ns.list('TaskQueues')]
servers = [Server(n, ns) for n in serverNames]
return render_to_response('samples/analysisTasks.html', {
'servers': servers,
})
def _loadQueue(self, filename):
"""Load data from a remote PYME.ParallelTasks.HDFTaskQueue queue using
Pyro.
Parameters:
-----------
filename : string
the name of the queue
"""
import Pyro.core
from PYME.IO.DataSources import TQDataSource
from PYME.misc.computerName import GetComputerName
compName = GetComputerName()
if self.queueURI is None:
#do a lookup
taskQueueName = 'TaskQueues.%s' % compName
try:
from PYME.misc import pyme_zeroconf
ns = pyme_zeroconf.getNS()
URI = ns.resolve(taskQueueName)
except:
URI = 'PYRONAME://' + taskQueueName
self.tq = Pyro.core.getProxyForURI(URI)
else:
self.tq = Pyro.core.getProxyForURI(self.queueURI)
self.seriesName = filename[len('QUEUE://'):]
self.dataSource = TQDataSource.DataSource(self.seriesName, self.tq)
self.data = self.dataSource #this will get replaced with a wrapped version
self.mdh = MetaDataHandler.QueueMDHandler(self.tq, self.seriesName)
MetaData.fillInBlanks(self.mdh, self.dataSource)
#self.timer.WantNotification.append(self.dsRefresh)
self.events = self.dataSource.getEvents()
self.mode = 'LM'
def getNodeInfo():
ns = pyme_zeroconf.getNS('_pyme-taskdist')
queueURLs = {}
for name, info in ns.get_advertised_services():
if name.startswith('PYMENodeServer'):
try:
queueURLs[name] = 'http://%s:%d/' % (socket.inet_ntoa(
info.address), info.port)
except TypeError:
if info.port is None:
logger.debug('Service info from %s has no port info' %
name)
else:
logger.debug('ValueError: %s %s, %s' %
(name, repr(info), info.port))
return queueURLs
def main():
global LOG_STREAMS
confFile = os.path.join(conf.user_config_dir, 'distributor.yaml')
with open(confFile) as f:
config = yaml.load(f)
serverAddr, serverPort = config['distributor']['http_endpoint'].split(':')
externalAddr = socket.gethostbyname(socket.gethostname())
#set up logging
data_root = conf.get('dataserver-root')
if data_root:
distr_log_dir = '%s/LOGS' % data_root
dist_log_err_file = os.path.join(distr_log_dir, 'distributor.log')
if os.path.exists(dist_log_err_file):
os.remove(dist_log_err_file)
dist_err_handler = logging.handlers.RotatingFileHandler(filename=dist_log_err_file, mode='w', maxBytes=1e6, backupCount=1)
#dist_err_handler.setFormatter(logging.Formatter('%(message)s'))
distLogErr = logging.getLogger('distributor')
distLogErr.setLevel(logging.DEBUG)
distLogErr.addHandler(dist_err_handler)
proc = ruleserver.ServerThread(serverPort, profile=False)
proc.start()
#proc = subprocess.Popen('python -m PYME.ParallelTasks.distributor 1234', shell=True)
ns = pyme_zeroconf.getNS('_pyme-taskdist')
ns.register_service('PYMERuleServer: ' + GetComputerName(), externalAddr, int(serverPort))
try:
while proc.is_alive():
time.sleep(1)
finally:
logger.debug('trying to shut down server')
proc.shutdown()
ns.unregister('PYMERuleServer: ' + GetComputerName())
def main():
op = ArgumentParser(
description=
"PYME node server for task distribution. This should run on every node of the cluster"
)
#NOTE - currently squatting on port 15347 for testing - TODO can we use an ephemeral port?
op.add_argument(
'-p',
'--port',
dest='port',
default=conf.get('nodeserver-port', 15347),
type=int,
help=
"port number to serve on (default: 15347, see also 'nodeserver-port' config entry)"
)
op.add_argument('-a',
'--advertisements',
dest='advertisements',
choices=['zeroconf', 'local'],
default='zeroconf',
help='Optionally restrict advertisements to local machine')
args = op.parse_args()
serverPort = args.port
externalAddr = socket.gethostbyname(socket.gethostname())
if args.advertisements == 'zeroconf':
ns = pyme_zeroconf.getNS('_pyme-taskdist')
else:
#assume local
ns = sqlite_ns.getNS('_pyme-taskdist')
externalAddr = '127.0.0.1' #bind to localhost
#TODO - move this into the nodeserver proper so that the ruleserver doesn't need to be up before we start
print(distribution.getDistributorInfo(ns).values())
distributors = [
u.lstrip('http://').rstrip('/')
for u in distribution.getDistributorInfo(ns).values()
]
#set up nodeserver logging
cluster_root = conf.get('dataserver-root')
if cluster_root:
nodeserver_log_dir = os.path.join(cluster_root, 'LOGS',
GetComputerName())
#remove old log files
try:
os.remove(os.path.join(nodeserver_log_dir, 'nodeserver.log'))
except OSError: # if we cant clear out old log files, we might not have a log directory set up
try:
if not os.path.exists(os.path.join(nodeserver_log_dir)):
os.makedirs(
os.path.join(nodeserver_log_dir)
) # NB - this will create all intermediate directories as well
except: # throw error because the RotatingFileHandler will fail to initialize
raise IOError('Unable to initialize log files at %s' %
nodeserver_log_dir)
try:
shutil.rmtree(os.path.join(nodeserver_log_dir, 'taskWorkerHTTP'))
except:
pass
nodeserver_log_handler = logging.handlers.RotatingFileHandler(
os.path.join(nodeserver_log_dir, 'nodeserver.log'),
'w',
maxBytes=1e6,
backupCount=0)
nodeserverLog = logging.getLogger('nodeserver')
nodeserverLog.setLevel(logging.DEBUG)
nodeserver_log_handler.setLevel(logging.DEBUG)
nodeserver_log_handler.setFormatter(formatter)
nodeserverLog.addHandler(nodeserver_log_handler)
nodeserverLog.addHandler(stream_handler)
#nodeserverLog.propagate=False
else:
nodeserver_log_dir = os.path.join(os.curdir, 'LOGS', GetComputerName())
proc = rulenodeserver.ServerThread(distributors[0],
serverPort,
externalAddr=externalAddr,
profile=False)
proc.start()
# TODO - do we need this advertisement
#get the actual adress (port) we bound to
time.sleep(0.5)
sa = proc.nodeserver.socket.getsockname()
serverPort = int(sa[1])
ns.register_service('PYMENodeServer: ' + GetComputerName(), externalAddr,
serverPort)
time.sleep(2)
logger.debug('Launching worker processors')
numWorkers = conf.get('nodeserver-num_workers', cpu_count())
workerProcs = [
subprocess.Popen('python -m PYME.cluster.taskWorkerHTTP -s %d' %
serverPort,
shell=True,
stdin=subprocess.PIPE) for i in range(numWorkers - 1)
]
#last worker has profiling enabled
profiledir = os.path.join(nodeserver_log_dir, 'mProf')
workerProcs.append(
subprocess.Popen(
'python -m PYME.cluster.taskWorkerHTTP -s % d -p --profile-dir=%s'
% (serverPort, profiledir),
shell=True,
stdin=subprocess.PIPE))
try:
while proc.is_alive():
time.sleep(1)
finally:
logger.info('Shutting down workers')
try:
ns.unregister('PYMENodeServer: ' + GetComputerName())
except:
pass
for p in workerProcs:
#ask the workers to quit (nicely)
try:
p.send_signal(1)
except:
pass
time.sleep(2)
for p in workerProcs:
#now kill them off
try:
p.kill()
except:
pass
logger.info('Shutting down nodeserver')
proc.shutdown()
proc.join()
logger.info('Workers and nodeserver are shut down')
sys.exit()
def main():
global LOG_STREAMS
op = ArgumentParser(description="PYME rule server for task distribution. This should run once per cluster.")
#NOTE - currently squatting on port 15346 for testing - TODO can we use an ephemeral port
op.add_argument('-p', '--port', dest='port', default=config.get('ruleserver-port', 15346), type=int,
help="port number to serve on (default: 15346, see also 'ruleserver-port' config entry)")
op.add_argument('-a','--advertisements', dest='advertisements', choices=['zeroconf', 'local'], default='zeroconf',
help='Optionally restrict advertisements to local machine')
args = op.parse_args()
serverPort = args.port
if args.advertisements == 'local':
#bind on localhost
bind_addr = '127.0.0.1'
else:
bind_addr = '' #bind all interfaces
#set up logging
data_root = config.get('dataserver-root')
if data_root:
distr_log_dir = '%s/LOGS' % data_root
try: # make sure the directory exists
os.makedirs(distr_log_dir) # exist_ok flag not present on py2
except OSError as e:
import errno
if e.errno != errno.EEXIST:
raise e
dist_log_err_file = os.path.join(distr_log_dir, 'distributor.log')
if os.path.exists(dist_log_err_file):
os.remove(dist_log_err_file)
dist_err_handler = logging.handlers.RotatingFileHandler(filename=dist_log_err_file, mode='w', maxBytes=1e6, backupCount=1)
#dist_err_handler.setFormatter(logging.Formatter('%(message)s'))
distLogErr = logging.getLogger('distributor')
distLogErr.setLevel(logging.DEBUG)
distLogErr.addHandler(dist_err_handler)
proc = ruleserver.ServerThread(serverPort, bind_addr=bind_addr, profile=False)
proc.start()
#proc = subprocess.Popen('python -m PYME.ParallelTasks.distributor 1234', shell=True)
if args.advertisements == 'zeroconf':
ns = pyme_zeroconf.getNS('_pyme-taskdist')
else:
#assume 'local'
ns = sqlite_ns.getNS('_pyme-taskdist')
time.sleep(0.5)
#get the actual adress (port) we bound to
sa = proc.distributor.socket.getsockname()
service_name = get_service_name('PYMERuleServer')
ns.register_service(service_name, proc.externalAddr, int(sa[1]))
try:
while proc.is_alive():
time.sleep(1)
finally:
logger.debug('trying to shut down server')
proc.shutdown()
ns.unregister(service_name)
import Pyro.core
import PYME.misc.pyme_zeroconf as pzc
import time
import numpy as np
import threading
import os
from PYME.misc.computerName import GetComputerName
compName = GetComputerName()
procName = compName + ' - PID:%d' % os.getpid()
ns = pzc.getNS('_pyme-pyro')
TARGETCOPIES = 2
def duplicateFiles(targetNCopies=TARGETCOPIES, serverfilter=clusterIO.local_serverfilter):
"""walks the cluster and mirrors any files with a count less than the target
for high duplication targets (e.g. NCopies = 3), each pass will only
increment the copy number by 1
"""
ncopied = 0
#print targetNCopies, serverfilter
#
##################
import tables
from PYME.IO import MetaDataHandler
import os
import time
import PYME.Acquire.Spooler as sp
#from PYME.Acquire import protocol as p
from PYME.IO.FileUtils import fileID
try:
from PYME.misc import pyme_zeroconf
ns = pyme_zeroconf.getNS()
except ImportError:
ns = None
#rom PYME.Acquire import eventLog
class SpoolEvent(tables.IsDescription):
EventName = tables.StringCol(32)
Time = tables.Time64Col()
EventDescr = tables.StringCol(256)
class EventLogger:
def __init__(self, spool, tq, queueName):
self.spooler = spool
def main():
global LOG_STREAMS
cluster_root = conf.get('dataserver-root', conf.user_config_dir)
confFile = os.path.join(conf.user_config_dir, 'nodeserver.yaml')
with open(confFile) as f:
config = yaml.load(f)
serverAddr, serverPort = config['nodeserver']['http_endpoint'].split(':')
externalAddr = socket.gethostbyname(socket.gethostname())
ns = pyme_zeroconf.getNS('_pyme-taskdist')
#
# #find distributor(s)
# distributors = []
# for name, info in ns.advertised_services.items():
# if name.startswith('PYMEDistributor'):
# distributors.append('%s:%d' % (socket.inet_ntoa(info.address), info.port))
distributors = [
u.lstrip('http://').rstrip('/')
for u in distribution.getDistributorInfo().values()
]
#modify the configuration to reflect the discovered distributor(s)
config['nodeserver']['distributors'] = distributors
#write a new config file for the nodeserver
with tempfile.NamedTemporaryFile(suffix='.yaml',
delete=False) as temp_conf_file:
temp_conf_file_name = temp_conf_file.name
temp_conf_file.write(yaml.dump(config))
logging.debug('Config file: ' + temp_conf_file_name)
#set up nodeserver logging
nodeserver_log_dir = os.path.join(cluster_root, 'LOGS', GetComputerName())
#remove old log files
try:
os.remove(os.path.join(nodeserver_log_dir, 'nodeserver.log'))
except OSError: # if we cant clear out old log files, we might not have a log directory set up
try:
os.makedirs(
os.path.join(nodeserver_log_dir)
) # NB - this will create all intermediate directories as well
except: # throw error because the RotatingFileHandler will fail to initialize
raise IOError('Unable to initialize log files at %s' %
nodeserver_log_dir)
pass
try:
shutil.rmtree(os.path.join(nodeserver_log_dir, 'taskWorkerHTTP'))
except:
pass
#nodeserverLog = open(os.path.join(nodeserver_log_dir, 'nodeserver.log'), 'w')
nodeserver_log_handler = logging.handlers.RotatingFileHandler(
os.path.join(nodeserver_log_dir, 'nodeserver.log'),
'w',
maxBytes=1e6,
backupCount=0)
nodeserver_log_handler.setFormatter(logging.Formatter('%(message)s'))
nodeserverLog = logging.getLogger('nodeserver')
nodeserverLog.addHandler(nodeserver_log_handler)
nodeserverLog.setLevel(logging.DEBUG)
nodeserverLog.propagate = False
if not (len(sys.argv) == 2 and sys.argv[1] == '-n'):
proc = subprocess.Popen('nodeserver -c %s' % temp_conf_file_name,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
proc = subprocess.Popen('python -m PYME.cluster.nodeserver %s %s' %
(distributors[0], serverPort),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
t_log_stderr = threading.Thread(target=log_stream,
args=(proc.stderr, nodeserverLog))
t_log_stderr.setDaemon(False)
t_log_stderr.start()
t_log_stdout = threading.Thread(target=log_stream,
args=(proc.stdout, nodeserverLog))
t_log_stdout.setDaemon(False)
t_log_stdout.start()
ns.register_service('PYMENodeServer: ' + GetComputerName(), externalAddr,
int(serverPort))
time.sleep(2)
logging.debug('Launching worker processors')
numWorkers = config.get('numWorkers', cpu_count())
workerProcs = [
subprocess.Popen('python -m PYME.cluster.taskWorkerHTTP',
shell=True,
stdin=subprocess.PIPE) for i in range(numWorkers - 1)
]
#last worker has profiling enabled
profiledir = os.path.join(nodeserver_log_dir, 'mProf')
workerProcs.append(
subprocess.Popen('python -m PYME.cluster.taskWorkerHTTP -p %s' %
profiledir,
shell=True,
stdin=subprocess.PIPE))
try:
while not proc.poll():
time.sleep(1)
#try to keep log size under control by doing crude rotation
#if nodeserverLog.tell() > 1e6:
# nodeserverLog.seek(0)
except KeyboardInterrupt:
pass
finally:
LOG_STREAMS = False
logging.info('Shutting down workers')
try:
ns.unregister('PYMENodeServer: ' + GetComputerName())
except:
pass
os.unlink(temp_conf_file_name)
for p in workerProcs:
#ask the workers to quit (nicely)
try:
p.send_signal(1)
except:
pass
time.sleep(2)
for p in workerProcs:
#now kill them off
try:
p.kill()
except:
pass
logging.info('Shutting down nodeserver')
try:
proc.kill()
except:
pass
logging.info('Workers and nodeserver are shut down')
sys.exit()
def main():
print('Starting PYME taskServer ...')
import socket
from PYME import config
ip_addr = socket.gethostbyname(socket.gethostname())
profile = False
if len(sys.argv) > 1 and sys.argv[1] == '-p':
print('profiling')
profile = True
from PYME.util.mProfile import mProfile
mProfile.profileOn(
['taskServerZC.py', 'HDFTaskQueue.py', 'TaskQueue.py'])
if len(sys.argv) > 1 and sys.argv[1] == '-fp':
print('profiling')
#profile = True
from PYME.util.fProfile import fProfile
tp = fProfile.thread_profiler()
tp.profileOn('.*taskServerZC.*|.*TaskQueue.*|.*h5rFile.*',
'taskServer_prof.txt')
Pyro.config.PYRO_MOBILE_CODE = 0
Pyro.core.initServer()
#ns=Pyro.naming.NameServerLocator().getNS()
ns = pzc.getNS()
daemon = Pyro.core.Daemon(host=ip_addr)
daemon.useNameServer(ns)
#check to see if we've got the TaskQueues group
#if not 'TaskQueues' in [n[0] for n in ns.list('')]:
# ns.createGroup('TaskQueues')
#get rid of any previous queue
#try:
# ns.unregister(taskQueueName)
#except Pyro.errors.NamingError:
# pass
tq = TaskQueueSet(process_queues_in_order=config.get(
'TaskServer.process_queues_in_order', True))
uri = daemon.connect(tq, taskQueueName)
logging.debug('taskserver URI -> %s, %s' % (uri, type(uri)))
tw = TaskWatcher(tq)
tw.start()
try:
daemon.requestLoop(tq.isAlive)
except (KeyboardInterrupt, SystemExit):
logging.debug(
'Got a keyboard interrupt, attempting to shut down cleanly')
#raise
finally:
daemon.shutdown(True)
tw.alive = False
#ns.unregister(taskQueueName)
logging.info('Task server is shut down')
if profile:
mProfile.report()
def main():
global LOG_STREAMS
confFile = os.path.join(conf.user_config_dir, 'distributor.yaml')
with open(confFile) as f:
config = yaml.load(f)
serverAddr, serverPort = config['distributor']['http_endpoint'].split(':')
externalAddr = socket.gethostbyname(socket.gethostname())
#set up logging
#logfile_error = None
#logfile_debug = None
data_root = conf.get('dataserver-root')
if data_root:
#logfile_error = open('%s/LOGS/distributor_error.log' % data_root, 'w')
#logfile_debug = open('%s/LOGS/distributor_debug.log' % data_root, 'w')
distr_log_dir = '%s/LOGS' % data_root
dist_log_err_file = os.path.join(distr_log_dir,
'distributor_error.log')
if os.path.exists(dist_log_err_file):
os.remove(dist_log_err_file)
dist_err_handler = logging.handlers.RotatingFileHandler(
dist_log_err_file, 'w', maxBytes=1e6, backupCount=1)
dist_err_handler.setFormatter(logging.Formatter('%(message)s'))
distLogErr = logging.getLogger('dist_err')
distLogErr.addHandler(dist_err_handler)
distLogErr.setLevel(logging.DEBUG)
distLogErr.propagate = False
dist_log_dbg_file = os.path.join(distr_log_dir,
'distributor_debug.log')
if os.path.exists(dist_log_dbg_file):
os.remove(dist_log_dbg_file)
dist_dbg_handler = logging.handlers.RotatingFileHandler(
dist_log_dbg_file, 'w', maxBytes=1e6, backupCount=1)
dist_dbg_handler.setFormatter(logging.Formatter('%(message)s'))
distLogDbg = logging.getLogger('dist_debug')
distLogDbg.addHandler(dist_dbg_handler)
distLogDbg.setLevel(logging.DEBUG)
distLogDbg.propagate = False
if not (len(sys.argv) == 2 and sys.argv[1] == '-n'):
proc = subprocess.Popen('distributor -c %s' % confFile,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
proc = subprocess.Popen('python -m PYME.cluster.distributor 1234',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
t_log_stderr = threading.Thread(target=log_stream,
args=(proc.stderr, distLogErr))
t_log_stderr.setDaemon(False)
t_log_stderr.start()
t_log_stdout = threading.Thread(target=log_stream,
args=(proc.stdout, distLogDbg))
t_log_stdout.setDaemon(False)
t_log_stdout.start()
else:
if not (len(sys.argv) == 2 and sys.argv[1] == '-n'):
proc = subprocess.Popen('distributor -c %s' % confFile, shell=True)
else:
proc = subprocess.Popen('python -m PYME.cluster.distributor 1234',
shell=True)
ns = pyme_zeroconf.getNS('_pyme-taskdist')
service_name = get_service_name('PYMEDistributor')
ns.register_service(service_name, externalAddr, int(serverPort))
try:
while not proc.poll():
time.sleep(1)
# if logfile_error:
# #do crude log rotation
# if logfile_error.tell() > 1e6:
# logfile_error.seek(0)
#
# if logfile_debug.tell() > 1e6:
# logfile_debug.seek(0)
finally:
ns.unregister(service_name)
#try and shut down the distributor cleanly
proc.send_signal(1)
time.sleep(2)
proc.kill()
LOG_STREAMS = False
def main(protocol="HTTP/1.0"):
global GPU_STATS
"""Test the HTTP request handler class.
This runs an HTTP server on port 8000 (or the first command line
argument).
"""
from optparse import OptionParser
op = OptionParser(usage='usage: %s [options]' % sys.argv[0])
op.add_option(
'-p',
'--port',
dest='port',
default=config.get('dataserver-port', 8080),
help=
"port number to serve on (default: 8080, see also 'dataserver-port' config entry)"
)
op.add_option('-t',
'--test',
dest='test',
help="Set up for bandwidth test (don't save files)",
action="store_true",
default=False)
op.add_option('-v',
'--protocol',
dest='protocol',
help="HTTP protocol version",
default="1.1")
op.add_option('-l',
'--log-requests',
dest='log_requests',
help="Display http request info",
default=False,
action="store_true")
default_root = config.get('dataserver-root', os.curdir)
op.add_option(
'-r',
'--root',
dest='root',
help=
"Root directory of virtual filesystem (default %s, see also 'dataserver-root' config entry)"
% dataserver_root,
default=default_root)
op.add_option('-k',
'--profile',
dest='profile',
help="Enable profiling",
default=False,
action="store_true")
default_server_filter = config.get('dataserver-filter', compName)
op.add_option(
'-f',
'--server-filter',
dest='server_filter',
help='Add a serverfilter for distinguishing between different clusters',
default=default_server_filter)
op.add_option(
'--timeout-test',
dest='timeout_test',
help=
'deliberately make requests timeout for testing error handling in calling modules',
default=0)
options, args = op.parse_args()
if options.profile:
from PYME.util import mProfile
mProfile.profileOn(['HTTPDataServer.py', 'clusterListing.py'])
profileOutDir = options.root + '/LOGS/%s/mProf' % compName
logger.info(
'========================================\nPYMEDataServer, running on python %s\n'
% sys.version)
#change to the dataserver root if given'
logger.info('Serving from directory: %s' % options.root)
os.chdir(options.root)
server_address = ('', int(options.port))
PYMEHTTPRequestHandler.protocol_version = 'HTTP/%s' % options.protocol
PYMEHTTPRequestHandler.bandwidthTesting = options.test
PYMEHTTPRequestHandler.timeoutTesting = options.timeout_test
PYMEHTTPRequestHandler.logrequests = options.log_requests
httpd = ThreadedHTTPServer(server_address, PYMEHTTPRequestHandler)
#httpd = http.server.HTTPServer(server_address, PYMEHTTPRequestHandler)
httpd.daemon_threads = True
sa = httpd.socket.getsockname()
try:
ip_addr = socket.gethostbyname(socket.gethostname())
except:
ip_addr = socket.gethostbyname(socket.gethostname() + '.local')
ns = pzc.getNS('_pyme-http')
ns.register_service(
'PYMEDataServer [%s]: ' % options.server_filter + procName, ip_addr,
sa[1])
status['IPAddress'] = ip_addr
status['BindAddress'] = server_address
status['Port'] = sa[1]
status['Protocol'] = options.protocol
status['TestMode'] = options.test
status['ComputerName'] = GetComputerName()
if GPU_STATS:
try:
pynvml.nvmlInit()
except:
GPU_STATS = False
sp = statusPoller()
sp.start()
logger.info("Serving HTTP on %s port %d ..." % (ip_addr, sa[1]))
try:
httpd.serve_forever()
finally:
logger.info('Shutting down ...')
httpd.shutdown()
httpd.server_close()
if options.profile:
mProfile.report(display=False, profiledir=profileOutDir)
sp.stop()
if GPU_STATS:
pynvml.nvmlShutdown()
sys.exit()
def main(protocol="HTTP/1.0"):
global GPU_STATS
"""Test the HTTP request handler class.
This runs an HTTP server on port 8000 (or the first command line
argument).
"""
from optparse import OptionParser
op = OptionParser(usage='usage: %s [options]' % sys.argv[0])
#NOTE - currently squatting on port 15348 for testing - TODO can we use an ephemeral port?
op.add_option(
'-p',
'--port',
dest='port',
default=config.get('dataserver-port', 15348),
help=
"port number to serve on (default: 15348, see also 'dataserver-port' config entry)"
)
op.add_option('-t',
'--test',
dest='test',
help="Set up for bandwidth test (don't save files)",
action="store_true",
default=False)
op.add_option('-v',
'--protocol',
dest='protocol',
help="HTTP protocol version",
default="1.1")
op.add_option('-l',
'--log-requests',
dest='log_requests',
help="Display http request info",
default=False,
action="store_true")
default_root = config.get('dataserver-root', os.curdir)
op.add_option(
'-r',
'--root',
dest='root',
help=
"Root directory of virtual filesystem (default %s, see also 'dataserver-root' config entry)"
% dataserver_root,
default=default_root)
op.add_option('-k',
'--profile',
dest='profile',
help="Enable profiling",
default=False,
action="store_true")
op.add_option('--thread-profile',
dest='thread_profile',
help="Enable thread profiling",
default=False,
action="store_true")
default_server_filter = config.get('dataserver-filter', compName)
op.add_option(
'-f',
'--server-filter',
dest='server_filter',
help='Add a serverfilter for distinguishing between different clusters',
default=default_server_filter)
op.add_option(
'--timeout-test',
dest='timeout_test',
help=
'deliberately make requests timeout for testing error handling in calling modules',
default=0)
op.add_option('-a',
'--advertisements',
dest='advertisements',
choices=['zeroconf', 'local'],
default='zeroconf',
help='Optionally restrict advertisements to local machine')
options, args = op.parse_args()
if options.profile:
from PYME.util import mProfile
mProfile.profileOn(['HTTPDataServer.py', 'clusterListing.py'])
profileOutDir = options.root + '/LOGS/%s/mProf' % compName
if options.thread_profile:
from PYME.util import fProfile
tp = fProfile.ThreadProfiler()
#tp.profile_on(subs=['PYME/', 'http/server', 'socketserver'],outfile=options.root + '/LOGS/%s/tProf/dataserver.txt' % compName)
tp.profile_on(subs=[
'PYME/',
],
outfile=options.root +
'/LOGS/%s/tProf/dataserver.txt' % compName)
# setup logging to file
log_dir = '%s/LOGS/%s' % (options.root, compName)
makedirs_safe(log_dir)
log_file = '%s/LOGS/%s/PYMEDataServer.log' % (options.root, compName)
fh = logging.handlers.RotatingFileHandler(filename=log_file,
mode='w',
maxBytes=1e6,
backupCount=1)
logger.addHandler(fh)
logger.info(
'========================================\nPYMEDataServer, running on python %s\n'
% sys.version)
#change to the dataserver root if given'
logger.info('Serving from directory: %s' % options.root)
os.chdir(options.root)
if options.advertisements == 'local':
# preference is to avoid zeroconf on clusterofone due to poor
# performance on crowded networks
if config.get('clusterIO-hybridns', True):
ns = sqlite_ns.getNS('_pyme-http')
else:
# if we aren't using the hybridns, we are using zeroconf in clusterIO
# TODO - warn that we might run into performance issues???
ns = pzc.getNS('_pyme-http')
server_address = ('127.0.0.1', int(options.port))
ip_addr = '127.0.0.1'
else:
#default
ns = pzc.getNS('_pyme-http')
server_address = ('', int(options.port))
try:
ip_addr = socket.gethostbyname(socket.gethostname())
except:
ip_addr = socket.gethostbyname(socket.gethostname() + '.local')
PYMEHTTPRequestHandler.protocol_version = 'HTTP/%s' % options.protocol
PYMEHTTPRequestHandler.bandwidthTesting = options.test
PYMEHTTPRequestHandler.timeoutTesting = options.timeout_test
PYMEHTTPRequestHandler.logrequests = options.log_requests
httpd = ThreadedHTTPServer(server_address, PYMEHTTPRequestHandler)
#httpd = http.server.HTTPServer(server_address, PYMEHTTPRequestHandler)
httpd.daemon_threads = True
#get the actual adress (port) we bound to
sa = httpd.socket.getsockname()
service_name = get_service_name('PYMEDataServer [%s]' %
options.server_filter)
ns.register_service(service_name, ip_addr, sa[1])
status['IPAddress'] = ip_addr
status['BindAddress'] = server_address
status['Port'] = sa[1]
status['Protocol'] = options.protocol
status['TestMode'] = options.test
status['ComputerName'] = GetComputerName()
if GPU_STATS:
try:
pynvml.nvmlInit()
except:
GPU_STATS = False
sp = statusPoller()
sp.start()
logger.info("Serving HTTP on %s port %d ..." % (ip_addr, sa[1]))
try:
httpd.serve_forever()
finally:
logger.info('Shutting down ...')
httpd.shutdown()
httpd.server_close()
ns.unregister(service_name)
if options.profile:
mProfile.report(display=False, profiledir=profileOutDir)
if options.thread_profile:
tp.profile_off()
sp.stop()
if GPU_STATS:
pynvml.nvmlShutdown()
try:
from pytest_cov.embed import cleanup
cleanup()
except:
pass
sys.exit()
def main():
#prof = fProfile.thread_profiler()
#prof.profileOn('.*PYME.*|.*zeroconf.*', 'ruleserver_prof.txt')
#mProfile.profileOn(['rulenodeserver.py', 'zeroconf.py'])
confFile = os.path.join(conf.user_config_dir, 'nodeserver.yaml')
with open(confFile) as f:
config = yaml.load(f)
serverAddr, serverPort = config['nodeserver']['http_endpoint'].split(':')
externalAddr = socket.gethostbyname(socket.gethostname())
print(distribution.getDistributorInfo().values())
distributors = [
u.lstrip('http://').rstrip('/')
for u in distribution.getDistributorInfo().values()
]
#set up nodeserver logging
cluster_root = conf.get('dataserver-root')
if cluster_root:
nodeserver_log_dir = os.path.join(cluster_root, 'LOGS',
GetComputerName())
#remove old log files
try:
os.remove(os.path.join(nodeserver_log_dir, 'nodeserver.log'))
except OSError: # if we cant clear out old log files, we might not have a log directory set up
try:
if not os.path.exists(os.path.join(nodeserver_log_dir)):
os.makedirs(
os.path.join(nodeserver_log_dir)
) # NB - this will create all intermediate directories as well
except: # throw error because the RotatingFileHandler will fail to initialize
raise IOError('Unable to initialize log files at %s' %
nodeserver_log_dir)
try:
shutil.rmtree(os.path.join(nodeserver_log_dir, 'taskWorkerHTTP'))
except:
pass
nodeserver_log_handler = logging.handlers.RotatingFileHandler(
os.path.join(nodeserver_log_dir, 'nodeserver.log'),
'w',
maxBytes=1e6,
backupCount=0)
nodeserverLog = logging.getLogger('nodeserver')
nodeserverLog.setLevel(logging.DEBUG)
nodeserver_log_handler.setLevel(logging.DEBUG)
nodeserver_log_handler.setFormatter(formatter)
nodeserverLog.addHandler(nodeserver_log_handler)
nodeserverLog.addHandler(stream_handler)
#nodeserverLog.propagate=False
else:
nodeserver_log_dir = os.path.join(os.curdir, 'LOGS', GetComputerName())
#proc = subprocess.Popen('python -m PYME.cluster.rulenodeserver %s %s' % (distributors[0], serverPort), shell=True,
# stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc = rulenodeserver.ServerThread(distributors[0],
serverPort,
profile=False)
proc.start()
ns = pyme_zeroconf.getNS('_pyme-taskdist')
ns.register_service('PYMENodeServer: ' + GetComputerName(), externalAddr,
int(serverPort))
time.sleep(2)
logger.debug('Launching worker processors')
numWorkers = config.get('nodeserver-num_workers', cpu_count())
workerProcs = [
subprocess.Popen('python -m PYME.cluster.taskWorkerHTTP',
shell=True,
stdin=subprocess.PIPE) for i in range(numWorkers - 1)
]
#last worker has profiling enabled
profiledir = os.path.join(nodeserver_log_dir, 'mProf')
workerProcs.append(
subprocess.Popen('python -m PYME.cluster.taskWorkerHTTP -p %s' %
profiledir,
shell=True,
stdin=subprocess.PIPE))
try:
while proc.is_alive():
time.sleep(1)
finally:
logger.info('Shutting down workers')
try:
ns.unregister('PYMENodeServer: ' + GetComputerName())
except:
pass
for p in workerProcs:
#ask the workers to quit (nicely)
try:
p.send_signal(1)
except:
pass
time.sleep(2)
for p in workerProcs:
#now kill them off
try:
p.kill()
except:
pass
logger.info('Shutting down nodeserver')
#try:
proc.shutdown()
proc.join()
#except:
# pass
logger.info('Workers and nodeserver are shut down')
#mProfile.report()
#prof.profileOff()
sys.exit()
def main(protocol="HTTP/1.0"):
"""Test the HTTP request handler class.
This runs an HTTP server on port 8000 (or the first command line
argument).
"""
from optparse import OptionParser
op = OptionParser(usage='usage: %s [options] [filename]' % sys.argv[0])
op.add_option('-p',
'--port',
dest='port',
default=config.get('dataserver-port', 8080),
help="port number to serve on")
op.add_option('-t',
'--test',
dest='test',
help="Set up for bandwidth test (don't save files)",
action="store_true",
default=False)
op.add_option('-v',
'--protocol',
dest='protocol',
help="HTTP protocol version",
default="1.1")
op.add_option('-l',
'--log-requests',
dest='log_requests',
help="Display http request info",
default=False,
action="store_true")
op.add_option('-r',
'--root',
dest='root',
help="Root directory of virtual filesystem",
default=config.get('dataserver-root', os.curdir))
options, args = op.parse_args()
#change to the dataserver root if given
logger.info('Serving from directory: %s' % options.root)
os.chdir(options.root)
#PYMEHTTPRequestHandler.protocol_version = 'HTTP/%s' % options.protocol
server_config['bandwidthTesting'] = options.test
#PYMEHTTPRequestHandler.logrequests = options.log_requests
#httpd = ThreadedHTTPServer(server_address, PYMEHTTPRequestHandler)
ip_addr = socket.gethostbyname(socket.gethostname())
server_address = ('', int(options.port))
global_status['IPAddress'] = ip_addr
global_status['BindAddress'] = server_address
global_status['Port'] = int(options.port)
global_status['Protocol'] = options.protocol
global_status['TestMode'] = options.test
global_status['ComputerName'] = GetComputerName()
ns = pzc.getNS('_pyme-http')
ns.register_service('PYMEDataServer: ' + procName, ip_addr,
int(options.port))
print("Serving HTTP on %s port %d ..." % (ip_addr, options.port))
#wsgiref_server(options)
cherrypy_server(options)
