def run(self):
""""""
if self._profile:
from PYME.util import mProfile
mProfile.profileOn([
'ruleserver.py',
])
profileOutDir = config.get(
'dataserver-root',
os.curdir) + '/LOGS/%s/mProf' % computerName.GetComputerName()
if self.bind_addr == '':
import socket
self.externalAddr = socket.gethostbyname(socket.gethostname())
else:
self.externalAddr = self.bind_addr
self.distributor = WFRuleServer(self.port, bind_addr=self.bind_addr)
logger.info('Starting ruleserver on %s:%d' %
(self.externalAddr, self.port))
try:
self.distributor.serve_forever()
finally:
self.distributor._do_poll = False
#logger.info('Shutting down ...')
#self.distributor.shutdown()
logger.info('Closing server ...')
self.distributor.server_close()
if self._profile:
mProfile.report(False, profiledir=profileOutDir)
PYME.config.config['VisGUI-new_layers'] = args.new_layers
if wx.GetApp() is None: #check to see if there's already a wxApp instance (running from ipython -pylab or -wthread)
main_(args.file, use_shaders=args.use_shaders, args=args)
else:
#time.sleep(1)
visFr = VisGUIFrame(None, args.file, args.use_shaders)
visFr.Show()
visFr.RefreshView()
if __name__ == '__main__':
from PYME.util import mProfile
mProfile.profileOn(['pipeline.py', 'tabular.py'])
main()
mProfile.report()
def ipython_visgui(filename=None, **kwargs):
import PYME.config
if wx.GetApp() is None:
raise RuntimeError('No wx App instance found. Start one using the `\%gui wx` magic in ipython before running this command')
PYME.config.config['VisGUI-new_layers'] = True
visFr = VisGUIFrame(None, filename=filename, pipeline_vars = kwargs)
visFr.Show()
return visFr
def ipython_pymevisualize(filename=None, **kwargs):
return ipython_visgui(filename, **kwargs)
def on_SIGHUP(signum, frame):
""""""
from PYME.util import mProfile
mProfile.report(False, profiledir=profileOutDir)
raise RuntimeError('Recieved SIGHUP')
if __name__ == '__main__':
import signal
port = sys.argv[1]
if (len(sys.argv) == 3) and (sys.argv[2] == '-k'):
profile = True
from PYME.util import mProfile
mProfile.profileOn([
'ruleserver.py',
])
profileOutDir = config.get(
'dataserver-root',
os.curdir) + '/LOGS/%s/mProf' % computerName.GetComputerName()
else:
profile = False
profileOutDir = None
if not sys.platform == 'win32':
#windows doesn't support handling signals ... don't catch and hope for the best.
#Note: This will make it hard to cleanly shutdown the distributor on Windows, but should be OK for testing and
#development
signal.signal(signal.SIGHUP, on_SIGHUP)
try:
run(int(port))
finally:
if profile:
mProfile.report(False, profiledir=profileOutDir)
def main():
print('Starting PYME taskServer ...')
import socket
from PYME import config
ip_addr = socket.gethostbyname(socket.gethostname())
profile = False
if len(sys.argv) > 1 and sys.argv[1] == '-p':
print('profiling')
profile = True
from PYME.util.mProfile import mProfile
mProfile.profileOn(
['taskServerZC.py', 'HDFTaskQueue.py', 'TaskQueue.py'])
if len(sys.argv) > 1 and sys.argv[1] == '-fp':
print('profiling')
#profile = True
from PYME.util.fProfile import fProfile
tp = fProfile.thread_profiler()
tp.profileOn('.*taskServerZC.*|.*TaskQueue.*|.*h5rFile.*',
'taskServer_prof.txt')
Pyro.config.PYRO_MOBILE_CODE = 0
Pyro.core.initServer()
#ns=Pyro.naming.NameServerLocator().getNS()
ns = pzc.getNS()
daemon = Pyro.core.Daemon(host=ip_addr)
daemon.useNameServer(ns)
#check to see if we've got the TaskQueues group
#if not 'TaskQueues' in [n[0] for n in ns.list('')]:
# ns.createGroup('TaskQueues')
#get rid of any previous queue
#try:
# ns.unregister(taskQueueName)
#except Pyro.errors.NamingError:
# pass
tq = TaskQueueSet(process_queues_in_order=config.get(
'TaskServer.process_queues_in_order', True))
uri = daemon.connect(tq, taskQueueName)
logging.debug('taskserver URI -> %s, %s' % (uri, type(uri)))
tw = TaskWatcher(tq)
tw.start()
try:
daemon.requestLoop(tq.isAlive)
except (KeyboardInterrupt, SystemExit):
logging.debug(
'Got a keyboard interrupt, attempting to shut down cleanly')
#raise
finally:
daemon.shutdown(True)
tw.alive = False
#ns.unregister(taskQueueName)
logging.info('Task server is shut down')
if profile:
mProfile.report()
def main(protocol="HTTP/1.0"):
global GPU_STATS
"""Test the HTTP request handler class.
This runs an HTTP server on port 8000 (or the first command line
argument).
"""
from optparse import OptionParser
op = OptionParser(usage='usage: %s [options]' % sys.argv[0])
op.add_option(
'-p',
'--port',
dest='port',
default=config.get('dataserver-port', 8080),
help=
"port number to serve on (default: 8080, see also 'dataserver-port' config entry)"
)
op.add_option('-t',
'--test',
dest='test',
help="Set up for bandwidth test (don't save files)",
action="store_true",
default=False)
op.add_option('-v',
'--protocol',
dest='protocol',
help="HTTP protocol version",
default="1.1")
op.add_option('-l',
'--log-requests',
dest='log_requests',
help="Display http request info",
default=False,
action="store_true")
default_root = config.get('dataserver-root', os.curdir)
op.add_option(
'-r',
'--root',
dest='root',
help=
"Root directory of virtual filesystem (default %s, see also 'dataserver-root' config entry)"
% dataserver_root,
default=default_root)
op.add_option('-k',
'--profile',
dest='profile',
help="Enable profiling",
default=False,
action="store_true")
default_server_filter = config.get('dataserver-filter', compName)
op.add_option(
'-f',
'--server-filter',
dest='server_filter',
help='Add a serverfilter for distinguishing between different clusters',
default=default_server_filter)
op.add_option(
'--timeout-test',
dest='timeout_test',
help=
'deliberately make requests timeout for testing error handling in calling modules',
default=0)
options, args = op.parse_args()
if options.profile:
from PYME.util import mProfile
mProfile.profileOn(['HTTPDataServer.py', 'clusterListing.py'])
profileOutDir = options.root + '/LOGS/%s/mProf' % compName
logger.info(
'========================================\nPYMEDataServer, running on python %s\n'
% sys.version)
#change to the dataserver root if given'
logger.info('Serving from directory: %s' % options.root)
os.chdir(options.root)
server_address = ('', int(options.port))
PYMEHTTPRequestHandler.protocol_version = 'HTTP/%s' % options.protocol
PYMEHTTPRequestHandler.bandwidthTesting = options.test
PYMEHTTPRequestHandler.timeoutTesting = options.timeout_test
PYMEHTTPRequestHandler.logrequests = options.log_requests
httpd = ThreadedHTTPServer(server_address, PYMEHTTPRequestHandler)
#httpd = http.server.HTTPServer(server_address, PYMEHTTPRequestHandler)
httpd.daemon_threads = True
sa = httpd.socket.getsockname()
try:
ip_addr = socket.gethostbyname(socket.gethostname())
except:
ip_addr = socket.gethostbyname(socket.gethostname() + '.local')
ns = pzc.getNS('_pyme-http')
ns.register_service(
'PYMEDataServer [%s]: ' % options.server_filter + procName, ip_addr,
sa[1])
status['IPAddress'] = ip_addr
status['BindAddress'] = server_address
status['Port'] = sa[1]
status['Protocol'] = options.protocol
status['TestMode'] = options.test
status['ComputerName'] = GetComputerName()
if GPU_STATS:
try:
pynvml.nvmlInit()
except:
GPU_STATS = False
sp = statusPoller()
sp.start()
logger.info("Serving HTTP on %s port %d ..." % (ip_addr, sa[1]))
try:
httpd.serve_forever()
finally:
logger.info('Shutting down ...')
httpd.shutdown()
httpd.server_close()
if options.profile:
mProfile.report(display=False, profiledir=profileOutDir)
sp.stop()
if GPU_STATS:
pynvml.nvmlShutdown()
sys.exit()
def main(protocol="HTTP/1.0"):
global GPU_STATS
"""Test the HTTP request handler class.
This runs an HTTP server on port 8000 (or the first command line
argument).
"""
from optparse import OptionParser
op = OptionParser(usage='usage: %s [options]' % sys.argv[0])
#NOTE - currently squatting on port 15348 for testing - TODO can we use an ephemeral port?
op.add_option(
'-p',
'--port',
dest='port',
default=config.get('dataserver-port', 15348),
help=
"port number to serve on (default: 15348, see also 'dataserver-port' config entry)"
)
op.add_option('-t',
'--test',
dest='test',
help="Set up for bandwidth test (don't save files)",
action="store_true",
default=False)
op.add_option('-v',
'--protocol',
dest='protocol',
help="HTTP protocol version",
default="1.1")
op.add_option('-l',
'--log-requests',
dest='log_requests',
help="Display http request info",
default=False,
action="store_true")
default_root = config.get('dataserver-root', os.curdir)
op.add_option(
'-r',
'--root',
dest='root',
help=
"Root directory of virtual filesystem (default %s, see also 'dataserver-root' config entry)"
% dataserver_root,
default=default_root)
op.add_option('-k',
'--profile',
dest='profile',
help="Enable profiling",
default=False,
action="store_true")
op.add_option('--thread-profile',
dest='thread_profile',
help="Enable thread profiling",
default=False,
action="store_true")
default_server_filter = config.get('dataserver-filter', compName)
op.add_option(
'-f',
'--server-filter',
dest='server_filter',
help='Add a serverfilter for distinguishing between different clusters',
default=default_server_filter)
op.add_option(
'--timeout-test',
dest='timeout_test',
help=
'deliberately make requests timeout for testing error handling in calling modules',
default=0)
op.add_option('-a',
'--advertisements',
dest='advertisements',
choices=['zeroconf', 'local'],
default='zeroconf',
help='Optionally restrict advertisements to local machine')
options, args = op.parse_args()
if options.profile:
from PYME.util import mProfile
mProfile.profileOn(['HTTPDataServer.py', 'clusterListing.py'])
profileOutDir = options.root + '/LOGS/%s/mProf' % compName
if options.thread_profile:
from PYME.util import fProfile
tp = fProfile.ThreadProfiler()
#tp.profile_on(subs=['PYME/', 'http/server', 'socketserver'],outfile=options.root + '/LOGS/%s/tProf/dataserver.txt' % compName)
tp.profile_on(subs=[
'PYME/',
],
outfile=options.root +
'/LOGS/%s/tProf/dataserver.txt' % compName)
# setup logging to file
log_dir = '%s/LOGS/%s' % (options.root, compName)
makedirs_safe(log_dir)
log_file = '%s/LOGS/%s/PYMEDataServer.log' % (options.root, compName)
fh = logging.handlers.RotatingFileHandler(filename=log_file,
mode='w',
maxBytes=1e6,
backupCount=1)
logger.addHandler(fh)
logger.info(
'========================================\nPYMEDataServer, running on python %s\n'
% sys.version)
#change to the dataserver root if given'
logger.info('Serving from directory: %s' % options.root)
os.chdir(options.root)
if options.advertisements == 'local':
# preference is to avoid zeroconf on clusterofone due to poor
# performance on crowded networks
if config.get('clusterIO-hybridns', True):
ns = sqlite_ns.getNS('_pyme-http')
else:
# if we aren't using the hybridns, we are using zeroconf in clusterIO
# TODO - warn that we might run into performance issues???
ns = pzc.getNS('_pyme-http')
server_address = ('127.0.0.1', int(options.port))
ip_addr = '127.0.0.1'
else:
#default
ns = pzc.getNS('_pyme-http')
server_address = ('', int(options.port))
try:
ip_addr = socket.gethostbyname(socket.gethostname())
except:
ip_addr = socket.gethostbyname(socket.gethostname() + '.local')
PYMEHTTPRequestHandler.protocol_version = 'HTTP/%s' % options.protocol
PYMEHTTPRequestHandler.bandwidthTesting = options.test
PYMEHTTPRequestHandler.timeoutTesting = options.timeout_test
PYMEHTTPRequestHandler.logrequests = options.log_requests
httpd = ThreadedHTTPServer(server_address, PYMEHTTPRequestHandler)
#httpd = http.server.HTTPServer(server_address, PYMEHTTPRequestHandler)
httpd.daemon_threads = True
#get the actual adress (port) we bound to
sa = httpd.socket.getsockname()
service_name = get_service_name('PYMEDataServer [%s]' %
options.server_filter)
ns.register_service(service_name, ip_addr, sa[1])
status['IPAddress'] = ip_addr
status['BindAddress'] = server_address
status['Port'] = sa[1]
status['Protocol'] = options.protocol
status['TestMode'] = options.test
status['ComputerName'] = GetComputerName()
if GPU_STATS:
try:
pynvml.nvmlInit()
except:
GPU_STATS = False
sp = statusPoller()
sp.start()
logger.info("Serving HTTP on %s port %d ..." % (ip_addr, sa[1]))
try:
httpd.serve_forever()
finally:
logger.info('Shutting down ...')
httpd.shutdown()
httpd.server_close()
ns.unregister(service_name)
if options.profile:
mProfile.report(display=False, profiledir=profileOutDir)
if options.thread_profile:
tp.profile_off()
sp.stop()
if GPU_STATS:
pynvml.nvmlShutdown()
try:
from pytest_cov.embed import cleanup
cleanup()
except:
pass
sys.exit()
default=False,
action='store_true',
help="enable profiling")
op.add_argument('--profile-dir', dest='profile_dir')
op.add_argument('-s',
'--server-port',
dest='server_port',
type=int,
default=config.get('nodeserver-port', 15347),
help='Optionally restrict advertisements to local machine')
args = op.parse_args()
if args.profile:
from PYME.util import mProfile
mProfile.profileOn(['taskWorkerHTTP.py', 'remFitBuf.py'])
if not platform.platform().startswith('Windows'):
signal.signal(signal.SIGHUP, on_SIGHUP)
try:
#main()
tW = taskWorker(nodeserver_port=args.server_port)
tW.loop_forever()
except KeyboardInterrupt:
#supress error message here - we only want to know if something bad happened
pass
finally:
if args.profile:
mProfile.report(display=False, profiledir=args.profile_dir)
raise RuntimeError('Recieved SIGHUP')
if __name__ == '__main__':
if len(sys.argv) > 1:
if sys.argv[1] == '-p':
profile = True
from PYME.util import mProfile
mProfile.profileOn(['taskWorkerHTTP.py', 'remFitBuf.py'])
if len(sys.argv) == 3:
profileOutDir = sys.argv[2]
else:
profileOutDir = None
else:
profile = False
if not platform.platform().startswith('Windows'):
signal.signal(signal.SIGHUP, on_SIGHUP)
try:
#main()
tW = taskWorker()
tW.loop_forever()
except KeyboardInterrupt:
#supress error message here - we only want to know if something bad happened
pass
finally:
if profile:
mProfile.report(display=False, profiledir=profileOutDir)
