def run(self): """""" if self._profile: from PYME.util import mProfile mProfile.profileOn([ 'ruleserver.py', ]) profileOutDir = config.get( 'dataserver-root', os.curdir) + '/LOGS/%s/mProf' % computerName.GetComputerName() if self.bind_addr == '': import socket self.externalAddr = socket.gethostbyname(socket.gethostname()) else: self.externalAddr = self.bind_addr self.distributor = WFRuleServer(self.port, bind_addr=self.bind_addr) logger.info('Starting ruleserver on %s:%d' % (self.externalAddr, self.port)) try: self.distributor.serve_forever() finally: self.distributor._do_poll = False #logger.info('Shutting down ...') #self.distributor.shutdown() logger.info('Closing server ...') self.distributor.server_close() if self._profile: mProfile.report(False, profiledir=profileOutDir)
PYME.config.config['VisGUI-new_layers'] = args.new_layers if wx.GetApp() is None: #check to see if there's already a wxApp instance (running from ipython -pylab or -wthread) main_(args.file, use_shaders=args.use_shaders, args=args) else: #time.sleep(1) visFr = VisGUIFrame(None, args.file, args.use_shaders) visFr.Show() visFr.RefreshView() if __name__ == '__main__': from PYME.util import mProfile mProfile.profileOn(['pipeline.py', 'tabular.py']) main() mProfile.report() def ipython_visgui(filename=None, **kwargs): import PYME.config if wx.GetApp() is None: raise RuntimeError('No wx App instance found. Start one using the `\%gui wx` magic in ipython before running this command') PYME.config.config['VisGUI-new_layers'] = True visFr = VisGUIFrame(None, filename=filename, pipeline_vars = kwargs) visFr.Show() return visFr def ipython_pymevisualize(filename=None, **kwargs): return ipython_visgui(filename, **kwargs)
def on_SIGHUP(signum, frame): """""" from PYME.util import mProfile mProfile.report(False, profiledir=profileOutDir) raise RuntimeError('Recieved SIGHUP')
if __name__ == '__main__': import signal port = sys.argv[1] if (len(sys.argv) == 3) and (sys.argv[2] == '-k'): profile = True from PYME.util import mProfile mProfile.profileOn([ 'ruleserver.py', ]) profileOutDir = config.get( 'dataserver-root', os.curdir) + '/LOGS/%s/mProf' % computerName.GetComputerName() else: profile = False profileOutDir = None if not sys.platform == 'win32': #windows doesn't support handling signals ... don't catch and hope for the best. #Note: This will make it hard to cleanly shutdown the distributor on Windows, but should be OK for testing and #development signal.signal(signal.SIGHUP, on_SIGHUP) try: run(int(port)) finally: if profile: mProfile.report(False, profiledir=profileOutDir)
def main(): print('Starting PYME taskServer ...') import socket from PYME import config ip_addr = socket.gethostbyname(socket.gethostname()) profile = False if len(sys.argv) > 1 and sys.argv[1] == '-p': print('profiling') profile = True from PYME.util.mProfile import mProfile mProfile.profileOn( ['taskServerZC.py', 'HDFTaskQueue.py', 'TaskQueue.py']) if len(sys.argv) > 1 and sys.argv[1] == '-fp': print('profiling') #profile = True from PYME.util.fProfile import fProfile tp = fProfile.thread_profiler() tp.profileOn('.*taskServerZC.*|.*TaskQueue.*|.*h5rFile.*', 'taskServer_prof.txt') Pyro.config.PYRO_MOBILE_CODE = 0 Pyro.core.initServer() #ns=Pyro.naming.NameServerLocator().getNS() ns = pzc.getNS() daemon = Pyro.core.Daemon(host=ip_addr) daemon.useNameServer(ns) #check to see if we've got the TaskQueues group #if not 'TaskQueues' in [n[0] for n in ns.list('')]: # ns.createGroup('TaskQueues') #get rid of any previous queue #try: # ns.unregister(taskQueueName) #except Pyro.errors.NamingError: # pass tq = TaskQueueSet(process_queues_in_order=config.get( 'TaskServer.process_queues_in_order', True)) uri = daemon.connect(tq, taskQueueName) logging.debug('taskserver URI -> %s, %s' % (uri, type(uri))) tw = TaskWatcher(tq) tw.start() try: daemon.requestLoop(tq.isAlive) except (KeyboardInterrupt, SystemExit): logging.debug( 'Got a keyboard interrupt, attempting to shut down cleanly') #raise finally: daemon.shutdown(True) tw.alive = False #ns.unregister(taskQueueName) logging.info('Task server is shut down') if profile: mProfile.report()
def main(protocol="HTTP/1.0"): global GPU_STATS """Test the HTTP request handler class. This runs an HTTP server on port 8000 (or the first command line argument). """ from optparse import OptionParser op = OptionParser(usage='usage: %s [options]' % sys.argv[0]) op.add_option( '-p', '--port', dest='port', default=config.get('dataserver-port', 8080), help= "port number to serve on (default: 8080, see also 'dataserver-port' config entry)" ) op.add_option('-t', '--test', dest='test', help="Set up for bandwidth test (don't save files)", action="store_true", default=False) op.add_option('-v', '--protocol', dest='protocol', help="HTTP protocol version", default="1.1") op.add_option('-l', '--log-requests', dest='log_requests', help="Display http request info", default=False, action="store_true") default_root = config.get('dataserver-root', os.curdir) op.add_option( '-r', '--root', dest='root', help= "Root directory of virtual filesystem (default %s, see also 'dataserver-root' config entry)" % dataserver_root, default=default_root) op.add_option('-k', '--profile', dest='profile', help="Enable profiling", default=False, action="store_true") default_server_filter = config.get('dataserver-filter', compName) op.add_option( '-f', '--server-filter', dest='server_filter', help='Add a serverfilter for distinguishing between different clusters', default=default_server_filter) op.add_option( '--timeout-test', dest='timeout_test', help= 'deliberately make requests timeout for testing error handling in calling modules', default=0) options, args = op.parse_args() if options.profile: from PYME.util import mProfile mProfile.profileOn(['HTTPDataServer.py', 'clusterListing.py']) profileOutDir = options.root + '/LOGS/%s/mProf' % compName logger.info( '========================================\nPYMEDataServer, running on python %s\n' % sys.version) #change to the dataserver root if given' logger.info('Serving from directory: %s' % options.root) os.chdir(options.root) server_address = ('', int(options.port)) PYMEHTTPRequestHandler.protocol_version = 'HTTP/%s' % options.protocol PYMEHTTPRequestHandler.bandwidthTesting = options.test PYMEHTTPRequestHandler.timeoutTesting = options.timeout_test PYMEHTTPRequestHandler.logrequests = options.log_requests httpd = ThreadedHTTPServer(server_address, PYMEHTTPRequestHandler) #httpd = http.server.HTTPServer(server_address, PYMEHTTPRequestHandler) httpd.daemon_threads = True sa = httpd.socket.getsockname() try: ip_addr = socket.gethostbyname(socket.gethostname()) except: ip_addr = socket.gethostbyname(socket.gethostname() + '.local') ns = pzc.getNS('_pyme-http') ns.register_service( 'PYMEDataServer [%s]: ' % options.server_filter + procName, ip_addr, sa[1]) status['IPAddress'] = ip_addr status['BindAddress'] = server_address status['Port'] = sa[1] status['Protocol'] = options.protocol status['TestMode'] = options.test status['ComputerName'] = GetComputerName() if GPU_STATS: try: pynvml.nvmlInit() except: GPU_STATS = False sp = statusPoller() sp.start() logger.info("Serving HTTP on %s port %d ..." % (ip_addr, sa[1])) try: httpd.serve_forever() finally: logger.info('Shutting down ...') httpd.shutdown() httpd.server_close() if options.profile: mProfile.report(display=False, profiledir=profileOutDir) sp.stop() if GPU_STATS: pynvml.nvmlShutdown() sys.exit()
def main(protocol="HTTP/1.0"): global GPU_STATS """Test the HTTP request handler class. This runs an HTTP server on port 8000 (or the first command line argument). """ from optparse import OptionParser op = OptionParser(usage='usage: %s [options]' % sys.argv[0]) #NOTE - currently squatting on port 15348 for testing - TODO can we use an ephemeral port? op.add_option( '-p', '--port', dest='port', default=config.get('dataserver-port', 15348), help= "port number to serve on (default: 15348, see also 'dataserver-port' config entry)" ) op.add_option('-t', '--test', dest='test', help="Set up for bandwidth test (don't save files)", action="store_true", default=False) op.add_option('-v', '--protocol', dest='protocol', help="HTTP protocol version", default="1.1") op.add_option('-l', '--log-requests', dest='log_requests', help="Display http request info", default=False, action="store_true") default_root = config.get('dataserver-root', os.curdir) op.add_option( '-r', '--root', dest='root', help= "Root directory of virtual filesystem (default %s, see also 'dataserver-root' config entry)" % dataserver_root, default=default_root) op.add_option('-k', '--profile', dest='profile', help="Enable profiling", default=False, action="store_true") op.add_option('--thread-profile', dest='thread_profile', help="Enable thread profiling", default=False, action="store_true") default_server_filter = config.get('dataserver-filter', compName) op.add_option( '-f', '--server-filter', dest='server_filter', help='Add a serverfilter for distinguishing between different clusters', default=default_server_filter) op.add_option( '--timeout-test', dest='timeout_test', help= 'deliberately make requests timeout for testing error handling in calling modules', default=0) op.add_option('-a', '--advertisements', dest='advertisements', choices=['zeroconf', 'local'], default='zeroconf', help='Optionally restrict advertisements to local machine') options, args = op.parse_args() if options.profile: from PYME.util import mProfile mProfile.profileOn(['HTTPDataServer.py', 'clusterListing.py']) profileOutDir = options.root + '/LOGS/%s/mProf' % compName if options.thread_profile: from PYME.util import fProfile tp = fProfile.ThreadProfiler() #tp.profile_on(subs=['PYME/', 'http/server', 'socketserver'],outfile=options.root + '/LOGS/%s/tProf/dataserver.txt' % compName) tp.profile_on(subs=[ 'PYME/', ], outfile=options.root + '/LOGS/%s/tProf/dataserver.txt' % compName) # setup logging to file log_dir = '%s/LOGS/%s' % (options.root, compName) makedirs_safe(log_dir) log_file = '%s/LOGS/%s/PYMEDataServer.log' % (options.root, compName) fh = logging.handlers.RotatingFileHandler(filename=log_file, mode='w', maxBytes=1e6, backupCount=1) logger.addHandler(fh) logger.info( '========================================\nPYMEDataServer, running on python %s\n' % sys.version) #change to the dataserver root if given' logger.info('Serving from directory: %s' % options.root) os.chdir(options.root) if options.advertisements == 'local': # preference is to avoid zeroconf on clusterofone due to poor # performance on crowded networks if config.get('clusterIO-hybridns', True): ns = sqlite_ns.getNS('_pyme-http') else: # if we aren't using the hybridns, we are using zeroconf in clusterIO # TODO - warn that we might run into performance issues??? ns = pzc.getNS('_pyme-http') server_address = ('127.0.0.1', int(options.port)) ip_addr = '127.0.0.1' else: #default ns = pzc.getNS('_pyme-http') server_address = ('', int(options.port)) try: ip_addr = socket.gethostbyname(socket.gethostname()) except: ip_addr = socket.gethostbyname(socket.gethostname() + '.local') PYMEHTTPRequestHandler.protocol_version = 'HTTP/%s' % options.protocol PYMEHTTPRequestHandler.bandwidthTesting = options.test PYMEHTTPRequestHandler.timeoutTesting = options.timeout_test PYMEHTTPRequestHandler.logrequests = options.log_requests httpd = ThreadedHTTPServer(server_address, PYMEHTTPRequestHandler) #httpd = http.server.HTTPServer(server_address, PYMEHTTPRequestHandler) httpd.daemon_threads = True #get the actual adress (port) we bound to sa = httpd.socket.getsockname() service_name = get_service_name('PYMEDataServer [%s]' % options.server_filter) ns.register_service(service_name, ip_addr, sa[1]) status['IPAddress'] = ip_addr status['BindAddress'] = server_address status['Port'] = sa[1] status['Protocol'] = options.protocol status['TestMode'] = options.test status['ComputerName'] = GetComputerName() if GPU_STATS: try: pynvml.nvmlInit() except: GPU_STATS = False sp = statusPoller() sp.start() logger.info("Serving HTTP on %s port %d ..." % (ip_addr, sa[1])) try: httpd.serve_forever() finally: logger.info('Shutting down ...') httpd.shutdown() httpd.server_close() ns.unregister(service_name) if options.profile: mProfile.report(display=False, profiledir=profileOutDir) if options.thread_profile: tp.profile_off() sp.stop() if GPU_STATS: pynvml.nvmlShutdown() try: from pytest_cov.embed import cleanup cleanup() except: pass sys.exit()
default=False, action='store_true', help="enable profiling") op.add_argument('--profile-dir', dest='profile_dir') op.add_argument('-s', '--server-port', dest='server_port', type=int, default=config.get('nodeserver-port', 15347), help='Optionally restrict advertisements to local machine') args = op.parse_args() if args.profile: from PYME.util import mProfile mProfile.profileOn(['taskWorkerHTTP.py', 'remFitBuf.py']) if not platform.platform().startswith('Windows'): signal.signal(signal.SIGHUP, on_SIGHUP) try: #main() tW = taskWorker(nodeserver_port=args.server_port) tW.loop_forever() except KeyboardInterrupt: #supress error message here - we only want to know if something bad happened pass finally: if args.profile: mProfile.report(display=False, profiledir=args.profile_dir)
raise RuntimeError('Recieved SIGHUP') if __name__ == '__main__': if len(sys.argv) > 1: if sys.argv[1] == '-p': profile = True from PYME.util import mProfile mProfile.profileOn(['taskWorkerHTTP.py', 'remFitBuf.py']) if len(sys.argv) == 3: profileOutDir = sys.argv[2] else: profileOutDir = None else: profile = False if not platform.platform().startswith('Windows'): signal.signal(signal.SIGHUP, on_SIGHUP) try: #main() tW = taskWorker() tW.loop_forever() except KeyboardInterrupt: #supress error message here - we only want to know if something bad happened pass finally: if profile: mProfile.report(display=False, profiledir=profileOutDir)