Exemple #1
0
    def Receive(self, replytype, chain=None, **kw):
        """This method allows code to act in a synchronous manner, it waits to 
        return until the deferred fires but it doesn't prevent other queued 
        calls from being executed.  Send must be called first, which sets up 
        the chain/factory.  
        
        WARNING: If defer is set to True, must either call Receive
        immediately after Send (ie. no intervening Sends) or pass
        chain in as a paramter.
        
        Parameters:
            replytype -- TypeCode
        KeyWord Parameters:
            chain -- processing chain, optional
            
        """
        chain = chain or self.chain
        d = chain.flow.deferred
        if self.trace:

            def trace(soapdata):
                print("_" * 33,
                      time.ctime(time.time()),
                      "RESPONSE:",
                      file=self.trace)
                print(soapdata, file=self.trace)
                return soapdata

            d.addCallback(trace)

        chain.processResponse(d, replytype, **kw)
        if self.defer:
            return d

        failure = []
        append = failure.append

        def errback(result):
            """Used with Response method to suppress 'Unhandled error in 
            Deferred' messages by adding an errback.
            """
            append(result)
            return None

        d.addErrback(errback)

        # spin reactor
        while not d.called:
            reactor.runUntilCurrent()
            t2 = reactor.timeout()
            t = reactor.running and t2
            reactor.doIteration(t)

        pyobj = d.result
        if len(failure):
            failure[0].raiseException()

        return pyobj
Exemple #2
0
 def mainLoop(self, reactor):
     Hub.state = 2
     # Unlike reactor's mainLoop, this function does not catch exceptions.
     # Anything raised goes into the main greenlet (because it is always the
     # parent of this one)
     while reactor.running:
         # Advance simulation time in delayed event processors.
         reactor.runUntilCurrent()
         t2 = reactor.timeout()
         t = reactor.running and t2
         reactor.doIteration(t)
Exemple #3
0
 def mainLoop(self, reactor):
     Hub.state = 2
     # Unlike reactor's mainLoop, this function does not catch exceptions.
     # Anything raised goes into the main greenlet (because it is always the
     # parent of this one)
     while reactor.running:
         # Advance simulation time in delayed event processors.
         reactor.runUntilCurrent()
         t2 = reactor.timeout()
         t = reactor.running and t2
         reactor.doIteration(t)
Exemple #4
0
    def Receive(self, replytype, chain=None, **kw):
        """This method allows code to act in a synchronous manner, it waits to
        return until the deferred fires but it doesn't prevent other queued
        calls from being executed.  Send must be called first, which sets up
        the chain/factory.

        WARNING: If defer is set to True, must either call Receive
        immediately after Send (ie. no intervening Sends) or pass
        chain in as a paramter.

        Parameters:
            replytype -- TypeCode
        KeyWord Parameters:
            chain -- processing chain, optional

        """
        chain = chain or self.chain
        d = chain.flow.deferred
        if self.trace:
            def trace(soapdata):
                print >>self.trace, "_" * 33, time.ctime(time.time()), "RESPONSE:"
                print >>self.trace, soapdata
                return soapdata

            d.addCallback(trace)

        chain.processResponse(d, replytype, **kw)
        if self.defer:
            return d

        failure = []
        append = failure.append
        def errback(result):
            """Used with Response method to suppress 'Unhandled error in
            Deferred' messages by adding an errback.
            """
            append(result)
            return None

        d.addErrback(errback)

        # spin reactor
        while not d.called:
            reactor.runUntilCurrent()
            t2 = reactor.timeout()
            t = reactor.running and t2
            reactor.doIteration(t)

        pyobj = d.result
        if len(failure):
            failure[0].raiseException()

        return pyobj
Exemple #5
0
def step(sleep = True, external = True):
    if reactor.running:
        try:
            t = sleep and reactor.running and reactor.timeout()
            reactor.doIteration(t)
            reactor.runUntilCurrent()
        except:
            log.error("problem running reactor - exiting")
            raise SystemExit
        if external:
            dispatch.dispatcher_run()
    else:
        log.info("reactor stopped - exiting")
        raise SystemExit
Exemple #6
0
def step(sleep=True, external=True):
    if reactor.running:
        try:
            t = sleep and reactor.running and reactor.timeout()
            reactor.doIteration(t)
            reactor.runUntilCurrent()
        except:
            log.error("problem running reactor - exiting")
            raise SystemExit
        if external:
            dispatch.dispatcher_run()
    else:
        log.info("reactor stopped - exiting")
        raise SystemExit
Exemple #7
0
 def _iterateTestReactor(self, debug=False):
     """
     Iterate the reactor.
     """
     reactor.runUntilCurrent()
     if debug:
         # When debug is enabled with iterate using a small delay in steps,
         # to have a much better debug output.
         # Otherwise the debug messages will flood the output.
         print (
             u'delayed: %s\n'
             u'threads: %s\n'
             u'writers: %s\n'
             u'readers: %s\n'
             u'threadpool size: %s\n'
             u'threadpool threads: %s\n'
             u'threadpool working: %s\n'
             u'\n' % (
                 self._reactorQueueToString(),
                 reactor.threadCallQueue,
                 reactor.getWriters(),
                 reactor.getReaders(),
                 self._threadPoolQueueSize(),
                 self._threadPoolThreads(),
                 self._threadPoolWorking(),
                 )
             )
         t2 = reactor.timeout()
         # For testing we want to force to reactor to wake at an
         # interval of at most 1 second.
         if t2 is None or t2 > 1:
             t2 = 0.1
         t = reactor.running and t2
         reactor.doIteration(t)
     else:
         reactor.doIteration(False)
Exemple #8
0
    def run(self):
        """Override Process run method to provide a custom wrapper for the API.

		This provides a continuous loop for watching the service while keeping
		an ear open to the main process from openContentPlatform, listening for
		any interrupt requests.

		"""
        ## Setup requested log handler
        try:
            ## Twisted imports here to avoid issues with epoll on Linux
            from twisted.internet import reactor, ssl
            from twisted.python.filepath import FilePath
            from twisted.web.server import Site
            from twisted.web.wsgi import WSGIResource
            from twisted.python.threadpool import ThreadPool

            print('Starting {}'.format(self.serviceName))
            self.getLocalLogger()
            self.logger.info('Starting {}'.format(self.serviceName))
            self.logger.info('Setting up the API application...')

            ## Setup shared resources for our WSGIResource instances to use
            self.getSharedLogger()
            self.getSharedDbPool()
            ## Create a PID file for system administration purposes
            pidEntryService(self.serviceName, env, self.pid)
            ## Reference the magic WSGI throwable from our root module using Hug
            application = apiResourceRoot.__hug_wsgi__

            ## Setup the WSGI to be hosted through Twisted's web server
            wsgiThreadPool = ThreadPool()
            wsgiThreadPool.start()
            ## For some reason the system event wasn't working all the time,
            ## so I'm adding an explicit wsgiThreadPool.stop() below as well,
            ## which was needed before reactor.stop() would properly cleanup.
            reactor.addSystemEventTrigger('after', 'shutdown',
                                          wsgiThreadPool.stop)
            resource = WSGIResource(reactor, wsgiThreadPool, application)
            self.logger.info('calling listener on {}:{}.'.format(
                str(self.serviceEndpoint), self.listeningPort))
            if self.useCertificates:
                ## Use TLS to encrypt the communication
                certData = FilePath(
                    os.path.join(
                        env.privateInternalCertPath,
                        self.globalSettings.get(
                            'ocpCertificateCaFile'))).getContent()
                certificate = ssl.PrivateCertificate.loadPEM(certData)
                reactor.listenSSL(self.listeningPort, Site(resource),
                                  certificate.options())
            else:
                ## Plain text communication
                reactor.listenTCP(self.listeningPort,
                                  Site(resource),
                                  interface=self.serviceEndpoint)
            ## Normally we'd just call reactor.run() here and let twisted handle
            ## the wait loop while watching for signals. The problem is that we
            ## need openContentPlatform (parent process) to manage this process.
            ## So this is a bit hacky in that I'm using the reactor code, but I
            ## am manually calling what would be called if I just called run():
            reactor.startRunning()
            ## Start event wait loop
            while reactor._started and not self.shutdownEvent.is_set():
                try:
                    ## Four lines from twisted.internet.main.mainloop:
                    reactor.runUntilCurrent()
                    t2 = reactor.timeout()
                    t = reactor.running and t2
                    reactor.doIteration(t)

                except:
                    exception = traceback.format_exception(
                        sys.exc_info()[0],
                        sys.exc_info()[1],
                        sys.exc_info()[2])
                    self.logger.error('Exception in {}: {}'.format(
                        self.serviceName, str(exception)))
                    break
            if self.shutdownEvent.is_set():
                self.logger.info('Process received shutdownEvent')
            with suppress(Exception):
                wsgiThreadPool.stop()
            with suppress(Exception):
                reactor.stop()

        except:
            exception = traceback.format_exception(sys.exc_info()[0],
                                                   sys.exc_info()[1],
                                                   sys.exc_info()[2])
            self.logger.error('Exception in {}: {}'.format(
                self.serviceName, str(exception)))

        ## Cleanup
        pidRemoveService(self.serviceName, env, self.pid)
        self.logger.info('Stopped {}'.format(self.serviceName))
        print('Stopped {}'.format(self.serviceName))

        ## end run
        return
    def run(self):
        """Override Process run method to provide a custom wrapper for service.

		Shared by all networked services. This provides a continuous loop for
		watching the child process while keeping an ear open to the main process
		from openContentPlatform, listening for any interrupt requests.

		"""
        try:
            ## Twisted import here to avoid issues with epoll on Linux
            from twisted.internet import reactor, ssl

            ## There are two types of event handlers being used here:
            ##   self.shutdownEvent : main process tells this one to shutdown
            ##                        (e.g. on a Ctrl+C type event)
            ##   self.canceledEvent : this process needs to restart
            serviceEndpoint = self.globalSettings.get('serviceIpAddress')
            useCertificates = self.globalSettings.get('useCertificates', True)

            ## Create a PID file for system administration purposes
            utils.pidEntryService(self.serviceName, env, self.pid)

            ## Network job services use networkService, which is a shared lib
            ## directed by additional input parameters; set args accordingly:
            factoryArgs = None
            if (self.serviceName == 'ContentGatheringService'
                    or self.serviceName == 'UniversalJobService'):
                factoryArgs = (self.serviceName, self.globalSettings,
                               self.canceledEvent, self.shutdownEvent,
                               self.moduleType, self.clientEndpointTable,
                               self.clientResultsTable,
                               self.serviceResultsTable, self.serviceJobTable,
                               self.serviceHealthTable, self.pkgPath,
                               self.serviceSettings, self.serviceLogSetup)
            else:
                factoryArgs = (self.serviceName, self.globalSettings,
                               self.canceledEvent, self.shutdownEvent)

            if useCertificates:
                ## Use TLS to encrypt the communication
                certData = FilePath(
                    os.path.join(
                        env.privateInternalCertPath,
                        self.globalSettings.get(
                            'ocpCertificateCaFile'))).getContent()
                certificate = ssl.PrivateCertificate.loadPEM(certData)
                print('Starting encrypted service: {}'.format(
                    self.serviceName))
                reactor.listenSSL(self.listeningPort,
                                  self.serviceFactory(*factoryArgs),
                                  certificate.options())
            else:
                ## Plain text communication
                print('Starting plain text service: {}'.format(
                    self.serviceName))
                reactor.listenTCP(self.listeningPort,
                                  self.serviceFactory(*factoryArgs),
                                  interface=serviceEndpoint)

            ## The following loop may look hacky at first glance, but solves a
            ## challenge with a mix of Python multi-processing, multi-threading,
            ## then Twisted's reactor and threading.

            ## Python threads that are not daemon types, cannot be forced closed
            ## when the controlling thread closes. So it's important to pass
            ## events all the way through, and have looping code catch/cleanup.

            ## Whenever the main process is being shut down, it must stop all
            ## sub-processes along with their work streams, which includes any
            ## Twisted reactors. We do that by passing shutdownEvent into the
            ## sub-processes. And whenever a service (sub-process) needs to
            ## restart, it notifies the other direction so the main process can
            ## verify it stopped and restart it. We do that by a canceledEvent.

            ## Now to the point of this verbose comment, so the reason we cannot
            ## call reactor.run() here, and instead cut/paste the same code, was
            ## to enhance 'while reactor._started' to watch for our events.

            reactor.startRunning()
            ## Start event wait loop
            while reactor._started and not self.shutdownEvent.is_set(
            ) and not self.canceledEvent.is_set():
                try:
                    ## Four lines from twisted.internet.base.mainloop:
                    reactor.runUntilCurrent()
                    t2 = reactor.timeout()
                    t = reactor.running and t2
                    reactor.doIteration(t)
                except:
                    exception = traceback.format_exception(
                        sys.exc_info()[0],
                        sys.exc_info()[1],
                        sys.exc_info()[2])
                    print('Exception in {}: {}'.format(self.serviceName,
                                                       exception))
                    break
            if self.shutdownEvent.is_set():
                print('Shutdown event received for {}'.format(
                    self.serviceName))
                self.canceledEvent.set()
                with suppress(Exception):
                    time.sleep(2)
                    print('Calling reactor stop for {}'.format(
                        self.serviceName))
                    reactor.stop()
                    time.sleep(.5)
            elif self.canceledEvent.is_set():
                print('Canceled event received for {}'.format(
                    self.serviceName))
                with suppress(Exception):
                    time.sleep(2)
                    reactor.stop()
                    time.sleep(.5)

        except PermissionError:
            exceptionOnly = traceback.format_exception_only(
                sys.exc_info()[0],
                sys.exc_info()[1])
            print('  {}'.format(exceptionOnly))
            print('  Stopping {}'.format(self.serviceName))
        except:
            exception = traceback.format_exception(sys.exc_info()[0],
                                                   sys.exc_info()[1],
                                                   sys.exc_info()[2])
            print('Exception in {}: {}'.format(self.serviceName, exception))

        ## Cleanup
        utils.pidRemoveService(self.serviceName, env, self.pid)
        with suppress(Exception):
            reactor.stop()
        print('Stopped {}'.format(self.serviceName))

        ## end ServiceProcess
        return
Exemple #10
0
	def run(self):
		"""Override Process run method to provide a custom wrapper for service.

		Shared by all local services. This provides a continuous loop for
		watching the child process while keeping an ear open to the main process
		from openContentPlatform, listening for any interrupt requests.

		"""
		try:
			## Twisted import here to avoid issues with epoll on Linux
			from twisted.internet import reactor

			## There are two types of event handlers being used here:
			##   self.shutdownEvent : main process tells this one to shutdown
			##                        (e.g. on a Ctrl+C type event)
			##   self.canceledEvent : this process needs to restart

			## Create a PID file for system administration purposes
			utils.pidEntryService(self.serviceName, env, self.pid)

			factoryArgs = (self.serviceName, self.globalSettings, self.canceledEvent, self.shutdownEvent)
			print('Starting local service: {}'.format(self.serviceName))

			## The following loop may look hacky at first glance, but solves a
			## challenge with a mix of Python multi-processing, multi-threading,
			## then Twisted's reactor and threading.

			## Python threads that are not daemon types, cannot be forced closed
			## when the controlling thread closes. So it's important to pass
			## events all the way through, and have looping code catch/cleanup.

			## Whenever the main process is being shut down, it must stop all
			## sub-processes along with their work streams, which includes any
			## Twisted reactors. We do that by passing shutdownEvent into the
			## sub-processes. And whenever a service (sub-process) needs to
			## restart, it notifies the other direction so the main process can
			## verify it stopped and restart it. We do that by a canceledEvent.

			## Now to the point of this verbose comment, so the reason we cannot
			## call reactor.run() here, and instead cut/paste the same code, was
			## to enhance 'while reactor._started' to watch for our events.

			reactor.callFromThread(self.serviceClass, *factoryArgs)
			#reactor.callWhenRunning(self.serviceClass *factoryArgs)

			reactor.startRunning()
			## Start event wait loop
			while reactor._started and not self.shutdownEvent.is_set() and not self.canceledEvent.is_set():
				try:
					## Four lines from twisted.internet.base.mainloop:
					reactor.runUntilCurrent()
					t2 = reactor.timeout()
					t = reactor.running and t2
					reactor.doIteration(t)
				except:
					exception = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
					print('Exception in {}: {}'.format(self.serviceName, exception))
					break
			if self.shutdownEvent.is_set():
				print('Shutdown event received for {}'.format(self.serviceName))
				self.canceledEvent.set()
				with suppress(Exception):
					time.sleep(2)
					print('Calling reactor stop for {}'.format(self.serviceName))
					reactor.stop()
					time.sleep(.5)
			elif self.canceledEvent.is_set():
				print('Canceled event received for {}'.format(self.serviceName))
				with suppress(Exception):
					time.sleep(2)
					reactor.stop()
					time.sleep(.5)

		except PermissionError:
			exceptionOnly = traceback.format_exception_only(sys.exc_info()[0], sys.exc_info()[1])
			print('  {}'.format(exceptionOnly))
			print('  Stopping {}'.format(self.serviceName))
		except:
			exception = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
			print('Exception in {}: {}'.format(self.serviceName, exception))

		## Cleanup
		utils.pidRemoveService(self.serviceName, env, self.pid)
		## Remove the handler to avoid duplicate lines the next time it runs
		with suppress(Exception):
			reactor.stop()
		print('Stopped {}'.format(self.serviceName))

		## end ServiceProcess
		return