def _start_logging(self): """ Configures and starts the logging of this operation """ # Remove the default handler if len(logging.getLogger('').handlers) > 0: logging.getLogger('').removeHandler( logging.getLogger('').handlers[0]) # define a handler which writes WARNING messages # or higher to the sys.stderr console = logging.StreamHandler() # Determining level of the logger based on pySPACE configuration file try: log_level = eval(pySPACE.configuration.console_log_level) \ if hasattr(pySPACE.configuration, "console_log_level") \ else logging.WARNING if not isinstance(log_level, int): raise NameError() except (AttributeError, NameError): import warnings warnings.warn("%s is not a valid log level! Falling back to " \ "logging.WARNING." % pySPACE.configuration.log_level) log_level = logging.WARNING console.setLevel(log_level) # set a format which is simpler for console use formatter = logging.Formatter( '%(asctime)s %(name)-40s %(levelname)-8s %(message)s') # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger logging.getLogger('').addHandler(console) logging.getLogger('').setLevel(logging.DEBUG) self._log("Logging started") # Starting the socket server from pySPACE.tools.socket_logger import LogRecordSocketReceiver self._log("Starting the TCP logging server...") # Determining host ip host, aliaslist, lan_ip = socket.gethostbyname_ex(socket.gethostname()) self.host = lan_ip[0] # Search for an available port self.port = logging.handlers.DEFAULT_TCP_LOGGING_PORT while True: try: self.tcpserver = LogRecordSocketReceiver(host=self.host, port=self.port) break except socket.error: self.port += 1 self.tcpserver.start() self._log("Started the TCP logging server on port %s!" % self.port)
def start_logging_server(): """ Start the socket server """ from pySPACE.tools.socket_logger import LogRecordSocketReceiver # Determining host ip host, aliaslist, lan_ip = socket.gethostbyname_ex(socket.gethostname()) host = lan_ip[0] # Search for an available port port = logging.handlers.DEFAULT_TCP_LOGGING_PORT while True: try: tcpserver = LogRecordSocketReceiver(host=host, port=port) break except socket.error: port += 1 tcpserver.start() online_logger.log(logging.info, "Started the TCP logging server on port %s!" % port)
def _start_logging(self): """ Configures and starts the logging of this operation """ # Remove the default handler if len(logging.getLogger("").handlers) > 0: logging.getLogger("").removeHandler(logging.getLogger("").handlers[0]) # define a handler which writes WARNING messages # or higher to the sys.stderr console = logging.StreamHandler() # Determining level of the logger based on pySPACE configuration file try: log_level = ( eval(pySPACE.configuration.console_log_level) if hasattr(pySPACE.configuration, "console_log_level") else logging.WARNING ) if not isinstance(log_level, int): raise NameError() except (AttributeError, NameError): import warnings warnings.warn( "%s is not a valid log level! Falling back to " "logging.WARNING." % pySPACE.configuration.log_level ) log_level = logging.WARNING console.setLevel(log_level) # set a format which is simpler for console use formatter = logging.Formatter("%(asctime)s %(name)-40s %(levelname)-8s %(message)s") # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger logging.getLogger("").addHandler(console) logging.getLogger("").setLevel(logging.DEBUG) self._log("Logging started") # Starting the socket server from pySPACE.tools.socket_logger import LogRecordSocketReceiver self._log("Starting the TCP logging server...") # Determining host ip host, aliaslist, lan_ip = socket.gethostbyname_ex(socket.gethostname()) self.host = lan_ip[0] # Search for an available port self.port = logging.handlers.DEFAULT_TCP_LOGGING_PORT while True: try: self.tcpserver = LogRecordSocketReceiver(host=self.host, port=self.port) break except socket.error: self.port += 1 self.tcpserver.start() self._log("Started the TCP logging server on port %s!" % self.port)
class Backend(object): """ Interface for backends All other backends must implement several methods of this interface in order to execute an operation on a specific modality. """ STATES = set(["idling", "staged", "executing", "retrieved", "consolidated"]) def __init__(self): # Start logging self._start_logging() # Install signal handlers #signal.signal(signal.SIGINT, self._grace) #signal.signal(signal.SIGTERM, self._grace) #signal.signal(signal.SIGQUIT, self._grace) #signal.signal(signal.SIGPIPE, self._grace) self.file_handler = None self.current_operation = None self.SERVER_IP = socket.gethostbyname(socket.gethostname()) self.SERVER_PORT = None self.listener = None def __del__(self): # Stop logging self._stop_logging() def __str__(self): return str(type(self).__name__) def stage_in(self, operation): """ Stage the current operation """ self.current_operation = operation # Add a handler that logs the operations output to a file log_path = self.current_operation.result_directory self.file_handler = logging.FileHandler(os.path.join(log_path, "operation.log")) # Determining level of the logger based on pySPACE configuration file try: log_level = eval(pySPACE.configuration.file_log_level) \ if hasattr(pySPACE.configuration, "file_log_level") \ else logging.INFO if not isinstance(log_level, int): raise NameError() except (AttributeError, NameError): import warnings warnings.warn( "%s is not a valid log level! Falling back to logging.INFO." % pySPACE.configuration.log_level) log_level = logging.INFO self.file_handler.setLevel(log_level) # set a format which is simpler for console use formatter = logging.Formatter( '%(asctime)s %(name)-40s %(levelname)-8s %(message)s') # tell the handler to use the formatter self.file_handler.setFormatter(formatter) # add the handler to the root logger logging.getLogger('').addHandler(self.file_handler) # prepare socket connection and listener thread self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP self.sock.bind((self.SERVER_IP,0)) self.sock.setblocking(0) self.SERVER_PORT = self.sock.getsockname()[1] def execute(self): """ Executes all processes specified in the currently staged operation. """ raise NotImplementedError( "Method execute has not been implemented in subclass %s" % self.__class__.__name__) def check_status(self): """ Returns a description of the current state of the operations execution. """ raise NotImplementedError( "Method check_status has not been implemented in subclass %s" % self.__class__.__name__) def retrieve(self): """ Fetches the results of the operation's processes. ... note:: This call might block until all processes are finished """ raise NotImplementedError( "Method retrieve has not been implemented in subclass %s" % self.__class__.__name__) def consolidate(self): """ Consolidates the results of the single processes into a consistent result of the whole operation """ raise NotImplementedError( "Method consolidate has not been implemented in subclass %s" % self.__class__.__name__) def cleanup(self): """ Remove the current operation and all potential results that have been stored in this object """ raise NotImplementedError( "Method cleanup has not been implemented in subclass %s" % self.__class__.__name__) def get_result_directory(self): """ Return the result directory of the current operation (if any) """ if self.current_operation == None: raise Exception("No operation staged!") return self.current_operation.result_directory def _start_logging(self): """ Configures and starts the logging of this operation """ # Remove the default handler if len(logging.getLogger('').handlers) > 0: logging.getLogger('').removeHandler(logging.getLogger('').handlers[0]) # define a handler which writes WARNING messages # or higher to the sys.stderr console = logging.StreamHandler() # Determining level of the logger based on pySPACE configuration file try: log_level = eval(pySPACE.configuration.console_log_level) \ if hasattr(pySPACE.configuration, "console_log_level") \ else logging.WARNING if not isinstance(log_level, int): raise NameError() except (AttributeError, NameError): import warnings warnings.warn("%s is not a valid log level! Falling back to " \ "logging.WARNING." % pySPACE.configuration.log_level) log_level = logging.WARNING console.setLevel(log_level) # set a format which is simpler for console use formatter = logging.Formatter( '%(asctime)s %(name)-40s %(levelname)-8s %(message)s') # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger logging.getLogger('').addHandler(console) logging.getLogger('').setLevel(logging.DEBUG) self._log("Logging started") # Starting the socket server from pySPACE.tools.socket_logger import LogRecordSocketReceiver self._log("Starting the TCP logging server...") # Determining host ip host, aliaslist, lan_ip = socket.gethostbyname_ex(socket.gethostname()) self.host = lan_ip[0] # Search for an available port self.port = logging.handlers.DEFAULT_TCP_LOGGING_PORT while True: try: self.tcpserver = LogRecordSocketReceiver(host=self.host, port=self.port) break except socket.error: self.port += 1 self.tcpserver.start() self._log("Started the TCP logging server on port %s!" % self.port) def _stop_logging(self): """ Stops the logging of this operation """ self._log("Stopping the TCP logging server...") self.tcpserver.abort = True self.tcpserver.join() self.tcpserver.shutdown() self._log("Stopping the TCP logging server... Done!") self._log("Logging stopped") def _log(self, message, level = logging.INFO): """ Logs the given message with the given logging level """ root_logger = logging.getLogger("%s-%s.%s" % (socket.gethostname(), os.getpid(), self)) if len(root_logger.handlers)==0: root_logger.addHandler(logging.handlers.SocketHandler('localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT)) root_logger.log(level, message)
class Backend(object): """ Interface for backends All other backends must implement several methods of this interface in order to execute an operation on a specific modality. """ STATES = set( ["idling", "staged", "executing", "retrieved", "consolidated"]) def __init__(self): # Start logging self._start_logging() # Install signal handlers #signal.signal(signal.SIGINT, self._grace) #signal.signal(signal.SIGTERM, self._grace) #signal.signal(signal.SIGQUIT, self._grace) #signal.signal(signal.SIGPIPE, self._grace) self.file_handler = None self.current_operation = None self.SERVER_IP = socket.gethostbyname(socket.gethostname()) self.SERVER_PORT = None self.listener = None def __del__(self): # Stop logging self._stop_logging() def __str__(self): return str(type(self).__name__) def stage_in(self, operation): """ Stage the current operation """ self.current_operation = operation # Add a handler that logs the operations output to a file log_path = self.current_operation.result_directory self.file_handler = logging.FileHandler( os.path.join(log_path, "operation.log")) # Determining level of the logger based on pySPACE configuration file try: log_level = eval(pySPACE.configuration.file_log_level) \ if hasattr(pySPACE.configuration, "file_log_level") \ else logging.INFO if not isinstance(log_level, int): raise NameError() except (AttributeError, NameError): import warnings warnings.warn( "%s is not a valid log level! Falling back to logging.INFO." % pySPACE.configuration.log_level) log_level = logging.INFO self.file_handler.setLevel(log_level) # set a format which is simpler for console use formatter = logging.Formatter( '%(asctime)s %(name)-40s %(levelname)-8s %(message)s') # tell the handler to use the formatter self.file_handler.setFormatter(formatter) # add the handler to the root logger logging.getLogger('').addHandler(self.file_handler) # prepare socket connection and listener thread self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP self.sock.bind((self.SERVER_IP, 0)) self.sock.setblocking(0) self.SERVER_PORT = self.sock.getsockname()[1] def execute(self): """ Executes all processes specified in the currently staged operation. """ raise NotImplementedError( "Method execute has not been implemented in subclass %s" % self.__class__.__name__) def check_status(self): """ Returns a description of the current state of the operations execution. """ raise NotImplementedError( "Method check_status has not been implemented in subclass %s" % self.__class__.__name__) def retrieve(self): """ Fetches the results of the operation's processes. ... note:: This call might block until all processes are finished """ raise NotImplementedError( "Method retrieve has not been implemented in subclass %s" % self.__class__.__name__) def consolidate(self): """ Consolidates the results of the single processes into a consistent result of the whole operation """ raise NotImplementedError( "Method consolidate has not been implemented in subclass %s" % self.__class__.__name__) def cleanup(self): """ Remove the current operation and all potential results that have been stored in this object """ raise NotImplementedError( "Method cleanup has not been implemented in subclass %s" % self.__class__.__name__) def get_result_directory(self): """ Return the result directory of the current operation (if any) """ if self.current_operation == None: raise Exception("No operation staged!") return self.current_operation.result_directory def _start_logging(self): """ Configures and starts the logging of this operation """ # Remove the default handler if len(logging.getLogger('').handlers) > 0: logging.getLogger('').removeHandler( logging.getLogger('').handlers[0]) # define a handler which writes WARNING messages # or higher to the sys.stderr console = logging.StreamHandler() # Determining level of the logger based on pySPACE configuration file try: log_level = eval(pySPACE.configuration.console_log_level) \ if hasattr(pySPACE.configuration, "console_log_level") \ else logging.WARNING if not isinstance(log_level, int): raise NameError() except (AttributeError, NameError): import warnings warnings.warn("%s is not a valid log level! Falling back to " \ "logging.WARNING." % pySPACE.configuration.log_level) log_level = logging.WARNING console.setLevel(log_level) # set a format which is simpler for console use formatter = logging.Formatter( '%(asctime)s %(name)-40s %(levelname)-8s %(message)s') # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger logging.getLogger('').addHandler(console) logging.getLogger('').setLevel(logging.DEBUG) self._log("Logging started") # Starting the socket server from pySPACE.tools.socket_logger import LogRecordSocketReceiver self._log("Starting the TCP logging server...") # Determining host ip host, aliaslist, lan_ip = socket.gethostbyname_ex(socket.gethostname()) self.host = lan_ip[0] # Search for an available port self.port = logging.handlers.DEFAULT_TCP_LOGGING_PORT while True: try: self.tcpserver = LogRecordSocketReceiver(host=self.host, port=self.port) break except socket.error: self.port += 1 self.tcpserver.start() self._log("Started the TCP logging server on port %s!" % self.port) def _stop_logging(self): """ Stops the logging of this operation """ self._log("Stopping the TCP logging server...") self.tcpserver.abort = True self.tcpserver.join() self.tcpserver.shutdown() self._log("Stopping the TCP logging server... Done!") self._log("Logging stopped") def _log(self, message, level=logging.INFO): """ Logs the given message with the given logging level """ root_logger = logging.getLogger( "%s-%s.%s" % (socket.gethostname(), os.getpid(), self)) if len(root_logger.handlers) == 0: root_logger.addHandler( logging.handlers.SocketHandler( 'localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT)) root_logger.log(level, message)