Esempio n. 1
0
def main_loop(conf, foreground=False, task_manager_class=None):
    """infinite daemon loop"""

    # define custom signal handlers
    signal.signal(signal.SIGTERM, daemon_shutdown)

    # initialize TaskManager
    try:
        log_file = conf.get("LOG_FILE", None)
        logger = logging.Logger("TaskManager")
        logger.setLevel(logging.DEBUG)
        if log_file:
            log_level = logging.getLevelName(conf.get("LOG_LEVEL", "DEBUG").upper())
            kobo.log.add_rotating_file_logger(logger, log_file, log_level=log_level)

        if not task_manager_class:
            tm = TaskManager(conf=conf, logger=logger)
        else:
            tm = task_manager_class(conf=conf, logger=logger)
    except Exception as ex:
        raise
        sys.stderr.write("Error initializing TaskManager: %s\n" % ex)
        sys.exit(1)

    if foreground and tm._logger is not None:
        kobo.log.add_stderr_logger(tm._logger)

    while 1:
        try:
            tm.log_debug(80 * '-')
            # poll hub for new tasks
            tm.hub._login()
            tm.update_worker_info()
            tm.update_tasks()
            tm.get_next_task()

            # write to stdout / stderr
            sys.stdout.flush()
            sys.stderr.flush()

            # sleep for some time
            tm.sleep()

        except (ShutdownException, KeyboardInterrupt):
            # ignore keyboard interrupts and sigterm
            signal.signal(signal.SIGINT, signal.SIG_IGN)
            signal.signal(signal.SIGTERM, signal.SIG_IGN)

            tm.log_info('Exiting...')
            tm.shutdown()
            tm.log_info('Cleanup done...')
            break

        except:
            # this is a little extreme: log the exception and continue
            traceback = Traceback()
            tm.log_error(traceback.get_traceback())
            tm.sleep()
Esempio n. 2
0
 def test_encoding(self):
     try:
         a = ''.join([chr(i) for i in range(256)])
         b = u''.join([unichr(i) for i in range(65536)])
         raise Exception()
     except:
         tb = Traceback(show_code=False, show_traceback=False)
     output = tb.get_traceback()
     self.assertIsInstance(output, six.binary_type)
Esempio n. 3
0
 def test_encoding(self):
     try:
         a = ''.join([chr(i) for i in range(256)])
         b = u''.join([unichr(i) for i in range(65536)])
         raise Exception()
     except:
         tb = Traceback(show_code = False, show_traceback = False)
     output = tb.get_traceback()
     self.assertIsInstance(output, six.binary_type)
Esempio n. 4
0
    def test_text(self):
        try:
            raise Exception('Simple text')
        except:
            str_regexp = re.compile('Traceback \(most recent call last\):\n *File ".*test_tback.py", line .+, in test_text\n *raise Exception\(\'Simple text\'\)\n *Exception: Simple text', re.M)
            bytes_regexp = re.compile(b'Traceback \(most recent call last\):\n *File ".*test_tback.py", line .+, in test_text\n *raise Exception\(\'Simple text\'\)\n *Exception: Simple text', re.M)

            self.assertRegex(get_traceback(), str_regexp)
            tb = Traceback(show_traceback = True, show_code = False, show_locals = False, show_environ = False, show_modules = False)
            self.assertRegex(tb.get_traceback(), bytes_regexp)
Esempio n. 5
0
File: main.py Progetto: kdudka/kobo
def main_loop(conf, foreground=False):
    """infinite daemon loop"""

    # define custom signal handlers
    signal.signal(signal.SIGTERM, daemon_shutdown)

    # initialize TaskManager
    try:
        log_file = conf.get("LOG_FILE", None)
        logger = logging.Logger("TaskManager")
        logger.setLevel(logging.DEBUG)
        if log_file:
            log_level = logging._levelNames.get(conf.get("LOG_LEVEL", "DEBUG").upper())
            kobo.log.add_rotating_file_logger(logger, log_file, log_level=log_level)

        tm = TaskManager(conf=conf, logger=logger)
    except Exception as ex:
        raise
        sys.stderr.write("Error initializing TaskManager: %s\n" % ex)
        sys.exit(1)

    if foreground and tm._logger is not None:
        kobo.log.add_stderr_logger(tm._logger)

    while 1:
        try:
            tm.log_debug(80 * '-')
            # poll hub for new tasks
            tm.hub._login()
            tm.update_worker_info()
            tm.update_tasks()
            tm.get_next_task()

            # write to stdout / stderr
            sys.stdout.flush()
            sys.stderr.flush()

            # sleep for some time
            tm.sleep()

        except (ShutdownException, KeyboardInterrupt):
            # ignore keyboard interrupts and sigterm
            signal.signal(signal.SIGINT, signal.SIG_IGN)
            signal.signal(signal.SIGTERM, signal.SIG_IGN)

            tm.log_info('Exiting...')
            tm.shutdown()
            tm.log_info('Cleanup done...')
            break

        except:
            # this is a little extreme: log the exception and continue
            traceback = Traceback()
            tm.log_error(traceback.get_traceback())
            tm.sleep()
Esempio n. 6
0
    def _new_function(request, *args, **kwargs):
        try:
            result = function(request, *args, **kwargs)
        except Exception as ex:
            # logdir must be absolute path
            if os.path.abspath(logdir) != logdir:
                raise

            if not os.path.isdir(logdir):
                os.makedirs(logdir)

            # create a file name from function name, date, time and some random characters
            file_name = "%s_%s_%s" % (
                function.__name__,
                datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d_%H-%M-%S"),
                random_string(32),
            )

            # create a file with 0600 perms (log can contain sensitive information like passwords)
            file_path = os.path.join(logdir, file_name)
            fd = os.open(file_path, os.O_CREAT | os.O_WRONLY, 0o600)
            os.write(fd, Traceback().get_traceback())
            os.close(fd)
            raise

        return result
Esempio n. 7
0
def main_loop(transfer, conf=None, foreground=False):
    """infinite daemon loop"""

    # define custom signal handlers
    signal.signal(signal.SIGTERM, daemon_shutdown)

    # set up logging
    log_level_string = conf.get("TRANSFER_LOG_LEVEL") or conf["LOG_LEVEL"]
    log_level = getattr(logging, log_level_string.upper(), logging.DEBUG)
    logging.getLogger().setLevel(log_level)
    if foreground:
        add_stderr_logger(logging.getLogger(), log_level=log_level)
    else:
        log_file = conf["TRANSFER_LOG_FILE"]
        add_rotating_file_logger(logging.getLogger(), log_file,
                log_level=log_level, format=conf["VERBOSE_LOG_FORMAT"])

    while True:
        try:
            transfer.hub._login()
            # Look for logs to transfer if none transfered then sleep
            if not transfer.transfer_logs():
                logger.debug(80 * '-')
                transfer.sleep()

            # write to stdout / stderr
            sys.stdout.flush()
            sys.stderr.flush()

        except socket.sslerror:
            pass # will try again..

        except (ShutdownException, KeyboardInterrupt):
            # ignore keyboard interrupts and sigterm
            signal.signal(signal.SIGINT, signal.SIG_IGN)
            signal.signal(signal.SIGTERM, signal.SIG_IGN)

            logger.info('Exiting...')
            break

        except:
            # this is a little extreme: log the exception and continue
            traceback = Traceback()
            logger.error(traceback.get_traceback())
            transfer.sleep()
Esempio n. 8
0
    def test_text(self):
        try:
            raise Exception('Simple text')
        except:
            str_regexp = re.compile(
                'Traceback \(most recent call last\):\n *File ".*test_tback.py", line .+, in test_text\n *raise Exception\(\'Simple text\'\)\n *Exception: Simple text',
                re.M)
            bytes_regexp = re.compile(
                b'Traceback \(most recent call last\):\n *File ".*test_tback.py", line .+, in test_text\n *raise Exception\(\'Simple text\'\)\n *Exception: Simple text',
                re.M)

            self.assertRegex(get_traceback(), str_regexp)
            tb = Traceback(show_traceback=True,
                           show_code=False,
                           show_locals=False,
                           show_environ=False,
                           show_modules=False)
            self.assertRegex(tb.get_traceback(), bytes_regexp)
Esempio n. 9
0
    def _marshaled_dispatch(self, request, dispatch_method=None):
        """Dispatches an XML-RPC method from marshalled (XML) data.

        XML-RPC methods are dispatched from the marshalled (XML) data
        using the _dispatch method and the result is returned as
        marshalled data. For backwards compatibility, a dispatch
        function can be provided as an argument (see comment in
        SimpleXMLRPCRequestHandler.do_POST) but overriding the
        existing method through subclassing is the prefered means
        of changing method dispatch behavior.
        """

        data = request.body
        params, method = xmlrpclib.loads(data)

        # add request to params
        params = (request, ) + params

        # generate response
        try:
            if dispatch_method is not None:
                response = dispatch_method(method, params)
            else:
                response = self._dispatch(method, params)
            # wrap response in a singleton tuple
            response = (response, )
            response = xmlrpclib.dumps(response,
                                       methodresponse=1,
                                       allow_none=self.allow_none,
                                       encoding=self.encoding)

        except xmlrpclib.Fault as fault:
            response = xmlrpclib.dumps(fault,
                                       allow_none=self.allow_none,
                                       encoding=self.encoding)

        except:
            # report exception back to server
            if settings.DEBUG:
                from kobo.tback import Traceback
                response = xmlrpclib.dumps(xmlrpclib.Fault(
                    1, u"%s" % Traceback().get_traceback()),
                                           allow_none=self.allow_none,
                                           encoding=self.encoding)
            else:
                exc_info = sys.exc_info()[1]
                exc_type = exc_info.__class__.__name__
                response = xmlrpclib.dumps(xmlrpclib.Fault(
                    1, "%s: %s" % (exc_type, exc_info)),
                                           allow_none=self.allow_none,
                                           encoding=self.encoding)

        return response
Esempio n. 10
0
 def test_Traceback(self):
     try:
         raise Exception('Simple text')
     except:
         tb = Traceback(show_traceback = False, show_code = False, show_locals = False, show_environ = False, show_modules = False)
     self.assertEqual(b'', tb.get_traceback())
     tb.show_code = True
     self.assertRegex(tb.get_traceback(), re.compile(b'<CODE>.*--> *\d+ *raise Exception.*<\/CODE>$', re.M | re.S))
     tb.show_code = False
     tb.show_locals = True
     self.assertRegex(tb.get_traceback(), re.compile(b'<LOCALS>.*tb = .*<\/LOCALS>$', re.M | re.S))
     tb.show_locals = False
     tb.show_environ = True
     self.assertRegex(tb.get_traceback(), re.compile(b'<ENVIRON>.*<\/ENVIRON>\n<GLOBALS>.*</GLOBALS>$', re.M | re.S))
     tb.show_environ = False
     tb.show_modules = True
     self.assertRegex(tb.get_traceback(), re.compile(b'<MODULES>.*<\/MODULES>$', re.M | re.S))
Esempio n. 11
0
    time_of_last_check = 0
    while True:
        try:
            now = time.time()
            # Poll for watchdogs
            if now - time_of_last_check > conf.get('SLEEP_TIME', 60):
                time_of_last_check = now
                watchdog.hub._login()

                try:
                    expired_watchdogs = watchdog.hub.recipes.tasks.watchdogs('expired')
                except xmlrpclib.Fault:
                    # catch any xmlrpc errors
                    expired_watchdogs = []
                    traceback = Traceback()
                    logger.error(traceback.get_traceback())
                watchdog.expire_watchdogs(expired_watchdogs)

                # Get active watchdogs *after* we finish running
                # expired_watchdogs, depending on the configuration
                # we may have extended the watchdog and its therefore
                # no longer expired!
                try:
                    active_watchdogs = watchdog.hub.recipes.tasks.watchdogs('active')
                except xmlrpclib.Fault:
                    # catch any xmlrpc errors
                    traceback = Traceback()
                    logger.error(traceback.get_traceback())
                    active_watchdogs = []
                watchdog.active_watchdogs(active_watchdogs)
Esempio n. 12
0
 def test(self):
     try:
         raise
     except:
         # bar is uninitialized
         return Traceback().get_traceback()
Esempio n. 13
0
 def test_empty(self):
     self.assertEqual('', get_traceback())
     self.assertEqual('', Traceback().get_traceback())
     self.assertEqual((None, None, None), Traceback().exc_info)
Esempio n. 14
0
            # sleep for some time
            tm.sleep()

        except (ShutdownException, KeyboardInterrupt):
            # ignore keyboard interrupts and sigterm
            signal.signal(signal.SIGINT, signal.SIG_IGN)
            signal.signal(signal.SIGTERM, signal.SIG_IGN)

            tm.log_info('Exiting...')
            tm.shutdown()
            tm.log_info('Cleanup done...')
            break

        except:
            # this is a little extreme: log the exception and continue
            traceback = Traceback()
            tm.log_error(traceback.get_traceback())
            tm.sleep()


def main(conf, argv=None):
    parser = optparse.OptionParser()
    parser.add_option(
        "-f",
        "--foreground",
        default=False,
        action="store_true",
        help="run in foreground (do not spawn a daemon)",
    )
    parser.add_option(
        "-k",
Esempio n. 15
0
                response = dispatch_method(method, params)
            else:
                response = self._dispatch(method, params)
            # wrap response in a singleton tuple
            response = (response, )
            response = xmlrpclib.dumps(response,
                                       methodresponse=1,
                                       allow_none=self.allow_none,
                                       encoding=self.encoding)

        except xmlrpclib.Fault, fault:
            response = xmlrpclib.dumps(fault,
                                       allow_none=self.allow_none,
                                       encoding=self.encoding)

        except:
            # report exception back to server
            if settings.DEBUG:
                from kobo.tback import Traceback
                response = xmlrpclib.dumps(xmlrpclib.Fault(
                    1, u"%s" % Traceback().get_traceback()),
                                           allow_none=self.allow_none,
                                           encoding=self.encoding)
            else:
                response = xmlrpclib.dumps(xmlrpclib.Fault(
                    1, "%s: %s" % (sys.exc_type.__name__, sys.exc_value)),
                                           allow_none=self.allow_none,
                                           encoding=self.encoding)

        return response
Esempio n. 16
0
 def test_Traceback(self):
     try:
         raise Exception('Simple text')
     except:
         tb = Traceback(show_traceback=False,
                        show_code=False,
                        show_locals=False,
                        show_environ=False,
                        show_modules=False)
     self.assertEqual(b'', tb.get_traceback())
     tb.show_code = True
     self.assertRegex(
         tb.get_traceback(),
         re.compile(b'<CODE>.*--> *\d+ *raise Exception.*<\/CODE>$',
                    re.M | re.S))
     tb.show_code = False
     tb.show_locals = True
     self.assertRegex(
         tb.get_traceback(),
         re.compile(b'<LOCALS>.*tb = .*<\/LOCALS>$', re.M | re.S))
     tb.show_locals = False
     tb.show_environ = True
     self.assertRegex(
         tb.get_traceback(),
         re.compile(b'<ENVIRON>.*<\/ENVIRON>\n<GLOBALS>.*</GLOBALS>$',
                    re.M | re.S))
     tb.show_environ = False
     tb.show_modules = True
     self.assertRegex(tb.get_traceback(),
                      re.compile(b'<MODULES>.*<\/MODULES>$', re.M | re.S))
Esempio n. 17
0
            # sleep for some time
            tm.sleep()

        except (ShutdownException, KeyboardInterrupt):
            # ignore keyboard interrupts and sigterm
            signal.signal(signal.SIGINT, signal.SIG_IGN)
            signal.signal(signal.SIGTERM, signal.SIG_IGN)

            tm.log_info('Exiting...')
            tm.shutdown()
            tm.log_info('Cleanup done...')
            break

        except:
            # this is a little extreme: log the exception and continue
            traceback = Traceback()
            tm.log_error(traceback.get_traceback())
            tm.sleep()


def main(conf, argv=None):
    parser = optparse.OptionParser()
    parser.add_option(
        "-f", "--foreground",
        default=False,
        action="store_true",
        help="run in foreground (do not spawn a daemon)",
    )
    parser.add_option(
        "-k", "--kill",
        default=False,