def main_loop(conf, foreground=False, task_manager_class=None):
"""infinite daemon loop"""
# define custom signal handlers
signal.signal(signal.SIGTERM, daemon_shutdown)
# initialize TaskManager
try:
log_file = conf.get("LOG_FILE", None)
logger = logging.Logger("TaskManager")
logger.setLevel(logging.DEBUG)
if log_file:
log_level = logging.getLevelName(conf.get("LOG_LEVEL", "DEBUG").upper())
kobo.log.add_rotating_file_logger(logger, log_file, log_level=log_level)
if not task_manager_class:
tm = TaskManager(conf=conf, logger=logger)
else:
tm = task_manager_class(conf=conf, logger=logger)
except Exception as ex:
raise
sys.stderr.write("Error initializing TaskManager: %s\n" % ex)
sys.exit(1)
if foreground and tm._logger is not None:
kobo.log.add_stderr_logger(tm._logger)
while 1:
try:
tm.log_debug(80 * '-')
# poll hub for new tasks
tm.hub._login()
tm.update_worker_info()
tm.update_tasks()
tm.get_next_task()
# write to stdout / stderr
sys.stdout.flush()
sys.stderr.flush()
# sleep for some time
tm.sleep()
except (ShutdownException, KeyboardInterrupt):
# ignore keyboard interrupts and sigterm
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
tm.log_info('Exiting...')
tm.shutdown()
tm.log_info('Cleanup done...')
break
except:
# this is a little extreme: log the exception and continue
traceback = Traceback()
tm.log_error(traceback.get_traceback())
tm.sleep()
def test_encoding(self):
try:
a = ''.join([chr(i) for i in range(256)])
b = u''.join([unichr(i) for i in range(65536)])
raise Exception()
except:
tb = Traceback(show_code=False, show_traceback=False)
output = tb.get_traceback()
self.assertIsInstance(output, six.binary_type)
def test_encoding(self):
try:
a = ''.join([chr(i) for i in range(256)])
b = u''.join([unichr(i) for i in range(65536)])
raise Exception()
except:
tb = Traceback(show_code = False, show_traceback = False)
output = tb.get_traceback()
self.assertIsInstance(output, six.binary_type)
def test_text(self):
try:
raise Exception('Simple text')
except:
str_regexp = re.compile('Traceback \(most recent call last\):\n *File ".*test_tback.py", line .+, in test_text\n *raise Exception\(\'Simple text\'\)\n *Exception: Simple text', re.M)
bytes_regexp = re.compile(b'Traceback \(most recent call last\):\n *File ".*test_tback.py", line .+, in test_text\n *raise Exception\(\'Simple text\'\)\n *Exception: Simple text', re.M)
self.assertRegex(get_traceback(), str_regexp)
tb = Traceback(show_traceback = True, show_code = False, show_locals = False, show_environ = False, show_modules = False)
self.assertRegex(tb.get_traceback(), bytes_regexp)
def main_loop(conf, foreground=False):
"""infinite daemon loop"""
# define custom signal handlers
signal.signal(signal.SIGTERM, daemon_shutdown)
# initialize TaskManager
try:
log_file = conf.get("LOG_FILE", None)
logger = logging.Logger("TaskManager")
logger.setLevel(logging.DEBUG)
if log_file:
log_level = logging._levelNames.get(conf.get("LOG_LEVEL", "DEBUG").upper())
kobo.log.add_rotating_file_logger(logger, log_file, log_level=log_level)
tm = TaskManager(conf=conf, logger=logger)
except Exception as ex:
raise
sys.stderr.write("Error initializing TaskManager: %s\n" % ex)
sys.exit(1)
if foreground and tm._logger is not None:
kobo.log.add_stderr_logger(tm._logger)
while 1:
try:
tm.log_debug(80 * '-')
# poll hub for new tasks
tm.hub._login()
tm.update_worker_info()
tm.update_tasks()
tm.get_next_task()
# write to stdout / stderr
sys.stdout.flush()
sys.stderr.flush()
# sleep for some time
tm.sleep()
except (ShutdownException, KeyboardInterrupt):
# ignore keyboard interrupts and sigterm
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
tm.log_info('Exiting...')
tm.shutdown()
tm.log_info('Cleanup done...')
break
except:
# this is a little extreme: log the exception and continue
traceback = Traceback()
tm.log_error(traceback.get_traceback())
tm.sleep()
def _new_function(request, *args, **kwargs):
try:
result = function(request, *args, **kwargs)
except Exception as ex:
# logdir must be absolute path
if os.path.abspath(logdir) != logdir:
raise
if not os.path.isdir(logdir):
os.makedirs(logdir)
# create a file name from function name, date, time and some random characters
file_name = "%s_%s_%s" % (
function.__name__,
datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d_%H-%M-%S"),
random_string(32),
)
# create a file with 0600 perms (log can contain sensitive information like passwords)
file_path = os.path.join(logdir, file_name)
fd = os.open(file_path, os.O_CREAT | os.O_WRONLY, 0o600)
os.write(fd, Traceback().get_traceback())
os.close(fd)
raise
return result
def main_loop(transfer, conf=None, foreground=False):
"""infinite daemon loop"""
# define custom signal handlers
signal.signal(signal.SIGTERM, daemon_shutdown)
# set up logging
log_level_string = conf.get("TRANSFER_LOG_LEVEL") or conf["LOG_LEVEL"]
log_level = getattr(logging, log_level_string.upper(), logging.DEBUG)
logging.getLogger().setLevel(log_level)
if foreground:
add_stderr_logger(logging.getLogger(), log_level=log_level)
else:
log_file = conf["TRANSFER_LOG_FILE"]
add_rotating_file_logger(logging.getLogger(), log_file,
log_level=log_level, format=conf["VERBOSE_LOG_FORMAT"])
while True:
try:
transfer.hub._login()
# Look for logs to transfer if none transfered then sleep
if not transfer.transfer_logs():
logger.debug(80 * '-')
transfer.sleep()
# write to stdout / stderr
sys.stdout.flush()
sys.stderr.flush()
except socket.sslerror:
pass # will try again..
except (ShutdownException, KeyboardInterrupt):
# ignore keyboard interrupts and sigterm
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
logger.info('Exiting...')
break
except:
# this is a little extreme: log the exception and continue
traceback = Traceback()
logger.error(traceback.get_traceback())
transfer.sleep()
def test_text(self):
try:
raise Exception('Simple text')
except:
str_regexp = re.compile(
'Traceback \(most recent call last\):\n *File ".*test_tback.py", line .+, in test_text\n *raise Exception\(\'Simple text\'\)\n *Exception: Simple text',
re.M)
bytes_regexp = re.compile(
b'Traceback \(most recent call last\):\n *File ".*test_tback.py", line .+, in test_text\n *raise Exception\(\'Simple text\'\)\n *Exception: Simple text',
re.M)
self.assertRegex(get_traceback(), str_regexp)
tb = Traceback(show_traceback=True,
show_code=False,
show_locals=False,
show_environ=False,
show_modules=False)
self.assertRegex(tb.get_traceback(), bytes_regexp)
def _marshaled_dispatch(self, request, dispatch_method=None):
"""Dispatches an XML-RPC method from marshalled (XML) data.
XML-RPC methods are dispatched from the marshalled (XML) data
using the _dispatch method and the result is returned as
marshalled data. For backwards compatibility, a dispatch
function can be provided as an argument (see comment in
SimpleXMLRPCRequestHandler.do_POST) but overriding the
existing method through subclassing is the prefered means
of changing method dispatch behavior.
"""
data = request.body
params, method = xmlrpclib.loads(data)
# add request to params
params = (request, ) + params
# generate response
try:
if dispatch_method is not None:
response = dispatch_method(method, params)
else:
response = self._dispatch(method, params)
# wrap response in a singleton tuple
response = (response, )
response = xmlrpclib.dumps(response,
methodresponse=1,
allow_none=self.allow_none,
encoding=self.encoding)
except xmlrpclib.Fault as fault:
response = xmlrpclib.dumps(fault,
allow_none=self.allow_none,
encoding=self.encoding)
except:
# report exception back to server
if settings.DEBUG:
from kobo.tback import Traceback
response = xmlrpclib.dumps(xmlrpclib.Fault(
1, u"%s" % Traceback().get_traceback()),
allow_none=self.allow_none,
encoding=self.encoding)
else:
exc_info = sys.exc_info()[1]
exc_type = exc_info.__class__.__name__
response = xmlrpclib.dumps(xmlrpclib.Fault(
1, "%s: %s" % (exc_type, exc_info)),
allow_none=self.allow_none,
encoding=self.encoding)
return response
def test_Traceback(self):
try:
raise Exception('Simple text')
except:
tb = Traceback(show_traceback = False, show_code = False, show_locals = False, show_environ = False, show_modules = False)
self.assertEqual(b'', tb.get_traceback())
tb.show_code = True
self.assertRegex(tb.get_traceback(), re.compile(b'<CODE>.*--> *\d+ *raise Exception.*<\/CODE>$', re.M | re.S))
tb.show_code = False
tb.show_locals = True
self.assertRegex(tb.get_traceback(), re.compile(b'<LOCALS>.*tb = .*<\/LOCALS>$', re.M | re.S))
tb.show_locals = False
tb.show_environ = True
self.assertRegex(tb.get_traceback(), re.compile(b'<ENVIRON>.*<\/ENVIRON>\n<GLOBALS>.*</GLOBALS>$', re.M | re.S))
tb.show_environ = False
tb.show_modules = True
self.assertRegex(tb.get_traceback(), re.compile(b'<MODULES>.*<\/MODULES>$', re.M | re.S))
time_of_last_check = 0
while True:
try:
now = time.time()
# Poll for watchdogs
if now - time_of_last_check > conf.get('SLEEP_TIME', 60):
time_of_last_check = now
watchdog.hub._login()
try:
expired_watchdogs = watchdog.hub.recipes.tasks.watchdogs('expired')
except xmlrpclib.Fault:
# catch any xmlrpc errors
expired_watchdogs = []
traceback = Traceback()
logger.error(traceback.get_traceback())
watchdog.expire_watchdogs(expired_watchdogs)
# Get active watchdogs *after* we finish running
# expired_watchdogs, depending on the configuration
# we may have extended the watchdog and its therefore
# no longer expired!
try:
active_watchdogs = watchdog.hub.recipes.tasks.watchdogs('active')
except xmlrpclib.Fault:
# catch any xmlrpc errors
traceback = Traceback()
logger.error(traceback.get_traceback())
active_watchdogs = []
watchdog.active_watchdogs(active_watchdogs)
def test(self):
try:
raise
except:
# bar is uninitialized
return Traceback().get_traceback()
def test_empty(self):
self.assertEqual('', get_traceback())
self.assertEqual('', Traceback().get_traceback())
self.assertEqual((None, None, None), Traceback().exc_info)
# sleep for some time
tm.sleep()
except (ShutdownException, KeyboardInterrupt):
# ignore keyboard interrupts and sigterm
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
tm.log_info('Exiting...')
tm.shutdown()
tm.log_info('Cleanup done...')
break
except:
# this is a little extreme: log the exception and continue
traceback = Traceback()
tm.log_error(traceback.get_traceback())
tm.sleep()
def main(conf, argv=None):
parser = optparse.OptionParser()
parser.add_option(
"-f",
"--foreground",
default=False,
action="store_true",
help="run in foreground (do not spawn a daemon)",
)
parser.add_option(
"-k",
response = dispatch_method(method, params)
else:
response = self._dispatch(method, params)
# wrap response in a singleton tuple
response = (response, )
response = xmlrpclib.dumps(response,
methodresponse=1,
allow_none=self.allow_none,
encoding=self.encoding)
except xmlrpclib.Fault, fault:
response = xmlrpclib.dumps(fault,
allow_none=self.allow_none,
encoding=self.encoding)
except:
# report exception back to server
if settings.DEBUG:
from kobo.tback import Traceback
response = xmlrpclib.dumps(xmlrpclib.Fault(
1, u"%s" % Traceback().get_traceback()),
allow_none=self.allow_none,
encoding=self.encoding)
else:
response = xmlrpclib.dumps(xmlrpclib.Fault(
1, "%s: %s" % (sys.exc_type.__name__, sys.exc_value)),
allow_none=self.allow_none,
encoding=self.encoding)
return response
def test_Traceback(self):
try:
raise Exception('Simple text')
except:
tb = Traceback(show_traceback=False,
show_code=False,
show_locals=False,
show_environ=False,
show_modules=False)
self.assertEqual(b'', tb.get_traceback())
tb.show_code = True
self.assertRegex(
tb.get_traceback(),
re.compile(b'<CODE>.*--> *\d+ *raise Exception.*<\/CODE>$',
re.M | re.S))
tb.show_code = False
tb.show_locals = True
self.assertRegex(
tb.get_traceback(),
re.compile(b'<LOCALS>.*tb = .*<\/LOCALS>$', re.M | re.S))
tb.show_locals = False
tb.show_environ = True
self.assertRegex(
tb.get_traceback(),
re.compile(b'<ENVIRON>.*<\/ENVIRON>\n<GLOBALS>.*</GLOBALS>$',
re.M | re.S))
tb.show_environ = False
tb.show_modules = True
self.assertRegex(tb.get_traceback(),
re.compile(b'<MODULES>.*<\/MODULES>$', re.M | re.S))
# sleep for some time
tm.sleep()
except (ShutdownException, KeyboardInterrupt):
# ignore keyboard interrupts and sigterm
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
tm.log_info('Exiting...')
tm.shutdown()
tm.log_info('Cleanup done...')
break
except:
# this is a little extreme: log the exception and continue
traceback = Traceback()
tm.log_error(traceback.get_traceback())
tm.sleep()
def main(conf, argv=None):
parser = optparse.OptionParser()
parser.add_option(
"-f", "--foreground",
default=False,
action="store_true",
help="run in foreground (do not spawn a daemon)",
)
parser.add_option(
"-k", "--kill",
default=False,