def scan_plugin_dir(self, dir):
self.logger.debug('Scanning plugin directory %s', dir)
for f in os.listdir(dir):
name, ext = os.path.splitext(os.path.basename(f))
if ext != '.py':
continue
try:
plugin = load_module_from_file(name, os.path.join(dir, f))
plugin._init(self)
except:
self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True)
def scan_plugin_dir(self, dir):
self.logger.debug('Scanning plugin directory %s', dir)
for f in os.listdir(dir):
name, ext = os.path.splitext(os.path.basename(f))
if ext != '.py':
continue
try:
plugin = load_module_from_file(name, os.path.join(dir, f))
plugin._init(self)
except:
self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True)
def main(self):
if len(sys.argv) != 2:
print("Invalid number of arguments", file=sys.stderr)
sys.exit(errno.EINVAL)
key = sys.argv[1]
configure_logging(None, logging.DEBUG)
self.datastore = get_datastore()
self.configstore = ConfigStore(self.datastore)
self.conn = Client()
self.conn.connect('unix:')
self.conn.login_service('task.{0}'.format(os.getpid()))
self.conn.enable_server()
self.conn.call_sync('management.enable_features',
['streaming_responses'])
self.conn.rpc.register_service_instance('taskproxy', self.service)
self.conn.register_event_handler('task.progress',
self.task_progress_handler)
self.conn.call_sync('task.checkin', key)
setproctitle.setproctitle('task executor (idle)')
while True:
try:
task = self.task.get()
logging.root.setLevel(
self.conn.call_sync('management.get_logging_level'))
setproctitle.setproctitle('task executor (tid {0})'.format(
task['id']))
if task['debugger']:
sys.path.append('/usr/local/lib/dispatcher/pydev')
import pydevd
host, port = task['debugger']
pydevd.settrace(host,
port=port,
stdoutToServer=True,
stderrToServer=True)
name, _ = os.path.splitext(os.path.basename(task['filename']))
module = self.module_cache.get(task['filename'])
if not module:
module = load_module_from_file(name, task['filename'])
self.module_cache[task['filename']] = module
setproctitle.setproctitle('task executor (tid {0})'.format(
task['id']))
fds = list(self.collect_fds(task['args']))
try:
dispatcher = DispatcherWrapper(self.conn)
self.instance = getattr(module,
task['class'])(dispatcher,
self.datastore)
self.instance.configstore = self.configstore
self.instance.user = task['user']
self.instance.environment = task['environment']
self.running.set()
self.run_task_hooks(self.instance, task, 'before')
result = self.instance.run(*task['args'])
self.run_task_hooks(self.instance,
task,
'after',
result=result)
except BaseException as err:
print("Task exception: {0}".format(str(err)),
file=sys.stderr)
traceback.print_exc(file=sys.stderr)
if hasattr(self.instance, 'rollback'):
self.put_status('ROLLBACK')
try:
self.instance.rollback(*task['args'])
except BaseException as rerr:
print("Task exception during rollback: {0}".format(
str(rerr)),
file=sys.stderr)
traceback.print_exc(file=sys.stderr)
# Main task is already failed at this point, so ignore hook errors
with contextlib.suppress(RpcException):
self.run_task_hooks(self.instance,
task,
'error',
error=serialize_error(err))
self.put_status('FAILED', exception=err)
else:
self.put_status('FINISHED', result=result)
finally:
self.close_fds(fds)
self.running.clear()
except RpcException as err:
print("RPC failed: {0}".format(str(err)), file=sys.stderr)
print(traceback.format_exc(), flush=True)
sys.exit(errno.EBADMSG)
except socket.error as err:
print("Cannot connect to dispatcher: {0}".format(str(err)),
file=sys.stderr)
sys.exit(errno.ETIMEDOUT)
if task['debugger']:
import pydevd
pydevd.stoptrace()
setproctitle.setproctitle('task executor (idle)')
def main(self):
if len(sys.argv) != 2:
print("Invalid number of arguments", file=sys.stderr)
sys.exit(errno.EINVAL)
key = sys.argv[1]
configure_logging(None, logging.DEBUG)
self.datastore = get_datastore()
self.configstore = ConfigStore(self.datastore)
self.conn = Client()
self.conn.connect('unix:')
self.conn.login_service('task.{0}'.format(os.getpid()))
self.conn.enable_server()
self.conn.rpc.register_service_instance('taskproxy', self.service)
self.conn.register_event_handler('task.progress', self.task_progress_handler)
self.conn.call_sync('task.checkin', key)
setproctitle.setproctitle('task executor (idle)')
while True:
try:
task = self.task.get()
logging.root.setLevel(self.conn.call_sync('management.get_logging_level'))
setproctitle.setproctitle('task executor (tid {0})'.format(task['id']))
if task['debugger']:
sys.path.append('/usr/local/lib/dispatcher/pydev')
import pydevd
host, port = task['debugger']
pydevd.settrace(host, port=port, stdoutToServer=True, stderrToServer=True)
name, _ = os.path.splitext(os.path.basename(task['filename']))
module = load_module_from_file(name, task['filename'])
setproctitle.setproctitle('task executor (tid {0})'.format(task['id']))
fds = list(self.collect_fds(task['args']))
try:
self.instance = getattr(module, task['class'])(DispatcherWrapper(self.conn), self.datastore)
self.instance.configstore = self.configstore
self.instance.user = task['user']
self.instance.environment = task['environment']
self.running.set()
result = self.instance.run(*task['args'])
except BaseException as err:
print("Task exception: {0}".format(str(err)), file=sys.stderr)
traceback.print_exc(file=sys.stderr)
if hasattr(self.instance, 'rollback'):
self.put_status('ROLLBACK')
try:
self.instance.rollback(*task['args'])
except BaseException as rerr:
print("Task exception during rollback: {0}".format(str(rerr)), file=sys.stderr)
traceback.print_exc(file=sys.stderr)
self.put_status('FAILED', exception=err)
else:
self.put_status('FINISHED', result=result)
finally:
self.close_fds(fds)
self.running.clear()
except RpcException as err:
print("RPC failed: {0}".format(str(err)), file=sys.stderr)
print(traceback.format_exc(), flush=True)
sys.exit(errno.EBADMSG)
except socket.error as err:
print("Cannot connect to dispatcher: {0}".format(str(err)), file=sys.stderr)
sys.exit(errno.ETIMEDOUT)
if task['debugger']:
import pydevd
pydevd.stoptrace()
setproctitle.setproctitle('task executor (idle)')
