def _run_namespace_tasks(namespace, tasks=None, worker_scheduler_factory=None, override_defaults=None):
"""
:param namespace:
:param worker_scheduler_factory:
:param override_defaults:
:return: True if all tasks and their dependencies were successfully run (or already completed);
False if any error occurred.
"""
if worker_scheduler_factory is None:
worker_scheduler_factory = _WorkerSchedulerFactory()
if override_defaults is None:
override_defaults = {}
env_params = core(**override_defaults)
# search for logging configuration path first on the command line, then
# in the application config file
logging_conf = env_params.logging_conf_file
if logging_conf != '' and not os.path.exists(logging_conf):
raise Exception(
"Error: Unable to locate specified logging configuration file!"
)
if not configuration.get_config().getboolean(
'core', 'no_configure_logging', False):
setup_interface_logging(logging_conf, env_params.log_level)
kill_signal = signal.SIGUSR1 if env_params.take_lock else None
if (not env_params.no_lock and
not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size, kill_signal))):
raise PidLockAlreadyTakenExit()
if env_params.local_scheduler:
raise RuntimeError('Cannot use namespace with local scheduler.')
else:
if env_params.scheduler_url != '':
url = env_params.scheduler_url
else:
url = 'http://{host}:{port:d}/'.format(
host=env_params.scheduler_host,
port=env_params.scheduler_port,
)
sch = worker_scheduler_factory.create_remote_scheduler(url=url)
worker = worker_scheduler_factory.create_namespace_worker(
scheduler=sch, worker_processes=env_params.workers, namespace=namespace, assistant=env_params.assistant)
success = True
logger = logging.getLogger('luigi-interface')
with worker:
if namespace == 'init':
for t in tasks:
success &= worker.add(
t, env_params.parallel_scheduling, env_params.parallel_scheduling_processes)
logger.info('Done scheduling tasks')
else:
logger.info('Worker setup for "{}" namespace'.format(namespace))
success &= worker.run()
logger.info(execution_summary.summary(worker))
return dict(success=success, worker=worker)
def _schedule_and_run(tasks,
worker_scheduler_factory=None,
override_defaults=None):
"""
:param tasks:
:param worker_scheduler_factory:
:param override_defaults:
:return: True if all tasks and their dependencies were successfully run (or already completed);
False if any error occurred.
"""
if worker_scheduler_factory is None:
worker_scheduler_factory = _WorkerSchedulerFactory()
if override_defaults is None:
override_defaults = {}
env_params = core(**override_defaults)
# search for logging configuration path first on the command line, then
# in the application config file
logging_conf = env_params.logging_conf_file
if logging_conf is not None and not os.path.exists(logging_conf):
raise Exception(
"Error: Unable to locate specified logging configuration file!")
if not configuration.get_config().getboolean(
'core', 'no_configure_logging', False):
setup_interface_logging(logging_conf)
kill_signal = signal.SIGUSR1 if env_params.take_lock else None
if (not env_params.no_lock and not (lock.acquire_for(
env_params.lock_pid_dir, env_params.lock_size, kill_signal))):
sys.exit(1)
if env_params.local_scheduler:
sch = worker_scheduler_factory.create_local_scheduler()
else:
if env_params.scheduler_url is not None:
url = env_params.scheduler_url
else:
url = 'http://{host}:{port:d}/'.format(
host=env_params.scheduler_host,
port=env_params.scheduler_port,
)
sch = worker_scheduler_factory.create_remote_scheduler(url=url)
w = worker_scheduler_factory.create_worker(
scheduler=sch,
worker_processes=env_params.workers,
assistant=env_params.assistant)
success = True
for t in tasks:
success &= w.add(t, env_params.parallel_scheduling)
logger = logging.getLogger('luigi-interface')
logger.info('Done scheduling tasks')
if env_params.workers != 0:
success &= w.run()
w.stop()
logger.info(execution_summary.summary(w))
return success
def _schedule_and_run(tasks, worker_scheduler_factory=None, override_defaults=None):
"""
:param tasks:
:param worker_scheduler_factory:
:param override_defaults:
:return: True if all tasks and their dependencies were successfully run (or already completed);
False if any error occurred.
"""
if worker_scheduler_factory is None:
worker_scheduler_factory = _WorkerSchedulerFactory()
if override_defaults is None:
override_defaults = {}
env_params = core(**override_defaults)
# search for logging configuration path first on the command line, then
# in the application config file
logging_conf = env_params.logging_conf_file
if logging_conf is not None and not os.path.exists(logging_conf):
raise Exception(
"Error: Unable to locate specified logging configuration file!"
)
if not configuration.get_config().getboolean(
'core', 'no_configure_logging', False):
setup_interface_logging(logging_conf)
kill_signal = signal.SIGUSR1 if env_params.take_lock else None
if (not env_params.no_lock and
not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size, kill_signal))):
raise PidLockAlreadyTakenExit()
if env_params.local_scheduler:
sch = worker_scheduler_factory.create_local_scheduler()
else:
if env_params.scheduler_url is not None:
url = env_params.scheduler_url
else:
url = 'http://{host}:{port:d}/'.format(
host=env_params.scheduler_host,
port=env_params.scheduler_port,
)
sch = worker_scheduler_factory.create_remote_scheduler(url=url)
w = worker_scheduler_factory.create_worker(
scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant)
success = True
for t in tasks:
success &= w.add(t, env_params.parallel_scheduling)
logger = logging.getLogger('luigi-interface')
logger.info('Done scheduling tasks')
if env_params.workers != 0:
try:
success &= w.run()
finally:
w.stop()
logger.info(execution_summary.summary(w))
return dict(success=success, worker=w)
def _schedule_and_run(tasks,
worker_scheduler_factory=None,
override_defaults=None):
"""
:param tasks:
:param worker_scheduler_factory:
:param override_defaults:
:return: True if all tasks and their dependencies were successfully run (or already completed);
False if any error occurred.
"""
if worker_scheduler_factory is None:
worker_scheduler_factory = _WorkerSchedulerFactory()
if override_defaults is None:
override_defaults = {}
env_params = core(**override_defaults)
InterfaceLogging.setup(env_params)
kill_signal = signal.SIGUSR1 if env_params.take_lock else None
if (not env_params.no_lock and not (lock.acquire_for(
env_params.lock_pid_dir, env_params.lock_size, kill_signal))):
raise PidLockAlreadyTakenExit()
if env_params.local_scheduler:
sch = worker_scheduler_factory.create_local_scheduler()
else:
if env_params.scheduler_url != '':
url = env_params.scheduler_url
else:
url = 'http://{host}:{port:d}/'.format(
host=env_params.scheduler_host,
port=env_params.scheduler_port,
)
sch = worker_scheduler_factory.create_remote_scheduler(url=url)
worker = worker_scheduler_factory.create_worker(
scheduler=sch,
worker_processes=env_params.workers,
assistant=env_params.assistant)
success = True
logger = logging.getLogger('luigi-interface')
with worker:
for t in tasks:
success &= worker.add(t, env_params.parallel_scheduling,
env_params.parallel_scheduling_processes)
logger.info('Done scheduling tasks')
success &= worker.run()
logger.info(execution_summary.summary(worker))
return dict(success=success, worker=worker)
def _schedule_and_run(tasks, worker_scheduler_factory=None, override_defaults=None):
"""
:param tasks:
:param worker_scheduler_factory:
:param override_defaults:
:return: True if all tasks and their dependencies were successfully run (or already completed);
False if any error occurred.
"""
if worker_scheduler_factory is None:
worker_scheduler_factory = _WorkerSchedulerFactory()
if override_defaults is None:
override_defaults = {}
env_params = core(**override_defaults)
InterfaceLogging.setup(env_params)
kill_signal = signal.SIGUSR1 if env_params.take_lock else None
if (not env_params.no_lock and
not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size, kill_signal))):
raise PidLockAlreadyTakenExit()
if env_params.local_scheduler:
sch = worker_scheduler_factory.create_local_scheduler()
else:
if env_params.scheduler_url != '':
url = env_params.scheduler_url
else:
url = 'http://{host}:{port:d}/'.format(
host=env_params.scheduler_host,
port=env_params.scheduler_port,
)
sch = worker_scheduler_factory.create_remote_scheduler(url=url)
worker = worker_scheduler_factory.create_worker(
scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant)
success = True
logger = logging.getLogger('luigi-interface')
with worker:
for t in tasks:
success &= worker.add(t, env_params.parallel_scheduling, env_params.parallel_scheduling_processes)
logger.info('Done scheduling tasks')
success &= worker.run()
logger.info(execution_summary.summary(worker))
return dict(success=success, worker=worker)
def add_worker(self):
"""Add a worker
"""
if self._local:
scheduler_ = self._worker_scheduler_factory.create_local_scheduler(
)
else:
url = 'http://{host}:{port:d}/'.format(host=self._host,
port=self._port)
scheduler_ = self._worker_scheduler_factory.create_remote_scheduler(
url)
assistant_worker = self._worker_scheduler_factory.create_worker(
scheduler_, self._workers, True)
with assistant_worker:
success = assistant_worker.run()
LOGGER.info(execution_summary.summary(assistant_worker))
if not success:
LOGGER.error('Error running spawn tasks - see logs for details')
