def _run_namespace_tasks(namespace, tasks=None, worker_scheduler_factory=None, override_defaults=None): """ :param namespace: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. """ if worker_scheduler_factory is None: worker_scheduler_factory = _WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) # search for logging configuration path first on the command line, then # in the application config file logging_conf = env_params.logging_conf_file if logging_conf != '' and not os.path.exists(logging_conf): raise Exception( "Error: Unable to locate specified logging configuration file!" ) if not configuration.get_config().getboolean( 'core', 'no_configure_logging', False): setup_interface_logging(logging_conf, env_params.log_level) kill_signal = signal.SIGUSR1 if env_params.take_lock else None if (not env_params.no_lock and not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size, kill_signal))): raise PidLockAlreadyTakenExit() if env_params.local_scheduler: raise RuntimeError('Cannot use namespace with local scheduler.') else: if env_params.scheduler_url != '': url = env_params.scheduler_url else: url = 'http://{host}:{port:d}/'.format( host=env_params.scheduler_host, port=env_params.scheduler_port, ) sch = worker_scheduler_factory.create_remote_scheduler(url=url) worker = worker_scheduler_factory.create_namespace_worker( scheduler=sch, worker_processes=env_params.workers, namespace=namespace, assistant=env_params.assistant) success = True logger = logging.getLogger('luigi-interface') with worker: if namespace == 'init': for t in tasks: success &= worker.add( t, env_params.parallel_scheduling, env_params.parallel_scheduling_processes) logger.info('Done scheduling tasks') else: logger.info('Worker setup for "{}" namespace'.format(namespace)) success &= worker.run() logger.info(execution_summary.summary(worker)) return dict(success=success, worker=worker)
def _schedule_and_run(tasks, worker_scheduler_factory=None, override_defaults=None): """ :param tasks: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. """ if worker_scheduler_factory is None: worker_scheduler_factory = _WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) # search for logging configuration path first on the command line, then # in the application config file logging_conf = env_params.logging_conf_file if logging_conf is not None and not os.path.exists(logging_conf): raise Exception( "Error: Unable to locate specified logging configuration file!") if not configuration.get_config().getboolean( 'core', 'no_configure_logging', False): setup_interface_logging(logging_conf) kill_signal = signal.SIGUSR1 if env_params.take_lock else None if (not env_params.no_lock and not (lock.acquire_for( env_params.lock_pid_dir, env_params.lock_size, kill_signal))): sys.exit(1) if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: if env_params.scheduler_url is not None: url = env_params.scheduler_url else: url = 'http://{host}:{port:d}/'.format( host=env_params.scheduler_host, port=env_params.scheduler_port, ) sch = worker_scheduler_factory.create_remote_scheduler(url=url) w = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant) success = True for t in tasks: success &= w.add(t, env_params.parallel_scheduling) logger = logging.getLogger('luigi-interface') logger.info('Done scheduling tasks') if env_params.workers != 0: success &= w.run() w.stop() logger.info(execution_summary.summary(w)) return success
def _schedule_and_run(tasks, worker_scheduler_factory=None, override_defaults=None): """ :param tasks: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. """ if worker_scheduler_factory is None: worker_scheduler_factory = _WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) # search for logging configuration path first on the command line, then # in the application config file logging_conf = env_params.logging_conf_file if logging_conf is not None and not os.path.exists(logging_conf): raise Exception( "Error: Unable to locate specified logging configuration file!" ) if not configuration.get_config().getboolean( 'core', 'no_configure_logging', False): setup_interface_logging(logging_conf) kill_signal = signal.SIGUSR1 if env_params.take_lock else None if (not env_params.no_lock and not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size, kill_signal))): raise PidLockAlreadyTakenExit() if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: if env_params.scheduler_url is not None: url = env_params.scheduler_url else: url = 'http://{host}:{port:d}/'.format( host=env_params.scheduler_host, port=env_params.scheduler_port, ) sch = worker_scheduler_factory.create_remote_scheduler(url=url) w = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant) success = True for t in tasks: success &= w.add(t, env_params.parallel_scheduling) logger = logging.getLogger('luigi-interface') logger.info('Done scheduling tasks') if env_params.workers != 0: try: success &= w.run() finally: w.stop() logger.info(execution_summary.summary(w)) return dict(success=success, worker=w)
def _schedule_and_run(tasks, worker_scheduler_factory=None, override_defaults=None): """ :param tasks: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. """ if worker_scheduler_factory is None: worker_scheduler_factory = _WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) InterfaceLogging.setup(env_params) kill_signal = signal.SIGUSR1 if env_params.take_lock else None if (not env_params.no_lock and not (lock.acquire_for( env_params.lock_pid_dir, env_params.lock_size, kill_signal))): raise PidLockAlreadyTakenExit() if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: if env_params.scheduler_url != '': url = env_params.scheduler_url else: url = 'http://{host}:{port:d}/'.format( host=env_params.scheduler_host, port=env_params.scheduler_port, ) sch = worker_scheduler_factory.create_remote_scheduler(url=url) worker = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant) success = True logger = logging.getLogger('luigi-interface') with worker: for t in tasks: success &= worker.add(t, env_params.parallel_scheduling, env_params.parallel_scheduling_processes) logger.info('Done scheduling tasks') success &= worker.run() logger.info(execution_summary.summary(worker)) return dict(success=success, worker=worker)
def _schedule_and_run(tasks, worker_scheduler_factory=None, override_defaults=None): """ :param tasks: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. """ if worker_scheduler_factory is None: worker_scheduler_factory = _WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) InterfaceLogging.setup(env_params) kill_signal = signal.SIGUSR1 if env_params.take_lock else None if (not env_params.no_lock and not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size, kill_signal))): raise PidLockAlreadyTakenExit() if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: if env_params.scheduler_url != '': url = env_params.scheduler_url else: url = 'http://{host}:{port:d}/'.format( host=env_params.scheduler_host, port=env_params.scheduler_port, ) sch = worker_scheduler_factory.create_remote_scheduler(url=url) worker = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant) success = True logger = logging.getLogger('luigi-interface') with worker: for t in tasks: success &= worker.add(t, env_params.parallel_scheduling, env_params.parallel_scheduling_processes) logger.info('Done scheduling tasks') success &= worker.run() logger.info(execution_summary.summary(worker)) return dict(success=success, worker=worker)
def add_worker(self): """Add a worker """ if self._local: scheduler_ = self._worker_scheduler_factory.create_local_scheduler( ) else: url = 'http://{host}:{port:d}/'.format(host=self._host, port=self._port) scheduler_ = self._worker_scheduler_factory.create_remote_scheduler( url) assistant_worker = self._worker_scheduler_factory.create_worker( scheduler_, self._workers, True) with assistant_worker: success = assistant_worker.run() LOGGER.info(execution_summary.summary(assistant_worker)) if not success: LOGGER.error('Error running spawn tasks - see logs for details')