def _schedule_and_run(tasks, worker_scheduler_factory=None, override_defaults=None): """ :param tasks: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. """ if worker_scheduler_factory is None: worker_scheduler_factory = _WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) # search for logging configuration path first on the command line, then # in the application config file logging_conf = env_params.logging_conf_file if logging_conf is not None and not os.path.exists(logging_conf): raise Exception( "Error: Unable to locate specified logging configuration file!") if not configuration.get_config().getboolean( 'core', 'no_configure_logging', False): setup_interface_logging(logging_conf) kill_signal = signal.SIGUSR1 if env_params.take_lock else None if (not env_params.no_lock and not (lock.acquire_for( env_params.lock_pid_dir, env_params.lock_size, kill_signal))): sys.exit(1) if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: if env_params.scheduler_url is not None: url = env_params.scheduler_url else: url = 'http://{host}:{port:d}/'.format( host=env_params.scheduler_host, port=env_params.scheduler_port, ) sch = worker_scheduler_factory.create_remote_scheduler(url=url) w = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant) success = True for t in tasks: success &= w.add(t, env_params.parallel_scheduling) logger = logging.getLogger('luigi-interface') logger.info('Done scheduling tasks') if env_params.workers != 0: success &= w.run() w.stop() logger.info(execution_summary.summary(w)) return success
def _run_namespace_tasks(namespace, tasks=None, worker_scheduler_factory=None, override_defaults=None): """ :param namespace: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. """ if worker_scheduler_factory is None: worker_scheduler_factory = _WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) # search for logging configuration path first on the command line, then # in the application config file logging_conf = env_params.logging_conf_file if logging_conf != '' and not os.path.exists(logging_conf): raise Exception( "Error: Unable to locate specified logging configuration file!" ) if not configuration.get_config().getboolean( 'core', 'no_configure_logging', False): setup_interface_logging(logging_conf, env_params.log_level) kill_signal = signal.SIGUSR1 if env_params.take_lock else None if (not env_params.no_lock and not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size, kill_signal))): raise PidLockAlreadyTakenExit() if env_params.local_scheduler: raise RuntimeError('Cannot use namespace with local scheduler.') else: if env_params.scheduler_url != '': url = env_params.scheduler_url else: url = 'http://{host}:{port:d}/'.format( host=env_params.scheduler_host, port=env_params.scheduler_port, ) sch = worker_scheduler_factory.create_remote_scheduler(url=url) worker = worker_scheduler_factory.create_namespace_worker( scheduler=sch, worker_processes=env_params.workers, namespace=namespace, assistant=env_params.assistant) success = True logger = logging.getLogger('luigi-interface') with worker: if namespace == 'init': for t in tasks: success &= worker.add( t, env_params.parallel_scheduling, env_params.parallel_scheduling_processes) logger.info('Done scheduling tasks') else: logger.info('Worker setup for "{}" namespace'.format(namespace)) success &= worker.run() logger.info(execution_summary.summary(worker)) return dict(success=success, worker=worker)
def _schedule_and_run(tasks, worker_scheduler_factory=None, override_defaults=None): """ :param tasks: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. """ if worker_scheduler_factory is None: worker_scheduler_factory = _WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) # search for logging configuration path first on the command line, then # in the application config file logging_conf = env_params.logging_conf_file if logging_conf is not None and not os.path.exists(logging_conf): raise Exception( "Error: Unable to locate specified logging configuration file!" ) if not configuration.get_config().getboolean( 'core', 'no_configure_logging', False): setup_interface_logging(logging_conf) kill_signal = signal.SIGUSR1 if env_params.take_lock else None if (not env_params.no_lock and not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size, kill_signal))): raise PidLockAlreadyTakenExit() if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: if env_params.scheduler_url is not None: url = env_params.scheduler_url else: url = 'http://{host}:{port:d}/'.format( host=env_params.scheduler_host, port=env_params.scheduler_port, ) sch = worker_scheduler_factory.create_remote_scheduler(url=url) w = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant) success = True for t in tasks: success &= w.add(t, env_params.parallel_scheduling) logger = logging.getLogger('luigi-interface') logger.info('Done scheduling tasks') if env_params.workers != 0: try: success &= w.run() finally: w.stop() logger.info(execution_summary.summary(w)) return dict(success=success, worker=w)
def _schedule_and_run(tasks, worker_scheduler_factory=None, override_defaults=None): """ :param tasks: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. It will return a detailed response of type LuigiRunResult instead of a boolean if detailed_summary=True. """ if worker_scheduler_factory is None: worker_scheduler_factory = _WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) InterfaceLogging.setup(env_params) kill_signal = signal.SIGUSR1 if env_params.take_lock else None if (not env_params.no_lock and not (lock.acquire_for( env_params.lock_pid_dir, env_params.lock_size, kill_signal))): raise PidLockAlreadyTakenExit() if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: if env_params.scheduler_url != '': url = env_params.scheduler_url else: url = 'http://{host}:{port:d}/'.format( host=env_params.scheduler_host, port=env_params.scheduler_port, ) sch = worker_scheduler_factory.create_remote_scheduler(url=url) worker = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant) success = True logger = logging.getLogger('luigi-interface') with worker: for t in tasks: success &= worker.add(t, env_params.parallel_scheduling, env_params.parallel_scheduling_processes) logger.info('Done scheduling tasks') success &= worker.run() luigi_run_result = LuigiRunResult(worker, success) logger.info(luigi_run_result.summary_text) if hasattr(sch, 'close'): sch.close() return luigi_run_result
def run(tasks, worker_scheduler_factory=None, override_defaults=None): """ :param tasks: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. """ if worker_scheduler_factory is None: worker_scheduler_factory = WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) # search for logging configuration path first on the command line, then # in the application config file logging_conf = env_params.logging_conf_file if logging_conf is not None and not os.path.exists(logging_conf): raise Exception( "Error: Unable to locate specified logging configuration file!" ) if not configuration.get_config().getboolean( 'core', 'no_configure_logging', False): setup_interface_logging(logging_conf, context=env_params.log_context) if (not env_params.no_lock and not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size))): sys.exit(1) if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: sch = worker_scheduler_factory.create_remote_scheduler( host=env_params.scheduler_host, port=env_params.scheduler_port) w = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant) success = True logger = logging.getLogger('luigi-interface') try: for t in tasks: success &= w.add(t, env_params.parallel_scheduling) logger.debug('Done scheduling tasks') if env_params.workers != 0: success &= w.run() w.stop() except Exception as e: logger.exception(e) sys.exit(1) return success
def _schedule_and_run(tasks, worker_scheduler_factory=None, override_defaults=None): """ :param tasks: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. It will return a detailed response of type LuigiRunResult instead of a boolean if detailed_summary=True. """ if worker_scheduler_factory is None: worker_scheduler_factory = _WorkerSchedulerFactory() if override_defaults is None: override_defaults = {} env_params = core(**override_defaults) InterfaceLogging.setup(env_params) kill_signal = signal.SIGUSR1 if env_params.take_lock else None if (not env_params.no_lock and not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size, kill_signal))): raise PidLockAlreadyTakenExit() if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: if env_params.scheduler_url != '': url = env_params.scheduler_url else: url = 'http://{host}:{port:d}/'.format( host=env_params.scheduler_host, port=env_params.scheduler_port, ) sch = worker_scheduler_factory.create_remote_scheduler(url=url) worker = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers, assistant=env_params.assistant) success = True logger = logging.getLogger('luigi-interface') with worker: for t in tasks: success &= worker.add(t, env_params.parallel_scheduling, env_params.parallel_scheduling_processes) logger.info('Done scheduling tasks') success &= worker.run() luigi_run_result = LuigiRunResult(worker, success) logger.info(luigi_run_result.summary_text) return luigi_run_result
def run(tasks, worker_scheduler_factory=None, override_defaults=None): """ :param tasks: :param worker_scheduler_factory: :param override_defaults: :return: True if all tasks and their dependencies were successfully run (or already completed); False if any error occurred. """ if worker_scheduler_factory is None: worker_scheduler_factory = WorkerSchedulerFactory() env_params = core(**override_defaults) # search for logging configuration path first on the command line, then # in the application config file logging_conf = env_params.logging_conf_file if logging_conf is not None and not os.path.exists(logging_conf): raise Exception( "Error: Unable to locate specified logging configuration file!" ) if not configuration.get_config().getboolean( 'core', 'no_configure_logging', False): setup_interface_logging(logging_conf) if (not env_params.no_lock and not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size))): sys.exit(1) if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: sch = worker_scheduler_factory.create_remote_scheduler( host=env_params.scheduler_host, port=env_params.scheduler_port) w = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers) success = True for t in tasks: success &= w.add(t, env_params.parallel_scheduling) logger = logging.getLogger('luigi-interface') logger.info('Done scheduling tasks') success &= w.run() w.stop() return success