def run(tasks, worker_scheduler_factory=None, override_defaults={}): if worker_scheduler_factory is None: worker_scheduler_factory = WorkerSchedulerFactory() env_params = EnvironmentParamsContainer.env_params(override_defaults) # search for logging configuration path first on the command line, then # in the application config file logging_conf = env_params.logging_conf_file or \ configuration.get_config().get('core', 'logging_conf_file', None) if logging_conf is not None and not os.path.exists(logging_conf): raise Exception("Error: Unable to locate specified logging configuration file!") if not configuration.get_config().getboolean('core', 'no_configure_logging', False): setup_interface_logging(logging_conf) if env_params.lock and not(lock.acquire_for(env_params.lock_pid_dir)): sys.exit(1) if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: sch = worker_scheduler_factory.create_remote_scheduler(host=env_params.scheduler_host, port=env_params.scheduler_port) w = worker_scheduler_factory.create_worker(scheduler=sch, worker_processes=env_params.workers) for task in tasks: w.add(task) logger = logging.getLogger('luigi-interface') logger.info('Done scheduling tasks') w.run() w.stop()
def run(tasks, worker_scheduler_factory=None, override_defaults={}): if worker_scheduler_factory is None: worker_scheduler_factory = WorkerSchedulerFactory() env_params = EnvironmentParamsContainer.env_params(override_defaults) if env_params.lock and not (lock.acquire_for(env_params.lock_pid_dir)): sys.exit(1) if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: sch = worker_scheduler_factory.create_remote_scheduler( host=env_params.scheduler_host, port=env_params.scheduler_port) w = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers) for task in tasks: w.add(task) logger = logging.getLogger('luigi-interface') logger.info('Done scheduling tasks') w.run() w.stop()
def run(tasks, worker_scheduler_factory=None, override_defaults={}): """ :return: True if all tasks and their dependencies were successfully run (or already completed) False if any error occurred """ if worker_scheduler_factory is None: worker_scheduler_factory = WorkerSchedulerFactory() env_params = EnvironmentParamsContainer.env_params(override_defaults) # search for logging configuration path first on the command line, then # in the application config file logging_conf = env_params.logging_conf_file if logging_conf is not None and not os.path.exists(logging_conf): raise Exception( "Error: Unable to locate specified logging configuration file!" ) if not configuration.get_config().getboolean( 'core', 'no_configure_logging', False): setup_interface_logging(logging_conf) if env_params.lock: warnings.warn( "The --lock flag is deprecated and will be removed." "Locking is now the default behavior." "Use --no-lock to override to not use lock", DeprecationWarning ) if (not env_params.no_lock and not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size))): sys.exit(1) if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: sch = worker_scheduler_factory.create_remote_scheduler( host=env_params.scheduler_host, port=env_params.scheduler_port) w = worker_scheduler_factory.create_worker( scheduler=sch, worker_processes=env_params.workers) success = True for t in tasks: success &= w.add(t) logger = logging.getLogger('luigi-interface') logger.info('Done scheduling tasks') success &= w.run() w.stop() return success
def run(tasks, worker_scheduler_factory=None, override_defaults={}): if worker_scheduler_factory is None: worker_scheduler_factory = WorkerSchedulerFactory() env_params = EnvironmentParamsContainer.env_params(override_defaults) if env_params.lock and not(lock.acquire_for(env_params.lock_pid_dir)): sys.exit(1) if env_params.local_scheduler: sch = worker_scheduler_factory.create_local_scheduler() else: sch = worker_scheduler_factory.create_remote_scheduler(host=env_params.scheduler_host, port=env_params.scheduler_port) w = worker_scheduler_factory.create_worker(scheduler=sch, worker_processes=env_params.workers) for task in tasks: w.add(task) logger = logging.getLogger('luigi-interface') logger.info('Done scheduling tasks') w.run() w.stop()