Example #1
0
    def run(tasks, worker_scheduler_factory=None, override_defaults={}):

        if worker_scheduler_factory is None:
            worker_scheduler_factory = WorkerSchedulerFactory()

        env_params = EnvironmentParamsContainer.env_params(override_defaults)
        # search for logging configuration path first on the command line, then
        # in the application config file
        logging_conf = env_params.logging_conf_file or \
            configuration.get_config().get('core', 'logging_conf_file', None)
        if logging_conf is not None and not os.path.exists(logging_conf):
            raise Exception("Error: Unable to locate specified logging configuration file!")

        if not configuration.get_config().getboolean('core', 'no_configure_logging', False):
            setup_interface_logging(logging_conf)

        if env_params.lock and not(lock.acquire_for(env_params.lock_pid_dir)):
            sys.exit(1)

        if env_params.local_scheduler:
            sch = worker_scheduler_factory.create_local_scheduler()
        else:
            sch = worker_scheduler_factory.create_remote_scheduler(host=env_params.scheduler_host, port=env_params.scheduler_port)

        w = worker_scheduler_factory.create_worker(scheduler=sch, worker_processes=env_params.workers)

        for task in tasks:
            w.add(task)
        logger = logging.getLogger('luigi-interface')
        logger.info('Done scheduling tasks')
        w.run()
        w.stop()
Example #2
0
    def run(tasks, worker_scheduler_factory=None, override_defaults={}):

        if worker_scheduler_factory is None:
            worker_scheduler_factory = WorkerSchedulerFactory()

        env_params = EnvironmentParamsContainer.env_params(override_defaults)

        if env_params.lock and not (lock.acquire_for(env_params.lock_pid_dir)):
            sys.exit(1)

        if env_params.local_scheduler:
            sch = worker_scheduler_factory.create_local_scheduler()
        else:
            sch = worker_scheduler_factory.create_remote_scheduler(
                host=env_params.scheduler_host, port=env_params.scheduler_port)

        w = worker_scheduler_factory.create_worker(
            scheduler=sch, worker_processes=env_params.workers)

        for task in tasks:
            w.add(task)
        logger = logging.getLogger('luigi-interface')
        logger.info('Done scheduling tasks')
        w.run()
        w.stop()
Example #3
0
    def run(tasks, worker_scheduler_factory=None, override_defaults={}):
        """
        :return: True if all tasks and their dependencies were successfully run (or already completed)
        False if any error occurred
        """

        if worker_scheduler_factory is None:
            worker_scheduler_factory = WorkerSchedulerFactory()
        env_params = EnvironmentParamsContainer.env_params(override_defaults)
        # search for logging configuration path first on the command line, then
        # in the application config file
        logging_conf = env_params.logging_conf_file
        if logging_conf is not None and not os.path.exists(logging_conf):
            raise Exception(
                "Error: Unable to locate specified logging configuration file!"
            )

        if not configuration.get_config().getboolean(
                'core', 'no_configure_logging', False):
            setup_interface_logging(logging_conf)

        if env_params.lock:
            warnings.warn(
                "The --lock flag is deprecated and will be removed."
                "Locking is now the default behavior."
                "Use --no-lock to override to not use lock",
                DeprecationWarning
            )

        if (not env_params.no_lock and
                not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size))):
            sys.exit(1)

        if env_params.local_scheduler:
            sch = worker_scheduler_factory.create_local_scheduler()
        else:
            sch = worker_scheduler_factory.create_remote_scheduler(
                host=env_params.scheduler_host,
                port=env_params.scheduler_port)

        w = worker_scheduler_factory.create_worker(
            scheduler=sch, worker_processes=env_params.workers)

        success = True
        for t in tasks:
            success &= w.add(t)
        logger = logging.getLogger('luigi-interface')
        logger.info('Done scheduling tasks')
        success &= w.run()
        w.stop()
        return success
Example #4
0
    def run(tasks, worker_scheduler_factory=None, override_defaults={}):
        """
        :return: True if all tasks and their dependencies were successfully run (or already completed)
        False if any error occurred
        """

        if worker_scheduler_factory is None:
            worker_scheduler_factory = WorkerSchedulerFactory()
        env_params = EnvironmentParamsContainer.env_params(override_defaults)
        # search for logging configuration path first on the command line, then
        # in the application config file
        logging_conf = env_params.logging_conf_file
        if logging_conf is not None and not os.path.exists(logging_conf):
            raise Exception(
                "Error: Unable to locate specified logging configuration file!"
            )

        if not configuration.get_config().getboolean(
                'core', 'no_configure_logging', False):
            setup_interface_logging(logging_conf)

        if env_params.lock:
            warnings.warn(
                "The --lock flag is deprecated and will be removed."
                "Locking is now the default behavior."
                "Use --no-lock to override to not use lock",
                DeprecationWarning
            )

        if (not env_params.no_lock and
                not(lock.acquire_for(env_params.lock_pid_dir, env_params.lock_size))):
            sys.exit(1)

        if env_params.local_scheduler:
            sch = worker_scheduler_factory.create_local_scheduler()
        else:
            sch = worker_scheduler_factory.create_remote_scheduler(
                host=env_params.scheduler_host,
                port=env_params.scheduler_port)

        w = worker_scheduler_factory.create_worker(
            scheduler=sch, worker_processes=env_params.workers)

        success = True
        for t in tasks:
            success &= w.add(t)
        logger = logging.getLogger('luigi-interface')
        logger.info('Done scheduling tasks')
        success &= w.run()
        w.stop()
        return success
Example #5
0
    def run(tasks, worker_scheduler_factory=None, override_defaults={}):

        if worker_scheduler_factory is None:
            worker_scheduler_factory = WorkerSchedulerFactory()

        env_params = EnvironmentParamsContainer.env_params(override_defaults)

        if env_params.lock and not(lock.acquire_for(env_params.lock_pid_dir)):
            sys.exit(1)

        if env_params.local_scheduler:
            sch = worker_scheduler_factory.create_local_scheduler()
        else:
            sch = worker_scheduler_factory.create_remote_scheduler(host=env_params.scheduler_host, port=env_params.scheduler_port)

        w = worker_scheduler_factory.create_worker(scheduler=sch, worker_processes=env_params.workers)

        for task in tasks:
            w.add(task)
        logger = logging.getLogger('luigi-interface')
        logger.info('Done scheduling tasks')
        w.run()
        w.stop()