class worker(Config): ping_interval = FloatParameter(default=1.0, config_path=dict( section='core', name='worker-ping-interval')) keep_alive = BoolParameter(default=False, config_path=dict(section='core', name='worker-keep-alive')) count_uniques = BoolParameter( default=False, config_path=dict(section='core', name='worker-count-uniques'), description='worker-count-uniques means that we will keep a ' 'worker alive only if it has a unique pending task, as ' 'well as having keep-alive true') wait_interval = FloatParameter(default=1.0, config_path=dict( section='core', name='worker-wait-interval')) max_reschedules = IntParameter(default=1, config_path=dict( section='core', name='worker-max-reschedules')) timeout = IntParameter(default=0, config_path=dict(section='core', name='worker-timeout')) task_limit = IntParameter(default=None, config_path=dict(section='core', name='worker-task-limit')) retry_external_tasks = BoolParameter( default=False, config_path=dict(section='core', name='retry-external-tasks'), description='If true, incomplete external tasks will be ' 'retested for completion while Luigi is running.')
class worker(Config): # NOTE: `section.config-variable` in the config_path argument is deprecated in favor of `worker.config_variable` ping_interval = FloatParameter(default=1.0, config_path=dict(section='core', name='worker-ping-interval')) keep_alive = BoolParameter(default=False, config_path=dict(section='core', name='worker-keep-alive')) count_uniques = BoolParameter(default=False, config_path=dict(section='core', name='worker-count-uniques'), description='worker-count-uniques means that we will keep a ' 'worker alive only if it has a unique pending task, as ' 'well as having keep-alive true') wait_interval = FloatParameter(default=1.0, config_path=dict(section='core', name='worker-wait-interval')) wait_jitter = FloatParameter(default=5.0) max_reschedules = IntParameter(default=1, config_path=dict(section='core', name='worker-max-reschedules')) timeout = IntParameter(default=0, config_path=dict(section='core', name='worker-timeout')) task_limit = IntParameter(default=None, config_path=dict(section='core', name='worker-task-limit')) retry_external_tasks = BoolParameter(default=False, config_path=dict(section='core', name='retry-external-tasks'), description='If true, incomplete external tasks will be ' 'retested for completion while Luigi is running.') no_install_shutdown_handler = BoolParameter(default=False, description='If true, the SIGUSR1 shutdown handler will' 'NOT be install on the worker')
class worker(Config): # NOTE: `section.config-variable` in the config_path argument is deprecated in favor of `worker.config_variable` ping_interval = FloatParameter(default=1.0, config_path=dict( section='core', name='worker-ping-interval')) keep_alive = BoolParameter(default=False, config_path=dict(section='core', name='worker-keep-alive')) count_uniques = BoolParameter( default=False, config_path=dict(section='core', name='worker-count-uniques'), description='worker-count-uniques means that we will keep a ' 'worker alive only if it has a unique pending task, as ' 'well as having keep-alive true') count_last_scheduled = BoolParameter( default=False, description='Keep a worker alive only if there are ' 'pending tasks which it was the last to ' 'schedule.') wait_interval = FloatParameter(default=1.0, config_path=dict( section='core', name='worker-wait-interval')) wait_jitter = FloatParameter(default=5.0) max_reschedules = IntParameter(default=1, config_path=dict( section='core', name='worker-max-reschedules')) timeout = IntParameter(default=0, config_path=dict(section='core', name='worker-timeout')) task_limit = IntParameter(default=None, config_path=dict(section='core', name='worker-task-limit')) retry_external_tasks = BoolParameter( default=False, config_path=dict(section='core', name='retry-external-tasks'), description='If true, incomplete external tasks will be ' 'retested for completion while Luigi is running.') send_failure_email = BoolParameter( default=True, description='If true, send e-mails directly from the worker' 'on failure') no_install_shutdown_handler = BoolParameter( default=False, description='If true, the SIGUSR1 shutdown handler will' 'NOT be install on the worker') check_unfulfilled_deps = BoolParameter( default=True, description='If true, check for completeness of ' 'dependencies before running a task')
class worker(Config): # NOTE: `section.config-variable` in the config_path argument is deprecated in favor of `worker.config_variable` ping_interval = FloatParameter(default=1.0, config_path=dict(section='core', name='worker-ping-interval')) keep_alive = BoolParameter(default=False, config_path=dict(section='core', name='worker-keep-alive')) count_uniques = BoolParameter(default=False, config_path=dict(section='core', name='worker-count-uniques'), description='worker-count-uniques means that we will keep a ' 'worker alive only if it has a unique pending task, as ' 'well as having keep-alive true') count_last_scheduled = BoolParameter(default=False, description='Keep a worker alive only if there are ' 'pending tasks which it was the last to ' 'schedule.') wait_interval = FloatParameter(default=1.0, config_path=dict(section='core', name='worker-wait-interval')) wait_jitter = FloatParameter(default=5.0) max_reschedules = IntParameter(default=1, config_path=dict(section='core', name='worker-max-reschedules')) timeout = IntParameter(default=0, config_path=dict(section='core', name='worker-timeout')) task_limit = IntParameter(default=None, config_path=dict(section='core', name='worker-task-limit')) retry_external_tasks = BoolParameter(default=False, config_path=dict(section='core', name='retry-external-tasks'), description='If true, incomplete external tasks will be ' 'retested for completion while Luigi is running.') send_failure_email = BoolParameter(default=True, description='If true, send e-mails directly from the worker' 'on failure') no_install_shutdown_handler = BoolParameter(default=False, description='If true, the SIGUSR1 shutdown handler will' 'NOT be install on the worker') check_unfulfilled_deps = BoolParameter(default=True, description='If true, check for completeness of ' 'dependencies before running a task') force_multiprocessing = BoolParameter(default=False, description='If true, use multiprocessing also when ' 'running with 1 worker') task_process_context = OptionalParameter(default=None, description='If set to a fully qualified class name, the class will ' 'be instantiated with a TaskProcess as its constructor parameter and ' 'applied as a context manager around its run() call, so this can be ' 'used for obtaining high level customizable monitoring or logging of ' 'each individual Task run.')
class worker(Config): ping_interval = FloatParameter(default=1.0, config_path=dict(section='core', name='retry-delay')) keep_alive = BoolParameter(default=False, config_path=dict(section='core', name='worker-keep-alive')) count_uniques = BoolParameter( default=False, config_path=dict(section='core', name='worker-count-uniques'), description='worker-count-uniques means that we will keep a ' 'worker alive only if it has a unique pending task, as ' 'well as having keep-alive true') wait_interval = IntParameter(default=1, config_path=dict(section='core', name='worker-wait-interval')) max_reschedules = IntParameter(default=1, config_path=dict( section='core', name='worker-max-reschedules')) timeout = IntParameter(default=0, config_path=dict(section='core', name='worker-timeout')) task_limit = IntParameter(default=None, config_path=dict(section='core', name='worker-task-limit'))
class ThePayneMixin(SlurmMixin, BaseTask): task_namespace = "ThePayne" n_steps = IntParameter( default=100000, config_path=dict(section=task_namespace, name="n_steps") ) n_neurons = IntParameter( default=300, config_path=dict(section=task_namespace, name="n_neurons") ) weight_decay = FloatParameter( default=0.0, config_path=dict(section=task_namespace, name="weight_decay") ) learning_rate = FloatParameter( default=0.001, config_path=dict(section=task_namespace, name="learning_rate") ) training_set_path = Parameter( config_path=dict(section=task_namespace, name="training_set_path") )
class TrainTheCannonBase(TheCannonMixin): """ A base task for training The Cannon. :param label_names: A list of label names. :param order: (optional) The polynomial order to use for this model (default: 2). :param regularization: (optional) The strength of L1-regularization to apply during training. :param threads: (optional) The number of threads to use (default: 1). :param plot: (optional) A boolean flag to indicate whether to produce post-training quality plots. """ regularization = FloatParameter(default=0.0) threads = IntParameter(default=1, significant=False) plot = BoolParameter(default=True, significant=False) def run(self): """ Execute this task. """ # Load training set labels and spectra. labels, dispersion, training_set_flux, training_set_ivar = read_training_set( self.input().path, ) # Set the vectorizer. # We sort the label names so that luigi doesn't re-train models if we alter the order. vectorizer = tc.vectorizer.PolynomialVectorizer( sorted(self.label_names), self.order) # Initiate model. model = tc.model.CannonModel(labels, training_set_flux, training_set_ivar, vectorizer=vectorizer, dispersion=dispersion, regularization=self.regularization) log.info(f"Training The Cannon model {model}") model.train(threads=self.threads) output_path = self.output().path log.info(f"Writing The Cannon model {model} to disk {output_path}") model.write(output_path) if self.plot: # Plot zeroth and first order coefficients. fig = plot.theta( model, indices=np.arange(1 + len(model.vectorizer.label_names)), normalize=False) fig.savefig(f"{self.task_id}-theta.png") # Plot scatter. fig = plot.scatter(model) fig.savefig(f"{self.task_id}-scatter.png") # Plot one-to-one. test_labels, test_cov, test_meta = model.test( training_set_flux, training_set_ivar, initial_labels=model.training_set_labels) fig = plot.one_to_one(model, test_labels, cov=test_cov) fig.savefig(f"{self.task_id}-one-to-one.png") def output(self): """ The output of this task. """ return LocalTarget( os.path.join(self.output_base_dir, f"{self.task_id}.pkl"))