class cors(Config):
enabled = parameter.BoolParameter(
default=False,
description='Enables CORS support.')
allowed_origins = parameter.ListParameter(
default=[],
description='A list of allowed origins. Used only if `allow_any_origin` is false.')
allow_any_origin = parameter.BoolParameter(
default=False,
description='Accepts requests from any origin.')
allow_null_origin = parameter.BoolParameter(
default=False,
description='Allows the request to set `null` value of the `Origin` header.')
max_age = parameter.IntParameter(
default=86400,
description='Content of `Access-Control-Max-Age`.')
allowed_methods = parameter.Parameter(
default='GET, OPTIONS',
description='Content of `Access-Control-Allow-Methods`.')
allowed_headers = parameter.Parameter(
default='Accept, Content-Type, Origin',
description='Content of `Access-Control-Allow-Headers`.')
exposed_headers = parameter.Parameter(
default='',
description='Content of `Access-Control-Expose-Headers`.')
allow_credentials = parameter.BoolParameter(
default=False,
description='Indicates that the actual request can include user credentials.')
def __init__(self, *args, **kwargs):
super(cors, self).__init__(*args, **kwargs)
self.allowed_origins = set(i for i in self.allowed_origins if i not in ['*', 'null'])
class scheduler(Config):
# TODO(erikbern): the config_path is needed for backwards compatilibity. We should drop the compatibility
# at some point (in particular this would force users to replace all dashes with underscores in the config)
retry_delay = parameter.FloatParameter(default=900.0)
remove_delay = parameter.FloatParameter(default=600.0)
worker_disconnect_delay = parameter.FloatParameter(default=60.0)
state_path = parameter.Parameter(
default='/var/lib/luigi-server/state.pickle')
# Jobs are disabled if we see more than disable_failures failures in disable_window seconds.
# These disables last for disable_persist seconds.
disable_window = parameter.IntParameter(default=3600,
config_path=dict(
section='scheduler',
name='disable-window-seconds'))
disable_failures = parameter.IntParameter(default=None,
config_path=dict(
section='scheduler',
name='disable-num-failures'))
disable_hard_timeout = parameter.IntParameter(
default=None,
config_path=dict(section='scheduler', name='disable-hard-timeout'))
disable_persist = parameter.IntParameter(
default=86400,
config_path=dict(section='scheduler', name='disable-persist-seconds'))
max_shown_tasks = parameter.IntParameter(default=100000)
prune_done_tasks = parameter.BoolParameter(default=False)
record_task_history = parameter.BoolParameter(default=False)
visualization_graph = parameter.Parameter(default="svg",
config_path=dict(
section='scheduler',
name='visualization-graph'))
class core(task.Config):
''' Keeps track of a bunch of environment params.
Uses the internal luigi parameter mechanism.
The nice thing is that we can instantiate this class
and get an object with all the environment variables set.
This is arguably a bit of a hack.
'''
use_cmdline_section = False
local_scheduler = parameter.BoolParameter(
default=False, description='Use local scheduling')
scheduler_host = parameter.Parameter(
default='localhost',
description='Hostname of machine running remote scheduler',
config_path=dict(section='core', name='default-scheduler-host'))
scheduler_port = parameter.IntParameter(
default=8082,
description='Port of remote scheduler api process',
config_path=dict(section='core', name='default-scheduler-port'))
scheduler_url = parameter.Parameter(
default=None,
description='Full path to remote scheduler',
config_path=dict(section='core', name='default-scheduler-url'),
)
lock_size = parameter.IntParameter(
default=1,
description="Maximum number of workers running the same command")
no_lock = parameter.BoolParameter(
default=False,
description='Ignore if similar process is already running')
lock_pid_dir = parameter.Parameter(
default=os.path.join(tempfile.gettempdir(), 'luigi'),
description='Directory to store the pid file')
take_lock = parameter.BoolParameter(
default=False,
description=
'Signal other processes to stop getting work if already running')
workers = parameter.IntParameter(
default=1, description='Maximum number of parallel tasks to run')
logging_conf_file = parameter.Parameter(
default=None, description='Configuration file for logging')
module = parameter.Parameter(
default=None, description='Used for dynamic loading of modules')
parallel_scheduling = parameter.BoolParameter(
default=False,
description='Use multiprocessing to do scheduling in parallel.')
assistant = parameter.BoolParameter(
default=False, description='Run any task from the scheduler.')
help = parameter.BoolParameter(default=False,
description='Help option flag, for --help')
class scheduler(Config):
# TODO(erikbern): the config_path is needed for backwards compatilibity. We
# should drop the compatibility at some point
retry_delay = parameter.FloatParameter(default=900.0)
remove_delay = parameter.FloatParameter(default=600.0)
worker_disconnect_delay = parameter.FloatParameter(default=60.0)
state_path = parameter.Parameter(
default='/var/lib/luigi-server/state.pickle')
# Jobs are disabled if we see more than disable_failures failures in disable_window seconds.
# These disables last for disable_persist seconds.
disable_window = parameter.IntParameter(default=3600,
config_path=dict(
section='scheduler',
name='disable-window-seconds'))
disable_failures = parameter.IntParameter(default=999999999,
config_path=dict(
section='scheduler',
name='disable-num-failures'))
disable_hard_timeout = parameter.IntParameter(
default=999999999,
config_path=dict(section='scheduler', name='disable-hard-timeout'))
disable_persist = parameter.IntParameter(
default=86400,
config_path=dict(section='scheduler', name='disable-persist-seconds'))
max_shown_tasks = parameter.IntParameter(default=100000)
max_graph_nodes = parameter.IntParameter(default=100000)
record_task_history = parameter.BoolParameter(default=False)
prune_on_get_work = parameter.BoolParameter(default=False)
class core(task.ConfigWithoutSection):
''' Keeps track of a bunch of environment params.
Uses the internal luigi parameter mechanism.
The nice thing is that we can instantiate this class
and get an object with all the environment variables set.
This is arguably a bit of a hack.
'''
local_scheduler = parameter.BoolParameter(
default=False,
description='Use local scheduling')
scheduler_host = parameter.Parameter(
default='localhost',
description='Hostname of machine running remote scheduler',
config_path=dict(section='core', name='default-scheduler-host'))
scheduler_port = parameter.IntParameter(
default=8082,
description='Port of remote scheduler api process',
config_path=dict(section='core', name='default-scheduler-port'))
lock_size = parameter.IntParameter(
default=1,
description="Maximum number of workers running the same command")
no_lock = parameter.BoolParameter(
default=False,
description='Ignore if similar process is already running')
lock_pid_dir = parameter.Parameter(
default=os.path.join(tempfile.gettempdir(), 'luigi'),
description='Directory to store the pid file')
workers = parameter.IntParameter(
default=1,
description='Maximum number of parallel tasks to run')
logging_conf_file = parameter.Parameter(
default=None,
description='Configuration file for logging')
module = parameter.Parameter(
default=None,
description='Used for dynamic loading of modules') # see DynamicArgParseInterface
parallel_scheduling = parameter.BoolParameter(
default=False,
description='Use multiprocessing to do scheduling in parallel.',
config_path={'section': 'core', 'name': 'parallel-scheduling'},
)
class datadog(Config):
api_key = parameter.Parameter(default='dummy_api_key',
description='API key provided by Datadog')
app_key = parameter.Parameter(default='dummy_app_key',
description='APP key provided by Datadog')
default_tags = parameter.Parameter(
default='application:luigi',
description='Default tags for every events and metrics sent to Datadog'
)
environment = parameter.Parameter(
default='development',
description=
"Environment of which the pipeline is ran from (eg: 'production', 'staging', ..."
)
metric_namespace = parameter.Parameter(
default='luigi',
description=
"Default namespace for events and metrics (eg: 'luigi' for 'luigi.task.started')"
)
statsd_host = parameter.Parameter(
default='localhost',
description='StatsD host implementing the Datadog service')
statsd_port = parameter.IntParameter(
default=8125,
description='StatsD port implementing the Datadog service')
class core(task.Config):
''' Keeps track of a bunch of environment params.
Uses the internal luigi parameter mechanism.
The nice thing is that we can instantiate this class
and get an object with all the environment variables set.
This is arguably a bit of a hack.
'''
use_cmdline_section = False
local_scheduler = parameter.BoolParameter(
default=False,
description='Use an in-memory central scheduler. Useful for testing.',
always_in_help=True)
scheduler_host = parameter.Parameter(
default='localhost',
description='Hostname of machine running remote scheduler',
config_path=dict(section='core', name='default-scheduler-host'))
scheduler_port = parameter.IntParameter(
default=8082,
description='Port of remote scheduler api process',
config_path=dict(section='core', name='default-scheduler-port'))
scheduler_url = parameter.Parameter(
default='',
description='Full path to remote scheduler',
config_path=dict(section='core', name='default-scheduler-url'),
)
lock_size = parameter.IntParameter(
default=1,
description="Maximum number of workers running the same command")
no_lock = parameter.BoolParameter(
default=False,
description='Ignore if similar process is already running')
lock_pid_dir = parameter.Parameter(
default=os.path.join(tempfile.gettempdir(), 'luigi'),
description='Directory to store the pid file')
take_lock = parameter.BoolParameter(
default=False,
description='Signal other processes to stop getting work if already running')
workers = parameter.IntParameter(
default=1,
description='Maximum number of parallel tasks to run')
logging_conf_file = parameter.Parameter(
default='',
description='Configuration file for logging')
log_level = parameter.ChoiceParameter(
default='DEBUG',
choices=['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
description="Default log level to use when logging_conf_file is not set")
module = parameter.Parameter(
default='',
description='Used for dynamic loading of modules',
always_in_help=True)
parallel_scheduling = parameter.BoolParameter(
default=False,
description='Use multiprocessing to do scheduling in parallel.')
parallel_scheduling_processes = parameter.IntParameter(
default=0,
description='The number of processes to use for scheduling in parallel.'
' By default the number of available CPUs will be used')
assistant = parameter.BoolParameter(
default=False,
description='Run any task from the scheduler.')
help = parameter.BoolParameter(
default=False,
description='Show most common flags and all task-specific flags',
always_in_help=True)
help_all = parameter.BoolParameter(
default=False,
description='Show all command line flags',
always_in_help=True)
class upstream(luigi.task.Config):
'''
Used to provide the parameter upstream-family
'''
family = parameter.Parameter(default=None)
