def submit(clusters, args):
"""
Submits a job (or multiple jobs) to cook scheduler. Assembles a list of jobs,
potentially getting data from configuration, the command line, and stdin.
"""
logging.debug('submit args: %s' % args)
job = args
raw = job.pop('raw', None)
command_from_command_line = job.pop('command', None)
application_name = job.pop('application_name', 'cook-scheduler-cli')
application_version = job.pop(
'application_version',
pkg_resources.require('cook_client')[0].version)
job['application'] = {
'name': application_name,
'version': application_version
}
if raw:
if command_from_command_line:
raise Exception(
'You cannot specify a command at the command line when using --raw/-r.'
)
jobs_json = read_jobs_from_stdin()
jobs = parse_raw_job_spec(job, jobs_json)
else:
commands = acquire_commands(command_from_command_line)
if job.get('uuid') and len(commands) > 1:
raise Exception(
'You cannot specify multiple subcommands with a single UUID.')
if job.get('env'):
job['env'] = dict([e.split('=', maxsplit=1) for e in job['env']])
jobs = [deep_merge(job, {'command': c}) for c in commands]
for j in jobs:
if not j.get('uuid'):
j['uuid'] = str(uuid.uuid4())
if not j.get('name'):
j['name'] = '%s_job' % current_user()
logging.debug('jobs: %s' % jobs)
return submit_federated(clusters, jobs)
def register(add_parser, add_defaults):
"""Adds this sub-command's parser and returns the action function"""
parser = add_parser(
'usage', help='show breakdown of usage by application and group')
parser.add_argument('--user', '-u', help='show usage for a user')
parser.add_argument('--pool',
'-p',
action='append',
help='filter by pool (can be repeated)')
parser.add_argument('--json',
help='show the data in JSON format',
dest='json',
action='store_true')
add_defaults('usage', {'user': current_user()})
return usage
def register(add_parser, add_defaults):
"""Adds this sub-command's parser and returns the action function"""
list_parser = add_parser('list',
help='list jobs by state / user / time / name')
list_parser.add_argument('--state',
'-s',
help='list jobs by status (can be repeated)',
action='append',
choices=('waiting', 'running', 'completed',
'failed', 'success', 'all'))
list_parser.add_argument('--user', '-u', help='list jobs for a user')
list_parser.add_argument('--lookback',
'-t',
help='list jobs for the last X hours',
type=float)
list_parser.add_argument(
'--name',
'-n',
help=
"list jobs with a particular name pattern (name filters can contain "
"alphanumeric characters, '.', '-', '_', and '*' as a wildcard)")
list_parser.add_argument('--limit',
'-l',
help='limit the number of results',
type=check_positive)
list_parser.add_argument('--json',
help='show the data in JSON format',
dest='json',
action='store_true')
add_defaults('list', {
'state': ['running'],
'user': current_user(),
'lookback': 6,
'limit': 150
})
return list_jobs
def register(add_parser, add_defaults):
"""Adds this sub-command's parser and returns the action function"""
parser = add_parser('jobs', help='list jobs by state / user / time / name')
parser.add_argument('--waiting',
'-w',
help='include waiting jobs',
dest='states',
action='append_const',
const='waiting')
parser.add_argument('--running',
'-r',
help='include running jobs',
dest='states',
action='append_const',
const='running')
parser.add_argument('--completed',
'-c',
help='include completed jobs',
dest='states',
action='append_const',
const='completed')
parser.add_argument('--failed',
'-f',
help='include failed jobs',
dest='states',
action='append_const',
const='failed')
parser.add_argument('--success',
'-s',
help='include successful jobs',
dest='states',
action='append_const',
const='success')
parser.add_argument('--all',
'-a',
help='include all jobs, regardless of status',
dest='states',
action='append_const',
const='all')
parser.add_argument('--user', '-u', help='list jobs for a user')
parser.add_argument('--pool', '-P', help='list jobs for a pool')
parser.add_argument(
'--lookback',
'-t',
help=
f'list jobs submitted in the last HOURS hours (default = {DEFAULT_LOOKBACK_HOURS})',
type=float,
metavar='HOURS')
parser.add_argument('--submitted-after',
'-A',
help=f'list jobs submitted after the given time')
parser.add_argument('--submitted-before',
'-B',
help=f'list jobs submitted before the given time')
parser.add_argument(
'--name',
'-n',
help=
"list jobs with a particular name pattern (name filters can contain "
"alphanumeric characters, '.', '-', '_', and '*' as a wildcard)")
parser.add_argument(
'--limit',
'-l',
help=f'limit the number of results (default = {DEFAULT_LIMIT})',
type=check_positive)
parser.add_argument('--exclude-custom-executor',
help=f'exclude jobs with a custom executor',
action='store_true')
group = parser.add_mutually_exclusive_group()
group.add_argument('--json',
help='show the data in JSON format',
dest='json',
action='store_true')
group.add_argument(
'--urls',
'-1',
help='list one job URL per line, without table formatting',
dest='one-per-line',
action='store_true')
add_defaults('jobs', {
'states': ['running'],
'user': current_user(),
'limit': DEFAULT_LIMIT
})
return jobs
def submit(clusters, args, _):
"""
Submits a job (or multiple jobs) to cook scheduler. Assembles a list of jobs,
potentially getting data from configuration, the command line, and stdin.
"""
guard_no_cluster(clusters)
logging.debug('submit args: %s' % args)
job_template = args
raw = job_template.pop('raw', None)
command_from_command_line = job_template.pop('command', None)
command_prefix = job_template.pop('command-prefix')
application_name = job_template.pop('application-name',
'cook-scheduler-cli')
application_version = job_template.pop('application-version',
version.VERSION)
job_template['application'] = {
'name': application_name,
'version': application_version
}
pool = job_template.pop('pool-name', None)
docker_image = job_template.pop('docker-image', None)
if docker_image:
job_template['container'] = {
'type': 'docker',
'docker': {
'image': docker_image,
'network': 'HOST',
'force-pull-image': False
}
}
group = None
if 'group-name' in job_template:
# If the user did not also specify a group uuid, generate
# one for them, and place the job(s) into the group
if 'group' not in job_template:
job_template['group'] = str(uuid.uuid4())
# The group name is specified on the group object
group = {
'name': job_template.pop('group-name'),
'uuid': job_template['group']
}
if raw:
if command_from_command_line:
raise Exception(
'You cannot specify a command at the command line when using --raw/-r.'
)
jobs_json = read_jobs_from_stdin()
jobs = parse_raw_job_spec(job_template, jobs_json)
else:
commands = acquire_commands(command_from_command_line)
if job_template.get('uuid') and len(commands) > 1:
raise Exception(
'You cannot specify multiple subcommands with a single UUID.')
if job_template.get('env'):
job_template['env'] = dict(
[e.split('=', maxsplit=1) for e in job_template['env']])
if job_template.get('label'):
labels = dict(
[l.split('=', maxsplit=1) for l in job_template['label']])
job_template.pop('label')
if 'labels' not in job_template:
job_template['labels'] = {}
job_template['labels'].update(labels)
jobs = [deep_merge(job_template, {'command': c}) for c in commands]
for job in jobs:
if not job.get('uuid'):
job['uuid'] = str(uuid.uuid4())
if not job.get('name'):
job['name'] = '%s_job' % current_user()
if command_prefix:
job['command'] = f'{command_prefix}{job["command"]}'
logging.debug('jobs: %s' % jobs)
return submit_federated(clusters, jobs, group, pool)
def submit(clusters, args, _):
"""
Submits a job (or multiple jobs) to cook scheduler. Assembles a list of jobs,
potentially getting data from configuration, the command line, and stdin.
"""
guard_no_cluster(clusters)
logging.debug('submit args: %s' % args)
job_template = args
raw = job_template.pop('raw', None)
command_from_command_line = job_template.pop('command', None)
command_prefix = job_template.pop('command-prefix')
application_name = job_template.pop('application-name',
'cook-scheduler-cli')
application_version = job_template.pop('application-version',
version.VERSION)
job_template['application'] = {
'name': application_name,
'version': application_version
}
pool = job_template.pop('pool-name', None)
checkpoint = job_template.pop('checkpoint', False)
checkpoint_mode = job_template.pop('checkpoint-mode', None)
checkpoint_preserve_paths = job_template.pop('checkpoint-preserve-paths',
None)
checkpoint_period_sec = job_template.pop('checkpoint-period-sec', None)
disk_request = job_template.pop('disk-request', None)
disk_limit = job_template.pop('disk-limit', None)
disk_type = job_template.pop('disk-type', None)
docker_image = job_template.pop('docker-image', None)
if docker_image:
job_template['container'] = {
'type': 'docker',
'docker': {
'image': docker_image,
'network': 'HOST',
'force-pull-image': False
}
}
group = None
if 'group-name' in job_template:
# If the user did not also specify a group uuid, generate
# one for them, and place the job(s) into the group
if 'group' not in job_template:
job_template['group'] = str(make_temporal_uuid())
# The group name is specified on the group object
group = {
'name': job_template.pop('group-name'),
'uuid': job_template['group']
}
if raw:
if command_from_command_line:
raise Exception(
'You cannot specify a command at the command line when using --raw/-r.'
)
jobs_json = read_jobs_from_stdin()
jobs = parse_raw_job_spec(job_template, jobs_json)
else:
commands = acquire_commands(command_from_command_line)
if job_template.get('uuid') and len(commands) > 1:
raise Exception(
'You cannot specify multiple subcommands with a single UUID.')
if job_template.get('env'):
job_template['env'] = dict(
[e.split('=', maxsplit=1) for e in job_template['env']])
if job_template.get('label'):
labels = dict(
[l.split('=', maxsplit=1) for l in job_template['label']])
job_template.pop('label')
if 'labels' not in job_template:
job_template['labels'] = {}
job_template['labels'].update(labels)
if job_template.get('constraint'):
constraints = []
for c in job_template['constraint']:
parts = c.split('=', maxsplit=1)
if len(parts) != 2:
raise Exception(
'Invalid constraint, must be of the form K=V')
constraints.append([parts[0], 'EQUALS', parts[1]])
job_template.pop('constraint')
job_template['constraints'] = constraints
jobs = [deep_merge(job_template, {'command': c}) for c in commands]
for job in jobs:
if not job.get('uuid'):
job['uuid'] = str(make_temporal_uuid())
if not job.get('name'):
job['name'] = '%s_job' % current_user()
if command_prefix:
job['command'] = f'{command_prefix}{job["command"]}'
if checkpoint or checkpoint_mode:
checkpoint = {
'mode': checkpoint_mode if checkpoint_mode else 'auto'
}
if checkpoint_preserve_paths:
checkpoint['options'] = {
'preserve-paths': checkpoint_preserve_paths
}
if checkpoint_period_sec:
checkpoint['periodic-options'] = {
'period-sec': checkpoint_period_sec
}
job['checkpoint'] = checkpoint
if disk_request or disk_limit or disk_type:
disk = {}
if disk_request:
disk['request'] = disk_request
if disk_limit:
disk['limit'] = disk_limit
if disk_type:
disk['type'] = disk_type
job['disk'] = disk
logging.debug('jobs: %s' % jobs)
return submit_federated(clusters, jobs, group, pool)
import logging
import socket
from cook.util import current_user
__line_formats = None
__conn = None
__host = socket.gethostname()
__user = current_user()
__disabled = True
def initialize(config):
"""
Initializes the metrics module using the given
config; note that metrics can be completely
disabled in which case this is essentially a no-op
"""
global __disabled
try:
metrics_config = config.get('metrics')
__disabled = metrics_config.get('disabled')
if __disabled:
return
global __conn
global __line_formats
__line_formats = metrics_config.get('line-formats')
__conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
__conn.settimeout(metrics_config.get('timeout'))
address = (metrics_config.get('host'), metrics_config.get('port'))
