def setup_job(service, instance, chronos_job_config, complete_job_config,
client, cluster):
job_id = complete_job_config['name']
# Sort this initial list since other lists of jobs will come from it (with
# their orders preserved by list comprehensions) and since we'll care about
# ordering by recency when we go calculate jobs_to_delete.
all_existing_jobs = chronos_tools.sort_jobs(
chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
))
old_jobs = [job for job in all_existing_jobs if job["name"] != job_id]
enabled_old_jobs = [job for job in old_jobs if not job["disabled"]]
if all_existing_jobs == old_jobs:
job_to_create = complete_job_config
else:
job_to_create = None
number_of_old_jobs_to_keep = 5
return bounce_chronos_job(
service=service,
instance=instance,
cluster=cluster,
jobs_to_disable=enabled_old_jobs,
jobs_to_delete=old_jobs[number_of_old_jobs_to_keep:],
job_to_create=job_to_create,
client=client,
)
def build_service_job_mapping(client, configured_jobs):
"""
:param client: A Chronos client used for getting the list of running jobs
:param configured_jobs: A list of jobs configured in Paasta, i.e. jobs we
expect to be able to find
:returns: A dict of {(service, instance): [(chronos job, lastrunstate)]}
where the chronos job is any with a matching (service, instance) in its
name and disabled == False
"""
service_job_mapping = {}
for job in configured_jobs:
# find all the jobs belonging to each service
matching_jobs = chronos_tools.lookup_chronos_jobs(
service=job[0],
instance=job[1],
client=client,
include_disabled=True,
)
matching_jobs = chronos_tools.sort_jobs(matching_jobs)
# Only consider the most recent one
if len(matching_jobs) > 0:
matching_jobs = [matching_jobs[0]]
with_states = last_run_state_for_jobs(matching_jobs)
service_job_mapping[job] = with_states
return service_job_mapping
def job_exists(context, service, instance):
matching_jobs = chronos_tools.lookup_chronos_jobs(
client=context.chronos_client,
service=service,
instance=instance,
)
assert len(matching_jobs) == 1
def should_be_disabled_jobs(context, disabled, job_count, service, instance):
is_disabled = True if disabled == "disabled" else False
all_jobs = chronos_tools.lookup_chronos_jobs(
service=service, instance=instance, client=context.chronos_client, include_disabled=True
)
filtered_jobs = [job for job in all_jobs if job["disabled"] is is_disabled]
assert len(filtered_jobs) == int(job_count)
def job_exists(context, service, instance):
matching_jobs = chronos_tools.lookup_chronos_jobs(
client=context.chronos_client,
service=service,
instance=instance,
)
assert len(matching_jobs) == 1
def setup_job(service, instance, chronos_job_config, complete_job_config, client, cluster):
job_id = complete_job_config['name']
all_existing_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
)
# TODO: Sort the jobs in the right order so we delete the least relevant
# This currently depends on implicit behavior that Chronos returns jobs
# "oldest first"
old_jobs = [job for job in all_existing_jobs if job["name"] != job_id]
enabled_old_jobs = [job for job in old_jobs if not job["disabled"]]
if all_existing_jobs == old_jobs:
job_to_create = complete_job_config
else:
job_to_create = None
number_of_old_jobs_to_keep = 5
return bounce_chronos_job(
service=service,
instance=instance,
cluster=cluster,
jobs_to_disable=enabled_old_jobs,
jobs_to_delete=old_jobs[number_of_old_jobs_to_keep:],
job_to_create=job_to_create,
client=client,
)
def setup_job(service, instance, complete_job_config, client, cluster):
# There should only ever be *one* job for a given service_instance
all_existing_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
include_disabled=True,
)
job_to_update = None
if len(all_existing_jobs) > 0:
# we store the md5 sum of the config in the description field.
if all_existing_jobs[0]['description'] != complete_job_config[
'description']:
job_to_update = complete_job_config
else:
job_to_update = complete_job_config
return bounce_chronos_job(
service=service,
instance=instance,
cluster=cluster,
job_to_update=job_to_update,
client=client,
)
def build_service_job_mapping(client, configured_jobs):
"""
:param client: A Chronos client used for getting the list of running jobs
:param configured_jobs: A list of jobs configured in Paasta, i.e. jobs we
expect to be able to find
:returns: A dict of {(service, instance): [(chronos job, lastrunstate)]}
where the chronos job is any with a matching (service, instance) in its
name and disabled == False
"""
service_job_mapping = {}
for job in configured_jobs:
# find all the jobs belonging to each service
matching_jobs = chronos_tools.lookup_chronos_jobs(
service=job[0],
instance=job[1],
client=client,
include_disabled=True,
)
matching_jobs = chronos_tools.sort_jobs(matching_jobs)
# Only consider the most recent one
if len(matching_jobs) > 0:
matching_jobs = [matching_jobs[0]]
with_states = last_run_state_for_jobs(matching_jobs)
service_job_mapping[job] = with_states
return service_job_mapping
def setup_job(service, instance, chronos_job_config, complete_job_config, client, cluster):
job_id = complete_job_config['name']
# Sort this initial list since other lists of jobs will come from it (with
# their orders preserved by list comprehensions) and since we'll care about
# ordering by recency when we go calculate jobs_to_delete.
all_existing_jobs = chronos_tools.sort_jobs(chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
))
old_jobs = [job for job in all_existing_jobs if job["name"] != job_id]
enabled_old_jobs = [job for job in old_jobs if not job["disabled"]]
if all_existing_jobs == old_jobs:
job_to_create = complete_job_config
else:
job_to_create = None
number_of_old_jobs_to_keep = 5
return bounce_chronos_job(
service=service,
instance=instance,
cluster=cluster,
jobs_to_disable=enabled_old_jobs,
jobs_to_delete=old_jobs[number_of_old_jobs_to_keep:],
job_to_create=job_to_create,
client=client,
)
def should_be_enabled_jobs(context, job_count):
enabled_jobs = chronos_tools.lookup_chronos_jobs(
service=fake_service_name,
instance=fake_instance_name,
client=context.chronos_client,
include_disabled=False,
)
assert len(enabled_jobs) == int(job_count)
def job_exists(context, service, instance):
matching_jobs = chronos_tools.lookup_chronos_jobs(
client=context.chronos_client,
service=service,
instance=instance,
include_temporary=True,
include_disabled=True,
)
assert len(matching_jobs) == 1
def job_exists(context, service, instance):
matching_jobs = chronos_tools.lookup_chronos_jobs(
client=context.chronos_client,
service=service,
instance=instance,
include_temporary=True,
include_disabled=True,
)
assert len(matching_jobs) == 1
def should_be_disabled_jobs(context, disabled, job_count, service, instance):
is_disabled = True if disabled == "disabled" else False
all_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=context.chronos_client,
include_disabled=True,
)
filtered_jobs = [job for job in all_jobs if job["disabled"] is is_disabled]
assert len(filtered_jobs) == int(job_count)
def chronos_check_job_state(context, field, job_name, value):
job_id = context.jobs[job_name]['name']
service, instance = chronos_tools.decompose_job_id(job_id)
jobs = chronos_tools.lookup_chronos_jobs(service=service,
instance=instance,
client=context.chronos_client,
include_disabled=True)
assert len(jobs) == 1
# we cast to a string so you can correctly assert that a value is True/False
assert str(jobs[0][field]) == value
def should_be_disabled_jobs(context, job_count):
all_related_jobs = chronos_tools.lookup_chronos_jobs(
service=fake_service_name,
instance=fake_instance_name,
client=context.chronos_client,
include_disabled=True,
)
disabled_jobs = [
job for job in all_related_jobs if job["disabled"] is True
]
assert len(disabled_jobs) == int(job_count)
def chronos_check_job_state(context, field, job_name, value):
job_id = context.jobs[job_name]['name']
service, instance = chronos_tools.decompose_job_id(job_id)
jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=context.chronos_client,
include_disabled=True
)
assert len(jobs) == 1
# we cast to a string so you can correctly assert that a value is True/False
assert str(jobs[0][field]) == value
def chronos_check_job_state(context, old_or_new_job, disabled):
desired_disabled = (disabled == 'disabled')
if old_or_new_job == 'old job':
job_id = context.old_chronos_job_name
else:
job_id = context.chronos_job_name
(service, instance, git_hash, config_hash) = chronos_tools.decompose_job_id(job_id)
jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
git_hash=git_hash,
config_hash=config_hash,
client=context.chronos_client,
include_disabled=desired_disabled,
)
assert len(jobs) == 1
for job in jobs:
assert job['disabled'] == desired_disabled
def build_service_job_mapping(client, configured_jobs):
"""
:param client: A Chronos client used for getting the list of running jobs
:param configured_jobs: A list of jobs configured in Paasta, i.e. jobs we
expect to be able to find
:returns: A dict of {(service, instance): [(chronos job, lastrunstate)]}
where the chronos job is any with a matching (service, instance) in its
name and disabled == False
"""
service_job_mapping = {}
for job in configured_jobs:
# find all the jobs belonging to each service
matching_jobs = chronos_tools.lookup_chronos_jobs(
service=job[0],
instance=job[1],
client=client,
)
filtered = chronos_tools.filter_non_temporary_chronos_jobs(matching_jobs)
with_states = last_run_state_for_jobs(filtered)
service_job_mapping[job] = with_states
return service_job_mapping
def build_service_job_mapping(client, configured_jobs):
"""
:param client: A Chronos client used for getting the list of running jobs
:param configured_jobs: A list of jobs configured in Paasta, i.e. jobs we
expect to be able to find
:returns: A dict of {(service, instance): last_chronos_job}
where last_chronos_job is the latest job matching (service, instance)
or None if there is no such job
"""
service_job_mapping = {}
for job in configured_jobs:
# find all the jobs belonging to each service
matching_jobs = chronos_tools.lookup_chronos_jobs(
service=job[0],
instance=job[1],
client=client,
include_disabled=True,
)
matching_jobs = chronos_tools.sort_jobs(matching_jobs)
# Only consider the most recent one
service_job_mapping[job] = matching_jobs[0] if len(
matching_jobs) > 0 else None
return service_job_mapping
def setup_job(service, instance, complete_job_config, client, cluster):
# There should only ever be *one* job for a given service_instance
all_existing_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
include_disabled=True,
)
job_to_update = None
if len(all_existing_jobs) > 0:
if all_existing_jobs[0] != complete_job_config:
job_to_update = complete_job_config
else:
job_to_update = complete_job_config
return bounce_chronos_job(
service=service,
instance=instance,
cluster=cluster,
job_to_update=job_to_update,
client=client,
)
def config_with_historical_stats(chronos_client, service, instance,
job_config):
"""
Given a complete_job_config, return a new chronos_job_config
to be posted with the historical 'stats' for the job are merged
into the new config to be posted. If the job is 'new', that is,
there is no job currently deployed to Chronos for the service+instance
in soa-configs, these values are left empty.
:param chronos_client: a chronos-python client object
:param service: the service for the job
:param instance: the instance for the job
:param job_config: the job config to be modified
:returns: a modified job config, with the historical stats set
"""
existing_matching_jobs = chronos_tools.lookup_chronos_jobs(
client=chronos_client,
service=service,
instance=instance,
include_disabled=True,
include_temporary=False,
)
if existing_matching_jobs:
# assume there is only one of them
existing_job = existing_matching_jobs[0]
# these keys should always exist, but I guess it's good to be
# defensive
historical_state = {
'lastSuccess': existing_job.get('lastSuccess', ''),
'lastError': existing_job.get('lastError', ''),
'successCount': existing_job.get('successCount', 0),
'errorCount': existing_job.get('errorCount', 0),
}
copied = copy.deepcopy(job_config)
copied.update(historical_state)
return copied
return job_config
def setup_job(service, instance, complete_job_config, client, cluster):
# There should only ever be *one* job for a given service_instance
all_existing_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
include_disabled=True,
)
job_to_update = None
if len(all_existing_jobs) > 0:
# we store the md5 sum of the config in the description field.
if all_existing_jobs[0]['description'] != complete_job_config['description']:
job_to_update = complete_job_config
else:
job_to_update = complete_job_config
return bounce_chronos_job(
service=service,
instance=instance,
cluster=cluster,
job_to_update=job_to_update,
client=client,
)
def perform_command(command, service, instance, cluster, verbose, soa_dir):
"""Performs a start/stop/restart/status on an instance
:param command: String of start, stop, restart, status or scale
:param service: service name
:param instance: instance name, like "main" or "canary"
:param cluster: cluster name
:param verbose: int verbosity level
:returns: A unix-style return code
"""
chronos_config = chronos_tools.load_chronos_config()
client = chronos_tools.get_chronos_client(chronos_config)
job_config = chronos_tools.load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
complete_job_config = chronos_tools.create_complete_config(service, instance, soa_dir=soa_dir)
job_id = complete_job_config["name"]
if command == "start":
start_chronos_job(
service=service,
instance=instance,
job_id=job_id,
client=client,
cluster=cluster,
job_config=job_config,
complete_job_config=complete_job_config,
emergency=True,
)
elif command == "stop":
matching_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
include_disabled=True,
include_temporary=True
)
stop_chronos_job(service, instance, client, cluster, matching_jobs, emergency=True)
elif command == "restart":
matching_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
include_disabled=True,
)
restart_chronos_job(
service=service,
instance=instance,
job_id=job_id,
client=client,
cluster=cluster,
matching_jobs=matching_jobs,
job_config=job_config,
complete_job_config=complete_job_config,
emergency=True,
)
elif command == "status":
# Verbose mode shows previous versions.
matching_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
include_disabled=True,
)
sorted_matching_jobs = chronos_tools.sort_jobs(matching_jobs)
job_config = chronos_tools.load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
paasta_print(status_chronos_jobs(client, sorted_matching_jobs, job_config, verbose))
else:
# The command parser shouldn't have let us get this far...
raise NotImplementedError("Command %s is not implemented!" % command)
return 0
def perform_command(command, service, instance, cluster, verbose, soa_dir):
"""Performs a start/stop/restart/status on an instance
:param command: String of start, stop, restart, status or scale
:param service: service name
:param instance: instance name, like "main" or "canary"
:param cluster: cluster name
:param verbose: int verbosity level
:returns: A unix-style return code
"""
chronos_config = chronos_tools.load_chronos_config()
client = chronos_tools.get_chronos_client(chronos_config)
job_config = chronos_tools.load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
complete_job_config = chronos_tools.create_complete_config(service,
instance,
soa_dir=soa_dir)
job_id = complete_job_config["name"]
if command == "start":
start_chronos_job(
service=service,
instance=instance,
job_id=job_id,
client=client,
cluster=cluster,
job_config=job_config,
complete_job_config=complete_job_config,
emergency=True,
)
elif command == "stop":
matching_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
include_disabled=True,
include_temporary=True)
stop_chronos_job(service,
instance,
client,
cluster,
matching_jobs,
emergency=True)
elif command == "restart":
matching_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
include_disabled=True,
)
restart_chronos_job(
service=service,
instance=instance,
job_id=job_id,
client=client,
cluster=cluster,
matching_jobs=matching_jobs,
job_config=job_config,
complete_job_config=complete_job_config,
emergency=True,
)
elif command == "status":
# Verbose mode shows previous versions.
matching_jobs = chronos_tools.lookup_chronos_jobs(
service=service,
instance=instance,
client=client,
include_disabled=True,
)
sorted_matching_jobs = chronos_tools.sort_jobs(matching_jobs)
job_config = chronos_tools.load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
paasta_print(
status_chronos_jobs(client, sorted_matching_jobs, job_config,
verbose))
else:
# The command parser shouldn't have let us get this far...
raise NotImplementedError("Command %s is not implemented!" % command)
return 0
