def clone_job(chronos_job, timestamp=None, force_disabled=False):
"""
Given a chronos job, create a 'rerun' clone that respects the parents relations.
If the job has his own schedule it will be executed once and only once, and as soon as possible.
:param chronos_job: a chronos job suitable for POSTing to Chronos
:param timestamp: timestamp to use for the generation of the tmp job name
:returns: the chronos_job parameter, modified to be submitted as a
temporary clone used to rerun a job in the context of a given date.
"""
clone = copy.deepcopy(chronos_job)
job_type = chronos_tools.get_job_type(clone)
# modify the name of the job
clone = set_tmp_naming_scheme(clone, timestamp)
# If the jobs is a dependent job rename the parents dependencies
# in order to make this job dependent from the temporary clone of the parents
if job_type == chronos_tools.JobType.Dependent:
clone['parents'] = [
'{}{}{}'.format(
get_tmp_naming_scheme_prefix(timestamp),
chronos_tools.SPACER,
parent,
) for parent in chronos_job['parents']
]
else:
# If the job is a scheduled one update the schedule to start it NOW
clone = set_default_schedule(clone)
# Set disabled to false if force_disabled is on
if force_disabled:
clone['disabled'] = False
return clone
def clone_job(chronos_job, date):
"""
Given a chronos job, create a 'rerun' clone, that is due to run once and
only once, and as soon as possible.
:param chronos_job: a chronos job suitable for POSTing to Chronos
:param date: the date for which the job is to be run.
:returns: the chronos_job parameter, modified to be submitted as a
temporary clone used to rerun a job in the context of a given date.
"""
clone = copy.deepcopy(chronos_job)
job_type = chronos_tools.get_job_type(clone)
# modify the name of the job
clone = set_tmp_naming_scheme(clone)
# give the job a schedule for it to run now
clone = set_default_schedule(clone)
# if the job is a dependent job
# then convert it to be a scheduled job
# that should run now
if job_type == chronos_tools.JobType.Dependent:
clone = remove_parents(clone)
# set the job to run now
clone = set_default_schedule(clone)
# modify the command to run commands
# for a given date
clone = modify_command_for_date(clone, date)
return clone
def chronos_instance_status(instance_status, service, instance, verbose):
cstatus = {}
chronos_config = chronos_tools.load_chronos_config()
client = chronos_tools.get_chronos_client(chronos_config)
job_config = chronos_tools.load_chronos_job_config(
service=service,
instance=instance,
cluster=settings.cluster,
soa_dir=settings.soa_dir,
)
cstatus['desired_state'] = job_config.get_desired_state()
job_type = chronos_tools.get_job_type(job_config.config_dict)
if job_type == chronos_tools.JobType.Scheduled:
schedule_type = 'schedule'
schedule = job_config.get_schedule()
epsilon = job_config.get_epsilon()
time_zone = job_config.get_schedule_time_zone()
if time_zone == 'null' or time_zone is None:
time_zone = 'UTC'
cstatus['schedule'] = {}
cstatus['schedule']['schedule'] = schedule
cstatus['schedule']['epsilon'] = epsilon
cstatus['schedule']['time_zone'] = time_zone
elif job_type == chronos_tools.JobType.Dependent:
schedule_type = 'parents'
parents = job_config.get_parents()
cstatus['parents'] = parents
else:
schedule_type = 'unknown'
cstatus['schedule_type'] = schedule_type
cstatus['status'] = {}
if verbose:
running_task_count = len(
select_tasks_by_id(
a_sync.block(get_cached_list_of_running_tasks_from_frameworks),
job_config.get_job_name(),
),
)
cstatus['status']['mesos_state'] = 'running' if running_task_count else 'not_running'
cstatus['status']['disabled_state'] = 'not_scheduled' if job_config.get_disabled() else 'scheduled'
cstatus['status']['chronos_state'] = chronos_tools.get_chronos_status_for_job(client, service, instance)
cstatus['command'] = job_config.get_cmd()
last_time, last_status = chronos_tools.get_status_last_run(job_config.config_dict)
if last_status == chronos_tools.LastRunState.Success:
last_status = 'success'
elif last_status == chronos_tools.LastRunState.Fail:
last_status = 'fail'
elif last_status == chronos_tools.LastRunState.NotRun:
last_status = 'not_run'
else:
last_status = ''
if last_status == 'not_run' or last_status == '':
last_time = 'never'
cstatus['last_status'] = {}
cstatus['last_status']['result'] = last_status
cstatus['last_status']['time'] = last_time
return cstatus
def format_chronos_job_status(client, job, running_tasks, verbose=0):
"""Given a job, returns a pretty-printed human readable output regarding
the status of the job.
:param job: dictionary of the job status
:param running_tasks: a list of Mesos tasks associated with ``job``, e.g. the
result of ``mesos_tools.get_running_tasks_from_frameworks()``.
:param verbose: int verbosity level
"""
job_name = _format_job_name(job)
is_temporary = chronos_tools.is_temporary_job(
job) if 'name' in job else 'UNKNOWN'
job_name = modify_string_for_rerun_status(job_name, is_temporary)
disabled_state = _format_disabled_status(job)
service, instance = chronos_tools.decompose_job_id(job['name'])
chronos_state = chronos_tools.get_chronos_status_for_job(
client, service, instance)
(last_result, formatted_time) = _format_last_result(job)
job_type = chronos_tools.get_job_type(job)
schedule_type = _get_schedule_field_for_job_type(job_type)
schedule_formatter = get_schedule_formatter(job_type, verbose)
schedule_value = schedule_formatter(job)
command = _format_command(job)
mesos_status = _format_mesos_status(running_tasks)
if verbose > 0:
tail_lines = calculate_tail_lines(verbose_level=verbose)
mesos_status_verbose = status_mesos_tasks_verbose(
job_id=job["name"],
get_short_task_id=get_short_task_id,
tail_lines=tail_lines,
)
mesos_status = "%s\n%s" % (mesos_status, mesos_status_verbose)
return ("Job: %(job_name)s\n"
" Status: %(disabled_state)s (%(chronos_state)s)"
" Last: %(last_result)s (%(formatted_time)s)\n"
" %(schedule_type)s: %(schedule_value)s\n"
" Command: %(command)s\n"
" Mesos: %(mesos_status)s" % {
"job_name": job_name,
"is_temporary": is_temporary,
"schedule_type": schedule_type,
"chronos_state": PaastaColors.grey(chronos_state),
"disabled_state": disabled_state,
"last_result": last_result,
"formatted_time": formatted_time,
"schedule_value": schedule_value,
"command": command,
"mesos_status": mesos_status,
})
def format_chronos_job_status(client, job, running_tasks, verbose=0):
"""Given a job, returns a pretty-printed human readable output regarding
the status of the job.
:param job: dictionary of the job status
:param running_tasks: a list of Mesos tasks associated with ``job``, e.g. the
result of ``mesos_tools.get_running_tasks_from_active_frameworks()``.
:param verbose: int verbosity level
"""
job_name = _format_job_name(job)
is_temporary = chronos_tools.is_temporary_job(job) if 'name' in job else 'UNKNOWN'
job_name = modify_string_for_rerun_status(job_name, is_temporary)
disabled_state = _format_disabled_status(job)
service, instance = chronos_tools.decompose_job_id(job['name'])
chronos_state = chronos_tools.get_chronos_status_for_job(client, service, instance)
(last_result, formatted_time) = _format_last_result(job)
job_type = chronos_tools.get_job_type(job)
schedule_type = _get_schedule_field_for_job_type(job_type)
schedule_formatter = get_schedule_formatter(job_type, verbose)
schedule_value = schedule_formatter(job)
command = _format_command(job)
mesos_status = _format_mesos_status(job, running_tasks)
if verbose > 0:
tail_lines = calculate_tail_lines(verbose_level=verbose)
mesos_status_verbose = status_mesos_tasks_verbose(
job_id=job["name"],
get_short_task_id=get_short_task_id,
tail_lines=tail_lines,
)
mesos_status = "%s\n%s" % (mesos_status, mesos_status_verbose)
return (
"Job: %(job_name)s\n"
" Status: %(disabled_state)s (%(chronos_state)s)"
" Last: %(last_result)s (%(formatted_time)s)\n"
" %(schedule_type)s: %(schedule_value)s\n"
" Command: %(command)s\n"
" Mesos: %(mesos_status)s" % {
"job_name": job_name,
"is_temporary": is_temporary,
"schedule_type": schedule_type,
"chronos_state": PaastaColors.grey(chronos_state),
"disabled_state": disabled_state,
"last_result": last_result,
"formatted_time": formatted_time,
"schedule_value": schedule_value,
"command": command,
"mesos_status": mesos_status,
}
)
def main():
args = parse_args()
system_paasta_config = load_system_paasta_config()
cluster = system_paasta_config.get_cluster()
service, instance = chronos_tools.decompose_job_id(args.service_instance)
config = chronos_tools.load_chronos_config()
client = chronos_tools.get_chronos_client(config)
related_jobs = chronos_tools.get_related_jobs_configs(cluster,
service,
instance,
soa_dir=args.soa_dir)
if not related_jobs:
error_msg = "No deployment found for {} in cluster {}. Has Jenkins run for it?".format(
args.service_instance,
cluster,
)
paasta_print(error_msg)
raise NoDeploymentsAvailable
if not args.run_all_related_jobs:
# Strip all the configuration for the related services
# those information will not be used by the rest of the flow
related_jobs = {
(service, instance): related_jobs[(service, instance)],
}
complete_job_configs = {}
for (srv, inst) in related_jobs:
try:
complete_job_configs.update(
{
(srv, inst):
chronos_tools.create_complete_config(
service=srv,
job_name=inst,
soa_dir=args.soa_dir,
),
}, )
except (NoDeploymentsAvailable, NoDockerImageError) as e:
error_msg = "No deployment found for {} in cluster {}. Has Jenkins run for it?".format(
chronos_tools.compose_job_id(srv, inst),
cluster,
)
paasta_print(error_msg)
raise e
except NoConfigurationForServiceError as e:
error_msg = (
"Could not read chronos configuration file for {} in cluster {}\nError was: {}"
.format(
chronos_tools.compose_job_id(srv, inst),
cluster,
str(e),
))
paasta_print(error_msg)
raise e
except chronos_tools.InvalidParentError as e:
raise e
if not args.run_all_related_jobs:
sorted_jobs = [(service, instance)]
else:
sorted_jobs = chronos_tools.topological_sort_related_jobs(
cluster, service, instance, soa_dir=args.soa_dir)
timestamp = datetime.datetime.utcnow().isoformat()
chronos_to_add = []
for (service, instance) in sorted_jobs:
# complete_job_config is a formatted version of the job,
# so the command is formatted in the context of 'now'
# replace it with the 'original' cmd so it can be re rendered
chronos_job_config = chronos_tools.load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=args.soa_dir,
)
original_command = chronos_job_config.get_cmd()
complete_job_config = complete_job_configs[(service, instance)]
complete_job_config['command'] = original_command
clone = clone_job(
chronos_job=complete_job_config,
timestamp=timestamp,
force_disabled=args.force_disabled,
)
# modify the command to run commands for a given date
clone = modify_command_for_date(
chronos_job=clone,
date=datetime.datetime.strptime(args.execution_date,
"%Y-%m-%dT%H:%M:%S"),
verbose=args.verbose,
)
if not args.run_all_related_jobs and chronos_tools.get_job_type(
clone) == chronos_tools.JobType.Dependent:
# If the job is a dependent job and we want to re-run only the specific instance
# remove the parents and update the schedule to start the job as soon as possible
clone = set_default_schedule(remove_parents(clone))
chronos_to_add.append(clone)
for job_to_add in chronos_to_add:
client.add(job_to_add)
