def list_previous_versions(
service: str,
deploy_groups: Collection[str],
any_given_deploy_groups: bool,
versions: Mapping[DeploymentVersion, Tuple],
) -> None:
def format_timestamp(tstamp: str) -> str:
return naturaltime(datetime_from_utc_to_local(parse_timestamp(tstamp)))
print("Below is a list of recent commits:")
# Latest 10 versions sorted by deployment time
list_of_versions = sorted(versions.items(),
key=lambda x: x[1],
reverse=True)[:10]
rows = [("Timestamp -- UTC", "Human time", "deploy_group", "Version")]
for version, (timestamp, deploy_group) in list_of_versions:
rows.extend([(timestamp, format_timestamp(timestamp), deploy_group,
repr(version))])
for line in format_table(rows):
print(line)
if len(list_of_versions) >= 2:
version, (timestamp, deploy_group) = list_of_versions[1]
deploy_groups_arg_line = ("-l %s " % ",".join(deploy_groups)
if any_given_deploy_groups else "")
version_arg = (f" --image-version {version.image_version}"
if version.image_version else "")
print(
"\nFor example, to use the second to last version from {} used on {}, run:"
.format(format_timestamp(timestamp),
PaastaColors.bold(deploy_group)))
print(
PaastaColors.bold(
f" paasta rollback -s {service} {deploy_groups_arg_line}-k {version.sha}{version_arg}"
))
def list_previous_commits(service, deploy_groups, any_given_deploy_groups, git_shas):
def format_timestamp(tstamp):
return naturaltime(datetime_from_utc_to_local(parse_timestamp(tstamp)))
print("Below is a list of recent commits:")
git_shas = sorted(git_shas.items(), key=lambda x: x[1], reverse=True)[:10]
rows = [("Timestamp -- UTC", "Human time", "deploy_group", "Git SHA")]
for sha, (timestamp, deploy_group) in git_shas:
rows.extend([(timestamp, format_timestamp(timestamp), deploy_group, sha)])
for line in format_table(rows):
print(line)
if len(git_shas) >= 2:
sha, (timestamp, deploy_group) = git_shas[1]
deploy_groups_arg_line = (
"-l %s " % ",".join(deploy_groups) if any_given_deploy_groups else ""
)
print(
"\nFor example, to use the second to last commit from {} used on {}, run:".format(
format_timestamp(timestamp), PaastaColors.bold(deploy_group)
)
)
print(
PaastaColors.bold(
f" paasta rollback -s {service} {deploy_groups_arg_line}-k {sha}"
)
)
def marathon_deployments_check(service):
"""Checks for consistency between deploy.yaml and the marathon yamls"""
the_return = True
pipeline_deployments = get_pipeline_config(service)
pipeline_steps = [step['instancename'] for step in pipeline_deployments]
pipeline_steps = [step for step in pipeline_steps if step not in DEPLOY_PIPELINE_NON_DEPLOY_STEPS]
marathon_steps = get_marathon_steps(service)
in_marathon_not_deploy = set(marathon_steps) - set(pipeline_steps)
if len(in_marathon_not_deploy) > 0:
print "%s There are some instance(s) you have asked to run in marathon that" % x_mark()
print " do not have a corresponding entry in deploy.yaml:"
print " %s" % PaastaColors.bold(", ".join(in_marathon_not_deploy))
print " You should probably add entries to deploy.yaml for them so they"
print " are deployed to those clusters."
the_return = False
in_deploy_not_marathon = set(pipeline_steps) - set(marathon_steps)
if len(in_deploy_not_marathon) > 0:
print "%s There are some instance(s) in deploy.yaml that are not referenced" % x_mark()
print " by any marathon instance:"
print " %s" % PaastaColors.bold((", ".join(in_deploy_not_marathon)))
print " You should probably delete these deploy.yaml entries if they are unused."
the_return = False
if the_return is True:
print success("All entries in deploy.yaml correspond to a marathon entry")
print success("All marathon instances have a corresponding deploy.yaml entry")
return the_return
def list_previous_commits(service, deploy_groups, any_given_deploy_groups,
soa_dir):
def format_timestamp(tstamp):
return naturaltime(datetime_from_utc_to_local(parse_timestamp(tstamp)))
print "Please specify a commit to mark for rollback (-k, --commit). Below is a list of recent commits:"
git_shas = sorted(get_git_shas_for_service(service, deploy_groups,
soa_dir),
key=lambda x: x[1],
reverse=True)[:10]
rows = [('Timestamp -- UTC', 'Human time', 'deploy_group', 'Git SHA')]
for sha, (timestamp, deploy_group) in git_shas:
print timestamp
rows.extend([(timestamp, format_timestamp(timestamp), deploy_group,
sha)])
for line in format_table(rows):
print line
if len(git_shas) >= 2:
print ""
sha, (timestamp, deploy_group) = git_shas[1]
deploy_groups_arg_line = '-d %s ' % ','.join(
deploy_groups) if any_given_deploy_groups else ''
print "For example, to use the second to last commit from %s used on %s, run:" % (
format_timestamp(timestamp), PaastaColors.bold(deploy_group))
print PaastaColors.bold(" paasta rollback -s %s %s-k %s" %
(service, deploy_groups_arg_line, sha))
def marathon_deployments_check(service):
"""Checks for consistency between deploy.yaml and the marathon yamls"""
the_return = True
pipeline_deployments = get_pipeline_config(service)
pipeline_steps = [step['instancename'] for step in pipeline_deployments]
pipeline_steps = [step for step in pipeline_steps if step not in DEPLOY_PIPELINE_NON_DEPLOY_STEPS]
marathon_steps = get_marathon_steps(service)
in_marathon_not_deploy = set(marathon_steps) - set(pipeline_steps)
if len(in_marathon_not_deploy) > 0:
print "%s There are some instance(s) you have asked to run in marathon that" % x_mark()
print " do not have a corresponding entry in deploy.yaml:"
print " %s" % PaastaColors.bold(", ".join(in_marathon_not_deploy))
print " You should probably add entries to deploy.yaml for them so they"
print " are deployed to those clusters."
the_return = False
in_deploy_not_marathon = set(pipeline_steps) - set(marathon_steps)
if len(in_deploy_not_marathon) > 0:
print "%s There are some instance(s) in deploy.yaml that are not referenced" % x_mark()
print " by any marathon instance:"
print " %s" % PaastaColors.bold((", ".join(in_deploy_not_marathon)))
print " You should probably delete these deploy.yaml entries if they are unused."
the_return = False
if the_return is True:
print success("All entries in deploy.yaml correspond to a marathon entry")
print success("All marathon instances have a corresponding deploy.yaml entry")
return the_return
def get_desired_state_human(self):
desired_state = self.get_desired_state()
if desired_state == "start":
return PaastaColors.bold("Scheduled")
elif desired_state == "stop":
return PaastaColors.bold("Disabled")
else:
return PaastaColors.red("Unknown (desired_state: %s)" % desired_state)
def get_desired_state_human(self):
desired_state = self.get_desired_state()
if desired_state == 'start':
return PaastaColors.bold('Scheduled')
elif desired_state == 'stop':
return PaastaColors.bold('Disabled')
else:
return PaastaColors.red('Unknown (desired_state: %s)' % desired_state)
def get_desired_state_human(self):
desired_state = self.get_desired_state()
if desired_state == 'start':
return PaastaColors.bold('Scheduled')
elif desired_state == 'stop':
return PaastaColors.bold('Disabled')
else:
return PaastaColors.red('Unknown (desired_state: %s)' % desired_state)
def desired_state_human(desired_state, instances):
if desired_state == 'start' and instances != 0:
return PaastaColors.bold('Started')
elif desired_state == 'start' and instances == 0:
return PaastaColors.bold('Stopped')
elif desired_state == 'stop':
return PaastaColors.red('Stopped')
else:
return PaastaColors.red('Unknown (desired_state: %s)' % desired_state)
def get_desired_state_human(self):
desired_state = self.get_desired_state()
if desired_state == "start":
return PaastaColors.bold("Scheduled")
elif desired_state == "stop":
return PaastaColors.bold("Disabled")
else:
return PaastaColors.red("Unknown (desired_state: %s)" %
desired_state)
def desired_state_human(desired_state, instances):
if desired_state == "start" and instances != 0:
return PaastaColors.bold("Started")
elif desired_state == "start" and instances == 0:
return PaastaColors.bold("Stopped")
elif desired_state == "stop":
return PaastaColors.red("Stopped")
else:
return PaastaColors.red("Unknown (desired_state: %s)" % desired_state)
def deployments_check(service, soa_dir):
"""Checks for consistency between deploy.yaml and the marathon/chronos yamls"""
the_return = True
pipeline_steps = [
step['step'] for step in get_pipeline_config(service, soa_dir)
]
pipeline_deploy_groups = [
step for step in pipeline_steps if is_deploy_step(step)
]
framework_deploy_groups = {}
in_deploy_not_frameworks = set(pipeline_deploy_groups)
for it in INSTANCE_TYPES:
framework_deploy_groups[it] = get_deploy_groups_used_by_framework(
it, service, soa_dir)
in_framework_not_deploy = set(
framework_deploy_groups[it]) - set(pipeline_deploy_groups)
in_deploy_not_frameworks -= set(framework_deploy_groups[it])
if len(in_framework_not_deploy) > 0:
paasta_print(
"%s There are some instance(s) you have asked to run in %s that"
% (x_mark(), it))
paasta_print(" do not have a corresponding entry in deploy.yaml:")
paasta_print(" %s" %
PaastaColors.bold(", ".join(in_framework_not_deploy)))
paasta_print(
" You should probably configure these to use a 'deploy_group' or"
)
paasta_print(
" add entries to deploy.yaml for them so they are deployed to those clusters."
)
the_return = False
if len(in_deploy_not_frameworks) > 0:
paasta_print(
"%s There are some instance(s) in deploy.yaml that are not referenced"
% x_mark())
paasta_print(" by any marathon, chronos or adhoc instance:")
paasta_print(" %s" % PaastaColors.bold(
(", ".join(in_deploy_not_frameworks))))
paasta_print(
" You should probably delete these deploy.yaml entries if they are unused."
)
the_return = False
if the_return is True:
paasta_print(
success(
"All entries in deploy.yaml correspond to a marathon, chronos or adhoc entry"
))
for it in INSTANCE_TYPES:
if len(framework_deploy_groups[it]) > 0:
paasta_print(
success(
"All %s instances have a corresponding deploy.yaml entry"
% it))
return the_return
def desired_state_human(desired_state, instances):
if desired_state == 'start' and instances != 0:
return PaastaColors.bold('Started')
elif desired_state == 'start' and instances == 0:
return PaastaColors.bold('Stopped')
elif desired_state == 'stop':
return PaastaColors.red('Stopped')
else:
return PaastaColors.red('Unknown (desired_state: %s)' % desired_state)
def get_desired_state_human(self):
desired_state = self.get_desired_state()
if desired_state == 'start' and self.get_instances() != 0:
return PaastaColors.bold('Started')
elif desired_state == 'start' and self.get_instances() == 0:
return PaastaColors.bold('Stopped')
elif desired_state == 'stop':
return PaastaColors.red('Stopped')
else:
return PaastaColors.red('Unknown (desired_state: %s)' % desired_state)
def get_desired_state_human(self):
desired_state = self.get_desired_state()
if desired_state == "start" and self.get_instances() != 0:
return PaastaColors.bold("Started")
elif desired_state == "start" and self.get_instances() == 0:
return PaastaColors.bold("Stopped")
elif desired_state == "stop":
return PaastaColors.red("Stopped")
else:
return PaastaColors.red("Unknown (desired_state: %s)" % desired_state)
def get_desired_state_human(self):
desired_state = self.get_desired_state()
if desired_state == 'start' and self.get_instances() != 0:
return PaastaColors.bold('Started')
elif desired_state == 'start' and self.get_instances() == 0:
return PaastaColors.bold('Stopped')
elif desired_state == 'stop':
return PaastaColors.red('Stopped')
else:
return PaastaColors.red('Unknown (desired_state: %s)' % desired_state)
def list_previous_commits(service, deploy_groups, any_given_deploy_groups, soa_dir):
def format_timestamp(tstamp):
return naturaltime(datetime_from_utc_to_local(parse_timestamp(tstamp)))
print "Please specify a commit to mark for rollback (-k, --commit). Below is a list of recent commits:"
git_shas = sorted(get_git_shas_for_service(service, deploy_groups, soa_dir), key=lambda x: x[1], reverse=True)[:10]
rows = [('Timestamp -- UTC', 'Git SHA')]
rows.extend([('%s (%s)' % (timestamp, format_timestamp(timestamp)), sha) for sha, timestamp in git_shas])
for line in format_table(rows):
print line
if len(git_shas) >= 2:
sha, tstamp = git_shas[1]
deploy_groups_arg_line = '-d %s ' % ','.join(deploy_groups) if any_given_deploy_groups else ''
print "For example, to roll back to the second to last commit from %s, run:" % format_timestamp(tstamp)
print PaastaColors.bold(" paasta rollback -s %s %s-k %s" % (service, deploy_groups_arg_line, sha))
def deployments_check(service, soa_dir):
"""Checks for consistency between deploy.yaml and the marathon/chronos yamls"""
the_return = True
pipeline_deployments = get_pipeline_config(service, soa_dir)
pipeline_steps = [step['step'] for step in pipeline_deployments]
pipeline_steps = [step for step in pipeline_steps if is_deploy_step(step)]
marathon_steps = get_marathon_steps(service, soa_dir)
chronos_steps = get_chronos_steps(service, soa_dir)
in_marathon_not_deploy = set(marathon_steps) - set(pipeline_steps)
in_chronos_not_deploy = set(chronos_steps) - set(pipeline_steps)
if len(in_marathon_not_deploy) > 0:
print "%s There are some instance(s) you have asked to run in marathon that" % x_mark(
)
print " do not have a corresponding entry in deploy.yaml:"
print " %s" % PaastaColors.bold(", ".join(in_marathon_not_deploy))
print " You should probably configure these to use a 'deploy_group' or"
print " add entries to deploy.yaml for them so they are deployed to those clusters."
the_return = False
if len(in_chronos_not_deploy) > 0:
print "%s There are some instance(s) you have asked to run in chronos that" % x_mark(
)
print " do not have a corresponding entry in deploy.yaml:"
print " %s" % PaastaColors.bold(", ".join(in_chronos_not_deploy))
print " You should probably configure these to use a 'deploy_group' or"
print " add entries to deploy.yaml for them so they are deployed to those clusters."
the_return = False
in_deploy_not_marathon_chronos = set(pipeline_steps) - set(
marathon_steps) - set(chronos_steps)
if len(in_deploy_not_marathon_chronos) > 0:
print "%s There are some instance(s) in deploy.yaml that are not referenced" % x_mark(
)
print " by any marathon or chronos instance:"
print " %s" % PaastaColors.bold(
(", ".join(in_deploy_not_marathon_chronos)))
print " You should probably delete these deploy.yaml entries if they are unused."
the_return = False
if the_return is True:
print success(
"All entries in deploy.yaml correspond to a marathon or chronos entry"
)
if len(marathon_steps) > 0:
print success(
"All marathon instances have a corresponding deploy.yaml entry"
)
if len(chronos_steps) > 0:
print success(
"All chronos instances have a corresponding deploy.yaml entry")
return the_return
def status_mesos_tasks(service, instance, normal_instance_count):
job_id = marathon_tools.format_job_id(service, instance)
# We have to add a spacer at the end to make sure we only return
# things for service.main and not service.main_foo
filter_string = "%s%s" % (job_id, marathon_tools.MESOS_TASK_SPACER)
try:
count = len(
select_tasks_by_id(
get_cached_list_of_running_tasks_from_frameworks(),
filter_string))
if count >= normal_instance_count:
status = PaastaColors.green("Healthy")
count = PaastaColors.green("(%d/%d)" %
(count, normal_instance_count))
elif count == 0:
status = PaastaColors.red("Critical")
count = PaastaColors.red("(%d/%d)" %
(count, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
count = PaastaColors.yellow("(%d/%d)" %
(count, normal_instance_count))
running_string = PaastaColors.bold('TASK_RUNNING')
return "Mesos: %s - %s tasks in the %s state." % (status, count,
running_string)
except ReadTimeout:
return "Error: talking to Mesos timed out. It may be overloaded."
def paasta_status(args):
"""Print the status of a Yelp service running on PaaSTA.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
actual_deployments = get_actual_deployments(service, soa_dir)
system_paasta_config = load_system_paasta_config()
if args.clusters is not None:
cluster_whitelist = args.clusters.split(",")
else:
cluster_whitelist = []
if args.instances is not None:
instance_whitelist = args.instances.split(",")
else:
instance_whitelist = []
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service, soa_dir))
try:
report_status(
service=service,
deploy_pipeline=deploy_pipeline,
actual_deployments=actual_deployments,
cluster_whitelist=cluster_whitelist,
instance_whitelist=instance_whitelist,
system_paasta_config=system_paasta_config,
verbose=args.verbose,
)
except CalledProcessError as e:
print PaastaColors.grey(PaastaColors.bold(e.cmd + " exited with non-zero return code."))
print PaastaColors.grey(e.output)
return e.returncode
else:
print missing_deployments_message(service)
def get_verbose_status_of_marathon_app(app):
"""Takes a given marathon app object and returns the verbose details
about the tasks, times, hosts, etc"""
output = []
create_datetime = datetime_from_utc_to_local(isodate.parse_datetime(app.version))
output.append(" Marathon app ID: %s" % PaastaColors.bold(app.id))
output.append(" App created: %s (%s)" % (str(create_datetime), humanize.naturaltime(create_datetime)))
output.append(" Tasks:")
rows = [("Mesos Task ID", "Host deployed to", "Deployed at what localtime")]
for task in app.tasks:
local_deployed_datetime = datetime_from_utc_to_local(task.staged_at)
if task.host is not None:
hostname = "%s:%s" % (task.host.split(".")[0], task.ports[0])
else:
hostname = "Unknown"
rows.append((
get_short_task_id(task.id),
hostname,
'%s (%s)' % (
local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(local_deployed_datetime),
)
))
output.append('\n'.join([" %s" % line for line in format_table(rows)]))
if len(app.tasks) == 0:
output.append(" No tasks associated with this marathon app")
return app.tasks, "\n".join(output)
def get_verbose_status_of_marathon_app(app):
"""Takes a given marathon app object and returns the verbose details
about the tasks, times, hosts, etc"""
output = []
create_datetime = datetime_from_utc_to_local(
isodate.parse_datetime(app.version))
output.append(" Marathon app ID: %s" % PaastaColors.bold(app.id))
output.append(
" App created: %s (%s)" %
(str(create_datetime), humanize.naturaltime(create_datetime)))
output.append(" Tasks:")
rows = [("Mesos Task ID", "Host deployed to", "Deployed at what localtime")
]
for task in app.tasks:
local_deployed_datetime = datetime_from_utc_to_local(task.staged_at)
if task.host is not None:
hostname = "%s:%s" % (task.host.split(".")[0], task.ports[0])
else:
hostname = "Unknown"
rows.append((get_short_task_id(task.id), hostname, '%s (%s)' % (
local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(local_deployed_datetime),
)))
output.append('\n'.join([" %s" % line
for line in format_table(rows)]))
if len(app.tasks) == 0:
output.append(" No tasks associated with this marathon app")
return app.tasks, "\n".join(output)
def status_marathon_job(service, instance, app_id, normal_instance_count,
client):
name = PaastaColors.cyan(compose_job_id(service, instance))
if marathon_tools.is_app_id_running(app_id, client):
app = client.get_app(app_id)
running_instances = app.tasks_running
if len(app.deployments) == 0:
deploy_status = PaastaColors.bold("Running")
else:
deploy_status = PaastaColors.yellow("Deploying")
if running_instances >= normal_instance_count:
status = PaastaColors.green("Healthy")
instance_count = PaastaColors.green(
"(%d/%d)" % (running_instances, normal_instance_count))
elif running_instances == 0:
status = PaastaColors.yellow("Critical")
instance_count = PaastaColors.red(
"(%d/%d)" % (running_instances, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
instance_count = PaastaColors.yellow(
"(%d/%d)" % (running_instances, normal_instance_count))
return "Marathon: %s - up with %s instances. Status: %s." % (
status, instance_count, deploy_status)
else:
red_not = PaastaColors.red("NOT")
status = PaastaColors.red("Critical")
return "Marathon: %s - %s (app %s) is %s running in Marathon." % (
status, name, app_id, red_not)
def get_marathon_dashboard(client, dashboards, app_id):
if dashboards is not None:
base_url = dashboards.get(client)
if base_url:
url = "{}/ui/#/apps/%2F{}".format(base_url.rstrip('/'),
app_id.lstrip('/'))
return " Marathon dashboard: %s" % PaastaColors.blue(url)
return " Marathon app ID: %s" % PaastaColors.bold(app_id)
def list_previous_commits(service, deploy_groups, any_given_deploy_groups, git_shas):
def format_timestamp(tstamp):
return naturaltime(datetime_from_utc_to_local(parse_timestamp(tstamp)))
paasta_print('Below is a list of recent commits:')
git_shas = sorted(git_shas.items(), key=lambda x: x[1], reverse=True)[:10]
rows = [('Timestamp -- UTC', 'Human time', 'deploy_group', 'Git SHA')]
for sha, (timestamp, deploy_group) in git_shas:
rows.extend([(timestamp, format_timestamp(timestamp), deploy_group, sha)])
for line in format_table(rows):
paasta_print(line)
if len(git_shas) >= 2:
sha, (timestamp, deploy_group) = git_shas[1]
deploy_groups_arg_line = '-l %s ' % ','.join(deploy_groups) if any_given_deploy_groups else ''
paasta_print("\nFor example, to use the second to last commit from %s used on %s, run:" % (
format_timestamp(timestamp), PaastaColors.bold(deploy_group),
))
paasta_print(PaastaColors.bold(" paasta rollback -s %s %s-k %s" % (service, deploy_groups_arg_line, sha)))
def get_verbose_status_of_marathon_app(marathon_client, app, service, instance,
cluster, soa_dir):
"""Takes a given marathon app object and returns the verbose details
about the tasks, times, hosts, etc"""
output = []
create_datetime = datetime_from_utc_to_local(
isodate.parse_datetime(app.version))
output.append(" Marathon app ID: %s" % PaastaColors.bold(app.id))
output.append(
" App created: %s (%s)" %
(str(create_datetime), humanize.naturaltime(create_datetime)))
autoscaling_info = get_autoscaling_info(marathon_client, service, instance,
cluster, soa_dir)
if autoscaling_info:
output.append(" Autoscaling Info:")
headers = [
field.replace("_", " ").capitalize()
for field in ServiceAutoscalingInfo._fields
]
table = [headers, autoscaling_info]
output.append('\n'.join(
[" %s" % line for line in format_table(table)]))
output.append(" Tasks:")
rows = [("Mesos Task ID", "Host deployed to", "Deployed at what localtime",
"Health")]
for task in app.tasks:
local_deployed_datetime = datetime_from_utc_to_local(task.staged_at)
if task.host is not None:
hostname = "%s:%s" % (task.host.split(".")[0], task.ports[0])
else:
hostname = "Unknown"
if not task.health_check_results:
health_check_status = PaastaColors.grey("N/A")
elif marathon_tools.is_task_healthy(task):
health_check_status = PaastaColors.green("Healthy")
else:
health_check_status = PaastaColors.red("Unhealthy")
rows.append((
get_short_task_id(task.id),
hostname,
'%s (%s)' % (
local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(local_deployed_datetime),
),
health_check_status,
))
output.append('\n'.join([" %s" % line
for line in format_table(rows)]))
if len(app.tasks) == 0:
output.append(" No tasks associated with this marathon app")
return app.tasks, "\n".join(output)
def get_marathon_dashboard(
client: marathon_tools.MarathonClient,
dashboards: Dict[marathon_tools.MarathonClient, str],
app_id: str,
) -> str:
if dashboards is not None:
base_url = dashboards.get(client)
if base_url:
url = "{}/ui/#/apps/%2F{}".format(base_url.rstrip("/"), app_id.lstrip("/"))
return " Marathon dashboard: %s" % PaastaColors.blue(url)
return " Marathon app ID: %s" % PaastaColors.bold(app_id)
def paasta_fsm(args):
validate_args(args)
(srvname, service_stanza, smartstack_stanza, monitoring_stanza,
deploy_stanza, marathon_stanza, cluster_stanza, team) = (
get_paasta_config(
args.yelpsoa_config_root,
args.srvname,
args.auto,
args.port,
args.team,
args.description,
args.external_link,
)
)
srv = Service(srvname, args.yelpsoa_config_root)
write_paasta_config(
srv,
service_stanza,
smartstack_stanza,
monitoring_stanza,
deploy_stanza,
marathon_stanza,
cluster_stanza,
)
print PaastaColors.yellow(" _ _(o)_(o)_ _")
print PaastaColors.red(" ._\`:_ F S M _:' \_,")
print PaastaColors.green(" / (`---'\ `-.")
print PaastaColors.cyan(" ,-` _) (_,")
print "With My Noodly Appendage I Have Written Configs For"
print
print PaastaColors.bold(" %s" % srvname)
print
print "Customize Them If It Makes You Happy -- http://y/paasta For Details"
print "Remember To Add, Commit, And Push When You're Done:"
print
print "cd %s" % join(args.yelpsoa_config_root, srvname)
print "# Review And/Or Customize Files"
print "git add ."
print "git commit -m'Initial Commit For %s'" % srvname
print "git push origin HEAD # Pushmaster Or Ops Deputy Privs Required"
print
def paasta_fsm(args):
validate_args(args)
(srvname, service_stanza, smartstack_stanza, monitoring_stanza,
deploy_stanza, marathon_stanza, cluster_stanza, team) = (
get_paasta_config(
args.yelpsoa_config_root,
args.srvname,
args.auto,
args.port,
args.team,
args.description,
args.external_link,
)
)
srv = Service(srvname, args.yelpsoa_config_root)
write_paasta_config(
srv,
service_stanza,
smartstack_stanza,
monitoring_stanza,
deploy_stanza,
marathon_stanza,
cluster_stanza,
)
print PaastaColors.yellow(" _ _(o)_(o)_ _")
print PaastaColors.red(" ._\`:_ F S M _:' \_,")
print PaastaColors.green(" / (`---'\ `-.")
print PaastaColors.cyan(" ,-` _) (_,")
print "With My Noodly Appendage I Have Written Configs For"
print
print PaastaColors.bold(" %s" % srvname)
print
print "Customize Them If It Makes You Happy -- http://y/paasta For Details"
print "Remember To Add, Commit, And Push When You're Done:"
print
print "cd %s" % join(args.yelpsoa_config_root, srvname)
print "# Review And/Or Customize Files"
print "git add ."
print "git commit -m'Initial Commit For %s'" % srvname
print "git push origin HEAD # Pushmaster Or Ops Deputy Privs Required"
print
def print_rollback_cmd(old_git_sha, commit, auto_rollback, service, deploy_group):
if old_git_sha is not None and old_git_sha != commit and not auto_rollback:
paasta_print()
paasta_print("If you wish to roll back, you can run:")
paasta_print()
paasta_print(
PaastaColors.bold(
" paasta rollback --service {} --deploy-group {} --commit {} ".format(
service, deploy_group, old_git_sha
)
)
)
def kubernetes_app_deploy_status_human(status, backoff_seconds=None):
status_string = kubernetes_tools.KubernetesDeployStatus.tostring(status)
if status == kubernetes_tools.KubernetesDeployStatus.Waiting:
deploy_status = "%s (new tasks waiting for capacity to become available)" % PaastaColors.red(status_string)
elif status == kubernetes_tools.KubernetesDeployStatus.Deploying:
deploy_status = PaastaColors.yellow(status_string)
elif status == kubernetes_tools.KubernetesDeployStatus.Running:
deploy_status = PaastaColors.bold(status_string)
else:
deploy_status = status_string
return deploy_status
def deployments_check(service, soa_dir):
"""Checks for consistency between deploy.yaml and the marathon/chronos yamls"""
the_return = True
pipeline_deployments = get_pipeline_config(service, soa_dir)
pipeline_steps = [step["step"] for step in pipeline_deployments]
pipeline_steps = [step for step in pipeline_steps if is_deploy_step(step)]
marathon_steps = get_marathon_steps(service, soa_dir)
chronos_steps = get_chronos_steps(service, soa_dir)
in_marathon_not_deploy = set(marathon_steps) - set(pipeline_steps)
in_chronos_not_deploy = set(chronos_steps) - set(pipeline_steps)
if len(in_marathon_not_deploy) > 0:
paasta_print("%s There are some instance(s) you have asked to run in marathon that" % x_mark())
paasta_print(" do not have a corresponding entry in deploy.yaml:")
paasta_print(" %s" % PaastaColors.bold(", ".join(in_marathon_not_deploy)))
paasta_print(" You should probably configure these to use a 'deploy_group' or")
paasta_print(" add entries to deploy.yaml for them so they are deployed to those clusters.")
the_return = False
if len(in_chronos_not_deploy) > 0:
paasta_print("%s There are some instance(s) you have asked to run in chronos that" % x_mark())
paasta_print(" do not have a corresponding entry in deploy.yaml:")
paasta_print(" %s" % PaastaColors.bold(", ".join(in_chronos_not_deploy)))
paasta_print(" You should probably configure these to use a 'deploy_group' or")
paasta_print(" add entries to deploy.yaml for them so they are deployed to those clusters.")
the_return = False
in_deploy_not_marathon_chronos = set(pipeline_steps) - set(marathon_steps) - set(chronos_steps)
if len(in_deploy_not_marathon_chronos) > 0:
paasta_print("%s There are some instance(s) in deploy.yaml that are not referenced" % x_mark())
paasta_print(" by any marathon or chronos instance:")
paasta_print(" %s" % PaastaColors.bold((", ".join(in_deploy_not_marathon_chronos))))
paasta_print(" You should probably delete these deploy.yaml entries if they are unused.")
the_return = False
if the_return is True:
paasta_print(success("All entries in deploy.yaml correspond to a marathon or chronos entry"))
if len(marathon_steps) > 0:
paasta_print(success("All marathon instances have a corresponding deploy.yaml entry"))
if len(chronos_steps) > 0:
paasta_print(success("All chronos instances have a corresponding deploy.yaml entry"))
return the_return
def status_mesos_tasks(service, instance, normal_instance_count):
job_id = marathon_tools.format_job_id(service, instance)
running_and_active_tasks = get_running_tasks_from_active_frameworks(job_id)
count = len(running_and_active_tasks)
if count >= normal_instance_count:
status = PaastaColors.green("Healthy")
count = PaastaColors.green("(%d/%d)" % (count, normal_instance_count))
elif count == 0:
status = PaastaColors.red("Critical")
count = PaastaColors.red("(%d/%d)" % (count, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
count = PaastaColors.yellow("(%d/%d)" % (count, normal_instance_count))
running_string = PaastaColors.bold('TASK_RUNNING')
return "Mesos: %s - %s tasks in the %s state." % (status, count, running_string)
def marathon_mesos_status_summary(mesos_task_count,
expected_instance_count) -> str:
if mesos_task_count >= expected_instance_count:
status = PaastaColors.green("Healthy")
count_str = PaastaColors.green(
"(%d/%d)" % (mesos_task_count, expected_instance_count))
elif mesos_task_count == 0:
status = PaastaColors.red("Critical")
count_str = PaastaColors.red(
"(%d/%d)" % (mesos_task_count, expected_instance_count))
else:
status = PaastaColors.yellow("Warning")
count_str = PaastaColors.yellow(
"(%d/%d)" % (mesos_task_count, expected_instance_count))
running_string = PaastaColors.bold("TASK_RUNNING")
return f"Mesos: {status} - {count_str} tasks in the {running_string} state."
def haproxy_backend_report(normal_instance_count, up_backends):
"""Given that a service is in smartstack, this returns a human readable
report of the up backends"""
# TODO: Take into account a configurable threshold, PAASTA-1102
crit_threshold = 50
under_replicated, ratio = is_under_replicated(num_available=up_backends,
expected_count=normal_instance_count,
crit_threshold=crit_threshold)
if under_replicated:
status = PaastaColors.red("Critical")
count = PaastaColors.red("(%d/%d, %d%%)" % (up_backends, normal_instance_count, ratio))
else:
status = PaastaColors.green("Healthy")
count = PaastaColors.green("(%d/%d)" % (up_backends, normal_instance_count))
up_string = PaastaColors.bold('UP')
return "%s - in haproxy with %s total backends %s in this namespace." % (status, count, up_string)
def status_mesos_tasks(service, instance, normal_instance_count):
job_id = marathon_tools.format_job_id(service, instance)
running_and_active_tasks = get_running_tasks_from_active_frameworks(job_id)
count = len(running_and_active_tasks)
if count >= normal_instance_count:
status = PaastaColors.green("Healthy")
count = PaastaColors.green("(%d/%d)" % (count, normal_instance_count))
elif count == 0:
status = PaastaColors.red("Critical")
count = PaastaColors.red("(%d/%d)" % (count, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
count = PaastaColors.yellow("(%d/%d)" % (count, normal_instance_count))
running_string = PaastaColors.bold('TASK_RUNNING')
return "Mesos: %s - %s tasks in the %s state." % (status, count,
running_string)
def prettify_level(level, requested_levels):
"""Colorize level. 'event' is special and gets bolded; everything else gets
lightened.
requested_levels is an iterable of levels that will be displayed. If only
one level will be displayed, don't bother to print it (return empty string).
If multiple levels will be displayed, emit the (prettified) level so the
resulting log output is not ambiguous.
"""
pretty_level = ''
if len(requested_levels) > 1:
if level == 'event':
pretty_level = PaastaColors.bold('[%s]' % level)
else:
pretty_level = PaastaColors.grey('[%s]' % level)
return pretty_level
def prettify_level(level, requested_levels):
"""Colorize level. 'event' is special and gets bolded; everything else gets
lightened.
requested_levels is an iterable of levels that will be displayed. If only
one level will be displayed, don't bother to print it (return empty string).
If multiple levels will be displayed, emit the (prettified) level so the
resulting log output is not ambiguous.
"""
pretty_level = ''
if len(requested_levels) > 1:
if level == 'event':
pretty_level = PaastaColors.bold('[%s]' % level)
else:
pretty_level = PaastaColors.grey('[%s]' % level)
return pretty_level
def haproxy_backend_report(normal_instance_count, up_backends):
"""Given that a service is in smartstack, this returns a human readable
report of the up backends"""
# TODO: Take into account a configurable threshold, PAASTA-1102
crit_threshold = 50
under_replicated, ratio = is_under_replicated(num_available=up_backends,
expected_count=normal_instance_count,
crit_threshold=crit_threshold)
if under_replicated:
status = PaastaColors.red("Critical")
count = PaastaColors.red("(%d/%d, %d%%)" % (up_backends, normal_instance_count, ratio))
else:
status = PaastaColors.green("Healthy")
count = PaastaColors.green("(%d/%d)" % (up_backends, normal_instance_count))
up_string = PaastaColors.bold('UP')
return "%s - in haproxy with %s total backends %s in this namespace." % (status, count, up_string)
def status_smartstack_backends(service, instance, job_config, cluster, tasks,
expected_count, soa_dir, verbose, synapse_port,
synapse_haproxy_url_format):
"""Returns detailed information about smartstack backends for a service
and instance.
return: A newline separated string of the smarststack backend status
"""
output = []
nerve_ns = marathon_tools.read_namespace_for_service_instance(
service, instance, cluster)
service_instance = compose_job_id(service, nerve_ns)
if instance != nerve_ns:
ns_string = PaastaColors.bold(nerve_ns)
output.append(
"Smartstack: N/A - %s is announced in the %s namespace." %
(instance, ns_string))
# If verbose mode is specified, then continue to show backends anyway, otherwise stop early
if not verbose:
return "\n".join(output)
service_namespace_config = marathon_tools.load_service_namespace_config(
service, instance, soa_dir=soa_dir)
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist()
unique_attributes = get_mesos_slaves_grouped_by_attribute(
attribute=discover_location_type, blacklist=monitoring_blacklist)
if len(unique_attributes) == 0:
output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" %
service_instance)
else:
output.append("Smartstack:")
if verbose:
output.append(" Haproxy Service Name: %s" % service_instance)
output.append(" Backends:")
output.extend(
pretty_print_smartstack_backends_for_locations(
service_instance,
tasks,
unique_attributes,
expected_count,
verbose,
synapse_port,
synapse_haproxy_url_format,
))
return "\n".join(output)
def get_marathon_app_deploy_status_human(app, app_id, client):
status, delay = get_marathon_app_deploy_status(app, app_id, client)
status_string = MarathonDeployStatus.tostring(status)
if status == MarathonDeployStatus.Waiting:
deploy_status = "%s (new tasks are not launching due to lack of capacity)" % PaastaColors.red(status_string)
elif status == MarathonDeployStatus.Delayed:
deploy_status = "%s (next task won't launch for %s seconds due to previous failures)" % (
PaastaColors.red(status_string), delay)
elif status == MarathonDeployStatus.Deploying:
deploy_status = PaastaColors.yellow(status_string)
elif status == MarathonDeployStatus.Stopped:
deploy_status = PaastaColors.grey(status_string)
else:
deploy_status = PaastaColors.bold(status_string)
return deploy_status
def marathon_app_deploy_status_human(status, backoff_seconds=None):
status_string = marathon_tools.MarathonDeployStatus.tostring(status)
if status == marathon_tools.MarathonDeployStatus.Waiting:
deploy_status = "%s (new tasks waiting for capacity to become available)" % PaastaColors.red(status_string)
elif status == marathon_tools.MarathonDeployStatus.Delayed:
deploy_status = "%s (tasks are crashing, next won't launch for another %s seconds)" % (
PaastaColors.red(status_string), backoff_seconds)
elif status == marathon_tools.MarathonDeployStatus.Deploying:
deploy_status = PaastaColors.yellow(status_string)
elif status == marathon_tools.MarathonDeployStatus.Stopped:
deploy_status = PaastaColors.grey(status_string)
elif status == marathon_tools.MarathonDeployStatus.Running:
deploy_status = PaastaColors.bold(status_string)
else:
deploy_status = status_string
return deploy_status
def marathon_app_deploy_status_human(status, backoff_seconds=None):
status_string = marathon_tools.MarathonDeployStatus.tostring(status)
if status == marathon_tools.MarathonDeployStatus.Waiting:
deploy_status = "%s (new tasks waiting for capacity to become available)" % PaastaColors.red(status_string)
elif status == marathon_tools.MarathonDeployStatus.Delayed:
deploy_status = "%s (tasks are crashing, next won't launch for another %s seconds)" % (
PaastaColors.red(status_string), backoff_seconds)
elif status == marathon_tools.MarathonDeployStatus.Deploying:
deploy_status = PaastaColors.yellow(status_string)
elif status == marathon_tools.MarathonDeployStatus.Stopped:
deploy_status = PaastaColors.grey(status_string)
elif status == marathon_tools.MarathonDeployStatus.Running:
deploy_status = PaastaColors.bold(status_string)
else:
deploy_status = status_string
return deploy_status
def status_mesos_tasks(service, instance, normal_instance_count):
job_id = marathon_tools.format_job_id(service, instance)
# We have to add a spacer at the end to make sure we only return
# things for service.main and not service.main_foo
filter_string = "%s%s" % (job_id, marathon_tools.MESOS_TASK_SPACER)
running_and_active_tasks = get_running_tasks_from_active_frameworks(filter_string)
count = len(running_and_active_tasks)
if count >= normal_instance_count:
status = PaastaColors.green("Healthy")
count = PaastaColors.green("(%d/%d)" % (count, normal_instance_count))
elif count == 0:
status = PaastaColors.red("Critical")
count = PaastaColors.red("(%d/%d)" % (count, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
count = PaastaColors.yellow("(%d/%d)" % (count, normal_instance_count))
running_string = PaastaColors.bold('TASK_RUNNING')
return "Mesos: %s - %s tasks in the %s state." % (status, count, running_string)
def status_mesos_tasks(service, instance, normal_instance_count):
job_id = marathon_tools.format_job_id(service, instance)
# We have to add a spacer at the end to make sure we only return
# things for service.main and not service.main_foo
filter_string = "%s%s" % (job_id, marathon_tools.MESOS_TASK_SPACER)
running_and_active_tasks = get_running_tasks_from_active_frameworks(filter_string)
count = len(running_and_active_tasks)
if count >= normal_instance_count:
status = PaastaColors.green("Healthy")
count = PaastaColors.green("(%d/%d)" % (count, normal_instance_count))
elif count == 0:
status = PaastaColors.red("Critical")
count = PaastaColors.red("(%d/%d)" % (count, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
count = PaastaColors.yellow("(%d/%d)" % (count, normal_instance_count))
running_string = PaastaColors.bold('TASK_RUNNING')
return "Mesos: %s - %s tasks in the %s state." % (status, count, running_string)
def status_marathon_job(service, instance, app_id, normal_instance_count,
client):
name = PaastaColors.cyan(compose_job_id(service, instance))
if marathon_tools.is_app_id_running(app_id, client):
app = client.get_app(app_id)
running_instances = app.tasks_running
if len(app.deployments) == 0:
deploy_status = PaastaColors.bold("Running")
elif app.instances == 0 and app.tasks_running == 0:
deploy_status = PaastaColors.grey("Stopped")
else:
# App is currently deploying so we should check the launch queue for more info
is_overdue, backoff_seconds = marathon_tools.get_app_queue_status(
client, app_id)
if is_overdue:
deploy_status = "%s (new tasks are not launching due to lack of capacity)" % PaastaColors.red(
"Waiting")
elif backoff_seconds:
deploy_status = "%s (next task won't launch for %s seconds due to previous failures)" % (
PaastaColors.red("Delayed"), backoff_seconds)
else:
deploy_status = PaastaColors.yellow("Deploying")
if running_instances >= normal_instance_count:
status = PaastaColors.green("Healthy")
instance_count = PaastaColors.green(
"(%d/%d)" % (running_instances, normal_instance_count))
elif running_instances == 0:
status = PaastaColors.yellow("Critical")
instance_count = PaastaColors.red(
"(%d/%d)" % (running_instances, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
instance_count = PaastaColors.yellow(
"(%d/%d)" % (running_instances, normal_instance_count))
return "Marathon: %s - up with %s instances. Status: %s" % (
status, instance_count, deploy_status)
else:
red_not = PaastaColors.red("NOT")
status = PaastaColors.red("Critical")
return "Marathon: %s - %s (app %s) is %s running in Marathon." % (
status, name, app_id, red_not)
def marathon_app_deploy_status_human(status, backoff_seconds=None):
status_string = marathon_tools.MarathonDeployStatus.tostring(status)
if status == marathon_tools.MarathonDeployStatus.Waiting:
deploy_status = "%s (new tasks are not launching due to lack of capacity)" % PaastaColors.red(
status_string)
elif status == marathon_tools.MarathonDeployStatus.Delayed:
deploy_status = "%s (next task won't launch for %s seconds due to previous failures)" % (
PaastaColors.red(status_string), backoff_seconds)
elif status == marathon_tools.MarathonDeployStatus.Deploying:
deploy_status = PaastaColors.yellow(status_string)
elif status == marathon_tools.MarathonDeployStatus.Stopped:
deploy_status = PaastaColors.grey(status_string)
elif status == marathon_tools.MarathonDeployStatus.Running:
deploy_status = PaastaColors.bold(status_string)
else:
deploy_status = status_string
return deploy_status
def status_smartstack_backends(service, instance, job_config, cluster, tasks, expected_count, soa_dir, verbose,
synapse_port, synapse_haproxy_url_format):
"""Returns detailed information about smartstack backends for a service
and instance.
return: A newline separated string of the smarststack backend status
"""
output = []
nerve_ns = marathon_tools.read_namespace_for_service_instance(service, instance, cluster)
service_instance = compose_job_id(service, nerve_ns)
if instance != nerve_ns:
ns_string = PaastaColors.bold(nerve_ns)
output.append("Smartstack: N/A - %s is announced in the %s namespace." % (instance, ns_string))
# If verbose mode is specified, then continue to show backends anyway, otherwise stop early
if not verbose:
return "\n".join(output)
service_namespace_config = marathon_tools.load_service_namespace_config(service, instance, soa_dir=soa_dir)
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist()
unique_attributes = get_mesos_slaves_grouped_by_attribute(
attribute=discover_location_type, blacklist=monitoring_blacklist)
if len(unique_attributes) == 0:
output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" % service_instance)
else:
output.append("Smartstack:")
if verbose:
output.append(" Haproxy Service Name: %s" % service_instance)
output.append(" Backends:")
output.extend(pretty_print_smartstack_backends_for_locations(
service_instance,
tasks,
unique_attributes,
expected_count,
verbose,
synapse_port,
synapse_haproxy_url_format,
))
return "\n".join(output)
def status_marathon_job(service, instance, app_id, normal_instance_count, client):
name = PaastaColors.cyan(compose_job_id(service, instance))
if marathon_tools.is_app_id_running(app_id, client):
app = client.get_app(app_id)
running_instances = app.tasks_running
if len(app.deployments) == 0:
deploy_status = PaastaColors.bold("Running")
else:
deploy_status = PaastaColors.yellow("Deploying")
if running_instances >= normal_instance_count:
status = PaastaColors.green("Healthy")
instance_count = PaastaColors.green("(%d/%d)" % (running_instances, normal_instance_count))
elif running_instances == 0:
status = PaastaColors.yellow("Critical")
instance_count = PaastaColors.red("(%d/%d)" % (running_instances, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
instance_count = PaastaColors.yellow("(%d/%d)" % (running_instances, normal_instance_count))
return "Marathon: %s - up with %s instances. Status: %s." % (status, instance_count, deploy_status)
else:
red_not = PaastaColors.red("NOT")
status = PaastaColors.red("Critical")
return "Marathon: %s - %s (app %s) is %s running in Marathon." % (status, name, app_id, red_not)
def status_marathon_job(service, instance, app_id, normal_instance_count, client):
name = PaastaColors.cyan(compose_job_id(service, instance))
if marathon_tools.is_app_id_running(app_id, client):
app = client.get_app(app_id)
running_instances = app.tasks_running
if len(app.deployments) == 0:
deploy_status = PaastaColors.bold("Running")
elif app.instances == 0 and app.tasks_running == 0:
deploy_status = PaastaColors.grey("Stopped")
else:
# App is currently deploying so we should check the launch queue for more info
is_overdue, backoff_seconds = marathon_tools.get_app_queue_status(client, app_id)
if is_overdue:
deploy_status = "%s (new tasks are not launching due to lack of capacity)" % PaastaColors.red("Waiting")
elif backoff_seconds:
deploy_status = "%s (next task won't launch for %s seconds due to previous failures)" % (
PaastaColors.red("Delayed"), backoff_seconds)
else:
deploy_status = PaastaColors.yellow("Deploying")
if running_instances >= normal_instance_count:
status = PaastaColors.green("Healthy")
instance_count = PaastaColors.green("(%d/%d)" % (running_instances, normal_instance_count))
elif running_instances == 0:
status = PaastaColors.yellow("Critical")
instance_count = PaastaColors.red("(%d/%d)" % (running_instances, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
instance_count = PaastaColors.yellow("(%d/%d)" % (running_instances, normal_instance_count))
return "Marathon: %s - up with %s instances. Status: %s" % (status, instance_count, deploy_status)
else:
red_not = PaastaColors.red("NOT")
status = PaastaColors.red("Critical")
return "Marathon: %s - %s (app %s) is %s running in Marathon." % (status, name, app_id, red_not)
def paasta_mark_for_deployment(args):
"""Wrapping mark_for_deployment"""
if args.verbose:
log.setLevel(level=logging.DEBUG)
else:
log.setLevel(level=logging.INFO)
service = args.service
if service and service.startswith('services-'):
service = service.split('services-', 1)[1]
validate_service_name(service, soa_dir=args.soa_dir)
in_use_deploy_groups = list_deploy_groups(
service=service,
soa_dir=args.soa_dir,
)
_, invalid_deploy_groups = validate_given_deploy_groups(in_use_deploy_groups, [args.deploy_group])
if len(invalid_deploy_groups) == 1:
paasta_print(PaastaColors.red(
"ERROR: These deploy groups are not currently used anywhere: %s.\n" % (",").join(invalid_deploy_groups)))
paasta_print(PaastaColors.red(
"This isn't technically wrong because you can mark-for-deployment before deploying there"))
paasta_print(PaastaColors.red("but this is probably a typo. Did you mean one of these in-use deploy groups?:"))
paasta_print(PaastaColors.red(" %s" % (",").join(in_use_deploy_groups)))
paasta_print()
paasta_print(PaastaColors.red("Continuing regardless..."))
if args.git_url is None:
args.git_url = get_git_url(service=service, soa_dir=args.soa_dir)
old_git_sha = get_currently_deployed_sha(service=service, deploy_group=args.deploy_group)
if old_git_sha == args.commit:
paasta_print("Warning: The sha asked to be deployed already matches what is set to be deployed:")
paasta_print(old_git_sha)
paasta_print("Continuing anyway.")
ret = mark_for_deployment(
git_url=args.git_url,
deploy_group=args.deploy_group,
service=service,
commit=args.commit,
)
if args.block:
try:
wait_for_deployment(service=service,
deploy_group=args.deploy_group,
git_sha=args.commit,
soa_dir=args.soa_dir,
timeout=args.timeout)
line = "Deployment of {0} for {1} complete".format(args.commit, args.deploy_group)
_log(
service=service,
component='deploy',
line=line,
level='event'
)
except (KeyboardInterrupt, TimeoutError):
if args.auto_rollback is True:
if old_git_sha == args.commit:
paasta_print("Error: --auto-rollback was requested, but the previous sha")
paasta_print("is the same that was requested with --commit. Can't rollback")
paasta_print("automatically.")
else:
paasta_print("Auto-Rollback requested. Marking the previous sha")
paasta_print("(%s) for %s as desired." % (args.deploy_group, old_git_sha))
mark_for_deployment(
git_url=args.git_url,
deploy_group=args.deploy_group,
service=service,
commit=old_git_sha,
)
else:
paasta_print("Waiting for deployment aborted. PaaSTA will continue to try to deploy this code.")
paasta_print("If you wish to see the status, run:")
paasta_print()
paasta_print(" paasta status -s %s -v" % service)
paasta_print()
ret = 1
except NoInstancesFound:
return 1
if old_git_sha is not None and old_git_sha != args.commit and not args.auto_rollback:
paasta_print()
paasta_print("If you wish to roll back, you can run:")
paasta_print()
paasta_print(PaastaColors.bold(" paasta rollback --service %s --deploy-group %s --commit %s " % (
service, args.deploy_group, old_git_sha))
)
return ret
