def paasta_status(args):
"""Print the status of a Yelp service running on PaaSTA.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
actual_deployments = get_actual_deployments(service, soa_dir)
system_paasta_config = load_system_paasta_config()
if args.clusters is not None:
cluster_whitelist = args.clusters.split(",")
else:
cluster_whitelist = []
if args.instances is not None:
instance_whitelist = args.instances.split(",")
else:
instance_whitelist = []
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service, soa_dir))
try:
report_status(
service=service,
deploy_pipeline=deploy_pipeline,
actual_deployments=actual_deployments,
cluster_whitelist=cluster_whitelist,
instance_whitelist=instance_whitelist,
system_paasta_config=system_paasta_config,
verbose=args.verbose,
)
except CalledProcessError as e:
print PaastaColors.grey(PaastaColors.bold(e.cmd + " exited with non-zero return code."))
print PaastaColors.grey(e.output)
return e.returncode
else:
print missing_deployments_message(service)
def paasta_rollback(args):
"""Call mark_for_deployment with rollback parameters
:param args: contains all the arguments passed onto the script: service,
deploy groups and sha. These arguments will be verified and passed onto
mark_for_deployment.
"""
service = figure_out_service_name(args)
git_url = get_git_url(service)
commit = args.commit
given_deploy_groups = [deploy_group for deploy_group in args.deploy_groups.split(",") if deploy_group]
service_deploy_groups = set(config.get_deploy_group() for config in get_instance_config_for_service(
soa_dir=DEFAULT_SOA_DIR,
service=service,
))
deploy_groups, invalid = validate_given_deploy_groups(service_deploy_groups, given_deploy_groups)
if len(invalid) > 0:
print PaastaColors.yellow("These deploy groups are not valid and will be skipped: %s.\n" % (",").join(invalid))
if len(deploy_groups) == 0:
print PaastaColors.red("ERROR: No valid deploy groups specified for %s.\n" % (service))
returncode = 1
for deploy_group in deploy_groups:
returncode = mark_for_deployment(
git_url=git_url,
service=service,
deploy_group=deploy_group,
commit=commit,
)
sys.exit(returncode)
def paasta_autoscale(args):
log.setLevel(logging.DEBUG)
service = figure_out_service_name(args)
api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
if not api:
paasta_print(
"Could not connect to paasta api. Maybe you misspelled the cluster?"
)
return 1
if args.set is None:
log.debug("Getting the current autoscaler count...")
res, http = api.autoscaler.get_autoscaler_count(
service=service, instance=args.instance).result()
else:
log.debug(f"Setting desired instances to {args.set}.")
body = {"desired_instances": int(args.set)}
res, http = api.autoscaler.update_autoscaler_count(
service=service, instance=args.instance, json_body=body).result()
_log_audit(
action="manual-scale",
action_details=body,
service=service,
instance=args.instance,
cluster=args.cluster,
)
log.debug(f"Res: {res} Http: {http}")
print(res["desired_instances"])
return 0
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
Warning: This command is not magic and cannot actually get a service to start if it couldn't
run before. This includes configurations that prevent the service from running,
such as 'instances: 0' (for Marathon apps).
All it does for Marathon apps is ask Marathon to resume normal operation by scaling up to
the instance count defined in the service's config.
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency start on %s..." % compose_job_id(service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand="start",
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
print "%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:])
print "Output: %s" % PaastaColors.grey(output)
print "Run this command to see the status:"
print "paasta status --service %s --clusters %s" % (service, args.cluster)
def paasta_logs(args):
"""Print the logs for as Paasta service.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
if args.clusters is None:
clusters = list_clusters(service, soa_dir=soa_dir)
else:
clusters = args.clusters.split(",")
if args.components is not None:
components = args.components.split(",")
else:
components = DEFAULT_COMPONENTS
if args.verbose:
log.setLevel(logging.DEBUG)
levels = [DEFAULT_LOGLEVEL, 'debug']
else:
log.setLevel(logging.WARNING)
levels = [DEFAULT_LOGLEVEL]
log.info("Going to get logs for %s on clusters %s" % (service, clusters))
log_reader = get_log_reader()
if args.tail:
log_reader.tail_logs(service, levels, components, clusters, raw_mode=args.raw_mode)
else:
print "Non-tailing actions are not yet supported"
def paasta_emergency_stop(args):
"""Performs an emergency stop on a given service instance on a given cluster
Warning: This command does not permanently stop the service. The next time the service is updated
(config change, deploy, bounce, etc.), those settings will override the emergency stop.
If you want this stop to be permanant, adjust the relevant config file to reflect that.
For example, this can be done for Marathon apps by setting 'instances: 0', or
for Chronos jobs by setting 'disabled: True'. Alternatively, remove the config yaml entirely.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency stop on %s..." % compose_job_id(service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand="stop",
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
app_id=args.appid,
)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
print "To start this service again asap, run:"
print "paasta emergency-start --service %s --instance %s --cluster %s" % (service, args.instance, args.cluster)
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
Warning: This command is not magic and cannot actually get a service to start if it couldn't
run before. This includes configurations that prevent the service from running,
such as 'instances: 0' (for Marathon apps).
All it does for Marathon apps is ask Marathon to resume normal operation by scaling up to
the instance count defined in the service's config.
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency start on %s..." % compose_job_id(
service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand='start',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config)
print "%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:])
print "Output: %s" % PaastaColors.grey(output)
print "Run this command to see the status:"
print "paasta status --service %s --clusters %s" % (service, args.cluster)
def paasta_status(args):
"""Print the status of a Yelp service running on PaaSTA.
:param args: argparse.Namespace obj created from sys.args by cli"""
service = figure_out_service_name(args)
actual_deployments = get_actual_deployments(service)
if args.clusters is not None:
cluster_whitelist = args.clusters.split(",")
else:
cluster_whitelist = []
if args.instances is not None:
instance_whitelist = args.instances.split(",")
else:
instance_whitelist = []
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service))
report_status(
service=service,
deploy_pipeline=deploy_pipeline,
actual_deployments=actual_deployments,
cluster_whitelist=cluster_whitelist,
instance_whitelist=instance_whitelist,
verbose=args.verbose,
)
else:
print missing_deployments_message(service)
def paasta_logs(args):
"""Print the logs for as Paasta service.
:param args: argparse.Namespace obj created from sys.args by cli"""
if scribereader is None:
sys.exit(
"Unfortunately, `paasta logs` is unavailable without Scribe."
" We're working to support alternative logging backends in PaaSTA:"
" follow https://github.com/Yelp/paasta/issues/64 for updates."
)
service = figure_out_service_name(args)
if args.clusters is None:
clusters = list_clusters(service)
else:
clusters = args.clusters.split(",")
if args.components is not None:
components = args.components.split(",")
else:
components = DEFAULT_COMPONENTS
if args.debug:
log.setLevel(logging.DEBUG)
levels = [DEFAULT_LOGLEVEL, 'debug']
else:
log.setLevel(logging.WARNING)
levels = [DEFAULT_LOGLEVEL]
log.info("Going to get logs for %s on clusters %s" % (service, clusters))
if args.tail:
tail_paasta_logs(service, levels, components, clusters, raw_mode=args.raw_mode)
else:
print "Non-tailing actions are not yet supported"
def paasta_emergency_stop(args):
"""Performs an emergency stop on a given service instance on a given cluster
Warning: This command does not permanently stop the service. The next time the service is updated
(config change, deploy, bounce, etc.), those settings will override the emergency stop.
If you want this stop to be permanant, adjust the relevant config file to reflect that.
For example, this can be done for Marathon apps by setting 'instances: 0', or
for Chronos jobs by setting 'disabled: True'. Alternatively, remove the config yaml entirely.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency stop on %s..." % compose_job_id(
service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand='stop',
cluster=args.cluster,
service=args.service,
instances=args.instance,
system_paasta_config=system_paasta_config,
app_id=args.appid)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
print "To start this service again asap, run:"
print "paasta emergency-start --service %s --instance %s --cluster %s" % (
service, args.instance, args.cluster)
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
instance = args.instance
cluster = args.cluster
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))
if 'refs/heads/paasta-%s' % service_config.get_deploy_group() not in remote_refs:
print "No branches found for %s in %s." % \
(service_config.get_deploy_group(), remote_refs)
print "Has it been deployed there yet?"
sys.exit(1)
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
paasta_print("Performing an emergency start on %s..." % compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='start',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
_log_audit(
action='emergency-start',
service=service,
cluster=args.cluster,
instance=args.instance,
)
paasta_print("%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:]))
paasta_print("Output: %s" % PaastaColors.grey(output))
paasta_print("Run this command to see the status:")
paasta_print(f"paasta status --service {service} --clusters {args.cluster}")
return return_code
def paasta_emergency_stop(args):
"""Performs an emergency stop on a given service instance on a given cluster
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
paasta_print("Performing an emergency stop on %s..." %
compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand="stop",
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
_log_audit(
action="emergency-stop",
service=service,
cluster=args.cluster,
instance=args.instance,
)
paasta_print("Output: %s" % output)
paasta_print("%s" %
"\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:]))
paasta_print("To start this service again asap, run:")
paasta_print(
f"paasta emergency-start --service {service} --instance {args.instance} --cluster {args.cluster}"
)
return return_code
def paasta_logs(args):
"""Print the logs for as Paasta service.
:param args: argparse.Namespace obj created from sys.args by cli"""
service = figure_out_service_name(args)
if args.clusters is None:
clusters = list_clusters(service)
else:
clusters = args.clusters.split(",")
if args.components is not None:
components = args.components.split(",")
else:
components = DEFAULT_COMPONENTS
if args.verbose:
log.setLevel(logging.DEBUG)
levels = [DEFAULT_LOGLEVEL, 'debug']
else:
log.setLevel(logging.WARNING)
levels = [DEFAULT_LOGLEVEL]
log.info("Going to get logs for %s on clusters %s" % (service, clusters))
log_reader = get_log_reader()
if args.tail:
log_reader.tail_logs(service,
levels,
components,
clusters,
raw_mode=args.raw_mode)
else:
print "Non-tailing actions are not yet supported"
def paasta_autoscale(args):
log.setLevel(logging.DEBUG)
service = figure_out_service_name(args)
api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
if not api:
paasta_print(
'Could not connect to paasta api. Maybe you misspelled the cluster?'
)
return 1
if args.set is None:
log.debug("Getting the current autoscaler count...")
res, http = api.autoscaler.get_autoscaler_count(
service=service, instance=args.instance).result()
else:
log.debug("Setting desired instances to {}.".format(args.set))
body = {'desired_instances': int(args.set)}
res, http = api.autoscaler.update_autoscaler_count(
service=service,
instance=args.instance,
json_body=body,
).result()
log.debug("Res: {} Http: {}".format(res, http))
print(res["desired_instances"])
return 0
def paasta_emergency_restart(args):
"""Performs an emergency restart on a given service instance on a given cluster
Warning: This command is only intended to be used in an emergency.
It should not be needed in normal circumstances.
"""
service = figure_out_service_name(args, args.soa_dir)
system_paasta_config = load_system_paasta_config()
paasta_print("Performing an emergency restart on %s...\n" %
compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='restart',
cluster=args.cluster,
service=args.service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
paasta_print("Output: %s" % output)
paasta_print("%s" %
"\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:]))
paasta_print("Run this to see the status:")
paasta_print(
f"paasta status --service {service} --clusters {args.cluster}")
return return_code
def paasta_status(args):
"""Print the status of a Yelp service running on PaaSTA.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
actual_deployments = get_actual_deployments(service, soa_dir)
if 'USE_API_ENDPOINT' in os.environ:
use_api_endpoint = strtobool(os.environ.get('USE_API_ENDPOINT'))
else:
use_api_endpoint = False
pargs = paasta_args_mixer(args, service)
if pargs is None:
return 1
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service, soa_dir))
return_code = report_status(
service=service,
deploy_pipeline=deploy_pipeline,
actual_deployments=actual_deployments,
cluster_whitelist=pargs.cluster_whitelist,
instance_whitelist=pargs.instance_whitelist,
system_paasta_config=load_system_paasta_config(),
verbose=args.verbose,
use_api_endpoint=use_api_endpoint)
return return_code
else:
paasta_print(missing_deployments_message(service))
return 1
def paasta_status(args):
"""Print the status of a Yelp service running on PaaSTA.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
actual_deployments = get_actual_deployments(service, soa_dir)
system_paasta_config = load_system_paasta_config()
if args.clusters is not None:
cluster_whitelist = args.clusters.split(",")
else:
cluster_whitelist = []
if args.instances is not None:
instance_whitelist = args.instances.split(",")
else:
instance_whitelist = []
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service, soa_dir))
report_status(
service=service,
deploy_pipeline=deploy_pipeline,
actual_deployments=actual_deployments,
cluster_whitelist=cluster_whitelist,
instance_whitelist=instance_whitelist,
system_paasta_config=system_paasta_config,
verbose=args.verbose,
)
else:
print missing_deployments_message(service)
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
instance = args.instance
cluster = args.cluster
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))
if 'refs/heads/paasta-%s' % service_config.get_deploy_group(
) not in remote_refs:
print "No branches found for %s in %s." % \
(service_config.get_deploy_group(), remote_refs)
print "Has it been deployed there yet?"
sys.exit(1)
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
def apply_args_filters(
args, ) -> Mapping[str, Mapping[str, Mapping[str, Type[InstanceConfig]]]]:
"""
Take an args object and returns the dict of cluster:service:instances
Currently, will filter by clusters, instances, services, and deploy_groups
If no instances are found, will print a message and try to find matching instances
for each service
:param args: args object containing attributes to filter by
:returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}}
"""
clusters_services_instances: DefaultDict[str, DefaultDict[str, Dict[
str, Type[InstanceConfig]]]] = defaultdict(lambda: defaultdict(dict))
if args.service is None and args.owner is None:
args.service = figure_out_service_name(args, soa_dir=args.soa_dir)
filters = get_filters(args)
all_services = list_services(soa_dir=args.soa_dir)
if args.service and args.service not in all_services:
paasta_print(
PaastaColors.red(f'The service "{args.service}" does not exist.'))
suggestions = difflib.get_close_matches(args.service,
all_services,
n=5,
cutoff=0.5)
if suggestions:
paasta_print(PaastaColors.red(f'Did you mean any of these?'))
for suggestion in suggestions:
paasta_print(PaastaColors.red(f' {suggestion}'))
return clusters_services_instances
i_count = 0
for service in all_services:
if args.service and service != args.service:
continue
for instance_conf in get_instance_configs_for_service(
service, soa_dir=args.soa_dir):
if all([f(instance_conf) for f in filters]):
cluster_service = clusters_services_instances[
instance_conf.get_cluster()][service]
cluster_service[
instance_conf.get_instance()] = instance_conf.__class__
i_count += 1
if i_count == 0 and args.service and args.instances:
if args.clusters:
clusters = args.clusters.split(',')
else:
clusters = list_clusters()
for service in args.service.split(','):
verify_instances(args.instances, service, clusters)
return clusters_services_instances
def extract_args(args):
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
paasta_print(
PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates"
"PaaSTA is not configured locally on this host, and remote-run may not behave"
"the same way it would behave on a server configured for PaaSTA.",
),
sep='\n',
)
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = args.cluster or system_paasta_config.get_local_run_config().get('default_cluster', None)
if not cluster:
paasta_print(
PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster."
"Please pass one using '-c'.",
),
sep='\n',
file=sys.stderr,
)
sys.exit(1)
soa_dir = args.yelpsoa_config_root
instance = args.instance
if instance is None:
instance_type = 'adhoc'
instance = 'remote'
else:
instance_type = validate_service_instance(
service, instance, cluster, soa_dir,
)
if instance_type != 'adhoc':
paasta_print(
PaastaColors.red(
(
"Please use instance declared in adhoc.yaml for use "
"with remote-run, {} is declared as {}"
).format(instance, instance_type),
),
)
sys.exit(1)
return (
system_paasta_config,
service,
cluster,
soa_dir,
instance,
instance_type,
)
def paasta_rollback(args):
"""Call mark_for_deployment with rollback parameters
:param args: contains all the arguments passed onto the script: service,
deploy groups and sha. These arguments will be verified and passed onto
mark_for_deployment.
"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
git_url = get_git_url(service, soa_dir)
given_deploy_groups = {
deploy_group
for deploy_group in args.deploy_groups.split(",") if deploy_group
}
service_deploy_groups = {
config.get_deploy_group()
for config in get_instance_config_for_service(
service=service,
soa_dir=soa_dir,
)
}
deploy_groups, invalid = validate_given_deploy_groups(
service_deploy_groups, given_deploy_groups)
if len(invalid) > 0:
print PaastaColors.yellow(
"These deploy groups are not valid and will be skipped: %s.\n" %
(",").join(invalid))
if len(deploy_groups) == 0:
print PaastaColors.red(
"ERROR: No valid deploy groups specified for %s.\n" % (service))
return 1
commit = args.commit
if not commit:
list_previous_commits(service, deploy_groups,
bool(given_deploy_groups), soa_dir)
return 1
returncode = 0
for deploy_group in deploy_groups:
returncode = max(
mark_for_deployment(
git_url=git_url,
service=service,
deploy_group=deploy_group,
commit=commit,
),
returncode,
)
return returncode
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
instance = args.instance
clusters = args.clusters
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
if args.clusters is not None:
clusters = args.clusters.split(",")
else:
clusters = list_clusters(service)
try:
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment."
) % str(e)
print msg
return 1
invalid_deploy_groups = []
for cluster in clusters:
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
deploy_group = service_config.get_deploy_group()
(deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
if invalid_deploy_groups:
print "No branches found for %s in %s." % \
(", ".join(invalid_deploy_groups), remote_refs)
print "Has %s been deployed there yet?" % service
return_val = 1
return return_val
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
instance = args.instance
clusters = args.clusters
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
if args.clusters is not None:
clusters = args.clusters.split(",")
else:
clusters = list_clusters(service)
try:
remote_refs = remote_git.list_remote_refs(
utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment.") % str(e)
print msg
return 1
invalid_deploy_groups = []
for cluster in clusters:
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
deploy_group = service_config.get_deploy_group()
(deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
if invalid_deploy_groups:
print "No branches found for %s in %s." % \
(", ".join(invalid_deploy_groups), remote_refs)
print "Has %s been deployed there yet?" % service
return_val = 1
return return_val
def paasta_autoscale(args):
log.setLevel(logging.DEBUG)
service = figure_out_service_name(args)
api = client.get_paasta_oapi_client(cluster=args.cluster, http_res=True)
if not api:
print(
"Could not connect to paasta api. Maybe you misspelled the cluster?"
)
return 1
try:
if args.set is None:
log.debug("Getting the current autoscaler count...")
res, status, _ = api.autoscaler.get_autoscaler_count(
service=service,
instance=args.instance,
_return_http_data_only=False)
else:
log.debug(f"Setting desired instances to {args.set}.")
msg = paastamodels.AutoscalerCountMsg(
desired_instances=int(args.set))
res, status, _ = api.autoscaler.update_autoscaler_count(
service=service,
instance=args.instance,
autoscaler_count_msg=msg,
_return_http_data_only=False,
)
_log_audit(
action="manual-scale",
action_details=str(msg),
service=service,
instance=args.instance,
cluster=args.cluster,
)
except api.api_error as exc:
status = exc.status
if not 200 <= status <= 299:
print(
PaastaColors.red(
f"ERROR: '{args.instance}' is not configured to autoscale, "
f"so paasta autoscale could not scale it up on demand. "
f"If you want to be able to boost this service, please configure autoscaling for the service "
f"in its config file by setting min and max instances. Example: \n"
f"{args.instance}:\n"
f" min_instances: 5\n"
f" max_instances: 50"))
return 0
log.debug(f"Res: {res} Http: {status}")
print(res.desired_instances)
return 0
def paasta_emergency_restart(args):
"""Performs an emergency restart on a given service instance on a given cluster
Warning: This command is only intended to be used in an emergency.
It should not be needed in normal circumstances.
"""
service = figure_out_service_name(args)
print "Performing an emergency restart on %s...\n" % compose_job_id(service, args.instance)
execute_paasta_serviceinit_on_remote_master('restart', args.cluster, service, args.instance)
print "%s" % "\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:])
print "Run this to see the status:"
print "paasta status --service %s --clusters %s" % (service, args.cluster)
def extract_args(args):
system_paasta_config = get_system_paasta_config()
soa_dir = args.yelpsoa_config_root
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = args.cluster or \
system_paasta_config.get_remote_run_config().get('default_cluster', None)
if not cluster:
paasta_print(
PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster."
"Please pass one using '-c'.", ))
emit_counter_metric('paasta.remote_run.' + args.action + '.failed',
service, 'UNKNOWN')
sys.exit(1)
instance = args.instance
if instance is None:
instance_type = 'adhoc'
instance = 'remote'
else:
try:
instance_type = validate_service_instance(
service,
instance,
cluster,
soa_dir,
)
except NoConfigurationForServiceError as e:
paasta_print(e)
emit_counter_metric('paasta.remote_run.' + args.action + '.failed',
service, instance)
sys.exit(1)
if instance_type != 'adhoc':
paasta_print(
PaastaColors.red(
"Please use instance declared in adhoc.yaml for use "
f"with remote-run, {instance} is declared as {instance_type}",
))
emit_counter_metric('paasta.remote_run.' + args.action + '.failed',
service, instance)
sys.exit(1)
return (
system_paasta_config,
service,
cluster,
soa_dir,
instance,
instance_type,
)
def paasta_rollback(args):
"""Call mark_for_deployment with rollback parameters
:param args: contains all the arguments passed onto the script: service,
deploy groups and sha. These arguments will be verified and passed onto
mark_for_deployment.
"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
git_url = get_git_url(service, soa_dir)
given_deploy_groups = {deploy_group for deploy_group in args.deploy_groups.split(",") if deploy_group}
all_deploy_groups = list_deploy_groups(service=service, soa_dir=soa_dir)
deploy_groups, invalid = validate_given_deploy_groups(all_deploy_groups, given_deploy_groups)
if len(invalid) > 0:
paasta_print(
PaastaColors.yellow(
"These deploy groups are not valid and will be skipped: %s.\n" % (",").join(invalid),
),
)
if len(deploy_groups) == 0:
paasta_print(PaastaColors.red("ERROR: No valid deploy groups specified for %s.\n" % (service)))
return 1
git_shas = get_git_shas_for_service(service, deploy_groups, soa_dir)
commit = args.commit
if not commit:
paasta_print("Please specify a commit to mark for rollback (-k, --commit).")
list_previous_commits(service, deploy_groups, bool(given_deploy_groups), git_shas)
return 1
elif commit not in git_shas and not args.force:
paasta_print(PaastaColors.red("This Git SHA has never been deployed before."))
paasta_print("Please double check it or use --force to skip this verification.\n")
list_previous_commits(service, deploy_groups, bool(given_deploy_groups), git_shas)
return 1
returncode = 0
for deploy_group in deploy_groups:
returncode = max(
mark_for_deployment(
git_url=git_url,
service=service,
deploy_group=deploy_group,
commit=commit,
),
returncode,
)
return returncode
def paasta_emergency_restart(args):
"""Performs an emergency restart on a given service instance on a given cluster
Warning: This command is only intended to be used in an emergency.
It should not be needed in normal circumstances.
"""
service = figure_out_service_name(args)
print "Performing an emergency restart on %s...\n" % compose_job_id(
service, args.instance)
execute_paasta_serviceinit_on_remote_master('restart', args.cluster,
service, args.instance)
print "%s" % "\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:])
print "Run this to see the status:"
print "paasta status --service %s --clusters %s" % (service, args.cluster)
def paasta_emergency_scale(args):
"""Performs an emergency scale on a given service instance on a given cluster
Warning: This command does not permanently scale the service. The next time the service is updated
(config change, deploy, bounce, etc.), those settings will override the emergency scale.
If you want this scale to be permanant, adjust the relevant config file to reflect that.
For example, this can be done for Marathon apps by setting 'instances: n'
"""
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
print "Performing an emergency scale on %s..." % compose_job_id(service, args.instance)
output = execute_paasta_serviceinit_on_remote_master('scale', args.cluster, service, args.instance,
app_id=args.appid, delta=args.delta)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_scale.__doc__.splitlines()[-7:])
def paasta_emergency_scale(args):
"""Performs an emergency scale on a given service instance on a given cluster
Warning: This command does not permanently scale the service. The next time the service is updated
(config change, deploy, bounce, etc.), those settings will override the emergency scale.
If you want this scale to be permanant, adjust the relevant config file to reflect that.
For example, this can be done for Marathon apps by setting 'instances: n'
"""
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
print "Performing an emergency scale on %s..." % compose_job_id(service, args.instance)
output = execute_paasta_serviceinit_on_remote_master('scale', args.cluster, service, args.instance,
app_id=args.appid, delta=args.delta)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_scale.__doc__.splitlines()[-7:])
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write(
"A local Makefile with a 'cook-image' target is required for --build\n"
)
sys.stderr.write(
"If you meant to pull the docker image from the registry, explicitly pass --pull\n"
)
return 1
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None,
service=service,
soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def paasta_check(args):
"""Analyze the service in the PWD to determine if it is paasta ready
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.yelpsoa_config_root
service = figure_out_service_name(args, soa_dir)
service_path = os.path.join(soa_dir, service)
service_dir_check(service, soa_dir)
deploy_check(service_path)
deploy_has_security_check(service, soa_dir)
git_repo_check(service, soa_dir)
docker_check()
makefile_check()
deployments_check(service, soa_dir)
sensu_check(service, service_path, soa_dir)
smartstack_check(service, service_path, soa_dir)
paasta_validate_soa_configs(service, service_path)
def paasta_autoscale(args):
log.setLevel(logging.DEBUG)
service = figure_out_service_name(args)
api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
if not api:
paasta_print(
"Could not connect to paasta api. Maybe you misspelled the cluster?"
)
return 1
try:
if args.set is None:
log.debug("Getting the current autoscaler count...")
res, http = api.autoscaler.get_autoscaler_count(
service=service, instance=args.instance
).result()
else:
log.debug(f"Setting desired instances to {args.set}.")
body = {"desired_instances": int(args.set)}
res, http = api.autoscaler.update_autoscaler_count(
service=service, instance=args.instance, json_body=body
).result()
_log_audit(
action="manual-scale",
action_details=body,
service=service,
instance=args.instance,
cluster=args.cluster,
)
except HTTPNotFound:
paasta_print(
PaastaColors.red(
f"ERROR: '{args.instance}' is not configured to autoscale, "
f"so paasta autoscale could not scale it up on demand. "
f"If you want to be able to boost this service, please configure autoscaling for the service "
f"in its config file by setting min and max instances. Example: \n"
f"{args.instance}:\n"
f" min_instances: 5\n"
f" max_instances: 50"
)
)
return 0
log.debug(f"Res: {res} Http: {http}")
print(res["desired_instances"])
return 0
def paasta_check(args):
"""Analyze the service in the PWD to determine if it is paasta ready
:param args: argparse.Namespace obj created from sys.args by cli"""
service = figure_out_service_name(args)
service_path = os.path.join('/nail/etc/services', service)
service_dir_check(service)
deploy_check(service_path)
deploy_has_security_check(service)
deploy_has_performance_check(service)
pipeline_check(service)
git_repo_check(service)
docker_check()
makefile_check()
yaml_check(service_path)
deployments_check(service_path)
sensu_check(service, service_path)
smartstack_check(service, service_path)
def paasta_local_run(args):
if args.pull or args.dry_run:
build = False
elif args.build:
build = True
else:
build = local_makefile_present()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if build:
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None,
service=service,
soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.dry_run:
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.dry_run,
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def paasta_rollback(args):
"""Call mark_for_deployment with rollback parameters
:param args: contains all the arguments passed onto the script: service,
deploy groups and sha. These arguments will be verified and passed onto
mark_for_deployment.
"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
git_url = get_git_url(service, soa_dir)
given_deploy_groups = {deploy_group for deploy_group in args.deploy_groups.split(",") if deploy_group}
service_deploy_groups = {config.get_deploy_group() for config in get_instance_config_for_service(
service=service,
soa_dir=soa_dir,
)}
deploy_groups, invalid = validate_given_deploy_groups(service_deploy_groups, given_deploy_groups)
if len(invalid) > 0:
print PaastaColors.yellow("These deploy groups are not valid and will be skipped: %s.\n" % (",").join(invalid))
if len(deploy_groups) == 0:
print PaastaColors.red("ERROR: No valid deploy groups specified for %s.\n" % (service))
return 1
commit = args.commit
if not commit:
list_previous_commits(service, deploy_groups, bool(given_deploy_groups), soa_dir)
return 1
returncode = 0
for deploy_group in deploy_groups:
returncode = max(
mark_for_deployment(
git_url=git_url,
service=service,
deploy_group=deploy_group,
commit=commit,
),
returncode,
)
return returncode
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
service = figure_out_service_name(args)
instance = args.instance
cluster = args.cluster
branch = "paasta-%s.%s" % (cluster, instance)
all_branches = list(get_branches(service))
if branch not in all_branches:
print "No branches found for %s in %s." % \
(branch, all_branches)
print "Has it been deployed there yet?"
sys.exit(1)
force_bounce = format_timestamp(datetime.datetime.utcnow())
branches = [branch]
issue_state_change_for_branches(service, instance, cluster, branches, force_bounce,
desired_state)
def paasta_check(args):
"""Analyze the service in the PWD to determine if it is paasta ready
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.yelpsoa_config_root
service = figure_out_service_name(args, soa_dir)
service_path = os.path.join(soa_dir, service)
service_dir_check(service, soa_dir)
deploy_check(service_path)
deploy_has_security_check(service, soa_dir)
deploy_has_performance_check(service, soa_dir)
git_repo_check(service)
docker_check()
makefile_check()
yaml_check(service_path)
deployments_check(service, soa_dir)
sensu_check(service, service_path, soa_dir)
smartstack_check(service, service_path, soa_dir)
paasta_validate_soa_configs(service_path)
def paasta_emergency_stop(args):
"""Performs an emergency stop on a given service instance on a given cluster
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency stop on %s..." % compose_job_id(service, args.instance)
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='stop',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
print "To start this service again asap, run:"
print "paasta emergency-start --service %s --instance %s --cluster %s" % (service, args.instance, args.cluster)
return return_code
def paasta_emergency_stop(args):
"""Performs an emergency stop on a given service instance on a given cluster
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency stop on %s..." % compose_job_id(
service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand='stop',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
print "To start this service again asap, run:"
print "paasta emergency-start --service %s --instance %s --cluster %s" % (
service, args.instance, args.cluster)
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
service = figure_out_service_name(args)
instance = args.instance
cluster = args.cluster
branch = "paasta-%s.%s" % (cluster, instance)
all_branches = list(get_branches(service))
if branch not in all_branches:
print "No branches found for %s in %s." % \
(branch, all_branches)
print "Has it been deployed there yet?"
sys.exit(1)
force_bounce = format_timestamp(datetime.datetime.utcnow())
branches = [branch]
issue_state_change_for_branches(service, instance, cluster, branches,
force_bounce, desired_state)
def paasta_local_run(args):
if args.pull or args.dry_run:
build = False
elif args.build:
build = True
else:
build = local_makefile_present()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if build:
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.dry_run:
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.dry_run,
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def paasta_rollback(args):
"""Call mark_for_deployment with rollback parameters
:param args: contains all the arguments passed onto the script: service,
cluster, instance and sha. These arguments will be verified and passed onto
mark_for_deployment.
"""
service = figure_out_service_name(args)
cluster = args.cluster
git_url = get_git_url(service)
commit = args.commit
given_instances = args.instances.split(",")
if cluster in list_clusters(service):
service_instances = list_all_instances_for_service(service)
instances, invalid = validate_given_instances(service_instances,
given_instances)
if len(invalid) > 0:
print PaastaColors.yellow(
"These instances are not valid and will be skipped: %s.\n" %
(",").join(invalid))
if len(instances) is 0:
print PaastaColors.red(
"ERROR: No valid instances specified for %s.\n" % (service))
returncode = 1
for instance in instances:
returncode = mark_for_deployment(
git_url=git_url,
cluster=cluster,
instance=instance,
service=service,
commit=commit,
)
else:
print PaastaColors.red(
"ERROR: The service %s is not deployed into cluster %s.\n" %
(service, cluster))
returncode = 1
sys.exit(returncode)
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
paasta_print("Performing an emergency start on %s..." % compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='start',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config
)
paasta_print("%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:]))
paasta_print("Output: %s" % PaastaColors.grey(output))
paasta_print("Run this command to see the status:")
paasta_print("paasta status --service %s --clusters %s" % (service, args.cluster))
return return_code
def paasta_rollback(args):
"""Call mark_for_deployment with rollback parameters
:param args: contains all the arguments passed onto the script: service,
deploy groups and sha. These arguments will be verified and passed onto
mark_for_deployment.
"""
service = figure_out_service_name(args)
git_url = get_git_url(service)
commit = args.commit
given_deploy_groups = [
deploy_group for deploy_group in args.deploy_groups.split(",")
if deploy_group
]
service_deploy_groups = set(config.get_deploy_group()
for config in get_instance_config_for_service(
soa_dir=DEFAULT_SOA_DIR,
service=service,
))
deploy_groups, invalid = validate_given_deploy_groups(
service_deploy_groups, given_deploy_groups)
if len(invalid) > 0:
print PaastaColors.yellow(
"These deploy groups are not valid and will be skipped: %s.\n" %
(",").join(invalid))
if len(deploy_groups) == 0:
print PaastaColors.red(
"ERROR: No valid deploy groups specified for %s.\n" % (service))
returncode = 1
for deploy_group in deploy_groups:
returncode = mark_for_deployment(
git_url=git_url,
service=service,
deploy_group=deploy_group,
commit=commit,
)
sys.exit(returncode)
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
return 1
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
instance = args.instance
cluster = args.cluster
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
try:
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment."
) % str(e)
print msg
return 1
if 'refs/heads/paasta-%s' % service_config.get_deploy_group() not in remote_refs:
print "No branches found for %s in %s." % \
(service_config.get_deploy_group(), remote_refs)
print "Has it been deployed there yet?"
return 1
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
def paasta_status(args):
"""Print the status of a Yelp service running on PaaSTA.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
actual_deployments = get_actual_deployments(service, soa_dir)
system_paasta_config = load_system_paasta_config()
if 'USE_API_ENDPOINT' in os.environ:
use_api_endpoint = strtobool(os.environ.get('USE_API_ENDPOINT'))
else:
use_api_endpoint = False
if args.clusters is not None:
cluster_whitelist = args.clusters.split(",")
else:
cluster_whitelist = []
if args.instances is not None:
instance_whitelist = args.instances.split(",")
else:
instance_whitelist = []
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service, soa_dir))
try:
report_status(service=service,
deploy_pipeline=deploy_pipeline,
actual_deployments=actual_deployments,
cluster_whitelist=cluster_whitelist,
instance_whitelist=instance_whitelist,
system_paasta_config=system_paasta_config,
verbose=args.verbose,
use_api_endpoint=use_api_endpoint)
except CalledProcessError as e:
print PaastaColors.grey(
PaastaColors.bold(e.cmd +
" exited with non-zero return code."))
print PaastaColors.grey(e.output)
return e.returncode
else:
print missing_deployments_message(service)
def paasta_emergency_restart(args):
"""Performs an emergency restart on a given service instance on a given cluster
Warning: This command is only intended to be used in an emergency.
It should not be needed in normal circumstances.
"""
service = figure_out_service_name(args, args.soa_dir)
system_paasta_config = load_system_paasta_config()
paasta_print("Performing an emergency restart on %s...\n" % compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='restart',
cluster=args.cluster,
service=args.service,
instances=args.instance,
system_paasta_config=system_paasta_config
)
paasta_print("Output: %s" % output)
paasta_print("%s" % "\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:]))
paasta_print("Run this to see the status:")
paasta_print("paasta status --service %s --clusters %s" % (service, args.cluster))
return return_code
def apply_args_filters(args):
"""
Take an args object and returns the dict of cluster:service:instances
Currently, will filter by clusters, instances, services, and deploy_groups
If no instances are found, will print a message and try to find matching instances
for each service
:param args: args object containing attributes to filter by
:returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}}
"""
clusters_services_instances = defaultdict(lambda: defaultdict(set))
if args.service is None and args.owner is None:
args.service = figure_out_service_name(args, soa_dir=args.soa_dir)
filters = get_filters(args)
i_count = 0
for service in list_services(soa_dir=args.soa_dir):
if args.service and service != args.service:
continue
for instance_conf in get_instance_configs_for_service(
service, soa_dir=args.soa_dir):
if all([f(instance_conf) for f in filters]):
clusters_services_instances[
instance_conf.get_cluster()][service].add(
instance_conf.get_instance())
i_count += 1
if i_count == 0 and args.service and args.instances:
if args.clusters:
clusters = args.clusters.split(',')
else:
clusters = list_clusters()
for service in args.service.split(','):
verify_instances(args.instances, service, clusters)
return clusters_services_instances
def paasta_local_run(args):
if args.pull:
build = False
elif args.build:
build = True
else:
build = local_makefile_present()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
base_docker_url = get_docker_host()
docker_client = Client(base_url=base_docker_url)
if build:
default_tag = "paasta-local-run-%s-%s" % (service, get_username())
tag = os.environ.get("DOCKER_TAG", default_tag)
os.environ["DOCKER_TAG"] = tag
pull_image = False
paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
)
except errors.APIError as e:
sys.stderr.write("Can't run Docker container. Error: %s\n" % str(e))
sys.exit(1)
def paasta_status(args):
"""Print the status of a Yelp service running on PaaSTA.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
actual_deployments = get_actual_deployments(service, soa_dir)
system_paasta_config = load_system_paasta_config()
if 'USE_API_ENDPOINT' in os.environ:
use_api_endpoint = strtobool(os.environ.get('USE_API_ENDPOINT'))
else:
use_api_endpoint = False
if args.clusters is not None:
cluster_whitelist = args.clusters.split(",")
else:
cluster_whitelist = []
if args.instances is not None:
instance_whitelist = args.instances.split(",")
else:
instance_whitelist = []
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service, soa_dir))
return_code = report_status(
service=service,
deploy_pipeline=deploy_pipeline,
actual_deployments=actual_deployments,
cluster_whitelist=cluster_whitelist,
instance_whitelist=instance_whitelist,
system_paasta_config=system_paasta_config,
verbose=args.verbose,
use_api_endpoint=use_api_endpoint
)
return return_code
else:
paasta_print(missing_deployments_message(service))
return 1
def paasta_rollback(args):
"""Call mark_for_deployment with rollback parameters
:param args: contains all the arguments passed onto the script: service,
cluster, instance and sha. These arguments will be verified and passed onto
mark_for_deployment.
"""
service = figure_out_service_name(args)
cluster = args.cluster
git_url = get_git_url(service)
commit = args.commit
given_instances = args.instances.split(",")
if cluster in list_clusters(service):
service_instances = list_all_instances_for_service(service)
instances, invalid = validate_given_instances(service_instances, given_instances)
if len(invalid) > 0:
print PaastaColors.yellow("These instances are not valid and will be skipped: %s.\n" % (",").join(invalid))
if len(instances) is 0:
print PaastaColors.red("ERROR: No valid instances specified for %s.\n" % (service))
returncode = 1
for instance in instances:
returncode = mark_for_deployment(
git_url=git_url,
cluster=cluster,
instance=instance,
service=service,
commit=commit,
)
else:
print PaastaColors.red("ERROR: The service %s is not deployed into cluster %s.\n" % (service, cluster))
returncode = 1
sys.exit(returncode)
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
if args.clusters is not None:
clusters = args.clusters.split(",")
else:
clusters = list_clusters(service)
if args.instances is not None:
instances = args.instances.split(",")
else:
instances = None
try:
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment."
) % str(e)
print msg
return 1
invalid_deploy_groups = []
marathon_message_printed, chronos_message_printed = False, False
for cluster in clusters:
# If they haven't specified what instances to act on, do it for all of them.
# If they have specified what instances, only iterate over them if they're
# actually within this cluster.
if instances is None:
cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
else:
all_cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
cluster_instances = all_cluster_instances.intersection(set(instances))
for instance in cluster_instances:
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
deploy_group = service_config.get_deploy_group()
(deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
if isinstance(service_config, MarathonServiceConfig) and not marathon_message_printed:
print_marathon_message(desired_state)
marathon_message_printed = True
elif isinstance(service_config, ChronosJobConfig) and not chronos_message_printed:
print_chronos_message(desired_state)
chronos_message_printed = True
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
if invalid_deploy_groups:
print "No branches found for %s in %s." % \
(", ".join(invalid_deploy_groups), remote_refs)
print "Has %s been deployed there yet?" % service
return_val = 1
return return_val
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
sys.stdout.write(PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates\n"
"PaaSTA is not configured locally on this host, and local-run may not behave\n"
"the same way it would behave on a server configured for PaaSTA.\n"
))
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config['default_cluster']
except KeyError:
sys.stderr.write(PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster.\n"
"Please pass one to local-run using '-c'.\n"))
return 1
instance = args.instance
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def paasta_logs(args):
"""Print the logs for as Paasta service.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
if args.clusters is None:
clusters = list_clusters(service, soa_dir=soa_dir)
else:
clusters = args.clusters.split(",")
if args.instances is None:
instances = None
else:
instances = args.instances.split(",")
if args.components is not None:
components = args.components.split(",")
else:
components = DEFAULT_COMPONENTS
components = set(components)
if "app_output" in components:
components.remove("app_output")
components.add("stdout")
components.add("stderr")
if args.verbose:
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.WARNING)
levels = [DEFAULT_LOGLEVEL, "debug"]
log.info("Going to get logs for %s on clusters %s" % (service, clusters))
log_reader = get_log_reader()
if not validate_filtering_args(args, log_reader):
return 1
# They haven't specified what kind of filtering they want, decide for them
if args.line_count is None and args.time_from is None and not args.tail:
return pick_default_log_mode(args, log_reader, service, levels, components, clusters, instances)
if args.tail:
paasta_print(PaastaColors.cyan("Tailing logs"), file=sys.stderr)
log_reader.tail_logs(
service=service,
levels=levels,
components=components,
clusters=clusters,
instances=instances,
raw_mode=args.raw_mode,
)
return 0
# If the logger doesn't support offsetting the number of lines by a particular line number
# there is no point in distinguishing between a positive/negative number of lines since it
# can only get the last N lines
if not log_reader.SUPPORTS_LINE_OFFSET and args.line_count is not None:
args.line_count = abs(args.line_count)
# Handle line based filtering
if args.line_count is not None and args.line_offset is None:
log_reader.print_last_n_logs(
service=service,
line_count=args.line_count,
levels=levels,
components=components,
clusters=clusters,
instances=instances,
raw_mode=args.raw_mode,
)
return 0
elif args.line_count is not None and args.line_offset is not None:
log_reader.print_logs_by_offset(
service=service,
line_count=args.line_count,
line_offset=args.line_offset,
levels=levels,
components=components,
cluters=clusters,
instances=instances,
raw_mode=args.raw_mode,
)
return 0
# Handle time based filtering
try:
start_time, end_time = generate_start_end_time(args.time_from, args.time_to)
except ValueError as e:
paasta_print(PaastaColors.red(e.message), file=sys.stderr)
return 1
log_reader.print_logs_by_time(
service=service,
start_time=start_time,
end_time=end_time,
levels=levels,
components=components,
clusters=clusters,
instances=instances,
raw_mode=args.raw_mode,
)
def paasta_info(args):
"""Prints general information about a service"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir=soa_dir)
print get_service_info(service, soa_dir)
def paasta_rerun(args):
"""Reruns a Chronos job.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir) # exit with an error if the service doesn't exist
if args.execution_date:
execution_date = args.execution_date
else:
execution_date = None
all_clusters = list_clusters(soa_dir=soa_dir)
actual_deployments = get_actual_deployments(service, soa_dir) # cluster.instance: sha
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service, soa_dir)) # cluster.instance
deployed_clusters = list_deployed_clusters(deploy_pipeline, actual_deployments)
deployed_cluster_instance = _get_cluster_instance(actual_deployments.keys())
if args.clusters is not None:
clusters = args.clusters.split(",")
else:
clusters = deployed_clusters
for cluster in clusters:
print "cluster: %s" % cluster
if cluster not in all_clusters:
print " Warning: \"%s\" does not look like a valid cluster." % cluster
continue
if cluster not in deployed_clusters:
print " Warning: service \"%s\" has not been deployed to \"%s\" yet." % (service, cluster)
continue
if not deployed_cluster_instance[cluster].get(args.instance, False):
print (" Warning: instance \"%s\" is either invalid "
"or has not been deployed to \"%s\" yet." % (args.instance, cluster))
continue
try:
chronos_job_config = chronos_tools.load_chronos_job_config(
service, args.instance, cluster, load_deployments=False, soa_dir=soa_dir)
if chronos_tools.uses_time_variables(chronos_job_config) and execution_date is None:
print (" Warning: \"%s\" uses time variables interpolation, "
"please supply a `--execution_date` argument." % args.instance)
continue
except chronos_tools.UnknownChronosJobError as e:
print " Warning: %s" % e.message
continue
if execution_date is None:
execution_date = _get_default_execution_date()
rc, output = execute_chronos_rerun_on_remote_master(
service=service,
instancename=args.instance,
cluster=cluster,
verbose=args.verbose,
execution_date=execution_date.strftime(chronos_tools.EXECUTION_DATE_FORMAT)
)
if rc == 0:
print PaastaColors.green(' successfully created job')
else:
print PaastaColors.red(' error')
print output
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
service = figure_out_service_name(args=args, soa_dir=soa_dir)
instances = args.instances.split(",") if args.instances else None
# assert that each of the clusters that the user specifies are 'valid'
# for the instance list provided; that is, assert that at least one of the instances
# provided in the -i argument is deployed there.
# if there are no instances defined in the args, then assert that the service
# is deployed to that cluster.
# If args.clusters is falsey, then default to *all* clusters that a service is deployed to,
# and we figure out which ones are needed for each service later.
if instances:
instance_clusters = [list_clusters(service, soa_dir, instance) for instance in args.instances]
valid_clusters = sorted(list(set([cluster for cluster_list in instance_clusters for cluster in cluster_list])))
else:
valid_clusters = list_clusters(service, soa_dir)
if args.clusters:
clusters = args.clusters.split(",")
invalid_clusters = [cluster for cluster in clusters if cluster not in valid_clusters]
if invalid_clusters:
print ("Invalid cluster name(s) specified: %s." "Valid options: %s") % (
" ".join(invalid_clusters),
" ".join(valid_clusters),
)
return 1
else:
clusters = valid_clusters
try:
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment."
) % str(e)
print msg
return 1
invalid_deploy_groups = []
marathon_message_printed, chronos_message_printed = False, False
for cluster in clusters:
# If they haven't specified what instances to act on, do it for all of them.
# If they have specified what instances, only iterate over them if they're
# actually within this cluster.
if instances is None:
cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
print ("no instances specified; restarting all instances for service")
else:
all_cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir)
cluster_instances = all_cluster_instances.intersection(set(instances))
for instance in cluster_instances:
service_config = get_instance_config(
service=service, cluster=cluster, instance=instance, soa_dir=soa_dir, load_deployments=False
)
deploy_group = service_config.get_deploy_group()
(deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
if isinstance(service_config, MarathonServiceConfig) and not marathon_message_printed:
print_marathon_message(desired_state)
marathon_message_printed = True
elif isinstance(service_config, ChronosJobConfig) and not chronos_message_printed:
print_chronos_message(desired_state)
chronos_message_printed = True
issue_state_change_for_service(
service_config=service_config, force_bounce=force_bounce, desired_state=desired_state
)
return_val = 0
if invalid_deploy_groups:
print "No branches found for %s in %s." % (", ".join(invalid_deploy_groups), remote_refs)
print "Has %s been deployed there yet?" % service
return_val = 1
return return_val
