def test_run_cancels_timer_thread_on_keyboard_interrupt():
mock_process = mock.Mock()
mock_timer_object = mock.Mock()
with contextlib.nested(
mock.patch("paasta_tools.utils.Popen", autospec=True, return_value=mock_process),
mock.patch("paasta_tools.utils.threading.Timer", autospec=True, return_value=mock_timer_object),
) as (mock_popen, mock_timer):
mock_process.stdout.readline.side_effect = KeyboardInterrupt
with raises(KeyboardInterrupt):
utils._run("sh echo foo", timeout=10)
assert mock_timer_object.cancel.call_count == 1
def paasta_sysdig(args):
if not args.local:
mesos_master = get_any_mesos_master(cluster=args.cluster)
ssh_cmd = 'ssh -At -o LogLevel=QUIET {0} "sudo paasta {1} --local"'.format(mesos_master, ' '.join(sys.argv[1:]))
return_code, output = _run(ssh_cmd)
if return_code != 0:
print output
sys.exit(return_code)
slave, command = output.split(':', 1)
subprocess.call(shlex.split("ssh -tA {0} '{1}'".format(slave, command.strip())))
return
status = get_status_for_instance(cluster=args.cluster,
service=args.service,
instance=args.instance)
slave = pick_slave_from_status(status=status,
host=args.host)
marathon_config = load_marathon_config()
marathon_url = marathon_config.get_url()[0]
marathon_user = marathon_config.get_username()
marathon_pass = marathon_config.get_password()
mesos_url = get_mesos_master().host
marathon_parsed_url = urlparse(marathon_url)
marathon_creds_url = marathon_parsed_url._replace(netloc="{0}:{1}@{2}".format(marathon_user, marathon_pass,
marathon_parsed_url.netloc))
print format_mesos_command(slave, status.marathon.app_id, mesos_url, marathon_creds_url.geturl())
def run_chronos_rerun(context, service_instance):
cmd = (
"python ../paasta_tools/chronos_rerun.py -d %s '%s' "
"2016-03-13T04:50:31"
) % (context.soa_dir, service_instance)
exit_code, output = _run(cmd)
context.exit_code, context.output = exit_code, output
def mark_for_deployment(git_url, cluster, instance, service, commit):
"""Mark a docker image for deployment"""
cmd = build_command(git_url, commit, cluster=cluster, instance=instance)
# Clusterinstance should be in cluster.instance format
returncode, output = _run(
cmd,
timeout=30,
)
loglines = get_loglines(
returncode=returncode,
cmd=cmd,
output=output,
commit=commit,
cluster=cluster,
instance=instance
)
for logline in loglines:
_log(
service=service,
line=logline,
component='deploy',
level='event',
cluster=cluster,
instance=instance,
)
return returncode
def paasta_push_to_registry(args):
"""Upload a docker image to a registry"""
service = args.service
if service and service.startswith('services-'):
service = service.split('services-', 1)[1]
validate_service_name(service)
cmd = build_command(service, args.commit)
loglines = []
returncode, output = _run(
cmd,
timeout=3600,
log=True,
component='build',
service=service,
loglevel='debug'
)
if returncode != 0:
loglines.append('ERROR: Failed to promote image for %s.' % args.commit)
output = get_jenkins_build_output_url()
if output:
loglines.append('See output: %s' % output)
else:
loglines.append('Successfully pushed image for %s to registry' % args.commit)
for logline in loglines:
_log(
service=service,
line=logline,
component='build',
level='event',
)
return returncode
def run_paasta_serviceinit(subcommand, master, service, instancename, cluster, **kwargs):
"""Run 'paasta_serviceinit <subcommand>'. Return the output from running it."""
if 'verbose' in kwargs and kwargs['verbose']:
verbose_flag = "-v "
timeout = 240
else:
verbose_flag = ''
timeout = 60
if 'app_id' in kwargs and kwargs['app_id']:
app_id_flag = "--appid %s " % kwargs['app_id']
else:
app_id_flag = ''
if 'delta' in kwargs and kwargs['delta']:
delta = "--delta %s" % kwargs['delta']
else:
delta = ''
command = 'ssh -A -n %s sudo paasta_serviceinit %s%s%s %s %s' % (
master,
verbose_flag,
app_id_flag,
compose_job_id(service, instancename),
subcommand,
delta
)
log.debug("Running Command: %s" % command)
_, output = _run(command, timeout=timeout)
return output
def paasta_itest(args):
"""Build and test a docker image"""
service = args.service
if service and service.startswith("services-"):
service = service.split("services-", 1)[1]
validate_service_name(service)
tag = build_docker_tag(service, args.commit)
run_env = os.environ.copy()
run_env["DOCKER_TAG"] = tag
cmd = "make itest"
loglines = []
_log(service=service, line="starting itest for %s." % args.commit, component="build", level="event")
returncode, output = _run(
cmd, env=run_env, timeout=3600, log=True, component="build", service=service, loglevel="debug"
)
if returncode != 0:
loglines.append("ERROR: itest failed for %s." % args.commit)
output = get_jenkins_build_output_url()
if output:
loglines.append("See output: %s" % output)
else:
loglines.append("itest passed for %s." % args.commit)
if not check_docker_image(service, args.commit):
loglines.append("ERROR: itest has not created %s" % tag)
returncode = 1
for logline in loglines:
_log(service=service, line=logline, component="build", level="event")
sys.exit(returncode)
def run_cleanup_marathon_job(context, flags, expected_return_code):
cmd = '../paasta_tools/cleanup_marathon_jobs.py --soa-dir %s %s' % (context.soa_dir, flags)
paasta_print('Running cmd %s' % (cmd))
exit_code, output = _run(cmd)
paasta_print(output)
assert exit_code == int(expected_return_code)
def check_ssh_and_sudo_on_master(master, timeout=10):
"""Given a master, attempt to ssh to the master and run a simple command
with sudo to verify that ssh and sudo work properly. Return a tuple of the
success status (True or False) and any output from attempting the check.
"""
check_command = 'ssh -A -n %s sudo paasta_serviceinit -h' % master
rc, output = _run(check_command, timeout=timeout)
if rc == 0:
return (True, None)
if rc == 255: # ssh error
reason = 'Return code was %d which probably means an ssh failure.' % rc
hint = 'HINT: Are you allowed to ssh to this machine %s?' % master
if rc == 1: # sudo error
reason = 'Return code was %d which probably means a sudo failure.' % rc
hint = 'HINT: Is your ssh agent forwarded? (ssh-add -l)'
if rc == -9: # timeout error
reason = 'Return code was %d which probably means ssh took too long and timed out.' % rc
hint = 'HINT: Is there network latency? Try running somewhere closer to the cluster.'
else: # unknown error
reason = 'Return code was %d which is an unknown failure.' % rc
hint = 'HINT: Talk to #operations and pastebin this output'
output = ('ERROR cannot run check command %(check_command)s\n'
'%(reason)s\n'
'%(hint)s\n'
'Output from check command: %(output)s' %
{
'check_command': check_command,
'reason': reason,
'hint': hint,
'output': output,
})
return (False, output)
def run_paasta_serviceinit(subcommand, master, service, instances, cluster, stream, **kwargs):
"""Run 'paasta_serviceinit <subcommand>'. Return the output from running it."""
if 'verbose' in kwargs and kwargs['verbose'] > 0:
verbose_flag = ' '.join(['-v' for i in range(kwargs['verbose'])])
timeout = 960 if subcommand == 'status' else 240
else:
verbose_flag = ''
timeout = 240 if subcommand == 'status' else 60
if 'app_id' in kwargs and kwargs['app_id']:
app_id_flag = "--appid %s" % kwargs['app_id']
else:
app_id_flag = ''
if 'delta' in kwargs and kwargs['delta']:
delta_flag = "--delta %s" % kwargs['delta']
else:
delta_flag = ''
ssh_flag = '-t' if stream else '-n'
command_parts = [
"ssh -A %s %s sudo paasta_serviceinit" % (ssh_flag, master),
"-s %s" % service,
"-i %s" % instances,
verbose_flag,
app_id_flag,
delta_flag,
subcommand
]
command_without_empty_strings = [part for part in command_parts if part != '']
command = ' '.join(command_without_empty_strings)
log.debug("Running Command: %s" % command)
_, output = _run(command, timeout=timeout, stream=stream)
return output
def paasta_serviceinit_command(context, command, job_id):
cmd = '../paasta_tools/paasta_serviceinit.py --soa-dir %s %s %s' % (context.soa_dir, job_id, command)
print 'Running cmd %s' % cmd
(exit_code, output) = _run(cmd)
print 'Got exitcode %s with output:\n%s' % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert exit_code == 0
def paasta_serviceinit_command_scale(context, delta, job_id):
cmd = "../paasta_tools/paasta_serviceinit.py --soa-dir %s %s scale --delta %s" % (context.soa_dir, job_id, delta)
print "Running cmd %s" % cmd
(exit_code, output) = _run(cmd)
print "Got exitcode %s with output:\n%s" % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert exit_code == 0
def chronos_emergency_restart_job(context):
cmd = '../paasta_tools/paasta_serviceinit.py --soa-dir %s test-service.job restart' % context.soa_dir
print 'Running cmd %s' % cmd
(exit_code, output) = _run(cmd)
print 'Got exitcode %s with output:\n%s' % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert exit_code == 0
def makefile_responds_to(target):
"""Runs `make --dry-run <target>` to detect if a makefile responds to the
specified target."""
cmd = "make --dry-run %s" % target
# According to http://www.gnu.org/software/make/manual/make.html#index-exit-status-of-make,
# 0 means OK, and 2 means error
returncode, _ = _run(cmd, timeout=5)
return returncode == 0
def git_repo_check(service):
git_url = get_git_url(service)
cmd = 'git ls-remote %s' % git_url
returncode, _ = _run(cmd, timeout=5)
if returncode == 0:
print PaastaCheckMessages.GIT_REPO_FOUND
else:
print PaastaCheckMessages.git_repo_missing(git_url)
def chronos_emergency_stop_job(context, service_instance):
cmd = '../paasta_tools/paasta_serviceinit.py --soa-dir %s %s stop' % (context.soa_dir, service_instance)
print 'Running cmd %s' % cmd
exit_code, output = _run(cmd)
print 'Got exitcode %s with output:\n%s' % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert exit_code == 0
def chronos_status_verbose_returns_healthy(context):
cmd = '../paasta_tools/paasta_serviceinit.py --soa-dir %s test-service.job status --verbose' % context.soa_dir
print 'Running cmd %s' % cmd
(exit_code, output) = _run(cmd)
print 'Got exitcode %s with output:\n%s' % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert exit_code == 0
assert "Running Tasks:" in output
def chronos_status_verbose_returns_healthy(context, service_instance):
cmd = "../paasta_tools/paasta_serviceinit.py --soa-dir %s %s status --verbose" % (context.soa_dir, service_instance)
print "Running cmd %s" % cmd
(exit_code, output) = _run(cmd)
print "Got exitcode %s with output:\n%s" % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert exit_code == 0
assert "Running Tasks:" in output
def run_cleanup_marathon_job(context, flags, expected_return_code):
cmd = '../paasta_tools/cleanup_marathon_jobs.py --soa-dir %s %s' % (context.soa_dir, flags)
env = dict(os.environ)
env['MESOS_CLI_CONFIG'] = '/nail/etc/mesos-cli.json' # context.mesos_cli_config_filename
print 'Running cmd %s with MESOS_CLI_CONFIG=%s' % (cmd, env['MESOS_CLI_CONFIG'])
exit_code, output = _run(cmd, env=env)
print output
assert exit_code == int(expected_return_code)
def paasta_serviceinit_tail_stdstreams(context, service_instance):
cmd = "../paasta_tools/paasta_serviceinit.py --soa-dir %s %s status -vv" % (context.soa_dir, service_instance)
print 'Running cmd %s' % cmd
exit_code, output = _run(cmd)
print 'Got exitcode %s with output:\n%s' % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert exit_code == 0
assert "stdout EOF" in output
def local_run_on_chronos_job_with_cmd(context):
with Path("fake_simple_service"):
local_run_cmd = ("paasta local-run "
"--yelpsoa-config-root ../fake_soa_configs_local_run/ "
"--service fake_simple_service "
"--cluster test-cluster "
"--instance chronos_job_with_cmd "
"--build ")
context.local_run_return_code, context.local_run_output = _run(command=local_run_cmd, timeout=60)
def paasta_serviceinit_status_single_instance(context, service, instances):
cmd = '../paasta_tools/paasta_serviceinit.py --soa-dir %s -s %s -i %s status' % \
(context.soa_dir, service, instances)
print 'Running cmd %s' % cmd
exit_code, output = _run(cmd)
print 'Got exitcode %s with output:\n%s' % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert "Running" in output
assert exit_code == 0
def run_paasta_metastatus(master, verbose=0):
if verbose > 0:
verbose_flag = " -%s" % "v" * verbose
timeout = 120
else:
verbose_flag = ""
timeout = 20
command = "ssh -A -n %s sudo paasta_metastatus%s" % (master, verbose_flag)
_, output = _run(command, timeout=timeout)
return output
def chronos_status_returns_healthy(context, service_instance):
cmd = '../paasta_tools/paasta_serviceinit.py --soa-dir %s %s status' % (context.soa_dir, service_instance)
print 'Running cmd %s' % cmd
exit_code, output = _run(cmd)
print 'Got exitcode %s with output:\n%s' % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert exit_code == 0
assert "Disabled" in output
assert "New" in output
def run_chronos_rerun(master, service, instancename, **kwargs):
timeout = 60
verbose_flags = '-v ' * kwargs['verbose']
command = 'ssh -A -n %s \'sudo chronos_rerun %s"%s %s" "%s"\'' % (
master,
verbose_flags,
service,
instancename,
kwargs['execution_date'],
)
return _run(command, timeout=timeout)
def paasta_serviceinit_status_multi_instances(context, service, instances):
cmd = '../paasta_tools/paasta_serviceinit.py --soa-dir %s -s %s -i %s status' % \
(context.soa_dir, service, instances)
paasta_print('Running cmd %s' % cmd)
exit_code, output = _run(cmd)
paasta_print('Got exitcode %s with output:\n%s' % (exit_code, output))
paasta_print() # sacrificial line for behave to eat instead of our output
# one service is deployed and the other is not
assert "Configured" in output
assert exit_code != 0
def docker_pull_image(docker_url):
"""Pull an image via ``docker pull``. Uses the actual pull command instead of the python
bindings due to the docker auth/registry transition. Once we are past Docker 1.6
we can use better credential management, but for now this function assumes the
user running the command has already been authorized for the registry"""
sys.stderr.write("Please wait while the image (%s) is pulled...\n" % docker_url)
DEVNULL = open(os.devnull, 'wb')
ret, output = _run('docker pull %s' % docker_url, stream=True, stdin=DEVNULL)
if ret != 0:
sys.stderr.write("\nPull failed. Are you authorized to run docker commands?\n")
sys.exit(ret)
def check_cleanup_chronos_jobs_output(context, expected_return_code):
cmd = '../paasta_tools/cleanup_chronos_jobs.py --soa-dir %s' % context.soa_dir
exit_code, output = _run(cmd)
paasta_print(context.unconfigured_job_names)
paasta_print('Got exitcode %s with output:\n%s' % (exit_code, output))
assert exit_code == int(expected_return_code)
assert "Successfully Removed Tasks (if any were running) for:" in output
assert "Successfully Removed Jobs:" in output
for job in context.unconfigured_job_names:
assert ' %s' % job in output
def paasta_serviceinit_command_appid(context, command, job_id):
(service, instance, _, __) = decompose_job_id(job_id)
app_id = marathon_tools.create_complete_config(service, instance, None, soa_dir=context.soa_dir)['id']
cmd = '../paasta_tools/paasta_serviceinit.py --soa-dir %s --appid %s %s %s' \
% (context.soa_dir, app_id, job_id, command)
print 'Running cmd %s' % cmd
(exit_code, output) = _run(cmd)
print 'Got exitcode %s with output:\n%s' % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert exit_code == 0
def paasta_serviceinit_tail_stdstreams(context, service_instance):
cmd = "../paasta_tools/paasta_serviceinit.py --soa-dir %s %s status -vv" % (context.soa_dir, service_instance)
print 'Running cmd %s' % cmd
exit_code, output = _run(cmd)
print 'Got exitcode %s with output:\n%s' % (exit_code, output)
print # sacrificial line for behave to eat instead of our output
assert exit_code == 0
# The container we run doesn't really have a stdout/stderr. The message below
# comes from mesos.cli, proving that paasta_serviceinit tried to read stdout/sdterr,
# caught the Exception and presented the right information to the user.
assert "No such task has the requested file or directory" in output
def docker_pull_image(docker_url):
"""Pull an image via ``docker pull``. Uses the actual pull command instead of the python
bindings due to the docker auth/registry transition. Once we are past Docker 1.6
we can use better credential management, but for now this function assumes the
user running the command has already been authorized for the registry"""
paasta_print(
"Please wait while the image (%s) is pulled (times out after 30m)..." %
docker_url,
file=sys.stderr)
DEVNULL = open(os.devnull, 'wb')
with open('/tmp/paasta-local-run-pull.lock', 'w') as f:
with timed_flock(f, seconds=1800):
ret, output = _run('docker pull %s' % docker_url,
stream=True,
stdin=DEVNULL)
if ret != 0:
paasta_print(
"\nPull failed. Are you authorized to run docker commands?",
file=sys.stderr,
)
sys.exit(ret)
def run_paasta_metastatus(
master: str,
groupings: Sequence[str],
verbose: int = 0,
autoscaling_info: bool = False,
use_mesos_cache: bool = False,
) -> Tuple[int, str]:
cmd_args, timeout = get_paasta_metastatus_cmd_args(
groupings=groupings,
verbose=verbose,
autoscaling_info=autoscaling_info,
use_mesos_cache=use_mesos_cache,
)
command = (
'ssh -A -n -o StrictHostKeyChecking=no {} sudo paasta_metastatus {}'.
format(
master,
" ".join(cmd_args),
)).strip()
return_code, output = _run(command, timeout=timeout)
return return_code, output
def run_paasta_cluster_boost(
master,
action,
pool,
duration,
override,
boost,
verbose,
):
timeout = 20
verbose_flag: Optional[str]
if verbose > 0:
verbose_flag = '-{}'.format('v' * verbose)
else:
verbose_flag = None
pool_flag = f'--pool {pool}'
duration_flag = f'--duration {duration}' if duration is not None else ''
boost_flag = f'--boost {boost}' if boost is not None else ''
override_flag = '--force' if override is not None else ''
cmd_args = " ".join(
filter(
None, [
action,
pool_flag,
duration_flag,
boost_flag,
override_flag,
verbose_flag,
],
),
)
command = ('ssh -A -n -o StrictHostKeyChecking=no {} paasta_cluster_boost {}'.format(
master,
cmd_args,
)).strip()
return_code, output = _run(command, timeout=timeout)
return return_code, output
def run_paasta_metastatus(
master,
humanize,
groupings,
verbose=0,
autoscaling_info=False,
use_mesos_cache=False,
):
if verbose > 0:
verbose_flag = "-%s" % ('v' * verbose)
timeout = 120
else:
verbose_flag = ''
timeout = 20
autoscaling_flag = "-a" if autoscaling_info else ""
if autoscaling_flag and verbose < 2:
verbose_flag = '-vv'
humanize_flag = "-H" if humanize else ''
groupings_flag = "-g %s" % " ".join(groupings) if groupings else ''
cache_flag = "--use-mesos-cache" if use_mesos_cache else ''
cmd_args = " ".join(
filter(
None,
[
verbose_flag,
humanize_flag,
groupings_flag,
autoscaling_flag,
cache_flag,
],
), )
command = (
'ssh -A -n -o StrictHostKeyChecking=no %s sudo paasta_metastatus %s' %
(
master,
cmd_args,
)).strip()
return_code, output = _run(command, timeout=timeout)
return return_code, output
def run_paasta_serviceinit(subcommand, master, service, instances, cluster, stream, ssh_flags='', **kwargs):
"""Run 'paasta_serviceinit <subcommand>'. Return the output from running it."""
if 'verbose' in kwargs and kwargs['verbose'] > 0:
verbose_flag = ' '.join(['-v' for i in range(kwargs['verbose'])])
timeout = 960 if subcommand == 'status' else 240
else:
verbose_flag = ''
timeout = 240 if subcommand == 'status' else 60
if 'app_id' in kwargs and kwargs['app_id']:
app_id_flag = "--appid %s" % kwargs['app_id']
else:
app_id_flag = ''
if 'delta' in kwargs and kwargs['delta']:
delta_flag = "--delta %s" % kwargs['delta']
else:
delta_flag = ''
ssh_flags += ' -t' if stream else ' -n'
ssh_flags = ssh_flags.strip()
command_parts = [
"ssh -A %s %s sudo paasta_serviceinit" % (ssh_flags, master),
"-s %s" % service,
"-i %s" % instances,
verbose_flag,
app_id_flag,
delta_flag,
subcommand
]
command_without_empty_strings = [part for part in command_parts if part != '']
command = ' '.join(command_without_empty_strings)
log.debug("Running Command: %s" % command)
return_code, output = _run(command, timeout=timeout, stream=stream)
if return_code != 0:
raise CalledProcessError(return_code, command, output)
return output
def perform_security_check(args):
"""It runs a few security tests, checks the return code and prints output that should help in fixing failures.
If you are at Yelp, please visit https://confluence.yelpcorp.com/display/PAASTA/PaaSTA+security-check+explained
to learn more.
:param args: service - the name of the service; commit - upstream git commit.
:return: 0 if the security-check passed, non-zero if it failed.
"""
security_check_command = load_system_paasta_config(
).get_security_check_command()
if not security_check_command:
paasta_print("Nothing to be executed during the security-check step")
return 0
command = f"{security_check_command} {args.service} {args.commit}"
ret_code, output = _run(command, timeout=3600, stream=True)
if ret_code != 0:
paasta_print(
"The security-check failed. Please visit y/security-check-runbook to learn how to fix it ("
"including whitelisting safe versions of packages and docker images)."
)
return ret_code
def run_paasta_serviceinit(subcommand, master, service, instancename, cluster,
**kwargs):
"""Run 'paasta_serviceinit <subcommand>'. Return the output from running it."""
if 'verbose' in kwargs and kwargs['verbose']:
verbose_flag = "-v "
timeout = 240
else:
verbose_flag = ''
timeout = 60
if 'app_id' in kwargs and kwargs['app_id']:
app_id_flag = "--appid %s " % kwargs['app_id']
else:
app_id_flag = ''
if 'delta' in kwargs and kwargs['delta']:
delta = "--delta %s" % kwargs['delta']
else:
delta = ''
command = 'ssh -A -n %s sudo paasta_serviceinit %s%s%s %s %s' % (
master, verbose_flag, app_id_flag, compose_job_id(
service, instancename), subcommand, delta)
log.debug("Running Command: %s" % command)
_, output = _run(command, timeout=timeout)
return output
def generate_pipeline(service):
email_address = get_team_email_address(service=service)
repo = get_git_repo_for_fab_repo(service)
if email_address is None:
owner = get_team(overrides={}, service=service)
else:
# fab_repo tacks on the domain, so we only want the first
# part of the email.
owner = re.sub('@.*', '', email_address)
cmds = [
'fab_repo setup_jenkins:services/%s,'
'profile=paasta,job_disabled=False,owner=%s,repo=%s' %
(service, owner, repo),
'fab_repo setup_jenkins:services/%s,'
'profile=paasta_boilerplate,owner=%s,repo=%s' % (service, owner, repo),
]
for cmd in cmds:
print "INFO: Executing %s" % cmd
returncode, output = _run(cmd, timeout=90)
if returncode != 0:
print "ERROR: Failed to generate Jenkins pipeline"
print output
sys.exit(returncode)
def check_ssh_on_master(master, timeout=10):
"""Given a master, attempt to ssh to the master and run a simple command
with sudo to verify that ssh and sudo work properly. Return a tuple of the
success status (True or False) and any output from attempting the check.
"""
check_command = "ssh -A -n -o StrictHostKeyChecking=no %s /bin/true" % master
rc, output = _run(check_command, timeout=timeout)
if rc == 0:
return (True, None)
if rc == 255: # ssh error
reason = "Return code was %d which probably means an ssh failure." % rc
hint = "HINT: Are you allowed to ssh to this machine %s?" % master
if rc == 1: # sudo error
reason = "Return code was %d which probably means a sudo failure." % rc
hint = "HINT: Is your ssh agent forwarded? (ssh-add -l)"
if rc == -9: # timeout error
reason = (
"Return code was %d which probably means ssh took too long and timed out."
% rc
)
hint = "HINT: Is there network latency? Try running somewhere closer to the cluster."
else: # unknown error
reason = "Return code was %d which is an unknown failure." % rc
hint = "HINT: Talk to #paasta and pastebin this output"
output = (
"ERROR cannot run check command %(check_command)s\n"
"%(reason)s\n"
"%(hint)s\n"
"Output from check command: %(output)s"
% {
"check_command": check_command,
"reason": reason,
"hint": hint,
"output": output,
}
)
return (False, output)
def run_on_master(cluster,
system_paasta_config,
cmd_parts,
timeout=None,
shell=False,
dry=False,
err_code=-1):
"""Find connectable master for :cluster: and :system_paasta_config: args and
invoke command from :cmd_parts:, wrapping it in ssh call.
:returns (exit code, output)
:param cluster: cluster to find master in
:param system_paasta_config: system configuration to lookup master data
:param cmd_parts: passed into paasta_tools.utils._run as command along with
ssh bits
:param timeout: see paasta_tools.utils._run documentation (default: None)
:param shell: prepend :cmd_parts: with 'sh -c' (default: False)
:param err_code: code to return along with error message when something goes
wrong (default: -1)
"""
try:
master = connectable_master(cluster, system_paasta_config)
except NoMasterError as e:
return (err_code, str(e))
ssh_parts = ['ssh', '-A', '-n', master]
if shell:
ssh_parts.extend('sh -c "%s"' % quote(' '.join(cmd_parts)))
else:
ssh_parts.append(' '.join(cmd_parts))
if dry:
return (0, "Would have run: %s" % ' '.join(ssh_parts))
else:
return _run(ssh_parts, timeout=timeout)
def perform_security_check(args):
"""It runs a few security tests, checks the return code and prints output that should help in fixing failures.
If you are at Yelp, please visit https://confluence.yelpcorp.com/display/PAASTA/PaaSTA+security-check+explained
to learn more.
:param args: service - the name of the service; commit - upstream git commit.
:return: 0 if the security-check passed, non-zero if it failed.
"""
security_check_command = load_system_paasta_config(
).get_security_check_command()
if not security_check_command:
paasta_print("Nothing to be executed during the security-check step")
return 0
command = "{} {} {}".format(security_check_command, args.service,
args.commit)
ret_code, output = _run(command, timeout=300, stream=True)
if ret_code != 0:
paasta_print(
"The security-check failed. Please visit y/security-check-runbook to learn how to fix it ("
"including whitelisting safe versions of packages and docker images).",
)
sensu_status = pysensu_yelp.Status.CRITICAL if ret_code != 0 else pysensu_yelp.Status.OK
send_event(
service=args.service,
check_name='%s.security_check' % args.service,
overrides={
'page': False,
'ticket': True
},
status=sensu_status,
output=output,
soa_dir=DEFAULT_SOA_DIR,
)
return ret_code
def paasta_sysdig(args):
if not args.local:
mesos_master = get_any_mesos_master(cluster=args.cluster)
ssh_cmd = ('ssh -At -o StrictHostKeyChecking=no -o LogLevel=QUIET {0} '
'"sudo paasta {1} --local"').format(mesos_master,
' '.join(sys.argv[1:]))
return_code, output = _run(ssh_cmd)
if return_code != 0:
paasta_print(output)
sys.exit(return_code)
slave, command = output.split(':', 1)
subprocess.call(
shlex.split("ssh -tA {} '{}'".format(slave, command.strip())))
return
status = get_status_for_instance(
cluster=args.cluster,
service=args.service,
instance=args.instance,
)
slave = pick_slave_from_status(
status=status,
host=args.host,
)
marathon_config = load_marathon_config()
marathon_url = marathon_config.get_url()[0]
marathon_user = marathon_config.get_username()
marathon_pass = marathon_config.get_password()
mesos_url = get_mesos_master().host
marathon_parsed_url = urlparse(marathon_url)
marathon_creds_url = marathon_parsed_url._replace(netloc="{}:{}@{}".format(
marathon_user,
marathon_pass,
marathon_parsed_url.netloc,
))
paasta_print(
format_mesos_command(slave, status.marathon.app_id, mesos_url,
marathon_creds_url.geturl()))
def run_paasta_metastatus(
master: str,
groupings: Sequence[str],
verbose: int = 0,
autoscaling_info: bool = False,
use_mesos_cache: bool = False,
) -> Tuple[int, str]:
if verbose > 0:
verbose_flag = "-%s" % ('v' * verbose)
timeout = 120
else:
verbose_flag = ''
timeout = 20
autoscaling_flag = "-a" if autoscaling_info else ""
if autoscaling_flag and verbose < 2:
verbose_flag = '-vv'
groupings_flag = "-g %s" % " ".join(groupings) if groupings else ''
cache_flag = "--use-mesos-cache" if use_mesos_cache else ''
cmd_args = " ".join(
filter(
None,
[
verbose_flag,
groupings_flag,
autoscaling_flag,
cache_flag,
],
), )
command = (
'ssh -A -n -o StrictHostKeyChecking=no {} sudo paasta_metastatus {}'.
format(
master,
cmd_args,
)).strip()
return_code, output = _run(command, timeout=timeout)
return return_code, output
def mark_for_deployment(git_url, cluster, instance, service, commit):
"""Mark a docker image for deployment"""
cmd = build_command(git_url, commit, cluster=cluster, instance=instance)
# Clusterinstance should be in cluster.instance format
returncode, output = _run(
cmd,
timeout=30,
)
loglines = get_loglines(returncode=returncode,
cmd=cmd,
output=output,
commit=commit,
cluster=cluster,
instance=instance)
for logline in loglines:
_log(
service=service,
line=logline,
component='deploy',
level='event',
cluster=cluster,
instance=instance,
)
return returncode
def run_setup_chronos_job(context, service_instance):
cmd = (
f"../paasta_tools/setup_chronos_job.py {service_instance} -d {context.soa_dir}"
)
exit_code, output = _run(cmd)
context.exit_code, context.output = exit_code, output
def test_run_returns_when_popen_fails():
fake_exception = OSError(1234, 'fake error')
with mock.patch('paasta_tools.utils.Popen', autospec=True, side_effect=fake_exception):
return_code, output = utils._run('nonexistant command', timeout=10)
assert return_code == 1234
assert 'fake error' in output
def run_command_in_container(context, code, task_id):
cmd = f'../paasta_tools/paasta_execute_docker_command.py -i {task_id} -c "exit {code}"'
paasta_print("Running cmd %s" % cmd)
exit_code, output = _run(cmd)
paasta_print(f"Got exitcode {exit_code} with output:\n{output}")
context.return_code = exit_code
def paasta_spark_run(args):
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
paasta_print(
PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates"
"PaaSTA is not configured locally on this host, and local-run may not behave"
"the same way it would behave on a server configured for PaaSTA.",
),
sep='\n',
)
system_paasta_config = SystemPaastaConfig({"volumes": []},
'/etc/paasta')
# Use the default spark:client instance configs if not provided
try:
instance_config = get_instance_config(
service=args.service,
instance=args.instance,
cluster=args.cluster,
load_deployments=args.build is False,
soa_dir=args.yelpsoa_config_root,
)
except NoConfigurationForServiceError as e:
paasta_print(str(e), file=sys.stderr)
return 1
except NoDeploymentsAvailable:
paasta_print(
PaastaColors.red(
"Error: No deployments.json found in %(soa_dir)s/%(service)s."
"You can generate this by running:"
"generate_deployments_for_service -d %(soa_dir)s -s %(service)s"
% {
'soa_dir': args.yelpsoa_config_root,
'service': args.service,
}, ),
sep='\n',
file=sys.stderr,
)
return 1
if args.build:
docker_url = build_and_push_docker_image(args)
if docker_url is None:
return 1
else:
try:
docker_url = instance_config.get_docker_url()
except NoDockerImageError:
paasta_print(
PaastaColors.red(
"Error: No sha has been marked for deployment for the %s deploy group.\n"
"Please ensure this service has either run through a jenkins pipeline "
"or paasta mark-for-deployment has been run for %s\n" % (
instance_config.get_deploy_group(),
args.service,
), ),
sep='',
file=sys.stderr,
)
return 1
paasta_print(
"Please wait while the image (%s) is pulled (times out after 5m)..."
% docker_url,
file=sys.stderr,
)
retcode, _ = _run('sudo -H docker pull %s' % docker_url,
stream=True,
timeout=300)
if retcode != 0:
paasta_print(
"\nPull failed. Are you authorized to run docker commands?",
file=sys.stderr,
)
return 1
return configure_and_run_docker_container(
args,
docker_img=docker_url,
instance_config=instance_config,
system_paasta_config=system_paasta_config,
)
def run_paasta_validate(context):
validate_cmd = ("paasta validate "
"--yelpsoa-config-root %s "
"--service %s " % (context.soa_dir, context.service))
context.validate_return_code, context.validate_output = _run(command=validate_cmd)
def run_setup_chronos_job(context, service_instance):
cmd = "../paasta_tools/setup_chronos_job.py %s -d %s" % (service_instance,
context.soa_dir)
exit_code, output = _run(cmd)
context.exit_code, context.output = exit_code, output
def step_run_paasta_setup_tron_namespace_dry_run(context):
cmd = (
f"paasta_setup_tron_namespace --dry-run -a --soa-dir {context.soa_dir}"
f" --cluster test-cluster")
context.return_code, context.output = _run(command=cmd)
def run_chronos_rerun_with_args(context, service_instance, cli_args):
cmd = ("python ../paasta_tools/chronos_rerun.py -d %s %s '%s' "
"2016-03-13T04:50:31") % (context.soa_dir, cli_args,
service_instance)
exit_code, output = _run(cmd)
context.exit_code, context.output = exit_code, output
def run_docker_container(docker_client,
service,
instance,
docker_hash,
volumes,
interactive,
command,
hostname,
healthcheck,
healthcheck_only,
instance_config,
soa_dir=DEFAULT_SOA_DIR,
dry_run=False,
json_dict=False):
"""docker-py has issues running a container with a TTY attached, so for
consistency we execute 'docker run' directly in both interactive and
non-interactive modes.
In non-interactive mode when the run is complete, stop the container and
remove it (with docker-py).
"""
if interactive:
sys.stderr.write(
PaastaColors.yellow(
"Warning! You're running a container in interactive mode.\n"
"This is *NOT* how Mesos runs containers.\n"
"To run the container exactly as Mesos does, omit the -I flag.\n\n"
))
else:
sys.stderr.write(
PaastaColors.yellow(
"You're running a container in non-interactive mode.\n"
"This is how Mesos runs containers.\n"
"Note that some programs behave differently when running with no\n"
"tty attached (as your program is about to run).\n\n"))
environment = instance_config.get_env_dictionary()
net = instance_config.get_net()
memory = instance_config.get_mem()
random_port = pick_random_port()
container_name = get_container_name()
docker_params = instance_config.format_docker_parameters()
docker_run_args = dict(
memory=memory,
random_port=random_port,
container_name=container_name,
volumes=volumes,
env=environment,
interactive=interactive,
docker_hash=docker_hash,
command=command,
hostname=hostname,
net=net,
docker_params=docker_params,
)
docker_run_cmd = get_docker_run_cmd(**docker_run_args)
joined_docker_run_cmd = ' '.join(docker_run_cmd)
healthcheck_mode, healthcheck_data = get_healthcheck_for_instance(
service, instance, instance_config, random_port, soa_dir=soa_dir)
if dry_run:
if json_dict:
sys.stdout.write(json.dumps(docker_run_args) + '\n')
else:
sys.stdout.write(json.dumps(docker_run_cmd) + '\n')
sys.exit(0)
else:
sys.stdout.write('Running docker command:\n%s\n' %
PaastaColors.grey(joined_docker_run_cmd))
if interactive:
# NOTE: This immediately replaces us with the docker run cmd. Docker
# run knows how to clean up the running container in this situation.
execlp('docker', *docker_run_cmd)
# For testing, when execlp is patched out and doesn't replace us, we
# still want to bail out.
return
container_started = False
container_id = None
try:
(returncode, output) = _run(joined_docker_run_cmd)
if returncode != 0:
sys.stdout.write('Failure trying to start your container!\n'
'Returncode: %d\n'
'Output:\n'
'%s\n'
'\n'
'Fix that problem and try again.\n'
'http://y/paasta-troubleshooting\n' %
(returncode, output))
# Container failed to start so no need to cleanup; just bail.
sys.exit(1)
container_started = True
container_id = get_container_id(docker_client, container_name)
sys.stdout.write('Found our container running with CID %s\n' %
container_id)
# If the service has a healthcheck, simulate it
if healthcheck_mode:
status, _ = simulate_healthcheck_on_service(
instance_config, docker_client, container_id, healthcheck_mode,
healthcheck_data, healthcheck)
else:
status = True
def _output_on_failure():
sys.stdout.write(
'Your service failed to start, here is the stdout and stderr\n'
)
sys.stdout.write(
PaastaColors.grey(
docker_client.attach(container_id,
stderr=True,
stream=False,
logs=True)))
if healthcheck_only:
sys.stdout.write(
'Detected --healthcheck-only flag, exiting now.\n')
if container_started:
_cleanup_container(docker_client, container_id)
if status:
sys.exit(0)
else:
_output_on_failure()
sys.exit(1)
running = docker_client.inspect_container(
container_id)['State']['Running']
if running:
sys.stdout.write(
'Your service is now running! Tailing stdout and stderr:\n')
for line in docker_client.attach(container_id,
stderr=True,
stream=True,
logs=True):
sys.stdout.write(PaastaColors.grey(line))
else:
_output_on_failure()
returncode = 3
except KeyboardInterrupt:
returncode = 3
# Cleanup if the container exits on its own or interrupted.
if container_started:
returncode = docker_client.inspect_container(
container_id)['State']['ExitCode']
_cleanup_container(docker_client, container_id)
sys.exit(returncode)
def run_command(context, timeout):
fake_cmd = 'sleep 1'
context.rc, context.output = _run(fake_cmd, timeout=float(timeout))
def run_on_master(
cluster,
system_paasta_config,
cmd_parts,
timeout=None,
err_code=-1,
graceful_exit=False,
stdin=None,
):
"""Find connectable master for :cluster: and :system_paasta_config: args and
invoke command from :cmd_parts:, wrapping it in ssh call.
:returns (exit code, output)
:param cluster: cluster to find master in
:param system_paasta_config: system configuration to lookup master data
:param cmd_parts: passed into paasta_tools.utils._run as command along with
ssh bits
:param timeout: see paasta_tools.utils._run documentation (default: None)
:param err_code: code to return along with error message when something goes
wrong (default: -1)
:param graceful_exit: wrap command in a bash script that waits for input and
kills the original command; trap SIGINT and send newline into stdin
"""
try:
master = connectable_master(cluster, system_paasta_config)
except NoMasterError as e:
return (err_code, str(e))
if graceful_exit:
# Signals don't travel over ssh, kill process when anything lands on stdin instead
# The procedure here is:
# 1. send process to background and capture it's pid
# 2. wait for stdin with timeout in a loop, exit when original process finished
# 3. kill original process if loop finished (something on stdin)
cmd_parts.append(
"& p=$!; " +
"while ! read -t1; do ! kill -0 $p 2>/dev/null && kill $$; done; "
+ "kill $p; wait")
stdin = subprocess.PIPE
stdin_interrupt = True
popen_kwargs = {"preexec_fn": os.setsid}
else:
stdin_interrupt = False
popen_kwargs = {}
cmd_parts = [
"ssh",
"-q",
"-t",
"-t",
"-A",
master,
"sudo /bin/bash -c %s" % quote(" ".join(cmd_parts)),
]
log.debug("Running %s" % " ".join(cmd_parts))
return _run(
cmd_parts,
timeout=timeout,
stream=True,
stdin=stdin,
stdin_interrupt=stdin_interrupt,
popen_kwargs=popen_kwargs,
)
def run_command_in_container(context, code, task_id):
cmd = '../paasta_tools/paasta_execute_docker_command.py -i %s -c "exit %s"' % (task_id, code)
print 'Running cmd %s' % cmd
exit_code, output = _run(cmd)
print 'Got exitcode %s with output:\n%s' % (exit_code, output)
context.return_code = exit_code
def run_docker_container(
docker_client,
service,
instance,
docker_url,
volumes,
interactive,
command,
healthcheck,
healthcheck_only,
user_port,
instance_config,
secret_provider_name,
soa_dir=DEFAULT_SOA_DIR,
dry_run=False,
json_dict=False,
framework=None,
secret_provider_kwargs={},
skip_secrets=False,
):
"""docker-py has issues running a container with a TTY attached, so for
consistency we execute 'docker run' directly in both interactive and
non-interactive modes.
In non-interactive mode when the run is complete, stop the container and
remove it (with docker-py).
"""
if user_port:
if check_if_port_free(user_port):
chosen_port = user_port
else:
paasta_print(
PaastaColors.red(
"The chosen port is already in use!\n"
"Try specifying another one, or omit (--port|-o) and paasta will find a free one for you"
),
file=sys.stderr,
)
sys.exit(1)
else:
chosen_port = pick_random_port(service)
environment = instance_config.get_env_dictionary()
if not skip_secrets:
secret_environment = decrypt_secret_environment_variables(
secret_provider_name=secret_provider_name,
environment=environment,
soa_dir=soa_dir,
service_name=service,
cluster_name=instance_config.cluster,
secret_provider_kwargs=secret_provider_kwargs,
)
environment.update(secret_environment)
local_run_environment = get_local_run_environment_vars(
instance_config=instance_config, port0=chosen_port, framework=framework
)
environment.update(local_run_environment)
net = instance_config.get_net()
memory = instance_config.get_mem()
container_name = get_container_name()
docker_params = instance_config.format_docker_parameters()
healthcheck_mode, healthcheck_data = get_healthcheck_for_instance(
service, instance, instance_config, chosen_port, soa_dir=soa_dir
)
if healthcheck_mode is None:
container_port = None
interactive = True
elif not user_port and not healthcheck and not healthcheck_only:
container_port = None
else:
try:
container_port = instance_config.get_container_port()
except AttributeError:
container_port = None
simulate_healthcheck = (
healthcheck_only or healthcheck
) and healthcheck_mode is not None
docker_run_args = dict(
memory=memory,
chosen_port=chosen_port,
container_port=container_port,
container_name=container_name,
volumes=volumes,
env=environment,
interactive=interactive,
detach=simulate_healthcheck,
docker_hash=docker_url,
command=command,
net=net,
docker_params=docker_params,
)
docker_run_cmd = get_docker_run_cmd(**docker_run_args)
joined_docker_run_cmd = " ".join(docker_run_cmd)
if dry_run:
if json_dict:
paasta_print(json.dumps(docker_run_args))
else:
paasta_print(json.dumps(docker_run_cmd))
return 0
else:
paasta_print(
"Running docker command:\n%s" % PaastaColors.grey(joined_docker_run_cmd)
)
merged_env = {**os.environ, **environment}
if interactive or not simulate_healthcheck:
# NOTE: This immediately replaces us with the docker run cmd. Docker
# run knows how to clean up the running container in this situation.
wrapper_path = shutil.which("paasta_docker_wrapper")
# To properly simulate mesos, we pop the PATH, which is not available to
# The executor
merged_env.pop("PATH")
execlpe(wrapper_path, *docker_run_cmd, merged_env)
# For testing, when execlpe is patched out and doesn't replace us, we
# still want to bail out.
return 0
container_started = False
container_id = None
try:
(returncode, output) = _run(docker_run_cmd, env=merged_env)
if returncode != 0:
paasta_print(
"Failure trying to start your container!"
"Returncode: %d"
"Output:"
"%s"
""
"Fix that problem and try again."
"http://y/paasta-troubleshooting" % (returncode, output),
sep="\n",
)
# Container failed to start so no need to cleanup; just bail.
sys.exit(1)
container_started = True
container_id = get_container_id(docker_client, container_name)
paasta_print("Found our container running with CID %s" % container_id)
if simulate_healthcheck:
healthcheck_result = simulate_healthcheck_on_service(
instance_config=instance_config,
docker_client=docker_client,
container_id=container_id,
healthcheck_mode=healthcheck_mode,
healthcheck_data=healthcheck_data,
healthcheck_enabled=healthcheck,
)
def _output_exit_code():
returncode = docker_client.inspect_container(container_id)["State"][
"ExitCode"
]
paasta_print(f"Container exited: {returncode})")
if healthcheck_only:
if container_started:
_output_exit_code()
_cleanup_container(docker_client, container_id)
if healthcheck_mode is None:
paasta_print(
"--healthcheck-only, but no healthcheck is defined for this instance!"
)
sys.exit(1)
elif healthcheck_result is True:
sys.exit(0)
else:
sys.exit(1)
running = docker_client.inspect_container(container_id)["State"]["Running"]
if running:
paasta_print("Your service is now running! Tailing stdout and stderr:")
for line in docker_client.attach(
container_id, stderr=True, stream=True, logs=True
):
paasta_print(line)
else:
_output_exit_code()
returncode = 3
except KeyboardInterrupt:
returncode = 3
# Cleanup if the container exits on its own or interrupted.
if container_started:
returncode = docker_client.inspect_container(container_id)["State"]["ExitCode"]
_cleanup_container(docker_client, container_id)
return returncode
def paasta_spark_run(args):
# argparse does not work as expected with both default and
# type=validate_work_dir.
validate_work_dir(args.work_dir)
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
paasta_print(
PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates"
"PaaSTA is not configured locally on this host, and local-run may not behave"
"the same way it would behave on a server configured for PaaSTA."
),
sep="\n",
)
system_paasta_config = SystemPaastaConfig({"volumes": []},
"/etc/paasta")
# Use the default spark:client instance configs if not provided
try:
instance_config = get_instance_config(
service=args.service,
instance=args.instance,
cluster=args.cluster,
load_deployments=args.build is False and args.image is None,
soa_dir=args.yelpsoa_config_root,
)
except NoConfigurationForServiceError as e:
paasta_print(str(e), file=sys.stderr)
return 1
except NoDeploymentsAvailable:
paasta_print(
PaastaColors.red(
"Error: No deployments.json found in %(soa_dir)s/%(service)s."
"You can generate this by running:"
"generate_deployments_for_service -d %(soa_dir)s -s %(service)s"
% {
"soa_dir": args.yelpsoa_config_root,
"service": args.service
}),
sep="\n",
file=sys.stderr,
)
return 1
if not args.cmd and not instance_config.get_cmd():
paasta_print(
"A command is required, pyspark, spark-shell, spark-submit or jupyter",
file=sys.stderr,
)
return 1
if args.build:
docker_url = build_and_push_docker_image(args)
if docker_url is None:
return 1
elif args.image:
docker_url = args.image
else:
if args.cmd == "jupyter-lab":
paasta_print(
PaastaColors.red(
"The jupyter-lab command requires a prebuilt image with -I or --image."
),
file=sys.stderr,
)
return 1
try:
docker_url = instance_config.get_docker_url()
except NoDockerImageError:
paasta_print(
PaastaColors.red(
"Error: No sha has been marked for deployment for the %s deploy group.\n"
"Please ensure this service has either run through a jenkins pipeline "
"or paasta mark-for-deployment has been run for %s\n" %
(instance_config.get_deploy_group(), args.service)),
sep="",
file=sys.stderr,
)
return 1
paasta_print(
"Please wait while the image (%s) is pulled (times out after 5m)..."
% docker_url,
file=sys.stderr,
)
retcode, _ = _run("sudo -H docker pull %s" % docker_url,
stream=True,
timeout=300)
if retcode != 0:
paasta_print(
"\nPull failed. Are you authorized to run docker commands?",
file=sys.stderr,
)
return 1
return configure_and_run_docker_container(
args,
docker_img=docker_url,
instance_config=instance_config,
system_paasta_config=system_paasta_config,
)
def run_docker_container(
docker_client,
service,
instance,
docker_hash,
volumes,
interactive,
command,
healthcheck,
healthcheck_only,
user_port,
instance_config,
soa_dir=DEFAULT_SOA_DIR,
dry_run=False,
json_dict=False,
framework=None,
):
"""docker-py has issues running a container with a TTY attached, so for
consistency we execute 'docker run' directly in both interactive and
non-interactive modes.
In non-interactive mode when the run is complete, stop the container and
remove it (with docker-py).
"""
if user_port:
if check_if_port_free(user_port):
chosen_port = user_port
else:
paasta_print(
PaastaColors.red(
"The chosen port is already in use!\n"
"Try specifying another one, or omit (--port|-o) and paasta will find a free one for you"
),
file=sys.stderr,
)
sys.exit(1)
else:
chosen_port = pick_random_port()
environment = instance_config.get_env_dictionary()
local_run_environment = get_local_run_environment_vars(
instance_config=instance_config,
port0=chosen_port,
framework=framework,
)
environment.update(local_run_environment)
net = instance_config.get_net()
memory = instance_config.get_mem()
container_name = get_container_name()
docker_params = instance_config.format_docker_parameters()
try:
container_port = instance_config.get_container_port()
except AttributeError:
container_port = None
docker_run_args = dict(
memory=memory,
chosen_port=chosen_port,
container_port=container_port,
container_name=container_name,
volumes=volumes,
env=environment,
interactive=interactive,
docker_hash=docker_hash,
command=command,
net=net,
docker_params=docker_params,
)
docker_run_cmd = get_docker_run_cmd(**docker_run_args)
joined_docker_run_cmd = ' '.join(docker_run_cmd)
healthcheck_mode, healthcheck_data = get_healthcheck_for_instance(
service, instance, instance_config, chosen_port, soa_dir=soa_dir)
if dry_run:
if json_dict:
paasta_print(json.dumps(docker_run_args))
else:
paasta_print(json.dumps(docker_run_cmd))
return 0
else:
paasta_print('Running docker command:\n%s' % PaastaColors.grey(joined_docker_run_cmd))
if interactive:
# NOTE: This immediately replaces us with the docker run cmd. Docker
# run knows how to clean up the running container in this situation.
execlp('paasta_docker_wrapper', *docker_run_cmd)
# For testing, when execlp is patched out and doesn't replace us, we
# still want to bail out.
return 0
container_started = False
container_id = None
try:
(returncode, output) = _run(docker_run_cmd)
if returncode != 0:
paasta_print(
'Failure trying to start your container!'
'Returncode: %d'
'Output:'
'%s'
''
'Fix that problem and try again.'
'http://y/paasta-troubleshooting'
% (returncode, output),
sep='\n',
)
# Container failed to start so no need to cleanup; just bail.
sys.exit(1)
container_started = True
container_id = get_container_id(docker_client, container_name)
paasta_print('Found our container running with CID %s' % container_id)
# If the service has a healthcheck, simulate it
if healthcheck_mode is not None:
healthcheck_result = simulate_healthcheck_on_service(
instance_config=instance_config,
docker_client=docker_client,
container_id=container_id,
healthcheck_mode=healthcheck_mode,
healthcheck_data=healthcheck_data,
healthcheck_enabled=healthcheck,
)
def _output_stdout_and_exit_code():
returncode = docker_client.inspect_container(container_id)['State']['ExitCode']
paasta_print('Container exited: %d)' % returncode)
paasta_print('Here is the stdout and stderr:\n\n')
paasta_print(
docker_client.attach(container_id, stderr=True, stream=False, logs=True)
)
if healthcheck_only:
if container_started:
_output_stdout_and_exit_code()
_cleanup_container(docker_client, container_id)
if healthcheck_mode is None:
paasta_print('--healthcheck-only, but no healthcheck is defined for this instance!')
sys.exit(1)
elif healthcheck_result is True:
sys.exit(0)
else:
sys.exit(1)
running = docker_client.inspect_container(container_id)['State']['Running']
if running:
paasta_print('Your service is now running! Tailing stdout and stderr:')
for line in docker_client.attach(container_id, stderr=True, stream=True, logs=True):
paasta_print(line)
else:
_output_stdout_and_exit_code()
returncode = 3
except KeyboardInterrupt:
returncode = 3
# Cleanup if the container exits on its own or interrupted.
if container_started:
returncode = docker_client.inspect_container(container_id)['State']['ExitCode']
_cleanup_container(docker_client, container_id)
return returncode
def run_cleanup_chronos_jobs(context):
cmd = '../paasta_tools/cleanup_chronos_jobs.py --soa-dir %s' % context.soa_dir
context.exit_code, context.output = _run(cmd)
paasta_print(context.output)
