def test_run_makefile_fail(
mock_run,
mock_makefile_responds_to,
mock_validate_service_name,
):
mock_run.return_value = (0, 'Output')
mock_makefile_responds_to.return_value = False
mock_validate_service_name.return_value = True
args = mock.MagicMock()
args.service = 'fake_service'
with raises(SystemExit) as excinfo:
paasta_cook_image(args)
assert excinfo.value.code == 1
def test_run_makefile_fail(
mock_run,
mock_makefile_responds_to,
mock_validate_service_name,
):
mock_run.return_value = (0, 'Output')
mock_makefile_responds_to.return_value = False
mock_validate_service_name.return_value = True
args = mock.MagicMock()
args.service = 'fake_service'
with raises(SystemExit) as excinfo:
paasta_cook_image(args)
assert excinfo.value.code == 1
def test_run_success_with_commit(mock_log_audit, mock_run,
mock_makefile_responds_to,
mock_validate_service_name):
mock_run.return_value = (0, "Output")
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
args = mock.MagicMock()
args.commit = "0" * 40
args.service = "fake_service"
args.image_version = None
with mock.patch(
"paasta_tools.utils.get_service_docker_registry",
autospec=True,
return_value="fake_registry",
):
assert paasta_cook_image(args) == 0
mock_log_audit.assert_called_once_with(
action="cook-image",
action_details={
"tag": f"fake_registry/services-fake_service:paasta-{args.commit}"
},
service="fake_service",
)
def test_run_keyboard_interrupt(
mock_run,
mock_makefile_responds_to,
mock_validate_service_name,
):
mock_run.return_value = (0, 'Output')
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
mock_run.side_effect = FakeKeyboardInterrupt
args = mock.MagicMock()
args.service = 'fake_service'
with raises(SystemExit) as excinfo:
paasta_cook_image(args)
assert excinfo.value.code == 2
def test_run_keyboard_interrupt(
mock_run,
mock_makefile_responds_to,
mock_validate_service_name,
):
mock_run.return_value = (0, 'Output')
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
mock_run.side_effect = FakeKeyboardInterrupt
args = mock.MagicMock()
args.service = 'fake_service'
with raises(SystemExit) as excinfo:
paasta_cook_image(args)
assert excinfo.value.code == 2
def test_run_makefile_fail(mock_log_audit, mock_run, mock_makefile_responds_to,
mock_validate_service_name):
mock_run.return_value = (0, "Output")
mock_makefile_responds_to.return_value = False
mock_validate_service_name.return_value = True
args = mock.MagicMock()
args.service = "fake_service"
assert paasta_cook_image(args) == 1
assert not mock_log_audit.called
def test_run_success(
mock_run,
mock_makefile_responds_to,
mock_validate_service_name,
):
mock_run.return_value = (0, 'Output')
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
args = mock.MagicMock()
args.service = 'fake_service'
assert paasta_cook_image(args) is 0
def test_run_success(
mock_run,
mock_makefile_responds_to,
mock_validate_service_name,
):
mock_run.return_value = (0, 'Output')
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
args = mock.MagicMock()
args.service = 'fake_service'
assert paasta_cook_image(args) is 0
def paasta_local_run(args):
if args.pull:
build = False
elif args.build:
build = True
else:
build = local_makefile_present()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
base_docker_url = get_docker_host()
docker_client = Client(base_url=base_docker_url)
if build:
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
paasta_cook_image(args=None,
service=service,
soa_dir=args.yelpsoa_config_root)
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
sys.exit(1)
def test_run_keyboard_interrupt(mock_log_audit, mock_run,
mock_makefile_responds_to,
mock_validate_service_name):
mock_run.return_value = (0, "Output")
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
mock_run.side_effect = FakeKeyboardInterrupt
args = mock.MagicMock()
args.service = "fake_service"
assert paasta_cook_image(args) == 2
assert not mock_log_audit.called
def test_run_keyboard_interrupt(
mock_run,
mock_makefile_responds_to,
mock_validate_service_name,
):
mock_run.return_value = (0, 'Output')
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
mock_run.side_effect = FakeKeyboardInterrupt
args = mock.MagicMock()
args.service = 'fake_service'
assert paasta_cook_image(args) == 2
def test_run_keyboard_interrupt(
mock_run,
mock_makefile_responds_to,
mock_validate_service_name,
):
mock_run.return_value = (0, 'Output')
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
mock_run.side_effect = FakeKeyboardInterrupt
args = mock.MagicMock()
args.service = 'fake_service'
assert paasta_cook_image(args) == 2
def paasta_local_run(args):
if args.pull:
build = False
elif args.build:
build = True
else:
build = local_makefile_present()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
base_docker_url = get_docker_host()
docker_client = Client(base_url=base_docker_url)
if build:
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
sys.exit(1)
def test_run_success(mock_log_audit, mock_run, mock_makefile_responds_to,
mock_validate_service_name):
mock_run.return_value = (0, "Output")
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
args = mock.MagicMock()
args.service = "fake_service"
assert paasta_cook_image(args) == 0
mock_log_audit.assert_called_once_with(
action="cook-image",
action_details={
"tag": "paasta-cook-image-fake_service-{}".format(get_username())
},
service="fake_service",
)
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write(
"A local Makefile with a 'cook-image' target is required for --build\n"
)
sys.stderr.write(
"If you meant to pull the docker image from the registry, explicitly pass --pull\n"
)
return 1
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None,
service=service,
soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def build_and_push_docker_image(args):
"""
Build an image if the default Spark service image is not preferred.
The image needs to be pushed to a registry for the Spark executors
to pull.
"""
if not makefile_responds_to('cook-image'):
paasta_print(
"A local Makefile with a 'cook-image' target is required for --build",
file=sys.stderr,
)
return None
default_tag = '{}-{}'.format(DEFAULT_SPARK_DOCKER_IMAGE_PREFIX,
get_username())
docker_tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = docker_tag
cook_return = paasta_cook_image(
args=None,
service=args.service,
soa_dir=args.yelpsoa_config_root,
)
if cook_return is not 0:
return None
docker_url = f'{args.docker_registry}/{docker_tag}'
command = f'docker tag {docker_tag} {docker_url}'
paasta_print(PaastaColors.grey(command))
retcode, _ = _run(command, stream=True)
if retcode is not 0:
return None
if args.docker_registry != DEFAULT_SPARK_DOCKER_REGISTRY:
command = 'sudo -H docker push %s' % docker_url
else:
command = 'docker push %s' % docker_url
paasta_print(PaastaColors.grey(command))
retcode, output = _run(command, stream=True)
if retcode is not 0:
return None
return docker_url
def test_run_success(
mock_log_audit,
mock_run,
mock_makefile_responds_to,
mock_validate_service_name,
):
mock_run.return_value = (0, 'Output')
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
args = mock.MagicMock()
args.service = 'fake_service'
assert paasta_cook_image(args) == 0
mock_log_audit.assert_called_once_with(
action='cook-image',
action_details={'tag': 'paasta-cook-image-fake_service-{}'.format(get_username())},
service='fake_service',
)
def paasta_local_run(args):
if args.pull or args.dry_run:
build = False
elif args.build:
build = True
else:
build = local_makefile_present()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if build:
default_tag = "paasta-local-run-%s-%s" % (service, get_username())
tag = os.environ.get("DOCKER_TAG", default_tag)
os.environ["DOCKER_TAG"] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.dry_run:
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.dry_run,
)
except errors.APIError as e:
sys.stderr.write("Can't run Docker container. Error: %s\n" % str(e))
return 1
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
return 1
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
paasta_print("A local Makefile with a 'cook-image' target is required for --build", file=sys.stderr)
paasta_print("If you meant to pull the docker image from the registry, explicitly pass --pull", file=sys.stderr)
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
paasta_print(
PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates"
"PaaSTA is not configured locally on this host, and local-run may not behave"
"the same way it would behave on a server configured for PaaSTA."
),
sep='\n',
)
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config['default_cluster']
except KeyError:
paasta_print(
PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster."
"Please pass one to local-run using '-c'."),
sep='\n',
file=sys.stderr,
)
return 1
instance = args.instance
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
return configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
paasta_print(
'Can\'t run Docker container. Error: %s' % str(e),
file=sys.stderr,
)
return 1
def paasta_local_run(args):
if args.action == "pull" and os.geteuid() != 0 and not docker_config_available():
paasta_print("Re-executing paasta local-run --pull with sudo..")
os.execvp("sudo", ["sudo", "-H"] + sys.argv)
if args.action == "build" and not makefile_responds_to("cook-image"):
paasta_print(
"A local Makefile with a 'cook-image' target is required for --build",
file=sys.stderr,
)
paasta_print(
"If you meant to pull the docker image from the registry, explicitly pass --pull",
file=sys.stderr,
)
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
paasta_print(
PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates"
"PaaSTA is not configured locally on this host, and local-run may not behave"
"the same way it would behave on a server configured for PaaSTA."
),
sep="\n",
)
system_paasta_config = SystemPaastaConfig({"volumes": []}, "/etc/paasta")
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config["default_cluster"]
except KeyError:
paasta_print(
PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster."
"Please pass one to local-run using '-c'."
),
sep="\n",
file=sys.stderr,
)
return 1
instance = args.instance
docker_client = get_docker_client()
docker_sha = None
docker_url = None
if args.action == "build":
default_tag = "paasta-local-run-{}-{}".format(service, get_username())
docker_url = os.environ.get("DOCKER_TAG", default_tag)
os.environ["DOCKER_TAG"] = docker_url
pull_image = False
cook_return = paasta_cook_image(
args=None, service=service, soa_dir=args.yelpsoa_config_root
)
if cook_return != 0:
return cook_return
elif args.action == "dry_run":
pull_image = False
docker_url = None
docker_sha = args.sha
else:
pull_image = True
docker_url = None
docker_sha = args.sha
try:
return configure_and_run_docker_container(
docker_client=docker_client,
docker_url=docker_url,
docker_sha=docker_sha,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == "dry_run",
)
except errors.APIError as e:
paasta_print("Can't run Docker container. Error: %s" % str(e), file=sys.stderr)
return 1
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
sys.stdout.write(PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates\n"
"PaaSTA is not configured locally on this host, and local-run may not behave\n"
"the same way it would behave on a server configured for PaaSTA.\n"
))
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config['default_cluster']
except KeyError:
sys.stderr.write(PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster.\n"
"Please pass one to local-run using '-c'.\n"))
return 1
instance = args.instance
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
