def local_makefile_present():
if makefile_responds_to('cook-image'):
sys.stderr.write("Local Makefile with 'cook-image' target detected. Assuming --build\n")
return True
else:
sys.stderr.write("No Makefile with 'cook-image' target detected. Assuming --pull\n")
return False
def local_makefile_present():
if makefile_responds_to('cook-image'):
sys.stderr.write("Local Makefile with 'cook-image' target detected. Assuming --build\n")
return True
else:
sys.stderr.write("No Makefile with 'cook-image' target detected. Assuming --pull\n")
return False
def paasta_cook_image(args, service=None, soa_dir=None):
"""Build a docker image"""
if not service:
service = args.service
if service.startswith('services-'):
service = service.split('services-', 1)[1]
if not soa_dir:
soa_dir = args.yelpsoa_config_root
validate_service_name(service, soa_dir)
run_env = os.environ.copy()
default_tag = 'paasta-cook-image-{}-{}'.format(service, get_username())
tag = run_env.get('DOCKER_TAG', default_tag)
run_env['DOCKER_TAG'] = tag
if not makefile_responds_to('cook-image'):
paasta_print(
'ERROR: local-run now requires a cook-image target to be present in the Makefile. See'
'http://paasta.readthedocs.io/en/latest/about/contract.html', file=sys.stderr,
)
return 1
try:
cmd = 'make cook-image'
returncode, output = _run(
cmd,
env=run_env,
log=True,
component='build',
service=service,
loglevel='debug',
)
if returncode != 0:
_log(
service=service,
line='ERROR: make cook-image failed for %s.' % service,
component='build',
level='event',
)
else:
action_details = {
'tag': tag,
}
_log_audit(
action='cook-image',
action_details=action_details,
service=service,
)
return returncode
except KeyboardInterrupt:
paasta_print('\nProcess interrupted by the user. Cancelling.', file=sys.stderr)
return 2
def paasta_cook_image(args, service=None, soa_dir=None):
"""Build a docker image"""
if not service:
service = args.service
if service.startswith("services-"):
service = service.split("services-", 1)[1]
if not soa_dir:
soa_dir = args.yelpsoa_config_root
validate_service_name(service, soa_dir)
run_env = os.environ.copy()
default_tag = "paasta-cook-image-{}-{}".format(service, get_username())
tag = run_env.get("DOCKER_TAG", default_tag)
run_env["DOCKER_TAG"] = tag
if not makefile_responds_to("cook-image"):
print(
"ERROR: local-run now requires a cook-image target to be present in the Makefile. See"
"http://paasta.readthedocs.io/en/latest/about/contract.html",
file=sys.stderr,
)
return 1
try:
cmd = "make cook-image"
returncode, output = _run(
cmd,
env=run_env,
log=True,
component="build",
service=service,
loglevel="debug",
)
if returncode != 0:
_log(
service=service,
line="ERROR: make cook-image failed for %s." % service,
component="build",
level="event",
)
else:
action_details = {"tag": tag}
_log_audit(action="cook-image",
action_details=action_details,
service=service)
return returncode
except KeyboardInterrupt:
print("\nProcess interrupted by the user. Cancelling.",
file=sys.stderr)
return 2
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write(
"A local Makefile with a 'cook-image' target is required for --build\n"
)
sys.stderr.write(
"If you meant to pull the docker image from the registry, explicitly pass --pull\n"
)
return 1
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None,
service=service,
soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def build_and_push_docker_image(args):
"""
Build an image if the default Spark service image is not preferred.
The image needs to be pushed to a registry for the Spark executors
to pull.
"""
if not makefile_responds_to('cook-image'):
paasta_print(
"A local Makefile with a 'cook-image' target is required for --build",
file=sys.stderr,
)
return None
default_tag = '{}-{}'.format(DEFAULT_SPARK_DOCKER_IMAGE_PREFIX,
get_username())
docker_tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = docker_tag
cook_return = paasta_cook_image(
args=None,
service=args.service,
soa_dir=args.yelpsoa_config_root,
)
if cook_return is not 0:
return None
docker_url = f'{args.docker_registry}/{docker_tag}'
command = f'docker tag {docker_tag} {docker_url}'
paasta_print(PaastaColors.grey(command))
retcode, _ = _run(command, stream=True)
if retcode is not 0:
return None
if args.docker_registry != DEFAULT_SPARK_DOCKER_REGISTRY:
command = 'sudo -H docker push %s' % docker_url
else:
command = 'docker push %s' % docker_url
paasta_print(PaastaColors.grey(command))
retcode, output = _run(command, stream=True)
if retcode is not 0:
return None
return docker_url
def paasta_cook_image(args, service=None, soa_dir=None):
"""Build a docker image"""
if service:
service = service
else:
service = args.service
if service and service.startswith('services-'):
service = service.split('services-', 1)[1]
validate_service_name(service, soa_dir)
run_env = os.environ.copy()
default_tag = 'paasta-cook-image-%s-%s' % (service, get_username())
tag = run_env.get('DOCKER_TAG', default_tag)
run_env['DOCKER_TAG'] = tag
if not makefile_responds_to('cook-image'):
sys.stderr.write('ERROR: local-run now requires a cook-image target to be present in the Makefile. See '
'http://paasta.readthedocs.io/en/latest/about/contract.html\n')
return 1
try:
cmd = 'make cook-image'
returncode, output = _run(
cmd,
env=run_env,
log=True,
component='build',
service=service,
loglevel='debug'
)
if returncode != 0:
_log(
service=service,
line='ERROR: make cook-image failed for %s.' % service,
component='build',
level='event',
)
return returncode
except KeyboardInterrupt:
sys.stderr.write('\nProcess interrupted by the user. Cancelling.\n')
return 2
def paasta_cook_image(args, service=None, soa_dir=None):
"""Build a docker image"""
if service:
service = service
else:
service = args.service
if service and service.startswith('services-'):
service = service.split('services-', 1)[1]
validate_service_name(service, soa_dir)
run_env = os.environ.copy()
default_tag = 'paasta-cook-image-%s-%s' % (service, get_username())
tag = run_env.get('DOCKER_TAG', default_tag)
run_env['DOCKER_TAG'] = tag
if not makefile_responds_to('cook-image'):
sys.stderr.write(
'ERROR: local-run now requires a cook-image target to be present in the Makefile. See '
'http://paasta.readthedocs.io/en/latest/about/contract.html\n')
return 1
try:
cmd = 'make cook-image'
returncode, output = _run(cmd,
env=run_env,
log=True,
component='build',
service=service,
loglevel='debug')
if returncode != 0:
_log(
service=service,
line='ERROR: make cook-image failed for %s.' % service,
component='build',
level='event',
)
return returncode
except KeyboardInterrupt:
sys.stderr.write('\nProcess interrupted by the user. Cancelling.\n')
return 2
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
return 1
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def paasta_local_run(args):
if args.action == "pull" and os.geteuid() != 0 and not docker_config_available():
paasta_print("Re-executing paasta local-run --pull with sudo..")
os.execvp("sudo", ["sudo", "-H"] + sys.argv)
if args.action == "build" and not makefile_responds_to("cook-image"):
paasta_print(
"A local Makefile with a 'cook-image' target is required for --build",
file=sys.stderr,
)
paasta_print(
"If you meant to pull the docker image from the registry, explicitly pass --pull",
file=sys.stderr,
)
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
paasta_print(
PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates"
"PaaSTA is not configured locally on this host, and local-run may not behave"
"the same way it would behave on a server configured for PaaSTA."
),
sep="\n",
)
system_paasta_config = SystemPaastaConfig({"volumes": []}, "/etc/paasta")
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config["default_cluster"]
except KeyError:
paasta_print(
PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster."
"Please pass one to local-run using '-c'."
),
sep="\n",
file=sys.stderr,
)
return 1
instance = args.instance
docker_client = get_docker_client()
docker_sha = None
docker_url = None
if args.action == "build":
default_tag = "paasta-local-run-{}-{}".format(service, get_username())
docker_url = os.environ.get("DOCKER_TAG", default_tag)
os.environ["DOCKER_TAG"] = docker_url
pull_image = False
cook_return = paasta_cook_image(
args=None, service=service, soa_dir=args.yelpsoa_config_root
)
if cook_return != 0:
return cook_return
elif args.action == "dry_run":
pull_image = False
docker_url = None
docker_sha = args.sha
else:
pull_image = True
docker_url = None
docker_sha = args.sha
try:
return configure_and_run_docker_container(
docker_client=docker_client,
docker_url=docker_url,
docker_sha=docker_sha,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == "dry_run",
)
except errors.APIError as e:
paasta_print("Can't run Docker container. Error: %s" % str(e), file=sys.stderr)
return 1
def test_makefile_responds_to_run(mock_run):
mock_run.return_value = (2, 'Output')
actual = makefile_responds_to('non-present-target')
assert actual is False
def test_makefile_responds_to_good(mock_run):
mock_run.return_value = (1, 'Output')
actual = makefile_responds_to('present-target')
assert actual is True
def test_makefile_responds_to_run(mock_run):
mock_run.return_value = (2, 'Output')
actual = makefile_responds_to('non-present-target')
assert actual is False
def test_makefile_responds_to_good(mock_run):
mock_run.return_value = (0, 'Output')
actual = makefile_responds_to('present-target')
assert actual is True
def paasta_cook_image(
args: Optional[argparse.Namespace],
service: Optional[str] = None,
soa_dir: Optional[str] = None,
) -> int:
"""Build a docker image"""
if not service:
if args is None:
print(
"ERROR: No arguments or service passed to cook-image - unable to determine what service to cook an image for",
file=sys.stderr,
)
return 1
service = args.service
if service and service.startswith("services-"):
service = service.split("services-", 1)[1]
if not soa_dir:
if args is None:
print(
"ERROR: No arguments or soadir passed to cook-image - unable to determine where to look for soa-configs",
file=sys.stderr,
)
return 1
soa_dir = args.yelpsoa_config_root
validate_service_name(service, soa_dir)
run_env = os.environ.copy()
if args is not None and args.commit is not None:
# if we're given a commit, we're likely being called by Jenkins or someone
# trying to push the cooked image to our registry - as such, we should tag
# the cooked image as `paasta itest` would.
tag = build_docker_tag(service, args.commit, args.image_version)
else:
default_tag = "paasta-cook-image-{}-{}".format(service, get_username())
tag = run_env.get("DOCKER_TAG", default_tag)
run_env["DOCKER_TAG"] = tag
if not makefile_responds_to("cook-image"):
print(
"ERROR: local-run now requires a cook-image target to be present in the Makefile. See "
"http://paasta.readthedocs.io/en/latest/about/contract.html.",
file=sys.stderr,
)
return 1
try:
cmd = "make cook-image"
returncode, output = _run(
cmd,
env=run_env,
log=True,
component="build",
service=service,
loglevel="debug",
)
if returncode != 0:
_log(
service=service,
line="ERROR: make cook-image failed for %s." % service,
component="build",
level="event",
)
else:
action_details = {"tag": tag}
_log_audit(
action="cook-image", action_details=action_details, service=service
)
return returncode
except KeyboardInterrupt:
print("\nProcess interrupted by the user. Cancelling.", file=sys.stderr)
return 2
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
paasta_print("A local Makefile with a 'cook-image' target is required for --build", file=sys.stderr)
paasta_print("If you meant to pull the docker image from the registry, explicitly pass --pull", file=sys.stderr)
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
paasta_print(
PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates"
"PaaSTA is not configured locally on this host, and local-run may not behave"
"the same way it would behave on a server configured for PaaSTA."
),
sep='\n',
)
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config['default_cluster']
except KeyError:
paasta_print(
PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster."
"Please pass one to local-run using '-c'."),
sep='\n',
file=sys.stderr,
)
return 1
instance = args.instance
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
return configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
paasta_print(
'Can\'t run Docker container. Error: %s' % str(e),
file=sys.stderr,
)
return 1
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
sys.stdout.write(PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates\n"
"PaaSTA is not configured locally on this host, and local-run may not behave\n"
"the same way it would behave on a server configured for PaaSTA.\n"
))
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config['default_cluster']
except KeyError:
sys.stderr.write(PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster.\n"
"Please pass one to local-run using '-c'.\n"))
return 1
instance = args.instance
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1