def paasta_local_run(args):
if args.pull:
build = False
elif args.build:
build = True
else:
build = local_makefile_present()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
base_docker_url = get_docker_host()
docker_client = Client(base_url=base_docker_url)
if build:
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
paasta_cook_image(None, service=service, soa_dir=args.yelpsoa_config_root)
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
args=args,
pull_image=pull_image,
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
sys.exit(1)
def log_event(service_config, desired_state):
user = utils.get_username()
host = socket.getfqdn()
line = "Issued request to change state of %s to '%s' by %s@%s" % (
service_config.get_instance(), desired_state, user, host)
utils._log(
service=service_config.get_service(),
level='event',
cluster=service_config.get_cluster(),
instance=service_config.get_instance(),
component='deploy',
line=line,
)
def submit_performance_check_job(service, commit, image):
performance_check_config = load_performance_check_config()
payload = {
'service': service,
'commit': commit,
'submitter': get_username(),
'image': image,
}
r = requests.post(
url=performance_check_config['endpoint'],
data=payload,
)
print "Posted a submission to the PaaSTA performance-check service:"
print r.text
def build_and_push_docker_image(args):
"""
Build an image if the default Spark service image is not preferred.
The image needs to be pushed to a registry for the Spark executors
to pull.
"""
if not makefile_responds_to('cook-image'):
paasta_print(
"A local Makefile with a 'cook-image' target is required for --build",
file=sys.stderr,
)
return None
default_tag = '{}-{}'.format(DEFAULT_SPARK_DOCKER_IMAGE_PREFIX, get_username())
docker_tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = docker_tag
cook_return = paasta_cook_image(
args=None,
service=args.service,
soa_dir=args.yelpsoa_config_root,
)
if cook_return is not 0:
return None
docker_url = f'{args.docker_registry}/{docker_tag}'
command = f'docker tag {docker_tag} {docker_url}'
paasta_print(PaastaColors.grey(command))
retcode, _ = _run(command, stream=True)
if retcode is not 0:
return None
if args.docker_registry != DEFAULT_SPARK_DOCKER_REGISTRY:
command = 'sudo -H docker push %s' % docker_url
else:
command = 'docker push %s' % docker_url
paasta_print(PaastaColors.grey(command))
retcode, output = _run(command, stream=True)
if retcode is not 0:
return None
return docker_url
def paasta_local_run(args):
if args.pull or args.dry_run:
build = False
elif args.build:
build = True
else:
build = local_makefile_present()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if build:
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.dry_run:
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.dry_run,
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def paasta_local_run(args):
if args.pull or args.dry_run:
build = False
elif args.build:
build = True
else:
build = local_makefile_present()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if build:
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.dry_run:
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.dry_run,
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def paasta_cook_image(args, service=None, soa_dir=None):
"""Build a docker image"""
if service:
service = service
else:
service = args.service
if service and service.startswith('services-'):
service = service.split('services-', 1)[1]
validate_service_name(service, soa_dir)
run_env = os.environ.copy()
default_tag = 'paasta-cook-image-%s-%s' % (service, get_username())
tag = run_env.get('DOCKER_TAG', default_tag)
run_env['DOCKER_TAG'] = tag
if not makefile_responds_to('cook-image'):
paasta_print('ERROR: local-run now requires a cook-image target to be present in the Makefile. See'
'http://paasta.readthedocs.io/en/latest/about/contract.html', file=sys.stderr)
return 1
try:
cmd = 'make cook-image'
returncode, output = _run(
cmd,
env=run_env,
log=True,
component='build',
service=service,
loglevel='debug'
)
if returncode != 0:
_log(
service=service,
line='ERROR: make cook-image failed for %s.' % service,
component='build',
level='event',
)
return returncode
except KeyboardInterrupt:
paasta_print('\nProcess interrupted by the user. Cancelling.', file=sys.stderr)
return 2
def paasta_cook_image(args, service=None, soa_dir=None):
"""Build a docker image"""
if service:
service = service
else:
service = args.service
if service and service.startswith('services-'):
service = service.split('services-', 1)[1]
validate_service_name(service, soa_dir)
run_env = os.environ.copy()
default_tag = 'paasta-cook-image-%s-%s' % (service, get_username())
tag = run_env.get('DOCKER_TAG', default_tag)
run_env['DOCKER_TAG'] = tag
if not makefile_responds_to('cook-image'):
sys.stderr.write('ERROR: local-run now requires a cook-image target to be present in the Makefile. See '
'http://paasta.readthedocs.io/en/latest/about/contract.html\n')
return 1
try:
cmd = 'make cook-image'
returncode, output = _run(
cmd,
env=run_env,
log=True,
component='build',
service=service,
loglevel='debug'
)
if returncode != 0:
_log(
service=service,
line='ERROR: make cook-image failed for %s.' % service,
component='build',
level='event',
)
return returncode
except KeyboardInterrupt:
sys.stderr.write('\nProcess interrupted by the user. Cancelling.\n')
return 2
def paasta_local_run(args):
validate_environment()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
base_docker_url = get_docker_host()
docker_client = Client(base_url=base_docker_url)
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
paasta_cook_image(None, service=service, soa_dir=args.yelpsoa_config_root)
try:
configure_and_run_docker_container(docker_client, tag, service, args)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
sys.exit(1)
def paasta_local_run(args):
validate_environment()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
base_docker_url = get_docker_host()
docker_client = Client(base_url=base_docker_url)
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
paasta_cook_image(None, service=service, soa_dir=args.yelpsoa_config_root)
try:
configure_and_run_docker_container(docker_client, tag, service, args)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
sys.exit(1)
def get_docker_cmd(args, instance_config, spark_conf_str):
original_docker_cmd = args.cmd or instance_config.get_cmd()
if args.mrjob:
return original_docker_cmd + " " + spark_conf_str
# Default cli options to start the jupyter notebook server.
elif original_docker_cmd == "jupyter-lab":
cull_opts = ("--MappingKernelManager.cull_idle_timeout=%s " %
args.cull_idle_timeout)
if args.not_cull_connected is False:
cull_opts += "--MappingKernelManager.cull_connected=True "
return "SHELL=bash USER={} /source/virtualenv_run_jupyter/bin/jupyter-lab -y --ip={} {}".format(
get_username(), socket.getfqdn(), cull_opts)
elif original_docker_cmd == "history-server":
return "start-history-server.sh"
# Spark options are passed as options to pyspark and spark-shell.
# For jupyter, environment variable SPARK_OPTS is set instead.
else:
return inject_spark_conf_str(original_docker_cmd, spark_conf_str)
def test_run_success(
mock_log_audit,
mock_run,
mock_makefile_responds_to,
mock_validate_service_name,
):
mock_run.return_value = (0, 'Output')
mock_makefile_responds_to.return_value = True
mock_validate_service_name.return_value = True
args = mock.MagicMock()
args.service = 'fake_service'
assert paasta_cook_image(args) is 0
mock_log_audit.assert_called_once_with(
action='cook-image',
action_details={
'tag': 'paasta-cook-image-fake_service-{}'.format(get_username())
},
service='fake_service',
)
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
return 1
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = guess_cluster(service=service, args=args)
instance = guess_instance(service=service, cluster=cluster, args=args)
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def get_spark_app_name(original_docker_cmd: Union[Any, str, List[str]],
spark_ui_port: int) -> str:
"""Use submitted batch name as default spark_run job name"""
docker_cmds = (shlex.split(original_docker_cmd) if isinstance(
original_docker_cmd, str) else original_docker_cmd)
spark_app_name = None
after_spark_submit = False
for arg in docker_cmds:
if arg == "spark-submit":
after_spark_submit = True
elif after_spark_submit and arg.endswith(".py"):
batch_name = arg.split("/")[-1].replace(".py", "")
spark_app_name = "paasta_" + batch_name
break
elif arg == "jupyter-lab":
spark_app_name = "paasta_jupyter"
break
if spark_app_name is None:
spark_app_name = "paasta_spark_run"
spark_app_name += "_{}_{}".format(get_username(), spark_ui_port)
return spark_app_name
def get_container_name():
return 'paasta_local_run_%s_%s' % (get_username(), randint(1, 999999))
def configure_and_run_docker_container(
args,
docker_img,
instance_config,
system_paasta_config,
):
volumes = list()
for volume in instance_config.get_volumes(
system_paasta_config.get_volumes()):
if os.path.exists(volume['hostPath']):
volumes.append('%s:%s:%s' %
(volume['hostPath'], volume['containerPath'],
volume['mode'].lower()))
else:
paasta_print(
PaastaColors.yellow(
"Warning: Path %s does not exist on this host. Skipping this binding."
% volume['hostPath'], ), )
volumes.append('%s:%s:rw' % (os.getcwd(), DEFAULT_SPARK_WORK_DIR))
volumes.append('/etc/passwd:/etc/passwd:ro')
volumes.append('/etc/group:/etc/group:ro')
if args.cmd is None:
docker_cmd = instance_config.get_cmd()
else:
docker_cmd = args.cmd
if docker_cmd is None:
paasta_print(
"A command is required, pyspark, spark-shell, spark-submit or jupyter",
file=sys.stderr)
return 1
# Changes at docker ENTRYPOINT or CMD does not work.
elif docker_cmd == 'jupyter':
docker_cmd = 'jupyter notebook -y --ip=%s --notebook-dir=%s' % (
socket.getfqdn(),
DEFAULT_SPARK_WORK_DIR,
)
spark_ui_port = pick_random_port(args.service)
container_name = 'paasta_spark_run_%s_%s' % (get_username(), spark_ui_port)
# Do not put memory and CPU limits on Spark driver for now.
# Toree won't work with the default memory-swap setting.
environment = instance_config.get_env_dictionary()
environment.update(
get_spark_configuration(
args,
container_name,
spark_ui_port,
docker_img,
system_paasta_config,
), )
return run_docker_container(
container_name=container_name,
volumes=volumes,
environment=environment,
docker_img=docker_img,
docker_cmd=docker_cmd,
dry_run=args.dry_run,
)
def configure_and_run_docker_container(
args,
docker_img,
instance_config,
system_paasta_config,
):
volumes = list()
for volume in instance_config.get_volumes(
system_paasta_config.get_volumes()):
if os.path.exists(volume['hostPath']):
volumes.append('%s:%s:%s' %
(volume['hostPath'], volume['containerPath'],
volume['mode'].lower()))
else:
paasta_print(
PaastaColors.yellow(
"Warning: Path %s does not exist on this host. Skipping this binding."
% volume['hostPath'], ), )
spark_ui_port = pick_random_port(args.service)
container_name = 'paasta_spark_run_%s_%s' % (get_username(), spark_ui_port)
spark_conf_str = get_spark_conf_str(
args=args,
container_name=container_name,
spark_ui_port=spark_ui_port,
docker_img=docker_img,
system_paasta_config=system_paasta_config,
volumes=volumes,
)
# Spark client specific volumes
volumes.append('%s:rw' % args.work_dir)
volumes.append('/etc/passwd:/etc/passwd:ro')
volumes.append('/etc/group:/etc/group:ro')
if args.cmd is None:
docker_cmd = instance_config.get_cmd()
else:
docker_cmd = args.cmd
if docker_cmd is None:
paasta_print(
"A command is required, pyspark, spark-shell, spark-submit or jupyter",
file=sys.stderr)
return 1
# Default cli options to start the jupyter notebook server.
if docker_cmd == 'jupyter':
docker_cmd = 'jupyter notebook -y --ip=%s --notebook-dir=%s' % (
socket.getfqdn(),
args.work_dir.split(':')[1],
)
# Spark options are passed as options to pyspark and spark-shell.
# For jupyter, environment variable SPARK_OPTS is set instead.
elif docker_cmd in ['pyspark', 'spark-shell']:
docker_cmd = docker_cmd + ' ' + spark_conf_str
elif docker_cmd.startswith('spark-submit'):
docker_cmd = 'spark-submit ' + spark_conf_str + docker_cmd[
len('spark-submit'):]
environment = instance_config.get_env_dictionary()
environment.update(get_spark_env(
args,
spark_conf_str,
), )
paasta_print('\nSpark Monitoring URL http://%s:%d\n' %
(socket.getfqdn(), spark_ui_port))
return run_docker_container(
container_name=container_name,
volumes=volumes,
environment=environment,
docker_img=docker_img,
docker_cmd=docker_cmd,
dry_run=args.dry_run,
)
def configure_and_run_docker_container(
args,
docker_img,
instance_config,
system_paasta_config,
):
volumes = list()
for volume in instance_config.get_volumes(
system_paasta_config.get_volumes()):
if os.path.exists(volume['hostPath']):
volumes.append('{}:{}:{}'.format(volume['hostPath'],
volume['containerPath'],
volume['mode'].lower()))
else:
paasta_print(
PaastaColors.yellow(
"Warning: Path %s does not exist on this host. Skipping this binding."
% volume['hostPath'], ), )
spark_ui_port = pick_random_port(args.service)
container_name = 'paasta_spark_run_{}_{}'.format(get_username(),
spark_ui_port)
spark_conf_str = get_spark_conf_str(
args=args,
container_name=container_name,
spark_ui_port=spark_ui_port,
docker_img=docker_img,
system_paasta_config=system_paasta_config,
volumes=volumes,
)
# Spark client specific volumes
volumes.append('%s:rw' % args.work_dir)
volumes.append('/etc/passwd:/etc/passwd:ro')
volumes.append('/etc/group:/etc/group:ro')
docker_cmd = get_docker_cmd(args, instance_config, spark_conf_str)
if docker_cmd is None:
paasta_print(
"A command is required, pyspark, spark-shell, spark-submit or jupyter",
file=sys.stderr)
return 1
environment = instance_config.get_env_dictionary()
environment.update(get_spark_env(
args,
spark_conf_str,
), )
paasta_print('\nSpark Monitoring URL http://%s:%d\n' %
(socket.getfqdn(), spark_ui_port))
return run_docker_container(
container_name=container_name,
volumes=volumes,
environment=environment,
docker_img=docker_img,
docker_cmd=docker_cmd,
dry_run=args.dry_run,
)
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
sys.stdout.write(PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates\n"
"PaaSTA is not configured locally on this host, and local-run may not behave\n"
"the same way it would behave on a server configured for PaaSTA.\n"
))
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config['default_cluster']
except KeyError:
sys.stderr.write(PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster.\n"
"Please pass one to local-run using '-c'.\n"))
return 1
instance = args.instance
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def paasta_local_run(args):
if args.action == "pull" and os.geteuid() != 0 and not docker_config_available():
paasta_print("Re-executing paasta local-run --pull with sudo..")
os.execvp("sudo", ["sudo", "-H"] + sys.argv)
if args.action == "build" and not makefile_responds_to("cook-image"):
paasta_print(
"A local Makefile with a 'cook-image' target is required for --build",
file=sys.stderr,
)
paasta_print(
"If you meant to pull the docker image from the registry, explicitly pass --pull",
file=sys.stderr,
)
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
paasta_print(
PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates"
"PaaSTA is not configured locally on this host, and local-run may not behave"
"the same way it would behave on a server configured for PaaSTA."
),
sep="\n",
)
system_paasta_config = SystemPaastaConfig({"volumes": []}, "/etc/paasta")
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config["default_cluster"]
except KeyError:
paasta_print(
PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster."
"Please pass one to local-run using '-c'."
),
sep="\n",
file=sys.stderr,
)
return 1
instance = args.instance
docker_client = get_docker_client()
docker_sha = None
docker_url = None
if args.action == "build":
default_tag = "paasta-local-run-{}-{}".format(service, get_username())
docker_url = os.environ.get("DOCKER_TAG", default_tag)
os.environ["DOCKER_TAG"] = docker_url
pull_image = False
cook_return = paasta_cook_image(
args=None, service=service, soa_dir=args.yelpsoa_config_root
)
if cook_return != 0:
return cook_return
elif args.action == "dry_run":
pull_image = False
docker_url = None
docker_sha = args.sha
else:
pull_image = True
docker_url = None
docker_sha = args.sha
try:
return configure_and_run_docker_container(
docker_client=docker_client,
docker_url=docker_url,
docker_sha=docker_sha,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == "dry_run",
)
except errors.APIError as e:
paasta_print("Can't run Docker container. Error: %s" % str(e), file=sys.stderr)
return 1
def get_container_name():
return 'paasta_local_run_%s_%s' % (get_username(), randint(1, 999999))
def paasta_cook_image(
args: Optional[argparse.Namespace],
service: Optional[str] = None,
soa_dir: Optional[str] = None,
) -> int:
"""Build a docker image"""
if not service:
if args is None:
print(
"ERROR: No arguments or service passed to cook-image - unable to determine what service to cook an image for",
file=sys.stderr,
)
return 1
service = args.service
if service and service.startswith("services-"):
service = service.split("services-", 1)[1]
if not soa_dir:
if args is None:
print(
"ERROR: No arguments or soadir passed to cook-image - unable to determine where to look for soa-configs",
file=sys.stderr,
)
return 1
soa_dir = args.yelpsoa_config_root
validate_service_name(service, soa_dir)
run_env = os.environ.copy()
if args is not None and args.commit is not None:
# if we're given a commit, we're likely being called by Jenkins or someone
# trying to push the cooked image to our registry - as such, we should tag
# the cooked image as `paasta itest` would.
tag = build_docker_tag(service, args.commit, args.image_version)
else:
default_tag = "paasta-cook-image-{}-{}".format(service, get_username())
tag = run_env.get("DOCKER_TAG", default_tag)
run_env["DOCKER_TAG"] = tag
if not makefile_responds_to("cook-image"):
print(
"ERROR: local-run now requires a cook-image target to be present in the Makefile. See "
"http://paasta.readthedocs.io/en/latest/about/contract.html.",
file=sys.stderr,
)
return 1
try:
cmd = "make cook-image"
returncode, output = _run(
cmd,
env=run_env,
log=True,
component="build",
service=service,
loglevel="debug",
)
if returncode != 0:
_log(
service=service,
line="ERROR: make cook-image failed for %s." % service,
component="build",
level="event",
)
else:
action_details = {"tag": tag}
_log_audit(
action="cook-image", action_details=action_details, service=service
)
return returncode
except KeyboardInterrupt:
print("\nProcess interrupted by the user. Cancelling.", file=sys.stderr)
return 2
def get_container_name():
return "paasta_local_run_{}_{}".format(get_username(), randint(1, 999999))
def configure_and_run_docker_container(args, docker_img, instance_config,
system_paasta_config):
volumes = list()
for volume in instance_config.get_volumes(
system_paasta_config.get_volumes()):
if os.path.exists(volume["hostPath"]):
volumes.append("{}:{}:{}".format(volume["hostPath"],
volume["containerPath"],
volume["mode"].lower()))
else:
paasta_print(
PaastaColors.yellow(
"Warning: Path %s does not exist on this host. Skipping this binding."
% volume["hostPath"]),
file=sys.stderr,
)
spark_ui_port = pick_random_port(args.service + str(os.getpid()))
spark_app_name = "paasta_spark_run_{}".format(get_username())
container_name = spark_app_name + "_" + str(spark_ui_port)
original_docker_cmd = args.cmd or instance_config.get_cmd()
if "jupyter" not in original_docker_cmd:
spark_app_name = container_name
access_key, secret_key = get_aws_credentials(args)
spark_config_dict = get_spark_config(
args=args,
spark_app_name=spark_app_name,
spark_ui_port=spark_ui_port,
docker_img=docker_img,
system_paasta_config=system_paasta_config,
volumes=volumes,
access_key=access_key,
secret_key=secret_key,
)
spark_conf_str = create_spark_config_str(spark_config_dict,
is_mrjob=args.mrjob)
# Spark client specific volumes
volumes.append("%s:rw" % args.work_dir)
volumes.append("/etc/passwd:/etc/passwd:ro")
volumes.append("/etc/group:/etc/group:ro")
volumes.append("/nail/home:/nail/home:rw")
environment = instance_config.get_env_dictionary()
environment.update(
get_spark_env(args, spark_conf_str, spark_ui_port, access_key,
secret_key))
webui_url = f"http://{socket.getfqdn()}:{spark_ui_port}"
docker_cmd = get_docker_cmd(args, instance_config, spark_conf_str)
if "history-server" in docker_cmd:
paasta_print(f"\nSpark history server URL {webui_url}\n")
elif any(c in docker_cmd
for c in ["pyspark", "spark-shell", "spark-submit"]):
paasta_print(f"\nSpark monitoring URL {webui_url}\n")
if clusterman_metrics and _should_emit_resource_requirements(
docker_cmd, args.mrjob):
try:
emit_resource_requirements(spark_config_dict, args.cluster,
webui_url)
except Boto3Error as e:
paasta_print(
PaastaColors.red(
f"Encountered {e} while attempting to send resource requirements to Clusterman."
))
if args.suppress_clusterman_metrics_errors:
paasta_print(
"Continuing anyway since --suppress-clusterman-metrics-errors was passed"
)
else:
raise
return run_docker_container(
container_name=container_name,
volumes=volumes,
environment=environment,
docker_img=docker_img,
docker_cmd=docker_cmd,
dry_run=args.dry_run,
nvidia=args.nvidia,
)
def get_spark_config(
args,
spark_app_name,
spark_ui_port,
docker_img,
system_paasta_config,
volumes,
):
# User configurable Spark options
user_args = {
'spark.app.name': spark_app_name,
'spark.cores.max': '4',
'spark.executor.cores': '2',
'spark.executor.memory': '4g',
# Use \; for multiple constraints. e.g.
# instance_type:m4.10xlarge\;pool:default
'spark.mesos.constraints': 'pool:%s' % args.pool,
'spark.mesos.executor.docker.forcePullImage': 'true',
}
# Spark options managed by PaaSTA
cluster_fqdn = system_paasta_config.get_cluster_fqdn_format().format(
cluster=args.cluster)
mesos_address = '{}:{}'.format(
find_mesos_leader(cluster_fqdn),
MESOS_MASTER_PORT,
)
non_user_args = {
'spark.master':
'mesos://%s' % mesos_address,
'spark.ui.port':
spark_ui_port,
'spark.executorEnv.PAASTA_SERVICE':
args.service,
'spark.executorEnv.PAASTA_INSTANCE':
'{}_{}'.format(args.instance, get_username()),
'spark.executorEnv.PAASTA_CLUSTER':
args.cluster,
'spark.mesos.executor.docker.parameters':
'label=paasta_service={},label=paasta_instance={}_{}'.format(
args.service,
args.instance,
get_username(),
),
'spark.mesos.executor.docker.volumes':
','.join(volumes),
'spark.mesos.executor.docker.image':
docker_img,
'spark.mesos.principal':
'spark',
'spark.mesos.secret':
_load_mesos_secret(),
# derby.system.home property defaulting to '.',
# which requires directory permission changes.
'spark.driver.extraJavaOptions':
'-Dderby.system.home=/tmp/derby',
}
if not args.build and not args.image:
non_user_args['spark.mesos.uris'] = 'file:///root/.dockercfg'
if args.spark_args:
spark_args = args.spark_args.split()
for spark_arg in spark_args:
fields = spark_arg.split('=')
if len(fields) != 2:
paasta_print(
PaastaColors.red(
"Spark option %s is not in format option=value." %
spark_arg, ),
file=sys.stderr,
)
sys.exit(1)
if fields[0] in non_user_args:
paasta_print(
PaastaColors.red(
"Spark option {} is set by PaaSTA with {}.".format(
fields[0],
non_user_args[fields[0]],
), ),
file=sys.stderr,
)
sys.exit(1)
# Update default configuration
user_args[fields[0]] = fields[1]
if int(user_args['spark.cores.max']) < int(
user_args['spark.executor.cores']):
paasta_print(
PaastaColors.red(
"Total number of cores {} is less than per-executor cores {}.".
format(
user_args['spark.cores.max'],
user_args['spark.executor.cores'],
), ),
file=sys.stderr,
)
sys.exit(1)
exec_mem = user_args['spark.executor.memory']
if exec_mem[-1] != 'g' or not exec_mem[:-1].isdigit() or int(
exec_mem[:-1]) > 32:
paasta_print(
PaastaColors.red(
"Executor memory {} not in format dg (d<=32).".format(
user_args['spark.executor.memory'], ), ),
file=sys.stderr,
)
sys.exit(1)
# Limit a container's cpu usage
non_user_args[
'spark.mesos.executor.docker.parameters'] += ',cpus={}'.format(
user_args['spark.executor.cores'])
return dict(non_user_args, **user_args)
def configure_and_run_docker_container(
args,
docker_img,
instance_config,
system_paasta_config,
):
volumes = list()
for volume in instance_config.get_volumes(
system_paasta_config.get_volumes()):
if os.path.exists(volume['hostPath']):
volumes.append('{}:{}:{}'.format(volume['hostPath'],
volume['containerPath'],
volume['mode'].lower()))
else:
paasta_print(
PaastaColors.yellow(
"Warning: Path %s does not exist on this host. Skipping this binding."
% volume['hostPath'], ), )
spark_ui_port = pick_random_port(args.service + str(os.getpid()))
spark_app_name = 'paasta_spark_run_{}'.format(get_username())
container_name = spark_app_name + "_" + str(spark_ui_port)
original_docker_cmd = args.cmd or instance_config.get_cmd()
if 'jupyter' not in original_docker_cmd:
spark_app_name = container_name
spark_config_dict = get_spark_config(
args=args,
spark_app_name=spark_app_name,
spark_ui_port=spark_ui_port,
docker_img=docker_img,
system_paasta_config=system_paasta_config,
volumes=volumes,
)
spark_conf_str = create_spark_config_str(spark_config_dict)
# Spark client specific volumes
volumes.append('%s:rw' % args.work_dir)
volumes.append('/etc/passwd:/etc/passwd:ro')
volumes.append('/etc/group:/etc/group:ro')
environment = instance_config.get_env_dictionary()
environment.update(get_spark_env(
args,
spark_conf_str,
spark_ui_port,
), )
webui_url = f'http://{socket.getfqdn()}:{spark_ui_port}'
docker_cmd = get_docker_cmd(args, instance_config, spark_conf_str)
if 'history-server' in docker_cmd:
paasta_print(f'\nSpark history server URL {webui_url}\n')
elif any(c in docker_cmd
for c in ['pyspark', 'spark-shell', 'spark-submit']):
paasta_print(f'\nSpark monitoring URL {webui_url}\n')
if clusterman_metrics and _should_emit_resource_requirements(docker_cmd):
try:
emit_resource_requirements(spark_config_dict, args.cluster,
webui_url)
except Boto3Error as e:
paasta_print(
PaastaColors.red(
f'Encountered {e} while attempting to send resource requirements to Clusterman.'
), )
if args.suppress_clusterman_metrics_errors:
paasta_print(
'Continuing anyway since --suppress-clusterman-metrics-errors was passed'
)
else:
raise
return run_docker_container(
container_name=container_name,
volumes=volumes,
environment=environment,
docker_img=docker_img,
docker_cmd=docker_cmd,
dry_run=args.dry_run,
)
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
paasta_print("A local Makefile with a 'cook-image' target is required for --build", file=sys.stderr)
paasta_print("If you meant to pull the docker image from the registry, explicitly pass --pull", file=sys.stderr)
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
paasta_print(
PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates"
"PaaSTA is not configured locally on this host, and local-run may not behave"
"the same way it would behave on a server configured for PaaSTA."
),
sep='\n',
)
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config['default_cluster']
except KeyError:
paasta_print(
PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster."
"Please pass one to local-run using '-c'."),
sep='\n',
file=sys.stderr,
)
return 1
instance = args.instance
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
return configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
paasta_print(
'Can\'t run Docker container. Error: %s' % str(e),
file=sys.stderr,
)
return 1
def get_spark_config(
args,
spark_app_name,
spark_ui_port,
docker_img,
system_paasta_config,
volumes,
access_key,
secret_key,
):
# User configurable Spark options
user_args = {
"spark.app.name": spark_app_name,
"spark.cores.max": "4",
"spark.executor.cores": "2",
"spark.executor.memory": "4g",
# Use \; for multiple constraints. e.g.
# instance_type:m4.10xlarge\;pool:default
"spark.mesos.constraints": "pool:%s" % args.pool,
"spark.mesos.executor.docker.forcePullImage": "true",
}
default_event_log_dir = get_default_event_log_dir(access_key, secret_key)
if default_event_log_dir is not None:
user_args["spark.eventLog.enabled"] = "true"
user_args["spark.eventLog.dir"] = default_event_log_dir
# Spark options managed by PaaSTA
cluster_fqdn = system_paasta_config.get_cluster_fqdn_format().format(
cluster=args.cluster)
mesos_address = "{}:{}".format(find_mesos_leader(cluster_fqdn),
MESOS_MASTER_PORT)
non_user_args = {
"spark.master":
"mesos://%s" % mesos_address,
"spark.ui.port":
spark_ui_port,
"spark.executorEnv.PAASTA_SERVICE":
args.service,
"spark.executorEnv.PAASTA_INSTANCE":
"{}_{}".format(args.instance, get_username()),
"spark.executorEnv.PAASTA_CLUSTER":
args.cluster,
"spark.executorEnv.PAASTA_INSTANCE_TYPE":
"spark",
"spark.mesos.executor.docker.parameters":
"label=paasta_service={},label=paasta_instance={}_{}".format(
args.service, args.instance, get_username()),
"spark.mesos.executor.docker.volumes":
",".join(volumes),
"spark.mesos.executor.docker.image":
docker_img,
"spark.mesos.principal":
"spark",
"spark.mesos.secret":
_load_mesos_secret(),
# derby.system.home property defaulting to '.',
# which requires directory permission changes.
"spark.driver.extraJavaOptions":
"-Dderby.system.home=/tmp/derby",
}
if not args.build and not args.image:
non_user_args["spark.mesos.uris"] = "file:///root/.dockercfg"
if args.spark_args:
spark_args = args.spark_args.split()
for spark_arg in spark_args:
fields = spark_arg.split("=")
if len(fields) != 2:
paasta_print(
PaastaColors.red(
"Spark option %s is not in format option=value." %
spark_arg),
file=sys.stderr,
)
sys.exit(1)
if fields[0] in non_user_args:
paasta_print(
PaastaColors.red(
"Spark option {} is set by PaaSTA with {}.".format(
fields[0], non_user_args[fields[0]])),
file=sys.stderr,
)
sys.exit(1)
# Update default configuration
user_args[fields[0]] = fields[1]
if "spark.sql.shuffle.partitions" not in user_args:
num_partitions = str(2 * int(user_args["spark.cores.max"]))
user_args["spark.sql.shuffle.partitions"] = num_partitions
paasta_print(
PaastaColors.yellow(
f"Warning: spark.sql.shuffle.partitions has been set to"
f" {num_partitions} to be equal to twice the number of "
f"requested cores, but you should consider setting a "
f"higher value if necessary."))
if int(user_args["spark.cores.max"]) < int(
user_args["spark.executor.cores"]):
paasta_print(
PaastaColors.red(
"Total number of cores {} is less than per-executor cores {}.".
format(user_args["spark.cores.max"],
user_args["spark.executor.cores"])),
file=sys.stderr,
)
sys.exit(1)
exec_mem = user_args["spark.executor.memory"]
if exec_mem[-1] != "g" or not exec_mem[:-1].isdigit() or int(
exec_mem[:-1]) > 32:
paasta_print(
PaastaColors.red(
"Executor memory {} not in format dg (d<=32).".format(
user_args["spark.executor.memory"])),
file=sys.stderr,
)
sys.exit(1)
# Limit a container's cpu usage
non_user_args[
"spark.mesos.executor.docker.parameters"] += ",cpus={}".format(
user_args["spark.executor.cores"])
return dict(non_user_args, **user_args)
