def main(): print('-------------------------------------------------------') print('Please run echo $$PAASTA_SYSTEM_CONFIG_DIR to continue') print( "Please set environment variable PAASTA_TEST_CLUSTER to the cluster you want to use." ) print("This is necessary for tron jobs") print('-------------------------------------------------------') cluster = os.environ.get('PAASTA_TEST_CLUSTER', 'norcal-devc') config_path = 'etc_paasta_for_development' copy_tree('/etc/paasta', os.path.join(os.getcwd(), config_path)) # Generate tron.json tron_config = {'tron': {'url': f'http://tron-{cluster}:8089'}} with open(config_path + '/tron.json', 'w') as f: json.dump(tron_config, f) # find unused port port = pick_random_port('paasta-dev-api') # Generate api endpoints api_endpoints = {'api_endpoints': {cluster: f'http://localhost:{port}'}} api_endpoints_path = os.path.join(os.getcwd(), config_path, 'api_endpoints.json') os.chmod(api_endpoints_path, 0o777) with open(api_endpoints_path, 'w') as f: json.dump(api_endpoints, f) # export config path os.environ['PAASTA_SYSTEM_CONFIG_DIR'] = config_path os.execl( '.tox/py36-linux/bin/python', '.tox/py36-linux/bin/python', '-m', 'paasta_tools.api.api', *['-D', '-c', cluster, str(port)], )
def configure_and_run_docker_container( args, docker_img, instance_config, system_paasta_config, ): volumes = list() for volume in instance_config.get_volumes( system_paasta_config.get_volumes()): if os.path.exists(volume['hostPath']): volumes.append('%s:%s:%s' % (volume['hostPath'], volume['containerPath'], volume['mode'].lower())) else: paasta_print( PaastaColors.yellow( "Warning: Path %s does not exist on this host. Skipping this binding." % volume['hostPath'], ), ) volumes.append('%s:%s:rw' % (os.getcwd(), DEFAULT_SPARK_WORK_DIR)) if args.cmd is None: docker_cmd = instance_config.get_cmd() else: docker_cmd = args.cmd if docker_cmd is None: paasta_print( "A command is required, pyspark, spark-shell, spark-submit or jupyter", file=sys.stderr) return 1 # Changes at docker ENTRYPOINT or CMD does not work. elif docker_cmd == 'jupyter': docker_cmd = 'jupyter notebook -y --ip=%s --notebook-dir=%s --allow-root' % ( socket.getfqdn(), DEFAULT_SPARK_WORK_DIR, ) spark_ui_port = pick_random_port(args.service) container_name = 'paasta_spark_run_%s_%s' % (get_username(), spark_ui_port) # Do not put memory and CPU limits on Spark driver for now. # Toree won't work with the default memory-swap setting. environment = instance_config.get_env_dictionary() environment.update( get_spark_configuration( args, container_name, spark_ui_port, docker_img, system_paasta_config, ), ) return run_docker_container( container_name=container_name, volumes=volumes, environment=environment, docker_img=docker_img, docker_cmd=docker_cmd, dry_run=args.dry_run, )
def test_pick_random_port(): def fake_epr(ip, port): return port with mock.patch.object( ephemeral_port_reserve, 'reserve', side_effect=fake_epr, ), mock.patch.object( getpass, 'getuser', return_value='nobody', autospec=True, ): # Two calls with the same service should try to reserve the same port. port1 = utils.pick_random_port('fake_service') port2 = utils.pick_random_port('fake_service') assert port1 == port2 assert 33000 <= port1 < 58000 # A third call with a different service should try to reserve a different port. port3 = utils.pick_random_port('different_fake_service') assert port1 != port3 assert 33000 <= port3 < 58000
def main(): print("-------------------------------------------------------") print( "Please run export PAASTA_SYSTEM_CONFIG_DIR=etc_paasta_for_development to continue" ) print( "Please set environment variable PAASTA_TEST_CLUSTER to the cluster you want to use." ) print("This is necessary for tron jobs") print("-------------------------------------------------------") cluster = os.environ.get("PAASTA_TEST_CLUSTER", "norcal-devc") config_path = "etc_paasta_for_development" copy_tree("/etc/paasta", os.path.join(os.getcwd(), config_path)) # Generate tron.json tron_config = {"tron": {"url": f"http://tron-{cluster}:8089"}} with open(config_path + "/tron.json", "w") as f: json.dump(tron_config, f) # find unused port port = pick_random_port("paasta-dev-api") # Generate api endpoints api_endpoints = {"api_endpoints": {cluster: f"http://localhost:{port}"}} api_endpoints_path = os.path.join(os.getcwd(), config_path, "api_endpoints.json") os.chmod(api_endpoints_path, 0o777) with open(api_endpoints_path, "w") as f: json.dump(api_endpoints, f) # export config path os.environ["PAASTA_SYSTEM_CONFIG_DIR"] = config_path os.execl( ".tox/py36-linux/bin/python", ".tox/py36-linux/bin/python", "-m", "paasta_tools.api.api", *["-D", "-c", cluster, str(port)], )
def run_docker_container( docker_client, service, instance, docker_url, volumes, interactive, command, healthcheck, healthcheck_only, user_port, instance_config, secret_provider_name, soa_dir=DEFAULT_SOA_DIR, dry_run=False, json_dict=False, framework=None, secret_provider_kwargs={}, skip_secrets=False, ): """docker-py has issues running a container with a TTY attached, so for consistency we execute 'docker run' directly in both interactive and non-interactive modes. In non-interactive mode when the run is complete, stop the container and remove it (with docker-py). """ if user_port: if check_if_port_free(user_port): chosen_port = user_port else: paasta_print( PaastaColors.red( "The chosen port is already in use!\n" "Try specifying another one, or omit (--port|-o) and paasta will find a free one for you" ), file=sys.stderr, ) sys.exit(1) else: chosen_port = pick_random_port(service) environment = instance_config.get_env_dictionary() if not skip_secrets: secret_environment = decrypt_secret_environment_variables( secret_provider_name=secret_provider_name, environment=environment, soa_dir=soa_dir, service_name=service, cluster_name=instance_config.cluster, secret_provider_kwargs=secret_provider_kwargs, ) environment.update(secret_environment) local_run_environment = get_local_run_environment_vars( instance_config=instance_config, port0=chosen_port, framework=framework ) environment.update(local_run_environment) net = instance_config.get_net() memory = instance_config.get_mem() container_name = get_container_name() docker_params = instance_config.format_docker_parameters() healthcheck_mode, healthcheck_data = get_healthcheck_for_instance( service, instance, instance_config, chosen_port, soa_dir=soa_dir ) if healthcheck_mode is None: container_port = None interactive = True elif not user_port and not healthcheck and not healthcheck_only: container_port = None else: try: container_port = instance_config.get_container_port() except AttributeError: container_port = None simulate_healthcheck = ( healthcheck_only or healthcheck ) and healthcheck_mode is not None docker_run_args = dict( memory=memory, chosen_port=chosen_port, container_port=container_port, container_name=container_name, volumes=volumes, env=environment, interactive=interactive, detach=simulate_healthcheck, docker_hash=docker_url, command=command, net=net, docker_params=docker_params, ) docker_run_cmd = get_docker_run_cmd(**docker_run_args) joined_docker_run_cmd = " ".join(docker_run_cmd) if dry_run: if json_dict: paasta_print(json.dumps(docker_run_args)) else: paasta_print(json.dumps(docker_run_cmd)) return 0 else: paasta_print( "Running docker command:\n%s" % PaastaColors.grey(joined_docker_run_cmd) ) merged_env = {**os.environ, **environment} if interactive or not simulate_healthcheck: # NOTE: This immediately replaces us with the docker run cmd. Docker # run knows how to clean up the running container in this situation. wrapper_path = shutil.which("paasta_docker_wrapper") # To properly simulate mesos, we pop the PATH, which is not available to # The executor merged_env.pop("PATH") execlpe(wrapper_path, *docker_run_cmd, merged_env) # For testing, when execlpe is patched out and doesn't replace us, we # still want to bail out. return 0 container_started = False container_id = None try: (returncode, output) = _run(docker_run_cmd, env=merged_env) if returncode != 0: paasta_print( "Failure trying to start your container!" "Returncode: %d" "Output:" "%s" "" "Fix that problem and try again." "http://y/paasta-troubleshooting" % (returncode, output), sep="\n", ) # Container failed to start so no need to cleanup; just bail. sys.exit(1) container_started = True container_id = get_container_id(docker_client, container_name) paasta_print("Found our container running with CID %s" % container_id) if simulate_healthcheck: healthcheck_result = simulate_healthcheck_on_service( instance_config=instance_config, docker_client=docker_client, container_id=container_id, healthcheck_mode=healthcheck_mode, healthcheck_data=healthcheck_data, healthcheck_enabled=healthcheck, ) def _output_exit_code(): returncode = docker_client.inspect_container(container_id)["State"][ "ExitCode" ] paasta_print(f"Container exited: {returncode})") if healthcheck_only: if container_started: _output_exit_code() _cleanup_container(docker_client, container_id) if healthcheck_mode is None: paasta_print( "--healthcheck-only, but no healthcheck is defined for this instance!" ) sys.exit(1) elif healthcheck_result is True: sys.exit(0) else: sys.exit(1) running = docker_client.inspect_container(container_id)["State"]["Running"] if running: paasta_print("Your service is now running! Tailing stdout and stderr:") for line in docker_client.attach( container_id, stderr=True, stream=True, logs=True ): paasta_print(line) else: _output_exit_code() returncode = 3 except KeyboardInterrupt: returncode = 3 # Cleanup if the container exits on its own or interrupted. if container_started: returncode = docker_client.inspect_container(container_id)["State"]["ExitCode"] _cleanup_container(docker_client, container_id) return returncode
def configure_and_run_docker_container( args, docker_img, instance_config, system_paasta_config, ): volumes = list() for volume in instance_config.get_volumes( system_paasta_config.get_volumes()): if os.path.exists(volume['hostPath']): volumes.append('{}:{}:{}'.format(volume['hostPath'], volume['containerPath'], volume['mode'].lower())) else: paasta_print( PaastaColors.yellow( "Warning: Path %s does not exist on this host. Skipping this binding." % volume['hostPath'], ), ) spark_ui_port = pick_random_port(args.service + str(os.getpid())) spark_app_name = 'paasta_spark_run_{}'.format(get_username()) container_name = spark_app_name + "_" + str(spark_ui_port) original_docker_cmd = args.cmd or instance_config.get_cmd() if 'jupyter' not in original_docker_cmd: spark_app_name = container_name spark_config_dict = get_spark_config( args=args, spark_app_name=spark_app_name, spark_ui_port=spark_ui_port, docker_img=docker_img, system_paasta_config=system_paasta_config, volumes=volumes, ) spark_conf_str = create_spark_config_str(spark_config_dict) # Spark client specific volumes volumes.append('%s:rw' % args.work_dir) volumes.append('/etc/passwd:/etc/passwd:ro') volumes.append('/etc/group:/etc/group:ro') environment = instance_config.get_env_dictionary() environment.update(get_spark_env( args, spark_conf_str, spark_ui_port, ), ) webui_url = f'http://{socket.getfqdn()}:{spark_ui_port}' docker_cmd = get_docker_cmd(args, instance_config, spark_conf_str) if 'history-server' in docker_cmd: paasta_print(f'\nSpark history server URL {webui_url}\n') elif any(c in docker_cmd for c in ['pyspark', 'spark-shell', 'spark-submit']): paasta_print(f'\nSpark monitoring URL {webui_url}\n') if clusterman_metrics and _should_emit_resource_requirements(docker_cmd): try: emit_resource_requirements(spark_config_dict, args.cluster, webui_url) except Boto3Error as e: paasta_print( PaastaColors.red( f'Encountered {e} while attempting to send resource requirements to Clusterman.' ), ) if args.suppress_clusterman_metrics_errors: paasta_print( 'Continuing anyway since --suppress-clusterman-metrics-errors was passed' ) else: raise return run_docker_container( container_name=container_name, volumes=volumes, environment=environment, docker_img=docker_img, docker_cmd=docker_cmd, dry_run=args.dry_run, )
def configure_and_run_docker_container(args, docker_img, instance_config, system_paasta_config): volumes = list() for volume in instance_config.get_volumes( system_paasta_config.get_volumes()): if os.path.exists(volume["hostPath"]): volumes.append("{}:{}:{}".format(volume["hostPath"], volume["containerPath"], volume["mode"].lower())) else: paasta_print( PaastaColors.yellow( "Warning: Path %s does not exist on this host. Skipping this binding." % volume["hostPath"]), file=sys.stderr, ) spark_ui_port = pick_random_port(args.service + str(os.getpid())) spark_app_name = "paasta_spark_run_{}".format(get_username()) container_name = spark_app_name + "_" + str(spark_ui_port) original_docker_cmd = args.cmd or instance_config.get_cmd() if "jupyter" not in original_docker_cmd: spark_app_name = container_name access_key, secret_key = get_aws_credentials(args) spark_config_dict = get_spark_config( args=args, spark_app_name=spark_app_name, spark_ui_port=spark_ui_port, docker_img=docker_img, system_paasta_config=system_paasta_config, volumes=volumes, access_key=access_key, secret_key=secret_key, ) spark_conf_str = create_spark_config_str(spark_config_dict, is_mrjob=args.mrjob) # Spark client specific volumes volumes.append("%s:rw" % args.work_dir) volumes.append("/etc/passwd:/etc/passwd:ro") volumes.append("/etc/group:/etc/group:ro") volumes.append("/nail/home:/nail/home:rw") environment = instance_config.get_env_dictionary() environment.update( get_spark_env(args, spark_conf_str, spark_ui_port, access_key, secret_key)) webui_url = f"http://{socket.getfqdn()}:{spark_ui_port}" docker_cmd = get_docker_cmd(args, instance_config, spark_conf_str) if "history-server" in docker_cmd: paasta_print(f"\nSpark history server URL {webui_url}\n") elif any(c in docker_cmd for c in ["pyspark", "spark-shell", "spark-submit"]): paasta_print(f"\nSpark monitoring URL {webui_url}\n") if clusterman_metrics and _should_emit_resource_requirements( docker_cmd, args.mrjob): try: emit_resource_requirements(spark_config_dict, args.cluster, webui_url) except Boto3Error as e: paasta_print( PaastaColors.red( f"Encountered {e} while attempting to send resource requirements to Clusterman." )) if args.suppress_clusterman_metrics_errors: paasta_print( "Continuing anyway since --suppress-clusterman-metrics-errors was passed" ) else: raise return run_docker_container( container_name=container_name, volumes=volumes, environment=environment, docker_img=docker_img, docker_cmd=docker_cmd, dry_run=args.dry_run, nvidia=args.nvidia, )
def configure_and_run_docker_container( args, docker_img, instance_config, system_paasta_config, ): volumes = list() for volume in instance_config.get_volumes( system_paasta_config.get_volumes()): if os.path.exists(volume['hostPath']): volumes.append('{}:{}:{}'.format(volume['hostPath'], volume['containerPath'], volume['mode'].lower())) else: paasta_print( PaastaColors.yellow( "Warning: Path %s does not exist on this host. Skipping this binding." % volume['hostPath'], ), ) spark_ui_port = pick_random_port(args.service) container_name = 'paasta_spark_run_{}_{}'.format(get_username(), spark_ui_port) spark_conf_str = get_spark_conf_str( args=args, container_name=container_name, spark_ui_port=spark_ui_port, docker_img=docker_img, system_paasta_config=system_paasta_config, volumes=volumes, ) # Spark client specific volumes volumes.append('%s:rw' % args.work_dir) volumes.append('/etc/passwd:/etc/passwd:ro') volumes.append('/etc/group:/etc/group:ro') docker_cmd = get_docker_cmd(args, instance_config, spark_conf_str) if docker_cmd is None: paasta_print( "A command is required, pyspark, spark-shell, spark-submit or jupyter", file=sys.stderr) return 1 environment = instance_config.get_env_dictionary() environment.update(get_spark_env( args, spark_conf_str, ), ) paasta_print('\nSpark Monitoring URL http://%s:%d\n' % (socket.getfqdn(), spark_ui_port)) return run_docker_container( container_name=container_name, volumes=volumes, environment=environment, docker_img=docker_img, docker_cmd=docker_cmd, dry_run=args.dry_run, )
def run_docker_container( docker_client, service, instance, docker_hash, volumes, interactive, command, healthcheck, healthcheck_only, user_port, instance_config, soa_dir=DEFAULT_SOA_DIR, dry_run=False, json_dict=False, framework=None, ): """docker-py has issues running a container with a TTY attached, so for consistency we execute 'docker run' directly in both interactive and non-interactive modes. In non-interactive mode when the run is complete, stop the container and remove it (with docker-py). """ if user_port: if check_if_port_free(user_port): chosen_port = user_port else: paasta_print( PaastaColors.red( "The chosen port is already in use!\n" "Try specifying another one, or omit (--port|-o) and paasta will find a free one for you", ), file=sys.stderr, ) sys.exit(1) else: chosen_port = pick_random_port(service) environment = instance_config.get_env_dictionary() local_run_environment = get_local_run_environment_vars( instance_config=instance_config, port0=chosen_port, framework=framework, ) environment.update(local_run_environment) net = instance_config.get_net() memory = instance_config.get_mem() container_name = get_container_name() docker_params = instance_config.format_docker_parameters() healthcheck_mode, healthcheck_data = get_healthcheck_for_instance( service, instance, instance_config, chosen_port, soa_dir=soa_dir, ) if healthcheck_mode is None: container_port = None interactive = True elif not user_port and not healthcheck and not healthcheck_only: container_port = None else: try: container_port = instance_config.get_container_port() except AttributeError: container_port = None simulate_healthcheck = (healthcheck_only or healthcheck) and healthcheck_mode is not None docker_run_args = dict( memory=memory, chosen_port=chosen_port, container_port=container_port, container_name=container_name, volumes=volumes, env=environment, interactive=interactive, detach=simulate_healthcheck, docker_hash=docker_hash, command=command, net=net, docker_params=docker_params, ) docker_run_cmd = get_docker_run_cmd(**docker_run_args) joined_docker_run_cmd = ' '.join(docker_run_cmd) if dry_run: if json_dict: paasta_print(json.dumps(docker_run_args)) else: paasta_print(json.dumps(docker_run_cmd)) return 0 else: paasta_print('Running docker command:\n%s' % PaastaColors.grey(joined_docker_run_cmd)) if interactive or not simulate_healthcheck: # NOTE: This immediately replaces us with the docker run cmd. Docker # run knows how to clean up the running container in this situation. execlp('paasta_docker_wrapper', *docker_run_cmd) # For testing, when execlp is patched out and doesn't replace us, we # still want to bail out. return 0 container_started = False container_id = None try: (returncode, output) = _run(docker_run_cmd) if returncode != 0: paasta_print( 'Failure trying to start your container!' 'Returncode: %d' 'Output:' '%s' '' 'Fix that problem and try again.' 'http://y/paasta-troubleshooting' % (returncode, output), sep='\n', ) # Container failed to start so no need to cleanup; just bail. sys.exit(1) container_started = True container_id = get_container_id(docker_client, container_name) paasta_print('Found our container running with CID %s' % container_id) if simulate_healthcheck: healthcheck_result = simulate_healthcheck_on_service( instance_config=instance_config, docker_client=docker_client, container_id=container_id, healthcheck_mode=healthcheck_mode, healthcheck_data=healthcheck_data, healthcheck_enabled=healthcheck, ) def _output_stdout_and_exit_code(): returncode = docker_client.inspect_container( container_id)['State']['ExitCode'] paasta_print('Container exited: %d)' % returncode) paasta_print('Here is the stdout and stderr:\n\n') paasta_print( docker_client.attach(container_id, stderr=True, stream=False, logs=True), ) if healthcheck_only: if container_started: _output_stdout_and_exit_code() _cleanup_container(docker_client, container_id) if healthcheck_mode is None: paasta_print( '--healthcheck-only, but no healthcheck is defined for this instance!' ) sys.exit(1) elif healthcheck_result is True: sys.exit(0) else: sys.exit(1) running = docker_client.inspect_container( container_id)['State']['Running'] if running: paasta_print( 'Your service is now running! Tailing stdout and stderr:') for line in docker_client.attach(container_id, stderr=True, stream=True, logs=True): paasta_print(line) else: _output_stdout_and_exit_code() returncode = 3 except KeyboardInterrupt: returncode = 3 # Cleanup if the container exits on its own or interrupted. if container_started: returncode = docker_client.inspect_container( container_id)['State']['ExitCode'] _cleanup_container(docker_client, container_id) return returncode
def configure_and_run_docker_container( args, docker_img, instance_config, system_paasta_config, ): volumes = list() for volume in instance_config.get_volumes( system_paasta_config.get_volumes()): if os.path.exists(volume['hostPath']): volumes.append('%s:%s:%s' % (volume['hostPath'], volume['containerPath'], volume['mode'].lower())) else: paasta_print( PaastaColors.yellow( "Warning: Path %s does not exist on this host. Skipping this binding." % volume['hostPath'], ), ) spark_ui_port = pick_random_port(args.service) container_name = 'paasta_spark_run_%s_%s' % (get_username(), spark_ui_port) spark_conf_str = get_spark_conf_str( args=args, container_name=container_name, spark_ui_port=spark_ui_port, docker_img=docker_img, system_paasta_config=system_paasta_config, volumes=volumes, ) # Spark client specific volumes volumes.append('%s:rw' % args.work_dir) volumes.append('/etc/passwd:/etc/passwd:ro') volumes.append('/etc/group:/etc/group:ro') if args.cmd is None: docker_cmd = instance_config.get_cmd() else: docker_cmd = args.cmd if docker_cmd is None: paasta_print( "A command is required, pyspark, spark-shell, spark-submit or jupyter", file=sys.stderr) return 1 # Default cli options to start the jupyter notebook server. if docker_cmd == 'jupyter': docker_cmd = 'jupyter notebook -y --ip=%s --notebook-dir=%s' % ( socket.getfqdn(), args.work_dir.split(':')[1], ) # Spark options are passed as options to pyspark and spark-shell. # For jupyter, environment variable SPARK_OPTS is set instead. elif docker_cmd in ['pyspark', 'spark-shell']: docker_cmd = docker_cmd + ' ' + spark_conf_str elif docker_cmd.startswith('spark-submit'): docker_cmd = 'spark-submit ' + spark_conf_str + docker_cmd[ len('spark-submit'):] environment = instance_config.get_env_dictionary() environment.update(get_spark_env( args, spark_conf_str, ), ) paasta_print('\nSpark Monitoring URL http://%s:%d\n' % (socket.getfqdn(), spark_ui_port)) return run_docker_container( container_name=container_name, volumes=volumes, environment=environment, docker_img=docker_img, docker_cmd=docker_cmd, dry_run=args.dry_run, )