Ejemplo n.º 1
0
def local_makefile_present():
    if makefile_responds_to('cook-image'):
        sys.stderr.write("Local Makefile with 'cook-image' target detected. Assuming --build\n")
        return True
    else:
        sys.stderr.write("No Makefile with 'cook-image' target detected. Assuming --pull\n")
        return False
Ejemplo n.º 2
0
def local_makefile_present():
    if makefile_responds_to('cook-image'):
        sys.stderr.write("Local Makefile with 'cook-image' target detected. Assuming --build\n")
        return True
    else:
        sys.stderr.write("No Makefile with 'cook-image' target detected. Assuming --pull\n")
        return False
Ejemplo n.º 3
0
def paasta_cook_image(args, service=None, soa_dir=None):
    """Build a docker image"""
    if not service:
        service = args.service
    if service.startswith('services-'):
        service = service.split('services-', 1)[1]
    if not soa_dir:
        soa_dir = args.yelpsoa_config_root
    validate_service_name(service, soa_dir)

    run_env = os.environ.copy()
    default_tag = 'paasta-cook-image-{}-{}'.format(service, get_username())
    tag = run_env.get('DOCKER_TAG', default_tag)
    run_env['DOCKER_TAG'] = tag

    if not makefile_responds_to('cook-image'):
        paasta_print(
            'ERROR: local-run now requires a cook-image target to be present in the Makefile. See'
            'http://paasta.readthedocs.io/en/latest/about/contract.html', file=sys.stderr,
        )
        return 1

    try:
        cmd = 'make cook-image'
        returncode, output = _run(
            cmd,
            env=run_env,
            log=True,
            component='build',
            service=service,
            loglevel='debug',
        )
        if returncode != 0:
            _log(
                service=service,
                line='ERROR: make cook-image failed for %s.' % service,
                component='build',
                level='event',
            )
        else:
            action_details = {
                'tag': tag,
            }
            _log_audit(
                action='cook-image',
                action_details=action_details,
                service=service,
            )
        return returncode

    except KeyboardInterrupt:
        paasta_print('\nProcess interrupted by the user. Cancelling.', file=sys.stderr)
        return 2
Ejemplo n.º 4
0
def paasta_cook_image(args, service=None, soa_dir=None):
    """Build a docker image"""
    if not service:
        service = args.service
    if service.startswith("services-"):
        service = service.split("services-", 1)[1]
    if not soa_dir:
        soa_dir = args.yelpsoa_config_root
    validate_service_name(service, soa_dir)

    run_env = os.environ.copy()
    default_tag = "paasta-cook-image-{}-{}".format(service, get_username())
    tag = run_env.get("DOCKER_TAG", default_tag)
    run_env["DOCKER_TAG"] = tag

    if not makefile_responds_to("cook-image"):
        print(
            "ERROR: local-run now requires a cook-image target to be present in the Makefile. See"
            "http://paasta.readthedocs.io/en/latest/about/contract.html",
            file=sys.stderr,
        )
        return 1

    try:
        cmd = "make cook-image"
        returncode, output = _run(
            cmd,
            env=run_env,
            log=True,
            component="build",
            service=service,
            loglevel="debug",
        )
        if returncode != 0:
            _log(
                service=service,
                line="ERROR: make cook-image failed for %s." % service,
                component="build",
                level="event",
            )
        else:
            action_details = {"tag": tag}
            _log_audit(action="cook-image",
                       action_details=action_details,
                       service=service)
        return returncode

    except KeyboardInterrupt:
        print("\nProcess interrupted by the user. Cancelling.",
              file=sys.stderr)
        return 2
Ejemplo n.º 5
0
def paasta_local_run(args):
    if args.action == 'build' and not makefile_responds_to('cook-image'):
        sys.stderr.write(
            "A local Makefile with a 'cook-image' target is required for --build\n"
        )
        sys.stderr.write(
            "If you meant to pull the docker image from the registry, explicitly pass --pull\n"
        )
        return 1

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    cluster = guess_cluster(service=service, args=args)
    instance = guess_instance(service=service, cluster=cluster, args=args)
    docker_client = get_docker_client()

    if args.action == 'build':
        default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
        tag = os.environ.get('DOCKER_TAG', default_tag)
        os.environ['DOCKER_TAG'] = tag
        pull_image = False
        cook_return = paasta_cook_image(args=None,
                                        service=service,
                                        soa_dir=args.yelpsoa_config_root)
        if cook_return != 0:
            return cook_return
    elif args.action == 'dry_run':
        pull_image = False
        tag = None
    else:
        pull_image = True
        tag = None

    try:
        configure_and_run_docker_container(
            docker_client=docker_client,
            docker_hash=tag,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
            dry_run=args.action == 'dry_run',
        )
    except errors.APIError as e:
        sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
        return 1
Ejemplo n.º 6
0
def build_and_push_docker_image(args):
    """
    Build an image if the default Spark service image is not preferred.
    The image needs to be pushed to a registry for the Spark executors
    to pull.
    """
    if not makefile_responds_to('cook-image'):
        paasta_print(
            "A local Makefile with a 'cook-image' target is required for --build",
            file=sys.stderr,
        )
        return None

    default_tag = '{}-{}'.format(DEFAULT_SPARK_DOCKER_IMAGE_PREFIX,
                                 get_username())
    docker_tag = os.environ.get('DOCKER_TAG', default_tag)
    os.environ['DOCKER_TAG'] = docker_tag

    cook_return = paasta_cook_image(
        args=None,
        service=args.service,
        soa_dir=args.yelpsoa_config_root,
    )
    if cook_return is not 0:
        return None

    docker_url = f'{args.docker_registry}/{docker_tag}'
    command = f'docker tag {docker_tag} {docker_url}'
    paasta_print(PaastaColors.grey(command))
    retcode, _ = _run(command, stream=True)
    if retcode is not 0:
        return None

    if args.docker_registry != DEFAULT_SPARK_DOCKER_REGISTRY:
        command = 'sudo -H docker push %s' % docker_url
    else:
        command = 'docker push %s' % docker_url

    paasta_print(PaastaColors.grey(command))
    retcode, output = _run(command, stream=True)
    if retcode is not 0:
        return None

    return docker_url
Ejemplo n.º 7
0
def paasta_cook_image(args, service=None, soa_dir=None):
    """Build a docker image"""
    if service:
        service = service
    else:
        service = args.service
    if service and service.startswith('services-'):
        service = service.split('services-', 1)[1]
    validate_service_name(service, soa_dir)

    run_env = os.environ.copy()
    default_tag = 'paasta-cook-image-%s-%s' % (service, get_username())
    tag = run_env.get('DOCKER_TAG', default_tag)
    run_env['DOCKER_TAG'] = tag

    if not makefile_responds_to('cook-image'):
        sys.stderr.write('ERROR: local-run now requires a cook-image target to be present in the Makefile. See '
                         'http://paasta.readthedocs.io/en/latest/about/contract.html\n')
        return 1

    try:
        cmd = 'make cook-image'
        returncode, output = _run(
            cmd,
            env=run_env,
            log=True,
            component='build',
            service=service,
            loglevel='debug'
        )
        if returncode != 0:
            _log(
                service=service,
                line='ERROR: make cook-image failed for %s.' % service,
                component='build',
                level='event',
            )
        return returncode

    except KeyboardInterrupt:
        sys.stderr.write('\nProcess interrupted by the user. Cancelling.\n')
        return 2
def paasta_cook_image(args, service=None, soa_dir=None):
    """Build a docker image"""
    if service:
        service = service
    else:
        service = args.service
    if service and service.startswith('services-'):
        service = service.split('services-', 1)[1]
    validate_service_name(service, soa_dir)

    run_env = os.environ.copy()
    default_tag = 'paasta-cook-image-%s-%s' % (service, get_username())
    tag = run_env.get('DOCKER_TAG', default_tag)
    run_env['DOCKER_TAG'] = tag

    if not makefile_responds_to('cook-image'):
        sys.stderr.write(
            'ERROR: local-run now requires a cook-image target to be present in the Makefile. See '
            'http://paasta.readthedocs.io/en/latest/about/contract.html\n')
        return 1

    try:
        cmd = 'make cook-image'
        returncode, output = _run(cmd,
                                  env=run_env,
                                  log=True,
                                  component='build',
                                  service=service,
                                  loglevel='debug')
        if returncode != 0:
            _log(
                service=service,
                line='ERROR: make cook-image failed for %s.' % service,
                component='build',
                level='event',
            )
        return returncode

    except KeyboardInterrupt:
        sys.stderr.write('\nProcess interrupted by the user. Cancelling.\n')
        return 2
Ejemplo n.º 9
0
def paasta_local_run(args):
    if args.action == 'build' and not makefile_responds_to('cook-image'):
        sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
        sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
        return 1

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    cluster = guess_cluster(service=service, args=args)
    instance = guess_instance(service=service, cluster=cluster, args=args)
    docker_client = get_docker_client()

    if args.action == 'build':
        default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
        tag = os.environ.get('DOCKER_TAG', default_tag)
        os.environ['DOCKER_TAG'] = tag
        pull_image = False
        cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
        if cook_return != 0:
            return cook_return
    elif args.action == 'dry_run':
        pull_image = False
        tag = None
    else:
        pull_image = True
        tag = None

    try:
        configure_and_run_docker_container(
            docker_client=docker_client,
            docker_hash=tag,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
            dry_run=args.action == 'dry_run',
        )
    except errors.APIError as e:
        sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
        return 1
Ejemplo n.º 10
0
def paasta_local_run(args):
    if args.action == "pull" and os.geteuid() != 0 and not docker_config_available():
        paasta_print("Re-executing paasta local-run --pull with sudo..")
        os.execvp("sudo", ["sudo", "-H"] + sys.argv)
    if args.action == "build" and not makefile_responds_to("cook-image"):
        paasta_print(
            "A local Makefile with a 'cook-image' target is required for --build",
            file=sys.stderr,
        )
        paasta_print(
            "If you meant to pull the docker image from the registry, explicitly pass --pull",
            file=sys.stderr,
        )
        return 1

    try:
        system_paasta_config = load_system_paasta_config()
    except PaastaNotConfiguredError:
        paasta_print(
            PaastaColors.yellow(
                "Warning: Couldn't load config files from '/etc/paasta'. This indicates"
                "PaaSTA is not configured locally on this host, and local-run may not behave"
                "the same way it would behave on a server configured for PaaSTA."
            ),
            sep="\n",
        )
        system_paasta_config = SystemPaastaConfig({"volumes": []}, "/etc/paasta")

    local_run_config = system_paasta_config.get_local_run_config()

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    if args.cluster:
        cluster = args.cluster
    else:
        try:
            cluster = local_run_config["default_cluster"]
        except KeyError:
            paasta_print(
                PaastaColors.red(
                    "PaaSTA on this machine has not been configured with a default cluster."
                    "Please pass one to local-run using '-c'."
                ),
                sep="\n",
                file=sys.stderr,
            )
            return 1
    instance = args.instance
    docker_client = get_docker_client()

    docker_sha = None
    docker_url = None

    if args.action == "build":
        default_tag = "paasta-local-run-{}-{}".format(service, get_username())
        docker_url = os.environ.get("DOCKER_TAG", default_tag)
        os.environ["DOCKER_TAG"] = docker_url
        pull_image = False
        cook_return = paasta_cook_image(
            args=None, service=service, soa_dir=args.yelpsoa_config_root
        )
        if cook_return != 0:
            return cook_return
    elif args.action == "dry_run":
        pull_image = False
        docker_url = None
        docker_sha = args.sha
    else:
        pull_image = True
        docker_url = None
        docker_sha = args.sha

    try:
        return configure_and_run_docker_container(
            docker_client=docker_client,
            docker_url=docker_url,
            docker_sha=docker_sha,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
            system_paasta_config=system_paasta_config,
            dry_run=args.action == "dry_run",
        )
    except errors.APIError as e:
        paasta_print("Can't run Docker container. Error: %s" % str(e), file=sys.stderr)
        return 1
Ejemplo n.º 11
0
def test_makefile_responds_to_run(mock_run):
    mock_run.return_value = (2, 'Output')
    actual = makefile_responds_to('non-present-target')
    assert actual is False
Ejemplo n.º 12
0
def test_makefile_responds_to_good(mock_run):
    mock_run.return_value = (1, 'Output')
    actual = makefile_responds_to('present-target')
    assert actual is True
Ejemplo n.º 13
0
def test_makefile_responds_to_run(mock_run):
    mock_run.return_value = (2, 'Output')
    actual = makefile_responds_to('non-present-target')
    assert actual is False
Ejemplo n.º 14
0
def test_makefile_responds_to_good(mock_run):
    mock_run.return_value = (0, 'Output')
    actual = makefile_responds_to('present-target')
    assert actual is True
Ejemplo n.º 15
0
def paasta_cook_image(
    args: Optional[argparse.Namespace],
    service: Optional[str] = None,
    soa_dir: Optional[str] = None,
) -> int:
    """Build a docker image"""
    if not service:
        if args is None:
            print(
                "ERROR: No arguments or service passed to cook-image - unable to determine what service to cook an image for",
                file=sys.stderr,
            )
            return 1
        service = args.service
    if service and service.startswith("services-"):
        service = service.split("services-", 1)[1]
    if not soa_dir:
        if args is None:
            print(
                "ERROR: No arguments or soadir passed to cook-image - unable to determine where to look for soa-configs",
                file=sys.stderr,
            )
            return 1
        soa_dir = args.yelpsoa_config_root

    validate_service_name(service, soa_dir)

    run_env = os.environ.copy()
    if args is not None and args.commit is not None:
        # if we're given a commit, we're likely being called by Jenkins or someone
        # trying to push the cooked image to our registry - as such, we should tag
        # the cooked image as `paasta itest` would.
        tag = build_docker_tag(service, args.commit, args.image_version)
    else:
        default_tag = "paasta-cook-image-{}-{}".format(service, get_username())
        tag = run_env.get("DOCKER_TAG", default_tag)
    run_env["DOCKER_TAG"] = tag

    if not makefile_responds_to("cook-image"):
        print(
            "ERROR: local-run now requires a cook-image target to be present in the Makefile. See "
            "http://paasta.readthedocs.io/en/latest/about/contract.html.",
            file=sys.stderr,
        )
        return 1

    try:
        cmd = "make cook-image"
        returncode, output = _run(
            cmd,
            env=run_env,
            log=True,
            component="build",
            service=service,
            loglevel="debug",
        )
        if returncode != 0:
            _log(
                service=service,
                line="ERROR: make cook-image failed for %s." % service,
                component="build",
                level="event",
            )
        else:
            action_details = {"tag": tag}
            _log_audit(
                action="cook-image", action_details=action_details, service=service
            )
        return returncode

    except KeyboardInterrupt:
        print("\nProcess interrupted by the user. Cancelling.", file=sys.stderr)
        return 2
Ejemplo n.º 16
0
def paasta_local_run(args):
    if args.action == 'build' and not makefile_responds_to('cook-image'):
        paasta_print("A local Makefile with a 'cook-image' target is required for --build", file=sys.stderr)
        paasta_print("If you meant to pull the docker image from the registry, explicitly pass --pull", file=sys.stderr)
        return 1

    try:
        system_paasta_config = load_system_paasta_config()
    except PaastaNotConfiguredError:
        paasta_print(
            PaastaColors.yellow(
                "Warning: Couldn't load config files from '/etc/paasta'. This indicates"
                "PaaSTA is not configured locally on this host, and local-run may not behave"
                "the same way it would behave on a server configured for PaaSTA."
            ),
            sep='\n',
        )
        system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')

    local_run_config = system_paasta_config.get_local_run_config()

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    if args.cluster:
        cluster = args.cluster
    else:
        try:
            cluster = local_run_config['default_cluster']
        except KeyError:
            paasta_print(
                PaastaColors.red(
                    "PaaSTA on this machine has not been configured with a default cluster."
                    "Please pass one to local-run using '-c'."),
                sep='\n',
                file=sys.stderr,
            )
            return 1
    instance = args.instance
    docker_client = get_docker_client()

    if args.action == 'build':
        default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
        tag = os.environ.get('DOCKER_TAG', default_tag)
        os.environ['DOCKER_TAG'] = tag
        pull_image = False
        cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
        if cook_return != 0:
            return cook_return
    elif args.action == 'dry_run':
        pull_image = False
        tag = None
    else:
        pull_image = True
        tag = None

    try:
        return configure_and_run_docker_container(
            docker_client=docker_client,
            docker_hash=tag,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
            system_paasta_config=system_paasta_config,
            dry_run=args.action == 'dry_run',
        )
    except errors.APIError as e:
        paasta_print(
            'Can\'t run Docker container. Error: %s' % str(e),
            file=sys.stderr,
        )
        return 1
Ejemplo n.º 17
0
def paasta_local_run(args):
    if args.action == 'build' and not makefile_responds_to('cook-image'):
        sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
        sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
        return 1

    try:
        system_paasta_config = load_system_paasta_config()
    except PaastaNotConfiguredError:
        sys.stdout.write(PaastaColors.yellow(
            "Warning: Couldn't load config files from '/etc/paasta'. This indicates\n"
            "PaaSTA is not configured locally on this host, and local-run may not behave\n"
            "the same way it would behave on a server configured for PaaSTA.\n"
        ))
        system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')

    local_run_config = system_paasta_config.get_local_run_config()

    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
    if args.cluster:
        cluster = args.cluster
    else:
        try:
            cluster = local_run_config['default_cluster']
        except KeyError:
            sys.stderr.write(PaastaColors.red(
                "PaaSTA on this machine has not been configured with a default cluster.\n"
                "Please pass one to local-run using '-c'.\n"))
            return 1
    instance = args.instance
    docker_client = get_docker_client()

    if args.action == 'build':
        default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
        tag = os.environ.get('DOCKER_TAG', default_tag)
        os.environ['DOCKER_TAG'] = tag
        pull_image = False
        cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
        if cook_return != 0:
            return cook_return
    elif args.action == 'dry_run':
        pull_image = False
        tag = None
    else:
        pull_image = True
        tag = None

    try:
        configure_and_run_docker_container(
            docker_client=docker_client,
            docker_hash=tag,
            service=service,
            instance=instance,
            cluster=cluster,
            args=args,
            pull_image=pull_image,
            system_paasta_config=system_paasta_config,
            dry_run=args.action == 'dry_run',
        )
    except errors.APIError as e:
        sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
        return 1