def build_cmd(target, version, skip_tests): client = utils.get_docker_client() project_dir = utils.get_project_dir() config = load_shub_config() image = config.get_image(target) _create_setup_py_if_not_exists() image_name = utils.format_image_name(image, version) if not os.path.exists(os.path.join(project_dir, 'Dockerfile')): raise shub_exceptions.BadParameterException( "Dockerfile is not found, please use shub image 'init' command") is_built = False for data in client.build(path=project_dir, tag=image_name, decode=True): if 'stream' in data: utils.debug_log("{}".format(data['stream'][:-1])) is_built = re.search(r'Successfully built ([0-9a-f]+)', data['stream']) elif 'error' in data: click.echo("Error {}:\n{}".format(data['error'], data['errorDetail'])) if not is_built: raise shub_exceptions.RemoteErrorException( "Build image operation failed") click.echo("The image {} build is completed.".format(image_name)) # Test the image content after building it if not skip_tests: test_cmd(target, version)
def push_cmd(target, version, username, password, email, apikey, insecure, skip_tests): # Test the image content after building it if not skip_tests: test_cmd(target, version) client = utils.get_docker_client() config = load_shub_config() image = config.get_image(target) username, password = utils.get_credentials( username=username, password=password, insecure=insecure, apikey=apikey, target_apikey=config.get_apikey(target)) if username: _execute_push_login(client, image, username, password, email) image_name = utils.format_image_name(image, version) click.echo("Pushing {} to the registry.".format(image_name)) events = client.push(image_name, stream=True, decode=True, insecure_registry=not bool(username)) if utils.is_verbose(): push_progress_cls = _LoggedPushProgress else: push_progress_cls = _PushProgress push_progress = push_progress_cls(events) push_progress.show() click.echo("The image {} pushed successfully.".format(image_name))
def build_cmd(target, version, skip_tests): config = load_shub_config() create_scrapinghub_yml_wizard(config, target=target, image=True) client = utils.get_docker_client() project_dir = utils.get_project_dir() image = config.get_image(target) image_name = utils.format_image_name(image, version) if not os.path.exists(os.path.join(project_dir, 'Dockerfile')): raise shub_exceptions.NotFoundException( "Dockerfile is not found and it is required because project '{}' is configured " "to deploy Docker images. Please add a Dockerfile that will be used to build " "the image and retry this command. If you want to migrate an existing Scrapy project " "you can use `shub image init` command to create a Dockerfile.". format(target)) if utils.is_verbose(): build_progress_cls = _LoggedBuildProgress else: build_progress_cls = _BuildProgress click.echo("Building {}.".format(image_name)) events = client.build(path=project_dir, tag=image_name, decode=True) build_progress = build_progress_cls(events) build_progress.show() click.echo("The image {} build is completed.".format(image_name)) # Test the image content after building it if not skip_tests: test_cmd(target, version)
def _run_cmd_in_docker_container(image_name, command, environment): """Run a command inside the image container.""" client = utils.get_docker_client() container = client.create_container( image=image_name, command=[command], environment=environment, ) if 'Id' not in container: raise ShubException("Create container error:\n %s" % container) try: client.start(container) except docker.errors.APIError as e: explanation = utils.ensure_unicode(e.explanation or '') if 'executable file not found' in explanation: # docker.errors.APIError: 500 Server Error: # Internal Server Error ("Cannot start container xxx: # [8] System error: exec: "shub-image-info": # executable file not found in $PATH") return 127, None raise statuscode = client.wait(container=container['Id'])['StatusCode'] logs = client.logs( container=container['Id'], stream=False, timestamps=False, stdout=True, stderr=True if statuscode else False, ) return statuscode, utils.ensure_unicode(logs)
def _run_cmd_in_docker_container(image_name, command, environment): """Run a command inside the image container.""" client = utils.get_docker_client() container = client.create_container( image=image_name, command=[command], environment=environment, ) if 'Id' not in container: raise ShubException("Create container error:\n %s" % container) try: client.start(container) except docker.errors.APIError as e: explanation = utils.ensure_unicode(e.explanation or '') if 'executable file not found' in explanation: # docker.errors.APIError: 500 Server Error: # Internal Server Error ("Cannot start container xxx: # [8] System error: exec: "shub-image-info": # executable file not found in $PATH") return 127, None raise statuscode = client.wait(container=container['Id']) logs = client.logs( container=container['Id'], stream=False, timestamps=False, stdout=True, stderr=True if statuscode else False, ) return statuscode, utils.ensure_unicode(logs)
def build_cmd(target, version, skip_tests, no_cache, filename='Dockerfile'): config = load_shub_config() create_scrapinghub_yml_wizard(config, target=target, image=True) client = utils.get_docker_client() project_dir = utils.get_project_dir() image = config.get_image(target) image_name = utils.format_image_name(image, version) if not os.path.exists(os.path.join(project_dir, filename)): raise shub_exceptions.NotFoundException( "Dockerfile is not found and it is required because project '{}' is configured " "to deploy Docker images. Please add a Dockerfile that will be used to build " "the image and retry this command. If you want to migrate an existing Scrapy project " "you can use `shub image init` command to create a Dockerfile.".format(target)) if utils.is_verbose(): build_progress_cls = _LoggedBuildProgress else: build_progress_cls = _BuildProgress click.echo("Building {}.".format(image_name)) events = client.build( path=project_dir, tag=image_name, decode=True, dockerfile=filename, nocache=no_cache ) build_progress = build_progress_cls(events) build_progress.show() click.echo("The image {} build is completed.".format(image_name)) # Test the image content after building it if not skip_tests: test_cmd(target, version)
def test_cmd(target, version): config = load_shub_config() image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) docker_client = utils.get_docker_client() for check in [_check_image_size, _check_start_crawl_entry, _check_shub_image_info_entry]: check(image_name, docker_client)
def test_cmd(target, version): config = load_shub_config() image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) docker_client = utils.get_docker_client() for check in [ _check_image_size, _check_start_crawl_entry, _check_shub_image_info_entry ]: check(image_name, docker_client)
def test_get_docker_client(self): mocked_docker = mock.Mock() sys.modules['docker'] = mocked_docker client_mock = mock.Mock() class DockerClientMock(object): def __init__(self, *args, **kwargs): client_mock(*args, **kwargs) def version(self): return {} mocked_docker.APIClient = DockerClientMock assert get_docker_client() client_mock.assert_called_with( base_url=None, tls=None, version=DEFAULT_DOCKER_API_VERSION) # set basic test environment os.environ['DOCKER_HOST'] = 'http://127.0.0.1' os.environ['DOCKER_API_VERSION'] = '1.40' assert get_docker_client() client_mock.assert_called_with( base_url='http://127.0.0.1', tls=None, version='1.40') # test for tls os.environ['DOCKER_TLS_VERIFY'] = '1' os.environ['DOCKER_CERT_PATH'] = 'some-path' mocked_tls = mock.Mock() mocked_docker.tls.TLSConfig.return_value = mocked_tls assert get_docker_client() client_mock.assert_called_with( base_url='http://127.0.0.1', tls=mocked_tls, version='1.40') mocked_docker.tls.TLSConfig.assert_called_with( client_cert=(os.path.join('some-path', 'cert.pem'), os.path.join('some-path', 'key.pem')), verify=os.path.join('some-path', 'ca.pem'), assert_hostname=False)
def run_cmd(spider, args, settings, environment, version, keep_volume): try: target, spider = spider.rsplit('/', 1) except ValueError: target = 'default' config = load_shub_config() image = config.get_image(target) version = version or config.get_version() image_name = utils.format_image_name(image, version) docker_client = utils.get_docker_client() env = _format_environment(spider, args, settings, environment) _run_with_docker(docker_client, image_name, env, keep_volume)