def build_cmd(target, version, skip_tests):
client = utils.get_docker_client()
project_dir = utils.get_project_dir()
config = load_shub_config()
image = config.get_image(target)
_create_setup_py_if_not_exists()
image_name = utils.format_image_name(image, version)
if not os.path.exists(os.path.join(project_dir, 'Dockerfile')):
raise shub_exceptions.BadParameterException(
"Dockerfile is not found, please use shub image 'init' command")
is_built = False
for data in client.build(path=project_dir, tag=image_name, decode=True):
if 'stream' in data:
utils.debug_log("{}".format(data['stream'][:-1]))
is_built = re.search(r'Successfully built ([0-9a-f]+)',
data['stream'])
elif 'error' in data:
click.echo("Error {}:\n{}".format(data['error'],
data['errorDetail']))
if not is_built:
raise shub_exceptions.RemoteErrorException(
"Build image operation failed")
click.echo("The image {} build is completed.".format(image_name))
# Test the image content after building it
if not skip_tests:
test_cmd(target, version)
def push_cmd(target, version, username, password, email, apikey, insecure, skip_tests):
# Test the image content after building it
if not skip_tests:
test_cmd(target, version)
client = utils.get_docker_client()
config = load_shub_config()
image = config.get_image(target)
username, password = utils.get_credentials(
username=username, password=password, insecure=insecure,
apikey=apikey, target_apikey=config.get_apikey(target))
if username:
_execute_push_login(client, image, username, password, email)
image_name = utils.format_image_name(image, version)
click.echo("Pushing {} to the registry.".format(image_name))
events = client.push(image_name, stream=True, decode=True,
insecure_registry=not bool(username))
if utils.is_verbose():
push_progress_cls = _LoggedPushProgress
else:
push_progress_cls = _PushProgress
push_progress = push_progress_cls(events)
push_progress.show()
click.echo("The image {} pushed successfully.".format(image_name))
def build_cmd(target, version, skip_tests):
config = load_shub_config()
create_scrapinghub_yml_wizard(config, target=target, image=True)
client = utils.get_docker_client()
project_dir = utils.get_project_dir()
image = config.get_image(target)
image_name = utils.format_image_name(image, version)
if not os.path.exists(os.path.join(project_dir, 'Dockerfile')):
raise shub_exceptions.NotFoundException(
"Dockerfile is not found and it is required because project '{}' is configured "
"to deploy Docker images. Please add a Dockerfile that will be used to build "
"the image and retry this command. If you want to migrate an existing Scrapy project "
"you can use `shub image init` command to create a Dockerfile.".
format(target))
if utils.is_verbose():
build_progress_cls = _LoggedBuildProgress
else:
build_progress_cls = _BuildProgress
click.echo("Building {}.".format(image_name))
events = client.build(path=project_dir, tag=image_name, decode=True)
build_progress = build_progress_cls(events)
build_progress.show()
click.echo("The image {} build is completed.".format(image_name))
# Test the image content after building it
if not skip_tests:
test_cmd(target, version)
def _run_cmd_in_docker_container(image_name, command, environment):
"""Run a command inside the image container."""
client = utils.get_docker_client()
container = client.create_container(
image=image_name,
command=[command],
environment=environment,
)
if 'Id' not in container:
raise ShubException("Create container error:\n %s" % container)
try:
client.start(container)
except docker.errors.APIError as e:
explanation = utils.ensure_unicode(e.explanation or '')
if 'executable file not found' in explanation:
# docker.errors.APIError: 500 Server Error:
# Internal Server Error ("Cannot start container xxx:
# [8] System error: exec: "shub-image-info":
# executable file not found in $PATH")
return 127, None
raise
statuscode = client.wait(container=container['Id'])['StatusCode']
logs = client.logs(
container=container['Id'],
stream=False,
timestamps=False,
stdout=True,
stderr=True if statuscode else False,
)
return statuscode, utils.ensure_unicode(logs)
def push_cmd(target, version, username, password, email, apikey, insecure,
skip_tests):
# Test the image content after building it
if not skip_tests:
test_cmd(target, version)
client = utils.get_docker_client()
config = load_shub_config()
image = config.get_image(target)
username, password = utils.get_credentials(
username=username,
password=password,
insecure=insecure,
apikey=apikey,
target_apikey=config.get_apikey(target))
if username:
_execute_push_login(client, image, username, password, email)
image_name = utils.format_image_name(image, version)
click.echo("Pushing {} to the registry.".format(image_name))
events = client.push(image_name,
stream=True,
decode=True,
insecure_registry=not bool(username))
if utils.is_verbose():
push_progress_cls = _LoggedPushProgress
else:
push_progress_cls = _PushProgress
push_progress = push_progress_cls(events)
push_progress.show()
click.echo("The image {} pushed successfully.".format(image_name))
def _run_cmd_in_docker_container(image_name, command, environment):
"""Run a command inside the image container."""
client = utils.get_docker_client()
container = client.create_container(
image=image_name,
command=[command],
environment=environment,
)
if 'Id' not in container:
raise ShubException("Create container error:\n %s" % container)
try:
client.start(container)
except docker.errors.APIError as e:
explanation = utils.ensure_unicode(e.explanation or '')
if 'executable file not found' in explanation:
# docker.errors.APIError: 500 Server Error:
# Internal Server Error ("Cannot start container xxx:
# [8] System error: exec: "shub-image-info":
# executable file not found in $PATH")
return 127, None
raise
statuscode = client.wait(container=container['Id'])
logs = client.logs(
container=container['Id'], stream=False, timestamps=False,
stdout=True, stderr=True if statuscode else False,
)
return statuscode, utils.ensure_unicode(logs)
def build_cmd(target, version, skip_tests, no_cache, filename='Dockerfile'):
config = load_shub_config()
create_scrapinghub_yml_wizard(config, target=target, image=True)
client = utils.get_docker_client()
project_dir = utils.get_project_dir()
image = config.get_image(target)
image_name = utils.format_image_name(image, version)
if not os.path.exists(os.path.join(project_dir, filename)):
raise shub_exceptions.NotFoundException(
"Dockerfile is not found and it is required because project '{}' is configured "
"to deploy Docker images. Please add a Dockerfile that will be used to build "
"the image and retry this command. If you want to migrate an existing Scrapy project "
"you can use `shub image init` command to create a Dockerfile.".format(target))
if utils.is_verbose():
build_progress_cls = _LoggedBuildProgress
else:
build_progress_cls = _BuildProgress
click.echo("Building {}.".format(image_name))
events = client.build(
path=project_dir,
tag=image_name,
decode=True,
dockerfile=filename,
nocache=no_cache
)
build_progress = build_progress_cls(events)
build_progress.show()
click.echo("The image {} build is completed.".format(image_name))
# Test the image content after building it
if not skip_tests:
test_cmd(target, version)
def test_cmd(target, version):
config = load_shub_config()
image = config.get_image(target)
version = version or config.get_version()
image_name = utils.format_image_name(image, version)
docker_client = utils.get_docker_client()
for check in [_check_image_size,
_check_start_crawl_entry,
_check_shub_image_info_entry]:
check(image_name, docker_client)
def test_cmd(target, version):
config = load_shub_config()
image = config.get_image(target)
version = version or config.get_version()
image_name = utils.format_image_name(image, version)
docker_client = utils.get_docker_client()
for check in [
_check_image_size, _check_start_crawl_entry,
_check_shub_image_info_entry
]:
check(image_name, docker_client)
def test_get_docker_client(self):
mocked_docker = mock.Mock()
sys.modules['docker'] = mocked_docker
client_mock = mock.Mock()
class DockerClientMock(object):
def __init__(self, *args, **kwargs):
client_mock(*args, **kwargs)
def version(self):
return {}
mocked_docker.APIClient = DockerClientMock
assert get_docker_client()
client_mock.assert_called_with(
base_url=None, tls=None, version=DEFAULT_DOCKER_API_VERSION)
# set basic test environment
os.environ['DOCKER_HOST'] = 'http://127.0.0.1'
os.environ['DOCKER_API_VERSION'] = '1.40'
assert get_docker_client()
client_mock.assert_called_with(
base_url='http://127.0.0.1', tls=None, version='1.40')
# test for tls
os.environ['DOCKER_TLS_VERIFY'] = '1'
os.environ['DOCKER_CERT_PATH'] = 'some-path'
mocked_tls = mock.Mock()
mocked_docker.tls.TLSConfig.return_value = mocked_tls
assert get_docker_client()
client_mock.assert_called_with(
base_url='http://127.0.0.1',
tls=mocked_tls,
version='1.40')
mocked_docker.tls.TLSConfig.assert_called_with(
client_cert=(os.path.join('some-path', 'cert.pem'),
os.path.join('some-path', 'key.pem')),
verify=os.path.join('some-path', 'ca.pem'),
assert_hostname=False)
def run_cmd(spider, args, settings, environment, version, keep_volume):
try:
target, spider = spider.rsplit('/', 1)
except ValueError:
target = 'default'
config = load_shub_config()
image = config.get_image(target)
version = version or config.get_version()
image_name = utils.format_image_name(image, version)
docker_client = utils.get_docker_client()
env = _format_environment(spider, args, settings, environment)
_run_with_docker(docker_client, image_name, env, keep_volume)
def run_cmd(spider, args, settings, environment, version, keep_volume):
try:
target, spider = spider.rsplit('/', 1)
except ValueError:
target = 'default'
config = load_shub_config()
image = config.get_image(target)
version = version or config.get_version()
image_name = utils.format_image_name(image, version)
docker_client = utils.get_docker_client()
env = _format_environment(spider, args, settings, environment)
_run_with_docker(docker_client, image_name, env, keep_volume)
