def scan_registry(registry_url):
"""Scan the docker registry and import the layers into Neo4J."""
client = DockerRegistryClient(registry_url)
try:
repositories = client.repositories()
except requests.HTTPError as e:
if e.response.status_code == requests.codes.not_found:
print("Catalog/Search not supported")
else:
raise
else:
print("Repositories:")
for repository in repositories:
repo = client.repository(repository)
for tag in repo.tags():
print("%s/%s:%s" % (registry_url, repository, tag))
assert client.api_version in [1, 2]
if client.api_version == 2:
manifest, digest = repo.manifest(tag)
layers = list(map(get_hash, manifest["fsLayers"]))
else:
image = repo.image(tag)
image_json = image.get_json()
layers = list(map(get_hash, image_json["fsLayers"]))
layer_fingerprint = "".join(layers)
with neo4j.session() as session:
session.run(
"MERGE ( i:Image {url: '%s', repo: '%s', tag: '%s'}) SET i.fingerprint='%s' "
% (registry_url, repository, tag, layer_fingerprint))
def _check_image(self):
if self._new_charts_image is None:
return False
address = self._new_charts_image.split("/")
if len(address) != 3:
logger.error("The format of the image %s can not be converted" %
self._new_charts_image)
return False
try:
ip, namespace, repository, tag = address[0], address[1], address[
2].split(":")[0], address[2].split(":")[1]
except Exception as e:
logger.error("The format of the image %s can not be converted" %
self._new_charts_image)
return False
try:
client = DockerRegistryClient("https://" + ip, verify_ssl=False)
charts_repo = client.repository(repository, namespace)
charts_tags = charts_repo.tags()
if tag in charts_tags:
logger.info("Find image: " + self._new_charts_image)
return True
else:
logger.error("Could not find image: " + self._new_charts_image)
return False
except Exception as e:
logger.error("Image query failed: {}".format(
self._new_charts_image))
return False
def get_tags(repository):
if '/' in sys.argv[1]:
registry, repository = repository.split('/', 1)
else:
registry, repository = \
'docker.io', 'library/{}'.format(repository)
if 'docker.io' in registry:
# XXX: more magic hackery
from six.moves.urllib.parse import urlsplit
REGISTRY_SERVICE_URL = "https://registry.docker.io"
HOST_SERVICE_URL = "https://registry-1.docker.io"
AUTH_SERVICE_URL = "https://auth.docker.io"
client = DockerRegistryClient(HOST_SERVICE_URL,
api_version=2,
auth_service_url=AUTH_SERVICE_URL)
client._base_client.auth.registry = urlsplit(
REGISTRY_SERVICE_URL).netloc
else:
client = DockerRegistryClient("https://{}".format(registry),
api_version=2)
repo = client.repository(repository)
tags = repo.tags()
# sort
tags = ['.'.join(tag) for tag in sorted([tag.split('.') for tag in tags])]
return tags
def get_docker_image_labels(image):
"""
Get labels from docker image
:param image: docker image
:type image: str
:return: dict[str, Any] -- image labels
"""
LOGGER.info('Getting labels for {} image'.format(image))
image_attributes = parse_docker_image_url(image)
# Get nexus registry host from ENV or image url
try:
if image_attributes.host == os.getenv('MODEL_IMAGES_REGISTRY_HOST'):
registry_host = os.getenv(legion.config.NEXUS_DOCKER_REGISTRY[0])
else:
if urllib3.util.parse_url(image_attributes.host).port == 443:
registry_host = 'https://{}'.format(image_attributes.host)
else:
registry_host = 'http://{}'.format(image_attributes.host)
except Exception as err:
LOGGER.error(
'Can\'t get registry host neither from ENV nor from image URL: {}'.
format(err))
raise err
try:
registry_client = DockerRegistryClient(
host=registry_host,
username=os.getenv(*legion.config.DOCKER_REGISTRY_USER),
password=os.getenv(*legion.config.DOCKER_REGISTRY_PASSWORD),
api_version=2)
manifest = registry_client.repository(image_attributes.repo).manifest(
image_attributes.ref)
labels = json.loads(manifest[0]["history"][0]
["v1Compatibility"])["container_config"]["Labels"]
except Exception as err:
raise Exception('Can\'t get image labels for {} image: {}'.format(
image, err))
required_headers = [
legion.containers.headers.DOMAIN_MODEL_ID,
legion.containers.headers.DOMAIN_MODEL_VERSION,
legion.containers.headers.DOMAIN_CONTAINER_TYPE
]
if any(header not in labels for header in required_headers):
raise Exception(
'Missed one of %s labels. Available labels: %s' %
(', '.join(required_headers), ', '.join(tuple(labels.keys()))))
return labels
def get_tags(self, image):
params = {}
for (k, v) in self.config.items():
if v == '' or v is None:
continue
# FOR AWS ECR User
if k == 'username' and v == 'AWS':
# get password
region_name = self._get_aws_region_from_host(self.config['host'])
password = self._get_aws_ecr_token(region_name)
if password:
params.update({'password': password})
params.update({k: v})
try:
client = DockerRegistryClient(**params)
r = client.repository(image)
tags = r.tags()
# return sorted list
return tags
except Exception as e:
# Hard to determine backend problems
_LOGGER.error(f"Error to get container tags: {e}")
raise ERROR_REPOSITORY_BACKEND(host=params['host'])
from docker_registry_client import DockerRegistryClient
import docker
client = DockerRegistryClient("http://127.0.0.1:5000")
print client.repositories()
r = client.repository("busybox")
tags = r.tags()
print tags
docker_client = docker.from_env()
reg = docker_client.containers.get("registry")
print reg
docker_client.login(registry="http://127.0.0.1:5000", username="", password="")
docker_client.images.pull("127.0.0.1:5000/busybox")
def test_repository_namespace_incorrect(self, version):
url = mock_registry(version)
client = DockerRegistryClient(url)
with pytest.raises(RuntimeError):
client.repository('{0}/{1}'.format(TEST_NAMESPACE, TEST_REPO),
namespace=TEST_NAMESPACE)
def test_repository(self, version, repository, namespace):
url = mock_registry(version)
client = DockerRegistryClient(url)
repository = client.repository(repository, namespace=namespace)
assert isinstance(repository, BaseRepository)
def test_repository_namespace_incorrect(self, version):
url = mock_registry(version)
client = DockerRegistryClient(url)
with pytest.raises(RuntimeError):
client.repository('{0}/{1}'.format(TEST_NAMESPACE, TEST_REPO),
namespace=TEST_NAMESPACE)
def test_repository(self, version, repository, namespace):
url = mock_registry(version)
client = DockerRegistryClient(url)
repository = client.repository(repository, namespace=namespace)
assert isinstance(repository, BaseRepository)