def login(self, registry, docker_secret_path):
"""
login to docker registry
:param registry: registry name
:param docker_secret_path: path to docker config directory
"""
logger.info("logging in: registry '%s', secret path '%s'", registry, docker_secret_path)
# Docker-py needs username
dockercfg = Dockercfg(docker_secret_path)
credentials = dockercfg.get_credentials(registry)
unpacked_auth = dockercfg.unpack_auth_b64(registry)
username = credentials.get('username')
if unpacked_auth:
username = unpacked_auth.username
if not username:
raise RuntimeError("Failed to extract a username from '%s'" % dockercfg)
logger.info("found username %s for registry %s", username, registry)
response = self.d.login(registry=registry, username=username,
dockercfg_path=dockercfg.json_secret_path)
if not response:
raise RuntimeError("Failed to login to '%s' with config '%s'" % (registry, dockercfg))
if u'Status' in response and response[u'Status'] == u'Login Succeeded':
logger.info("login succeeded")
else:
if not(isinstance(response, dict) and 'password' in response.keys()):
# for some reason docker-py returns the contents of the dockercfg - we shouldn't
# be displaying that
logger.debug("response: %r", response)
def login(self, registry, docker_secret_path):
"""
login to docker registry
:param registry: registry name
:param docker_secret_path: path to docker config directory
"""
logger.info("logging in: registry '%s', secret path '%s'", registry,
docker_secret_path)
# Docker-py needs username
dockercfg = Dockercfg(docker_secret_path)
username = dockercfg.get_credentials(registry)['username']
logger.info("found username %s for registry %s", username, registry)
response = self.d.login(registry=registry,
username=username,
dockercfg_path=dockercfg.json_secret_path)
if not response:
raise RuntimeError("Failed to login to '%s' with config '%s'" %
(registry, dockercfg))
if u'Status' in response and response[u'Status'] == u'Login Succeeded':
logger.info("login succeeded")
else:
if not (isinstance(response, dict)
and 'password' in response.keys()):
# for some reason docker-py returns the contents of the dockercfg - we shouldn't
# be displaying that
logger.debug("response: %r", response)
def test_dockercfg(tmpdir, in_config, lookup, expected):
temp_dir = mkdtemp(dir=str(tmpdir))
with open(os.path.join(temp_dir, '.dockercfg'), 'w+') as dockerconfig:
dockerconfig.write(json.dumps({
in_config: {
'username': '******', 'password': '******'
}
}))
creds = Dockercfg(temp_dir).get_credentials(lookup)
found = creds.get('username') == 'john.doe' and creds.get('password') == 'letmein'
assert found == expected
def test_dockercfg(tmpdir, in_config, lookup, expected):
temp_dir = mkdtemp(dir=str(tmpdir))
with open(os.path.join(temp_dir, '.dockercfg'), 'w+') as dockerconfig:
dockerconfig.write(json.dumps({
in_config: {
'username': '******', 'password': '******'
}
}))
creds = Dockercfg(temp_dir).get_credentials(lookup)
found = creds.get('username') == 'john.doe' and creds.get('password') == 'letmein'
assert found == expected
def push_with_skopeo(self, image: Dict[str, str], registry_image: ImageName, insecure: bool,
docker_push_secret: str) -> None:
cmd = ['skopeo', 'copy']
if docker_push_secret is not None:
dockercfg = Dockercfg(docker_push_secret)
cmd.append('--authfile=' + dockercfg.json_secret_path)
if insecure:
cmd.append('--dest-tls-verify=false')
if image['type'] == IMAGE_TYPE_OCI:
# ref_name is added by 'flatpak_create_oci'
# we have to be careful when changing the source container image type
# since assumption here is that source container image will always be 'docker-archive'
source_img = 'oci:{path}:{ref_name}'.format(**image)
cmd.append('--format=v2s2')
elif image['type'] == IMAGE_TYPE_DOCKER_ARCHIVE:
source_img = 'docker-archive://{path}'.format(**image)
else:
raise RuntimeError("Attempt to push unsupported image type %s with skopeo" %
image['type'])
dest_img = 'docker://' + registry_image.to_str()
cmd += [source_img, dest_img]
try:
retries.run_cmd(cmd)
except subprocess.CalledProcessError as e:
self.log.error("push failed with output:\n%s", e.output)
raise
def get_dockercfg_credentials(self, docker_registry):
"""
Read the .dockercfg file and return an empty dict, or else a dict
with keys 'basic_auth_username' and 'basic_auth_password'.
"""
if not self.registry_secret_path:
return {}
dockercfg = Dockercfg(self.registry_secret_path)
registry_creds = dockercfg.get_credentials(docker_registry)
if 'username' not in registry_creds:
return {}
return {
'basic_auth_username': registry_creds['username'],
'basic_auth_password': registry_creds['password'],
}
def get_dockercfg_credentials(self, docker_registry):
"""
Read the .dockercfg file and return an empty dict, or else a dict
with keys 'basic_auth_username' and 'basic_auth_password'.
"""
if not self.registry_secret_path:
return {}
dockercfg = Dockercfg(self.registry_secret_path)
registry_creds = dockercfg.get_credentials(docker_registry)
if 'username' not in registry_creds:
return {}
return {
'basic_auth_username': registry_creds['username'],
'basic_auth_password': registry_creds['password'],
}
def push_with_skopeo(self, registry_image, insecure, docker_push_secret):
# If the last image has type OCI_TAR, then hunt back and find the
# the untarred version, since skopeo only supports OCI's as an
# untarred directory
image = [
x for x in self.workflow.exported_image_sequence
if x['type'] != IMAGE_TYPE_OCI_TAR
][-1]
cmd = ['skopeo', 'copy']
if docker_push_secret is not None:
dockercfg = Dockercfg(docker_push_secret)
credentials = dockercfg.get_credentials(registry_image.registry)
username = credentials['username']
password = credentials['password']
cmd.append('--dest-creds=' + username + ':' + password)
if insecure:
cmd.append('--dest-tls-verify=false')
if image['type'] == IMAGE_TYPE_OCI:
source_img = 'oci:{path}:{ref_name}'.format(**image)
elif image['type'] == IMAGE_TYPE_DOCKER_ARCHIVE:
source_img = 'docker-archive://{path}'.format(**image)
else:
raise RuntimeError(
"Attempt to push unsupported image type %s with skopeo",
image['type'])
dest_img = 'docker://' + registry_image.to_str()
# Make sure we don't log the credentials
cmd += [source_img, dest_img]
log_cmd = [
re.sub(r'^--dest-creds=.*', '--dest-creds=<HIDDEN>', arg)
for arg in cmd
]
self.log.info("Calling: %s", ' '.join(log_cmd))
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
self.log.error("push failed with output:\n%s", e.output)
e.cmd = log_cmd # hide credentials
raise
def push_with_skopeo(self, registry_image, insecure, docker_push_secret):
# If the last image has type OCI_TAR, then hunt back and find the
# the untarred version, since skopeo only supports OCI's as an
# untarred directory
image = [x for x in self.workflow.exported_image_sequence if
x['type'] != IMAGE_TYPE_OCI_TAR][-1]
cmd = ['skopeo', 'copy']
if docker_push_secret is not None:
dockercfg = Dockercfg(docker_push_secret)
credentials = dockercfg.get_credentials(registry_image.registry)
username = credentials['username']
password = credentials['password']
cmd.append('--dest-creds=' + username + ':' + password)
if insecure:
cmd.append('--dest-tls-verify=false')
if image['type'] == IMAGE_TYPE_OCI:
source_img = 'oci:{path}:{ref_name}'.format(**image)
elif image['type'] == IMAGE_TYPE_DOCKER_ARCHIVE:
source_img = 'docker-archive://{path}'.format(**image)
else:
raise RuntimeError("Attempt to push unsupported image type %s with skopeo",
image['type'])
dest_img = 'docker://' + registry_image.to_str()
# Make sure we don't log the credentials
cmd += [source_img, dest_img]
log_cmd = [re.sub(r'^--dest-creds=.*', '--dest-creds=<HIDDEN>', arg)
for arg in cmd]
self.log.info("Calling: %s", ' '.join(log_cmd))
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
self.log.error("push failed with output:\n%s", e.output)
e.cmd = log_cmd # hide credentials
raise
def run(self):
for registry, registry_conf in self.registries.items():
if not registry.startswith('http://') and not registry.startswith('https://'):
registry = 'https://' + registry
registry_noschema = urlparse(registry).netloc
auth = None
secret_path = registry_conf.get('secret')
if secret_path:
self.log.debug("registry %s secret %s", registry_noschema, secret_path)
dockercfg = Dockercfg(secret_path).get_credentials(registry_noschema)
try:
username = dockercfg['username']
password = dockercfg['password']
except KeyError:
self.log.error("credentials for registry %s not found in %s",
registry_noschema, secret_path)
else:
self.log.debug("found user %s for registry %s", username, registry_noschema)
auth = requests.auth.HTTPBasicAuth(username, password)
for push_conf_registry in self.workflow.push_conf.docker_registries:
if push_conf_registry.uri == registry_noschema:
break
else:
self.log.warning("requested deleting image from %s but we haven't pushed there",
registry_noschema)
continue
for tag, digest in push_conf_registry.digests.items():
repo = tag.split(':')[0]
url = registry + "/v2/" + repo + "/manifests/" + digest
insecure = push_conf_registry.insecure
response = requests.delete(url, verify=not insecure, auth=auth)
if response.status_code == requests.codes.ACCEPTED:
self.log.info("deleted manifest %s/%s@%s", registry_noschema, repo, digest)
elif response.status_code == requests.codes.NOT_FOUND:
self.log.warning("cannot delete %s/%s@%s: not found",
registry_noschema, repo, digest)
elif response.status_code == requests.codes.METHOD_NOT_ALLOWED:
self.log.warning("cannot delete %s/%s@%s: image deletion disabled on registry",
registry_noschema, repo, digest)
else:
msg = "failed to delete %s/%s@%s: %s" % (registry_noschema, repo, digest,
response.reason)
self.log.error("%s\n%s", msg, response.text)
raise PluginFailedException(msg)
def setup_secret(self, registry, secret_path):
auth = None
if secret_path:
self.log.debug("registry %s secret %s", registry, secret_path)
dockercfg = Dockercfg(secret_path).get_credentials(registry)
try:
username = dockercfg['username']
password = dockercfg['password']
except KeyError:
self.log.error("credentials for registry %s not found in %s",
registry, secret_path)
else:
self.log.debug("found user %s for registry %s", username, registry)
auth = requests.auth.HTTPBasicAuth(username, password)
return auth
def push_with_skopeo(self,
registry_image,
insecure,
docker_push_secret,
source_oci_image_path=None):
cmd = ['skopeo', 'copy']
if docker_push_secret is not None:
dockercfg = Dockercfg(docker_push_secret)
cmd.append('--authfile=' + dockercfg.json_secret_path)
if insecure:
cmd.append('--dest-tls-verify=false')
if not source_oci_image_path:
# If the last image has type OCI_TAR, then hunt back and find the
# the untarred version, since skopeo only supports OCI's as an
# untarred directory
image = [
x for x in self.workflow.exported_image_sequence
if x['type'] != IMAGE_TYPE_OCI_TAR
][-1]
if image['type'] == IMAGE_TYPE_OCI:
source_img = 'oci:{path}:{ref_name}'.format(**image)
elif image['type'] == IMAGE_TYPE_DOCKER_ARCHIVE:
source_img = 'docker-archive://{path}'.format(**image)
else:
raise RuntimeError(
"Attempt to push unsupported image type %s with skopeo" %
image['type'])
else:
source_img = 'oci:{}'.format(source_oci_image_path)
dest_img = 'docker://' + registry_image.to_str()
cmd += [source_img, dest_img]
self.log.info("Calling: %s", ' '.join(cmd))
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
self.log.error("push failed with output:\n%s", e.output)
raise
def get_worker_manifest(self, worker_data):
worker_digests = worker_data['digests']
msg = "worker_registries {0}".format(self.worker_registries)
self.log.debug(msg)
for registry, registry_conf in self.registries.items():
if registry_conf.get('version') == 'v1':
continue
if not registry.startswith('http://') and not registry.startswith(
'https://'):
registry = 'https://' + registry
registry_noschema = urlparse(registry).netloc
self.log.debug("evaluating registry %s", registry_noschema)
insecure = registry_conf.get('insecure', False)
auth = None
secret_path = registry_conf.get('secret')
if secret_path:
self.log.debug("registry %s secret %s", registry_noschema,
secret_path)
dockercfg = Dockercfg(secret_path).get_credentials(
registry_noschema)
try:
username = dockercfg['username']
password = dockercfg['password']
except KeyError:
self.log.error(
"credentials for registry %s not found in %s",
registry_noschema, secret_path)
else:
self.log.debug("found user %s for registry %s", username,
registry_noschema)
auth = requests.auth.HTTPBasicAuth(username, password)
if registry_noschema in self.worker_registries:
self.log.debug("getting manifests from %s", registry_noschema)
digest = worker_digests[0]['digest']
repo = worker_digests[0]['repository']
# get a v2 schemav2 response for now
v2schema2 = 'application/vnd.docker.distribution.manifest.v2+json'
headers = {'accept': v2schema2}
kwargs = {
'verify': not insecure,
'headers': headers,
'auth': auth
}
url = '{0}/v2/{1}/manifests/{2}'.format(registry, repo, digest)
self.log.debug("attempting get from %s", url)
response = requests.get(url, **kwargs)
if response.json()['schemaVersion'] == '1':
msg = 'invalid schema from {0}'.format(url)
raise PluginFailedException(msg)
image_manifest = response.content
headers = {'Content-Type': v2schema2}
kwargs = {
'verify': not insecure,
'headers': headers,
'auth': auth
}
push_conf_registry = self.workflow.push_conf.add_docker_registry(
registry, insecure=insecure)
for image in self.workflow.tag_conf.images:
url = '{0}/v2/{1}/manifests/{2}'.format(
registry, repo, image.tag)
self.log.debug("for image_tag %s, putting at %s",
image.tag, url)
response = requests.put(url, data=image_manifest, **kwargs)
if not response.ok:
msg = "PUT failed: {0},\n manifest was: {1}".format(
response.json(), image_manifest)
self.log.error(msg)
response.raise_for_status()
# add a tag for any plugins running later that expect it
push_conf_registry.digests[image.tag] = digest
break
