def __init__(self, url, username=None, password=None, method=None, ignore_cert=False):
self.url = url.rstrip('/')
self.username = username
self.password = password
self.ignore_cert = ignore_cert
self.CHUNK = 16 * 1024
self.log = logging.getLogger(__name__)
self.link_reg_ex = re.compile('<(.*)>;.*rel="next"')
self.anon_access = HTTPAccess(url=self.url, ignore_cert=self.ignore_cert)
self.valid_methods = ['token', 'basic']
if not method:
self.method = 'basic'
else:
if method not in self.valid_methods:
raise ValueError("Invalid method '%s' for DockerRegistryAccess" % method)
self.method = method
if self.method == 'token':
self.token_access = DockerTokenAccess(url=self.url, username=self.username, password=self.password,
ignore_cert=self.ignore_cert)
self.access = self.token_access
elif self.method == 'basic':
self.basic_access = HTTPAccess(url=self.url, username=username, password=password,
ignore_cert=self.ignore_cert)
self.access = self.basic_access
class QuayAccess:
def __init__(self, namespace, token):
self.log = logging.getLogger(__name__)
self.namespace = namespace
self.token = token
self.headers = {'Authorization': 'Bearer %s' % self.token}
self.access = HTTPAccess(url='https://quay.io')
'''
Gets the catalog of repositories using Quay specific API
@return None if there w as an error, else the a list of available repositories
'''
def get_catalog(self):
repos = None
path = "api/v1/repository?public=true&namespace=%s" % self.namespace
resp, stat = self.access.do_unprocessed_request(method='GET',
path=path,
headers=self.headers)
if stat == 200:
processed_response = self.access.process_response(resp)
if 'repositories' in processed_response:
repos = [
"%s/%s" % (entry['namespace'], entry['name'])
for entry in processed_response['repositories']
]
else:
self.log.error(
"Error getting catalog from Quay, received code %d." %
self.stat)
return repos
def get_tags(self, image):
pass
def __get_token(self, auth_header):
url = self.__get_token_url(auth_header)
if url:
scheme, netloc, path, query, frag = urlparse.urlsplit(url)
# If the user provided credentials, use them to get the token
access = HTTPAccess(url=scheme + "://" + netloc,
username=self.username,
password=self.password,
ignore_cert=self.ignore_cert)
token_response = access.dorequest('GET', path + '?' + query)
if token_response and token_response['token']:
return token_response['token']
return None
def __init__(self, url, username=None, password=None, ignore_cert=False):
self.url = url.rstrip('/')
self.username = username
self.password = password
self.ignore_cert = ignore_cert
self.CHUNK = 16 * 1024
self.log = logging.getLogger(__name__)
self.link_reg_ex = re.compile('<(.*)>;.*rel="next"')
self.access = HTTPAccess(url=self.url, ignore_cert=self.ignore_cert)
self.token_access = DockerTokenAccess(url=self.url,
username=self.username,
password=self.password,
ignore_cert=self.ignore_cert)
class DockerRegistryAccess:
def __init__(self, url, username=None, password=None, ignore_cert=False):
self.url = url.rstrip('/')
self.username = username
self.password = password
self.ignore_cert = ignore_cert
self.CHUNK = 16 * 1024
self.log = logging.getLogger(__name__)
self.link_reg_ex = re.compile('<(.*)>;.*rel="next"')
self.access = HTTPAccess(url=self.url, ignore_cert=self.ignore_cert)
self.token_access = DockerTokenAccess(url=self.url,
username=self.username,
password=self.password,
ignore_cert=self.ignore_cert)
'''
Verifies that the repository is a valid V2 repository
'''
def verify_is_v2(self):
self.log.info("Verify Registry is V2")
try:
# Perform request without credentials
response, status = self.access.do_unprocessed_request(
'GET', '/v2/')
return response and response.headers[
'Docker-Distribution-API-Version']
except Exception as ex:
self.log.info(ex.message)
return False
'''
Returns a full listing of all the catalog.
If the upstream repository uses pagination, aggregates the results
@param path - (optional) The endpoint for the catalog (used for paging)
'''
def get_catalog(self, path='/v2/_catalog'):
# Some registries allow access to the catalog anonymously but if the user provided credentials, we need to try
# it with the credentials or we may be missing certain images only that user can use.
if self.username and self.password and not self.token_access.has_token(
):
self.token_access.populate_generic_token()
out = self.token_access.get_call_wrapper(path)
if not out:
return False
output, response = out
code = response.getcode()
if code != 200:
self.log.error(
"Unable to get a listing of the upstream images. Catalog call returned: %d."
% code)
return False
if response and 'link' in response.headers:
link_value = response.headers['link']
results = self.link_reg_ex.findall(link_value)
if results and len(results) == 1:
return output['repositories'] + self.get_catalog(
self.access.get_relative_url(results[0]))
return output['repositories']
'''
Returns a full listing of all the tags for a particular image.
If the upstream repository uses pagination, aggregates the results
@param image - The image to get the tags for
@param path - (optional) The endpoint for the tags (used for paging)
'''
def get_tags(self, image, path=None):
if not path:
out = self.token_access.get_call_wrapper("/v2/" + image +
"/tags/list")
else:
out = self.token_access.get_call_wrapper(path)
if not out:
return False
output, response = out
code = response.getcode()
if code != 200:
self.log.error(
"Unable to get a listing of the tags for image '%s'. Tags call returned: %d."
% (image, code))
return False
if response and 'link' in response.headers:
link_value = response.headers['link']
results = self.link_reg_ex.findall(link_value)
if results and len(results) == 1:
return output['tags'] + self.get_tags(
image, self.access.get_relative_url(results[0]))
return output['tags']
'''
Downloads a manifest for the specified image/tag or image/digest
@param image - The image name
@param reference - The reference, which can be either a tag name or a digest
@param file - The file to store the contents into
'''
def download_manifest(self, image, reference, file):
# For now only accept v2.2 and not fat manifest
headers = {
'Accept':
'application/vnd.docker.distribution.manifest.v2+json, '
'application/vnd.docker.distribution.manifest.v1+json, application/json'
}
response = self.token_access.get_raw_call_wrapper(
url="/v2/" + image + "/manifests/" + reference, headers=headers)
if response.getcode() == 200:
try:
with open(file, 'wb') as f:
contents = response.read()
f.write(contents)
return True
except Exception as ex:
self.log.error("Failed to download manifest for image " +
image)
return False
return False
'''
Downloads the specified layer from the specified image and stores it in the specified path
@param image - The image name
@param layer - The layer (in the format 'sha256:03....')
@param file - The file to store the contents into
'''
def download_layer(self, image, layer, file):
response = self.token_access.get_raw_call_wrapper(url="/v2/" + image +
"/blobs/" + layer)
# Write the contents into a file and verify the sha256 while we are at it
if response.getcode() == 200:
try:
hash_256 = hashlib.sha256()
hash_1 = hashlib.sha1()
with open(file, 'wb') as f:
while True:
chunk = response.read(self.CHUNK)
if not chunk:
break
hash_256.update(chunk)
hash_1.update(chunk)
f.write(chunk)
expected_sha = layer.replace('sha256:', '')
found_sha = hash_256.hexdigest()
if found_sha == expected_sha:
return hash_1.hexdigest()
else:
self.log.error(
"Layer did not match expected sha. Expected " +
expected_sha + " but got " + found_sha)
except Exception as ex:
self.log.error(ex.message)
self.log.error("Failed to download layer %s for image %s" %
(layer, image))
return False
# Extracts the layers of the manifest file as well as the type
# Returns (type, layers)
def interpret_manifest(self, manif):
type = None
layers = []
try:
with open(manif, 'r') as m:
js = json.load(m)
# V2-1 (https://docs.docker.com/registry/spec/manifest-v2-1)
if js['schemaVersion'] == 1:
# According to official spec: '"application/json" will also be accepted for schema1'
type = "application/json"
for layer in js['fsLayers']:
layers.append(layer['blobSum'])
else: # V2-2 (https://docs.docker.com/registry/spec/manifest-v2-2)
if 'mediaType' in js:
type = js['mediaType']
else:
type = 'application/vnd.docker.distribution.manifest.v2+json'
layers.append(js['config']['digest'])
for layer in js['layers']:
# Don't try to grab foreign layers
if 'mediaType' not in layer \
or layer['mediaType'] != 'application/vnd.docker.image.rootfs.foreign.diff.tar.gzip':
layers.append(layer['digest'])
except:
self.log.exception("Error reading Docker manifest %s:", manif)
return type, layers
'''
Create a deep copy of this object
'''
def __deepcopy__(self, memo):
return DockerRegistryAccess(self.url, self.username, self.password,
self.ignore_cert)
def __init__(self, url, token, ignore_cert=False, exlog=False):
self.log = logging.getLogger(__name__)
self.url = url
self.token = token
self.headers = {'Authorization': 'Bearer %s' % self.token}
self.access = HTTPAccess(url=url, ignore_cert=ignore_cert, exlog=exlog)
class QuayEEAccess:
def __init__(self, url, token, ignore_cert=False, exlog=False):
self.log = logging.getLogger(__name__)
self.url = url
self.token = token
self.headers = {'Authorization': 'Bearer %s' % self.token}
self.access = HTTPAccess(url=url, ignore_cert=ignore_cert, exlog=exlog)
'''
Returns true if and only if the HEAD api/v1/discovery returns a 200
'''
def is_quay_ee(self):
path = "api/v1/superuser/users/"
try:
resp, stat = self.access.do_unprocessed_request(
method='GET', path=path, headers=self.headers)
return stat == 200
except:
return False
'''
Returns a list of all the repositories this user has access to
'''
def get_repositories(self, page=1):
path = "/api/v1/find/repositories?page=%d" % page
results = []
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'results' in resp:
results.extend(resp['results'])
if resp['has_additional']:
results.extend(self.get_repositories(page + 1))
return results
'''
Returns a list of all the users
@param disabled - If set to true, returns disabled users as well(Default to False)
'''
def get_users(self, disabled=False):
path = "api/v1/superuser/users/?disabled=%s" % disabled
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'users' in resp:
return resp['users']
return []
'''
Returns a list of organizations the current user has admin access to
'''
def get_organizations(self):
path = "api/v1/user/"
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'organizations' in resp:
return resp['organizations']
return []
'''
Returns a list of teams for the given organization
@param organization - The name of the organization
'''
def get_teams_in_org(self, organization):
path = "api/v1/organization/%s" % urllib.quote_plus(organization)
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'teams' in resp:
return resp['teams']
return []
'''
Returns a list of users in a given organization/team
@param organization - The name of the organization
@param team - The name of the team
'''
def get_users_in_team(self, organization, team):
path = "/api/v1/organization/%s/team/%s/members" % (
urllib.quote_plus(organization), urllib.quote_plus(team))
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'members' in resp:
return resp['members']
return []
'''
Returns a list of robots in a given organization
@param organization - The name of the organization
'''
def get_robots_in_org(self, organization):
path = "/api/v1/organization/%s/robots" % urllib.quote_plus(
organization)
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'robots' in resp:
return resp['robots']
return {}
'''
Returns a list of collaborators (users without a team) in a given organization
@param organization - The name of the organization
'''
def get_collaborators_in_org(self, organization):
path = "/api/v1/organization/%s/collaborators" % urllib.quote_plus(
organization)
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'collaborators' in resp:
return resp['collaborators']
return []
'''
Get a list of the permissions for all users on a particular repository
@param repository - The repository name
'''
def get_user_permissions_for_repo(self, repository):
path = "/api/v1/repository/%s/permissions/user/" % urllib.quote_plus(
repository)
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'permissions' in resp:
return resp['permissions']
return {}
'''
Get a list of the permissions for all teams on a particular repository
@param repository - The repository name
'''
def get_team_permissions_for_repo(self, repository):
path = "/api/v1/repository/%s/permissions/team/" % urllib.quote_plus(
repository)
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'permissions' in resp:
return resp['permissions']
return {}
'''
Get a list of the permissions for a particular team for all repositories in an org
@param organization - The organization name
@param team - The team name
'''
def get_team_permissions_for_org(self, organization, team):
path = "/api/v1/organization/%s/team/%s/permissions" % (
urllib.quote_plus(organization), urllib.quote_plus(team))
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'permissions' in resp:
return resp['permissions']
return {}
'''
Get a list of the permissions for a robot user on a particular organization
@param organization - The organization name
@param robot - The robot name
'''
def get_robot_permissions_for_organization(self, organization, robot):
path = "/api/v1/organization/}%s/robots/%s/permissions" % (
urllib.quote_plus(organization), urllib.quote_plus(robot))
resp = self.access.dorequest(method='GET',
path=path,
headers=self.headers)
if resp and 'permissions' in resp:
return resp['permissions']
return {}
def __init__(self, namespace, token):
self.log = logging.getLogger(__name__)
self.namespace = namespace
self.token = token
self.headers = {'Authorization': 'Bearer %s' % self.token}
self.access = HTTPAccess(url='https://quay.io')