def get_token(namespace, repo_name, registry=None, auth=None): '''get_token uses HTTP basic authentication to get a token for Docker registry API V2 operations :param namespace: the namespace for the image :param repo_name: the name of the repo, eg "ubuntu" :param registry: the docker registry to use :param auth: authorization header (default None) :: note # https://docs.docker.com/registry/spec/auth/token/ ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url # Check if we need a token at all by probing the tags/list endpoint. This # is an arbitrary choice, ideally we should always attempt without a token # and then retry with a token if we received a 401. base = "%s/%s/%s/%s/tags/list" % (registry, api_version, namespace, repo_name) response = api_get(base, default_header=False) if not isinstance(response, HTTPError): # No token required for registry. return None if response.code != 401 or not response.headers.has_key( "WWW-Authenticate"): logger.error("Authentication error for registry %s, exiting.", registry) sys.exit(1) challenge = response.headers["WWW-Authenticate"] match = re.match( '^Bearer\s+realm="([^"]+)",service="([^"]+)",scope="([^"]+)"\s*$', challenge) if not match: logger.error( "Unrecognized authentication challenge from registry %s, exiting.", registry) sys.exit(1) realm = match.group(1) service = match.group(2) scope = match.group(3) base = "%s?service=%s&scope=%s" % (realm, service, scope) headers = dict() if auth is not None: headers.update(auth) response = api_get(base, default_header=False, headers=headers) try: token = json.loads(response)["token"] token = {"Authorization": "Bearer %s" % (token)} return token except: logger.error("Error getting token for repository %s/%s, exiting.", namespace, repo_name) sys.exit(1)
def get_token(namespace,repo_name,registry=None,auth=None): '''get_token uses HTTP basic authentication to get a token for Docker registry API V2 operations :param namespace: the namespace for the image :param repo_name: the name of the repo, eg "ubuntu" :param registry: the docker registry to use :param auth: authorization header (default None) :: note # https://docs.docker.com/registry/spec/auth/token/ ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url # Check if we need a token at all by probing the tags/list endpoint. This # is an arbitrary choice, ideally we should always attempt without a token # and then retry with a token if we received a 401. base = "%s/%s/%s/%s/tags/list" %(registry,api_version,namespace,repo_name) response = api_get(base, default_header=False) if not isinstance(response, HTTPError): # No token required for registry. return None if response.code != 401 or "WWW-Authenticate" not in response.headers: logger.error("Authentication error for registry %s, exiting.", registry) sys.exit(1) challenge = response.headers["WWW-Authenticate"] match = re.match('^Bearer\s+realm="([^"]+)",service="([^"]+)",scope="([^"]+)"\s*$', challenge) if not match: logger.error("Unrecognized authentication challenge from registry %s, exiting.", registry) sys.exit(1) realm = match.group(1) service = match.group(2) scope = match.group(3) base = "%s?service=%s&scope=%s" % (realm, service, scope) headers = dict() if auth is not None: headers.update(auth) response = api_get(base,default_header=False,headers=headers) try: token = json.loads(response)["token"] token = {"Authorization": "Bearer %s" %(token) } return token except: logger.error("Error getting token for repository %s/%s, exiting.", namespace,repo_name) sys.exit(1)
def get_layer(image_id,namespace,repo_name,download_folder=None,registry=None,auth=None): '''get_layer will download an image layer (.tar.gz) to a specified download folder. :param image_id: the (full) image id to get the manifest for, required :param namespace: the namespace (eg, "library") :param repo_name: the repo name, (eg, "ubuntu") :param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended) :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url # The <name> variable is the namespace/repo_name base = "%s/%s/%s/%s/blobs/%s" %(registry,api_version,namespace,repo_name,image_id) logger.info("Downloading layers from %s", base) # To get the image layers, we need a valid token to read the repo token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) if download_folder != None: download_folder = "%s/%s.tar.gz" %(download_folder,image_id) # Update user what we are doing print("Downloading layer %s" %image_id) response = api_get(base,headers=token,stream=download_folder) if isinstance(response, HTTPError): logger.error("Error downloading layer %s, exiting.", base) sys.exit(1) return response
def get_layer(image_id,namespace,repo_name,download_folder=None,registry=None,auth=True): '''get_layer will download an image layer (.tar.gz) to a specified download folder. :param image_id: the (full) image id to get the manifest for, required :param namespace: the namespace (eg, "library") :param repo_name: the repo name, (eg, "ubuntu") :param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended) :param registry: the docker registry to use (default will use registry-1.docker.io :param auth: does the API require obtaining an authentication Token? (default True) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url # The <name> variable is the namespace/repo_name base = "%s/%s/%s/%s/blobs/%s" %(registry,api_version,namespace,repo_name,image_id) # To get the image layers, we need a valid token to read the repo token = None if auth == True: token = get_token(repo_name=repo_name, permission="pull") if download_folder != None: download_folder = "%s/%s.tar.gz" %(download_folder,image_id) # Update user what we are doing print("Downloading layer: %s" %(image_id)) return api_get(base,headers=token,stream=download_folder)
def get_manifest(repo_name, namespace, repo_tag="latest"): '''get_manifest should return an image manifest for a particular repo and tag. The token is expected to be from version 2.0 (function above) :param repo_name: the name of the repo, eg "ubuntu" :param namespace: the namespace for the image, default is "library" :param repo_tag: the repo tag, default is "latest" ''' base = "%s/%s/%s/%s/manifests/%s" % (api_base, api_version, namespace, repo_name, repo_tag) # Format the token, and prepare a header token = get_token(repo_name=repo_name, permission="pull") response = api_get(base, headers=token, default_header=True) try: response = json.loads(response) except: # If the call fails, give the user a list of acceptable tags tags = get_tags(namespace=namespace, repo_name=repo_name) print("\n".join(tags)) print( "Error getting manifest for %s/%s:%s. Acceptable tags are listed above." % (namespace, repo_name, repo_tag)) sys.exit(1) return response
def get_layer(image_id,namespace,repo_name,download_folder=None,registry=None,auth=True): '''get_layer will download an image layer (.tar.gz) to a specified download folder. :param image_id: the (full) image id to get the manifest for, required :param namespace: the namespace (eg, "library") :param repo_name: the repo name, (eg, "ubuntu") :param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended) :param registry: the docker registry to use (default will use registry-1.docker.io :param auth: does the API require obtaining an authentication Token? (default True) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url # The <name> variable is the namespace/repo_name base = "%s/%s/%s/%s/blobs/%s" %(registry,api_version,namespace,repo_name,image_id) # To get the image layers, we need a valid token to read the repo token = None if auth == True: token = get_token(repo_name=repo_name, namespace=namespace, permission="pull") if download_folder != None: download_folder = "%s/%s.tar.gz" %(download_folder,image_id) # Update user what we are doing print("Downloading layer: %s" %(image_id)) return api_get(base,headers=token,stream=download_folder)
def get_manifest(repo_name,namespace,repo_tag="latest",registry=None,auth=True): '''get_manifest should return an image manifest for a particular repo and tag. The token is expected to be from version 2.0 (function above) :param repo_name: the name of the repo, eg "ubuntu" :param namespace: the namespace for the image, default is "library" :param repo_tag: the repo tag, default is "latest" :param registry: the docker registry to use (default will use registry-1.docker.io :param auth: does the API require obtaining an authentication Token? (default True) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/manifests/%s" %(registry,api_version,namespace,repo_name,repo_tag) # Format the token, and prepare a header token = None if auth == True: token = get_token(repo_name=repo_name, namespace=namespace, permission="pull") response = api_get(base,headers=token,default_header=True) try: response = json.loads(response) except: # If the call fails, give the user a list of acceptable tags tags = get_tags(namespace=namespace, repo_name=repo_name, registry=registry) print("\n".join(tags)) print("Error getting manifest for %s/%s:%s. Acceptable tags are listed above." %(namespace,repo_name,repo_tag)) sys.exit(1) return response
def get_tags(namespace,repo_name,registry=None,auth=True): '''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API :param namespace: the namespace (eg, "library") :param repo_name: the name for the repo (eg, "ubuntu") :param registry: the docker registry to use (default will use registry-1.docker.io :param auth: does the API require obtaining an authentication token? (default True) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/tags/list" %(registry,api_version,namespace,repo_name) # Does the api need an auth token? token = None if auth == True: token = get_token(repo_name=repo_name, namespace=namespace, permission="pull") response = api_get(base,headers=token) try: response = json.loads(response) return response['tags'] except: print("Error getting tags using url %s" %(base)) sys.exit(1)
def get_tags(namespace, repo_name, registry=None, auth=True): '''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API :param namespace: the namespace (eg, "library") :param repo_name: the name for the repo (eg, "ubuntu") :param registry: the docker registry to use (default will use registry-1.docker.io :param auth: does the API require obtaining an authentication token? (default True) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/tags/list" % (registry, api_version, namespace, repo_name) # Does the api need an auth token? token = None if auth == True: token = get_token(repo_name=repo_name, namespace=namespace, permission="pull") response = api_get(base, headers=token) try: response = json.loads(response) return response['tags'] except: print("Error getting tags using url %s" % (base)) sys.exit(1)
def get_token(repo_name, namespace="library", scope="repository", permission="pull"): '''get_token will use version 2.0 of Docker's service to return a token with given permission and scope - this function does work, but the token doesn't seem to work when used with other functions below for authentication :param repo_name: the name of the repo, eg "ubuntu" :param repo_tag: the name of a tag for the repo, default is "latest" :param scope: scope of the request, default is "repository" :param permission: permission for the request, default is "read" :: note # https://docs.docker.com/registry/spec/auth/token/ ''' base = "https://auth.docker.io/token?service=registry.docker.io&scope=%s:%s/%s:%s" % ( scope, namespace, repo_name, permission) response = api_get(base, default_header=False) try: token = json.loads(response)["token"] token = {"Authorization": "Bearer %s" % (token)} return token except: print("Error getting %s token for repository %s/%s, exiting." % (permission, namespace, repo_name)) sys.exit(1)
def get_tags(namespace, repo_name, registry=None, auth=None): '''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API :param namespace: the namespace (eg, "library") :param repo_name: the name for the repo (eg, "ubuntu") :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/tags/list" % (registry, api_version, namespace, repo_name) logger.info("Obtaining tags: %s", base) token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) response = api_get(base, headers=token) try: response = json.loads(response) return response['tags'] except: logger.error("Error obtaining tags: %s", base) sys.exit(1)
def get_tags(namespace,repo_name,registry=None,auth=None): '''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API :param namespace: the namespace (eg, "library") :param repo_name: the name for the repo (eg, "ubuntu") :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/tags/list" %(registry,api_version,namespace,repo_name) logger.info("Obtaining tags: %s", base) token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) response = api_get(base,headers=token) try: response = json.loads(response) return response['tags'] except: logger.error("Error obtaining tags: %s", base) sys.exit(1)
def get_manifest(image, registry=None): '''get_image will return a json object with image metadata, based on a unique id. :param image: the image name, either an id, or a repo name, tag, etc. :param registry: the registry (hub) to use, if not defined, default is used ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url # Numeric images have slightly different endpoint from named if is_number(image) == True: base = "%s/containers/%s" % (registry, image) else: base = "%s/container/%s" % (registry, image) # --------------------------------------------------------------- # If we eventually have private images, need to authenticate here # --------------------------------------------------------------- response = api_get(base) try: response = json.loads(response) except: print("Error getting image manifest using url %s" % (base)) sys.exit(1) return response
def get_manifest(repo_name,namespace,repo_tag="latest",registry=None,auth=True): '''get_manifest should return an image manifest for a particular repo and tag. The token is expected to be from version 2.0 (function above) :param repo_name: the name of the repo, eg "ubuntu" :param namespace: the namespace for the image, default is "library" :param repo_tag: the repo tag, default is "latest" :param registry: the docker registry to use (default will use registry-1.docker.io :param auth: does the API require obtaining an authentication Token? (default True) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/manifests/%s" %(registry,api_version,namespace,repo_name,repo_tag) # Format the token, and prepare a header token = None if auth == True: token = get_token(repo_name=repo_name, permission="pull") response = api_get(base,headers=token,default_header=True) try: response = json.loads(response) except: # If the call fails, give the user a list of acceptable tags tags = get_tags(namespace=namespace, repo_name=repo_name, registry=registry) print("\n".join(tags)) print("Error getting manifest for %s/%s:%s. Acceptable tags are listed above." %(namespace,repo_name,repo_tag)) sys.exit(1) return response
def get_manifest(repo_name, namespace, repo_tag="latest", registry=None, auth=None, headers=None): '''get_manifest should return an image manifest for a particular repo and tag. The token is expected to be from version 2.0 (function above) :param repo_name: the name of the repo, eg "ubuntu" :param namespace: the namespace for the image, default is "library" :param repo_tag: the repo tag, default is "latest" :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) :param headers: dictionary of custom headers to add to token header (to get more specific manifest) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/manifests/%s" % (registry, api_version, namespace, repo_name, repo_tag) logger.info("Obtaining manifest: %s", base) # Format the token, and prepare a header token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) # Add ['Accept'] header to specify version 2 of manifest if headers != None: if token != None: token.update(headers) else: token = headers response = api_get(base, headers=token, default_header=True) try: response = json.loads(response) except: # If the call fails, give the user a list of acceptable tags tags = get_tags(namespace=namespace, repo_name=repo_name, registry=registry, auth=auth) print("\n".join(tags)) logger.error("Error getting manifest for %s/%s:%s, exiting.", namespace, repo_name, repo_tag) print( "Error getting manifest for %s/%s:%s. Acceptable tags are listed above." % (namespace, repo_name, repo_tag)) sys.exit(1) return response
def get_layer(image_id,namespace,repo_name,download_folder=None,registry=None,auth=None): '''get_layer will download an image layer (.tar.gz) to a specified download folder. :param image_id: the (full) image id to get the manifest for, required :param namespace: the namespace (eg, "library") :param repo_name: the repo name, (eg, "ubuntu") :param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended) :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url # The <name> variable is the namespace/repo_name base = "%s/%s/%s/%s/blobs/%s" %(registry,api_version,namespace,repo_name,image_id) logger.info("Downloading layers from %s", base) # To get the image layers, we need a valid token to read the repo token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) if download_folder != None: download_folder = "%s/%s.tar.gz" %(download_folder,image_id) # Update user what we are doing print("Downloading layer %s" %image_id) try: # Create temporary file with format .tar.gz.tmp.XXXXX fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % download_folder)) os.close(fd) response = api_get(base,headers=token,stream=tmp_file) if isinstance(response, HTTPError): logger.error("Error downloading layer %s, exiting.", base) sys.exit(1) os.rename(tmp_file, download_folder) except: logger.error("Removing temporary download file %s", tmp_file) try: os.remove(tmp_file) except: pass sys.exit(1) return download_folder
def get_manifest(repo_name,namespace,repo_tag="latest",registry=None,auth=None,headers=None): '''get_manifest should return an image manifest for a particular repo and tag. The token is expected to be from version 2.0 (function above) :param repo_name: the name of the repo, eg "ubuntu" :param namespace: the namespace for the image, default is "library" :param repo_tag: the repo tag, default is "latest" :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) :param headers: dictionary of custom headers to add to token header (to get more specific manifest) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/manifests/%s" %(registry,api_version,namespace,repo_name,repo_tag) logger.info("Obtaining manifest: %s", base) # Format the token, and prepare a header token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) # Add ['Accept'] header to specify version 2 of manifest if headers != None: if token != None: token.update(headers) else: token = headers response = api_get(base,headers=token,default_header=True) try: response = json.loads(response) except: # If the call fails, give the user a list of acceptable tags tags = get_tags(namespace=namespace, repo_name=repo_name, registry=registry, auth=auth) print("\n".join(tags)) logger.error("Error getting manifest for %s/%s:%s, exiting.", namespace, repo_name, repo_tag) print("Error getting manifest for %s/%s:%s. Acceptable tags are listed above." %(namespace,repo_name,repo_tag)) sys.exit(1) return response
def get_tags(namespace, repo_name): '''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API :param namespace: the namespace (eg, "library") :param repo_name: the name for the repo (eg, "ubuntu") ''' base = "%s/%s/%s/%s/tags/list" % (api_base, api_version, namespace, repo_name) token = get_token(repo_name=repo_name, permission="pull") response = api_get(base, headers=token) try: response = json.loads(response) return response['tags'] except: print("Error getting tags using url %s" % (base)) sys.exit(1)
def download_image(manifest, download_folder=None, extract=True): '''download_image will download a singularity image from singularity hub to a download_folder, named based on the image version (commit id) :param manifest: the manifest obtained with get_manifest :param download_folder: the folder to download to, if None, will be pwd :param extract: if True, will extract image to .img and return that. ''' image_file = get_image_name(manifest) print("Found image %s" % (manifest['name'])) print("Downloading image... %s" % (image_file)) if download_folder != None: image_file = "%s/%s" % (download_folder, image_file) url = manifest['image'] image_file = api_get(url, stream=image_file) if extract == True: print("Decompressing", image_file) os.system('gzip -d -f %s' % (image_file)) image_file = image_file.replace('.gz', '') return image_file
def download_image(manifest,download_folder=None,extract=True): '''download_image will download a singularity image from singularity hub to a download_folder, named based on the image version (commit id) :param manifest: the manifest obtained with get_manifest :param download_folder: the folder to download to, if None, will be pwd :param extract: if True, will extract image to .img and return that. ''' image_file = get_image_name(manifest) print("Found image %s" %(manifest['name'])) print("Downloading image... %s" %(image_file)) if download_folder != None: image_file = "%s/%s" %(download_folder,image_file) url = manifest['image'] image_file = api_get(url,stream=image_file) if extract == True: print("Decompressing", image_file) os.system('gzip -d -f %s' %(image_file)) image_file = image_file.replace('.gz','') return image_file
def get_manifest(image_id, registry=None): '''get_image will return a json object with image metadata, based on a unique id. :param image_id: the image_id :param registry: the registry (hub) to use, if not defined, default is used ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/containers/%s" % (registry, image_id) # --------------------------------------------------------------- # If we eventually have private images, need to authenticate here # --------------------------------------------------------------- response = api_get(base) try: response = json.loads(response) except: print("Error getting image manifest using url %s" % (base)) sys.exit(1) return response
def get_manifest(image_id,registry=None): '''get_image will return a json object with image metadata, based on a unique id. :param image_id: the image_id :param registry: the registry (hub) to use, if not defined, default is used ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/containers/%s" %(registry,image_id) # --------------------------------------------------------------- # If we eventually have private images, need to authenticate here # --------------------------------------------------------------- response = api_get(base) try: response = json.loads(response) except: print("Error getting image manifest using url %s" %(base)) sys.exit(1) return response
def get_layer(image_id, namespace, repo_name, download_folder=None): '''get_layer will download an image layer (.tar.gz) to a specified download folder. :param image_id: the (full) image id to get the manifest for, required :param namespace: the namespace (eg, "library") :param repo_name: the repo name, (eg, "ubuntu") :param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended) ''' # The <name> variable is the namespace/repo_name base = "%s/%s/%s/%s/blobs/%s" % (api_base, api_version, namespace, repo_name, image_id) # To get the image layers, we need a valid token to read the repo token = get_token(repo_name=repo_name, permission="pull") if download_folder != None: download_folder = "%s/%s.tar.gz" % (download_folder, image_id) # Update user what we are doing print("Downloading layer %s.tar.gz to %s" % (image_id, download_folder)) return api_get(base, headers=token, stream=download_folder)
def get_token(repo_name,namespace="library",scope="repository",permission="pull"): '''get_token will use version 2.0 of Docker's service to return a token with given permission and scope - this function does work, but the token doesn't seem to work when used with other functions below for authentication :param repo_name: the name of the repo, eg "ubuntu" :param repo_tag: the name of a tag for the repo, default is "latest" :param scope: scope of the request, default is "repository" :param permission: permission for the request, default is "read" :: note # https://docs.docker.com/registry/spec/auth/token/ ''' base = "https://auth.docker.io/token?service=registry.docker.io&scope=%s:%s/%s:%s" %(scope, namespace, repo_name, permission) response = api_get(base,default_header=False) try: token = json.loads(response)["token"] token = {"Authorization": "Bearer %s" %(token) } return token except: print("Error getting %s token for repository %s/%s, exiting." %(permission,namespace,repo_name)) sys.exit(1)
def is_brave_in_playing_state(port): response = utils.api_get('/api/all', port=port) assert response.status_code == 200 response_json = response.json() return response_json['inputs'][0]['state'] == 'PLAYING' and response_json[ 'outputs'][0]['state'] == 'PLAYING'