def __init__(self, localrepo):
self.localrepo = localrepo # LocalRepository object
self._autoinstall = Config.conf['autoinstall'] # True / False
self._tarball = Config.conf['tarball'] # URL or file
self._installinfo = Config.conf['installinfo'] # URL or file
self._tarball_release = Config.conf['tarball_release']
self._installretry = Config.conf['installretry']
self._install_json = dict()
self.curl = GetURL()
def __init__(self, localrepo):
self.index_url = Config.conf['dockerio_index_url']
self.registry_url = Config.conf['dockerio_registry_url']
self.v1_auth_header = ""
self.v2_auth_header = ""
self.v2_auth_token = ""
self.localrepo = localrepo
self.curl = GetURL()
self.search_pause = True
self.search_page = 0
self.search_ended = False
def test_04_set_insecure(self, mock_gupycurl):
"""Test04 GetURL().set_insecure()."""
mock_gupycurl.return_value = True
geturl = GetURL()
geturl.set_insecure()
self.assertEqual(geturl.insecure, True)
geturl = GetURL()
geturl.set_insecure(False)
self.assertEqual(geturl.insecure, False)
def test_08_get_status_code(self):
"""Test08 GetURL().get_status_code()."""
sline = "HTTP-Version 400 Reason-Phrase"
geturl = GetURL()
status = geturl.get_status_code(sline)
self.assertEqual(status, 400)
sline = "HTTP-Version Reason-Phrase"
geturl = GetURL()
status = geturl.get_status_code(sline)
self.assertEqual(status, 400)
sline = ""
geturl = GetURL()
status = geturl.get_status_code(sline)
self.assertEqual(status, 404)
def test_01_init(self, mock_msg, mock_gupycurl, mock_guexecurl,
mock_select):
"""Test01 GetURL() constructor."""
mock_msg.level = 0
mock_gupycurl.return_value = False
mock_guexecurl.return_value = True
geturl = GetURL()
mock_select.assert_called()
self.assertEqual(geturl.ctimeout, Config().conf['ctimeout'])
self.assertEqual(geturl.insecure, Config().conf['http_insecure'])
self.assertFalse(geturl.cache_support)
def test_02__select_implementation(self, mock_gupycurl, mock_guexecurl,
mock_msg):
"""Test02 GetURL()._select_implementation()."""
mock_msg.level = 0
mock_gupycurl.return_value = True
geturl = GetURL()
geturl._select_implementation()
# self.assertTrue(geturl.cache_support)
mock_gupycurl.return_value = False
geturl = GetURL()
geturl._select_implementation()
self.assertFalse(geturl.cache_support)
mock_guexecurl.return_value = False
with self.assertRaises(NameError):
GetURL()
def test_06_get(self):
"""Test06 GetURL().get()."""
geturl = GetURL()
self.assertRaises(TypeError, geturl.get)
geturl = GetURL()
geturl._geturl = type('test', (object, ), {})()
geturl._geturl.get = self._get
self.assertEqual(geturl.get("http://host"), "http://host")
def test_07_post(self):
"""Test07 GetURL().post()."""
geturl = GetURL()
self.assertRaises(TypeError, geturl.post)
self.assertRaises(TypeError, geturl.post, "http://host")
geturl = GetURL()
geturl._geturl = type('test', (object, ), {})()
geturl._geturl.get = self._get
status = geturl.post("http://host", {
"DATA": 1,
})
self.assertEqual(status, "http://host")
def test_03_get_content_length(self):
"""Test03 GetURL().get_content_length()."""
hdr = type('test', (object, ), {})()
hdr.data = {
"content-length": 10,
}
geturl = GetURL()
self.assertEqual(geturl.get_content_length(hdr), 10)
hdr = type('test', (object, ), {})()
hdr.data = {
"content-length": dict(),
}
geturl = GetURL()
self.assertEqual(geturl.get_content_length(hdr), -1)
class DockerIoAPI(object):
"""Class to encapsulate the access to the Docker Hub service
Allows to search and download images from Docker Hub
"""
def __init__(self, localrepo):
self.index_url = Config.conf['dockerio_index_url']
self.registry_url = Config.conf['dockerio_registry_url']
self.v1_auth_header = ""
self.v2_auth_header = ""
self.v2_auth_token = ""
self.localrepo = localrepo
self.curl = GetURL()
self.search_pause = True
self.search_page = 0
self.search_ended = False
def set_proxy(self, http_proxy):
"""Select a socks http proxy for API access and file download"""
self.curl.set_proxy(http_proxy)
def set_registry(self, registry_url):
"""Change docker registry url"""
self.registry_url = registry_url
def set_index(self, index_url):
"""Change docker index url"""
self.index_url = index_url
def is_repo_name(self, imagerepo):
"""Check if name matches authorized characters for a docker repo"""
if imagerepo and re.match("^[a-zA-Z0-9][a-zA-Z0-9-_./:]+$", imagerepo):
return True
return False
def _get_url(self, *args, **kwargs):
"""Encapsulates the call to GetURL.get() so that authentication
for v1 and v2 repositories can be treated differently.
Example:
_get_url(url, ctimeout=5, timeout=5, header=[]):
"""
url = str(args[0])
if "RETRY" not in kwargs:
kwargs["RETRY"] = 3
if "FOLLOW" not in kwargs:
kwargs["FOLLOW"] = 3
kwargs["RETRY"] -= 1
(hdr, buf) = self.curl.get(*args, **kwargs)
Msg().out("Info: header: %s" % (hdr.data), l=Msg.DBG)
Msg().out("Info: buffer: %s" % (buf.getvalue()), l=Msg.DBG)
status_code = self.curl.get_status_code(hdr.data["X-ND-HTTPSTATUS"])
if status_code == 200:
return (hdr, buf)
if not kwargs["RETRY"]:
hdr.data["X-ND-CURLSTATUS"] = 13 # Permission denied
return (hdr, buf)
auth_kwargs = kwargs.copy()
if "location" not in hdr.data:
kwargs["FOLLOW"] = 3
if "location" in hdr.data and hdr.data['location']:
if not kwargs["FOLLOW"]:
hdr.data["X-ND-CURLSTATUS"] = 13
return (hdr, buf)
kwargs["FOLLOW"] -= 1
args = [hdr.data['location']]
if "header" in auth_kwargs:
del auth_kwargs["header"]
elif status_code == 401:
if "www-authenticate" in hdr.data:
www_authenticate = hdr.data["www-authenticate"]
if not "realm" in www_authenticate:
return (hdr, buf)
if 'error="insufficient_scope"' in www_authenticate:
return (hdr, buf)
auth_header = ""
if "/v2/" in url:
auth_header = self._get_v2_auth(www_authenticate,
kwargs["RETRY"])
if "/v1/" in url:
auth_header = self._get_v1_auth(www_authenticate)
auth_kwargs.update({"header": [auth_header]})
(hdr, buf) = self._get_url(*args, **auth_kwargs)
return (hdr, buf)
def _get_file(self, url, filename, cache_mode):
"""Get a file and check its size. Optionally enable other
capabilities such as caching to check if the
file already exists locally and whether its size is the
same to avoid downloaded it again.
"""
match = re.search("/([^/:]+):(\\S+)$", filename)
if match:
layer_f_chksum = ChkSUM().hash(filename, match.group(1))
if layer_f_chksum == match.group(2):
return True # is cached skip download
cache_mode = 0
if self.curl.cache_support and cache_mode:
if cache_mode == 1:
(hdr, dummy) = self._get_url(url, nobody=1)
elif cache_mode == 3:
(hdr, dummy) = self._get_url(url, sizeonly=True)
remote_size = self.curl.get_content_length(hdr)
if remote_size == FileUtil(filename).size():
return True # is cached skip download
else:
remote_size = -1
resume = False
if filename.endswith("layer"):
resume = True
(hdr, dummy) = self._get_url(url, ofile=filename, resume=resume)
if self.curl.get_status_code(hdr.data["X-ND-HTTPSTATUS"]) != 200:
return False
if remote_size == -1:
remote_size = self.curl.get_content_length(hdr)
if (remote_size != FileUtil(filename).size()
and hdr.data["X-ND-CURLSTATUS"]):
Msg().err("Error: file size mismatch:", filename, remote_size,
FileUtil(filename).size())
return False
return True
def _split_fields(self, buf):
"""Split fields, used in the web authentication"""
all_fields = dict()
for field in buf.split(','):
pair = field.split('=', 1)
if len(pair) == 2:
all_fields[pair[0]] = pair[1].strip('"')
return all_fields
def is_v1(self):
"""Check if registry is of type v1"""
for prefix in ("/v1", "/v1/_ping"):
(hdr, dummy) = self._get_url(self.index_url + prefix)
try:
if ("200" in hdr.data["X-ND-HTTPSTATUS"]
or "401" in hdr.data["X-ND-HTTPSTATUS"]):
return True
except (KeyError, AttributeError, TypeError):
pass
return False
def has_search_v1(self, url=None):
"""Check if registry has search capabilities in v1"""
if url is None:
url = self.index_url
(hdr, dummy) = self._get_url(url + "/v1/search")
try:
if ("200" in hdr.data["X-ND-HTTPSTATUS"]
or "401" in hdr.data["X-ND-HTTPSTATUS"]):
return True
except (KeyError, AttributeError, TypeError):
pass
return False
def get_v1_repo(self, imagerepo):
"""Get list of images in a repo from Docker Hub"""
url = self.index_url + "/v1/repositories/" + imagerepo + "/images"
Msg().out("Info: repo url", url, l=Msg.DBG)
(hdr, buf) = self._get_url(url, header=["X-Docker-Token: true"])
try:
self.v1_auth_header = "Authorization: Token " + \
hdr.data["x-docker-token"]
return hdr.data, json.loads(buf.getvalue())
except (IOError, OSError, AttributeError, ValueError, TypeError,
KeyError):
self.v1_auth_header = ""
return hdr.data, []
def _get_v1_auth(self, www_authenticate):
"""Authentication for v1 API"""
if "Token" in www_authenticate:
return self.v1_auth_header
return ""
def get_v1_image_tags(self, imagerepo, tags_only=False):
"""Get list of tags in a repo from Docker Hub"""
(hdr_data, dummy) = self.get_v1_repo(imagerepo)
try:
endpoint = "http://" + hdr_data["x-docker-endpoints"]
except KeyError:
endpoint = self.index_url
url = endpoint + "/v1/repositories/" + imagerepo + "/tags"
Msg().out("Info: tags url", url, l=Msg.DBG)
(dummy, buf) = self._get_url(url)
tags = []
try:
if tags_only:
for tag in json.loads(buf.getvalue()):
tags.append(tag["name"])
return tags
return json.loads(buf.getvalue())
except (IOError, OSError, AttributeError, ValueError, TypeError):
return []
def get_v1_image_tag(self, endpoint, imagerepo, tag):
"""Get list of tags in a repo from Docker Hub"""
url = endpoint + "/v1/repositories/" + imagerepo + "/tags/" + tag
Msg().out("Info: tags url", url, l=Msg.DBG)
(hdr, buf) = self._get_url(url)
try:
return (hdr.data, json.loads(buf.getvalue()))
except (IOError, OSError, AttributeError, ValueError, TypeError):
return (hdr.data, [])
def get_v1_image_ancestry(self, endpoint, image_id):
"""Get the ancestry which is an ordered list of layers"""
url = endpoint + "/v1/images/" + image_id + "/ancestry"
Msg().out("Info: ancestry url", url, l=Msg.DBG)
(hdr, buf) = self._get_url(url)
try:
return (hdr.data, json.loads(buf.getvalue()))
except (IOError, OSError, AttributeError, ValueError, TypeError):
return (hdr.data, [])
def get_v1_image_json(self, endpoint, layer_id):
"""Get the JSON metadata for a specific layer"""
url = endpoint + "/v1/images/" + layer_id + "/json"
Msg().out("Info: json url", url, l=Msg.DBG)
filename = self.localrepo.layersdir + '/' + layer_id + ".json"
if self._get_file(url, filename, 0):
self.localrepo.add_image_layer(filename)
return True
return False
def get_v1_image_layer(self, endpoint, layer_id):
"""Get a specific layer data file (layer files are tarballs)"""
url = endpoint + "/v1/images/" + layer_id + "/layer"
Msg().out("Info: layer url", url, l=Msg.DBG)
filename = self.localrepo.layersdir + '/' + layer_id + ".layer"
if self._get_file(url, filename, 3):
self.localrepo.add_image_layer(filename)
return True
return False
def get_v1_layers_all(self, endpoint, layer_list):
"""Using a layer list download data and metadata files"""
files = []
if layer_list:
for layer_id in reversed(layer_list):
Msg().out("Info: downloading layer", layer_id, l=Msg.INF)
filesize = self.get_v1_image_json(endpoint, layer_id)
if not filesize:
return []
files.append(layer_id + ".json")
filesize = self.get_v1_image_layer(endpoint, layer_id)
if not filesize:
return []
files.append(layer_id + ".layer")
return files
def _get_v2_auth(self, www_authenticate, retry):
"""Authentication for v2 API"""
auth_header = ""
(bearer, auth_data) = www_authenticate.rsplit(' ', 1)
if bearer == "Bearer":
auth_fields = self._split_fields(auth_data)
if "realm" in auth_fields:
auth_url = auth_fields["realm"] + '?'
for field in auth_fields:
if field != "realm":
auth_url += field + '=' + auth_fields[field] + '&'
header = []
if self.v2_auth_token:
header = ["Authorization: Basic %s" % (self.v2_auth_token)]
(dummy, auth_buf) = self._get_url(auth_url,
header=header,
RETRY=retry)
if sys.version_info[0] >= 3:
token_buf = auth_buf.getvalue().decode()
else:
token_buf = auth_buf.getvalue()
if token_buf and "token" in token_buf:
try:
auth_token = json.loads(token_buf)
except (IOError, OSError, AttributeError, ValueError,
TypeError):
return auth_header
auth_header = "Authorization: Bearer " + \
auth_token["token"]
self.v2_auth_header = auth_header
# PR #126
elif 'BASIC' in bearer or 'Basic' in bearer:
auth_header = "Authorization: Basic %s" % (self.v2_auth_token)
self.v2_auth_header = auth_header
return auth_header
def get_v2_login_token(self, username, password):
"""Get a login token from username and password"""
if not (username and password):
return ""
try:
self.v2_auth_token = \
base64.b64encode(("%s:%s" % \
(username, password)).encode("utf-8")).decode("ascii")
except (KeyError, AttributeError, TypeError, ValueError, NameError):
self.v2_auth_token = ""
return self.v2_auth_token
def set_v2_login_token(self, v2_auth_token):
"""Load previously created login token"""
self.v2_auth_token = v2_auth_token
def is_v2(self):
"""Check if registry is of type v2"""
(hdr, dummy) = self._get_url(self.registry_url + "/v2/")
try:
if ("200" in hdr.data["X-ND-HTTPSTATUS"]
or "401" in hdr.data["X-ND-HTTPSTATUS"]):
return True
except (KeyError, AttributeError, TypeError):
pass
return False
def has_search_v2(self, url=None):
"""Check if registry has search capabilities in v2"""
if url is None:
url = self.registry_url
(hdr, dummy) = self._get_url(url + "/v2/search/repositories")
try:
if ("200" in hdr.data["X-ND-HTTPSTATUS"]
or "401" in hdr.data["X-ND-HTTPSTATUS"]):
return True
except (KeyError, AttributeError, TypeError):
pass
return False
def get_v2_image_tags(self, imagerepo, tags_only=False):
"""Get list of tags in a repo from Docker Hub"""
url = self.registry_url + "/v2/" + imagerepo + "/tags/list"
Msg().out("Info: tags url", url, l=Msg.DBG)
(dummy, buf) = self._get_url(url)
tags = []
try:
if tags_only:
for tag in json.loads(buf.getvalue())["tags"]:
tags.append(tag)
return tags
return json.loads(buf.getvalue())
except (IOError, OSError, AttributeError, ValueError, TypeError):
return []
def get_v2_image_manifest(self, imagerepo, tag):
"""Get the image manifest which contains JSON metadata
that is common to all layers in this image tag
"""
url = self.registry_url + "/v2/" + imagerepo + \
"/manifests/" + tag
Msg().out("Info: manifest url", url, l=Msg.DBG)
(hdr, buf) = self._get_url(url)
try:
return (hdr.data, json.loads(buf.getvalue()))
except (IOError, OSError, AttributeError, ValueError, TypeError):
return (hdr.data, [])
def get_v2_image_layer(self, imagerepo, layer_id):
"""Get one image layer data file (tarball)"""
url = self.registry_url + "/v2/" + imagerepo + \
"/blobs/" + layer_id
Msg().out("Info: layer url", url, l=Msg.DBG)
filename = self.localrepo.layersdir + '/' + layer_id
if self._get_file(url, filename, 3):
self.localrepo.add_image_layer(filename)
return True
return False
def get_v2_layers_all(self, imagerepo, fslayers):
"""Get all layer data files belonging to a image tag"""
files = []
blob = ""
if fslayers:
for layer in reversed(fslayers):
if "blobSum" in layer:
blob = layer["blobSum"]
elif "digest" in layer:
blob = layer["digest"]
Msg().out("Info: downloading layer", blob, l=Msg.INF)
if not self.get_v2_image_layer(imagerepo, blob):
return []
files.append(blob)
return files
def get_v2(self, imagerepo, tag):
"""Pull container with v2 API"""
files = []
(hdr_data, manifest) = self.get_v2_image_manifest(imagerepo, tag)
status = self.curl.get_status_code(hdr_data["X-ND-HTTPSTATUS"])
if status == 401:
Msg().err("Error: manifest not found or not authorized")
return []
if status != 200:
Msg().err("Error: pulling manifest:")
return []
try:
if not (self.localrepo.setup_tag(tag)
and self.localrepo.set_version("v2")):
Msg().err("Error: setting localrepo v2 tag and version")
return []
self.localrepo.save_json("manifest", manifest)
Msg().out("Info: v2 layers: %s" % (imagerepo), l=Msg.DBG)
if "fsLayers" in manifest:
files = self.get_v2_layers_all(imagerepo, manifest["fsLayers"])
elif "layers" in manifest:
if "config" in manifest:
manifest["layers"].append(manifest["config"])
files = self.get_v2_layers_all(imagerepo, manifest["layers"])
else:
Msg().err("Error: layers section missing in manifest")
except (KeyError, AttributeError, IndexError, ValueError, TypeError):
pass
return files
def _get_v1_id_from_tags(self, tags_obj, tag):
"""Get image id from array of tags"""
if isinstance(tags_obj, dict):
try:
return tags_obj[tag]
except KeyError:
pass
elif isinstance(tags_obj, list):
try:
for tag_dict in tags_obj:
if tag_dict["name"] == tag:
return tag_dict["layer"]
except KeyError:
pass
return ""
def _get_v1_id_from_images(self, images_array, short_id):
"""Get long image id from array of images using the short id"""
try:
for image_dict in images_array:
if image_dict["id"][0:8] == short_id:
return image_dict["id"]
except KeyError:
pass
return ""
def get_v1(self, imagerepo, tag):
"""Pull container with v1 API"""
Msg().out("Info: v1 image id: %s" % (imagerepo), l=Msg.DBG)
(hdr_data, images_array) = self.get_v1_repo(imagerepo)
status = self.curl.get_status_code(hdr_data["X-ND-HTTPSTATUS"])
if status == 401 or not images_array:
Msg().err("Error: image not found or not authorized")
return []
try:
endpoint = "http://" + hdr_data["x-docker-endpoints"]
except KeyError:
endpoint = self.index_url
(tags_array) = self.get_v1_image_tags(endpoint, imagerepo)
image_id = self._get_v1_id_from_tags(tags_array, tag)
if not image_id:
Msg().err("Error: image tag not found")
return []
if len(image_id) <= 8:
image_id = self._get_v1_id_from_images(images_array, image_id)
if not image_id:
Msg().err("Error: image id not found")
return []
if not (self.localrepo.setup_tag(tag)
and self.localrepo.set_version("v1")):
Msg().err("Error: setting localrepo v1 tag and version")
return []
Msg().out("Info: v1 ancestry", image_id, l=Msg.DBG)
(dummy, ancestry) = self.get_v1_image_ancestry(endpoint, image_id)
if not ancestry:
Msg().err("Error: ancestry not found")
return []
self.localrepo.save_json("ancestry", ancestry)
Msg().out("Info: v1 layers", image_id, l=Msg.DBG)
files = self.get_v1_layers_all(endpoint, ancestry)
return files
def _parse_imagerepo(self, imagerepo):
"""Parse imagerepo to extract registry"""
remoterepo = imagerepo
registry = ""
registry_url = ""
index_url = ""
components = imagerepo.split('/')
if '.' in components[0] and len(components) >= 2:
registry = components[0]
del components[0]
elif ('.' not in components[0] and components[0] != "library"
and len(components) == 1):
components.insert(0, "library")
remoterepo = '/'.join(components)
if registry:
try:
registry_url = Config.conf['docker_registries'][registry][0]
index_url = Config.conf['docker_registries'][registry][1]
except (KeyError, NameError, TypeError):
registry_url = registry
if "://" not in registry:
registry_url = "https://%s" % registry
index_url = registry_url
if registry_url:
self.registry_url = registry_url
if index_url:
self.index_url = index_url
return (imagerepo, remoterepo)
def get(self, imagerepo, tag):
"""Pull a docker image from a v2 registry or v1 index"""
Msg().out("Info: get imagerepo: %s tag: %s" % (imagerepo, tag),
l=Msg.DBG)
(imagerepo, remoterepo) = self._parse_imagerepo(imagerepo)
if self.localrepo.cd_imagerepo(imagerepo, tag):
new_repo = False
else:
self.localrepo.setup_imagerepo(imagerepo)
new_repo = True
if self.is_v2():
files = self.get_v2(remoterepo, tag) # try v2
else:
files = self.get_v1(remoterepo, tag) # try v1
if new_repo and not files:
self.localrepo.del_imagerepo(imagerepo, tag, False)
return files
def get_tags(self, imagerepo):
"""List tags from a v2 or v1 repositories"""
Msg().out("Info: get tags", imagerepo, l=Msg.DBG)
(dummy, remoterepo) = self._parse_imagerepo(imagerepo)
if self.is_v2():
return self.get_v2_image_tags(remoterepo, True) # try v2
return self.get_v1_image_tags(remoterepo, True) # try v1
def search_init(self, pause):
"""Setup new search"""
self.search_pause = pause
self.search_page = 0
self.search_ended = False
def search_get_page_v1(self, expression, url):
"""Get search results from Docker Hub using v1 API"""
if expression:
url = url + "/v1/search?q=%s" % expression
else:
url = url + "/v1/search?"
url += "&page=%s" % str(self.search_page)
(dummy, buf) = self._get_url(url)
try:
repo_list = json.loads(buf.getvalue())
if repo_list["page"] == repo_list["num_pages"]:
self.search_ended = True
return repo_list
except (IOError, OSError, AttributeError, ValueError, TypeError):
self.search_ended = True
return []
def search_get_page_v2(self, expression, url, lines=22, official=None):
"""Search results from Docker Hub using v2 API"""
if not expression:
expression = '*'
if expression and official is None:
url = url + \
"/v2/search/repositories?query=%s" % (expression)
elif expression and official is True:
url = url + \
"/v2/search/repositories?query=%s&is_official=%s" \
% (expression, "true")
elif expression and official is False:
url = url + \
"/v2/search/repositories?query=%s&is_official=%s" \
% (expression, "false")
else:
return []
url += "&page_size=%d" % (lines)
if self.search_page != 1:
url += "&page=%d" % (self.search_page)
(dummy, buf) = self._get_url(url)
try:
repo_list = json.loads(buf.getvalue())
if repo_list["count"] == self.search_page:
self.search_ended = True
return repo_list
except (IOError, OSError, AttributeError, KeyError, ValueError,
TypeError):
self.search_ended = True
return []
def search_get_page(self, expression, lines=22):
"""Get search results from Docker Hub"""
if self.search_ended:
return []
self.search_page += 1
if self.has_search_v2(self.index_url):
return self.search_get_page_v2(expression, self.index_url, lines)
if self.has_search_v1():
return self.search_get_page_v1(expression, self.index_url)
if self.has_search_v2(self.registry_url):
return self.search_get_page_v2(expression, self.registry_url,
lines)
if self.has_search_v1(self.registry_url):
return self.search_get_page_v1(expression, self.registry_url)
return []
def test_02__select_implementation(self, mock_gupycurl, mock_guexecurl,
mock_msg):
"""Test02 GetURL()._select_implementation()."""
Config.conf['use_curl_executable'] = ""
mock_msg.level = 0
mock_gupycurl.return_value = True
geturl = GetURL()
geturl._select_implementation()
self.assertTrue(geturl.cache_support)
self.assertTrue(mock_gupycurl.called)
mock_gupycurl.return_value = False
geturl = GetURL()
geturl._select_implementation()
self.assertFalse(geturl.cache_support)
mock_gupycurl.return_value = False
mock_guexecurl.return_value = False
with self.assertRaises(NameError) as nameerr:
geturl = GetURL()
geturl._select_implementation()
self.assertEqual(nameerr.exception.code, 1)
def test_05_set_proxy(self, mock_gupycurl):
"""Test05 GetURL().set_proxy()."""
mock_gupycurl.return_value = True
geturl = GetURL()
geturl.set_proxy("http://host")
self.assertEqual(geturl.http_proxy, "http://host")
class UdockerTools(object):
"""Download and setup of the udocker supporting tools
Includes: proot and alternative python modules, these
are downloaded to facilitate the installation by the
end-user.
"""
def __init__(self, localrepo):
self.localrepo = localrepo # LocalRepository object
self._autoinstall = Config.conf['autoinstall'] # True / False
self._tarball = Config.conf['tarball'] # URL or file
self._installinfo = Config.conf['installinfo'] # URL or file
self._tarball_release = Config.conf['tarball_release']
self._installretry = Config.conf['installretry']
self._install_json = dict()
self.curl = GetURL()
def _instructions(self):
"""
Udocker installation instructions are available at:
https://github.com/indigo-dc/udocker/tree/master/doc
https://github.com/indigo-dc/udocker/tree/devel/doc
Udocker requires additional tools to run. These are available
in the udocker tarball. The tarballs are available at several
locations. By default udocker will install from the locations
defined in Config.tarball.
To install from files or other URLs use these instructions:
1) set UDOCKER_TARBALL to a remote URL or local filename:
$ export UDOCKER_TARBALL=http://host/path
or
$ export UDOCKER_TARBALL=/tmp/filename
2) run udocker with the install command or optionally using
the option --force to overwrite the local installation:
./udocker install
or
./udocker install --force
3) once installed the binaries and containers will be placed
by default under $HOME/.udocker
"""
Msg().out(self._instructions.__doc__, l=Msg.ERR)
Msg().out("udocker command line interface version:",
__version__,
"\nrequires udockertools tarball release :",
self._tarball_release,
l=Msg.ERR)
def _version2int(self, version):
"""Convert version string to integer"""
version_int = 0
factor = 1000 * 1000
for vitem in _str(version).strip().split('.'):
try:
version_int = version_int + (int(vitem) * factor)
except (TypeError, ValueError):
pass
factor = factor / 1000
return int(version_int)
def _version_isok(self, version):
"""Is version >= than the minimum required tarball release"""
if not (version and self._tarball_release):
return False
tarball_version_int = self._version2int(version)
required_version_int = self._version2int(self._tarball_release)
return tarball_version_int >= required_version_int
def is_available(self):
"""Are the tools already installed"""
version = \
FileUtil(self.localrepo.libdir + "/VERSION").getdata('r').strip()
return self._version_isok(version)
def purge(self):
"""Remove existing files in bin, lib and doc"""
for f_name in os.listdir(self.localrepo.bindir):
f_path = self.localrepo.bindir + '/' + f_name
FileUtil(f_path).register_prefix()
FileUtil(f_path).remove(recursive=True)
for f_name in os.listdir(self.localrepo.libdir):
f_path = self.localrepo.libdir + '/' + f_name
FileUtil(f_path).register_prefix()
FileUtil(f_path).remove(recursive=True)
for f_name in os.listdir(self.localrepo.docdir):
f_path = self.localrepo.docdir + '/' + f_name
FileUtil(f_path).register_prefix()
FileUtil(f_path).remove(recursive=True)
def _download(self, url):
"""Download a file """
download_file = FileUtil("udockertools").mktmp()
if Msg.level <= Msg.DEF:
Msg().setlevel(Msg.NIL)
(hdr, dummy) = self.curl.get(url, ofile=download_file, follow=True)
if Msg.level == Msg.NIL:
Msg().setlevel()
try:
if "200" in hdr.data["X-ND-HTTPSTATUS"]:
return download_file
except (KeyError, TypeError, AttributeError):
pass
FileUtil(download_file).remove()
return ""
def _get_file(self, url):
"""Get file from list of possible locations file or internet"""
filename = ""
if "://" in url:
filename = self._download(url)
elif os.path.exists(url):
filename = os.path.realpath(url)
if filename and os.path.isfile(filename):
return filename
return ""
def _verify_version(self, tarball_file):
"""verify the tarball version"""
if not (tarball_file and os.path.isfile(tarball_file)):
return (False, "")
tmpdir = FileUtil("VERSION").mktmpdir()
if not tmpdir:
return (False, "")
try:
tfile = tarfile.open(tarball_file, "r:gz")
for tar_in in tfile.getmembers():
if tar_in.name.startswith("udocker_dir/lib/VERSION"):
tar_in.name = os.path.basename(tar_in.name)
tfile.extract(tar_in, path=tmpdir)
tfile.close()
except tarfile.TarError:
FileUtil(tmpdir).remove(recursive=True)
return (False, "")
tarball_version = FileUtil(tmpdir + "/VERSION").getdata('r').strip()
status = self._version_isok(tarball_version)
FileUtil(tmpdir).remove(recursive=True)
return (status, tarball_version)
def _install(self, tarball_file):
"""Install the tarball"""
if not (tarball_file and os.path.isfile(tarball_file)):
return False
FileUtil(self.localrepo.topdir).chmod()
self.localrepo.create_repo()
try:
tfile = tarfile.open(tarball_file, "r:gz")
FileUtil(self.localrepo.bindir).rchmod()
for tar_in in tfile.getmembers():
if tar_in.name.startswith("udocker_dir/bin/"):
tar_in.name = os.path.basename(tar_in.name)
Msg().out("Info: extrating", tar_in.name, l=Msg.DBG)
tfile.extract(tar_in, self.localrepo.bindir)
FileUtil(self.localrepo.bindir).rchmod(stat.S_IRUSR | stat.S_IWUSR
| stat.S_IXUSR)
FileUtil(self.localrepo.libdir).rchmod()
for tar_in in tfile.getmembers():
if tar_in.name.startswith("udocker_dir/lib/"):
tar_in.name = os.path.basename(tar_in.name)
Msg().out("Info: extrating", tar_in.name, l=Msg.DBG)
tfile.extract(tar_in, self.localrepo.libdir)
FileUtil(self.localrepo.libdir).rchmod()
FileUtil(self.localrepo.docdir).rchmod()
for tar_in in tfile.getmembers():
if tar_in.name.startswith("udocker_dir/doc/"):
tar_in.name = os.path.basename(tar_in.name)
Msg().out("Info: extrating", tar_in.name, l=Msg.DBG)
tfile.extract(tar_in, self.localrepo.docdir)
FileUtil(self.localrepo.docdir).rchmod()
tfile.close()
except tarfile.TarError:
return False
return True
def _get_mirrors(self, mirrors):
"""Get shuffled list of tarball mirrors"""
if is_genstr(mirrors):
mirrors = mirrors.split(' ')
try:
random.shuffle(mirrors)
except NameError:
pass
return mirrors
def get_installinfo(self):
"""Get json containing installation info"""
Msg().out("Info: searching for messages:", l=Msg.VER)
for url in self._get_mirrors(self._installinfo):
infofile = self._get_file(url)
try:
with open(infofile, 'r') as filep:
self._install_json = json.load(filep)
for msg in self._install_json["messages"]:
Msg().out("Info:", msg)
except (KeyError, AttributeError, ValueError, OSError, IOError):
Msg().out("Info: no messages:", infofile, url, l=Msg.VER)
return self._install_json
def _install_logic(self, force=False):
"""Obtain random mirror, download, verify and install"""
for url in self._get_mirrors(self._tarball):
Msg().out("Info: install using:", url, l=Msg.VER)
tarballfile = self._get_file(url)
(status, version) = self._verify_version(tarballfile)
if status:
Msg().out("Info: installing udockertools", version)
status = self._install(tarballfile)
elif force:
Msg().out("Info: forcing install of udockertools", version)
status = self._install(tarballfile)
else:
Msg().err("Error: version is", version, "for", url, l=Msg.VER)
if "://" in url and tarballfile:
FileUtil(tarballfile).remove()
if status:
return True
return False
def install(self, force=False):
"""Get the udocker tools tarball and install the binaries"""
if self.is_available() and not force:
return True
if not self._autoinstall and not force:
Msg().out("Warning: installation missing and autoinstall disabled",
l=Msg.WAR)
return None
if not self._tarball:
Msg().out("Info: UDOCKER_TARBALL not set, installation skipped",
l=Msg.VER)
return True
Msg().out("Info: udocker command line interface", __version__)
Msg().out("Info: searching for udockertools",
self._tarball_release,
l=Msg.INF)
retry = self._installretry
while retry:
if self._install_logic(force):
self.get_installinfo()
Msg().out("Info: installation of udockertools successful")
return True
retry = retry - 1
Msg().err("Error: installation failure retrying ...", l=Msg.VER)
self._instructions()
self.get_installinfo()
Msg().err("Error: installation of udockertools failed")
return False