def remove_github_token(self):
'''
If for some reason an ansible-galaxy token was left from a prior login, remove it. We cannot
retrieve the token after creation, so we are forced to create a new one.
'''
try:
tokens = json.load(
open_url(self.GITHUB_AUTH,
url_username=self.github_username,
url_password=self.github_password,
force_basic_auth=True,
validate_certs=self._validate_certs,
http_agent=user_agent()))
except HTTPError as e:
res = json.load(e)
raise AnsibleError(res['message'])
for token in tokens:
if token['note'] == 'ansible-galaxy login':
display.vvvvv('removing token: %s' % token['token_last_eight'])
try:
open_url('https://api.github.com/authorizations/%d' %
token['id'],
url_username=self.github_username,
url_password=self.github_password,
method='DELETE',
force_basic_auth=True,
validate_certs=self._validate_certs,
http_agent=user_agent())
except HTTPError as e:
res = json.load(e)
raise AnsibleError(res['message'])
def _download_file(url, b_path, expected_hash, validate_certs, headers=None):
bufsize = 65536
digest = sha256()
urlsplit = os.path.splitext(to_text(url.rsplit('/', 1)[1]))
b_file_name = to_bytes(urlsplit[0], errors='surrogate_or_strict')
b_file_ext = to_bytes(urlsplit[1], errors='surrogate_or_strict')
b_file_path = tempfile.NamedTemporaryFile(dir=b_path, prefix=b_file_name, suffix=b_file_ext, delete=False).name
display.vvv("Downloading %s to %s" % (url, to_text(b_path)))
# Galaxy redirs downloads to S3 which reject the request if an Authorization header is attached so don't redir that
resp = open_url(to_native(url, errors='surrogate_or_strict'), validate_certs=validate_certs, headers=headers,
unredirected_headers=['Authorization'], http_agent=user_agent())
with open(b_file_path, 'wb') as download_file:
data = resp.read(bufsize)
while data:
digest.update(data)
download_file.write(data)
data = resp.read(bufsize)
if expected_hash:
actual_hash = digest.hexdigest()
display.vvvv("Validating downloaded file hash %s with expected hash %s" % (actual_hash, expected_hash))
if expected_hash != actual_hash:
raise AnsibleError("Mismatch artifact hash with downloaded file")
return b_file_path
def fetch(self, role_data):
"""
Downloads the archived role to a temp location based on role data
"""
if role_data:
# first grab the file and save it to a temp location
if self.download_url is not None:
archive_url = self.download_url
elif "github_user" in role_data and "github_repo" in role_data:
archive_url = 'https://github.com/%s/%s/archive/%s.tar.gz' % (
role_data["github_user"], role_data["github_repo"],
self.version)
else:
archive_url = self.src
display.display("- downloading role from %s" % archive_url)
try:
url_file = open_url(archive_url,
validate_certs=self._validate_certs,
http_agent=user_agent())
temp_file = tempfile.NamedTemporaryFile(delete=False)
data = url_file.read()
while data:
temp_file.write(data)
data = url_file.read()
temp_file.close()
return temp_file.name
except Exception as e:
display.error(u"failed to download the file: %s" % to_text(e))
return False
def get(self):
if self._token:
return self._token
# - build a request to POST to auth_url
# - body is form encoded
# - 'request_token' is the offline token stored in ansible.cfg
# - 'grant_type' is 'refresh_token'
# - 'client_id' is 'cloud-services'
# - should probably be based on the contents of the
# offline_ticket's JWT payload 'aud' (audience)
# or 'azp' (Authorized party - the party to which the ID Token was issued)
payload = self._form_payload()
resp = open_url(to_native(self.auth_url),
data=payload,
validate_certs=self.validate_certs,
method='POST',
http_agent=user_agent())
# TODO: handle auth errors
data = json.loads(to_text(resp.read(), errors='surrogate_or_strict'))
# - extract 'access_token'
self._token = data.get('access_token')
return self._token
def _call_galaxy(self,
url,
args=None,
headers=None,
method=None,
auth_required=False,
error_context_msg=None):
headers = headers or {}
self._add_auth_token(headers, url, required=auth_required)
try:
display.vvvv("Calling Galaxy at %s" % url)
resp = open_url(to_native(url),
data=args,
validate_certs=self.validate_certs,
headers=headers,
method=method,
timeout=20,
http_agent=user_agent())
except HTTPError as e:
raise GalaxyError(e, error_context_msg)
except Exception as e:
raise AnsibleError(
"Unknown error when attempting to call Galaxy at '%s': %s" %
(url, to_native(e)))
resp_data = to_text(resp.read(), errors='surrogate_or_strict')
try:
data = json.loads(resp_data)
except ValueError:
raise AnsibleError(
"Failed to parse Galaxy response from '%s' as JSON:\n%s" %
(resp.url, to_native(resp_data)))
return data
def _call_galaxy(
self,
url,
args=None,
headers=None,
method=None,
auth_required=False,
error_context_msg=None,
):
headers = headers or {}
self._add_auth_token(headers, url, required=auth_required)
retries = 0
while retries < int(os.environ.get('ANSIBLE_CONNECTION_RETRIES',
'20')):
try:
display.vvvv("Calling Galaxy at %s" % url)
resp = open_url(
to_native(url),
data=args,
validate_certs=self.validate_certs,
headers=headers,
method=method,
timeout=int(os.environ.get('ANSIBLE_TIMEOUT', '120')),
http_agent=user_agent(),
follow_redirects='safe',
)
break
except HTTPError as e:
if e.code in (
504,
503,
):
retries += 1
display.vvvv("Calling Galaxy at %s, attempt %s" %
(url, retries))
continue
raise GalaxyError(e, error_context_msg)
except Exception as e:
if 'timed out' in str(e):
retries += 1
display.vvvv(
"Calling Galaxy at %s after read timeout, attempt %s" %
(url, retries))
continue
raise AnsibleError(
"Unknown error when attempting to call Galaxy at '%s': %s"
% (url, to_native(e)))
resp_data = to_text(resp.read(), errors='surrogate_or_strict')
try:
data = json.loads(resp_data)
except ValueError:
raise AnsibleError(
"Failed to parse Galaxy response from '%s' as JSON:\n%s" %
(resp.url, to_native(resp_data)))
return data
def authenticate(self, github_token):
"""
Retrieve an authentication token
"""
url = _urljoin(self.api_server, self.available_api_versions['v1'],
"tokens") + '/'
args = urlencode({"github_token": github_token})
resp = open_url(url,
data=args,
validate_certs=self.validate_certs,
method="POST",
http_agent=user_agent())
data = json.loads(to_text(resp.read(), errors='surrogate_or_strict'))
return data
def get_signature_from_source(source, display=None): # type: (str, t.Optional[Display]) -> str
if display is not None:
display.vvvv(f"Using signature at {source}")
try:
with open_url(
source,
http_agent=user_agent(),
validate_certs=True,
follow_redirects='safe'
) as resp:
signature = resp.read()
except (HTTPError, URLError) as e:
raise AnsibleError(
f"Failed to get signature for collection verification from '{source}': {e}"
) from e
return signature
def _download_file(url,
b_path,
expected_hash,
validate_certs,
token=None,
timeout=60):
# type: (str, bytes, t.Optional[str], bool, GalaxyToken, int) -> bytes
# ^ NOTE: used in download and verify_collections ^
b_tarball_name = to_bytes(
url.rsplit('/', 1)[1],
errors='surrogate_or_strict',
)
b_file_name = b_tarball_name[:-len('.tar.gz')]
b_tarball_dir = mkdtemp(
dir=b_path,
prefix=b'-'.join((b_file_name, b'')),
) # type: bytes
b_file_path = os.path.join(b_tarball_dir, b_tarball_name)
display.display("Downloading %s to %s" % (url, to_text(b_tarball_dir)))
# NOTE: Galaxy redirects downloads to S3 which rejects the request
# NOTE: if an Authorization header is attached so don't redirect it
resp = open_url(to_native(url, errors='surrogate_or_strict'),
validate_certs=validate_certs,
headers=None if token is None else token.headers(),
unredirected_headers=['Authorization'],
http_agent=user_agent(),
timeout=timeout)
with open(b_file_path, 'wb') as download_file: # type: t.BinaryIO
actual_hash = _consume_file(resp, write_to=download_file)
if expected_hash:
display.vvvv('Validating downloaded file hash {actual_hash!s} with '
'expected hash {expected_hash!s}'.format(
actual_hash=actual_hash, expected_hash=expected_hash))
if expected_hash != actual_hash:
raise AnsibleError('Mismatch artifact hash with downloaded file')
return b_file_path
def create_github_token(self):
'''
Create a personal authorization token with a note of 'ansible-galaxy login'
'''
self.remove_github_token()
args = json.dumps({
"scopes": ["public_repo"],
"note": "ansible-galaxy login"
})
try:
data = json.load(
open_url(self.GITHUB_AUTH,
url_username=self.github_username,
url_password=self.github_password,
force_basic_auth=True,
data=args,
validate_certs=self._validate_certs,
http_agent=user_agent()))
except HTTPError as e:
res = json.load(e)
raise AnsibleError(res['message'])
return data['token']
def authenticate(self, github_token):
"""
Retrieve an authentication token
"""
url = _urljoin(self.api_server, self.available_api_versions['v1'],
"tokens") + '/'
args = urlencode({"github_token": github_token})
try:
resp = open_url(url,
data=args,
validate_certs=self.validate_certs,
method="POST",
http_agent=user_agent(),
timeout=self._server_timeout)
except HTTPError as e:
raise GalaxyError(e, 'Attempting to authenticate to galaxy')
except Exception as e:
raise AnsibleError('Unable to authenticate to galaxy: %s' %
to_native(e),
orig_exc=e)
data = json.loads(to_text(resp.read(), errors='surrogate_or_strict'))
return data
def _call_galaxy(self,
url,
args=None,
headers=None,
method=None,
auth_required=False,
error_context_msg=None,
cache=False,
cache_key=None):
url_info = urlparse(url)
cache_id = get_cache_id(url)
if not cache_key:
cache_key = url_info.path
query = parse_qs(url_info.query)
if cache and self._cache:
server_cache = self._cache.setdefault(cache_id, {})
iso_datetime_format = '%Y-%m-%dT%H:%M:%SZ'
valid = False
if cache_key in server_cache:
expires = datetime.datetime.strptime(
server_cache[cache_key]['expires'], iso_datetime_format)
valid = datetime.datetime.utcnow() < expires
is_paginated_url = 'page' in query or 'offset' in query
if valid and not is_paginated_url:
# Got a hit on the cache and we aren't getting a paginated response
path_cache = server_cache[cache_key]
if path_cache.get('paginated'):
if '/v3/' in cache_key:
res = {'links': {'next': None}}
else:
res = {'next': None}
# Technically some v3 paginated APIs return in 'data' but the caller checks the keys for this so
# always returning the cache under results is fine.
res['results'] = []
for result in path_cache['results']:
res['results'].append(result)
else:
res = path_cache['results']
return res
elif not is_paginated_url:
# The cache entry had expired or does not exist, start a new blank entry to be filled later.
expires = datetime.datetime.utcnow()
expires += datetime.timedelta(days=1)
server_cache[cache_key] = {
'expires': expires.strftime(iso_datetime_format),
'paginated': False,
}
headers = headers or {}
self._add_auth_token(headers, url, required=auth_required)
try:
display.vvvv("Calling Galaxy at %s" % url)
resp = open_url(to_native(url),
data=args,
validate_certs=self.validate_certs,
headers=headers,
method=method,
timeout=self._server_timeout,
http_agent=user_agent(),
follow_redirects='safe')
except HTTPError as e:
raise GalaxyError(e, error_context_msg)
except Exception as e:
raise AnsibleError(
"Unknown error when attempting to call Galaxy at '%s': %s" %
(url, to_native(e)))
resp_data = to_text(resp.read(), errors='surrogate_or_strict')
try:
data = json.loads(resp_data)
except ValueError:
raise AnsibleError(
"Failed to parse Galaxy response from '%s' as JSON:\n%s" %
(resp.url, to_native(resp_data)))
if cache and self._cache:
path_cache = self._cache[cache_id][cache_key]
# v3 can return data or results for paginated results. Scan the result so we can determine what to cache.
paginated_key = None
for key in ['data', 'results']:
if key in data:
paginated_key = key
break
if paginated_key:
path_cache['paginated'] = True
results = path_cache.setdefault('results', [])
for result in data[paginated_key]:
results.append(result)
else:
path_cache['results'] = data
return data
def test_user_agent():
res = user_agent.user_agent()
assert res.startswith('ansible-galaxy/%s' % ansible_version)
assert platform.system() in res
assert 'python:' in res
