def remove_github_token(self): ''' If for some reason an ansible-galaxy token was left from a prior login, remove it. We cannot retrieve the token after creation, so we are forced to create a new one. ''' try: tokens = json.load( open_url(self.GITHUB_AUTH, url_username=self.github_username, url_password=self.github_password, force_basic_auth=True, validate_certs=self._validate_certs, http_agent=user_agent())) except HTTPError as e: res = json.load(e) raise AnsibleError(res['message']) for token in tokens: if token['note'] == 'ansible-galaxy login': display.vvvvv('removing token: %s' % token['token_last_eight']) try: open_url('https://api.github.com/authorizations/%d' % token['id'], url_username=self.github_username, url_password=self.github_password, method='DELETE', force_basic_auth=True, validate_certs=self._validate_certs, http_agent=user_agent()) except HTTPError as e: res = json.load(e) raise AnsibleError(res['message'])
def _download_file(url, b_path, expected_hash, validate_certs, headers=None): bufsize = 65536 digest = sha256() urlsplit = os.path.splitext(to_text(url.rsplit('/', 1)[1])) b_file_name = to_bytes(urlsplit[0], errors='surrogate_or_strict') b_file_ext = to_bytes(urlsplit[1], errors='surrogate_or_strict') b_file_path = tempfile.NamedTemporaryFile(dir=b_path, prefix=b_file_name, suffix=b_file_ext, delete=False).name display.vvv("Downloading %s to %s" % (url, to_text(b_path))) # Galaxy redirs downloads to S3 which reject the request if an Authorization header is attached so don't redir that resp = open_url(to_native(url, errors='surrogate_or_strict'), validate_certs=validate_certs, headers=headers, unredirected_headers=['Authorization'], http_agent=user_agent()) with open(b_file_path, 'wb') as download_file: data = resp.read(bufsize) while data: digest.update(data) download_file.write(data) data = resp.read(bufsize) if expected_hash: actual_hash = digest.hexdigest() display.vvvv("Validating downloaded file hash %s with expected hash %s" % (actual_hash, expected_hash)) if expected_hash != actual_hash: raise AnsibleError("Mismatch artifact hash with downloaded file") return b_file_path
def fetch(self, role_data): """ Downloads the archived role to a temp location based on role data """ if role_data: # first grab the file and save it to a temp location if self.download_url is not None: archive_url = self.download_url elif "github_user" in role_data and "github_repo" in role_data: archive_url = 'https://github.com/%s/%s/archive/%s.tar.gz' % ( role_data["github_user"], role_data["github_repo"], self.version) else: archive_url = self.src display.display("- downloading role from %s" % archive_url) try: url_file = open_url(archive_url, validate_certs=self._validate_certs, http_agent=user_agent()) temp_file = tempfile.NamedTemporaryFile(delete=False) data = url_file.read() while data: temp_file.write(data) data = url_file.read() temp_file.close() return temp_file.name except Exception as e: display.error(u"failed to download the file: %s" % to_text(e)) return False
def get(self): if self._token: return self._token # - build a request to POST to auth_url # - body is form encoded # - 'request_token' is the offline token stored in ansible.cfg # - 'grant_type' is 'refresh_token' # - 'client_id' is 'cloud-services' # - should probably be based on the contents of the # offline_ticket's JWT payload 'aud' (audience) # or 'azp' (Authorized party - the party to which the ID Token was issued) payload = self._form_payload() resp = open_url(to_native(self.auth_url), data=payload, validate_certs=self.validate_certs, method='POST', http_agent=user_agent()) # TODO: handle auth errors data = json.loads(to_text(resp.read(), errors='surrogate_or_strict')) # - extract 'access_token' self._token = data.get('access_token') return self._token
def _call_galaxy(self, url, args=None, headers=None, method=None, auth_required=False, error_context_msg=None): headers = headers or {} self._add_auth_token(headers, url, required=auth_required) try: display.vvvv("Calling Galaxy at %s" % url) resp = open_url(to_native(url), data=args, validate_certs=self.validate_certs, headers=headers, method=method, timeout=20, http_agent=user_agent()) except HTTPError as e: raise GalaxyError(e, error_context_msg) except Exception as e: raise AnsibleError( "Unknown error when attempting to call Galaxy at '%s': %s" % (url, to_native(e))) resp_data = to_text(resp.read(), errors='surrogate_or_strict') try: data = json.loads(resp_data) except ValueError: raise AnsibleError( "Failed to parse Galaxy response from '%s' as JSON:\n%s" % (resp.url, to_native(resp_data))) return data
def _call_galaxy( self, url, args=None, headers=None, method=None, auth_required=False, error_context_msg=None, ): headers = headers or {} self._add_auth_token(headers, url, required=auth_required) retries = 0 while retries < int(os.environ.get('ANSIBLE_CONNECTION_RETRIES', '20')): try: display.vvvv("Calling Galaxy at %s" % url) resp = open_url( to_native(url), data=args, validate_certs=self.validate_certs, headers=headers, method=method, timeout=int(os.environ.get('ANSIBLE_TIMEOUT', '120')), http_agent=user_agent(), follow_redirects='safe', ) break except HTTPError as e: if e.code in ( 504, 503, ): retries += 1 display.vvvv("Calling Galaxy at %s, attempt %s" % (url, retries)) continue raise GalaxyError(e, error_context_msg) except Exception as e: if 'timed out' in str(e): retries += 1 display.vvvv( "Calling Galaxy at %s after read timeout, attempt %s" % (url, retries)) continue raise AnsibleError( "Unknown error when attempting to call Galaxy at '%s': %s" % (url, to_native(e))) resp_data = to_text(resp.read(), errors='surrogate_or_strict') try: data = json.loads(resp_data) except ValueError: raise AnsibleError( "Failed to parse Galaxy response from '%s' as JSON:\n%s" % (resp.url, to_native(resp_data))) return data
def authenticate(self, github_token): """ Retrieve an authentication token """ url = _urljoin(self.api_server, self.available_api_versions['v1'], "tokens") + '/' args = urlencode({"github_token": github_token}) resp = open_url(url, data=args, validate_certs=self.validate_certs, method="POST", http_agent=user_agent()) data = json.loads(to_text(resp.read(), errors='surrogate_or_strict')) return data
def get_signature_from_source(source, display=None): # type: (str, t.Optional[Display]) -> str if display is not None: display.vvvv(f"Using signature at {source}") try: with open_url( source, http_agent=user_agent(), validate_certs=True, follow_redirects='safe' ) as resp: signature = resp.read() except (HTTPError, URLError) as e: raise AnsibleError( f"Failed to get signature for collection verification from '{source}': {e}" ) from e return signature
def _download_file(url, b_path, expected_hash, validate_certs, token=None, timeout=60): # type: (str, bytes, t.Optional[str], bool, GalaxyToken, int) -> bytes # ^ NOTE: used in download and verify_collections ^ b_tarball_name = to_bytes( url.rsplit('/', 1)[1], errors='surrogate_or_strict', ) b_file_name = b_tarball_name[:-len('.tar.gz')] b_tarball_dir = mkdtemp( dir=b_path, prefix=b'-'.join((b_file_name, b'')), ) # type: bytes b_file_path = os.path.join(b_tarball_dir, b_tarball_name) display.display("Downloading %s to %s" % (url, to_text(b_tarball_dir))) # NOTE: Galaxy redirects downloads to S3 which rejects the request # NOTE: if an Authorization header is attached so don't redirect it resp = open_url(to_native(url, errors='surrogate_or_strict'), validate_certs=validate_certs, headers=None if token is None else token.headers(), unredirected_headers=['Authorization'], http_agent=user_agent(), timeout=timeout) with open(b_file_path, 'wb') as download_file: # type: t.BinaryIO actual_hash = _consume_file(resp, write_to=download_file) if expected_hash: display.vvvv('Validating downloaded file hash {actual_hash!s} with ' 'expected hash {expected_hash!s}'.format( actual_hash=actual_hash, expected_hash=expected_hash)) if expected_hash != actual_hash: raise AnsibleError('Mismatch artifact hash with downloaded file') return b_file_path
def create_github_token(self): ''' Create a personal authorization token with a note of 'ansible-galaxy login' ''' self.remove_github_token() args = json.dumps({ "scopes": ["public_repo"], "note": "ansible-galaxy login" }) try: data = json.load( open_url(self.GITHUB_AUTH, url_username=self.github_username, url_password=self.github_password, force_basic_auth=True, data=args, validate_certs=self._validate_certs, http_agent=user_agent())) except HTTPError as e: res = json.load(e) raise AnsibleError(res['message']) return data['token']
def authenticate(self, github_token): """ Retrieve an authentication token """ url = _urljoin(self.api_server, self.available_api_versions['v1'], "tokens") + '/' args = urlencode({"github_token": github_token}) try: resp = open_url(url, data=args, validate_certs=self.validate_certs, method="POST", http_agent=user_agent(), timeout=self._server_timeout) except HTTPError as e: raise GalaxyError(e, 'Attempting to authenticate to galaxy') except Exception as e: raise AnsibleError('Unable to authenticate to galaxy: %s' % to_native(e), orig_exc=e) data = json.loads(to_text(resp.read(), errors='surrogate_or_strict')) return data
def _call_galaxy(self, url, args=None, headers=None, method=None, auth_required=False, error_context_msg=None, cache=False, cache_key=None): url_info = urlparse(url) cache_id = get_cache_id(url) if not cache_key: cache_key = url_info.path query = parse_qs(url_info.query) if cache and self._cache: server_cache = self._cache.setdefault(cache_id, {}) iso_datetime_format = '%Y-%m-%dT%H:%M:%SZ' valid = False if cache_key in server_cache: expires = datetime.datetime.strptime( server_cache[cache_key]['expires'], iso_datetime_format) valid = datetime.datetime.utcnow() < expires is_paginated_url = 'page' in query or 'offset' in query if valid and not is_paginated_url: # Got a hit on the cache and we aren't getting a paginated response path_cache = server_cache[cache_key] if path_cache.get('paginated'): if '/v3/' in cache_key: res = {'links': {'next': None}} else: res = {'next': None} # Technically some v3 paginated APIs return in 'data' but the caller checks the keys for this so # always returning the cache under results is fine. res['results'] = [] for result in path_cache['results']: res['results'].append(result) else: res = path_cache['results'] return res elif not is_paginated_url: # The cache entry had expired or does not exist, start a new blank entry to be filled later. expires = datetime.datetime.utcnow() expires += datetime.timedelta(days=1) server_cache[cache_key] = { 'expires': expires.strftime(iso_datetime_format), 'paginated': False, } headers = headers or {} self._add_auth_token(headers, url, required=auth_required) try: display.vvvv("Calling Galaxy at %s" % url) resp = open_url(to_native(url), data=args, validate_certs=self.validate_certs, headers=headers, method=method, timeout=self._server_timeout, http_agent=user_agent(), follow_redirects='safe') except HTTPError as e: raise GalaxyError(e, error_context_msg) except Exception as e: raise AnsibleError( "Unknown error when attempting to call Galaxy at '%s': %s" % (url, to_native(e))) resp_data = to_text(resp.read(), errors='surrogate_or_strict') try: data = json.loads(resp_data) except ValueError: raise AnsibleError( "Failed to parse Galaxy response from '%s' as JSON:\n%s" % (resp.url, to_native(resp_data))) if cache and self._cache: path_cache = self._cache[cache_id][cache_key] # v3 can return data or results for paginated results. Scan the result so we can determine what to cache. paginated_key = None for key in ['data', 'results']: if key in data: paginated_key = key break if paginated_key: path_cache['paginated'] = True results = path_cache.setdefault('results', []) for result in data[paginated_key]: results.append(result) else: path_cache['results'] = data return data
def test_user_agent(): res = user_agent.user_agent() assert res.startswith('ansible-galaxy/%s' % ansible_version) assert platform.system() in res assert 'python:' in res