def remove_github_token(self): ''' If for some reason an ansible-galaxy token was left from a prior login, remove it. We cannot retrieve the token after creation, so we are forced to create a new one. ''' try: tokens = json.load( open_url( self.GITHUB_AUTH, url_username=self.github_username, url_password=self.github_password, force_basic_auth=True, )) except HTTPError as e: res = json.load(e) raise exceptions.GalaxyClientError(res['message']) for token in tokens: if token['note'] == 'ansible-galaxy login': self.log.debug('removing token: %s', token['token_last_eight']) try: open_url('https://api.github.com/authorizations/%d' % token['id'], url_username=self.github_username, url_password=self.github_password, method='DELETE', force_basic_auth=True) except HTTPError as e: self.log.exception(e) res = json.load(e) raise exceptions.GalaxyClientError(res['message'])
def _get_server_api_version(self): """ Fetches the Galaxy API current version to ensure the API server is up and reachable. """ url = '%s/api/' % self._api_server try: return_data = open_url(url, validate_certs=self._validate_certs) except Exception as e: raise exceptions.GalaxyClientError( "Failed to get data from the API server (%s): %s " % (url, to_native(e))) try: data = json.loads( to_text(return_data.read(), errors='surrogate_or_strict')) except Exception as e: raise exceptions.GalaxyClientError( "Could not process data from the API server (%s): %s " % (url, to_native(e))) if 'current_version' not in data: raise exceptions.GalaxyClientError( "missing required 'current_version' from server response (%s)" % url) self.log.debug('Server API version of URL %s is "%s"', url, data['current_version']) return data['current_version']
def remove(self): """ Removes the specified content from the content path. There is a sanity check to make sure there's a meta/main.yml or ansible-galaxy.yml file at this path so the user doesn't blow away random directories. """ log.debug('remove content_type: %s', self.content_type) log.debug('remove metadata: %s', self.metadata) log.debug('remove path: %s', self.path) # FIXME - not yet implemented for non-role types if self.content_type == "role": if self.metadata: try: rmtree(self.path) return True except Exception as e: log.warn('unable to rmtree for path=%s', self.path) log.exception(e) else: raise exceptions.GalaxyClientError("Removing Galaxy Content not yet implemented") return False
def __auth_header(self): token = self.token.get() if token is None: raise exceptions.GalaxyClientError( "No access token. You must first use login to authenticate and obtain an access token." ) return {'Authorization': 'Token ' + token}
def find(self): real_path = os.path.abspath(self.requirement_spec.src) log.debug( 'Searching for a repository to link to as an editable install at %s', real_path) if not os.path.isdir(real_path): log.warning( "%s needs to be a local directory for an editable install" % self.repository.src) raise exceptions.GalaxyClientError( 'Error finding an editable install of %s because %s is not a directory', self.requirement_spec.src, real_path) results = { 'content': { 'galaxy_namespace': self.requirement_spec.namespace, 'repo_name': self.requirement_spec.name }, 'custom': { 'real_path': self.requirement_spec.src } } return results
def __call_galaxy(self, url, args=None, headers=None, method=None): if args and not headers: headers = self.__auth_header() try: http_log.info('%s %s', method, url) request_log.debug('%s %s args=%s', method, url, args) request_log.debug('%s %s headers=%s', method, url, headers) resp = open_url(url, data=args, validate_certs=self._validate_certs, headers=headers, method=method, timeout=20) http_log.info('%s %s http_status=%s', method, url, resp.getcode()) final_url = resp.geturl() if final_url != url: http_log.debug('%s %s Redirected to: %s', method, url, resp.geturl()) resp_info = resp.info() response_log.debug('%s %s info:\n%s', method, url, resp_info) # FIXME: making the request and loading the response should be sep try/except blocks response_body = to_text(resp.read(), errors='surrogate_or_strict') # debug log the raw response body response_log.debug('%s %s response body:\n%s', method, url, response_body) data = json.loads(response_body) # debug log a json version of the data that was created from the response response_log.debug('%s %s data:\n%s', method, url, json.dumps(data, indent=2)) except HTTPError as e: self.log.debug('Exception on %s %s', method, url) self.log.exception(e) # FIXME: probably need a try/except here if the response body isnt json which # can happen if a proxy mangles the response res = json.loads(to_text(e.fp.read(), errors='surrogate_or_strict')) http_log.error('%s %s data from server error response:\n%s', method, url, res) raise exceptions.GalaxyClientError(res['detail']) except (ssl.SSLError, socket.error) as e: self.log.debug( 'Connection error to Galaxy API for request "%s %s": %s', method, url, e) self.log.exception(e) raise exceptions.GalaxyClientAPIConnectionError( 'Connection error to Galaxy API for request "%s %s": %s' % (method, url, e)) return data
def get_list(self, what): """ Fetch the list of items specified. """ self.log.debug('what=%s', what) try: url = '%s/%s/?page_size' % (self.baseurl, what) data = self.__call_galaxy(url) if "results" in data: results = data['results'] else: results = data done = True if "next" in data: done = (data.get('next_link', None) is None) while not done: url = '%s%s' % (self._api_server, data['next_link']) data = self.__call_galaxy(url) results += data['results'] done = (data.get('next_link', None) is None) return results except Exception as error: self.log.exception(error) raise exceptions.GalaxyClientError( "Failed to download the %s list: %s" % (what, str(error)))
def __call_galaxy(self, url, args=None, headers=None, method=None): if args and not headers: headers = self.__auth_header() try: # self.log.info('%s %s', method, url) # self.log.debug('%s %s args=%s', method, url, args) # self.log.debug('%s %s headers=%s', method, url, headers) resp = open_url(url, data=args, validate_certs=self._validate_certs, headers=headers, method=method, timeout=20) self.log.debug('%s %s http_status=%s', method, url, resp.getcode()) final_url = resp.geturl() if final_url != url: self.log.debug('%s %s Redirected to: %s', method, url, resp.geturl()) # self.log.debug('%s %s info:\n%s', method, url, resp.info()) data = json.loads( to_text(resp.read(), errors='surrogate_or_strict')) # self.log.debug('%s %s data: \n%s', method, url, json.dumps(data, indent=2)) except HTTPError as e: self.log.debug('Exception on %s %s', method, url) self.log.exception(e) res = json.loads(to_text(e.fp.read(), errors='surrogate_or_strict')) raise exceptions.GalaxyClientError(res['detail']) return data
def lookup_role_by_name(self, role_name, notify=True): """ Find a role by name. """ role_name = urlquote(role_name) try: parts = role_name.split(".") user_name = ".".join(parts[0:-1]) role_name = parts[-1] if notify: self.log.info("- downloading role '%s', owned by %s", role_name, user_name) except Exception as e: self.log.exception(e) raise exceptions.GalaxyClientError( "Invalid role name (%s). Specify role as format: username.rolename" % role_name) url = '%s/roles/?owner__username=%s&name=%s' % (self.baseurl, user_name, role_name) data = self.__call_galaxy(url) if len(data["results"]) != 0: return data["results"][0] return None
def fetch(self, find_results=None): find_results = find_results or {} real_path = find_results.get('custom', {}).get('real_path', None) if not real_path: raise exceptions.GalaxyClientError('Error fetching an editable install of %s because no "real_path" was found in find_results', self.repository_spec.src, real_path) dst_ns_root = os.path.join(self.galaxy_context.content_path, self.repository_spec.namespace) dst_repo_root = os.path.join(dst_ns_root, self.repository_spec.name) if not os.path.exists(dst_ns_root): os.makedirs(dst_ns_root) if not os.path.exists(dst_repo_root): os.symlink(real_path, dst_repo_root) repository_archive_path = self.local_path log.debug('repository_archive_path=%s (inplace) synlink to %s', repository_archive_path, real_path) results = {'archive_path': repository_archive_path, 'fetch_method': self.fetch_method} results['custom'] = {'local_path': self.local_path, 'real_path': real_path, 'symlinked_repo_root': dst_repo_root} results['content'] = find_results['content'] return results
def split_kwarg(spec_string, valid_keywords): if '=' not in spec_string: return (None, spec_string) parts = spec_string.split('=', 1) if parts[0] in valid_keywords: return (parts[0], parts[1]) raise exceptions.GalaxyClientError( 'The repository spec uses an unsuppoted keyword: %s' % spec_string)
def load(data_or_file_object, klass=None): data_dict = yaml.safe_load(data_or_file_object) try: collection_info = CollectionInfo(**data_dict) except ValueError: raise except Exception as exc: raise exceptions.GalaxyClientError( "Error parsing collection metadata: %s" % str(exc)) return collection_info
def load_collections_lockfile(lockfile_path): try: log.debug('Opening the collections lockfile %s', lockfile_path) with open(lockfile_path, 'r') as lffd: return collections_lockfile.load(lffd) except EnvironmentError as exc: log.exception(exc) msg = 'Error opening the collections lockfile "%s": %s' % (lockfile_path, exc) log.error(msg) raise exceptions.GalaxyClientError(msg)
def _get_server_api_version(self): """ Fetches the Galaxy API current version to ensure the API server is up and reachable. """ url = '%s/api/' % self._api_server try: resp = self.session.get(url, verify=self._validate_certs) except requests.exceptions.RequestException as e: raise exceptions.GalaxyClientError( "Failed to get data from the API server (%s): %s " % (url, to_native(e))) try: # data = json.loads(to_text(return_data.read(), errors='surrogate_or_strict')) data = resp.json() except Exception as e: raise exceptions.GalaxyClientError( "Could not process data from the API server (%s): %s " % (url, to_native(e))) # Don't raise connection indicating errors unless we dont have valid error json try: resp.raise_for_status() except Exception as e: raise exceptions.GalaxyClientError( "Failed to get data from the API server (%s): %s " % (url, to_native(e))) if 'current_version' not in data: raise exceptions.GalaxyClientError( "missing required 'current_version' from server response (%s)" % url) self.log.debug('Server API version of URL %s is "%s"', url, data['current_version']) return data['current_version']
def wrapped(self, *args, **kwargs): if not self.initialized: log.debug("Initial connection to galaxy_server: %s", self._api_server) server_version = self._get_server_api_version() if server_version not in self.SUPPORTED_VERSIONS: raise exceptions.GalaxyClientError( "Unsupported Galaxy server API version: %s" % server_version) self.initialized = True return method(self, *args, **kwargs)
def load(data_or_file_object): data_dict = yaml.safe_load(data_or_file_object) log.debug('data_dict: %s', data_dict) try: collections_lockfile = CollectionsLockfile(dependencies=data_dict) except ValueError: raise except Exception as exc: log.exception(exc) raise exceptions.GalaxyClientError( "Error parsing collections lockfile: %s" % str(exc)) return collections_lockfile
def install_deps_by_galaxy_metadata(content_deps, parent_content_meta, display_callback=None): installed = [] for content_dep in content_deps: if 'src' not in content_dep: raise exceptions.GalaxyClientError("ansible-galaxy.yml dependencies must provide a src") res = install_dep_by_galaxy_metadata(content_dep, parent_content_meta=parent_content_meta, display_callback=display_callback) installed.extend(res) return installed
def extract_file(tar_file, file_to_extract): # TODO: should just be a object? ContentArchiveMember? ContentArchiveExtractData ? archive_member = file_to_extract['archive_member'] dest_dir = file_to_extract['dest_dir'] dest_filename = file_to_extract['dest_filename'] force_overwrite = file_to_extract['force_overwrite'] orig_name = archive_member.name if not archive_member.isreg() and not archive_member.issym(): return None # TODO: raise from up a level in the stack? dest_path = os.path.join(dest_dir, dest_filename) # log.debug('dest_dir: %s, dest_filename: %s, dest_path: %s orig_name: %s', # dest_dir, dest_filename, dest_path, orig_name) if os.path.exists(dest_path): if not force_overwrite: message = "The Galaxy content %s appears to already exist." % dest_path raise exceptions.GalaxyClientError(message) try: tar_file.getmember(archive_member.name) except KeyError: raise exceptions.GalaxyArchiveError( 'The archive "%s" has no file "%s"' % (tar_file.name, archive_member.name), archive_path=tar_file.name) # TODO: set a default owner/group # MAYBE TODO: pick a 'install time' and make sure all the mtime/ctime values of extracted files # match the 'install time' used in .galaxy_install_info ? # change the tar file member name in place to just the filename ('myfoo.py') so that extract places that file in # dest_dir directly instead of using adding the archive path as well # like '$dest_dir/archive-roles/library/myfoo.py' archive_member.name = dest_filename # log.debug('tar member: %s dest_dir: %s', archive_member, dest_dir) tar_file.extract(archive_member, dest_dir) installed_path = os.path.join(dest_dir, dest_filename) # reset the tar info object's name attr to the origin value in # case something else references this archive_member.name = orig_name return installed_path
def parse_repository_name(name): "split a full repository_name into namespace, repo_name, content_name" repo_name = None try: parts = name.split(".") user_name = parts[0] if len(parts) > 2: repo_name = parts[1] name = '.'.join(parts[2:]) else: name = '.'.join(parts[1:]) except Exception as e: log.exception(e) raise exceptions.GalaxyClientError("Invalid content name (%s). Specify content as format: username.contentname" % name) return (user_name, repo_name, name)
def load_archive(archive_path): archive_parent_dir = None if not tarfile.is_tarfile(archive_path): raise exceptions.GalaxyClientError( "the file downloaded was not a tar.gz") if archive_path.endswith('.gz'): content_tar_file = tarfile.open(archive_path, "r:gz") else: content_tar_file = tarfile.open(archive_path, "r") members = content_tar_file.getmembers() archive_parent_dir = members[0].name archive_type = detect_content_archive_type(archive_path, members) log.debug('archive_type: %s', archive_type) # log.debug('self.content_type: %s', self.content_type) # if not archive_parent_dir: # archive_parent_dir = archive.find_archive_parent_dir(members, # content_type=content_meta.content_type, # content_dir=content_meta.content_dir) log.debug('archive_type: %s', archive_type) log.debug("archive_parent_dir: %s", archive_parent_dir) # looks like we are a role, update the default content_type from all -> role if archive_type == 'role': # Look for top level role metadata # archive_role_metadata = \ # archive.load_archive_role_metadata(content_tar_file, # os.path.join(archive_parent_dir, archive.META_MAIN)) log.debug( 'Found role metadata in the archive, so installing it as role content_type' ) archive_meta = content_archive.ContentArchiveMeta( top_dir=archive_parent_dir, archive_type=archive_type, archive_path=archive_path) log.debug('role archive_meta: %s', archive_meta) return content_tar_file, archive_meta
def load(data_or_file_object, klass=None): log.debug('loading collection info from %s', pf(data_or_file_object)) data_dict = yaml.safe_load(data_or_file_object) log.debug('data: %s', pf(data_dict)) try: collection_info = CollectionInfo(**data_dict) except ValueError: raise except Exception as exc: raise exceptions.GalaxyClientError( "Error parsing collection metadata: %s" % str(exc)) log.debug('artifact_manifest from_kwargs: %s', collection_info) return collection_info
def choose_content_fetch_method(scm_url=None, src=None): log.debug('scm_url=%s, src=%s', scm_url, src) if scm_url: # create tar file from scm url return FetchMethods.SCM_URL if not src: raise exceptions.GalaxyClientError("No valid content data found") if os.path.isfile(src): # installing a local tar.gz return FetchMethods.LOCAL_FILE if '://' in src: return FetchMethods.REMOTE_URL # if it doesnt look like anything else, assume it's galaxy return FetchMethods.GALAXY_URL
def get_import_task(self, task_id=None, github_user=None, github_repo=None): """ Check the status of an import task. """ url = '%s/imports/' % self.baseurl if task_id is not None: url = "%s?id=%d" % (url, task_id) elif github_user is not None and github_repo is not None: url = "%s?github_user=%s&github_repo=%s" % (url, github_user, github_repo) else: raise exceptions.GalaxyClientError( "Expected task_id or github_user and github_repo") data = self.__call_galaxy(url) return data['results']
def _get_server_api_version(self): """ Fetches the Galaxy API current version to ensure the API server is up and reachable. """ url = '%s/api/' % self._api_server # Call without the @g_connect wrapper to avoid recursion data = self._get_object(href=url) if 'current_version' not in data: # The Galaxy API info at http://bogus.invalid:9443/api/ is missing the required 'current_version' field and the API version could not be determined. error_msg = "The Galaxy API version could not be determined. The required 'current_version' field is missing from info at %s" % url raise exceptions.GalaxyClientError(error_msg) self.log.debug('Server API version of URL %s is "%s"', url, data['current_version']) return data['current_version']
def create_github_token(self): ''' Create a personal authorization token with a note of 'ansible-galaxy login' ''' self.remove_github_token() args = json.dumps({ "scopes": ["public_repo"], "note": "ansible-galaxy login" }) try: data = json.load( open_url(self.GITHUB_AUTH, url_username=self.github_username, url_password=self.github_password, force_basic_auth=True, data=args)) except HTTPError as e: self.log.exception(e) res = json.load(e) raise exceptions.GalaxyClientError(res['message']) return data['token']
def load_tarfile_archive_info(archive_path, repository_spec): # if not tarfile.is_tarfile(archive_path): # raise exceptions.GalaxyClientError("the file downloaded was not a tar.gz") if archive_path.endswith('.gz'): tar_flags = "r:gz" else: tar_flags = "r" try: repository_tar_file = tarfile.open(archive_path, tar_flags) except tarfile.TarError as e: log.exception(e) raise exceptions.GalaxyClientError( "Error opening the tar file %s with flags: %s for repo: %s" % (archive_path, tar_flags, repository_spec)) members = repository_tar_file.getmembers() archive_info = build_archive_info(archive_path, [m.name for m in members]) return archive_info, repository_tar_file
def find_archive_parent_dir(archive_members, content_meta): # archive_parent_dir wasn't found when checking for metadata files archive_parent_dir = None shortest_dir = None for member in archive_members: # This is either a new-type Galaxy Content that doesn't have an # ansible-galaxy.yml file and the type desired is specified and # we check parent dir based on the correct subdir existing or # we need to just scan the subdirs heuristically and figure out # what to do member_dir = os.path.dirname(os.path.dirname(member.name)) shortest_dir = shortest_dir or member_dir if len(member_dir) < len(shortest_dir): shortest_dir = member_dir if content_meta.content_type != "all": if content_meta.content_dir and content_meta.content_dir in member.name: archive_parent_dir = os.path.dirname(member.name) return archive_parent_dir else: for plugin_dir in CONTENT_TYPE_DIR_MAP.values(): if plugin_dir in member.name: archive_parent_dir = os.path.dirname(member.name) return archive_parent_dir if content_meta.content_type not in CONTENT_TYPES: log.debug( 'did not find a content_dir or plugin_dir, so using shortest_dir %s for archive_parent_dir', shortest_dir) return shortest_dir # TODO: archive format exception? msg = "No content metadata provided, nor content directories found for content_type: %s" % \ content_meta.content_type log.debug('content_meta: %s', content_meta) raise exceptions.GalaxyClientError(msg)
def get_latest_version(available_normalized_versions, content_data): # and sort them to get the latest version. If there # are no versions in the list, we'll grab the head # of the master branch if available_normalized_versions: try: sorted_versions = sort_versions(available_normalized_versions) except (TypeError, ValueError) as e: log.exception(e) raise exceptions.GalaxyClientError( 'Unable to compare content versions (%s) to determine the most recent version due to incompatible version formats. ' 'Please contact the content author to resolve versioning conflicts, or specify an explicit content version to install. ' 'The version error was: "%s"' % (', '.join(available_normalized_versions), e)) content_version = sorted_versions[-1] # FIXME: follow 'repository' branch and it's ['import_branch'] ? elif content_data.get('github_branch', None): content_version = content_data['github_branch'] else: content_version = 'master' return content_version
def get_credentials(self): # FIXME(alikins) replace with a display callback print(u'\n\n' + "We need your " + 'Github login' + " to identify you.") print("This information will " + "not be sent to Galaxy" + ", only to " + "api.github.com.") print("The password will not be displayed." + u'\n\n') print("Use " + "--github-token" + " if you do not want to enter your password." + u'\n\n') try: self.github_username = six.input("Github Username: "******"Password for %s: " % self.github_username) except: pass if not self.github_username or not self.github_password: raise exceptions.GalaxyClientError( "Invalid Github credentials. Username and password are required." )
def __call_galaxy(self, url, args=None, headers=None, http_method=None): http_method = http_method or 'GET' headers = headers or {} request_id = uuid.uuid4().hex headers['X-Request-ID'] = request_id # The slug we use to identify a request by method, url and request id # For ex, '"GET https://galaxy.ansible.com/api/v1/repositories" c48937f4e8e849828772c4a0ce0fd5ed' request_slug = '"%s %s" %s' % (http_method, url, request_id) try: # log the http request_slug with request_id to the main log and # to the http log, both at INFO level for now. http_log.info('%s', request_slug) self.log.info('%s', request_slug) request_log.debug('%s args=%s', request_slug, args) request_log.debug('%s headers=%s', request_slug, headers) resp = open_url(url, data=args, validate_certs=self._validate_certs, headers=headers, method=http_method, http_agent=self.user_agent, timeout=20) response_log.info('%s http_status=%s', request_slug, resp.getcode()) final_url = resp.geturl() if final_url != url: request_log.debug('%s Redirected to: %s', request_slug, resp.geturl()) resp_info = resp.info() response_log.debug('%s info:\n%s', request_slug, resp_info) # FIXME: making the request and loading the response should be sep try/except blocks response_body = to_text(resp.read(), errors='surrogate_or_strict') # debug log the raw response body response_log.debug('%s response body:\n%s', request_slug, response_body) data = json.loads(response_body) # debug log a json version of the data that was created from the response response_log.debug('%s data:\n%s', request_slug, json.dumps(data, indent=2)) except HTTPError as http_exc: self.log.debug('Exception on %s', request_slug) self.log.exception("%s: %s", request_slug, http_exc) # FIXME: probably need a try/except here if the response body isnt json which # can happen if a proxy mangles the response res = json.loads( to_text(http_exc.fp.read(), errors='surrogate_or_strict')) http_log.error('%s data from server error response:\n%s', request_slug, res) try: error_msg = 'HTTP error on request %s: %s' % (request_slug, res['detail']) raise exceptions.GalaxyClientError(error_msg) except (KeyError, TypeError) as detail_parse_exc: self.log.exception("%s: %s", request_slug, detail_parse_exc) self.log.warning( 'Unable to parse error detail from response for request: %s response: %s', request_slug, detail_parse_exc) # TODO: great place to be able to use 'raise from' # FIXME: this needs to be tweaked so the raise exceptions.GalaxyClientError(http_exc) except (ssl.SSLError, socket.error) as e: self.log.debug('Connection error to Galaxy API for request %s: %s', request_slug, e) self.log.exception("%s: %s", request_slug, e) raise exceptions.GalaxyClientAPIConnectionError( 'Connection error to Galaxy API for request %s: %s' % (request_slug, e)) return data