def remove_github_token(self):
'''
If for some reason an ansible-galaxy token was left from a prior login, remove it. We cannot
retrieve the token after creation, so we are forced to create a new one.
'''
try:
tokens = json.load(
open_url(
self.GITHUB_AUTH,
url_username=self.github_username,
url_password=self.github_password,
force_basic_auth=True,
))
except HTTPError as e:
res = json.load(e)
raise exceptions.GalaxyClientError(res['message'])
for token in tokens:
if token['note'] == 'ansible-galaxy login':
self.log.debug('removing token: %s', token['token_last_eight'])
try:
open_url('https://api.github.com/authorizations/%d' %
token['id'],
url_username=self.github_username,
url_password=self.github_password,
method='DELETE',
force_basic_auth=True)
except HTTPError as e:
self.log.exception(e)
res = json.load(e)
raise exceptions.GalaxyClientError(res['message'])
def _get_server_api_version(self):
"""
Fetches the Galaxy API current version to ensure
the API server is up and reachable.
"""
url = '%s/api/' % self._api_server
try:
return_data = open_url(url, validate_certs=self._validate_certs)
except Exception as e:
raise exceptions.GalaxyClientError(
"Failed to get data from the API server (%s): %s " %
(url, to_native(e)))
try:
data = json.loads(
to_text(return_data.read(), errors='surrogate_or_strict'))
except Exception as e:
raise exceptions.GalaxyClientError(
"Could not process data from the API server (%s): %s " %
(url, to_native(e)))
if 'current_version' not in data:
raise exceptions.GalaxyClientError(
"missing required 'current_version' from server response (%s)"
% url)
self.log.debug('Server API version of URL %s is "%s"', url,
data['current_version'])
return data['current_version']
def remove(self):
"""
Removes the specified content from the content path.
There is a sanity check to make sure there's a meta/main.yml or
ansible-galaxy.yml file at this path so the user doesn't blow away
random directories.
"""
log.debug('remove content_type: %s', self.content_type)
log.debug('remove metadata: %s', self.metadata)
log.debug('remove path: %s', self.path)
# FIXME - not yet implemented for non-role types
if self.content_type == "role":
if self.metadata:
try:
rmtree(self.path)
return True
except Exception as e:
log.warn('unable to rmtree for path=%s', self.path)
log.exception(e)
else:
raise exceptions.GalaxyClientError("Removing Galaxy Content not yet implemented")
return False
def __auth_header(self):
token = self.token.get()
if token is None:
raise exceptions.GalaxyClientError(
"No access token. You must first use login to authenticate and obtain an access token."
)
return {'Authorization': 'Token ' + token}
def find(self):
real_path = os.path.abspath(self.requirement_spec.src)
log.debug(
'Searching for a repository to link to as an editable install at %s',
real_path)
if not os.path.isdir(real_path):
log.warning(
"%s needs to be a local directory for an editable install" %
self.repository.src)
raise exceptions.GalaxyClientError(
'Error finding an editable install of %s because %s is not a directory',
self.requirement_spec.src, real_path)
results = {
'content': {
'galaxy_namespace': self.requirement_spec.namespace,
'repo_name': self.requirement_spec.name
},
'custom': {
'real_path': self.requirement_spec.src
}
}
return results
def __call_galaxy(self, url, args=None, headers=None, method=None):
if args and not headers:
headers = self.__auth_header()
try:
http_log.info('%s %s', method, url)
request_log.debug('%s %s args=%s', method, url, args)
request_log.debug('%s %s headers=%s', method, url, headers)
resp = open_url(url,
data=args,
validate_certs=self._validate_certs,
headers=headers,
method=method,
timeout=20)
http_log.info('%s %s http_status=%s', method, url, resp.getcode())
final_url = resp.geturl()
if final_url != url:
http_log.debug('%s %s Redirected to: %s', method, url,
resp.geturl())
resp_info = resp.info()
response_log.debug('%s %s info:\n%s', method, url, resp_info)
# FIXME: making the request and loading the response should be sep try/except blocks
response_body = to_text(resp.read(), errors='surrogate_or_strict')
# debug log the raw response body
response_log.debug('%s %s response body:\n%s', method, url,
response_body)
data = json.loads(response_body)
# debug log a json version of the data that was created from the response
response_log.debug('%s %s data:\n%s', method, url,
json.dumps(data, indent=2))
except HTTPError as e:
self.log.debug('Exception on %s %s', method, url)
self.log.exception(e)
# FIXME: probably need a try/except here if the response body isnt json which
# can happen if a proxy mangles the response
res = json.loads(to_text(e.fp.read(),
errors='surrogate_or_strict'))
http_log.error('%s %s data from server error response:\n%s',
method, url, res)
raise exceptions.GalaxyClientError(res['detail'])
except (ssl.SSLError, socket.error) as e:
self.log.debug(
'Connection error to Galaxy API for request "%s %s": %s',
method, url, e)
self.log.exception(e)
raise exceptions.GalaxyClientAPIConnectionError(
'Connection error to Galaxy API for request "%s %s": %s' %
(method, url, e))
return data
def get_list(self, what):
"""
Fetch the list of items specified.
"""
self.log.debug('what=%s', what)
try:
url = '%s/%s/?page_size' % (self.baseurl, what)
data = self.__call_galaxy(url)
if "results" in data:
results = data['results']
else:
results = data
done = True
if "next" in data:
done = (data.get('next_link', None) is None)
while not done:
url = '%s%s' % (self._api_server, data['next_link'])
data = self.__call_galaxy(url)
results += data['results']
done = (data.get('next_link', None) is None)
return results
except Exception as error:
self.log.exception(error)
raise exceptions.GalaxyClientError(
"Failed to download the %s list: %s" % (what, str(error)))
def __call_galaxy(self, url, args=None, headers=None, method=None):
if args and not headers:
headers = self.__auth_header()
try:
# self.log.info('%s %s', method, url)
# self.log.debug('%s %s args=%s', method, url, args)
# self.log.debug('%s %s headers=%s', method, url, headers)
resp = open_url(url,
data=args,
validate_certs=self._validate_certs,
headers=headers,
method=method,
timeout=20)
self.log.debug('%s %s http_status=%s', method, url, resp.getcode())
final_url = resp.geturl()
if final_url != url:
self.log.debug('%s %s Redirected to: %s', method, url,
resp.geturl())
# self.log.debug('%s %s info:\n%s', method, url, resp.info())
data = json.loads(
to_text(resp.read(), errors='surrogate_or_strict'))
# self.log.debug('%s %s data: \n%s', method, url, json.dumps(data, indent=2))
except HTTPError as e:
self.log.debug('Exception on %s %s', method, url)
self.log.exception(e)
res = json.loads(to_text(e.fp.read(),
errors='surrogate_or_strict'))
raise exceptions.GalaxyClientError(res['detail'])
return data
def lookup_role_by_name(self, role_name, notify=True):
"""
Find a role by name.
"""
role_name = urlquote(role_name)
try:
parts = role_name.split(".")
user_name = ".".join(parts[0:-1])
role_name = parts[-1]
if notify:
self.log.info("- downloading role '%s', owned by %s",
role_name, user_name)
except Exception as e:
self.log.exception(e)
raise exceptions.GalaxyClientError(
"Invalid role name (%s). Specify role as format: username.rolename"
% role_name)
url = '%s/roles/?owner__username=%s&name=%s' % (self.baseurl,
user_name, role_name)
data = self.__call_galaxy(url)
if len(data["results"]) != 0:
return data["results"][0]
return None
def fetch(self, find_results=None):
find_results = find_results or {}
real_path = find_results.get('custom', {}).get('real_path', None)
if not real_path:
raise exceptions.GalaxyClientError('Error fetching an editable install of %s because no "real_path" was found in find_results',
self.repository_spec.src, real_path)
dst_ns_root = os.path.join(self.galaxy_context.content_path, self.repository_spec.namespace)
dst_repo_root = os.path.join(dst_ns_root,
self.repository_spec.name)
if not os.path.exists(dst_ns_root):
os.makedirs(dst_ns_root)
if not os.path.exists(dst_repo_root):
os.symlink(real_path, dst_repo_root)
repository_archive_path = self.local_path
log.debug('repository_archive_path=%s (inplace) synlink to %s',
repository_archive_path,
real_path)
results = {'archive_path': repository_archive_path,
'fetch_method': self.fetch_method}
results['custom'] = {'local_path': self.local_path,
'real_path': real_path,
'symlinked_repo_root': dst_repo_root}
results['content'] = find_results['content']
return results
def split_kwarg(spec_string, valid_keywords):
if '=' not in spec_string:
return (None, spec_string)
parts = spec_string.split('=', 1)
if parts[0] in valid_keywords:
return (parts[0], parts[1])
raise exceptions.GalaxyClientError(
'The repository spec uses an unsuppoted keyword: %s' % spec_string)
def load(data_or_file_object, klass=None):
data_dict = yaml.safe_load(data_or_file_object)
try:
collection_info = CollectionInfo(**data_dict)
except ValueError:
raise
except Exception as exc:
raise exceptions.GalaxyClientError(
"Error parsing collection metadata: %s" % str(exc))
return collection_info
def load_collections_lockfile(lockfile_path):
try:
log.debug('Opening the collections lockfile %s', lockfile_path)
with open(lockfile_path, 'r') as lffd:
return collections_lockfile.load(lffd)
except EnvironmentError as exc:
log.exception(exc)
msg = 'Error opening the collections lockfile "%s": %s' % (lockfile_path, exc)
log.error(msg)
raise exceptions.GalaxyClientError(msg)
def _get_server_api_version(self):
"""
Fetches the Galaxy API current version to ensure
the API server is up and reachable.
"""
url = '%s/api/' % self._api_server
try:
resp = self.session.get(url, verify=self._validate_certs)
except requests.exceptions.RequestException as e:
raise exceptions.GalaxyClientError(
"Failed to get data from the API server (%s): %s " %
(url, to_native(e)))
try:
# data = json.loads(to_text(return_data.read(), errors='surrogate_or_strict'))
data = resp.json()
except Exception as e:
raise exceptions.GalaxyClientError(
"Could not process data from the API server (%s): %s " %
(url, to_native(e)))
# Don't raise connection indicating errors unless we dont have valid error json
try:
resp.raise_for_status()
except Exception as e:
raise exceptions.GalaxyClientError(
"Failed to get data from the API server (%s): %s " %
(url, to_native(e)))
if 'current_version' not in data:
raise exceptions.GalaxyClientError(
"missing required 'current_version' from server response (%s)"
% url)
self.log.debug('Server API version of URL %s is "%s"', url,
data['current_version'])
return data['current_version']
def wrapped(self, *args, **kwargs):
if not self.initialized:
log.debug("Initial connection to galaxy_server: %s",
self._api_server)
server_version = self._get_server_api_version()
if server_version not in self.SUPPORTED_VERSIONS:
raise exceptions.GalaxyClientError(
"Unsupported Galaxy server API version: %s" %
server_version)
self.initialized = True
return method(self, *args, **kwargs)
def load(data_or_file_object):
data_dict = yaml.safe_load(data_or_file_object)
log.debug('data_dict: %s', data_dict)
try:
collections_lockfile = CollectionsLockfile(dependencies=data_dict)
except ValueError:
raise
except Exception as exc:
log.exception(exc)
raise exceptions.GalaxyClientError(
"Error parsing collections lockfile: %s" % str(exc))
return collections_lockfile
def install_deps_by_galaxy_metadata(content_deps,
parent_content_meta,
display_callback=None):
installed = []
for content_dep in content_deps:
if 'src' not in content_dep:
raise exceptions.GalaxyClientError("ansible-galaxy.yml dependencies must provide a src")
res = install_dep_by_galaxy_metadata(content_dep,
parent_content_meta=parent_content_meta,
display_callback=display_callback)
installed.extend(res)
return installed
def extract_file(tar_file, file_to_extract):
# TODO: should just be a object? ContentArchiveMember? ContentArchiveExtractData ?
archive_member = file_to_extract['archive_member']
dest_dir = file_to_extract['dest_dir']
dest_filename = file_to_extract['dest_filename']
force_overwrite = file_to_extract['force_overwrite']
orig_name = archive_member.name
if not archive_member.isreg() and not archive_member.issym():
return None
# TODO: raise from up a level in the stack?
dest_path = os.path.join(dest_dir, dest_filename)
# log.debug('dest_dir: %s, dest_filename: %s, dest_path: %s orig_name: %s',
# dest_dir, dest_filename, dest_path, orig_name)
if os.path.exists(dest_path):
if not force_overwrite:
message = "The Galaxy content %s appears to already exist." % dest_path
raise exceptions.GalaxyClientError(message)
try:
tar_file.getmember(archive_member.name)
except KeyError:
raise exceptions.GalaxyArchiveError(
'The archive "%s" has no file "%s"' %
(tar_file.name, archive_member.name),
archive_path=tar_file.name)
# TODO: set a default owner/group
# MAYBE TODO: pick a 'install time' and make sure all the mtime/ctime values of extracted files
# match the 'install time' used in .galaxy_install_info ?
# change the tar file member name in place to just the filename ('myfoo.py') so that extract places that file in
# dest_dir directly instead of using adding the archive path as well
# like '$dest_dir/archive-roles/library/myfoo.py'
archive_member.name = dest_filename
# log.debug('tar member: %s dest_dir: %s', archive_member, dest_dir)
tar_file.extract(archive_member, dest_dir)
installed_path = os.path.join(dest_dir, dest_filename)
# reset the tar info object's name attr to the origin value in
# case something else references this
archive_member.name = orig_name
return installed_path
def parse_repository_name(name):
"split a full repository_name into namespace, repo_name, content_name"
repo_name = None
try:
parts = name.split(".")
user_name = parts[0]
if len(parts) > 2:
repo_name = parts[1]
name = '.'.join(parts[2:])
else:
name = '.'.join(parts[1:])
except Exception as e:
log.exception(e)
raise exceptions.GalaxyClientError("Invalid content name (%s). Specify content as format: username.contentname" % name)
return (user_name, repo_name, name)
def load_archive(archive_path):
archive_parent_dir = None
if not tarfile.is_tarfile(archive_path):
raise exceptions.GalaxyClientError(
"the file downloaded was not a tar.gz")
if archive_path.endswith('.gz'):
content_tar_file = tarfile.open(archive_path, "r:gz")
else:
content_tar_file = tarfile.open(archive_path, "r")
members = content_tar_file.getmembers()
archive_parent_dir = members[0].name
archive_type = detect_content_archive_type(archive_path, members)
log.debug('archive_type: %s', archive_type)
# log.debug('self.content_type: %s', self.content_type)
# if not archive_parent_dir:
# archive_parent_dir = archive.find_archive_parent_dir(members,
# content_type=content_meta.content_type,
# content_dir=content_meta.content_dir)
log.debug('archive_type: %s', archive_type)
log.debug("archive_parent_dir: %s", archive_parent_dir)
# looks like we are a role, update the default content_type from all -> role
if archive_type == 'role':
# Look for top level role metadata
# archive_role_metadata = \
# archive.load_archive_role_metadata(content_tar_file,
# os.path.join(archive_parent_dir, archive.META_MAIN))
log.debug(
'Found role metadata in the archive, so installing it as role content_type'
)
archive_meta = content_archive.ContentArchiveMeta(
top_dir=archive_parent_dir,
archive_type=archive_type,
archive_path=archive_path)
log.debug('role archive_meta: %s', archive_meta)
return content_tar_file, archive_meta
def load(data_or_file_object, klass=None):
log.debug('loading collection info from %s', pf(data_or_file_object))
data_dict = yaml.safe_load(data_or_file_object)
log.debug('data: %s', pf(data_dict))
try:
collection_info = CollectionInfo(**data_dict)
except ValueError:
raise
except Exception as exc:
raise exceptions.GalaxyClientError(
"Error parsing collection metadata: %s" % str(exc))
log.debug('artifact_manifest from_kwargs: %s', collection_info)
return collection_info
def choose_content_fetch_method(scm_url=None, src=None):
log.debug('scm_url=%s, src=%s', scm_url, src)
if scm_url:
# create tar file from scm url
return FetchMethods.SCM_URL
if not src:
raise exceptions.GalaxyClientError("No valid content data found")
if os.path.isfile(src):
# installing a local tar.gz
return FetchMethods.LOCAL_FILE
if '://' in src:
return FetchMethods.REMOTE_URL
# if it doesnt look like anything else, assume it's galaxy
return FetchMethods.GALAXY_URL
def get_import_task(self,
task_id=None,
github_user=None,
github_repo=None):
"""
Check the status of an import task.
"""
url = '%s/imports/' % self.baseurl
if task_id is not None:
url = "%s?id=%d" % (url, task_id)
elif github_user is not None and github_repo is not None:
url = "%s?github_user=%s&github_repo=%s" % (url, github_user,
github_repo)
else:
raise exceptions.GalaxyClientError(
"Expected task_id or github_user and github_repo")
data = self.__call_galaxy(url)
return data['results']
def _get_server_api_version(self):
"""
Fetches the Galaxy API current version to ensure
the API server is up and reachable.
"""
url = '%s/api/' % self._api_server
# Call without the @g_connect wrapper to avoid recursion
data = self._get_object(href=url)
if 'current_version' not in data:
# The Galaxy API info at http://bogus.invalid:9443/api/ is missing the required 'current_version' field and the API version could not be determined.
error_msg = "The Galaxy API version could not be determined. The required 'current_version' field is missing from info at %s" % url
raise exceptions.GalaxyClientError(error_msg)
self.log.debug('Server API version of URL %s is "%s"', url,
data['current_version'])
return data['current_version']
def create_github_token(self):
'''
Create a personal authorization token with a note of 'ansible-galaxy login'
'''
self.remove_github_token()
args = json.dumps({
"scopes": ["public_repo"],
"note": "ansible-galaxy login"
})
try:
data = json.load(
open_url(self.GITHUB_AUTH,
url_username=self.github_username,
url_password=self.github_password,
force_basic_auth=True,
data=args))
except HTTPError as e:
self.log.exception(e)
res = json.load(e)
raise exceptions.GalaxyClientError(res['message'])
return data['token']
def load_tarfile_archive_info(archive_path, repository_spec):
# if not tarfile.is_tarfile(archive_path):
# raise exceptions.GalaxyClientError("the file downloaded was not a tar.gz")
if archive_path.endswith('.gz'):
tar_flags = "r:gz"
else:
tar_flags = "r"
try:
repository_tar_file = tarfile.open(archive_path, tar_flags)
except tarfile.TarError as e:
log.exception(e)
raise exceptions.GalaxyClientError(
"Error opening the tar file %s with flags: %s for repo: %s" %
(archive_path, tar_flags, repository_spec))
members = repository_tar_file.getmembers()
archive_info = build_archive_info(archive_path, [m.name for m in members])
return archive_info, repository_tar_file
def find_archive_parent_dir(archive_members, content_meta):
# archive_parent_dir wasn't found when checking for metadata files
archive_parent_dir = None
shortest_dir = None
for member in archive_members:
# This is either a new-type Galaxy Content that doesn't have an
# ansible-galaxy.yml file and the type desired is specified and
# we check parent dir based on the correct subdir existing or
# we need to just scan the subdirs heuristically and figure out
# what to do
member_dir = os.path.dirname(os.path.dirname(member.name))
shortest_dir = shortest_dir or member_dir
if len(member_dir) < len(shortest_dir):
shortest_dir = member_dir
if content_meta.content_type != "all":
if content_meta.content_dir and content_meta.content_dir in member.name:
archive_parent_dir = os.path.dirname(member.name)
return archive_parent_dir
else:
for plugin_dir in CONTENT_TYPE_DIR_MAP.values():
if plugin_dir in member.name:
archive_parent_dir = os.path.dirname(member.name)
return archive_parent_dir
if content_meta.content_type not in CONTENT_TYPES:
log.debug(
'did not find a content_dir or plugin_dir, so using shortest_dir %s for archive_parent_dir',
shortest_dir)
return shortest_dir
# TODO: archive format exception?
msg = "No content metadata provided, nor content directories found for content_type: %s" % \
content_meta.content_type
log.debug('content_meta: %s', content_meta)
raise exceptions.GalaxyClientError(msg)
def get_latest_version(available_normalized_versions, content_data):
# and sort them to get the latest version. If there
# are no versions in the list, we'll grab the head
# of the master branch
if available_normalized_versions:
try:
sorted_versions = sort_versions(available_normalized_versions)
except (TypeError, ValueError) as e:
log.exception(e)
raise exceptions.GalaxyClientError(
'Unable to compare content versions (%s) to determine the most recent version due to incompatible version formats. '
'Please contact the content author to resolve versioning conflicts, or specify an explicit content version to install. '
'The version error was: "%s"' %
(', '.join(available_normalized_versions), e))
content_version = sorted_versions[-1]
# FIXME: follow 'repository' branch and it's ['import_branch'] ?
elif content_data.get('github_branch', None):
content_version = content_data['github_branch']
else:
content_version = 'master'
return content_version
def get_credentials(self):
# FIXME(alikins) replace with a display callback
print(u'\n\n' + "We need your " + 'Github login' + " to identify you.")
print("This information will " + "not be sent to Galaxy" +
", only to " + "api.github.com.")
print("The password will not be displayed." + u'\n\n')
print("Use " + "--github-token" +
" if you do not want to enter your password." + u'\n\n')
try:
self.github_username = six.input("Github Username: "******"Password for %s: " %
self.github_username)
except:
pass
if not self.github_username or not self.github_password:
raise exceptions.GalaxyClientError(
"Invalid Github credentials. Username and password are required."
)
def __call_galaxy(self, url, args=None, headers=None, http_method=None):
http_method = http_method or 'GET'
headers = headers or {}
request_id = uuid.uuid4().hex
headers['X-Request-ID'] = request_id
# The slug we use to identify a request by method, url and request id
# For ex, '"GET https://galaxy.ansible.com/api/v1/repositories" c48937f4e8e849828772c4a0ce0fd5ed'
request_slug = '"%s %s" %s' % (http_method, url, request_id)
try:
# log the http request_slug with request_id to the main log and
# to the http log, both at INFO level for now.
http_log.info('%s', request_slug)
self.log.info('%s', request_slug)
request_log.debug('%s args=%s', request_slug, args)
request_log.debug('%s headers=%s', request_slug, headers)
resp = open_url(url,
data=args,
validate_certs=self._validate_certs,
headers=headers,
method=http_method,
http_agent=self.user_agent,
timeout=20)
response_log.info('%s http_status=%s', request_slug,
resp.getcode())
final_url = resp.geturl()
if final_url != url:
request_log.debug('%s Redirected to: %s', request_slug,
resp.geturl())
resp_info = resp.info()
response_log.debug('%s info:\n%s', request_slug, resp_info)
# FIXME: making the request and loading the response should be sep try/except blocks
response_body = to_text(resp.read(), errors='surrogate_or_strict')
# debug log the raw response body
response_log.debug('%s response body:\n%s', request_slug,
response_body)
data = json.loads(response_body)
# debug log a json version of the data that was created from the response
response_log.debug('%s data:\n%s', request_slug,
json.dumps(data, indent=2))
except HTTPError as http_exc:
self.log.debug('Exception on %s', request_slug)
self.log.exception("%s: %s", request_slug, http_exc)
# FIXME: probably need a try/except here if the response body isnt json which
# can happen if a proxy mangles the response
res = json.loads(
to_text(http_exc.fp.read(), errors='surrogate_or_strict'))
http_log.error('%s data from server error response:\n%s',
request_slug, res)
try:
error_msg = 'HTTP error on request %s: %s' % (request_slug,
res['detail'])
raise exceptions.GalaxyClientError(error_msg)
except (KeyError, TypeError) as detail_parse_exc:
self.log.exception("%s: %s", request_slug, detail_parse_exc)
self.log.warning(
'Unable to parse error detail from response for request: %s response: %s',
request_slug, detail_parse_exc)
# TODO: great place to be able to use 'raise from'
# FIXME: this needs to be tweaked so the
raise exceptions.GalaxyClientError(http_exc)
except (ssl.SSLError, socket.error) as e:
self.log.debug('Connection error to Galaxy API for request %s: %s',
request_slug, e)
self.log.exception("%s: %s", request_slug, e)
raise exceptions.GalaxyClientAPIConnectionError(
'Connection error to Galaxy API for request %s: %s' %
(request_slug, e))
return data
