def __call_galaxy(self, url, args=None, headers=None, method=None):
if args and not headers:
headers = self.__auth_header()
try:
http_log.info('%s %s', method, url)
request_log.debug('%s %s args=%s', method, url, args)
request_log.debug('%s %s headers=%s', method, url, headers)
resp = open_url(url,
data=args,
validate_certs=self._validate_certs,
headers=headers,
method=method,
timeout=20)
http_log.info('%s %s http_status=%s', method, url, resp.getcode())
final_url = resp.geturl()
if final_url != url:
http_log.debug('%s %s Redirected to: %s', method, url,
resp.geturl())
resp_info = resp.info()
response_log.debug('%s %s info:\n%s', method, url, resp_info)
# FIXME: making the request and loading the response should be sep try/except blocks
response_body = to_text(resp.read(), errors='surrogate_or_strict')
# debug log the raw response body
response_log.debug('%s %s response body:\n%s', method, url,
response_body)
data = json.loads(response_body)
# debug log a json version of the data that was created from the response
response_log.debug('%s %s data:\n%s', method, url,
json.dumps(data, indent=2))
except HTTPError as e:
self.log.debug('Exception on %s %s', method, url)
self.log.exception(e)
# FIXME: probably need a try/except here if the response body isnt json which
# can happen if a proxy mangles the response
res = json.loads(to_text(e.fp.read(),
errors='surrogate_or_strict'))
http_log.error('%s %s data from server error response:\n%s',
method, url, res)
raise exceptions.GalaxyClientError(res['detail'])
except (ssl.SSLError, socket.error) as e:
self.log.debug(
'Connection error to Galaxy API for request "%s %s": %s',
method, url, e)
self.log.exception(e)
raise exceptions.GalaxyClientAPIConnectionError(
'Connection error to Galaxy API for request "%s %s": %s' %
(method, url, e))
return data
def __call_galaxy(self, url, args=None, headers=None, method=None):
if args and not headers:
headers = self.__auth_header()
try:
# self.log.info('%s %s', method, url)
# self.log.debug('%s %s args=%s', method, url, args)
# self.log.debug('%s %s headers=%s', method, url, headers)
resp = open_url(url,
data=args,
validate_certs=self._validate_certs,
headers=headers,
method=method,
timeout=20)
self.log.debug('%s %s http_status=%s', method, url, resp.getcode())
final_url = resp.geturl()
if final_url != url:
self.log.debug('%s %s Redirected to: %s', method, url,
resp.geturl())
# self.log.debug('%s %s info:\n%s', method, url, resp.info())
data = json.loads(
to_text(resp.read(), errors='surrogate_or_strict'))
# self.log.debug('%s %s data: \n%s', method, url, json.dumps(data, indent=2))
except HTTPError as e:
self.log.debug('Exception on %s %s', method, url)
self.log.exception(e)
res = json.loads(to_text(e.fp.read(),
errors='surrogate_or_strict'))
raise exceptions.GalaxyClientError(res['detail'])
return data
def execute_import(self):
""" used to import a role into Ansible Galaxy """
# FIXME/TODO(alikins): replace with logging or display callback
colors = {
'INFO': 'normal',
'WARNING': runtime.COLOR_WARN,
'ERROR': runtime.COLOR_ERROR,
'SUCCESS': runtime.COLOR_OK,
'FAILED': runtime.COLOR_ERROR,
'DEBUG': runtime.COLOR_DEBUG,
}
if len(self.args) < 2:
raise cli_exceptions.GalaxyCliError("Expected a github_username and github_repository. Use --help.")
github_repo = to_text(self.args.pop(), errors='surrogate_or_strict')
github_user = to_text(self.args.pop(), errors='surrogate_or_strict')
if self.options.check_status:
task = self.api.get_import_task(github_user=github_user, github_repo=github_repo)
else:
# Submit an import request
task = self.api.create_import_task(github_user, github_repo, reference=self.options.reference, role_name=self.options.role_name)
if len(task) > 1:
# found multiple roles associated with github_user/github_repo
self.display("WARNING: More than one Galaxy role associated with Github repo %s/%s." % (github_user, github_repo),
color='yellow')
self.display("The following Galaxy roles are being updated:" + u'\n', color=runtime.COLOR_CHANGED)
for t in task:
self.display('%s.%s' % (t['summary_fields']['role']['namespace'], t['summary_fields']['role']['name']), color=runtime.COLOR_CHANGED)
self.display(u'\nTo properly namespace this role, remove each of the above and re-import %s/%s from scratch' % (github_user, github_repo),
color=runtime.COLOR_CHANGED)
return 0
# found a single role as expected
self.display("Successfully submitted import request %d" % task[0]['id'])
if not self.options.wait:
self.display("Role name: %s" % task[0]['summary_fields']['role']['name'])
self.display("Repo: %s/%s" % (task[0]['github_user'], task[0]['github_repo']))
if self.options.check_status or self.options.wait:
# Get the status of the import
msg_list = []
finished = False
while not finished:
task = self.api.get_import_task(task_id=task[0]['id'])
for msg in task[0]['summary_fields']['task_messages']:
if msg['id'] not in msg_list:
self.display(msg['message_text'], color=colors[msg['message_type']])
msg_list.append(msg['id'])
if task[0]['state'] in ['SUCCESS', 'FAILED']:
finished = True
else:
time.sleep(10)
return 0
def _text_encoding():
for valid_string, txtrpr, encoding in VALID_STRINGS:
log.debug('valid_string: %s txtrpr: %s encoding: %s', valid_string,
txtrpr, encoding)
exc = exceptions.GalaxyDownloadError(to_text(valid_string))
log.debug('exc: %s', exc)
log.debug(exc)
log.debug('exc(str): %s', str(exc))
log.debug('exc(repr): %s', repr(exc))
ttext = to_text(valid_string)
log.debug('%s in %s: %s', ttext, '%s' % exc, ttext in '%s' % exc)
assert ttext in '%s' % exc
def _get_server_api_version(self):
"""
Fetches the Galaxy API current version to ensure
the API server is up and reachable.
"""
url = '%s/api/' % self._api_server
try:
return_data = open_url(url, validate_certs=self._validate_certs)
except Exception as e:
raise exceptions.GalaxyClientError(
"Failed to get data from the API server (%s): %s " %
(url, to_native(e)))
try:
data = json.loads(
to_text(return_data.read(), errors='surrogate_or_strict'))
except Exception as e:
raise exceptions.GalaxyClientError(
"Could not process data from the API server (%s): %s " %
(url, to_native(e)))
if 'current_version' not in data:
raise exceptions.GalaxyClientError(
"missing required 'current_version' from server response (%s)"
% url)
self.log.debug('Server API version of URL %s is "%s"', url,
data['current_version'])
return data['current_version']
def execute_version(self):
self.display('Ansible Galaxy CLI, version', galaxy_cli_version)
self.display(', '.join(os.uname()))
self.display(sys.version, sys.executable)
if self.config_file_path:
self.display(u"Using %s as config file", to_text(self.config_file_path))
else:
self.display(u"No config file found; using defaults")
return True
def authenticate(self, github_token):
"""
Retrieve an authentication token
"""
url = '%s/tokens/' % self.baseurl
args = urlencode({"github_token": github_token})
resp = open_url(url,
data=args,
validate_certs=self._validate_certs,
method="POST")
data = json.loads(to_text(resp.read(), errors='surrogate_or_strict'))
return data
def run(self):
"""Run the ansible command
Subclasses must implement this method. It does the actual work of
running an Ansible command.
"""
log.debug('self.args: %s', self.args)
if self.config_file_path:
log.debug(u"Using %s as config file",
to_text(self.config_file_path))
else:
log.debug(u"No config file found; using defaults")
def run(self):
"""Run the ansible command
Subclasses must implement this method. It does the actual work of
running an Ansible command.
"""
# FIXME: why is self.parser none?
# display.vv(to_text(self.parser.get_version()))
if runtime.CONFIG_FILE:
log.info(u"Using %s as config file", to_text(runtime.CONFIG_FILE))
else:
log.info(u"No config file found; using defaults")
def _publish(galaxy_context,
archive_path,
publish_api_key=None,
display_callback=None):
results = {'errors': [], 'success': True}
archive = tarfile.open(archive_path, 'r')
top_dir = os.path.commonprefix(archive.getnames())
manifest_path = os.path.join(
top_dir, collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME)
try:
manifest_file = archive.extractfile(manifest_path)
except tarfile.TarError as exc:
raise exceptions.GalaxyPublishError(str(exc),
archive_path=archive_path)
try:
manifest = collection_artifact_manifest.load(manifest_file)
except Exception as exc:
raise exceptions.GalaxyPublishError(str(exc),
archive_path=archive_path)
# display_callback('Creating publish task for %s from artifact archive %s' %
# (manifest.collection_info.label, archive_path))
# display_callback(json.dumps(attr.asdict(manifest.collection_info)))
api = GalaxyAPI(galaxy_context)
collection_name = manifest.collection_info.name
data = {
'sha256': _get_file_checksum(archive_path),
'name': collection_name,
'version': manifest.collection_info.version
}
log.debug("Publishing file %s with data: %s" %
(archive_path, json.dumps(data)))
b_response_body = api.publish_file(data, archive_path, publish_api_key)
response_body = to_text(b_response_body, errors='surrogate_or_strict')
# TODO: try/except on json error but let bubble up for now
response_data = json.loads(response_body)
results['response_data'] = response_data
return results
def version_data(config_file_path, cli_version, argv):
data = {}
data['name'] = 'mazer'
data['version'] = cli_version
data['executable_location'] = argv[0]
data['uname'] = u', '.join(os.uname())
sys_ver = u"%s" % ''.join(sys.version.splitlines())
data['python_version'] = sys_ver
data['python_executable'] = sys.executable
if config_file_path:
data['config_file'] = to_text(config_file_path)
else:
data['config_file'] = u'No config file found; using defaults'
return data
def _repr_role_info(role_info):
text = [u"", u"Role: %s" % to_text(role_info['name'])]
text.append(u"\tdescription: %s" % role_info.get('description', ''))
for k in sorted(role_info.keys()):
if k in SKIP_INFO_KEYS:
continue
if isinstance(role_info[k], dict):
text.append(u"\t%s:" % (k))
for key in sorted(role_info[k].keys()):
if key in SKIP_INFO_KEYS:
continue
text.append(u"\t\t%s: %s" % (key, role_info[k][key]))
else:
text.append(u"\t%s: %s" % (k, role_info[k]))
return u'\n'.join(text)
def _publish(galaxy_context,
archive_path,
publish_api_key=None,
display_callback=None):
results = {'errors': [], 'success': True}
api = GalaxyAPI(galaxy_context)
data = {
'sha256': _get_file_checksum(archive_path),
}
log.debug("Publishing file %s with data: %s" %
(archive_path, json.dumps(data)))
# TODO: get a requests.Response here and use it's status/.json()
b_response_body = api.publish_file(data, archive_path, publish_api_key)
response_body = to_text(b_response_body, errors='surrogate_or_strict')
response_data = json.loads(response_body)
results['response_data'] = response_data
return results
def __call_galaxy(self, url, args=None, headers=None, http_method=None):
http_method = http_method or 'GET'
headers = headers or {}
request_id = uuid.uuid4().hex
headers['X-Request-ID'] = request_id
# The slug we use to identify a request by method, url and request id
# For ex, '"GET https://galaxy.ansible.com/api/v1/repositories" c48937f4e8e849828772c4a0ce0fd5ed'
request_slug = '"%s %s" %s' % (http_method, url, request_id)
try:
# log the http request_slug with request_id to the main log and
# to the http log, both at INFO level for now.
http_log.info('%s', request_slug)
self.log.info('%s', request_slug)
request_log.debug('%s args=%s', request_slug, args)
request_log.debug('%s headers=%s', request_slug, headers)
resp = open_url(url,
data=args,
validate_certs=self._validate_certs,
headers=headers,
method=http_method,
http_agent=self.user_agent,
timeout=20)
response_log.info('%s http_status=%s', request_slug,
resp.getcode())
final_url = resp.geturl()
if final_url != url:
request_log.debug('%s Redirected to: %s', request_slug,
resp.geturl())
resp_info = resp.info()
response_log.debug('%s info:\n%s', request_slug, resp_info)
# FIXME: making the request and loading the response should be sep try/except blocks
response_body = to_text(resp.read(), errors='surrogate_or_strict')
# debug log the raw response body
response_log.debug('%s response body:\n%s', request_slug,
response_body)
data = json.loads(response_body)
# debug log a json version of the data that was created from the response
response_log.debug('%s data:\n%s', request_slug,
json.dumps(data, indent=2))
except HTTPError as http_exc:
self.log.debug('Exception on %s', request_slug)
self.log.exception("%s: %s", request_slug, http_exc)
# FIXME: probably need a try/except here if the response body isnt json which
# can happen if a proxy mangles the response
res = json.loads(
to_text(http_exc.fp.read(), errors='surrogate_or_strict'))
http_log.error('%s data from server error response:\n%s',
request_slug, res)
try:
error_msg = 'HTTP error on request %s: %s' % (request_slug,
res['detail'])
raise exceptions.GalaxyClientError(error_msg)
except (KeyError, TypeError) as detail_parse_exc:
self.log.exception("%s: %s", request_slug, detail_parse_exc)
self.log.warning(
'Unable to parse error detail from response for request: %s response: %s',
request_slug, detail_parse_exc)
# TODO: great place to be able to use 'raise from'
# FIXME: this needs to be tweaked so the
raise exceptions.GalaxyClientError(http_exc)
except (ssl.SSLError, socket.error) as e:
self.log.debug('Connection error to Galaxy API for request %s: %s',
request_slug, e)
self.log.exception("%s: %s", request_slug, e)
raise exceptions.GalaxyClientAPIConnectionError(
'Connection error to Galaxy API for request %s: %s' %
(request_slug, e))
return data
def get_ca_certs(self):
# tries to find a valid CA cert in one of the
# standard locations for the current distribution
ca_certs = []
paths_checked = []
system = to_text(platform.system(), errors='surrogate_or_strict')
# build a list of paths to check for .crt/.pem files
# based on the platform type
paths_checked.append('/etc/ssl/certs')
if system == u'Linux':
paths_checked.append('/etc/pki/ca-trust/extracted/pem')
paths_checked.append('/etc/pki/tls/certs')
paths_checked.append('/usr/share/ca-certificates/cacert.org')
elif system == u'FreeBSD':
paths_checked.append('/usr/local/share/certs')
elif system == u'OpenBSD':
paths_checked.append('/etc/ssl')
elif system == u'NetBSD':
ca_certs.append('/etc/openssl/certs')
elif system == u'SunOS':
paths_checked.append('/opt/local/etc/openssl/certs')
# fall back to a user-deployed cert in a standard
# location if the OS platform one is not available
paths_checked.append('/etc/ansible')
tmp_fd, tmp_path = tempfile.mkstemp()
to_add_fd, to_add_path = tempfile.mkstemp()
to_add = False
# Write the dummy ca cert if we are running on Mac OS X
if system == u'Darwin':
os.write(tmp_fd, b_DUMMY_CA_CERT)
# Default Homebrew path for OpenSSL certs
paths_checked.append('/usr/local/etc/openssl')
# for all of the paths, find any .crt or .pem files
# and compile them into single temp file for use
# in the ssl check to speed up the test
for path in paths_checked:
if os.path.exists(path) and os.path.isdir(path):
dir_contents = os.listdir(path)
for f in dir_contents:
full_path = os.path.join(path, f)
if os.path.isfile(full_path) and os.path.splitext(
f)[1] in ('.crt', '.pem'):
try:
cert_file = open(full_path, 'rb')
cert = cert_file.read()
cert_file.close()
os.write(tmp_fd, cert)
os.write(tmp_fd, b'\n')
if full_path not in LOADED_VERIFY_LOCATIONS:
to_add = True
os.write(to_add_fd, cert)
os.write(to_add_fd, b'\n')
LOADED_VERIFY_LOCATIONS.add(full_path)
except (OSError, IOError):
pass
if not to_add:
try:
os.remove(to_add_path)
except OSError:
pass
to_add_path = None
return (tmp_path, to_add_path, paths_checked)
