def load_configuration(self):
if not os.path.isfile(self.src):
self.module.fail_json(msg="Source file {} does not exist".format(self.src))
url = self.host.configManager.firmwareSystem.QueryFirmwareConfigUploadURL()
url = url.replace('*', self.host.name)
# find manually the url if there is a redirect because urllib2 -per RFC- doesn't do automatic redirects for PUT requests
try:
request = open_url(url=url, method='HEAD', validate_certs=self.validate_certs)
except HTTPError as e:
url = e.geturl()
try:
with open(self.src, 'rb') as file:
data = file.read()
request = open_url(url=url, data=data, method='PUT', validate_certs=self.validate_certs,
url_username=self.username, url_password=self.password, force_basic_auth=True)
except Exception as e:
self.module.fail_json(msg=to_native(e))
if not self.host.runtime.inMaintenanceMode:
self.enter_maintenance()
try:
self.host.configManager.firmwareSystem.RestoreFirmwareConfiguration(force=True)
self.module.exit_json(changed=True)
except Exception as e:
self.exit_maintenance()
self.module.fail_json(msg=to_native(e))
def get_client_by_id(self, id, realm='master'):
""" Obtain client representation by id
:param id: id (not clientId) of client to be queried
:param realm: client from this realm
:return: dict of client representation or None if none matching exist
"""
client_url = URL_CLIENT.format(url=self.baseurl, realm=realm, id=id)
try:
return json.load(open_url(client_url, method='GET', headers=self.restheaders,
validate_certs=self.validate_certs))
except HTTPError as e:
if e.code == 404:
return None
else:
self.module.fail_json(msg='Could not obtain client %s for realm %s: %s'
% (id, realm, str(e)))
except ValueError as e:
self.module.fail_json(msg='API returned incorrect JSON when trying to obtain client %s for realm %s: %s'
% (id, realm, str(e)))
except Exception as e:
self.module.fail_json(msg='Could not obtain client %s for realm %s: %s'
% (id, realm, str(e)))
def run(self, terms, inject=None, **kwargs):
if isinstance(terms, basestring):
terms = [ terms ]
validate_certs = kwargs.get('validate_certs', True)
f_handle, templ_path = tempfile.mkstemp(prefix='ansible_template', dir='/tmp/')
temp_file = open(templ_path, 'w+r')
ret = []
for term in terms:
try:
response = open_url(term, validate_certs=validate_certs)
except urllib2.URLError as e:
utils.warnings("Failed lookup url for %s : %s" % (term, str(e)))
except urllib2.HTTPError as e:
utils.warnings("Received HTTP error for %s : %s" % (term, str(e)))
except SSLValidationError as e:
utils.warnings("Error validating the server's certificate for %s: %s" % (term, str(e)))
except ConnectionError as e:
raise utils.warnings("Error connecting to %s: %s" % (term, str(e)))
for line in response.read().splitlines():
temp_file.write(line)
ret.append(templ_path)
return ret
def run(self, terms, inject=None, **kwargs):
terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
if isinstance(terms, basestring):
terms = [ terms ]
validate_certs = kwargs.get('validate_certs', True)
ret = []
for term in terms:
try:
response = open_url(term, validate_certs=validate_certs)
except urllib2.URLError as e:
utils.warning("Failed lookup url for %s : %s" % (term, str(e)))
continue
except urllib2.HTTPError as e:
utils.warning("Received HTTP error for %s : %s" % (term, str(e)))
continue
except SSLValidationError as e:
utils.warning("Error validating the server's certificate for %s: %s" % (term, str(e)))
continue
except ConnectionError as e:
utils.warning("Error connecting to %s: %s" % (term, str(e)))
continue
for line in response.read().splitlines():
ret.append(to_unicode(line))
return ret
def _connect(self):
""" Obtains an access_token and saves it for use in API accesses
"""
self.baseurl = self.module.params.get('auth_keycloak_url')
self.validate_certs = self.module.params.get('validate_certs')
auth_url = URL_TOKEN.format(url=self.baseurl, realm=self.module.params.get('auth_realm'))
payload = {'grant_type': 'password',
'client_id': self.module.params.get('auth_client_id'),
'client_secret': self.module.params.get('auth_client_secret'),
'username': self.module.params.get('auth_username'),
'password': self.module.params.get('auth_password')}
# Remove empty items, for instance missing client_secret
payload = dict((k, v) for k, v in payload.items() if v is not None)
try:
r = json.load(open_url(auth_url, method='POST',
validate_certs=self.validate_certs, data=urlencode(payload)))
except ValueError as e:
self.module.fail_json(msg='API returned invalid JSON when trying to obtain access token from %s: %s'
% (auth_url, str(e)))
except Exception as e:
self.module.fail_json(msg='Could not obtain access token from %s: %s'
% (auth_url, str(e)))
if 'access_token' in r:
self.token = r['access_token']
self.restheaders = {'Authorization': 'Bearer ' + self.token,
'Content-Type': 'application/json'}
else:
self.module.fail_json(msg='Could not obtain access token from %s' % auth_url)
def send(self, request, **kwargs):
response = Response()
params = dict(
method=request.method,
data=request.body,
timeout=kwargs.get('timeout', None) or self.timeout,
validate_certs=kwargs.get('verify', None) or self.verify,
headers=request.headers
)
try:
result = open_url(request.url, **params)
response._content = result.read().decode('utf-8')
response.status = result.getcode()
response.url = result.geturl()
response.msg = "OK (%s bytes)" % result.headers.get('Content-Length', 'unknown')
response.headers = self._normalize_headers(result.headers.items())
response.request = request
except urllib_error.HTTPError as e:
try:
response._content = e.read()
except AttributeError:
response._content = ''
response.reason = to_native(e)
response.status_code = e.code
return response
def search_roles(self, search, platforms=None, tags=None):
search_url = self.baseurl + '/roles/?page=1'
if search:
search_url += '&search=' + urlquote(search)
if tags is None:
tags = []
elif isinstance(tags, basestring):
tags = tags.split(',')
for tag in tags:
search_url += '&chain__tags__name=' + urlquote(tag)
if platforms is None:
platforms = []
elif isinstance(platforms, basestring):
platforms = platforms.split(',')
for plat in platforms:
search_url += '&chain__platforms__name=' + urlquote(plat)
self.galaxy.display.debug("Executing query: %s" % search_url)
try:
data = json.load(open_url(search_url, validate_certs=self.galaxy.options.validate_certs))
except HTTPError as e:
raise AnsibleError("Unsuccessful request to server: %s" % str(e))
return data
def test_open_url_custom_method(urlopen_mock, install_opener_mock):
r = open_url('https://ansible.com/', method='DELETE')
args = urlopen_mock.call_args[0]
req = args[0]
assert isinstance(req, RequestWithMethod)
def test_open_url_client_cert(urlopen_mock, install_opener_mock):
here = os.path.dirname(__file__)
client_cert = os.path.join(here, 'fixtures/client.pem')
client_key = os.path.join(here, 'fixtures/client.key')
r = open_url('https://ansible.com/', client_cert=client_cert, client_key=client_key)
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
ssl_handler = None
for handler in handlers:
if isinstance(handler, HTTPSClientAuthHandler):
ssl_handler = handler
break
assert ssl_handler is not None
assert ssl_handler.client_cert == client_cert
assert ssl_handler.client_key == client_key
https_connection = ssl_handler._build_https_connection('ansible.com')
assert https_connection.key_file == client_key
assert https_connection.cert_file == client_cert
def test_open_url_user_agent(urlopen_mock, install_opener_mock):
r = open_url('https://ansible.com/', http_agent='ansible-tests')
args = urlopen_mock.call_args[0]
req = args[0]
assert req.headers.get('User-agent') == 'ansible-tests'
def get(self, key):
url = "%s/%s?recursive=true" % (self.baseurl, key)
data = None
value = {}
try:
r = open_url(url, validate_certs=self.validate_certs)
data = r.read()
except:
return None
try:
# I will not support Version 1 of etcd for folder parsing
item = json.loads(data)
if self.version == 'v1':
# When ETCD are working with just v1
if 'value' in item:
value = item['value']
else:
if 'node' in item:
# When a usual result from ETCD
value = self._parse_node(item['node'])
if 'errorCode' in item:
# Here return an error when an unknown entry responds
value = "ENOENT"
except:
raise
return value
def fetch(self, role_data):
"""
Downloads the archived role from github to a temp location
"""
if role_data:
# first grab the file and save it to a temp location
if "github_user" in role_data and "github_repo" in role_data:
archive_url = 'https://github.com/%s/%s/archive/%s.tar.gz' % (role_data["github_user"], role_data["github_repo"], self.version)
else:
archive_url = self.src
display.display("- downloading role from %s" % archive_url)
try:
url_file = open_url(archive_url, validate_certs=self._validate_certs)
temp_file = tempfile.NamedTemporaryFile(delete=False)
data = url_file.read()
while data:
temp_file.write(data)
data = url_file.read()
temp_file.close()
return temp_file.name
except Exception as e:
display.error("failed to download the file: %s" % str(e))
return False
def run(self, terms, variables=None, **kwargs):
validate_certs = kwargs.get('validate_certs', True)
split_lines = kwargs.get('split_lines', True)
use_proxy = kwargs.get('use_proxy', True)
ret = []
for term in terms:
display.vvvv("url lookup connecting to %s" % term)
try:
response = open_url(term, validate_certs=validate_certs, use_proxy=use_proxy)
except HTTPError as e:
raise AnsibleError("Received HTTP error for %s : %s" % (term, str(e)))
except URLError as e:
raise AnsibleError("Failed lookup url for %s : %s" % (term, str(e)))
except SSLValidationError as e:
raise AnsibleError("Error validating the server's certificate for %s: %s" % (term, str(e)))
except ConnectionError as e:
raise AnsibleError("Error connecting to %s: %s" % (term, str(e)))
if split_lines:
for line in response.read().splitlines():
ret.append(to_text(line))
else:
ret.append(to_text(response.read()))
return ret
def get(self, key):
url = "%s/%s" % (self.baseurl, key)
data = None
try:
r = open_url(url, validate_certs=self.validate_certs)
data = r.read()
except:
return data
try:
# {"action":"get","key":"/name","value":"Jane Jolie","index":5}
value=[]
nodes = json.loads(data)['node']
if 'nodes' in nodes:
nodes=nodes['nodes']
for node in nodes:
if 'key' in node:
value.append(node['key'])
elif 'value' in node:
value.append(node['value'])
elif 'errorCode' in node:
value.append("ENOENT")
break
else:
if 'value' in nodes:
value = nodes['value']
elif 'key' in nodes:
value = nodes['key']
elif 'errorCode' in nodes:
value = "ENOENT"
except:
raise
pass
return value
def request(url, data=None, headers=None, method='GET', use_proxy=True,
force=False, last_mod_time=None, timeout=10, validate_certs=True,
url_username=None, url_password=None, http_agent=None, force_basic_auth=True, ignore_errors=False):
try:
r = open_url(url=url, data=data, headers=headers, method=method, use_proxy=use_proxy,
force=force, last_mod_time=last_mod_time, timeout=timeout, validate_certs=validate_certs,
url_username=url_username, url_password=url_password, http_agent=http_agent,
force_basic_auth=force_basic_auth)
except HTTPError:
err = get_exception()
r = err.fp
try:
raw_data = r.read()
if raw_data:
data = json.loads(raw_data)
else:
raw_data = None
except:
if ignore_errors:
pass
else:
raise Exception(raw_data)
resp_code = r.getcode()
if resp_code >= 400 and not ignore_errors:
raise Exception(resp_code, data)
else:
return resp_code, data
def run(self, terms, variables, **kwargs):
try:
resp = open_url('https://ip-ranges.amazonaws.com/ip-ranges.json')
amazon_response = json.load(resp)['prefixes']
except getattr(json.decoder, 'JSONDecodeError', ValueError) as e:
# on Python 3+, json.decoder.JSONDecodeError is raised for bad
# JSON. On 2.x it's a ValueError
raise AnsibleError("Could not decode AWS IP ranges: %s" % to_native(e))
except HTTPError as e:
raise AnsibleError("Received HTTP error while pulling IP ranges: %s" % to_native(e))
except SSLValidationError as e:
raise AnsibleError("Error validating the server's certificate for: %s" % to_native(e))
except URLError as e:
raise AnsibleError("Failed look up IP range service: %s" % to_native(e))
except ConnectionError as e:
raise AnsibleError("Error connecting to IP range service: %s" % to_native(e))
if 'region' in kwargs:
region = kwargs['region']
amazon_response = (item for item in amazon_response if item['region'] == region)
if 'service' in kwargs:
service = str.upper(kwargs['service'])
amazon_response = (item for item in amazon_response if item['service'] == service)
return [item['ip_prefix'] for item in amazon_response]
def get(self, key):
url = "%s/%s" % (self.baseurl, key)
data = None
value = ""
try:
r = open_url(url, validate_certs=self.validate_certs)
data = r.read()
except:
return value
try:
# {"action":"get","key":"/name","value":"Jane Jolie","index":5}
item = json.loads(data)
if self.version == 'v1':
if 'value' in item:
value = item['value']
else:
if 'node' in item:
value = item['node']['value']
if 'errorCode' in item:
value = "ENOENT"
except:
raise
pass
return value
def lookup_role_by_name(self, role_name, notify=True):
"""
Find a role by name
"""
role_name = urlquote(role_name)
try:
parts = role_name.split(".")
user_name = ".".join(parts[0:-1])
role_name = parts[-1]
if notify:
self.galaxy.display.display("- downloading role '%s', owned by %s" % (role_name, user_name))
except:
raise AnsibleError("- invalid role name (%s). Specify role as format: username.rolename" % role_name)
url = '%s/roles/?owner__username=%s&name=%s' % (self.baseurl, user_name, role_name)
self.galaxy.display.vvvv("- %s" % (url))
try:
data = json.load(open_url(url, validate_certs=self.galaxy.options.validate_certs))
if len(data["results"]) != 0:
return data["results"][0]
except:
# TODO: report on connection/availability errors
pass
return None
def test_open_url(urlopen_mock, install_opener_mock):
r = open_url('https://ansible.com/')
args = urlopen_mock.call_args[0]
assert args[1] is None # data, this is handled in the Request not urlopen
assert args[2] == 10 # timeout
req = args[0]
assert req.headers == {}
assert req.data is None
assert req.get_method() == 'GET'
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
expected_handlers = (
SSLValidationHandler,
RedirectHandlerFactory(), # factory, get handler
)
found_handlers = []
for handler in handlers:
if isinstance(handler, SSLValidationHandler) or handler.__class__.__name__ == 'RedirectHandler':
found_handlers.append(handler)
assert len(found_handlers) == 2
def test_open_url_last_mod(urlopen_mock, install_opener_mock):
now = datetime.datetime.now()
r = open_url('https://ansible.com/', last_mod_time=now)
args = urlopen_mock.call_args[0]
req = args[0]
assert req.headers.get('If-modified-since') == now.strftime('%a, %d %b %Y %H:%M:%S +0000')
def get(self, url, data=None):
request_path = '{}{}'.format(self.options.url, url)
headers = {'Cookie': 'PVEAuthCookie={}'.format(self.credentials['ticket'])}
request = open_url(request_path, data=data, headers=headers)
response = json.load(request)
return response['data']
def test_open_url_force(urlopen_mock, install_opener_mock):
r = open_url('https://ansible.com/', force=True, last_mod_time=datetime.datetime.now())
args = urlopen_mock.call_args[0]
req = args[0]
assert req.headers.get('Cache-control') == 'no-cache'
assert 'If-modified-since' not in req.headers
def authenticate(self, github_token):
"""
Retrieve an authentication token
"""
url = '%s/tokens/' % self.baseurl
args = urllib.urlencode({"github_token": github_token})
resp = open_url(url, data=args, validate_certs=self._validate_certs, method="POST")
data = json.load(resp)
return data
def _send_annotation(self, annotation):
try:
response = open_url(self.grafana_url, data=annotation, headers=self.headers,
method="POST",
validate_certs=self.validate_grafana_certs,
url_username=self.grafana_user, url_password=self.grafana_password,
http_agent=self.http_agent, force_basic_auth=self.force_basic_auth)
except Exception as e:
self._display.error('Could not submit message to Grafana: %s' % str(e))
def user_add_to_group(module):
# Get username, and groupname from module parameters, and api base url
# along with validate_certs from the cyberark_session established
username = module.params["username"]
group_name = module.params["group_name"]
cyberark_session = module.params["cyberark_session"]
api_base_url = cyberark_session["api_base_url"]
validate_certs = cyberark_session["validate_certs"]
# Prepare result, end_point, headers and payload
result = {}
end_point = "/PasswordVault/WebServices/PIMServices.svc//Groups/{0}/Users".format(
group_name)
headers = {'Content-Type': 'application/json'}
headers["Authorization"] = cyberark_session["token"]
payload = {"UserName": username}
try:
# execute REST action
response = open_url(
api_base_url + end_point,
method="POST",
headers=headers,
data=json.dumps(payload),
validate_certs=validate_certs)
result = {"result": {}}
return (True, result, response.getcode())
except (HTTPError, httplib.HTTPException) as http_exception:
exception_text = to_text(http_exception)
if http_exception.code == 409 and "ITATS262E" in exception_text:
# User is already member of Group
return (False, None, http_exception.code)
else:
module.fail_json(
msg=("Error while performing user_add_to_group."
"Please validate parameters provided."
"\n*** end_point=%s%s\n ==> %s" % (api_base_url, end_point, exception_text)),
payload=payload,
headers=headers,
exception=traceback.format_exc(),
status_code=http_exception.code)
except Exception as unknown_exception:
module.fail_json(
msg=("Unknown error while performing user_add_to_group."
"\n*** end_point=%s%s\n%s" % (api_base_url, end_point, to_text(unknown_exception))),
payload=payload,
headers=headers,
status_code=-1)
def authenticate(self, github_token):
"""
Retrieve an authentication token
"""
url = '%s/tokens/' % self.baseurl
args = urlencode({"github_token": github_token})
resp = open_url(url, data=args, validate_certs=self._validate_certs, method="POST")
data = json.loads(to_text(resp.read(), errors='surrogate_or_strict'))
return data
def test_open_url_netrc(urlopen_mock, install_opener_mock, monkeypatch):
here = os.path.dirname(__file__)
monkeypatch.setenv('NETRC', os.path.join(here, 'fixtures/netrc'))
r = open_url('http://ansible.com/')
args = urlopen_mock.call_args[0]
req = args[0]
assert req.headers.get('Authorization') == b'Basic dXNlcjpwYXNzd2Q='
r = open_url('http://foo.ansible.com/')
args = urlopen_mock.call_args[0]
req = args[0]
assert 'Authorization' not in req.headers
monkeypatch.setenv('NETRC', os.path.join(here, 'fixtures/netrc.nonexistant'))
r = open_url('http://ansible.com/')
args = urlopen_mock.call_args[0]
req = args[0]
assert 'Authorization' not in req.headers
def __call_galaxy(self, url, args=None, headers=None, method=None):
if args and not headers:
headers = self.__auth_header()
try:
display.vvv(url)
resp = open_url(url, data=args, validate_certs=self._validate_certs, headers=headers, method=method)
data = json.load(resp)
except HTTPError as e:
res = json.load(e)
raise AnsibleError(res['detail'])
return data
def get_server_api_version(self):
"""
Fetches the Galaxy API current version to ensure
the API server is up and reachable.
"""
try:
url = '%s/api/' % self._api_server
data = json.load(open_url(url, validate_certs=self._validate_certs))
return data['current_version']
except Exception as e:
raise AnsibleError("The API server (%s) is not responding, please try again later." % url)
def _fetch_conjur_token(conjur_url, account, username, api_key):
conjur_url = '{0}/authn/{1}/{2}/authenticate'.format(conjur_url, account, username)
display.vvvv('Authentication request to Conjur at: {0}, with user: {1}'.format(conjur_url, username))
response = open_url(conjur_url, data=api_key, method='POST')
code = response.getcode()
if code != 200:
raise AnsibleError('Failed to authenticate as \'{0}\' (got {1} response)'
.format(username, code))
return response.read()
def run(self, terms, **kwargs):
validate_certs = kwargs.get('validate_certs', True)
ret = []
api = kwargs.pop('api', ANSIBLE_API)
url = kwargs.pop('url', RANCHER_URL)
url_username = kwargs.pop('url_username', RANCHER_ACCESS_KEY)
url_password = kwargs.pop('url_password', RANCHER_SECRET_KEY)
validate_certs = kwargs.get('validate_certs', True)
for term in terms:
url = '%s/%s/projects' % (url, api)
display.vvvv('rancher_project lookup of %s on %s' % (term, url))
try:
response = open_url(url=url,
url_username=url_username,
url_password=url_password,
validate_certs=validate_certs)
except HTTPError as e:
raise AnsibleError("Received HTTP error for %s : %s" %
(term, str(e)))
except URLError as e:
raise AnsibleError("Failed lookup url for %s : %s" %
(term, str(e)))
except SSLValidationError as e:
raise AnsibleError(
"Error validating the server's certificate for %s: %s" %
(term, str(e)))
except ConnectionError as e:
raise AnsibleError("Error connecting to %s: %s" %
(term, str(e)))
body = json.loads(response.read())
data = body.get('data')
if terms:
for item in data:
if item.get('name') == term and item.get(
'type') == 'project':
ret.append(item)
else:
ret.append(data)
return ret
def token(self):
"""Get a new token from BIG-IP and store it internally.
This method will be called automatically if a request is attempted
but there is no authentication token, or the authentication token
is expired. It is usually not necessary for users to call it, but
it can be called if it is known that the authentication token has
been invalidated by other means.
"""
if self._token:
return self._token
login_body = {
'username': self._username,
'password': self._password,
}
if self._auth_provider:
login_body['loginProviderName'] = self.auth_provider
url = "https://{0}:{1}/mgmt/shared/authn/login".format(
self._server, self._server_port)
response = open_url(url,
method='POST',
data=json.dumps(login_body),
validate_certs=self._validate_certs)
if response.code not in [200]:
raise F5ModuleError(
'{0} Unexpected Error: {1} for uri: {2}\nText: {3}'.format(
response.status_code, response.reason, response.url,
response.text))
resp = json.loads(response.read())
self._token = resp.get('token', None)
if self._token.get('token') is not None:
# BIG-IQ stores tokens in the 'token' key
result = self._token.get('token', None)
self._token = result
return result
else:
# BIG-IP stores tokens in the token dict, 'name' key
result = self._token.get('name', None)
self._token = result
return result
def register_source(module, self):
server = module.params.get('cluster')
validate_certs = module.params.get('validate_certs')
token = self['token']
try:
uri = "https://" + server + "/irisservices/api/v1/public/protectionSources/register"
headers = {
"Accept": "application/json",
"Authorization": "Bearer " + token
}
payload = self.copy()
payload['environment'] = "k" + self['environment']
if self['environment'] == "Physical":
payload['hostType'] = "k" + self['hostType']
payload['physicalType'] = "k" + self['physicalType']
elif self['environment'] == "VMware":
payload['vmwareType'] = "k" + self['vmwareType']
data = json.dumps(payload)
response = open_url(url=uri,
data=data,
headers=headers,
validate_certs=validate_certs,
timeout=REQUEST_TIMEOUT)
response = json.loads(response.read())
# => This switcher will allow us to return a standardized output
# => for all Protection Sources.
if self['environment'] == 'Physical':
response = dict(
ProtectionSource=response['physicalProtectionSource'])
elif self['environment'] == 'VMware':
response = dict(
ProtectionSource=response['vmWareProtectionSource'])
elif self['environment'] == 'GenericNas':
response = dict(ProtectionSource=response['nasProtectionSource'])
return response
except urllib_error.URLError as e:
# => Capture and report any error messages.
raise__cohesity_exception__handler(e.read(), module)
except Exception as error:
raise__cohesity_exception__handler(error, module)
def get_component_by_id(self, id, deployment_attrs='omitted'):
self._ensure_loggedin()
url = urllib.parse.urljoin(
self.api_base_url,
f'./components/{id}?deployment_attrs={deployment_attrs}&cache=skip'
)
headers = {
'Accepts': 'application/json',
'Authorization': self.authorization
}
for attempt in range(1, self.retries + 1):
try:
response = open_url(url,
None,
headers,
'GET',
validate_certs=False,
timeout=self.api_timeout)
return json.load(response)
except Exception as e:
# The API will return HTTP 404 Not Found if the component exists in the IBM Blockchain Platform
# console, but not in Kubernetes. Try again without requesting the deployment attributes, and
# add a value to the result that will trigger the calling module to delete the component.
if isinstance(e, urllib.error.HTTPError
) and deployment_attrs == 'included':
is_404 = e.code == 404
# Sometimes the HTTP 404 Not Found is buried in a HTTP 503 Service Unavailable message, so
# we need to check for that as well.
if e.code == 503:
try:
error = json.load(e)
is_404 = error.get('response',
dict()).get('status', 0) == 404
except Exception:
pass
if is_404:
result = self.get_component_by_id(id, 'omitted')
result['deployment_attrs_missing'] = True
return result
if self.should_retry_error(e, attempt):
continue
return self.handle_error('Failed to get component by ID', e)
def login_api_mm(module):
# Define variables from module arguments
username = module.params.get('username')
password = module.params.get('password')
host = module.params.get('host')
session_key = ""
resp = ""
# Variables required for open_url
url = "https://" + str(host) + ":4343/v1/api/login"
headers = {'Accept': 'application/json', 'Content-Type': 'application/x-www-form-urlencoded'}
data = urlencode({'username': username, 'password': password})
cookies = cookiejar.LWPCookieJar()
validate_certs = module.params.get('validate_certs')
client_cert = module.params.get('client_cert')
client_key = module.params.get('client_key')
http_agent = 'ansible-httpget'
follow_redirects = 'urllib2'
method = "POST"
# Store the url to module, so we can print the details in case of error
module.api_call = {'host':host,'username':username,'password':password, 'url':''}
module.api_call['url'] = url
try:
resp = open_url(url, data=data, headers=headers, method=method, validate_certs=validate_certs,
http_agent=http_agent, follow_redirects=follow_redirects, cookies=cookies,
client_cert=client_cert, client_key=client_key)
resp = resp.read()
result = json.loads(resp)['_global_result']
if result['status'] == "0":
session_key = result['UIDARUBA']
else:
module.fail_json(changed=False,
msg="Login Failed! Recheck the credentials you provided",
reason=str(result['status_str']),
api_call=module.api_call,
response="content: " + str(resp) + " login status code: " + str(result['status']) + " login error: " + str(result['status_str']))
except Exception as e:
module.fail_json(changed=False, msg="API Call failed! Exception during login", reason=str(e), api_call=module.api_call)
session_dict = {'host':host,
'session_token': session_key}
return session_dict
def download_screenshot_file(self, file_url, local_file_path, file_name):
response = None
download_size = 0
# file is downloaded as local_file_name when specified, or use original file name
if local_file_path.endswith('.png'):
local_file_name = local_file_path.split('/')[-1]
local_file_path = local_file_path.rsplit('/', 1)[0]
else:
local_file_name = file_name
if not os.path.exists(local_file_path):
try:
os.makedirs(local_file_path)
except OSError as err:
self.module.fail_json(
msg=
"Exception caught when create folder %s on local machine, with error %s"
% (local_file_path, to_native(err)))
local_file = os.path.join(local_file_path, local_file_name)
with open(local_file, 'wb') as handle:
try:
response = open_url(file_url,
url_username=self.params.get('username'),
url_password=self.params.get('password'),
validate_certs=False)
except Exception as err:
self.module.fail_json(
msg="Download screenshot file from URL %s, failed due to %s"
% (file_url, to_native(err)))
if not response or response.getcode() >= 400:
self.module.fail_json(
msg=
"Download screenshot file from URL %s, failed with response %s, response code %s"
% (file_url, response, response.getcode()))
bytes_read = response.read(2**20)
while bytes_read:
handle.write(bytes_read)
handle.flush()
os.fsync(handle.fileno())
download_size += len(bytes_read)
bytes_read = response.read(2**20)
return download_size
def register_oracle_source(module, self, _id):
'''
: To register a Oracle source, it should be already registered as Physical
: source in the cluster.
: Register a physical source as a Oracle source using physical source id.
'''
server = module.params.get('cluster')
validate_certs = module.params.get('validate_certs')
token = self['token']
endpoint = self['endpoint']
source_id = _id
db_user = module.params.get('db_username')
db_pwd = module.params.get('db_password')
try:
uri = 'https://' + server + '/irisservices/api/v1/applicationSourceRegistration'
headers = {
'Accept': 'application/json',
'Authorization': 'Bearer ' + token
}
# Payload to register Oracle source.
payload = dict(appEnvVec=[19],
usesPersistentAgent=True,
ownerEntity=dict(type=6))
payload['ownerEntity']['id'] = source_id
payload['ownerEntity']['displayName'] = endpoint
if db_user and db_pwd:
cred = dict(username=db_user, password=db_pwd)
payload['appCredentialsVec'] = list()
payload['appCredentialsVec'].append(
dict(credentials=cred, envType=19))
data = json.dumps(payload)
response = open_url(url=uri,
data=data,
headers=headers,
validate_certs=validate_certs,
timeout=REQUEST_TIMEOUT)
response = json.loads(response.read())
return response
except Exception as err:
return payload
def _download_file(url,
b_path,
expected_hash,
validate_certs,
token=None,
timeout=60):
# type: (str, bytes, t.Optional[str], bool, GalaxyToken, int) -> bytes
# ^ NOTE: used in download and verify_collections ^
b_tarball_name = to_bytes(
url.rsplit('/', 1)[1],
errors='surrogate_or_strict',
)
b_file_name = b_tarball_name[:-len('.tar.gz')]
b_tarball_dir = mkdtemp(
dir=b_path,
prefix=b'-'.join((b_file_name, b'')),
) # type: bytes
b_file_path = os.path.join(b_tarball_dir, b_tarball_name)
display.display("Downloading %s to %s" % (url, to_text(b_tarball_dir)))
# NOTE: Galaxy redirects downloads to S3 which rejects the request
# NOTE: if an Authorization header is attached so don't redirect it
resp = open_url(to_native(url, errors='surrogate_or_strict'),
validate_certs=validate_certs,
headers=None if token is None else token.headers(),
unredirected_headers=['Authorization'],
http_agent=user_agent(),
timeout=timeout)
with open(b_file_path, 'wb') as download_file: # type: t.BinaryIO
actual_hash = _consume_file(resp, write_to=download_file)
if expected_hash:
display.vvvv('Validating downloaded file hash {actual_hash!s} with '
'expected hash {expected_hash!s}'.format(
actual_hash=actual_hash, expected_hash=expected_hash))
if expected_hash != actual_hash:
raise AnsibleError('Mismatch artifact hash with downloaded file')
return b_file_path
def get_source_details(module, restore_to_source):
'''
Get VMware protection source details
:param module: object that holds parameters passed to the module
:param restore_to_source: boolean flag to get target source details or
vm's parent source details
:return:
'''
server = module.params.get('cluster')
validate_certs = module.params.get('validate_certs')
token = get__cohesity_auth__token(module)
try:
uri = "https://" + server + \
"/irisservices/api/v1/public/protectionSources/rootNodes?environments=kVMware"
headers = {
"Accept": "application/json",
"Authorization": "Bearer " + token,
"user-agent": "cohesity-ansible/v2.3.4"
}
response = open_url(url=uri,
headers=headers,
validate_certs=validate_certs,
method="GET",
timeout=REQUEST_TIMEOUT)
response = json.loads(response.read())
source_details = dict()
for source in response:
if not restore_to_source and source['protectionSource'][
'name'] == module.params.get('endpoint'):
source_details['id'] = source['protectionSource']['id']
elif restore_to_source and source['protectionSource'][
'name'] == module.params.get('restore_to_source'):
source_details['id'] = source['protectionSource']['id']
if not source_details:
module.fail_json(changed=False,
msg="Can't find the endpoint on the cluster")
return source_details
except urllib_error.URLError as e:
# => Capture and report any error messages.
raise__cohesity_exception__handler(e.read(), module)
except Exception as error:
raise__cohesity_exception__handler(error, module)
def _get(self, chroot):
"""Send a get request to the server to obtain the necessary data.
Args:
chroot: Chroot in the form of distribution-version.
Returns:
Info about a repository and status code of the get request.
"""
repo_info = None
url = "{0}://{1}/coprs/{2}/repo/{3}/dnf.repo?arch={4}".format(
self.protocol, self.host, self.name, chroot, self.arch
)
try:
r = open_url(url)
status_code = r.getcode()
repo_info = r.read().decode("utf-8")
except HTTPError as e:
status_code = e.getcode()
return repo_info, status_code
def _retrieve_servers(api_key):
api_url = '%s/v1/server/list' % VULTR_API_ENDPOINT
try:
response = open_url(
api_url,
headers={
'API-Key': api_key,
'Content-type': 'application/json'
},
http_agent=VULTR_USER_AGENT,
)
servers_list = json.loads(response.read())
return servers_list.values() if servers_list else []
except ValueError:
raise AnsibleError("Incorrect JSON payload")
except Exception as e:
raise AnsibleError("Error while fetching %s: %s" %
(api_url, to_native(e)))
def delete_request(self, uri, pyld, hdrs):
try:
resp = open_url(uri,
data=json.dumps(pyld),
headers=hdrs,
method="DELETE",
url_username=self.creds['user'],
url_password=self.creds['pswd'],
force_basic_auth=True,
validate_certs=False,
follow_redirects='all',
use_proxy=False)
except HTTPError as e:
return {'ret': False, 'msg': "HTTP Error: %s" % e.code}
except URLError as e:
return {'ret': False, 'msg': "URL Error: %s" % e.reason}
# Almost all errors should be caught above, but just in case
except:
return {'ret': False, 'msg': "Unknown error"}
return {'ret': True, 'resp': resp}
def send_msg_v2(self, msg, msg_format='text', color='yellow', notify=False):
"""Method for sending a message to HipChat"""
headers = {'Authorization': 'Bearer %s' % self.token, 'Content-Type': 'application/json'}
body = {}
body['room_id'] = self.room
body['from'] = self.from_name[:15] # max length is 15
body['message'] = msg
body['message_format'] = msg_format
body['color'] = color
body['notify'] = self.allow_notify and notify
data = json.dumps(body)
url = self.API_V2_URL + "room/{room_id}/notification".format(room_id=self.room)
try:
response = open_url(url, data=data, headers=headers, method='POST')
return response.read()
except Exception as ex:
self._display.warning('Could not submit message to hipchat: {0}'.format(ex))
def _get_server_api_version(self):
"""
Fetches the Galaxy API current version to ensure
the API server is up and reachable.
"""
url = '%s/api/' % self._api_server
try:
return_data = open_url(url, validate_certs=self._validate_certs)
except Exception as e:
raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_native(e)))
try:
data = json.loads(to_text(return_data.read(), errors='surrogate_or_strict'))
except Exception as e:
raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e)))
if 'current_version' not in data:
raise AnsibleError("missing required 'current_version' from server response (%s)" % url)
return data['current_version']
def get_health(self):
self._ensure_loggedin()
url = urllib.parse.urljoin(self.api_base_url, './health')
headers = {
'Accepts': 'application/json',
'Authorization': self.authorization
}
for attempt in range(1, self.retries + 1):
try:
response = open_url(url,
None,
headers,
'GET',
validate_certs=False,
timeout=self.api_timeout)
return json.load(response)
except Exception as e:
if self.should_retry_error(e, attempt):
continue
return self.handle_error('Failed to get console health', e)
def test_open_url_no_validate_certs(urlopen_mock, install_opener_mock):
r = open_url('https://ansible.com/', validate_certs=False)
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
ssl_handler = None
for handler in handlers:
if isinstance(handler, HTTPSClientAuthHandler):
ssl_handler = handler
break
assert ssl_handler is not None
context = ssl_handler._context
assert context.protocol == ssl.PROTOCOL_SSLv23
if ssl.OP_NO_SSLv2:
assert context.options & ssl.OP_NO_SSLv2
assert context.options & ssl.OP_NO_SSLv3
assert context.verify_mode == ssl.CERT_NONE
assert context.check_hostname is False
def update_group(self, grouprep, realm="master"):
""" Update an existing group.
:param grouprep: A GroupRepresentation of the updated group.
:return HTTPResponse object on success
"""
group_url = URL_GROUP.format(url=self.baseurl,
realm=realm,
groupid=grouprep['id'])
try:
return open_url(group_url,
method='PUT',
headers=self.restheaders,
data=json.dumps(grouprep),
validate_certs=self.validate_certs)
except Exception as e:
self.module.fail_json(
msg='Could not update group %s in realm %s: %s' %
(grouprep['name'], realm, str(e)))
def wait_for(self, timeout):
started = False
for x in range(timeout):
try:
url = urllib.parse.urljoin(self.operations_url, '/healthz')
response = open_url(url,
None,
None,
method='GET',
validate_certs=False)
if response.code == 200:
healthz = json.load(response)
if healthz['status'] == 'OK':
started = True
break
except Exception:
pass
time.sleep(1)
if not started:
raise Exception(f'Peer failed to start within {timeout} seconds')
def get_settings(self):
self._ensure_loggedin()
url = urllib.parse.urljoin(self.api_endpoint, '/ak/api/v2/settings')
headers = {
'Accepts': 'application/json',
'Authorization': self.authorization
}
for attempt in range(self.retries):
try:
response = open_url(url,
None,
headers,
'GET',
validate_certs=False,
timeout=self.api_timeout)
return json.load(response)
except Exception as e:
if self.should_retry_error(e):
continue
return self.handle_error('Failed to get console settings', e)
def save_configuration(self):
url = self.host.configManager.firmwareSystem.BackupFirmwareConfiguration()
url = url.replace('*', self.host.name)
if os.path.isdir(self.dest):
filename = url.rsplit('/', 1)[1]
self.dest = os.path.join(self.dest, filename)
else:
filename, file_extension = os.path.splitext(self.dest)
if file_extension != ".tgz":
self.dest = filename + ".tgz"
try:
request = open_url(url=url, validate_certs=self.validate_certs)
with open(self.dest, "wb") as file:
file.write(request.read())
self.module.exit_json(changed=True, dest_file=self.dest)
except IOError as e:
self.module.fail_json(msg="Failed to write backup file. Ensure that "
"the dest path exists and is writable. Details : %s" % to_native(e))
except Exception as e:
self.module.fail_json(msg=to_native(e))
def get_groups(self, realm="master"):
""" Fetch the name and ID of all groups on the Keycloak server.
To fetch the full data of the group, make a subsequent call to
get_group_by_groupid, passing in the ID of the group you wish to return.
:param realm: Return the groups of this realm (default "master").
"""
groups_url = URL_GROUPS.format(url=self.baseurl, realm=realm)
try:
return json.loads(
to_native(
open_url(groups_url,
method="GET",
headers=self.restheaders,
validate_certs=self.validate_certs).read()))
except Exception as e:
self.module.fail_json(
msg="Could not fetch list of groups in realm %s: %s" %
(realm, str(e)))
def send(self, path, data, **kwargs):
'''
Sends the command to the device over api
'''
url_kwargs = dict(
url_username=self.get_option('remote_user'),
url_password=self.get_option('password'),
timeout=self.get_option('timeout'),
)
url_kwargs.update(kwargs)
try:
response = open_url(self._url + path, data=data, **url_kwargs)
except URLError as exc:
raise AnsibleConnectionFailure(
'Could not connect to {0}: {1}'.format(self._url, exc.reason))
self._auth = response.info().get('Set-Cookie')
return response
def _get_endpoints_by_group(group_id, page, result):
url = url_builder(ssl, server, port, ISE_URL['endpoint'] + "?filter=groupId.EQ." + group_id + "&page=" + str(page) + "&size=100")
headers = {'Accept': ISE_NSPC['endpoint']}
method = "GET"
try:
con = open_url(url, headers=headers, method=method, use_proxy=False,force_basic_auth=True,
validate_certs=validate_certs, url_username=username, url_password=password)
except urllib.error.HTTPError:
return False
if con.code == 200:
tree = ET.fromstring(con.read())
for e in tree.iter():
if (e.tag == '{ers.ise.cisco.com}resource'):
ep = endpoint(e.get('id'),e.get('name'),e.get('description'))
result[e.get('name')] = ep
if (e.attrib.get('total') != None):
if (e.attrib.get('total') == '0'):
return False
return True
return False
def get_request(self, uri):
if 'token' in self.creds:
headers = {"X-Auth-Token": self.creds['token']}
try:
resp = open_url(uri,
method="GET",
url_username=self.creds['user'],
url_password=self.creds['pswd'],
force_basic_auth=True,
validate_certs=False,
timeout=10,
use_proxy=False)
data = json.loads(resp.read())
except HTTPError as e:
return {'ret': False, 'msg': "HTTP Error: %s" % e.code}
except URLError as e:
return {'ret': False, 'msg': "URL Error: %s" % e.reason}
except Exception as e:
return {'ret': False, 'msg': "Other error: %s" % e.message}
return {'ret': True, 'data': data}
def _request_spotinst(self, endpoint, method='GET'):
'''
Interract with Spotinst API.
:param endpoint: The spotinst API endpoint to query
'''
spotinst_api = "https://api.spotinst.io"
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer {}".format(self.spotinst_api_token)
}
uri = "{}/{}".format(spotinst_api, endpoint)
req = open_url(uri,
method=method,
headers=headers,
validate_certs=True)
res = json.loads(req.read())
return res
def delete_request(self, uri, pyld, hdrs):
if 'token' in self.creds:
headers = {"X-Auth-Token": self.creds['token']}
try:
resp = open_url(uri,
data=json.dumps(pyld),
headers=hdrs,
method="DELETE",
url_username=self.creds['user'],
url_password=self.creds['pswd'],
force_basic_auth=True,
validate_certs=False,
use_proxy=False)
except HTTPError as e:
return {'ret': False, 'msg': "HTTP Error: %s" % e.code}
except URLError as e:
return {'ret': False, 'msg': "URL Error: %s" % e.reason}
except Exception as e:
return {'ret': False, 'msg': "Other error: %s" % e.message}
return {'ret': True, 'resp': resp}
def test_open_url_auth_in_netloc(urlopen_mock, install_opener_mock):
r = open_url('http://*****:*****@ansible.com/')
args = urlopen_mock.call_args[0]
req = args[0]
assert req.get_full_url() == 'http://ansible.com/'
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
expected_handlers = (
urllib_request.HTTPBasicAuthHandler,
urllib_request.HTTPDigestAuthHandler,
)
found_handlers = []
for handler in handlers:
if isinstance(handler, expected_handlers):
found_handlers.append(handler)
assert len(found_handlers) == 2
def __init__(self, module):
self.module = module
self.params = module.params
self.url = self.params['url'].rstrip('/')
self.user = self.params['user']
self.password = self.params['password']
self.namespace = self.params['namespace']
try:
auth = open_url(
'{0}/auth'.format(self.url),
url_username=self.user,
url_password=self.password,
force_basic_auth=True,
)
except (HTTPError, URLError) as e:
module.fail_json(msg='authentication failed: {0}'.format(e.reason))
parsed = json.loads(auth.read())
self.token = parsed['access_token']
def _get_computer_info(self, computer):
computer_url = 'computer/{0}/config.xml'.format(computer)
jenkins_host = self._get_jenkins_host()
config_url = '{0}/{1}'.format(jenkins_host, computer_url)
r = open_url(config_url,
use_proxy=False,
validate_certs=False,
method='GET',
timeout=self._options.get('timeout', None),
force_basic_auth=(not self._must_login()),
url_username=self._get_jenkins_user(),
url_password=self._get_jenkins_pass(),
force=False,
headers=self._get_headers())
xml_config = r.read()
computer = objectify.fromstring(xml_config)
return computer
def create_github_token(self):
'''
Create a personal authorization token with a note of 'ansible-galaxy login'
'''
self.remove_github_token()
args = json.dumps({
"scopes": ["public_repo"],
"note": "ansible-galaxy login"
})
try:
data = json.load(
open_url(self.GITHUB_AUTH,
url_username=self.github_username,
url_password=self.github_password,
force_basic_auth=True,
data=args))
except HTTPError as e:
res = json.load(e)
raise AnsibleError(res['message'])
return data['token']
