def get_s3_connection(aws_connect_kwargs, location, rgw, s3_url):
if s3_url and rgw:
rgw = urlparse(s3_url)
s3 = boto.connect_s3(
is_secure=rgw.scheme == 'https',
host=rgw.hostname,
port=rgw.port,
calling_format=OrdinaryCallingFormat(),
**aws_connect_kwargs
)
elif is_fakes3(s3_url):
fakes3 = urlparse(s3_url)
s3 = S3Connection(
is_secure=fakes3.scheme == 'fakes3s',
host=fakes3.hostname,
port=fakes3.port,
calling_format=OrdinaryCallingFormat(),
**aws_connect_kwargs
)
elif is_walrus(s3_url):
walrus = urlparse(s3_url).hostname
s3 = boto.connect_walrus(walrus, **aws_connect_kwargs)
else:
aws_connect_kwargs['is_secure'] = True
try:
s3 = connect_to_aws(boto.s3, location, **aws_connect_kwargs)
except AnsibleAWSError:
# use this as fallback because connect_to_region seems to fail in boto + non 'classic' aws accounts in some cases
s3 = boto.connect_s3(**aws_connect_kwargs)
return s3
def test_generic_urlparse():
url = 'https://ansible.com/blog'
parts = urlparse(url)
generic_parts = generic_urlparse(parts)
assert generic_parts.as_list() == list(parts)
assert urlunparse(generic_parts.as_list()) == url
def run(self, terms, variables=None, **kwargs):
if not HAS_CONSUL:
raise AnsibleError('python-consul is required for consul_kv lookup. see http://python-consul.readthedocs.org/en/latest/#installation')
values = []
try:
for term in terms:
params = self.parse_params(term)
try:
url = os.environ['ANSIBLE_CONSUL_URL']
u = urlparse(url)
consul_api = consul.Consul(host=u.hostname, port=u.port)
except KeyError:
port = kwargs.get('port', '8500')
host = kwargs.get('host', 'localhost')
consul_api = consul.Consul(host=host, port=port)
results = consul_api.kv.get(params['key'],
token=params['token'],
index=params['index'],
recurse=params['recurse'])
if results[1]:
# responds with a single or list of result maps
if isinstance(results[1], list):
for r in results[1]:
values.append(r['Value'])
else:
values.append(results[1]['Value'])
except Exception as e:
raise AnsibleError(
"Error locating '%s' in kv store. Error was %s" % (term, e))
return values
def get_fqdn_and_port(repo_url):
""" chop the hostname and port out of a url """
fqdn = None
port = None
ipv6_re = re.compile(r'(\[[^]]*\])(?::([0-9]+))?')
if "@" in repo_url and "://" not in repo_url:
# most likely an user@host:path or user@host/path type URL
repo_url = repo_url.split("@", 1)[1]
match = ipv6_re.match(repo_url)
# For this type of URL, colon specifies the path, not the port
if match:
fqdn, path = match.groups()
elif ":" in repo_url:
fqdn = repo_url.split(":")[0]
elif "/" in repo_url:
fqdn = repo_url.split("/")[0]
elif "://" in repo_url:
# this should be something we can parse with urlparse
parts = urlparse(repo_url)
# parts[1] will be empty on python2.4 on ssh:// or git:// urls, so
# ensure we actually have a parts[1] before continuing.
if parts[1] != '':
fqdn = parts[1]
if "@" in fqdn:
fqdn = fqdn.split("@", 1)[1]
match = ipv6_re.match(fqdn)
if match:
fqdn, port = match.groups()
elif ":" in fqdn:
fqdn, port = fqdn.split(":")[0:2]
return fqdn, port
def __init__(self, args):
self._args = args
self._cloud_environment = None
self._compute_client = None
self._resource_client = None
self._network_client = None
self.debug = False
if args.debug:
self.debug = True
self.credentials = self._get_credentials(args)
if not self.credentials:
self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
"or define a profile in ~/.azure/credentials.")
# if cloud_environment specified, look up/build Cloud object
raw_cloud_env = self.credentials.get('cloud_environment')
if not raw_cloud_env:
self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default
else:
# try to look up "well-known" values via the name attribute on azure_cloud members
all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)]
matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
if len(matched_clouds) == 1:
self._cloud_environment = matched_clouds[0]
elif len(matched_clouds) > 1:
self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env))
else:
if not urlparse.urlparse(raw_cloud_env).scheme:
self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds]))
try:
self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env)
except Exception as e:
self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message))
if self.credentials.get('subscription_id', None) is None:
self.fail("Credentials did not include a subscription_id value.")
self.log("setting subscription_id")
self.subscription_id = self.credentials['subscription_id']
if self.credentials.get('credentials'):
self.azure_credentials = self.credentials.get('credentials')
elif self.credentials.get('client_id') and self.credentials.get('secret') and self.credentials.get('tenant'):
self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=self.credentials['tenant'],
cloud_environment=self._cloud_environment)
elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
tenant = self.credentials.get('tenant')
if not tenant:
tenant = 'common'
self.azure_credentials = UserPassCredentials(self.credentials['ad_user'],
self.credentials['password'],
tenant=tenant,
cloud_environment=self._cloud_environment)
else:
self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
"Credentials must include client_id, secret and tenant or ad_user and password.")
def test_generic_urlparse_no_netloc_no_host():
url = '/blog'
parts = list(urlparse(url))
generic_parts = generic_urlparse(parts)
assert generic_parts.username is None
assert generic_parts.password is None
assert generic_parts.port is None
assert generic_parts.hostname == ''
def test_generic_urlparse_netloc():
url = 'https://ansible.com:443/blog'
parts = urlparse(url)
generic_parts = generic_urlparse(parts)
assert generic_parts.hostname == parts.hostname
assert generic_parts.hostname == 'ansible.com'
assert generic_parts.port == 443
assert urlunparse(generic_parts.as_list()) == url
def is_walrus(s3_url):
""" Return True if it's Walrus endpoint, not S3
We assume anything other than *.amazonaws.com is Walrus"""
if s3_url is not None:
o = urlparse.urlparse(s3_url)
return not o.hostname.endswith('amazonaws.com')
else:
return False
def test_generic_urlparse_no_netloc():
url = 'https://*****:*****@ansible.com:443/blog'
parts = list(urlparse(url))
generic_parts = generic_urlparse(parts)
assert generic_parts.hostname == 'ansible.com'
assert generic_parts.port == 443
assert generic_parts.username == 'user'
assert generic_parts.password == 'passwd'
assert urlunparse(generic_parts.as_list()) == url
def do_notify_bearychat(module, url, payload):
response, info = fetch_url(module, url, data=payload)
if info['status'] != 200:
url_info = urlparse(url)
obscured_incoming_webhook = urlunparse(
(url_info.scheme, url_info.netloc, '[obscured]', '', '', ''))
module.fail_json(
msg=" failed to send %s to %s: %s" % (
payload, obscured_incoming_webhook, info['msg']))
def verify_md5(self, file, remote_url):
if os.path.exists(file):
local_md5 = self._local_md5(file)
if self.local:
parsed_url = urlparse(remote_url)
remote_md5 = self._local_md5(parsed_url.path)
else:
remote_md5 = self._getContent(remote_url + '.md5', "Failed to retrieve MD5", False)
return local_md5 == remote_md5
return False
def is_registry_match(item, pattern):
"""returns True if the registry matches the given whitelist pattern
Unlike in OpenShift, the comparison is done solely on hostname part
(excluding the port part) since the latter is much more difficult due to
vague definition of port defaulting based on insecure flag. Moreover, most
of the registries will be listed without the port and insecure flag.
"""
item = "schema://" + item.split('://', 1)[-1]
return is_match(urlparse(item).hostname, pattern.rsplit(':', 1)[0])
def _get_policy_id_from_response(response):
"""
Method to parse out the policy id from creation response
:param response: response from firewall creation API call
:return: policy_id: firewall policy id from creation call
"""
url = response.get('links')[0]['href']
path = urlparse(url).path
path_list = os.path.split(path)
policy_id = path_list[-1]
return policy_id
def get_s3_connection(module, aws_connect_kwargs, location, rgw, s3_url):
if s3_url and rgw: # TODO - test this
rgw = urlparse(s3_url)
params = dict(module=module, conn_type='client', resource='s3', use_ssl=rgw.scheme == 'https', region=location, endpoint=s3_url, **aws_connect_kwargs)
elif is_fakes3(s3_url):
for kw in ['is_secure', 'host', 'port'] and list(aws_connect_kwargs.keys()):
del aws_connect_kwargs[kw]
fakes3 = urlparse(s3_url)
if fakes3.scheme == 'fakes3s':
protocol = "https"
else:
protocol = "http"
params = dict(service_name='s3', endpoint_url="%s://%s:%s" % (protocol, fakes3.hostname, to_text(fakes3.port)),
use_ssl=fakes3.scheme == 'fakes3s', region_name=None, **aws_connect_kwargs)
elif is_walrus(s3_url):
walrus = urlparse(s3_url).hostname
params = dict(module=module, conn_type='client', resource='s3', region=location, endpoint=walrus, **aws_connect_kwargs)
else:
params = dict(module=module, conn_type='client', resource='s3', region=location, endpoint=s3_url, **aws_connect_kwargs)
return boto3_conn(**params)
def lib_utils_oo_hostname_from_url(url):
""" Returns the hostname contained in a URL
Ex: https://ose3-master.example.com/v1/api -> ose3-master.example.com
"""
if not isinstance(url, string_types):
raise errors.AnsibleFilterError("|failed expects a string or unicode")
parse_result = urlparse(url)
if parse_result.netloc != '':
return parse_result.netloc
else:
# netloc wasn't parsed, assume url was missing scheme and path
return parse_result.path
def _getContent(self, url, failmsg, force=True):
if self.local:
parsed_url = urlparse(url)
if os.path.isfile(parsed_url.path):
with io.open(parsed_url.path, 'rb') as f:
return f.read()
if force:
raise ValueError(failmsg + " because can not find file: " + url)
return None
response = self._request(url, failmsg, force)
if response:
return response.read()
return None
def update_qsl(url, params):
''' Add or update a URL query string '''
if HAS_URLPARSE:
url_parts = list(urlparse(url))
query = dict(parse_qsl(url_parts[4]))
query.update(params)
url_parts[4] = urlencode(query)
return urlunparse(url_parts)
elif '?' in url:
return url + '&' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()])
else:
return url + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()])
def _request(self, url, failmsg, force=True):
url_to_use = url
parsed_url = urlparse(url)
if parsed_url.scheme == 's3':
parsed_url = urlparse(url)
bucket_name = parsed_url.netloc
key_name = parsed_url.path[1:]
client = boto3.client('s3', aws_access_key_id=self.module.params.get('username', ''), aws_secret_access_key=self.module.params.get('password', ''))
url_to_use = client.generate_presigned_url('get_object', Params={'Bucket': bucket_name, 'Key': key_name}, ExpiresIn=10)
req_timeout = self.module.params.get('timeout')
# Hack to add parameters in the way that fetch_url expects
self.module.params['url_username'] = self.module.params.get('username', '')
self.module.params['url_password'] = self.module.params.get('password', '')
self.module.params['http_agent'] = self.module.params.get('user_agent', None)
response, info = fetch_url(self.module, url_to_use, timeout=req_timeout)
if info['status'] == 200:
return response
if force:
raise ValueError(failmsg + " because of " + info['msg'] + "for URL " + url_to_use)
return None
def detect_no_proxy(self, url):
'''
Detect if the 'no_proxy' environment variable is set and honor those locations.
'''
env_no_proxy = os.environ.get('no_proxy')
if env_no_proxy:
env_no_proxy = env_no_proxy.split(',')
netloc = urlparse(url).netloc
for host in env_no_proxy:
if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
# Our requested URL matches something in no_proxy, so don't
# use the proxy for this
return False
return True
def auth_params(self):
# Get authentication credentials.
# Precedence: module parameters-> environment variables-> defaults.
self.log('Getting credentials')
params = dict()
for key in DOCKER_COMMON_ARGS:
params[key] = self.module.params.get(key)
if self.module.params.get('use_tls'):
# support use_tls option in docker_image.py. This will be deprecated.
use_tls = self.module.params.get('use_tls')
if use_tls == 'encrypt':
params['tls'] = True
if use_tls == 'verify':
params['tls_verify'] = True
result = dict(
docker_host=self._get_value('docker_host', params['docker_host'], 'DOCKER_HOST',
DEFAULT_DOCKER_HOST),
tls_hostname=self._get_value('tls_hostname', params['tls_hostname'],
'DOCKER_TLS_HOSTNAME', 'localhost'),
api_version=self._get_value('api_version', params['api_version'], 'DOCKER_API_VERSION',
'auto'),
cacert_path=self._get_value('cacert_path', params['cacert_path'], 'DOCKER_CERT_PATH', None),
cert_path=self._get_value('cert_path', params['cert_path'], 'DOCKER_CERT_PATH', None),
key_path=self._get_value('key_path', params['key_path'], 'DOCKER_CERT_PATH', None),
ssl_version=self._get_value('ssl_version', params['ssl_version'], 'DOCKER_SSL_VERSION', None),
tls=self._get_value('tls', params['tls'], 'DOCKER_TLS', DEFAULT_TLS),
tls_verify=self._get_value('tls_verfy', params['tls_verify'], 'DOCKER_TLS_VERIFY',
DEFAULT_TLS_VERIFY),
timeout=self._get_value('timeout', params['timeout'], 'DOCKER_TIMEOUT',
DEFAULT_TIMEOUT_SECONDS),
)
if result['tls_hostname'] is None:
# get default machine name from the url
parsed_url = urlparse(result['docker_host'])
if ':' in parsed_url.netloc:
result['tls_hostname'] = parsed_url.netloc[:parsed_url.netloc.rindex(':')]
else:
result['tls_hostname'] = parsed_url
return result
def maybe_add_ssl_handler(url, validate_certs):
parsed = generic_urlparse(urlparse(url))
if parsed.scheme == 'https' and validate_certs:
if not HAS_SSL:
raise NoSSLError('SSL validation is not available in your version of python. You can use validate_certs=False,'
' however this is unsafe and not recommended')
# do the cert validation
netloc = parsed.netloc
if '@' in netloc:
netloc = netloc.split('@', 1)[1]
if ':' in netloc:
hostname, port = netloc.split(':', 1)
port = int(port)
else:
hostname = netloc
port = 443
# create the SSL validation handler and
# add it to the list of handlers
return SSLValidationHandler(hostname, port)
def download(self, artifact, verify_download, filename=None):
filename = artifact.get_filename(filename)
if not artifact.version or artifact.version == "latest":
artifact = Artifact(artifact.group_id, artifact.artifact_id, self.find_latest_version_available(artifact),
artifact.classifier, artifact.extension)
url = self.find_uri_for_artifact(artifact)
if self.local:
parsed_url = urlparse(url)
if os.path.isfile(parsed_url.path):
shutil.copy2(parsed_url.path, filename)
else:
return "Can not find local file: " + parsed_url.path
else:
response = self._request(url, "Failed to download artifact " + str(artifact))
with io.open(filename, 'wb') as f:
self._write_chunks(response, f, report_hook=self.chunk_report)
if verify_download and not self.verify_md5(filename, url):
# if verify_change was set, the previous file would be deleted
os.remove(filename)
return "Checksum verification failed"
return None
def maybe_add_ssl_handler(url, validate_certs):
# FIXME: change the following to use the generic_urlparse function
# to remove the indexed references for 'parsed'
parsed = urlparse(url)
if parsed[0] == 'https' and validate_certs:
if not HAS_SSL:
raise NoSSLError('SSL validation is not available in your version of python. You can use validate_certs=False,'
' however this is unsafe and not recommended')
# do the cert validation
netloc = parsed[1]
if '@' in netloc:
netloc = netloc.split('@', 1)[1]
if ':' in netloc:
hostname, port = netloc.split(':', 1)
port = int(port)
else:
hostname = netloc
port = 443
# create the SSL validation handler and
# add it to the list of handlers
return SSLValidationHandler(hostname, port)
def handle_uri(self):
'''Allow uri module on localhost if it doesn't touch unsafe files.
The :ansible:module:`uri` can be used from the executor to do
things like pinging readthedocs.org that otherwise don't need a node.
However, it can also download content to a local file, or be used to
read from file:/// urls.
Block any use of url schemes other than https, http and ftp. Further,
block any local file interaction that falls outside of the zuul
work dir.
'''
# uri takes all the file arguments, so just let handle_file validate
# them for us.
self.handle_file()
scheme = urlparse(self._task.args['url']).scheme
if scheme not in ALLOWED_URL_SCHEMES:
raise AnsibleError(
"{scheme} urls are not allowed from localhost."
" Only {allowed_schemes} are allowed".format(
scheme=scheme,
allowed_schemes=ALLOWED_URL_SCHEMES))
def get_consul_api(self):
'''get an instance of the api based on the supplied configuration'''
host = 'localhost'
port = 8500
token = None
scheme = 'http'
if hasattr(self, 'url'):
from ansible.module_utils.six.moves.urllib.parse import urlparse
o = urlparse(self.url)
if o.hostname:
host = o.hostname
if o.port:
port = o.port
if o.scheme:
scheme = o.scheme
if hasattr(self, 'token'):
token = self.token
if not token:
token = 'anonymous'
return consul.Consul(host=host, port=port, token=token, scheme=scheme)
def api_call(self, path):
''' Performs an API request '''
headers = {
'X-API-Token': self.token,
'X-API-Version': self.version,
'Content-Type': 'application/json;charset=utf-8'
}
target = urlparse(self.uri + path)
method = 'GET'
body = ''
try:
response, content = self.conn.request(target.geturl(), method, body, headers)
except:
self.fail_with_error('Error connecting to Rudder server')
try:
data = json.loads(content)
except ValueError as e:
self.fail_with_error('Could not parse JSON response from Rudder API', 'reading API response')
return data
def is_fakes3(s3_url):
""" Return True if s3_url has scheme fakes3:// """
if s3_url is not None:
return urlparse.urlparse(s3_url).scheme in ('fakes3', 'fakes3s')
else:
return False
def main():
# Validates the dependence of the utility module
if HAS_IDG_DEPS:
module_args = dict(
state=dict(type='str',
required=False,
default='directory',
choices=['absent', 'directory', 'move',
'show']), # State alternatives
path=dict(type='str', required=True), # Path to resource
source=dict(
type='str',
required=False), # Source. Only valid when state = move
overwrite=dict(
type='bool', required=False,
default=False), # overwrite target. Valid when state = move
domain=dict(type='str', required=True), # Domain name
idg_connection=dict(type='dict',
options=idg_endpoint_spec,
required=True) # IDG connection
)
# AnsibleModule instantiation
module = AnsibleModule(
argument_spec=module_args,
supports_check_mode=True,
required_if=[["state", "move", ["source", "overwrite"]]])
else:
# Failure AnsibleModule instance
module = AnsibleModule(argument_spec={}, check_invalid_arguments=False)
module.fail_json(msg="The IDG utils modules is required")
# Parse arguments to dict
idg_data_spec = IDGUtils.parse_to_dict(module,
module.params['idg_connection'],
'IDGConnection',
IDGUtils.ANSIBLE_VERSION)
path = module.params['path']
state = module.params['state']
domain_name = module.params['domain']
# Init IDG API connect
idg_mgmt = IDGApi(ansible_module=module,
idg_host="https://{0}:{1}".format(
idg_data_spec['server'],
idg_data_spec['server_port']),
headers=IDGUtils.BASIC_HEADERS,
http_agent=IDGUtils.HTTP_AGENT_SPEC,
use_proxy=idg_data_spec['use_proxy'],
timeout=idg_data_spec['timeout'],
validate_certs=idg_data_spec['validate_certs'],
user=idg_data_spec['user'],
password=idg_data_spec['password'],
force_basic_auth=IDGUtils.BASIC_AUTH_SPEC)
# Intermediate values for result
tmp_result = {
"domain": domain_name,
"msg": None,
"path": None,
"changed": None,
"output": None
}
#
# Here the action begins
#
pdb.set_trace()
try:
# Do request
parse = urlparse(path)
ldir = parse.scheme # Local directory
rpath = parse.path # Relative path
path_as_list = [d for d in rpath.split('/') if d.strip() != '']
td = '/'.join([IDGApi.URI_FILESTORE.format(domain_name),
ldir]) # Path prefix
if state == 'directory':
# Create directory recursively
for d in path_as_list:
idg_mgmt.api_call(td, method='GET', id="get_remote_directory")
if idg_mgmt.is_ok(idg_mgmt.last_call()):
td = '/'.join([td, d])
tmp_result['path'] = idg_mgmt.apifilestore_uri2path(td)
if 'directory' in idg_mgmt.last_call(
)["data"]['filestore']['location'].keys():
filestore_abst = AbstractListDict(
idg_mgmt.last_call()["data"]['filestore']
['location']['directory']) # Get directories
else:
filestore_abst = AbstractListDict(
{}) # Not contain directories
if ('href' in filestore_abst.keys()) and (
td in filestore_abst.values(
key='href')): # if directory exist
tmp_result['msg'] = IDGUtils.IMMUTABLE_MESSAGE
else: # Not exist, create it
# If the user is working in only check mode we do not want to make any changes
IDGUtils.implement_check_mode(module)
create_dir_msg = {"directory": {"name": d}}
idg_mgmt.api_call(td,
method='PUT',
data=json.dumps(create_dir_msg),
id="create_directory")
if idg_mgmt.is_created(idg_mgmt.last_call()):
tmp_result['msg'] = idg_mgmt.last_call(
)["data"]['result']
tmp_result['changed'] = True
else:
module.fail_json(
msg=IDGApi.ERROR_REACH_STATE.format(
state, domain_name) + str(
ErrorHandler(idg_mgmt.last_call()
["data"]['error'])))
else:
module.fail_json(msg=IDGApi.GENERAL_ERROR.format(
__MODULE_FULLNAME, state, domain_name) + str(
ErrorHandler(
idg_mgmt.call_by_id("get_remote_directory")
["data"]['error'])))
elif state == 'move': # Move remote files
# If the user is working in only check mode we do not want to make any changes
IDGUtils.implement_check_mode(module)
move_file_msg = {
"MoveFile": {
"sURL": module.params['source'].strip('/'),
"dURL": path.strip('/'),
"Overwrite":
IDGUtils.str_on_off(module.params['overwrite'])
}
}
tmp_result['path'] = move_file_msg['MoveFile']['dURL']
idg_mgmt.api_call(IDGApi.URI_ACTION.format(domain_name),
method='POST',
data=json.dumps(move_file_msg),
id="move_file")
if idg_mgmt.is_ok(idg_mgmt.last_call()):
tmp_result['msg'] = idg_mgmt.last_call()["data"]['MoveFile']
tmp_result['changed'] = True
else:
module.fail_json(
msg=IDGApi.ERROR_REACH_STATE.format(state, domain_name) +
str(ErrorHandler(idg_mgmt.last_call()["data"]['error'])))
elif state == 'show': # Show details of file or content of directories
# If the user is working in only check mode we do not want to make any changes
IDGUtils.implement_check_mode(module)
list_target = '/'.join([td] + path_as_list)
idg_mgmt.api_call(list_target,
method='GET',
id="get_remote_target")
if idg_mgmt.is_ok(idg_mgmt.last_call()):
output = {}
if 'filestore' in idg_mgmt.last_call()["data"].keys(
): # is directory
if 'directory' in idg_mgmt.last_call(
)["data"]['filestore']['location'].keys():
output['directory'] = [{
"name": i["name"]
} for i in AbstractListDict(
idg_mgmt.last_call()["data"]['filestore']
['location']['directory']).raw_data()]
if 'file' in idg_mgmt.last_call(
)["data"]['filestore']['location'].keys():
output['file'] = [{
"name": i["name"],
"size": i["size"],
"modified": i["modified"]
} for i in AbstractListDict(idg_mgmt.last_call(
)["data"]['filestore']['location']['file']).raw_data()]
else:
idg_mgmt.api_call('/'.join([td] + path_as_list[:-1]),
method='GET',
id="get_file_detail")
if idg_mgmt.is_ok(idg_mgmt.last_call()):
output = [{
"name": i["name"],
"size": i["size"],
"modified": i["modified"]
} for i in idg_mgmt.last_call()["data"]['filestore']
['location']['file']
if i['name'] == path_as_list[-1]]
else:
module.fail_json(msg=IDGApi.ERROR_REACH_STATE.format(
state, domain_name) + str(
ErrorHandler(idg_mgmt.last_call()["data"]
['error'])))
tmp_result['msg'] = IDGUtils.COMPLETED_MESSAGE
tmp_result['path'] = idg_mgmt.apifilestore_uri2path(
list_target)
tmp_result['output'] = output
else:
module.fail_json(msg=IDGApi.GENERAL_ERROR.format(
__MODULE_FULLNAME, state, domain_name) + str(
ErrorHandler(
idg_mgmt.call_by_id("get_remote_target")["data"]
['error'])))
else: # Remove
# Remove directory recursively
# If the user is working in only check mode we do not want to make any changes
IDGUtils.implement_check_mode(module)
td = '/'.join([td] + path_as_list)
idg_mgmt.api_call(td, method='DELETE', id="remove_remote_target")
tmp_result['path'] = idg_mgmt.apifilestore_uri2path(td)
if idg_mgmt.is_ok(idg_mgmt.last_call()):
tmp_result['msg'] = idg_mgmt.last_call()["data"]['result']
tmp_result['changed'] = True
elif idg_mgmt.is_notfound(idg_mgmt.last_call()):
tmp_result['msg'] = IDGUtils.IMMUTABLE_MESSAGE
else:
module.fail_json(
msg=IDGApi.ERROR_REACH_STATE.format(state, domain_name) +
str(ErrorHandler(idg_mgmt.last_call()["data"]['error'])))
#
# Finish
#
# Customize
del result['name']
# Update
for k, v in tmp_result.items():
if v is not None:
result[k] = v
except Exception as e:
# Uncontrolled exception
module.fail_json(msg=(IDGUtils.UNCONTROLLED_EXCEPTION +
'. {0}').format(to_native(e)))
else:
# That's all folks!
module.exit_json(**result)
def open_url(url,
data=None,
headers=None,
method=None,
use_proxy=True,
force=False,
last_mod_time=None,
timeout=10,
validate_certs=True,
url_username=None,
url_password=None,
http_agent=None,
force_basic_auth=False,
follow_redirects='urllib2'):
'''
Fetches a file from an HTTP/FTP server using urllib2
Does not require the module environment
'''
handlers = []
ssl_handler = maybe_add_ssl_handler(url, validate_certs)
if ssl_handler:
handlers.append(ssl_handler)
# FIXME: change the following to use the generic_urlparse function
# to remove the indexed references for 'parsed'
parsed = urlparse(url)
if parsed[0] != 'ftp':
username = url_username
if headers is None:
headers = {}
if username:
password = url_password
netloc = parsed[1]
elif '@' in parsed[1]:
credentials, netloc = parsed[1].split('@', 1)
if ':' in credentials:
username, password = credentials.split(':', 1)
else:
username = credentials
password = ''
parsed = list(parsed)
parsed[1] = netloc
# reconstruct url without credentials
url = urlunparse(parsed)
if username and not force_basic_auth:
passman = urllib_request.HTTPPasswordMgrWithDefaultRealm()
# this creates a password manager
passman.add_password(None, netloc, username, password)
# because we have put None at the start it will always
# use this username/password combination for urls
# for which `theurl` is a super-url
authhandler = urllib_request.HTTPBasicAuthHandler(passman)
# create the AuthHandler
handlers.append(authhandler)
elif username and force_basic_auth:
headers["Authorization"] = basic_auth_header(username, password)
else:
try:
rc = netrc.netrc(os.environ.get('NETRC'))
login = rc.authenticators(parsed[1])
except IOError:
login = None
if login:
username, _, password = login
if username and password:
headers["Authorization"] = basic_auth_header(
username, password)
if not use_proxy:
proxyhandler = urllib_request.ProxyHandler({})
handlers.append(proxyhandler)
if HAS_SSLCONTEXT and not validate_certs:
# In 2.7.9, the default context validates certificates
context = SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.options |= ssl.OP_NO_SSLv3
context.verify_mode = ssl.CERT_NONE
context.check_hostname = False
handlers.append(urllib_request.HTTPSHandler(context=context))
# pre-2.6 versions of python cannot use the custom https
# handler, since the socket class is lacking create_connection.
# Some python builds lack HTTPS support.
if hasattr(socket, 'create_connection') and CustomHTTPSHandler:
handlers.append(CustomHTTPSHandler)
handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs))
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
if method:
if method.upper() not in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT',
'DELETE', 'TRACE', 'CONNECT', 'PATCH'):
raise ConnectionError('invalid HTTP request method; %s' %
method.upper())
request = RequestWithMethod(url, method.upper(), data)
else:
request = urllib_request.Request(url, data)
# add the custom agent header, to help prevent issues
# with sites that block the default urllib agent string
request.add_header('User-agent', http_agent)
# if we're ok with getting a 304, set the timestamp in the
# header, otherwise make sure we don't get a cached copy
if last_mod_time and not force:
tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000')
request.add_header('If-Modified-Since', tstamp)
else:
request.add_header('cache-control', 'no-cache')
# user defined headers now, which may override things we've set above
if headers:
if not isinstance(headers, dict):
raise ValueError("headers provided to fetch_url() must be a dict")
for header in headers:
request.add_header(header, headers[header])
urlopen_args = [request, None]
if sys.version_info >= (2, 6, 0):
# urlopen in python prior to 2.6.0 did not
# have a timeout parameter
urlopen_args.append(timeout)
r = urllib_request.urlopen(*urlopen_args)
return r
def parse_rommon_file_location(rommon_file_location):
rommon_url = urlparse(rommon_file_location)
if rommon_url.scheme != 'tftp':
raise ValueError('The ROMMON image must be downloaded from TFTP server, other protocols are not supported.')
return rommon_url.netloc, rommon_url.path
def create_virtual_machine(module, azure):
"""
Create new virtual machine
module : AnsibleModule object
azure: authenticated azure ServiceManagementService object
Returns:
True if a new virtual machine and/or cloud service was created, false otherwise
"""
name = module.params.get('name')
os_type = module.params.get('os_type')
hostname = module.params.get('hostname') or name + ".cloudapp.net"
endpoints = module.params.get('endpoints').split(',')
ssh_cert_path = module.params.get('ssh_cert_path')
user = module.params.get('user')
password = module.params.get('password')
location = module.params.get('location')
role_size = module.params.get('role_size')
storage_account = module.params.get('storage_account')
image = module.params.get('image')
virtual_network_name = module.params.get('virtual_network_name')
wait = module.params.get('wait')
wait_timeout = int(module.params.get('wait_timeout'))
changed = False
# Check if a deployment with the same name already exists
cloud_service_name_available = azure.check_hosted_service_name_availability(name)
if cloud_service_name_available.result:
# cloud service does not exist; create it
try:
result = azure.create_hosted_service(service_name=name, label=name, location=location)
_wait_for_completion(azure, result, wait_timeout, "create_hosted_service")
changed = True
except AzureException as e:
module.fail_json(msg="failed to create the new service, error was: %s" % str(e))
try:
# check to see if a vm with this name exists; if so, do nothing
azure.get_role(name, name, name)
except AzureMissingException:
# vm does not exist; create it
if os_type == 'linux':
# Create linux configuration
disable_ssh_password_authentication = not password
vm_config = LinuxConfigurationSet(hostname, user, password, disable_ssh_password_authentication)
else:
# Create Windows Config
vm_config = WindowsConfigurationSet(hostname, password, None, module.params.get('auto_updates'), None, user)
vm_config.domain_join = None
if module.params.get('enable_winrm'):
listener = Listener('Http')
vm_config.win_rm.listeners.listeners.append(listener)
else:
vm_config.win_rm = None
# Add ssh certificates if specified
if ssh_cert_path:
fingerprint, pkcs12_base64 = get_ssh_certificate_tokens(module, ssh_cert_path)
# Add certificate to cloud service
result = azure.add_service_certificate(name, pkcs12_base64, 'pfx', '')
_wait_for_completion(azure, result, wait_timeout, "add_service_certificate")
# Create ssh config
ssh_config = SSH()
ssh_config.public_keys = PublicKeys()
authorized_keys_path = u'/home/%s/.ssh/authorized_keys' % user
ssh_config.public_keys.public_keys.append(PublicKey(path=authorized_keys_path, fingerprint=fingerprint))
# Append ssh config to linux machine config
vm_config.ssh = ssh_config
# Create network configuration
network_config = ConfigurationSetInputEndpoints()
network_config.configuration_set_type = 'NetworkConfiguration'
network_config.subnet_names = []
network_config.public_ips = None
for port in endpoints:
network_config.input_endpoints.append(ConfigurationSetInputEndpoint(name='TCP-%s' % port,
protocol='TCP',
port=port,
local_port=port))
# First determine where to store disk
today = datetime.date.today().strftime('%Y-%m-%d')
disk_prefix = u'%s-%s' % (name, name)
media_link = u'http://%s.blob.core.windows.net/vhds/%s-%s.vhd' % (storage_account, disk_prefix, today)
# Create system hard disk
os_hd = OSVirtualHardDisk(image, media_link)
# Spin up virtual machine
try:
result = azure.create_virtual_machine_deployment(service_name=name,
deployment_name=name,
deployment_slot='production',
label=name,
role_name=name,
system_config=vm_config,
network_config=network_config,
os_virtual_hard_disk=os_hd,
role_size=role_size,
role_type='PersistentVMRole',
virtual_network_name=virtual_network_name)
_wait_for_completion(azure, result, wait_timeout, "create_virtual_machine_deployment")
changed = True
except AzureException as e:
module.fail_json(msg="failed to create the new virtual machine, error was: %s" % str(e))
try:
deployment = azure.get_deployment_by_name(service_name=name, deployment_name=name)
return (changed, urlparse(deployment.url).hostname, deployment)
except AzureException as e:
module.fail_json(msg="failed to lookup the deployment information for %s, error was: %s" % (name, str(e)))
def open_url(url,
data=None,
headers=None,
method=None,
use_proxy=True,
force=False,
last_mod_time=None,
timeout=10,
validate_certs=True,
url_username=None,
url_password=None,
http_agent=None,
force_basic_auth=False,
follow_redirects='urllib2',
client_cert=None,
client_key=None,
cookies=None):
'''
Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
Does not require the module environment
'''
handlers = []
ssl_handler = maybe_add_ssl_handler(url, validate_certs)
if ssl_handler:
handlers.append(ssl_handler)
parsed = generic_urlparse(urlparse(url))
if parsed.scheme != 'ftp':
username = url_username
if headers is None:
headers = {}
if username:
password = url_password
netloc = parsed.netloc
elif '@' in parsed.netloc:
credentials, netloc = parsed.netloc.split('@', 1)
if ':' in credentials:
username, password = credentials.split(':', 1)
else:
username = credentials
password = ''
parsed_list = parsed.as_list()
parsed_list[1] = netloc
# reconstruct url without credentials
url = urlunparse(parsed_list)
if username and not force_basic_auth:
passman = urllib_request.HTTPPasswordMgrWithDefaultRealm()
# this creates a password manager
passman.add_password(None, netloc, username, password)
# because we have put None at the start it will always
# use this username/password combination for urls
# for which `theurl` is a super-url
authhandler = urllib_request.HTTPBasicAuthHandler(passman)
digest_authhandler = urllib_request.HTTPDigestAuthHandler(passman)
# create the AuthHandler
handlers.append(authhandler)
handlers.append(digest_authhandler)
elif username and force_basic_auth:
headers["Authorization"] = basic_auth_header(username, password)
else:
try:
rc = netrc.netrc(os.environ.get('NETRC'))
login = rc.authenticators(parsed.hostname)
except IOError:
login = None
if login:
username, _, password = login
if username and password:
headers["Authorization"] = basic_auth_header(
username, password)
if not use_proxy:
proxyhandler = urllib_request.ProxyHandler({})
handlers.append(proxyhandler)
if HAS_SSLCONTEXT and not validate_certs:
# In 2.7.9, the default context validates certificates
context = SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.options |= ssl.OP_NO_SSLv3
context.verify_mode = ssl.CERT_NONE
context.check_hostname = False
handlers.append(
HTTPSClientAuthHandler(client_cert=client_cert,
client_key=client_key,
context=context))
elif client_cert:
handlers.append(
HTTPSClientAuthHandler(client_cert=client_cert,
client_key=client_key))
# pre-2.6 versions of python cannot use the custom https
# handler, since the socket class is lacking create_connection.
# Some python builds lack HTTPS support.
if hasattr(socket, 'create_connection') and CustomHTTPSHandler:
handlers.append(CustomHTTPSHandler)
handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs))
# add some nicer cookie handling
if cookies is not None:
handlers.append(urllib_request.HTTPCookieProcessor(cookies))
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
data = to_bytes(data, nonstring='passthru')
if method:
if method.upper() not in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT',
'DELETE', 'TRACE', 'CONNECT', 'PATCH'):
raise ConnectionError('invalid HTTP request method; %s' %
method.upper())
request = RequestWithMethod(url, method.upper(), data)
else:
request = urllib_request.Request(url, data)
# add the custom agent header, to help prevent issues
# with sites that block the default urllib agent string
if http_agent:
request.add_header('User-agent', http_agent)
# Cache control
# Either we directly force a cache refresh
if force:
request.add_header('cache-control', 'no-cache')
# or we do it if the original is more recent than our copy
elif last_mod_time:
tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000')
request.add_header('If-Modified-Since', tstamp)
# user defined headers now, which may override things we've set above
if headers:
if not isinstance(headers, dict):
raise ValueError("headers provided to open_url() must be a dict")
for header in headers:
request.add_header(header, headers[header])
urlopen_args = [request, None]
if sys.version_info >= (2, 6, 0):
# urlopen in python prior to 2.6.0 did not
# have a timeout parameter
urlopen_args.append(timeout)
r = urllib_request.urlopen(*urlopen_args)
return r
def is_fakes3(s3_url):
""" Return True if s3_url has scheme fakes3:// """
if s3_url is not None:
return urlparse(s3_url).scheme in ('fakes3', 'fakes3s')
else:
return False
def __init__(self, derived_arg_spec, bypass_checks=False, no_log=False,
check_invalid_arguments=True, mutually_exclusive=None, required_together=None,
required_one_of=None, add_file_common_args=False, supports_check_mode=False,
required_if=None, supports_tags=True, facts_module=False, skip_exec=False):
merged_arg_spec = dict()
merged_arg_spec.update(AZURE_COMMON_ARGS)
if supports_tags:
merged_arg_spec.update(AZURE_TAG_ARGS)
if derived_arg_spec:
merged_arg_spec.update(derived_arg_spec)
merged_required_if = list(AZURE_COMMON_REQUIRED_IF)
if required_if:
merged_required_if += required_if
self.module = AnsibleModule(argument_spec=merged_arg_spec,
bypass_checks=bypass_checks,
no_log=no_log,
check_invalid_arguments=check_invalid_arguments,
mutually_exclusive=mutually_exclusive,
required_together=required_together,
required_one_of=required_one_of,
add_file_common_args=add_file_common_args,
supports_check_mode=supports_check_mode,
required_if=merged_required_if)
if not HAS_PACKAGING_VERSION:
self.fail("Do you have packaging installed? Try `pip install packaging`"
"- {0}".format(HAS_PACKAGING_VERSION_EXC))
if not HAS_MSRESTAZURE:
self.fail("Do you have msrestazure installed? Try `pip install msrestazure`"
"- {0}".format(HAS_MSRESTAZURE_EXC))
if not HAS_AZURE:
self.fail("Do you have azure>={1} installed? Try `pip install 'azure>={1}' --upgrade`"
"- {0}".format(HAS_AZURE_EXC, AZURE_MIN_RELEASE))
self._cloud_environment = None
self._network_client = None
self._storage_client = None
self._resource_client = None
self._compute_client = None
self._dns_client = None
self._web_client = None
self._containerservice_client = None
self.check_mode = self.module.check_mode
self.facts_module = facts_module
# self.debug = self.module.params.get('debug')
# authenticate
self.credentials = self._get_credentials(self.module.params)
if not self.credentials:
self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
"or define a profile in ~/.azure/credentials or be logged using AzureCLI.")
# if cloud_environment specified, look up/build Cloud object
raw_cloud_env = self.credentials.get('cloud_environment')
if not raw_cloud_env:
self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default
else:
# try to look up "well-known" values via the name attribute on azure_cloud members
all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)]
matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
if len(matched_clouds) == 1:
self._cloud_environment = matched_clouds[0]
elif len(matched_clouds) > 1:
self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env))
else:
if not urlparse.urlparse(raw_cloud_env).scheme:
self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds]))
try:
self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env)
except Exception as e:
self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message), exception=traceback.format_exc(e))
if self.credentials.get('subscription_id', None) is None:
self.fail("Credentials did not include a subscription_id value.")
self.log("setting subscription_id")
self.subscription_id = self.credentials['subscription_id']
if self.credentials.get('client_id') is not None and \
self.credentials.get('secret') is not None and \
self.credentials.get('tenant') is not None:
self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=self.credentials['tenant'],
cloud_environment=self._cloud_environment)
elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
tenant = self.credentials.get('tenant')
if not tenant:
tenant = 'common' # SDK default
self.azure_credentials = UserPassCredentials(self.credentials['ad_user'],
self.credentials['password'],
tenant=tenant,
cloud_environment=self._cloud_environment)
else:
self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
"Credentials must include client_id, secret and tenant or ad_user and password or "
"be logged using AzureCLI.")
# common parameter validation
if self.module.params.get('tags'):
self.validate_tags(self.module.params['tags'])
if not skip_exec:
res = self.exec_module(**self.module.params)
self.module.exit_json(**res)
def __init__(self, argument_spec, **kwargs):
args = dict(
tower_host=dict(required=False,
fallback=(env_fallback, ['TOWER_HOST'])),
tower_username=dict(required=False,
fallback=(env_fallback, ['TOWER_USERNAME'])),
tower_password=dict(no_log=True,
required=False,
fallback=(env_fallback, ['TOWER_PASSWORD'])),
validate_certs=dict(type='bool',
aliases=['tower_verify_ssl'],
required=False,
fallback=(env_fallback, ['TOWER_VERIFY_SSL'])),
tower_oauthtoken=dict(type='str',
no_log=True,
required=False,
fallback=(env_fallback,
['TOWER_OAUTH_TOKEN'])),
tower_config_file=dict(type='path', required=False, default=None),
)
args.update(argument_spec)
kwargs['supports_check_mode'] = True
self.json_output = {'changed': False}
super(TowerModule, self).__init__(argument_spec=args, **kwargs)
self.load_config_files()
# Parameters specified on command line will override settings in any config
if self.params.get('tower_host'):
self.host = self.params.get('tower_host')
if self.params.get('tower_username'):
self.username = self.params.get('tower_username')
if self.params.get('tower_password'):
self.password = self.params.get('tower_password')
if self.params.get('validate_certs') is not None:
self.verify_ssl = self.params.get('validate_certs')
if self.params.get('tower_oauthtoken'):
self.oauth_token = self.params.get('tower_oauthtoken')
# Perform some basic validation
if not re.match('^https{0,1}://', self.host):
self.host = "https://{0}".format(self.host)
# Try to parse the hostname as a url
try:
self.url = urlparse(self.host)
except Exception as e:
self.fail_json(
msg="Unable to parse tower_host as a URL ({1}): {0}".format(
self.host, e))
# Try to resolve the hostname
hostname = self.url.netloc.split(':')[0]
try:
gethostbyname(hostname)
except Exception as e:
self.fail_json(
msg="Unable to resolve tower_host ({1}): {0}".format(
hostname, e))
self.session = Request(cookies=CookieJar(),
validate_certs=self.verify_ssl)
def transform_ansible_unicode_to_str(value):
parsed_url = urlparse(str(value))
if OpenTelemetrySource.is_valid_url(parsed_url):
return OpenTelemetrySource.redact_user_password(
parsed_url).geturl()
return str(value)
def http_request(self, req):
tmp_ca_cert_path, paths_checked = self.get_ca_certs()
https_proxy = os.environ.get('https_proxy')
context = None
if HAS_SSLCONTEXT:
context = self._make_context(tmp_ca_cert_path)
# Detect if 'no_proxy' environment variable is set and if our URL is included
use_proxy = self.detect_no_proxy(req.get_full_url())
if not use_proxy:
# ignore proxy settings for this host request
return req
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if https_proxy:
proxy_parts = generic_urlparse(urlparse(https_proxy))
port = proxy_parts.get('port') or 443
s.connect((proxy_parts.get('hostname'), port))
if proxy_parts.get('scheme') == 'http':
s.sendall(self.CONNECT_COMMAND %
(self.hostname, self.port))
if proxy_parts.get('username'):
credentials = "%s:%s" % (proxy_parts.get(
'username', ''), proxy_parts.get('password', ''))
s.sendall('Proxy-Authorization: Basic %s\r\n' %
credentials.encode('base64').strip())
s.sendall('\r\n')
connect_result = s.recv(4096)
self.validate_proxy_response(connect_result)
if context:
ssl_s = context.wrap_socket(
s, server_hostname=self.hostname)
elif HAS_URLLIB3_SNI_SUPPORT:
ssl_s = ssl_wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL,
server_hostname=self.hostname)
else:
ssl_s = ssl.wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL)
match_hostname(ssl_s.getpeercert(), self.hostname)
else:
raise ProxyError(
'Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.'
% proxy_parts.get('scheme'))
else:
s.connect((self.hostname, self.port))
if context:
ssl_s = context.wrap_socket(s,
server_hostname=self.hostname)
elif HAS_URLLIB3_SNI_SUPPORT:
ssl_s = ssl_wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL,
server_hostname=self.hostname)
else:
ssl_s = ssl.wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL)
match_hostname(ssl_s.getpeercert(), self.hostname)
# close the ssl connection
#ssl_s.unwrap()
s.close()
except (ssl.SSLError, socket.error):
e = get_exception()
# fail if we tried all of the certs but none worked
if 'connection refused' in str(e).lower():
raise ConnectionError('Failed to connect to %s:%s.' %
(self.hostname, self.port))
else:
build_ssl_validation_error(self.hostname, self.port,
paths_checked)
except CertificateError:
build_ssl_validation_error(self.hostname, self.port, paths_checked)
try:
# cleanup the temp file created, don't worry
# if it fails for some reason
os.remove(tmp_ca_cert_path)
except:
pass
return req
def intersight_call(self,
http_method="",
resource_path="",
query_params=None,
body=None,
moid=None,
name=None):
"""
Invoke the Intersight API
:param resource_path: intersight resource path e.g. '/ntp/Policies'
:param query_params: dictionary object with query string parameters as key/value pairs
:param body: dictionary object with intersight data
:param moid: intersight object moid
:param name: intersight object name
:return: json http response object
"""
target_host = urlparse(self.host).netloc
target_path = urlparse(self.host).path
query_path = ""
method = http_method.upper()
bodyString = ""
# Verify an accepted HTTP verb was chosen
if (method not in ['GET', 'POST', 'PATCH', 'DELETE']):
raise ValueError(
'Please select a valid HTTP verb (GET/POST/PATCH/DELETE)')
# Verify the resource path isn't empy & is a valid <str> object
if (resource_path != "" and not (resource_path, str)):
raise TypeError(
'The *resource_path* value is required and must be of type "<str>"'
)
# Verify the query parameters isn't empy & is a valid <dict> object
if (query_params is not None and not isinstance(query_params, dict)):
raise TypeError(
'The *query_params* value must be of type "<dict>"')
# Verify the MOID is not null & of proper length
if (moid is not None and len(moid.encode('utf-8')) != 24):
raise ValueError('Invalid *moid* value!')
# Check for query_params, encode, and concatenate onto URL
if query_params:
query_path = "?" + urlencode(query_params)
# Handle PATCH/DELETE by Object "name" instead of "moid"
if method in ('PATCH', 'DELETE'):
if moid is None:
if name is not None:
if isinstance(name, str):
moid = self.get_moid_by_name(resource_path, name)
else:
raise TypeError(
'The *name* value must be of type "<str>"')
else:
raise ValueError(
'Must set either *moid* or *name* with "PATCH/DELETE!"'
)
# Check for moid and concatenate onto URL
if moid is not None:
resource_path += "/" + moid
# Check for GET request to properly form body
if method != "GET":
bodyString = json.dumps(body)
# Concatenate URLs for headers
target_url = self.host + resource_path + query_path
request_target = method.lower(
) + " " + target_path + resource_path + query_path
# Get the current GMT Date/Time
cdate = get_gmt_date()
# Generate the body digest
body_digest = get_sha256_digest(bodyString)
b64_body_digest = b64encode(body_digest.digest())
# Generate the authorization header
auth_header = {
'Host': target_host,
'Date': cdate,
'Digest': "SHA-256=" + b64_body_digest.decode('ascii'),
}
string_to_sign = prepare_str_to_sign(request_target, auth_header)
b64_signed_msg = self.get_sig_b64encode(string_to_sign)
auth_header = self.get_auth_header(auth_header, b64_signed_msg)
# Generate the HTTP requests header
request_header = {
'Accept': 'application/json',
'Content-Type': 'application/json',
'Host': '{0}'.format(target_host),
'Date': '{0}'.format(cdate),
'Digest': 'SHA-256={0}'.format(b64_body_digest.decode('ascii')),
'Authorization': '{0}'.format(auth_header),
}
response, info = fetch_url(self.module,
target_url,
data=bodyString,
headers=request_header,
method=method,
use_proxy=self.module.params['use_proxy'])
return response, info
def get_collection_versions(self, namespace, name):
"""
Gets a list of available versions for a collection on a Galaxy server.
:param namespace: The collection namespace.
:param name: The collection name.
:return: A list of versions that are available.
"""
relative_link = False
if 'v3' in self.available_api_versions:
api_path = self.available_api_versions['v3']
pagination_path = ['links', 'next']
relative_link = True # AH pagination results are relative an not an absolute URI.
else:
api_path = self.available_api_versions['v2']
pagination_path = ['next']
page_size_name = 'limit' if 'v3' in self.available_api_versions else 'page_size'
versions_url = _urljoin(
self.api_server, api_path, 'collections', namespace, name,
'versions', '/?%s=%d' % (page_size_name, COLLECTION_PAGE_SIZE))
versions_url_info = urlparse(versions_url)
# We should only rely on the cache if the collection has not changed. This may slow things down but it ensures
# we are not waiting a day before finding any new collections that have been published.
if self._cache:
server_cache = self._cache.setdefault(get_cache_id(versions_url),
{})
modified_cache = server_cache.setdefault('modified', {})
try:
modified_date = self.get_collection_metadata(namespace,
name).modified_str
except GalaxyError as err:
if err.http_code != 404:
raise
# No collection found, return an empty list to keep things consistent with the various APIs
return []
cached_modified_date = modified_cache.get(
'%s.%s' % (namespace, name), None)
if cached_modified_date != modified_date:
modified_cache['%s.%s' % (namespace, name)] = modified_date
if versions_url_info.path in server_cache:
del server_cache[versions_url_info.path]
self._set_cache()
error_context_msg = 'Error when getting available collection versions for %s.%s from %s (%s)' \
% (namespace, name, self.name, self.api_server)
try:
data = self._call_galaxy(versions_url,
error_context_msg=error_context_msg,
cache=True)
except GalaxyError as err:
if err.http_code != 404:
raise
# v3 doesn't raise a 404 so we need to mimick the empty response from APIs that do.
return []
if 'data' in data:
# v3 automation-hub is the only known API that uses `data`
# since v3 pulp_ansible does not, we cannot rely on version
# to indicate which key to use
results_key = 'data'
else:
results_key = 'results'
versions = []
while True:
versions += [v['version'] for v in data[results_key]]
next_link = data
for path in pagination_path:
next_link = next_link.get(path, {})
if not next_link:
break
elif relative_link:
# TODO: This assumes the pagination result is relative to the root server. Will need to be verified
# with someone who knows the AH API.
# Remove the query string from the versions_url to use the next_link's query
versions_url = urljoin(versions_url,
urlparse(versions_url).path)
next_link = versions_url.replace(versions_url_info.path,
next_link)
data = self._call_galaxy(to_native(next_link,
errors='surrogate_or_strict'),
error_context_msg=error_context_msg,
cache=True)
self._set_cache()
return versions
def _call_galaxy(self,
url,
args=None,
headers=None,
method=None,
auth_required=False,
error_context_msg=None,
cache=False):
url_info = urlparse(url)
cache_id = get_cache_id(url)
query = parse_qs(url_info.query)
if cache and self._cache:
server_cache = self._cache.setdefault(cache_id, {})
iso_datetime_format = '%Y-%m-%dT%H:%M:%SZ'
valid = False
if url_info.path in server_cache:
expires = datetime.datetime.strptime(
server_cache[url_info.path]['expires'],
iso_datetime_format)
valid = datetime.datetime.utcnow() < expires
is_paginated_url = 'page' in query or 'offset' in query
if valid and not is_paginated_url:
# Got a hit on the cache and we aren't getting a paginated response
path_cache = server_cache[url_info.path]
if path_cache.get('paginated'):
if '/v3/' in url_info.path:
res = {'links': {'next': None}}
else:
res = {'next': None}
# Technically some v3 paginated APIs return in 'data' but the caller checks the keys for this so
# always returning the cache under results is fine.
res['results'] = []
for result in path_cache['results']:
res['results'].append(result)
else:
res = path_cache['results']
return res
elif not is_paginated_url:
# The cache entry had expired or does not exist, start a new blank entry to be filled later.
expires = datetime.datetime.utcnow()
expires += datetime.timedelta(days=1)
server_cache[url_info.path] = {
'expires': expires.strftime(iso_datetime_format),
'paginated': False,
}
headers = headers or {}
self._add_auth_token(headers, url, required=auth_required)
try:
display.vvvv("Calling Galaxy at %s" % url)
resp = open_url(to_native(url),
data=args,
validate_certs=self.validate_certs,
headers=headers,
method=method,
timeout=20,
http_agent=user_agent(),
follow_redirects='safe')
except HTTPError as e:
raise GalaxyError(e, error_context_msg)
except Exception as e:
raise AnsibleError(
"Unknown error when attempting to call Galaxy at '%s': %s" %
(url, to_native(e)))
resp_data = to_text(resp.read(), errors='surrogate_or_strict')
try:
data = json.loads(resp_data)
except ValueError:
raise AnsibleError(
"Failed to parse Galaxy response from '%s' as JSON:\n%s" %
(resp.url, to_native(resp_data)))
if cache and self._cache:
path_cache = self._cache[cache_id][url_info.path]
# v3 can return data or results for paginated results. Scan the result so we can determine what to cache.
paginated_key = None
for key in ['data', 'results']:
if key in data:
paginated_key = key
break
if paginated_key:
path_cache['paginated'] = True
results = path_cache.setdefault('results', [])
for result in data[paginated_key]:
results.append(result)
else:
path_cache['results'] = data
return data
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(
dict(
force=dict(required=False, default='no', type='bool'),
policy=dict(required=False, default=None, type='json'),
name=dict(required=True, type='str'),
requester_pays=dict(default='no', type='bool'),
s3_url=dict(aliases=['S3_URL'], type='str'),
state=dict(default='present', type='str', choices=['present', 'absent']),
tags=dict(required=False, default=None, type='dict'),
versioning=dict(default=None, type='bool'),
ceph=dict(default='no', type='bool')
)
)
module = AnsibleModule(argument_spec=argument_spec)
if not HAS_BOTO:
module.fail_json(msg='boto required for this module')
region, ec2_url, aws_connect_params = get_aws_connection_info(module)
if region in ('us-east-1', '', None):
# S3ism for the US Standard region
location = Location.DEFAULT
else:
# Boto uses symbolic names for locations but region strings will
# actually work fine for everything except us-east-1 (US Standard)
location = region
s3_url = module.params.get('s3_url')
# allow eucarc environment variables to be used if ansible vars aren't set
if not s3_url and 'S3_URL' in os.environ:
s3_url = os.environ['S3_URL']
ceph = module.params.get('ceph')
if ceph and not s3_url:
module.fail_json(msg='ceph flavour requires s3_url')
flavour = 'aws'
# bucket names with .'s in them need to use the calling_format option,
# otherwise the connection will fail. See https://github.com/boto/boto/issues/2836
# for more details.
aws_connect_params['calling_format'] = OrdinaryCallingFormat()
# Look at s3_url and tweak connection settings
# if connecting to Walrus or fakes3
try:
if s3_url and ceph:
ceph = urlparse.urlparse(s3_url)
connection = boto.connect_s3(
host=ceph.hostname,
port=ceph.port,
is_secure=ceph.scheme == 'https',
**aws_connect_params
)
flavour = 'ceph'
elif is_fakes3(s3_url):
fakes3 = urlparse.urlparse(s3_url)
connection = S3Connection(
is_secure=fakes3.scheme == 'fakes3s',
host=fakes3.hostname,
port=fakes3.port,
**aws_connect_params
)
elif is_walrus(s3_url):
del aws_connect_params['calling_format']
walrus = urlparse.urlparse(s3_url).hostname
connection = boto.connect_walrus(walrus, **aws_connect_params)
else:
connection = boto.s3.connect_to_region(location, is_secure=True, **aws_connect_params)
# use this as fallback because connect_to_region seems to fail in boto + non 'classic' aws accounts in some cases
if connection is None:
connection = boto.connect_s3(**aws_connect_params)
except boto.exception.NoAuthHandlerFound as e:
module.fail_json(msg='No Authentication Handler found: %s ' % str(e))
except Exception as e:
module.fail_json(msg='Failed to connect to S3: %s' % str(e))
if connection is None: # this should never happen
module.fail_json(msg='Unknown error, failed to create s3 connection, no information from boto.')
state = module.params.get("state")
if state == 'present':
create_or_update_bucket(connection, module, location, flavour=flavour)
elif state == 'absent':
destroy_bucket(connection, module, flavour=flavour)
def main():
module = AnsibleModule(
argument_spec=dict(
group_id=dict(required=True),
artifact_id=dict(required=True),
version=dict(default="latest"),
classifier=dict(default=''),
extension=dict(default='jar'),
repository_url=dict(default=None),
username=dict(default=None, aliases=['aws_secret_key']),
password=dict(default=None, no_log=True, aliases=['aws_secret_access_key']),
state=dict(default="present", choices=["present", "absent"]), # TODO - Implement a "latest" state
timeout=dict(default=10, type='int'),
dest=dict(type="path", required=True),
validate_certs=dict(required=False, default=True, type='bool'),
keep_name=dict(required=False, default=False, type='bool'),
verify_checksum=dict(required=False, default='download', choices=['never', 'download', 'change', 'always'])
),
add_file_common_args=True
)
if not HAS_LXML_ETREE:
module.fail_json(msg='module requires the lxml python library installed on the managed machine')
repository_url = module.params["repository_url"]
if not repository_url:
repository_url = "http://repo1.maven.org/maven2"
try:
parsed_url = urlparse(repository_url)
except AttributeError as e:
module.fail_json(msg='url parsing went wrong %s' % e)
local = parsed_url.scheme == "file"
if parsed_url.scheme == 's3' and not HAS_BOTO:
module.fail_json(msg='boto3 required for this module, when using s3:// repository URLs')
group_id = module.params["group_id"]
artifact_id = module.params["artifact_id"]
version = module.params["version"]
classifier = module.params["classifier"]
extension = module.params["extension"]
state = module.params["state"]
dest = module.params["dest"]
b_dest = to_bytes(dest, errors='surrogate_or_strict')
keep_name = module.params["keep_name"]
verify_checksum = module.params["verify_checksum"]
verify_download = verify_checksum in ['download', 'always']
verify_change = verify_checksum in ['change', 'always']
downloader = MavenDownloader(module, repository_url, local)
try:
artifact = Artifact(group_id, artifact_id, version, classifier, extension)
except ValueError as e:
module.fail_json(msg=e.args[0])
changed = False
prev_state = "absent"
if dest.endswith(os.sep):
b_dest = to_bytes(dest, errors='surrogate_or_strict')
if not os.path.exists(b_dest):
(pre_existing_dir, new_directory_list) = split_pre_existing_dir(dest)
os.makedirs(b_dest)
directory_args = module.load_file_common_arguments(module.params)
directory_mode = module.params["directory_mode"]
if directory_mode is not None:
directory_args['mode'] = directory_mode
else:
directory_args['mode'] = None
changed = adjust_recursive_directory_permissions(pre_existing_dir, new_directory_list, module, directory_args, changed)
if os.path.isdir(b_dest):
version_part = version
if keep_name and version == 'latest':
version_part = downloader.find_latest_version_available(artifact)
if classifier:
dest = posixpath.join(dest, "%s-%s-%s.%s" % (artifact_id, version_part, classifier, extension))
else:
dest = posixpath.join(dest, "%s-%s.%s" % (artifact_id, version_part, extension))
b_dest = to_bytes(dest, errors='surrogate_or_strict')
if os.path.lexists(b_dest) and ((not verify_change) or not downloader.is_invalid_md5(dest, downloader.find_uri_for_artifact(artifact))):
prev_state = "present"
if prev_state == "absent":
try:
download_error = downloader.download(module.tmpdir, artifact, verify_download, b_dest)
if download_error is None:
changed = True
else:
module.fail_json(msg="Cannot retrieve the artifact to destination: " + download_error)
except ValueError as e:
module.fail_json(msg=e.args[0])
module.params['dest'] = dest
file_args = module.load_file_common_arguments(module.params)
changed = module.set_fs_attributes_if_different(file_args, changed)
if changed:
module.exit_json(state=state, dest=dest, group_id=group_id, artifact_id=artifact_id, version=version, classifier=classifier,
extension=extension, repository_url=repository_url, changed=changed)
else:
module.exit_json(state=state, dest=dest, changed=changed)
def main():
module = AnsibleModule(argument_spec=dict(
group_id=dict(default=None),
artifact_id=dict(default=None),
version=dict(default="latest"),
classifier=dict(default=None),
extension=dict(default='jar'),
repository_url=dict(default=None),
username=dict(default=None, aliases=['aws_secret_key']),
password=dict(
default=None, no_log=True, aliases=['aws_secret_access_key']),
state=dict(default="present", choices=["present", "absent"]
), # TODO - Implement a "latest" state
timeout=dict(default=10, type='int'),
dest=dict(type="path", default=None),
validate_certs=dict(required=False, default=True, type='bool'),
keep_name=dict(required=False, default=False, type='bool'),
))
repository_url = module.params["repository_url"]
if not repository_url:
repository_url = "http://repo1.maven.org/maven2"
try:
parsed_url = urlparse(repository_url)
except AttributeError as e:
module.fail_json(msg='url parsing went wrong %s' % e)
if parsed_url.scheme == 's3' and not HAS_BOTO:
module.fail_json(
msg=
'boto3 required for this module, when using s3:// repository URLs')
group_id = module.params["group_id"]
artifact_id = module.params["artifact_id"]
version = module.params["version"]
classifier = module.params["classifier"]
extension = module.params["extension"]
state = module.params["state"]
dest = module.params["dest"]
keep_name = module.params["keep_name"]
#downloader = MavenDownloader(module, repository_url, repository_username, repository_password)
downloader = MavenDownloader(module, repository_url)
try:
artifact = Artifact(group_id, artifact_id, version, classifier,
extension)
except ValueError as e:
module.fail_json(msg=e.args[0])
prev_state = "absent"
if os.path.isdir(dest):
version_part = version
if keep_name and version == 'latest':
version_part = downloader.find_latest_version_available(artifact)
dest = posixpath.join(
dest, "%s-%s.%s" % (artifact_id, version_part, extension))
if os.path.lexists(dest) and downloader.verify_md5(
dest,
downloader.find_uri_for_artifact(artifact) + '.md5'):
prev_state = "present"
else:
path = os.path.dirname(dest)
if not os.path.exists(path):
os.makedirs(path)
if prev_state == "present":
module.exit_json(dest=dest, state=state, changed=False)
try:
if downloader.download(artifact, dest):
module.exit_json(state=state,
dest=dest,
group_id=group_id,
artifact_id=artifact_id,
version=version,
classifier=classifier,
extension=extension,
repository_url=repository_url,
changed=True)
else:
module.fail_json(msg="Unable to download the artifact")
except ValueError as e:
module.fail_json(msg=e.args[0])
def __init__(self,
derived_arg_spec,
bypass_checks=False,
no_log=False,
check_invalid_arguments=None,
mutually_exclusive=None,
required_together=None,
required_one_of=None,
add_file_common_args=False,
supports_check_mode=False,
required_if=None,
supports_tags=True,
facts_module=False,
skip_exec=False):
merged_arg_spec = dict()
merged_arg_spec.update(AZURE_COMMON_ARGS)
if supports_tags:
merged_arg_spec.update(AZURE_TAG_ARGS)
if derived_arg_spec:
merged_arg_spec.update(derived_arg_spec)
merged_required_if = list(AZURE_COMMON_REQUIRED_IF)
if required_if:
merged_required_if += required_if
self.module = AnsibleModule(
argument_spec=merged_arg_spec,
bypass_checks=bypass_checks,
no_log=no_log,
check_invalid_arguments=check_invalid_arguments,
mutually_exclusive=mutually_exclusive,
required_together=required_together,
required_one_of=required_one_of,
add_file_common_args=add_file_common_args,
supports_check_mode=supports_check_mode,
required_if=merged_required_if)
if not HAS_PACKAGING_VERSION:
self.fail(
"Do you have packaging installed? Try `pip install packaging`"
"- {0}".format(HAS_PACKAGING_VERSION_EXC))
if not HAS_MSRESTAZURE:
self.fail(
"Do you have msrestazure installed? Try `pip install msrestazure`"
"- {0}".format(HAS_MSRESTAZURE_EXC))
if not HAS_AZURE:
self.fail(
"Do you have azure>={1} installed? Try `pip install ansible[azure]`"
"- {0}".format(HAS_AZURE_EXC, AZURE_MIN_RELEASE))
self._cloud_environment = None
self._network_client = None
self._storage_client = None
self._resource_client = None
self._compute_client = None
self._dns_client = None
self._web_client = None
self._containerservice_client = None
self._mysql_client = None
self._postgresql_client = None
self._adfs_authority_url = None
self._resource = None
self.check_mode = self.module.check_mode
self.api_profile = self.module.params.get('api_profile')
self.facts_module = facts_module
# authenticate
self.credentials = self._get_credentials(self.module.params)
if not self.credentials:
if HAS_AZURE_CLI_CORE:
self.fail(
"Failed to get credentials. Either pass as parameters, set environment variables, "
"define a profile in ~/.azure/credentials, or log in with Azure CLI (`az login`)."
)
else:
self.fail(
"Failed to get credentials. Either pass as parameters, set environment variables, "
"define a profile in ~/.azure/credentials, or install Azure CLI and log in (`az login`)."
)
# cert validation mode precedence: module-arg, credential profile, env, "validate"
self._cert_validation_mode = self.module.params['cert_validation_mode'] or self.credentials.get('cert_validation_mode') or \
os.environ.get('AZURE_CERT_VALIDATION_MODE') or 'validate'
if self._cert_validation_mode not in ['validate', 'ignore']:
self.fail('invalid cert_validation_mode: {0}'.format(
self._cert_validation_mode))
# if cloud_environment specified, look up/build Cloud object
raw_cloud_env = self.credentials.get('cloud_environment')
if self.credentials.get(
'credentials') is not None and raw_cloud_env is not None:
self._cloud_environment = raw_cloud_env
elif not raw_cloud_env:
self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default
else:
# try to look up "well-known" values via the name attribute on azure_cloud members
all_clouds = [
x[1] for x in inspect.getmembers(azure_cloud)
if isinstance(x[1], azure_cloud.Cloud)
]
matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
if len(matched_clouds) == 1:
self._cloud_environment = matched_clouds[0]
elif len(matched_clouds) > 1:
self.fail(
"Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'"
.format(raw_cloud_env))
else:
if not urlparse.urlparse(raw_cloud_env).scheme:
self.fail(
"cloud_environment must be an endpoint discovery URL or one of {0}"
.format([x.name for x in all_clouds]))
try:
self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(
raw_cloud_env)
except Exception as e:
self.fail(
"cloud_environment {0} could not be resolved: {1}".
format(raw_cloud_env, e.message),
exception=traceback.format_exc(e))
if self.credentials.get(
'subscription_id',
None) is None and self.credentials.get('credentials') is None:
self.fail("Credentials did not include a subscription_id value.")
self.log("setting subscription_id")
self.subscription_id = self.credentials['subscription_id']
# get authentication authority
# for adfs, user could pass in authority or not.
# for others, use default authority from cloud environment
if self.credentials.get('adfs_authority_url') is None:
self._adfs_authority_url = self._cloud_environment.endpoints.active_directory
else:
self._adfs_authority_url = self.credentials.get(
'adfs_authority_url')
# get resource from cloud environment
self._resource = self._cloud_environment.endpoints.active_directory_resource_id
if self.credentials.get('credentials') is not None:
# AzureCLI credentials
self.azure_credentials = self.credentials['credentials']
elif self.credentials.get('client_id') is not None and \
self.credentials.get('secret') is not None and \
self.credentials.get('tenant') is not None:
self.azure_credentials = ServicePrincipalCredentials(
client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=self.credentials['tenant'],
cloud_environment=self._cloud_environment,
verify=self._cert_validation_mode == 'validate')
elif self.credentials.get('ad_user') is not None and \
self.credentials.get('password') is not None and \
self.credentials.get('client_id') is not None and \
self.credentials.get('tenant') is not None:
self.azure_credentials = self.acquire_token_with_username_password(
self._adfs_authority_url, self._resource,
self.credentials['ad_user'], self.credentials['password'],
self.credentials['client_id'], self.credentials['tenant'])
elif self.credentials.get(
'ad_user') is not None and self.credentials.get(
'password') is not None:
tenant = self.credentials.get('tenant')
if not tenant:
tenant = 'common' # SDK default
self.azure_credentials = UserPassCredentials(
self.credentials['ad_user'],
self.credentials['password'],
tenant=tenant,
cloud_environment=self._cloud_environment,
verify=self._cert_validation_mode == 'validate')
else:
self.fail(
"Failed to authenticate with provided credentials. Some attributes were missing. "
"Credentials must include client_id, secret and tenant or ad_user and password, or "
"ad_user, password, client_id, tenant and adfs_authority_url(optional) for ADFS authentication, or "
"be logged in using AzureCLI.")
# common parameter validation
if self.module.params.get('tags'):
self.validate_tags(self.module.params['tags'])
if not skip_exec:
res = self.exec_module(**self.module.params)
self.module.exit_json(**res)
def __init__(self, args):
self._args = args
self._cloud_environment = None
self._compute_client = None
self._resource_client = None
self._network_client = None
self.debug = False
if args.debug:
self.debug = True
self.credentials = self._get_credentials(args)
if not self.credentials:
self.fail(
"Failed to get credentials. Either pass as parameters, set environment variables, "
"or define a profile in ~/.azure/credentials.")
# if cloud_environment specified, look up/build Cloud object
raw_cloud_env = self.credentials.get('cloud_environment')
if not raw_cloud_env:
self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default
else:
# try to look up "well-known" values via the name attribute on azure_cloud members
all_clouds = [
x[1] for x in inspect.getmembers(azure_cloud)
if isinstance(x[1], azure_cloud.Cloud)
]
matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
if len(matched_clouds) == 1:
self._cloud_environment = matched_clouds[0]
elif len(matched_clouds) > 1:
self.fail(
"Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'"
.format(raw_cloud_env))
else:
if not urlparse.urlparse(raw_cloud_env).scheme:
self.fail(
"cloud_environment must be an endpoint discovery URL or one of {0}"
.format([x.name for x in all_clouds]))
try:
self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(
raw_cloud_env)
except Exception as e:
self.fail(
"cloud_environment {0} could not be resolved: {1}".
format(raw_cloud_env, e.message))
if self.credentials.get('subscription_id', None) is None:
self.fail("Credentials did not include a subscription_id value.")
self.log("setting subscription_id")
self.subscription_id = self.credentials['subscription_id']
if self.credentials.get('client_id') is not None and \
self.credentials.get('secret') is not None and \
self.credentials.get('tenant') is not None:
self.azure_credentials = ServicePrincipalCredentials(
client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=self.credentials['tenant'],
cloud_environment=self._cloud_environment)
elif self.credentials.get(
'ad_user') is not None and self.credentials.get(
'password') is not None:
tenant = self.credentials.get('tenant')
if not tenant:
tenant = 'common'
self.azure_credentials = UserPassCredentials(
self.credentials['ad_user'],
self.credentials['password'],
tenant=tenant,
cloud_environment=self._cloud_environment)
else:
self.fail(
"Failed to authenticate with provided credentials. Some attributes were missing. "
"Credentials must include client_id, secret and tenant or ad_user and password."
)
def serialize_vm(self, vm):
'''
Convert a VirtualMachine object to dict.
:param vm: VirtualMachine object
:return: dict
'''
result = self.serialize_obj(vm,
AZURE_OBJECT_CLASS,
enum_modules=AZURE_ENUM_MODULES)
resource_group = parse_resource_id(result['id']).get('resource_group')
instance = None
power_state = None
try:
instance = self.compute_client.virtual_machines.instance_view(
resource_group, vm.name)
instance = self.serialize_obj(instance,
AZURE_OBJECT_CLASS,
enum_modules=AZURE_ENUM_MODULES)
except Exception as exc:
self.fail(
"Error getting virtual machine {0} instance view - {1}".format(
vm.name, str(exc)))
for index in range(len(instance['statuses'])):
code = instance['statuses'][index]['code'].split('/')
if code[0] == 'PowerState':
power_state = code[1]
elif code[0] == 'OSState' and code[1] == 'generalized':
power_state = 'generalized'
break
new_result = {}
new_result['power_state'] = power_state
new_result['id'] = vm.id
new_result['resource_group'] = resource_group
new_result['name'] = vm.name
new_result['state'] = 'present'
new_result['location'] = vm.location
new_result['vm_size'] = result['properties']['hardwareProfile'][
'vmSize']
os_profile = result['properties'].get('osProfile')
if os_profile is not None:
new_result['admin_username'] = os_profile.get('adminUsername')
image = result['properties']['storageProfile'].get('imageReference')
if image is not None:
if image.get('publisher', None) is not None:
new_result['image'] = {
'publisher': image['publisher'],
'sku': image['sku'],
'offer': image['offer'],
'version': image['version']
}
else:
new_result['image'] = {'id': image.get('id', None)}
new_result['boot_diagnostics'] = {
'enabled':
'diagnosticsProfile' in result['properties']
and 'bootDiagnostics' in result['properties']['diagnosticsProfile']
and result['properties']['diagnosticsProfile']['bootDiagnostics']
['enabled'] or False,
'storage_uri':
'diagnosticsProfile' in result['properties']
and 'bootDiagnostics' in result['properties']['diagnosticsProfile']
and result['properties']['diagnosticsProfile']['bootDiagnostics']
['storageUri'] or None
}
if new_result['boot_diagnostics']['enabled']:
new_result['boot_diagnostics']['console_screenshot_uri'] = result[
'properties']['instanceView']['bootDiagnostics'][
'consoleScreenshotBlobUri']
new_result['boot_diagnostics']['serial_console_log_uri'] = result[
'properties']['instanceView']['bootDiagnostics'][
'serialConsoleLogBlobUri']
vhd = result['properties']['storageProfile']['osDisk'].get('vhd')
if vhd is not None:
url = urlparse(vhd['uri'])
new_result['storage_account_name'] = url.netloc.split('.')[0]
new_result['storage_container_name'] = url.path.split('/')[1]
new_result['storage_blob_name'] = url.path.split('/')[-1]
new_result['os_disk_caching'] = result['properties']['storageProfile'][
'osDisk']['caching']
new_result['os_type'] = result['properties']['storageProfile'][
'osDisk']['osType']
new_result['data_disks'] = []
disks = result['properties']['storageProfile']['dataDisks']
for disk_index in range(len(disks)):
new_result['data_disks'].append({
'lun':
disks[disk_index].get('lun'),
'disk_size_gb':
disks[disk_index].get('diskSizeGB'),
'managed_disk_type':
disks[disk_index].get('managedDisk',
{}).get('storageAccountType'),
'caching':
disks[disk_index].get('caching')
})
new_result['network_interface_names'] = []
nics = result['properties']['networkProfile']['networkInterfaces']
for nic_index in range(len(nics)):
new_result['network_interface_names'].append(
re.sub('.*networkInterfaces/', '', nics[nic_index]['id']))
new_result['tags'] = vm.tags
return new_result
def __init__(self, url):
parts = urlparse(url)
_query = frozenset(parse_qs(parts.query))
_path = unquote_plus(parts.path)
self.parts = parts._replace(query=_query, path=_path)
def __init__(self,
argument_spec,
direct_params=None,
error_callback=None,
warn_callback=None,
**kwargs):
full_argspec = {}
full_argspec.update(TowerModule.AUTH_ARGSPEC)
full_argspec.update(argument_spec)
kwargs['supports_check_mode'] = True
self.error_callback = error_callback
self.warn_callback = warn_callback
self.json_output = {'changed': False}
if direct_params is not None:
self.params = direct_params
else:
super(TowerModule, self).__init__(argument_spec=full_argspec,
**kwargs)
self.load_config_files()
# Parameters specified on command line will override settings in any config
for short_param, long_param in self.short_params.items():
direct_value = self.params.get(long_param)
if direct_value is not None:
setattr(self, short_param, direct_value)
# Perform magic depending on whether tower_oauthtoken is a string or a dict
if self.params.get('tower_oauthtoken'):
token_param = self.params.get('tower_oauthtoken')
if type(token_param) is dict:
if 'token' in token_param:
self.oauth_token = self.params.get(
'tower_oauthtoken')['token']
else:
self.fail_json(
msg=
"The provided dict in tower_oauthtoken did not properly contain the token entry"
)
elif isinstance(token_param, string_types):
self.oauth_token = self.params.get('tower_oauthtoken')
else:
error_msg = "The provided tower_oauthtoken type was not valid ({0}). Valid options are str or dict.".format(
type(token_param).__name__)
self.fail_json(msg=error_msg)
# Perform some basic validation
if not re.match('^https{0,1}://', self.host):
self.host = "https://{0}".format(self.host)
# Try to parse the hostname as a url
try:
self.url = urlparse(self.host)
except Exception as e:
self.fail_json(
msg="Unable to parse tower_host as a URL ({1}): {0}".format(
self.host, e))
# Try to resolve the hostname
hostname = self.url.netloc.split(':')[0]
try:
gethostbyname(hostname)
except Exception as e:
self.fail_json(
msg="Unable to resolve tower_host ({1}): {0}".format(
hostname, e))
self.session = Request(cookies=CookieJar(),
validate_certs=self.verify_ssl)
def terminate_virtual_machine(module, azure):
"""
Terminates a virtual machine
module : AnsibleModule object
azure: authenticated azure ServiceManagementService object
Returns:
True if a new virtual machine was deleted, false otherwise
"""
# Whether to wait for termination to complete before returning
wait = module.params.get('wait')
wait_timeout = int(module.params.get('wait_timeout'))
name = module.params.get('name')
delete_empty_services = module.params.get('delete_empty_services')
changed = False
deployment = None
public_dns_name = None
disk_names = []
try:
deployment = azure.get_deployment_by_name(service_name=name, deployment_name=name)
except AzureMissingException as e:
pass # no such deployment or service
except AzureException as e:
module.fail_json(msg="failed to find the deployment, error was: %s" % str(e))
# Delete deployment
if deployment:
changed = True
try:
# gather disk info
results = []
for role in deployment.role_list:
role_props = azure.get_role(name, deployment.name, role.role_name)
if role_props.os_virtual_hard_disk.disk_name not in disk_names:
disk_names.append(role_props.os_virtual_hard_disk.disk_name)
except AzureException as e:
module.fail_json(msg="failed to get the role %s, error was: %s" % (role.role_name, str(e)))
try:
result = azure.delete_deployment(name, deployment.name)
_wait_for_completion(azure, result, wait_timeout, "delete_deployment")
except AzureException as e:
module.fail_json(msg="failed to delete the deployment %s, error was: %s" % (deployment.name, str(e)))
# It's unclear when disks associated with terminated deployment get detached.
# Thus, until the wait_timeout is reached, we continue to delete disks as they
# become detached by polling the list of remaining disks and examining the state.
try:
_delete_disks_when_detached(azure, wait_timeout, disk_names)
except (AzureException, TimeoutError) as e:
module.fail_json(msg=str(e))
try:
# Now that the vm is deleted, remove the cloud service
result = azure.delete_hosted_service(service_name=name)
_wait_for_completion(azure, result, wait_timeout, "delete_hosted_service")
except AzureException as e:
module.fail_json(msg="failed to delete the service %s, error was: %s" % (name, str(e)))
public_dns_name = urlparse(deployment.url).hostname
return changed, public_dns_name, deployment
def test_urlparse(self):
actual = urlparse("http://test.com/here")
self.assertEqual("http", actual.scheme)
self.assertEqual("test.com", actual.netloc)
self.assertEqual("/here", actual.path)
def update_firmware_redfish(idrac, module):
"""Update firmware from a network share and return the job details."""
msg = {}
msg['changed'], msg['failed'] = False, False
msg['update_msg'] = "Successfully triggered the job to update the firmware."
try:
share_name = module.params['share_name']
catalog_file_name = module.params['catalog_file_name']
share_user = module.params['share_user']
share_pwd = module.params['share_password']
reboot = module.params['reboot']
job_wait = module.params['job_wait']
ignore_cert_warning = module.params['ignore_cert_warning']
apply_update = module.params['apply_update']
payload = {
"RebootNeeded": reboot,
"CatalogFile": catalog_file_name,
"ApplyUpdate": str(apply_update),
"IgnoreCertWarning": CERT_WARN[ignore_cert_warning]
}
if share_user is not None:
payload['UserName'] = share_user
if share_pwd is not None:
payload['Password'] = share_pwd
if share_name.lower().startswith(('http://', 'https://', 'ftp://')):
msg['update_status'], job_details = update_firmware_url_redfish(
module, idrac, share_name, catalog_file_name, apply_update,
reboot, ignore_cert_warning, job_wait, payload)
if job_details:
msg['update_status']['job_details'] = job_details
else:
if share_name.startswith('\\\\'):
cifs = share_name.split('\\')
payload['IPAddress'] = cifs[2]
payload['ShareName'] = '\\'.join(cifs[3:])
payload['ShareType'] = 'CIFS'
else:
nfs = urlparse(share_name)
payload['IPAddress'] = nfs.scheme
payload['ShareName'] = nfs.path.strip('/')
payload['ShareType'] = 'NFS'
resp = idrac.invoke_request(PATH, method="POST", data=payload)
job_id = get_jobid(module, resp)
resp, mesg = wait_for_job_completion(module,
JOB_URI.format(job_id=job_id),
job_wait, reboot,
apply_update)
if not mesg:
msg['update_status'] = resp.json_data
else:
msg['update_status'] = mesg
repo_based_update_list = idrac.invoke_request(
GET_REPO_BASED_UPDATE_LIST_PATH,
method="POST",
data="{}",
dump=False)
msg['update_status'][
'job_details'] = repo_based_update_list.json_data
json_data, repo_status, failed = msg['update_status'][
'job_details'], False, False
if "PackageList" not in json_data:
job_data = json_data.get('Data')
pkglst = job_data['body'] if 'body' in job_data else job_data.get(
'GetRepoBasedUpdateList_OUTPUT')
if 'PackageList' in pkglst:
pkglst[
'PackageList'], repo_status, failed = _convert_xmltojson(
module, pkglst, idrac)
else:
json_data['PackageList'], repo_status, failed = _convert_xmltojson(
module, json_data, None)
if not apply_update and not failed:
msg['update_msg'] = "Successfully fetched the applicable firmware update package list."
elif apply_update and not reboot and not job_wait and not failed:
msg['update_msg'] = "Successfully triggered the job to stage the firmware."
elif apply_update and job_wait and not reboot and not failed:
msg['update_msg'] = "Successfully staged the applicable firmware update packages."
msg['changed'] = True
elif apply_update and job_wait and not reboot and failed:
msg['update_msg'] = "Successfully staged the applicable firmware update packages with error(s)."
msg['failed'] = True
except RuntimeError as e:
module.fail_json(msg=str(e))
if module.check_mode and not (json_data.get('PackageList') or json_data.get('Data')) and \
msg['update_status']['JobStatus'] == 'OK':
module.exit_json(msg="No changes found to commit!")
elif module.check_mode and (json_data.get('PackageList') or json_data.get('Data')) and \
msg['update_status']['JobStatus'] == 'OK':
module.exit_json(msg="Changes found to commit!",
changed=True,
update_status=msg['update_status'])
elif module.check_mode and not msg['update_status']['JobStatus'] == 'OK':
msg['update_status'].pop('job_details')
module.fail_json(
msg="Unable to complete the firmware repository download.",
update_status=msg['update_status'])
elif not module.check_mode and "JobStatus" in msg['update_status']:
if not msg['update_status']['JobStatus'] == "Critical":
if module.params['job_wait'] and module.params['apply_update'] and module.params['reboot'] and \
('job_details' in msg['update_status'] and repo_status) and not failed:
msg['changed'] = True
msg['update_msg'] = "Successfully updated the firmware."
elif module.params['job_wait'] and module.params['apply_update'] and module.params['reboot'] and \
('job_details' in msg['update_status'] and repo_status) and failed:
msg['failed'], msg['changed'] = True, False
msg['update_msg'] = "Firmware update failed."
else:
failed_msg = "Firmware update failed."
if not apply_update:
failed_msg = "Unable to complete the repository update."
module.fail_json(msg=failed_msg,
update_status=msg['update_status'])
return msg
def _is_http_url(tested_str):
return urlparse(tested_str).scheme.lower() in {'http', 'https'}
def http_request(self, req):
tmp_ca_cert_path, paths_checked = self.get_ca_certs()
https_proxy = os.environ.get('https_proxy')
context = None
if HAS_SSLCONTEXT:
context = self._make_context(tmp_ca_cert_path)
# Detect if 'no_proxy' environment variable is set and if our URL is included
use_proxy = self.detect_no_proxy(req.get_full_url())
if not use_proxy:
# ignore proxy settings for this host request
return req
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if https_proxy:
proxy_parts = generic_urlparse(urlparse(https_proxy))
port = proxy_parts.get('port') or 443
s.connect((proxy_parts.get('hostname'), port))
if proxy_parts.get('scheme') == 'http':
s.sendall(self.CONNECT_COMMAND % (self.hostname, self.port))
if proxy_parts.get('username'):
credentials = "%s:%s" % (proxy_parts.get('username',''), proxy_parts.get('password',''))
s.sendall('Proxy-Authorization: Basic %s\r\n' % credentials.encode('base64').strip())
s.sendall('\r\n')
connect_result = s.recv(4096)
self.validate_proxy_response(connect_result)
if context:
ssl_s = context.wrap_socket(s, server_hostname=self.hostname)
elif HAS_URLLIB3_SNI_SUPPORT:
ssl_s = ssl_wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL, server_hostname=self.hostname)
else:
ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL)
match_hostname(ssl_s.getpeercert(), self.hostname)
else:
raise ProxyError('Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme'))
else:
s.connect((self.hostname, self.port))
if context:
ssl_s = context.wrap_socket(s, server_hostname=self.hostname)
elif HAS_URLLIB3_SNI_SUPPORT:
ssl_s = ssl_wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL, server_hostname=self.hostname)
else:
ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED, ssl_version=PROTOCOL)
match_hostname(ssl_s.getpeercert(), self.hostname)
# close the ssl connection
#ssl_s.unwrap()
s.close()
except (ssl.SSLError, socket.error):
e = get_exception()
# fail if we tried all of the certs but none worked
if 'connection refused' in str(e).lower():
raise ConnectionError('Failed to connect to %s:%s.' % (self.hostname, self.port))
else:
build_ssl_validation_error(self.hostname, self.port, paths_checked)
except CertificateError:
build_ssl_validation_error(self.hostname, self.port, paths_checked)
try:
# cleanup the temp file created, don't worry
# if it fails for some reason
os.remove(tmp_ca_cert_path)
except:
pass
return req
def main():
module = AnsibleModule(
argument_spec=dict(
group_id=dict(required=True),
artifact_id=dict(required=True),
version=dict(default=None),
version_by_spec=dict(default=None),
classifier=dict(default=''),
extension=dict(default='jar'),
repository_url=dict(default='https://repo1.maven.org/maven2'),
username=dict(default=None, aliases=['aws_secret_key']),
password=dict(default=None,
no_log=True,
aliases=['aws_secret_access_key']),
headers=dict(type='dict'),
force_basic_auth=dict(default=False, type='bool'),
state=dict(default="present",
choices=["present", "absent"
]), # TODO - Implement a "latest" state
timeout=dict(default=10, type='int'),
dest=dict(type="path", required=True),
validate_certs=dict(required=False, default=True, type='bool'),
client_cert=dict(type="path", required=False),
client_key=dict(type="path", required=False),
keep_name=dict(required=False, default=False, type='bool'),
verify_checksum=dict(
required=False,
default='download',
choices=['never', 'download', 'change', 'always']),
directory_mode=dict(type='str'),
),
add_file_common_args=True,
mutually_exclusive=([('version', 'version_by_spec')]))
if not HAS_LXML_ETREE:
module.fail_json(msg=missing_required_lib('lxml'),
exception=LXML_ETREE_IMP_ERR)
if module.params['version_by_spec'] and not HAS_SEMANTIC_VERSION:
module.fail_json(msg=missing_required_lib('semantic_version'),
exception=SEMANTIC_VERSION_IMP_ERR)
repository_url = module.params["repository_url"]
if not repository_url:
repository_url = "https://repo1.maven.org/maven2"
try:
parsed_url = urlparse(repository_url)
except AttributeError as e:
module.fail_json(msg='url parsing went wrong %s' % e)
local = parsed_url.scheme == "file"
if parsed_url.scheme == 's3' and not HAS_BOTO:
module.fail_json(msg=missing_required_lib(
'boto3', reason='when using s3:// repository URLs'),
exception=BOTO_IMP_ERR)
group_id = module.params["group_id"]
artifact_id = module.params["artifact_id"]
version = module.params["version"]
version_by_spec = module.params["version_by_spec"]
classifier = module.params["classifier"]
extension = module.params["extension"]
headers = module.params['headers']
state = module.params["state"]
dest = module.params["dest"]
b_dest = to_bytes(dest, errors='surrogate_or_strict')
keep_name = module.params["keep_name"]
verify_checksum = module.params["verify_checksum"]
verify_download = verify_checksum in ['download', 'always']
verify_change = verify_checksum in ['change', 'always']
downloader = MavenDownloader(module, repository_url, local, headers)
if not version_by_spec and not version:
version = "latest"
try:
artifact = Artifact(group_id, artifact_id, version, version_by_spec,
classifier, extension)
except ValueError as e:
module.fail_json(msg=e.args[0])
changed = False
prev_state = "absent"
if dest.endswith(os.sep):
b_dest = to_bytes(dest, errors='surrogate_or_strict')
if not os.path.exists(b_dest):
(pre_existing_dir,
new_directory_list) = split_pre_existing_dir(dest)
os.makedirs(b_dest)
directory_args = module.load_file_common_arguments(module.params)
directory_mode = module.params["directory_mode"]
if directory_mode is not None:
directory_args['mode'] = directory_mode
else:
directory_args['mode'] = None
changed = adjust_recursive_directory_permissions(
pre_existing_dir, new_directory_list, module, directory_args,
changed)
if os.path.isdir(b_dest):
version_part = version
if version == 'latest':
version_part = downloader.find_latest_version_available(artifact)
elif version_by_spec:
version_part = downloader.find_version_by_spec(artifact)
filename = "{artifact_id}{version_part}{classifier}.{extension}".format(
artifact_id=artifact_id,
version_part="-{0}".format(version_part) if keep_name else "",
classifier="-{0}".format(classifier) if classifier else "",
extension=extension)
dest = posixpath.join(dest, filename)
b_dest = to_bytes(dest, errors='surrogate_or_strict')
if os.path.lexists(b_dest) and (
(not verify_change) or not downloader.is_invalid_md5(
dest, downloader.find_uri_for_artifact(artifact))):
prev_state = "present"
if prev_state == "absent":
try:
download_error = downloader.download(module.tmpdir, artifact,
verify_download, b_dest)
if download_error is None:
changed = True
else:
module.fail_json(
msg="Cannot retrieve the artifact to destination: " +
download_error)
except ValueError as e:
module.fail_json(msg=e.args[0])
try:
file_args = module.load_file_common_arguments(module.params, path=dest)
except TypeError:
# The path argument is only supported in Ansible-base 2.10+. Fall back to
# pre-2.10 behavior for older Ansible versions.
module.params['path'] = dest
file_args = module.load_file_common_arguments(module.params)
changed = module.set_fs_attributes_if_different(file_args, changed)
if changed:
module.exit_json(state=state,
dest=dest,
group_id=group_id,
artifact_id=artifact_id,
version=version,
classifier=classifier,
extension=extension,
repository_url=repository_url,
changed=changed)
else:
module.exit_json(state=state, dest=dest, changed=changed)
def open_url(url, data=None, headers=None, method=None, use_proxy=True,
force=False, last_mod_time=None, timeout=10, validate_certs=True,
url_username=None, url_password=None, http_agent=None,
force_basic_auth=False, follow_redirects='urllib2'):
'''
Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
Does not require the module environment
'''
handlers = []
ssl_handler = maybe_add_ssl_handler(url, validate_certs)
if ssl_handler:
handlers.append(ssl_handler)
# FIXME: change the following to use the generic_urlparse function
# to remove the indexed references for 'parsed'
parsed = urlparse(url)
if parsed[0] != 'ftp':
username = url_username
if headers is None:
headers = {}
if username:
password = url_password
netloc = parsed[1]
elif '@' in parsed[1]:
credentials, netloc = parsed[1].split('@', 1)
if ':' in credentials:
username, password = credentials.split(':', 1)
else:
username = credentials
password = ''
parsed = list(parsed)
parsed[1] = netloc
# reconstruct url without credentials
url = urlunparse(parsed)
if username and not force_basic_auth:
passman = urllib_request.HTTPPasswordMgrWithDefaultRealm()
# this creates a password manager
passman.add_password(None, netloc, username, password)
# because we have put None at the start it will always
# use this username/password combination for urls
# for which `theurl` is a super-url
authhandler = urllib_request.HTTPBasicAuthHandler(passman)
# create the AuthHandler
handlers.append(authhandler)
elif username and force_basic_auth:
headers["Authorization"] = basic_auth_header(username, password)
else:
try:
rc = netrc.netrc(os.environ.get('NETRC'))
login = rc.authenticators(parsed[1])
except IOError:
login = None
if login:
username, _, password = login
if username and password:
headers["Authorization"] = basic_auth_header(username, password)
if not use_proxy:
proxyhandler = urllib_request.ProxyHandler({})
handlers.append(proxyhandler)
if HAS_SSLCONTEXT and not validate_certs:
# In 2.7.9, the default context validates certificates
context = SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.options |= ssl.OP_NO_SSLv3
context.verify_mode = ssl.CERT_NONE
context.check_hostname = False
handlers.append(urllib_request.HTTPSHandler(context=context))
# pre-2.6 versions of python cannot use the custom https
# handler, since the socket class is lacking create_connection.
# Some python builds lack HTTPS support.
if hasattr(socket, 'create_connection') and CustomHTTPSHandler:
handlers.append(CustomHTTPSHandler)
handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs))
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
if method:
if method.upper() not in ('OPTIONS','GET','HEAD','POST','PUT','DELETE','TRACE','CONNECT','PATCH'):
raise ConnectionError('invalid HTTP request method; %s' % method.upper())
request = RequestWithMethod(url, method.upper(), data)
else:
request = urllib_request.Request(url, data)
# add the custom agent header, to help prevent issues
# with sites that block the default urllib agent string
request.add_header('User-agent', http_agent)
# if we're ok with getting a 304, set the timestamp in the
# header, otherwise make sure we don't get a cached copy
if last_mod_time and not force:
tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000')
request.add_header('If-Modified-Since', tstamp)
else:
request.add_header('cache-control', 'no-cache')
# user defined headers now, which may override things we've set above
if headers:
if not isinstance(headers, dict):
raise ValueError("headers provided to fetch_url() must be a dict")
for header in headers:
request.add_header(header, headers[header])
urlopen_args = [request, None]
if sys.version_info >= (2,6,0):
# urlopen in python prior to 2.6.0 did not
# have a timeout parameter
urlopen_args.append(timeout)
r = urllib_request.urlopen(*urlopen_args)
return r
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(
dict(
force=dict(required=False, default='no', type='bool'),
policy=dict(required=False, default=None, type='json'),
name=dict(required=True, type='str'),
requester_pays=dict(default='no', type='bool'),
s3_url=dict(aliases=['S3_URL'], type='str'),
state=dict(default='present', type='str', choices=['present', 'absent']),
tags=dict(required=False, default=None, type='dict'),
versioning=dict(default=None, type='bool'),
ceph=dict(default='no', type='bool')
)
)
module = AnsibleModule(argument_spec=argument_spec)
if not HAS_BOTO:
module.fail_json(msg='boto required for this module')
region, ec2_url, aws_connect_params = get_aws_connection_info(module)
if region in ('us-east-1', '', None):
# S3ism for the US Standard region
location = Location.DEFAULT
else:
# Boto uses symbolic names for locations but region strings will
# actually work fine for everything except us-east-1 (US Standard)
location = region
s3_url = module.params.get('s3_url')
# allow eucarc environment variables to be used if ansible vars aren't set
if not s3_url and 'S3_URL' in os.environ:
s3_url = os.environ['S3_URL']
ceph = module.params.get('ceph')
if ceph and not s3_url:
module.fail_json(msg='ceph flavour requires s3_url')
flavour = 'aws'
# bucket names with .'s in them need to use the calling_format option,
# otherwise the connection will fail. See https://github.com/boto/boto/issues/2836
# for more details.
aws_connect_params['calling_format'] = OrdinaryCallingFormat()
# Look at s3_url and tweak connection settings
# if connecting to Walrus or fakes3
try:
if s3_url and ceph:
ceph = urlparse.urlparse(s3_url)
connection = boto.connect_s3(
host=ceph.hostname,
port=ceph.port,
is_secure=ceph.scheme == 'https',
**aws_connect_params
)
flavour = 'ceph'
elif is_fakes3(s3_url):
fakes3 = urlparse.urlparse(s3_url)
connection = S3Connection(
is_secure=fakes3.scheme == 'fakes3s',
host=fakes3.hostname,
port=fakes3.port,
**aws_connect_params
)
elif is_walrus(s3_url):
del aws_connect_params['calling_format']
walrus = urlparse.urlparse(s3_url).hostname
connection = boto.connect_walrus(walrus, **aws_connect_params)
else:
connection = boto.s3.connect_to_region(location, is_secure=True, **aws_connect_params)
# use this as fallback because connect_to_region seems to fail in boto + non 'classic' aws accounts in some cases
if connection is None:
connection = boto.connect_s3(**aws_connect_params)
except boto.exception.NoAuthHandlerFound as e:
module.fail_json(msg='No Authentication Handler found: %s ' % str(e))
except Exception as e:
module.fail_json(msg='Failed to connect to S3: %s' % str(e))
if connection is None: # this should never happen
module.fail_json(msg='Unknown error, failed to create s3 connection, no information from boto.')
state = module.params.get("state")
if state == 'present':
create_or_update_bucket(connection, module, location, flavour=flavour)
elif state == 'absent':
destroy_bucket(connection, module, flavour=flavour)
def main():
# Validates the dependence of the utility module
if HAS_IDG_DEPS:
module_args = dict(
backup=dict(type='bool', required=False,
default=False), # Create a backup file
domain=dict(type='str', required=True), # Domain name
src=dict(type='str',
required=True), # Local path to a file or directory
dest=dict(type='str', required=True), # Remote absolute path
recursive=dict(type='bool', required=False,
default=False), # Download recursively
idg_connection=dict(type='dict',
options=idg_endpoint_spec,
required=True) # IDG connection
)
# AnsibleModule instantiation
module = AnsibleModule(argument_spec=module_args,
supports_check_mode=True)
else:
# Failure AnsibleModule instance
module = AnsibleModule(argument_spec={}, check_invalid_arguments=False)
module.fail_json(msg="The IDG utils modules is required")
# Parse arguments to dict
idg_data_spec = IDGUtils.parse_to_dict(module,
module.params['idg_connection'],
'IDGConnection',
IDGUtils.ANSIBLE_VERSION)
domain_name = module.params['domain']
backup = module.params['backup']
recursive = module.params['recursive']
tmp_dir = '' # Directory for processing on the control host
src = module.params['src']
dest = module.params['dest']
_dest_parse = urlparse(dest)
_dest_ldir = _dest_parse.scheme # Local directory
if _dest_ldir + ':' not in IDGUtils.IDG_DIRS:
module.fail_json(
msg=
"Base directory of the destination file {0} does not correspond to what is specified for datapower."
.format(dest))
_dest_path_list = [
d for d in _dest_parse.path.split('/') if d.strip() != ''
]
# Init IDG API connect
idg_mgmt = IDGApi(ansible_module=module,
idg_host="https://{0}:{1}".format(
idg_data_spec['server'],
idg_data_spec['server_port']),
headers=IDGUtils.BASIC_HEADERS,
http_agent=IDGUtils.HTTP_AGENT_SPEC,
use_proxy=idg_data_spec['use_proxy'],
timeout=idg_data_spec['timeout'],
validate_certs=idg_data_spec['validate_certs'],
user=idg_data_spec['user'],
password=idg_data_spec['password'],
force_basic_auth=IDGUtils.BASIC_AUTH_SPEC)
# Intermediate values for result
tmp_result = {
"msg": IDGUtils.COMPLETED_MESSAGE,
"domain": domain_name,
"backup_file": None,
"changed": True
}
#
# Here the action begins
#
pdb.set_trace()
try:
remote_home_path = '/'.join(
[IDGApi.URI_FILESTORE.format(domain_name), _dest_ldir] +
_dest_path_list)
idg_path = '/'.join([_dest_ldir + ':'] + _dest_path_list)
if os.path.isdir(src): # The source is a directory
# If the user is working in only check mode we do not want to make any changes
IDGUtils.implement_check_mode(module)
if recursive:
for home, subdirs, files in os.walk(
src): # Loop over directory
home_dir = home.strip('/').split(
os.sep)[-1 *
((len(home.strip('/').split(os.sep)) -
len(src.strip('/').split(os.sep))) + 1):]
remote_home_path_uri = '/'.join(
[remote_home_path] + home_dir) # Update root path
idg_path_upd = '/'.join([idg_path] +
home_dir) # Update path inside IDG
create_directory(module, idg_mgmt, remote_home_path_uri,
domain_name)
for file_name in files: # files in home
uri_file = '/'.join([remote_home_path_uri,
file_name]) # Update URI for file
remote_file = '/'.join([idg_path_upd, file_name
]) # Update path inside IDG
if backup: # Backup required
dummy = do_backup(module, idg_mgmt, uri_file,
remote_file, domain_name)
local_file_path = os.path.join(home, file_name)
upload_file(module, idg_mgmt, local_file_path,
uri_file, domain_name)
else: # Not recursive
for home, dummy, files in os.walk(src): # Loop over directory
home_dir = home.split(os.sep)[-1]
remote_home_path = '/'.join([remote_home_path,
home_dir]) # Update root path
idg_path = '/'.join([idg_path,
home_dir]) # Update path inside IDG
create_directory(module, idg_mgmt, remote_home_path,
domain_name)
for file_name in files: # files in home
uri_file = '/'.join([remote_home_path,
file_name]) # Update URI for file
remote_file = '/'.join([idg_path,
file_name]) # Path inside IDG
if backup: # check backup
dummy = do_backup(module, idg_mgmt, uri_file,
remote_file, domain_name)
local_file_path = os.path.join(home, file_name)
upload_file(module, idg_mgmt, local_file_path,
uri_file, domain_name)
break # Prevent continue touring directories
elif os.path.isfile(src): # The source is a local file
file_name = src.split(os.sep)[-1]
uri_file = '/'.join([remote_home_path,
file_name]) # Update URI for file
remote_file = '/'.join([idg_path, file_name]) # Path inside IDG
idg_mgmt.api_call(remote_home_path,
method='GET',
id="get_remote_path")
if idg_mgmt.is_ok(idg_mgmt.last_call()) or idg_mgmt.is_notfound(
idg_mgmt.last_call()):
# If the user is working in only check mode we do not want to make any changes
IDGUtils.implement_check_mode(module)
if 'filestore' not in idg_mgmt.last_call()["data"].keys(
) or idg_mgmt.is_notfound(idg_mgmt.last_call()
): # Is not a directory or not found
create_directory(module, idg_mgmt, remote_home_path,
domain_name)
if backup: # check backup
tmp_result["backup_file"] = do_backup(
module, idg_mgmt, uri_file, remote_file, domain_name)
upload_file(module, idg_mgmt, src, uri_file,
domain_name) # Upload file
else:
# Other Errors
module.fail_json(msg=IDGApi.GENERAL_STATELESS_ERROR.format(
__MODULE_FULLNAME, domain_name) + str(
ErrorHandler(
idg_mgmt.call_by_id("get_remote_path")["data"]
['error'])))
else:
module.fail_json(msg='Source "{0}" is not supported.'.format(src))
#
# Finish
#
# Customize
del result['name']
# Update
for k, v in tmp_result.items():
if v is not None:
result[k] = v
except Exception as e:
# Uncontrolled exception
module.fail_json(msg=(IDGUtils.UNCONTROLLED_EXCEPTION +
'. {0}').format(to_native(e)))
else:
# That's all folks!
module.exit_json(**result)
def http_request(self, req):
tmp_ca_cert_path, to_add_ca_cert_path, paths_checked = self.get_ca_certs(
)
https_proxy = os.environ.get('https_proxy')
context = None
try:
context = self._make_context(to_add_ca_cert_path)
except Exception:
# We'll make do with no context below
pass
# Detect if 'no_proxy' environment variable is set and if our URL is included
use_proxy = self.detect_no_proxy(req.get_full_url())
if not use_proxy:
# ignore proxy settings for this host request
if tmp_ca_cert_path:
try:
os.remove(tmp_ca_cert_path)
except OSError:
pass
if to_add_ca_cert_path:
try:
os.remove(to_add_ca_cert_path)
except OSError:
pass
return req
try:
if https_proxy:
proxy_parts = generic_urlparse(urlparse(https_proxy))
port = proxy_parts.get('port') or 443
s = socket.create_connection(
(proxy_parts.get('hostname'), port))
if proxy_parts.get('scheme') == 'http':
s.sendall(
to_bytes(self.CONNECT_COMMAND %
(self.hostname, self.port),
errors='surrogate_or_strict'))
if proxy_parts.get('username'):
credentials = "%s:%s" % (proxy_parts.get(
'username', ''), proxy_parts.get('password', ''))
s.sendall(
b'Proxy-Authorization: Basic %s\r\n' %
base64.b64encode(
to_bytes(
credentials,
errors='surrogate_or_strict')).strip())
s.sendall(b'\r\n')
connect_result = b""
while connect_result.find(b"\r\n\r\n") <= 0:
connect_result += s.recv(4096)
# 128 kilobytes of headers should be enough for everyone.
if len(connect_result) > 131072:
raise ProxyError(
'Proxy sent too verbose headers. Only 128KiB allowed.'
)
self.validate_proxy_response(connect_result)
if context:
ssl_s = context.wrap_socket(
s, server_hostname=self.hostname)
elif HAS_URLLIB3_SSL_WRAP_SOCKET:
ssl_s = ssl_wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL,
server_hostname=self.hostname)
else:
ssl_s = ssl.wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL)
match_hostname(ssl_s.getpeercert(), self.hostname)
else:
raise ProxyError(
'Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.'
% proxy_parts.get('scheme'))
else:
s = socket.create_connection((self.hostname, self.port))
if context:
ssl_s = context.wrap_socket(s,
server_hostname=self.hostname)
elif HAS_URLLIB3_SSL_WRAP_SOCKET:
ssl_s = ssl_wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL,
server_hostname=self.hostname)
else:
ssl_s = ssl.wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL)
match_hostname(ssl_s.getpeercert(), self.hostname)
# close the ssl connection
# ssl_s.unwrap()
s.close()
except (ssl.SSLError, CertificateError) as e:
build_ssl_validation_error(self.hostname, self.port, paths_checked,
e)
except socket.error as e:
raise ConnectionError('Failed to connect to %s at port %s: %s' %
(self.hostname, self.port, to_native(e)))
try:
# cleanup the temp file created, don't worry
# if it fails for some reason
os.remove(tmp_ca_cert_path)
except:
pass
try:
# cleanup the temp file created, don't worry
# if it fails for some reason
if to_add_ca_cert_path:
os.remove(to_add_ca_cert_path)
except:
pass
return req
def transfer(connection, module, direction, transfer_func):
transfers_service = connection.system_service().image_transfers_service()
transfer = transfers_service.add(
otypes.ImageTransfer(
image=otypes.Image(
id=module.params['id'],
),
direction=direction,
)
)
transfer_service = transfers_service.image_transfer_service(transfer.id)
try:
# After adding a new transfer for the disk, the transfer's status will be INITIALIZING.
# Wait until the init phase is over. The actual transfer can start when its status is "Transferring".
while transfer.phase == otypes.ImageTransferPhase.INITIALIZING:
time.sleep(module.params['poll_interval'])
transfer = transfer_service.get()
proxy_url = urlparse(transfer.proxy_url)
context = ssl.create_default_context()
auth = module.params['auth']
if auth.get('insecure'):
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
elif auth.get('ca_file'):
context.load_verify_locations(cafile=auth.get('ca_file'))
proxy_connection = HTTPSConnection(
proxy_url.hostname,
proxy_url.port,
context=context,
)
transfer_func(
transfer_service,
proxy_connection,
proxy_url,
transfer.signed_ticket
)
return True
finally:
transfer_service.finalize()
while transfer.phase in [
otypes.ImageTransferPhase.TRANSFERRING,
otypes.ImageTransferPhase.FINALIZING_SUCCESS,
]:
time.sleep(module.params['poll_interval'])
transfer = transfer_service.get()
if transfer.phase in [
otypes.ImageTransferPhase.UNKNOWN,
otypes.ImageTransferPhase.FINISHED_FAILURE,
otypes.ImageTransferPhase.FINALIZING_FAILURE,
otypes.ImageTransferPhase.CANCELLED,
]:
raise Exception(
"Error occurred while uploading image. The transfer is in %s" % transfer.phase
)
if module.params.get('logical_unit'):
disks_service = connection.system_service().disks_service()
wait(
service=disks_service.service(module.params['id']),
condition=lambda d: d.status == otypes.DiskStatus.OK,
wait=module.params['wait'],
timeout=module.params['timeout'],
)
def parse_gcp_url(url):
"""
Parse GCP urls and return dict of parts.
Supported URL structures:
/SERVICE/VERSION/'projects'/PROJECT_ID/RESOURCE
/SERVICE/VERSION/'projects'/PROJECT_ID/RESOURCE/ENTITY_NAME
/SERVICE/VERSION/'projects'/PROJECT_ID/RESOURCE/ENTITY_NAME/METHOD_NAME
/SERVICE/VERSION/'projects'/PROJECT_ID/'global'/RESOURCE
/SERVICE/VERSION/'projects'/PROJECT_ID/'global'/RESOURCE/ENTITY_NAME
/SERVICE/VERSION/'projects'/PROJECT_ID/'global'/RESOURCE/ENTITY_NAME/METHOD_NAME
/SERVICE/VERSION/'projects'/PROJECT_ID/LOCATION_TYPE/LOCATION/RESOURCE
/SERVICE/VERSION/'projects'/PROJECT_ID/LOCATION_TYPE/LOCATION/RESOURCE/ENTITY_NAME
/SERVICE/VERSION/'projects'/PROJECT_ID/LOCATION_TYPE/LOCATION/RESOURCE/ENTITY_NAME/METHOD_NAME
:param url: GCP-generated URL, such as a selflink or resource location.
:type url: ``str``
:return: dictionary of parts. Includes stanard components of urlparse, plus
GCP-specific 'service', 'api_version', 'project' and
'resource_name' keys. Optionally, 'zone', 'region', 'entity_name'
and 'method_name', if applicable.
:rtype: ``dict``
"""
p = urlparse.urlparse(url)
if not p:
return None
else:
# we add extra items such as
# zone, region and resource_name
url_parts = {}
url_parts['scheme'] = p.scheme
url_parts['host'] = p.netloc
url_parts['path'] = p.path
if p.path.find('/') == 0:
url_parts['path'] = p.path[1:]
url_parts['params'] = p.params
url_parts['fragment'] = p.fragment
url_parts['query'] = p.query
url_parts['project'] = None
url_parts['service'] = None
url_parts['api_version'] = None
path_parts = url_parts['path'].split('/')
url_parts['service'] = path_parts[0]
url_parts['api_version'] = path_parts[1]
if path_parts[2] == 'projects':
url_parts['project'] = path_parts[3]
else:
# invalid URL
raise GCPInvalidURLError('unable to parse: %s' % url)
if 'global' in path_parts:
url_parts['global'] = True
idx = path_parts.index('global')
if len(path_parts) - idx == 4:
# we have a resource, entity and method_name
url_parts['resource_name'] = path_parts[idx + 1]
url_parts['entity_name'] = path_parts[idx + 2]
url_parts['method_name'] = path_parts[idx + 3]
if len(path_parts) - idx == 3:
# we have a resource and entity
url_parts['resource_name'] = path_parts[idx + 1]
url_parts['entity_name'] = path_parts[idx + 2]
if len(path_parts) - idx == 2:
url_parts['resource_name'] = path_parts[idx + 1]
if len(path_parts) - idx < 2:
# invalid URL
raise GCPInvalidURLError('unable to parse: %s' % url)
elif 'regions' in path_parts or 'zones' in path_parts:
idx = -1
if 'regions' in path_parts:
idx = path_parts.index('regions')
url_parts['region'] = path_parts[idx + 1]
else:
idx = path_parts.index('zones')
url_parts['zone'] = path_parts[idx + 1]
if len(path_parts) - idx == 5:
# we have a resource, entity and method_name
url_parts['resource_name'] = path_parts[idx + 2]
url_parts['entity_name'] = path_parts[idx + 3]
url_parts['method_name'] = path_parts[idx + 4]
if len(path_parts) - idx == 4:
# we have a resource and entity
url_parts['resource_name'] = path_parts[idx + 2]
url_parts['entity_name'] = path_parts[idx + 3]
if len(path_parts) - idx == 3:
url_parts['resource_name'] = path_parts[idx + 2]
if len(path_parts) - idx < 3:
# invalid URL
raise GCPInvalidURLError('unable to parse: %s' % url)
else:
# no location in URL.
idx = path_parts.index('projects')
if len(path_parts) - idx == 5:
# we have a resource, entity and method_name
url_parts['resource_name'] = path_parts[idx + 2]
url_parts['entity_name'] = path_parts[idx + 3]
url_parts['method_name'] = path_parts[idx + 4]
if len(path_parts) - idx == 4:
# we have a resource and entity
url_parts['resource_name'] = path_parts[idx + 2]
url_parts['entity_name'] = path_parts[idx + 3]
if len(path_parts) - idx == 3:
url_parts['resource_name'] = path_parts[idx + 2]
if len(path_parts) - idx < 3:
# invalid URL
raise GCPInvalidURLError('unable to parse: %s' % url)
return url_parts
