def query(action=None, command=None, args=None, method="GET", data=None):
"""
Make a web call to a Parallels provider
"""
path = config.get_config_value("url", get_configured_provider(), __opts__, search_global=False)
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(
realm="Parallels Instance Manager",
uri=path,
user=config.get_config_value("user", get_configured_provider(), __opts__, search_global=False),
passwd=config.get_config_value("password", get_configured_provider(), __opts__, search_global=False),
)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
if action:
path += action
if command:
path += "/{0}".format(command)
if type(args) is not dict:
args = {}
kwargs = {"data": data}
if type(data) is str and "<?xml" in data:
kwargs["headers"] = {"Content-type": "application/xml"}
if args:
path += "?%s"
params = urllib.urlencode(args)
req = urllib2.Request(url=path % params, **kwargs)
else:
req = urllib2.Request(url=path, **kwargs)
req.get_method = lambda: method
log.debug("{0} {1}".format(method, req.get_full_url()))
if data:
log.debug(data)
try:
result = urllib2.urlopen(req)
log.debug("PARALLELS Response Status Code: {0}".format(result.getcode()))
if "content-length" in result.headers:
content = result.read()
result.close()
items = ET.fromstring(content)
return items
return {}
except urllib2.URLError as exc:
log.error("PARALLELS Response Status Code: {0} {1}".format(exc.code, exc.msg))
root = ET.fromstring(exc.read())
log.error(root)
return {"error": root}
def _get_error(error):
# Converts boto exception to string that can be used to output error.
error = '\n'.join(error.split('\n')[1:])
error = ET.fromstring(error)
code = error[0][1].text
message = error[0][2].text
return code, message
def list_users(order_by='id'):
'''
Show all users for this company.
CLI Example:
salt myminion bamboohr.list_users
By default, the return data will be keyed by ID. However, it can be ordered
by any other field. Keep in mind that if the field that is chosen contains
duplicate values (i.e., location is used, for a company which only has one
location), then each duplicate value will be overwritten by the previous.
Therefore, it is advisable to only sort by fields that are guaranteed to be
unique.
CLI Examples:
salt myminion bamboohr.list_users order_by=id
salt myminion bamboohr.list_users order_by=email
'''
ret = {}
status, result = _query(action='meta', command='users')
root = ET.fromstring(result)
users = root.getchildren()
for user in users:
user_id = None
user_ret = {}
for item in user.items():
user_ret[item[0]] = item[1]
if item[0] == 'id':
user_id = item[1]
for item in user.getchildren():
user_ret[item.tag] = item.text
ret[user_ret[order_by]] = user_ret
return ret
def list_employees(order_by='id'):
'''
Show all employees for this company.
CLI Example:
salt myminion bamboohr.list_employees
By default, the return data will be keyed by ID. However, it can be ordered
by any other field. Keep in mind that if the field that is chosen contains
duplicate values (i.e., location is used, for a company which only has one
location), then each duplicate value will be overwritten by the previous.
Therefore, it is advisable to only sort by fields that are guaranteed to be
unique.
CLI Examples:
salt myminion bamboohr.list_employees order_by=id
salt myminion bamboohr.list_employees order_by=displayName
salt myminion bamboohr.list_employees order_by=workEmail
'''
ret = {}
status, result = _query(action='employees', command='directory')
root = ET.fromstring(result)
directory = root.getchildren()
for cat in directory:
if cat.tag != 'employees':
continue
for item in cat:
emp_id = item.items()[0][1]
emp_ret = {'id': emp_id}
for details in item.getchildren():
emp_ret[details.items()[0][1]] = details.text
ret[emp_ret[order_by]] = emp_ret
return ret
def test_gen_vol_xml_for_esxi(self):
xml_data = virt._gen_vol_xml('vmname', 'system', 8192, 'esxi')
root = ET.fromstring(xml_data)
self.assertEqual(root.find('name').text, 'vmname/system.vmdk')
self.assertEqual(root.find('key').text, 'vmname/system')
self.assertEqual(root.find('capacity').attrib['unit'], 'KiB')
self.assertEqual(root.find('capacity').text, str(8192 * 1024))
def create_node(vm_):
"""
Build and submit the XML to create a node
"""
# Start the tree
content = ET.Element("ve")
# Name of the instance
name = ET.SubElement(content, "name")
name.text = vm_["name"]
# Description, defaults to name
desc = ET.SubElement(content, "description")
desc.text = config.get_config_value("desc", vm_, __opts__, default=vm_["name"], search_global=False)
# How many CPU cores, and how fast they are
cpu = ET.SubElement(content, "cpu")
cpu.attrib["number"] = config.get_config_value("cpu_number", vm_, __opts__, default="1", search_global=False)
cpu.attrib["power"] = config.get_config_value("cpu_power", vm_, __opts__, default="1000", search_global=False)
# How many megabytes of RAM
ram = ET.SubElement(content, "ram-size")
ram.text = config.get_config_value("ram", vm_, __opts__, default="256", search_global=False)
# Bandwidth available, in kbps
bandwidth = ET.SubElement(content, "bandwidth")
bandwidth.text = config.get_config_value("bandwidth", vm_, __opts__, default="100", search_global=False)
# How many public IPs will be assigned to this instance
ip_num = ET.SubElement(content, "no-of-public-ip")
ip_num.text = config.get_config_value("ip_num", vm_, __opts__, default="1", search_global=False)
# Size of the instance disk
disk = ET.SubElement(content, "ve-disk")
disk.attrib["local"] = "true"
disk.attrib["size"] = config.get_config_value("disk_size", vm_, __opts__, default="10", search_global=False)
# Attributes for the image
vm_image = config.get_config_value("image", vm_, __opts__, search_global=False)
image = show_image({"image": vm_image}, call="function")
platform = ET.SubElement(content, "platform")
template = ET.SubElement(platform, "template-info")
template.attrib["name"] = vm_image
os_info = ET.SubElement(platform, "os-info")
os_info.attrib["technology"] = image[vm_image]["technology"]
os_info.attrib["type"] = image[vm_image]["osType"]
# Username and password
admin = ET.SubElement(content, "admin")
admin.attrib["login"] = config.get_config_value("ssh_username", vm_, __opts__, default="root")
admin.attrib["password"] = config.get_config_value("password", vm_, __opts__, search_global=False)
data = ET.tostring(content, encoding="UTF-8")
salt.cloud.utils.fire_event(
"event", "requesting instance", "salt/cloud/{0}/requesting".format(vm_["name"]), {"kwargs": data}
)
node = query(action="ve", method="POST", data=data)
return node
def _get_artifact_metadata(artifactory_url, repository, group_id, artifact_id, headers):
metadata_xml = _get_artifact_metadata_xml(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, headers=headers)
root = ET.fromstring(metadata_xml)
assert group_id == root.find('groupId').text
assert artifact_id == root.find('artifactId').text
latest_version = root.find('versioning').find('latest').text
return {
'latest_version': latest_version
}
def mksls(src, dst=None):
'''
Convert an AutoYAST file to an SLS file
'''
with salt.utils.fopen(src, 'r') as fh_:
ps_opts = xml.to_dict(ET.fromstring(fh_.read()))
if dst is not None:
with salt.utils.fopen(dst, 'w') as fh_:
fh_.write(yaml.safe_dump(ps_opts, default_flow_style=False))
else:
return yaml.safe_dump(ps_opts, default_flow_style=False)
def test_boot_default_dev(self):
diskp = virt._disk_profile('default', 'kvm')
nicp = virt._nic_profile('default', 'kvm')
xml_data = virt._gen_xml(
'hello',
1,
512,
diskp,
nicp,
'kvm'
)
root = ET.fromstring(xml_data)
self.assertEqual(root.find('os/boot').attrib['dev'], 'hd')
def _get_artifact_metadata(artifactory_url, repository, group_id, artifact_id, headers):
metadata_xml = _get_artifact_metadata_xml(
artifactory_url=artifactory_url,
repository=repository,
group_id=group_id,
artifact_id=artifact_id,
headers=headers,
)
root = ET.fromstring(metadata_xml)
assert group_id == root.find("groupId").text
assert artifact_id == root.find("artifactId").text
latest_version = root.find("versioning").find("latest").text
return {"latest_version": latest_version}
def test_controller_for_kvm(self):
diskp = virt._disk_profile('default', 'kvm')
nicp = virt._nic_profile('default', 'kvm')
xml_data = virt._gen_xml(
'hello',
1,
512,
diskp,
nicp,
'kvm'
)
root = ET.fromstring(xml_data)
controllers = root.findall('.//devices/controller')
# There should be no controller
self.assertTrue(len(controllers) == 0)
def test_boot_multiple_devs(self):
diskp = virt._disk_profile('default', 'kvm')
nicp = virt._nic_profile('default', 'kvm')
xml_data = virt._gen_xml(
'hello',
1,
512,
diskp,
nicp,
'kvm',
boot_dev='cdrom network'
)
root = ET.fromstring(xml_data)
devs = root.findall('.//boot')
self.assertTrue(len(devs) == 2)
def test_controller_for_esxi(self):
diskp = virt._disk_profile('default', 'esxi')
nicp = virt._nic_profile('default', 'esxi')
xml_data = virt._gen_xml(
'hello',
1,
512,
diskp,
nicp,
'esxi'
)
root = ET.fromstring(xml_data)
controllers = root.findall('.//devices/controller')
self.assertTrue(len(controllers) == 1)
controller = controllers[0]
self.assertEqual(controller.attrib['model'], 'lsilogic')
def test_gen_xml_for_serial_console(self):
diskp = virt._disk_profile('default', 'kvm')
nicp = virt._nic_profile('default', 'kvm')
xml_data = virt._gen_xml(
'hello',
1,
512,
diskp,
nicp,
'kvm',
serial_type='pty',
console=True
)
root = ET.fromstring(xml_data)
self.assertEqual(root.find('devices/serial').attrib['type'], 'pty')
self.assertEqual(root.find('devices/console').attrib['type'], 'pty')
def test_gen_xml_for_telnet_console_unspecified_port(self):
diskp = virt._disk_profile('default', 'kvm')
nicp = virt._nic_profile('default', 'kvm')
xml_data = virt._gen_xml(
'hello',
1,
512,
diskp,
nicp,
'kvm',
serial_type='tcp',
console=True
)
root = ET.fromstring(xml_data)
self.assertEqual(root.find('devices/serial').attrib['type'], 'tcp')
self.assertEqual(root.find('devices/console').attrib['type'], 'tcp')
self.assertIsInstance(int(root.find('devices/console/source').attrib['service']), int)
def get_config(instance=_DEFAULT_INSTANCE):
'''
Determine the configuration of the provided instance.
:param str instance: The name of the Tentacle instance.
:return: A dictionary containing the configuration data.
:rtype: dict
CLI Example:
.. code-block:: bash
salt '*' octopus_tentacle.get_config instance='Tentacle'
'''
ret = dict()
name_mapping = {'Octopus.Home': 'home_path',
'Octopus.Communications.Squid': 'squid',
'Tentacle.CertificateThumbprint': 'thumbprint',
'Tentacle.Communication.TrustedOctopusServers': 'servers',
'Tentacle.Deployment.ApplicationDirectory': 'app_path',
'Tentacle.Services.NoListen': 'comms',
'Tentacle.Services.PortNumber': 'port'}
config_path = get_config_path(instance)
if not os.path.isfile(config_path):
_LOG.error('Unable to get configuration file for instance: %s', instance)
return ret
with salt.utils.fopen(config_path, 'r') as fh_:
config = _parse_config(ElementTree.fromstring(fh_.read()))
for item in config:
# Skip keys that we aren't specifically looking for.
if item in name_mapping:
# Convert the NoListen value to a friendly value.
if name_mapping[item] == 'comms':
for comms_style in _COMMS_STYLES:
if config[item] == _COMMS_STYLES[comms_style]:
ret[name_mapping[item]] = comms_style
break
else:
ret[name_mapping[item]] = config[item]
return ret
def _get_snapshot_version_metadata(artifactory_url, repository, group_id, artifact_id, version, headers):
metadata_xml = _get_snapshot_version_metadata_xml(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, headers=headers)
metadata = ET.fromstring(metadata_xml)
assert group_id == metadata.find('groupId').text
assert artifact_id == metadata.find('artifactId').text
assert version == metadata.find('version').text
snapshot_versions = metadata.find('versioning').find('snapshotVersions')
extension_version_dict = {}
for snapshot_version in snapshot_versions:
extension = snapshot_version.find('extension').text
value = snapshot_version.find('value').text
extension_version_dict[extension] = value
return {
'snapshot_versions': extension_version_dict
}
def test_gen_xml_for_esxi_custom_profile(self, disk_profile, nic_profile):
diskp_yaml = '''
- first:
size: 8192
format: vmdk
model: scsi
pool: datastore1
- second:
size: 4096
format: vmdk # FIX remove line, currently test fails
model: scsi # FIX remove line, currently test fails
pool: datastore2
'''
nicp_yaml = '''
- type: bridge
name: eth1
source: ONENET
model: e1000
mac: '00:00:00:00:00:00'
- name: eth2
type: bridge
source: TWONET
model: e1000
mac: '00:00:00:00:00:00'
'''
disk_profile.return_value = yaml.load(diskp_yaml)
nic_profile.return_value = yaml.load(nicp_yaml)
diskp = virt._disk_profile('noeffect', 'esxi')
nicp = virt._nic_profile('noeffect', 'esxi')
xml_data = virt._gen_xml(
'hello',
1,
512,
diskp,
nicp,
'esxi',
)
root = ET.fromstring(xml_data)
self.assertEqual(root.attrib['type'], 'vmware')
self.assertEqual(root.find('vcpu').text, '1')
self.assertEqual(root.find('memory').text, str(512 * 1024))
self.assertEqual(root.find('memory').attrib['unit'], 'KiB')
self.assertTrue(len(root.findall('.//disk')) == 2)
self.assertTrue(len(root.findall('.//interface')) == 2)
def __execute_cmd(name, xml):
'''
Execute ilom commands
'''
ret = {name.replace('_', ' '): {}}
id_num = 0
tmp_dir = os.path.join(__opts__['cachedir'], 'tmp')
if not os.path.isdir(tmp_dir):
os.mkdir(tmp_dir)
with tempfile.NamedTemporaryFile(dir=tmp_dir,
prefix=name+str(os.getpid()),
suffix='.xml',
delete=False) as fh:
tmpfilename = fh.name
fh.write(xml)
cmd = __salt__['cmd.run_all']('hponcfg -f {0}'.format(tmpfilename))
# Clean up the temp file
__salt__['file.remove'](tmpfilename)
if cmd['retcode'] != 0:
for i in cmd['stderr'].splitlines():
if i.startswith(' MESSAGE='):
return {'Failed': i.split('=')[-1]}
return False
try:
for i in ET.fromstring(''.join(cmd['stdout'].splitlines()[3:-1])):
# Make sure dict keys don't collide
if ret[name.replace('_', ' ')].get(i.tag, False):
ret[name.replace('_', ' ')].update(
{i.tag + '_' + str(id_num): i.attrib}
)
id_num += 1
else:
ret[name.replace('_', ' ')].update(
{i.tag: i.attrib}
)
except SyntaxError:
return True
return ret
def test_gen_xml_for_kvm_custom_profile(self, disk_profile, nic_profile):
diskp_yaml = '''
- first:
size: 8192
format: qcow2
model: virtio
pool: /var/lib/images
- second:
size: 4096
format: qcow2 # FIX remove line, currently test fails
model: virtio # FIX remove line, currently test fails
pool: /var/lib/images
'''
nicp_yaml = '''
- type: bridge
name: eth1
source: b2
model: virtio
mac: '00:00:00:00:00:00'
- name: eth2
type: bridge
source: b2
model: virtio
mac: '00:00:00:00:00:00'
'''
disk_profile.return_value = yaml.load(diskp_yaml)
nic_profile.return_value = yaml.load(nicp_yaml)
diskp = virt._disk_profile('noeffect', 'kvm')
nicp = virt._nic_profile('noeffect', 'kvm')
xml_data = virt._gen_xml(
'hello',
1,
512,
diskp,
nicp,
'kvm',
)
root = ET.fromstring(xml_data)
self.assertEqual(root.attrib['type'], 'kvm')
self.assertEqual(root.find('vcpu').text, '1')
self.assertEqual(root.find('memory').text, str(512 * 1024))
self.assertEqual(root.find('memory').attrib['unit'], 'KiB')
self.assertTrue(len(root.findall('.//disk')) == 2)
self.assertTrue(len(root.findall('.//interface')) == 2)
def list_meta_fields():
'''
Show all meta data fields for this company.
CLI Example:
salt myminion bamboohr.list_meta_fields
'''
ret = {}
status, result = _query(action='meta', command='fields')
root = ET.fromstring(result)
fields = root.getchildren()
for field in fields:
field_id = None
field_ret = {'name': field.text}
for item in field.items():
field_ret[item[0]] = item[1]
if item[0] == 'id':
field_id = item[1]
ret[field_id] = field_ret
return ret
def _get_snapshot_version_metadata(artifactory_url, repository, group_id, artifact_id, version, headers):
metadata_xml = _get_snapshot_version_metadata_xml(
artifactory_url=artifactory_url,
repository=repository,
group_id=group_id,
artifact_id=artifact_id,
version=version,
headers=headers,
)
metadata = ET.fromstring(metadata_xml)
assert group_id == metadata.find("groupId").text
assert artifact_id == metadata.find("artifactId").text
assert version == metadata.find("version").text
snapshot_versions = metadata.find("versioning").find("snapshotVersions")
extension_version_dict = {}
for snapshot_version in snapshot_versions:
extension = snapshot_version.find("extension").text
value = snapshot_version.find("value").text
extension_version_dict[extension] = value
return {"snapshot_versions": extension_version_dict}
def test_gen_xml_for_kvm_default_profile(self):
diskp = virt._disk_profile('default', 'kvm')
nicp = virt._nic_profile('default', 'kvm')
xml_data = virt._gen_xml(
'hello',
1,
512,
diskp,
nicp,
'kvm',
)
root = ET.fromstring(xml_data)
self.assertEqual(root.attrib['type'], 'kvm')
self.assertEqual(root.find('vcpu').text, '1')
self.assertEqual(root.find('memory').text, str(512 * 1024))
self.assertEqual(root.find('memory').attrib['unit'], 'KiB')
disks = root.findall('.//disk')
self.assertEqual(len(disks), 1)
disk = disks[0]
self.assertTrue(disk.find('source').attrib['file'].startswith('/'))
self.assertTrue('hello/system' in disk.find('source').attrib['file'])
self.assertEqual(disk.find('target').attrib['dev'], 'vda')
self.assertEqual(disk.find('target').attrib['bus'], 'virtio')
self.assertEqual(disk.find('driver').attrib['name'], 'qemu')
self.assertEqual(disk.find('driver').attrib['type'], 'qcow2')
interfaces = root.findall('.//interface')
self.assertEqual(len(interfaces), 1)
iface = interfaces[0]
self.assertEqual(iface.attrib['type'], 'bridge')
self.assertEqual(iface.find('source').attrib['bridge'], 'br0')
self.assertEqual(iface.find('model').attrib['type'], 'virtio')
mac = iface.find('mac').attrib['address']
self.assertTrue(
re.match('^([0-9A-F]{2}[:-]){5}([0-9A-F]{2})$', mac, re.I))
def test_gen_xml_for_esxi_default_profile(self):
diskp = virt._disk_profile('default', 'esxi')
nicp = virt._nic_profile('default', 'esxi')
xml_data = virt._gen_xml(
'hello',
1,
512,
diskp,
nicp,
'esxi',
)
root = ET.fromstring(xml_data)
self.assertEqual(root.attrib['type'], 'vmware')
self.assertEqual(root.find('vcpu').text, '1')
self.assertEqual(root.find('memory').text, str(512 * 1024))
self.assertEqual(root.find('memory').attrib['unit'], 'KiB')
disks = root.findall('.//disk')
self.assertEqual(len(disks), 1)
disk = disks[0]
self.assertTrue('[0]' in disk.find('source').attrib['file'])
self.assertTrue('hello/system' in disk.find('source').attrib['file'])
self.assertEqual(disk.find('target').attrib['dev'], 'sda')
self.assertEqual(disk.find('target').attrib['bus'], 'scsi')
self.assertEqual(disk.find('address').attrib['unit'], '0')
interfaces = root.findall('.//interface')
self.assertEqual(len(interfaces), 1)
iface = interfaces[0]
self.assertEqual(iface.attrib['type'], 'bridge')
self.assertEqual(iface.find('source').attrib['bridge'], 'DEFAULT')
self.assertEqual(iface.find('model').attrib['type'], 'e1000')
mac = iface.find('mac').attrib['address']
self.assertTrue(
re.match('^([0-9A-F]{2}[:-]){5}([0-9A-F]{2})$', mac, re.I))
def test_update_hostdev_changes(running, live, make_mock_device, make_mock_vm,
test):
"""
Test the virt.update function with host device changes
"""
xml_def = """
<domain type='kvm'>
<name>my_vm</name>
<memory unit='KiB'>524288</memory>
<currentMemory unit='KiB'>524288</currentMemory>
<vcpu placement='static'>1</vcpu>
<os>
<type arch='x86_64'>hvm</type>
</os>
<on_reboot>restart</on_reboot>
<devices>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</hostdev>
</devices>
</domain>"""
domain_mock = make_mock_vm(xml_def, running)
make_mock_device("""
<device>
<name>usb_3_1_3</name>
<path>/sys/devices/pci0000:00/0000:00:1d.6/0000:06:00.0/0000:07:02.0/0000:3e:00.0/usb3/3-1/3-1.3</path>
<devnode type='dev'>/dev/bus/usb/003/004</devnode>
<parent>usb_3_1</parent>
<driver>
<name>usb</name>
</driver>
<capability type='usb_device'>
<bus>3</bus>
<device>4</device>
<product id='0x6006'>AUKEY PC-LM1E Camera</product>
<vendor id='0x0458'>KYE Systems Corp. (Mouse Systems)</vendor>
</capability>
</device>
""")
make_mock_device("""
<device>
<name>pci_1002_71c4</name>
<parent>pci_8086_27a1</parent>
<capability type='pci'>
<class>0xffffff</class>
<domain>0</domain>
<bus>1</bus>
<slot>0</slot>
<function>0</function>
<product id='0x71c4'>M56GL [Mobility FireGL V5200]</product>
<vendor id='0x1002'>ATI Technologies Inc</vendor>
<numa node='1'/>
</capability>
</device>
""")
ret = virt.update("my_vm",
host_devices=["usb_3_1_3"],
test=test,
live=live)
define_mock = virt.libvirt.openAuth().defineXML
assert_called(define_mock, not test)
# Test that the XML is updated with the proper devices
usb_device_xml = strip_xml("""
<hostdev mode="subsystem" type="usb">
<source>
<vendor id="0x0458" />
<product id="0x6006" />
</source>
</hostdev>
""")
if not test:
set_xml = ET.fromstring(define_mock.call_args[0][0])
actual_hostdevs = [
ET.tostring(xmlutil.strip_spaces(node))
for node in set_xml.findall("./devices/hostdev")
]
assert [usb_device_xml] == actual_hostdevs
if not test and live:
attach_xml = strip_xml(domain_mock.attachDevice.call_args[0][0])
assert usb_device_xml == attach_xml
pci_device_xml = strip_xml("""
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</hostdev>
""")
detach_xml = strip_xml(domain_mock.detachDevice.call_args[0][0])
assert pci_device_xml == detach_xml
else:
domain_mock.attachDevice.assert_not_called()
domain_mock.detachDevice.assert_not_called()
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
backend=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
header_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
raise_error=True,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = salt.config.master_config(
os.path.join(salt.syspaths.CONFIG_DIR, 'master')
)
elif node == 'minion':
opts = salt.config.minion_config(
os.path.join(salt.syspaths.CONFIG_DIR, 'minion')
)
else:
opts = {}
if not backend:
backend = opts.get('backend', 'tornado')
match = re.match(r'https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)', url)
if not match:
salt.utils.network.refresh_dns()
if backend == 'requests':
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = tornado.httputil.url_concat(url, params) if params else url
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(
data_file, data_render, data_renderer, template_dict, opts
)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug('Requesting URL %s using %s method', log_url, method)
log.debug("Using backend: %s", backend)
if method == 'POST' and log.isEnabledFor(logging.TRACE):
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = 'XXXXXXXXXX'
log.trace('Request POST Data: %s', pprint.pformat(log_data))
else:
log.trace('Request POST Data: %s', pprint.pformat(data))
if header_file is not None:
header_tpl = _render(
header_file, header_render, header_renderer, template_dict, opts
)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.txt')
if session_cookie_jar is None:
session_cookie_jar = os.path.join(opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.session.p')
if persist_session is True and HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if backend == 'requests':
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: %s', sess.headers)
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == 'urllib2':
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if backend == 'requests':
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error('The client-side certificate path that'
' was passed is not valid: %s', cert)
result = sess.request(
method, url, params=params, data=data, **req_kwargs
)
result.raise_for_status()
if stream is True:
# fake a HTTP response header
header_callback('HTTP/1.0 {0} MESSAGE'.format(result.status_code))
# fake streaming the content
streaming_callback(result.content)
return {
'handle': result,
}
if handle is True:
return {
'handle': result,
'body': result.content,
}
log.debug('Final URL location of Response: %s',
sanitize_url(result.url, hide_fields))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
body = result.content
if not isinstance(body, six.text_type):
body = body.decode(result.encoding or 'utf-8')
ret['body'] = body
elif backend == 'urllib2':
request = urllib_request.Request(url_full, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https'):
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warning('match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!')
elif verify_ssl is False:
log.warning('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!')
else:
if ':' in hostname:
hostname, port = hostname.split(':')
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(
sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED
)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = (
'The certificate was invalid. '
'Error returned was: %s',
pprint.pformat(exc)
)
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error('The client-side certificate path that was '
'passed is not valid: %s', cert)
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[0] = salt.ext.six.moves.http_client.HTTPSConnection(**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {'Error': six.text_type(exc)}
if stream is True or handle is True:
return {
'handle': result,
'body': result.content,
}
result_status_code = result.code
result_headers = dict(result.info())
result_text = result.read()
if 'Content-Type' in result_headers:
res_content_type, res_params = cgi.parse_header(result_headers['Content-Type'])
if res_content_type.startswith('text/') and \
'charset' in res_params and \
not isinstance(result_text, six.text_type):
result_text = result_text.decode(res_params['charset'])
if six.PY3 and isinstance(result_text, bytes):
result_text = result_text.decode('utf-8')
ret['body'] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['client_cert'] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['client_cert'] = cert[0]
req_kwargs['client_key'] = cert[1]
else:
log.error('The client-side certificate path that '
'was passed is not valid: %s', cert)
if isinstance(data, dict):
data = _urlencode(data)
if verify_ssl:
req_kwargs['ca_certs'] = ca_bundle
max_body = opts.get('http_max_body', salt.config.DEFAULT_MINION_OPTS['http_max_body'])
connect_timeout = opts.get('http_connect_timeout', salt.config.DEFAULT_MINION_OPTS['http_connect_timeout'])
timeout = opts.get('http_request_timeout', salt.config.DEFAULT_MINION_OPTS['http_request_timeout'])
client_argspec = None
proxy_host = opts.get('proxy_host', None)
if proxy_host:
# tornado requires a str for proxy_host, cannot be a unicode str in py2
proxy_host = salt.utils.stringutils.to_str(proxy_host)
proxy_port = opts.get('proxy_port', None)
proxy_username = opts.get('proxy_username', None)
if proxy_username:
# tornado requires a str, cannot be unicode str in py2
proxy_username = salt.utils.stringutils.to_str(proxy_username)
proxy_password = opts.get('proxy_password', None)
if proxy_password:
# tornado requires a str, cannot be unicode str in py2
proxy_password = salt.utils.stringutils.to_str(proxy_password)
no_proxy = opts.get('no_proxy', [])
# Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones
# except we remove the valid ones if a url has a no_proxy hostname in it
if urlparse(url_full).hostname in no_proxy:
proxy_host = None
proxy_port = None
# We want to use curl_http if we have a proxy defined
if proxy_host and proxy_port:
if HAS_CURL_HTTPCLIENT is False:
ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl and tornado, '
'but the libraries does not seem to be installed')
log.error(ret['error'])
return ret
tornado.httpclient.AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient')
client_argspec = salt.utils.args.get_function_argspec(
tornado.curl_httpclient.CurlAsyncHTTPClient.initialize)
else:
client_argspec = salt.utils.args.get_function_argspec(
tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize)
supports_max_body_size = 'max_body_size' in client_argspec.args
req_kwargs.update({
'method': method,
'headers': header_dict,
'auth_username': username,
'auth_password': password,
'body': data,
'validate_cert': verify_ssl,
'allow_nonstandard_methods': True,
'streaming_callback': streaming_callback,
'header_callback': header_callback,
'connect_timeout': connect_timeout,
'request_timeout': timeout,
'proxy_host': proxy_host,
'proxy_port': proxy_port,
'proxy_username': proxy_username,
'proxy_password': proxy_password,
'raise_error': raise_error,
'decompress_response': False,
})
# Unicode types will cause a TypeError when Tornado's curl HTTPClient
# invokes setopt. Therefore, make sure all arguments we pass which
# contain strings are str types.
req_kwargs = salt.utils.data.decode(req_kwargs, to_str=True)
try:
download_client = HTTPClient(max_body_size=max_body) \
if supports_max_body_size \
else HTTPClient()
result = download_client.fetch(url_full, **req_kwargs)
except tornado.httpclient.HTTPError as exc:
ret['status'] = exc.code
ret['error'] = six.text_type(exc)
return ret
except socket.gaierror as exc:
if status is True:
ret['status'] = 0
ret['error'] = six.text_type(exc)
return ret
if stream is True or handle is True:
return {
'handle': result,
'body': result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
if 'Content-Type' in result_headers:
res_content_type, res_params = cgi.parse_header(result_headers['Content-Type'])
if res_content_type.startswith('text/') and \
'charset' in res_params and \
not isinstance(result_text, six.text_type):
result_text = result_text.decode(res_params['charset'])
if six.PY3 and isinstance(result_text, bytes):
result_text = result_text.decode('utf-8')
ret['body'] = result_text
if 'Set-Cookie' in result_headers and cookies is not None:
result_cookies = parse_cookie_header(result_headers['Set-Cookie'])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: %s', result_status_code)
log.trace('Response Headers: %s', result_headers)
log.trace('Response Cookies: %s', sess_cookies)
# log.trace("Content: %s", result_text)
coding = result_headers.get('Content-Encoding', "identity")
# Requests will always decompress the content, and working around that is annoying.
if backend != 'requests':
result_text = __decompressContent(coding, result_text)
try:
log.trace('Response Text: %s', result_text)
except UnicodeEncodeError as exc:
log.trace('Cannot Trace Log Response Text: %s. This may be due to '
'incompatibilities between requests and logging.', exc)
if text_out is not None:
with salt.utils.files.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.files.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get(
'content-type', 'application/json'
)
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
elif 'yaml' in content_type:
decode_type = 'yaml'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'yaml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = (
'Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)
)
)
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = salt.utils.json.loads(result_text)
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
elif decode_type == 'yaml':
ret['dict'] = salt.utils.data.decode(salt.utils.yaml.safe_load(result_text))
else:
text = True
if decode_out:
with salt.utils.files.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
def query(params=None, setname=None, requesturl=None, location=None,
return_url=False, return_root=False, opts=None, provider=None,
endpoint=None, product='ec2', sigver='2'):
'''
Perform a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
Regions and endpoints are documented at:
http://docs.aws.amazon.com/general/latest/gr/rande.html
Default ``product`` is ``ec2``. Valid ``product`` names are:
.. code-block: yaml
- autoscaling (Auto Scaling)
- cloudformation (CloudFormation)
- ec2 (Elastic Compute Cloud)
- elasticache (ElastiCache)
- elasticbeanstalk (Elastic BeanStalk)
- elasticloadbalancing (Elastic Load Balancing)
- elasticmapreduce (Elastic MapReduce)
- iam (Identity and Access Management)
- importexport (Import/Export)
- monitoring (CloudWatch)
- rds (Relational Database Service)
- simpledb (SimpleDB)
- sns (Simple Notification Service)
- sqs (Simple Queue Service)
'''
if params is None:
params = {}
if opts is None:
opts = {}
function = opts.get('function', (None, product))
providers = opts.get('providers', {})
if provider is None:
prov_dict = providers.get(function[1], {}).get(product, {})
if prov_dict:
driver = list(list(prov_dict.keys()))[0]
provider = providers.get(driver, product)
else:
prov_dict = providers.get(provider, {}).get(product, {})
service_url = prov_dict.get('service_url', 'amazonaws.com')
if not location:
location = get_location(opts, provider)
if endpoint is None:
if not requesturl:
endpoint = prov_dict.get(
'endpoint',
'{0}.{1}.{2}'.format(product, location, service_url)
)
requesturl = 'https://{0}/'.format(endpoint)
else:
endpoint = urlparse(requesturl).netloc
if endpoint == '':
endpoint_err = ('Could not find a valid endpoint in the '
'requesturl: {0}. Looking for something '
'like https://some.aws.endpoint/?args').format(
requesturl
)
LOG.error(endpoint_err)
if return_url is True:
return {'error': endpoint_err}, requesturl
return {'error': endpoint_err}
LOG.debug('Using AWS endpoint: {0}'.format(endpoint))
method = 'GET'
aws_api_version = prov_dict.get(
'aws_api_version', prov_dict.get(
'{0}_api_version'.format(product),
DEFAULT_AWS_API_VERSION
)
)
if sigver == '4':
headers, requesturl = sig4(
method, endpoint, params, prov_dict, aws_api_version, location, product, requesturl=requesturl
)
params_with_headers = {}
else:
params_with_headers = sig2(
method, endpoint, params, prov_dict, aws_api_version
)
headers = {}
attempts = 5
while attempts > 0:
LOG.debug('AWS Request: {0}'.format(requesturl))
LOG.trace('AWS Request Parameters: {0}'.format(params_with_headers))
try:
result = requests.get(requesturl, headers=headers, params=params_with_headers)
LOG.debug(
'AWS Response Status Code: {0}'.format(
result.status_code
)
)
LOG.trace(
'AWS Response Text: {0}'.format(
result.text
)
)
result.raise_for_status()
break
except requests.exceptions.HTTPError as exc:
root = ET.fromstring(exc.response.content)
data = xml.to_dict(root)
# check to see if we should retry the query
err_code = data.get('Errors', {}).get('Error', {}).get('Code', '')
if attempts > 0 and err_code and err_code in AWS_RETRY_CODES:
attempts -= 1
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}; '
'Attempts remaining: {3}'.format(
exc.response.status_code, exc, data, attempts
)
)
# Wait a bit before continuing to prevent throttling
time.sleep(2)
continue
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data
)
)
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
else:
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data
)
)
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
response = result.text
root = ET.fromstring(response)
items = root[1]
if return_root is True:
items = root
if setname:
if sys.version_info < (2, 7):
children_len = len(root.getchildren())
else:
children_len = len(root)
for item in range(0, children_len):
comps = root[item].tag.split('}')
if comps[1] == setname:
items = root[item]
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
return ret
def test_xml_case_f_full(self):
xmldata = ET.fromstring(self.cases['f']['xml'])
defaultdict = xml.to_dict(xmldata, True)
self.assertEqual(defaultdict, self.cases['f']['full'])
def create_node(vm_):
'''
Build and submit the XML to create a node
'''
# Start the tree
content = ET.Element('ve')
# Name of the instance
name = ET.SubElement(content, 'name')
name.text = vm_['name']
# Description, defaults to name
desc = ET.SubElement(content, 'description')
desc.text = config.get_cloud_config_value('desc',
vm_,
__opts__,
default=vm_['name'],
search_global=False)
# How many CPU cores, and how fast they are
cpu = ET.SubElement(content, 'cpu')
cpu.attrib['number'] = config.get_cloud_config_value('cpu_number',
vm_,
__opts__,
default='1',
search_global=False)
cpu.attrib['power'] = config.get_cloud_config_value('cpu_power',
vm_,
__opts__,
default='1000',
search_global=False)
# How many megabytes of RAM
ram = ET.SubElement(content, 'ram-size')
ram.text = config.get_cloud_config_value('ram',
vm_,
__opts__,
default='256',
search_global=False)
# Bandwidth available, in kbps
bandwidth = ET.SubElement(content, 'bandwidth')
bandwidth.text = config.get_cloud_config_value('bandwidth',
vm_,
__opts__,
default='100',
search_global=False)
# How many public IPs will be assigned to this instance
ip_num = ET.SubElement(content, 'no-of-public-ip')
ip_num.text = config.get_cloud_config_value('ip_num',
vm_,
__opts__,
default='1',
search_global=False)
# Size of the instance disk
disk = ET.SubElement(content, 've-disk')
disk.attrib['local'] = 'true'
disk.attrib['size'] = config.get_cloud_config_value('disk_size',
vm_,
__opts__,
default='10',
search_global=False)
# Attributes for the image
vm_image = config.get_cloud_config_value('image',
vm_,
__opts__,
search_global=False)
image = show_image({'image': vm_image}, call='function')
platform = ET.SubElement(content, 'platform')
template = ET.SubElement(platform, 'template-info')
template.attrib['name'] = vm_image
os_info = ET.SubElement(platform, 'os-info')
os_info.attrib['technology'] = image[vm_image]['technology']
os_info.attrib['type'] = image[vm_image]['osType']
# Username and password
admin = ET.SubElement(content, 'admin')
admin.attrib['login'] = config.get_cloud_config_value('ssh_username',
vm_,
__opts__,
default='root')
admin.attrib['password'] = config.get_cloud_config_value(
'password', vm_, __opts__, search_global=False)
data = ET.tostring(content, encoding='UTF-8')
salt.utils.cloud.fire_event('event',
'requesting instance',
'salt/cloud/{0}/requesting'.format(
vm_['name']), {'kwargs': data},
transport=__opts__['transport'])
node = query(action='ve', method='POST', data=data)
return node
def query(action=None, command=None, args=None, method='GET', data=None):
'''
Make a web call to a Parallels provider
'''
path = config.get_cloud_config_value(
'url', get_configured_provider(), __opts__, search_global=False
)
auth_handler = _HTTPBasicAuthHandler()
auth_handler.add_password(
realm='Parallels Instance Manager',
uri=path,
user=config.get_cloud_config_value(
'user', get_configured_provider(), __opts__, search_global=False
),
passwd=config.get_cloud_config_value(
'password', get_configured_provider(), __opts__,
search_global=False
)
)
opener = _build_opener(auth_handler)
_install_opener(opener)
if action:
path += action
if command:
path += '/{0}'.format(command)
if not type(args, dict):
args = {}
kwargs = {'data': data}
if isinstance(data, str) and '<?xml' in data:
kwargs['headers'] = {
'Content-type': 'application/xml',
}
if args:
params = _urlencode(args)
req = _Request(url='{0}?{1}'.format(path, params), **kwargs)
else:
req = _Request(url=path, **kwargs)
req.get_method = lambda: method
log.debug('{0} {1}'.format(method, req.get_full_url()))
if data:
log.debug(data)
try:
result = _urlopen(req)
log.debug(
'PARALLELS Response Status Code: {0}'.format(
result.getcode()
)
)
if 'content-length' in result.headers:
content = result.read()
result.close()
items = ET.fromstring(content)
return items
return {}
except URLError as exc:
log.error(
'PARALLELS Response Status Code: {0} {1}'.format(
exc.code,
exc.msg
)
)
root = ET.fromstring(exc.read())
log.error(root)
return {'error': root}
def test_update_clock(make_mock_vm):
"""
test virt.update with clock parameter
"""
xml_def = """
<domain type="kvm">
<name>my_vm</name>
<memory unit='KiB'>524288</memory>
<currentMemory unit='KiB'>524288</currentMemory>
<vcpu placement='static'>1</vcpu>
<os>
<type arch='x86_64'>linux</type>
<kernel>/usr/lib/grub2/x86_64-xen/grub.xen</kernel>
</os>
<clock offset="localtime" adjustment="-3600">
<timer name="tsc" frequency="3504000000" mode="native" />
<timer name="kvmclock" present="no" />
</clock>
<on_reboot>restart</on_reboot>
</domain>
"""
domain_mock = make_mock_vm(xml_def)
# Update with no change to the features
ret = virt.update(
"my_vm",
clock={
"utc": False,
"adjustment": -3600,
"timers": {
"tsc": {
"frequency": 3504000000,
"mode": "native"
},
"kvmclock": {
"present": False
},
},
},
)
assert not ret["definition"]
# Update
ret = virt.update(
"my_vm",
clock={
"timezone": "CEST",
"timers": {
"rtc": {
"track": "wall",
"tickpolicy": "catchup",
"slew": 4636,
"threshold": 123,
"limit": 2342,
},
"hpet": {
"present": True
},
},
},
)
assert ret["definition"]
setxml = ET.fromstring(virt.libvirt.openAuth().defineXML.call_args[0][0])
assert "timezone" == setxml.find("clock").get("offset")
assert "CEST" == setxml.find("clock").get("timezone")
assert {"rtc",
"hpet"} == {t.get("name")
for t in setxml.findall("clock/timer")}
assert "catchup" == setxml.find("clock/timer[@name='rtc']").get(
"tickpolicy")
assert "wall" == setxml.find("clock/timer[@name='rtc']").get("track")
assert {
"slew": "4636",
"threshold": "123",
"limit": "2342"
} == setxml.find("clock/timer[@name='rtc']/catchup").attrib
assert "yes" == setxml.find("clock/timer[@name='hpet']").get("present")
# Revert to UTC
ret = virt.update("my_vm",
clock={
"utc": True,
"adjustment": None,
"timers": None
})
assert ret["definition"]
setxml = ET.fromstring(virt.libvirt.openAuth().defineXML.call_args[0][0])
assert {"offset": "utc"} == setxml.find("clock").attrib
assert setxml.find("clock/timer") is None
def show_employee(emp_id, fields=None):
'''
Show all employees for this company.
CLI Example:
salt myminion bamboohr.show_employee 1138
By default, the fields normally returned from bamboohr.list_employees are
returned. These fields are:
- canUploadPhoto
- department
- displayName
- firstName
- id
- jobTitle
- lastName
- location
- mobilePhone
- nickname
- photoUploaded
- photoUrl
- workEmail
- workPhone
- workPhoneExtension
If needed, a different set of fields may be specified, separated by commas:
CLI Example:
salt myminion bamboohr.show_employee 1138 displayName,dateOfBirth
A list of available fields can be found at
http://www.bamboohr.com/api/documentation/employees.php
'''
ret = {}
if fields is None:
fields = ','.join((
'canUploadPhoto',
'department',
'displayName',
'firstName',
'id',
'jobTitle',
'lastName',
'location',
'mobilePhone',
'nickname',
'photoUploaded',
'photoUrl',
'workEmail',
'workPhone',
'workPhoneExtension',
))
status, result = _query(
action='employees',
command=emp_id,
args={'fields': fields}
)
root = ET.fromstring(result)
items = root.getchildren()
ret = {'id': emp_id}
for item in items:
ret[item.items()[0][1]] = item.text
return ret
def query(key, keyid, method='GET', params=None, headers=None,
requesturl=None, return_url=False, bucket=None, service_url=None,
path='', return_bin=False, action=None, local_file=None,
verify_ssl=True, location=None, full_headers=False):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
If keyid or key is not specified, an attempt to fetch them from EC2 IAM
metadata service will be made.
A service_url may also be specified in the configuration:
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
A region may be specified:
s3.location: eu-central-1
If region is not specified, an attempt to fetch the region from EC2 IAM
metadata service will be made. Failing that, default is us-east-1
'''
if not HAS_REQUESTS:
log.error('There was an error: requests is required for s3 access')
if not headers:
headers = {}
if not params:
params = {}
if not service_url:
service_url = 's3.amazonaws.com'
if bucket:
endpoint = '{0}.{1}'.format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
if not key or not keyid:
key = salt.utils.aws.IROLE_CODE
keyid = salt.utils.aws.IROLE_CODE
if not location:
location = iam.get_iam_region()
if not location:
location = DEFAULT_LOCATION
data = ''
if method == 'PUT':
if local_file:
with salt.utils.fopen(local_file, 'r') as ifile:
data = ifile.read()
if not requesturl:
requesturl = 'https://{0}/{1}'.format(endpoint, path)
headers, requesturl = salt.utils.aws.sig4(
method,
endpoint,
params,
data=data,
uri='/{0}'.format(path),
prov_dict={'id': keyid, 'key': key},
location=location,
product='s3',
requesturl=requesturl,
)
log.debug('S3 Request: {0}'.format(requesturl))
log.debug('S3 Headers::')
log.debug(' Authorization: {0}'.format(headers['Authorization']))
if not data:
data = None
try:
result = requests.request(method, requesturl, headers=headers,
data=data,
verify=verify_ssl)
response = result.content
except requests.exceptions.HTTPError as exc:
log.error('There was an error::')
if hasattr(exc, 'code') and hasattr(exc, 'msg'):
log.error(' Code: {0}: {1}'.format(exc.code, exc.msg))
log.error(' Content: \n{0}'.format(exc.read()))
return False
log.debug('S3 Response Status Code: {0}'.format(result.status_code))
if method == 'PUT':
if result.status_code == 200:
if local_file:
log.debug('Uploaded from {0} to {1}'.format(local_file, path))
else:
log.debug('Created bucket {0}'.format(bucket))
else:
if local_file:
log.debug('Failed to upload from {0} to {1}: {2}'.format(
local_file,
path,
result.status_code,
))
else:
log.debug('Failed to create bucket {0}'.format(bucket))
return
if method == 'DELETE':
if str(result.status_code).startswith('2'):
if path:
log.debug('Deleted {0} from bucket {1}'.format(path, bucket))
else:
log.debug('Deleted bucket {0}'.format(bucket))
else:
if path:
log.debug('Failed to delete {0} from bucket {1}: {2}'.format(
path,
bucket,
result.status_code,
))
else:
log.debug('Failed to delete bucket {0}'.format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'w') as out:
out.write(response)
return 'Saved to local file: {0}'.format(local_file)
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {'headers': []}
if full_headers:
ret['headers'] = dict(result.headers)
else:
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def query(key, keyid, method='GET', params=None, headers=None,
requesturl=None, return_url=False, bucket=None, service_url=None,
path='', return_bin=False, action=None, local_file=None,
verify_ssl=True, location=None, full_headers=False):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
If keyid or key is not specified, an attempt to fetch them from EC2 IAM
metadata service will be made.
A service_url may also be specified in the configuration:
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
A region may be specified:
s3.location: eu-central-1
If region is not specified, an attempt to fetch the region from EC2 IAM
metadata service will be made. Failing that, default is us-east-1
'''
if not HAS_REQUESTS:
log.error('There was an error: requests is required for s3 access')
if not headers:
headers = {}
if not params:
params = {}
if not service_url:
service_url = 's3.amazonaws.com'
if bucket:
endpoint = '{0}.{1}'.format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
if not key or not keyid:
key = salt.utils.aws.IROLE_CODE
keyid = salt.utils.aws.IROLE_CODE
data = ''
if method == 'PUT':
if local_file:
with salt.utils.fopen(local_file, 'r') as ifile:
data = ifile.read()
if not requesturl:
requesturl = 'https://{0}/{1}'.format(endpoint, path)
headers, requesturl = salt.utils.aws.sig4(
method,
endpoint,
params,
data=data,
uri='/{0}'.format(path),
prov_dict={'id': keyid, 'key': key},
location=location,
product='s3',
requesturl=requesturl,
)
log.debug('S3 Request: {0}'.format(requesturl))
log.debug('S3 Headers::')
log.debug(' Authorization: {0}'.format(headers['Authorization']))
if not data:
data = None
try:
result = requests.request(method, requesturl, headers=headers,
data=data,
verify=verify_ssl)
response = result.content
except requests.exceptions.HTTPError as exc:
log.error('There was an error::')
if hasattr(exc, 'code') and hasattr(exc, 'msg'):
log.error(' Code: {0}: {1}'.format(exc.code, exc.msg))
log.error(' Content: \n{0}'.format(exc.read()))
return False
log.debug('S3 Response Status Code: {0}'.format(result.status_code))
if method == 'PUT':
if result.status_code == 200:
if local_file:
log.debug('Uploaded from {0} to {1}'.format(local_file, path))
else:
log.debug('Created bucket {0}'.format(bucket))
else:
if local_file:
log.debug('Failed to upload from {0} to {1}: {2}'.format(
local_file,
path,
result.status_code,
))
else:
log.debug('Failed to create bucket {0}'.format(bucket))
return
if method == 'DELETE':
if str(result.status_code).startswith('2'):
if path:
log.debug('Deleted {0} from bucket {1}'.format(path, bucket))
else:
log.debug('Deleted bucket {0}'.format(bucket))
else:
if path:
log.debug('Failed to delete {0} from bucket {1}: {2}'.format(
path,
bucket,
result.status_code,
))
else:
log.debug('Failed to delete bucket {0}'.format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'w') as out:
out.write(response)
return 'Saved to local file: {0}'.format(local_file)
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {'headers': []}
if full_headers:
ret['headers'] = dict(result.headers)
else:
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def query(key, keyid, method='GET', params=None, headers=None,
requesturl=None, return_url=False, bucket=None, service_url=None,
path='', return_bin=False, action=None, local_file=None,
verify_ssl=True, full_headers=False, kms_keyid=None,
location=None, role_arn=None, chunk_size=16384, path_style=False,
https_enable=True):
"""
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
If keyid or key is not specified, an attempt to fetch them from EC2 IAM
metadata service will be made.
A service_url may also be specified in the configuration:
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
Path style can be enabled:
s3.path_style: True
This can be useful if you need to use salt with a proxy for an s3 compatible storage
You can use either https protocol or http protocol:
s3.https_enable: True
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
A region may be specified:
s3.location: eu-central-1
If region is not specified, an attempt to fetch the region from EC2 IAM
metadata service will be made. Failing that, default is us-east-1
"""
if not HAS_REQUESTS:
log.error('There was an error: requests is required for s3 access')
if not headers:
headers = {}
if not params:
params = {}
if not service_url:
service_url = 's3.amazonaws.com'
if not bucket or path_style:
endpoint = service_url
else:
endpoint = '{0}.{1}'.format(bucket, service_url)
if path_style and bucket:
path = '{0}/{1}'.format(bucket, path)
# Try grabbing the credentials from the EC2 instance IAM metadata if available
if not key:
key = salt.utils.aws.IROLE_CODE
if not keyid:
keyid = salt.utils.aws.IROLE_CODE
if kms_keyid is not None and method in ('PUT', 'POST'):
headers['x-amz-server-side-encryption'] = 'aws:kms'
headers['x-amz-server-side-encryption-aws-kms-key-id'] = kms_keyid
if not location:
location = salt.utils.aws.get_location()
data = ''
fh = None
payload_hash = None
if method == 'PUT':
if local_file:
payload_hash = salt.utils.hashutils.get_hash(local_file, form='sha256')
if path is None:
path = ''
path = _quote(path)
if not requesturl:
requesturl = (('https' if https_enable else 'http')+'://{0}/{1}').format(endpoint, path)
headers, requesturl = salt.utils.aws.sig4(
method,
endpoint,
params,
data=data,
uri='/{0}'.format(path),
prov_dict={'id': keyid, 'key': key},
role_arn=role_arn,
location=location,
product='s3',
requesturl=requesturl,
headers=headers,
payload_hash=payload_hash,
)
log.debug('S3 Request: %s', requesturl)
log.debug('S3 Headers::')
log.debug(' Authorization: %s', headers['Authorization'])
if not data:
data = None
try:
if method == 'PUT':
if local_file:
fh = salt.utils.files.fopen(local_file, 'rb') # pylint: disable=resource-leakage
data = fh.read() # pylint: disable=resource-leakage
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True,
timeout=300)
elif method == 'GET' and local_file and not return_bin:
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True,
timeout=300)
else:
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
timeout=300)
finally:
if fh is not None:
fh.close()
err_code = None
err_msg = None
if result.status_code >= 400:
# On error the S3 API response should contain error message
err_text = result.content or 'Unknown error'
log.debug(' Response content: %s', err_text)
# Try to get err info from response xml
try:
err_data = xml.to_dict(ET.fromstring(err_text))
err_code = err_data['Code']
err_msg = err_data['Message']
except (KeyError, ET.ParseError) as err:
log.debug(
'Failed to parse s3 err response. %s: %s',
type(err).__name__, err
)
err_code = 'http-{0}'.format(result.status_code)
err_msg = err_text
if os.environ.get('MOCK_SLOW_DOWN'):
result.status_code = 503
err_code = 'SlowDown'
err_msg = 'MOCK_SLOW_DOWN environment variable set. All S3 queries will fail for testing purposes.'
log.debug('S3 Response Status Code: %s', result.status_code)
if method == 'PUT':
if result.status_code != 200:
if local_file:
raise CommandExecutionError(
'Failed to upload from {0} to {1}. {2}: {3}'.format(
local_file, path, err_code, err_msg))
raise CommandExecutionError(
'Failed to create bucket {0}. {1}: {2}'.format(
bucket, err_code, err_msg))
if local_file:
log.debug('Uploaded from %s to %s', local_file, path)
else:
log.debug('Created bucket %s', bucket)
return None
if method == 'DELETE':
if not six.text_type(result.status_code).startswith('2'):
if path:
raise CommandExecutionError(
'Failed to delete {0} from bucket {1}. {2}: {3}'.format(
path, bucket, err_code, err_msg))
raise CommandExecutionError(
'Failed to delete bucket {0}. {1}: {2}'.format(
bucket, err_code, err_msg))
if path:
log.debug('Deleted %s from bucket %s', path, bucket)
else:
log.debug('Deleted bucket %s', bucket)
return None
sortof_ok = ['SlowDown', 'ServiceUnavailable', 'RequestTimeTooSkewed',
'RequestTimeout', 'OperationAborted', 'InternalError',
'AccessDenied']
# This can be used to save a binary object to disk
if local_file and method == 'GET':
if result.status_code < 200 or result.status_code >= 300:
if err_code in sortof_ok:
log.error('Failed to get file=%s. %s: %s', path, err_code, err_msg)
return None
raise CommandExecutionError(
'Failed to get file=%s. {0}: {1}'.format(path, err_code, err_msg))
log.debug('Saving to local file: %s', local_file)
with salt.utils.files.fopen(local_file, 'wb') as out:
for chunk in result.iter_content(chunk_size=chunk_size):
out.write(chunk)
return 'Saved to local file: {0}'.format(local_file)
if result.status_code < 200 or result.status_code >= 300:
if err_code in sortof_ok:
log.error('Failed s3 operation. %s: %s', err_code, err_msg)
return None
raise CommandExecutionError(
'Failed s3 operation. {0}: {1}'.format(err_code, err_msg))
# This can be used to return a binary object wholesale
if return_bin:
return result.content
if result.content:
items = ET.fromstring(result.content)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return None
ret = {'headers': []}
if full_headers:
ret['headers'] = dict(result.headers)
else:
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def query(key, keyid, method='GET', params=None, headers=None,
requesturl=None, return_url=False, bucket=None, service_url=None,
path=None, return_bin=False, action=None, local_file=None,
verify_ssl=True):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
A service_url may also be specified in the configuration::
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
'''
if not headers:
headers = {}
if not params:
params = {}
if path is None:
path = ''
if not service_url:
service_url = 's3.amazonaws.com'
if bucket:
endpoint = '{0}.{1}'.format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
token = None
if not key or not keyid:
iam_creds = iam.get_iam_metadata()
key = iam_creds['secret_key']
keyid = iam_creds['access_key']
token = iam_creds['security_token']
if not requesturl:
x_amz_date = datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
content_type = 'text/plain'
if method == 'GET':
if bucket:
can_resource = '/{0}/{1}'.format(bucket, path)
else:
can_resource = '/'
elif method == 'PUT' or method == 'HEAD' or method == 'DELETE':
if path:
can_resource = '/{0}/{1}'.format(bucket, path)
else:
can_resource = '/{0}/'.format(bucket)
if action:
can_resource += '?{0}'.format(action)
log.debug('CanonicalizedResource: {0}'.format(can_resource))
headers['Host'] = endpoint
headers['Content-type'] = content_type
headers['Date'] = x_amz_date
if token:
headers['x-amz-security-token'] = token
string_to_sign = '{0}\n'.format(method)
new_headers = []
for header in sorted(headers):
if header.lower().startswith('x-amz'):
log.debug(header.lower())
new_headers.append('{0}:{1}'.format(header.lower(),
headers[header]))
can_headers = '\n'.join(new_headers)
log.debug('CanonicalizedAmzHeaders: {0}'.format(can_headers))
string_to_sign += '\n{0}'.format(content_type)
string_to_sign += '\n{0}'.format(x_amz_date)
if can_headers:
string_to_sign += '\n{0}'.format(can_headers)
string_to_sign += '\n{0}'.format(can_resource)
log.debug('String To Sign:: \n{0}'.format(string_to_sign))
hashed = hmac.new(key, string_to_sign, hashlib.sha1)
sig = binascii.b2a_base64(hashed.digest())
headers['Authorization'] = 'AWS {0}:{1}'.format(keyid, sig.strip())
querystring = urllib.urlencode(params)
if action:
if querystring:
querystring = '{0}&{1}'.format(action, querystring)
else:
querystring = action
requesturl = 'https://{0}/'.format(endpoint)
if path:
requesturl += path
if querystring:
requesturl += '?{0}'.format(querystring)
if method == 'PUT':
if local_file:
with salt.utils.fopen(local_file, 'r') as ifile:
data = ifile.read()
log.debug('S3 Request: {0}'.format(requesturl))
log.debug('S3 Headers::')
log.debug(' Authorization: {0}'.format(headers['Authorization']))
try:
result = requests.request(method, requesturl, headers=headers,
verify=verify_ssl)
response = result.content
except requests.exceptions.HTTPError as exc:
log.error('There was an error::')
if hasattr(exc, 'code') and hasattr(exc, 'msg'):
log.error(' Code: {0}: {1}'.format(exc.code, exc.msg))
log.error(' Content: \n{0}'.format(exc.read()))
return False
log.debug('S3 Response Status Code: {0}'.format(result.status_code))
if method == 'PUT':
if result.getcode() == 200:
if local_file:
log.debug('Uploaded from {0} to {1}'.format(local_file, path))
else:
log.debug('Created bucket {0}'.format(bucket))
else:
if local_file:
log.debug('Failed to upload from {0} to {1}: {2}'.format(
local_file,
path,
result.getcode(),
))
else:
log.debug('Failed to create bucket {0}'.format(bucket))
return
if method == 'DELETE':
if str(result.getcode()).startswith('2'):
if path:
log.debug('Deleted {0} from bucket {1}'.format(path, bucket))
else:
log.debug('Deleted bucket {0}'.format(bucket))
else:
if path:
log.debug('Failed to delete {0} from bucket {1}: {2}'.format(
path,
bucket,
result.getcode(),
))
else:
log.debug('Failed to delete bucket {0}'.format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'w') as out:
out.write(response)
return 'Saved to local file: {0}'.format(local_file)
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if method == 'GET' or method == 'HEAD':
return
ret = {'headers': []}
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def query(key,
keyid,
method='GET',
params=None,
headers=None,
requesturl=None,
return_url=False,
bucket=None,
service_url=None,
path='',
return_bin=False,
action=None,
local_file=None,
verify_ssl=True,
full_headers=False,
kms_keyid=None,
location=None,
role_arn=None,
chunk_size=16384):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
If keyid or key is not specified, an attempt to fetch them from EC2 IAM
metadata service will be made.
A service_url may also be specified in the configuration:
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
A region may be specified:
s3.location: eu-central-1
If region is not specified, an attempt to fetch the region from EC2 IAM
metadata service will be made. Failing that, default is us-east-1
'''
if not HAS_REQUESTS:
log.error('There was an error: requests is required for s3 access')
if not headers:
headers = {}
if not params:
params = {}
if not service_url:
service_url = 's3.amazonaws.com'
if bucket:
endpoint = '{0}.{1}'.format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
if not key:
key = salt.utils.aws.IROLE_CODE
if not keyid:
keyid = salt.utils.aws.IROLE_CODE
if kms_keyid is not None and method in ('PUT', 'POST'):
headers['x-amz-server-side-encryption'] = 'aws:kms'
headers['x-amz-server-side-encryption-aws-kms-key-id'] = kms_keyid
if not location:
location = __utils__['aws.get_location']()
data = ''
payload_hash = None
if method == 'PUT':
if local_file:
payload_hash = salt.utils.get_hash(local_file, form='sha256')
if path is None:
path = ''
if not requesturl:
requesturl = 'https://{0}/{1}'.format(endpoint, path)
headers, requesturl = salt.utils.aws.sig4(
method,
endpoint,
params,
data=data,
uri='/{0}'.format(path),
prov_dict={
'id': keyid,
'key': key
},
role_arn=role_arn,
location=location,
product='s3',
requesturl=requesturl,
headers=headers,
payload_hash=payload_hash,
)
log.debug('S3 Request: {0}'.format(requesturl))
log.debug('S3 Headers::')
log.debug(' Authorization: {0}'.format(headers['Authorization']))
if not data:
data = None
response = None
if method == 'PUT':
if local_file:
data = salt.utils.fopen(local_file, 'r')
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True)
response = result.content
elif method == 'GET' and local_file and not return_bin:
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True)
else:
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl)
response = result.content
err_code = None
err_msg = None
if result.status_code >= 400:
# On error the S3 API response should contain error message
err_text = response or result.content or 'Unknown error'
log.debug(' Response content: {0}'.format(err_text))
# Try to get err info from response xml
try:
err_data = xml.to_dict(ET.fromstring(err_text))
err_code = err_data['Code']
err_msg = err_data['Message']
except (KeyError, ET.ParseError) as err:
log.debug('Failed to parse s3 err response. {0}: {1}'.format(
type(err).__name__, err))
err_code = 'http-{0}'.format(result.status_code)
err_msg = err_text
log.debug('S3 Response Status Code: {0}'.format(result.status_code))
if method == 'PUT':
if result.status_code != 200:
if local_file:
raise CommandExecutionError(
'Failed to upload from {0} to {1}. {2}: {3}'.format(
local_file, path, err_code, err_msg))
raise CommandExecutionError(
'Failed to create bucket {0}. {1}: {2}'.format(
bucket, err_code, err_msg))
if local_file:
log.debug('Uploaded from {0} to {1}'.format(local_file, path))
else:
log.debug('Created bucket {0}'.format(bucket))
return
if method == 'DELETE':
if not str(result.status_code).startswith('2'):
if path:
raise CommandExecutionError(
'Failed to delete {0} from bucket {1}. {2}: {3}'.format(
path, bucket, err_code, err_msg))
raise CommandExecutionError(
'Failed to delete bucket {0}. {1}: {2}'.format(
bucket, err_code, err_msg))
if path:
log.debug('Deleted {0} from bucket {1}'.format(path, bucket))
else:
log.debug('Deleted bucket {0}'.format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
if result.status_code < 200 or result.status_code >= 300:
raise CommandExecutionError('Failed to get file. {0}: {1}'.format(
err_code, err_msg))
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'wb') as out:
for chunk in result.iter_content(chunk_size=chunk_size):
out.write(chunk)
return 'Saved to local file: {0}'.format(local_file)
if result.status_code < 200 or result.status_code >= 300:
raise CommandExecutionError('Failed s3 operation. {0}: {1}'.format(
err_code, err_msg))
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {'headers': []}
if full_headers:
ret['headers'] = dict(result.headers)
else:
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def test_boot_default_dev(self):
diskp = virt._disk_profile('default', 'kvm')
nicp = virt._nic_profile('default', 'kvm')
xml_data = virt._gen_xml('hello', 1, 512, diskp, nicp, 'kvm')
root = ET.fromstring(xml_data)
self.assertEqual(root.find('os/boot').attrib['dev'], 'hd')
def call(payload=None):
'''
This function captures the query string and sends it to the Palo Alto device.
'''
r = None
try:
if DETAILS['method'] == 'dev_key':
# Pass the api key without the target declaration
conditional_payload = {'key': DETAILS['apikey']}
payload.update(conditional_payload)
r = __utils__['http.query'](DETAILS['url'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=DETAILS["verify_ssl"],
raise_error=True)
elif DETAILS['method'] == 'dev_pass':
# Pass credentials without the target declaration
r = __utils__['http.query'](DETAILS['url'],
username=DETAILS['username'],
password=DETAILS['password'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=DETAILS["verify_ssl"],
raise_error=True)
elif DETAILS['method'] == 'pan_key':
# Pass the api key with the target declaration
conditional_payload = {
'key': DETAILS['apikey'],
'target': DETAILS['serial']
}
payload.update(conditional_payload)
r = __utils__['http.query'](DETAILS['url'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=DETAILS["verify_ssl"],
raise_error=True)
elif DETAILS['method'] == 'pan_pass':
# Pass credentials with the target declaration
conditional_payload = {'target': DETAILS['serial']}
payload.update(conditional_payload)
r = __utils__['http.query'](DETAILS['url'],
username=DETAILS['username'],
password=DETAILS['password'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=DETAILS["verify_ssl"],
raise_error=True)
except KeyError as err:
raise salt.exceptions.CommandExecutionError(
"Did not receive a valid response from host.")
if not r:
raise salt.exceptions.CommandExecutionError(
"Did not receive a valid response from host.")
xmldata = ET.fromstring(r['text'])
# If we are pulling the candidate configuration, we need to strip the dirtyId
if payload['type'] == 'config' and payload['action'] == 'get':
xmldata = (_strip_dirty(xmldata))
return xml.to_dict(xmldata, True)
def show_employee(emp_id, fields=None):
'''
Show all employees for this company.
CLI Example:
salt myminion bamboohr.show_employee 1138
By default, the fields normally returned from bamboohr.list_employees are
returned. These fields are:
- canUploadPhoto
- department
- displayName
- firstName
- id
- jobTitle
- lastName
- location
- mobilePhone
- nickname
- photoUploaded
- photoUrl
- workEmail
- workPhone
- workPhoneExtension
If needed, a different set of fields may be specified, separated by commas:
CLI Example:
salt myminion bamboohr.show_employee 1138 displayName,dateOfBirth
A list of available fields can be found at
http://www.bamboohr.com/api/documentation/employees.php
'''
ret = {}
if fields is None:
fields = ','.join((
'canUploadPhoto',
'department',
'displayName',
'firstName',
'id',
'jobTitle',
'lastName',
'location',
'mobilePhone',
'nickname',
'photoUploaded',
'photoUrl',
'workEmail',
'workPhone',
'workPhoneExtension',
))
status, result = _query(action='employees',
command=emp_id,
args={'fields': fields})
root = ET.fromstring(result)
items = root.getchildren()
ret = {'id': emp_id}
for item in items:
ret[item.items()[0][1]] = item.text
return ret
def security_rule_exists(
name,
rulename=None,
vsys="1",
action=None,
disabled=None,
sourcezone=None,
destinationzone=None,
source=None,
destination=None,
application=None,
service=None,
description=None,
logsetting=None,
logstart=None,
logend=None,
negatesource=None,
negatedestination=None,
profilegroup=None,
datafilter=None,
fileblock=None,
spyware=None,
urlfilter=None,
virus=None,
vulnerability=None,
wildfire=None,
move=None,
movetarget=None,
commit=False,
):
"""
Ensures that a security rule exists on the device. Also, ensure that all configurations are set appropriately.
This method will create the rule if it does not exist. If the rule does exist, it will ensure that the
configurations are set appropriately.
If the rule does not exist and is created, any value that is not provided will be provided as the default.
The action, to, from, source, destination, application, and service fields are mandatory and must be provided.
This will enforce the exact match of the rule. For example, if the rule is currently configured with the log-end
option, but this option is not specified in the state method, it will be removed and reset to the system default.
It is strongly recommended to specify all options to ensure proper operation.
When defining the profile group settings, the device can only support either a profile group or individual settings.
If both are specified, the profile group will be preferred and the individual settings are ignored. If neither are
specified, the value will be set to system default of none.
name: The name of the module function to execute.
rulename(str): The name of the security rule. The name is case-sensitive and can have up to 31 characters, which
can be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on Panorama,
unique within its device group and any ancestor or descendant device groups.
vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1.
action(str): The action that the security rule will enforce. Valid options are: allow, deny, drop, reset-client,
reset-server, reset-both.
disabled(bool): Controls if the rule is disabled. Set 'True' to disable and 'False' to enable.
sourcezone(str, list): The source zone(s). The value 'any' will match all zones.
destinationzone(str, list): The destination zone(s). The value 'any' will match all zones.
source(str, list): The source address(es). The value 'any' will match all addresses.
destination(str, list): The destination address(es). The value 'any' will match all addresses.
application(str, list): The application(s) matched. The value 'any' will match all applications.
service(str, list): The service(s) matched. The value 'any' will match all services. The value
'application-default' will match based upon the application defined ports.
description(str): A description for the policy (up to 255 characters).
logsetting(str): The name of a valid log forwarding profile.
logstart(bool): Generates a traffic log entry for the start of a session (disabled by default).
logend(bool): Generates a traffic log entry for the end of a session (enabled by default).
negatesource(bool): Match all but the specified source addresses.
negatedestination(bool): Match all but the specified destination addresses.
profilegroup(str): A valid profile group name.
datafilter(str): A valid data filter profile name. Ignored with the profilegroup option set.
fileblock(str): A valid file blocking profile name. Ignored with the profilegroup option set.
spyware(str): A valid spyware profile name. Ignored with the profilegroup option set.
urlfilter(str): A valid URL filtering profile name. Ignored with the profilegroup option set.
virus(str): A valid virus profile name. Ignored with the profilegroup option set.
vulnerability(str): A valid vulnerability profile name. Ignored with the profilegroup option set.
wildfire(str): A valid vulnerability profile name. Ignored with the profilegroup option set.
move(str): An optional argument that ensure the rule is moved to a specific location. Valid options are 'top',
'bottom', 'before', or 'after'. The 'before' and 'after' options require the use of the 'movetarget' argument
to define the location of the move request.
movetarget(str): An optional argument that defines the target of the move operation if the move argument is
set to 'before' or 'after'.
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/rulebase/security/rule01:
panos.security_rule_exists:
- rulename: rule01
- vsys: 1
- action: allow
- disabled: False
- sourcezone: untrust
- destinationzone: trust
- source:
- 10.10.10.0/24
- 1.1.1.1
- destination:
- 2.2.2.2-2.2.2.4
- application:
- any
- service:
- tcp-25
- description: My test security rule
- logsetting: logprofile
- logstart: False
- logend: True
- negatesource: False
- negatedestination: False
- profilegroup: myprofilegroup
- move: top
- commit: False
panos/rulebase/security/rule01:
panos.security_rule_exists:
- rulename: rule01
- vsys: 1
- action: allow
- disabled: False
- sourcezone: untrust
- destinationzone: trust
- source:
- 10.10.10.0/24
- 1.1.1.1
- destination:
- 2.2.2.2-2.2.2.4
- application:
- any
- service:
- tcp-25
- description: My test security rule
- logsetting: logprofile
- logstart: False
- logend: False
- datafilter: foobar
- fileblock: foobar
- spyware: foobar
- urlfilter: foobar
- virus: foobar
- vulnerability: foobar
- wildfire: foobar
- move: after
- movetarget: rule02
- commit: False
"""
ret = _default_ret(name)
if not rulename:
return ret
# Check if rule currently exists
rule = __salt__["panos.get_security_rule"](rulename, vsys)["result"]
if rule and "entry" in rule:
rule = rule["entry"]
else:
rule = {}
# Build the rule element
element = ""
if sourcezone:
element += "<from>{0}</from>".format(_build_members(sourcezone, True))
else:
ret.update({"comment": "The sourcezone field must be provided."})
return ret
if destinationzone:
element += "<to>{0}</to>".format(_build_members(destinationzone, True))
else:
ret.update({"comment": "The destinationzone field must be provided."})
return ret
if source:
element += "<source>{0}</source>".format(_build_members(source, True))
else:
ret.update({"comment": "The source field must be provided."})
return
if destination:
element += "<destination>{0}</destination>".format(
_build_members(destination, True))
else:
ret.update({"comment": "The destination field must be provided."})
return ret
if application:
element += "<application>{0}</application>".format(
_build_members(application, True))
else:
ret.update({"comment": "The application field must be provided."})
return ret
if service:
element += "<service>{0}</service>".format(
_build_members(service, True))
else:
ret.update({"comment": "The service field must be provided."})
return ret
if action:
element += "<action>{0}</action>".format(action)
else:
ret.update({"comment": "The action field must be provided."})
return ret
if disabled is not None:
if disabled:
element += "<disabled>yes</disabled>"
else:
element += "<disabled>no</disabled>"
if description:
element += "<description>{0}</description>".format(description)
if logsetting:
element += "<log-setting>{0}</log-setting>".format(logsetting)
if logstart is not None:
if logstart:
element += "<log-start>yes</log-start>"
else:
element += "<log-start>no</log-start>"
if logend is not None:
if logend:
element += "<log-end>yes</log-end>"
else:
element += "<log-end>no</log-end>"
if negatesource is not None:
if negatesource:
element += "<negate-source>yes</negate-source>"
else:
element += "<negate-source>no</negate-source>"
if negatedestination is not None:
if negatedestination:
element += "<negate-destination>yes</negate-destination>"
else:
element += "<negate-destination>no</negate-destination>"
# Build the profile settings
profile_string = None
if profilegroup:
profile_string = "<group><member>{0}</member></group>".format(
profilegroup)
else:
member_string = ""
if datafilter:
member_string += "<data-filtering><member>{0}</member></data-filtering>".format(
datafilter)
if fileblock:
member_string += "<file-blocking><member>{0}</member></file-blocking>".format(
fileblock)
if spyware:
member_string += "<spyware><member>{0}</member></spyware>".format(
spyware)
if urlfilter:
member_string += "<url-filtering><member>{0}</member></url-filtering>".format(
urlfilter)
if virus:
member_string += "<virus><member>{0}</member></virus>".format(
virus)
if vulnerability:
member_string += "<vulnerability><member>{0}</member></vulnerability>".format(
vulnerability)
if wildfire:
member_string += "<wildfire-analysis><member>{0}</member></wildfire-analysis>".format(
wildfire)
if member_string != "":
profile_string = "<profiles>{0}</profiles>".format(member_string)
if profile_string:
element += "<profile-setting>{0}</profile-setting>".format(
profile_string)
full_element = "<entry name='{0}'>{1}</entry>".format(rulename, element)
new_rule = xml.to_dict(ET.fromstring(full_element), True)
config_change = False
if rule == new_rule:
ret.update(
{"comment": "Security rule already exists. No changes required."})
else:
config_change = True
xpath = (
"/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys{0}']/rulebase/"
"security/rules/entry[@name='{1}']".format(vsys, rulename))
result, msg = _edit_config(xpath, full_element)
if not result:
ret.update({"comment": msg})
return ret
ret.update({
"changes": {
"before": rule,
"after": new_rule
},
"comment": "Security rule verified successfully.",
})
if move:
movepath = (
"/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys{0}']/rulebase/"
"security/rules/entry[@name='{1}']".format(vsys, rulename))
move_result = False
move_msg = ""
if move == "before" and movetarget:
move_result, move_msg = _move_before(movepath, movetarget)
elif move == "after":
move_result, move_msg = _move_after(movepath, movetarget)
elif move == "top":
move_result, move_msg = _move_top(movepath)
elif move == "bottom":
move_result, move_msg = _move_bottom(movepath)
if config_change:
ret.update({
"changes": {
"before": rule,
"after": new_rule,
"move": move_msg
}
})
else:
ret.update({"changes": {"move": move_msg}})
if not move_result:
ret.update({"comment": move_msg})
return ret
if commit is True:
ret.update({"commit": __salt__["panos.commit"](), "result": True})
else:
ret.update({"result": True})
return ret
def test_gen_xml_isolated():
"""
Test the virt._gen_net_xml() function for an isolated network
"""
xml_data = virt._gen_net_xml("network", "main", None, None)
assert ET.fromstring(xml_data).find("forward") is None
def test_gen_clock():
"""
Test the virt._gen_xml clock property
"""
# Localtime with adjustment
xml_data = virt._gen_xml(
virt.libvirt.openAuth.return_value,
"hello",
1,
512,
{},
{},
"kvm",
"hvm",
"x86_64",
clock={
"adjustment": 3600,
"utc": False
},
)
root = ET.fromstring(xml_data)
assert "localtime" == root.find("clock").get("offset")
assert "3600" == root.find("clock").get("adjustment")
# Specific timezone
xml_data = virt._gen_xml(
virt.libvirt.openAuth.return_value,
"hello",
1,
512,
{},
{},
"kvm",
"hvm",
"x86_64",
clock={"timezone": "CEST"},
)
root = ET.fromstring(xml_data)
assert "timezone" == root.find("clock").get("offset")
assert "CEST" == root.find("clock").get("timezone")
# UTC
xml_data = virt._gen_xml(
virt.libvirt.openAuth.return_value,
"hello",
1,
512,
{},
{},
"kvm",
"hvm",
"x86_64",
clock={"utc": True},
)
root = ET.fromstring(xml_data)
assert "utc" == root.find("clock").get("offset")
# Timers
xml_data = virt._gen_xml(
virt.libvirt.openAuth.return_value,
"hello",
1,
512,
{},
{},
"kvm",
"hvm",
"x86_64",
clock={
"timers": {
"tsc": {
"frequency": 3504000000,
"mode": "native"
},
"rtc": {
"tickpolicy": "catchup",
"slew": 4636,
"threshold": 123,
"limit": 2342,
},
"hpet": {
"present": False
},
},
},
)
root = ET.fromstring(xml_data)
assert "utc" == root.find("clock").get("offset")
assert "3504000000" == root.find("clock/timer[@name='tsc']").get(
"frequency")
assert "native" == root.find("clock/timer[@name='tsc']").get("mode")
assert "catchup" == root.find("clock/timer[@name='rtc']").get("tickpolicy")
assert {
"slew": "4636",
"threshold": "123",
"limit": "2342"
} == root.find("clock/timer[@name='rtc']/catchup").attrib
assert "no" == root.find("clock/timer[@name='hpet']").get("present")
def test_gen_xml_nat():
"""
Test virt._get_net_xml() in a nat setup
"""
xml_data = virt._gen_net_xml(
"network",
"main",
"nat",
None,
ip_configs=[
{
"cidr":
"192.168.2.0/24",
"dhcp_ranges": [
{
"start": "192.168.2.10",
"end": "192.168.2.25"
},
{
"start": "192.168.2.110",
"end": "192.168.2.125"
},
],
"hosts": {
"192.168.2.10": {
"mac": "00:16:3e:77:e2:ed",
"name": "foo.example.com",
},
},
"bootp": {
"file": "pxeboot.img",
"server": "192.168.2.1"
},
"tftp":
"/path/to/tftp",
},
{
"cidr": "2001:db8:ca2:2::/64",
"hosts": {
"2001:db8:ca2:2:3::1": {
"name": "paul"
},
"2001:db8:ca2:2:3::2": {
"id": "0:3:0:1:0:16:3e:11:22:33",
"name": "ralph",
},
},
},
],
nat={
"address": {
"start": "1.2.3.4",
"end": "1.2.3.10"
},
"port": {
"start": 500,
"end": 1000
},
},
domain={
"name": "acme.lab",
"localOnly": True
},
mtu=9000,
)
root = ET.fromstring(xml_data)
assert "network" == root.find("name").text
assert "main" == root.find("bridge").attrib["name"]
assert "nat" == root.find("forward").attrib["mode"]
expected_ipv4 = ET.fromstring("""
<ip family='ipv4' address='192.168.2.1' prefix='24'>
<dhcp>
<range start='192.168.2.10' end='192.168.2.25'/>
<range start='192.168.2.110' end='192.168.2.125'/>
<host ip='192.168.2.10' mac='00:16:3e:77:e2:ed' name='foo.example.com'/>
<bootp file='pxeboot.img' server='192.168.2.1'/>
</dhcp>
<tftp root='/path/to/tftp'/>
</ip>
""")
assert_xml_equals(expected_ipv4, root.find("./ip[@address='192.168.2.1']"))
expected_ipv6 = ET.fromstring("""
<ip family='ipv6' address='2001:db8:ca2:2::1' prefix='64'>
<dhcp>
<host ip='2001:db8:ca2:2:3::1' name='paul'/>
<host ip='2001:db8:ca2:2:3::2' id='0:3:0:1:0:16:3e:11:22:33' name='ralph'/>
</dhcp>
</ip>
""")
assert_xml_equals(expected_ipv6,
root.find("./ip[@address='2001:db8:ca2:2::1']"))
actual_nat = ET.tostring(xmlutil.strip_spaces(root.find("./forward/nat")))
expected_nat = strip_xml("""
<nat>
<address start='1.2.3.4' end='1.2.3.10'/>
<port start='500' end='1000'/>
</nat>
""")
assert expected_nat == actual_nat
assert {
"name": "acme.lab",
"localOnly": "yes"
} == root.find("./domain").attrib
assert "9000" == root.find("mtu").get("size")
def test_update_xen_disk_volumes(make_mock_vm, make_mock_storage_pool):
xml_def = """
<domain type='xen'>
<name>my_vm</name>
<memory unit='KiB'>524288</memory>
<currentMemory unit='KiB'>524288</currentMemory>
<vcpu placement='static'>1</vcpu>
<os>
<type arch='x86_64'>linux</type>
<kernel>/usr/lib/grub2/x86_64-xen/grub.xen</kernel>
</os>
<devices>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2' cache='none' io='native'/>
<source file='/path/to/default/my_vm_system'/>
<target dev='xvda' bus='xen'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/path/to/my-iscsi/unit:0:0:1'/>
<target dev='xvdb' bus='xen'/>
</disk>
<controller type='xenbus' index='0'/>
</devices>
</domain>"""
domain_mock = make_mock_vm(xml_def)
make_mock_storage_pool("default", "dir", ["my_vm_system"])
make_mock_storage_pool("my-iscsi", "iscsi", ["unit:0:0:1"])
make_mock_storage_pool("vdb", "disk", ["vdb1"])
ret = virt.update(
"my_vm",
disks=[
{
"name": "system",
"pool": "default"
},
{
"name": "iscsi-data",
"pool": "my-iscsi",
"source_file": "unit:0:0:1"
},
{
"name": "vdb-data",
"pool": "vdb",
"source_file": "vdb1"
},
{
"name": "file-data",
"pool": "default",
"size": "10240"
},
],
)
assert ret["definition"]
define_mock = virt.libvirt.openAuth().defineXML
setxml = ET.fromstring(define_mock.call_args[0][0])
assert "block" == setxml.find(".//disk[3]").get("type")
assert "/path/to/vdb/vdb1" == setxml.find(".//disk[3]/source").get("dev")
# Note that my_vm-file-data was not an existing volume before the update
assert "file" == setxml.find(".//disk[4]").get("type")
assert "/path/to/default/my_vm_file-data" == setxml.find(
".//disk[4]/source").get("file")
def query(
key,
keyid,
method="GET",
params=None,
headers=None,
requesturl=None,
return_url=False,
bucket=None,
service_url=None,
path="",
return_bin=False,
action=None,
local_file=None,
verify_ssl=True,
full_headers=False,
kms_keyid=None,
location=None,
role_arn=None,
chunk_size=16384,
path_style=False,
https_enable=True,
):
"""
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
If keyid or key is not specified, an attempt to fetch them from EC2 IAM
metadata service will be made.
A service_url may also be specified in the configuration:
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
Path style can be enabled:
s3.path_style: True
This can be useful if you need to use salt with a proxy for an s3 compatible storage
You can use either https protocol or http protocol:
s3.https_enable: True
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
A region may be specified:
s3.location: eu-central-1
If region is not specified, an attempt to fetch the region from EC2 IAM
metadata service will be made. Failing that, default is us-east-1
"""
if not HAS_REQUESTS:
log.error("There was an error: requests is required for s3 access")
if not headers:
headers = {}
if not params:
params = {}
if not service_url:
service_url = "s3.amazonaws.com"
if not bucket or path_style:
endpoint = service_url
else:
endpoint = "{}.{}".format(bucket, service_url)
if path_style and bucket:
path = "{}/{}".format(bucket, path)
# Try grabbing the credentials from the EC2 instance IAM metadata if available
if not key:
key = salt.utils.aws.IROLE_CODE
if not keyid:
keyid = salt.utils.aws.IROLE_CODE
if kms_keyid is not None and method in ("PUT", "POST"):
headers["x-amz-server-side-encryption"] = "aws:kms"
headers["x-amz-server-side-encryption-aws-kms-key-id"] = kms_keyid
if not location:
location = salt.utils.aws.get_location()
data = ""
fh = None
payload_hash = None
if method == "PUT":
if local_file:
payload_hash = salt.utils.hashutils.get_hash(local_file, form="sha256")
if path is None:
path = ""
path = urllib.parse.quote(path)
if not requesturl:
requesturl = (("https" if https_enable else "http") + "://{0}/{1}").format(
endpoint, path
)
headers, requesturl = salt.utils.aws.sig4(
method,
endpoint,
params,
data=data,
uri="/{}".format(path),
prov_dict={"id": keyid, "key": key},
role_arn=role_arn,
location=location,
product="s3",
requesturl=requesturl,
headers=headers,
payload_hash=payload_hash,
)
log.debug("S3 Request: %s", requesturl)
log.debug("S3 Headers::")
log.debug(" Authorization: %s", headers["Authorization"])
if not data:
data = None
try:
if method == "PUT":
if local_file:
# pylint: disable=resource-leakage
fh = salt.utils.files.fopen(local_file, "rb")
# pylint: enable=resource-leakage
data = fh.read() # pylint: disable=resource-leakage
result = requests.request(
method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True,
timeout=300,
)
elif method == "GET" and local_file and not return_bin:
result = requests.request(
method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True,
timeout=300,
)
else:
result = requests.request(
method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
timeout=300,
)
finally:
if fh is not None:
fh.close()
err_code = None
err_msg = None
if result.status_code >= 400:
# On error the S3 API response should contain error message
err_text = result.content or "Unknown error"
log.debug(" Response content: %s", err_text)
# Try to get err info from response xml
try:
err_data = xml.to_dict(ET.fromstring(err_text))
err_code = err_data["Code"]
err_msg = err_data["Message"]
except (KeyError, ET.ParseError) as err:
log.debug(
"Failed to parse s3 err response. %s: %s", type(err).__name__, err
)
err_code = "http-{}".format(result.status_code)
err_msg = err_text
log.debug("S3 Response Status Code: %s", result.status_code)
if method == "PUT":
if result.status_code != 200:
if local_file:
raise CommandExecutionError(
"Failed to upload from {} to {}. {}: {}".format(
local_file, path, err_code, err_msg
)
)
raise CommandExecutionError(
"Failed to create bucket {}. {}: {}".format(bucket, err_code, err_msg)
)
if local_file:
log.debug("Uploaded from %s to %s", local_file, path)
else:
log.debug("Created bucket %s", bucket)
return
if method == "DELETE":
if not str(result.status_code).startswith("2"):
if path:
raise CommandExecutionError(
"Failed to delete {} from bucket {}. {}: {}".format(
path, bucket, err_code, err_msg
)
)
raise CommandExecutionError(
"Failed to delete bucket {}. {}: {}".format(bucket, err_code, err_msg)
)
if path:
log.debug("Deleted %s from bucket %s", path, bucket)
else:
log.debug("Deleted bucket %s", bucket)
return
# This can be used to save a binary object to disk
if local_file and method == "GET":
if result.status_code < 200 or result.status_code >= 300:
raise CommandExecutionError(
"Failed to get file. {}: {}".format(err_code, err_msg)
)
log.debug("Saving to local file: %s", local_file)
with salt.utils.files.fopen(local_file, "wb") as out:
for chunk in result.iter_content(chunk_size=chunk_size):
out.write(chunk)
return "Saved to local file: {}".format(local_file)
if result.status_code < 200 or result.status_code >= 300:
raise CommandExecutionError(
"Failed s3 operation. {}: {}".format(err_code, err_msg)
)
# This can be used to return a binary object wholesale
if return_bin:
return result.content
if result.content:
items = ET.fromstring(result.content)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {"headers": []}
if full_headers:
ret["headers"] = dict(result.headers)
else:
for header in result.headers:
ret["headers"].append(header.strip())
return ret
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
backend='tornado',
requests_lib=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = salt.config.master_config(
os.path.join(syspaths.CONFIG_DIR, 'master'))
elif node == 'minion':
opts = salt.config.minion_config(
os.path.join(syspaths.CONFIG_DIR, 'minion'))
else:
opts = {}
if requests_lib is None:
requests_lib = opts.get('requests_lib', False)
if requests_lib is True:
log.warn('Please set "backend" to "requests" instead of setting '
'"requests_lib" to "True"')
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
backend = 'requests'
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = tornado.httputil.url_concat(url, params)
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(data_file, data_render, data_renderer, template_dict,
opts)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug('Requesting URL {0} using {1} method'.format(log_url, method))
if method == 'POST':
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = 'XXXXXXXXXX'
log.trace('Request POST Data: {0}'.format(
pprint.pformat(log_data)))
else:
log.trace('Request POST Data: {0}'.format(pprint.pformat(data)))
if header_file is not None:
header_tpl = _render(header_file, header_render, header_renderer,
template_dict, opts)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(opts.get('cachedir', syspaths.CACHE_DIR),
'cookies.txt')
if session_cookie_jar is None:
session_cookie_jar = os.path.join(
opts.get('cachedir', syspaths.CACHE_DIR), 'cookies.session.p')
if persist_session is True and HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.fopen(session_cookie_jar, 'rb') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.fopen(session_cookie_jar, 'wb') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
else:
auth = None
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if backend == 'requests':
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: {0}'.format(sess.headers))
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == 'urllib2':
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(
cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(
cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if backend == 'requests':
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error(
'The client-side certificate path that was passed is '
'not valid: {0}'.format(cert))
result = sess.request(method,
url,
params=params,
data=data,
**req_kwargs)
result.raise_for_status()
if stream is True or handle is True:
return {'handle': result}
log.debug('Final URL location of Response: {0}'.format(
sanitize_url(result.url, hide_fields)))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.text
result_cookies = result.cookies
elif backend == 'urllib2':
request = urllib_request.Request(url_full, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https') or port == 443:
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warn(
('match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!'))
elif verify_ssl is False:
log.warn(('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!'))
else:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, 443))
sockwrap = ssl.wrap_socket(sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = ('The certificate was invalid. '
'Error returned was: {0}'.format(
pprint.pformat(exc)))
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error('The client-side certificate path that was '
'passed is not valid: {0}'.format(cert))
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(
urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[
0] = salt.ext.six.moves.http_client.HTTPSConnection(
**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {'Error': str(exc)}
if stream is True or handle is True:
return {'handle': result}
result_status_code = result.code
result_headers = result.headers.headers
result_text = result.read()
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['client_cert'] = cert
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['client_cert'] = cert[0]
req_kwargs['client_key'] = cert[1]
else:
log.error(
'The client-side certificate path that was passed is '
'not valid: {0}'.format(cert))
if isinstance(data, dict):
data = urllib.urlencode(data)
max_body = opts.get('http_max_body',
salt.config.DEFAULT_MINION_OPTS['http_max_body'])
timeout = opts.get(
'http_request_timeout',
salt.config.DEFAULT_MINION_OPTS['http_request_timeout'])
try:
result = HTTPClient(max_body_size=max_body).fetch(
url_full,
method=method,
headers=header_dict,
auth_username=username,
auth_password=password,
body=data,
validate_cert=verify_ssl,
allow_nonstandard_methods=True,
streaming_callback=streaming_callback,
request_timeout=timeout,
**req_kwargs)
except tornado.httpclient.HTTPError as exc:
ret['status'] = exc.code
ret['error'] = str(exc)
return ret
if stream is True or handle is True:
return {'handle': result}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
if 'Set-Cookie' in result_headers.keys() and cookies is not None:
result_cookies = parse_cookie_header(result_headers['Set-Cookie'])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: {0}'.format(result_status_code))
log.trace('Response Headers: {0}'.format(result_headers))
log.trace('Response Cookies: {0}'.format(sess_cookies))
try:
log.trace('Response Text: {0}'.format(result_text))
except UnicodeEncodeError as exc:
log.trace(
('Cannot Trace Log Response Text: {0}. This may be due to '
'incompatibilities between requests and logging.').format(exc))
if text_out is not None and os.path.exists(text_out):
with salt.utils.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with salt.utils.fopen(session_cookie_jar, 'wb') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get('content-type',
'application/json')
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = ('Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)))
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = json.loads(salt.utils.to_str(result_text))
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
else:
text = True
if decode_out and os.path.exists(decode_out):
with salt.utils.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
def test_pool(self):
xml_data = virt._gen_pool_xml('pool', 'logical', 'base')
root = ET.fromstring(xml_data)
self.assertEqual(root.find('name').text, 'pool')
self.assertEqual(root.attrib['type'], 'logical')
self.assertEqual(root.find('target/path').text, '/dev/base')
def query(action=None, command=None, args=None, method='GET', data=None):
'''
Make a web call to a Parallels provider
'''
path = config.get_cloud_config_value('url',
get_configured_provider(),
__opts__,
search_global=False)
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(
realm='Parallels Instance Manager',
uri=path,
user=config.get_cloud_config_value('user',
get_configured_provider(),
__opts__,
search_global=False),
passwd=config.get_cloud_config_value('password',
get_configured_provider(),
__opts__,
search_global=False))
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
if action:
path += action
if command:
path += '/{0}'.format(command)
if type(args) is not dict:
args = {}
kwargs = {'data': data}
if type(data) is str and '<?xml' in data:
kwargs['headers'] = {
'Content-type': 'application/xml',
}
if args:
path += '?%s'
params = urllib.urlencode(args)
req = urllib2.Request(url=path % params, **kwargs)
else:
req = urllib2.Request(url=path, **kwargs)
req.get_method = lambda: method
log.debug('{0} {1}'.format(method, req.get_full_url()))
if data:
log.debug(data)
try:
result = urllib2.urlopen(req)
log.debug('PARALLELS Response Status Code: {0}'.format(
result.getcode()))
if 'content-length' in result.headers:
content = result.read()
result.close()
items = ET.fromstring(content)
return items
return {}
except urllib2.URLError as exc:
log.error('PARALLELS Response Status Code: {0} {1}'.format(
exc.code, exc.msg))
root = ET.fromstring(exc.read())
log.error(root)
return {'error': root}
def address_exists(
name,
addressname=None,
vsys=1,
ipnetmask=None,
iprange=None,
fqdn=None,
description=None,
commit=False,
):
"""
Ensures that an address object exists in the configured state. If it does not exist or is not configured with the
specified attributes, it will be adjusted to match the specified values.
This module will only process a single address type (ip-netmask, ip-range, or fqdn). It will process the specified
value if the following order: ip-netmask, ip-range, fqdn. For proper execution, only specify a single address
type.
name: The name of the module function to execute.
addressname(str): The name of the address object. The name is case-sensitive and can have up to 31 characters,
which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on
Panorama, unique within its device group and any ancestor or descendant device groups.
vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1.
ipnetmask(str): The IPv4 or IPv6 address or IP address range using the format ip_address/mask or ip_address where
the mask is the number of significant binary digits used for the network portion of the address. Ideally, for IPv6,
you specify only the network portion, not the host portion.
iprange(str): A range of addresses using the format ip_address–ip_address where both addresses can be IPv4 or both
can be IPv6.
fqdn(str): A fully qualified domain name format. The FQDN initially resolves at commit time. Entries are
subsequently refreshed when the firewall performs a check every 30 minutes; all changes in the IP address for the
entries are picked up at the refresh cycle.
description(str): A description for the policy (up to 255 characters).
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/address/h-10.10.10.10:
panos.address_exists:
- addressname: h-10.10.10.10
- vsys: 1
- ipnetmask: 10.10.10.10
- commit: False
panos/address/10.0.0.1-10.0.0.50:
panos.address_exists:
- addressname: r-10.0.0.1-10.0.0.50
- vsys: 1
- iprange: 10.0.0.1-10.0.0.50
- commit: False
panos/address/foo.bar.com:
panos.address_exists:
- addressname: foo.bar.com
- vsys: 1
- fqdn: foo.bar.com
- description: My fqdn object
- commit: False
"""
ret = _default_ret(name)
if not addressname:
ret.update({"comment": "The service name field must be provided."})
return ret
# Check if address object currently exists
address = __salt__["panos.get_address"](addressname, vsys)["result"]
if address and "entry" in address:
address = address["entry"]
else:
address = {}
element = ""
# Verify the arguments
if ipnetmask:
element = "<ip-netmask>{0}</ip-netmask>".format(ipnetmask)
elif iprange:
element = "<ip-range>{0}</ip-range>".format(iprange)
elif fqdn:
element = "<fqdn>{0}</fqdn>".format(fqdn)
else:
ret.update({"comment": "A valid address type must be specified."})
return ret
if description:
element += "<description>{0}</description>".format(description)
full_element = "<entry name='{0}'>{1}</entry>".format(addressname, element)
new_address = xml.to_dict(ET.fromstring(full_element), True)
if address == new_address:
ret.update({
"comment": "Address object already exists. No changes required.",
"result": True,
})
return ret
else:
xpath = (
"/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys{0}']/address/"
"entry[@name='{1}']".format(vsys, addressname))
result, msg = _edit_config(xpath, full_element)
if not result:
ret.update({"comment": msg})
return ret
if commit is True:
ret.update({
"changes": {
"before": address,
"after": new_address
},
"commit": __salt__["panos.commit"](),
"comment": "Address object successfully configured.",
"result": True,
})
else:
ret.update({
"changes": {
"before": address,
"after": new_address
},
"comment": "Service object successfully configured.",
"result": True,
})
return ret
def query(params=None,
setname=None,
requesturl=None,
location=None,
return_url=False,
return_root=False,
opts=None,
provider=None,
endpoint=None,
product='ec2',
sigver='2'):
'''
Perform a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
Regions and endpoints are documented at:
http://docs.aws.amazon.com/general/latest/gr/rande.html
Default ``product`` is ``ec2``. Valid ``product`` names are:
.. code-block: yaml
- autoscaling (Auto Scaling)
- cloudformation (CloudFormation)
- ec2 (Elastic Compute Cloud)
- elasticache (ElastiCache)
- elasticbeanstalk (Elastic BeanStalk)
- elasticloadbalancing (Elastic Load Balancing)
- elasticmapreduce (Elastic MapReduce)
- iam (Identity and Access Management)
- importexport (Import/Export)
- monitoring (CloudWatch)
- rds (Relational Database Service)
- simpledb (SimpleDB)
- sns (Simple Notification Service)
- sqs (Simple Queue Service)
'''
if params is None:
params = {}
if opts is None:
opts = {}
function = opts.get('function', (None, product))
providers = opts.get('providers', {})
if provider is None:
prov_dict = providers.get(function[1], {}).get(product, {})
if prov_dict:
driver = list(list(prov_dict.keys()))[0]
provider = providers.get(driver, product)
else:
prov_dict = providers.get(provider, {}).get(product, {})
service_url = prov_dict.get('service_url', 'amazonaws.com')
if not location:
location = get_location(opts, provider)
if endpoint is None:
if not requesturl:
endpoint = prov_dict.get(
'endpoint', '{0}.{1}.{2}'.format(product, location,
service_url))
requesturl = 'https://{0}/'.format(endpoint)
else:
endpoint = urlparse(requesturl).netloc
if endpoint == '':
endpoint_err = (
'Could not find a valid endpoint in the '
'requesturl: {0}. Looking for something '
'like https://some.aws.endpoint/?args').format(requesturl)
LOG.error(endpoint_err)
if return_url is True:
return {'error': endpoint_err}, requesturl
return {'error': endpoint_err}
LOG.debug('Using AWS endpoint: {0}'.format(endpoint))
method = 'GET'
aws_api_version = prov_dict.get(
'aws_api_version',
prov_dict.get('{0}_api_version'.format(product),
DEFAULT_AWS_API_VERSION))
# Fallback to ec2's id & key if none is found, for this component
if not prov_dict.get('id', None):
prov_dict['id'] = providers.get(provider, {}).get('ec2',
{}).get('id', {})
prov_dict['key'] = providers.get(provider, {}).get('ec2',
{}).get('key', {})
if sigver == '4':
headers, requesturl = sig4(method,
endpoint,
params,
prov_dict,
aws_api_version,
location,
product,
requesturl=requesturl)
params_with_headers = {}
else:
params_with_headers = sig2(method, endpoint, params, prov_dict,
aws_api_version)
headers = {}
attempts = 5
while attempts > 0:
LOG.debug('AWS Request: {0}'.format(requesturl))
LOG.trace('AWS Request Parameters: {0}'.format(params_with_headers))
try:
result = requests.get(requesturl,
headers=headers,
params=params_with_headers)
LOG.debug('AWS Response Status Code: {0}'.format(
result.status_code))
LOG.trace('AWS Response Text: {0}'.format(result.text))
result.raise_for_status()
break
except requests.exceptions.HTTPError as exc:
root = ET.fromstring(exc.response.content)
data = xml.to_dict(root)
# check to see if we should retry the query
err_code = data.get('Errors', {}).get('Error', {}).get('Code', '')
if attempts > 0 and err_code and err_code in AWS_RETRY_CODES:
attempts -= 1
LOG.error('AWS Response Status Code and Error: [{0} {1}] {2}; '
'Attempts remaining: {3}'.format(
exc.response.status_code, exc, data, attempts))
# Wait a bit before continuing to prevent throttling
time.sleep(2)
continue
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data))
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
else:
LOG.error('AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data))
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
response = result.text
root = ET.fromstring(response)
items = root[1]
if return_root is True:
items = root
if setname:
if sys.version_info < (2, 7):
children_len = len(root.getchildren())
else:
children_len = len(root)
for item in range(0, children_len):
comps = root[item].tag.split('}')
if comps[1] == setname:
items = root[item]
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
return ret
def address_group_exists(name,
groupname=None,
vsys=1,
members=None,
description=None,
commit=False):
"""
Ensures that an address group object exists in the configured state. If it does not exist or is not configured with
the specified attributes, it will be adjusted to match the specified values.
This module will enforce group membership. If a group exists and contains members this state does not include,
those members will be removed and replaced with the specified members in the state.
name: The name of the module function to execute.
groupname(str): The name of the address group object. The name is case-sensitive and can have up to 31 characters,
which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on
Panorama, unique within its device group and any ancestor or descendant device groups.
vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1.
members(str, list): The members of the address group. These must be valid address objects or address groups on the
system that already exist prior to the execution of this state.
description(str): A description for the policy (up to 255 characters).
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/address-group/my-group:
panos.address_group_exists:
- groupname: my-group
- vsys: 1
- members:
- my-address-object
- my-other-address-group
- description: A group that needs to exist
- commit: False
"""
ret = _default_ret(name)
if not groupname:
ret.update({"comment": "The group name field must be provided."})
return ret
# Check if address group object currently exists
group = __salt__["panos.get_address_group"](groupname, vsys)["result"]
if group and "entry" in group:
group = group["entry"]
else:
group = {}
# Verify the arguments
if members:
element = "<static>{0}</static>".format(_build_members(members, True))
else:
ret.update({"comment": "The group members must be provided."})
return ret
if description:
element += "<description>{0}</description>".format(description)
full_element = "<entry name='{0}'>{1}</entry>".format(groupname, element)
new_group = xml.to_dict(ET.fromstring(full_element), True)
if group == new_group:
ret.update({
"comment":
"Address group object already exists. No changes required.",
"result": True,
})
return ret
else:
xpath = (
"/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys{0}']/address-group/"
"entry[@name='{1}']".format(vsys, groupname))
result, msg = _edit_config(xpath, full_element)
if not result:
ret.update({"comment": msg})
return ret
if commit is True:
ret.update({
"changes": {
"before": group,
"after": new_group
},
"commit": __salt__["panos.commit"](),
"comment": "Address group object successfully configured.",
"result": True,
})
else:
ret.update({
"changes": {
"before": group,
"after": new_group
},
"comment": "Address group object successfully configured.",
"result": True,
})
return ret
def test_xml_case_f_legacy(self):
xmldata = ET.fromstring(self.cases['f']['xml'])
defaultdict = xml.to_dict(xmldata, False)
self.assertEqual(defaultdict, self.cases['f']['legacy'])
def edit_config(name, xpath=None, value=None, commit=False):
"""
Edits a Palo Alto XPATH to a specific value. This will always overwrite the existing value, even if it is not
changed.
You can replace an existing object hierarchy at a specified location in the configuration with a new value. Use
the xpath parameter to specify the location of the object, including the node to be replaced.
This is the recommended state to enforce configurations on a xpath.
name: The name of the module function to execute.
xpath(str): The XPATH of the configuration API tree to control.
value(str): The XML value to edit. This must be a child to the XPATH.
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/addressgroup:
panos.edit_config:
- xpath: /config/devices/entry/vsys/entry[@name='vsys1']/address-group/entry[@name='test']
- value: <static><entry name='test'><member>abc</member><member>xyz</member></entry></static>
- commit: True
"""
ret = _default_ret(name)
# Verify if the current XPATH is equal to the specified value.
# If we are equal, no changes required.
xpath_split = xpath.split("/")
# Retrieve the head of the xpath for validation.
if xpath_split:
head = xpath_split[-1]
if "[" in head:
head = head.split("[")[0]
current_element = __salt__["panos.get_xpath"](xpath)["result"]
if head and current_element and head in current_element:
current_element = current_element[head]
else:
current_element = {}
new_element = xml.to_dict(ET.fromstring(value), True)
if current_element == new_element:
ret.update({
"comment": "XPATH is already equal to the specified value.",
"result": True,
})
return ret
result, msg = _edit_config(xpath, value)
ret.update({"comment": msg, "result": result})
if not result:
return ret
if commit is True:
ret.update({
"changes": {
"before": current_element,
"after": new_element
},
"commit": __salt__["panos.commit"](),
"result": True,
})
else:
ret.update({
"changes": {
"before": current_element,
"after": new_element
},
"result": True,
})
return ret
def service_exists(
name,
servicename=None,
vsys=1,
protocol=None,
port=None,
description=None,
commit=False,
):
"""
Ensures that a service object exists in the configured state. If it does not exist or is not configured with the
specified attributes, it will be adjusted to match the specified values.
name: The name of the module function to execute.
servicename(str): The name of the security object. The name is case-sensitive and can have up to 31 characters,
which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on
Panorama, unique within its device group and any ancestor or descendant device groups.
vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1.
protocol(str): The protocol that is used by the service object. The only valid options are tcp and udp.
port(str): The port number that is used by the service object. This can be specified as a single integer or a
valid range of ports.
description(str): A description for the policy (up to 255 characters).
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/service/tcp-80:
panos.service_exists:
- servicename: tcp-80
- vsys: 1
- protocol: tcp
- port: 80
- description: Hypertext Transfer Protocol
- commit: False
panos/service/udp-500-550:
panos.service_exists:
- servicename: udp-500-550
- vsys: 3
- protocol: udp
- port: 500-550
- commit: False
"""
ret = _default_ret(name)
if not servicename:
ret.update({"comment": "The service name field must be provided."})
return ret
# Check if service object currently exists
service = __salt__["panos.get_service"](servicename, vsys)["result"]
if service and "entry" in service:
service = service["entry"]
else:
service = {}
# Verify the arguments
if not protocol and protocol not in ["tcp", "udp"]:
ret.update({
"comment":
"The protocol must be provided and must be tcp or udp."
})
return ret
if not port:
ret.update({"comment": "The port field must be provided."})
return ret
element = "<protocol><{0}><port>{1}</port></{0}></protocol>".format(
protocol, port)
if description:
element += "<description>{0}</description>".format(description)
full_element = "<entry name='{0}'>{1}</entry>".format(servicename, element)
new_service = xml.to_dict(ET.fromstring(full_element), True)
if service == new_service:
ret.update({
"comment": "Service object already exists. No changes required.",
"result": True,
})
return ret
else:
xpath = (
"/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys{0}']/service/"
"entry[@name='{1}']".format(vsys, servicename))
result, msg = _edit_config(xpath, full_element)
if not result:
ret.update({"comment": msg})
return ret
if commit is True:
ret.update({
"changes": {
"before": service,
"after": new_service
},
"commit": __salt__["panos.commit"](),
"comment": "Service object successfully configured.",
"result": True,
})
else:
ret.update({
"changes": {
"before": service,
"after": new_service
},
"comment": "Service object successfully configured.",
"result": True,
})
return ret
def query(url,
method="GET",
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type="auto",
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format="lwp",
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node="minion",
port=80,
opts=None,
backend=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
header_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
raise_error=True,
formdata=False,
formdata_fieldname=None,
formdata_filename=None,
decode_body=True,
**kwargs):
"""
Query a resource, and decode the return data
"""
ret = {}
if opts is None:
if node == "master":
opts = salt.config.master_config(
os.path.join(salt.syspaths.CONFIG_DIR, "master"))
elif node == "minion":
opts = salt.config.minion_config(
os.path.join(salt.syspaths.CONFIG_DIR, "minion"))
else:
opts = {}
if not backend:
backend = opts.get("backend", "tornado")
match = re.match(
r"https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)",
url,
)
if not match:
salt.utils.network.refresh_dns()
if backend == "requests":
if HAS_REQUESTS is False:
ret["error"] = ("http.query has been set to use requests, but the "
"requests library does not seem to be installed")
log.error(ret["error"])
return ret
else:
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = salt.ext.tornado.httputil.url_concat(url,
params) if params else url
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get("verify_ssl", True)
if cert is None:
cert = opts.get("cert", None)
if data_file is not None:
data = _render(data_file, data_render, data_renderer, template_dict,
opts)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug("Requesting URL %s using %s method", log_url, method)
log.debug("Using backend: %s", backend)
if method == "POST" and log.isEnabledFor(logging.TRACE):
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = "XXXXXXXXXX"
log.trace("Request POST Data: %s", pprint.pformat(log_data))
else:
log.trace("Request POST Data: %s", pprint.pformat(data))
if header_file is not None:
header_tpl = _render(header_file, header_render, header_renderer,
template_dict, opts)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(
opts.get("cachedir", salt.syspaths.CACHE_DIR), "cookies.txt")
if session_cookie_jar is None:
session_cookie_jar = os.path.join(
opts.get("cachedir", salt.syspaths.CACHE_DIR), "cookies.session.p")
if persist_session is True and salt.utils.msgpack.HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.files.fopen(session_cookie_jar, "rb") as fh_:
session_cookies = salt.utils.msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.files.fopen(session_cookie_jar, "wb") as fh_:
salt.utils.msgpack.dump("", fh_)
for header in header_list:
comps = header.split(":")
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
agent = "{0} http.query()".format(agent)
header_dict["User-agent"] = agent
if backend == "requests":
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace("Request Headers: %s", sess.headers)
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == "urllib2":
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == "mozilla":
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(
cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(
cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret["test"] = True
if backend == "requests":
req_kwargs = {}
if stream is True:
if requests.__version__[0] == "0":
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs["prefetch"] = False
else:
req_kwargs["stream"] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs["cert"] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs["cert"] = cert
else:
log.error(
"The client-side certificate path that"
" was passed is not valid: %s",
cert,
)
if formdata:
if not formdata_fieldname:
ret["error"] = "formdata_fieldname is required when formdata=True"
log.error(ret["error"])
return ret
result = sess.request(method,
url,
params=params,
files={
formdata_fieldname:
(formdata_filename, StringIO(data))
},
**req_kwargs)
else:
result = sess.request(method,
url,
params=params,
data=data,
**req_kwargs)
result.raise_for_status()
if stream is True:
# fake a HTTP response header
header_callback("HTTP/1.0 {0} MESSAGE".format(result.status_code))
# fake streaming the content
streaming_callback(result.content)
return {
"handle": result,
}
if handle is True:
return {
"handle": result,
"body": result.content,
}
log.debug("Final URL location of Response: %s",
sanitize_url(result.url, hide_fields))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
body = result.content
if not isinstance(body, six.text_type) and decode_body:
body = body.decode(result.encoding or "utf-8")
ret["body"] = body
elif backend == "urllib2":
request = urllib_request.Request(url_full, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies),
]
if url.startswith("https"):
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warning(
"match_hostname() not available, SSL hostname checking "
"not available. THIS CONNECTION MAY NOT BE SECURE!")
elif verify_ssl is False:
log.warning("SSL certificate verification has been explicitly "
"disabled. THIS CONNECTION MAY NOT BE SECURE!")
else:
if ":" in hostname:
hostname, port = hostname.split(":")
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret["error"] = (
"The certificate was invalid. Error returned was: %s",
pprint.pformat(exc),
)
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error(
"The client-side certificate path that was "
"passed is not valid: %s",
cert,
)
return
if hasattr(ssl, "SSLContext"):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(
urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
"host": request.get_host(),
"port": port,
"cert_file": cert_chain[0],
}
if len(cert_chain) > 1:
cert_kwargs["key_file"] = cert_chain[1]
handlers[
0] = salt.ext.six.moves.http_client.HTTPSConnection(
**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {"Error": six.text_type(exc)}
if stream is True or handle is True:
return {
"handle": result,
"body": result.content,
}
result_status_code = result.code
result_headers = dict(result.info())
result_text = result.read()
if "Content-Type" in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers["Content-Type"])
if (res_content_type.startswith("text/")
and "charset" in res_params
and not isinstance(result_text, six.text_type)):
result_text = result_text.decode(res_params["charset"])
if six.PY3 and isinstance(result_text, bytes) and decode_body:
result_text = result_text.decode("utf-8")
ret["body"] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs["client_cert"] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs["client_cert"] = cert[0]
req_kwargs["client_key"] = cert[1]
else:
log.error(
"The client-side certificate path that "
"was passed is not valid: %s",
cert,
)
if isinstance(data, dict):
data = _urlencode(data)
if verify_ssl:
req_kwargs["ca_certs"] = ca_bundle
max_body = opts.get("http_max_body",
salt.config.DEFAULT_MINION_OPTS["http_max_body"])
connect_timeout = opts.get(
"http_connect_timeout",
salt.config.DEFAULT_MINION_OPTS["http_connect_timeout"],
)
timeout = opts.get(
"http_request_timeout",
salt.config.DEFAULT_MINION_OPTS["http_request_timeout"],
)
client_argspec = None
proxy_host = opts.get("proxy_host", None)
if proxy_host:
# tornado requires a str for proxy_host, cannot be a unicode str in py2
proxy_host = salt.utils.stringutils.to_str(proxy_host)
proxy_port = opts.get("proxy_port", None)
proxy_username = opts.get("proxy_username", None)
if proxy_username:
# tornado requires a str, cannot be unicode str in py2
proxy_username = salt.utils.stringutils.to_str(proxy_username)
proxy_password = opts.get("proxy_password", None)
if proxy_password:
# tornado requires a str, cannot be unicode str in py2
proxy_password = salt.utils.stringutils.to_str(proxy_password)
no_proxy = opts.get("no_proxy", [])
# Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones
# except we remove the valid ones if a url has a no_proxy hostname in it
if urlparse(url_full).hostname in no_proxy:
proxy_host = None
proxy_port = None
# We want to use curl_http if we have a proxy defined
if proxy_host and proxy_port:
if HAS_CURL_HTTPCLIENT is False:
ret["error"] = (
"proxy_host and proxy_port has been set. This requires pycurl and tornado, "
"but the libraries does not seem to be installed")
log.error(ret["error"])
return ret
salt.ext.tornado.httpclient.AsyncHTTPClient.configure(
"tornado.curl_httpclient.CurlAsyncHTTPClient")
client_argspec = salt.utils.args.get_function_argspec(
salt.ext.tornado.curl_httpclient.CurlAsyncHTTPClient.initialize
)
else:
client_argspec = salt.utils.args.get_function_argspec(
salt.ext.tornado.simple_httpclient.SimpleAsyncHTTPClient.
initialize)
supports_max_body_size = "max_body_size" in client_argspec.args
req_kwargs.update({
"method": method,
"headers": header_dict,
"auth_username": username,
"auth_password": password,
"body": data,
"validate_cert": verify_ssl,
"allow_nonstandard_methods": True,
"streaming_callback": streaming_callback,
"header_callback": header_callback,
"connect_timeout": connect_timeout,
"request_timeout": timeout,
"proxy_host": proxy_host,
"proxy_port": proxy_port,
"proxy_username": proxy_username,
"proxy_password": proxy_password,
"raise_error": raise_error,
"decompress_response": False,
})
# Unicode types will cause a TypeError when Tornado's curl HTTPClient
# invokes setopt. Therefore, make sure all arguments we pass which
# contain strings are str types.
req_kwargs = salt.utils.data.decode(req_kwargs, to_str=True)
try:
download_client = (HTTPClient(max_body_size=max_body)
if supports_max_body_size else HTTPClient())
result = download_client.fetch(url_full, **req_kwargs)
except salt.ext.tornado.httpclient.HTTPError as exc:
ret["status"] = exc.code
ret["error"] = six.text_type(exc)
return ret
except (socket.herror, socket.error, socket.timeout,
socket.gaierror) as exc:
if status is True:
ret["status"] = 0
ret["error"] = six.text_type(exc)
log.debug("Cannot perform 'http.query': {0} - {1}".format(
url_full, ret["error"]))
return ret
if stream is True or handle is True:
return {
"handle": result,
"body": result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
if "Content-Type" in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers["Content-Type"])
if (res_content_type.startswith("text/")
and "charset" in res_params
and not isinstance(result_text, six.text_type)):
result_text = result_text.decode(res_params["charset"])
if six.PY3 and isinstance(result_text, bytes) and decode_body:
result_text = result_text.decode("utf-8")
ret["body"] = result_text
if "Set-Cookie" in result_headers and cookies is not None:
result_cookies = parse_cookie_header(result_headers["Set-Cookie"])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(":")
result_headers_dict[comps[0].strip()] = ":".join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug("Response Status Code: %s", result_status_code)
log.trace("Response Headers: %s", result_headers)
log.trace("Response Cookies: %s", sess_cookies)
# log.trace("Content: %s", result_text)
coding = result_headers.get("Content-Encoding", "identity")
# Requests will always decompress the content, and working around that is annoying.
if backend != "requests":
result_text = __decompressContent(coding, result_text)
try:
log.trace("Response Text: %s", result_text)
except UnicodeEncodeError as exc:
log.trace(
"Cannot Trace Log Response Text: %s. This may be due to "
"incompatibilities between requests and logging.",
exc,
)
if text_out is not None:
with salt.utils.files.fopen(text_out, "w") as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.files.fopen(headers_out, "w") as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and salt.utils.msgpack.HAS_MSGPACK:
# TODO: See persist_session above
if "set-cookie" in result_headers:
with salt.utils.files.fopen(session_cookie_jar, "wb") as fh_:
session_cookies = result_headers.get("set-cookie", None)
if session_cookies is not None:
salt.utils.msgpack.dump({"Cookie": session_cookies}, fh_)
else:
salt.utils.msgpack.dump("", fh_)
if status is True:
ret["status"] = result_status_code
if headers is True:
ret["headers"] = result_headers
if decode is True:
if decode_type == "auto":
content_type = result_headers.get("content-type",
"application/json")
if "xml" in content_type:
decode_type = "xml"
elif "json" in content_type:
decode_type = "json"
elif "yaml" in content_type:
decode_type = "yaml"
else:
decode_type = "plain"
valid_decodes = ("json", "xml", "yaml", "plain")
if decode_type not in valid_decodes:
ret["error"] = ("Invalid decode_type specified. "
"Valid decode types are: {0}".format(
pprint.pformat(valid_decodes)))
log.error(ret["error"])
return ret
if decode_type == "json":
ret["dict"] = salt.utils.json.loads(result_text)
elif decode_type == "xml":
ret["dict"] = []
items = ET.fromstring(result_text)
for item in items:
ret["dict"].append(xml.to_dict(item))
elif decode_type == "yaml":
ret["dict"] = salt.utils.data.decode(
salt.utils.yaml.safe_load(result_text))
else:
text = True
if decode_out:
with salt.utils.files.fopen(decode_out, "w") as dof:
dof.write(result_text)
if text is True:
ret["text"] = result_text
return ret
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
backend='tornado',
requests_lib=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = salt.config.master_config(
os.path.join(syspaths.CONFIG_DIR, 'master')
)
elif node == 'minion':
opts = salt.config.minion_config(
os.path.join(syspaths.CONFIG_DIR, 'minion')
)
else:
opts = {}
if requests_lib is None:
requests_lib = opts.get('requests_lib', False)
if requests_lib is True:
log.warn('Please set "backend" to "requests" instead of setting '
'"requests_lib" to "True"')
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
backend = 'requests'
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = tornado.httputil.url_concat(url, params)
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(
data_file, data_render, data_renderer, template_dict, opts
)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug('Requesting URL {0} using {1} method'.format(log_url, method))
if method == 'POST':
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = 'XXXXXXXXXX'
log.trace('Request POST Data: {0}'.format(pprint.pformat(log_data)))
else:
log.trace('Request POST Data: {0}'.format(pprint.pformat(data)))
if header_file is not None:
header_tpl = _render(
header_file, header_render, header_renderer, template_dict, opts
)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(opts.get('cachedir', syspaths.CACHE_DIR), 'cookies.txt')
if session_cookie_jar is None:
session_cookie_jar = os.path.join(opts.get('cachedir', syspaths.CACHE_DIR), 'cookies.session.p')
if persist_session is True and HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.fopen(session_cookie_jar, 'rb') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.fopen(session_cookie_jar, 'wb') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if backend == 'requests':
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: {0}'.format(sess.headers))
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == 'urllib2':
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if backend == 'requests':
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error('The client-side certificate path that was passed is '
'not valid: {0}'.format(cert))
result = sess.request(
method, url, params=params, data=data, **req_kwargs
)
result.raise_for_status()
if stream is True or handle is True:
return {
'handle': result,
'body': result.content,
}
log.debug('Final URL location of Response: {0}'.format(sanitize_url(result.url, hide_fields)))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
ret['body'] = result.content
elif backend == 'urllib2':
request = urllib_request.Request(url_full, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https'):
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warn(('match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!'))
elif verify_ssl is False:
log.warn(('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!'))
else:
if ':' in hostname:
hostname, port = hostname.split(':')
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(
sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED
)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = (
'The certificate was invalid. '
'Error returned was: {0}'.format(
pprint.pformat(exc)
)
)
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error('The client-side certificate path that was '
'passed is not valid: {0}'.format(cert))
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[0] = salt.ext.six.moves.http_client.HTTPSConnection(**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {'Error': str(exc)}
if stream is True or handle is True:
return {
'handle': result,
'body': result.content,
}
result_status_code = result.code
result_headers = result.headers.headers
result_text = result.read()
ret['body'] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['client_cert'] = cert
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['client_cert'] = cert[0]
req_kwargs['client_key'] = cert[1]
else:
log.error('The client-side certificate path that was passed is '
'not valid: {0}'.format(cert))
if isinstance(data, dict):
data = urllib.urlencode(data)
if verify_ssl:
req_kwargs['ca_certs'] = ca_bundle
max_body = opts.get('http_max_body', salt.config.DEFAULT_MINION_OPTS['http_max_body'])
timeout = opts.get('http_request_timeout', salt.config.DEFAULT_MINION_OPTS['http_request_timeout'])
client_argspec = None
proxy_host = opts.get('proxy_host', None)
proxy_port = opts.get('proxy_port', None)
proxy_username = opts.get('proxy_username', None)
proxy_password = opts.get('proxy_password', None)
# We want to use curl_http if we have a proxy defined
if proxy_host and proxy_port:
if HAS_CURL_HTTPCLIENT is False:
ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl, but the '
'pycurl library does not seem to be installed')
log.error(ret['error'])
return ret
tornado.httpclient.AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient')
client_argspec = inspect.getargspec(tornado.curl_httpclient.CurlAsyncHTTPClient.initialize)
else:
client_argspec = inspect.getargspec(tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize)
supports_max_body_size = 'max_body_size' in client_argspec.args
try:
if supports_max_body_size:
result = HTTPClient(max_body_size=max_body).fetch(
url_full,
method=method,
headers=header_dict,
auth_username=username,
auth_password=password,
body=data,
validate_cert=verify_ssl,
allow_nonstandard_methods=True,
streaming_callback=streaming_callback,
request_timeout=timeout,
proxy_host=proxy_host,
proxy_port=proxy_port,
proxy_username=proxy_username,
proxy_password=proxy_password,
**req_kwargs
)
else:
result = HTTPClient().fetch(
url_full,
method=method,
headers=header_dict,
auth_username=username,
auth_password=password,
body=data,
validate_cert=verify_ssl,
allow_nonstandard_methods=True,
streaming_callback=streaming_callback,
request_timeout=timeout,
proxy_host=proxy_host,
proxy_port=proxy_port,
proxy_username=proxy_username,
proxy_password=proxy_password,
**req_kwargs
)
except tornado.httpclient.HTTPError as exc:
ret['status'] = exc.code
ret['error'] = str(exc)
return ret
if stream is True or handle is True:
return {
'handle': result,
'body': result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
ret['body'] = result.body
if 'Set-Cookie' in result_headers.keys() and cookies is not None:
result_cookies = parse_cookie_header(result_headers['Set-Cookie'])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: {0}'.format(result_status_code))
log.trace('Response Headers: {0}'.format(result_headers))
log.trace('Response Cookies: {0}'.format(sess_cookies))
try:
log.trace('Response Text: {0}'.format(result_text))
except UnicodeEncodeError as exc:
log.trace(('Cannot Trace Log Response Text: {0}. This may be due to '
'incompatibilities between requests and logging.').format(exc))
if text_out is not None and os.path.exists(text_out):
with salt.utils.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with salt.utils.fopen(session_cookie_jar, 'wb') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get(
'content-type', 'application/json'
)
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
elif 'yaml' in content_type:
decode_type = 'yaml'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'yaml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = (
'Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)
)
)
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = json.loads(salt.utils.to_str(result_text))
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
elif decode_type == 'yaml':
ret['dict'] = yaml.safe_load(result_text)
else:
text = True
if decode_out and os.path.exists(decode_out):
with salt.utils.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
def test_gen_xml_dns():
"""
Test virt._get_net_xml() with DNS configuration
"""
xml_data = virt._gen_net_xml(
"network",
"main",
"nat",
None,
ip_configs=[{
"cidr":
"192.168.2.0/24",
"dhcp_ranges": [{
"start": "192.168.2.10",
"end": "192.168.2.25"
}],
}],
dns={
"forwarders": [
{
"domain": "example.com",
"addr": "192.168.1.1"
},
{
"addr": "8.8.8.8"
},
{
"domain": "www.example.com"
},
],
"txt": {
"host.widgets.com.": "printer=lpr5",
"example.com.": "reserved for doc",
},
"hosts": {
"192.168.1.2": ["mirror.acme.lab", "test.acme.lab"]
},
"srvs": [
{
"name": "srv1",
"protocol": "tcp",
"domain": "test-domain-name",
"target": ".",
"port": 1024,
"priority": 10,
"weight": 10,
},
{
"name": "srv2",
"protocol": "udp"
},
],
},
)
root = ET.fromstring(xml_data)
expected_xml = ET.fromstring("""
<dns>
<forwarder domain='example.com' addr='192.168.1.1'/>
<forwarder addr='8.8.8.8'/>
<forwarder domain='www.example.com'/>
<txt name='example.com.' value='reserved for doc'/>
<txt name='host.widgets.com.' value='printer=lpr5'/>
<host ip='192.168.1.2'>
<hostname>mirror.acme.lab</hostname>
<hostname>test.acme.lab</hostname>
</host>
<srv name='srv1' protocol='tcp' port='1024' target='.' priority='10' weight='10' domain='test-domain-name'/>
<srv name='srv2' protocol='udp'/>
</dns>
""")
assert_xml_equals(expected_xml, root.find("./dns"))
def query(key, keyid, method='GET', params=None, headers=None,
requesturl=None, return_url=False, bucket=None, service_url=None,
path=None, return_bin=False, action=None, local_file=None,
verify_ssl=True):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
A service_url may also be specified in the configuration::
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
'''
if not headers:
headers = {}
if not params:
params = {}
if path is None:
path = ''
if not service_url:
service_url = 's3.amazonaws.com'
if bucket:
endpoint = '{0}.{1}'.format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
token = None
if not key or not keyid:
iam_creds = iam.get_iam_metadata()
key = iam_creds['secret_key']
keyid = iam_creds['access_key']
token = iam_creds['security_token']
if not requesturl:
x_amz_date = datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
content_type = 'text/plain'
if method == 'GET':
if bucket:
can_resource = '/{0}/{1}'.format(bucket, path)
else:
can_resource = '/'
elif method == 'PUT' or method == 'HEAD' or method == 'DELETE':
if path:
can_resource = '/{0}/{1}'.format(bucket, path)
else:
can_resource = '/{0}/'.format(bucket)
if action:
can_resource += '?{0}'.format(action)
log.debug('CanonicalizedResource: {0}'.format(can_resource))
headers['Host'] = endpoint
headers['Content-type'] = content_type
headers['Date'] = x_amz_date
if token:
headers['x-amz-security-token'] = token
string_to_sign = '{0}\n'.format(method)
new_headers = []
for header in sorted(headers):
if header.lower().startswith('x-amz'):
log.debug(header.lower())
new_headers.append('{0}:{1}'.format(header.lower(),
headers[header]))
can_headers = '\n'.join(new_headers)
log.debug('CanonicalizedAmzHeaders: {0}'.format(can_headers))
string_to_sign += '\n{0}'.format(content_type)
string_to_sign += '\n{0}'.format(x_amz_date)
if can_headers:
string_to_sign += '\n{0}'.format(can_headers)
string_to_sign += '\n{0}'.format(can_resource)
log.debug('String To Sign:: \n{0}'.format(string_to_sign))
hashed = hmac.new(key, string_to_sign, hashlib.sha1)
sig = binascii.b2a_base64(hashed.digest())
headers['Authorization'] = 'AWS {0}:{1}'.format(keyid, sig.strip())
querystring = urllib.urlencode(params)
if action:
if querystring:
querystring = '{0}&{1}'.format(action, querystring)
else:
querystring = action
requesturl = 'https://{0}/'.format(endpoint)
if path:
requesturl += path
if querystring:
requesturl += '?{0}'.format(querystring)
data = None
if method == 'PUT':
if local_file:
with salt.utils.fopen(local_file, 'r') as ifile:
data = ifile.read()
log.debug('S3 Request: {0}'.format(requesturl))
log.debug('S3 Headers::')
log.debug(' Authorization: {0}'.format(headers['Authorization']))
try:
result = requests.request(method, requesturl, headers=headers,
data=data,
verify=verify_ssl)
response = result.content
except requests.exceptions.HTTPError as exc:
log.error('There was an error::')
if hasattr(exc, 'code') and hasattr(exc, 'msg'):
log.error(' Code: {0}: {1}'.format(exc.code, exc.msg))
log.error(' Content: \n{0}'.format(exc.read()))
return False
log.debug('S3 Response Status Code: {0}'.format(result.status_code))
if method == 'PUT':
if result.status_code == 200:
if local_file:
log.debug('Uploaded from {0} to {1}'.format(local_file, path))
else:
log.debug('Created bucket {0}'.format(bucket))
else:
if local_file:
log.debug('Failed to upload from {0} to {1}: {2}'.format(
local_file,
path,
result.status_code,
))
else:
log.debug('Failed to create bucket {0}'.format(bucket))
return
if method == 'DELETE':
if str(result.status_code).startswith('2'):
if path:
log.debug('Deleted {0} from bucket {1}'.format(path, bucket))
else:
log.debug('Deleted bucket {0}'.format(bucket))
else:
if path:
log.debug('Failed to delete {0} from bucket {1}: {2}'.format(
path,
bucket,
result.status_code,
))
else:
log.debug('Failed to delete bucket {0}'.format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'w') as out:
out.write(response)
return 'Saved to local file: {0}'.format(local_file)
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {'headers': []}
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def query(
params=None,
setname=None,
requesturl=None,
location=None,
return_url=False,
return_root=False,
opts=None,
provider=None,
endpoint=None,
product="ec2",
sigver="2",
):
"""
Perform a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
Regions and endpoints are documented at:
http://docs.aws.amazon.com/general/latest/gr/rande.html
Default ``product`` is ``ec2``. Valid ``product`` names are:
.. code-block:: yaml
- autoscaling (Auto Scaling)
- cloudformation (CloudFormation)
- ec2 (Elastic Compute Cloud)
- elasticache (ElastiCache)
- elasticbeanstalk (Elastic BeanStalk)
- elasticloadbalancing (Elastic Load Balancing)
- elasticmapreduce (Elastic MapReduce)
- iam (Identity and Access Management)
- importexport (Import/Export)
- monitoring (CloudWatch)
- rds (Relational Database Service)
- simpledb (SimpleDB)
- sns (Simple Notification Service)
- sqs (Simple Queue Service)
"""
if params is None:
params = {}
if opts is None:
opts = {}
function = opts.get("function", (None, product))
providers = opts.get("providers", {})
if provider is None:
prov_dict = providers.get(function[1], {}).get(product, {})
if prov_dict:
driver = list(list(prov_dict.keys()))[0]
provider = providers.get(driver, product)
else:
prov_dict = providers.get(provider, {}).get(product, {})
service_url = prov_dict.get("service_url", "amazonaws.com")
if not location:
location = get_location(opts, prov_dict)
if endpoint is None:
if not requesturl:
endpoint = prov_dict.get(
"endpoint", "{0}.{1}.{2}".format(product, location, service_url)
)
requesturl = "https://{0}/".format(endpoint)
else:
endpoint = urlparse(requesturl).netloc
if endpoint == "":
endpoint_err = (
"Could not find a valid endpoint in the "
"requesturl: {0}. Looking for something "
"like https://some.aws.endpoint/?args"
).format(requesturl)
log.error(endpoint_err)
if return_url is True:
return {"error": endpoint_err}, requesturl
return {"error": endpoint_err}
log.debug("Using AWS endpoint: %s", endpoint)
method = "GET"
aws_api_version = prov_dict.get(
"aws_api_version",
prov_dict.get("{0}_api_version".format(product), DEFAULT_AWS_API_VERSION),
)
# Fallback to ec2's id & key if none is found, for this component
if not prov_dict.get("id", None):
prov_dict["id"] = providers.get(provider, {}).get("ec2", {}).get("id", {})
prov_dict["key"] = providers.get(provider, {}).get("ec2", {}).get("key", {})
if sigver == "4":
headers, requesturl = sig4(
method,
endpoint,
params,
prov_dict,
aws_api_version,
location,
product,
requesturl=requesturl,
)
params_with_headers = {}
else:
params_with_headers = sig2(method, endpoint, params, prov_dict, aws_api_version)
headers = {}
attempts = 0
while attempts < AWS_MAX_RETRIES:
log.debug("AWS Request: %s", requesturl)
log.trace("AWS Request Parameters: %s", params_with_headers)
try:
result = requests.get(
requesturl, headers=headers, params=params_with_headers
)
log.debug("AWS Response Status Code: %s", result.status_code)
log.trace("AWS Response Text: %s", result.text)
result.raise_for_status()
break
except requests.exceptions.HTTPError as exc:
root = ET.fromstring(exc.response.content)
data = xml.to_dict(root)
# check to see if we should retry the query
err_code = data.get("Errors", {}).get("Error", {}).get("Code", "")
if attempts < AWS_MAX_RETRIES and err_code and err_code in AWS_RETRY_CODES:
attempts += 1
log.error(
"AWS Response Status Code and Error: [%s %s] %s; "
"Attempts remaining: %s",
exc.response.status_code,
exc,
data,
attempts,
)
sleep_exponential_backoff(attempts)
continue
log.error(
"AWS Response Status Code and Error: [%s %s] %s",
exc.response.status_code,
exc,
data,
)
if return_url is True:
return {"error": data}, requesturl
return {"error": data}
else:
log.error(
"AWS Response Status Code and Error: [%s %s] %s",
exc.response.status_code,
exc,
data,
)
if return_url is True:
return {"error": data}, requesturl
return {"error": data}
root = ET.fromstring(result.text)
items = root[1]
if return_root is True:
items = root
if setname:
if sys.version_info < (2, 7):
children_len = len(root.getchildren())
else:
children_len = len(root)
for item in range(0, children_len):
comps = root[item].tag.split("}")
if comps[1] == setname:
items = root[item]
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
return ret
