def tables(schema):
'''
Displays all the tables that are
present in the given schema
CLI Example::
salt '*' drizzle.tables schema_name
'''
# Initializing the required variables
ret_val = {}
drizzle_db = _connect()
cursor = drizzle_db.cursor()
# Fetching tables
try:
cursor.execute('SHOW TABLES IN {0}'.format(schema))
except MySQLdb.OperationalError:
return 'Unknown Schema'
for iter, count in zip(list(range(cursor.rowcount)), list(range(1, cursor.rowcount+1))):
table = cursor.fetchone()
ret_val[count] = table[0]
cursor.close()
drizzle_db.close()
return ret_val
def plugins():
'''
Fetches the plugins added to the database server
CLI Example::
salt '*' drizzle.plugins
'''
# Initializing the required variables
ret_val = {}
count = 1
drizzle_db = _connect()
cursor = drizzle_db.cursor()
# Fetching the plugins
query = 'SELECT PLUGIN_NAME FROM DATA_DICTIONARY.PLUGINS WHERE IS_ACTIVE LIKE "YES"'
cursor.execute(query)
for iter, count in zip(list(range(cursor.rowcount)), list(range(1, cursor.rowcount+1))):
table = cursor.fetchone()
ret_val[count] = table[0]
cursor.close()
drizzle_db.close()
return ret_val
def test_reload(self):
# ensure it doesn't exist
self.assertNotIn(self.module_key, self.loader)
# update both the module and the lib
for x in range(1, 3):
self.update_module()
self.update_lib()
self.loader.clear()
self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count))
# update just the module
for x in range(1, 3):
self.update_module()
self.loader.clear()
self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count))
# update just the lib
for x in range(1, 3):
self.update_lib()
self.loader.clear()
self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count))
self.rm_module()
# make sure that even if we remove the module, its still loaded until a clear
self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count))
self.loader.clear()
self.assertNotIn(self.module_key, self.loader)
def _xfs_inventory_output(out):
'''
Transform xfsrestore inventory data output to a Python dict source and evaluate it.
'''
data = []
out = [line for line in out.split("\n") if line.strip()]
# No inventory yet
if len(out) == 1 and 'restore status' in out[0].lower():
return {'restore_status': out[0]}
ident = 0
data.append("{")
for line in out[:-1]:
if len([elm for elm in line.strip().split(":") if elm]) == 1:
n_ident = len(re.sub("[^\t]", "", line))
if ident > n_ident:
for step in range(ident):
data.append("},")
ident = n_ident
data.append(_xr_to_keyset(line))
data.append("{")
else:
data.append(_xr_to_keyset(line))
for step in range(ident + 1):
data.append("},")
data.append("},")
# We are evaling into a python dict, a json load
# would be safer
data = eval('\n'.join(data))[0] # pylint: disable=W0123
data['restore_status'] = out[-1]
return data
def hex2ip(hex_ip, invert=False):
'''
Convert a hex string to an ip, if a failure occurs the original hex is
returned. If 'invert=True' assume that ip from /proc/net/<proto>
'''
if len(hex_ip) == 32: # ipv6
ip = []
for i in range(0, 32, 8):
ip_part = hex_ip[i:i + 8]
ip_part = [ip_part[x:x + 2] for x in range(0, 8, 2)]
if invert:
ip.append("{0[3]}{0[2]}:{0[1]}{0[0]}".format(ip_part))
else:
ip.append("{0[0]}{0[1]}:{0[2]}{0[3]}".format(ip_part))
try:
return ipaddress.IPv6Address(":".join(ip)).compressed
except ipaddress.AddressValueError as ex:
log.error('hex2ip - ipv6 address error: {0}'.format(ex))
return hex_ip
try:
hip = int(hex_ip, 16)
except ValueError:
return hex_ip
if invert:
return '{3}.{2}.{1}.{0}'.format(hip >> 24 & 255,
hip >> 16 & 255,
hip >> 8 & 255,
hip & 255)
return '{0}.{1}.{2}.{3}'.format(hip >> 24 & 255,
hip >> 16 & 255,
hip >> 8 & 255,
hip & 255)
def sunos_netdev():
'''
sunos specific implementation of netdev
'''
ret = {}
##NOTE: we cannot use hwaddr_interfaces here, so we grab both ip4 and ip6
for dev in __grains__['ip4_interfaces'].keys() + __grains__['ip6_interfaces'].keys():
# fetch device info
netstat_ipv4 = __salt__['cmd.run']('netstat -i -I {dev} -n -f inet'.format(dev=dev)).splitlines()
netstat_ipv6 = __salt__['cmd.run']('netstat -i -I {dev} -n -f inet6'.format(dev=dev)).splitlines()
# prepare data
netstat_ipv4[0] = netstat_ipv4[0].split()
netstat_ipv4[1] = netstat_ipv4[1].split()
netstat_ipv6[0] = netstat_ipv6[0].split()
netstat_ipv6[1] = netstat_ipv6[1].split()
# add data
ret[dev] = {}
for i in range(len(netstat_ipv4[0])-1):
if netstat_ipv4[0][i] == 'Name':
continue
if netstat_ipv4[0][i] in ['Address', 'Net/Dest']:
ret[dev]['IPv4 {field}'.format(field=netstat_ipv4[0][i])] = netstat_ipv4[1][i]
else:
ret[dev][netstat_ipv4[0][i]] = _number(netstat_ipv4[1][i])
for i in range(len(netstat_ipv6[0])-1):
if netstat_ipv6[0][i] == 'Name':
continue
if netstat_ipv6[0][i] in ['Address', 'Net/Dest']:
ret[dev]['IPv6 {field}'.format(field=netstat_ipv6[0][i])] = netstat_ipv6[1][i]
else:
ret[dev][netstat_ipv6[0][i]] = _number(netstat_ipv6[1][i])
return ret
def test_event_many_backlog(self):
'''Test a large number of events, send all then recv all'''
with eventpublisher_process():
me = event.MasterEvent(SOCK_DIR, listen=True)
# Must not exceed zmq HWM
for i in range(500):
me.fire_event({'data': '{0}'.format(i)}, 'testevents')
for i in range(500):
evt = me.get_event(tag='testevents')
self.assertGotEvent(evt, {'data': '{0}'.format(i)}, 'Event {0}'.format(i))
def _gather_update_categories(updateCollection):
categories = []
for i in range(updateCollection.Count):
update = updateCollection.Item(i)
for j in range(update.Categories.Count):
name = update.Categories.Item(j).Name
if name not in categories:
log.debug('found category: {0}'.format(name))
categories.append(name)
return categories
def linux_netstats():
'''
freebsd specific netstats implementation
'''
procf = '/proc/net/netstat'
if not os.path.isfile(procf):
return {}
stats = salt.utils.fopen(procf, 'r').read().splitlines()
ret = {}
headers = ['']
for line in stats:
if not line:
continue
comps = line.split()
if comps[0] == headers[0]:
index = len(headers) - 1
row = {}
for field in range(index):
if field < 1:
continue
else:
row[headers[field]] = _number(comps[field])
rowname = headers[0].replace(':', '')
ret[rowname] = row
else:
headers = comps
return ret
def GetInstallationResults(self):
'''
this gets results of installation process.
'''
# if the blugger is empty, the results are nil.
log.debug('blugger has {0} updates in it'.format(self.install_collection.Count))
if self.install_collection.Count == 0:
return {}
updates = []
log.debug('repairing update list')
for i in range(self.install_collection.Count):
# this gets the result from install_results, but the title comes from the update
# collection install_collection.
updates.append('{0}: {1}'.format(
self.install_results.GetUpdateResult(i).ResultCode,
self.install_collection.Item(i).Title))
log.debug('Update results enumerated, now making a library to pass back')
results = {}
# translates the list of update results into a library that salt expects.
for i, update in enumerate(updates):
results['update {0}'.format(i)] = update
log.debug('Update information complied. returning')
return results
def rm_special(user, special, cmd):
'''
Remove a special cron job for a specified user.
CLI Example:
.. code-block:: bash
salt '*' cron.rm_job root @hourly /usr/bin/foo
'''
lst = list_tab(user)
ret = 'absent'
rm_ = None
for ind in range(len(lst['special'])):
if lst['special'][ind]['cmd'] == cmd and \
lst['special'][ind]['spec'] == special:
lst['special'].pop(ind)
rm_ = ind
if rm_ is not None:
ret = 'removed'
comdat = _write_cron_lines(user, _render_tab(lst))
if comdat['retcode']:
# Failed to commit
return comdat['stderr']
return ret
def update_function_config(FunctionName, Role=None, Handler=None,
Description=None, Timeout=None, MemorySize=None,
region=None, key=None, keyid=None, profile=None, VpcConfig=None,
WaitForRole=False, RoleRetries=5):
'''
Update the named lambda function to the configuration.
Returns {updated: true} if the function was updated and returns
{updated: False} if the function was not updated.
CLI Example:
.. code-block:: bash
salt myminion boto_lamba.update_function_config my_function my_role my_file.my_function "my lambda function"
'''
args = dict(FunctionName=FunctionName)
options = {'Handler': Handler,
'Description': Description,
'Timeout': Timeout,
'MemorySize': MemorySize,
'VpcConfig': VpcConfig}
for val, var in six.iteritems(options):
if var:
args[val] = var
if Role:
role_arn = _get_role_arn(Role, region, key, keyid, profile)
args['Role'] = role_arn
try:
conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
if WaitForRole:
retrycount = RoleRetries
else:
retrycount = 1
for retry in range(retrycount, 0, -1):
try:
r = conn.update_function_configuration(**args)
except ClientError as e:
if retry > 1 and e.response.get('Error', {}).get('Code') == 'InvalidParameterValueException':
log.info('Function not updated but IAM role may not have propagated, will retry')
# exponential backoff
time.sleep((2 ** (RoleRetries - retry)) + (random.randint(0, 1000) / 1000))
continue
else:
raise
else:
break
if r:
keys = ('FunctionName', 'Runtime', 'Role', 'Handler', 'CodeSha256',
'CodeSize', 'Description', 'Timeout', 'MemorySize', 'FunctionArn',
'LastModified', 'VpcConfig')
return {'updated': True, 'function': dict([(k, r.get(k)) for k in keys])}
else:
log.warning('Function was not updated')
return {'updated': False}
except ClientError as e:
return {'updated': False, 'error': salt.utils.boto3.get_error(e)}
def _retry_get_url(url, num_retries=10, timeout=5):
'''
Retry grabbing a URL.
Based heavily on boto.utils.retry_url
'''
for i in range(0, num_retries):
try:
result = requests.get(url, timeout=timeout, proxies={'http': ''})
if hasattr(result, 'text'):
return result.text
elif hasattr(result, 'content'):
return result.content
else:
return ''
except requests.exceptions.HTTPError as exc:
return ''
except Exception as exc:
pass
log.warning(
'Caught exception reading from URL. Retry no. {0}'.format(i)
)
log.warning(pprint.pformat(exc))
time.sleep(2 ** i)
log.error(
'Failed to read from URL for {0} times. Giving up.'.format(num_retries)
)
return ''
def check_inheritance(path, objectType):
'''
check a specified path to verify if inheritance is enabled
returns 'Inheritance' of True/False
hkey: HKEY_LOCAL_MACHINE, HKEY_CURRENT_USER, etc
path: path of the registry key to check
'''
ret = {'result': False,
'Inheritance': False,
'comment': []}
dc = daclConstants()
objectType = dc.getObjectTypeBit(objectType)
path = dc.processPath(path, objectType)
try:
sd = win32security.GetNamedSecurityInfo(path, objectType, win32security.DACL_SECURITY_INFORMATION)
dacls = sd.GetSecurityDescriptorDacl()
except Exception as e:
ret['result'] = False
ret['comment'].append((
'Error obtaining the Security Descriptor or DACL of the path: {0}'
).format(e))
return ret
for counter in range(0, dacls.GetAceCount()):
ace = dacls.GetAce(counter)
if (ace[0][1] & win32security.INHERITED_ACE) == win32security.INHERITED_ACE:
ret['Inheritance'] = True
ret['result'] = True
return ret
def exactly_n(l, n=1):
'''
Tests that exactly N items in an iterable are "truthy" (neither None,
False, nor 0).
'''
i = iter(l)
return all(any(i) for j in range(n)) and not any(i)
def nameservers(ns,
host=None,
admin_username=None,
admin_password=None,
module=None):
'''
Configure the nameservers on the DRAC
CLI Example:
.. code-block:: bash
salt dell dracr.nameservers [NAMESERVERS]
salt dell dracr.nameservers ns1.example.com ns2.example.com
admin_username=root admin_password=calvin module=server-1
host=192.168.1.1
'''
if len(ns) > 2:
log.warning('racadm only supports two nameservers')
return False
for i in range(1, len(ns) + 1):
if not __execute_cmd('config -g cfgLanNetworking -o '
'cfgDNSServer{0} {1}'.format(i, ns[i - 1]),
host=host,
admin_username=admin_username,
admin_password=admin_password,
module=module):
return False
return True
def test_any_future(self):
'''
Test that the Any Future does what we think it does
'''
# create a few futures
futures = []
for x in range(0, 3):
future = tornado.concurrent.Future()
future.add_done_callback(self.stop)
futures.append(future)
# create an any future, make sure it isn't immediately done
any_ = saltnado.Any(futures)
self.assertIs(any_.done(), False)
# finish one, lets see who finishes
futures[0].set_result('foo')
self.wait()
self.assertIs(any_.done(), True)
self.assertIs(futures[0].done(), True)
self.assertIs(futures[1].done(), False)
self.assertIs(futures[2].done(), False)
# make sure it returned the one that finished
self.assertEqual(any_.result(), futures[0])
futures = futures[1:]
# re-wait on some other futures
any_ = saltnado.Any(futures)
futures[0].set_result('foo')
self.wait()
self.assertIs(any_.done(), True)
self.assertIs(futures[0].done(), True)
self.assertIs(futures[1].done(), False)
def _makepass(password, hasher='sha256'):
'''
Create a znc compatible hashed password
'''
# Setup the hasher
if hasher == 'sha256':
h = hashlib.sha256(password)
elif hasher == 'md5':
h = hashlib.md5(password)
else:
return NotImplemented
c = "abcdefghijklmnopqrstuvwxyz" \
"ABCDEFGHIJKLMNOPQRSTUVWXYZ" \
"0123456789!?.,:;/*-+_()"
r = {
'Method': h.name,
'Salt': ''.join(random.SystemRandom().choice(c) for x in range(20)),
}
# Salt the password hash
h.update(r['Salt'])
r['Hash'] = h.hexdigest()
return r
def _pretty_hex(hex_str):
'''
Nicely formats hex strings
'''
if len(hex_str) % 2 != 0:
hex_str = '0' + hex_str
return ':'.join([hex_str[i:i+2] for i in range(0, len(hex_str), 2)]).upper()
def test_mutually_exclusive_list_options(self):
test_options = ['--list-locations', '--list-images', '--list-sizes']
while True:
for idx in range(1, len(test_options)):
output = self.run_cloud(
'{0} ec2 {1} ec2'.format(
test_options[0], test_options[idx]
), catch_stderr=True
)
try:
self.assertIn(
'salt-cloud: error: The options {0}/{1} are mutually '
'exclusive. Please only choose one of them'.format(
test_options[0], test_options[idx]
),
output[1]
)
except AssertionError:
print(output)
raise
# Remove the first option from the list
test_options.pop(0)
if len(test_options) <= 1:
# Only one left? Stop iterating
break
def find_room(name, api_key=None):
'''
Find a room by name and return it.
:param name: The room name.
:param api_key: The Slack admin api key.
:return: The room object.
CLI Example:
.. code-block:: bash
salt '*' slack.find_room name="random"
salt '*' slack.find_room name="random" api_key=peWcBiMOS9HrZG15peWcBiMOS9HrZG15
'''
if not api_key:
api_key = _get_api_key()
# search results don't include the name of the
# channel with a hash, if the passed channel name
# has a hash we remove it.
if name.startswith('#'):
name = name[1:]
ret = list_rooms(api_key)
if ret['res']:
rooms = ret['message']
if rooms:
for room in range(0, len(rooms)):
if rooms[room]['name'] == name:
return rooms[room]
return False
def find_user(name, api_key=None):
'''
Find a user by name and return it.
:param name: The user name.
:param api_key: The Slack admin api key.
:return: The user object.
CLI Example:
.. code-block:: bash
salt '*' slack.find_user name="ThomasHatch"
salt '*' slack.find_user name="ThomasHatch" api_key=peWcBiMOS9HrZG15peWcBiMOS9HrZG15
'''
if not api_key:
api_key = _get_api_key()
ret = list_users(api_key)
if ret['res']:
users = ret['message']
if users:
for user in range(0, len(users)):
if users[user]['name'] == name:
return users[user]
return False
def _dmi_isclean(key, val):
'''
Clean out well-known bogus values
'''
if val is None or not len(val) or re.match('none', val, flags=re.IGNORECASE):
# log.debug('DMI {0} value {1} seems invalid or empty'.format(key, val))
return False
elif 'uuid' in key:
# Try each version (1-5) of RFC4122 to check if it's actually a UUID
for uuidver in range(1, 5):
try:
uuid.UUID(val, version=uuidver)
return True
except ValueError:
continue
log.trace('DMI {0} value {1} is an invalid UUID'.format(key, val.replace('\n', ' ')))
return False
elif re.search('serial|part|version', key):
# 'To be filled by O.E.M.
# 'Not specified' etc.
# 0000000, 1234667 etc.
# begone!
return not re.match(r'^[0]+$', val) \
and not re.match(r'[0]?1234567[8]?[9]?[0]?', val) \
and not re.search(r'sernum|part[_-]?number|specified|filled', val, flags=re.IGNORECASE)
elif re.search('asset|manufacturer', key):
# AssetTag0. Manufacturer04. Begone.
return not re.search(r'manufacturer|to be filled|available|asset|^no(ne|t)', val, flags=re.IGNORECASE)
else:
# map unspecified, undefined, unknown & whatever to None
return not re.search(r'to be filled', val, flags=re.IGNORECASE) \
and not re.search(r'un(known|specified)|no(t|ne)? (asset|provided|defined|available|present|specified)',
val, flags=re.IGNORECASE)
def test_update_secret(self):
name = self.name
filename = "/tmp/{0}.json".format(name)
with open(filename, 'w') as f:
json.dump(self.request, f)
create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE)
# wee need to give kubernetes time save data in etcd
time.sleep(0.1)
expected_data = {}
names = []
for i in range(3):
names.append("/tmp/{0}-{1}-updated".format(name, i))
with open("/tmp/{0}-{1}-updated".format(name, i), 'w') as f:
expected_data["{0}-{1}-updated".format(name, i)] = base64.b64encode("{0}{1}-updated".format(name, i))
f.write("{0}{1}-updated".format(name, i))
res = k8s.update_secret("default", name, names, apiserver_url="http://127.0.0.1:8080")
# if creation is failed, kubernetes return non json error message
proc = Popen(["kubectl", "--namespace=default", "get", "secrets", name, "-o", "json"], stdout=PIPE)
kubectl_out = json.loads(proc.communicate()[0])
# if creation is failed, kubernetes return non json error message
b = kubectl_out.get("data", {})
self.assertTrue(isinstance(kubectl_out, dict))
self.assertEqual(expected_data, b)
def generate_selectors(labels=None, *fields, **kwargs):
'''
Create an element list based in another objects labels that will create
a value of True in the corresponding element if in either selectors
or kwargs, otherwise False.
Args:
labels:
Example:
>>> labels = ['one', 'two', 'three', 'four']
>>> fields = ['two', 'three']
>>> generate_selectors(labels, fields)
[False, True, True, False]
'''
if not labels:
return []
enabled = True if 'all' in fields or 'all' in kwargs else False
selectors = [enabled for i in range(len(labels))] # pylint: disable=W0612
if enabled:
return selectors
for index, selector in enumerate(labels):
if selector in fields or selector in kwargs:
selectors[index] = True
return selectors
def netstats():
'''
Return the network stats for this minion
CLI Example:
.. code-block:: bash
salt '*' status.netstats
'''
procf = '/proc/net/netstat'
if not os.path.isfile(procf):
return {}
stats = salt.utils.fopen(procf, 'r').read().splitlines()
ret = {}
headers = ['']
for line in stats:
if not line:
continue
comps = line.split()
if comps[0] == headers[0]:
index = len(headers) - 1
row = {}
for field in range(index):
if field < 1:
continue
else:
row[headers[field]] = _number(comps[field])
rowname = headers[0].replace(':', '')
ret[rowname] = row
else:
headers = comps
return ret
def stop(name):
'''
Stop the specified service
CLI Example:
.. code-block:: bash
salt '*' service.stop <service name>
'''
# net stop issues a stop command and waits briefly (~30s), but will give
# up if the service takes too long to stop with a misleading
# "service could not be stopped" message and RC 0.
cmd = ['net', 'stop', name]
res = __salt__['cmd.run'](cmd, python_shell=False)
if 'service was stopped' in res:
return True
# we requested a stop, but the service is still thinking about it.
# poll for the real status
for attempt in range(SERVICE_STOP_POLL_MAX_ATTEMPTS):
if not status(name):
return True
log.debug('Waiting for %s to stop', name)
time.sleep(SERVICE_STOP_DELAY_SECONDS)
log.warning('Giving up on waiting for service `%s` to stop', name)
return False
def waitfor_job(conn=None, LinodeID=None, JobID=None, timeout=300, quiet=True):
if not conn:
conn = get_conn()
interval = 5
iterations = int(timeout / interval)
for i in range(0, iterations):
try:
result = conn.linode_job_list(LinodeID=LinodeID, JobID=JobID)
except linode.ApiError as exc:
log.info('Waiting for job {0} on host {1} returned {2}'.
format(LinodeID, JobID, exc))
return False
if result[0]['HOST_SUCCESS'] == 1:
return True
time.sleep(interval)
if not quiet:
log.info('Still waiting on Job {0} for {1}'.format(JobID,
LinodeID))
else:
log.debug('Still waiting on Job {0} for {1}'.format(JobID,
LinodeID))
return False
def test_start_stop(self):
for i in range(2):
machine = vb_start_vm(BOOTABLE_BASE_BOX_NAME, 20000)
self.assertEqual(machine_get_machinestate_str(machine), "Running")
machine = vb_stop_vm(BOOTABLE_BASE_BOX_NAME)
self.assertEqual(machine_get_machinestate_str(machine), "PoweredOff")
def waitfor_status(conn=None, LinodeID=None, status=None,
timeout=300, quiet=True):
'''
Wait for a certain status
'''
if not conn:
conn = get_conn()
if status is None:
status = 'Brand New'
interval = 5
iterations = int(timeout / interval)
for i in range(0, iterations):
result = get_node(LinodeID)
if result['state'] == status:
return True
time.sleep(interval)
if not quiet:
log.info('Status for {0} is {1}'.format(LinodeID, result['state']))
else:
log.debug('Status for {0} is {1}'.format(LinodeID, result))
return False
def list_values(hive, key=None, use_32bit_registry=False, include_default=True):
'''
Enumerates the values in a registry key or hive.
:param str hive: The name of the hive. Can be one of the following
- HKEY_LOCAL_MACHINE or HKLM
- HKEY_CURRENT_USER or HKCU
- HKEY_USER or HKU
- HKEY_CLASSES_ROOT or HKCR
- HKEY_CURRENT_CONFIG or HKCC
:param str key: The key (looks like a path) to the value name. If a key is
not passed, the values under the hive will be returned.
:param bool use_32bit_registry: Accesses the 32bit portion of the registry
on 64 bit installations. On 32bit machines this is ignored.
:param bool include_default: Toggle whether to include the '(Default)' value.
:return: A list of values under the hive or key.
:rtype: list
CLI Example:
.. code-block:: bash
salt '*' reg.list_values HKLM 'SYSTEM\\CurrentControlSet\\Services\\Tcpip'
'''
local_hive = _to_unicode(hive)
local_key = _to_unicode(key)
registry = Registry()
hkey = registry.hkeys[local_hive]
access_mask = registry.registry_32[use_32bit_registry]
handle = None
values = list()
try:
handle = win32api.RegOpenKeyEx(hkey, local_key, 0, access_mask)
for i in range(win32api.RegQueryInfoKey(handle)[1]):
vname, vdata, vtype = win32api.RegEnumValue(handle, i)
if not vname:
vname = "(Default)"
value = {'hive': local_hive,
'key': local_key,
'vname': _to_mbcs(vname),
'vtype': registry.vtype_reverse[vtype],
'success': True}
# Only convert text types to unicode
if vtype == win32con.REG_MULTI_SZ:
value['vdata'] = [_to_mbcs(i) for i in vdata]
elif vtype in [win32con.REG_SZ, win32con.REG_EXPAND_SZ]:
value['vdata'] = _to_mbcs(vdata)
else:
value['vdata'] = vdata
values.append(value)
except pywintypes.error as exc: # pylint: disable=E0602
log.debug(r'Cannot find key: %s\%s', hive, key, exc_info=True)
return False, r'Cannot find key: {0}\{1}'.format(hive, key)
finally:
if handle:
handle.Close()
return values
def port_bindings(val, **kwargs):
'''
On the CLI, these are passed as multiple instances of a given CLI option.
In Salt, we accept these as a comma-delimited list but the API expects a
Python dictionary mapping ports to their bindings. The format the API
expects is complicated depending on whether or not the external port maps
to a different internal port, or if the port binding is for UDP instead of
TCP (the default). For reference, see the "Port bindings" section in the
docker-py documentation at the following URL:
http://docker-py.readthedocs.io/en/stable/api.html
'''
validate_ip_addrs = kwargs.get('validate_ip_addrs', True)
if not isinstance(val, dict):
if not isinstance(val, list):
try:
val = helpers.split(val)
except AttributeError:
val = helpers.split(six.text_type(val))
for idx in range(len(val)):
if not isinstance(val[idx], six.string_types):
val[idx] = six.text_type(val[idx])
def _format_port(port_num, proto):
return six.text_type(port_num) + '/udp' if proto.lower(
) == 'udp' else port_num
bindings = {}
for binding in val:
bind_parts = helpers.split(binding, ':')
num_bind_parts = len(bind_parts)
if num_bind_parts == 1:
# Single port or port range being passed through (no
# special mapping)
container_port = six.text_type(bind_parts[0])
if container_port == '':
raise SaltInvocationError(
'Empty port binding definition found')
container_port, _, proto = container_port.partition('/')
try:
start, end = helpers.get_port_range(container_port)
except ValueError as exc:
# Using __str__() to avoid deprecation warning for using
# the message attribute of the ValueError.
raise SaltInvocationError(exc.__str__())
bind_vals = [(_format_port(port_num, proto), None)
for port_num in range(start, end + 1)]
elif num_bind_parts == 2:
if bind_parts[0] == '':
raise SaltInvocationError(
'Empty host port in port binding definition '
'\'{0}\''.format(binding))
if bind_parts[1] == '':
raise SaltInvocationError(
'Empty container port in port binding definition '
'\'{0}\''.format(binding))
container_port, _, proto = bind_parts[1].partition('/')
try:
cport_start, cport_end = \
helpers.get_port_range(container_port)
hport_start, hport_end = \
helpers.get_port_range(bind_parts[0])
except ValueError as exc:
# Using __str__() to avoid deprecation warning for
# using the message attribute of the ValueError.
raise SaltInvocationError(exc.__str__())
if (hport_end - hport_start) != (cport_end - cport_start):
# Port range is mismatched
raise SaltInvocationError(
'Host port range ({0}) does not have the same '
'number of ports as the container port range '
'({1})'.format(bind_parts[0], container_port))
cport_list = list(range(cport_start, cport_end + 1))
hport_list = list(range(hport_start, hport_end + 1))
bind_vals = [(_format_port(cport_list[x],
proto), hport_list[x])
for x in range(len(cport_list))]
elif num_bind_parts == 3:
host_ip, host_port = bind_parts[0:2]
if validate_ip_addrs:
helpers.validate_ip(host_ip)
container_port, _, proto = bind_parts[2].partition('/')
try:
cport_start, cport_end = \
helpers.get_port_range(container_port)
except ValueError as exc:
# Using __str__() to avoid deprecation warning for
# using the message attribute of the ValueError.
raise SaltInvocationError(exc.__str__())
cport_list = list(range(cport_start, cport_end + 1))
if host_port == '':
hport_list = [None] * len(cport_list)
else:
try:
hport_start, hport_end = \
helpers.get_port_range(host_port)
except ValueError as exc:
# Using __str__() to avoid deprecation warning for
# using the message attribute of the ValueError.
raise SaltInvocationError(exc.__str__())
hport_list = list(range(hport_start, hport_end + 1))
if (hport_end - hport_start) != (cport_end - cport_start):
# Port range is mismatched
raise SaltInvocationError(
'Host port range ({0}) does not have the same '
'number of ports as the container port range '
'({1})'.format(host_port, container_port))
bind_vals = [(_format_port(val, proto),
(host_ip, ) if hport_list[idx] is None else
(host_ip, hport_list[idx]))
for idx, val in enumerate(cport_list)]
else:
raise SaltInvocationError(
'\'{0}\' is an invalid port binding definition (at most '
'3 components are allowed, found {1})'.format(
binding, num_bind_parts))
for cport, bind_def in bind_vals:
if cport not in bindings:
bindings[cport] = bind_def
else:
if isinstance(bindings[cport], list):
# Append to existing list of bindings for this
# container port.
bindings[cport].append(bind_def)
else:
bindings[cport] = [bindings[cport], bind_def]
for idx in range(len(bindings[cport])):
if bindings[cport][idx] is None:
# Now that we are adding multiple
# bindings
try:
# Convert 1234/udp to 1234
bindings[cport][idx] = int(cport.split('/')[0])
except AttributeError:
# Port was tcp, the AttributeError
# signifies that the split failed
# because the port number was
# already defined as an integer.
# Just use the cport.
bindings[cport][idx] = cport
val = bindings
return val
def process_results(self, rows):
'''
This function takes a list of database results and iterates over,
merging them into a dict form.
'''
listify = OrderedDict()
listify_dicts = OrderedDict()
for ret in rows:
# crd is the Current Return Data level, to make this non-recursive.
crd = self.focus
# Walk and create dicts above the final layer
for i in range(0, self.depth - 1):
# At the end we'll use listify to find values to make a list of
if i + 1 in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if ret[i] not in listify[id(crd)]:
listify[id(crd)].append(ret[i])
if ret[i] not in crd:
# Key missing
crd[ret[i]] = {}
crd = crd[ret[i]]
else:
# Check type of collision
ty = type(crd[ret[i]])
if ty is list:
# Already made list
temp = {}
crd[ret[i]].append(temp)
crd = temp
elif ty is not dict:
# Not a list, not a dict
if self.as_list:
# Make list
temp = {}
crd[ret[i]] = [crd[ret[i]], temp]
crd = temp
else:
# Overwrite
crd[ret[i]] = {}
crd = crd[ret[i]]
else:
# dict, descend.
crd = crd[ret[i]]
# If this test is true, the penultimate field is the key
if self.depth == self.num_fields - 1:
nk = self.num_fields - 2 # Aka, self.depth-1
# Should we and will we have a list at the end?
if ((self.as_list and (ret[nk] in crd))
or (nk + 1 in self.with_lists)):
if ret[nk] in crd:
if not isinstance(crd[ret[nk]], list):
crd[ret[nk]] = [crd[ret[nk]]]
# if it's already a list, do nothing
else:
crd[ret[nk]] = []
crd[ret[nk]].append(ret[self.num_fields - 1])
else:
if not self.ignore_null or ret[self.num_fields -
1] is not None:
crd[ret[nk]] = ret[self.num_fields - 1]
else:
# Otherwise, the field name is the key but we have a spare.
# The spare results because of {c: d} vs {c: {"d": d, "e": e }}
# So, make that last dict
if ret[self.depth - 1] not in crd:
crd[ret[self.depth - 1]] = {}
# This bit doesn't escape listify
if self.depth in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if ret[self.depth - 1] not in listify[id(crd)]:
listify[id(crd)].append(ret[self.depth - 1])
crd = crd[ret[self.depth - 1]]
# Now for the remaining keys, we put them into the dict
for i in range(self.depth, self.num_fields):
nk = self.field_names[i]
# Listify
if i + 1 in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if nk not in listify[id(crd)]:
listify[id(crd)].append(nk)
# Collision detection
if self.as_list and (nk in crd):
# Same as before...
if isinstance(crd[nk], list):
crd[nk].append(ret[i])
else:
crd[nk] = [crd[nk], ret[i]]
else:
if not self.ignore_null or ret[i] is not None:
crd[nk] = ret[i]
# Get key list and work backwards. This is inner-out processing
ks = list(listify_dicts.keys())
ks.reverse()
for i in ks:
d = listify_dicts[i]
for k in listify[i]:
if isinstance(d[k], dict):
d[k] = list(d[k].values())
elif isinstance(d[k], list):
d[k] = [d[k]]
def clear(self):
for i in self._by_level:
self._by_level[i] = []
for i in range(len(self._msgs)):
self._msgs.pop()
def managed(name,
dns_proto=None,
dns_servers=None,
ip_proto=None,
ip_addrs=None,
gateway=None,
enabled=True,
**kwargs):
'''
Ensure that the named interface is configured properly.
Args:
name (str):
The name of the interface to manage
dns_proto (str): None
Set to ``static`` and use the ``dns_servers`` parameter to provide a
list of DNS nameservers. set to ``dhcp`` to use DHCP to get the DNS
servers.
dns_servers (list): None
A list of static DNS servers. To clear the list of DNS servers pass
an empty list (``[]``). ``None`` will make no changes.
ip_proto (str): None
Set to ``static`` and use the ``ip_addrs`` and (optionally)
``gateway`` parameters to provide a list of static IP addresses and
the default gateway. Set to ``dhcp`` to use DHCP.
ip_addrs (list): None
A list of static IP addresses with netmask flag, ie: 192.168.0.11/24
gateway (str): None
The gateway to set for the interface
enabled (bool): True
Set to ``False`` to ensure that this interface is disabled.
Returns:
dict: A dictionary of old and new settings
Example:
.. code-block:: yaml
Ethernet1:
network.managed:
- dns_proto: static
- dns_servers:
- 8.8.8.8
- 8.8.8.4
- ip_proto: static
- ip_addrs:
- 192.168.0.100/24
Clear DNS entries example:
.. code-block:: yaml
Ethernet1:
network.managed:
- dns_proto: static
- dns_servers: []
- ip_proto: dhcp
'''
ret = {
'name': name,
'changes': {},
'result': True,
'comment': 'Interface \'{0}\' is up to date'.format(name)
}
dns_proto = six.text_type(dns_proto).lower()
ip_proto = six.text_type(ip_proto).lower()
errors = []
if dns_proto not in __VALID_PROTO:
ret['result'] = False
errors.append('dns_proto must be one of the following: {0}'.format(
', '.join(__VALID_PROTO)))
if ip_proto not in __VALID_PROTO:
errors.append('ip_proto must be one of the following: {0}'.format(
', '.join(__VALID_PROTO)))
if errors:
ret['result'] = False
ret['comment'] = '\n'.join(errors)
return ret
try:
currently_enabled = __salt__['ip.is_enabled'](name)
except CommandExecutionError:
currently_enabled = False
if not enabled:
if currently_enabled:
if __opts__['test']:
ret['result'] = None
ret['comment'] = (
'Interface \'{0}\' will be disabled'.format(name))
else:
ret['result'] = __salt__['ip.disable'](name)
if not ret['result']:
ret['comment'] = (
'Failed to disable interface \'{0}\''.format(name))
else:
ret['comment'] += ' (already disabled)'
return ret
else:
if not currently_enabled:
if __opts__['test']:
ret['result'] = None
ret['comment'] = (
'Interface \'{0}\' will be enabled'.format(name))
else:
if not __salt__['ip.enable'](name):
ret['result'] = False
ret['comment'] = ('Failed to enable interface \'{0}\' to '
'make changes'.format(name))
return ret
errors = _validate(dns_proto, dns_servers, ip_proto, ip_addrs, gateway)
if errors:
ret['result'] = False
ret['comment'] = ('The following SLS configuration errors were '
'detected:\n- {0}'.format('\n- '.join(errors)))
return ret
old = __salt__['ip.get_interface'](name)
if not old:
ret['result'] = False
ret['comment'] = ('Unable to get current configuration for '
'interface \'{0}\''.format(name))
return ret
changes = _changes(old, dns_proto, dns_servers, ip_proto, ip_addrs,
gateway)
# If dns_servers is the default `None` make no changes
# To clear the list, pass an empty dict
if str(dns_servers).lower() == 'none':
changes.pop('dns_servers', None)
if not changes:
return ret
if __opts__['test']:
comments = []
if 'dns_proto' in changes:
comments.append('DNS protocol will be changed to: {0}'.format(
changes['dns_proto']))
if dns_proto == 'static' and 'dns_servers' in changes:
if len(changes['dns_servers']) == 0:
comments.append('The list of DNS servers will be cleared')
else:
comments.append(
'DNS servers will be set to the following: {0}'.format(
', '.join(changes['dns_servers'])))
if 'ip_proto' in changes:
comments.append('IP protocol will be changed to: {0}'.format(
changes['ip_proto']))
if ip_proto == 'static':
if 'ip_addrs' in changes:
comments.append(
'IP addresses will be set to the following: {0}'.
format(', '.join(changes['ip_addrs'])))
if 'gateway' in changes:
if changes['gateway'] is None:
comments.append('Default gateway will be removed')
else:
comments.append(
'Default gateway will be set to {0}'.format(
changes['gateway']))
ret['result'] = None
ret['comment'] = ('The following changes will be made to '
'interface \'{0}\':\n- {1}'.format(
name, '\n- '.join(comments)))
return ret
if changes.get('dns_proto') == 'dhcp':
__salt__['ip.set_dhcp_dns'](name)
elif 'dns_servers' in changes:
if len(changes['dns_servers']) == 0:
# To clear the list of DNS servers you have to pass []. Later
# changes gets passed like *args and a single empty list is
# converted to an empty tuple. So, you have to add [] here
changes['dns_servers'] = [[]]
__salt__['ip.set_static_dns'](name, *changes['dns_servers'])
if changes.get('ip_proto') == 'dhcp':
__salt__['ip.set_dhcp_ip'](name)
elif changes.get('ip_addrs') or changes.get('gateway') or changes.get(
'ip_proto') == 'static':
if changes.get('gateway') and not changes.get('ip_addrs'):
changes['ip_addrs'] = ip_addrs
if changes.get(
'ip_proto') == 'static' and not changes.get('ip_addrs'):
changes['ip_addrs'] = ip_addrs
for idx in range(len(changes['ip_addrs'])):
if idx == 0:
__salt__['ip.set_static_ip'](name,
changes['ip_addrs'][idx],
gateway=gateway,
append=False)
else:
__salt__['ip.set_static_ip'](name,
changes['ip_addrs'][idx],
gateway=None,
append=True)
new = __salt__['ip.get_interface'](name)
ret['changes'] = salt.utils.data.compare_dicts(old, new)
if _changes(new, dns_proto, dns_servers, ip_proto, ip_addrs, gateway):
ret['result'] = False
ret['comment'] = ('Failed to set desired configuration settings '
'for interface \'{0}\''.format(name))
else:
ret['comment'] = ('Successfully updated configuration for '
'interface \'{0}\''.format(name))
return ret
def _first_avail_uid():
uids = set(x.pw_uid for x in pwd.getpwall())
for idx in range(501, 2**24):
if idx not in uids:
return idx
def test_very_big_message(self):
long_str = ''.join([six.text_type(num) for num in range(10**5)])
msg = {'long_str': long_str, 'stop': True}
self.channel.send(msg)
self.wait()
self.assertEqual(msg, self.payloads[0])
def setUp(self):
boto_s3_bucket.__context__ = {}
context.clear()
# connections keep getting cached from prior tests, can't find the
# correct context object to clear it. So randomize the cache key, to prevent any
# cache hits
conn_parameters['key'] = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(50))
self.patcher = patch('boto3.session.Session')
self.addCleanup(self.patcher.stop)
mock_session = self.patcher.start()
session_instance = mock_session.return_value
self.conn = MagicMock()
session_instance.client.return_value = self.conn
def _run(name, **kwargs):
'''
.. deprecated:: 2017.7.0
Function name stays the same, behaviour will change.
Run a single module function
``name``
The module function to execute
``returner``
Specify the returner to send the return of the module execution to
``kwargs``
Pass any arguments needed to execute the function
'''
ret = {'name': name, 'changes': {}, 'comment': '', 'result': None}
if name not in __salt__:
ret['comment'] = 'Module function {0} is not available'.format(name)
ret['result'] = False
return ret
if __opts__['test']:
ret['comment'] = 'Module function {0} is set to execute'.format(name)
return ret
aspec = salt.utils.args.get_function_argspec(__salt__[name])
args = []
defaults = {}
arglen = 0
deflen = 0
if isinstance(aspec.args, list):
arglen = len(aspec.args)
if isinstance(aspec.defaults, tuple):
deflen = len(aspec.defaults)
# Match up the defaults with the respective args
for ind in range(arglen - 1, -1, -1):
minus = arglen - ind
if deflen - minus > -1:
defaults[aspec.args[ind]] = aspec.defaults[-minus]
# overwrite passed default kwargs
for arg in defaults:
if arg == 'name':
if 'm_name' in kwargs:
defaults[arg] = kwargs.pop('m_name')
elif arg == 'fun':
if 'm_fun' in kwargs:
defaults[arg] = kwargs.pop('m_fun')
elif arg == 'state':
if 'm_state' in kwargs:
defaults[arg] = kwargs.pop('m_state')
elif arg == 'saltenv':
if 'm_saltenv' in kwargs:
defaults[arg] = kwargs.pop('m_saltenv')
if arg in kwargs:
defaults[arg] = kwargs.pop(arg)
missing = set()
for arg in aspec.args:
if arg == 'name':
rarg = 'm_name'
elif arg == 'fun':
rarg = 'm_fun'
elif arg == 'names':
rarg = 'm_names'
elif arg == 'state':
rarg = 'm_state'
elif arg == 'saltenv':
rarg = 'm_saltenv'
else:
rarg = arg
if rarg not in kwargs and arg not in defaults:
missing.add(rarg)
continue
if arg in defaults:
args.append(defaults[arg])
else:
args.append(kwargs.pop(rarg))
if missing:
comment = 'The following arguments are missing:'
for arg in missing:
comment += ' {0}'.format(arg)
ret['comment'] = comment
ret['result'] = False
return ret
if aspec.varargs:
if aspec.varargs == 'name':
rarg = 'm_name'
elif aspec.varargs == 'fun':
rarg = 'm_fun'
elif aspec.varargs == 'names':
rarg = 'm_names'
elif aspec.varargs == 'state':
rarg = 'm_state'
elif aspec.varargs == 'saltenv':
rarg = 'm_saltenv'
else:
rarg = aspec.varargs
if rarg in kwargs:
varargs = kwargs.pop(rarg)
if not isinstance(varargs, list):
msg = "'{0}' must be a list."
ret['comment'] = msg.format(aspec.varargs)
ret['result'] = False
return ret
args.extend(varargs)
nkwargs = {}
if aspec.keywords and aspec.keywords in kwargs:
nkwargs = kwargs.pop(aspec.keywords)
if not isinstance(nkwargs, dict):
msg = "'{0}' must be a dict."
ret['comment'] = msg.format(aspec.keywords)
ret['result'] = False
return ret
try:
if aspec.keywords:
mret = __salt__[name](*args, **nkwargs)
else:
mret = __salt__[name](*args)
except Exception as e:
ret['comment'] = 'Module function {0} threw an exception. Exception: {1}'.format(
name, e)
ret['result'] = False
return ret
else:
if mret is not None or mret is not {}:
ret['changes']['ret'] = mret
if 'returner' in kwargs:
ret_ret = {
'id': __opts__['id'],
'ret': mret,
'fun': name,
'jid': salt.utils.jid.gen_jid(__opts__)
}
returners = salt.loader.returners(__opts__, __salt__)
if kwargs['returner'] in returners:
returners[kwargs['returner']](ret_ret)
ret['comment'] = 'Module function {0} executed'.format(name)
ret['result'] = _get_result(mret, ret['changes'])
return ret
def subdict_match(data,
expr,
delimiter=DEFAULT_TARGET_DELIM,
regex_match=False,
exact_match=False):
'''
Check for a match in a dictionary using a delimiter character to denote
levels of subdicts, and also allowing the delimiter character to be
matched. Thus, 'foo:bar:baz' will match data['foo'] == 'bar:baz' and
data['foo']['bar'] == 'baz'. The former would take priority over the
latter.
'''
def _match(target, pattern, regex_match=False, exact_match=False):
if regex_match:
try:
return re.match(pattern.lower(), six.text_type(target).lower())
except Exception:
log.error('Invalid regex \'%s\' in match', pattern)
return False
elif exact_match:
return six.text_type(target).lower() == pattern.lower()
else:
return fnmatch.fnmatch(
six.text_type(target).lower(), pattern.lower())
def _dict_match(target, pattern, regex_match=False, exact_match=False):
wildcard = pattern.startswith('*:')
if wildcard:
pattern = pattern[2:]
if pattern == '*':
# We are just checking that the key exists
return True
elif pattern in target:
# We might want to search for a key
return True
elif subdict_match(target,
pattern,
regex_match=regex_match,
exact_match=exact_match):
return True
if wildcard:
for key in target:
if _match(key,
pattern,
regex_match=regex_match,
exact_match=exact_match):
return True
if isinstance(target[key], dict):
if _dict_match(target[key],
pattern,
regex_match=regex_match,
exact_match=exact_match):
return True
elif isinstance(target[key], list):
for item in target[key]:
if _match(item,
pattern,
regex_match=regex_match,
exact_match=exact_match):
return True
return False
for idx in range(1, expr.count(delimiter) + 1):
splits = expr.split(delimiter)
key = delimiter.join(splits[:idx])
matchstr = delimiter.join(splits[idx:])
log.debug("Attempting to match '%s' in '%s' using delimiter '%s'",
matchstr, key, delimiter)
match = traverse_dict_and_list(data, key, {}, delimiter=delimiter)
if match == {}:
continue
if isinstance(match, dict):
if _dict_match(match,
matchstr,
regex_match=regex_match,
exact_match=exact_match):
return True
continue
if isinstance(match, (list, tuple)):
# We are matching a single component to a single list member
for member in match:
if isinstance(member, dict):
if _dict_match(member,
matchstr,
regex_match=regex_match,
exact_match=exact_match):
return True
if _match(member,
matchstr,
regex_match=regex_match,
exact_match=exact_match):
return True
continue
if _match(match,
matchstr,
regex_match=regex_match,
exact_match=exact_match):
return True
return False
def create_user(username, password, permissions, users=None):
"""
Create user accounts
CLI Example:
.. code-block:: bash
salt dell drac.create_user [USERNAME] [PASSWORD] [PRIVILEGES]
salt dell drac.create_user diana secret login,test_alerts,clear_logs
DRAC Privileges
* login : Login to iDRAC
* drac : Configure iDRAC
* user_management : Configure Users
* clear_logs : Clear Logs
* server_control_commands : Execute Server Control Commands
* console_redirection : Access Console Redirection
* virtual_media : Access Virtual Media
* test_alerts : Test Alerts
* debug_commands : Execute Debug Commands
"""
_uids = set()
if users is None:
users = list_users()
if username in users:
log.warning("'{0}' already exists".format(username))
return False
for idx in six.iterkeys(users):
_uids.add(users[idx]["index"])
uid = sorted(list(set(range(2, 12)) - _uids), reverse=True).pop()
# Create user accountvfirst
if not __execute_cmd("config -g cfgUserAdmin -o \
cfgUserAdminUserName -i {0} {1}".format(uid, username)):
delete_user(username, uid)
return False
# Configure users permissions
if not set_permissions(username, permissions, uid):
log.warning("unable to set user permissions")
delete_user(username, uid)
return False
# Configure users password
if not change_password(username, password, uid):
log.warning("unable to set user password")
delete_user(username, uid)
return False
# Enable users admin
if not __execute_cmd("config -g cfgUserAdmin -o \
cfgUserAdminEnable -i {0} 1".format(uid)):
delete_user(username, uid)
return False
return True
def create(vm_):
'''
Create a single VM from a data dict
CLI Example:
.. code-block:: bash
salt-cloud -p proxmox-ubuntu vmhostname
'''
try:
# Check for required profile parameters before sending any API calls.
if vm_['profile'] and config.is_profile_configured(
__opts__,
__active_provider_name__ or 'proxmox',
vm_['profile'],
vm_=vm_) is False:
return False
except AttributeError:
pass
ret = {}
__utils__['cloud.fire_event'](
'event',
'starting create',
'salt/cloud/{0}/creating'.format(vm_['name']),
args=__utils__['cloud.filter_event'](
'creating', vm_, ['name', 'profile', 'provider', 'driver']),
sock_dir=__opts__['sock_dir'],
transport=__opts__['transport'])
log.info('Creating Cloud VM %s', vm_['name'])
if 'use_dns' in vm_ and 'ip_address' not in vm_:
use_dns = vm_['use_dns']
if use_dns:
from socket import gethostbyname, gaierror
try:
ip_address = gethostbyname(six.text_type(vm_['name']))
except gaierror:
log.debug('Resolving of %s failed', vm_['name'])
else:
vm_['ip_address'] = six.text_type(ip_address)
try:
newid = _get_next_vmid()
data = create_node(vm_, newid)
except Exception as exc:
log.error(
'Error creating %s on PROXMOX\n\n'
'The following exception was thrown when trying to '
'run the initial deployment: \n%s',
vm_['name'],
exc,
# Show the traceback if the debug logging level is enabled
exc_info_on_loglevel=logging.DEBUG)
return False
ret['creation_data'] = data
name = vm_['name'] # hostname which we know
if 'clone' in vm_ and vm_['clone'] is True:
vmid = newid
else:
vmid = data['vmid'] # vmid which we have received
host = data['node'] # host which we have received
nodeType = data['technology'] # VM tech (Qemu / OpenVZ)
# Determine which IP to use in order of preference:
if 'ip_address' in vm_:
ip_address = six.text_type(vm_['ip_address'])
elif 'public_ips' in data:
ip_address = six.text_type(data['public_ips'][0]) # first IP
elif 'private_ips' in data:
ip_address = six.text_type(data['private_ips'][0]) # first IP
else:
raise SaltCloudExecutionFailure(
"Could not determine an IP address to use")
log.debug('Using IP address %s', ip_address)
# wait until the vm has been created so we can start it
if not wait_for_created(data['upid'], timeout=300):
return {
'Error': 'Unable to create {0}, command timed out'.format(name)
}
if 'clone' in vm_ and vm_['clone'] is True and vm_['technology'] == 'qemu':
# If we cloned a machine, see if we need to reconfigure any of the options such as net0,
# ide2, etc. This enables us to have a different cloud-init ISO mounted for each VM that's
# brought up
# TODO: Support other settings here too as these are not the only ones that can be modified
# after a clone operation
log.info('Configuring cloned VM')
# Modify the settings for the VM one at a time so we can see any problems with the values
# as quickly as possible
for setting_number in range(3):
setting = 'ide{0}'.format(setting_number)
if setting in vm_:
postParams = {}
postParams[setting] = vm_[setting]
query('post',
'nodes/{0}/qemu/{1}/config'.format(vm_['host'],
vmid), postParams)
for setting_number in range(5):
setting = 'sata{0}'.format(setting_number)
if setting in vm_:
postParams = {}
postParams[setting] = vm_[setting]
query('post',
'nodes/{0}/qemu/{1}/config'.format(vm_['host'],
vmid), postParams)
for setting_number in range(13):
setting = 'scsi{0}'.format(setting_number)
if setting in vm_:
postParams = {}
postParams[setting] = vm_[setting]
query('post',
'nodes/{0}/qemu/{1}/config'.format(vm_['host'],
vmid), postParams)
# net strings are a list of comma seperated settings. We need to merge the settings so that
# the setting in the profile only changes the settings it touches and the other settings
# are left alone. An example of why this is necessary is because the MAC address is set
# in here and generally you don't want to alter or have to know the MAC address of the new
# instance, but you may want to set the VLAN bridge for example
for setting_number in range(20):
setting = 'net{0}'.format(setting_number)
if setting in vm_:
data = query(
'get',
'nodes/{0}/qemu/{1}/config'.format(vm_['host'], vmid))
# Generate a dictionary of settings from the existing string
new_setting = {}
if setting in data:
new_setting.update(_stringlist_to_dictionary(
data[setting]))
# Merge the new settings (as a dictionary) into the existing dictionary to get the
# new merged settings
new_setting.update(_stringlist_to_dictionary(vm_[setting]))
# Convert the dictionary back into a string list
postParams = {setting: _dictionary_to_stringlist(new_setting)}
query('post',
'nodes/{0}/qemu/{1}/config'.format(vm_['host'],
vmid), postParams)
# VM has been created. Starting..
if not start(name, vmid, call='action'):
log.error('Node %s (%s) failed to start!', name, vmid)
raise SaltCloudExecutionFailure
# Wait until the VM has fully started
log.debug('Waiting for state "running" for vm %s on %s', vmid, host)
if not wait_for_state(vmid, 'running'):
return {'Error': 'Unable to start {0}, command timed out'.format(name)}
ssh_username = config.get_cloud_config_value('ssh_username',
vm_,
__opts__,
default='root')
ssh_password = config.get_cloud_config_value(
'password',
vm_,
__opts__,
)
ret['ip_address'] = ip_address
ret['username'] = ssh_username
ret['password'] = ssh_password
vm_['ssh_host'] = ip_address
vm_['password'] = ssh_password
ret = __utils__['cloud.bootstrap'](vm_, __opts__)
# Report success!
log.info('Created Cloud VM \'%s\'', vm_['name'])
log.debug('\'%s\' VM creation details:\n%s', vm_['name'],
pprint.pformat(data))
__utils__['cloud.fire_event'](
'event',
'created instance',
'salt/cloud/{0}/created'.format(vm_['name']),
args=__utils__['cloud.filter_event'](
'created', vm_, ['name', 'profile', 'provider', 'driver']),
sock_dir=__opts__['sock_dir'],
)
return ret
def _netstat_bsd():
'''
Return netstat information for BSD flavors
'''
ret = []
if __grains__['kernel'] == 'NetBSD':
for addr_family in ('inet', 'inet6'):
cmd = 'netstat -f {0} -an | tail -n+3'.format(addr_family)
out = __salt__['cmd.run'](cmd, python_shell=True)
for line in out.splitlines():
comps = line.split()
entry = {
'proto': comps[0],
'recv-q': comps[1],
'send-q': comps[2],
'local-address': comps[3],
'remote-address': comps[4]
}
if entry['proto'].startswith('tcp'):
entry['state'] = comps[5]
ret.append(entry)
else:
# Lookup TCP connections
cmd = 'netstat -p tcp -an | tail -n+3'
out = __salt__['cmd.run'](cmd, python_shell=True)
for line in out.splitlines():
comps = line.split()
ret.append({
'proto': comps[0],
'recv-q': comps[1],
'send-q': comps[2],
'local-address': comps[3],
'remote-address': comps[4],
'state': comps[5]
})
# Lookup UDP connections
cmd = 'netstat -p udp -an | tail -n+3'
out = __salt__['cmd.run'](cmd, python_shell=True)
for line in out.splitlines():
comps = line.split()
ret.append({
'proto': comps[0],
'recv-q': comps[1],
'send-q': comps[2],
'local-address': comps[3],
'remote-address': comps[4]
})
# Add in user and program info
ppid = _ppid()
if __grains__['kernel'] == 'OpenBSD':
netinfo = _netinfo_openbsd()
elif __grains__['kernel'] in ('FreeBSD', 'NetBSD'):
netinfo = _netinfo_freebsd_netbsd()
for idx in range(len(ret)):
local = ret[idx]['local-address']
remote = ret[idx]['remote-address']
proto = ret[idx]['proto']
try:
# Make a pointer to the info for this connection for easier
# reference below
ptr = netinfo[local][remote][proto]
except KeyError:
continue
# Get the pid-to-ppid mappings for this connection
conn_ppid = dict((x, y) for x, y in six.iteritems(ppid) if x in ptr)
try:
# Master pid for this connection will be the pid whose ppid isn't
# in the subset dict we created above
master_pid = next(
iter(x for x, y in six.iteritems(conn_ppid) if y not in ptr))
except StopIteration:
continue
ret[idx]['user'] = ptr[master_pid]['user']
ret[idx]['program'] = '/'.join((master_pid, ptr[master_pid]['cmd']))
return ret
def traceroute(host):
'''
Performs a traceroute to a 3rd party host
CLI Example:
.. code-block:: bash
salt '*' network.traceroute archlinux.org
'''
ret = []
if not salt.utils.which('traceroute'):
log.info('This minion does not have traceroute installed')
return ret
cmd = 'traceroute {0}'.format(salt.utils.network.sanitize_host(host))
out = __salt__['cmd.run'](cmd)
# Parse version of traceroute
cmd2 = 'traceroute --version'
out2 = __salt__['cmd.run'](cmd2)
try:
# Linux traceroute version looks like:
# Modern traceroute for Linux, version 2.0.19, Dec 10 2012
# Darwin and FreeBSD traceroute version looks like: Version 1.4a12+[FreeBSD|Darwin]
traceroute_version_raw = re.findall(
r'.*[Vv]ersion (\d+)\.([\w\+]+)\.*(\w*)', out2)[0]
log.debug('traceroute_version_raw: {0}'.format(traceroute_version_raw))
traceroute_version = []
for t in traceroute_version_raw:
try:
traceroute_version.append(int(t))
except ValueError:
traceroute_version.append(t)
if len(traceroute_version) < 3:
traceroute_version.append(0)
log.debug('traceroute_version: {0}'.format(traceroute_version))
except IndexError:
traceroute_version = [0, 0, 0]
for line in out.splitlines():
if ' ' not in line:
continue
if line.startswith('traceroute'):
continue
if 'Darwin' in str(traceroute_version[1]) or 'FreeBSD' in str(
traceroute_version[1]):
try:
traceline = re.findall(r'\s*(\d*)\s+(.*)\s+\((.*)\)\s+(.*)$',
line)[0]
except IndexError:
traceline = re.findall(r'\s*(\d*)\s+(\*\s+\*\s+\*)', line)[0]
log.debug('traceline: {0}'.format(traceline))
delays = re.findall(r'(\d+\.\d+)\s*ms', str(traceline))
try:
if traceline[1] == '* * *':
result = {'count': traceline[0], 'hostname': '*'}
else:
result = {
'count': traceline[0],
'hostname': traceline[1],
'ip': traceline[2],
}
for idx in range(0, len(delays)):
result['ms{0}'.format(idx + 1)] = delays[idx]
except IndexError:
result = {}
elif (traceroute_version[0] >= 2 and traceroute_version[2] >= 14
or traceroute_version[0] >= 2 and traceroute_version[1] > 0):
comps = line.split(' ')
if comps[1] == '* * *':
result = {'count': int(comps[0]), 'hostname': '*'}
else:
result = {
'count': int(comps[0]),
'hostname': comps[1].split()[0],
'ip': comps[1].split()[1].strip('()'),
'ms1': float(comps[2].split()[0]),
'ms2': float(comps[3].split()[0]),
'ms3': float(comps[4].split()[0])
}
else:
comps = line.split()
result = {
'count': comps[0],
'hostname': comps[1],
'ip': comps[2],
'ms1': comps[4],
'ms2': comps[6],
'ms3': comps[8],
'ping1': comps[3],
'ping2': comps[5],
'ping3': comps[7]
}
ret.append(result)
return ret
def _run(name, **kwargs):
"""
.. deprecated:: 2017.7.0
Function name stays the same, behaviour will change.
Run a single module function
``name``
The module function to execute
``returner``
Specify the returner to send the return of the module execution to
``kwargs``
Pass any arguments needed to execute the function
"""
ret = {"name": name, "changes": {}, "comment": "", "result": None}
if name not in __salt__:
ret["comment"] = "Module function {0} is not available".format(name)
ret["result"] = False
return ret
if __opts__["test"]:
ret["comment"] = "Module function {0} is set to execute".format(name)
return ret
aspec = salt.utils.args.get_function_argspec(__salt__[name])
args = []
defaults = {}
arglen = 0
deflen = 0
if isinstance(aspec.args, list):
arglen = len(aspec.args)
if isinstance(aspec.defaults, tuple):
deflen = len(aspec.defaults)
# Match up the defaults with the respective args
for ind in range(arglen - 1, -1, -1):
minus = arglen - ind
if deflen - minus > -1:
defaults[aspec.args[ind]] = aspec.defaults[-minus]
# overwrite passed default kwargs
for arg in defaults:
if arg == "name":
if "m_name" in kwargs:
defaults[arg] = kwargs.pop("m_name")
elif arg == "fun":
if "m_fun" in kwargs:
defaults[arg] = kwargs.pop("m_fun")
elif arg == "state":
if "m_state" in kwargs:
defaults[arg] = kwargs.pop("m_state")
elif arg == "saltenv":
if "m_saltenv" in kwargs:
defaults[arg] = kwargs.pop("m_saltenv")
if arg in kwargs:
defaults[arg] = kwargs.pop(arg)
missing = set()
for arg in aspec.args:
if arg == "name":
rarg = "m_name"
elif arg == "fun":
rarg = "m_fun"
elif arg == "names":
rarg = "m_names"
elif arg == "state":
rarg = "m_state"
elif arg == "saltenv":
rarg = "m_saltenv"
else:
rarg = arg
if rarg not in kwargs and arg not in defaults:
missing.add(rarg)
continue
if arg in defaults:
args.append(defaults[arg])
else:
args.append(kwargs.pop(rarg))
if missing:
comment = "The following arguments are missing:"
for arg in missing:
comment += " {0}".format(arg)
ret["comment"] = comment
ret["result"] = False
return ret
if aspec.varargs:
if aspec.varargs == "name":
rarg = "m_name"
elif aspec.varargs == "fun":
rarg = "m_fun"
elif aspec.varargs == "names":
rarg = "m_names"
elif aspec.varargs == "state":
rarg = "m_state"
elif aspec.varargs == "saltenv":
rarg = "m_saltenv"
else:
rarg = aspec.varargs
if rarg in kwargs:
varargs = kwargs.pop(rarg)
if not isinstance(varargs, list):
msg = "'{0}' must be a list."
ret["comment"] = msg.format(aspec.varargs)
ret["result"] = False
return ret
args.extend(varargs)
nkwargs = {}
if aspec.keywords and aspec.keywords in kwargs:
nkwargs = kwargs.pop(aspec.keywords)
if not isinstance(nkwargs, dict):
msg = "'{0}' must be a dict."
ret["comment"] = msg.format(aspec.keywords)
ret["result"] = False
return ret
try:
if aspec.keywords:
mret = __salt__[name](*args, **nkwargs)
else:
mret = __salt__[name](*args)
except Exception as e: # pylint: disable=broad-except
ret["comment"] = "Module function {0} threw an exception. Exception: {1}".format(
name, e)
ret["result"] = False
return ret
else:
if mret is not None or mret is not {}:
ret["changes"]["ret"] = mret
if "returner" in kwargs:
ret_ret = {
"id": __opts__["id"],
"ret": mret,
"fun": name,
"jid": salt.utils.jid.gen_jid(__opts__),
}
returners = salt.loader.returners(__opts__, __salt__)
if kwargs["returner"] in returners:
returners[kwargs["returner"]](ret_ret)
ret["comment"] = "Module function {0} executed".format(name)
ret["result"] = _get_result(mret, ret["changes"])
return ret
def __random_string(self, size=6):
return ''.join(
random.choice(string.ascii_uppercase + string.digits)
for x in range(size))
def present(name,
skip_translate=None,
ignore_collisions=False,
validate_ip_addrs=True,
containers=None,
reconnect=True,
**kwargs):
'''
.. versionchanged:: 2018.3.0
Support added for network configuration options other than ``driver``
and ``driver_opts``, as well as IPAM configuration.
Ensure that a network is present
.. note::
This state supports all arguments for network and IPAM pool
configuration which are available for the release of docker-py
installed on the minion. For that reason, the arguments described below
in the :ref:`NETWORK CONFIGURATION
<salt-states-docker-network-present-netconf>` and :ref:`IP ADDRESS
MANAGEMENT (IPAM) <salt-states-docker-network-present-ipam>` sections
may not accurately reflect what is available on the minion. The
:py:func:`docker.get_client_args
<salt.modules.dockermod.get_client_args>` function can be used to check
the available arguments for the installed version of docker-py (they
are found in the ``network_config`` and ``ipam_config`` sections of the
return data), but Salt will not prevent a user from attempting to use
an argument which is unsupported in the release of Docker which is
installed. In those cases, network creation be attempted but will fail.
name
Network name
skip_translate
This function translates Salt SLS input into the format which
docker-py expects. However, in the event that Salt's translation logic
fails (due to potential changes in the Docker Remote API, or to bugs in
the translation code), this argument can be used to exert granular
control over which arguments are translated and which are not.
Pass this argument as a comma-separated list (or Python list) of
arguments, and translation for each passed argument name will be
skipped. Alternatively, pass ``True`` and *all* translation will be
skipped.
Skipping tranlsation allows for arguments to be formatted directly in
the format which docker-py expects. This allows for API changes and
other issues to be more easily worked around. See the following links
for more information:
- `docker-py Low-level API`_
- `Docker Engine API`_
.. versionadded:: 2018.3.0
.. _`docker-py Low-level API`: http://docker-py.readthedocs.io/en/stable/api.html#docker.api.container.ContainerApiMixin.create_container
.. _`Docker Engine API`: https://docs.docker.com/engine/api/v1.33/#operation/ContainerCreate
ignore_collisions : False
Since many of docker-py's arguments differ in name from their CLI
counterparts (with which most Docker users are more familiar), Salt
detects usage of these and aliases them to the docker-py version of
that argument. However, if both the alias and the docker-py version of
the same argument (e.g. ``options`` and ``driver_opts``) are used, an error
will be raised. Set this argument to ``True`` to suppress these errors
and keep the docker-py version of the argument.
.. versionadded:: 2018.3.0
validate_ip_addrs : True
For parameters which accept IP addresses/subnets as input, validation
will be performed. To disable, set this to ``False``.
.. versionadded:: 2018.3.0
containers
A list of containers which should be connected to this network.
.. note::
As of the 2018.3.0 release, this is not the recommended way of
managing a container's membership in a network, for a couple
reasons:
1. It does not support setting static IPs, aliases, or links in the
container's IP configuration.
2. If a :py:func:`docker_container.running
<salt.states.docker_container.running>` state replaces a
container, it will not be reconnected to the network until the
``docker_network.present`` state is run again. Since containers
often have ``require`` requisites to ensure that the network
is present, this means that the ``docker_network.present`` state
ends up being run *before* the :py:func:`docker_container.running
<salt.states.docker_container.running>`, leaving the container
unattached at the end of the Salt run.
For these reasons, it is recommended to use
:ref:`docker_container.running's network management support
<salt-states-docker-container-network-management>`.
reconnect : True
If ``containers`` is not used, and the network is replaced, then Salt
will keep track of the containers which were connected to the network
and reconnect them to the network after it is replaced. Salt will first
attempt to reconnect using the same IP the container had before the
network was replaced. If that fails (for instance, if the network was
replaced because the subnet was modified), then the container will be
reconnected without an explicit IP address, and its IP will be assigned
by Docker.
Set this option to ``False`` to keep Salt from trying to reconnect
containers. This can be useful in some cases when :ref:`managing static
IPs in docker_container.running
<salt-states-docker-container-network-management>`. For instance, if a
network's subnet is modified, it is likely that the static IP will need
to be updated in the ``docker_container.running`` state as well. When
the network is replaced, the initial reconnect attempt would fail, and
the container would be reconnected with an automatically-assigned IP
address. Then, when the ``docker_container.running`` state executes, it
would disconnect the network *again* and reconnect using the new static
IP. Disabling the reconnect behavior in these cases would prevent the
unnecessary extra reconnection.
.. versionadded:: 2018.3.0
.. _salt-states-docker-network-present-netconf:
**NETWORK CONFIGURATION ARGUMENTS**
driver
Network driver
.. code-block:: yaml
mynet:
docker_network.present:
- driver: macvlan
driver_opts (or *driver_opt*, or *options*)
Options for the network driver. Either a dictionary of option names and
values or a Python list of strings in the format ``varname=value``. The
below three examples are equivalent:
.. code-block:: yaml
mynet:
docker_network.present:
- driver: macvlan
- driver_opts: macvlan_mode=bridge,parent=eth0
.. code-block:: yaml
mynet:
docker_network.present:
- driver: macvlan
- driver_opts:
- macvlan_mode=bridge
- parent=eth0
.. code-block:: yaml
mynet:
docker_network.present:
- driver: macvlan
- driver_opts:
- macvlan_mode: bridge
- parent: eth0
The options can also simply be passed as a dictionary, though this can
be error-prone due to some :ref:`idiosyncrasies <yaml-idiosyncrasies>`
with how PyYAML loads nested data structures:
.. code-block:: yaml
mynet:
docker_network.present:
- driver: macvlan
- driver_opts:
macvlan_mode: bridge
parent: eth0
check_duplicate : True
If ``True``, checks for networks with duplicate names. Since networks
are primarily keyed based on a random ID and not on the name, and
network name is strictly a user-friendly alias to the network which is
uniquely identified using ID, there is no guaranteed way to check for
duplicates. This option providess a best effort, checking for any
networks which have the same name, but it is not guaranteed to catch
all name collisions.
.. code-block:: yaml
mynet:
docker_network.present:
- check_duplicate: False
internal : False
If ``True``, restricts external access to the network
.. code-block:: yaml
mynet:
docker_network.present:
- internal: True
labels
Add metadata to the network. Labels can be set both with and without
values, and labels with values can be passed either as ``key=value`` or
``key: value`` pairs. For example, while the below would be very
confusing to read, it is technically valid, and demonstrates the
different ways in which labels can be passed:
.. code-block:: yaml
mynet:
docker_network.present:
- labels:
- foo
- bar=baz
- hello: world
The labels can also simply be passed as a YAML dictionary, though this
can be error-prone due to some :ref:`idiosyncrasies
<yaml-idiosyncrasies>` with how PyYAML loads nested data structures:
.. code-block:: yaml
foo:
docker_network.present:
- labels:
foo: ''
bar: baz
hello: world
.. versionchanged:: 2018.3.0
Methods for specifying labels can now be mixed. Earlier releases
required either labels with or without values.
enable_ipv6 (or *ipv6*) : False
Enable IPv6 on the network
.. code-block:: yaml
mynet:
docker_network.present:
- enable_ipv6: True
.. note::
While it should go without saying, this argument must be set to
``True`` to :ref:`configure an IPv6 subnet
<salt-states-docker-network-present-ipam>`. Also, if this option is
turned on without an IPv6 subnet explicitly configured, you will
get an error unless you have set up a fixed IPv6 subnet. Consult
the `Docker IPv6 docs`_ for information on how to do this.
.. _`Docker IPv6 docs`: https://docs.docker.com/v17.09/engine/userguide/networking/default_network/ipv6/
attachable : False
If ``True``, and the network is in the global scope, non-service
containers on worker nodes will be able to connect to the network.
.. code-block:: yaml
mynet:
docker_network.present:
- attachable: True
.. note::
This option cannot be reliably managed on CentOS 7. This is because
while support for this option was added in API version 1.24, its
value was not added to the inpsect results until API version 1.26.
The version of Docker which is available for CentOS 7 runs API
version 1.24, meaning that while Salt can pass this argument to the
API, it has no way of knowing the value of this config option in an
existing Docker network.
scope
Specify the network's scope (``local``, ``global`` or ``swarm``)
.. code-block:: yaml
mynet:
docker_network.present:
- scope: local
ingress : False
If ``True``, create an ingress network which provides the routing-mesh in
swarm mode
.. code-block:: yaml
mynet:
docker_network.present:
- ingress: True
.. _salt-states-docker-network-present-ipam:
**IP ADDRESS MANAGEMENT (IPAM)**
This state supports networks with either IPv4, or both IPv4 and IPv6. If
configuring IPv4, then you can pass the :ref:`IPAM pool arguments
<salt-states-docker-network-present-ipam-pool-arguments>` below as
individual arguments. However, if configuring IPv4 and IPv6, the arguments
must be passed as a list of dictionaries, in the ``ipam_pools`` argument
(click :ref:`here <salt-states-docker-network-present-ipam-examples>` for
some examples). `These docs`_ also have more information on these
arguments.
.. _`These docs`: http://docker-py.readthedocs.io/en/stable/api.html#docker.types.IPAMPool
*IPAM ARGUMENTS*
ipam_driver
IPAM driver to use, if different from the default one
.. code-block:: yaml
mynet:
docker_network.present:
- ipam_driver: foo
ipam_opts
Options for the IPAM driver. Either a dictionary of option names and
values or a Python list of strings in the format ``varname=value``. The
below three examples are equivalent:
.. code-block:: yaml
mynet:
docker_network.present:
- ipam_driver: foo
- ipam_opts: foo=bar,baz=qux
.. code-block:: yaml
mynet:
docker_network.present:
- ipam_driver: foo
- ipam_opts:
- foo=bar
- baz=qux
.. code-block:: yaml
mynet:
docker_network.present:
- ipam_driver: foo
- ipam_opts:
- foo: bar
- baz: qux
The options can also simply be passed as a dictionary, though this can
be error-prone due to some :ref:`idiosyncrasies <yaml-idiosyncrasies>`
with how PyYAML loads nested data structures:
.. code-block:: yaml
mynet:
docker_network.present:
- ipam_driver: macvlan
- ipam_opts:
foo: bar
baz: qux
.. _salt-states-docker-network-present-ipam-pool-arguments:
*IPAM POOL ARGUMENTS*
subnet
Subnet in CIDR format that represents a network segment
iprange (or *ip_range*)
Allocate container IP from a sub-range within the subnet
Subnet in CIDR format that represents a network segment
gateway
IPv4 or IPv6 gateway for the master subnet
aux_addresses (or *aux_address*)
A dictionary of mapping container names to IP addresses which should be
allocated for them should they connect to the network. Either a
dictionary of option names and values or a Python list of strings in
the format ``host=ipaddr``.
.. _salt-states-docker-network-present-ipam-examples:
*IPAM CONFIGURATION EXAMPLES*
Below is an example of an IPv4-only network (keep in mind that ``subnet``
is the only required argument).
.. code-block:: yaml
mynet:
docker_network.present:
- subnet: 10.0.20.0/24
- iprange: 10.0.20.128/25
- gateway: 10.0.20.254
- aux_addresses:
- foo.bar.tld: 10.0.20.50
- hello.world.tld: 10.0.20.51
.. note::
The ``aux_addresses`` can be passed differently, in the same way that
``driver_opts`` and ``ipam_opts`` can.
This same network could also be configured this way:
.. code-block:: yaml
mynet:
docker_network.present:
- ipam_pools:
- subnet: 10.0.20.0/24
iprange: 10.0.20.128/25
gateway: 10.0.20.254
aux_addresses:
foo.bar.tld: 10.0.20.50
hello.world.tld: 10.0.20.51
Here is an example of a mixed IPv4/IPv6 subnet.
.. code-block:: yaml
mynet:
docker_network.present:
- ipam_pools:
- subnet: 10.0.20.0/24
gateway: 10.0.20.1
- subnet: fe3f:2180:26:1::/123
gateway: fe3f:2180:26:1::1
'''
ret = {'name': name, 'changes': {}, 'result': False, 'comment': ''}
try:
network = __salt__['docker.inspect_network'](name)
except CommandExecutionError as exc:
msg = exc.__str__()
if '404' in msg:
# Network not present
network = None
else:
ret['comment'] = msg
return ret
# map container's IDs to names
to_connect = {}
missing_containers = []
stopped_containers = []
for cname in __utils__['args.split_input'](containers or []):
try:
cinfo = __salt__['docker.inspect_container'](cname)
except CommandExecutionError:
missing_containers.append(cname)
else:
try:
cid = cinfo['Id']
except KeyError:
missing_containers.append(cname)
else:
if not cinfo.get('State', {}).get('Running', False):
stopped_containers.append(cname)
else:
to_connect[cid] = {'Name': cname}
if missing_containers:
ret.setdefault('warnings', []).append(
'The following containers do not exist: {0}.'.format(
', '.join(missing_containers)))
if stopped_containers:
ret.setdefault('warnings', []).append(
'The following containers are not running: {0}.'.format(
', '.join(stopped_containers)))
# We might disconnect containers in the process of recreating the network,
# we'll need to keep track these containers so we can reconnect them later.
disconnected_containers = {}
try:
kwargs = __utils__['docker.translate_input'](
salt.utils.docker.translate.network,
skip_translate=skip_translate,
ignore_collisions=ignore_collisions,
validate_ip_addrs=validate_ip_addrs,
**__utils__['args.clean_kwargs'](**kwargs))
except Exception as exc:
ret['comment'] = exc.__str__()
return ret
# Separate out the IPAM config options and build the IPAM config dict
ipam_kwargs = {}
ipam_kwarg_names = ['ipam', 'ipam_driver', 'ipam_opts', 'ipam_pools']
ipam_kwarg_names.extend(
__salt__['docker.get_client_args']('ipam_config')['ipam_config'])
for key in ipam_kwarg_names:
try:
ipam_kwargs[key] = kwargs.pop(key)
except KeyError:
pass
if 'ipam' in ipam_kwargs:
if len(ipam_kwargs) > 1:
ret['comment'] = (
'Cannot mix the \'ipam\' argument with any of the IPAM config '
'arguments. See documentation for details.')
return ret
ipam_config = ipam_kwargs['ipam']
else:
ipam_pools = ipam_kwargs.pop('ipam_pools', ())
try:
ipam_config = __utils__['docker.create_ipam_config'](*ipam_pools,
**ipam_kwargs)
except Exception as exc:
ret['comment'] = exc.__str__()
return ret
# We'll turn this off if we decide below that creating the network is not
# necessary.
create_network = True
if network is not None:
log.debug('Docker network \'%s\' already exists', name)
# Set the comment now to say that it already exists, if we need to
# recreate the network with new config we'll update the comment later.
ret['comment'] = ('Network \'{0}\' already exists, and is configured '
'as specified'.format(name))
log.trace('Details of docker network \'%s\': %s', name, network)
temp_net_name = ''.join(
random.choice(string.ascii_lowercase) for _ in range(20))
try:
# When using enable_ipv6, you *must* provide a subnet. But we don't
# care about the subnet when we make our temp network, we only care
# about the non-IPAM values in the network. And we also do not want
# to try some hacky workaround where we choose a small IPv6 subnet
# to pass when creating the temp network, that may end up
# overlapping with a large IPv6 subnet already in use by Docker.
# So, for purposes of comparison we will create the temp network
# with enable_ipv6=False and then munge the inspect results before
# performing the comparison. Note that technically it is not
# required that one specify both v4 and v6 subnets when creating a
# network, but not specifying IPv4 makes it impossible for us to
# reliably compare the SLS input to the existing network, as we
# wouldng't know if the IPv4 subnet in the existing network was
# explicitly configured or was automatically assigned by Docker.
enable_ipv6 = kwargs.pop('enable_ipv6', None)
__salt__['docker.create_network'](
temp_net_name,
skip_translate=True, # No need to translate (already did)
enable_ipv6=False,
**kwargs)
except CommandExecutionError as exc:
ret['comment'] = (
'Failed to create temp network for comparison: {0}'.format(
exc.__str__()))
return ret
else:
# Replace the value so we can use it later
if enable_ipv6 is not None:
kwargs['enable_ipv6'] = enable_ipv6
try:
try:
temp_net_info = __salt__['docker.inspect_network'](
temp_net_name)
except CommandExecutionError as exc:
ret['comment'] = 'Failed to inspect temp network: {0}'.format(
exc.__str__())
return ret
else:
temp_net_info['EnableIPv6'] = bool(enable_ipv6)
# Replace the IPAM configuration in the temp network with the IPAM
# config dict we created earlier, for comparison purposes. This is
# necessary because we cannot create two networks that have
# overlapping subnets (the Docker Engine will throw an error).
temp_net_info['IPAM'] = ipam_config
existing_pool_count = len(network['IPAM']['Config'])
desired_pool_count = len(temp_net_info['IPAM']['Config'])
is_default_pool = lambda x: True \
if sorted(x) == ['Gateway', 'Subnet'] \
else False
if desired_pool_count == 0 \
and existing_pool_count == 1 \
and is_default_pool(network['IPAM']['Config'][0]):
# If we're not explicitly configuring an IPAM pool, then we
# don't care what the subnet is. Docker networks created with
# no explicit IPAM configuration are assigned a single IPAM
# pool containing just a subnet and gateway. If the above if
# statement resolves as True, then we know that both A) we
# aren't explicitly configuring IPAM, and B) the existing
# network appears to be one that was created without an
# explicit IPAM configuration (since it has the default pool
# config values). Of course, it could be possible that the
# existing network was created with a single custom IPAM pool,
# with just a subnet and gateway. But even if this was the
# case, the fact that we aren't explicitly enforcing IPAM
# configuration means we don't really care what the existing
# IPAM configuration is. At any rate, to avoid IPAM differences
# when comparing the existing network to the temp network, we
# need to clear the existing network's IPAM configuration.
network['IPAM']['Config'] = []
changes = __salt__['docker.compare_networks'](
network, temp_net_info, ignore='Name,Id,Created,Containers')
if not changes:
# No changes to the network, so we'll be keeping the existing
# network and at most just connecting containers to it.
create_network = False
else:
ret['changes'][name] = changes
if __opts__['test']:
ret['result'] = None
ret['comment'] = 'Network would be recreated with new config'
return ret
if network['Containers']:
# We've removed the network, so there are now no containers
# attached to it. However, once we recreate the network
# with the new configuration we may need to reconnect the
# containers that were previously connected. Even if we're
# not reconnecting, we still need to track the containers
# so that we can report on which were disconnected.
disconnected_containers = copy.deepcopy(
network['Containers'])
if not containers and reconnect:
# Grab the links and aliases from each connected
# container so that we have them when we attempt to
# reconnect later
for cid in disconnected_containers:
try:
cinfo = __salt__['docker.inspect_container'](
cid)
netinfo = cinfo['NetworkSettings']['Networks'][
name]
# Links and Aliases will be None if not
# explicitly set, hence using "or" instead of
# placing the empty list inside the dict.get
net_links = netinfo.get('Links') or []
net_aliases = netinfo.get('Aliases') or []
if net_links:
disconnected_containers[cid][
'Links'] = net_links
if net_aliases:
disconnected_containers[cid][
'Aliases'] = net_aliases
except (CommandExecutionError, KeyError,
ValueError):
continue
remove_result = _remove_network(network)
if not remove_result['result']:
return remove_result
# Replace the Containers key with an empty dict so that when we
# check for connnected containers below, we correctly see that
# there are none connected.
network['Containers'] = {}
finally:
try:
__salt__['docker.remove_network'](temp_net_name)
except CommandExecutionError as exc:
ret.setdefault('warnings', []).append(
'Failed to remove temp network \'{0}\': {1}.'.format(
temp_net_name, exc.__str__()))
if create_network:
log.debug('Network \'%s\' will be created', name)
if __opts__['test']:
# NOTE: if the container already existed and needed to be
# recreated, and we were in test mode, we would have already exited
# above with a comment about the network needing to be recreated.
# So, even though the below block to create the network would be
# executed to create the network both when it's being recreated and
# when it's being created for the first time, the below comment is
# still accurate.
ret['result'] = None
ret['comment'] = 'Network will be created'
return ret
kwargs['ipam'] = ipam_config
try:
__salt__['docker.create_network'](
name,
skip_translate=True, # No need to translate (already did)
**kwargs)
except Exception as exc:
ret['comment'] = 'Failed to create network \'{0}\': {1}'.format(
name, exc.__str__())
return ret
else:
action = 'recreated' if network is not None else 'created'
ret['changes'][action] = True
ret['comment'] = 'Network \'{0}\' {1}'.format(
name, 'created'
if network is None else 'was replaced with updated config')
# Make sure the "Containers" key exists for logic below
network = {'Containers': {}}
# If no containers were specified in the state but we have disconnected
# some in the process of recreating the network, we should reconnect those
# containers.
if containers is None and reconnect and disconnected_containers:
to_connect = disconnected_containers
# Don't try to connect any containers which are already connected. If we
# created/re-created the network, then network['Containers'] will be empty
# and no containers will be deleted from the to_connect dict (the result
# being that we will reconnect all containers in the to_connect dict).
# list() is used here because we will potentially be modifying the
# dictionary during iteration.
for cid in list(to_connect):
if cid in network['Containers']:
del to_connect[cid]
errors = []
if to_connect:
for cid, connect_info in six.iteritems(to_connect):
connect_kwargs = {}
if cid in disconnected_containers:
for key_name, arg_name in (('IPv4Address', 'ipv4_address'),
('IPV6Address', 'ipv6_address'),
('Links', 'links'), ('Aliases',
'aliases')):
try:
connect_kwargs[arg_name] = connect_info[key_name]
except (KeyError, AttributeError):
continue
else:
if key_name.endswith('Address'):
connect_kwargs[arg_name] = \
connect_kwargs[arg_name].rsplit('/', 1)[0]
try:
__salt__['docker.connect_container_to_network'](
cid, name, **connect_kwargs)
except CommandExecutionError as exc:
if not connect_kwargs:
errors.append(exc.__str__())
else:
# We failed to reconnect with the container's old IP
# configuration. Reconnect using automatic IP config.
try:
__salt__['docker.connect_container_to_network'](cid,
name)
except CommandExecutionError as exc:
errors.append(exc.__str__())
else:
ret['changes'].setdefault(
'reconnected' if cid in disconnected_containers
else 'connected', []).append(connect_info['Name'])
else:
ret['changes'].setdefault(
'reconnected' if cid in disconnected_containers else
'connected', []).append(connect_info['Name'])
if errors:
if ret['comment']:
ret['comment'] += '. '
ret['comment'] += '. '.join(errors) + '.'
else:
ret['result'] = True
# Figure out if we removed any containers as a result of replacing the
# network and did not reconnect them. We only would not have reconnected if
# a list of containers was passed in the "containers" argument, and there
# were containers connected to the network prior to its replacement which
# were not part of that list.
for cid, c_info in six.iteritems(disconnected_containers):
if cid not in to_connect:
ret['changes'].setdefault('disconnected',
[]).append(c_info['Name'])
return ret
def run(self):
'''
Execute the batch run
'''
args = [[],
self.opts['fun'],
self.opts['arg'],
self.opts['timeout'],
'list',
]
bnum = self.get_bnum()
# No targets to run
if not self.minions:
return
to_run = copy.deepcopy(self.minions)
active = []
ret = {}
iters = []
# wait the specified time before decide a job is actually done
bwait = self.opts.get('batch_wait', 0)
wait = []
if self.options:
show_jid = self.options.show_jid
show_verbose = self.options.verbose
else:
show_jid = False
show_verbose = False
# the minion tracker keeps track of responses and iterators
# - it removes finished iterators from iters[]
# - if a previously detected minion does not respond, its
# added with an empty answer to ret{} once the timeout is reached
# - unresponsive minions are removed from active[] to make
# sure that the main while loop finishes even with unresp minions
minion_tracker = {}
# We already know some minions didn't respond to the ping, so inform
# the user we won't be attempting to run a job on them
for down_minion in self.down_minions:
print_cli('Minion {0} did not respond. No job will be sent.'.format(down_minion))
# Iterate while we still have things to execute
while len(ret) < len(self.minions):
next_ = []
if bwait and wait:
self.__update_wait(wait)
if len(to_run) <= bnum - len(wait) and not active:
# last bit of them, add them all to next iterator
while to_run:
next_.append(to_run.pop())
else:
for i in range(bnum - len(active) - len(wait)):
if to_run:
minion_id = to_run.pop()
if isinstance(minion_id, dict):
next_.append(minion_id.keys()[0])
else:
next_.append(minion_id)
active += next_
args[0] = next_
if next_:
if not self.quiet:
print_cli('\nExecuting run on {0}\n'.format(next_))
# create a new iterator for this batch of minions
new_iter = self.local.cmd_iter_no_block(
*args,
raw=self.opts.get('raw', False),
ret=self.opts.get('return', ''),
show_jid=show_jid,
verbose=show_verbose,
**self.eauth)
# add it to our iterators and to the minion_tracker
iters.append(new_iter)
minion_tracker[new_iter] = {}
# every iterator added is 'active' and has its set of minions
minion_tracker[new_iter]['minions'] = next_
minion_tracker[new_iter]['active'] = True
else:
time.sleep(0.02)
parts = {}
# see if we found more minions
for ping_ret in self.ping_gen:
if ping_ret is None:
break
m = next(ping_ret.iterkeys())
if m not in self.minions:
self.minions.append(m)
to_run.append(m)
for queue in iters:
try:
# Gather returns until we get to the bottom
ncnt = 0
while True:
part = next(queue)
if part is None:
time.sleep(0.01)
ncnt += 1
if ncnt > 5:
break
continue
if self.opts.get('raw'):
parts.update({part['data']['id']: part})
if part['data']['id'] in minion_tracker[queue]['minions']:
minion_tracker[queue]['minions'].remove(part['data']['id'])
else:
print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(part['id']))
else:
parts.update(part)
for id in part.keys():
if id in minion_tracker[queue]['minions']:
minion_tracker[queue]['minions'].remove(id)
else:
print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(id))
except StopIteration:
# if a iterator is done:
# - set it to inactive
# - add minions that have not responded to parts{}
# check if the tracker contains the iterator
if queue in minion_tracker:
minion_tracker[queue]['active'] = False
# add all minions that belong to this iterator and
# that have not responded to parts{} with an empty response
for minion in minion_tracker[queue]['minions']:
if minion not in parts:
parts[minion] = {}
parts[minion]['ret'] = {}
for minion, data in six.iteritems(parts):
if minion in active:
active.remove(minion)
if bwait:
wait.append(datetime.now() + timedelta(seconds=bwait))
if self.opts.get('raw'):
ret[minion] = data
yield data
else:
ret[minion] = data
yield {minion: data}
if not self.quiet:
ret[minion] = data['ret']
data[minion] = data.pop('ret')
if 'out' in data:
out = data.pop('out')
else:
out = None
salt.output.display_output(
data,
out,
self.opts)
# remove inactive iterators from the iters list
for queue in minion_tracker:
# only remove inactive queues
if not minion_tracker[queue]['active'] and queue in iters:
iters.remove(queue)
# also remove the iterator's minions from the active list
for minion in minion_tracker[queue]['minions']:
if minion in active:
active.remove(minion)
if bwait:
wait.append(datetime.now() + timedelta(seconds=bwait))
def _connect(contact_points=None,
port=None,
cql_user=None,
cql_pass=None,
protocol_version=4):
'''
Connect to a Cassandra cluster.
:param contact_points: The Cassandra cluster addresses, can either be a string or a list of IPs.
:type contact_points: str or list of str
:param cql_user: The Cassandra user if authentication is turned on.
:type cql_user: str
:param cql_pass: The Cassandra user password if authentication is turned on.
:type cql_pass: str
:param port: The Cassandra cluster port, defaults to None.
:type port: int
:param protocol_version: Cassandra protocol version to use.
:type port: int
:return: The session and cluster objects.
:rtype: cluster object, session object
'''
# Lazy load the Cassandra cluster and session for this module by creating a
# cluster and session when cql_query is called the first time. Get the
# Cassandra cluster and session from this module's __context__ after it is
# loaded the first time cql_query is called.
#
# TODO: Call cluster.shutdown() when the module is unloaded on
# master/minion shutdown. Currently, Master.shutdown() and Minion.shutdown()
# do nothing to allow loaded modules to gracefully handle resources stored
# in __context__ (i.e. connection pools). This means that the the connection
# pool is orphaned and Salt relies on Cassandra to reclaim connections.
# Perhaps if Master/Minion daemons could be enhanced to call an "__unload__"
# function, or something similar for each loaded module, connection pools
# and the like can be gracefully reclaimed/shutdown.
if (__context__ and 'cassandra_cql_returner_cluster' in __context__
and 'cassandra_cql_returner_session' in __context__):
return __context__['cassandra_cql_returner_cluster'], __context__[
'cassandra_cql_returner_session']
else:
contact_points = _load_properties(property_name=contact_points,
config_option='cluster')
contact_points = contact_points if isinstance(
contact_points, list) else contact_points.split(',')
port = _load_properties(property_name=port,
config_option='port',
set_default=True,
default=9042)
cql_user = _load_properties(property_name=cql_user,
config_option='username',
set_default=True,
default="cassandra")
cql_pass = _load_properties(property_name=cql_pass,
config_option='password',
set_default=True,
default="cassandra")
protocol_version = _load_properties(property_name=protocol_version,
config_option='protocol_version',
set_default=True,
default=4)
try:
auth_provider = PlainTextAuthProvider(username=cql_user,
password=cql_pass)
ssl_opts = _get_ssl_opts()
if ssl_opts:
cluster = Cluster(contact_points,
port=port,
auth_provider=auth_provider,
ssl_options=ssl_opts,
protocol_version=protocol_version,
compression=True)
else:
cluster = Cluster(contact_points,
port=port,
auth_provider=auth_provider,
protocol_version=protocol_version,
compression=True)
for recontimes in range(1, 4):
try:
session = cluster.connect()
break
except OperationTimedOut:
log.warning(
'Cassandra cluster.connect timed out, try {0}'.format(
recontimes))
if recontimes >= 3:
raise
# TODO: Call cluster.shutdown() when the module is unloaded on shutdown.
__context__['cassandra_cql_returner_cluster'] = cluster
__context__['cassandra_cql_returner_session'] = session
__context__['cassandra_cql_prepared'] = {}
log.debug(
'Successfully connected to Cassandra cluster at {0}'.format(
contact_points))
return cluster, session
except TypeError:
pass
except (ConnectionException, ConnectionShutdown, NoHostAvailable):
log.error('Could not connect to Cassandra cluster at {0}'.format(
contact_points))
raise CommandExecutionError(
'ERROR: Could not connect to Cassandra cluster.')
def subdict_match(data,
expr,
delimiter=DEFAULT_TARGET_DELIM,
regex_match=False,
exact_match=False):
'''
Check for a match in a dictionary using a delimiter character to denote
levels of subdicts, and also allowing the delimiter character to be
matched. Thus, 'foo:bar:baz' will match data['foo'] == 'bar:baz' and
data['foo']['bar'] == 'baz'. The latter would take priority over the
former, as more deeply-nested matches are tried first.
'''
def _match(target, pattern, regex_match=False, exact_match=False):
# The reason for using six.text_type first and _then_ using
# to_unicode as a fallback is because we want to eventually have
# unicode types for comparison below. If either value is numeric then
# six.text_type will turn it into a unicode string. However, if the
# value is a PY2 str type with non-ascii chars, then the result will be
# a UnicodeDecodeError. In those cases, we simply use to_unicode to
# decode it to unicode. The reason we can't simply use to_unicode to
# begin with is that (by design) to_unicode will raise a TypeError if a
# non-string/bytestring/bytearray value is passed.
try:
target = six.text_type(target).lower()
except UnicodeDecodeError:
target = salt.utils.stringutils.to_unicode(target).lower()
try:
pattern = six.text_type(pattern).lower()
except UnicodeDecodeError:
pattern = salt.utils.stringutils.to_unicode(pattern).lower()
if regex_match:
try:
return re.match(pattern, target)
except Exception:
log.error('Invalid regex \'%s\' in match', pattern)
return False
else:
return target == pattern if exact_match \
else fnmatch.fnmatch(target, pattern)
def _dict_match(target, pattern, regex_match=False, exact_match=False):
wildcard = pattern.startswith('*:')
if wildcard:
pattern = pattern[2:]
if pattern == '*':
# We are just checking that the key exists
return True
elif pattern in target:
# We might want to search for a key
return True
elif subdict_match(target,
pattern,
regex_match=regex_match,
exact_match=exact_match):
return True
if wildcard:
for key in target:
if isinstance(target[key], dict):
if _dict_match(target[key],
pattern,
regex_match=regex_match,
exact_match=exact_match):
return True
elif isinstance(target[key], list):
for item in target[key]:
if _match(item,
pattern,
regex_match=regex_match,
exact_match=exact_match):
return True
elif _match(target[key],
pattern,
regex_match=regex_match,
exact_match=exact_match):
return True
return False
splits = expr.split(delimiter)
num_splits = len(splits)
if num_splits == 1:
# Delimiter not present, this can't possibly be a match
return False
splits = expr.split(delimiter)
num_splits = len(splits)
if num_splits == 1:
# Delimiter not present, this can't possibly be a match
return False
# If we have 4 splits, then we have three delimiters. Thus, the indexes we
# want to use are 3, 2, and 1, in that order.
for idx in range(num_splits - 1, 0, -1):
key = delimiter.join(splits[:idx])
if key == '*':
# We are matching on everything under the top level, so we need to
# treat the match as the entire data being passed in
matchstr = expr
match = data
else:
matchstr = delimiter.join(splits[idx:])
match = traverse_dict_and_list(data, key, {}, delimiter=delimiter)
log.debug("Attempting to match '%s' in '%s' using delimiter '%s'",
matchstr, key, delimiter)
if match == {}:
continue
if isinstance(match, dict):
if _dict_match(match,
matchstr,
regex_match=regex_match,
exact_match=exact_match):
return True
continue
if isinstance(match, (list, tuple)):
# We are matching a single component to a single list member
for member in match:
if isinstance(member, dict):
if _dict_match(member,
matchstr,
regex_match=regex_match,
exact_match=exact_match):
return True
if _match(member,
matchstr,
regex_match=regex_match,
exact_match=exact_match):
return True
continue
if _match(match,
matchstr,
regex_match=regex_match,
exact_match=exact_match):
return True
return False
def _test_hwclock_sync(self):
"""
Check that hw and sw clocks are sync'd.
"""
if not self.run_function("system.has_settable_hwclock"):
return None
if not self._hwclock_has_compare():
return None
class CompareTimeout(BaseException):
pass
def _alrm_handler(sig, frame):
log.warning(
"hwclock --compare failed to produce output after 3 seconds")
raise CompareTimeout
for _ in range(2):
try:
orig_handler = signal.signal(signal.SIGALRM, _alrm_handler)
signal.alarm(3)
rpipeFd, wpipeFd = os.pipe()
log.debug("Comparing hwclock to sys clock")
with os.fdopen(rpipeFd, "r") as rpipe:
with os.fdopen(wpipeFd, "w") as wpipe:
with salt.utils.files.fopen(os.devnull, "r") as nulFd:
p = subprocess.Popen(
args=["hwclock", "--compare"],
stdin=nulFd,
stdout=wpipeFd,
stderr=subprocess.PIPE,
)
p.communicate()
# read header
rpipe.readline()
# read first time comparison
timeCompStr = rpipe.readline()
# stop
p.terminate()
timeComp = timeCompStr.split()
hwTime = float(timeComp[0])
swTime = float(timeComp[1])
diff = abs(hwTime - swTime)
self.assertTrue(
diff <= 2.0,
msg=("hwclock difference too big: " +
six.text_type(timeCompStr)),
)
break
except CompareTimeout:
p.terminate()
finally:
signal.alarm(0)
signal.signal(signal.SIGALRM, orig_handler)
else:
log.error("Failed to check hwclock sync")
def test_call_success(self):
'''
test module calling inside containers
'''
ret = None
docker_run_all_mock = MagicMock(
return_value={
'retcode': 0,
'stdout': '{"retcode": 0, "comment": "container cmd"}',
'stderr': 'err',
})
docker_copy_to_mock = MagicMock(
return_value={
'retcode': 0
})
docker_config_mock = MagicMock(
return_value=''
)
client = Mock()
client.put_archive = Mock()
get_client_mock = MagicMock(return_value=client)
context = {'docker.exec_driver': 'docker-exec'}
salt_dunder = {'config.option': docker_config_mock}
with patch.object(docker_mod, 'run_all', docker_run_all_mock), \
patch.object(docker_mod, 'copy_to', docker_copy_to_mock), \
patch.object(docker_mod, '_get_client', get_client_mock), \
patch.dict(docker_mod.__opts__, {'cachedir': '/tmp'}), \
patch.dict(docker_mod.__salt__, salt_dunder), \
patch.dict(docker_mod.__context__, context):
# call twice to verify tmp path later
for i in range(2):
ret = docker_mod.call('ID', 'test.arg', 1, 2, arg1='val1')
# Check that the directory is different each time
# [ call(name, [args]), ...
self.maxDiff = None
self.assertIn('mkdir', docker_run_all_mock.mock_calls[0][1][1])
self.assertIn('mkdir', docker_run_all_mock.mock_calls[4][1][1])
self.assertNotEqual(docker_run_all_mock.mock_calls[0][1][1],
docker_run_all_mock.mock_calls[4][1][1])
self.assertIn('salt-call', docker_run_all_mock.mock_calls[2][1][1])
self.assertIn('salt-call', docker_run_all_mock.mock_calls[6][1][1])
self.assertNotEqual(docker_run_all_mock.mock_calls[2][1][1],
docker_run_all_mock.mock_calls[6][1][1])
# check thin untar
self.assertIn('tarfile', docker_run_all_mock.mock_calls[1][1][1])
self.assertIn('tarfile', docker_run_all_mock.mock_calls[5][1][1])
self.assertNotEqual(docker_run_all_mock.mock_calls[1][1][1],
docker_run_all_mock.mock_calls[5][1][1])
# check directory cleanup
self.assertIn('rm -rf', docker_run_all_mock.mock_calls[3][1][1])
self.assertIn('rm -rf', docker_run_all_mock.mock_calls[7][1][1])
self.assertNotEqual(docker_run_all_mock.mock_calls[3][1][1],
docker_run_all_mock.mock_calls[7][1][1])
self.assertEqual({"retcode": 0, "comment": "container cmd"}, ret)
def diff(*args, **kwargs):
'''
Return the DIFFERENCE of the result sets returned by each matching minion
pool
.. versionadded:: 2014.7.0
These pools are determined from the aggregated and sorted results of
a salt command.
This command displays the "diffs" as a series of 2-way differences --
namely the difference between the FIRST displayed minion pool
(according to sort order) and EACH SUBSEQUENT minion pool result set.
Differences are displayed according to the Python ``difflib.unified_diff()``
as in the case of the salt execution module ``file.get_diff``.
This command is submitted via a salt runner using the general form::
salt-run survey.diff [survey_sort=up/down] <target>
<salt-execution-module> <salt-execution-module parameters>
Optionally accept a ``survey_sort=`` parameter. Default:
``survey_sort=down``
CLI Example #1: (Example to display the "differences of files")
.. code-block:: bash
salt-run survey.diff survey_sort=up "*" cp.get_file_str file:///etc/hosts
'''
# TODO: The salt execution module "cp.get_file_str file:///..." is a
# non-obvious way to display the differences between files using
# survey.diff . A more obvious method needs to be found or developed.
import difflib
bulk_ret = _get_pool_results(*args, **kwargs)
is_first_time = True
for k in bulk_ret:
print('minion pool :\n' '------------')
print(k['pool'])
print('pool size :\n' '----------')
print(' ' + str(len(k['pool'])))
if is_first_time:
is_first_time = False
print('pool result :\n' '------------')
print(' ' + bulk_ret[0]['result'])
print()
continue
outs = ('differences from "{0}" results :').format(
bulk_ret[0]['pool'][0])
print(outs)
print('-' * (len(outs) - 1))
from_result = bulk_ret[0]['result'].splitlines()
for i in range(0, len(from_result)):
from_result[i] += '\n'
to_result = k['result'].splitlines()
for i in range(0, len(to_result)):
to_result[i] += '\n'
outs = ''
outs += ''.join(
difflib.unified_diff(from_result,
to_result,
fromfile=bulk_ret[0]['pool'][0],
tofile=k['pool'][0],
n=0))
print(outs)
print()
return bulk_ret
def query(params=None,
setname=None,
requesturl=None,
location=None,
return_url=False,
return_root=False,
opts=None,
provider=None,
endpoint=None,
product='ec2',
sigver='2'):
'''
Perform a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
Regions and endpoints are documented at:
http://docs.aws.amazon.com/general/latest/gr/rande.html
Default ``product`` is ``ec2``. Valid ``product`` names are:
.. code-block: yaml
- autoscaling (Auto Scaling)
- cloudformation (CloudFormation)
- ec2 (Elastic Compute Cloud)
- elasticache (ElastiCache)
- elasticbeanstalk (Elastic BeanStalk)
- elasticloadbalancing (Elastic Load Balancing)
- elasticmapreduce (Elastic MapReduce)
- iam (Identity and Access Management)
- importexport (Import/Export)
- monitoring (CloudWatch)
- rds (Relational Database Service)
- simpledb (SimpleDB)
- sns (Simple Notification Service)
- sqs (Simple Queue Service)
'''
if params is None:
params = {}
if opts is None:
opts = {}
function = opts.get('function', (None, product))
providers = opts.get('providers', {})
if provider is None:
prov_dict = providers.get(function[1], {}).get(product, {})
if prov_dict:
driver = list(list(prov_dict.keys()))[0]
provider = providers.get(driver, product)
else:
prov_dict = providers.get(provider, {}).get(product, {})
service_url = prov_dict.get('service_url', 'amazonaws.com')
if not location:
location = get_location(opts, provider)
if endpoint is None:
if not requesturl:
endpoint = prov_dict.get(
'endpoint', '{0}.{1}.{2}'.format(product, location,
service_url))
requesturl = 'https://{0}/'.format(endpoint)
else:
endpoint = urlparse(requesturl).netloc
if endpoint == '':
endpoint_err = (
'Could not find a valid endpoint in the '
'requesturl: {0}. Looking for something '
'like https://some.aws.endpoint/?args').format(requesturl)
LOG.error(endpoint_err)
if return_url is True:
return {'error': endpoint_err}, requesturl
return {'error': endpoint_err}
LOG.debug('Using AWS endpoint: {0}'.format(endpoint))
method = 'GET'
aws_api_version = prov_dict.get(
'aws_api_version',
prov_dict.get('{0}_api_version'.format(product),
DEFAULT_AWS_API_VERSION))
if sigver == '4':
headers, requesturl = sig4(method,
endpoint,
params,
prov_dict,
aws_api_version,
location,
product,
requesturl=requesturl)
params_with_headers = {}
else:
params_with_headers = sig2(method, endpoint, params, prov_dict,
aws_api_version)
headers = {}
attempts = 5
while attempts > 0:
LOG.debug('AWS Request: {0}'.format(requesturl))
LOG.trace('AWS Request Parameters: {0}'.format(params_with_headers))
try:
result = requests.get(requesturl,
headers=headers,
params=params_with_headers)
LOG.debug('AWS Response Status Code: {0}'.format(
result.status_code))
LOG.trace('AWS Response Text: {0}'.format(result.text))
result.raise_for_status()
break
except requests.exceptions.HTTPError as exc:
root = ET.fromstring(exc.response.content)
data = xml.to_dict(root)
# check to see if we should retry the query
err_code = data.get('Errors', {}).get('Error', {}).get('Code', '')
if attempts > 0 and err_code and err_code in AWS_RETRY_CODES:
attempts -= 1
LOG.error('AWS Response Status Code and Error: [{0} {1}] {2}; '
'Attempts remaining: {3}'.format(
exc.response.status_code, exc, data, attempts))
# Wait a bit before continuing to prevent throttling
time.sleep(2)
continue
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data))
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
else:
LOG.error('AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data))
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
response = result.text
root = ET.fromstring(response)
items = root[1]
if return_root is True:
items = root
if setname:
if sys.version_info < (2, 7):
children_len = len(root.getchildren())
else:
children_len = len(root)
for item in range(0, children_len):
comps = root[item].tag.split('}')
if comps[1] == setname:
items = root[item]
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
return ret
def wrap(cls):
result = None
for attempt in range(1, times + 1):
log.info('%s test run %d of %s times', cls, attempt, times)
caller(cls)
return cls
def managed(name,
dns_proto=None,
dns_servers=None,
ip_proto=None,
ip_addrs=None,
gateway=None,
enabled=True,
**kwargs):
"""
Ensure that the named interface is configured properly.
Args:
name (str):
The name of the interface to manage
dns_proto (str): None
Set to ``static`` and use the ``dns_servers`` parameter to provide a
list of DNS nameservers. set to ``dhcp`` to use DHCP to get the DNS
servers.
dns_servers (list): None
A list of static DNS servers. To clear the list of DNS servers pass
an empty list (``[]``). ``None`` will make no changes.
ip_proto (str): None
Set to ``static`` and use the ``ip_addrs`` and (optionally)
``gateway`` parameters to provide a list of static IP addresses and
the default gateway. Set to ``dhcp`` to use DHCP.
ip_addrs (list): None
A list of static IP addresses with netmask flag, ie: 192.168.0.11/24
gateway (str): None
The gateway to set for the interface
enabled (bool): True
Set to ``False`` to ensure that this interface is disabled.
Returns:
dict: A dictionary of old and new settings
Example:
.. code-block:: yaml
Ethernet1:
network.managed:
- dns_proto: static
- dns_servers:
- 8.8.8.8
- 8.8.8.4
- ip_proto: static
- ip_addrs:
- 192.168.0.100/24
Clear DNS entries example:
.. code-block:: yaml
Ethernet1:
network.managed:
- dns_proto: static
- dns_servers: []
- ip_proto: dhcp
"""
ret = {
"name": name,
"changes": {},
"result": True,
"comment": "Interface '{0}' is up to date".format(name),
}
dns_proto = six.text_type(dns_proto).lower()
ip_proto = six.text_type(ip_proto).lower()
errors = []
if dns_proto not in __VALID_PROTO:
ret["result"] = False
errors.append("dns_proto must be one of the following: {0}".format(
", ".join(__VALID_PROTO)))
if ip_proto not in __VALID_PROTO:
errors.append("ip_proto must be one of the following: {0}".format(
", ".join(__VALID_PROTO)))
if errors:
ret["result"] = False
ret["comment"] = "\n".join(errors)
return ret
try:
currently_enabled = __salt__["ip.is_enabled"](name)
except CommandExecutionError:
currently_enabled = False
if not enabled:
if currently_enabled:
if __opts__["test"]:
ret["result"] = None
ret["comment"] = "Interface '{0}' will be disabled".format(
name)
else:
ret["result"] = __salt__["ip.disable"](name)
if not ret["result"]:
ret["comment"] = "Failed to disable interface '{0}'".format(
name)
else:
ret["comment"] += " (already disabled)"
return ret
else:
if not currently_enabled:
if __opts__["test"]:
ret["result"] = None
ret["comment"] = "Interface '{0}' will be enabled".format(name)
else:
if not __salt__["ip.enable"](name):
ret["result"] = False
ret["comment"] = ("Failed to enable interface '{0}' to "
"make changes".format(name))
return ret
errors = _validate(dns_proto, dns_servers, ip_proto, ip_addrs, gateway)
if errors:
ret["result"] = False
ret["comment"] = ("The following SLS configuration errors were "
"detected:\n- {0}".format("\n- ".join(errors)))
return ret
old = __salt__["ip.get_interface"](name)
if not old:
ret["result"] = False
ret["comment"] = ("Unable to get current configuration for "
"interface '{0}'".format(name))
return ret
changes = _changes(old, dns_proto, dns_servers, ip_proto, ip_addrs,
gateway)
# If dns_servers is the default `None` make no changes
# To clear the list, pass an empty dict
if str(dns_servers).lower() == "none":
changes.pop("dns_servers", None)
if not changes:
return ret
if __opts__["test"]:
comments = []
if "dns_proto" in changes:
comments.append("DNS protocol will be changed to: {0}".format(
changes["dns_proto"]))
if dns_proto == "static" and "dns_servers" in changes:
if len(changes["dns_servers"]) == 0:
comments.append("The list of DNS servers will be cleared")
else:
comments.append(
"DNS servers will be set to the following: {0}".format(
", ".join(changes["dns_servers"])))
if "ip_proto" in changes:
comments.append("IP protocol will be changed to: {0}".format(
changes["ip_proto"]))
if ip_proto == "static":
if "ip_addrs" in changes:
comments.append(
"IP addresses will be set to the following: {0}".
format(", ".join(changes["ip_addrs"])))
if "gateway" in changes:
if changes["gateway"] is None:
comments.append("Default gateway will be removed")
else:
comments.append(
"Default gateway will be set to {0}".format(
changes["gateway"]))
ret["result"] = None
ret["comment"] = ("The following changes will be made to "
"interface '{0}':\n- {1}".format(
name, "\n- ".join(comments)))
return ret
if changes.get("dns_proto") == "dhcp":
__salt__["ip.set_dhcp_dns"](name)
elif "dns_servers" in changes:
if len(changes["dns_servers"]) == 0:
# To clear the list of DNS servers you have to pass []. Later
# changes gets passed like *args and a single empty list is
# converted to an empty tuple. So, you have to add [] here
changes["dns_servers"] = [[]]
__salt__["ip.set_static_dns"](name, *changes["dns_servers"])
if changes.get("ip_proto") == "dhcp":
__salt__["ip.set_dhcp_ip"](name)
elif (changes.get("ip_addrs") or changes.get("gateway")
or changes.get("ip_proto") == "static"):
if changes.get("gateway") and not changes.get("ip_addrs"):
changes["ip_addrs"] = ip_addrs
if changes.get(
"ip_proto") == "static" and not changes.get("ip_addrs"):
changes["ip_addrs"] = ip_addrs
for idx in range(len(changes["ip_addrs"])):
if idx == 0:
__salt__["ip.set_static_ip"](name,
changes["ip_addrs"][idx],
gateway=gateway,
append=False)
else:
__salt__["ip.set_static_ip"](name,
changes["ip_addrs"][idx],
gateway=None,
append=True)
new = __salt__["ip.get_interface"](name)
ret["changes"] = salt.utils.data.compare_dicts(old, new)
if _changes(new, dns_proto, dns_servers, ip_proto, ip_addrs, gateway):
ret["result"] = False
ret["comment"] = ("Failed to set desired configuration settings "
"for interface '{0}'".format(name))
else:
ret["comment"] = "Successfully updated configuration for " "interface '{0}'".format(
name)
return ret
def create_ca_signed_cert(ca_name, CN, days=365, cacert_path=None, cert_filename=None, digest='sha256'):
'''
Create a Certificate (CERT) signed by a named Certificate Authority (CA)
If the certificate file already exists, the function just returns assuming
the CERT already exists.
The CN *must* match an existing CSR generated by create_csr. If it
does not, this method does nothing.
ca_name
name of the CA
CN
common name matching the certificate signing request
days
number of days certificate is valid, default is 365 (1 year)
cacert_path
absolute path to ca certificates root directory
cert_filename
alternative filename for the certificate, useful when using special characters in the CN
digest
The message digest algorithm. Must be a string describing a digest
algorithm supported by OpenSSL (by EVP_get_digestbyname, specifically).
For example, "md5" or "sha1". Default: 'sha256'
If the following values were set:
.. code-block:: text
ca.cert_base_path='/etc/pki'
ca_name='koji'
CN='test.egavas.org'
the resulting signed certificate would be written in the following
location:
.. code-block:: text
/etc/pki/koji/certs/test.egavas.org.crt
CLI Example:
.. code-block:: bash
salt '*' tls.create_ca_signed_cert test localhost
'''
set_ca_path(cacert_path)
if not cert_filename:
cert_filename = CN
if os.path.exists(
'{0}/{1}/{2}.crt'.format(cert_base_path(),
ca_name, cert_filename)
):
return 'Certificate "{0}" already exists'.format(cert_filename)
try:
maybe_fix_ssl_version(ca_name)
with salt.utils.fopen('{0}/{1}/{2}_ca_cert.crt'.format(cert_base_path(),
ca_name,
ca_name)) as fhr:
ca_cert = OpenSSL.crypto.load_certificate(
OpenSSL.crypto.FILETYPE_PEM, fhr.read()
)
with salt.utils.fopen('{0}/{1}/{2}_ca_cert.key'.format(cert_base_path(),
ca_name,
ca_name)) as fhr:
ca_key = OpenSSL.crypto.load_privatekey(
OpenSSL.crypto.FILETYPE_PEM,
fhr.read()
)
except IOError:
return 'There is no CA named "{0}"'.format(ca_name)
try:
with salt.utils.fopen('{0}/{1}/certs/{2}.csr'.format(cert_base_path(),
ca_name,
cert_filename)) as fhr:
req = OpenSSL.crypto.load_certificate_request(
OpenSSL.crypto.FILETYPE_PEM,
fhr.read()
)
except IOError:
return 'There is no CSR that matches the CN "{0}"'.format(cert_filename)
exts = []
try:
# see: http://bazaar.launchpad.net/~exarkun/pyopenssl/master/revision/189
# support is there from quite a long time, but without API
# so we mimic the newly get_extensions method present in ultra
# recent pyopenssl distros
native_exts_obj = OpenSSL._util.lib.X509_REQ_get_extensions(req._req)
for i in range(OpenSSL._util.lib.sk_X509_EXTENSION_num(native_exts_obj)):
ext = OpenSSL.crypto.X509Extension.__new__(OpenSSL.crypto.X509Extension)
ext._extension = OpenSSL._util.lib.sk_X509_EXTENSION_value(native_exts_obj, i)
exts.append(ext)
except Exception:
log.error('Support for extensions is not available, upgrade PyOpenSSL')
cert = OpenSSL.crypto.X509()
cert.set_version(2)
cert.set_subject(req.get_subject())
cert.gmtime_adj_notBefore(0)
cert.gmtime_adj_notAfter(int(days) * 24 * 60 * 60)
if exts:
cert.add_extensions(exts)
cert.set_serial_number(_new_serial(ca_name, CN))
cert.set_issuer(ca_cert.get_subject())
cert.set_pubkey(req.get_pubkey())
cert.sign(ca_key, digest)
with salt.utils.fopen('{0}/{1}/certs/{2}.crt'.format(cert_base_path(),
ca_name,
cert_filename), 'w+') as crt:
crt.write(
OpenSSL.crypto.dump_certificate(
OpenSSL.crypto.FILETYPE_PEM,
cert
)
)
_write_cert_to_database(ca_name, cert)
return ('Created Certificate for "{0}": '
'"{1}/{2}/certs/{3}.crt"').format(
CN,
cert_base_path(),
ca_name,
cert_filename
)
def keys_present(name, number, save_dir, region=None, key=None, keyid=None, profile=None):
'''
.. versionadded:: 2015.8
Ensure the IAM access keys are present.
name (string)
The name of the new user.
number (int)
Number of keys that user should have.
save_dir (string)
The directory that the key/keys will be saved. Keys are saved to a file named according
to the username privided.
region (string)
Region to connect to.
key (string)
Secret key to be used.
keyid (string)
Access key to be used.
profile (dict)
A dict with region, key and keyid, or a pillar key (string)
that contains a dict with region, key and keyid.
'''
ret = {'name': name, 'result': True, 'comment': '', 'changes': {}}
if not __salt__['boto_iam.get_user'](name, region, key, keyid, profile):
ret['result'] = False
ret['comment'] = 'IAM User {0} does not exist.'.format(name)
return ret
if not isinstance(number, int):
ret['comment'] = 'The number of keys must be an integer.'
ret['result'] = False
return ret
if not os.path.isdir(save_dir):
ret['comment'] = 'The directory {0} does not exist.'.format(save_dir)
ret['result'] = False
return ret
keys = __salt__['boto_iam.get_all_access_keys'](user_name=name, region=region, key=key,
keyid=keyid, profile=profile)
if isinstance(keys, str):
log.debug('keys are : false {0}'.format(keys))
error, message = _get_error(keys)
ret['comment'] = 'Could not get keys.\n{0}\n{1}'.format(error, message)
ret['result'] = False
return ret
keys = keys['list_access_keys_response']['list_access_keys_result']['access_key_metadata']
log.debug('Keys are : {0}.'.format(keys))
if len(keys) >= number:
ret['comment'] = 'The number of keys exist for user {0}'.format(name)
ret['result'] = True
return ret
if __opts__['test']:
ret['comment'] = 'Access key is set to be created for {0}.'.format(name)
ret['result'] = None
return ret
new_keys = {}
for i in range(number-len(keys)):
created = __salt__['boto_iam.create_access_key'](name, region, key, keyid, profile)
if isinstance(created, str):
error, message = _get_error(created)
ret['comment'] = 'Could not create keys.\n{0}\n{1}'.format(error, message)
ret['result'] = False
return ret
log.debug('Created is : {0}'.format(created))
response = 'create_access_key_response'
result = 'create_access_key_result'
new_keys['key-{0}'.format(i)] = created[response][result]['access_key']['access_key_id']
new_keys['key_id-{0}'.format(i)] = created[response][result]['access_key']['secret_access_key']
try:
with salt.utils.fopen('{0}/{1}'.format(save_dir, name), 'a') as _wrf:
for key_id, access_key in new_keys.items():
_wrf.write('{0}\n{1}\n'.format(key_id, access_key))
ret['comment'] = 'Keys have been written to file {0}/{1}.'.format(save_dir, name)
ret['result'] = True
ret['changes'] = new_keys
return ret
except IOError:
ret['comment'] = 'Could not write to file {0}/{1}.'.format(save_dir, name)
ret['result'] = False
return ret
def read_certificate(certificate):
'''
Returns a dict containing details of a certificate. Input can be a PEM string or file path.
certificate:
The certificate to be read. Can be a path to a certificate file, or a string containing
the PEM formatted text of the certificate.
CLI Example:
.. code-block:: bash
salt '*' x509.read_certificate /etc/pki/mycert.crt
'''
if isinstance(certificate, M2Crypto.X509.X509):
cert = certificate
else:
cert = _get_certificate_obj(certificate)
ret = {
# X509 Verison 3 has a value of 2 in the field.
# Version 2 has a value of 1.
# https://tools.ietf.org/html/rfc5280#section-4.1.2.1
'Version':
cert.get_version() + 1,
# Get size returns in bytes. The world thinks of key sizes in bits.
'Key Size':
cert.get_pubkey().size() * 8,
'Serial Number':
_dec2hex(cert.get_serial_number()),
'SHA-256 Finger Print':
_pretty_hex(cert.get_fingerprint(md='sha256')),
'MD5 Finger Print':
_pretty_hex(cert.get_fingerprint(md='md5')),
'SHA1 Finger Print':
_pretty_hex(cert.get_fingerprint(md='sha1')),
'Subject':
_parse_subject(cert.get_subject()),
'Subject Hash':
_dec2hex(cert.get_subject().as_hash()),
'Issuer':
_parse_subject(cert.get_issuer()),
'Issuer Hash':
_dec2hex(cert.get_issuer().as_hash()),
'Not Before':
cert.get_not_before().get_datetime().strftime('%Y-%m-%d %H:%M:%S'),
'Not After':
cert.get_not_after().get_datetime().strftime('%Y-%m-%d %H:%M:%S'),
'Public Key':
get_public_key(cert.as_pem())
}
exts = OrderedDict()
for ext_index in range(0, cert.get_ext_count()):
ext = cert.get_ext_at(ext_index)
name = ext.get_name()
val = ext.get_value()
if ext.get_critical():
val = 'critical ' + val
exts[name] = val
if exts:
ret['X509v3 Extensions'] = exts
return ret
def create_table(
table_name,
region=None,
key=None,
keyid=None,
profile=None,
read_capacity_units=None,
write_capacity_units=None,
hash_key=None,
hash_key_data_type=None,
range_key=None,
range_key_data_type=None,
local_indexes=None,
global_indexes=None,
):
"""
Creates a DynamoDB table.
CLI Example:
.. code-block:: bash
salt myminion boto_dynamodb.create_table table_name /
region=us-east-1 /
hash_key=id /
hash_key_data_type=N /
range_key=created_at /
range_key_data_type=N /
read_capacity_units=1 /
write_capacity_units=1
"""
schema = []
primary_index_fields = []
primary_index_name = ""
if hash_key:
hash_key_obj = HashKey(hash_key, data_type=hash_key_data_type)
schema.append(hash_key_obj)
primary_index_fields.append(hash_key_obj)
primary_index_name += hash_key
if range_key:
range_key_obj = RangeKey(range_key, data_type=range_key_data_type)
schema.append(range_key_obj)
primary_index_fields.append(range_key_obj)
primary_index_name += "_"
primary_index_name += range_key
primary_index_name += "_index"
throughput = {"read": read_capacity_units, "write": write_capacity_units}
local_table_indexes = []
if local_indexes:
for index in local_indexes:
local_table_indexes.append(extract_index(index))
global_table_indexes = []
if global_indexes:
for index in global_indexes:
global_table_indexes.append(extract_index(index,
global_index=True))
conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
Table.create(
table_name,
schema=schema,
throughput=throughput,
indexes=local_table_indexes,
global_indexes=global_table_indexes,
connection=conn,
)
# Table creation can take several seconds to propagate.
# We will check MAX_ATTEMPTS times.
MAX_ATTEMPTS = 30
for i in range(MAX_ATTEMPTS):
if exists(table_name, region, key, keyid, profile):
return True
else:
time.sleep(1) # sleep for one second and try again
return False
