def get_minion_data(minion, opts):
'''
Get the grains/pillar for a specific minion. If minion is None, it
will return the grains/pillar for the first minion it finds.
Return value is a tuple of the minion ID, grains, and pillar
'''
grains = None
pillar = None
if opts.get('minion_data_cache', False):
cache = salt.cache.Cache(opts)
if minion is None:
for id_ in cache.list('minions'):
data = cache.fetch('minions/{0}'.format(id_), 'data')
if data is None:
continue
else:
data = cache.fetch('minions/{0}'.format(minion), 'data')
if data is not None:
grains = data['grains']
pillar = data['pillar']
return minion if minion else None, grains, pillar
def get_minion_data(minion, opts):
"""
Get the grains/pillar for a specific minion. If minion is None, it
will return the grains/pillar for the first minion it finds.
Return value is a tuple of the minion ID, grains, and pillar
"""
grains = None
pillar = None
if opts.get("minion_data_cache", False):
cache = salt.cache.factory(opts)
if minion is None:
for id_ in cache.list("minions"):
data = cache.fetch("minions/{0}".format(id_), "data")
if data is None:
continue
else:
data = cache.fetch("minions/{0}".format(minion), "data")
if data is not None:
grains = data.get("grains", None)
pillar = data.get("pillar", None)
return minion if minion else None, grains, pillar
def get_minion_data(minion, opts):
'''
Get the grains/pillar for a specific minion. If minion is None, it
will return the grains/pillar for the first minion it finds.
Return value is a tuple of the minion ID, grains, and pillar
'''
grains = None
pillar = None
if opts.get('minion_data_cache', False):
cache = salt.cache.factory(opts)
if minion is None:
for id_ in cache.list('minions'):
data = cache.fetch('minions/{0}'.format(id_), 'data')
if data is None:
continue
else:
data = cache.fetch('minions/{0}'.format(minion), 'data')
if data is not None:
grains = data.get('grains', None)
pillar = data.get('pillar', None)
return minion if minion else None, grains, pillar
def show_rsa(minion_id, dns_name):
'''
Show a private RSA key
CLI Example:
.. code-block:: bash
salt-run venafi.show_rsa myminion domain.example.com
'''
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
bank = 'venafi/domains'
data = cache.fetch(bank, dns_name)
return data['private_key']
def show_rsa(minion_id, dns_name):
"""
Show a private RSA key
CLI Example:
.. code-block:: bash
salt-run digicert.show_rsa myminion domain.example.com
"""
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
bank = "digicert/domains"
data = cache.fetch(bank, dns_name)
return data["private_key"]
def ext_pillar(
minion_id,
pillar, # pylint: disable=W0613
url=None,
ca_bundle=None,
username=None,
password=None):
'''
Read pillar data from HTTP response.
:param url String to make request
:param ca_bundle Path to CA bundle
:param username Username for basic-auth
:param password Password for basic-auth
:returns dict with pillar data to add
:returns empty if error
'''
log = logging.getLogger(__name__)
cache = salt.cache.Cache(__opts__)
if username is None:
with open(os.environ['VELUM_INTERNAL_API_USERNAME_FILE'], 'r') as f:
username = f.read().strip()
if password is None:
with open(os.environ['VELUM_INTERNAL_API_PASSWORD_FILE'], 'r') as f:
password = f.read().strip()
data = __salt__['http.query'](url=url,
ca_bundle=ca_bundle,
username=username,
password=password,
decode=True,
decode_type='json')
if 'dict' in data:
cache.store('caasp/pillar', minion_id, data['dict'])
return data['dict']
elif cache.contains('caasp/pillar', minion_id):
log.warning(
'Serving pillar from cache for minion {0}, since {1} was not available'
.format(minion_id, url))
return cache.fetch('caasp/pillar', minion_id)
log.error('Error caught on query to {0}. More Info:\n'.format(url))
for k, v in six.iteritems(data):
log.error(k + ' : ' + v)
return {}
def show_cert(dns_name):
"""
Show issued certificate for domain
CLI Example:
.. code-block:: bash
salt-run venafi.show_cert example.com
"""
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
domain_data = cache.fetch(CACHE_BANK_NAME, dns_name) or {}
cert = domain_data.get("cert")
return cert
def _cache(bank, key, fun, **kwargs):
'''
Cache an Azure ARM object
'''
items = cache.fetch(bank, key)
if items is None:
items = {}
try:
item_list = fun(**kwargs)
except CloudError as exc:
log.warn(
'There was a cloud error calling {0} with kwargs {1}: {2}'.
format(fun, kwargs, exc))
for item in item_list:
items[item.name] = object_to_dict(item)
cache.store(bank, key, items)
return items
def fetch(bank, key, cachedir=None):
'''
Fetch data from a salt.cache bank.
CLI Example:
.. code-block:: bash
salt-run cache.fetch cloud/active/ec2/myec2 myminion cachedir=/var/cache/salt/
'''
if cachedir is None:
cachedir = __opts__['cachedir']
try:
cache = salt.cache.Cache(__opts__, cachedir=cachedir)
except TypeError:
cache = salt.cache.Cache(__opts__)
return cache.fetch(bank, key)
def mine_get(tgt, fun, tgt_type='glob', opts=None):
'''
Gathers the data from the specified minions' mine, pass in the target,
function to look up and the target type
'''
ret = {}
serial = salt.payload.Serial(opts)
checker = CkMinions(opts)
minions = checker.check_minions(tgt, tgt_type)
cache = salt.cache.Cache(opts)
for minion in minions:
mdata = cache.fetch('minions/{0}'.format(minion), 'mine')
if mdata is None:
continue
fdata = mdata.get(fun)
if fdata:
ret[minion] = fdata
return ret
def _process_hostkeys(pki, cache, minion_id, ca_config, keygen_info):
log.info("Loading host key certificates for minion '%s'", minion_id)
try:
try:
principals = ca_config['hostkey_by_minion'][minion_id][
'principals']
except (KeyError, TypeError):
principals = ca_config['hostkey']['principals']
except (KeyError, TypeError):
try:
try:
principals = [
ca_config['hostkey_by_minion'][minion_id]['principal']
]
except (KeyError, TypeError):
principals = [ca_config['hostkey']['principal']]
except (KeyError, TypeError):
try:
patterns = ca_config['hostkey_by_minion'][minion_id][
'principal_patterns']
except (KeyError, TypeError):
try:
patterns = ca_config['hostkey']['principal_patterns']
except (KeyError, TypeError):
patterns = ['{}']
principals = [
pattern.format(__grains__['localhost']) for pattern in patterns
]
log.debug("Checking cache for host keys for minion '%s'", minion_id)
host_keys = cache.fetch('sshpki/hostkeys', minion_id)
log.trace("Found host keys: %s", host_keys)
host_key_certs = _get_key_certs(pki,
host_keys,
"host",
minion_id,
principals,
keygen_info,
host_keys=True)
log.trace("Loaded certificate data: %s", host_key_certs)
return host_key_certs
def _process_users(pki, cache, minion_id, ca_config, keygen_info):
log.info("Loading user certificates for minion '%s'", minion_id)
try:
try:
users = ca_config['users_by_minion'][minion_id]
except (KeyError, TypeError):
users = ca_config['users']
except (KeyError, TypeError):
users = {}
if not users:
log.debug("No user keys needed for minion '%s'", minion_id)
return {}
log.trace("Found user data: %s", users)
user_certs = {}
for user, options in users.iteritems():
if options is None:
options = {}
principals = options.get('principals')
if not principals:
principals = [options.get('principal', user)]
keygen_info['options'] = options.get('options')
if keygen_info['options'] is None:
keygen_info['options'] = []
log.trace("Found user '%s' with options: %s", user, options)
log.debug("Checking cache for user keys for '%s' on minion '%s'", user,
minion_id)
user_keys = cache.fetch('sshpki/userkeys/{}'.format(minion_id), user)
log.trace("Found user keys: %s", user_keys)
certs = _get_key_certs(pki, user_keys, "user",
'{0}@{1}'.format(user, minion_id), principals,
keygen_info)
if options.get('pubkey_path'):
for t in certs:
certs[t]['path_opt'] = options.get('pubkey_path')
log.trace("Loaded user certificate data: %s", certs)
if certs:
user_certs[user] = certs
log.trace("Loaded certificate data: %s", user_certs)
return user_certs
def mine_get(tgt, fun, tgt_type='glob', opts=None):
'''
Gathers the data from the specified minions' mine, pass in the target,
function to look up and the target type
'''
ret = {}
serial = salt.payload.Serial(opts)
checker = CkMinions(opts)
minions = checker.check_minions(
tgt,
tgt_type)
cache = salt.cache.Cache(opts)
for minion in minions:
mdata = cache.fetch('minions/{0}'.format(minion), 'mine')
if mdata is None:
continue
fdata = mdata.get(fun)
if fdata:
ret[minion] = fdata
return ret
def _load_minion(minion_id, cache):
data_minion, grains, pillar = salt.utils.minions.get_minion_data(minion_id, __opts__)
if minion_id != data_minion:
log.error('Asked for minion %s, got %s', minion_id, data_minion)
raise LookupError
if not grains:
log.warning('No grain data for minion id %s', minion_id)
grains = {}
if not pillar:
log.warning('No pillar data for minion id %s', minion_id)
pillar = {}
addrs = {
4: sorted([ipaddress.IPv4Address(addr) for addr in grains.get('ipv4', [])]),
6: sorted([ipaddress.IPv6Address(addr) for addr in grains.get('ipv6', [])])
}
mine = cache.fetch('minions/{0}'.format(minion_id), 'mine')
return grains, pillar, addrs, mine
def mine_get(tgt, fun, tgt_type='glob', opts=None):
'''
Gathers the data from the specified minions' mine, pass in the target,
function to look up and the target type
'''
ret = {}
serial = salt.payload.Serial(opts)
checker = CkMinions(opts)
_res = checker.check_minions(
tgt,
tgt_type)
minions = _res['minions']
cache = salt.cache.factory(opts)
if isinstance(fun, six.string_types):
functions = list(set(fun.split(',')))
_ret_dict = len(functions) > 1
elif isinstance(fun, list):
functions = fun
_ret_dict = True
else:
return {}
for minion in minions:
mdata = cache.fetch('minions/{0}'.format(minion), 'mine')
if not isinstance(mdata, dict):
continue
if not _ret_dict and functions and functions[0] in mdata:
ret[minion] = mdata.get(functions)
elif _ret_dict:
for fun in functions:
if fun in mdata:
ret.setdefault(fun, {})[minion] = mdata.get(fun)
return ret
def order_certificate(
minion_id,
common_name,
organization_id,
validity_years,
cert_key_passphrase=None,
signature_hash=None,
key_len=2048,
dns_names=None,
organization_units=None,
server_platform=None,
custom_expiration_date=None,
comments=None,
disable_renewal_notifications=False,
product_type_hint=None,
renewal_of_order_id=None,
):
"""
Order a certificate. Requires that an Organization has been created inside Digicert's CertCentral.
See here for API documentation:
https://www.digicert.com/services/v2/documentation/order/order-ssl-determinator
CLI Example:
.. code-block:: bash
salt-run digicert.order_certificate my_minionid my.domain.com 10 \
3 signature_hash=sha256 \
dns_names=['this.domain.com', 'that.domain.com'] \
organization_units='My Domain Org Unit' \
comments='Comment goes here for the approver'
This runner can also be used to renew a certificate by passing `renewal_of_order_id`.
Previous order details can be retrieved with digicertapi.list_orders.
"""
if dns_names and isinstance(dns_names, six.string_types):
dns_names = [dns_names]
if dns_names and not isinstance(dns_names, collections.Sequence):
raise SaltRunnerError(
"order_certificate needs a single dns_name, or an array of dns_names."
)
certificate = {"common_name": common_name}
certificate["dns_names"] = dns_names
if signature_hash:
certificate["signature_hash"] = signature_hash
else:
certificate["signature_hash"] = __opts__.get("digicert", {}).get(
"shatype", "sha256")
body = {}
if organization_units and isinstance(organization_units, six.string_types):
organization_units = [organization_units]
if organization_units and not isinstance(organization_units,
collections.Sequence):
raise SaltRunnerError("Organization_units is not a valid data type.")
if organization_units:
certificate["organization_units"] = organization_units
if organization_units:
# Currently the Digicert API requires organization units to be an array
# but only pays attention to the first one.
csr = gen_csr(
minion_id,
common_name,
organization_id,
ou_name=organization_units[0],
shatype=certificate["signature_hash"],
key_len=key_len,
password=cert_key_passphrase,
)
else:
csr = gen_csr(
minion_id,
common_name,
organization_id,
shatype=certificate["signature_hash"],
key_len=key_len,
password=cert_key_passphrase,
)
certificate["csr"] = csr
if server_platform:
certificate["server_platform"]["id"] = server_platform
body["organization"] = {"id": organization_id}
if custom_expiration_date:
body["custom_expiration_date"] = custom_expiration_date
if validity_years:
body["validity_years"] = validity_years
if comments:
body["comments"] = comments
body["disable_renewal_notifications"] = disable_renewal_notifications
if product_type_hint:
body["product"] = {"type_hint": product_type_hint}
if renewal_of_order_id:
body["renewal_of_order_id"] = renewal_of_order_id
body["certificate"] = certificate
encoded_body = salt.utils.json.dumps(body)
qdata = salt.utils.http.query(
"{0}/order/certificate/ssl".format(_base_url()),
method="POST",
data=encoded_body,
decode=True,
decode_type="json",
header_dict={
"X-DC-DEVKEY": _api_key(),
"Content-Type": "application/json"
},
raise_error=False,
)
if "errors" not in qdata["dict"]:
bank = "digicert/domains"
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
data = cache.fetch(bank, common_name)
if data is None:
data = {}
data.update({
"minion_id": minion_id,
"order_id": qdata["dict"]["requests"][0]["id"],
"csr": csr,
})
cache.store(bank, common_name, data)
_id_map(minion_id, common_name)
return {"order": qdata["dict"]}
def gen_csr(
minion_id,
dns_name,
organization_id,
ou_name=None,
key_len=2048,
shatype="sha256",
password=None,
):
"""
CLI Example:
.. code-block:: bash
salt-run digicert.gen_csr <minion_id> <dns_name>
"""
org_details = get_org_details(organization_id)
if "error" in org_details:
raise SaltRunnerError(
"Problem getting organization details for organization_id={0} ({1})"
.format(organization_id, org_details["error"]))
if org_details["dict"].get("status", "active") == "inactive":
raise SaltRunnerError(
"Organization with organization_id={0} is marked inactive".format(
organization_id))
tmpdir = tempfile.mkdtemp()
os.chmod(tmpdir, 0o700)
bank = "digicert/domains"
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
data = cache.fetch(bank, dns_name)
if data is None:
data = {}
if "private_key" not in data:
data["private_key"] = gen_key(minion_id,
dns_name,
password,
key_len=key_len)
tmppriv = "{0}/priv".format(tmpdir)
tmpcsr = "{0}/csr".format(tmpdir)
with salt.utils.files.fopen(tmppriv, "w") as if_:
if_.write(salt.utils.stringutils.to_str(data["private_key"]))
subject = "/C={0}/ST={1}/L={2}/O={3}".format(
org_details["dict"]["country"],
org_details["dict"]["state"],
org_details["dict"]["city"],
org_details["dict"]["display_name"],
)
if ou_name:
subject = subject + "/OU={0}".format(ou_name)
subject = subject + "/CN={0}".format(dns_name)
cmd = "openssl req -new -{0} -key {1} -out {2} -subj '{3}'".format(
shatype, tmppriv, tmpcsr, subject)
output = __salt__["salt.cmd"]("cmd.run", cmd)
if "problems making Certificate Request" in output:
raise CommandExecutionError(
"There was a problem generating the CSR. Please ensure that you "
"have a valid Organization established inside CertCentral")
with salt.utils.files.fopen(tmpcsr, "r") as of_:
csr = salt.utils.stringutils.to_unicode(of_.read())
data["minion_id"] = minion_id
data["csr"] = csr
cache.store(bank, dns_name, data)
return csr
def get_certificate(
order_id=None,
certificate_id=None,
minion_id=None,
cert_format="pem_all",
filename=None,
):
"""
Retrieve a certificate by order_id or certificate_id and write it to stdout or a filename.
A list of permissible cert_formats is here:
https://www.digicert.com/services/v2/documentation/appendix-certificate-formats
CLI Example:
.. code-block:: bash
salt-run digicert.get_certificate order_id=48929454 cert_format=apache
Including a 'filename' will write the certificate to the desired file.
Note that some cert formats are zipped files, and some are binary.
If the certificate has not been issued, this function will return the order details
inside of which will be a status (one of pending, rejected, processing, issued,
revoked, canceled, needs_csr, and needs_approval)
If for some reason you want to pipe the output of this command to a file or other
command you will want to leave off the ``filename`` argument and make sure to include
``--no-color`` so there will be no terminal ANSI escape sequences.
"""
if order_id:
order_cert = salt.utils.http.query(
"{0}/order/certificate/{1}".format(_base_url(), order_id),
method="GET",
raise_error=False,
decode=True,
decode_type="json",
header_dict={
"X-DC-DEVKEY": _api_key(),
"Content-Type": "application/json",
},
)
if order_cert["dict"].get("status") != "issued":
return {"certificate": order_cert["dict"]}
if order_cert["dict"].get("errors", False):
return {"certificate": order_cert["dict"]}
certificate_id = order_cert["dict"].get("certificate").get("id", None)
common_name = order_cert["dict"].get("certificate").get("common_name")
if not certificate_id:
return {
"certificate": {
"errors": {
"code":
"unknown",
"message":
"Unknown error, no certificate ID passed on command line or in body returned from API",
}
}
}
if filename:
ret_cert = salt.utils.http.query(
"{0}/certificate/{1}/download/format/{2}".format(
_base_url(), certificate_id, cert_format),
method="GET",
decode=False,
text=False,
headers=True,
text_out=filename,
raise_error=False,
header_dict={"X-DC-DEVKEY": _api_key()},
)
else:
ret_cert = salt.utils.http.query(
"{0}/certificate/{1}/download/format/{2}".format(
_base_url(), certificate_id, cert_format),
method="GET",
text=False,
decode=False,
raise_error=False,
header_dict={"X-DC-DEVKEY": _api_key()},
)
if "errors" in ret_cert:
return {"certificate": ret_cert}
if "body" not in ret_cert:
ret = {"certificate": ret_cert}
cert = ret_cert
if isinstance(ret_cert, dict):
ret = ret_cert["body"]
cert = ret
else:
ret = ret_cert
cert = ret
tmpfilename = None
if not filename:
fd, tmpfilename = tempfile.mkstemp()
filename = tmpfilename
os.write(fd, cert)
os.close(fd)
cmd = [
"openssl",
"x509",
"-noout",
"-subject",
"-nameopt",
"multiline",
"-in",
filename,
]
out = subprocess.check_output(cmd)
common_name = None
for l in out.splitlines():
common_name_match = re.search(" *commonName *= *(.*)", l)
if common_name_match:
common_name = common_name_match.group(1)
break
if tmpfilename:
os.unlink(tmpfilename)
if common_name:
bank = "digicert/domains"
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
try:
data = cache.fetch(bank, common_name)
except TypeError:
data = {"certificate": cert}
cache.store(bank, common_name, data)
if "headers" in ret_cert:
return {
"certificate": {
"filename":
filename,
"original_filename":
ret_cert["headers"].get("Content-Disposition", "Not provided"),
"Content-Type":
ret_cert["headers"].get("Content-Type", "Not provided"),
}
}
return {"certificate": cert}
def load_cache(pool, __runner__, opts, tgt, tgt_type=None):
'''
Load the Pillar and Grain cache, as required, and merge the Roster Grains
and Pillar into.
'''
if opts.get('grains'):
for device, device_opts in six.iteritems(pool):
if 'minion_opts' not in device_opts:
device_opts['minion_opts'] = {}
if 'grains' not in device_opts['minion_opts']:
device_opts['minion_opts']['grains'] = {}
device_opts['minion_opts']['grains'] = salt.utils.dictupdate.merge(
opts['grains'],
device_opts['minion_opts']['grains'],
merge_lists=True,
)
if tgt_type in ('glob', 'pcre', 'list'):
# When the target type is glob, pcre, or list, we don't require grains
# or pillar loaded from the cache, because the targeting won't depend on
# those.
return pool
if not opts.get('use_cached_grains', True) and not opts.get(
'use_cached_pillar', True):
return pool
# NOTE: It wouldn't be feasible to use the cache.grains or cache.pillar
# Runners as they rely on fetching data from the Master, for Minions that
# are accepted. What we're doing here is reading straight from the cache.
log.debug('Loading cached and merging into the Roster data')
cache = salt.cache.factory(opts)
cache_pool = cache.list('minions')
for device in cache_pool:
if device not in pool:
log.trace('%s has cache, but is not in the Roster pool', device)
continue
if 'minion_opts' not in pool[device]:
pool[device]['minion_opts'] = {'grains': {}, 'pillar': {}}
cache_key = 'minions/{}/data'.format(device)
if opts.get('target_use_cached_grains', True) and tgt_type in (
'compound',
'grain',
'grain_pcre',
'nodegroup',
):
log.debug('Fetching cached Grains for %s', device)
cached_grains = cache.fetch(cache_key, 'grains')
if cached_grains:
pool[device]['minion_opts'][
'grains'] = salt.utils.dictupdate.merge(
cached_grains,
pool[device]['minion_opts'].get('grains', {}),
merge_lists=True,
)
if opts.get('target_use_cached_pillar', True) and tgt_type in (
'compound',
'pillar',
'pillar_pcre',
'pillar_target',
'nodegroup',
):
log.debug('Fetching cached Pillar for %s', device)
cached_pillar = cache.fetch(cache_key, 'pillar')
if cached_pillar:
pool[device]['minion_opts'][
'pillar'] = salt.utils.dictupdate.merge(
cached_pillar,
pool[device]['minion_opts'].get('pillar', {}),
merge_lists=True,
)
log.debug('The device pool with the cached data')
log.debug(pool)
return pool
def run_common_cache_tests(subtests, cache):
bank = "fnord/kevin/stuart"
# ^^^^ This bank can be just fnord, or fnord/foo, or any mildly reasonable
# or possibly unreasonably nested names.
#
# No. Seriously. Try import string; bank = '/'.join(string.ascii_letters)
# - it works!
# import string; bank = "/".join(string.ascii_letters)
good_key = "roscivs"
bad_key = "monkey"
with subtests.test("non-existent bank should be empty on cache start"):
assert not cache.contains(bank=bank)
assert cache.list(bank=bank) == []
with subtests.test("after storing key in bank it should be in cache list"):
cache.store(bank=bank, key=good_key, data=b"\x01\x04\x05fnordy data")
assert cache.list(bank) == [good_key]
with subtests.test("after storing value, it should be fetchable"):
expected_data = "trombone pleasantry"
cache.store(bank=bank, key=good_key, data=expected_data)
assert cache.fetch(bank=bank, key=good_key) == expected_data
with subtests.test("bad key should still be absent from cache"):
assert cache.fetch(bank=bank, key=bad_key) == {}
with subtests.test("storing new value should update it"):
# Double check that the data was still the old stuff
old_data = expected_data
assert cache.fetch(bank=bank, key=good_key) == old_data
new_data = "stromboli"
cache.store(bank=bank, key=good_key, data=new_data)
assert cache.fetch(bank=bank, key=good_key) == new_data
with subtests.test("storing complex object works"):
new_thing = {
"some": "data",
42: "wheee",
"some other": {"sub": {"objects": "here"}},
}
cache.store(bank=bank, key=good_key, data=new_thing)
actual_thing = cache.fetch(bank=bank, key=good_key)
if isinstance(cache, salt.cache.MemCache):
# MemCache should actually store the object - everything else
# should create a copy of it.
assert actual_thing is new_thing
else:
assert actual_thing is not new_thing
assert actual_thing == new_thing
with subtests.test("contains returns true if key in bank"):
assert cache.contains(bank=bank, key=good_key)
with subtests.test("contains returns true if bank exists and key is None"):
assert cache.contains(bank=bank, key=None)
with subtests.test(
"contains returns False when bank not in cache and/or key not in bank"
):
assert not cache.contains(bank=bank, key=bad_key)
assert not cache.contains(bank="nonexistent", key=good_key)
assert not cache.contains(bank="nonexistent", key=bad_key)
assert not cache.contains(bank="nonexistent", key=None)
with subtests.test("flushing nonexistent key should not remove other keys"):
cache.flush(bank=bank, key=bad_key)
assert cache.contains(bank=bank, key=good_key)
with subtests.test(
"flushing existing key should not remove bank if no more keys exist"
):
pytest.skip(
"This is impossible with redis. Should we make localfs behave the same way?"
)
cache.flush(bank=bank, key=good_key)
assert cache.contains(bank=bank)
assert cache.list(bank=bank) == []
with subtests.test(
"after existing key is flushed updated should not return a timestamp for that key"
):
cache.store(bank=bank, key=good_key, data="fnord")
cache.flush(bank=bank, key=good_key)
timestamp = cache.updated(bank=bank, key=good_key)
assert timestamp is None
with subtests.test(
"after flushing bank containing a good key, updated should not return a timestamp for that key"
):
cache.store(bank=bank, key=good_key, data="fnord")
cache.flush(bank=bank, key=None)
timestamp = cache.updated(bank=bank, key=good_key)
assert timestamp is None
with subtests.test("flushing bank with None as key should remove bank"):
cache.flush(bank=bank, key=None)
assert not cache.contains(bank=bank)
with subtests.test("Exception should happen when flushing None bank"):
# This bit is maybe an accidental API, but currently there is no
# protection at least with the localfs cache when bank is None. If
# bank is None we try to `os.path.normpath` the bank, which explodes
# and is at least the current behavior. If we want to change that
# this test should change. Or be removed altogether.
# TODO: this should actually not raise. Not sure if there's a test that we can do here... or just call the code which will fail if there's actually an exception. -W. Werner, 2021-09-28
pytest.skip(
"Skipping for now - etcd, redis, and mysql do not raise. Should ensure all backends behave consistently"
)
with pytest.raises(Exception):
cache.flush(bank=None, key=None)
with subtests.test("Updated for non-existent key should return None"):
timestamp = cache.updated(bank="nonexistent", key="whatever")
assert timestamp is None
with subtests.test("Updated for key should return a reasonable time"):
before_storage = int(time.time())
cache.store(bank="fnord", key="updated test part 2", data="fnord")
after_storage = int(time.time())
timestamp = cache.updated(bank="fnord", key="updated test part 2")
assert before_storage <= timestamp <= after_storage
with subtests.test(
"If the module raises SaltCacheError then it should make it out of updated"
):
with patch.dict(
cache.modules._dict,
{"{}.updated".format(cache.driver): MagicMock(side_effect=SaltCacheError)},
), pytest.raises(SaltCacheError):
cache.updated(bank="kaboom", key="oops")
with subtests.test(
"cache.cache right after a value is cached should not update the cache"
):
expected_value = "some cool value yo"
cache.store(bank=bank, key=good_key, data=expected_value)
result = cache.cache(
bank=bank,
key=good_key,
fun=lambda **kwargs: "bad bad value no good",
value="some other value?",
loop_fun=lambda x: "super very no good bad",
)
fetch_result = cache.fetch(bank=bank, key=good_key)
assert result == fetch_result == expected_value
with subtests.test(
"cache.cache should update the value with the result of fun when value was updated longer than expiration",
), patch(
"salt.cache.Cache.updated",
return_value=42, # Dec 31, 1969... time to update the cache!
autospec=True,
):
expected_value = "this is the return value woo woo woo"
cache.store(bank=bank, key=good_key, data="not this value")
cache_result = cache.cache(
bank=bank, key=good_key, fun=lambda *args, **kwargs: expected_value
)
fetch_result = cache.fetch(bank=bank, key=good_key)
assert cache_result == fetch_result == expected_value
with subtests.test(
"cache.cache should update the value with all of the outputs from loop_fun if loop_fun was provided",
), patch(
"salt.cache.Cache.updated",
return_value=42,
autospec=True,
):
expected_value = "SOME HUGE STRING OKAY?"
cache.store(bank=bank, key=good_key, data="nope, not me")
cache_result = cache.cache(
bank=bank,
key=good_key,
fun=lambda **kwargs: "some huge string okay?",
loop_fun=str.upper,
)
fetch_result = cache.fetch(bank=bank, key=good_key)
assert cache_result == fetch_result
assert "".join(fetch_result) == expected_value
with subtests.test(
"cache.cache should update the value if the stored value is empty but present and expiry is way in the future"
), patch(
"salt.cache.Cache.updated",
return_value=time.time() * 2,
autospec=True,
):
# Unclear if this was intended behavior: currently any falsey data will
# be updated by cache.cache. If this is incorrect, this test should
# be updated or removed.
expected_data = "some random string whatever"
for empty in ("", (), [], {}, 0, 0.0, False, None):
with subtests.test(empty=empty):
cache.store(
bank=bank, key=good_key, data=empty
) # empty chairs and empty data
cache_result = cache.cache(
bank=bank, key=good_key, fun=lambda **kwargs: expected_data
)
fetch_result = cache.fetch(bank=bank, key=good_key)
assert cache_result == fetch_result == expected_data
with subtests.test("cache.cache should store a value if it does not exist"):
expected_result = "some result plz"
cache.flush(bank=bank, key=None)
assert cache.fetch(bank=bank, key=good_key) == {}
cache_result = cache.cache(
bank=bank, key=good_key, fun=lambda **kwargs: expected_result
)
fetch_result = cache.fetch(bank=bank, key=good_key)
assert cache_result == fetch_result
assert fetch_result == expected_result
assert cache_result == fetch_result == expected_result
def gen_csr(minion_id,
dns_name,
organization_id,
ou_name=None,
key_len=2048,
shatype='sha256',
password=None):
'''
CLI Example:
.. code-block:: bash
salt-run digicert.gen_csr <minion_id> <dns_name>
'''
org_details = get_org_details(organization_id)
if 'error' in org_details:
raise SaltRunnerError(
'Problem getting organization details for organization_id={0} ({1})'
.format(organization_id, org_details['error']))
if org_details['dict'].get('status', 'active') == 'inactive':
raise SaltRunnerError(
'Organization with organization_id={0} is marked inactive'.format(
organization_id))
tmpdir = tempfile.mkdtemp()
os.chmod(tmpdir, 0o700)
bank = 'digicert/domains'
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
data = cache.fetch(bank, dns_name)
if data is None:
data = {}
if 'private_key' not in data:
data['private_key'] = gen_key(minion_id,
dns_name,
password,
key_len=key_len)
tmppriv = '{0}/priv'.format(tmpdir)
tmpcsr = '{0}/csr'.format(tmpdir)
with salt.utils.files.fopen(tmppriv, 'w') as if_:
if_.write(salt.utils.stringutils.to_str(data['private_key']))
subject = '/C={0}/ST={1}/L={2}/O={3}'.format(
org_details['dict']['country'], org_details['dict']['state'],
org_details['dict']['city'], org_details['dict']['display_name'])
if ou_name:
subject = subject + '/OU={0}'.format(ou_name)
subject = subject + '/CN={0}'.format(dns_name)
cmd = "openssl req -new -{0} -key {1} -out {2} -subj '{3}'".format(
shatype, tmppriv, tmpcsr, subject)
output = __salt__['salt.cmd']('cmd.run', cmd)
if 'problems making Certificate Request' in output:
raise CommandExecutionError(
'There was a problem generating the CSR. Please ensure that you '
'have a valid Organization established inside CertCentral')
with salt.utils.files.fopen(tmpcsr, 'r') as of_:
csr = salt.utils.stringutils.to_unicode(of_.read())
data['minion_id'] = minion_id
data['csr'] = csr
cache.store(bank, dns_name, data)
return csr
def request(
minion_id,
dns_name=None,
zone='default',
request_id=None,
country='US',
state='California',
loc='Palo Alto',
org='Beta Organization',
org_unit='Beta Group',
password=None,
zone_id=None,
):
'''
Request a new certificate
Uses the following command:
.. code-block:: bash
VCert enroll -z <zone> -k <api key> -cn <domain name>
CLI Example:
.. code-block:: bash
salt-run venafi.request <minion_id> <dns_name>
'''
if password is not None:
if password.startswith('sdb://'):
password = __salt__['sdb.get'](password)
if zone_id is None:
zone_id = __opts__.get('venafi', {}).get('zone_id')
if zone_id is None and zone is not None:
zone_id = get_zone_id(zone)
if zone_id is None:
raise CommandExecutionError(
'Either a zone or a zone_id must be passed in or '
'configured in the master file. This id can be retreived using '
'venafi.show_company <domain>'
)
private_key = gen_key(minion_id, dns_name, zone, password)
csr = gen_csr(
minion_id,
dns_name,
zone=zone,
country=country,
state=state,
loc=loc,
org=org,
org_unit=org_unit,
password=password,
)
pdata = salt.utils.json.dumps({
'zoneId': zone_id,
'certificateSigningRequest': csr,
})
qdata = __utils__['http.query'](
'{0}/certificaterequests'.format(_base_url()),
method='POST',
data=pdata,
decode=True,
decode_type='json',
header_dict={
'tppl-api-key': _api_key(),
'Content-Type': 'application/json',
},
)
request_id = qdata['dict']['certificateRequests'][0]['id']
ret = {
'request_id': request_id,
'private_key': private_key,
'csr': csr,
'zone': zone,
}
bank = 'venafi/domains'
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
data = cache.fetch(bank, dns_name)
if data is None:
data = {}
data.update({
'minion_id': minion_id,
'request_id': request_id,
'private_key': private_key,
'zone': zone,
'csr': csr,
})
cache.store(bank, dns_name, data)
_id_map(minion_id, dns_name)
return ret
def gen_csr(
minion_id,
dns_name,
zone='default',
country=None,
state=None,
loc=None,
org=None,
org_unit=None,
password=None,
):
'''
Generate a csr using the host's private_key.
Analogous to:
.. code-block:: bash
VCert gencsr -cn [CN Value] -o "Beta Organization" -ou "Beta Group" \
-l "Palo Alto" -st "California" -c US
CLI Example:
.. code-block:: bash
salt-run venafi.gen_csr <minion_id> <dns_name>
'''
tmpdir = tempfile.mkdtemp()
os.chmod(tmpdir, 0o700)
bank = 'venafi/domains'
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
data = cache.fetch(bank, dns_name)
if data is None:
data = {}
if 'private_key' not in data:
data['private_key'] = gen_key(minion_id, dns_name, zone, password)
tmppriv = '{0}/priv'.format(tmpdir)
tmpcsr = '{0}/csr'.format(tmpdir)
with salt.utils.files.fopen(tmppriv, 'w') as if_:
if_.write(salt.utils.stringutils.to_str(data['private_key']))
if country is None:
country = __opts__.get('venafi', {}).get('country')
if state is None:
state = __opts__.get('venafi', {}).get('state')
if loc is None:
loc = __opts__.get('venafi', {}).get('loc')
if org is None:
org = __opts__.get('venafi', {}).get('org')
if org_unit is None:
org_unit = __opts__.get('venafi', {}).get('org_unit')
subject = '/C={0}/ST={1}/L={2}/O={3}/OU={4}/CN={5}'.format(
country,
state,
loc,
org,
org_unit,
dns_name,
)
cmd = "openssl req -new -sha256 -key {0} -out {1} -subj '{2}'".format(
tmppriv,
tmpcsr,
subject
)
if password is not None:
cmd += ' -passin pass:{0}'.format(password)
output = __salt__['salt.cmd']('cmd.run', cmd)
if 'problems making Certificate Request' in output:
raise CommandExecutionError(
'There was a problem generating the CSR. Please ensure that you '
'have the following variables set either on the command line, or '
'in the venafi section of your master configuration file: '
'country, state, loc, org, org_unit'
)
with salt.utils.files.fopen(tmpcsr, 'r') as of_:
csr = salt.utils.stringutils.to_unicode(of_.read())
data['minion_id'] = minion_id
data['csr'] = csr
cache.store(bank, dns_name, data)
return csr
def get_certificate(order_id=None,
certificate_id=None,
minion_id=None,
cert_format='pem_all',
filename=None):
'''
Retrieve a certificate by order_id or certificate_id and write it to stdout or a filename.
A list of permissible cert_formats is here:
https://www.digicert.com/services/v2/documentation/appendix-certificate-formats
CLI Example:
.. code-block:: bash
salt-run digicert.get_certificate order_id=48929454 cert_format=apache
Including a 'filename' will write the certificate to the desired file.
Note that some cert formats are zipped files, and some are binary.
If the certificate has not been issued, this function will return the order details
inside of which will be a status (one of pending, rejected, processing, issued,
revoked, canceled, needs_csr, and needs_approval)
If for some reason you want to pipe the output of this command to a file or other
command you will want to leave off the ``filename`` argument and make sure to include
``--no-color`` so there will be no terminal ANSI escape sequences.
'''
if order_id:
order_cert = salt.utils.http.query('{0}/order/certificate/{1}'.format(
_base_url(), order_id),
method='GET',
raise_error=False,
decode=True,
decode_type='json',
header_dict={
'X-DC-DEVKEY': _api_key(),
'Content-Type':
'application/json',
})
if order_cert['dict'].get('status') != 'issued':
return {'certificate': order_cert['dict']}
if order_cert['dict'].get('errors', False):
return {'certificate': order_cert['dict']}
certificate_id = order_cert['dict'].get('certificate').get('id', None)
common_name = order_cert['dict'].get('certificate').get('common_name')
if not certificate_id:
return {
'certificate': {
'errors': {
'code':
'unknown',
'message':
'Unknown error, no certificate ID passed on command line or in body returned from API'
}
}
}
if filename:
ret_cert = salt.utils.http.query(
'{0}/certificate/{1}/download/format/{2}'.format(
_base_url(), certificate_id, cert_format),
method='GET',
decode=False,
text=False,
headers=True,
text_out=filename,
raise_error=False,
header_dict={
'X-DC-DEVKEY': _api_key(),
})
else:
ret_cert = salt.utils.http.query(
'{0}/certificate/{1}/download/format/{2}'.format(
_base_url(), certificate_id, cert_format),
method='GET',
text=False,
decode=False,
raise_error=False,
header_dict={
'X-DC-DEVKEY': _api_key(),
})
if 'errors' in ret_cert:
return {'certificate': ret_cert}
if 'body' not in ret_cert:
ret = {'certificate': ret_cert}
cert = ret_cert
if isinstance(ret_cert, dict):
ret = ret_cert['body']
cert = ret
else:
ret = ret_cert
cert = ret
tmpfilename = None
if not filename:
fd, tmpfilename = tempfile.mkstemp()
filename = tmpfilename
os.write(fd, cert)
os.close(fd)
cmd = [
'openssl', 'x509', '-noout', '-subject', '-nameopt', 'multiline',
'-in', filename
]
out = subprocess.check_output(cmd)
common_name = None
for l in out.splitlines():
common_name_match = re.search(' *commonName *= *(.*)', l)
if common_name_match:
common_name = common_name_match.group(1)
break
if tmpfilename:
os.unlink(tmpfilename)
if common_name:
bank = 'digicert/domains'
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
try:
data = cache.fetch(bank, common_name)
except TypeError:
data = {'certificate': cert}
cache.store(bank, common_name, data)
if 'headers' in ret_cert:
return {
'certificate': {
'filename':
filename,
'original_filename':
ret_cert['headers'].get('Content-Disposition', 'Not provided'),
'Content-Type':
ret_cert['headers'].get('Content-Type', 'Not provided')
}
}
return {'certificate': cert}
def gen_key(minion_id, dns_name=None, zone='default', password=None):
'''
Generate and return an private_key. If a ``dns_name`` is passed in, the
private_key will be cached under that name. The type of key and the
parameters used to generate the key are based on the default certificate
use policy associated with the specified zone.
CLI Example:
.. code-block:: bash
salt-run venafi.gen_key <minion_id> [dns_name] [zone] [password]
'''
# Get the default certificate use policy associated with the zone
# so we can generate keys that conform with policy
# The /v1/zones/tag/{name} API call is a shortcut to get the zoneID
# directly from the name
qdata = __utils__['http.query'](
'{0}/zones/tag/{1}'.format(_base_url(), zone),
method='GET',
decode=True,
decode_type='json',
header_dict={
'tppl-api-key': _api_key(),
'Content-Type': 'application/json',
},
)
zone_id = qdata['dict']['id']
# the /v1/certificatepolicies?zoneId API call returns the default
# certificate use and certificate identity policies
qdata = __utils__['http.query'](
'{0}/certificatepolicies?zoneId={1}'.format(_base_url(), zone_id),
method='GET',
decode=True,
decode_type='json',
header_dict={
'tppl-api-key': _api_key(),
'Content-Type': 'application/json',
},
)
policies = qdata['dict']['certificatePolicies']
# Extract the key length and key type from the certificate use policy
# and generate the private key accordingly
for policy in policies:
if policy['certificatePolicyType'] == "CERTIFICATE_USE":
keyTypes = policy['keyTypes']
# in case multiple keytypes and key lengths are supported
# always use the first key type and key length
keygen_type = keyTypes[0]['keyType']
key_len = keyTypes[0]['keyLengths'][0]
if int(key_len) < 2048:
key_len = 2048
if keygen_type == "RSA":
gen = RSA.generate(bits=key_len)
private_key = gen.exportKey('PEM', password)
if dns_name is not None:
bank = 'venafi/domains'
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
try:
data = cache.fetch(bank, dns_name)
data['private_key'] = private_key
data['minion_id'] = minion_id
except TypeError:
data = {'private_key': private_key,
'minion_id': minion_id}
cache.store(bank, dns_name, data)
return private_key
def order_certificate(minion_id,
common_name,
organization_id,
validity_years,
cert_key_passphrase=None,
signature_hash=None,
key_len=2048,
dns_names=None,
organization_units=None,
server_platform=None,
custom_expiration_date=None,
comments=None,
disable_renewal_notifications=False,
product_type_hint=None,
renewal_of_order_id=None):
'''
Order a certificate. Requires that an Organization has been created inside Digicert's CertCentral.
See here for API documentation:
https://www.digicert.com/services/v2/documentation/order/order-ssl-determinator
CLI Example:
.. code-block:: bash
salt-run digicert.order_certificate my_minionid my.domain.com 10 \
3 signature_hash=sha256 \
dns_names=['this.domain.com', 'that.domain.com'] \
organization_units='My Domain Org Unit' \
comments='Comment goes here for the approver'
This runner can also be used to renew a certificate by passing `renewal_of_order_id`.
Previous order details can be retrieved with digicertapi.list_orders.
'''
if dns_names and isinstance(dns_names, six.string_types):
dns_names = [dns_names]
if dns_names and not isinstance(dns_names, collections.Sequence):
raise SaltRunnerError(
'order_certificate needs a single dns_name, or an array of dns_names.'
)
certificate = {'common_name': common_name}
certificate['dns_names'] = dns_names
if signature_hash:
certificate['signature_hash'] = signature_hash
else:
certificate['signature_hash'] = __opts__.get('digicert', {}).get(
'shatype', 'sha256')
body = {}
if organization_units and isinstance(organization_units, six.string_types):
organization_units = [organization_units]
if organization_units and not isinstance(organization_units,
collections.Sequence):
raise SaltRunnerError('Organization_units is not a valid data type.')
if organization_units:
certificate['organization_units'] = organization_units
if organization_units:
# Currently the Digicert API requires organization units to be an array
# but only pays attention to the first one.
csr = gen_csr(minion_id,
common_name,
organization_id,
ou_name=organization_units[0],
shatype=certificate['signature_hash'],
key_len=key_len,
password=cert_key_passphrase)
else:
csr = gen_csr(minion_id,
common_name,
organization_id,
shatype=certificate['signature_hash'],
key_len=key_len,
password=cert_key_passphrase)
certificate['csr'] = csr
if server_platform:
certificate['server_platform']['id'] = server_platform
body['organization'] = {'id': organization_id}
if custom_expiration_date:
body['custom_expiration_date'] = custom_expiration_date
if validity_years:
body['validity_years'] = validity_years
if comments:
body['comments'] = comments
body['disable_renewal_notifications'] = disable_renewal_notifications
if product_type_hint:
body['product'] = {'type_hint': product_type_hint}
if renewal_of_order_id:
body['renewal_of_order_id'] = renewal_of_order_id
body['certificate'] = certificate
encoded_body = salt.utils.json.dumps(body)
qdata = salt.utils.http.query('{0}/order/certificate/ssl'.format(
_base_url()),
method='POST',
data=encoded_body,
decode=True,
decode_type='json',
header_dict={
'X-DC-DEVKEY': _api_key(),
'Content-Type': 'application/json',
},
raise_error=False)
if 'errors' not in qdata['dict']:
bank = 'digicert/domains'
cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR)
data = cache.fetch(bank, common_name)
if data is None:
data = {}
data.update({
'minion_id': minion_id,
'order_id': qdata['dict']['requests'][0]['id'],
'csr': csr,
})
cache.store(bank, common_name, data)
_id_map(minion_id, common_name)
return {'order': qdata['dict']}
