def store(bank, key, data, cachedir):
"""
Store information in a file.
"""
base = os.path.join(cachedir, os.path.normpath(bank))
try:
os.makedirs(base)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise SaltCacheError(
"The cache directory, {}, could not be created: {}".format(
base, exc))
outfile = os.path.join(base, "{}.p".format(key))
tmpfh, tmpfname = tempfile.mkstemp(dir=base)
os.close(tmpfh)
try:
with salt.utils.files.fopen(tmpfname, "w+b") as fh_:
salt.payload.dump(data, fh_)
# On Windows, os.rename will fail if the destination file exists.
salt.utils.atomicfile.atomic_rename(tmpfname, outfile)
except OSError as exc:
raise SaltCacheError(
"There was an error writing the cache file, {}: {}".format(
base, exc))
def store(bank, key, data, cachedir):
'''
Store information in a file.
'''
base = os.path.join(cachedir, os.path.normpath(bank))
if not os.path.isdir(base):
try:
os.makedirs(base)
except OSError as exc:
raise SaltCacheError(
'The cache directory, {0}, does not exist and could not be '
'created: {1}'.format(base, exc)
)
outfile = os.path.join(base, '{0}.p'.format(key))
tmpfh, tmpfname = tempfile.mkstemp(dir=base)
os.close(tmpfh)
try:
with salt.utils.fopen(tmpfname, 'w+b') as fh_:
fh_.write(__context__['serial'].dumps(data))
# On Windows, os.rename will fail if the destination file exists.
salt.utils.atomicfile.atomic_rename(tmpfname, outfile)
except IOError as exc:
raise SaltCacheError(
'There was an error writing the cache file, {0}: {1}'.format(
base, exc
)
)
def fetch(bank, key, cachedir):
"""
Fetch information from a file.
"""
inkey = False
key_file = os.path.join(cachedir, os.path.normpath(bank),
"{}.p".format(key))
if not os.path.isfile(key_file):
# The bank includes the full filename, and the key is inside the file
key_file = os.path.join(cachedir, os.path.normpath(bank) + ".p")
inkey = True
if not os.path.isfile(key_file):
log.debug('Cache file "%s" does not exist', key_file)
return {}
try:
with salt.utils.files.fopen(key_file, "rb") as fh_:
if inkey:
return salt.payload.load(fh_)[key]
else:
return salt.payload.load(fh_)
except OSError as exc:
raise SaltCacheError(
'There was an error reading the cache file "{}": {}'.format(
key_file, exc))
def run_query(conn, query, retries=3):
'''
Get a cursor and run a query. Reconnect up to `retries` times if
needed.
Returns: cursor, affected rows counter
Raises: SaltCacheError, AttributeError, OperationalError
'''
try:
cur = conn.cursor()
out = cur.execute(query)
return cur, out
except (AttributeError, OperationalError) as e:
if retries == 0:
raise
# reconnect creating new client
sleep(_RECONNECT_INTERVAL_SEC)
if conn is None:
log.debug("mysql_cache: creating db connection")
else:
log.info("mysql_cache: recreating db connection due to: %r", e)
global client
client = MySQLdb.connect(**_mysql_kwargs)
return run_query(client, query, retries - 1)
except Exception as e: # pylint: disable=broad-except
if len(query) > 150:
query = query[:150] + "<...>"
raise SaltCacheError("Error running {0}: {1}".format(query, e))
def run_query(conn, query, retries=3, args=None):
"""
Get a cursor and run a query. Reconnect up to `retries` times if
needed.
Returns: cursor, affected rows counter
Raises: SaltCacheError, AttributeError, OperationalError
"""
try:
cur = conn.cursor()
if args is None or args == {}:
log.debug("Doing query: %s", query)
out = cur.execute(query)
else:
log.debug("Doing query: %s args: %s ", query, repr(args))
out = cur.execute(query, args)
return cur, out
except (AttributeError, OperationalError) as e:
if retries == 0:
raise
# reconnect creating new client
time.sleep(_RECONNECT_INTERVAL_SEC)
if conn is None:
log.debug("mysql_cache: creating db connection")
else:
log.info("mysql_cache: recreating db connection due to: %r", e)
__context__["mysql_client"] = MySQLdb.connect(**__context__["mysql_kwargs"])
return run_query(__context__["mysql_client"], query, args, retries - 1)
except Exception as e: # pylint: disable=broad-except
if len(query) > 150:
query = query[:150] + "<...>"
raise SaltCacheError("Error running {}: {}".format(query, e))
def store(bank, key, data):
"""
Store the data in a Redis key.
"""
redis_server = _get_redis_server()
redis_pipe = redis_server.pipeline()
redis_key = _get_key_redis_key(bank, key)
redis_bank_keys = _get_bank_keys_redis_key(bank)
try:
_build_bank_hier(bank, redis_pipe)
value = salt.payload.dumps(data)
redis_pipe.set(redis_key, value)
log.debug("Setting the value for %s under %s (%s)", key, bank, redis_key)
redis_pipe.sadd(redis_bank_keys, key)
# localfs cache truncates the timestamp to int only. We'll do the same.
redis_pipe.set(
_get_timestamp_key(bank=bank, key=key),
salt.payload.dumps(int(time.time())),
)
log.debug("Adding %s to %s", key, redis_bank_keys)
redis_pipe.execute()
except (RedisConnectionError, RedisResponseError) as rerr:
mesg = "Cannot set the Redis cache key {rkey}: {rerr}".format(
rkey=redis_key, rerr=rerr
)
log.error(mesg)
raise SaltCacheError(mesg)
def flush(bank, key=None, cachedir=None):
'''
Remove the key from the cache bank with all the key content.
'''
if cachedir is None:
cachedir = __cachedir()
try:
if key is None:
target = os.path.join(cachedir, os.path.normpath(bank))
if not os.path.isdir(target):
return False
shutil.rmtree(target)
else:
target = os.path.join(cachedir, os.path.normpath(bank), '{0}.p'.format(key))
if not os.path.isfile(target):
return False
os.remove(target)
except OSError as exc:
raise SaltCacheError(
'There was an error removing "{0}": {1}'.format(
target, exc
)
)
return True
def fetch(bank, key, cachedir):
'''
Fetch information from a file.
'''
inkey = False
key_file = os.path.join(cachedir, os.path.normpath(bank), '{0}.p'.format(key))
if not os.path.isfile(key_file):
# The bank includes the full filename, and the key is inside the file
key_file = os.path.join(cachedir, os.path.normpath(bank) + '.p')
inkey = True
if not os.path.isfile(key_file):
log.debug('Cache file "%s" does not exist', key_file)
return {}
try:
with salt.utils.fopen(key_file, 'rb') as fh_:
if inkey:
return __context__['serial'].load(fh_)[key]
else:
return __context__['serial'].load(fh_)
except IOError as exc:
raise SaltCacheError(
'There was an error reading the cache file "{0}": {1}'.format(
key_file, exc
)
)
def store(bank, key, data):
'''
Store the data in a Redis key.
'''
redis_server = _get_redis_server()
redis_pipe = redis_server.pipeline()
redis_key = _get_key_redis_key(bank, key)
redis_bank_keys = _get_bank_keys_redis_key(bank)
try:
_build_bank_hier(bank, redis_pipe)
value = __context__['serial'].dumps(data)
redis_pipe.set(redis_key, value)
log.debug('Setting the value for {key} under {bank} ({redis_key})'.format(
key=key,
bank=bank,
redis_key=redis_key
))
redis_pipe.sadd(redis_bank_keys, key)
log.debug('Adding {key} to {bkey}'.format(key=key, bkey=redis_bank_keys))
redis_pipe.execute()
except (RedisConnectionError, RedisResponseError) as rerr:
mesg = 'Cannot set the Redis cache key {rkey}: {rerr}'.format(rkey=redis_key,
rerr=rerr)
log.error(mesg)
raise SaltCacheError(mesg)
def contains(bank, key):
"""
Checks if the specified bank contains the specified key.
"""
try:
c_key = "{}/{}".format(bank, key or "")
_, value = api.kv.get(c_key, keys=True)
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
"There was an error getting the key, {}: {}".format(c_key, exc))
return value is not None
def store(bank, key, data):
'''
Store a key value.
'''
c_key = '{0}/{1}'.format(bank, key)
try:
c_data = __context__['serial'].dumps(data)
api.kv.put(c_key, c_data)
except Exception as exc:
raise SaltCacheError(
'There was an error writing the key, {0}: {1}'.format(c_key, exc))
def store(bank, key, data):
"""
Store a key value.
"""
c_key = "{}/{}".format(bank, key)
try:
c_data = __context__["serial"].dumps(data)
api.kv.put(c_key, c_data)
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
"There was an error writing the key, {}: {}".format(c_key, exc))
def ls(bank):
"""
Return an iterable object containing all entries stored in the specified
bank.
"""
_init_client()
path = "{}/{}".format(path_prefix, bank)
try:
return _walk(client.read(path))
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
'There was an error getting the key "{}": {}'.format(bank, exc))
def ls(bank):
'''
Return an iterable object containing all entries stored in the specified
bank.
'''
_init_client()
path = '{0}/{1}'.format(path_prefix, bank)
try:
return _walk(client.read(path))
except Exception as exc:
raise SaltCacheError(
'There was an error getting the key "{0}": {1}'.format(bank, exc))
def store(bank, key, data):
"""
Store a key value.
"""
_init_client()
etcd_key = "{}/{}/{}".format(path_prefix, bank, key)
try:
value = __context__["serial"].dumps(data)
client.write(etcd_key, base64.b64encode(value))
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
"There was an error writing the key, {}: {}".format(etcd_key, exc))
def contains(bank, key):
'''
Checks if the specified bank contains the specified key.
'''
redis_server = _get_redis_server()
bank_redis_key = _get_bank_redis_key(bank)
try:
return redis_server.sismember(bank_redis_key, key)
except (RedisConnectionError, RedisResponseError) as rerr:
mesg = 'Cannot retrieve the Redis cache key {rkey}: {rerr}'.format(
rkey=bank_redis_key, rerr=rerr)
log.error(mesg)
raise SaltCacheError(mesg)
def flush(bank, key=None):
'''
Remove the key from the cache bank with all the key content.
'''
if key is None:
c_key = bank
else:
c_key = '{0}/{1}'.format(bank, key)
try:
return api.kv.delete(c_key, recurse=key is None)
except Exception as exc:
raise SaltCacheError(
'There was an error removing the key, {0}: {1}'.format(c_key, exc))
def getlist(bank):
'''
Return an iterable object containing all entries stored in the specified bank.
'''
base = os.path.join(__opts__['cachedir'], os.path.normpath(bank))
if not os.path.isdir(base):
return []
try:
return os.listdir(base)
except OSError as exc:
raise SaltCacheError(
'There was an error accessing directory "{0}": {1}'.format(
base, exc))
def store(bank, key, data):
'''
Store a key value.
'''
_init_client()
data = __context__['serial'].dumps(data)
query = "REPLACE INTO {0} (bank, etcd_key, data) values('{1}', '{2}', " \
"'{3}')".format(_table_name, bank, key, data)
cur, cnt = run_query(client, query)
cur.close()
if cnt not in (1, 2):
raise SaltCacheError('Error storing {0} {1} returned {2}'.format(
bank, key, cnt))
def fetch(bank, key):
"""
Fetch a key value.
"""
c_key = "{}/{}".format(bank, key)
try:
_, value = api.kv.get(c_key)
if value is None:
return {}
return salt.payload.loads(value["Value"])
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
"There was an error reading the key, {}: {}".format(c_key, exc))
def fetch(bank, key):
'''
Fetch a key value.
'''
c_key = '{0}/{1}'.format(bank, key)
try:
_, value = api.kv.get(c_key)
if value is None:
return {}
return __context__['serial'].loads(value['Value'])
except Exception as exc:
raise SaltCacheError(
'There was an error reading the key, {0}: {1}'.format(c_key, exc))
def store(bank, key, data):
'''
Store a key value.
'''
_init_client()
etcd_key = '{0}/{1}/{2}'.format(path_prefix, bank, key)
try:
value = __context__['serial'].dumps(data)
client.write(etcd_key, base64.b64encode(value))
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
'There was an error writing the key, {0}: {1}'.format(
etcd_key, exc))
def flush(bank, key=None):
"""
Remove the key from the cache bank with all the key content.
"""
if key is None:
c_key = bank
else:
c_key = "{}/{}".format(bank, key)
try:
return api.kv.delete(c_key, recurse=key is None)
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
"There was an error removing the key, {}: {}".format(c_key, exc))
def store(bank, key, data):
'''
Store a key value.
'''
_init_client()
etcd_key = '{0}/{1}/{2}'.format(path_prefix, bank, key)
try:
value = __context__['serial'].dumps(data)
client.set(etcd_key, value)
except Exception as exc:
raise SaltCacheError(
'There was an error writing the key, {0}: {1}'.format(
etcd_key, exc))
def store(bank, key, data):
"""
Store a key value.
"""
_init_client()
data = __context__["serial"].dumps(data)
query = b"REPLACE INTO {} (bank, etcd_key, data) values('{}', '{}', '{}')".format(
_table_name, bank, key, data)
cur, cnt = run_query(client, query)
cur.close()
if cnt not in (1, 2):
raise SaltCacheError("Error storing {} {} returned {}".format(
bank, key, cnt))
def store(bank, key, data):
"""
Store a key value.
"""
c_key = "{}/{}".format(bank, key)
tstamp_key = "{}/{}{}".format(bank, key, _tstamp_suffix)
try:
c_data = salt.payload.dumps(data)
api.kv.put(c_key, c_data)
api.kv.put(tstamp_key, salt.payload.dumps(int(time.time())))
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
"There was an error writing the key, {}: {}".format(c_key, exc))
def updated(bank, key):
"""
Return the Unix Epoch timestamp of when the key was last updated. Return
None if key is not found.
"""
c_key = "{}/{}{}".format(bank, key, _tstamp_suffix)
try:
_, value = api.kv.get(c_key)
if value is None:
return None
return salt.payload.loads(value["Value"])
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
"There was an error reading the key, {}: {}".format(c_key, exc))
def fetch(bank, key):
"""
Fetch a key value.
"""
_init_client()
etcd_key = "{}/{}/{}".format(path_prefix, bank, key)
try:
value = client.read(etcd_key).value
return __context__["serial"].loads(base64.b64decode(value))
except etcd.EtcdKeyNotFound:
return {}
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
"There was an error reading the key, {}: {}".format(etcd_key, exc))
def updated(bank, key, cachedir):
'''
Return the epoch of the mtime for this cache file
'''
key_file = os.path.join(cachedir, os.path.normpath(bank),
'{0}.p'.format(key))
if not os.path.isfile(key_file):
log.warning('Cache file "%s" does not exist', key_file)
return None
try:
return int(os.path.getmtime(key_file))
except IOError as exc:
raise SaltCacheError(
'There was an error reading the mtime for "{0}": {1}'.format(
key_file, exc))
def contains(bank, key):
"""
Checks if the specified bank contains the specified key.
"""
_init_client()
etcd_key = "{}/{}/{}".format(path_prefix, bank, key)
try:
r = client.read(etcd_key)
# return True for keys, not dirs
return r.dir is False
except etcd.EtcdKeyNotFound:
return False
except Exception as exc: # pylint: disable=broad-except
raise SaltCacheError(
"There was an error getting the key, {}: {}".format(etcd_key, exc))
