def __init__(self,
*args,
enter_reconnect: Optional[bool] = None,
exit_close: Optional[bool] = None,
**kwargs):
self.ins_enter_reconnect = empty_if(enter_reconnect,
self.adapter_enter_reconnect)
self.ins_exit_close = empty_if(exit_close, self.adapter_exit_close)
def get_ssl_context(
verify_cert: bool = False, check_hostname: Optional[bool] = None, verify_mode: Optional[int] = None, **kwargs
) -> ssl.SSLContext:
check_hostname = empty_if(check_hostname, is_true(verify_cert))
verify_mode = empty_if(verify_mode, ssl.CERT_REQUIRED if verify_cert else ssl.CERT_NONE)
ctx = ssl.create_default_context()
ctx.check_hostname = check_hostname
ctx.verify_mode = verify_mode
return ctx
def __init__(self,
db_file: str = None,
memory_persist=False,
use_pickle: bool = None,
connection_kwargs: dict = None,
*args,
**kwargs):
"""
:class:`.AsyncSqliteCache` uses an auto-generated database filename / path by default, based on the name of the currently running
script ( retrieved from ``sys.argv[0]`` ), allowing for persistent caching - without any manual configuration of the adapter,
nor the requirement for any running background services such as ``redis`` / ``memcached``.
:param str db_file: (Optional) Name of / path to Sqlite3 database file to create/use for the cache.
:param bool memory_persist: Use a shared in-memory database, which can be accessed by other instances of this class (in this
process) - which is cleared after all memory connections are closed.
Shortcut for ``db_file='file::memory:?cache=shared'``
:param bool use_pickle: (Default: ``True``) Use the built-in ``pickle`` to serialise values before
storing in Sqlite3, and un-serialise when loading from Sqlite3
:param dict connection_kwargs: (Optional) Additional / overriding kwargs to pass to :meth:`sqlite3.connect` when
:class:`.AsyncSqliteCacheManager` initialises it's sqlite3 connection.
:keyword int purge_every: (Default: 300) Expired + abandoned cache records are purged using the DB manager method
:meth:`.AsyncSqliteCacheManager.purge_expired` during :meth:`.get` / :meth:`.set` calls. To avoid
performance issues, the actual :meth:`.AsyncSqliteCacheManager.purge_expired` method is only called
if at least ``purge_every`` seconds have passed since the last purge was
triggered ( :attr:`.last_purged_expired` )
"""
from privex.helpers.cache.post_deps import AsyncSqliteCacheManager
super().__init__(*args, **kwargs)
self.db_file: str = empty_if(db_file,
AsyncSqliteCacheManager.DEFAULT_DB)
self.db_folder = None
if ':memory:' not in self.db_file:
if not isabs(self.db_file):
self.db_file = join(AsyncSqliteCacheManager.DEFAULT_DB_FOLDER,
self.db_file)
self.db_folder = dirname(self.db_file)
if not exists(self.db_folder):
log.debug("Folder for database doesn't exist. Creating: %s",
self.db_folder)
makedirs(self.db_folder)
self.connection_kwargs = empty_if(connection_kwargs, {}, itr=True)
self.memory_persist = is_true(memory_persist)
self._wrapper = None
self.purge_every = kwargs.get('purge_every', 300)
self.use_pickle = self.pickle_default if use_pickle is None else use_pickle
def _connect(self,
db=None,
*args,
connection_kwargs=None,
memory_persist=None,
**kwargs):
c_kwargs = dict(connection_kwargs=empty_if(connection_kwargs,
self.connection_kwargs),
memory_persist=empty_if(memory_persist,
self.memory_persist))
c_kwargs = {**c_kwargs, **kwargs}
from privex.helpers.cache.post_deps import AsyncSqliteCacheManager
self._wrapper = AsyncSqliteCacheManager(empty_if(db, self.db_file),
*args, **c_kwargs)
return self._wrapper
def _get_threadstore(name=None, fallback=None, thread_id=None) -> Any:
thread_id = empty_if(thread_id, threading.get_ident())
if thread_id not in __STORE['threads']:
__STORE['threads'][thread_id] = {}
thread_store: dict = __STORE['threads'][thread_id]
if name is None:
return thread_store
return thread_store.get(name, fallback)
def reset_adapter(cls,
default: CLSCacheAdapter = default_adapter,
*args,
**kwargs) -> INSCacheAdapter:
"""
Re-create the adapter instance at :attr:`.cache_instance` with the same adapter class (assuming it's set)
"""
adp = cls.get_adapter(default, *args, **kwargs)
n_args, n_kwargs = empty_if(args, cls.instance_args, itr=True), {
**cls.instance_kwargs,
**kwargs
}
c = adp.__class__
cls.cache_instance = c.__init__(c(), *n_args, **n_kwargs)
return cls.cache_instance
def _pfx_key(cls,
key: Union[str, Callable[[Any], str]],
auto_prefix: bool = True,
_auto_cache=True,
call_args: list = None,
call_kwargs: dict = None,
_lock: ANY_LCK = None) -> str:
"""
Add this class's cache key prefix to ``key`` if it isn't already prefixed
:param str|callable key: The key to prepend the cache key prefix onto - if not already prefixed.
This may optionally be a function (e.g. a lambda) which returns a cache key to be auto-prefixed,
and any necessary positional/keyword arguments for the function may be specified using the
arguments ``call_args`` and ``call_kwargs``
:param bool auto_prefix: This argument is mainly used by internal methods to reduce the need to copy/paste handling code
which allows users to request that a method does not attempt to auto-prefix the key they entered.
:param bool _auto_cache: This is a boolean key, which controls whether or not keys are automatically logged to the cache key log,
which is a list of all known cache keys for a given class. Uses :meth:`.log_cache_key`
:param list call_args: If ``key`` is a callable (e.g. a lambda), ``call_args`` can be set to a list of positional arguments
to pass to the callable function ``key``
:param dict call_kwargs: If ``key`` is a callable (e.g. a lambda), ``call_kwargs`` can be set to a :class:`.dict` of
keyword arguments to pass to the callable function ``key``
:param ANY_LCK _lock: This method itself does not use a lock, but it calls upon :meth:`.log_cache_key` which does use
a lock. You may optionally pass a :class:`.Lock` instance if needed, e.g. to prevent a conflict
where the calling function/method has already acquired the class-level lock.
It can also be set to the dummy type :class:`.NO_LOCK` ``NO_LOCK`` to prevent using a lock.
:return str new_key: The original ``key`` after it may or may not have had a prefix prepended to it.
"""
if callable(key):
call_args, call_kwargs = auto_list(call_args), empty_if(
call_kwargs, {}, itr=True, zero=True)
key = key(*call_args, **call_kwargs)
key = key if key.startswith(
cls.cache_prefix) or not auto_prefix else cls.cache_sep.join(
[cls.cache_prefix, key])
if _auto_cache: # By default, _auto_cache is enabled, which means we log all created keys to the cache key log
cls.log_cache_key(key, _lock=_lock)
return key
async def get_current_commit(self,
version: str = None,
repo: str = None) -> str:
"""
Get current commit hash. Optionally specify ``version`` to get the current commit hash for a branch or tag.
**Examples**::
>>> g = Git()
>>> await g.get_current_commit()
'ac52b28f551825160785f9ea7e96f86ccc869cc1'
>>> await g.get_current_commit('2.0.0') # Get the commit hash for the tag 2.0.0
'598584a447ba63212ac3fe798c01941badf1c194'
:param str version: Optionally specify a branch / tag to get the current commit hash for.
:param str repo: Optionally specify a specific local repository path to run ``git`` within.
:return str commit_hash: The current Git commit hash
"""
return await self.git("rev-parse",
empty_if(version, self.default_version),
repo=repo)
async def get_current_tag(self,
version: str = None,
repo: str = None) -> str:
"""
Get the latest tag on this branch - useful for detecting current version of your python application.
**Examples**::
>>> g = Git()
>>> await g.get_current_tag() # Get the latest tag on the active branch
'2.5.0'
>>> await g.get_current_tag('develop') # Get the latest tag on the branch 'develop'
'2.5.3'
:param str version: Optionally specify a branch / tag to get the latest tag for.
:param str repo: Optionally specify a specific local repository path to run ``git`` within.
:return str current_tag: The name of the latest tag on this branch.
"""
return await self.git("describe",
"--abbrev=0",
"--tags",
empty_if(version, self.default_version),
repo=repo)
def set_sleep_time(secs: int = None):
CacheManagerExample.sleep_multiplier = float(
empty_if(secs, CacheManagerExample.default_sleep_multiplier))
def test_hosts(hosts: List[str] = None,
ipver: str = 'any',
timeout: AnyNum = None,
**kwargs) -> bool:
randomise = is_true(kwargs.get('randomise', True))
max_hosts = kwargs.get('max_hosts', settings.NET_CHECK_HOST_COUNT_TRY)
if max_hosts is not None: max_hosts = int(max_hosts)
timeout = empty_if(timeout,
empty_if(socket.getdefaulttimeout(), 4, zero=True),
zero=True)
v4h, v6h = list(settings.V4_TEST_HOSTS), list(settings.V6_TEST_HOSTS)
if randomise: random.shuffle(v4h)
if randomise: random.shuffle(v6h)
if empty(hosts, True, True):
# if empty(ipver, True, True) or ipver in ['any', 'all', 'both', 10, '10', '46', 46]:
# settings.V4_CHECKED_AT
if isinstance(ipver, str): ipver = ipver.lower()
if ipver in [4, '4', 'v4', 'ipv4']:
hosts = v4h
ipver = 4
elif ipver in [6, '6', 'v6', 'ipv6']:
hosts = v6h
ipver = 6
else:
ipver = 'any'
if max_hosts:
hosts = v4h[:int(ceil(max_hosts /
2))] + v6h[:int(ceil(max_hosts / 2))]
else:
hosts = v4h + v6h
if max_hosts: hosts = hosts[:max_hosts]
# st4_empty = any([empty(settings.HAS_WORKING_V4, True, True), empty(settings.V4_CHECKED_AT, True, True)])
# st6_empty = any([empty(settings.HAS_WORKING_V6, True, True), empty(settings.V6_CHECKED_AT, True, True)])
# if ipver == 6 and not st6_empty and settings.V6_CHECKED_AT > datetime.utcnow():
# # if settings.V6_CHECKED_AT > datetime.utcnow()
# log.debug("Returning cached IPv6 status: working = %s", settings.HAS_WORKING_V6)
# return settings.HAS_WORKING_V6
# if ipver == 4 and not st4_empty and settings.V4_CHECKED_AT > datetime.utcnow():
# # if settings.V6_CHECKED_AT > datetime.utcnow()
# log.debug("Returning cached IPv4 status: working = %s", settings.HAS_WORKING_V4)
# return settings.HAS_WORKING_V4
# if ipver == 'any' and any([not st4_empty, not st6_empty]) and settings.V4_CHECKED_AT > datetime.utcnow():
# # if settings.V6_CHECKED_AT > datetime.utcnow()
# if st4_empty:
# log.debug("test_hosts being requested for 'any' ip ver. IPv6 status cached, but not IPv4 status. Checking IPv4 status...")
# check_v4()
# if st6_empty:
# log.debug("test_hosts being requested for 'any' ip ver. IPv4 status cached, but not IPv6 status. Checking IPv6 status...")
# check_v6()
# # if not st4_empty and not st6_empty:
# log.debug(
# "Returning status %s based on: Working IPv4 = %s || Working IPv6 = %s",
# settings.HAS_WORKING_V4 or settings.HAS_WORKING_V6, settings.HAS_WORKING_V4, settings.HAS_WORKING_V6
# )
# return settings.HAS_WORKING_V4 or settings.HAS_WORKING_V6
# max_hosts = int(kwargs.get('max_hosts', settings.NET_CHECK_HOST_COUNT_TRY))
min_hosts_pos = int(
kwargs.get('required_positive', settings.NET_CHECK_HOST_COUNT))
# hosts = empty_if(hosts, settings.V4_TEST_HOSTS, itr=True)
hosts = [x for x in hosts]
if randomise: random.shuffle(hosts)
if len(hosts) > max_hosts: hosts = hosts[:max_hosts]
total_hosts = len(hosts)
total_working, total_broken = 0, 0
log.debug("Testing %s hosts with IP version '%s' - timeout: %s",
total_hosts, ipver, timeout)
port = 80
for h in hosts:
try:
nh = h.split(':')
if len(nh) > 1:
port = int(nh[-1])
h = ':'.join(nh[:-1])
else:
h = ':'.join(nh)
log.warning(
"Host is missing port: %s - falling back to port 80")
port = 80
log.debug("Checking host %s via port %s + IP version '%s'", h,
port, ipver)
if port == 80:
res = check_host_http(h,
port,
ipver,
throw=False,
timeout=timeout)
else:
res = check_host(h, port, ipver, throw=False, timeout=timeout)
if res:
total_working += 1
log.debug(
"check_host for %s came back true. incremented working hosts: %s",
h, total_working)
else:
total_broken += 1
log.debug(
"check_host for %s came back false. incremented broken hosts: %s",
h, total_broken)
except Exception as e:
log.warning("Exception while checking host %s port %s", h, port)
working = total_working >= min_hosts_pos
log.info(
"test_hosts - proto: %s - protocol working? %s || total hosts: %s || working hosts: %s || broken hosts: %s",
ipver, working, total_hosts, total_working, total_broken)
return working
async def test_hosts_async(hosts: List[str] = None,
ipver: str = 'any',
timeout: AnyNum = None,
**kwargs) -> bool:
randomise = is_true(kwargs.get('randomise', True))
max_hosts = kwargs.get('max_hosts', settings.NET_CHECK_HOST_COUNT_TRY)
if max_hosts is not None: max_hosts = int(max_hosts)
timeout = empty_if(timeout,
empty_if(socket.getdefaulttimeout(), 4, zero=True),
zero=True)
v4h, v6h = list(settings.V4_TEST_HOSTS), list(settings.V6_TEST_HOSTS)
if randomise: random.shuffle(v4h)
if randomise: random.shuffle(v6h)
if empty(hosts, True, True):
# if empty(ipver, True, True) or ipver in ['any', 'all', 'both', 10, '10', '46', 46]:
# settings.V4_CHECKED_AT
if isinstance(ipver, str): ipver = ipver.lower()
if ipver in [4, '4', 'v4', 'ipv4']:
hosts = v4h
ipver = 4
elif ipver in [6, '6', 'v6', 'ipv6']:
hosts = v6h
ipver = 6
else:
ipver = 'any'
if max_hosts:
hosts = v4h[:int(ceil(max_hosts /
2))] + v6h[:int(ceil(max_hosts / 2))]
else:
hosts = v4h + v6h
if max_hosts: hosts = hosts[:max_hosts]
# st4_empty = any([empty(settings.HAS_WORKING_V4, True, True), empty(settings.V4_CHECKED_AT, True, True)])
# st6_empty = any([empty(settings.HAS_WORKING_V6, True, True), empty(settings.V6_CHECKED_AT, True, True)])
# if ipver == 6 and not st6_empty and settings.V6_CHECKED_AT > datetime.utcnow():
# # if settings.V6_CHECKED_AT > datetime.utcnow()
# log.debug("Returning cached IPv6 status: working = %s", settings.HAS_WORKING_V6)
# return settings.HAS_WORKING_V6
# if ipver == 4 and not st4_empty and settings.V4_CHECKED_AT > datetime.utcnow():
# # if settings.V6_CHECKED_AT > datetime.utcnow()
# log.debug("Returning cached IPv4 status: working = %s", settings.HAS_WORKING_V4)
# return settings.HAS_WORKING_V4
#
# if ipver == 'any' and any([not st4_empty, not st6_empty]) and settings.V4_CHECKED_AT > datetime.utcnow():
# # if settings.V6_CHECKED_AT > datetime.utcnow()
# if st4_empty:
# log.debug("test_hosts being requested for 'any' ip ver. IPv6 status cached, but not IPv4 status. Checking IPv4 status...")
# await check_v4_async()
# if st6_empty:
# log.debug("test_hosts being requested for 'any' ip ver. IPv4 status cached, but not IPv6 status. Checking IPv6 status...")
# await check_v6_async(hosts)
# # if not st4_empty and not st6_empty:
# log.debug(
# "Returning status %s based on: Working IPv4 = %s || Working IPv6 = %s",
# settings.HAS_WORKING_V4 or settings.HAS_WORKING_V6, settings.HAS_WORKING_V4, settings.HAS_WORKING_V6
# )
# return settings.HAS_WORKING_V4 or settings.HAS_WORKING_V6
# max_hosts = int(kwargs.get('max_hosts', settings.NET_CHECK_HOST_COUNT_TRY))
min_hosts_pos = int(
kwargs.get('required_positive', settings.NET_CHECK_HOST_COUNT))
# hosts = empty_if(hosts, settings.V4_TEST_HOSTS, itr=True)
hosts = [x for x in hosts]
if randomise: random.shuffle(hosts)
if len(hosts) > max_hosts: hosts = hosts[:max_hosts]
# port = empty_if(port, 80, zero=True)
total_hosts = len(hosts)
total_working, total_broken = 0, 0
working_list, broken_list = [], []
log.debug("Testing %s hosts with IP version '%s' - timeout: %s",
total_hosts, ipver, timeout)
host_checks = []
host_checks_hosts = []
for h in hosts:
# host_checks.append(
# asyncio.create_task(_test_host_async(h, ipver=ipver, timeout=timeout))
# )
host_checks.append(
asyncio.create_task(
run_coro_thread_async(_test_host_async,
h,
ipver=ipver,
timeout=timeout)))
host_checks_hosts.append(h)
host_checks_res = await asyncio.gather(*host_checks,
return_exceptions=True)
for i, _res in enumerate(host_checks_res):
h = host_checks_hosts[i]
if isinstance(_res, Exception):
log.warning("Exception while checking host %s", h)
total_broken += 1
continue
res, h, port = _res
if res:
total_working += 1
working_list.append(f"{h}:{port}")
log.debug(
"check_host for %s (port %s) came back True (WORKING). incremented working hosts: %s",
h, port, total_working)
else:
total_broken += 1
broken_list.append(f"{h}:{port}")
log.debug(
"check_host for %s (port %s) came back False (! BROKEN !). incremented broken hosts: %s",
h, port, total_broken)
# port = 80
# for h in hosts:
# try:
# h, port, res = await _test_host_async(h, ipver, timeout)
# if res:
# total_working += 1
# log.debug("check_host for %s came back true. incremented working hosts: %s", h, total_working)
# else:
# total_broken += 1
# log.debug("check_host for %s came back false. incremented broken hosts: %s", h, total_broken)
#
# except Exception as e:
# log.warning("Exception while checking host %s port %s", h, port)
working = total_working >= min_hosts_pos
log.info(
"test_hosts - proto: %s - protocol working? %s || total hosts: %s || working hosts: %s || broken hosts: %s",
ipver, working, total_hosts, total_working, total_broken)
log.debug("working hosts: %s", working_list)
log.debug("broken hosts: %s", broken_list)
return working
def _unpfx_key(cls, key: str, prefix: str = None, sep: str = None):
"""Remove the cache key prefix from a given cache key"""
prefix, sep = empty_if(prefix,
cls.cache_prefix), empty_if(sep, cls.cache_sep)
j = key.split(f"{prefix}{sep}")
return j[0] if len(j) == 1 else ''.join(j[1:])
def sock_ver(version) -> Optional[int]:
version = empty_if(version, 'any', zero=True, itr=True)
version = str(version).lower()
if ip_ver_to_int(version) == 4: return socket.AF_INET
if ip_ver_to_int(version) == 6: return socket.AF_INET6
return None
def clean_threadstore(thread_id=None,
name=None,
clean_all: bool = False) -> bool:
"""
Remove the per-thread instance storage in :attr:`.__STORE`, usually called when a thread is exiting.
Can also be used to clear a certain key in all thread stores, or completely clear every key from every thread store.
Example::
>>> def some_thread():
... r = get_redis()
... print('doing something')
... print('cleaning up...')
... clean_threadstore() # With no arguments, it cleans the thread store for the thread that called it.
>>> t = threading.Thread(target=some_thread)
>>> t.start()
>>> t.join()
Usage outside of a thread::
>>> t = threading.Thread(target=some_thread)
>>> t.start()
>>> thread_id = t.ident # Get the thread ID for the started thread
>>> t.join() # Wait for the thread to finish
>>> if thread_id is not None: # Make sure the thread ID isn't None
... clean_threadstore(thread_id) # Cleanup any leftover instances, if there are any.
...
Removing an individual item from thread store::
>>> def some_thread():
... r = get_redis()
... print('doing something')
... print('cleaning up...')
... clean_threadstore(name='redis') # Delete only the key 'redis' from the thread store
Removing an individual item from the thread store for **ALL thread ID's**::
>>> # Remove the redis instance from every thread store
>>> clean_threadstore(name='redis', clean_all=True)
Clearing the entire thread store for every thread ID::
>>> clean_threadstore(clean_all=True)
:param thread_id: The ID of the thread (usually from :func:`threading.get_ident`) to clean the storage for.
If left as None, will use the ID returned by :func:`threading.get_ident`.
:param name: If specified, then only the key ``name`` will be deleted from the thread store, instead of the entire thread store.
:param bool clean_all: (default: ``False``) If ``True`` - when ``name`` is non-empty - that key will be removed from every
thread ID's thread store - while if ``name`` is empty, every single thread ID's thread store is cleared.
"""
thread_id = empty_if(thread_id, threading.get_ident())
if clean_all:
for t_id, ts in __STORE['threads'].items():
ts: dict
if empty(name):
log.debug(
"[clean_threadstore] (clean_all True) Cleaning entire thread store for thread ID '%s'",
t_id)
for n in ts.keys():
log.debug(
"[clean_threadstore] Cleaning '%s' key for thread ID '%s'",
name, t_id)
del ts[n]
continue
log.debug(
"[clean_threadstore] (clean_all True) Cleaning '%s' key for thread ID '%s'",
name, t_id)
if name in ts:
log.debug(
"[clean_threadstore] Found %s in thread ID %s - deleting...",
name, ts)
del ts[name]
return True
if thread_id in __STORE['threads']:
ts = __STORE['threads'][thread_id]
if name is None:
del __STORE['threads'][thread_id]
return True
elif name in ts:
del ts[name]
return True
return False
