def _delete_item(self, key: str) -> bool:
status = False
path = Path(self.context.PATH)
if not path.exists():
logger.debug(
"Tried to delete item in directory that does not exist")
else:
count = 0
for filename in path.glob(f'{key}*'):
if filename.exists():
filename.unlink()
count += 1
if count > 0:
status = True
return status
def run(self):
"""
run is responsible for the flow WorkManager. It iterates as long as there is work being processed
and will shutdown when either the work is complete or no work has been picked up.
If new work is found while the ProducerConsumer is still running, run will add it to the work being
done. Otherwise, if ProducerConsumer is finishing up, run will return the work to the queue.
If no work is found, the loop should break.
:return:
"""
logger.info('WorkManager has started...')
while True:
work = self.work_queue.get()
if work is not None:
if not self.producer_consumer:
self.build_producer_consumer()
self.register_producer_for_work(work)
if self.producer_consumer_thread and self.producer_consumer_thread.is_alive(
):
if not self.producer_consumer.event_manager.is_set():
event = self.producer_consumer.event_manager.get(
work, None)
if event is None or event.is_set():
self.producer_consumer.start_producer(work)
else:
logger.warn(
f"Somehow received {work} work while already processing that work"
)
else:
logger.warn(
"ProducerConsumer thread is already shutting down, replacing work in queue"
)
self.work_queue.reset_work(work)
break
else:
self.start_producer_consumer()
if self.producer_consumer_thread is None:
break
elif not self.producer_consumer_thread.is_alive():
logger.info(
"ProducerConsumer is dead, shutting down WorkManager")
break
logger.debug(
"Waiting for 10 seconds before checking for more work")
sleep(self.time_to_wait)
def _get_mysql_connection(dict_cursor=False, unbuffered=False, **kwargs):
"""
Obtain `pymysql.Connection <pymysql.connections.Connection>` instance
for given DB url.
:param bool dict_cursor: use `~pymysql.cursors.DictCursor` for cursors,
so that cursor will return records as dicts rather than tuples.
Shorthand for `cursorclass = pymysql.cursors.DictCursor`.
:param bool unbuffered:
use `~pymysql.cursors.SSCursor`
which does not buffer all response data in memory.
The downside is that it cannot report number of records
and navigate them only forwards.
This option can be combined with ``dict_cursor``
which will result in `~pymysql.cursors.SSDictCursor`.
:param kwargs: additional attributes to pass to `pymysql`.
Common examples are ``password`` or ``database``.
"""
import pymysql
logger.info('Obtaining database connection...')
params = dict()
params['cursorclass'] = getattr(
pymysql.cursors, '{}{}Cursor'.format(
'SS' if unbuffered else '',
'Dict' if dict_cursor else '',
))
params.update(kwargs)
if not all(params.get(k) for k in ('host', 'user', 'password')):
raise ValueError(
'Host, user and password are mandatory and should be provided in kwargs'
)
print_params = params.copy()
print_params['password'] = '******'
logger.debug(
'Connecting to DB using these credentials (password masked): %s',
print_params)
conn = pymysql.connect(**params)
logger.debug('Obtained DB connection: %s', conn)
return conn
def _put_item(self, key: str) -> (bool, str):
if not Path(self.context.PATH).exists():
logger.debug("Creating queue path")
Path(self.context.PATH).mkdir(parents=True)
if Path(self.context.PATH, f'{key}.lock').exists():
logger.info("key exists in queue and is locked")
return False, ErrorMessages.LOCKED_EXISTS
path = Path(self.context.PATH, key)
if path.exists():
logger.info("key exists in queue and is unlocked")
return False, ErrorMessages.UNLOCKED_EXISTS
try:
logger.info(f"attempting to create item for {key} in queue")
path.touch()
except Exception as e:
logger.exception(
"Something unexpected happened when writing the file")
return False, e
return True, None
def get_connection(context: DatabaseContext = None,
connection: Connection = None,
**kwargs) -> PyMySQLConnection:
"""
Given a DatabaseContext object or a Connection object, return an instance of a PyMySQLConnection
For a DatabaseContext argument, _get_mysql_connection will be called and kwargs will be passed
:param context: optional DatabaseContext object
:param connection: optional Connection object
:param kwargs: to be passed to the connecting function
:return: None if no arguments provided else Connection object
"""
warnings.warn("Use get_connection_object", DeprecationWarning)
if connection:
# This allows get_connection to be used with mock connections, allowing for testing of units that call it
logger.debug('Using connection object received through arguments.')
return connection
elif context:
db_dict = {**_parse_context_attrs(context), **kwargs}
return _get_mysql_connection(**db_dict)
return None
def get_connection_object(context: DatabaseContext = None,
connection: Connection = None,
**_kwargs):
"""
Given a DatabaseContext object or a Connection object, return an instance of a
kasasa_common Connection interface
:param context: optional DatabaseContext object
:param connection: optional Connection object
:return: None if no arguments provided else Connection object
"""
from .mysql_connection import MySQLConnection
if connection:
# This allows get_connection to be used with mock connections, allowing for testing of units that call it
logger.debug('Using connection object received through arguments.')
return connection
elif context:
retval = dict(mysql=MySQLConnection).get(context.DATABASE_TYPE,
None)(context)
if not retval:
raise ValueError("No valid DATABASE_TYPE existed in context")
return retval
raise ValueError("No context or connection provided")