def _delete_item(self, key: str) -> bool: status = False path = Path(self.context.PATH) if not path.exists(): logger.debug( "Tried to delete item in directory that does not exist") else: count = 0 for filename in path.glob(f'{key}*'): if filename.exists(): filename.unlink() count += 1 if count > 0: status = True return status
def run(self): """ run is responsible for the flow WorkManager. It iterates as long as there is work being processed and will shutdown when either the work is complete or no work has been picked up. If new work is found while the ProducerConsumer is still running, run will add it to the work being done. Otherwise, if ProducerConsumer is finishing up, run will return the work to the queue. If no work is found, the loop should break. :return: """ logger.info('WorkManager has started...') while True: work = self.work_queue.get() if work is not None: if not self.producer_consumer: self.build_producer_consumer() self.register_producer_for_work(work) if self.producer_consumer_thread and self.producer_consumer_thread.is_alive( ): if not self.producer_consumer.event_manager.is_set(): event = self.producer_consumer.event_manager.get( work, None) if event is None or event.is_set(): self.producer_consumer.start_producer(work) else: logger.warn( f"Somehow received {work} work while already processing that work" ) else: logger.warn( "ProducerConsumer thread is already shutting down, replacing work in queue" ) self.work_queue.reset_work(work) break else: self.start_producer_consumer() if self.producer_consumer_thread is None: break elif not self.producer_consumer_thread.is_alive(): logger.info( "ProducerConsumer is dead, shutting down WorkManager") break logger.debug( "Waiting for 10 seconds before checking for more work") sleep(self.time_to_wait)
def _get_mysql_connection(dict_cursor=False, unbuffered=False, **kwargs): """ Obtain `pymysql.Connection <pymysql.connections.Connection>` instance for given DB url. :param bool dict_cursor: use `~pymysql.cursors.DictCursor` for cursors, so that cursor will return records as dicts rather than tuples. Shorthand for `cursorclass = pymysql.cursors.DictCursor`. :param bool unbuffered: use `~pymysql.cursors.SSCursor` which does not buffer all response data in memory. The downside is that it cannot report number of records and navigate them only forwards. This option can be combined with ``dict_cursor`` which will result in `~pymysql.cursors.SSDictCursor`. :param kwargs: additional attributes to pass to `pymysql`. Common examples are ``password`` or ``database``. """ import pymysql logger.info('Obtaining database connection...') params = dict() params['cursorclass'] = getattr( pymysql.cursors, '{}{}Cursor'.format( 'SS' if unbuffered else '', 'Dict' if dict_cursor else '', )) params.update(kwargs) if not all(params.get(k) for k in ('host', 'user', 'password')): raise ValueError( 'Host, user and password are mandatory and should be provided in kwargs' ) print_params = params.copy() print_params['password'] = '******' logger.debug( 'Connecting to DB using these credentials (password masked): %s', print_params) conn = pymysql.connect(**params) logger.debug('Obtained DB connection: %s', conn) return conn
def _put_item(self, key: str) -> (bool, str): if not Path(self.context.PATH).exists(): logger.debug("Creating queue path") Path(self.context.PATH).mkdir(parents=True) if Path(self.context.PATH, f'{key}.lock').exists(): logger.info("key exists in queue and is locked") return False, ErrorMessages.LOCKED_EXISTS path = Path(self.context.PATH, key) if path.exists(): logger.info("key exists in queue and is unlocked") return False, ErrorMessages.UNLOCKED_EXISTS try: logger.info(f"attempting to create item for {key} in queue") path.touch() except Exception as e: logger.exception( "Something unexpected happened when writing the file") return False, e return True, None
def get_connection(context: DatabaseContext = None, connection: Connection = None, **kwargs) -> PyMySQLConnection: """ Given a DatabaseContext object or a Connection object, return an instance of a PyMySQLConnection For a DatabaseContext argument, _get_mysql_connection will be called and kwargs will be passed :param context: optional DatabaseContext object :param connection: optional Connection object :param kwargs: to be passed to the connecting function :return: None if no arguments provided else Connection object """ warnings.warn("Use get_connection_object", DeprecationWarning) if connection: # This allows get_connection to be used with mock connections, allowing for testing of units that call it logger.debug('Using connection object received through arguments.') return connection elif context: db_dict = {**_parse_context_attrs(context), **kwargs} return _get_mysql_connection(**db_dict) return None
def get_connection_object(context: DatabaseContext = None, connection: Connection = None, **_kwargs): """ Given a DatabaseContext object or a Connection object, return an instance of a kasasa_common Connection interface :param context: optional DatabaseContext object :param connection: optional Connection object :return: None if no arguments provided else Connection object """ from .mysql_connection import MySQLConnection if connection: # This allows get_connection to be used with mock connections, allowing for testing of units that call it logger.debug('Using connection object received through arguments.') return connection elif context: retval = dict(mysql=MySQLConnection).get(context.DATABASE_TYPE, None)(context) if not retval: raise ValueError("No valid DATABASE_TYPE existed in context") return retval raise ValueError("No context or connection provided")