def command_whoosh(action=None): if action == "drop": write_handlers.drop_indexes() elif action == "reindex": # drop root privileges if root, obviously the index needs to be # created with the same permissions as the app is running with. drop_privileges() from opmuse.library import library_dao from opmuse.database import database_data, get_session database_data.database = get_session() write_handlers.init_indexes() for artist in library_dao.get_artists(): search.add_artist(artist) for album in library_dao.get_albums(): search.add_album(album) for track in library_dao.get_tracks(): search.add_track(track) write_handlers.commit() database_data.database.remove() database_data.database = None else: parser.error('Needs to provide a valid action (drop, reindex).')
def _unserialize(self): """ Unserialize / load cache objects from database. """ database = get_session() item_count = 0 for object in database.query(CacheObject).all(): item_count += 1 if object.type == 'object': value = pickle.loads(object.value) elif object.type == 'str': value = object.value.decode() elif object.type == 'dict' or object.type == 'list': value = json.loads(object.value.decode()) else: value = object.value cache.storage.set(object.key, object.updated, value) database.remove() log("Unserialized %d cache objects" % item_count)
def _serialize(self): """ Serialize cache objects to database. TODO actually use "updated" var to see if we even need to serialize/persist the changes... TODO lock cache storage while serializing? """ database = get_session() item_count = 0 for key, item in cache.storage.values(): item_count += 1 value = item['value'] updated = item['updated'] count = (database.query(func.count(CacheObject.id)) .filter(CacheObject.key == key).scalar()) # Dont serialize Keep values, this is for example useful for bgtasks # that was added to the queue but never run because of a restart. # This way they will just be triggered again on the next start/use # and we dont have to wait for the Keep time to run out. if value is Keep: continue elif isinstance(value, object): value_type = 'object' else: value_type = type(value).__name__ if value_type == 'object': value = pickle.dumps(value) elif value_type == 'str': value = value.encode() elif value_type == 'dict' or value_type == 'list': value = json.dumps(value).encode() if count > 0: parameters = {'updated': updated} parameters['value'] = value parameters['type'] = value_type database.query(CacheObject).filter(CacheObject.key == key).update(parameters) else: parameters = {'key': key, 'value': value, 'updated': updated, 'type': value_type} database.execute(CacheObject.__table__.insert(), parameters) database.commit() database.remove() log("Serialized %d cache objects" % item_count)
def _gc(self): database = get_session() total_bytes_before = self._total_bytes() old_item_count = 0 now = int(time.time()) keys = [] # remove old objects from memory for key, in (database.query(CacheObject.key) .filter(text("(%d - updated) > %d" % (now, CachePlugin.GC_AGE))).all()): cache.delete(key) keys.append(key) old_item_count += 1 # remove old objects from database if len(keys) > 0: database.query(CacheObject).filter(CacheObject.key.in_(keys)).delete(synchronize_session='fetch') database.commit() debug('Garbage collected %d old cache objects because of age' % old_item_count) big_item_count = 0 # remove the 10 oldest cache objects until total cache size is below limit while True: total_bytes = self._total_bytes() if total_bytes > CachePlugin.GC_SIZE: keys = [] # remove old oversized objects from memory for key, in database.query(CacheObject.key).order_by(CacheObject.updated.asc()).limit(10).all(): cache.delete(key) keys.append(key) big_item_count += 1 # remove old oversized objects from database database.query(CacheObject).filter(CacheObject.key.in_(keys)).delete(synchronize_session='fetch') database.commit() else: break debug('Garbage collected %d old cache objects because of total cache size' % big_item_count) database.remove() log("Garbage collected %d cache objects, total cache size was %d kb and is now %d kb" % (old_item_count + big_item_count, total_bytes_before / 1024, total_bytes / 1024))
def run(self, number): debug("Starting bgtask thread #%d" % number) while self._running: if self._running == "stop": break item = None try: try: item = self.queue.get(block=True, timeout=2) name, priority, func, args, kwargs = item.values() except queue.Empty: if self._running == "drain": break continue else: thread = threading.current_thread() item.started = time.time() thread.item = item thread.name = name self.running += 1 debug("Running bgtask in thread #%d %r with priority %d, args %r and kwargs %r." % (number, func, priority, args, kwargs)) database_data.database = get_session() func(*args, **kwargs) try: database_data.database.commit() except: database_data.database.rollback() raise except Exception as error: log("Error in bgtask thread #%d %r, args %r and kwargs %r." % (number, func, args, kwargs), traceback=True) item.fail(error) name, text, html = get_pretty_errors(sys.exc_info()) name = "bgtask %r: %s" % (func, name) mail_pretty_errors(name, text, html) finally: if item is not None: database_data.database.remove() database_data.database = None self.queue.task_done() self.queue.done(item) item.done = time.time() thread.name = 'idle' thread.item = None self.done.put(item) # store max 50 items in done queue if self.done.qsize() > 50: self.done.get() self.running -= 1 debug("Stopping bgtask thread #%d" % number)
def clean_up(self): # uses get_session() because this runs in a background thread and needs to # setup its own connection session = get_session() session.query(Session).filter(Session.expiration_time < self.now()).delete() session.commit()