Example #1
0
    def load(self):
        """ Load caches from specified file """

        # Nothing to do when persistent caching is off
        if not self._filename or get_cache_level() < config.CACHE_PERSISTENT:
            return

        # Load the saved cache from file
        try:
            log.debug("Loading persistent cache from {0}".format(
                    self._filename))
            input_file = gzip.open(self._filename, 'rb')
            data = pickle.load(input_file)
            input_file.close()
        except EOFError:
            log.cache("Cache file empty, will fill it upon exit")
            return
        except (IOError, zlib.error), error:
            if getattr(error, "errno", None) == 2:
                log.warn("Cache file not found, will create one on exit")
                return
            else:
                log.error("Failed to load the cache ({0})".format(error))
                log.warn("Going on but switching to the CACHE_OBJECTS level")
                set_cache_level(config.CACHE_OBJECTS)
                self.unlock()
                return
Example #2
0
 def _fetch(self, inset=None):
     """ Fetch currently linked test cases from the server """
     # If data initialized from the inset ---> we're done
     if Container._fetch(self, inset):
         return
     # Initialize all plan-case tags (skip when caching persistently
     # as this an additional/unnecessary call in that case)
     if config.get_cache_level() == config.CACHE_OBJECTS:
         log.info("Fetching tags for all {0}'s test cases".format(self._object.identifier))
         for tag in self._server.TestPlan.get_all_cases_tags(self.id):
             Tag(tag)
     # Fetch test cases from the server
     log.info("Fetching {0}'s cases".format(self._identifier))
     injects = self._server.TestPlan.get_test_cases(self.id)
     log.data("Fetched {0}".format(listed(injects, "inject")))
     self._current = set([TestCase(inject) for inject in injects])
     self._original = set(self._current)
     # Initialize case plans if not already cached
     if not PlanCasePlans._is_cached(self._object.caseplans):
         inset = [
             CasePlan(
                 {
                     # Fake our own internal id from testplan & testcase
                     "id": _idify([self._object.id, inject["case_id"]]),
                     "case_id": inject["case_id"],
                     "plan_id": self._object.id,
                     "sortkey": inject["sortkey"],
                 }
             )
             for inject in injects
         ]
         self._object.caseplans._fetch(inset)
Example #3
0
    def save(self):
        """ Save caches to specified file """

        # Nothing to do when persistent caching is off
        if not self._filename or get_cache_level() < config.CACHE_PERSISTENT:
            return

        # Clear expired items and gather all caches into a single object
        self.expire()
        log.cache("Cache dump stats:\n" + self.stats().strip())
        data = {}
        for current_class in self._classes:
            # Put container classes into id-sleep
            if issubclass(current_class, containers.Container):
                for container in current_class._cache.values():
                    container._sleep()
            data[current_class.__name__] = current_class._cache

        # Dump the cache object into file
        try:
            # Use temporary file to minimize the time during which
            # the real cache is inconsistent
            output_file = tempfile.NamedTemporaryFile(
                    mode="wb", delete=False, prefix="nitrate-cache.",
                    dir=os.path.dirname(self._filename))
            log.cache("Temporary cache file: {0}".format(output_file.name))
            output_file = gzip.open(output_file.name, "wb")
            log.debug("Saving persistent cache into {0}".format(
                    self._filename))
            pickle.dump(data, output_file)
            output_file.close()
            os.rename(output_file.name, self._filename)
            log.debug("Persistent cache successfully saved")
        except IOError as error:
            log.error("Failed to save persistent cache ({0})".format(error))
Example #4
0
 def __new__(cls, id=None, *args, **kwargs):
     """ Create a new object, handle caching if enabled """
     # No caching when turned of or class does not support it
     if (config.get_cache_level() < config.CACHE_OBJECTS or
             getattr(cls, "_cache", None) is None):
         return super(Nitrate, cls).__new__(cls)
     # Make sure that cache has been initialized
     Cache()
     # Look up cached object by id (or other arguments in kwargs)
     try:
         # If found, we get instance and key by which it was found
         instance, key = cls._cache_lookup(id, **kwargs)
         if isinstance(key, int):
             log.cache("Using cached {0} ID#{1}".format(cls.__name__, key))
         else:
             log.cache("Using cached {0} '{1}'".format(cls.__name__, key))
         return instance
     # Object not cached yet, create a new one and cache it
     except KeyError:
         new = super(Nitrate, cls).__new__(cls)
         if isinstance(id, int):
             log.cache("Caching {0} ID#{1}".format(cls.__name__, id))
             cls._cache[id] = new
         elif isinstance(id, six.string_types) or "name" in kwargs:
             log.cache("Caching {0} '{1}'".format(
                     cls.__name__, (id or kwargs.get("name"))))
         return new
Example #5
0
    def save(self):
        """ Save caches to specified file """

        # Nothing to do when persistent caching is off
        if not self._filename or get_cache_level() < config.CACHE_PERSISTENT:
            return

        # Clear expired items and gather all caches into a single object
        self.expire()
        log.cache("Cache dump stats:\n" + self.stats().strip())
        data = {}
        for current_class in self._classes:
            # Put container classes into id-sleep
            if issubclass(current_class, containers.Container):
                for container in current_class._cache.itervalues():
                    container._sleep()
            data[current_class.__name__] = current_class._cache

        # Dump the cache object into file
        try:
            # Use temporary file to minimize the time during which
            # the real cache is inconsistent
            output_file = tempfile.NamedTemporaryFile(
                    mode="wb", delete=False, prefix="nitrate-cache.",
                    dir=os.path.dirname(self._filename))
            log.cache("Temporary cache file: {0}".format(output_file.name))
            output_file = gzip.open(output_file.name, "wb")
            log.debug("Saving persistent cache into {0}".format(
                    self._filename))
            pickle.dump(data, output_file)
            output_file.close()
            os.rename(output_file.name, self._filename)
            log.debug("Persistent cache successfully saved")
        except IOError, error:
            log.error("Failed to save persistent cache ({0})".format(error))
Example #6
0
 def _fetch(self, inset=None):
     """ Fetch currently linked test cases from the server """
     # If data initialized from the inset ---> we're done
     if Container._fetch(self, inset):
         return
     # Initialize all plan-case tags (skip when caching persistently
     # as this an additional/unnecessary call in that case)
     if config.get_cache_level() == config.CACHE_OBJECTS:
         log.info("Fetching tags for all {0}'s test cases".format(
             self._object.identifier))
         for tag in self._server.TestPlan.get_all_cases_tags(self.id):
             Tag(tag)
     # Fetch test cases from the server
     log.info("Fetching {0}'s cases".format(self._identifier))
     injects = self._server.TestPlan.get_test_cases(self.id)
     log.data("Fetched {0}".format(listed(injects, "inject")))
     self._current = set([TestCase(inject) for inject in injects])
     self._original = set(self._current)
     # Initialize case plans if not already cached
     if not PlanCasePlans._is_cached(self._object.caseplans):
         inset = [
             CasePlan({
                 # Fake our own internal id from testplan & testcase
                 "id": _idify([self._object.id, inject["case_id"]]),
                 "case_id": inject["case_id"],
                 "plan_id": self._object.id,
                 "sortkey": inject["sortkey"]
             }) for inject in injects
         ]
         self._object.caseplans._fetch(inset)
Example #7
0
 def __new__(cls, id=None, *args, **kwargs):
     """ Create a new object, handle caching if enabled """
     # No caching when turned of or class does not support it
     if (config.get_cache_level() < config.CACHE_OBJECTS or
             getattr(cls, "_cache", None) is None):
         return super(Nitrate, cls).__new__(cls)
     # Make sure that cache has been initialized
     Cache()
     # Look up cached object by id (or other arguments in kwargs)
     try:
         # If found, we get instance and key by which it was found
         instance, key = cls._cache_lookup(id, **kwargs)
         if isinstance(key, int):
             log.cache("Using cached {0} ID#{1}".format(cls.__name__, key))
         else:
             log.cache("Using cached {0} '{1}'".format(cls.__name__, key))
         return instance
     # Object not cached yet, create a new one and cache it
     except KeyError:
         new = super(Nitrate, cls).__new__(cls)
         if isinstance(id, int):
             log.cache("Caching {0} ID#{1}".format(cls.__name__, id))
             cls._cache[id] = new
         elif isinstance(id, basestring) or "name" in kwargs:
             log.cache("Caching {0} '{1}'".format(
                     cls.__name__, (id or kwargs.get("name"))))
         return new
Example #8
0
 def enter(self, filename=None):
     """ Perform setup, create lock, load the cache """
     # Nothing to do when persistent caching is off
     if get_cache_level() < config.CACHE_PERSISTENT:
         return
     # Setup the cache
     self.setup(filename)
     # Create the lock and load the cache
     self.lock()
     self.load()
Example #9
0
 def enter(self, filename=None):
     """ Perform setup, create lock, load the cache """
     # Nothing to do when persistent caching is off
     if get_cache_level() < config.CACHE_PERSISTENT:
         return
     # Setup the cache
     self.setup(filename)
     # Create the lock and load the cache
     self.lock()
     self.load()
Example #10
0
    def load(self):
        """ Load caches from specified file """

        # Nothing to do when persistent caching is off
        if not self._filename or get_cache_level() < config.CACHE_PERSISTENT:
            return

        # Load the saved cache from file
        try:
            log.debug("Loading persistent cache from {0}".format(
                    self._filename))
            input_file = gzip.open(self._filename, 'rb')
            data = pickle.load(input_file)
            input_file.close()
        except EOFError:
            log.cache("Cache file empty, will fill it upon exit")
            return
        except (IOError, zlib.error) as error:
            if getattr(error, "errno", None) == 2:
                log.warn("Cache file not found, will create one on exit")
                return
            else:
                log.error("Failed to load the cache ({0})".format(error))
                log.warn("Going on but switching to the CACHE_OBJECTS level")
                set_cache_level(config.CACHE_OBJECTS)
                self.unlock()
                return

        # Restore cache for immutable & mutable classes first
        for current_class in self._immutable + self._mutable:
            try:
                log.cache("Loading cache for {0}".format(
                        current_class.__name__))
                current_class._cache = data[current_class.__name__]
            except KeyError:
                log.cache("Failed to load cache for {0}, starting "
                        "with empty".format(current_class.__name__))
                current_class._cache = {}
        # Containers to be loaded last (to prevent object duplicates)
        for current_class in self._containers:
            try:
                log.cache("Loading cache for {0}".format(
                        current_class.__name__))
                current_class._cache = data[current_class.__name__]
            except KeyError:
                log.cache("Failed to load cache for {0}, starting "
                        "with empty".format(current_class.__name__))
                current_class._cache = {}
            # Wake up container objects from the id-sleep
            for container in current_class._cache.itervalues():
                container._wake()
        # Clear expired items and give a short summary for debugging
        self.expire()
        log.cache("Cache restore stats:\n" + self.stats().strip())
Example #11
0
 def _index(self, *keys):
     """ Index self into the class cache if caching enabled """
     # Skip indexing completely when caching off
     if config.get_cache_level() < config.CACHE_OBJECTS:
         return
     # Index by ID
     if self._id is not NitrateNone:
         self.__class__._cache[self._id] = self
     # Index each given key
     for key in keys:
         self.__class__._cache[key] = self
Example #12
0
 def _index(self, *keys):
     """ Index self into the class cache if caching enabled """
     # Skip indexing completely when caching off
     if config.get_cache_level() < config.CACHE_OBJECTS:
         return
     # Index by ID
     if self._id is not NitrateNone:
         self.__class__._cache[self._id] = self
     # Index each given key
     for key in keys:
         self.__class__._cache[key] = self
Example #13
0
    def load(self):
        """ Load caches from specified file """

        # Nothing to do when persistent caching is off
        if not self._filename or get_cache_level() < config.CACHE_PERSISTENT:
            return

        # Load the saved cache from file
        try:
            log.debug("Loading persistent cache from {0}".format(
                    self._filename))
            input_file = gzip.open(self._filename, 'rb')
            data = pickle.load(input_file)
            input_file.close()
        except EOFError:
            log.cache("Cache file empty, will fill it upon exit")
            return
        except (IOError, zlib.error) as error:
            if getattr(error, "errno", None) == 2:
                log.warn("Cache file not found, will create one on exit")
                return
            else:
                log.error("Failed to load the cache ({0})".format(error))
                log.warn("Going on but switching to the CACHE_OBJECTS level")
                set_cache_level(config.CACHE_OBJECTS)
                self.unlock()
                return

        # Restore cache for immutable & mutable classes first
        for current_class in self._immutable + self._mutable:
            try:
                log.cache("Loading cache for {0}".format(
                        current_class.__name__))
                current_class._cache = data[current_class.__name__]
            except KeyError:
                log.cache("Failed to load cache for {0}, starting "
                        "with empty".format(current_class.__name__))
                current_class._cache = {}
        # Containers to be loaded last (to prevent object duplicates)
        for current_class in self._containers:
            try:
                log.cache("Loading cache for {0}".format(
                        current_class.__name__))
                current_class._cache = data[current_class.__name__]
            except KeyError:
                log.cache("Failed to load cache for {0}, starting "
                        "with empty".format(current_class.__name__))
                current_class._cache = {}
            # Wake up container objects from the id-sleep
            for container in current_class._cache.values():
                container._wake()
        # Clear expired items and give a short summary for debugging
        self.expire()
        log.cache("Cache restore stats:\n" + self.stats().strip())
Example #14
0
 def exit(self):
     """ Save the cache and remove the lock """
     # Nothing to do when persistent caching is off
     if get_cache_level() < config.CACHE_PERSISTENT:
         return
     # Skip cache save in read-only mode
     if self._mode == "read-only":
         log.cache("Skipping persistent cache save in read-only mode")
         return
     # Save the cache and remove the lock
     self.save()
     self.unlock()
Example #15
0
 def exit(self):
     """ Save the cache and remove the lock """
     # Nothing to do when persistent caching is off
     if get_cache_level() < config.CACHE_PERSISTENT:
         return
     # Skip cache save in read-only mode
     if self._mode == "read-only":
         log.cache("Skipping persistent cache save in read-only mode")
         return
     # Save the cache and remove the lock
     self.save()
     self.unlock()
Example #16
0
 def _fetch(self, inset=None):
     """ Save cache timestamp and initialize from inset if given """
     Nitrate._fetch(self)
     # Create copies of the initial set (if given)
     if inset is not None:
         log.debug("Initializing {0} for {1} from the inset".format(self.__class__.__name__, self._identifier))
         log.debug(pretty(inset))
         self._current = set(inset)
         self._original = set(inset)
     # Cache into container class
     if config.get_cache_level() >= config.CACHE_OBJECTS:
         self.__class__._cache[self._id] = self
     # Return True if the data are already initialized
     return inset is not None
Example #17
0
 def setter(self, value):
     # Initialize the attribute unless already done
     if getattr(self, "_" + field) is NitrateNone:
         self._fetch()
     # Update only if changed
     if getattr(self, "_" + field) != value:
         setattr(self, "_" + field, value)
         log.info(u"Updating {0}'s {1} to '{2}'".format(
                 self.identifier, field, value))
         # Remember modified state if caching
         if config.get_cache_level() != config.CACHE_NONE:
             self._modified = True
         # Save the changes immediately otherwise
         else:
             self._update()
Example #18
0
 def setter(self, value):
     # Initialize the attribute unless already done
     if getattr(self, "_" + field) is NitrateNone:
         self._fetch()
     # Update only if changed
     if getattr(self, "_" + field) != value:
         setattr(self, "_" + field, value)
         log.info(u"Updating {0}'s {1} to '{2}'".format(
                 self.identifier, field, value))
         # Remember modified state if caching
         if config.get_cache_level() != config.CACHE_NONE:
             self._modified = True
         # Save the changes immediately otherwise
         else:
             self._update()
Example #19
0
 def _fetch(self, inset=None):
     """ Save cache timestamp and initialize from inset if given """
     Nitrate._fetch(self)
     # Create copies of the initial set (if given)
     if inset is not None:
         log.debug("Initializing {0} for {1} from the inset".format(
             self.__class__.__name__, self._identifier))
         log.debug(pretty(inset))
         self._current = set(inset)
         self._original = set(inset)
     # Cache into container class
     if config.get_cache_level() >= config.CACHE_OBJECTS:
         self.__class__._cache[self._id] = self
     # Return True if the data are already initialized
     return inset is not None
Example #20
0
    def setup(self, filename=None):
        """ Set cache filename and initialize expiration times """
        # Nothing to do when persistent caching is off
        if get_cache_level() < config.CACHE_PERSISTENT:
            return

        # Detect cache filename, argument first, then config
        if filename is not None:
            self._filename = filename
        else:
            try:
                self._filename = config.Config().cache.file
            except AttributeError:
                log.warn("Persistent caching off "
                         "(cache filename not found in the config)")
        self._lock = "{0}.lock".format(self._filename)

        # Initialize user-defined expiration times from the config
        for klass in self._classes + [
                Nitrate, mutable.Mutable, containers.Container
        ]:
            try:
                expiration = getattr(config.Config().expiration,
                                     klass.__name__.lower())
            except AttributeError:
                continue
            # Convert from seconds, handle special values
            if isinstance(expiration, int):
                expiration = datetime.timedelta(seconds=expiration)
            elif expiration == "NEVER_EXPIRE":
                expiration = config.NEVER_EXPIRE
            elif expiration == "NEVER_CACHE":
                expiration = config.NEVER_CACHE
            # Give warning for invalid expiration values
            if isinstance(expiration, datetime.timedelta):
                klass._expiration = expiration
                log.debug("User defined expiration for {0}: {1}".format(
                    klass.__name__, expiration))
            else:
                log.warn("Invalid expiration time '{0}'".format(expiration))
Example #21
0
    def setup(self, filename=None):
        """ Set cache filename and initialize expiration times """
        # Nothing to do when persistent caching is off
        if get_cache_level() < config.CACHE_PERSISTENT:
            return

        # Detect cache filename, argument first, then config
        if filename is not None:
            self._filename = filename
        else:
            try:
                self._filename = config.Config().cache.file
            except AttributeError:
                log.warn("Persistent caching off "
                        "(cache filename not found in the config)")
        self._lock = "{0}.lock".format(self._filename)

        # Initialize user-defined expiration times from the config
        for klass in self._classes + [Nitrate, mutable.Mutable,
                containers.Container]:
            try:
                expiration = getattr(
                        config.Config().expiration, klass.__name__.lower())
            except AttributeError:
                continue
            # Convert from seconds, handle special values
            if isinstance(expiration, int):
                expiration = datetime.timedelta(seconds=expiration)
            elif expiration == "NEVER_EXPIRE":
                expiration = config.NEVER_EXPIRE
            elif expiration == "NEVER_CACHE":
                expiration = config.NEVER_CACHE
            # Give warning for invalid expiration values
            if isinstance(expiration, datetime.timedelta):
                klass._expiration = expiration
                log.debug("User defined expiration for {0}: {1}".format(
                        klass.__name__, expiration))
            else:
                log.warn("Invalid expiration time '{0}'".format(expiration))
Example #22
0
    def remove(self, items):
        """ Remove an item or a list of items from the container """

        # Convert to set representation
        if isinstance(items, list):
            items = set(items)
        else:
            items = set([items])

        # If there are any items to be removed
        remove_items = items.intersection(self._items)
        if remove_items:
            log.info("Removing {0} from {1}'s {2}".format(
                    listed([item.identifier for item in remove_items],
                        self._class.__name__, max=10),
                    self._object.identifier,
                    self.__class__.__name__))
            self._items.difference_update(items)
            if config.get_cache_level() != config.CACHE_NONE:
                self._modified = True
            else:
                self._update()
Example #23
0
    def remove(self, items):
        """ Remove an item or a list of items from the container """

        # Convert to set representation
        if isinstance(items, list):
            items = set(items)
        else:
            items = set([items])

        # If there are any items to be removed
        remove_items = items.intersection(self._items)
        if remove_items:
            log.info("Removing {0} from {1}'s {2}".format(
                listed([item.identifier for item in remove_items],
                       self._class.__name__,
                       max=10), self._object.identifier,
                self.__class__.__name__))
            self._items.difference_update(items)
            if config.get_cache_level() != config.CACHE_NONE:
                self._modified = True
            else:
                self._update()
Example #24
0
    def add(self, items):
        """ Add an item or a list of items to the container """

        # Convert to set representation
        if isinstance(items, list):
            items = set(items)
        else:
            items = set([items])

        # If there are any new items
        add_items = items - self._items
        if add_items:
            log.info("Adding {0} to {1}'s {2}".format(
                listed([item.identifier for item in add_items],
                       self._class.__name__,
                       max=10), self._object.identifier,
                self.__class__.__name__))
            self._items.update(items)
            if config.get_cache_level() != config.CACHE_NONE:
                self._modified = True
            else:
                self._update()
Example #25
0
    def add(self, items):
        """ Add an item or a list of items to the container """

        # Convert to set representation
        if isinstance(items, list):
            items = set(items)
        else:
            items = set([items])

        # If there are any new items
        add_items = items - self._items
        if add_items:
            log.info("Adding {0} to {1}'s {2}".format(
                    listed([item.identifier for item in add_items],
                        self._class.__name__, max=10),
                    self._object.identifier,
                    self.__class__.__name__))
            self._items.update(items)
            if config.get_cache_level() != config.CACHE_NONE:
                self._modified = True
            else:
                self._update()