Esempio n. 1
0
    def test_locache(self):
        acache = LocalCache(timeout=5)

        self.assertEquals(None, acache.get("key"))

        acache.set("key", "value")
        self.assertEquals("value", acache.get("key"))

        time.sleep(5)
        self.assertEquals(None, acache.get("key"))

        acache.set("akey", "avalue", timeout=2)
        self.assertEquals("avalue", acache.get("akey"))
        time.sleep(2)
        self.assertEquals(None, acache.get("akey"))
        
        acache = LocalCache(timeout=300)
        for i in xrange(500):
            acache.set("%03d" % i, "%03d")

        end = start = -1
        for i in xrange(500, 0, -1):
            if acache.get("%03d" % i):
                end = i
                break

        for i in xrange(500):
            if acache.get("%03d" % i):
                start = i
                break

        self.assertTrue((end-start) < 300)
Esempio n. 2
0
def test_bust():
    backend = LocalCache()
    cache = Cache(backend, bust=True)

    c = cache("counter")(CallCounter())

    assert c() == 1
    assert c() == 2
    assert c.cached() == 2, 'not called if you just get the cached val'
    assert backend.get("counter")
Esempio n. 3
0
def test_basic():
    backend = LocalCache()
    cache = Cache(backend, enabled=True)

    c = cache("counter")(CallCounter())

    assert c() == 1, 'called the first time'
    assert c() == 1, 'not called the second time'
    assert c.refresh() == 2
    assert backend.get("counter")
Esempio n. 4
0
def test_disable():
    backend = LocalCache()
    cache = Cache(backend, enabled=False)

    c = cache("counter")(CallCounter())

    assert c() == 1, 'called the first time'
    assert c() == 2, 'called the second time too'
    assert c.cached() == 3, 'called even when you get the cached val'
    assert not backend.get("counter")
Esempio n. 5
0
def test_bust():
    backend = LocalCache()
    cache = Cache(backend, bust=True)

    c = cache("counter")(CallCounter())

    assert c() == 1
    assert c() == 2
    assert c.cached() == 2, 'not called if you just get the cached val'
    assert backend.get("counter")
Esempio n. 6
0
def test_disable():
    backend = LocalCache()
    cache = Cache(backend, enabled=False)

    c = cache("counter")(CallCounter())

    assert c() == 1, 'called the first time'
    assert c() == 2, 'called the second time too'
    assert c.cached() == 3, 'called even when you get the cached val'
    assert not backend.get("counter")
Esempio n. 7
0
def test_basic():
    backend = LocalCache()
    cache = Cache(backend, enabled=True)

    c = cache("counter")(CallCounter())

    assert c() == 1, 'called the first time'
    assert c() == 1, 'not called the second time'
    assert c.refresh() == 2
    assert backend.get("counter")
Esempio n. 8
0
    def run(self,kwargs):

        #Verify argument validity
        self.__verify_arguments__()

        #Get cache name from remote url
        cache_name = self.get_cache_name()

        #Get cache path
        cache_path = os.path.join(self.cache_root, cache_name)

        #Get cache object (will raise an exception if it doesn't exist)
        cache = LocalCache(cache_path, host_id=default_host_id)

        cache.DEBUG = self.DEBUG

        cache.lock(os.path.join(cache_path,self.dependency_platform, self.dependency_name, self.dependency_version))
        #Try to get entry from cache
        entry = cache.get(self.dependency_platform, self.dependency_name, self.dependency_version)

        #Set state
        if entry is None:
            state = EntryState.DOWNLOADED
        else:
            state = EntryState.CACHE

        #Verify Remote MD5

        #We need to download the file, it wasn't in the cache
        if entry is None:
            print (self.format_entry_name() + ": Not in cache")
            full_url = s3.join_s3_url(self.dependency_repo, self.dependency_platform, self.dependency_name, self.dependency_version)

            print (self.format_entry_name() + ": Downloading " + full_url)

            #Get Downloader
            downloader = s3.AsyncS3Downloader(None)

            #Set Downloader arguments
            downloader.source_url = full_url
            downloader.destination_path = os.path.join(self.cache_root, "downloading") + os.sep
            downloader.start()

            #Wait for downloader to finish #TODO: Do something with the reported progress
            while downloader.status != module.DONE:
                time.sleep(0.5)

            #Check for an exception, if so bubble it up
            if downloader.exception is not None:
                raise downloader.exception

            print self.format_entry_name() + ": Download complete"

            if downloader.result is None or len(downloader.result) == 0:
                raise EntryError(self.format_entry_name() + ": Unable to find remote entry '" + full_url + "'")

            #Iterate of the result (downloaded files)
            for item, checksum in downloader.result:
                local_file_checksum = mfile.md5_checksum(item)
                if checksum != local_file_checksum:
                    print(self.format_entry_name() + ": WARNING: Downloaded file does not match the checksum on the server")
                    print(self.format_entry_name() + ": WARNING: local:\t" + str(local_file_checksum))
                    print(self.format_entry_name() + ": WARNING: server:\t" + str(checksum))
                if self.dependency_unpack:
                    print (self.format_entry_name() + ": Unpacking...")
                #Put will unlock
                cache.put(item,self.dependency_platform, self.dependency_name, self.dependency_version, unpack=self.dependency_unpack)
            entry = cache.get(self.dependency_platform, self.dependency_name, self.dependency_version)
            if entry is None:
                raise EntryError(self.format_entry_name() + ": Error retrieving entry from cache.")
        cache.unlock(os.path.join(cache_path,self.dependency_platform, self.dependency_name, self.dependency_version))
        #Entry is not None, return all the files listed in the entry that aren't the configuration files
        return [os.path.abspath(os.path.join(entry.path,f)) for f in os.listdir(entry.path) if f != ".umpire"], state
Esempio n. 9
0
    def run(self, kwargs):

        # Verify argument validity
        self.__verify_arguments__()

        # Get cache name from remote url
        cache_name = self.get_cache_name()

        # Get cache path
        cache_path = os.path.join(self.cache_root, cache_name)

        # Get cache object (will raise an exception if it doesn't exist)
        cache = LocalCache(cache_path, host_id=default_host_id)

        cache.DEBUG = self.DEBUG

        cache.lock(os.path.join(cache_path, self.dependency_platform, self.dependency_name, self.dependency_version))
        # Try to get entry from cache
        entry = cache.get(self.dependency_platform, self.dependency_name, self.dependency_version)

        # Set state
        if entry is None:
            state = EntryState.DOWNLOADED
        else:
            state = EntryState.CACHE

        # Verify Remote MD5

        # We need to download the file, it wasn't in the cache
        if entry is None:
            print (self.format_entry_name() + ": Not in cache")
            full_url = s3.join_s3_url(
                self.dependency_repo, self.dependency_platform, self.dependency_name, self.dependency_version
            )

            print (self.format_entry_name() + ": Downloading " + full_url)

            # Get Downloader
            downloader = s3.AsyncS3Downloader(None)

            # Set Downloader arguments
            downloader.source_url = full_url
            downloader.destination_path = os.path.join(self.cache_root, "downloading") + os.sep
            downloader.start()

            # Wait for downloader to finish #TODO: Do something with the reported progress
            while downloader.status != module.DONE:
                time.sleep(0.5)

            # Check for an exception, if so bubble it up
            if downloader.exception is not None:
                raise downloader.exception

            print self.format_entry_name() + ": Download complete"

            if downloader.result is None or len(downloader.result) == 0:
                raise EntryError(self.format_entry_name() + ": Unable to find remote entry '" + full_url + "'")

            # Iterate of the result (downloaded files)
            for item, checksum in downloader.result:
                local_file_checksum = mfile.md5_checksum(item)
                if checksum != local_file_checksum:
                    print (
                        self.format_entry_name()
                        + ": WARNING: Downloaded file does not match the checksum on the server"
                    )
                    print (self.format_entry_name() + ": WARNING: local:\t" + str(local_file_checksum))
                    print (self.format_entry_name() + ": WARNING: server:\t" + str(checksum))
                if self.dependency_unpack:
                    print (self.format_entry_name() + ": Unpacking...")
                # Put will unlock
                cache.put(
                    item,
                    self.dependency_platform,
                    self.dependency_name,
                    self.dependency_version,
                    unpack=self.dependency_unpack,
                )
            entry = cache.get(self.dependency_platform, self.dependency_name, self.dependency_version)
            if entry is None:
                raise EntryError(self.format_entry_name() + ": Error retrieving entry from cache.")
        cache.unlock(os.path.join(cache_path, self.dependency_platform, self.dependency_name, self.dependency_version))
        # Entry is not None, return all the files listed in the entry that aren't the configuration files
        return [os.path.abspath(os.path.join(entry.path, f)) for f in os.listdir(entry.path) if f != ".umpire"], state