Пример #1
0
    def remove(self, file_handle_id, path=None, delete=None):
        """
        Remove a file from the cache.

        :param file_handle_id:
        :param path: If the given path is None, remove (and potentially delete)
                     all cached copies. If the path is that of a file in the
                     .cacheMap file, remove it.

        :returns: A list of files removed
        """
        removed = []
        cache_dir = self.get_cache_dir(file_handle_id)
        with Lock(self.cache_map_file_name, dir=cache_dir):
            cache_map = self._read_cache_map(cache_dir)

            if path is None:
                if delete is True:
                    for cached_file_path in cache_map:
                        os.remove(cached_file_path)
                        removed.append(cached_file_path)
                cache_map = {}
            else:
                path = utils.normalize_path(path)
                if path in cache_map:
                    if delete is True and os.path.exists(path):
                        os.remove(path)
                    del cache_map[path]
                    removed.append(path)

            self._write_cache_map(cache_dir, cache_map)

        return removed
Пример #2
0
def test_lock_timeout():
    user1_lock = Lock("foo", max_age=timedelta(seconds=1))
    user2_lock = Lock("foo", max_age=timedelta(seconds=1))

    with user1_lock:
        assert user1_lock.held == True
        assert user1_lock.get_age() < 1.0
        assert not user2_lock.acquire(break_old_locks=True)
        time.sleep(1)
        assert user1_lock.get_age() > 1.0
        assert user2_lock.acquire(break_old_locks=True)
Пример #3
0
def test_with_lock():
    user1_lock = Lock("foo", max_age=timedelta(seconds=5))
    user2_lock = Lock("foo", max_age=timedelta(seconds=5))

    with user1_lock:
        assert_less(user1_lock.get_age(), 5)
        assert_false(user2_lock.acquire())

    with user2_lock:
        assert_true(user2_lock.acquire())
        assert_false(user1_lock.acquire())
Пример #4
0
def test_lock_timeout():
    user1_lock = Lock("foo", max_age=timedelta(seconds=1))
    user2_lock = Lock("foo", max_age=timedelta(seconds=1))

    with user1_lock:
        assert_true(user1_lock.held)
        assert_less(user1_lock.get_age(), 1.0)
        assert_false(user2_lock.acquire(break_old_locks=True))
        time.sleep(1.1)
        assert_greater(user1_lock.get_age(), 1.0)
        assert_true(user2_lock.acquire(break_old_locks=True))
Пример #5
0
def test_with_lock():
    user1_lock = Lock("foo", max_age=timedelta(seconds=5))
    user2_lock = Lock("foo", max_age=timedelta(seconds=5))

    with user1_lock:
        assert user1_lock.get_age() < 5
        assert not user2_lock.acquire()

    with user2_lock:
        assert user2_lock.acquire()
        assert not user1_lock.acquire()
Пример #6
0
    def get(self, file_handle_id, path=None):
        """
        Retrieve a file with the given file handle from the cache.

        :param file_handle_id:
        :param path: If the given path is None, look for a cached copy of the
                     file in the cache directory. If the path is a directory,
                     look there for a cached copy. If a full file-path is
                     given, only check whether that exact file exists and is
                     unmodified since it was cached.

        :returns: Either a file path, if an unmodified cached copy of the file
                  exists in the specified location or None if it does not
        """
        cache_dir = self.get_cache_dir(file_handle_id)
        if not os.path.exists(cache_dir):
            return None

        with Lock(self.cache_map_file_name, dir=cache_dir):
            cache_map = self._read_cache_map(cache_dir)

            path = utils.normalize_path(path)

            ## If the caller specifies a path and that path exists in the cache
            ## but has been modified, we need to indicate no match by returning
            ## None. The logic for updating a synapse entity depends on this to
            ## determine the need to upload a new file.

            if path is not None:
                ## If we're given a path to a directory, look for a cached file in that directory
                if os.path.isdir(path):
                    for cached_file_path, cached_time in six.iteritems(
                            cache_map):
                        if path == os.path.dirname(cached_file_path):
                            return cached_file_path if compare_timestamps(
                                _get_modified_time(cached_file_path),
                                cached_time) else None

                ## if we're given a full file path, look up a matching file in the cache
                else:
                    cached_time = cache_map.get(path, None)
                    if cached_time:
                        return path if compare_timestamps(
                            _get_modified_time(path), cached_time) else None

            ## return most recently cached and unmodified file OR
            ## None if there are no unmodified files
            for cached_file_path, cached_time in sorted(
                    cache_map.items(), key=operator.itemgetter(1),
                    reverse=True):
                if compare_timestamps(_get_modified_time(cached_file_path),
                                      cached_time):
                    return cached_file_path

            return None
def test_with_lock():
    user1_lock = Lock("foo", max_age=timedelta(seconds=5))
    user2_lock = Lock("foo", max_age=timedelta(seconds=5))

    with user1_lock:
        assert user1_lock.get_age() < 5
        assert not user2_lock.acquire()

    with user2_lock:
        assert user2_lock.acquire()
        assert not user1_lock.acquire()
def test_lock_timeout():
    user1_lock = Lock("foo", max_age=timedelta(seconds=1))
    user2_lock = Lock("foo", max_age=timedelta(seconds=1))

    with user1_lock:
        assert_true(user1_lock.held)
        assert_less(user1_lock.get_age(), 1.0)
        assert_false(user2_lock.acquire(break_old_locks=True))
        time.sleep(1.1)
        assert_greater(user1_lock.get_age(), 1.0)
        assert_true(user2_lock.acquire(break_old_locks=True))
def test_with_lock():
    user1_lock = Lock("foo", max_age=timedelta(seconds=5))
    user2_lock = Lock("foo", max_age=timedelta(seconds=5))

    with user1_lock:
        assert_less(user1_lock.get_age(), 5)
        assert_false(user2_lock.acquire())

    with user2_lock:
        assert_true(user2_lock.acquire())
        assert_false(user1_lock.acquire())
Пример #10
0
def test_lock_timeout():
    user1_lock = Lock("foo", max_age=timedelta(seconds=1))
    user2_lock = Lock("foo", max_age=timedelta(seconds=1))

    with user1_lock:
        assert user1_lock.held == True
        assert user1_lock.get_age() < 1.0
        assert not user2_lock.acquire(break_old_locks=True)
        time.sleep(1)
        assert user1_lock.get_age() > 1.0
        assert user2_lock.acquire(break_old_locks=True)
Пример #11
0
    def add(self, file_handle_id, path):
        """
        Add a file to the cache
        """
        if not path or not os.path.exists(path):
            raise ValueError("Can't find file \"%s\"" % path)

        cache_dir = self.get_cache_dir(file_handle_id)
        with Lock(self.cache_map_file_name, dir=cache_dir):
            cache_map = self._read_cache_map(cache_dir)

            path = utils.normalize_path(path)
            ## write .000 milliseconds for backward compatibility
            cache_map[path] = epoch_time_to_iso(floor(_get_modified_time(path)))
            self._write_cache_map(cache_dir, cache_map)

        return cache_map
Пример #12
0
    def contains(self, file_handle_id, path):
        """
        Given a file and file_handle_id, return True if an unmodified cached
        copy of the file exists at the exact path given or False otherwise.
        :param file_handle_id:
        :param path: file path at which to look for a cached copy
        """
        cache_dir = self.get_cache_dir(file_handle_id)
        if not os.path.exists(cache_dir):
            return False

        with Lock(self.cache_map_file_name, dir=cache_dir):
            cache_map = self._read_cache_map(cache_dir)

            path = utils.normalize_path(path)

            cached_time = cache_map.get(path, None)
            if cached_time:
                return True if compare_timestamps(_get_modified_time(path), cached_time) else False
Пример #13
0
    def remove(self, file_handle_id, path=None, delete=None):
        """
        Remove a file from the cache.

        :param file_handle_id: Will also extract file handle id from either a File or file handle
        :param path: If the given path is None, remove (and potentially delete)
                     all cached copies. If the path is that of a file in the
                     .cacheMap file, remove it.

        :returns: A list of files removed
        """
        removed = []
        cache_dir = self.get_cache_dir(file_handle_id)

        ## if we've passed an entity and not a path, get path from entity
        if path is None and isinstance(
                file_handle_id,
                collections.Mapping) and 'path' in file_handle_id:
            path = file_handle_id['path']

        with Lock(self.cache_map_file_name, dir=cache_dir):
            cache_map = self._read_cache_map(cache_dir)

            if path is None:
                for path in cache_map:
                    if delete is True and os.path.exists(path):
                        os.remove(path)
                    removed.append(path)
                cache_map = {}
            else:
                path = utils.normalize_path(path)
                if path in cache_map:
                    if delete is True and os.path.exists(path):
                        os.remove(path)
                    del cache_map[path]
                    removed.append(path)

            self._write_cache_map(cache_dir, cache_map)

        return removed
Пример #14
0
def test_lock():
    user1_lock = Lock("foo", max_age=timedelta(seconds=5))
    user2_lock = Lock("foo", max_age=timedelta(seconds=5))

    assert user1_lock.acquire()
    assert user1_lock.get_age() < 5
    assert not user2_lock.acquire()

    user1_lock.release()

    assert user2_lock.acquire()
    assert not user1_lock.acquire()

    user2_lock.release()
Пример #15
0
def do_stuff_with_a_locked_resource(name, event_log):
    lock = Lock("foo", max_age=timedelta(seconds=5))
    for i in range(NUMBER_OF_TIMES_PER_THREAD):
        with lock:
            event_log.append((name, i))
        time.sleep(random.betavariate(2, 5))
Пример #16
0
def test_lock():
    user1_lock = Lock("foo", max_age=timedelta(seconds=5))
    user2_lock = Lock("foo", max_age=timedelta(seconds=5))

    assert user1_lock.acquire()
    assert user1_lock.get_age() < 5
    assert not user2_lock.acquire()

    user1_lock.release()

    assert user2_lock.acquire()
    assert not user1_lock.acquire()

    user2_lock.release()
Пример #17
0
    def get(self, file_handle_id, path=None):
        """
        Retrieve a file with the given file handle from the cache.

        :param file_handle_id:
        :param path: If the given path is None, look for a cached copy of the
                     file in the cache directory. If the path is a directory,
                     look there for a cached copy. If a full file-path is
                     given, only check whether that exact file exists and is
                     unmodified since it was cached.

        :returns: Either a file path, if an unmodified cached copy of the file
                  exists in the specified location or None if it does not
        """
        cache_dir = self.get_cache_dir(file_handle_id)
        if not os.path.exists(cache_dir):
            return None

        with Lock(self.cache_map_file_name, dir=cache_dir):
            cache_map = self._read_cache_map(cache_dir)

            path = utils.normalize_path(path)

            # If the caller specifies a path and that path exists in the cache
            # but has been modified, we need to indicate no match by returning
            # None. The logic for updating a synapse entity depends on this to
            # determine the need to upload a new file.

            if path is not None:
                # If we're given a path to a directory, look for a cached file in that directory
                if os.path.isdir(path):
                    matching_unmodified_directory = None
                    removed_entry_from_cache = False  # determines if cache_map needs to be rewritten to disk

                    # iterate a copy of cache_map to allow modifying original cache_map
                    for cached_file_path, cached_time in six.iteritems(
                            dict(cache_map)):
                        if path == os.path.dirname(cached_file_path):
                            # compare_timestamps has an implicit check for whether the path exists
                            if compare_timestamps(
                                    _get_modified_time(cached_file_path),
                                    cached_time):
                                # "break" instead of "return" to write removed invalid entries to disk if necessary
                                matching_unmodified_directory = cached_file_path
                                break
                            else:
                                # remove invalid cache entries pointing to files that that no longer exist
                                # or have been modified
                                del cache_map[cached_file_path]
                                removed_entry_from_cache = True

                    if removed_entry_from_cache:
                        # write cache_map with non-existent entries removed
                        self._write_cache_map(cache_dir, cache_map)

                    if matching_unmodified_directory is not None:
                        return matching_unmodified_directory

                # if we're given a full file path, look up a matching file in the cache
                else:
                    cached_time = cache_map.get(path, None)
                    if cached_time:
                        return path if compare_timestamps(
                            _get_modified_time(path), cached_time) else None

            # return most recently cached and unmodified file OR
            # None if there are no unmodified files
            for cached_file_path, cached_time in sorted(
                    cache_map.items(), key=operator.itemgetter(1),
                    reverse=True):
                if compare_timestamps(_get_modified_time(cached_file_path),
                                      cached_time):
                    return cached_file_path
            return None