def remove(self, file_handle_id, path=None, delete=None):
"""
Remove a file from the cache.
:param file_handle_id:
:param path: If the given path is None, remove (and potentially delete)
all cached copies. If the path is that of a file in the
.cacheMap file, remove it.
:returns: A list of files removed
"""
removed = []
cache_dir = self.get_cache_dir(file_handle_id)
with Lock(self.cache_map_file_name, dir=cache_dir):
cache_map = self._read_cache_map(cache_dir)
if path is None:
if delete is True:
for cached_file_path in cache_map:
os.remove(cached_file_path)
removed.append(cached_file_path)
cache_map = {}
else:
path = utils.normalize_path(path)
if path in cache_map:
if delete is True and os.path.exists(path):
os.remove(path)
del cache_map[path]
removed.append(path)
self._write_cache_map(cache_dir, cache_map)
return removed
def test_lock_timeout():
user1_lock = Lock("foo", max_age=timedelta(seconds=1))
user2_lock = Lock("foo", max_age=timedelta(seconds=1))
with user1_lock:
assert user1_lock.held == True
assert user1_lock.get_age() < 1.0
assert not user2_lock.acquire(break_old_locks=True)
time.sleep(1)
assert user1_lock.get_age() > 1.0
assert user2_lock.acquire(break_old_locks=True)
def test_with_lock():
user1_lock = Lock("foo", max_age=timedelta(seconds=5))
user2_lock = Lock("foo", max_age=timedelta(seconds=5))
with user1_lock:
assert_less(user1_lock.get_age(), 5)
assert_false(user2_lock.acquire())
with user2_lock:
assert_true(user2_lock.acquire())
assert_false(user1_lock.acquire())
def test_lock_timeout():
user1_lock = Lock("foo", max_age=timedelta(seconds=1))
user2_lock = Lock("foo", max_age=timedelta(seconds=1))
with user1_lock:
assert_true(user1_lock.held)
assert_less(user1_lock.get_age(), 1.0)
assert_false(user2_lock.acquire(break_old_locks=True))
time.sleep(1.1)
assert_greater(user1_lock.get_age(), 1.0)
assert_true(user2_lock.acquire(break_old_locks=True))
def test_with_lock():
user1_lock = Lock("foo", max_age=timedelta(seconds=5))
user2_lock = Lock("foo", max_age=timedelta(seconds=5))
with user1_lock:
assert user1_lock.get_age() < 5
assert not user2_lock.acquire()
with user2_lock:
assert user2_lock.acquire()
assert not user1_lock.acquire()
def get(self, file_handle_id, path=None):
"""
Retrieve a file with the given file handle from the cache.
:param file_handle_id:
:param path: If the given path is None, look for a cached copy of the
file in the cache directory. If the path is a directory,
look there for a cached copy. If a full file-path is
given, only check whether that exact file exists and is
unmodified since it was cached.
:returns: Either a file path, if an unmodified cached copy of the file
exists in the specified location or None if it does not
"""
cache_dir = self.get_cache_dir(file_handle_id)
if not os.path.exists(cache_dir):
return None
with Lock(self.cache_map_file_name, dir=cache_dir):
cache_map = self._read_cache_map(cache_dir)
path = utils.normalize_path(path)
## If the caller specifies a path and that path exists in the cache
## but has been modified, we need to indicate no match by returning
## None. The logic for updating a synapse entity depends on this to
## determine the need to upload a new file.
if path is not None:
## If we're given a path to a directory, look for a cached file in that directory
if os.path.isdir(path):
for cached_file_path, cached_time in six.iteritems(
cache_map):
if path == os.path.dirname(cached_file_path):
return cached_file_path if compare_timestamps(
_get_modified_time(cached_file_path),
cached_time) else None
## if we're given a full file path, look up a matching file in the cache
else:
cached_time = cache_map.get(path, None)
if cached_time:
return path if compare_timestamps(
_get_modified_time(path), cached_time) else None
## return most recently cached and unmodified file OR
## None if there are no unmodified files
for cached_file_path, cached_time in sorted(
cache_map.items(), key=operator.itemgetter(1),
reverse=True):
if compare_timestamps(_get_modified_time(cached_file_path),
cached_time):
return cached_file_path
return None
def test_with_lock():
user1_lock = Lock("foo", max_age=timedelta(seconds=5))
user2_lock = Lock("foo", max_age=timedelta(seconds=5))
with user1_lock:
assert user1_lock.get_age() < 5
assert not user2_lock.acquire()
with user2_lock:
assert user2_lock.acquire()
assert not user1_lock.acquire()
def test_lock_timeout():
user1_lock = Lock("foo", max_age=timedelta(seconds=1))
user2_lock = Lock("foo", max_age=timedelta(seconds=1))
with user1_lock:
assert_true(user1_lock.held)
assert_less(user1_lock.get_age(), 1.0)
assert_false(user2_lock.acquire(break_old_locks=True))
time.sleep(1.1)
assert_greater(user1_lock.get_age(), 1.0)
assert_true(user2_lock.acquire(break_old_locks=True))
def test_with_lock():
user1_lock = Lock("foo", max_age=timedelta(seconds=5))
user2_lock = Lock("foo", max_age=timedelta(seconds=5))
with user1_lock:
assert_less(user1_lock.get_age(), 5)
assert_false(user2_lock.acquire())
with user2_lock:
assert_true(user2_lock.acquire())
assert_false(user1_lock.acquire())
def test_lock_timeout():
user1_lock = Lock("foo", max_age=timedelta(seconds=1))
user2_lock = Lock("foo", max_age=timedelta(seconds=1))
with user1_lock:
assert user1_lock.held == True
assert user1_lock.get_age() < 1.0
assert not user2_lock.acquire(break_old_locks=True)
time.sleep(1)
assert user1_lock.get_age() > 1.0
assert user2_lock.acquire(break_old_locks=True)
def add(self, file_handle_id, path):
"""
Add a file to the cache
"""
if not path or not os.path.exists(path):
raise ValueError("Can't find file \"%s\"" % path)
cache_dir = self.get_cache_dir(file_handle_id)
with Lock(self.cache_map_file_name, dir=cache_dir):
cache_map = self._read_cache_map(cache_dir)
path = utils.normalize_path(path)
## write .000 milliseconds for backward compatibility
cache_map[path] = epoch_time_to_iso(floor(_get_modified_time(path)))
self._write_cache_map(cache_dir, cache_map)
return cache_map
def contains(self, file_handle_id, path):
"""
Given a file and file_handle_id, return True if an unmodified cached
copy of the file exists at the exact path given or False otherwise.
:param file_handle_id:
:param path: file path at which to look for a cached copy
"""
cache_dir = self.get_cache_dir(file_handle_id)
if not os.path.exists(cache_dir):
return False
with Lock(self.cache_map_file_name, dir=cache_dir):
cache_map = self._read_cache_map(cache_dir)
path = utils.normalize_path(path)
cached_time = cache_map.get(path, None)
if cached_time:
return True if compare_timestamps(_get_modified_time(path), cached_time) else False
def remove(self, file_handle_id, path=None, delete=None):
"""
Remove a file from the cache.
:param file_handle_id: Will also extract file handle id from either a File or file handle
:param path: If the given path is None, remove (and potentially delete)
all cached copies. If the path is that of a file in the
.cacheMap file, remove it.
:returns: A list of files removed
"""
removed = []
cache_dir = self.get_cache_dir(file_handle_id)
## if we've passed an entity and not a path, get path from entity
if path is None and isinstance(
file_handle_id,
collections.Mapping) and 'path' in file_handle_id:
path = file_handle_id['path']
with Lock(self.cache_map_file_name, dir=cache_dir):
cache_map = self._read_cache_map(cache_dir)
if path is None:
for path in cache_map:
if delete is True and os.path.exists(path):
os.remove(path)
removed.append(path)
cache_map = {}
else:
path = utils.normalize_path(path)
if path in cache_map:
if delete is True and os.path.exists(path):
os.remove(path)
del cache_map[path]
removed.append(path)
self._write_cache_map(cache_dir, cache_map)
return removed
def test_lock():
user1_lock = Lock("foo", max_age=timedelta(seconds=5))
user2_lock = Lock("foo", max_age=timedelta(seconds=5))
assert user1_lock.acquire()
assert user1_lock.get_age() < 5
assert not user2_lock.acquire()
user1_lock.release()
assert user2_lock.acquire()
assert not user1_lock.acquire()
user2_lock.release()
def do_stuff_with_a_locked_resource(name, event_log):
lock = Lock("foo", max_age=timedelta(seconds=5))
for i in range(NUMBER_OF_TIMES_PER_THREAD):
with lock:
event_log.append((name, i))
time.sleep(random.betavariate(2, 5))
def test_lock():
user1_lock = Lock("foo", max_age=timedelta(seconds=5))
user2_lock = Lock("foo", max_age=timedelta(seconds=5))
assert user1_lock.acquire()
assert user1_lock.get_age() < 5
assert not user2_lock.acquire()
user1_lock.release()
assert user2_lock.acquire()
assert not user1_lock.acquire()
user2_lock.release()
def get(self, file_handle_id, path=None):
"""
Retrieve a file with the given file handle from the cache.
:param file_handle_id:
:param path: If the given path is None, look for a cached copy of the
file in the cache directory. If the path is a directory,
look there for a cached copy. If a full file-path is
given, only check whether that exact file exists and is
unmodified since it was cached.
:returns: Either a file path, if an unmodified cached copy of the file
exists in the specified location or None if it does not
"""
cache_dir = self.get_cache_dir(file_handle_id)
if not os.path.exists(cache_dir):
return None
with Lock(self.cache_map_file_name, dir=cache_dir):
cache_map = self._read_cache_map(cache_dir)
path = utils.normalize_path(path)
# If the caller specifies a path and that path exists in the cache
# but has been modified, we need to indicate no match by returning
# None. The logic for updating a synapse entity depends on this to
# determine the need to upload a new file.
if path is not None:
# If we're given a path to a directory, look for a cached file in that directory
if os.path.isdir(path):
matching_unmodified_directory = None
removed_entry_from_cache = False # determines if cache_map needs to be rewritten to disk
# iterate a copy of cache_map to allow modifying original cache_map
for cached_file_path, cached_time in six.iteritems(
dict(cache_map)):
if path == os.path.dirname(cached_file_path):
# compare_timestamps has an implicit check for whether the path exists
if compare_timestamps(
_get_modified_time(cached_file_path),
cached_time):
# "break" instead of "return" to write removed invalid entries to disk if necessary
matching_unmodified_directory = cached_file_path
break
else:
# remove invalid cache entries pointing to files that that no longer exist
# or have been modified
del cache_map[cached_file_path]
removed_entry_from_cache = True
if removed_entry_from_cache:
# write cache_map with non-existent entries removed
self._write_cache_map(cache_dir, cache_map)
if matching_unmodified_directory is not None:
return matching_unmodified_directory
# if we're given a full file path, look up a matching file in the cache
else:
cached_time = cache_map.get(path, None)
if cached_time:
return path if compare_timestamps(
_get_modified_time(path), cached_time) else None
# return most recently cached and unmodified file OR
# None if there are no unmodified files
for cached_file_path, cached_time in sorted(
cache_map.items(), key=operator.itemgetter(1),
reverse=True):
if compare_timestamps(_get_modified_time(cached_file_path),
cached_time):
return cached_file_path
return None
