def test_unparseable_cache_map(): """ Verify that a cache map file with contents that are not parseable as JSON will be treated as if it does not exist (i.e. it will be overwritten with fresh contents on use) """ tmp_dir = tempfile.mkdtemp() my_cache = cache.Cache(cache_root_dir=tmp_dir) file_handle_id = 101201 # path normalization for windows path1 = os.path.normcase( os.path.normpath( utils.touch( os.path.join(my_cache.get_cache_dir(file_handle_id), "file1.ext")))) cache_dir = my_cache.get_cache_dir(file_handle_id) cache_map_file = os.path.join(cache_dir, my_cache.cache_map_file_name) # empty cache map file will not be parseable as json utils.touch(cache_map_file) my_cache.add(file_handle_id=101201, path=path1) assert os.path.normcase( os.path.normpath(my_cache.get(file_handle_id=101201, path=path1))) == path1 with open(cache_map_file, mode='r') as cache_map_in: cache_map = json.loads(cache_map_in.read()) assert os.path.normpath(os.path.normcase(next(iter( cache_map.keys())))) == path1
def test_cache_modified_time(): tmp_dir = tempfile.mkdtemp() my_cache = cache.Cache(cache_root_dir=tmp_dir) path1 = utils.touch(os.path.join(my_cache.get_cache_dir(101201), "file1.ext")) my_cache.add(file_handle_id=101201, path=path1) new_time_stamp = cache._get_modified_time(path1) + 1 utils.touch(path1, (new_time_stamp, new_time_stamp)) a_file = my_cache.get(file_handle_id=101201, path=path1) assert_is_none(a_file)
def add_file_to_cache(i, cache_root_dir): """ Helper function for use in test_cache_concurrent_access """ my_cache = cache.Cache(cache_root_dir=cache_root_dir) file_handle_ids = [1001, 1002, 1003, 1004, 1005] random.shuffle(file_handle_ids) for file_handle_id in file_handle_ids: cache_dir = my_cache.get_cache_dir(file_handle_id) file_path = os.path.join(cache_dir, "file_handle_%d_process_%02d.junk" % (file_handle_id, i)) utils.touch(file_path) my_cache.add(file_handle_id, file_path)
def test_cache_store_get(): tmp_dir = tempfile.mkdtemp() my_cache = cache.Cache(cache_root_dir=tmp_dir) path1 = utils.touch( os.path.join(my_cache.get_cache_dir(101201), "file1.ext")) my_cache.add(file_handle_id=101201, path=path1) path2 = utils.touch( os.path.join(my_cache.get_cache_dir(101202), "file2.ext")) my_cache.add(file_handle_id=101202, path=path2) # set path3's mtime to be later than path2's new_time_stamp = cache._get_modified_time(path2) + 2 path3 = utils.touch(os.path.join(tmp_dir, "foo", "file2.ext"), (new_time_stamp, new_time_stamp)) my_cache.add(file_handle_id=101202, path=path3) a_file = my_cache.get(file_handle_id=101201) assert utils.equal_paths(a_file, path1) a_file = my_cache.get(file_handle_id=101201, path=path1) assert utils.equal_paths(a_file, path1) a_file = my_cache.get(file_handle_id=101201, path=my_cache.get_cache_dir(101201)) assert utils.equal_paths(a_file, path1) b_file = my_cache.get(file_handle_id=101202, path=os.path.dirname(path2)) assert utils.equal_paths(b_file, path2) b_file = my_cache.get(file_handle_id=101202, path=os.path.dirname(path3)) assert utils.equal_paths(b_file, path3) not_in_cache_file = my_cache.get(file_handle_id=101203, path=tmp_dir) assert not_in_cache_file is None removed = my_cache.remove(file_handle_id=101201, path=path1, delete=True) assert utils.normalize_path(path1) in removed assert len(removed) == 1 assert my_cache.get(file_handle_id=101201) is None removed = my_cache.remove(file_handle_id=101202, path=path3, delete=True) b_file = my_cache.get(file_handle_id=101202) assert utils.normalize_path(path3) in removed assert len(removed) == 1 assert utils.equal_paths(b_file, path2) removed = my_cache.remove(file_handle_id=101202, delete=True) assert utils.normalize_path(path2) in removed assert len(removed) == 1 assert my_cache.get(file_handle_id=101202) is None
def test_cache_remove(): tmp_dir = tempfile.mkdtemp() my_cache = cache.Cache(cache_root_dir=tmp_dir) path1 = utils.touch(os.path.join(my_cache.get_cache_dir(101201), "file1.ext")) my_cache.add(file_handle_id=101201, path=path1) alt_dir = tempfile.mkdtemp() path2 = utils.touch(os.path.join(alt_dir, "file2.ext")) my_cache.add(file_handle_id=101201, path=path2) # remove the cached copy at path1 rp = my_cache.remove({'dataFileHandleId': 101201, 'path': path1}) assert len(rp) == 1 assert utils.equal_paths(rp[0], path1) assert utils.equal_paths(my_cache.get(101201), path2) my_cache.remove(101201) assert_is_none(my_cache.get(101201))
def test_subsecond_timestamps(): tmp_dir = tempfile.mkdtemp() my_cache = cache.Cache(cache_root_dir=tmp_dir) path = utils.touch(os.path.join(tmp_dir, "some_crazy_location", "file1.ext")) my_cache.add(file_handle_id=1234, path=path) with patch.object(cache, "_get_modified_time") as _get_modified_time_mock, \ patch.object(cache.Cache, "_read_cache_map") as _read_cache_map_mock: # this should be a match, 'cause we round microseconds to milliseconds _read_cache_map_mock.return_value = {path: "2015-05-05T21:34:55.001Z"} _get_modified_time_mock.return_value = 1430861695.001111 assert_equal(path, my_cache.get(file_handle_id=1234, path=path)) # The R client always writes .000 for milliseconds, for compatibility, # we should match .000 with any number of milliseconds _read_cache_map_mock.return_value = {path: "2015-05-05T21:34:55.000Z"} assert_equal(path, my_cache.get(file_handle_id=1234, path=path))
def test_cache_rules(): # Cache should (in order of preference): # # 1. DownloadLocation specified: # a. return exact match (unmodified file at the same path) # b. return an unmodified file at another location, # copy to downloadLocation subject to ifcollision # c. download file to downloadLocation subject to ifcollision # # 2. DownloadLocation *not* specified: # a. return an unmodified file at another location # b. download file to cache_dir overwritting any existing file tmp_dir = tempfile.mkdtemp() my_cache = cache.Cache(cache_root_dir=tmp_dir) # put file in cache dir path1 = utils.touch(os.path.join(my_cache.get_cache_dir(101201), "file1.ext")) my_cache.add(file_handle_id=101201, path=path1) new_time_stamp = cache._get_modified_time(path1) + 1 path2 = utils.touch(os.path.join(tmp_dir, "not_in_cache", "file1.ext"), (new_time_stamp, new_time_stamp)) my_cache.add(file_handle_id=101201, path=path2) new_time_stamp = cache._get_modified_time(path2) + 1 path3 = utils.touch(os.path.join(tmp_dir, "also_not_in_cache", "file1.ext"), (new_time_stamp, new_time_stamp)) my_cache.add(file_handle_id=101201, path=path3) # DownloadLocation specified, found exact match assert utils.equal_paths(my_cache.get(file_handle_id=101201, path=path2), path2) # DownloadLocation specified, no match, get most recent path = my_cache.get(file_handle_id=101201, path=os.path.join(tmp_dir, "file_is_not_here", "file1.ext")) assert utils.equal_paths(path, path3) # DownloadLocation specified as a directory, not in cache, get most recent empty_dir = os.path.join(tmp_dir, "empty_directory") os.makedirs(empty_dir) path = my_cache.get(file_handle_id=101201, path=empty_dir) assert utils.equal_paths(path, path3) # path2 is now modified new_time_stamp = cache._get_modified_time(path2) + 2 utils.touch(path2, (new_time_stamp, new_time_stamp)) # test cache.contains assert_false(my_cache.contains(file_handle_id=101201, path=empty_dir)) assert_false(my_cache.contains(file_handle_id=101201, path=path2)) assert_false(my_cache.contains(file_handle_id=101999, path=path2)) assert_true(my_cache.contains(file_handle_id=101201, path=path1)) assert_true(my_cache.contains(file_handle_id=101201, path=path3)) # Get file from alternate location. Do we care which file we get? assert_is_none(my_cache.get(file_handle_id=101201, path=path2)) assert_in(my_cache.get(file_handle_id=101201), [utils.normalize_path(path1), utils.normalize_path(path3)]) # Download uncached file to a specified download location assert_is_none(my_cache.get(file_handle_id=101202, path=os.path.join(tmp_dir, "not_in_cache"))) # No downloadLocation specified, get file from alternate location. Do we care which file we get? assert_is_not_none(my_cache.get(file_handle_id=101201)) assert_in(my_cache.get(file_handle_id=101201), [utils.normalize_path(path1), utils.normalize_path(path3)]) # test case 2b. assert_is_none(my_cache.get(file_handle_id=101202))