def __init__(self, cache_path: str = ""): self._data = dc.FanoutCache(cache_path, shards=8, size_limit=int(float(cache_size.get()))) self._hit = 0 self._miss = 0 if self.version < cache_version: self._data.clear() self.version = cache_version
def test_incr_concurrent(): count = 16 limit = 500 with dc.FanoutCache('tmp', timeout=0.001) as cache: threads = [ threading.Thread(target=stress_incr, args=(cache, limit)) for _ in range(count) ] for thread in threads: thread.start() for thread in threads: thread.join() with dc.FanoutCache('tmp') as cache: assert cache.get(b'key') == count * limit cache.check() shutil.rmtree('tmp', ignore_errors=True)
def test_copy(): cache_dir1 = op.join('tmp', 'foo') with dc.FanoutCache(cache_dir1) as cache1: for count in range(10): cache1[count] = str(count) for count in range(10, 20): cache1[count] = str(count) * int(1e5) cache_dir2 = op.join('tmp', 'bar') shutil.copytree(cache_dir1, cache_dir2) with dc.FanoutCache(cache_dir2) as cache2: for count in range(10): assert cache2[count] == str(count) for count in range(10, 20): assert cache2[count] == str(count) * int(1e5) shutil.rmtree('tmp', ignore_errors=True)
def test_copy(): cache_dir1 = tempfile.mkdtemp() with dc.FanoutCache(cache_dir1) as cache1: for count in range(10): cache1[count] = str(count) for count in range(10, 20): cache1[count] = str(count) * int(1e5) cache_dir2 = tempfile.mkdtemp() shutil.rmtree(cache_dir2) shutil.copytree(cache_dir1, cache_dir2) with dc.FanoutCache(cache_dir2) as cache2: for count in range(10): assert cache2[count] == str(count) for count in range(10, 20): assert cache2[count] == str(count) * int(1e5) shutil.rmtree(cache_dir1, ignore_errors=True) shutil.rmtree(cache_dir2, ignore_errors=True)
def test_custom_filename_disk(): with dc.FanoutCache(disk=SHA256FilenameDisk) as cache: for count in range(100, 200): key = str(count).encode('ascii') cache[key] = str(count) * int(1e5) disk = SHA256FilenameDisk(cache.directory) for count in range(100, 200): key = str(count).encode('ascii') subdir = '%03d' % (disk.hash(key) % 8) filename = hashlib.sha256(key).hexdigest()[:32] full_path = op.join(cache.directory, subdir, filename) with open(full_path) as reader: content = reader.read() assert content == str(count) * int(1e5) shutil.rmtree(cache.directory, ignore_errors=True)
def test_add_concurrent(): with dc.FanoutCache(shards=1) as cache: results = co.deque() limit = 1000 threads = [ threading.Thread(target=stress_add, args=(cache, limit, results)) for _ in range(16) ] for thread in threads: thread.start() for thread in threads: thread.join() assert sum(results) == limit cache.check() shutil.rmtree(cache.directory, ignore_errors=True)
def read(clear=False): "Read item from reading list." import biplist # pylint: disable=import-error import diskcache # pylint: disable=import-error import os.path import webbrowser home_dir = os.path.expanduser('~') gj_dir = os.path.join(home_dir, '.gj') cache_dir = os.path.join(gj_dir, 'diskcache') cache = diskcache.FanoutCache(cache_dir) reading_list_index = cache.index('reading-list') if clear: reading_list_index.clear() return data = biplist.readPlist('/Users/grantj/Library/Safari/Bookmarks.plist') reading_list_all = lookup( data, 'Children', ('Title', 'com.apple.ReadingList'), 'Children' ) reading_list = [ item for item in reading_list_all if 'DateLastViewed' not in item['ReadingList'] ] reading_list.sort(key=lambda item: item['ReadingList']['DateAdded']) for item in reading_list: url = item['URLString'] if url not in reading_list_index: reading_list_index[url] = 1 webbrowser.open(url) break else: print(u'Nothing to read \N{white frowning face}')
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--cache-dir', default='/tmp/test') parser.add_argument('--iterations', type=int, default=100) parser.add_argument('--sleep', type=float, default=0.1) parser.add_argument('--size', type=int, default=25) args = parser.parse_args() data = dc.FanoutCache(args.cache_dir) delays = [] values = {str(num): num for num in range(args.size)} iterations = args.iterations for i in range(args.iterations): print(f'Iteration {i + 1}/{iterations}', end='\r') time.sleep(args.sleep) for key, value in values.items(): start = time.monotonic() data[key] = value stop = time.monotonic() diff = stop - start delays.append(diff) # Discard warmup delays, first two iterations. del delays[:(len(values) * 2)] # Convert seconds to microseconds. delays = sorted(delay * 1e6 for delay in delays) # Display performance. print() print(f'Total #: {len(delays)}') print(f'Min delay (us): {delays[0]:>8.3f}') print(f'50th %ile (us): {delays[int(len(delays) * 0.50)]:>8.3f}') print(f'90th %ile (us): {delays[int(len(delays) * 0.90)]:>8.3f}') print(f'99th %ile (us): {delays[int(len(delays) * 0.99)]:>8.3f}') print(f'Max delay (us): {delays[-1]:>8.3f}')
def wrapper(): shutil.rmtree('tmp', ignore_errors=True) with dc.FanoutCache('tmp') as cache: func(cache) shutil.rmtree('tmp', ignore_errors=True)
""" See https://docs.docker.com/registry/spec/api/#overview and https://docs.docker.com/registry/spec/manifest-v2-2/#example-manifest-list """ import flask import pathlib import json import subprocess import hashlib import diskcache app = flask.Flask(__name__) dcache = diskcache.FanoutCache('/tmp/simci-cache/', shards=4, timeout=2, eviction_policy='least-recently-used') EMPTY_MANIFESTS = { "schemaVersion": 2, "mediaType": "application/vnd.docker.distribution.manifest.v2+json", } def _git_checkout(sha): # TODO(tom): ensure clone if repo not yet cloned. # TODO(tom): cache tar = subprocess.check_output([ 'git', '--git-dir', '/home/tom/src/nixpkgs/.git', 'archive', '--prefix', 'ar/', sha ])
def prepare(self): cache = diskcache.FanoutCache(self.args.cache_path, shards=self.shards, eviction_policy='none') self.cache = diskcache.Index.fromcache(cache)
def test_rsync(): try: run(['rsync', '--version']) except OSError: return # No rsync installed. Skip test. rsync_args = ['rsync', '-a', '--checksum', '--delete', '--stats'] cache_dir1 = tempfile.mkdtemp() + os.sep cache_dir2 = tempfile.mkdtemp() + os.sep # Store some items in cache_dir1. with dc.FanoutCache(cache_dir1) as cache1: for count in range(100): cache1[count] = str(count) for count in range(100, 200): cache1[count] = str(count) * int(1e5) # Rsync cache_dir1 to cache_dir2. run(rsync_args + [cache_dir1, cache_dir2]) # Validate items in cache_dir2. with dc.FanoutCache(cache_dir2) as cache2: for count in range(100): assert cache2[count] == str(count) for count in range(100, 200): assert cache2[count] == str(count) * int(1e5) # Store more items in cache_dir2. with dc.FanoutCache(cache_dir2) as cache2: for count in range(200, 300): cache2[count] = str(count) for count in range(300, 400): cache2[count] = str(count) * int(1e5) # Rsync cache_dir2 to cache_dir1. run(rsync_args + [cache_dir2, cache_dir1]) # Validate items in cache_dir1. with dc.FanoutCache(cache_dir1) as cache1: for count in range(100): assert cache1[count] == str(count) for count in range(100, 200): assert cache1[count] == str(count) * int(1e5) for count in range(200, 300): assert cache1[count] == str(count) for count in range(300, 400): assert cache1[count] == str(count) * int(1e5) shutil.rmtree(cache_dir1, ignore_errors=True) shutil.rmtree(cache_dir2, ignore_errors=True)
def cache(): with dc.FanoutCache() as cache: yield cache shutil.rmtree(cache.directory, ignore_errors=True)
$ python -m IPython tests/benchmark_kv_store.py """ from IPython import get_ipython import diskcache ipython = get_ipython() assert ipython is not None, 'No IPython! Run with $ ipython ...' value = 'value' print('diskcache set') dc = diskcache.FanoutCache('/tmp/diskcache') ipython.magic("timeit -n 100 -r 7 dc['key'] = value") print('diskcache get') ipython.magic("timeit -n 100 -r 7 dc['key']") print('diskcache set/delete') ipython.magic("timeit -n 100 -r 7 dc['key'] = value; del dc['key']") try: import dbm.gnu # Only trust GNU DBM except ImportError: print('Error: Cannot import dbm.gnu') print('Error: Skipping import shelve') else: print('dbm set') d = dbm.gnu.open('/tmp/dbm', 'c') ipython.magic("timeit -n 100 -r 7 d['key'] = value; d.sync()")
malingnant_tensor = torch.from_numpy( np.array([isMalignant_bool], dtype=np.float32)) # dim=4, Channel x Index x Row x Col ct_tensor = ct_tensor.unsqueeze(0) # Unpacked as: input_tensor, answer_int, series_uid, center_irc return ct_tensor, malignant_tensor, series_uid, center_irc # In[8]: import functools import diskcache cache = diskcache.FanoutCache('/tmp/diskcache/fanoutcache') @functools.lru_cache(1, typed=True) def getCt(series_uid): return Ct(series_uid) @cache.memoize(typed=True) def getCtInputChunk(series_uid, center_xyz, width_irc): ct = getCt(series_uid) ct_chunk, center_irc = ct.getInputChunk(center_xyz, width_irc) return ct_chunk, center_irc # In[3]: