def __init__(self):
self.cache = dc.Cache('umlsCache')
def cache():
with dc.Cache() as cache:
yield cache
shutil.rmtree(cache.directory, ignore_errors=True)
def test_disk_valueerror():
with pytest.raises(ValueError):
with dc.Cache(disk=dc.Disk('test')):
pass
# Obtain configuration dictionary.
config = get_config()
# Physical constants.
# TODO: Move physical constants to model definition.
water_density = 998.31 # [kg/m^3]
water_kinematic_viscosity = 1.3504e-6 # [m^2/s]
gravitational_acceleration = 9.81 # [m^2/s]
# Instantiate multiprocessing / parallel computing pool.
# - Pool is instantiated as None and only created on first use in `fledge.utils.starmap`.
parallel_pool = None
# Instantiate / reload cache.
if config['caching']['enable']:
cache = diskcache.Cache(os.path.join(base_path, 'cache'))
if config['caching']['reset_cache']:
cache.clear()
# Modify matplotlib default settings.
plt.style.use(config['plots']['matplotlib_style'])
matplotlib.rc('axes',
axisbelow=True) # Ensure that axis grid is behind plot elements.
matplotlib.rc('figure', figsize=config['plots']['matplotlib_figure_size'])
matplotlib.rc('font', family=config['plots']['matplotlib_font_family'])
matplotlib.rc('image', cmap=config['plots']['matplotlib_colormap'])
matplotlib.rc(
'pdf',
fonttype=42) # Avoid "Type 3 fonts" in PDFs for better compatibility.
matplotlib.rc('ps', fonttype=42) # See: http://phyletica.org/matplotlib-fonts/
matplotlib.rc('savefig', format=config['plots']['file_format'])
def memoized(func, *, size_limit=10**8, eviction_policy='least-recently-used', cache_dir=CACHE_DIR,
typed=False, round_digits=15, ignore_args=None):
"""Persistent memoization function decorator with argument normalization and ignore list.
:func: a callable object that is not a method
:size_limit: (int, in bytes) approximate size limit of cache - default 100 MB
:eviction_policy: rule to evict cache if size_limit is reached, any of
diskcache.EVICTION_POLICY
:cache_dir: location (directory path) of persistent cache files
:typed: wheter to consider lists of identically valued arguments of different types as
different arguments lists
:round_digits: number of digits to round to, pass False to disable rounding
:ignore_args: name or list of names of parameters to ignore
:returns: a memoized version of function 'func'
"""
func_hash = hashlib.md5(func.__code__.co_code).hexdigest()
func_id = "{}.{:0>4s}".format(func.__qualname__, func_hash[-4:])
cache_dir = os.path.join(cache_dir, func_id)
func.cache = diskcache.Cache(cache_dir, size_limit=size_limit, eviction_policy=eviction_policy)
func.async_results = {}
atexit.register(func.cache.close)
@atexit.register
def consolidate_async():
for key, result in func.async_results.items():
try:
if result.successful():
func.cache[dict(sorted(key))] = result.get()
# Exception class changed in Python 3.7:
# https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.AsyncResult.successful
except (AssertionError, ValueError):
pass
arg_names = inspect.getfullargspec(func).args
if ignore_args is not None:
ignore_args = frozenset([ignore_args] if isinstance(ignore_args, str) else ignore_args)
assert all(arg in arg_names for arg in ignore_args), "Unknown argument name passed to 'ignore_args' option."
@wraps(func)
def wrapper(*args, **kwargs):
key = kwargs.copy()
key.update(zip(arg_names, args))
if ignore_args is not None:
key = {k: v for k, v in key.items() if k not in ignore_args}
if not typed:
key = {k: _normalize_type(v, round_digits) for k, v in key.items()}
key = dict(sorted(key.items()))
try:
return func.cache[key]
except KeyError:
try:
return func.async_results[tuple(key.items())]
except KeyError:
logging.debug("%s: cache miss on key %s", wrapper.__qualname__, repr(key))
value = func(*args, **kwargs)
if isinstance(value, pool.AsyncResult):
func.async_results[tuple(key.items())] = value
else:
func.cache[key] = value
return value
return wrapper
def _set_creds(self, creds):
with diskcache.Cache(directory=self._cachedir) as cache:
cache[self._CREDS_STORAGE_KEY] = creds
return creds
from typing import Dict
import bs4
import diskcache
import requests
_CACHE = diskcache.Cache('.cache/html')
def fetch_static_soup(url: str,
params: Dict[str, str] = None) -> bs4.BeautifulSoup:
html = _fetch_static(url, params)
return bs4.BeautifulSoup(html, 'html.parser')
@_CACHE.memoize()
def _fetch_static(url: str, params: Dict[str, str] = None) -> str:
return _fetch(url, params)
def _fetch(url: str, params: Dict[str, str] = None) -> str:
response = requests.get(url, params=params)
return response.content
import diskcache as dc
cache = dc.Cache('stage_1_db')
crops = dc.Cache('stage_1_images')
from tqdm import tqdm
import glob, os
import cv2
size = 32
storage = "./images/"
os.chdir(storage)
step = 10
print(len(crops))
cached=set(list(cache))
for file in tqdm(glob.glob("*.*")):
try:
if not file in cached:
img = cv2.imread(file)
shape=img.shape
h = shape[0]
w = shape[1]
for x in tqdm(range(0,w-size, step)):
for y in range(0,h-size, step):
crop_img = img[y:y+size, x:x+size]
target = f"crop-{str(x)}-{str(y)}-{str(file)}"
crops[target]=crop_img
cache[file]="OK"
except KeyboardInterrupt:
cache.close()
break
finally:
pass
import diskcache
import pickle
import hashlib
r = diskcache.Cache('tmp')
class Cache:
CACHED_SECONDS = 768
@classmethod
def get(cls, path):
if cls.has(path):
return pickle.loads(r.get(cls._get_key(path)))
return False
@classmethod
def has(cls, path):
return r.get(cls._get_key(path)) is not None
@classmethod
def set(cls, path, entity, expire=CACHED_SECONDS):
return r.set(cls._get_key(path), pickle.dumps(entity), expire)
@classmethod
def rem(cls, path):
return r.delete(cls._get_key(path))
@staticmethod
def _get_key(path):
return 'onelist:' + hashlib.md5(path.encode()).hexdigest()
import feets
PATH = "/home/jbcabral/carpyncho3/correctors/"
import sys
sys.path.insert(0, PATH)
from parfeets import extract
# =============================================================================
# SOME VARS
# =============================================================================
host_name = socket.gethostname()
cache = diskcache.Cache("production_data/_cache" + host_name)
CPUS = joblib.cpu_count()
COLUMNS_TO_REMOVE = [
'scls_h', 'scls_j', 'scls_k', "AndersonDarling", "AmplitudeJ",
"AmplitudeH", "AmplitudeJH", "AmplitudeJK", 'Freq1_harmonics_rel_phase_0',
'Freq2_harmonics_rel_phase_0', 'Freq3_harmonics_rel_phase_0', "CAR_mean",
"CAR_tau", "CAR_sigma"
]
COLUMNS_NO_FEATURES = [
'id', 'tile', 'cnt', 'ra_k', 'dec_k', 'vs_type', 'vs_catalog', 'cls'
]
TILES_BY_HOST = {
def __init__(self):
self.cache = diskcache.Cache(Defaults.cache_dir)
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>
"""This is just a Test."""
import diskcache as dc
from enum import Enum
from .exceptions import DataverseError, MissingRequiredArgumentError, UnknownAPIEndpointError
__version__ = "0.2.0-dev"
_USER_AGENT = f"b4dataverse/{__version__}"
_cache = dc.Cache("tmp")
class APIEndpoint(Enum):
EVE_ONLINE = "https://esi.evetech.net/latest"
ZKILLBOARD = "https://zkillboard.com/api"
REQUEST_HEADERS = {"User-Agent": _USER_AGENT}
import numpy as np
import matplotlib as mpl
mpl.use('agg')
import matplotlib.pylab as plt
import matplotlib.cm as cm
import png
import io
import base64
import traceback
import diskcache as dc
cache = dc.Cache('tmp-cache')
def d_(x, n=10):
d_filter = np.zeros(n)
d_filter[0] = 1
d_filter[-1] = -1
R = np.convolve(x, d_filter, mode='same')
R[:1] = 0
R[-1:] = 0
return R
def speed_estim_for_grade(grade, lut_speed_grade, peak=(0.1, 0.9), plot=False):
i_grade = np.abs(grade - lut_speed_grade[2][:-1]).argmin()
dp = np.copy(lut_speed_grade[0][:, i_grade])
def __init__(self):
self.cache = dc.Cache('snomedctCache')
def __init__(self, conn):
self.cache = dc.Cache('tmp')
self.conn = conn
self.hits = 0
self.total = 0
import dash
import dash_labs as dl
import dash_bootstrap_components as dbc
from dash import html
from dash.long_callback import DiskcacheLongCallbackManager
from dash.dependencies import Input, Output
## Diskcache
import diskcache
cache = diskcache.Cache("./cache")
long_callback_manager = DiskcacheLongCallbackManager(cache)
import time
app = dash.Dash(__name__,
plugins=[dl.plugins.pages],
external_stylesheets=[dbc.themes.BOOTSTRAP])
navbar = dbc.NavbarSimple(
dbc.DropdownMenu(
[
dbc.DropdownMenuItem(page["name"], href=page["path"])
for page in dash.page_registry.values()
],
nav=True,
label="More Pages",
),
brand="Multi Page App Plugin Demo",
color="primary",
dark=True,
def _get_creds(self):
with diskcache.Cache(directory=self._cachedir) as cache:
return cache.get(self._CREDS_STORAGE_KEY)
def main():
url = 'https://cods.colorado.edu/api/gpslidar/'
msg_dict = {
NavTimeUTC.id: NavTimeUTC,
NavHPPOSLLH.id: NavHPPOSLLH,
AckAck.id: AckAck,
AckNak.id: AckNak,
CfgValgetRec.id: CfgValgetRec,
RxmRawx.id: RxmRawx,
InfDebug.id: InfDebug,
InfError.id: InfError,
InfNotice.id: InfNotice,
InfTest.id: InfTest,
InfWarning.id: InfWarning
} # Dictionary of implemented packet formats
def_loc = socket.gethostname()[0:4]
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--comm',
type=str,
default="USB",
help='Communication type ("USB" or "UART"). Default is "USB"')
parser.add_argument(
'-f',
'--configfile',
type=str,
default='/home/ccaruser/ublox/default.ini',
help='Location of configuration file to use. Default is "default.ini"')
parser.add_argument(
'-l',
'--location',
type=str,
default=def_loc,
help='GPS location. Default is first four letters of hostname (' +
def_loc + ')')
parser.add_argument('--led',
type=int,
default=21,
help='LED pin. Default is 21.')
args = parser.parse_args()
if args.comm == "USB":
port = '/dev/serial/by-id/usb-u-blox_AG_-_www.u-blox.com_u-blox_GNSS_receiver-if00' # Serial port TODO: UART
elif args.comm == "UART":
port = '/dev/ttyS0'
else:
logging.critical("Bad communication type: " + args.comm)
sys.exit(0)
dev = serial.Serial(port,
timeout=5,
baudrate=38400,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
bytesize=serial.EIGHTBITS) # Open serial port
# Configure
config = ConfigParser(inline_comment_prefixes=('#', ';'))
config.read(args.configfile) # Read configuration packets to be sent
packet = CfgValsetSend(config[args.comm]) # Create configuration packets
# Write config packet
wrtr = UBXWriter(dev, msg_dict) # ublox writer
wrtr.write_packet(packet.payload(), packet.id) # Write ublox packets
try:
dev.baudrate = config[args.comm][
'CFG-UART1-BAUDRATE'] # Set baud rate to desired rate in configuration file
except KeyError: # If there is no baud rate in configuration file
pass
# Read packets
loc = args.location
key = read_key('/home/ccaruser/.keys/' + loc +
'.key') # Private key for sending
led = LED(args.led) # LED class initialization
led.set_high() # Turn on LED
next_raw, next_pos = [], []
leapS = None
week = None
cache_raw = dc.Cache('/var/tmp/unsent_gpsraw')
cache_pos = dc.Cache('/var/tmp/unsent_gpspos')
# Send old data
t2 = Thread(target=send_old, args=(cache_raw, url + 'rawgps/' + loc, key))
t2.start()
t3 = Thread(target=send_old, args=(cache_pos, url + 'posgps/' + loc, key))
t3.start()
logging.info('Starting ' + loc + ' GPS at: ' + str(dt.datetime.utcnow()))
try:
while True:
led_timer = dt.datetime.utcnow()
raw, hp_pos = next_raw, next_pos # Initialization of vectors
next_raw, next_pos = [], []
prev_raw, prev_pos = 0, 0
while True:
rdr = UBXReader(dev, msg_dict) # Initialize reader
packet = rdr.read_packet() # Read packet
if isinstance(packet, RxmRawx): # If raw gps position packet
mod_raw = (packet.rcvTow - packet.leapS) % 60
if mod_raw >= prev_raw:
raw.append(packet)
week = packet.week
leapS = packet.leapS
prev_raw = mod_raw
else:
next_raw.append(packet)
break
elif isinstance(
packet, NavHPPOSLLH
) and leapS: # If high precision gps position packet
mod_pos = ((packet.iTOW / 1000) - leapS) % 60
if mod_pos >= prev_pos:
hp_pos.append(packet)
prev_pos = mod_pos
else:
next_pos.append(packet)
break
elif isinstance(packet, NavTimeUTC): # If time packet
#if packet.nano < 0:
# time = dt.datetime(packet.year, packet.month, packet.day, packet.hour, packet.min,
# packet.sec, -packet.nano // 10**3)
#else:
# time = dt.datetime(packet.year, packet.month, packet.day, packet.hour, packet.min,
# packet.sec, packet.nano // 10**3)
#cmd = 'sudo date -s "' + time.strftime('%Y-%m-%d %H:%M:%S') + 'UTC"'
#logging.info(cmd)
#os.system(cmd)
pass
else:
pass
if (dt.datetime.utcnow() - led_timer
).total_seconds() >= 1: # Switch led every second
led.switch()
led_timer = dt.datetime.utcnow()
# Get packets to send and start threads to send packets through api
if raw:
p_raw = raw_packet(raw)
if not t2.isAlive():
t2 = Thread(
target=call_send,
args=(url + 'rawgps/' + loc, key, p_raw,
(dt.datetime.utcnow() -
dt.datetime(1970, 1, 1)).total_seconds(),
cache_raw))
t2.start()
else:
save_to_dc(cache_raw,
(dt.datetime.utcnow() -
dt.datetime(1970, 1, 1)).total_seconds(),
p_raw)
if hp_pos and week and leapS and not t3.is_alive():
p_pos = pos_packet(hp_pos, week, leapS)
if not t3.isAlive():
t3 = Thread(
target=call_send,
args=(url + 'posgps/' + loc, key, p_pos,
(dt.datetime.utcnow() -
dt.datetime(1970, 1, 1)).total_seconds(),
cache_pos))
t3.start()
else:
save_to_dc(cache_pos,
(dt.datetime.utcnow() -
dt.datetime(1970, 1, 1)).total_seconds(),
p_pos)
finally:
# At the end turn LED off
led.set_low()
def clear_cached_credentials(self):
with diskcache.Cache(directory=self._cachedir) as cache:
cache.delete(self._CREDS_STORAGE_KEY)
from collections import Counter
from requests import ConnectionError
from requests.exceptions import Timeout
from six.moves.urllib.parse import quote
import tripleoci.config as config
from tripleoci.config import log
try:
from urlparse import urljoin
except ImportError:
from urllib.parse import urljoin
requests.packages.urllib3.disable_warnings()
cache = diskcache.Cache(config.CACHE_DIR)
cache.expire()
class SSH(object):
"""SSH
SSH class, just for any connection
"""
def __init__(self,
host, port, user, timeout=None, key=None, key_path=None):
self.ssh_cl = paramiko.SSHClient()
self.ssh_cl.set_missing_host_key_policy(paramiko.AutoAddPolicy())
log.debug("Executing ssh {user}@{host}:{port}".format(
user=user, host=host, port=port))
def __init__(self) -> None:
self._cache = diskcache.Cache(CACHE_ROOT_DIR)
def get_cache(settings: Settings = settings) -> diskcache.Cache:
return diskcache.Cache(
directory=str(settings.cache.dir),
size_limit=settings.cache.size_limit_gb * 1e9
)
def load_with_cache(
path_or_array: typing.Union[np.ndarray, io.IOBase, str, pathlib.Path],
grayscale=False,
use_cache=True,
max_size=None,
use_temp_dir=None,
) -> np.ndarray:
"""画像の読み込み。
Args:
path_or_array: 画像ファイルへのパス or npy/npzファイルへのパス or ndarray
grascale: Trueならグレースケールで読み込み、FalseならRGB
use_cache: 読み込み結果をdiskcacheライブラリでキャッシュするならTrue
max_size: このサイズを超えるなら縮小する。int or tuple。tupleは(height, width)
use_temp_dir: キャッシュを保存する場所。Noneだったらtempfile.gettempdir()の場所を使う
Returns:
読み込み結果のndarray。
"""
max_size = tk.utils.normalize_tuple(max_size,
2) if max_size is not None else None
def _load():
img = load(path_or_array, grayscale=grayscale)
if max_size is not None and (img.shape[0] > max_size[0]
or img.shape[1] > max_size[1]):
r0 = max_size[0] / img.shape[0]
r1 = max_size[1] / img.shape[1]
r = min(r0, r1)
img = resize(img, int(round(img.shape[1] * r)),
int(round(img.shape[0] * r)))
return img
if use_cache and isinstance(path_or_array, (str, pathlib.Path)):
global _load_cache
global _diskcache_load_failed
if _load_cache is None and not _diskcache_load_failed:
# 何回も呼び出されないかこれ?
# マルチプロセスのときとか怖い
temp_dir = tempfile.mkdtemp(
suffix="pytoolkit",
dir=use_temp_dir) if use_temp_dir is None else use_temp_dir
try:
import diskcache
_load_cache = diskcache.Cache(temp_dir)
atexit.register(_clear_cache, _load_cache)
except BaseException:
pathlib.Path(temp_dir).rmdir()
_diskcache_load_failed = True
tk.log.get(__name__).warning("diskcache load failed.",
exc_info=True)
if _load_cache is not None:
key = f"{path_or_array}::{max_size}"
img = _load_cache.get(key)
if img is None:
img = _load()
_load_cache.set(key, img)
return img
return _load()
def storage_disk(request):
client = diskcache.Cache('/tmp/ring-test/diskcache')
return client, ring.disk
def test_rsync():
try:
run(['rsync', '--version'])
except OSError:
return # No rsync installed. Skip test.
rsync_args = ['rsync', '-a', '--checksum', '--delete', '--stats']
cache_dir1 = tempfile.mkdtemp() + os.sep
cache_dir2 = tempfile.mkdtemp() + os.sep
# Store some items in cache_dir1.
with dc.Cache(cache_dir1) as cache1:
for count in range(100):
cache1[count] = str(count)
for count in range(100, 200):
cache1[count] = str(count) * int(1e5)
# Rsync cache_dir1 to cache_dir2.
run(rsync_args + [cache_dir1, cache_dir2])
# Validate items in cache_dir2.
with dc.Cache(cache_dir2) as cache2:
for count in range(100):
assert cache2[count] == str(count)
for count in range(100, 200):
assert cache2[count] == str(count) * int(1e5)
# Store more items in cache_dir2.
with dc.Cache(cache_dir2) as cache2:
for count in range(200, 300):
cache2[count] = str(count)
for count in range(300, 400):
cache2[count] = str(count) * int(1e5)
# Rsync cache_dir2 to cache_dir1.
run(rsync_args + [cache_dir2, cache_dir1])
# Validate items in cache_dir1.
with dc.Cache(cache_dir1) as cache1:
for count in range(100):
assert cache1[count] == str(count)
for count in range(100, 200):
assert cache1[count] == str(count) * int(1e5)
for count in range(200, 300):
assert cache1[count] == str(count)
for count in range(300, 400):
assert cache1[count] == str(count) * int(1e5)
shutil.rmtree(cache_dir1, ignore_errors=True)
shutil.rmtree(cache_dir2, ignore_errors=True)
def get_cache(self):
if self.cache is None:
self.cache_path.parent.mkdir(parents=True, exist_ok=True)
self.cache = dc.Cache(self.cache_path)
return self.cache
def test_tag_index():
with dc.Cache(tag_index=True) as cache:
assert cache.tag_index == 1
shutil.rmtree(cache.directory, ignore_errors=True)
import bson.json_util
import diskcache
# Populate cache defaults here instead of config.py because they are
# used in function decorators before the config.py is loaded into the
# Flask api environment
CACHE_DIRECTORY = os.getenv('CACHE_DIRECTORY', '/var/cache/app')
CACHE_SETTINGS = json.loads(os.getenv('CACHE_SETTINGS', '{}'))
CACHE_GET_DEFAULT = os.getenv("CACHE_GET_DEFAULT", '{"expire": 600}')
CACHE_GET_APPCONFIGS = json.loads(os.getenv("CACHE_GET_APPCONFIGS", CACHE_GET_DEFAULT))
CACHE_GET_APPCONFIG = json.loads(os.getenv("CACHE_GET_APPCONFIG", CACHE_GET_DEFAULT))
CACHE_SETTINGS.setdefault('size_limit', 3.5 * 1025 * 1024 * 1024) # 3.5GB cache
cache = diskcache.Cache(
directory=CACHE_DIRECTORY,
**CACHE_SETTINGS,
)
def memoize(*args, **kwargs):
"""
Basic memoize function that uses our cache and wraps memoize_stampede.
It takes all the same arguments as memoize_stampede, except do not
specify the cache argument.
"""
def decorator(func):
return diskcache.memoize_stampede(cache, *args, **kwargs)(func)
return decorator
def ping_diskcache(cachedb):
ping = cachedb.get("ping")
if not ping:
res = cachedb.set("ping", "pong")
if not res:
return False
ping = cachedb.get("ping")
if ping != "pong":
return False
return True
print("[Migrate] Initializing connection...")
cachedb = diskcache.Cache(settings["diskcache_path"])
redisdb = redis.Redis(**settings["redis"])
print("[Migrate] Checking connection to both database...")
if not ping_diskcache(cachedb):
print("[Migrate:diskcache] Failed to ping diskcache server, exiting...")
sys.exit(1)
try:
test = redisdb.get("ping")
except Exception:
print("[Migrate:redis] Failed to ping redis server, exiting...")
sys.exit(1)
print("[Migrate:diskcache] Fetching all keys and old data...")
all_old_keys = list(cachedb.iterkeys())
print(all_old_keys)
def _init_cache(self):
self.cachedir = tempfile.mkdtemp(prefix="wtdm")
self.cache = CacheWrapper(diskcache.Cache(self.cachedir))
