def prepare(self):
self.prepared = True
# Cache
if self.config.CACHE_TYPE == 'redis':
self.cache = RedisCache(host=self.config.CACHE_SERV)
elif self.config.CACHE_TYPE == 'memcached':
self.cache = MemcachedCache(servers=[self.config.CACHE_SERV])
else:
self.cache = FileSystemCache(self.config.CACHE_SERV)
# Options
from .admin import Option
self.options = Option.auto_load()
# Timer
@self.app.before_request
def before_request():
g.start = time.time()
# Medias
self.app.add_url_rule(self.app.config['UPLOAD_DIRECTORY_URL'] +
'<filename>',
'FyPress.uploaded_file',
build_only=True)
self.app.wsgi_app = SharedDataMiddleware(
self.app.wsgi_app, {
self.app.config['UPLOAD_DIRECTORY_URL']:
self.app.config['UPLOAD_DIRECTORY']
})
def load_session_hack(cli_token):
"""
Loads a session manually for a certain cli_token
"""
# Attempt to load local session - this only works for file system sessions
# See https://github.com/pallets/werkzeug/blob/master/werkzeug/contrib/cache.py for format
# XXX FIXME replace this by a custom session handler for Flask
from werkzeug.contrib.cache import FileSystemCache
import pickle
cache = FileSystemCache(app.config['SESSION_FILE_DIR'],
threshold=app.config['SESSION_FILE_THRESHOLD'],
mode=app.config['SESSION_FILE_MODE'])
found = False
for fn in cache._list_dir():
with open(fn, 'rb') as f:
fot = pickle.load(f)
del fot # Unused
local_session = pickle.load(f)
if type(local_session) is not dict:
continue
if local_session.get('cli_token') == cli_token:
found = True
break
if not found:
return None
return local_session
def setUp(self):
self.cache = FileSystemCache("./cache")
self.saxon = SaxonShellTransform(
"./jars/saxon.jar",
"./tests/data/xsl/ciham.xsl",
cache=self.cache
)
self.nautilus = NautilusRetriever(
folders=[
"./tests/data/repo"
]
)
self.nautilus.logger.setLevel(logging.ERROR)
app = Flask("Nemo")
app.debug = True
nemo = Nemo(
app=app,
base_url="",
retriever=self.nautilus,
transform={
"default": self.saxon.transform
}
)
self.client = app.test_client()
def __init__(self, cache_dir='cache', default_timeout=300):
"""初始化
cache_dir是缓存目录
"""
self.cache = FileSystemCache(cache_dir,
default_timeout=default_timeout)
def wipe_old_sessions_hack(ap_session, cli_token):
"""
Finds all prior sessions for this cli_token that do not match the current session
(current session, aka 'ap_session') and drop them as they're now invalid
"""
from werkzeug.contrib.cache import FileSystemCache
import pickle
cache = FileSystemCache(app.config['SESSION_FILE_DIR'],
threshold=app.config['SESSION_FILE_THRESHOLD'],
mode=app.config['SESSION_FILE_MODE'])
for fn in cache._list_dir():
with open(fn, 'rb') as f:
fot = pickle.load(f)
del fot # Unused
local_session = pickle.load(f)
if type(local_session) is not dict:
continue
if (local_session.get('cli_token') == cli_token) and (
local_session.get('ap_session') != ap_session):
os.remove(fn)
app.logger.debug(
'Removed now invalidated session data for cli_token {}'.
format(cli_token))
continue
return
def setUp(self):
# Full creation of app
self.cache = FileSystemCache(subprocess_cache_dir, default_timeout=0)
self.resolver = NautilusCTSResolver(subprocess_repository,
dispatcher=make_dispatcher(),
cache=self.cache)
self.__app__ = Flask("Nautilus")
self.http_cache = Cache(self.app,
config={
'CACHE_TYPE': "filesystem",
"CACHE_DIR": http_cache_dir,
"CACHE_DEFAULT_TIMEOUT": 0
})
self.nautilus = FlaskNautilus(app=self.app,
prefix="/api",
name="nautilus",
resolver=self.resolver,
flask_caching=self.http_cache)
self.test_client = self.app.test_client()
# Option to ensure cache works
self.former_parse = self.resolver.parse
def x(*k, **kw):
raise self.ParsingCalled("Parse should not be called")
self.resolver.parse = x
def cache_get_or_set(cache_name, function, timeout=3600):
cache = FileSystemCache(cache_dir=abspath('tmp'))
cache_data = cache.get(cache_name)
if cache_data is None:
cache_data = function()
cache.set(cache_name, cache_data, timeout=timeout)
return cache_data
def get_last_digests():
cache = FileSystemCache(Config.CACHE_DIR, default_timeout=0)
cache_key = 'last_digests'
api = Api(host=Config.HOST)
contract = {
'args': '["5"]',
'function': 'getLast',
}
try:
response = api.call(
from_addr=Config.ADDRESS,
to_addr=Config.ADDRESS,
value='0',
nonce=0,
gasprice=Config.GASPRICE,
gaslimit=Config.GASLIMIT,
contract=contract)
except RequestException:
result = cache.get(cache_key)
else:
if response.status_code == 200:
result = response.json().get('result', {}).get('result', '{}')
result = json.loads(result)
if isinstance(result, str):
result = json.loads(result)
cache.set(cache_key, result)
return result
def __init__(self, cache_dir, threshold, mode, key_prefix,
use_signer=False, permanent=True):
from werkzeug.contrib.cache import FileSystemCache
self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode)
self.key_prefix = key_prefix
self.use_signer = use_signer
self.permanent = permanent
def get_configuration():
sheet_scope = ['https://www.googleapis.com/auth/spreadsheets.readonly']
credentials = ServiceAccountCredentials.from_json_keyfile_name(
SERVICE_KEY, sheet_scope)
http_auth = credentials.authorize(Http())
sheets_service = build('sheets', 'v4', http=http_auth)
fileId = '1I8un3pP8aE3b2ixeQTN9mVPwMzTsE_TEkaFzAHEWZ-A'
data_range = 'configuration'
data_result = sheets_service.spreadsheets().values().get(
spreadsheetId=fileId, range=data_range).execute()
return_data = data_result.get('values', [])
if not return_data:
return -1
configuration = {}
for row in return_data:
configuration[row[0]] = row[1]
cc = FileSystemCache('/var/www/html/gs-cookies/cache/config',
default_timeout=300)
if cc.get('first_monday') is None:
for key, value in configuration.iteritems():
cc.set(key, value)
return configuration
def _get_cache():
_cache = g.get('_oauth_cache')
if _cache:
return _cache
_cache = FileSystemCache(current_app.config['OAUTH_CACHE_DIR'])
g._oauth_cache = _cache
return _cache
def __init__(self, api_id, api_key, plugins):
self.plugins = plugins
self.ai = AiPlat(api_id, api_key)
self._bootstrap_plugins()
self.cache = FileSystemCache('/tmp/wechat-bot-cache')
self.available_cmds = OrderedDict()
self.client = None
self.register_default_cmd()
def __init__(self, cache_dir, threshold, mode, key_prefix,
use_signer=False, permanent=True):
from werkzeug.contrib.cache import FileSystemCache
self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode)
self.key_prefix = key_prefix
self.use_signer = use_signer
self.permanent = permanent
self.has_same_site_capability = hasattr(self, "get_cookie_samesite")
def __init__(self):
self.sendkey = '1562-20254d9020f4f5883c56d0836c8bf5cb'
self.text = 'Lazy'
self.push_url = 'https://pushbear.ftqq.com/sub'
# 缓存
self.cache = FileSystemCache('/tmp/jjz_cache')
# push次数
self.cache_push_times_key = 'today_push_times'
def init_cache(app):
cache_timeout = 24 * 60 * 60
temp_dir_path = get_cache_dir()
if temp_dir_path:
app.logger.info(temp_dir_path)
app.cache = FileSystemCache(temp_dir_path,
default_timeout=cache_timeout)
else:
app.cache = SimpleCache(default_timeout=cache_timeout)
def setUp(self):
"""BaseModel test set up"""
if os.path.isfile('/tmp/box.db'):
os.unlink('/tmp/box.db')
DBHelper().set_db('/tmp/box.db')
InstallHelper.reset()
cache = FileSystemCache('/tmp/werkzeug')
cache.clear()
BaseModel.set_cache(cache)
SampleModel.install()
def test_filesystemcache_prune():
"""
test if FileSystemCache._prune works and keeps the cache entry count
below the given threshold.
"""
THRESHOLD = 13
tmp_dir = tempfile.mkdtemp()
cache = FileSystemCache(cache_dir=tmp_dir, threshold=THRESHOLD)
for i in range(2 * THRESHOLD):
cache.set(str(i), i)
cache_files = os.listdir(tmp_dir)
shutil.rmtree(tmp_dir)
assert len(cache_files) <= THRESHOLD
def test_filesystemcache_clear():
"""
test if FileSystemCache.clear works
"""
tmp_dir = tempfile.mkdtemp()
cache = FileSystemCache(cache_dir=tmp_dir)
cache.set('foo', 'bar')
cache_files = os.listdir(tmp_dir)
assert len(cache_files) == 1
cache.clear()
cache_files = os.listdir(tmp_dir)
assert len(cache_files) == 0
shutil.rmtree(tmp_dir)
def _commandline(repositories,
port=8000,
host="127.0.0.1",
debug=False,
cache=None,
cache_path="./cache",
redis=None):
""" Run a CTS API from command line.
.. warning:: This function should not be used in the production context
:param repositories:
:param port:
:param ip:
:param debug:
:param cache:
:param cache_path:
:return:
"""
if cache == "redis":
nautilus_cache = RedisCache(redis)
cache_type = "redis"
elif cache == "filesystem":
nautilus_cache = FileSystemCache(cache_path)
cache_type = "simple"
else:
nautilus_cache = NullCache()
cache_type = "simple"
app = Flask("Nautilus")
if debug:
app.logger.setLevel(logging.INFO)
resolver = NautilusCTSResolver(resource=repositories)
nautilus = FlaskNautilus(
app=app,
resolver=resolver
#parser_cache=WerkzeugCacheWrapper(nautilus_cache),
#logger=None
)
nautilus.resolver.parse()
if debug:
app.run(debug=debug, port=port, host=host)
else:
app.debug = debug
http_server = HTTPServer(WSGIContainer(app))
http_server.bind(port=port, address=host)
http_server.start(0)
IOLoop.current().start()
def test_filesystemcache_set_get():
"""
test if FileSystemCache.set/get works
"""
tmp_dir = tempfile.mkdtemp()
try:
cache = FileSystemCache(cache_dir=tmp_dir)
for i in range(3):
cache.set(str(i), i * i)
for i in range(3):
result = cache.get(str(i))
assert result == i * i
finally:
shutil.rmtree(tmp_dir)
def __init__(self,
path,
tolerance,
expiration,
max_values,
run_tests=True,
max_file_size=0):
"""Constructor method.
:param path: The path of the cache database file.
:param tolerance: The tolerance, in seconds to which a TimeMap is
considered young enough to be used as is.
:param expiration: How long, in seconds, the cache entries are stored
every get will be a CACHE MISS.
:param max_values: The maximum number of TimeMaps stored in cache
before some are deleted
:param run_tests: (Optional) Tests the cache at initialization.
:param max_file_size: (Optional) The maximum size (in Bytes) for a
TimeMap cache value. When max_file_size=0, there is no limit to
a cache value. When max_file_size=X > 0, the cache will not
store TimeMap that require more than X Bytes on disk.
"""
# Parameters Check
if tolerance <= 0 or expiration <= 0 or max_values <= 0:
raise CacheError('Cannot create cache: all parameters must be > 0')
self.tolerance = relativedelta(seconds=tolerance)
self.path = path.rstrip('/')
self.max_file_size = max(max_file_size, 0)
self.CHECK_SIZE = self.max_file_size > 0
self.max_values = max_values
self.backend = FileSystemCache(path,
threshold=self.max_values,
default_timeout=expiration)
# Testing cache
if run_tests:
try:
key = b'1'
val = 1
self.backend.set(key, val)
assert (not self.CHECK_SIZE) or self._check_size(key) > 0
assert self.backend.get(key) == val
os.remove(os.path.join(self.path, md5(key).hexdigest()))
except Exception as e:
raise CacheError('Error testing cache: %s' % e)
logging.debug('Cache created. max_files = %d. Expiration = %d. '
'max_file_size = %d' %
(self.max_values, expiration, self.max_file_size))
def get_content(url, params, cache):
c = FileSystemCache('cache',
threshold=cache['threshold'],
default_timeout=cache['default_timeout'])
cache_id = url + str(params)
cache_content = c.get(cache_id)
if cache_content is not None:
return cache_content
headers = {'user-agent': UserAgent().chrome}
resp = requests.get(url, params=params, headers=headers)
resp.raise_for_status()
content = resp.text
c.set(cache_id, content)
return content
def setUp(self):
output = call([python, "./tests/cts/run_cache.py"], cwd=cwd)
if output != 0:
raise Exception("Creating cache failed")
self.cache = FileSystemCache(subprocess_cache_dir)
self.resolver = NautilusCTSResolver(resource=subprocess_repository,
cache=self.cache)
self.resolver.logger.disabled = True
def x(*k, **kw):
raise Exception("Parse should not be called")
self.resolver.parse = x
def weather_data(latitude, longitude):
cache = FileSystemCache('./cache')
cache_key = '{}.{}'.format(latitude, longitude)
data = cache.get(cache_key)
if data is None:
apikey = '3a4c7bb6f812d00b3bb9d19ceace7222'
fio = ForecastIO.ForecastIO(apikey,
units=ForecastIO.ForecastIO.UNITS_SI,
lang=ForecastIO.ForecastIO.LANG_ENGLISH,
latitude=latitude,
longitude=longitude)
currently = fio.currently
data = currently
cache.set(cache_key, data, timeout=300) # 5 minutes
return data
def make_resolver(directories=None, cache_directory=None):
""" Generate the CapiTainS Resolver and add metadata to it
"""
if directories is None:
directories = glob.glob("data/raw/corpora/**/**")
logger = logging.getLogger(__name__)
logger.setLevel(logging.CRITICAL)
kwargs = dict(resource=directories, logger=logger)
if cache_directory:
kwargs["cache"] = FileSystemCache(cache_directory)
print("Clearing cache")
kwargs["cache"].clear()
resolver = NautilusCTSResolver(**kwargs)
return resolver
def setUp(self):
""" Set up a dummy application with a manager """
nautilus_cache = FileSystemCache("cache_dir")
nautilus_cache.clear()
app = Flask("Nautilus")
resolver = NautilusCTSResolver(["./tests/test_data/latinLit"], cache=nautilus_cache, logger=logger)
flask_nautilus = FlaskNautilus(
app=app,
resolver=resolver,
flask_caching=Cache(config={'CACHE_TYPE': 'filesystem'}),
logger=logger
)
self.cache_manager = nautilus_cache
self.nautilus = flask_nautilus
self.resolver = resolver
self.resolver.logger.disabled = True
self.manager = FlaskNautilusManager(resolver, flask_nautilus)
def gunicorn():
ec = EnjoliverConfig(importer=__file__)
engine = create_engine(ec.db_uri,
echo=bool(os.environ.get('SQLALCHEMY_ECHO', False)))
sess_maker = sessionmaker(bind=engine)
cache = FileSystemCache(ec.werkzeug_fs_cache_dir)
app = create_app(
name='enjoliver-api',
ec=ec,
)
registry = RepositoryRegistry(sess_maker)
register_routes(app=app,
ec=ec,
cache=cache,
sess_maker=sess_maker,
registry=registry)
return app
def parse(url):
cache = FileSystemCache('.cachedir', threshold=100000)
# get page
page_source = cache.get(url)
if page_source is None:
# use firefox to get page with javascript generated content
with closing(Firefox()) as browser:
browser.get(url)
page_source = browser.page_source
cache.set(url, page_source,
timeout=60 * 60 * 24 * 7) # week in seconds
# extract text
root = html.document_fromstring(page_source)
# remove flash, images, <script>,<style>, etc
Cleaner(kill_tags=['noscript'], style=True)(root) # lxml >= 2.3.1
return root.text_content() # extract text
def init_cache(cache_type="simple",
memcached_servers=[],
cache_dir=None,
timeout=259200):
''' init_cache creates the oembed cache with the given cache type
cache_type - 'simple', 'memcached', or 'file'. Determines which type of cache to use
memcached_servers - List of memcached servers. Must be set if cache_type is 'memcached'.
cache_dir - Directory for a file system cache. Must be set if cache_type is 'file'.
timeout - Timeout in seconds. Default is 3 days.
'''
global cache
if cache_type == 'simple':
cache = SimpleCache(default_timeout=timeout)
elif cache_type == 'memcached':
cache = MemcachedCache(servers=memcached_servers,
default_timeout=timeout)
elif cache_type == 'file':
cache = FileSystemCache(cache_dir, default_timeout=timeout)
def __init__(self):
#获取当前目录
cur_path = os.path.split(os.path.realpath(__file__))[0]
#获取父级目录
parent_path = os.path.dirname(cur_path)
#配置文件路径
ini_file = "%s/conf.ini" % parent_path
#缓存文件路径
cache_path = "%s/cache/" % parent_path
cf = ConfigParser.ConfigParser()
cf.read(ini_file)
self.sendkey = cf.get('pushbear', 'sendkey')
self.text = cf.get('pushbear', 'sendname')
self.push_url = 'https://pushbear.ftqq.com/sub'
#缓存
self.cache = FileSystemCache(cache_path)
#push次数key
self.cache_push_times_key = 'today_push_times'
