class FileSystemSessionInterface(SessionInterface): """Uses the :class:`werkzeug.contrib.cache.FileSystemCache` as a session backend. :param cache_dir: the directory where session files are stored. :param threshold: the maximum number of items the session stores before it starts deleting some. :param mode: the file mode wanted for the session files, default 0600 :param key_prefix: A prefix that is added to FileSystemCache store keys. """ session_class = FileSystemSession def __init__(self, cache_dir, threshold, mode, key_prefix): from werkzeug.contrib.cache import FileSystemCache self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode) self.key_prefix = key_prefix def _generate_sid(self): return str(uuid4()) def open_session(self, app, request): sid = request.cookies.get(app.session_cookie_name) if not sid: sid = self._generate_sid() return self.session_class(sid=sid) data = self.cache.get(self.key_prefix + sid) if data is not None: return self.session_class(data, sid=sid) return self.session_class(sid=sid) def save_session(self, app, session, response): domain = self.get_cookie_domain(app) path = self.get_cookie_path(app) if not session: if session.modified: self.cache.delete(self.key_prefix + session.sid) response.delete_cookie(app.session_cookie_name, domain=domain, path=path) return # Modification case. There are upsides and downsides to # emitting a set-cookie header each request. The behavior # is controlled by the :meth:`should_set_cookie` method # which performs a quick check to figure out if the cookie # should be set or not. This is controlled by the # SESSION_REFRESH_EACH_REQUEST config flag as well as # the permanent flag on the session itself. # if not self.should_set_cookie(app, session): # return httponly = self.get_cookie_httponly(app) secure = self.get_cookie_secure(app) expires = self.get_expiration_time(app, session) data = dict(session) self.cache.set(self.key_prefix + session.sid, data, int(app.permanent_session_lifetime.total_seconds())) response.set_cookie(app.session_cookie_name, session.sid, expires=expires, httponly=httponly, domain=domain, path=path, secure=secure)
def number_gen(first_part, start_num, end_num, semester, dept, subjects): driver = webdriver.Firefox() # Firefox used for testing. Change it to PhantomJS driver.implicitly_wait(30) base_url = "http://result.pondiuni.edu.in/candidate.asp" url = base_url driver.get(base_url) # os.mkdir(str(first_part)) os.chdir("results") os.chdir(str(first_part)) cache = FileSystemCache('.cachedir', threshold=100000) for number in range(start_num, end_num + 1): current_num = "%04d" % number numb = first_part + str(current_num) driver.find_element_by_id("txtregno").clear() driver.find_element_by_id("txtregno").send_keys(numb) Select(driver.find_element_by_id("cmbdegree")).select_by_visible_text(dept) Select(driver.find_element_by_id("cmbexamno")).select_by_visible_text(semester) driver.find_element_by_id("button1").click() # copying the content page_source = cache.get(url) page_source = driver.page_source cache.set(url, page_source, timeout=60 * 60 * 24 * 7) # week in seconds root = html.document_fromstring(page_source) Cleaner(kill_tags=['noscript'], style=True)(root) # lxml >= 2.3.1 # pasting to file filename = str(numb) + ".txt" fp = open(filename, 'w') fp.write((root.text_content()).encode('utf-8')) fp.close() driver.back() driver.close() return analyze(subjects)
def check_cache( self, bib_id ): """ Checks cache for marc. """ cache = FileSystemCache( self.cache_dir, threshold=500, default_timeout=self.cache_hours, mode=0664 ) # http://werkzeug.pocoo.org/docs/0.9/contrib/cache/ cache_key = bib_id marc = cache.get( cache_key ) if marc == None: self.log.debug( u'in app_helper.Helper.check_cache(); marc not found in cache' ) else: self.log.debug( u'in app_helper.Helper.check_cache(); marc found in cache' ) return ( marc, cache )
def setUp(self): """BaseModel test set up""" if os.path.isfile("/tmp/box.db"): os.unlink("/tmp/box.db") DBHelper().set_db("/tmp/box.db") InstallHelper.reset() cache = FileSystemCache("/tmp/werkzeug") cache.clear() BaseModel.set_cache(cache) SampleModel.install()
def test_filesystemcache_clear(): """ test if FileSystemCache.clear works """ tmp_dir = tempfile.mkdtemp() cache = FileSystemCache(cache_dir=tmp_dir) cache.set("foo", "bar") cache_files = os.listdir(tmp_dir) assert len(cache_files) == 1 cache.clear() cache_files = os.listdir(tmp_dir) assert len(cache_files) == 0 shutil.rmtree(tmp_dir)
def test_filesystemcache_prune(): """ test if FileSystemCache._prune works and keeps the cache entry count below the given threshold. """ THRESHOLD = 13 tmp_dir = tempfile.mkdtemp() cache = FileSystemCache(cache_dir=tmp_dir, threshold=THRESHOLD) for i in range(2 * THRESHOLD): cache.set(str(i), i) cache_files = os.listdir(tmp_dir) shutil.rmtree(tmp_dir) assert len(cache_files) <= THRESHOLD
def test_filesystemcache_set_get(): """ test if FileSystemCache.set/get works """ tmp_dir = tempfile.mkdtemp() try: cache = FileSystemCache(cache_dir=tmp_dir) for i in range(3): cache.set(str(i), i * i) for i in range(3): result = cache.get(str(i)) assert result == i * i finally: shutil.rmtree(tmp_dir)
class BaseProvider(object): __metaclass__ = ABCMeta def __init__(self, cache_dir=None, default_timeout=60 * 60 * 24, api_key=None): # store it in cache for 1 day. using file system cache because # memcached is too mainstream. :) self.cache = FileSystemCache(cache_dir=cache_dir or '/tmp/__arcoiro__', default_timeout=default_timeout) self._api_key = api_key @abstractproperty def name(self): pass @abstractproperty def url(self): pass @abstractmethod def get_urls_from_tag(self, tag): pass @property def display_name(self): return self.name @property def api_key(self): if self._api_key is not None: return self._api_key config_key = '%s_API_KEY' % self.name.upper() key = current_app.config.get(config_key) if key is None: raise RuntimeError('%s not defined!' % config_key) return key def get_cached_urls_from_tag(self, tag): cache_key = '%s:%s' % (self.name, tag) urls = self.cache.get(cache_key) if urls is not None: return urls urls = self.get_urls_from_tag(tag) if urls is None: return None self.cache.set(cache_key, urls) return urls
def __init__(self, cache_dir=None, default_timeout=60 * 60 * 24, api_key=None): # store it in cache for 1 day. using file system cache because # memcached is too mainstream. :) self.cache = FileSystemCache(cache_dir=cache_dir or '/tmp/__arcoiro__', default_timeout=default_timeout) self._api_key = api_key
def __init__(self, cache_dir, threshold, mode, key_prefix, use_signer=False, permanent=True): from werkzeug.contrib.cache import FileSystemCache self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode) self.key_prefix = key_prefix self.use_signer = use_signer self.permanent = permanent
class WechatCache(object): """基于文件的缓存 """ def __init__(self, cache_dir='cache', default_timeout=300): """初始化 cache_dir是缓存目录 """ self.cache = FileSystemCache(cache_dir, default_timeout=default_timeout) def clear(self): """清空缓存 """ return self.cache.clear() def get(self, key): """获取缓存 获取键值key的缓存值 如果没有对应缓存,返回None """ return self.cache.get(key) def add(self, key, value, timeout=None): """增加缓存 如果键值key对应的缓存不存在,那么增加值value到键值key,过期时间timeout,默认300秒 否则返回False(即不能覆盖设置缓存) """ return self.cache.add(key, value, timeout) def set(self, key, value, timeout=None): """设置缓存 设置键值key的缓存为value,过期时间300秒 """ return self.cache.set(key, value, timeout) def delete(self, key): """删除缓存 删除键值key存储的缓存 """ return self.cache.delete(key)
def parse(url): cache = FileSystemCache('.cachedir', threshold=100000) # get page page_source = cache.get(url) if page_source is None: # use firefox to get page with javascript generated content with closing(Firefox()) as browser: browser.get(url) page_source = browser.page_source cache.set(url, page_source, timeout=60*60*24*7) # week in seconds # extract text root = html.document_fromstring(page_source) # remove flash, images, <script>,<style>, etc Cleaner(kill_tags=['noscript'], style=True)(root) # lxml >= 2.3.1 return root.text_content() # extract text
def getSpaceInfo(spaceId): global spaceCache if spaceCache is None: mkdir_p(DURACLOUD_SPACE_CACHE_DIR) spaceCache = FileSystemCache(DURACLOUD_SPACE_CACHE_DIR, threshold=50, default_timeout=(24*3600), mode=384) # check spaceCache, otherwise fetch info from DuraCloud result = spaceCache.get(spaceId) if result is None: url = DURACLOUD_URL+ "/duradmin/download/contentItem" auth = HTTPBasicAuth(DURACLOUD_USERNAME, DURACLOUD_PASSWORD) payload = {'spaceId': spaceId, 'contentId': 'info.json'} try: response = requests.get(url, params=payload, auth=auth) result = response.json() spaceCache.set(spaceId, result) except RequestException as e: print e raise return result
def __init__(self, app, cache_dir='/tmp/cache', lock_file=None): if not os.path.isdir(cache_dir): os.mkdir(cache_dir) self.app = app self.cache = FileSystemCache(cache_dir, default_timeout=600) self.t_local = threading.local() if lock_file is not None: lock_file = os.path.abspath(os.path.realpath(lock_file)) else: lock_file = '/tmp/cache.lock' self.lock_file = open(lock_file, 'wb+') self.t_lock = threading.Lock()
def __init__(self, path, tolerance, expiration, max_values, run_tests=True, max_file_size=0): """Constructor method. :param path: The path of the cache database file. :param tolerance: The tolerance, in seconds to which a TimeMap is considered young enough to be used as is. :param expiration: How long, in seconds, the cache entries are stored every get will be a CACHE MISS. :param max_values: The maximum number of TimeMaps stored in cache before some are deleted :param run_tests: (Optional) Tests the cache at initialization. :param max_file_size: (Optional) The maximum size (in Bytes) for a TimeMap cache value. When max_file_size=0, there is no limit to a cache value. When max_file_size=X > 0, the cache will not store TimeMap that require more than X Bytes on disk. """ # Parameters Check if tolerance <= 0 or expiration <= 0 or max_values <= 0: raise CacheError("Cannot create cache: all parameters must be > 0") self.tolerance = relativedelta(seconds=tolerance) self.path = path.rstrip('/') self.max_file_size = max(max_file_size, 0) self.CHECK_SIZE = self.max_file_size > 0 self.max_values = max_values self.backend = FileSystemCache(path, threshold=self.max_values, default_timeout=expiration) # Testing cache if run_tests: try: key = '1' val = 1 self.backend.set(key, val) assert (not self.CHECK_SIZE) or self._check_size(key) > 0 assert self.backend.get(key) == val os.remove(self.path + '/' + md5(key).hexdigest()) except Exception as e: raise CacheError("Error testing cache: %s" % e) logging.debug( "Cache created. max_files = %d. Expiration = %d. " "max_file_size = %d" % ( self.max_values, expiration, self.max_file_size))
def _set_cache(self): if self.app.config['TESTING']: self.cache = NullCache() else: if self.app.config['CACHE_TYPE'] == 'Null': self.cache = NullCache() elif self.app.config['CACHE_TYPE'] == 'Simple': self.cache = SimpleCache( threshold=self.app.config['CACHE_THRESHOLD'], default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT']) elif self.app.config['CACHE_TYPE'] == 'Memcached': self.cache = MemcachedCache( self.app.config['CACHE_MEMCACHED_SERVERS'], default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'], key_prefix=self.app.config['CACHE_KEY_PREFIX']) elif self.app.config['CACHE_TYPE'] == 'GAE': self.cache = GAEMemcachedCache( default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'], key_prefix=self.app.config['CACHE_KEY_PREFIX']) elif self.app.config['CACHE_TYPE'] == 'FileSystem': self.cache = FileSystemCache( self.app.config['CACHE_DIR'], threshold=self.app.config['CACHE_THRESHOLD'], default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'])
def filesystem(app, config, args, kwargs): args.insert(0, config['CACHE_DIR']) kwargs.update(dict(threshold=config['CACHE_THRESHOLD'])) return FileSystemCache(*args, **kwargs)
class CacheMiddleware: def __init__(self, app, cache_dir='/tmp/cache', lock_file=None): if not os.path.isdir(cache_dir): os.mkdir(cache_dir) self.app = app self.cache = FileSystemCache(cache_dir, default_timeout=600) self.t_local = threading.local() if lock_file is not None: lock_file = os.path.abspath(os.path.realpath(lock_file)) else: lock_file = '/tmp/cache.lock' self.lock_file = open(lock_file, 'wb+') self.t_lock = threading.Lock() def __call__(self, environ, start_response): self.t_local.start_server_response = start_response # get keys content, header and status self.t_local.content_key = environ['PATH_INFO'] if environ.get('QUERY_STRING'): self.t_local.content_key = '%s?%s' % (self.t_local.content_key, environ['QUERY_STRING']) self.t_local.header_key = '%s|%s' % (self.t_local.content_key, 'header') self.t_local.status_key = '%s|%s' % (self.t_local.content_key, 'status') # get cached response keys = (self.t_local.content_key, self.t_local.header_key, self.t_local.status_key) with self.t_lock: try: fcntl.lockf(self.lock_file, fcntl.LOCK_SH) content, headers, status = self.cache.get_many(*keys) finally: fcntl.lockf(self.lock_file, fcntl.LOCK_UN) if (content is not None and headers is not None and status is not None): # return cached response self.t_local.start_server_response(status, headers) return content else: # call app and cache the response content = self.app(environ, self.start_response) if self.do_cache(): content = list(content) cached = {self.t_local.content_key: content, self.t_local.header_key: self.t_local.headers, self.t_local.status_key: self.t_local.status} with self.t_lock: try: fcntl.lockf(self.lock_file, fcntl.LOCK_EX) self.cache.set_many(cached) finally: fcntl.lockf(self.lock_file, fcntl.LOCK_UN) return content def start_response(self, status, response_headers, exc_info=None): self.t_local.status = status self.t_local.headers = response_headers self.t_local.start_server_response(status, response_headers, exc_info) def do_cache(self): if 200 <= int(self.t_local.status.split(' ')[0]) < 300: return True else: return False
from flask import Flask, render_template from cinemas import get_complete_info from werkzeug.contrib.cache import FileSystemCache import tempfile app = Flask(__name__) cache = FileSystemCache(cache_dir=tempfile.gettempdir()) @app.route('/') def films_list(): movie_list = cache.get('movie_list') if movie_list is None: movie_list = get_complete_info() cache.set('movie_list', movie_list, timeout=30 * 30) return render_template('films_list.html', movie_list=movie_list) if __name__ == "__main__": app.run()
from flask import Flask, flash, render_template, \ request, redirect, send_from_directory, make_response import os, generate_identifiers, hgvs_conversion, json_conversion, sqlite3 from werkzeug.contrib.cache import FileSystemCache from hashlib import sha256 # create the application object APP = Flask(__name__, static_folder='static') APP.secret_key = "vmcsuite" APP.config['UPLOAD_FOLDER'] = 'static/uploads' cache = FileSystemCache('/tmp/vmc-suite') def get_json_schema(): """Returns the example JSON schema file as a string""" with open("static/schema.json") as f_in: return f_in.read() def get_filename(): """Returns the name of the uploaded VCF file""" return request.cookies.get("filename") def get_fileKey(): """Returns the key for the uploaded VCF file""" return request.cookies.get("fileKey") def get_chrs_intervals_and_states(vcf_upload): """Takes the VCF file and returns lists of chromosomes, intervals, and states for each variant"""
def __init__(self): self._cache = FileSystemCache(Cache.cache_dir(), default_timeout=10 * 60)
def create_app(config_name): app = Flask(__name__, static_url_path='') app.config.from_object(config[config_name]) app.config.from_envvar("OPENTARGETS_API_LOCAL_SETTINGS", silent=True) config[config_name].init_app(app) api_version = app.config['API_VERSION'] api_version_minor = app.config['API_VERSION_MINOR'] # log_level = logging.INFO # if app.config['DEBUG']: # log_level = logging.DEBUG # Flask has a default logger which works well and pushes to stderr # if you want to add different handlers (to file, or logstash, or whatever) # you can use code similar to the one below and set the error level accordingly. # logHandler = logging.StreamHandler() # formatter = jsonlogger.JsonFormatter() # logHandler.setFormatter(formatter) # loghandler.setLevel(logging.INFO) # app.logger.addHandler(logHandler) # or for LOGSTASH # app.logger.addHandler(logstash.LogstashHandler(app.config['LOGSTASH_HOST'], app.config['LOGSTASH_PORT'], version=1)) app.logger.info('looking for elasticsearch at: %s' % app.config['ELASTICSEARCH_URL']) print('looking for elasticsearch at: %s' % app.config['ELASTICSEARCH_URL']) app.extensions['redis-core'] = Redis(app.config['REDIS_SERVER_PATH'], db=0) #served data app.extensions['redis-service'] = Redis( app.config['REDIS_SERVER_PATH'], db=1) #cache, rate limit and internal things app.extensions['redis-user'] = Redis(app.config['REDIS_SERVER_PATH'], db=2) # user info '''setup cache''' app.extensions['redis-service'].config_set('save', '') app.extensions['redis-service'].config_set('appendonly', 'no') icache = InternalCache(app.extensions['redis-service'], str(api_version_minor)) ip2org = IP2Org(icache) es = Elasticsearch( app.config['ELASTICSEARCH_URL'], # # sniff before doing anything # sniff_on_start=True, # # refresh nodes after a node fails to respond # sniff_on_connection_fail=True, # # and also every 60 seconds # sniffer_timeout=60 timeout=60 * 20, maxsize=100, ) '''elasticsearch handlers''' app.extensions['esquery'] = esQuery( es, DataTypes(app), DataSourceScoring(app), index_data=app.config['ELASTICSEARCH_DATA_INDEX_NAME'], index_efo=app.config['ELASTICSEARCH_EFO_LABEL_INDEX_NAME'], index_eco=app.config['ELASTICSEARCH_ECO_INDEX_NAME'], index_genename=app.config['ELASTICSEARCH_GENE_NAME_INDEX_NAME'], index_expression=app.config['ELASTICSEARCH_EXPRESSION_INDEX_NAME'], index_reactome=app.config['ELASTICSEARCH_REACTOME_INDEX_NAME'], index_association=app. config['ELASTICSEARCH_DATA_ASSOCIATION_INDEX_NAME'], index_search=app.config['ELASTICSEARCH_DATA_SEARCH_INDEX_NAME'], index_relation=app.config['ELASTICSEARCH_DATA_RELATION_INDEX_NAME'], docname_data=app.config['ELASTICSEARCH_DATA_DOC_NAME'], docname_efo=app.config['ELASTICSEARCH_EFO_LABEL_DOC_NAME'], docname_eco=app.config['ELASTICSEARCH_ECO_DOC_NAME'], docname_genename=app.config['ELASTICSEARCH_GENE_NAME_DOC_NAME'], docname_expression=app.config['ELASTICSEARCH_EXPRESSION_DOC_NAME'], docname_reactome=app. config['ELASTICSEARCH_REACTOME_REACTION_DOC_NAME'], docname_association=app. config['ELASTICSEARCH_DATA_ASSOCIATION_DOC_NAME'], docname_search=app.config['ELASTICSEARCH_DATA_SEARCH_DOC_NAME'], # docname_search_target=app.config['ELASTICSEARCH_DATA_SEARCH_TARGET_DOC_NAME'], # docname_search_disease=app.config['ELASTICSEARCH_DATA_SEARCH_DISEASE_DOC_NAME'], docname_relation=app.config['ELASTICSEARCH_DATA_RELATION_DOC_NAME'], log_level=app.logger.getEffectiveLevel(), cache=icache) app.extensions['es_access_store'] = esStore( es, eventlog_index=app.config['ELASTICSEARCH_LOG_EVENT_INDEX_NAME'], ip2org=ip2org, ) '''mixpanel handlers''' if Config.MIXPANEL_TOKEN: mp = Mixpanel(Config.MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer()) app.extensions['mixpanel'] = mp app.extensions['mp_access_store'] = MixPanelStore( mp, ip2org=ip2org, ) app.extensions['proxy'] = ProxyHandler( allowed_targets=app.config['PROXY_SETTINGS']['allowed_targets'], allowed_domains=app.config['PROXY_SETTINGS']['allowed_domains'], allowed_request_domains=app.config['PROXY_SETTINGS'] ['allowed_request_domains']) basepath = app.config['PUBLIC_API_BASE_PATH'] + api_version # cors = CORS(app, resources=r'/api/*', allow_headers='Content-Type,Auth-Token') ''' define cache''' # cache = Cache(config={'CACHE_TYPE': 'simple'}) # cache.init_app(latest_blueprint) # latest_blueprint.cache = cache # latest_blueprint.extensions['cache'] = cache # app.cache = SimpleCache() app.cache = FileSystemCache('/tmp/cttv-rest-api-cache', threshold=100000, default_timeout=60 * 60, mode=777) '''Set usage limiter ''' # limiter = Limiter(global_limits=["2000 per hour", "20 per second"]) # limiter.init_app(app)# use redis to store limits '''Load api keys in redis''' rate_limit_file = app.config['USAGE_LIMIT_PATH'] if not os.path.exists(rate_limit_file): rate_limit_file = '../' + rate_limit_file if os.path.exists(rate_limit_file): with open(rate_limit_file) as csvfile: reader = csv.DictReader(csvfile) for row in reader: auth_key = AuthKey(**row) app.extensions['redis-user'].hmset(auth_key.get_key(), auth_key.__dict__) print('INFO - succesfully loaded rate limit file') else: print('ERROR - cannot find rate limit file') app.logger.error( 'cannot find rate limit file: %s. RATE LIMIT QUOTA LOAD SKIPPED!' % rate_limit_file) '''load ip name resolution''' ip_resolver = defaultdict(lambda: "PUBLIC") ip_list_file = app.config['IP_RESOLVER_LIST_PATH'] if not os.path.exists(ip_list_file): ip_list_file = '../' + ip_list_file if os.path.exists(ip_list_file): with open(ip_list_file) as csvfile: reader = csv.DictReader(csvfile) for row in reader: net = IPNetwork(row['ip']) ip_resolver[net] = row['org'] else: app.logger.warning( 'cannot find IP list for IP resolver. All traffic will be logged as PUBLIC' ) app.config['IP_RESOLVER'] = ip_resolver '''compress http response''' compress = Compress() compress.init_app(app) latest_blueprint = Blueprint('latest', __name__) current_version_blueprint = Blueprint(str(api_version), __name__) current_minor_version_blueprint = Blueprint(str(api_version_minor), __name__) specpath = '/cttv' if app.config['PROFILE'] == True: from werkzeug.contrib.profiler import ProfilerMiddleware app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30]) create_api(latest_blueprint, api_version, specpath) create_api(current_version_blueprint, api_version, specpath) create_api(current_minor_version_blueprint, api_version_minor, specpath) app.register_blueprint(latest_blueprint, url_prefix='/api/latest') app.register_blueprint(current_version_blueprint, url_prefix='/api/' + str(api_version)) app.register_blueprint(current_minor_version_blueprint, url_prefix='/api/' + str(api_version_minor)) @app.route('/api-docs/%s' % str(api_version_minor)) def docs_current_minor_version(): return redirect('/api/swagger/index.html') @app.route('/api-docs/%s' % str(api_version)) def docs_current_version(): return redirect('/api/swagger/index.html') @app.route('/api-docs') def docs(): return redirect('/api/swagger/index.html') def serve_swagger(): return app.send_static_file('docs/swagger/swagger.yaml') @app.route('/api/docs/swagger.yaml') def send_swagger(): return serve_swagger() @app.route('/api/latest/docs/swagger.yaml') def send_swagger_latest(): return serve_swagger() @app.route('/api/' + str(api_version) + '/docs/swagger.yaml') def send_swagger_current_cersion(): return serve_swagger() @app.before_request def before_request(): g.request_start = datetime.now() @app.after_request def after(resp): rate_limiter = RateLimiter() now = datetime.now() took = (now - g.request_start).total_seconds() * 1000 if took > 500: cache_time = str( int(3600 * took) ) # set cache to last one our for each second spent in the request resp.headers.add('X-Accel-Expires', cache_time) took = int(round(took)) LogApiCallWeight(took) # if took < RateLimiter.DEFAULT_CALL_WEIGHT: # took = RateLimiter.DEFAULT_CALL_WEIGHT current_values = increment_call_rate(took, rate_limiter) now = datetime.now() ceil10s = round(ceil_dt_to_future_time(now, 10), 2) ceil1h = round(ceil_dt_to_future_time(now, 3600), 2) usage_left_10s = rate_limiter.short_window_rate - current_values[ 'short'] usage_left_1h = rate_limiter.long_window_rate - current_values['long'] min_ceil = ceil10s if usage_left_1h < 0: min_ceil = ceil1h if (usage_left_10s < 0) or (usage_left_1h < 0): resp.headers.add('Retry-After', min_ceil) resp.headers.add('X-API-Took', took) resp.headers.add('X-Usage-Limit-10s', rate_limiter.short_window_rate) resp.headers.add('X-Usage-Limit-1h', rate_limiter.long_window_rate) resp.headers.add('X-Usage-Remaining-10s', usage_left_10s) resp.headers.add('X-Usage-Remaining-1h', usage_left_1h) # resp.headers.add('X-Usage-Limit-Reset-10s', ceil10s) # resp.headers.add('X-Usage-Limit-Reset-1h', ceil1h) resp.headers.add('Access-Control-Allow-Origin', '*') resp.headers.add('Access-Control-Allow-Headers', 'Content-Type,Auth-Token') if do_not_cache(request): # do not cache in the browser resp.headers.add('Cache-Control', "no-cache, must-revalidate, max-age=0") else: resp.headers.add( 'Cache-Control', "no-transform, public, max-age=%i, s-maxage=%i" % (took * 1800 / 1000, took * 9000 / 1000)) return resp return app
from unsw_ldap import authenticate # config.variables contains appropriate pathnames for this course & session import config app = Flask(__name__, template_folder=os.path.join( config.variables['public_html_session_directory'], '.'), static_folder=os.path.join( config.variables['public_html_session_directory'], 'static')) app.config.from_object(__name__) cache = None if os.path.exists(config.variables['flask_cache_directory']): try: cache = FileSystemCache(config.variables['flask_cache_directory']) except (OSError, ImportError): pass # cache & gzip requests def before_request(): if not cache or 'gzip' not in request.headers.get('Accept-Encoding', '').lower(): return None # tutors served different content so include is_tutor in key cache_key = str(is_tutor()) + str(request.path) return cache.get(cache_key) def after_request(response):
class Cache(object): """Base class for TimeGate caches.""" def __init__(self, path, tolerance, expiration, max_values, run_tests=True, max_file_size=0): """Constructor method. :param path: The path of the cache database file. :param tolerance: The tolerance, in seconds to which a TimeMap is considered young enough to be used as is. :param expiration: How long, in seconds, the cache entries are stored every get will be a CACHE MISS. :param max_values: The maximum number of TimeMaps stored in cache before some are deleted :param run_tests: (Optional) Tests the cache at initialization. :param max_file_size: (Optional) The maximum size (in Bytes) for a TimeMap cache value. When max_file_size=0, there is no limit to a cache value. When max_file_size=X > 0, the cache will not store TimeMap that require more than X Bytes on disk. """ # Parameters Check if tolerance <= 0 or expiration <= 0 or max_values <= 0: raise CacheError("Cannot create cache: all parameters must be > 0") self.tolerance = relativedelta(seconds=tolerance) self.path = path.rstrip('/') self.max_file_size = max(max_file_size, 0) self.CHECK_SIZE = self.max_file_size > 0 self.max_values = max_values self.backend = FileSystemCache(path, threshold=self.max_values, default_timeout=expiration) # Testing cache if run_tests: try: key = '1' val = 1 self.backend.set(key, val) assert (not self.CHECK_SIZE) or self._check_size(key) > 0 assert self.backend.get(key) == val os.remove(self.path + '/' + md5(key).hexdigest()) except Exception as e: raise CacheError("Error testing cache: %s" % e) logging.debug( "Cache created. max_files = %d. Expiration = %d. " "max_file_size = %d" % ( self.max_values, expiration, self.max_file_size)) def get_until(self, uri_r, date): """Returns the TimeMap (memento,datetime)-list for the requested Memento. The TimeMap is guaranteed to span at least until the 'date' parameter, within the tolerance. :param uri_r: The URI-R of the resource as a string. :param date: The target date. It is the accept-datetime for TimeGate requests, and the current date. The cache will return all Mementos prior to this date (within cache.tolerance parameter) :return: [(memento_uri_string, datetime_obj),...] list if it is in cache and if it is within the cache tolerance for *date*, None otherwise. """ # Query the backend for stored cache values to that memento key = uri_r try: val = self.backend.get(key) except Exception as e: logging.error("Exception loading cache content: %s" % e) return None if val: # There is a value in the cache timestamp, timemap = val logging.info("Cached value exists for %s" % uri_r) if date > timestamp + self.tolerance: logging.info("Cache MISS: value outdated for %s" % uri_r) timemap = None else: logging.info("Cache HIT: found value for %s" % uri_r) else: # Cache MISS: No value logging.info("Cache MISS: No cached value for %s" % uri_r) timemap = None return timemap def get_all(self, uri_r): """Request the whole TimeMap for that uri. :param uri_r: the URI-R of the resource. :return: [(memento_uri_string, datetime_obj),...] list if it is in cache and if it is within the cache tolerance, None otherwise. """ return self.get_until(uri_r, timegate_utils.now()) def refresh(self, uri_r, getter, *args, **kwargs): """Refreshes the cached TimeMap for a specific resource and returns it. :param uri_r: The original resource URI to refresh the TimeMap :param getter: The function to call to get a fresh TimeMap :param args: *getter* arguments :param kwargs: *getter* keywords arguments :return: The fresh TimeMap """ timemap = parsed_request(getter, *args, **kwargs) # timemap,new_uri = parsed_request(getter, *args, **kwargs) # if new_uri: # uri_r = new_uri # Creates or refreshes the new timemap for that URI-R self._set(uri_r, timemap) return timemap def _set(self, uri_r, timemap): """Sets / refreshes the cached TimeMap for that URI-R. And appends it with a timestamp of when it is stored. :param uri_r: The URI-R of the original resource. :param timemap: The value to cache. :return: The backend setter method return value. """ logging.info("Updating cache for %s" % uri_r) timestamp = timegate_utils.now() val = (timestamp, timemap) key = uri_r try: self.backend.set(key, val) if self.CHECK_SIZE: self._check_size(uri_r) except Exception as e: logging.error("Error setting cache value: %s" % e) def _check_size(self, key, delete=True): """Check the size that a specific TimeMap value is using on disk. It deletes if it is more than the maximum size. :param key: The TimeMap original resource. :param delete: (Optional) When true, the value is deleted. Else only a warning is raised. :return: The size of the value on disk (0 if it was deleted). """ try: fname = md5(key).hexdigest() # werkzeug key fpath = self.path + '/' + fname size = os.path.getsize(fpath) if size > self.max_file_size and delete: message = ("Cache value too big (%dB, max %dB) " "for the TimeMap of %s") if delete: message += ". Deleting cached value." os.remove(fpath) size = 0 logging.warning(message % (size, self.max_file_size, key)) return size except Exception as e: logging.error( "Exception checking cache value size for TimeMap of %s " "Exception: %s" % (key, e)) return 0
def runCouchPotato(options, base_path, args): # Load settings from couchpotato.environment import Env settings = Env.get('settings') settings.setFile(options.config_file) # Create data dir if needed data_dir = os.path.expanduser(Env.setting('data_dir')) if data_dir == '': data_dir = os.path.join(base_path, '_data') if not os.path.isdir(data_dir): os.makedirs(data_dir) # Create logging dir log_dir = os.path.join(data_dir, 'logs') if not os.path.isdir(log_dir): os.mkdir(log_dir) # Daemonize app if options.daemonize: createDaemon() # Register environment settings Env.set('uses_git', not options.git) Env.set('app_dir', base_path) Env.set('data_dir', data_dir) Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log')) Env.set('db_path', 'sqlite:///' + os.path.join(data_dir, 'couchpotato.db')) Env.set('cache_dir', os.path.join(data_dir, 'cache')) Env.set('cache', FileSystemCache(os.path.join(Env.get('cache_dir'), 'python'))) Env.set('quiet', options.quiet) Env.set('daemonize', options.daemonize) Env.set('args', args) # Determine debug debug = options.debug or Env.setting('debug', default=False) Env.set('debug', debug) # Only run once when debugging if os.environ.get('WERKZEUG_RUN_MAIN') or not debug: # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) # To screen if debug and not options.quiet and not options.daemonize: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Disable server access log server_log = logging.getLogger('werkzeug') server_log.disabled = True # Start logging from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s' % options) # Load configs & plugins loader = Env.get('loader') loader.preload(root=base_path) loader.run() # Load migrations from migrate.versioning.api import version_control, db_version, version, upgrade db = Env.get('db_path') repo = os.path.join(base_path, 'couchpotato', 'core', 'migration') logging.getLogger('migrate').setLevel( logging.WARNING) # Disable logging for migration latest_db_version = version(repo) initialize = True try: current_db_version = db_version(db, repo) initialize = False except: version_control(db, repo, version=latest_db_version) current_db_version = db_version(db, repo) if current_db_version < latest_db_version and not debug: log.info('Doing database upgrade. From %d to %d' % (current_db_version, latest_db_version)) upgrade(db, repo) # Configure Database from couchpotato.core.settings.model import setup setup() fireEventAsync('app.load') if initialize: fireEventAsync('app.initialize') # Create app from couchpotato import app api_key = Env.setting('api_key') url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting( 'url_base') else '' reloader = debug and not options.daemonize # Basic config app.secret_key = api_key config = { 'use_reloader': reloader, 'host': Env.setting('host', default='0.0.0.0'), 'port': Env.setting('port', default=5000) } # Static path web.add_url_rule(url_base + '/static/<path:filename>', endpoint='static', view_func=app.send_static_file) # Register modules app.register_blueprint(web, url_prefix='%s/' % url_base) app.register_blueprint(api, url_prefix='%s/%s/' % (url_base, api_key)) # Go go go! app.run(**config)
def getCache(name): return FileSystemCache(os.path.join("cache", name))
def __init__(self, cache_dir='cache', default_timeout=300): """初始化 cache_dir是缓存目录 """ self.cache = FileSystemCache(cache_dir, default_timeout=default_timeout)
from flask import Flask from werkzeug.contrib.cache import FileSystemCache import os import logging logging.basicConfig(level=logging.INFO) root = os.getenv("MEDIABROWSER_ROOT") if not root: raise Exception('Must set MEDIABROWSER_ROOT variable') cache_dir = os.getenv("MEDIABROWSER_CACHEDIR") if not cache_dir: raise Exception('Must set MEDIABROWSER_CACHEDIR variable') # default_timeout=0 doesn't work with FileSystemCache cache = FileSystemCache(cache_dir, default_timeout=9999999999, threshold=5000) application = Flask("mediabrowser-demo") application.register_blueprint(mediabrowser.build(root, cache)) def reverse_proxified(app): """ Configure apache as: RequestHeader set X-Script-Name /videos """ def wsgi_call(environ, start_response): script_name = environ.get('HTTP_X_SCRIPT_NAME', '') if script_name: environ['SCRIPT_NAME'] = script_name path_info = environ['PATH_INFO'] if path_info.startswith(script_name): environ['PATH_INFO'] = path_info[len(script_name):]
def __init__(self, cache_dir, threshold, mode, key_prefix): from werkzeug.contrib.cache import FileSystemCache self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode) self.key_prefix = key_prefix
def configure(configFile=None, baseConfig="ProductionConfig", port=8000, extraConfig={}): """ TODO Document this critical function! What does it do? What does it assume? """ file_handler = StreamHandler() file_handler.setLevel(logging.WARNING) app.logger.addHandler(file_handler) configStr = 'ga4gh.server.serverconfig:{0}'.format(baseConfig) app.config.from_object(configStr) if os.environ.get('GA4GH_CONFIGURATION') is not None: app.config.from_envvar('GA4GH_CONFIGURATION') if configFile is not None: app.config.from_pyfile(configFile) app.config.update(extraConfig.items()) # Setup file handle cache max size datamodel.fileHandleCache.setMaxCacheSize( app.config["FILE_HANDLE_CACHE_MAX_SIZE"]) # Setup CORS cors.CORS(app, allow_headers='Content-Type') app.serverStatus = ServerStatus() app.backend = _configure_backend(app) if app.config.get('SECRET_KEY'): app.secret_key = app.config['SECRET_KEY'] elif app.config.get('OIDC_PROVIDER'): raise exceptions.ConfigurationException( 'OIDC configuration requires a secret key') if app.config.get('CACHE_DIRECTORY'): app.cache_dir = app.config['CACHE_DIRECTORY'] else: app.cache_dir = '/tmp/ga4gh' app.cache = FileSystemCache(app.cache_dir, threshold=5000, default_timeout=600, mode=384) app.oidcClient = None app.myPort = port if "OIDC_PROVIDER" in app.config: # The oic client. If we're testing, we don't want to verify # SSL certificates app.oidcClient = oic.oic.Client( verify_ssl=('TESTING' not in app.config)) try: app.oidcClient.provider_config(app.config['OIDC_PROVIDER']) except requests.exceptions.ConnectionError: configResponse = message.ProviderConfigurationResponse( issuer=app.config['OIDC_PROVIDER'], authorization_endpoint=app.config['OIDC_AUTHZ_ENDPOINT'], token_endpoint=app.config['OIDC_TOKEN_ENDPOINT'], revocation_endpoint=app.config['OIDC_TOKEN_REV_ENDPOINT']) app.oidcClient.handle_provider_config(configResponse, app.config['OIDC_PROVIDER']) # The redirect URI comes from the configuration. # If we are testing, then we allow the automatic creation of a # redirect uri if none is configured redirectUri = app.config.get('OIDC_REDIRECT_URI') if redirectUri is None and app.config.get('TESTING'): redirectUri = 'https://{0}:{1}/oauth2callback'.format( socket.gethostname(), app.myPort) app.oidcClient.redirect_uris = [redirectUri] if redirectUri is []: raise exceptions.ConfigurationException( 'OIDC configuration requires a redirect uri') # We only support dynamic registration while testing. if ('registration_endpoint' in app.oidcClient.provider_info and app.config.get('TESTING')): app.oidcClient.register( app.oidcClient.provider_info["registration_endpoint"], redirect_uris=[redirectUri]) else: response = message.RegistrationResponse( client_id=app.config['OIDC_CLIENT_ID'], client_secret=app.config['OIDC_CLIENT_SECRET'], redirect_uris=[redirectUri], verify_ssl=False) app.oidcClient.store_registration_info(response)
import os import logging from flask import Flask, render_template, jsonify, redirect, request, json, send_from_directory from werkzeug.contrib.cache import FileSystemCache import cinemas ENV_DEBUG = os.environ.get('DEBUG') == 'True' PORT = int(os.environ.get('PORT', 5000)) CACHE_TIMEOUT = 60 * 60 * 12 cache = FileSystemCache('.cachedir', default_timeout=CACHE_TIMEOUT) app = Flask(__name__) @app.route('/') def output_static_index_page(): return render_template('films_list.html') @app.route('/api') def output_static_api_page(): return render_template('api.html') @app.route('/movies', methods=['GET']) def output_movies(): movies_json = cache.get('movies_json') if movies_json is None: afisha_page = cinemas.fetch_afisha_page() movies_from_afisha = cinemas.parse_afisha_list(afisha_page)
class Cache(object): """ This class is used to control the cache objects. If TESTING is True it will use NullCache. """ def __init__(self, app=None): self.cache = None if app is not None: self.init_app(app) else: self.app = None def init_app(self, app): "This is used to initialize cache with your app object" app.config.setdefault('CACHE_DEFAULT_TIMEOUT', 300) app.config.setdefault('CACHE_THRESHOLD', 500) app.config.setdefault('CACHE_KEY_PREFIX', None) app.config.setdefault('CACHE_MEMCACHED_SERVERS', None) app.config.setdefault('CACHE_DIR', None) app.config.setdefault('CACHE_TYPE', 'NullCache') self.app = app self._set_cache() def _set_cache(self): if self.app.config['TESTING']: self.cache = NullCache() else: if self.app.config['CACHE_TYPE'] == 'Null': self.cache = NullCache() elif self.app.config['CACHE_TYPE'] == 'Simple': self.cache = SimpleCache( threshold=self.app.config['CACHE_THRESHOLD'], default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT']) elif self.app.config['CACHE_TYPE'] == 'Memcached': self.cache = MemcachedCache( self.app.config['CACHE_MEMCACHED_SERVERS'], default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'], key_prefix=self.app.config['CACHE_KEY_PREFIX']) elif self.app.config['CACHE_TYPE'] == 'GAE': self.cache = GAEMemcachedCache( default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'], key_prefix=self.app.config['CACHE_KEY_PREFIX']) elif self.app.config['CACHE_TYPE'] == 'FileSystem': self.cache = FileSystemCache( self.app.config['CACHE_DIR'], threshold=self.app.config['CACHE_THRESHOLD'], default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT']) def get(self, *args, **kwargs): "Proxy function for internal cache object." return self.cache.get(*args, **kwargs) def set(self, *args, **kwargs): "Proxy function for internal cache object." self.cache.set(*args, **kwargs) def add(self, *args, **kwargs): "Proxy function for internal cache object." self.cache.add(*args, **kwargs) def delete(self, *args, **kwargs): "Proxy function for internal cache object." self.cache.delete(*args, **kwargs) def cached(self, timeout=None, key_prefix='view/%s', unless=None): """ Decorator. Use this to cache a function. By default the cache key is `view/request.path`. You are able to use this decorator with any function by changing the `key_prefix`. If the token `%s` is located within the `key_prefix` then it will replace that with `request.path` Example:: # An example view function @cache.cached(timeout=50) def big_foo(): return big_bar_calc() # An example misc function to cache. @cache.cached(key_prefix='MyCachedList') def get_list(): return [random.randrange(0, 1) for i in range(50000)] .. code-block:: pycon >>> my_list = get_list() :param timeout: Default None. If set to an integer, will cache for that amount of time. :param key_prefix: Default 'view/%(request.path)s'. Beginning key to . use for the cache key. :param unless: Default None. Cache will *always* execute the caching facilities unless this callable is true. This will bypass the caching entirely. """ def decorator(f): @wraps(f) def decorated_function(*args, **kwargs): #: Bypass the cache entirely. if callable(unless) and unless() is True: return f(*args, **kwargs) if '%s' in key_prefix: cache_key = key_prefix % request.path else: cache_key = key_prefix rv = self.cache.get(cache_key) if not rv or current_app.debug: rv = f(*args, **kwargs) self.cache.set(cache_key, rv, timeout=timeout) return rv return decorated_function return decorator def memoize(self, timeout=None): """ Use this to cache the result of a function, taking its arguments into account in the cache key. Information on `Memoization <http://en.wikipedia.org/wiki/Memoization>`_. Example:: @cache.memoize(timeout=50) def big_foo(a, b): return a + b + random.randrange(0, 1000) .. code-block:: pycon >>> big_foo(5, 2) 753 >>> big_foo(5, 3) 234 >>> big_foo(5, 2) 753 :param timeout: Default None. If set to an integer, will cache for that amount of time. """ def memoize(f): @wraps(f) def decorated_function(*args, **kwargs): cache_key = (f.__name__, id(f), args, str(kwargs)) rv = self.cache.get(cache_key) if rv is None: rv = f(*args, **kwargs) self.cache.set(cache_key, rv) return rv return decorated_function return memoize
import json import os import requests from datetime import datetime from flask import Flask, render_template, request from socket import gethostname from urllib.parse import quote, urlparse from werkzeug.contrib.cache import FileSystemCache app = Flask(__name__) cache = FileSystemCache(os.path.join(app.root_path, '_cache')) def parse_date(timestamp): return datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d') def parse_google_sb(site): base = 'https://www.google.com/transparencyreport/safebrowsing/diagnostic/index.html#url=' # previously 'https://www.google.com/safebrowsing/diagnostic?site=' query = quote(site, safe='') page = base + query resp_url = 'https://www.google.com/safebrowsing/diagnostic?output=jsonp&site=' + query resp = requests.get(resp_url).text resp_str = resp.split('processResponse(')[1].split(');')[0] resp_obj = json.loads(resp_str) url = resp_obj['website'].get('name') num_tested = resp_obj.get('numTested') if num_tested: status_set = set() for k in resp_obj['website'].keys(): if k[-6:] == 'Status':
from flask import Flask, request from werkzeug.contrib.cache import FileSystemCache CACHE_TIMEOUT = 300 cache = FileSystemCache('summit/cache') class cached(object): def __init__(self, timeout=None): self.timeout = timeout or CACHE_TIMEOUT def __call__(self, f): def decorator(*args, **kwargs): response = cache.get(request.path) if response is None: response = f(*args, **kwargs) cache.set(request.path, response, self.timeout) return response return decorator
tic = CtsTextInventoryCollection() latin = CtsTextInventoryMetadata("urn:perseus:latinLit", parent=tic) latin.set_label("Classical Latin", "eng") latin.set_label("Latin Classique", "fre") dispatcher = CollectionDispatcher(tic) @dispatcher.inventory("urn:perseus:latinLit") def dispatchLatinLit(collection, path=None, **kwargs): if collection.id.startswith("urn:cts:latinLit:"): return True return False return dispatcher nautilus_cache = FileSystemCache(subprocess_cache_dir, default_timeout=0) resolver = NautilusCTSResolver(subprocess_repository, dispatcher=make_dispatcher(), cache=nautilus_cache) app = Flask("Nautilus") http_cache = Cache(app, config={ 'CACHE_TYPE': "filesystem", "CACHE_DIR": http_cache_dir, "CACHE_DEFAULT_TIMEOUT": 0 }) nautilus = FlaskNautilus(app=app, prefix="/api", name="nautilus",
#!/usr/bin/python #coding=utf-8 # imports from flask import Flask from flask.ext.sqlalchemy import SQLAlchemy from werkzeug.contrib.cache import FileSystemCache app = Flask(__name__) app.config.from_object( 'config.config.DevelopmentConfig') # 导入所有配置,后续都需要用app.config['key']获取。 # app.config.from_object('config.config.ProductionConfig') # 导入所有配置,后续都需要用app.config['key']获取。 app.config.from_envvar('WEBSTH_CONFIG', silent=True) # 如果有环境变量,就导入此文件覆盖相应设置。 db = SQLAlchemy(app) cache = FileSystemCache(app.config['CACHE_PATH']) # import views from views import * # 开发环境运行部分,开始运行程序 if __name__ == '__main__': app.run()
#!/usr/bin/env python # -*- coding: utf-8 -*- from linklist import parse_linklist from progressbar import Bar, ProgressBar, SimpleProgress from requests import get, head, ConnectionError, Timeout from sys import stdin, stderr from werkzeug.contrib.cache import FileSystemCache from urllib2 import urlparse url_status_cache = FileSystemCache('.cache_dir', \ threshold=10000, default_timeout=3600) def validate_links(data): widgets = [Bar(), SimpleProgress()] pbar = ProgressBar(widgets=widgets, maxval=len(data)).start() for i, element in enumerate(data): url = element['url'] if url == '': continue scheme = urlparse.urlsplit(url).scheme host = urlparse.urlsplit(url).netloc if scheme in ('http', 'https') and \ url_status_cache.get(url) is not True: try: request = head(url, timeout=10) # some web sites cannot into head requests if request.status_code in (403, 405, 500) or \ host in ('mobil.morgenpost.de'):
class FileSystemSessionInterface(SessionInterface): """Uses the :class:`werkzeug.contrib.cache.FileSystemCache` as a session backend. .. versionadded:: 0.2 The `use_signer` parameter was added. :param cache_dir: the directory where session files are stored. :param threshold: the maximum number of items the session stores before it starts deleting some. :param mode: the file mode wanted for the session files, default 0600 :param key_prefix: A prefix that is added to FileSystemCache store keys. :param use_signer: Whether to sign the session id cookie or not. :param permanent: Whether to use permanent session or not. """ session_class = FileSystemSession def __init__(self, cache_dir, threshold, mode, key_prefix, use_signer=False, permanent=True): from werkzeug.contrib.cache import FileSystemCache self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode) self.key_prefix = key_prefix self.use_signer = use_signer self.permanent = permanent def open_session(self, app, request): sid = request.cookies.get(app.session_cookie_name) if not sid: sid = self._generate_sid() return self.session_class(sid=sid, permanent=self.permanent) if self.use_signer: signer = self._get_signer(app) if signer is None: return None try: sid = signer.unsign(sid) except BadSignature: sid = self._generate_sid() return self.session_class(sid=sid, permanent=self.permanent) data = self.cache.get(self.key_prefix + sid) if data is not None: return self.session_class(data, sid=sid) return self.session_class(sid=sid, permanent=self.permanent) def save_session(self, app, session, response): domain = self.get_cookie_domain(app) path = self.get_cookie_path(app) if not session: if session.modified: self.cache.delete(self.key_prefix + session.sid) response.delete_cookie(app.session_cookie_name, domain=domain, path=path) return httponly = self.get_cookie_httponly(app) secure = self.get_cookie_secure(app) expires = self.get_expiration_time(app, session) data = dict(session) self.cache.set(self.key_prefix + session.sid, data, total_seconds(app.permanent_session_lifetime)) if self.use_signer: session_id = self._get_signer(app).sign(session.sid) else: session_id = session.sid response.set_cookie(app.session_cookie_name, session_id, expires=expires, httponly=httponly, domain=domain, path=path, secure=secure)
__author__ = 'hadware' from os.path import dirname, join, isdir from werkzeug.contrib.cache import FileSystemCache CACHE_TIMEOUT = 300 CACHE_DIRNAME = join(dirname(__file__), "cache") if not isdir(CACHE_DIRNAME): try: makedirs(CACHE_DIRNAME) except OSError: pass cache = FileSystemCache(CACHE_DIRNAME) class cached(object): def __init__(self, cache_key, timeout=None): self.timeout = timeout or CACHE_TIMEOUT self.cache_key = cache_key def __call__(self, f): def decorator(*args, **kwargs): cached_value = cache.get(self.cache_key) if cached_value is None: cached_value = f(*args, **kwargs) cache.set(self.cache_key, cached_value, self.timeout) return cached_value
# -*- coding: utf-8 -*- from flask import Flask from flask.ext.nemo import Nemo from nautilus.flask_ext import FlaskNautilus from werkzeug.contrib.cache import FileSystemCache from flask_cache import Cache app = Flask("Nautilus") nautilus_cache = FileSystemCache("/code/cache") nautilus = FlaskNautilus(app=app, prefix="/api/cts", resources=["/code/data/canonical-latinLit-master"], parser_cache=nautilus_cache, http_cache=Cache(config={'CACHE_TYPE': "simple"})) if __name__ == "__main__": app.run(debug=True, host='0.0.0.0')
class FileSystemSessionInterface(SessionInterface): """Uses the :class:`werkzeug.contrib.cache.FileSystemCache` as a session backend. .. versionadded:: 0.2 The `use_signer` parameter was added. :param cache_dir: the directory where session files are stored. :param threshold: the maximum number of items the session stores before it starts deleting some. :param mode: the file mode wanted for the session files, default 0600 :param key_prefix: A prefix that is added to FileSystemCache store keys. :param use_signer: Whether to sign the session id cookie or not. :param permanent: Whether to use permanent session or not. """ session_class = FileSystemSession def __init__(self, cache_dir, threshold, mode, key_prefix, use_signer=False, permanent=True): from werkzeug.contrib.cache import FileSystemCache self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode) self.key_prefix = key_prefix self.use_signer = use_signer self.permanent = permanent def open_session(self, app, request): sid = request.cookies.get(app.session_cookie_name) if not sid: sid = self._generate_sid() return self.session_class(sid=sid, permanent=self.permanent) if self.use_signer: signer = self._get_signer(app) if signer is None: return None try: sid_as_bytes = signer.unsign(sid) sid = sid_as_bytes.decode() except BadSignature: sid = self._generate_sid() return self.session_class(sid=sid, permanent=self.permanent) data = self.cache.get(self.key_prefix + sid) if data is not None: return self.session_class(data, sid=sid) return self.session_class(sid=sid, permanent=self.permanent) def save_session(self, app, session, response): domain = self.get_cookie_domain(app) path = self.get_cookie_path(app) if not session: if session.modified: self.cache.delete(self.key_prefix + session.sid) response.delete_cookie(app.session_cookie_name, domain=domain, path=path) return httponly = self.get_cookie_httponly(app) secure = self.get_cookie_secure(app) expires = self.get_expiration_time(app, session) data = dict(session) self.cache.set(self.key_prefix + session.sid, data, total_seconds(app.permanent_session_lifetime)) if self.use_signer: session_id = self._get_signer(app).sign(want_bytes(session.sid)) else: session_id = session.sid response.set_cookie(app.session_cookie_name, session_id, expires=expires, httponly=httponly, domain=domain, path=path, secure=secure)
class FileSystemSessionInterface(SessionInterface): """Uses the :class:`werkzeug.contrib.cache.FileSystemCache` as a session backend. .. versionadded:: 0.2 The `use_signer` parameter was added. :param cache_dir: the directory where session files are stored. :param threshold: the maximum number of items the session stores before it starts deleting some. :param mode: the file mode wanted for the session files, default 0600 :param key_prefix: A prefix that is added to FileSystemCache store keys. :param use_signer: Whether to sign the session id cookie or not. """ session_class = FileSystemSession def __init__(self, cache_dir, threshold, mode, key_prefix, use_signer=False): from werkzeug.contrib.cache import FileSystemCache self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode) #FileSystemCache是从上句导入的,threshold代表最多纪录条数 self.key_prefix = key_prefix self.use_signer = use_signer def open_session(self, app, request): sid = request.cookies.get(app.session_cookie_name) if not sid: sid = self._generate_sid() return self.session_class(sid=sid) if self.use_signer: signer = self._get_signer(app) if signer is None: return None try: sid = signer.unsign(sid) except BadSignature: sid = None data = self.cache.get(self.key_prefix + sid) #这是根据session_id获取session值 if data is not None: return self.session_class(data, sid=sid) return self.session_class(sid=sid) def save_session(self, app, session, response): domain = self.get_cookie_domain(app) path = self.get_cookie_path(app) if not session: #验证过了,就是判断session是否为空(空字典) if session.modified: self.cache.delete(self.key_prefix + session.sid) response.delete_cookie(app.session_cookie_name, domain=domain, path=path) return # Modification case. There are upsides and downsides to # emitting a set-cookie header each request. The behavior # is controlled by the :meth:`should_set_cookie` method # which performs a quick check to figure out if the cookie # should be set or not. This is controlled by the # SESSION_REFRESH_EACH_REQUEST config flag as well as # the permanent flag on the session itself. #if not self.should_set_cookie(app, session): # return httponly = self.get_cookie_httponly(app) secure = self.get_cookie_secure(app) expires = self.get_expiration_time(app, session) #data = dict(session) #self.cache.set(self.key_prefix + session.sid, data, # int(app.permanent_session_lifetime.total_seconds())) if self.use_signer: #签名加密 session_id = self._get_signer(app).sign(session.sid) else: session_id = session.sid response.set_cookie(app.session_cookie_name, session_id, expires=expires, httponly=httponly, domain=domain, path=path, secure=secure) def save_session_without_response(self, app, session): #httponly = self.get_cookie_httponly(app) #secure = self.get_cookie_secure(app) #expires = self.get_expiration_time(app, session) data = dict(session) self.cache.set(self.key_prefix + session.sid, data, int(app.permanent_session_lifetime.total_seconds())) def judge_attack(self, app, request): #判断两次间隔的访问是否大于三秒 sid = request.cookies.get(app.session_cookie_name) if not sid: return False current_time = time.time() try: last_time = self.cache.get(self.key_prefix + sid)['time'] #这是根据session_id获取session值 except: return False if current_time-last_time < 3: return True return False
sys.path.append(os.getcwd()) app = Flask(__name__) app.config.update( PERSONA_JS='https://login.persona.org/include.js', PERSONA_VERIFIER='https://verifier.login.persona.org/verify' ) app.config.from_object("simple_settings") app.secret_key = app.config["SECRET_KEY"] if not app.config["DEBUG"]: app.static_folder = os.path.join(os.getcwd(), "static") db = orm.Database('sqlite', os.path.join(os.getcwd(), app.config["DATABASE_FILE"]), create_db=True) cache = FileSystemCache(app.config["CACHE_DIR"]) csrf = SeaSurf(app) basic_auth = BasicAuth(app) app.config['UPLOAD_FOLDER'] = os.path.join(os.getcwd(), "uploads") ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'gif'} app.jinja_env.globals.update(now=datetime.datetime.now) md = markdown.Markdown( extensions=['fenced_code', 'toc'], output_format="html5" ) class Post(db.Entity): _table_ = "posts"
Server: Werkzeug/0.9.4 Python/2.7.5+ X-Cache: miss 1000 """ from __future__ import division, unicode_literals from six.moves.http_client import BAD_REQUEST, OK from time import sleep from werkzeug.contrib.cache import FileSystemCache from flask import Flask, render_template, request from flask.ext.webcache import easy_setup, modifiers app = Flask(__name__) werkzeug_cache = FileSystemCache('/tmp/.sleepycalc') easy_setup(app, werkzeug_cache) PLAINTEXT = (('Content-Type', 'text/plain'), ) @app.route("/") def index(): return render_template("index.html") @app.route("/addition") @modifiers.cache_for(seconds=30) def addition(): try: term1, term2 = int(request.args['term1']), int(request.args['term2'])
def runCouchPotato(options, base_path, args, data_dir=None, log_dir=None, Env=None, desktop=None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' # Do db stuff db_path = os.path.join(data_dir, 'couchpotato.db') # Backup before start and cleanup old databases new_backup = os.path.join(data_dir, 'db_backup', str(int(time.time()))) # Create path and copy if not os.path.isdir(new_backup): os.makedirs(new_backup) src_files = [ options.config_file, db_path, db_path + '-shm', db_path + '-wal' ] for src_file in src_files: if os.path.isfile(src_file): shutil.copy2(src_file, os.path.join(new_backup, os.path.basename(src_file))) # Remove older backups, keep backups 3 days or at least 3 backups = [] for directory in os.listdir(os.path.dirname(new_backup)): backup = os.path.join(os.path.dirname(new_backup), directory) if os.path.isdir(backup): backups.append(backup) total_backups = len(backups) for backup in backups: if total_backups > 3: if int(os.path.basename(backup)) < time.time() - 259200: for src_file in src_files: b_file = os.path.join(backup, os.path.basename(src_file)) if os.path.isfile(b_file): os.remove(b_file) os.rmdir(backup) total_backups -= 1 # Register environment settings Env.set('encoding', encoding) Env.set('app_dir', base_path) Env.set('data_dir', data_dir) Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log')) Env.set('db_path', 'sqlite:///' + db_path) Env.set('cache_dir', os.path.join(data_dir, 'cache')) Env.set('cache', FileSystemCache(os.path.join(Env.get('cache_dir'), 'python'))) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default=False, type='bool') Env.set('debug', debug) # Development development = Env.setting('development', default=False, type='bool') Env.set('dev', development) # Disable logging for some modules for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp', 'migrate']: logging.getLogger(logger_name).setLevel(logging.WARNING) # Use reloader reloader = debug is True and development and not Env.get( 'desktop') and not options.daemon # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) logging.addLevelName(19, 'INFO') # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s', options) def customwarn(message, category, filename, lineno, file=None, line=None): log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn # Check if database exists db = Env.get('db_path') db_exists = os.path.isfile(db_path) # Load configs & plugins loader = Env.get('loader') loader.preload(root=base_path) loader.run() # Load migrations if db_exists: from migrate.versioning.api import version_control, db_version, version, upgrade repo = os.path.join(base_path, 'couchpotato', 'core', 'migration') latest_db_version = version(repo) try: current_db_version = db_version(db, repo) except: version_control(db, repo, version=latest_db_version) current_db_version = db_version(db, repo) if current_db_version < latest_db_version and not debug: log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version)) upgrade(db, repo) # Configure Database from couchpotato.core.settings.model import setup setup() # Fill database with needed stuff if not db_exists: fireEvent('app.initialize', in_order=True) # Create app from couchpotato import app api_key = Env.setting('api_key') url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting( 'url_base') else '' # Basic config app.secret_key = api_key # app.debug = development config = { 'use_reloader': reloader, 'host': Env.setting('host', default='0.0.0.0'), 'port': tryInt(Env.setting('port', default=5000)) } # Static path app.static_folder = os.path.join(base_path, 'couchpotato', 'static') web.add_url_rule('api/%s/static/<path:filename>' % api_key, endpoint='static', view_func=app.send_static_file) # Register modules app.register_blueprint(web, url_prefix='%s/' % url_base) app.register_blueprint(api, url_prefix='%s/api/%s/' % (url_base, api_key)) # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') # Go go go! web_container = WSGIContainer(app) web_container._log = _log loop = IOLoop.instance() application = Application([ (r'%s/api/%s/nonblock/(.*)/' % (url_base, api_key), NonBlockHandler), (r'.*', FallbackHandler, dict(fallback=web_container)), ], log_function=lambda x: None, debug=config['use_reloader']) try_restart = True restart_tries = 5 while try_restart: try: application.listen(config['port'], config['host'], no_keep_alive=True) loop.start() except Exception, e: try: nr, msg = e if nr == 48: log.info( 'Already in use, try %s more time after few seconds', restart_tries) time.sleep(1) restart_tries -= 1 if restart_tries > 0: continue else: return except: pass raise try_restart = False
from bs4 import BeautifulSoup from threading import Thread from werkzeug.contrib.cache import FileSystemCache import requests cache = FileSystemCache('static/cache/', threshold=100, default_timeout=24 * 60 * 60) def add_additional_film_info(film): title = film.get('film_title') film_page = fetch_kinopoisk_page(title) additional_information = parse_kinopoisk_page(film_page.text) film.update(additional_information) def sort_film_list(item_list): return sorted(item_list, key=lambda item: item['film_rating'], reverse=True) def list_of_films(): number_of_top_films = 10 afisha_page_url = 'https://www.afisha.ru/msk/schedule_cinema/' afisha_page_raw = fetch_page(afisha_page_url) film_list = parse_afisha_page(afisha_page_raw.text) threads_list = [] for film in film_list: thread = Thread(target=add_additional_film_info,
def __init__(self, cache_dir, threshold, mode, key_prefix, use_signer=False): from werkzeug.contrib.cache import FileSystemCache self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode) #FileSystemCache是从上句导入的,threshold代表最多纪录条数 self.key_prefix = key_prefix self.use_signer = use_signer
class TestSaxonJar(TestCase): def setUp(self): self.cache = FileSystemCache("./cache") self.saxon = SaxonShellTransform( "./jars/saxon.jar", "./tests/data/xsl/ciham.xsl", cache=self.cache ) self.nautilus = NautilusRetriever( folders=[ "./tests/data/repo" ] ) self.nautilus.logger.setLevel(logging.ERROR) app = Flask("Nemo") app.debug = True nemo = Nemo( app=app, base_url="", retriever=self.nautilus, transform={ "default": self.saxon.transform } ) self.client = app.test_client() def tearDown(self): # We clean the cache folder to ensure that no cache is passed from one test to the other self.cache.clear() def test_simple_transformation(self): """ Test transformation works fine""" read = self.client.get("/read/froLit/jns915/jns1856/ciham-fro1/1") data = read.data.decode() self.assertIn( '<span class="expan">et </span>', data, "Text content should be transformed" ) self.assertIn( 'Facsimilaire', data, "Other content should be added" ) cached = self.cache.get("urn:cts:froLit:jns915.jns1856.ciham-fro1:1").decode() self.assertIn('<aside class="text-left">', cached, "Assert cache is made") def test_cache_retrieved(self): """ Test that cache is nicely used and built """ read = self.client.get("/read/froLit/jns915/jns1856/ciham-fro1/1") data = read.data.decode() self.assertIn( '<span class="expan">et </span>', data, "Text content should be transformed" ) self.assertIn( 'Facsimilaire', data, "Other content should be added" ) cached = self.cache.get("urn:cts:froLit:jns915.jns1856.ciham-fro1:1").decode() self.assertIn('<aside class="text-left">', cached, "Assert cache is made") with mock.patch("nemo_xslttwo_plugin.shell") as shell: read = self.client.get("/read/froLit/jns915/jns1856/ciham-fro1/1") cached_response = read.data.decode() self.assertEqual( cached_response, data, "Text content should the same in cache" ) self.assertEqual( shell.call_count, 0, "Shell should not be called because we use cache" )
import urllib2 import datetime from bs4 import BeautifulSoup from urlparse import urljoin from flask import Flask, Markup, request from werkzeug.contrib.atom import AtomFeed from werkzeug.contrib.cache import FileSystemCache app = Flask(__name__) cache = FileSystemCache(cache_dir='/tmp/') CACHE_TIMEOUT = 60 def get_url(url, cache_timeout): """ Returns the contents of the URL as a string. """ html = cache.get("cache") if html is None: response = urllib2.urlopen(url) html = response.read() cache.set("cache", html, timeout=cache_timeout) return html def get_articles(url): """yields a tuple of (title, url, published)""" html = get_url(url, cache_timeout=CACHE_TIMEOUT) soup = BeautifulSoup(html) for event in soup.select('.content-view-listing__single-item'):
from unicodedata import normalize # web stuff and markdown imports import markdown from flask.ext.sqlalchemy import SQLAlchemy from werkzeug.security import check_password_hash from flask import render_template, request, Flask, flash, redirect, url_for, \ abort, jsonify, Response, make_response from werkzeug.contrib.cache import FileSystemCache, NullCache app = Flask(__name__) app.config.from_object('settings') db = SQLAlchemy(app) cache_directory = os.path.dirname(__file__) try: cache = FileSystemCache(os.path.join(cache_directory, "cache")) except Exception,e: print "Could not create cache folder, caching will be disabled." print "Error: %s"%e cache = NullCache() _punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+') MARKDOWN_PARSER = markdown.Markdown(extensions=['fenced_code'], output_format="html5", safe_mode=True) class Post(db.Model): def __init__(self, title=None, created_at=None):
import requests from werkzeug.contrib.cache import FileSystemCache from pythoncz import app __all__ = ('get_issues', ) get_issues_graphql_filename = (os.path.join(os.path.dirname(__file__), 'github_get_issues.graphql')) with open(get_issues_graphql_filename) as f: GET_ISSUES_GRAPHQL = f.read() SIX_HOURS_AS_SECONDS = 21600 cache = FileSystemCache(app.config['CACHE_DIR'], default_timeout=SIX_HOURS_AS_SECONDS) def get_issues(org_names): issues = cache.get('github-issues') if issues is None: session = _create_api_session() issues = itertools.chain(*(_get_issues_for_org(session, org_name) for org_name in org_names)) issues = _sort_issues(issues) cache.set('github-issues', issues) return issues def _create_api_session(): user_agent = ('pythoncz/{now.year}-{now.month} '
import sys from contextlib import closing import lxml.html as html # pip install 'lxml>=2.3.1' from lxml.html.clean import Cleaner from selenium.webdriver import Firefox # pip install selenium from werkzeug.contrib.cache import FileSystemCache # pip install werkzeug cache = FileSystemCache('.cachedir', threshold=100000) url = sys.argv[1] if len(sys.argv) > 1 else "http://www.schibsted.cl/testqa/" # get page page_source = cache.get(url) if page_source is None: # use firefox to get page with javascript generated content with closing(Firefox()) as browser: browser.get(url) page_source = browser.page_source cache.set(url, page_source, timeout=60*60*24*7) # week in seconds # extract text root = html.document_fromstring(page_source) # remove flash, images, <script>,<style>, etc Cleaner(kill_tags=['noscript'], style=True)(root) # lxml >= 2.3.1 webtext = root.text_content() # extract text f = open("C:/schibsted/data/Test.txt", "w"); print f value = (webtext)
class Cache(object): """Cache module based on werkzeug.contrib.cache. This is a mixed version of NullCache, SimpleCache, FileSystemCache, MemcachedCache, and RedisCache. :param app: Flask app instance. :param config_prefix: Define a prefix for Flask app config. :param kwargs: Extra parameters. You need to configure a type of the cache, and its related configurations. The default ``config_prefix`` is ``AUTHLIB``, so it requires a config of:: AUTHLIB_CACHE_TYPE = 'simple' If ``config_prefix`` is something else, like ``EXAMPLE``, it would be:: EXAMPLE_CACHE_TYPE = 'simple' The available cache types are: * null: It will not cache anything. No configuration. * simple: It caches things in memory. The only configuration is ``threshold``:: AUTHLIB_CACHE_THRESHOLD = 500 * memcache: It caches things in Memcache. Available configurations:: AUTHLIB_CACHE_MEMCACHED_SERVERS = [] AUTHLIB_CACHE_KEY_PREFIX = None * redis: It caches things in Redis. Available configurations:: AUTHLIB_CACHE_REDIS_HOST = 'localhost' AUTHLIB_CACHE_REDIS_PORT = 6379 AUTHLIB_CACHE_REDIS_PASSWORD = None AUTHLIB_CACHE_REDIS_DB = 0 AUTHLIB_CACHE_KEY_PREFIX = None * filesystem: It caches things in local filesystem. Available configurations:: AUTHLIB_CACHE_DIR = '' # required AUTHLIB_CACHE_THRESHOLD = 500 """ def __init__(self, app, config_prefix='AUTHLIB', **kwargs): self.config_prefix = config_prefix self.config = app.config cache_type = self._config('type') kwargs.update( dict(default_timeout=self._config('DEFAULT_TIMEOUT', 100))) if cache_type == 'null': self.cache = NullCache() elif cache_type == 'simple': kwargs.update(dict(threshold=self._config('threshold', 500))) self.cache = SimpleCache(**kwargs) elif cache_type == 'memcache': kwargs.update( dict( servers=self._config('MEMCACHED_SERVERS'), key_prefix=self._config('KEY_PREFIX', None), )) self.cache = MemcachedCache(**kwargs) elif cache_type == 'redis': kwargs.update( dict( host=self._config('REDIS_HOST', 'localhost'), port=self._config('REDIS_PORT', 6379), password=self._config('REDIS_PASSWORD', None), db=self._config('REDIS_DB', 0), key_prefix=self._config('KEY_PREFIX', None), )) self.cache = RedisCache(**kwargs) elif cache_type == 'filesystem': kwargs.update(dict(threshold=self._config('threshold', 500), )) self.cache = FileSystemCache(self._config('dir'), **kwargs) else: raise RuntimeError('`%s` is not a valid cache type!' % cache_type) app.extensions[config_prefix.lower() + '_cache'] = self.cache def _config(self, key, default=_missing): key = key.upper() prior = '%s_CACHE_%s' % (self.config_prefix, key) if prior in self.config: return self.config[prior] fallback = 'CACHE_%s' % key if fallback in self.config: return self.config[fallback] if default is _missing: raise RuntimeError('%s is missing.' % prior) return default def get(self, key): """Look up key in the cache and return the value for it. :param key: the key to be looked up. :returns: The value if it exists and is readable, else ``None``. """ return self.cache.get(key) def delete(self, key): """Delete `key` from the cache. :param key: the key to delete. :returns: Whether the key existed and has been deleted. """ return self.cache.delete(key) def get_many(self, *keys): """Returns a list of values for the given keys. For each key a item in the list is created:: foo, bar = cache.get_many("foo", "bar") Has the same error handling as :meth:`get`. :param keys: The function accepts multiple keys as positional arguments. """ return [self.cache.get(k) for k in keys] def get_dict(self, *keys): """Like :meth:`get_many` but return a dict:: d = cache.get_dict("foo", "bar") foo = d["foo"] bar = d["bar"] :param keys: The function accepts multiple keys as positional arguments. """ return self.cache.get_dict(*keys) def set(self, key, value, timeout=None): """Add a new key/value to the cache (overwrites value, if key already exists in the cache). :param key: the key to set :param value: the value for the key :param timeout: the cache timeout for the key in seconds (if not specified, it uses the default timeout). A timeout of 0 idicates that the cache never expires. :returns: ``True`` if key has been updated, ``False`` for backend errors. Pickling errors, however, will raise a subclass of ``pickle.PickleError``. """ return self.cache.set(key, value, timeout) def add(self, key, value, timeout=None): """Works like :meth:`set` but does not overwrite the values of already existing keys. :param key: the key to set :param value: the value for the key :param timeout: the cache timeout for the key in seconds (if not specified, it uses the default timeout). A timeout of 0 idicates that the cache never expires. :returns: Same as :meth:`set`, but also ``False`` for already existing keys. """ return self.cache.add(key, value, timeout) def set_many(self, mapping, timeout=None): """Sets multiple keys and values from a mapping. :param mapping: a mapping with the keys/values to set. :param timeout: the cache timeout for the key in seconds (if not specified, it uses the default timeout). A timeout of 0 idicates that the cache never expires. :returns: Whether all given keys have been set. """ return self.cache.set_many(mapping, timeout) def delete_many(self, *keys): """Deletes multiple keys at once. :param keys: The function accepts multiple keys as positional arguments. :returns: Whether all given keys have been deleted. :rtype: boolean """ return self.cache.delete_many(*keys) def has(self, key): """Checks if a key exists in the cache without returning it. This is a cheap operation that bypasses loading the actual data on the backend. This method is optional and may not be implemented on all caches. :param key: the key to check """ return self.cache.has(key) def clear(self): """Clears the cache. Keep in mind that not all caches support completely clearing the cache. :returns: Whether the cache has been cleared. """ return self.cache.clear() def inc(self, key, delta=1): """Increments the value of a key by `delta`. If the key does not yet exist it is initialized with `delta`. For supporting caches this is an atomic operation. :param key: the key to increment. :param delta: the delta to add. :returns: The new value or ``None`` for backend errors. """ return self.cache.inc(key, delta=delta) def dec(self, key, delta=1): """Decrements the value of a key by `delta`. If the key does not yet exist it is initialized with `-delta`. For supporting caches this is an atomic operation. :param key: the key to increment. :param delta: the delta to subtract. :returns: The new value or `None` for backend errors. """ return self.cache.dec(key, delta=delta)
# -*- coding: utf-8 -*- import itertools from datetime import datetime import requests import grequests from werkzeug.contrib.cache import FileSystemCache from .. import app __all__ = ('get_issues', ) cache = FileSystemCache(app.config['CACHE_DIR'], default_timeout=3600) def get_issues(org_names): issues = cache.get('github-issues') if issues is None: session = _create_github_api_session() issues = itertools.chain(*(_get_issues_for_org(session, org) for org in org_names)) # Using grequests here to get the issues details asynchronously # Note that it does not support session, so we set the headers manually reqs = (grequests.get(_issue_url(issue), headers=session.headers) for issue in issues) issues = [_enhance_issue(response) for response in grequests.map(reqs)] issues = sorted(issues, key=_get_issue_sort_key, reverse=True) cache.set('github-issues', issues) return issues
# You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. from sys import argv, stderr, stdout from werkzeug.contrib.cache import FileSystemCache from progressbar import ProgressBar from requests import get from cStringIO import StringIO from xml.etree.cElementTree import dump, ElementTree from iso8601 import parse_date from geopy import Point, distance from math import sqrt from cluster import HierarchicalClustering feed_cache = FileSystemCache('feed_cache', 3600) name_cache = FileSystemCache('name_cache', 360000) class Event(): def __init__(self, name, uri, datetime, latitude, longitude): self.name = name self.uri = uri self.datetime = datetime self.latitude = latitude self.longitude = longitude def __repr__(self): return "%s <%s>, %s @ %s, %s" % (self.name, self.uri, self.datetime, self.latitude, self.longitude) def spatialDistance(a, b): """