class FileSystemSessionInterface(SessionInterface):
"""Uses the :class:`werkzeug.contrib.cache.FileSystemCache` as a session
backend.
:param cache_dir: the directory where session files are stored.
:param threshold: the maximum number of items the session stores before it
starts deleting some.
:param mode: the file mode wanted for the session files, default 0600
:param key_prefix: A prefix that is added to FileSystemCache store keys.
"""
session_class = FileSystemSession
def __init__(self, cache_dir, threshold, mode, key_prefix):
from werkzeug.contrib.cache import FileSystemCache
self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode)
self.key_prefix = key_prefix
def _generate_sid(self):
return str(uuid4())
def open_session(self, app, request):
sid = request.cookies.get(app.session_cookie_name)
if not sid:
sid = self._generate_sid()
return self.session_class(sid=sid)
data = self.cache.get(self.key_prefix + sid)
if data is not None:
return self.session_class(data, sid=sid)
return self.session_class(sid=sid)
def save_session(self, app, session, response):
domain = self.get_cookie_domain(app)
path = self.get_cookie_path(app)
if not session:
if session.modified:
self.cache.delete(self.key_prefix + session.sid)
response.delete_cookie(app.session_cookie_name,
domain=domain, path=path)
return
# Modification case. There are upsides and downsides to
# emitting a set-cookie header each request. The behavior
# is controlled by the :meth:`should_set_cookie` method
# which performs a quick check to figure out if the cookie
# should be set or not. This is controlled by the
# SESSION_REFRESH_EACH_REQUEST config flag as well as
# the permanent flag on the session itself.
# if not self.should_set_cookie(app, session):
# return
httponly = self.get_cookie_httponly(app)
secure = self.get_cookie_secure(app)
expires = self.get_expiration_time(app, session)
data = dict(session)
self.cache.set(self.key_prefix + session.sid, data,
int(app.permanent_session_lifetime.total_seconds()))
response.set_cookie(app.session_cookie_name, session.sid,
expires=expires, httponly=httponly,
domain=domain, path=path, secure=secure)
def number_gen(first_part, start_num, end_num, semester, dept, subjects):
driver = webdriver.Firefox() # Firefox used for testing. Change it to PhantomJS
driver.implicitly_wait(30)
base_url = "http://result.pondiuni.edu.in/candidate.asp"
url = base_url
driver.get(base_url)
# os.mkdir(str(first_part))
os.chdir("results")
os.chdir(str(first_part))
cache = FileSystemCache('.cachedir', threshold=100000)
for number in range(start_num, end_num + 1):
current_num = "%04d" % number
numb = first_part + str(current_num)
driver.find_element_by_id("txtregno").clear()
driver.find_element_by_id("txtregno").send_keys(numb)
Select(driver.find_element_by_id("cmbdegree")).select_by_visible_text(dept)
Select(driver.find_element_by_id("cmbexamno")).select_by_visible_text(semester)
driver.find_element_by_id("button1").click()
# copying the content
page_source = cache.get(url)
page_source = driver.page_source
cache.set(url, page_source, timeout=60 * 60 * 24 * 7) # week in seconds
root = html.document_fromstring(page_source)
Cleaner(kill_tags=['noscript'], style=True)(root) # lxml >= 2.3.1
# pasting to file
filename = str(numb) + ".txt"
fp = open(filename, 'w')
fp.write((root.text_content()).encode('utf-8'))
fp.close()
driver.back()
driver.close()
return analyze(subjects)
def check_cache( self, bib_id ):
""" Checks cache for marc. """
cache = FileSystemCache( self.cache_dir, threshold=500, default_timeout=self.cache_hours, mode=0664 ) # http://werkzeug.pocoo.org/docs/0.9/contrib/cache/
cache_key = bib_id
marc = cache.get( cache_key )
if marc == None:
self.log.debug( u'in app_helper.Helper.check_cache(); marc not found in cache' )
else:
self.log.debug( u'in app_helper.Helper.check_cache(); marc found in cache' )
return ( marc, cache )
def setUp(self):
"""BaseModel test set up"""
if os.path.isfile("/tmp/box.db"):
os.unlink("/tmp/box.db")
DBHelper().set_db("/tmp/box.db")
InstallHelper.reset()
cache = FileSystemCache("/tmp/werkzeug")
cache.clear()
BaseModel.set_cache(cache)
SampleModel.install()
def test_filesystemcache_clear():
"""
test if FileSystemCache.clear works
"""
tmp_dir = tempfile.mkdtemp()
cache = FileSystemCache(cache_dir=tmp_dir)
cache.set("foo", "bar")
cache_files = os.listdir(tmp_dir)
assert len(cache_files) == 1
cache.clear()
cache_files = os.listdir(tmp_dir)
assert len(cache_files) == 0
shutil.rmtree(tmp_dir)
def test_filesystemcache_prune():
"""
test if FileSystemCache._prune works and keeps the cache entry count
below the given threshold.
"""
THRESHOLD = 13
tmp_dir = tempfile.mkdtemp()
cache = FileSystemCache(cache_dir=tmp_dir, threshold=THRESHOLD)
for i in range(2 * THRESHOLD):
cache.set(str(i), i)
cache_files = os.listdir(tmp_dir)
shutil.rmtree(tmp_dir)
assert len(cache_files) <= THRESHOLD
def test_filesystemcache_set_get():
"""
test if FileSystemCache.set/get works
"""
tmp_dir = tempfile.mkdtemp()
try:
cache = FileSystemCache(cache_dir=tmp_dir)
for i in range(3):
cache.set(str(i), i * i)
for i in range(3):
result = cache.get(str(i))
assert result == i * i
finally:
shutil.rmtree(tmp_dir)
class BaseProvider(object):
__metaclass__ = ABCMeta
def __init__(self, cache_dir=None, default_timeout=60 * 60 * 24,
api_key=None):
# store it in cache for 1 day. using file system cache because
# memcached is too mainstream. :)
self.cache = FileSystemCache(cache_dir=cache_dir or '/tmp/__arcoiro__',
default_timeout=default_timeout)
self._api_key = api_key
@abstractproperty
def name(self):
pass
@abstractproperty
def url(self):
pass
@abstractmethod
def get_urls_from_tag(self, tag):
pass
@property
def display_name(self):
return self.name
@property
def api_key(self):
if self._api_key is not None:
return self._api_key
config_key = '%s_API_KEY' % self.name.upper()
key = current_app.config.get(config_key)
if key is None:
raise RuntimeError('%s not defined!' % config_key)
return key
def get_cached_urls_from_tag(self, tag):
cache_key = '%s:%s' % (self.name, tag)
urls = self.cache.get(cache_key)
if urls is not None:
return urls
urls = self.get_urls_from_tag(tag)
if urls is None:
return None
self.cache.set(cache_key, urls)
return urls
def __init__(self, cache_dir=None, default_timeout=60 * 60 * 24,
api_key=None):
# store it in cache for 1 day. using file system cache because
# memcached is too mainstream. :)
self.cache = FileSystemCache(cache_dir=cache_dir or '/tmp/__arcoiro__',
default_timeout=default_timeout)
self._api_key = api_key
def __init__(self, cache_dir, threshold, mode, key_prefix,
use_signer=False, permanent=True):
from werkzeug.contrib.cache import FileSystemCache
self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode)
self.key_prefix = key_prefix
self.use_signer = use_signer
self.permanent = permanent
class WechatCache(object):
"""基于文件的缓存
"""
def __init__(self, cache_dir='cache', default_timeout=300):
"""初始化
cache_dir是缓存目录
"""
self.cache = FileSystemCache(cache_dir, default_timeout=default_timeout)
def clear(self):
"""清空缓存
"""
return self.cache.clear()
def get(self, key):
"""获取缓存
获取键值key的缓存值
如果没有对应缓存，返回None
"""
return self.cache.get(key)
def add(self, key, value, timeout=None):
"""增加缓存
如果键值key对应的缓存不存在，那么增加值value到键值key，过期时间timeout，默认300秒
否则返回False（即不能覆盖设置缓存）
"""
return self.cache.add(key, value, timeout)
def set(self, key, value, timeout=None):
"""设置缓存
设置键值key的缓存为value,过期时间300秒
"""
return self.cache.set(key, value, timeout)
def delete(self, key):
"""删除缓存
删除键值key存储的缓存
"""
return self.cache.delete(key)
def parse(url):
cache = FileSystemCache('.cachedir', threshold=100000)
# get page
page_source = cache.get(url)
if page_source is None:
# use firefox to get page with javascript generated content
with closing(Firefox()) as browser:
browser.get(url)
page_source = browser.page_source
cache.set(url, page_source, timeout=60*60*24*7) # week in seconds
# extract text
root = html.document_fromstring(page_source)
# remove flash, images, <script>,<style>, etc
Cleaner(kill_tags=['noscript'], style=True)(root) # lxml >= 2.3.1
return root.text_content() # extract text
def getSpaceInfo(spaceId):
global spaceCache
if spaceCache is None:
mkdir_p(DURACLOUD_SPACE_CACHE_DIR)
spaceCache = FileSystemCache(DURACLOUD_SPACE_CACHE_DIR, threshold=50, default_timeout=(24*3600), mode=384)
# check spaceCache, otherwise fetch info from DuraCloud
result = spaceCache.get(spaceId)
if result is None:
url = DURACLOUD_URL+ "/duradmin/download/contentItem"
auth = HTTPBasicAuth(DURACLOUD_USERNAME, DURACLOUD_PASSWORD)
payload = {'spaceId': spaceId, 'contentId': 'info.json'}
try:
response = requests.get(url, params=payload, auth=auth)
result = response.json()
spaceCache.set(spaceId, result)
except RequestException as e:
print e
raise
return result
def __init__(self, app, cache_dir='/tmp/cache', lock_file=None):
if not os.path.isdir(cache_dir):
os.mkdir(cache_dir)
self.app = app
self.cache = FileSystemCache(cache_dir, default_timeout=600)
self.t_local = threading.local()
if lock_file is not None:
lock_file = os.path.abspath(os.path.realpath(lock_file))
else:
lock_file = '/tmp/cache.lock'
self.lock_file = open(lock_file, 'wb+')
self.t_lock = threading.Lock()
def __init__(self, path, tolerance, expiration, max_values, run_tests=True,
max_file_size=0):
"""Constructor method.
:param path: The path of the cache database file.
:param tolerance: The tolerance, in seconds to which a TimeMap is
considered young enough to be used as is.
:param expiration: How long, in seconds, the cache entries are stored
every get will be a CACHE MISS.
:param max_values: The maximum number of TimeMaps stored in cache
before some are deleted
:param run_tests: (Optional) Tests the cache at initialization.
:param max_file_size: (Optional) The maximum size (in Bytes) for a
TimeMap cache value. When max_file_size=0, there is no limit to
a cache value. When max_file_size=X > 0, the cache will not
store TimeMap that require more than X Bytes on disk.
"""
# Parameters Check
if tolerance <= 0 or expiration <= 0 or max_values <= 0:
raise CacheError("Cannot create cache: all parameters must be > 0")
self.tolerance = relativedelta(seconds=tolerance)
self.path = path.rstrip('/')
self.max_file_size = max(max_file_size, 0)
self.CHECK_SIZE = self.max_file_size > 0
self.max_values = max_values
self.backend = FileSystemCache(path,
threshold=self.max_values,
default_timeout=expiration)
# Testing cache
if run_tests:
try:
key = '1'
val = 1
self.backend.set(key, val)
assert (not self.CHECK_SIZE) or self._check_size(key) > 0
assert self.backend.get(key) == val
os.remove(self.path + '/' + md5(key).hexdigest())
except Exception as e:
raise CacheError("Error testing cache: %s" % e)
logging.debug(
"Cache created. max_files = %d. Expiration = %d. "
"max_file_size = %d" % (
self.max_values, expiration, self.max_file_size))
def _set_cache(self):
if self.app.config['TESTING']:
self.cache = NullCache()
else:
if self.app.config['CACHE_TYPE'] == 'Null':
self.cache = NullCache()
elif self.app.config['CACHE_TYPE'] == 'Simple':
self.cache = SimpleCache(
threshold=self.app.config['CACHE_THRESHOLD'],
default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'])
elif self.app.config['CACHE_TYPE'] == 'Memcached':
self.cache = MemcachedCache(
self.app.config['CACHE_MEMCACHED_SERVERS'],
default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'],
key_prefix=self.app.config['CACHE_KEY_PREFIX'])
elif self.app.config['CACHE_TYPE'] == 'GAE':
self.cache = GAEMemcachedCache(
default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'],
key_prefix=self.app.config['CACHE_KEY_PREFIX'])
elif self.app.config['CACHE_TYPE'] == 'FileSystem':
self.cache = FileSystemCache(
self.app.config['CACHE_DIR'],
threshold=self.app.config['CACHE_THRESHOLD'],
default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'])
def filesystem(app, config, args, kwargs):
args.insert(0, config['CACHE_DIR'])
kwargs.update(dict(threshold=config['CACHE_THRESHOLD']))
return FileSystemCache(*args, **kwargs)
class CacheMiddleware:
def __init__(self, app, cache_dir='/tmp/cache', lock_file=None):
if not os.path.isdir(cache_dir):
os.mkdir(cache_dir)
self.app = app
self.cache = FileSystemCache(cache_dir, default_timeout=600)
self.t_local = threading.local()
if lock_file is not None:
lock_file = os.path.abspath(os.path.realpath(lock_file))
else:
lock_file = '/tmp/cache.lock'
self.lock_file = open(lock_file, 'wb+')
self.t_lock = threading.Lock()
def __call__(self, environ, start_response):
self.t_local.start_server_response = start_response
# get keys content, header and status
self.t_local.content_key = environ['PATH_INFO']
if environ.get('QUERY_STRING'):
self.t_local.content_key = '%s?%s' % (self.t_local.content_key,
environ['QUERY_STRING'])
self.t_local.header_key = '%s|%s' % (self.t_local.content_key, 'header')
self.t_local.status_key = '%s|%s' % (self.t_local.content_key, 'status')
# get cached response
keys = (self.t_local.content_key, self.t_local.header_key,
self.t_local.status_key)
with self.t_lock:
try:
fcntl.lockf(self.lock_file, fcntl.LOCK_SH)
content, headers, status = self.cache.get_many(*keys)
finally:
fcntl.lockf(self.lock_file, fcntl.LOCK_UN)
if (content is not None
and headers is not None
and status is not None):
# return cached response
self.t_local.start_server_response(status, headers)
return content
else:
# call app and cache the response
content = self.app(environ, self.start_response)
if self.do_cache():
content = list(content)
cached = {self.t_local.content_key: content,
self.t_local.header_key: self.t_local.headers,
self.t_local.status_key: self.t_local.status}
with self.t_lock:
try:
fcntl.lockf(self.lock_file, fcntl.LOCK_EX)
self.cache.set_many(cached)
finally:
fcntl.lockf(self.lock_file, fcntl.LOCK_UN)
return content
def start_response(self, status, response_headers, exc_info=None):
self.t_local.status = status
self.t_local.headers = response_headers
self.t_local.start_server_response(status, response_headers, exc_info)
def do_cache(self):
if 200 <= int(self.t_local.status.split(' ')[0]) < 300:
return True
else:
return False
from flask import Flask, render_template
from cinemas import get_complete_info
from werkzeug.contrib.cache import FileSystemCache
import tempfile
app = Flask(__name__)
cache = FileSystemCache(cache_dir=tempfile.gettempdir())
@app.route('/')
def films_list():
movie_list = cache.get('movie_list')
if movie_list is None:
movie_list = get_complete_info()
cache.set('movie_list', movie_list, timeout=30 * 30)
return render_template('films_list.html', movie_list=movie_list)
if __name__ == "__main__":
app.run()
from flask import Flask, flash, render_template, \
request, redirect, send_from_directory, make_response
import os, generate_identifiers, hgvs_conversion, json_conversion, sqlite3
from werkzeug.contrib.cache import FileSystemCache
from hashlib import sha256
# create the application object
APP = Flask(__name__, static_folder='static')
APP.secret_key = "vmcsuite"
APP.config['UPLOAD_FOLDER'] = 'static/uploads'
cache = FileSystemCache('/tmp/vmc-suite')
def get_json_schema():
"""Returns the example JSON schema file as a string"""
with open("static/schema.json") as f_in:
return f_in.read()
def get_filename():
"""Returns the name of the uploaded VCF file"""
return request.cookies.get("filename")
def get_fileKey():
"""Returns the key for the uploaded VCF file"""
return request.cookies.get("fileKey")
def get_chrs_intervals_and_states(vcf_upload):
"""Takes the VCF file and returns lists of chromosomes, intervals, and states for each variant"""
def __init__(self):
self._cache = FileSystemCache(Cache.cache_dir(),
default_timeout=10 * 60)
def create_app(config_name):
app = Flask(__name__, static_url_path='')
app.config.from_object(config[config_name])
app.config.from_envvar("OPENTARGETS_API_LOCAL_SETTINGS", silent=True)
config[config_name].init_app(app)
api_version = app.config['API_VERSION']
api_version_minor = app.config['API_VERSION_MINOR']
# log_level = logging.INFO
# if app.config['DEBUG']:
# log_level = logging.DEBUG
# Flask has a default logger which works well and pushes to stderr
# if you want to add different handlers (to file, or logstash, or whatever)
# you can use code similar to the one below and set the error level accordingly.
# logHandler = logging.StreamHandler()
# formatter = jsonlogger.JsonFormatter()
# logHandler.setFormatter(formatter)
# loghandler.setLevel(logging.INFO)
# app.logger.addHandler(logHandler)
# or for LOGSTASH
# app.logger.addHandler(logstash.LogstashHandler(app.config['LOGSTASH_HOST'], app.config['LOGSTASH_PORT'], version=1))
app.logger.info('looking for elasticsearch at: %s' %
app.config['ELASTICSEARCH_URL'])
print('looking for elasticsearch at: %s' % app.config['ELASTICSEARCH_URL'])
app.extensions['redis-core'] = Redis(app.config['REDIS_SERVER_PATH'],
db=0) #served data
app.extensions['redis-service'] = Redis(
app.config['REDIS_SERVER_PATH'],
db=1) #cache, rate limit and internal things
app.extensions['redis-user'] = Redis(app.config['REDIS_SERVER_PATH'],
db=2) # user info
'''setup cache'''
app.extensions['redis-service'].config_set('save', '')
app.extensions['redis-service'].config_set('appendonly', 'no')
icache = InternalCache(app.extensions['redis-service'],
str(api_version_minor))
ip2org = IP2Org(icache)
es = Elasticsearch(
app.config['ELASTICSEARCH_URL'],
# # sniff before doing anything
# sniff_on_start=True,
# # refresh nodes after a node fails to respond
# sniff_on_connection_fail=True,
# # and also every 60 seconds
# sniffer_timeout=60
timeout=60 * 20,
maxsize=100,
)
'''elasticsearch handlers'''
app.extensions['esquery'] = esQuery(
es,
DataTypes(app),
DataSourceScoring(app),
index_data=app.config['ELASTICSEARCH_DATA_INDEX_NAME'],
index_efo=app.config['ELASTICSEARCH_EFO_LABEL_INDEX_NAME'],
index_eco=app.config['ELASTICSEARCH_ECO_INDEX_NAME'],
index_genename=app.config['ELASTICSEARCH_GENE_NAME_INDEX_NAME'],
index_expression=app.config['ELASTICSEARCH_EXPRESSION_INDEX_NAME'],
index_reactome=app.config['ELASTICSEARCH_REACTOME_INDEX_NAME'],
index_association=app.
config['ELASTICSEARCH_DATA_ASSOCIATION_INDEX_NAME'],
index_search=app.config['ELASTICSEARCH_DATA_SEARCH_INDEX_NAME'],
index_relation=app.config['ELASTICSEARCH_DATA_RELATION_INDEX_NAME'],
docname_data=app.config['ELASTICSEARCH_DATA_DOC_NAME'],
docname_efo=app.config['ELASTICSEARCH_EFO_LABEL_DOC_NAME'],
docname_eco=app.config['ELASTICSEARCH_ECO_DOC_NAME'],
docname_genename=app.config['ELASTICSEARCH_GENE_NAME_DOC_NAME'],
docname_expression=app.config['ELASTICSEARCH_EXPRESSION_DOC_NAME'],
docname_reactome=app.
config['ELASTICSEARCH_REACTOME_REACTION_DOC_NAME'],
docname_association=app.
config['ELASTICSEARCH_DATA_ASSOCIATION_DOC_NAME'],
docname_search=app.config['ELASTICSEARCH_DATA_SEARCH_DOC_NAME'],
# docname_search_target=app.config['ELASTICSEARCH_DATA_SEARCH_TARGET_DOC_NAME'],
# docname_search_disease=app.config['ELASTICSEARCH_DATA_SEARCH_DISEASE_DOC_NAME'],
docname_relation=app.config['ELASTICSEARCH_DATA_RELATION_DOC_NAME'],
log_level=app.logger.getEffectiveLevel(),
cache=icache)
app.extensions['es_access_store'] = esStore(
es,
eventlog_index=app.config['ELASTICSEARCH_LOG_EVENT_INDEX_NAME'],
ip2org=ip2org,
)
'''mixpanel handlers'''
if Config.MIXPANEL_TOKEN:
mp = Mixpanel(Config.MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer())
app.extensions['mixpanel'] = mp
app.extensions['mp_access_store'] = MixPanelStore(
mp,
ip2org=ip2org,
)
app.extensions['proxy'] = ProxyHandler(
allowed_targets=app.config['PROXY_SETTINGS']['allowed_targets'],
allowed_domains=app.config['PROXY_SETTINGS']['allowed_domains'],
allowed_request_domains=app.config['PROXY_SETTINGS']
['allowed_request_domains'])
basepath = app.config['PUBLIC_API_BASE_PATH'] + api_version
# cors = CORS(app, resources=r'/api/*', allow_headers='Content-Type,Auth-Token')
''' define cache'''
# cache = Cache(config={'CACHE_TYPE': 'simple'})
# cache.init_app(latest_blueprint)
# latest_blueprint.cache = cache
# latest_blueprint.extensions['cache'] = cache
# app.cache = SimpleCache()
app.cache = FileSystemCache('/tmp/cttv-rest-api-cache',
threshold=100000,
default_timeout=60 * 60,
mode=777)
'''Set usage limiter '''
# limiter = Limiter(global_limits=["2000 per hour", "20 per second"])
# limiter.init_app(app)# use redis to store limits
'''Load api keys in redis'''
rate_limit_file = app.config['USAGE_LIMIT_PATH']
if not os.path.exists(rate_limit_file):
rate_limit_file = '../' + rate_limit_file
if os.path.exists(rate_limit_file):
with open(rate_limit_file) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
auth_key = AuthKey(**row)
app.extensions['redis-user'].hmset(auth_key.get_key(),
auth_key.__dict__)
print('INFO - succesfully loaded rate limit file')
else:
print('ERROR - cannot find rate limit file')
app.logger.error(
'cannot find rate limit file: %s. RATE LIMIT QUOTA LOAD SKIPPED!' %
rate_limit_file)
'''load ip name resolution'''
ip_resolver = defaultdict(lambda: "PUBLIC")
ip_list_file = app.config['IP_RESOLVER_LIST_PATH']
if not os.path.exists(ip_list_file):
ip_list_file = '../' + ip_list_file
if os.path.exists(ip_list_file):
with open(ip_list_file) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
net = IPNetwork(row['ip'])
ip_resolver[net] = row['org']
else:
app.logger.warning(
'cannot find IP list for IP resolver. All traffic will be logged as PUBLIC'
)
app.config['IP_RESOLVER'] = ip_resolver
'''compress http response'''
compress = Compress()
compress.init_app(app)
latest_blueprint = Blueprint('latest', __name__)
current_version_blueprint = Blueprint(str(api_version), __name__)
current_minor_version_blueprint = Blueprint(str(api_version_minor),
__name__)
specpath = '/cttv'
if app.config['PROFILE'] == True:
from werkzeug.contrib.profiler import ProfilerMiddleware
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
create_api(latest_blueprint, api_version, specpath)
create_api(current_version_blueprint, api_version, specpath)
create_api(current_minor_version_blueprint, api_version_minor, specpath)
app.register_blueprint(latest_blueprint, url_prefix='/api/latest')
app.register_blueprint(current_version_blueprint,
url_prefix='/api/' + str(api_version))
app.register_blueprint(current_minor_version_blueprint,
url_prefix='/api/' + str(api_version_minor))
@app.route('/api-docs/%s' % str(api_version_minor))
def docs_current_minor_version():
return redirect('/api/swagger/index.html')
@app.route('/api-docs/%s' % str(api_version))
def docs_current_version():
return redirect('/api/swagger/index.html')
@app.route('/api-docs')
def docs():
return redirect('/api/swagger/index.html')
def serve_swagger():
return app.send_static_file('docs/swagger/swagger.yaml')
@app.route('/api/docs/swagger.yaml')
def send_swagger():
return serve_swagger()
@app.route('/api/latest/docs/swagger.yaml')
def send_swagger_latest():
return serve_swagger()
@app.route('/api/' + str(api_version) + '/docs/swagger.yaml')
def send_swagger_current_cersion():
return serve_swagger()
@app.before_request
def before_request():
g.request_start = datetime.now()
@app.after_request
def after(resp):
rate_limiter = RateLimiter()
now = datetime.now()
took = (now - g.request_start).total_seconds() * 1000
if took > 500:
cache_time = str(
int(3600 * took)
) # set cache to last one our for each second spent in the request
resp.headers.add('X-Accel-Expires', cache_time)
took = int(round(took))
LogApiCallWeight(took)
# if took < RateLimiter.DEFAULT_CALL_WEIGHT:
# took = RateLimiter.DEFAULT_CALL_WEIGHT
current_values = increment_call_rate(took, rate_limiter)
now = datetime.now()
ceil10s = round(ceil_dt_to_future_time(now, 10), 2)
ceil1h = round(ceil_dt_to_future_time(now, 3600), 2)
usage_left_10s = rate_limiter.short_window_rate - current_values[
'short']
usage_left_1h = rate_limiter.long_window_rate - current_values['long']
min_ceil = ceil10s
if usage_left_1h < 0:
min_ceil = ceil1h
if (usage_left_10s < 0) or (usage_left_1h < 0):
resp.headers.add('Retry-After', min_ceil)
resp.headers.add('X-API-Took', took)
resp.headers.add('X-Usage-Limit-10s', rate_limiter.short_window_rate)
resp.headers.add('X-Usage-Limit-1h', rate_limiter.long_window_rate)
resp.headers.add('X-Usage-Remaining-10s', usage_left_10s)
resp.headers.add('X-Usage-Remaining-1h', usage_left_1h)
# resp.headers.add('X-Usage-Limit-Reset-10s', ceil10s)
# resp.headers.add('X-Usage-Limit-Reset-1h', ceil1h)
resp.headers.add('Access-Control-Allow-Origin', '*')
resp.headers.add('Access-Control-Allow-Headers',
'Content-Type,Auth-Token')
if do_not_cache(request): # do not cache in the browser
resp.headers.add('Cache-Control',
"no-cache, must-revalidate, max-age=0")
else:
resp.headers.add(
'Cache-Control',
"no-transform, public, max-age=%i, s-maxage=%i" %
(took * 1800 / 1000, took * 9000 / 1000))
return resp
return app
from unsw_ldap import authenticate
# config.variables contains appropriate pathnames for this course & session
import config
app = Flask(__name__,
template_folder=os.path.join(
config.variables['public_html_session_directory'], '.'),
static_folder=os.path.join(
config.variables['public_html_session_directory'], 'static'))
app.config.from_object(__name__)
cache = None
if os.path.exists(config.variables['flask_cache_directory']):
try:
cache = FileSystemCache(config.variables['flask_cache_directory'])
except (OSError, ImportError):
pass
# cache & gzip requests
def before_request():
if not cache or 'gzip' not in request.headers.get('Accept-Encoding',
'').lower():
return None
# tutors served different content so include is_tutor in key
cache_key = str(is_tutor()) + str(request.path)
return cache.get(cache_key)
def after_request(response):
class Cache(object):
"""Base class for TimeGate caches."""
def __init__(self, path, tolerance, expiration, max_values, run_tests=True,
max_file_size=0):
"""Constructor method.
:param path: The path of the cache database file.
:param tolerance: The tolerance, in seconds to which a TimeMap is
considered young enough to be used as is.
:param expiration: How long, in seconds, the cache entries are stored
every get will be a CACHE MISS.
:param max_values: The maximum number of TimeMaps stored in cache
before some are deleted
:param run_tests: (Optional) Tests the cache at initialization.
:param max_file_size: (Optional) The maximum size (in Bytes) for a
TimeMap cache value. When max_file_size=0, there is no limit to
a cache value. When max_file_size=X > 0, the cache will not
store TimeMap that require more than X Bytes on disk.
"""
# Parameters Check
if tolerance <= 0 or expiration <= 0 or max_values <= 0:
raise CacheError("Cannot create cache: all parameters must be > 0")
self.tolerance = relativedelta(seconds=tolerance)
self.path = path.rstrip('/')
self.max_file_size = max(max_file_size, 0)
self.CHECK_SIZE = self.max_file_size > 0
self.max_values = max_values
self.backend = FileSystemCache(path,
threshold=self.max_values,
default_timeout=expiration)
# Testing cache
if run_tests:
try:
key = '1'
val = 1
self.backend.set(key, val)
assert (not self.CHECK_SIZE) or self._check_size(key) > 0
assert self.backend.get(key) == val
os.remove(self.path + '/' + md5(key).hexdigest())
except Exception as e:
raise CacheError("Error testing cache: %s" % e)
logging.debug(
"Cache created. max_files = %d. Expiration = %d. "
"max_file_size = %d" % (
self.max_values, expiration, self.max_file_size))
def get_until(self, uri_r, date):
"""Returns the TimeMap (memento,datetime)-list for the requested
Memento. The TimeMap is guaranteed to span at least until the 'date'
parameter, within the tolerance.
:param uri_r: The URI-R of the resource as a string.
:param date: The target date. It is the accept-datetime for TimeGate
requests, and the current date. The cache will return all
Mementos prior to this date (within cache.tolerance parameter)
:return: [(memento_uri_string, datetime_obj),...] list if it is
in cache and if it is within the cache tolerance for *date*,
None otherwise.
"""
# Query the backend for stored cache values to that memento
key = uri_r
try:
val = self.backend.get(key)
except Exception as e:
logging.error("Exception loading cache content: %s" % e)
return None
if val:
# There is a value in the cache
timestamp, timemap = val
logging.info("Cached value exists for %s" % uri_r)
if date > timestamp + self.tolerance:
logging.info("Cache MISS: value outdated for %s" % uri_r)
timemap = None
else:
logging.info("Cache HIT: found value for %s" % uri_r)
else:
# Cache MISS: No value
logging.info("Cache MISS: No cached value for %s" % uri_r)
timemap = None
return timemap
def get_all(self, uri_r):
"""Request the whole TimeMap for that uri.
:param uri_r: the URI-R of the resource.
:return: [(memento_uri_string, datetime_obj),...] list if it is in
cache and if it is within the cache tolerance, None otherwise.
"""
return self.get_until(uri_r, timegate_utils.now())
def refresh(self, uri_r, getter, *args, **kwargs):
"""Refreshes the cached TimeMap for a specific resource and returns it.
:param uri_r: The original resource URI to refresh the TimeMap
:param getter: The function to call to get a fresh TimeMap
:param args: *getter* arguments
:param kwargs: *getter* keywords arguments
:return: The fresh TimeMap
"""
timemap = parsed_request(getter, *args, **kwargs)
# timemap,new_uri = parsed_request(getter, *args, **kwargs)
# if new_uri:
# uri_r = new_uri
# Creates or refreshes the new timemap for that URI-R
self._set(uri_r, timemap)
return timemap
def _set(self, uri_r, timemap):
"""Sets / refreshes the cached TimeMap for that URI-R. And appends it
with a timestamp of when it is stored.
:param uri_r: The URI-R of the original resource.
:param timemap: The value to cache.
:return: The backend setter method return value.
"""
logging.info("Updating cache for %s" % uri_r)
timestamp = timegate_utils.now()
val = (timestamp, timemap)
key = uri_r
try:
self.backend.set(key, val)
if self.CHECK_SIZE:
self._check_size(uri_r)
except Exception as e:
logging.error("Error setting cache value: %s" % e)
def _check_size(self, key, delete=True):
"""Check the size that a specific TimeMap value is using on disk.
It deletes if it is more than the maximum size.
:param key: The TimeMap original resource.
:param delete: (Optional) When true, the value is deleted.
Else only a warning is raised.
:return: The size of the value on disk (0 if it was deleted).
"""
try:
fname = md5(key).hexdigest() # werkzeug key
fpath = self.path + '/' + fname
size = os.path.getsize(fpath)
if size > self.max_file_size and delete:
message = ("Cache value too big (%dB, max %dB) "
"for the TimeMap of %s")
if delete:
message += ". Deleting cached value."
os.remove(fpath)
size = 0
logging.warning(message % (size, self.max_file_size, key))
return size
except Exception as e:
logging.error(
"Exception checking cache value size for TimeMap of %s "
"Exception: %s" % (key, e))
return 0
def runCouchPotato(options, base_path, args):
# Load settings
from couchpotato.environment import Env
settings = Env.get('settings')
settings.setFile(options.config_file)
# Create data dir if needed
data_dir = os.path.expanduser(Env.setting('data_dir'))
if data_dir == '':
data_dir = os.path.join(base_path, '_data')
if not os.path.isdir(data_dir):
os.makedirs(data_dir)
# Create logging dir
log_dir = os.path.join(data_dir, 'logs')
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
# Daemonize app
if options.daemonize:
createDaemon()
# Register environment settings
Env.set('uses_git', not options.git)
Env.set('app_dir', base_path)
Env.set('data_dir', data_dir)
Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log'))
Env.set('db_path', 'sqlite:///' + os.path.join(data_dir, 'couchpotato.db'))
Env.set('cache_dir', os.path.join(data_dir, 'cache'))
Env.set('cache',
FileSystemCache(os.path.join(Env.get('cache_dir'), 'python')))
Env.set('quiet', options.quiet)
Env.set('daemonize', options.daemonize)
Env.set('args', args)
# Determine debug
debug = options.debug or Env.setting('debug', default=False)
Env.set('debug', debug)
# Only run once when debugging
if os.environ.get('WERKZEUG_RUN_MAIN') or not debug:
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
'%H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
# To screen
if debug and not options.quiet and not options.daemonize:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000,
10)
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Disable server access log
server_log = logging.getLogger('werkzeug')
server_log.disabled = True
# Start logging
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s' % options)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root=base_path)
loader.run()
# Load migrations
from migrate.versioning.api import version_control, db_version, version, upgrade
db = Env.get('db_path')
repo = os.path.join(base_path, 'couchpotato', 'core', 'migration')
logging.getLogger('migrate').setLevel(
logging.WARNING) # Disable logging for migration
latest_db_version = version(repo)
initialize = True
try:
current_db_version = db_version(db, repo)
initialize = False
except:
version_control(db, repo, version=latest_db_version)
current_db_version = db_version(db, repo)
if current_db_version < latest_db_version and not debug:
log.info('Doing database upgrade. From %d to %d' %
(current_db_version, latest_db_version))
upgrade(db, repo)
# Configure Database
from couchpotato.core.settings.model import setup
setup()
fireEventAsync('app.load')
if initialize:
fireEventAsync('app.initialize')
# Create app
from couchpotato import app
api_key = Env.setting('api_key')
url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting(
'url_base') else ''
reloader = debug and not options.daemonize
# Basic config
app.secret_key = api_key
config = {
'use_reloader': reloader,
'host': Env.setting('host', default='0.0.0.0'),
'port': Env.setting('port', default=5000)
}
# Static path
web.add_url_rule(url_base + '/static/<path:filename>',
endpoint='static',
view_func=app.send_static_file)
# Register modules
app.register_blueprint(web, url_prefix='%s/' % url_base)
app.register_blueprint(api, url_prefix='%s/%s/' % (url_base, api_key))
# Go go go!
app.run(**config)
def getCache(name):
return FileSystemCache(os.path.join("cache", name))
def __init__(self, cache_dir='cache', default_timeout=300):
"""初始化
cache_dir是缓存目录
"""
self.cache = FileSystemCache(cache_dir, default_timeout=default_timeout)
from flask import Flask
from werkzeug.contrib.cache import FileSystemCache
import os
import logging
logging.basicConfig(level=logging.INFO)
root = os.getenv("MEDIABROWSER_ROOT")
if not root:
raise Exception('Must set MEDIABROWSER_ROOT variable')
cache_dir = os.getenv("MEDIABROWSER_CACHEDIR")
if not cache_dir:
raise Exception('Must set MEDIABROWSER_CACHEDIR variable')
# default_timeout=0 doesn't work with FileSystemCache
cache = FileSystemCache(cache_dir, default_timeout=9999999999, threshold=5000)
application = Flask("mediabrowser-demo")
application.register_blueprint(mediabrowser.build(root, cache))
def reverse_proxified(app):
"""
Configure apache as:
RequestHeader set X-Script-Name /videos
"""
def wsgi_call(environ, start_response):
script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
if script_name:
environ['SCRIPT_NAME'] = script_name
path_info = environ['PATH_INFO']
if path_info.startswith(script_name):
environ['PATH_INFO'] = path_info[len(script_name):]
def __init__(self, cache_dir, threshold, mode, key_prefix):
from werkzeug.contrib.cache import FileSystemCache
self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode)
self.key_prefix = key_prefix
def configure(configFile=None,
baseConfig="ProductionConfig",
port=8000,
extraConfig={}):
"""
TODO Document this critical function! What does it do? What does
it assume?
"""
file_handler = StreamHandler()
file_handler.setLevel(logging.WARNING)
app.logger.addHandler(file_handler)
configStr = 'ga4gh.server.serverconfig:{0}'.format(baseConfig)
app.config.from_object(configStr)
if os.environ.get('GA4GH_CONFIGURATION') is not None:
app.config.from_envvar('GA4GH_CONFIGURATION')
if configFile is not None:
app.config.from_pyfile(configFile)
app.config.update(extraConfig.items())
# Setup file handle cache max size
datamodel.fileHandleCache.setMaxCacheSize(
app.config["FILE_HANDLE_CACHE_MAX_SIZE"])
# Setup CORS
cors.CORS(app, allow_headers='Content-Type')
app.serverStatus = ServerStatus()
app.backend = _configure_backend(app)
if app.config.get('SECRET_KEY'):
app.secret_key = app.config['SECRET_KEY']
elif app.config.get('OIDC_PROVIDER'):
raise exceptions.ConfigurationException(
'OIDC configuration requires a secret key')
if app.config.get('CACHE_DIRECTORY'):
app.cache_dir = app.config['CACHE_DIRECTORY']
else:
app.cache_dir = '/tmp/ga4gh'
app.cache = FileSystemCache(app.cache_dir,
threshold=5000,
default_timeout=600,
mode=384)
app.oidcClient = None
app.myPort = port
if "OIDC_PROVIDER" in app.config:
# The oic client. If we're testing, we don't want to verify
# SSL certificates
app.oidcClient = oic.oic.Client(
verify_ssl=('TESTING' not in app.config))
try:
app.oidcClient.provider_config(app.config['OIDC_PROVIDER'])
except requests.exceptions.ConnectionError:
configResponse = message.ProviderConfigurationResponse(
issuer=app.config['OIDC_PROVIDER'],
authorization_endpoint=app.config['OIDC_AUTHZ_ENDPOINT'],
token_endpoint=app.config['OIDC_TOKEN_ENDPOINT'],
revocation_endpoint=app.config['OIDC_TOKEN_REV_ENDPOINT'])
app.oidcClient.handle_provider_config(configResponse,
app.config['OIDC_PROVIDER'])
# The redirect URI comes from the configuration.
# If we are testing, then we allow the automatic creation of a
# redirect uri if none is configured
redirectUri = app.config.get('OIDC_REDIRECT_URI')
if redirectUri is None and app.config.get('TESTING'):
redirectUri = 'https://{0}:{1}/oauth2callback'.format(
socket.gethostname(), app.myPort)
app.oidcClient.redirect_uris = [redirectUri]
if redirectUri is []:
raise exceptions.ConfigurationException(
'OIDC configuration requires a redirect uri')
# We only support dynamic registration while testing.
if ('registration_endpoint' in app.oidcClient.provider_info
and app.config.get('TESTING')):
app.oidcClient.register(
app.oidcClient.provider_info["registration_endpoint"],
redirect_uris=[redirectUri])
else:
response = message.RegistrationResponse(
client_id=app.config['OIDC_CLIENT_ID'],
client_secret=app.config['OIDC_CLIENT_SECRET'],
redirect_uris=[redirectUri],
verify_ssl=False)
app.oidcClient.store_registration_info(response)
import os
import logging
from flask import Flask, render_template, jsonify, redirect, request, json, send_from_directory
from werkzeug.contrib.cache import FileSystemCache
import cinemas
ENV_DEBUG = os.environ.get('DEBUG') == 'True'
PORT = int(os.environ.get('PORT', 5000))
CACHE_TIMEOUT = 60 * 60 * 12
cache = FileSystemCache('.cachedir', default_timeout=CACHE_TIMEOUT)
app = Flask(__name__)
@app.route('/')
def output_static_index_page():
return render_template('films_list.html')
@app.route('/api')
def output_static_api_page():
return render_template('api.html')
@app.route('/movies', methods=['GET'])
def output_movies():
movies_json = cache.get('movies_json')
if movies_json is None:
afisha_page = cinemas.fetch_afisha_page()
movies_from_afisha = cinemas.parse_afisha_list(afisha_page)
class Cache(object):
"""
This class is used to control the cache objects.
If TESTING is True it will use NullCache.
"""
def __init__(self, app=None):
self.cache = None
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app):
"This is used to initialize cache with your app object"
app.config.setdefault('CACHE_DEFAULT_TIMEOUT', 300)
app.config.setdefault('CACHE_THRESHOLD', 500)
app.config.setdefault('CACHE_KEY_PREFIX', None)
app.config.setdefault('CACHE_MEMCACHED_SERVERS', None)
app.config.setdefault('CACHE_DIR', None)
app.config.setdefault('CACHE_TYPE', 'NullCache')
self.app = app
self._set_cache()
def _set_cache(self):
if self.app.config['TESTING']:
self.cache = NullCache()
else:
if self.app.config['CACHE_TYPE'] == 'Null':
self.cache = NullCache()
elif self.app.config['CACHE_TYPE'] == 'Simple':
self.cache = SimpleCache(
threshold=self.app.config['CACHE_THRESHOLD'],
default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'])
elif self.app.config['CACHE_TYPE'] == 'Memcached':
self.cache = MemcachedCache(
self.app.config['CACHE_MEMCACHED_SERVERS'],
default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'],
key_prefix=self.app.config['CACHE_KEY_PREFIX'])
elif self.app.config['CACHE_TYPE'] == 'GAE':
self.cache = GAEMemcachedCache(
default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'],
key_prefix=self.app.config['CACHE_KEY_PREFIX'])
elif self.app.config['CACHE_TYPE'] == 'FileSystem':
self.cache = FileSystemCache(
self.app.config['CACHE_DIR'],
threshold=self.app.config['CACHE_THRESHOLD'],
default_timeout=self.app.config['CACHE_DEFAULT_TIMEOUT'])
def get(self, *args, **kwargs):
"Proxy function for internal cache object."
return self.cache.get(*args, **kwargs)
def set(self, *args, **kwargs):
"Proxy function for internal cache object."
self.cache.set(*args, **kwargs)
def add(self, *args, **kwargs):
"Proxy function for internal cache object."
self.cache.add(*args, **kwargs)
def delete(self, *args, **kwargs):
"Proxy function for internal cache object."
self.cache.delete(*args, **kwargs)
def cached(self, timeout=None, key_prefix='view/%s', unless=None):
"""
Decorator. Use this to cache a function. By default the cache key
is `view/request.path`. You are able to use this decorator with any
function by changing the `key_prefix`. If the token `%s` is located
within the `key_prefix` then it will replace that with `request.path`
Example::
# An example view function
@cache.cached(timeout=50)
def big_foo():
return big_bar_calc()
# An example misc function to cache.
@cache.cached(key_prefix='MyCachedList')
def get_list():
return [random.randrange(0, 1) for i in range(50000)]
.. code-block:: pycon
>>> my_list = get_list()
:param timeout: Default None. If set to an integer, will cache for that
amount of time.
:param key_prefix: Default 'view/%(request.path)s'. Beginning key to .
use for the cache key.
:param unless: Default None. Cache will *always* execute the caching
facilities unless this callable is true.
This will bypass the caching entirely.
"""
def decorator(f):
@wraps(f)
def decorated_function(*args, **kwargs):
#: Bypass the cache entirely.
if callable(unless) and unless() is True:
return f(*args, **kwargs)
if '%s' in key_prefix:
cache_key = key_prefix % request.path
else:
cache_key = key_prefix
rv = self.cache.get(cache_key)
if not rv or current_app.debug:
rv = f(*args, **kwargs)
self.cache.set(cache_key, rv, timeout=timeout)
return rv
return decorated_function
return decorator
def memoize(self, timeout=None):
"""
Use this to cache the result of a function, taking its arguments into
account in the cache key.
Information on
`Memoization <http://en.wikipedia.org/wiki/Memoization>`_.
Example::
@cache.memoize(timeout=50)
def big_foo(a, b):
return a + b + random.randrange(0, 1000)
.. code-block:: pycon
>>> big_foo(5, 2)
753
>>> big_foo(5, 3)
234
>>> big_foo(5, 2)
753
:param timeout: Default None. If set to an integer, will cache for that
amount of time.
"""
def memoize(f):
@wraps(f)
def decorated_function(*args, **kwargs):
cache_key = (f.__name__, id(f), args, str(kwargs))
rv = self.cache.get(cache_key)
if rv is None:
rv = f(*args, **kwargs)
self.cache.set(cache_key, rv)
return rv
return decorated_function
return memoize
import json
import os
import requests
from datetime import datetime
from flask import Flask, render_template, request
from socket import gethostname
from urllib.parse import quote, urlparse
from werkzeug.contrib.cache import FileSystemCache
app = Flask(__name__)
cache = FileSystemCache(os.path.join(app.root_path, '_cache'))
def parse_date(timestamp):
return datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d')
def parse_google_sb(site):
base = 'https://www.google.com/transparencyreport/safebrowsing/diagnostic/index.html#url=' # previously 'https://www.google.com/safebrowsing/diagnostic?site='
query = quote(site, safe='')
page = base + query
resp_url = 'https://www.google.com/safebrowsing/diagnostic?output=jsonp&site=' + query
resp = requests.get(resp_url).text
resp_str = resp.split('processResponse(')[1].split(');')[0]
resp_obj = json.loads(resp_str)
url = resp_obj['website'].get('name')
num_tested = resp_obj.get('numTested')
if num_tested:
status_set = set()
for k in resp_obj['website'].keys():
if k[-6:] == 'Status':
from flask import Flask, request
from werkzeug.contrib.cache import FileSystemCache
CACHE_TIMEOUT = 300
cache = FileSystemCache('summit/cache')
class cached(object):
def __init__(self, timeout=None):
self.timeout = timeout or CACHE_TIMEOUT
def __call__(self, f):
def decorator(*args, **kwargs):
response = cache.get(request.path)
if response is None:
response = f(*args, **kwargs)
cache.set(request.path, response, self.timeout)
return response
return decorator
tic = CtsTextInventoryCollection()
latin = CtsTextInventoryMetadata("urn:perseus:latinLit", parent=tic)
latin.set_label("Classical Latin", "eng")
latin.set_label("Latin Classique", "fre")
dispatcher = CollectionDispatcher(tic)
@dispatcher.inventory("urn:perseus:latinLit")
def dispatchLatinLit(collection, path=None, **kwargs):
if collection.id.startswith("urn:cts:latinLit:"):
return True
return False
return dispatcher
nautilus_cache = FileSystemCache(subprocess_cache_dir, default_timeout=0)
resolver = NautilusCTSResolver(subprocess_repository,
dispatcher=make_dispatcher(),
cache=nautilus_cache)
app = Flask("Nautilus")
http_cache = Cache(app,
config={
'CACHE_TYPE': "filesystem",
"CACHE_DIR": http_cache_dir,
"CACHE_DEFAULT_TIMEOUT": 0
})
nautilus = FlaskNautilus(app=app,
prefix="/api",
name="nautilus",
#!/usr/bin/python
#coding=utf-8
# imports
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
from werkzeug.contrib.cache import FileSystemCache
app = Flask(__name__)
app.config.from_object(
'config.config.DevelopmentConfig') # 导入所有配置，后续都需要用app.config['key']获取。
# app.config.from_object('config.config.ProductionConfig') # 导入所有配置，后续都需要用app.config['key']获取。
app.config.from_envvar('WEBSTH_CONFIG', silent=True) # 如果有环境变量，就导入此文件覆盖相应设置。
db = SQLAlchemy(app)
cache = FileSystemCache(app.config['CACHE_PATH'])
# import views
from views import *
# 开发环境运行部分，开始运行程序
if __name__ == '__main__':
app.run()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from linklist import parse_linklist
from progressbar import Bar, ProgressBar, SimpleProgress
from requests import get, head, ConnectionError, Timeout
from sys import stdin, stderr
from werkzeug.contrib.cache import FileSystemCache
from urllib2 import urlparse
url_status_cache = FileSystemCache('.cache_dir', \
threshold=10000, default_timeout=3600)
def validate_links(data):
widgets = [Bar(), SimpleProgress()]
pbar = ProgressBar(widgets=widgets, maxval=len(data)).start()
for i, element in enumerate(data):
url = element['url']
if url == '':
continue
scheme = urlparse.urlsplit(url).scheme
host = urlparse.urlsplit(url).netloc
if scheme in ('http', 'https') and \
url_status_cache.get(url) is not True:
try:
request = head(url, timeout=10)
# some web sites cannot into head requests
if request.status_code in (403, 405, 500) or \
host in ('mobil.morgenpost.de'):
class FileSystemSessionInterface(SessionInterface):
"""Uses the :class:`werkzeug.contrib.cache.FileSystemCache` as a session
backend.
.. versionadded:: 0.2
The `use_signer` parameter was added.
:param cache_dir: the directory where session files are stored.
:param threshold: the maximum number of items the session stores before it
starts deleting some.
:param mode: the file mode wanted for the session files, default 0600
:param key_prefix: A prefix that is added to FileSystemCache store keys.
:param use_signer: Whether to sign the session id cookie or not.
:param permanent: Whether to use permanent session or not.
"""
session_class = FileSystemSession
def __init__(self, cache_dir, threshold, mode, key_prefix,
use_signer=False, permanent=True):
from werkzeug.contrib.cache import FileSystemCache
self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode)
self.key_prefix = key_prefix
self.use_signer = use_signer
self.permanent = permanent
def open_session(self, app, request):
sid = request.cookies.get(app.session_cookie_name)
if not sid:
sid = self._generate_sid()
return self.session_class(sid=sid, permanent=self.permanent)
if self.use_signer:
signer = self._get_signer(app)
if signer is None:
return None
try:
sid = signer.unsign(sid)
except BadSignature:
sid = self._generate_sid()
return self.session_class(sid=sid, permanent=self.permanent)
data = self.cache.get(self.key_prefix + sid)
if data is not None:
return self.session_class(data, sid=sid)
return self.session_class(sid=sid, permanent=self.permanent)
def save_session(self, app, session, response):
domain = self.get_cookie_domain(app)
path = self.get_cookie_path(app)
if not session:
if session.modified:
self.cache.delete(self.key_prefix + session.sid)
response.delete_cookie(app.session_cookie_name,
domain=domain, path=path)
return
httponly = self.get_cookie_httponly(app)
secure = self.get_cookie_secure(app)
expires = self.get_expiration_time(app, session)
data = dict(session)
self.cache.set(self.key_prefix + session.sid, data,
total_seconds(app.permanent_session_lifetime))
if self.use_signer:
session_id = self._get_signer(app).sign(session.sid)
else:
session_id = session.sid
response.set_cookie(app.session_cookie_name, session_id,
expires=expires, httponly=httponly,
domain=domain, path=path, secure=secure)
__author__ = 'hadware'
from os.path import dirname, join, isdir
from werkzeug.contrib.cache import FileSystemCache
CACHE_TIMEOUT = 300
CACHE_DIRNAME = join(dirname(__file__), "cache")
if not isdir(CACHE_DIRNAME):
try:
makedirs(CACHE_DIRNAME)
except OSError:
pass
cache = FileSystemCache(CACHE_DIRNAME)
class cached(object):
def __init__(self, cache_key, timeout=None):
self.timeout = timeout or CACHE_TIMEOUT
self.cache_key = cache_key
def __call__(self, f):
def decorator(*args, **kwargs):
cached_value = cache.get(self.cache_key)
if cached_value is None:
cached_value = f(*args, **kwargs)
cache.set(self.cache_key, cached_value, self.timeout)
return cached_value
# -*- coding: utf-8 -*-
from flask import Flask
from flask.ext.nemo import Nemo
from nautilus.flask_ext import FlaskNautilus
from werkzeug.contrib.cache import FileSystemCache
from flask_cache import Cache
app = Flask("Nautilus")
nautilus_cache = FileSystemCache("/code/cache")
nautilus = FlaskNautilus(app=app,
prefix="/api/cts",
resources=["/code/data/canonical-latinLit-master"],
parser_cache=nautilus_cache,
http_cache=Cache(config={'CACHE_TYPE': "simple"}))
if __name__ == "__main__":
app.run(debug=True, host='0.0.0.0')
class FileSystemSessionInterface(SessionInterface):
"""Uses the :class:`werkzeug.contrib.cache.FileSystemCache` as a session
backend.
.. versionadded:: 0.2
The `use_signer` parameter was added.
:param cache_dir: the directory where session files are stored.
:param threshold: the maximum number of items the session stores before it
starts deleting some.
:param mode: the file mode wanted for the session files, default 0600
:param key_prefix: A prefix that is added to FileSystemCache store keys.
:param use_signer: Whether to sign the session id cookie or not.
:param permanent: Whether to use permanent session or not.
"""
session_class = FileSystemSession
def __init__(self,
cache_dir,
threshold,
mode,
key_prefix,
use_signer=False,
permanent=True):
from werkzeug.contrib.cache import FileSystemCache
self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode)
self.key_prefix = key_prefix
self.use_signer = use_signer
self.permanent = permanent
def open_session(self, app, request):
sid = request.cookies.get(app.session_cookie_name)
if not sid:
sid = self._generate_sid()
return self.session_class(sid=sid, permanent=self.permanent)
if self.use_signer:
signer = self._get_signer(app)
if signer is None:
return None
try:
sid_as_bytes = signer.unsign(sid)
sid = sid_as_bytes.decode()
except BadSignature:
sid = self._generate_sid()
return self.session_class(sid=sid, permanent=self.permanent)
data = self.cache.get(self.key_prefix + sid)
if data is not None:
return self.session_class(data, sid=sid)
return self.session_class(sid=sid, permanent=self.permanent)
def save_session(self, app, session, response):
domain = self.get_cookie_domain(app)
path = self.get_cookie_path(app)
if not session:
if session.modified:
self.cache.delete(self.key_prefix + session.sid)
response.delete_cookie(app.session_cookie_name,
domain=domain,
path=path)
return
httponly = self.get_cookie_httponly(app)
secure = self.get_cookie_secure(app)
expires = self.get_expiration_time(app, session)
data = dict(session)
self.cache.set(self.key_prefix + session.sid, data,
total_seconds(app.permanent_session_lifetime))
if self.use_signer:
session_id = self._get_signer(app).sign(want_bytes(session.sid))
else:
session_id = session.sid
response.set_cookie(app.session_cookie_name,
session_id,
expires=expires,
httponly=httponly,
domain=domain,
path=path,
secure=secure)
class FileSystemSessionInterface(SessionInterface):
"""Uses the :class:`werkzeug.contrib.cache.FileSystemCache` as a session
backend.
.. versionadded:: 0.2
The `use_signer` parameter was added.
:param cache_dir: the directory where session files are stored.
:param threshold: the maximum number of items the session stores before it
starts deleting some.
:param mode: the file mode wanted for the session files, default 0600
:param key_prefix: A prefix that is added to FileSystemCache store keys.
:param use_signer: Whether to sign the session id cookie or not.
"""
session_class = FileSystemSession
def __init__(self, cache_dir, threshold, mode, key_prefix,
use_signer=False):
from werkzeug.contrib.cache import FileSystemCache
self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode) #FileSystemCache是从上句导入的,threshold代表最多纪录条数
self.key_prefix = key_prefix
self.use_signer = use_signer
def open_session(self, app, request):
sid = request.cookies.get(app.session_cookie_name)
if not sid:
sid = self._generate_sid()
return self.session_class(sid=sid)
if self.use_signer:
signer = self._get_signer(app)
if signer is None:
return None
try:
sid = signer.unsign(sid)
except BadSignature:
sid = None
data = self.cache.get(self.key_prefix + sid) #这是根据session_id获取session值
if data is not None:
return self.session_class(data, sid=sid)
return self.session_class(sid=sid)
def save_session(self, app, session, response):
domain = self.get_cookie_domain(app)
path = self.get_cookie_path(app)
if not session: #验证过了，就是判断session是否为空(空字典)
if session.modified:
self.cache.delete(self.key_prefix + session.sid)
response.delete_cookie(app.session_cookie_name,
domain=domain, path=path)
return
# Modification case. There are upsides and downsides to
# emitting a set-cookie header each request. The behavior
# is controlled by the :meth:`should_set_cookie` method
# which performs a quick check to figure out if the cookie
# should be set or not. This is controlled by the
# SESSION_REFRESH_EACH_REQUEST config flag as well as
# the permanent flag on the session itself.
#if not self.should_set_cookie(app, session):
# return
httponly = self.get_cookie_httponly(app)
secure = self.get_cookie_secure(app)
expires = self.get_expiration_time(app, session)
#data = dict(session)
#self.cache.set(self.key_prefix + session.sid, data,
# int(app.permanent_session_lifetime.total_seconds()))
if self.use_signer: #签名加密
session_id = self._get_signer(app).sign(session.sid)
else:
session_id = session.sid
response.set_cookie(app.session_cookie_name, session_id,
expires=expires, httponly=httponly,
domain=domain, path=path, secure=secure)
def save_session_without_response(self, app, session):
#httponly = self.get_cookie_httponly(app)
#secure = self.get_cookie_secure(app)
#expires = self.get_expiration_time(app, session)
data = dict(session)
self.cache.set(self.key_prefix + session.sid, data,
int(app.permanent_session_lifetime.total_seconds()))
def judge_attack(self, app, request): #判断两次间隔的访问是否大于三秒
sid = request.cookies.get(app.session_cookie_name)
if not sid:
return False
current_time = time.time()
try:
last_time = self.cache.get(self.key_prefix + sid)['time'] #这是根据session_id获取session值
except:
return False
if current_time-last_time < 3:
return True
return False
sys.path.append(os.getcwd())
app = Flask(__name__)
app.config.update(
PERSONA_JS='https://login.persona.org/include.js',
PERSONA_VERIFIER='https://verifier.login.persona.org/verify'
)
app.config.from_object("simple_settings")
app.secret_key = app.config["SECRET_KEY"]
if not app.config["DEBUG"]:
app.static_folder = os.path.join(os.getcwd(), "static")
db = orm.Database('sqlite',
os.path.join(os.getcwd(), app.config["DATABASE_FILE"]), create_db=True)
cache = FileSystemCache(app.config["CACHE_DIR"])
csrf = SeaSurf(app)
basic_auth = BasicAuth(app)
app.config['UPLOAD_FOLDER'] = os.path.join(os.getcwd(), "uploads")
ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'gif'}
app.jinja_env.globals.update(now=datetime.datetime.now)
md = markdown.Markdown(
extensions=['fenced_code', 'toc'],
output_format="html5"
)
class Post(db.Entity):
_table_ = "posts"
Server: Werkzeug/0.9.4 Python/2.7.5+
X-Cache: miss
1000
"""
from __future__ import division, unicode_literals
from six.moves.http_client import BAD_REQUEST, OK
from time import sleep
from werkzeug.contrib.cache import FileSystemCache
from flask import Flask, render_template, request
from flask.ext.webcache import easy_setup, modifiers
app = Flask(__name__)
werkzeug_cache = FileSystemCache('/tmp/.sleepycalc')
easy_setup(app, werkzeug_cache)
PLAINTEXT = (('Content-Type', 'text/plain'), )
@app.route("/")
def index():
return render_template("index.html")
@app.route("/addition")
@modifiers.cache_for(seconds=30)
def addition():
try:
term1, term2 = int(request.args['term1']), int(request.args['term2'])
def runCouchPotato(options,
base_path,
args,
data_dir=None,
log_dir=None,
Env=None,
desktop=None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
# Do db stuff
db_path = os.path.join(data_dir, 'couchpotato.db')
# Backup before start and cleanup old databases
new_backup = os.path.join(data_dir, 'db_backup', str(int(time.time())))
# Create path and copy
if not os.path.isdir(new_backup): os.makedirs(new_backup)
src_files = [
options.config_file, db_path, db_path + '-shm', db_path + '-wal'
]
for src_file in src_files:
if os.path.isfile(src_file):
shutil.copy2(src_file,
os.path.join(new_backup, os.path.basename(src_file)))
# Remove older backups, keep backups 3 days or at least 3
backups = []
for directory in os.listdir(os.path.dirname(new_backup)):
backup = os.path.join(os.path.dirname(new_backup), directory)
if os.path.isdir(backup):
backups.append(backup)
total_backups = len(backups)
for backup in backups:
if total_backups > 3:
if int(os.path.basename(backup)) < time.time() - 259200:
for src_file in src_files:
b_file = os.path.join(backup, os.path.basename(src_file))
if os.path.isfile(b_file):
os.remove(b_file)
os.rmdir(backup)
total_backups -= 1
# Register environment settings
Env.set('encoding', encoding)
Env.set('app_dir', base_path)
Env.set('data_dir', data_dir)
Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log'))
Env.set('db_path', 'sqlite:///' + db_path)
Env.set('cache_dir', os.path.join(data_dir, 'cache'))
Env.set('cache',
FileSystemCache(os.path.join(Env.get('cache_dir'), 'python')))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default=False, type='bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default=False, type='bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp', 'migrate']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Use reloader
reloader = debug is True and development and not Env.get(
'desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
'%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or
options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10)
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file=None, line=None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Check if database exists
db = Env.get('db_path')
db_exists = os.path.isfile(db_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root=base_path)
loader.run()
# Load migrations
if db_exists:
from migrate.versioning.api import version_control, db_version, version, upgrade
repo = os.path.join(base_path, 'couchpotato', 'core', 'migration')
latest_db_version = version(repo)
try:
current_db_version = db_version(db, repo)
except:
version_control(db, repo, version=latest_db_version)
current_db_version = db_version(db, repo)
if current_db_version < latest_db_version and not debug:
log.info('Doing database upgrade. From %d to %d',
(current_db_version, latest_db_version))
upgrade(db, repo)
# Configure Database
from couchpotato.core.settings.model import setup
setup()
# Fill database with needed stuff
if not db_exists:
fireEvent('app.initialize', in_order=True)
# Create app
from couchpotato import app
api_key = Env.setting('api_key')
url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting(
'url_base') else ''
# Basic config
app.secret_key = api_key
# app.debug = development
config = {
'use_reloader': reloader,
'host': Env.setting('host', default='0.0.0.0'),
'port': tryInt(Env.setting('port', default=5000))
}
# Static path
app.static_folder = os.path.join(base_path, 'couchpotato', 'static')
web.add_url_rule('api/%s/static/<path:filename>' % api_key,
endpoint='static',
view_func=app.send_static_file)
# Register modules
app.register_blueprint(web, url_prefix='%s/' % url_base)
app.register_blueprint(api, url_prefix='%s/api/%s/' % (url_base, api_key))
# Some logging and fire load event
try:
log.info('Starting server on port %(port)s', config)
except:
pass
fireEventAsync('app.load')
# Go go go!
web_container = WSGIContainer(app)
web_container._log = _log
loop = IOLoop.instance()
application = Application([
(r'%s/api/%s/nonblock/(.*)/' % (url_base, api_key), NonBlockHandler),
(r'.*', FallbackHandler, dict(fallback=web_container)),
],
log_function=lambda x: None,
debug=config['use_reloader'])
try_restart = True
restart_tries = 5
while try_restart:
try:
application.listen(config['port'],
config['host'],
no_keep_alive=True)
loop.start()
except Exception, e:
try:
nr, msg = e
if nr == 48:
log.info(
'Already in use, try %s more time after few seconds',
restart_tries)
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except:
pass
raise
try_restart = False
from bs4 import BeautifulSoup
from threading import Thread
from werkzeug.contrib.cache import FileSystemCache
import requests
cache = FileSystemCache('static/cache/',
threshold=100,
default_timeout=24 * 60 * 60)
def add_additional_film_info(film):
title = film.get('film_title')
film_page = fetch_kinopoisk_page(title)
additional_information = parse_kinopoisk_page(film_page.text)
film.update(additional_information)
def sort_film_list(item_list):
return sorted(item_list,
key=lambda item: item['film_rating'],
reverse=True)
def list_of_films():
number_of_top_films = 10
afisha_page_url = 'https://www.afisha.ru/msk/schedule_cinema/'
afisha_page_raw = fetch_page(afisha_page_url)
film_list = parse_afisha_page(afisha_page_raw.text)
threads_list = []
for film in film_list:
thread = Thread(target=add_additional_film_info,
def __init__(self, cache_dir, threshold, mode, key_prefix,
use_signer=False):
from werkzeug.contrib.cache import FileSystemCache
self.cache = FileSystemCache(cache_dir, threshold=threshold, mode=mode) #FileSystemCache是从上句导入的,threshold代表最多纪录条数
self.key_prefix = key_prefix
self.use_signer = use_signer
class TestSaxonJar(TestCase):
def setUp(self):
self.cache = FileSystemCache("./cache")
self.saxon = SaxonShellTransform(
"./jars/saxon.jar",
"./tests/data/xsl/ciham.xsl",
cache=self.cache
)
self.nautilus = NautilusRetriever(
folders=[
"./tests/data/repo"
]
)
self.nautilus.logger.setLevel(logging.ERROR)
app = Flask("Nemo")
app.debug = True
nemo = Nemo(
app=app,
base_url="",
retriever=self.nautilus,
transform={
"default": self.saxon.transform
}
)
self.client = app.test_client()
def tearDown(self):
# We clean the cache folder to ensure that no cache is passed from one test to the other
self.cache.clear()
def test_simple_transformation(self):
""" Test transformation works fine"""
read = self.client.get("/read/froLit/jns915/jns1856/ciham-fro1/1")
data = read.data.decode()
self.assertIn(
'<span class="expan">et </span>', data,
"Text content should be transformed"
)
self.assertIn(
'Facsimilaire', data,
"Other content should be added"
)
cached = self.cache.get("urn:cts:froLit:jns915.jns1856.ciham-fro1:1").decode()
self.assertIn('<aside class="text-left">', cached, "Assert cache is made")
def test_cache_retrieved(self):
""" Test that cache is nicely used and built """
read = self.client.get("/read/froLit/jns915/jns1856/ciham-fro1/1")
data = read.data.decode()
self.assertIn(
'<span class="expan">et </span>', data,
"Text content should be transformed"
)
self.assertIn(
'Facsimilaire', data,
"Other content should be added"
)
cached = self.cache.get("urn:cts:froLit:jns915.jns1856.ciham-fro1:1").decode()
self.assertIn('<aside class="text-left">', cached, "Assert cache is made")
with mock.patch("nemo_xslttwo_plugin.shell") as shell:
read = self.client.get("/read/froLit/jns915/jns1856/ciham-fro1/1")
cached_response = read.data.decode()
self.assertEqual(
cached_response, data,
"Text content should the same in cache"
)
self.assertEqual(
shell.call_count, 0,
"Shell should not be called because we use cache"
)
import urllib2
import datetime
from bs4 import BeautifulSoup
from urlparse import urljoin
from flask import Flask, Markup, request
from werkzeug.contrib.atom import AtomFeed
from werkzeug.contrib.cache import FileSystemCache
app = Flask(__name__)
cache = FileSystemCache(cache_dir='/tmp/')
CACHE_TIMEOUT = 60
def get_url(url, cache_timeout):
"""
Returns the contents of the URL as a string.
"""
html = cache.get("cache")
if html is None:
response = urllib2.urlopen(url)
html = response.read()
cache.set("cache", html, timeout=cache_timeout)
return html
def get_articles(url):
"""yields a tuple of (title, url, published)"""
html = get_url(url, cache_timeout=CACHE_TIMEOUT)
soup = BeautifulSoup(html)
for event in soup.select('.content-view-listing__single-item'):
from unicodedata import normalize
# web stuff and markdown imports
import markdown
from flask.ext.sqlalchemy import SQLAlchemy
from werkzeug.security import check_password_hash
from flask import render_template, request, Flask, flash, redirect, url_for, \
abort, jsonify, Response, make_response
from werkzeug.contrib.cache import FileSystemCache, NullCache
app = Flask(__name__)
app.config.from_object('settings')
db = SQLAlchemy(app)
cache_directory = os.path.dirname(__file__)
try:
cache = FileSystemCache(os.path.join(cache_directory, "cache"))
except Exception,e:
print "Could not create cache folder, caching will be disabled."
print "Error: %s"%e
cache = NullCache()
_punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+')
MARKDOWN_PARSER = markdown.Markdown(extensions=['fenced_code'],
output_format="html5",
safe_mode=True)
class Post(db.Model):
def __init__(self, title=None, created_at=None):
def __init__(self, cache_dir='cache', default_timeout=300):
"""初始化
cache_dir是缓存目录
"""
self.cache = FileSystemCache(cache_dir, default_timeout=default_timeout)
import requests
from werkzeug.contrib.cache import FileSystemCache
from pythoncz import app
__all__ = ('get_issues', )
get_issues_graphql_filename = (os.path.join(os.path.dirname(__file__),
'github_get_issues.graphql'))
with open(get_issues_graphql_filename) as f:
GET_ISSUES_GRAPHQL = f.read()
SIX_HOURS_AS_SECONDS = 21600
cache = FileSystemCache(app.config['CACHE_DIR'],
default_timeout=SIX_HOURS_AS_SECONDS)
def get_issues(org_names):
issues = cache.get('github-issues')
if issues is None:
session = _create_api_session()
issues = itertools.chain(*(_get_issues_for_org(session, org_name)
for org_name in org_names))
issues = _sort_issues(issues)
cache.set('github-issues', issues)
return issues
def _create_api_session():
user_agent = ('pythoncz/{now.year}-{now.month} '
import sys
from contextlib import closing
import lxml.html as html # pip install 'lxml>=2.3.1'
from lxml.html.clean import Cleaner
from selenium.webdriver import Firefox # pip install selenium
from werkzeug.contrib.cache import FileSystemCache # pip install werkzeug
cache = FileSystemCache('.cachedir', threshold=100000)
url = sys.argv[1] if len(sys.argv) > 1 else "http://www.schibsted.cl/testqa/"
# get page
page_source = cache.get(url)
if page_source is None:
# use firefox to get page with javascript generated content
with closing(Firefox()) as browser:
browser.get(url)
page_source = browser.page_source
cache.set(url, page_source, timeout=60*60*24*7) # week in seconds
# extract text
root = html.document_fromstring(page_source)
# remove flash, images, <script>,<style>, etc
Cleaner(kill_tags=['noscript'], style=True)(root) # lxml >= 2.3.1
webtext = root.text_content() # extract text
f = open("C:/schibsted/data/Test.txt", "w");
print f
value = (webtext)
class Cache(object):
"""Cache module based on werkzeug.contrib.cache. This is a mixed
version of NullCache, SimpleCache, FileSystemCache, MemcachedCache,
and RedisCache.
:param app: Flask app instance.
:param config_prefix: Define a prefix for Flask app config.
:param kwargs: Extra parameters.
You need to configure a type of the cache, and its related configurations.
The default ``config_prefix`` is ``AUTHLIB``, so it requires a config of::
AUTHLIB_CACHE_TYPE = 'simple'
If ``config_prefix`` is something else, like ``EXAMPLE``, it would be::
EXAMPLE_CACHE_TYPE = 'simple'
The available cache types are:
* null: It will not cache anything. No configuration.
* simple: It caches things in memory.
The only configuration is ``threshold``::
AUTHLIB_CACHE_THRESHOLD = 500
* memcache: It caches things in Memcache. Available configurations::
AUTHLIB_CACHE_MEMCACHED_SERVERS = []
AUTHLIB_CACHE_KEY_PREFIX = None
* redis: It caches things in Redis. Available configurations::
AUTHLIB_CACHE_REDIS_HOST = 'localhost'
AUTHLIB_CACHE_REDIS_PORT = 6379
AUTHLIB_CACHE_REDIS_PASSWORD = None
AUTHLIB_CACHE_REDIS_DB = 0
AUTHLIB_CACHE_KEY_PREFIX = None
* filesystem: It caches things in local filesystem. Available
configurations::
AUTHLIB_CACHE_DIR = '' # required
AUTHLIB_CACHE_THRESHOLD = 500
"""
def __init__(self, app, config_prefix='AUTHLIB', **kwargs):
self.config_prefix = config_prefix
self.config = app.config
cache_type = self._config('type')
kwargs.update(
dict(default_timeout=self._config('DEFAULT_TIMEOUT', 100)))
if cache_type == 'null':
self.cache = NullCache()
elif cache_type == 'simple':
kwargs.update(dict(threshold=self._config('threshold', 500)))
self.cache = SimpleCache(**kwargs)
elif cache_type == 'memcache':
kwargs.update(
dict(
servers=self._config('MEMCACHED_SERVERS'),
key_prefix=self._config('KEY_PREFIX', None),
))
self.cache = MemcachedCache(**kwargs)
elif cache_type == 'redis':
kwargs.update(
dict(
host=self._config('REDIS_HOST', 'localhost'),
port=self._config('REDIS_PORT', 6379),
password=self._config('REDIS_PASSWORD', None),
db=self._config('REDIS_DB', 0),
key_prefix=self._config('KEY_PREFIX', None),
))
self.cache = RedisCache(**kwargs)
elif cache_type == 'filesystem':
kwargs.update(dict(threshold=self._config('threshold', 500), ))
self.cache = FileSystemCache(self._config('dir'), **kwargs)
else:
raise RuntimeError('`%s` is not a valid cache type!' % cache_type)
app.extensions[config_prefix.lower() + '_cache'] = self.cache
def _config(self, key, default=_missing):
key = key.upper()
prior = '%s_CACHE_%s' % (self.config_prefix, key)
if prior in self.config:
return self.config[prior]
fallback = 'CACHE_%s' % key
if fallback in self.config:
return self.config[fallback]
if default is _missing:
raise RuntimeError('%s is missing.' % prior)
return default
def get(self, key):
"""Look up key in the cache and return the value for it.
:param key: the key to be looked up.
:returns: The value if it exists and is readable, else ``None``.
"""
return self.cache.get(key)
def delete(self, key):
"""Delete `key` from the cache.
:param key: the key to delete.
:returns: Whether the key existed and has been deleted.
"""
return self.cache.delete(key)
def get_many(self, *keys):
"""Returns a list of values for the given keys.
For each key a item in the list is created::
foo, bar = cache.get_many("foo", "bar")
Has the same error handling as :meth:`get`.
:param keys: The function accepts multiple keys as positional
arguments.
"""
return [self.cache.get(k) for k in keys]
def get_dict(self, *keys):
"""Like :meth:`get_many` but return a dict::
d = cache.get_dict("foo", "bar")
foo = d["foo"]
bar = d["bar"]
:param keys: The function accepts multiple keys as positional
arguments.
"""
return self.cache.get_dict(*keys)
def set(self, key, value, timeout=None):
"""Add a new key/value to the cache (overwrites value, if key already
exists in the cache).
:param key: the key to set
:param value: the value for the key
:param timeout: the cache timeout for the key in seconds (if not
specified, it uses the default timeout). A timeout of
0 idicates that the cache never expires.
:returns: ``True`` if key has been updated, ``False`` for backend
errors. Pickling errors, however, will raise a subclass of
``pickle.PickleError``.
"""
return self.cache.set(key, value, timeout)
def add(self, key, value, timeout=None):
"""Works like :meth:`set` but does not overwrite the values of already
existing keys.
:param key: the key to set
:param value: the value for the key
:param timeout: the cache timeout for the key in seconds (if not
specified, it uses the default timeout). A timeout of
0 idicates that the cache never expires.
:returns: Same as :meth:`set`, but also ``False`` for already
existing keys.
"""
return self.cache.add(key, value, timeout)
def set_many(self, mapping, timeout=None):
"""Sets multiple keys and values from a mapping.
:param mapping: a mapping with the keys/values to set.
:param timeout: the cache timeout for the key in seconds (if not
specified, it uses the default timeout). A timeout of
0 idicates that the cache never expires.
:returns: Whether all given keys have been set.
"""
return self.cache.set_many(mapping, timeout)
def delete_many(self, *keys):
"""Deletes multiple keys at once.
:param keys: The function accepts multiple keys as positional
arguments.
:returns: Whether all given keys have been deleted.
:rtype: boolean
"""
return self.cache.delete_many(*keys)
def has(self, key):
"""Checks if a key exists in the cache without returning it. This is a
cheap operation that bypasses loading the actual data on the backend.
This method is optional and may not be implemented on all caches.
:param key: the key to check
"""
return self.cache.has(key)
def clear(self):
"""Clears the cache. Keep in mind that not all caches support
completely clearing the cache.
:returns: Whether the cache has been cleared.
"""
return self.cache.clear()
def inc(self, key, delta=1):
"""Increments the value of a key by `delta`. If the key does
not yet exist it is initialized with `delta`.
For supporting caches this is an atomic operation.
:param key: the key to increment.
:param delta: the delta to add.
:returns: The new value or ``None`` for backend errors.
"""
return self.cache.inc(key, delta=delta)
def dec(self, key, delta=1):
"""Decrements the value of a key by `delta`. If the key does
not yet exist it is initialized with `-delta`.
For supporting caches this is an atomic operation.
:param key: the key to increment.
:param delta: the delta to subtract.
:returns: The new value or `None` for backend errors.
"""
return self.cache.dec(key, delta=delta)
# -*- coding: utf-8 -*-
import itertools
from datetime import datetime
import requests
import grequests
from werkzeug.contrib.cache import FileSystemCache
from .. import app
__all__ = ('get_issues', )
cache = FileSystemCache(app.config['CACHE_DIR'], default_timeout=3600)
def get_issues(org_names):
issues = cache.get('github-issues')
if issues is None:
session = _create_github_api_session()
issues = itertools.chain(*(_get_issues_for_org(session, org)
for org in org_names))
# Using grequests here to get the issues details asynchronously
# Note that it does not support session, so we set the headers manually
reqs = (grequests.get(_issue_url(issue), headers=session.headers)
for issue in issues)
issues = [_enhance_issue(response) for response in grequests.map(reqs)]
issues = sorted(issues, key=_get_issue_sort_key, reverse=True)
cache.set('github-issues', issues)
return issues
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from sys import argv, stderr, stdout
from werkzeug.contrib.cache import FileSystemCache
from progressbar import ProgressBar
from requests import get
from cStringIO import StringIO
from xml.etree.cElementTree import dump, ElementTree
from iso8601 import parse_date
from geopy import Point, distance
from math import sqrt
from cluster import HierarchicalClustering
feed_cache = FileSystemCache('feed_cache', 3600)
name_cache = FileSystemCache('name_cache', 360000)
class Event():
def __init__(self, name, uri, datetime, latitude, longitude):
self.name = name
self.uri = uri
self.datetime = datetime
self.latitude = latitude
self.longitude = longitude
def __repr__(self):
return "%s <%s>, %s @ %s, %s" % (self.name, self.uri, self.datetime, self.latitude, self.longitude)
def spatialDistance(a, b):
"""
