class QuickAuth: def __init__(self, dbfile='auth.db'): self.conn = Redis(dbfile) def register(self, key=None): if key is None: key = str(uuid1(clock_seq=int(datetime.now().timestamp()))) value = str(uuid4()) if self.conn.set(key, value): return { 'key': key, 'secret': value } else: raise ValueError('write to database failed: {} = {}'.format(key, value)) def authorize(self, key, value): c = self.conn.get(key) if c is not None: return c.decode() == value return False def update(self, key): if self.conn.get(key): return self.register(key=key) else: raise ValueError('key is not found: {}'.format(key))
def __init__(self, db_path): os.makedirs(os.path.dirname(db_path), exist_ok=True) try: if not self.conn: self.conn = Redis(db_path) except: raise Exception("Unable to create connection with Redis server")
def __init__(self): super().__init__() if config.settings.stackl_redis_type == "fake": logger.info("Using fake client") self.redis = Redis() else: self.redis = redis.Redis( host=config.settings.stackl_redis_host, port=config.settings.stackl_redis_port, password=config.settings.stackl_redis_password, db=0)
class RedisManager(): def __init__(self, remote, host, port): self.logger = logging.getLogger(__name__) self.r_instance = None self.remote = remote self.host = host self.port = port def redis_server_is_up(self): is_up = False try: c = new_redis_client(self.host, self.port) c.ping() is_up = True self.logger.debug( 'detected a redis-server instance running, so attaching to it') except redis_ex.ConnectionError: self.logger.warning('not detected a redis-server running') finally: return is_up def create_redislite_if_needed(self): # check if redis server is already running it will be checked if we dont want # remote enabled but the local redis server instance is still running and we want # things implicit and stop bothering other developers forced to kill local redis if not self.remote and self.redis_server_is_up(): raise RuntimeError( "Able to connect to redis when should be creating one!") elif self.remote and not self.redis_server_is_up(): #we asked to use an external redis, but it doesn't exist raise RuntimeError("Unable to connect to redis") if not self.remote and not self.r_instance: self.redis_db_file = tmp.mktemp(suffix='.rdb', dir='/tmp') self.r_instance = Redis(dbfilename=self.redis_db_file, serverconfig={ 'save': [], 'maxclients': 10000, 'bind': str(self.host), 'port': str(self.port) }) def __enter__(self): self.create_redislite_if_needed() return self def __exit__(self, type, value, traceback): if self.r_instance: self.r_instance.shutdown() self.r_instance = None os.remove(self.redis_db_file + '.settings')
def __init__(self, dbPath): """ Parameters __________ dbPath : str path of the redis DB """ self.conn = Redis(dbPath)
def tests(): b = Banco(lambda: Redis('/tmp/redis.db')) b.set(1, 2) assert b.get(1) == 2, "falhou em recuperar b[1]: %s" % str(b.get(1)) print("assert b.get(1) == 2", str(b.get(1))) b.set(1, 3) assert b.get(1) == 3, "falhou em recuperar b[1]: %s" % str(b.get(1)) c = b.save(4) assert b.get(c) == 4, "falhou em recuperar b[1]: %s" % str(b.get(c))
def create_redislite_if_needed(self): # check if redis server is already running it will be checked if we dont want # remote enabled but the local redis server instance is still running and we want # things implicit and stop bothering other developers forced to kill local redis if not self.remote and self.redis_server_is_up(): raise RuntimeError( "Able to connect to redis when should be creating one!") elif self.remote and not self.redis_server_is_up(): #we asked to use an external redis, but it doesn't exist raise RuntimeError("Unable to connect to redis") if not self.remote and not self.r_instance: self.redis_db_file = tmp.mktemp(suffix='.rdb', dir='/tmp') self.r_instance = Redis(dbfilename=self.redis_db_file, serverconfig={ 'save': [], 'maxclients': 10000, 'bind': str(self.host), 'port': str(self.port) })
def __init__( self, name, serializer=pickle, redis=None, max_queue_length=None, **kwargs ): self.name = name self.serializer = serializer self.max_queue_length = max_queue_length if redis: self.__redis = redis else: self.__redis = Redis(**kwargs)
def handler(self, sub_command, args): if sub_command == "watch": # We are connecting as consumer to Redis pubsub channel here. In production we don't want to expose our redis. It's best to # register client socket connection and push to socket when we get an key update event on Redis. This seems fairly complex and # out of scope for this assignment. with Redis("/tmp/redis/data.db") as conn: pubsub = conn.pubsub() pubsub.subscribe("key-update") while True: data = pubsub.get_message() data and print(data["data"]) sleep(1)
def redis(): rd = Redis() keys = set(rd.keys()) yield rd to_del = [k for k in rd.keys() if k not in keys] if to_del: rd.delete(*to_del)
class RedisClient: conn = "" CHANNEL = "key-update" pubsub = "" def __init__(self, db_path): os.makedirs(os.path.dirname(db_path), exist_ok=True) try: if not self.conn: self.conn = Redis(db_path) except: raise Exception("Unable to create connection with Redis server") def get_key(self, key): return self.conn.get(key) def set_key(self, key, value, publish_update=False): if publish_update: self.publish_update(message=json.dumps({key: value})) return self.conn.set(key, value) def remove_key(self, key): return self.conn.delete(key) def publish_update(self, channel=CHANNEL, message=""): self.conn.pubsub() self.conn.publish(channel, message) def get_update(self, channel=CHANNEL): if not self.pubsub: self.pubsub = self.conn.pubsub() self.pubsub.subscribe(channel) return self.pubsub.get_message(channel)
def create_app(config_name): app = Flask(__name__, static_url_path='') # This first loads the configuration from eg. config['development'] which corresponds to the DevelopmentConfig class in the config.py app.config.from_object(config[config_name]) # Then you can override the values with the contents of the file the OPENTARGETS_API_LOCAL_SETTINGS environment variable points to. # For eg: # $ export OPENTARGETS_API_LOCAL_SETTINGS=/path/to/settings.cfg # # where settings.cfg looks like: # # DEBUG = False # SECRET_KEY = 'foo' # app.config.from_envvar("OPENTARGETS_API_LOCAL_SETTINGS", silent=True) config[config_name].init_app(app) api_version = app.config['API_VERSION'] api_version_minor = app.config['API_VERSION_MINOR'] app.logger.info('looking for elasticsearch at: %s' % app.config['ELASTICSEARCH_URL']) app.extensions['redis-core'] = Redis(app.config['REDIS_SERVER_PATH'], db=0) #served data app.extensions['redis-service'] = Redis(app.config['REDIS_SERVER_PATH'], db=1) #cache, rate limit and internal things app.extensions['redis-user'] = Redis(app.config['REDIS_SERVER_PATH'], db=2)# user info '''setup cache''' app.extensions['redis-service'].config_set('save','') app.extensions['redis-service'].config_set('appendonly', 'no') icache = InternalCache(app.extensions['redis-service'], str(api_version_minor)) ip2org = IP2Org(icache) if app.config['ELASTICSEARCH_URL']: es = Elasticsearch(app.config['ELASTICSEARCH_URL'], # # sniff before doing anything # sniff_on_start=True, # # refresh nodes after a node fails to respond # sniff_on_connection_fail=True, # # and also every 60 seconds # sniffer_timeout=60 timeout=60 * 20, maxsize=32, ) else: es = None '''elasticsearch handlers''' app.extensions['esquery'] = esQuery(es, DataTypes(app), DataSourceScoring(app), index_data=app.config['ELASTICSEARCH_DATA_INDEX_NAME'], index_efo=app.config['ELASTICSEARCH_EFO_LABEL_INDEX_NAME'], index_eco=app.config['ELASTICSEARCH_ECO_INDEX_NAME'], index_genename=app.config['ELASTICSEARCH_GENE_NAME_INDEX_NAME'], index_expression=app.config['ELASTICSEARCH_EXPRESSION_INDEX_NAME'], index_reactome=app.config['ELASTICSEARCH_REACTOME_INDEX_NAME'], index_association=app.config['ELASTICSEARCH_DATA_ASSOCIATION_INDEX_NAME'], index_search=app.config['ELASTICSEARCH_DATA_SEARCH_INDEX_NAME'], index_relation=app.config['ELASTICSEARCH_DATA_RELATION_INDEX_NAME'], docname_data=app.config['ELASTICSEARCH_DATA_DOC_NAME'], docname_efo=app.config['ELASTICSEARCH_EFO_LABEL_DOC_NAME'], docname_eco=app.config['ELASTICSEARCH_ECO_DOC_NAME'], docname_genename=app.config['ELASTICSEARCH_GENE_NAME_DOC_NAME'], docname_expression=app.config['ELASTICSEARCH_EXPRESSION_DOC_NAME'], docname_reactome=app.config['ELASTICSEARCH_REACTOME_REACTION_DOC_NAME'], docname_association=app.config['ELASTICSEARCH_DATA_ASSOCIATION_DOC_NAME'], docname_search=app.config['ELASTICSEARCH_DATA_SEARCH_DOC_NAME'], # docname_search_target=app.config['ELASTICSEARCH_DATA_SEARCH_TARGET_DOC_NAME'], # docname_search_disease=app.config['ELASTICSEARCH_DATA_SEARCH_DISEASE_DOC_NAME'], docname_relation=app.config['ELASTICSEARCH_DATA_RELATION_DOC_NAME'], log_level=app.logger.getEffectiveLevel(), cache=icache ) app.extensions['es_access_store'] = esStore(es, eventlog_index=app.config['ELASTICSEARCH_LOG_EVENT_INDEX_NAME'], ip2org=ip2org, ) '''mixpanel handlers''' if Config.MIXPANEL_TOKEN: mp = Mixpanel(Config.MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer()) app.extensions['mixpanel']= mp app.extensions['mp_access_store'] = MixPanelStore(mp, ip2org=ip2org, ) app.extensions['proxy'] = ProxyHandler(allowed_targets=app.config['PROXY_SETTINGS']['allowed_targets'], allowed_domains=app.config['PROXY_SETTINGS']['allowed_domains'], allowed_request_domains=app.config['PROXY_SETTINGS']['allowed_request_domains']) # basepath = app.config['PUBLIC_API_BASE_PATH']+api_version # cors = CORS(app, resources=r'/api/*', allow_headers='Content-Type,Auth-Token') ''' define cache''' # cache = Cache(config={'CACHE_TYPE': 'simple'}) # cache.init_app(latest_blueprint) # latest_blueprint.cache = cache # latest_blueprint.extensions['cache'] = cache # app.cache = SimpleCache() app.cache = FileSystemCache('/tmp/cttv-rest-api-cache', threshold=100000, default_timeout=60*60, mode=777) '''Set usage limiter ''' # limiter = Limiter(global_limits=["2000 per hour", "20 per second"]) # limiter.init_app(app)# use redis to store limits '''Load api keys in redis''' rate_limit_file = app.config['USAGE_LIMIT_PATH'] if not os.path.exists(rate_limit_file): rate_limit_file = '../'+rate_limit_file csvfile = None if Config.GITHUB_AUTH_TOKEN: r = requests.get('https://api.github.com/repos/opentargets/rest_api_auth/contents/rate_limit.csv', headers = {'Authorization': 'token %s'%Config.GITHUB_AUTH_TOKEN, 'Accept': 'application/vnd.github.v3.raw'}) if r.ok: csvfile = r.text.split('\n') app.logger.info('Retrieved rate limit file from github remote') else: app.logger.warning('Cannot retrieve rate limit file from remote, SKIPPED!') elif os.path.exists(rate_limit_file): csvfile = open(rate_limit_file) app.logger.info('Using dummy rate limit file') if csvfile is None: app.logger.error('cannot find rate limit file: %s. RATE LIMIT QUOTA LOAD SKIPPED!'%rate_limit_file) else: reader = csv.DictReader(csvfile) for row in reader: auth_key = AuthKey(**row) app.extensions['redis-user'].hmset(auth_key.get_key(), auth_key.__dict__) try: csvfile.close() except: pass app.logger.info('succesfully loaded rate limit file') '''load ip name resolution''' ip_resolver = defaultdict(lambda: "PUBLIC") ip_list_file = app.config['IP_RESOLVER_LIST_PATH'] if not os.path.exists(ip_list_file): ip_list_file = '../' + ip_list_file if os.path.exists(ip_list_file): with open(ip_list_file) as csvfile: reader = csv.DictReader(csvfile) for row in reader: net = IPNetwork(row['ip']) ip_resolver[net] = row['org'] else: app.logger.warning('cannot find IP list for IP resolver. All traffic will be logged as PUBLIC') app.config['IP_RESOLVER'] = ip_resolver '''compress http response''' compress = Compress() compress.init_app(app) latest_blueprint = Blueprint('latest', __name__) current_version_blueprint = Blueprint(str(api_version), __name__) current_minor_version_blueprint = Blueprint(str(api_version_minor), __name__) specpath = '/cttv' if app.config['PROFILE'] == True: from werkzeug.contrib.profiler import ProfilerMiddleware app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30]) '''set the right prefixes''' create_api(latest_blueprint, api_version, specpath) create_api(current_version_blueprint, api_version, specpath) create_api(current_minor_version_blueprint, api_version_minor, specpath) # app.register_blueprint(latest_blueprint, url_prefix='/latest/platform') app.register_blueprint(current_version_blueprint, url_prefix='/v'+str(api_version) + '/platform') app.register_blueprint(current_minor_version_blueprint, url_prefix='/v'+str(api_version_minor) + '/platform') '''serve the static docs''' try: ''' NOTE: this file gets created only at deployment time ''' openapi_def = yaml.load(file('app/static/openapi.yaml', 'r')) app.logger.info('parsing swagger from static/openapi.yaml') except IOError: '''if we are not deployed, then simply use the template''' openapi_def = yaml.load(file('openapi.template.yaml', 'r')) app.logger.error('parsing swagger from openapi.template.yaml') with open("api-description.md", "r") as f: desc = f.read() openapi_def['info']['description'] = desc openapi_def['basePath'] = '/v%s' % str(api_version) @app.route('/v%s/platform/swagger' % str(api_version)) def serve_swagger(apiversion=api_version): return jsonify(openapi_def) @app.route('/v%s/platform/docs' % str(api_version)) def render_redoc(apiversion=api_version): return render_template('docs.html',api_version=apiversion) '''pre and post-request''' @app.before_request def before_request(): g.request_start = datetime.now() @app.after_request def after(resp): try: rate_limiter = RateLimiter() now = datetime.now() took = (now - g.request_start).total_seconds()*1000 if took > 500: cache_time = str(int(3600*took))# set cache to last one our for each second spent in the request resp.headers.add('X-Accel-Expires', cache_time) took = int(round(took)) LogApiCallWeight(took) # if took < RateLimiter.DEFAULT_CALL_WEIGHT: # took = RateLimiter.DEFAULT_CALL_WEIGHT current_values = increment_call_rate(took,rate_limiter) now = datetime.now() ceil10s=round(ceil_dt_to_future_time(now, 10),2) ceil1h=round(ceil_dt_to_future_time(now, 3600),2) usage_left_10s = rate_limiter.short_window_rate-current_values['short'] usage_left_1h = rate_limiter.long_window_rate - current_values['long'] min_ceil = ceil10s if usage_left_1h <0: min_ceil = ceil1h if (usage_left_10s < 0) or (usage_left_1h <0): resp.headers.add('Retry-After', min_ceil) resp.headers.add('X-API-Took', took) resp.headers.add('X-Usage-Limit-10s', rate_limiter.short_window_rate) resp.headers.add('X-Usage-Limit-1h', rate_limiter.long_window_rate) resp.headers.add('X-Usage-Remaining-10s', usage_left_10s) resp.headers.add('X-Usage-Remaining-1h', usage_left_1h) # resp.headers.add('X-Usage-Limit-Reset-10s', ceil10s) # resp.headers.add('X-Usage-Limit-Reset-1h', ceil1h) resp.headers.add('Access-Control-Allow-Origin', '*') resp.headers.add('Access-Control-Allow-Headers','Content-Type,Auth-Token') resp.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS') if do_not_cache(request):# do not cache in the browser resp.headers.add('Cache-Control', "no-cache, must-revalidate, max-age=0") else: resp.headers.add('Cache-Control', "no-transform, public, max-age=%i, s-maxage=%i"%(took*1800/1000, took*9000/1000)) return resp except Exception as e: app.logger.exception('failed request teardown function', str(e)) return resp # Override the HTTP exception handler. app.handle_http_exception = get_http_exception_handler(app) return app
def __init__(self, dbfile='auth.db'): self.conn = Redis(dbfile)
USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.2/howto/static-files/ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, 'static') LOGIN_REDIRECT_URL = '/' EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' #Celery-Redis settings #CELERY_BROKER_URL = 'redis://localhost:6379' # Create a Redis instance using redislite REDIS_DB_PATH = os.path.join('/tmp/redis.rdb') rdb = Redis(REDIS_DB_PATH) REDIS_SOCKET_PATH = 'redis+socket://%s' % (rdb.socket_file, ) # Use redislite for the Celery broker CELERY_BROKER_URL = REDIS_SOCKET_PATH CELERY_ACCEPT_CONTENT = ['json'] CELERY_TASK_SERIALIZER = 'json' CELERY_BEAT_SCHEDULE = { 'add_unical_pages_for_users': { 'task': 'wiki.tasks.add_unical_pages_for_users', 'schedule': 10.0 #crontab(minute = 0, hour='*/1'), }, }
def redis(): try: redis_connection = Redis("ace.rdb") yield redis_connection finally: redis_connection.close()
def _get_redis(db=JSONDB): return Redis(db)
import os from flask import Flask, render_template, request, redirect from werkzeug.utils import secure_filename from cgi import escape from flask_dropzone import Dropzone import PyPDF2 import docx2txt from tempfile import mkdtemp from shutil import rmtree import csv from redislite import Redis app = Flask(__name__) redis_connection = Redis('/tmp/redis.db') collegeDataset = open('collegeDataset.csv', 'r', encoding='utf-8', newline='\n') collegeDatasetReader = csv.reader(collegeDataset) collegeData = [] for row in collegeDatasetReader: collegeData.append(row) app.config.update( # Flask-Dropzone config: DROPZONE_ALLOWED_FILE_CUSTOM=True, DROPZONE_ALLOWED_FILE_TYPE='.pdf, .docx, .txt', DROPZONE_MAX_FILE_SIZE=3, DROPZONE_MAX_FILES=20, DROPZONE_UPLOAD_MULTIPLE = True, ) dropzone = Dropzone(app)
NOSE_ARGS = [ '--with-coverage', '--cover-package=general,project,employee', ] PAGINATION_PER_PAGE = 20 try: from .local_settings import * except ImportError: pass # CELERY REDIS_DB_PATH = os.path.join(DATA_DIR,'my_redis.db') rdb = Redis(REDIS_DB_PATH, serverconfig={'port': '1116'}) REDIS_SOCKET_PATH = 'redis+socket://%s' % (rdb.socket_file, ) BROKER_URL = REDIS_SOCKET_PATH CELERY_RESULT_BACKEND = REDIS_SOCKET_PATH CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_IMPORTS = ['general.tasks'] CELERY_TIMEZONE = 'UTC' # EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' EMAIL_USE_TLS = True
STATIC_URL = '/static/' STATICFILES_DIRS = [ os.path.join(BASE_DIR, "static"), ] SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') STATIC_TMP = os.path.join(BASE_DIR, 'static') STATIC_URL = '/static/' os.makedirs(STATIC_TMP, exist_ok=True) os.makedirs(STATIC_ROOT, exist_ok=True) LOGIN_REDIRECT_URL = '/reception' LOGIN_URL = '/admin' rdb = Redis('/tmp/redis.db', serverconfig={'port': '6379'}) CHANNEL_LAYERS = { 'default': { 'BACKEND': 'channels_redis.core.RedisChannelLayer', 'CONFIG': { "hosts": [os.environ.get('REDIS_URL', 'redis://localhost:6379')], }, }, }
def process(self): logging.basicConfig( filename='output_mongo.log', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO) r_server = Redis(dbfilename=str(REDISLITE_DB_PATH), serverconfig={ 'save': [], 'maxclients': 10000 }) mongo_client = MongoClient('host:port') db = mongo_client['xxx'] db.authenticate('uname', 'pwd') gwas23andme = db['xxxx'] no_of_workers = NO_OF_WORKERS or multiprocessing.cpu_count() mongo_q = RedisQueue(queue_id=UNIQUE_RUN_ID + '|mongo_processor_q', max_size=MAX_CHUNKS * no_of_workers, job_timeout=120) transformer_q = RedisQueue(queue_id=UNIQUE_RUN_ID + '|data_transformer_q', max_size=MAX_CHUNKS * no_of_workers, job_timeout=120) writer_q = RedisQueue(queue_id=UNIQUE_RUN_ID + '|writer_q', max_size=MAX_CHUNKS * no_of_workers, job_timeout=120) workers = [ MongoWorkerProcess( mongo_q, r_server.db, transformer_q, ) for i in range(no_of_workers) ] for w in workers: w.start() transformers = [ DataTransformerProcess( transformer_q, r_server.db, writer_q, ) for i in range(no_of_workers) ] for t in transformers: t.start() # writers = [FileWriterProcess(writer_q, # r_server.db # )for i in range(no_of_workers)] # # for writer in writers: # writer.start() writer = FileWriterProcess(writer_q, r_server.db) writer.start() logging.info('Getting distinct genes and phenotypes') distinct_phenotypes = list(gwas23andme.distinct("source")) distinct_genes = list(gwas23andme.distinct("ingenes")) logging.info('Start the real deal here!!!! ') for phenotype in distinct_phenotypes: logging.info('Processing phenotype {} '.format(phenotype)) logging.info('Total docs for phenotype {} are {}'.format( phenotype, gwas23andme.count({"source": phenotype}))) mongo_q.put((phenotype, 'test'), r_server) # for gene in distinct_genes: # mongo_q.put((phenotype,gene),r_server) mongo_q.set_submission_finished(r_server=r_server) for a in workers: a.join() for a in transformers: a.join() writer.join()
self.banco = base() def save(self, value): key = str(uuid1()) self.set(key, value) return key def get(self, key): return loads(self.banco.get(key).decode('utf8')) def set(self, key, value): self.banco.set(key, dumps(value)) return key DBT = lambda: Redis('/tmp/redis.db') def tests(): b = Banco(DBT) b.set(1, 2) assert int(b.get(1)) == 2, "falhou em recuperar b[1]: %s" % str(b.get(1)) print("assert b.get(1) == 2", str(b.get(1))) b.set(1, 3) assert int(b.get(1)) == 3, "falhou em recuperar b[1]: %s" % str(b.get(1)) c = b.save('oi maçã') assert b.get(c) == 'oi maçã', "falhou em recuperar b[1]: %s" % str(b.get(c)) if __name__ == "__main__":
__author__ = 'carlo' import os from redislite import Redis # from tinydb.storages import MemoryStorage from uuid import uuid1 # DBM = lambda :TinyDB(storage=MemoryStorage) JSONDB = os.path.dirname(__file__) + '/eica.json' DBF = lambda: Redis(JSONDB) class Banco: def __init__(self, base=DBF): self.banco = base() def save(self, value): key = str(uuid1()) self.banco.set(key, value) return key def get(self, key): return self.banco.get(key) def set(self, key, value): self.banco.set(key, value) return key def tests():
def create_app(config_name): app = Flask(__name__, static_url_path='') # This first loads the configuration from eg. config['development'] which corresponds to the DevelopmentConfig class in the config.py app.config.from_object(config[config_name]) # Then you can override the values with the contents of the file the OPENTARGETS_API_LOCAL_SETTINGS environment variable points to. # For eg: # $ export OPENTARGETS_API_LOCAL_SETTINGS=/path/to/settings.cfg # # where settings.cfg looks like: # # DEBUG = False # SECRET_KEY = 'foo' # app.config.from_envvar("OPENTARGETS_API_LOCAL_SETTINGS", silent=True) config[config_name].init_app(app) api_version = app.config['API_VERSION'] api_version_minor = app.config['API_VERSION_MINOR'] app.logger.info('looking for elasticsearch at: %s' % app.config['ELASTICSEARCH_URL']) app.extensions['redis-core'] = Redis(app.config['REDIS_SERVER_PATH'], db=0) #served data app.extensions['redis-service'] = Redis(app.config['REDIS_SERVER_PATH'], db=1) #cache, rate limit and internal things app.extensions['redis-user'] = Redis(app.config['REDIS_SERVER_PATH'], db=2)# user info '''setup cache''' app.extensions['redis-service'].config_set('save','') app.extensions['redis-service'].config_set('appendonly', 'no') icache = InternalCache(app.extensions['redis-service'], str(api_version_minor)) ip2org = IP2Org(icache) if app.config['ELASTICSEARCH_URL']: es = Elasticsearch(app.config['ELASTICSEARCH_URL'], # # sniff before doing anything # sniff_on_start=True, # # refresh nodes after a node fails to respond # sniff_on_connection_fail=True, # # and also every 60 seconds # sniffer_timeout=60 timeout=60 * 20, maxsize=32, ) else: es = None '''elasticsearch handlers''' app.extensions['esquery'] = esQuery( es, DataTypes(app), DataSourceScoring(app), index_data=app.config['ELASTICSEARCH_DATA_INDEX_NAME'], index_drug=app.config['ELASTICSEARCH_DRUG_INDEX_NAME'], index_efo=app.config['ELASTICSEARCH_EFO_LABEL_INDEX_NAME'], index_eco=app.config['ELASTICSEARCH_ECO_INDEX_NAME'], index_genename=app.config['ELASTICSEARCH_GENE_NAME_INDEX_NAME'], index_expression=app.config['ELASTICSEARCH_EXPRESSION_INDEX_NAME'], index_reactome=app.config['ELASTICSEARCH_REACTOME_INDEX_NAME'], index_association=app.config['ELASTICSEARCH_DATA_ASSOCIATION_INDEX_NAME'], index_search=app.config['ELASTICSEARCH_DATA_SEARCH_INDEX_NAME'], index_relation=app.config['ELASTICSEARCH_DATA_RELATION_INDEX_NAME'], docname_data=app.config['ELASTICSEARCH_DATA_DOC_NAME'], docname_drug=app.config['ELASTICSEARCH_DRUG_DOC_NAME'], docname_efo=app.config['ELASTICSEARCH_EFO_LABEL_DOC_NAME'], docname_eco=app.config['ELASTICSEARCH_ECO_DOC_NAME'], docname_genename=app.config['ELASTICSEARCH_GENE_NAME_DOC_NAME'], docname_expression=app.config['ELASTICSEARCH_EXPRESSION_DOC_NAME'], docname_reactome=app.config['ELASTICSEARCH_REACTOME_REACTION_DOC_NAME'], docname_association=app.config['ELASTICSEARCH_DATA_ASSOCIATION_DOC_NAME'], docname_search=app.config['ELASTICSEARCH_DATA_SEARCH_DOC_NAME'], # docname_search_target=app.config['ELASTICSEARCH_DATA_SEARCH_TARGET_DOC_NAME'], # docname_search_disease=app.config['ELASTICSEARCH_DATA_SEARCH_DISEASE_DOC_NAME'], docname_relation=app.config['ELASTICSEARCH_DATA_RELATION_DOC_NAME'], log_level=app.logger.getEffectiveLevel(), cache=icache ) app.extensions['es_access_store'] = esStore(es, eventlog_index=app.config['ELASTICSEARCH_LOG_EVENT_INDEX_NAME'], ip2org=ip2org, ) '''mixpanel handlers''' if Config.MIXPANEL_TOKEN: mp = Mixpanel(Config.MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer()) app.extensions['mixpanel']= mp app.extensions['mp_access_store'] = MixPanelStore( mp, ip2org=ip2org, ) app.extensions['proxy'] = ProxyHandler( allowed_targets=app.config['PROXY_SETTINGS']['allowed_targets'], allowed_domains=app.config['PROXY_SETTINGS']['allowed_domains'], allowed_request_domains=app.config['PROXY_SETTINGS']['allowed_request_domains']) # basepath = app.config['PUBLIC_API_BASE_PATH']+api_version # cors = CORS(app, resources=r'/api/*', allow_headers='Content-Type,Auth-Token') ''' define cache''' # cache = Cache(config={'CACHE_TYPE': 'simple'}) # cache.init_app(latest_blueprint) # latest_blueprint.cache = cache # latest_blueprint.extensions['cache'] = cache # app.cache = SimpleCache() app.cache = FileSystemCache('/tmp/cttv-rest-api-cache', threshold=100000, default_timeout=60*60, mode=777) '''load ip name resolution''' ip_resolver = defaultdict(lambda: "PUBLIC") ip_list_file = app.config['IP_RESOLVER_LIST_PATH'] if not os.path.exists(ip_list_file): ip_list_file = '../' + ip_list_file if os.path.exists(ip_list_file): with open(ip_list_file) as csvfile: reader = csv.DictReader(csvfile) for row in reader: net = IPNetwork(row['ip']) ip_resolver[net] = row['org'] else: app.logger.warning('cannot find IP list for IP resolver. All traffic will be logged as PUBLIC') app.config['IP_RESOLVER'] = ip_resolver '''compress http response''' compress = Compress() compress.init_app(app) latest_blueprint = Blueprint('latest', __name__) current_version_blueprint = Blueprint(str(api_version), __name__) current_minor_version_blueprint = Blueprint(str(api_version_minor), __name__) specpath = '/cttv' if app.config['PROFILE'] == True: from werkzeug.contrib.profiler import ProfilerMiddleware app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30]) '''set the right prefixes''' create_api(latest_blueprint, api_version, specpath) create_api(current_version_blueprint, api_version, specpath) create_api(current_minor_version_blueprint, api_version_minor, specpath) # app.register_blueprint(latest_blueprint, url_prefix='/latest/platform') app.register_blueprint(current_version_blueprint, url_prefix='/v'+str(api_version) + '/platform') app.register_blueprint(current_minor_version_blueprint, url_prefix='/v'+str(api_version_minor) + '/platform') '''serve the static docs''' openapi_def = yaml.load(file('app/static/openapi.template.yaml', 'r')) app.logger.info('parsing swagger from app/static/openapi.template.yaml') #inject the description into the docs with open("api-description.md", "r") as f: desc = f.read() openapi_def['info']['description'] = desc openapi_def['basePath'] = '/v%s' % str(api_version) @app.route('/v%s/platform/swagger' % str(api_version)) def serve_swagger(apiversion=api_version): return jsonify(openapi_def) @app.route('/v%s/platform/docs/swagger-ui' % str(api_version)) def render_swaggerui(apiversion=api_version): return render_template('swaggerui.html',api_version=apiversion) '''pre and post-request''' @app.before_request def before_request(): g.request_start = datetime.now() @app.after_request def after(resp): try: now = datetime.now() took = int(round((now - g.request_start).total_seconds())) resp.headers.add('Access-Control-Allow-Origin', '*') resp.headers.add('Access-Control-Allow-Headers','Content-Type,Auth-Token') resp.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS') if do_not_cache(request):# do not cache in the browser resp.headers.add('Cache-Control', "no-cache, must-revalidate, max-age=0") else: cache = 30 * 24 * 60 * 60 #cache for seven days resp.headers.add('Cache-Control', "no-transform, max-age=%i"%(cache)) return resp except Exception as e: app.logger.exception('failed request teardown function', str(e)) return resp # Override the HTTP exception handler. app.handle_http_exception = get_http_exception_handler(app) return app
from redislite import Redis from redislite import patch from settings import REDIS_PORT # Original intention was to use Redis for caching and task queue. # But I found out that the server has both memcached and gearman installed, # so this is no longer needed, as it will require installing another server. # Also, since each process importing this file will try to launch its own # instance of Redis on the same local port, this will be a problem once worker # processes are started. patch.patch_redis() redis = Redis(serverconfig={'port': REDIS_PORT})
# ================================================================================ CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache' } } # ================================================================================ # static files (CSS, JavaScript, Images) # # References: # * https://docs.djangoproject.com/en/1.9/howto/static-files/ # ================================================================================ STATIC_ROOT = os.path.join(BASE_DIR, 'temporary', 'static') # ================================================================================ # Celery # # References: # * http://docs.celeryproject.org/en/master/getting-started/index.html # ================================================================================ CELERY_REDIS_PORT = os.environ.get('CELERY_REDIS_PORT', '8001') CELERY_REDIS = Redis(os.path.join(BASE_DIR, 'temporary', 'celery.redis'), serverconfig={'port': CELERY_REDIS_PORT}) BROKER_URL = 'redis://127.0.0.1:{}/0'.format(CELERY_REDIS_PORT) CELERY_RESULT_BACKEND = BROKER_URL
UI_CHANNEL: ui_data_handler, TTS_CHANNEL: tts_data_handler, LEDS_CHANNEL: leds_data_handler } channels = list(handlers.keys()) # Main Listener Event Handler def main_event_handler(self, event): print("Logic main event handler called") handlers[event['channel']](self.redis, event['data']) if __name__ == '__main__': print("Channel logic started") redis_object = Redis(REDIS_DB_FILE) listener = RedisListener('Logic Listener', redis_object, channels, main_event_handler) listener.start() time.sleep(10) print('Pushing to ui channel') redis_object.publish(UI_CHANNEL, 'Test message!') time.sleep(5) print('Pushing to TTS channel') redis_object.publish(TTS_CHANNEL, 'Test speech message') time.sleep(10) print('Pushing to ui channel again') redis_object.publish(UI_CHANNEL, 'Another test message!') # emotions & speech to text need pubsub objects # ui & text to speech & leds need to subscribe & publish
class HotQueue(object): """Simple FIFO message queue stored in a Redis list. Parameters ---------- name : str name of the queue max_queue_length : int Maximum length the queue can grow to (default is None allows the queue to grow without any limits. serializer : class, module, optional the class or module to serialize msgs with, must have methods or functions named ``dumps`` and ``loads``, `pickle <http://docs.python.org/library/pickle.html>`_ is the default, use ``None`` to store messages in plain text (suitable for strings, integers, etc) redis : redis.Redis, redislite.Redis, optional redis connection object, defaults to redislite.Redis with fallback to redis.Redis. **kwargs Additional kwargs to pass to :class:`redislite.Redis`, most commonly :attr:`dbfilename`. Examples -------- >>> from hotqueue import HotQueue >>> queue = HotQueue("myqueue", dbfilename="queue.rdb") """ def __init__( self, name, serializer=pickle, redis=None, max_queue_length=None, **kwargs ): self.name = name self.serializer = serializer self.max_queue_length = max_queue_length if redis: self.__redis = redis else: self.__redis = Redis(**kwargs) def __len__(self): return self.__redis.llen(self.key) @property def key(self): """ Key in Redis to store the queue Returns ------- str The name of the key containing the queue in redis. """ return key_for_name(self.name) def clear(self): """ Clear the queue of all messages, by deleting the Redis key. """ self.__redis.delete(self.key) def consume(self, **kwargs): """ A blocking generator that yields whenever a message is waiting in the queue. Parameters ---------- **kwargs any arguments that :meth:`~hotqueue.HotQueue.get` can accept (:attr:`block` will default to ``True`` if not given) Yields ------ object The deserialized object from the queue. Examples -------- >>> queue = HotQueue("example") >>> for msg in queue.consume(timeout=1): ... print(msg) my message another message """ kwargs.setdefault('block', True) try: while True: msg = self.get(**kwargs) if msg is None: break yield msg except KeyboardInterrupt: # pragma: no cover print() return def get(self, block=False, timeout=None): """ Get a message from the queue. Parameters ---------- block : bool whether or not to wait until a msg is available in the queue before returning; ``False`` by default timeout : int When using :attr:`block`, if no msg is available for :attr:`timeout` in seconds, give up and return Returns ------- object The deserialized object from the queue. Examples -------- >>> queue.get() 'my message' >>> queue.get() 'another message' """ if block: if timeout is None: timeout = 0 msg = self.__redis.blpop(self.key, timeout=timeout) if msg is not None: msg = msg[1] else: msg = self.__redis.lpop(self.key) if msg is not None and self.serializer is not None: msg = self.serializer.loads(msg) if isinstance(msg, bytes): msg = msg.decode() return msg def put(self, *msgs): """Put one or more messages onto the queue. Example: >>> queue.put("my message") >>> queue.put("another message") To put messages onto the queue in bulk, which can be significantly faster if you have a large number of messages: >>> queue.put("my message", "another message", "third message") """ if self.serializer is not None: msgs = [self.serializer.dumps(m) for m in msgs] self.__redis.rpush(self.key, *msgs) if self.max_queue_length: self.__redis.ltrim(self.key, 0, int(self.max_queue_length) - 1) def worker(self, *args, **kwargs): """Decorator for using a function as a queue worker. Example: >>> @queue.worker(timeout=1) ... def printer(msg): ... print(msg) >>> printer() my message another message You can also use it without passing any keyword arguments: >>> @queue.worker ... def printer(msg): ... print(msg) >>> printer() my message another message :param kwargs: any arguments that :meth:`~hotqueue.HotQueue.get` can accept (:attr:`block` will default to ``True`` if not given) """ def decorator(worker): """ Worker decorator :param worker: :return: """ @wraps(worker) def wrapper(*args): """ Inner wrapper :param args: :return: """ for msg in self.consume(**kwargs): worker(*args + (msg,)) return wrapper if args: return decorator(*args) return decorator
def __call__(self): redis_instance = Redis('/tmp/redis.db') return redis_instance
from redislite import Redis from utils.url_utils import get_filtered_links redis_client = Redis(dbfilename="./redis.db", decode_responses=True) def redis_cleanup(website_full_url): """removed invalid entries from redis caches""" # remove intersections for anchor in redis_client.sinter("new_urls", "processed_urls"): redis_client.srem("new_urls", anchor) print("Removed processed URL from redis: {}!\n".format(anchor)) for anchor in redis_client.smembers("new_urls"): if len(get_filtered_links([anchor], website_full_url)) < 1: redis_client.srem("new_urls", anchor) if __name__ == '__main__': redis_cleanup("https://www.wikipedia.org/") pass
class RedisStore(DataStore): """Implementation of Redis datastore""" def __init__(self): super().__init__() if config.settings.stackl_redis_type == "fake": logger.info("Using fake client") self.redis = Redis() else: self.redis = redis.Redis( host=config.settings.stackl_redis_host, port=config.settings.stackl_redis_port, password=config.settings.stackl_redis_password, db=0) def get(self, **keys): """Gets a document from a redis instance""" document_key = keys.get("category") + '/' + keys.get( "type") + '/' + keys.get("name") logger.debug(f"[RedisStore] get on key '{document_key}'") redis_value = self.redis.get(document_key) if redis_value is None: response = self._create_store_response( status_code=StatusCode.NOT_FOUND, content={}) else: content = json.loads(self.redis.get(document_key)) response = self._create_store_response(status_code=StatusCode.OK, content=content) logger.debug(f"[RedisStore] StoreResponse for get: {response}") return response def get_all(self, category, document_type, wildcard_prefix=""): """Gets all documents of a type from a Redis""" document_key = f"{category}/{document_type}/{wildcard_prefix}*" logger.debug( f"[RedisStore] get_all in '{document_key}' for type '{document_type}'" ) content = [] for key in self.redis.scan_iter(document_key): content.append(json.loads(self.redis.get(key))) response = self._create_store_response(status_code=StatusCode.OK, content=content) logger.debug(f"[RedisStore] StoreResponse for get: {response}") return response def get_history(self, category, document_type, name): """Gets the snapshots of document from Redis""" document_key = category + '/' + document_type + '/' + name logger.debug( f"[RedisStore] get_history in '{document_key}' for type '{document_type}'" ) content = [] for key in self.redis.scan_iter(document_key): content.append(json.loads(self.redis.get(key))) response = self._create_store_response(status_code=StatusCode.OK, content=content) logger.debug(f"[RedisStore] StoreResponse for get: {response}") return response def put(self, file): """Puts a document in Redis""" document_key = file.get("category") + '/' + file.get( "type") + '/' + file["name"] logger.debug(f"[RedisStore] put on '{document_key}' with file {file}") self.redis.set(document_key, json.dumps(file)) response = self._create_store_response( status_code=StatusCode.CREATED, content=json.loads(self.redis.get(document_key))) logger.debug(f"[RedisStore] StoreResponse for put: {response}") return response def delete(self, **keys): """Deletes a document in Redis""" document_key = keys.get("category") + '/' + keys.get( "type") + '/' + keys.get("name") self.redis.delete(document_key) response = self._create_store_response(status_code=200, content={}) logger.debug(f"[RedisStore] StoreResponse for delete: {response}") return response
from multiprocessing import Pool from redislite import Redis from piston.steem import Steem from piston.blockchain import Blockchain from django.core.management.base import BaseCommand from django.db.utils import InterfaceError from django import db from backend import settings from apps.blockchains.sync import BaseUpdater logger = logging.getLogger('mapala.fetch') pool = Pool(processes=4) redis = Redis('redis.db') def get_block(steem, blockchain_name): last_block = redis.get('%s_last_block' % blockchain_name) if last_block is None: # Fitst app fetch last_block = Blockchain(steem).get_current_block_num() print(last_block, settings.LOCALE) return int(last_block) class Command(BaseCommand): def add_arguments(self, parser):
def create_app(config_name): app = Flask(__name__, static_url_path='') app.config.from_object(config[config_name]) app.config.from_envvar("OPENTARGETS_API_LOCAL_SETTINGS", silent=True) config[config_name].init_app(app) api_version = app.config['API_VERSION'] api_version_minor = app.config['API_VERSION_MINOR'] # log_level = logging.INFO # if app.config['DEBUG']: # log_level = logging.DEBUG # Flask has a default logger which works well and pushes to stderr # if you want to add different handlers (to file, or logstash, or whatever) # you can use code similar to the one below and set the error level accordingly. # logHandler = logging.StreamHandler() # formatter = jsonlogger.JsonFormatter() # logHandler.setFormatter(formatter) # loghandler.setLevel(logging.INFO) # app.logger.addHandler(logHandler) # or for LOGSTASH # app.logger.addHandler(logstash.LogstashHandler(app.config['LOGSTASH_HOST'], app.config['LOGSTASH_PORT'], version=1)) app.logger.info('looking for elasticsearch at: %s' % app.config['ELASTICSEARCH_URL']) print('looking for elasticsearch at: %s' % app.config['ELASTICSEARCH_URL']) app.extensions['redis-core'] = Redis(app.config['REDIS_SERVER_PATH'], db=0) #served data app.extensions['redis-service'] = Redis( app.config['REDIS_SERVER_PATH'], db=1) #cache, rate limit and internal things app.extensions['redis-user'] = Redis(app.config['REDIS_SERVER_PATH'], db=2) # user info '''setup cache''' app.extensions['redis-service'].config_set('save', '') app.extensions['redis-service'].config_set('appendonly', 'no') icache = InternalCache(app.extensions['redis-service'], str(api_version_minor)) ip2org = IP2Org(icache) es = Elasticsearch( app.config['ELASTICSEARCH_URL'], # # sniff before doing anything # sniff_on_start=True, # # refresh nodes after a node fails to respond # sniff_on_connection_fail=True, # # and also every 60 seconds # sniffer_timeout=60 timeout=60 * 20, maxsize=100, ) '''elasticsearch handlers''' app.extensions['esquery'] = esQuery( es, DataTypes(app), DataSourceScoring(app), index_data=app.config['ELASTICSEARCH_DATA_INDEX_NAME'], index_efo=app.config['ELASTICSEARCH_EFO_LABEL_INDEX_NAME'], index_eco=app.config['ELASTICSEARCH_ECO_INDEX_NAME'], index_genename=app.config['ELASTICSEARCH_GENE_NAME_INDEX_NAME'], index_expression=app.config['ELASTICSEARCH_EXPRESSION_INDEX_NAME'], index_reactome=app.config['ELASTICSEARCH_REACTOME_INDEX_NAME'], index_association=app. config['ELASTICSEARCH_DATA_ASSOCIATION_INDEX_NAME'], index_search=app.config['ELASTICSEARCH_DATA_SEARCH_INDEX_NAME'], index_relation=app.config['ELASTICSEARCH_DATA_RELATION_INDEX_NAME'], docname_data=app.config['ELASTICSEARCH_DATA_DOC_NAME'], docname_efo=app.config['ELASTICSEARCH_EFO_LABEL_DOC_NAME'], docname_eco=app.config['ELASTICSEARCH_ECO_DOC_NAME'], docname_genename=app.config['ELASTICSEARCH_GENE_NAME_DOC_NAME'], docname_expression=app.config['ELASTICSEARCH_EXPRESSION_DOC_NAME'], docname_reactome=app. config['ELASTICSEARCH_REACTOME_REACTION_DOC_NAME'], docname_association=app. config['ELASTICSEARCH_DATA_ASSOCIATION_DOC_NAME'], docname_search=app.config['ELASTICSEARCH_DATA_SEARCH_DOC_NAME'], # docname_search_target=app.config['ELASTICSEARCH_DATA_SEARCH_TARGET_DOC_NAME'], # docname_search_disease=app.config['ELASTICSEARCH_DATA_SEARCH_DISEASE_DOC_NAME'], docname_relation=app.config['ELASTICSEARCH_DATA_RELATION_DOC_NAME'], log_level=app.logger.getEffectiveLevel(), cache=icache) app.extensions['es_access_store'] = esStore( es, eventlog_index=app.config['ELASTICSEARCH_LOG_EVENT_INDEX_NAME'], ip2org=ip2org, ) '''mixpanel handlers''' if Config.MIXPANEL_TOKEN: mp = Mixpanel(Config.MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer()) app.extensions['mixpanel'] = mp app.extensions['mp_access_store'] = MixPanelStore( mp, ip2org=ip2org, ) app.extensions['proxy'] = ProxyHandler( allowed_targets=app.config['PROXY_SETTINGS']['allowed_targets'], allowed_domains=app.config['PROXY_SETTINGS']['allowed_domains'], allowed_request_domains=app.config['PROXY_SETTINGS'] ['allowed_request_domains']) basepath = app.config['PUBLIC_API_BASE_PATH'] + api_version # cors = CORS(app, resources=r'/api/*', allow_headers='Content-Type,Auth-Token') ''' define cache''' # cache = Cache(config={'CACHE_TYPE': 'simple'}) # cache.init_app(latest_blueprint) # latest_blueprint.cache = cache # latest_blueprint.extensions['cache'] = cache # app.cache = SimpleCache() app.cache = FileSystemCache('/tmp/cttv-rest-api-cache', threshold=100000, default_timeout=60 * 60, mode=777) '''Set usage limiter ''' # limiter = Limiter(global_limits=["2000 per hour", "20 per second"]) # limiter.init_app(app)# use redis to store limits '''Load api keys in redis''' rate_limit_file = app.config['USAGE_LIMIT_PATH'] if not os.path.exists(rate_limit_file): rate_limit_file = '../' + rate_limit_file if os.path.exists(rate_limit_file): with open(rate_limit_file) as csvfile: reader = csv.DictReader(csvfile) for row in reader: auth_key = AuthKey(**row) app.extensions['redis-user'].hmset(auth_key.get_key(), auth_key.__dict__) print('INFO - succesfully loaded rate limit file') else: print('ERROR - cannot find rate limit file') app.logger.error( 'cannot find rate limit file: %s. RATE LIMIT QUOTA LOAD SKIPPED!' % rate_limit_file) '''load ip name resolution''' ip_resolver = defaultdict(lambda: "PUBLIC") ip_list_file = app.config['IP_RESOLVER_LIST_PATH'] if not os.path.exists(ip_list_file): ip_list_file = '../' + ip_list_file if os.path.exists(ip_list_file): with open(ip_list_file) as csvfile: reader = csv.DictReader(csvfile) for row in reader: net = IPNetwork(row['ip']) ip_resolver[net] = row['org'] else: app.logger.warning( 'cannot find IP list for IP resolver. All traffic will be logged as PUBLIC' ) app.config['IP_RESOLVER'] = ip_resolver '''compress http response''' compress = Compress() compress.init_app(app) latest_blueprint = Blueprint('latest', __name__) current_version_blueprint = Blueprint(str(api_version), __name__) current_minor_version_blueprint = Blueprint(str(api_version_minor), __name__) specpath = '/cttv' if app.config['PROFILE'] == True: from werkzeug.contrib.profiler import ProfilerMiddleware app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30]) create_api(latest_blueprint, api_version, specpath) create_api(current_version_blueprint, api_version, specpath) create_api(current_minor_version_blueprint, api_version_minor, specpath) app.register_blueprint(latest_blueprint, url_prefix='/api/latest') app.register_blueprint(current_version_blueprint, url_prefix='/api/' + str(api_version)) app.register_blueprint(current_minor_version_blueprint, url_prefix='/api/' + str(api_version_minor)) @app.route('/api-docs/%s' % str(api_version_minor)) def docs_current_minor_version(): return redirect('/api/swagger/index.html') @app.route('/api-docs/%s' % str(api_version)) def docs_current_version(): return redirect('/api/swagger/index.html') @app.route('/api-docs') def docs(): return redirect('/api/swagger/index.html') def serve_swagger(): return app.send_static_file('docs/swagger/swagger.yaml') @app.route('/api/docs/swagger.yaml') def send_swagger(): return serve_swagger() @app.route('/api/latest/docs/swagger.yaml') def send_swagger_latest(): return serve_swagger() @app.route('/api/' + str(api_version) + '/docs/swagger.yaml') def send_swagger_current_cersion(): return serve_swagger() @app.before_request def before_request(): g.request_start = datetime.now() @app.after_request def after(resp): rate_limiter = RateLimiter() now = datetime.now() took = (now - g.request_start).total_seconds() * 1000 if took > 500: cache_time = str( int(3600 * took) ) # set cache to last one our for each second spent in the request resp.headers.add('X-Accel-Expires', cache_time) took = int(round(took)) LogApiCallWeight(took) # if took < RateLimiter.DEFAULT_CALL_WEIGHT: # took = RateLimiter.DEFAULT_CALL_WEIGHT current_values = increment_call_rate(took, rate_limiter) now = datetime.now() ceil10s = round(ceil_dt_to_future_time(now, 10), 2) ceil1h = round(ceil_dt_to_future_time(now, 3600), 2) usage_left_10s = rate_limiter.short_window_rate - current_values[ 'short'] usage_left_1h = rate_limiter.long_window_rate - current_values['long'] min_ceil = ceil10s if usage_left_1h < 0: min_ceil = ceil1h if (usage_left_10s < 0) or (usage_left_1h < 0): resp.headers.add('Retry-After', min_ceil) resp.headers.add('X-API-Took', took) resp.headers.add('X-Usage-Limit-10s', rate_limiter.short_window_rate) resp.headers.add('X-Usage-Limit-1h', rate_limiter.long_window_rate) resp.headers.add('X-Usage-Remaining-10s', usage_left_10s) resp.headers.add('X-Usage-Remaining-1h', usage_left_1h) # resp.headers.add('X-Usage-Limit-Reset-10s', ceil10s) # resp.headers.add('X-Usage-Limit-Reset-1h', ceil1h) resp.headers.add('Access-Control-Allow-Origin', '*') resp.headers.add('Access-Control-Allow-Headers', 'Content-Type,Auth-Token') if do_not_cache(request): # do not cache in the browser resp.headers.add('Cache-Control', "no-cache, must-revalidate, max-age=0") else: resp.headers.add( 'Cache-Control', "no-transform, public, max-age=%i, s-maxage=%i" % (took * 1800 / 1000, took * 9000 / 1000)) return resp return app