class Cache(object): timeout = 604800 #week cache = None def __init__(self, timeout=None): self.timeout = timeout or self.timeout self.cache = SimpleCache() def __call__(self, f): @wraps(f) def decorator(*args, **kwargs): key = request.data or request.path response = self.cache.get(key) if response is None: response = f(*args, **kwargs) self.cache.set(key, response, self.timeout) return response return decorator def get(self, key): return self.cache.get(key) def set(self, key, val): return self.cache.set(key, val, self.timeout)
def test_simplecache_set_many(): """Make sure set_many works with sequences as well as dicts""" cache = SimpleCache() cache.set_many({0: 0, 1: 1, 2: 4}) assert cache.get(2) == 4 cache.set_many((i, i*i) for i in xrange(3)) assert cache.get(2) == 4
class HybridCache(object): """ A hybrid cache class that provide in-process cache that is backup up by out-of-process cache When setting a key it will set it in both in-process and out-of-process When getting a key it will try to retrieve first from in-process and if not, from out-of-process """ def __init__(self, remote_addresses): self.in_proc_cache = SimpleCache(1000, 3600) #todo - pass these are arguments self.remote_addresses = remote_addresses self.out_proc_cache = MemcachedCache(remote_addresses) def get(self, key): """ get an item from the hybrid cache first in the in-process and then in the out-of-process in case the key does not exist in both return None if the value exist in out of process but not in-process then it is added """ val = self.in_proc_cache.get(key) if val is None: val = MemcachedCache(self.remote_addresses).get(key) if val is not None: self.in_proc_cache.add(key, val, None) #todo: for how long to cache? return val def add(self, key, value, timeout = 300): """ store a key-value in the hybrid cache - both in and out-off process """ #self.out_proc_cache.add(key, value, timeout = timeout) MemcachedCache(self.remote_addresses).add(key, value, timeout) self.in_proc_cache.add(key, value, timeout = timeout)
class ImageSimpleCache(ImageCache): """Simple image cache.""" def __init__(self): """Initialize the cache.""" super(ImageSimpleCache, self).__init__() self.cache = SimpleCache() def get(self, key): """Return the key value. :param key: the object's key :return: the stored object :rtype: `BytesIO` object """ return self.cache.get(key) def set(self, key, value, timeout=None): """Cache the object. :param key: the object's key :param value: the stored object :type value: `BytesIO` object :param timeout: the cache timeout in seconds """ timeout = timeout if timeout else self.timeout self.cache.set(key, value, timeout) def delete(self, key): """Delete the specific key.""" self.cache.delete(key) def flush(self): """Flush the cache.""" self.cache.clear()
class JanuaCriticalHandler(Handler): def __init__(self, email, queue): Handler.__init__(self) self.cache = SimpleCache() self.mailobj = MailObj() self.mailobj.template = 'critical_report' self.mailobj.to = email self.queue = queue def emit(self, record): # to avoid spam if its the same error notify only every hour log = '%s:%s' % (record.pathname, record.lineno) cached_log = self.cache.get(log) if not cached_log: self.cache.set(log, log, timeout=3600) try: self.mailobj.template_args = { 'levelname': record.levelname, 'pathname': record.pathname, 'lineno': record.lineno, 'module': record.module, 'funcName': record.funcName, 'asctime': record.asctime, 'message': record.message } self.queue.put(self.mailobj) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record)
def getImagePaths(self, create=True): cache = SimpleCache() if not create: rv = cache.get('image-paths') if rv is not None: return rv try: with MySQL.conn(MySQL) as cursor: categories = {} sql = "SELECT id, category_id, alias FROM `categories` WHERE id > 0 ORDER BY id ASC" cursor.execute(sql) for row in cursor: idx = row['id'] category_id = row['category_id'] alias = row['alias'] if category_id == 0: categories[idx] = alias else: categories[idx] = categories[category_id] + '/' + alias if create: if not os.path.exists(Config.product_image_path() + '/' + categories[idx]): os.makedirs(Config.product_image_path() + '/' + categories[idx]) cache.set('image-paths', categories, timeout=60 * 60 * 24 * 365) return categories except: return False
class Metrics(object): def __init__(self): self._cache = SimpleCache() self._metrics = [] def build_keyspace(self, fields): """ Given: ["one", "two", "three"] Yield: "one", "one.two", "one.two.three" """ current = set() for field in fields: current.add(field) yield ".".join(current) def index(self): if not self._metrics: r = requests.get("%s/metrics/index.json" % graphite_url) self._metrics = json.loads(r.text) def search(self, term): cache_key = term rv = self._cache.get(cache_key) if not rv: rv = [metric for metric in self._metrics if term in metric] self._cache.set(cache_key, rv, timeout=86400) return rv
def find(self, query, index): from flask import g from werkzeug.contrib.cache import SimpleCache cache = SimpleCache() CACHE_TIMEOUT = 86400 index = cache.get('bokbok:metrics_index') if not cache.get('bokbok:metrics_list'): metrics_list = list(g.redis.smembers('bokbok:metrics_cache')) cache.set('bokbok:metrics_list', metrics_list, CACHE_TIMEOUT) if not index: index = self.index(metrics_list) cache.set('bokbok:metrics_index', index, CACHE_TIMEOUT) return [metric for metric in sorted(index) if query in metric]
class _UserSessions(object): def __init__(self): self.cache = SimpleCache() def create(self, user_id): user = User.find(User.id == user_id) if user is None: return None sess = os.urandom(24) self.cache.set(sess, user_id) session['key'] = sess return sess def get(self): if 'key' not in session: return None key = session['key'] user_id = self.cache.get(key) user = User.find(User.id == user_id) session['user'] = user return user def delete(self): if 'key' in session: self.cache.delete(session['key']) session.pop('key', None) session.pop('user', None)
def getPage(page): assert isinstance(page, str) urls = [] cache = SimpleCache() cached_object = cache.get(page) if cached_object is not None: logging.info("Using cached result for page %s." % page) title, urls = cached_object return title, urls logging.info("Page %s not found in cache" % page) title, content = getRawPage(page) if title: title = urldefrag(unquote(title))[0] title = title.replace("_", " ") if not content: cache.set(page, (title, None), timeout=5 * 60) return title, None soup = BeautifulSoup(content, 'html.parser') for paragraph in soup.find_all('p', recursive=False): for link in paragraph.find_all('a'): url = link.get('href') if url.startswith("/wiki/"): url = url[6:] url = unquote(url) urls.append(url) urls = list(set(urls)) urls = urls[:10] cache.set(page, (title, urls), timeout=5 * 60) return title, urls
class __Store: def __init__(self): ''' Cache structure { 'reviews': { <id>: <review> }, 'word_to_review_ids': { <word>: <list_of_review_ids> } } ''' self.cache = SimpleCache() def _get_reviews(self): return self.cache.get('reviews') or {} def get_review_by_id(self, id): return Review(**self._get_reviews().get(id)) def _get_word_to_review_ids(self): return self.cache.get('word_to_review_ids') or {} def get_review_ids_by_word(self, word): return self._get_word_to_review_ids().get(word) def add_reviews(self, reviews): ''' :param reviews: { <review_id>: Review object, ... } :return: ''' reviews = {review_id: review.to_dict() for review_id, review in reviews.items()} self.cache.set('reviews', reviews) def add_word_to_review_ids(self, word_to_review_ids): ''' :param word_to_review_ids: { <word>: <list_of_review_ids>. ... } :return: ''' self.cache.set('word_to_review_ids', word_to_review_ids)
def get_my_item(todate): cache = SimpleCache() cache1 = SimpleCache() rv = cache.get('my-item') rv1 = cache1.get('todate') if rv1 != todate: print("not equal") if rv is None and rv1 != todate: data = bl.get_val(todate) #print(data) js = json.dumps(data, default=date_to_string, indent=4) rv = js rv1 = todate cache.set('my-item', rv, timeout=5 * 60) cache1.set('todate', rv1, timeout=5 * 60) #print("in" + rv1) return rv
class ZopeTemplateLoader(jinja2.BaseLoader): def __init__(self, parent_loader, base_path, cache_templates=True, template_list=[]): self.parent_loader = parent_loader self.cache = SimpleCache() self.cache_templates = cache_templates self.path = base_path self.template_list = template_list def get_source(self, environment, template): def passthrough(cachable=True): up = self.parent_loader source, path, uptodate = up.get_source(environment, template) if not cachable: uptodate = lambda: False return source, path, uptodate if not template in self.template_list: return passthrough() path = "%s%s" % (self.path, template) source = self.cache.get(path) if not source: try: response = requests.get(path) except requests.exceptions.ConnectionError: return passthrough(cachable=False) if response.status_code != 200: return passthrough(cachable=False) # escape jinja tags source = response.text source = source.strip() source = source.replace("{%", "{{ '{%' }}").replace("%}", "{{ '%}' }}") source = source.replace("{{", "{{ '{{' }}").replace("}}", "{{ '}}' }}") # template blocks source = source.replace( "<!-- block_content -->", "{% block natura2000_content %}{% endblock %}") source = source.replace("<!-- block_head -->", "{% block head %}{% endblock %}") # fix breadcrumb link source = source.replace( '"%s"' % self.path, '"%s"' % flask.url_for('naturasites.index')) if self.cache_templates: self.cache.set(path, source) return source, path, lambda: False
class PostalBot(TelegramBot): STR_USERNAME = '******' STR_MESSAGE_SENT = "Message sent" SPAM_TIMEOUT = 10*60 def __init__(self, tg_api_key, tg_lounge_id): super(PostalBot, self).__init__(tg_api_key, tg_lounge_id) self.cache = SimpleCache() def handle_stream_publish(self, data): keys = data.keys() if 'watch_url' in keys and 'username' in keys: c_key = self.STR_USERNAME + ':' + data['username'] if not self.cache.get(c_key): message = '{username} went live on Postal\n{url}'.format( username=data['username'], url=data['watch_url'] ) self.send_tg_message(self.tg_lounge_id, message) self.cache.set(c_key, True, timeout=self.SPAM_TIMEOUT) return self.STR_MESSAGE_SENT return '' def handle_postal_new_post(self, data): keys = data.keys() if 'username' in keys and 'title' in keys: message = "New post by {username}\n{title}".format( username=data['username'], title=data['title'] ) if 'image_url' in keys and 'image_size' in keys: message += "\n{url} {size}".format( url=data['image_url'], size=humanize.naturalsize(data['image_size'], gnu=True) ) if 'file_url' in keys and 'file_size' in keys: message += "\n{url} {size}".format( url=data['file_url'], size=humanize.naturalsize(data['file_size'], gnu=True) ) self.send_tg_message(self.tg_lounge_id, message) return self.STR_MESSAGE_SENT return '' def handle_tg_update(self, data): keys = data.keys() if 'message' in keys: self.handle_tg_message(data['message']) return '' def handle_tg_message(self, message): # print "%s %s: %s" % (message['from']['first_name'], message['from']['last_name'], message['text']) pass
class DropdownCache: def __init__(self, timeout): self.cache_timeout = timeout self.dd_cache = SimpleCache() self.load_data() def load_data(self): print 'Updating cache' time.sleep(2) repos = ['one', 'two', 'three'] templates = ['a', 'b', 'c'] self.dd_cache.set('repos', repos, self.cache_timeout) self.dd_cache.set('templates', templates, self.cache_timeout) def get_repo_data(self): return self.dd_cache.get('repos') def get_template_data(self): return self.dd_cache.get('templates')
def get_text_data(todate): flag = cbf.testdate(string_to_date(todate)) if flag == 1: cache = SimpleCache() rv = cache.get('textdata') if rv is None: data = generate_text_data(todate) #cache.set('textdata', data, timeout=5 * 60) else: data = 'Data not present' return Response(data, status=200, mimetype='text/text')
class PostalBot(TelegramBot): STR_USERNAME = '******' STR_MESSAGE_SENT = "Message sent" SPAM_TIMEOUT = 10 * 60 def __init__(self, tg_api_key, tg_lounge_id): super(PostalBot, self).__init__(tg_api_key, tg_lounge_id) self.cache = SimpleCache() def handle_stream_publish(self, data): keys = data.keys() if 'watch_url' in keys and 'username' in keys: c_key = self.STR_USERNAME + ':' + data['username'] if not self.cache.get(c_key): message = '{username} went live on Postal\n{url}'.format( username=data['username'], url=data['watch_url']) self.send_tg_message(self.tg_lounge_id, message) self.cache.set(c_key, True, timeout=self.SPAM_TIMEOUT) return self.STR_MESSAGE_SENT return '' def handle_postal_new_post(self, data): keys = data.keys() if 'username' in keys and 'title' in keys: message = "New post by {username}\n{title}".format( username=data['username'], title=data['title']) if 'image_url' in keys and 'image_size' in keys: message += "\n{url} {size}".format(url=data['image_url'], size=humanize.naturalsize( data['image_size'], gnu=True)) if 'file_url' in keys and 'file_size' in keys: message += "\n{url} {size}".format(url=data['file_url'], size=humanize.naturalsize( data['file_size'], gnu=True)) self.send_tg_message(self.tg_lounge_id, message) return self.STR_MESSAGE_SENT return '' def handle_tg_update(self, data): keys = data.keys() if 'message' in keys: self.handle_tg_message(data['message']) return '' def handle_tg_message(self, message): # print "%s %s: %s" % (message['from']['first_name'], message['from']['last_name'], message['text']) pass
class ZopeTemplateLoader(jinja2.BaseLoader): def __init__(self, parent_loader, base_path, cache_templates=True, template_list=[]): self.parent_loader = parent_loader self.cache = SimpleCache() self.cache_templates = cache_templates self.path = base_path self.template_list = template_list def get_source(self, environment, template): def passthrough(cachable=True): up = self.parent_loader source, path, uptodate = up.get_source(environment, template) if not cachable: uptodate = lambda: False return source, path, uptodate if not template in self.template_list: return passthrough() path = "%s%s" % (self.path, template) source = self.cache.get(path) if not source: try: response = requests.get(path) except requests.exceptions.ConnectionError: return passthrough(cachable=False) if response.status_code != 200: return passthrough(cachable=False) # escape jinja tags source = response.text source = source.strip() source = source.replace("{%", "{{ '{%' }}").replace("%}", "{{ '%}' }}") source = source.replace("{{", "{{ '{{' }}").replace("}}", "{{ '}}' }}") # template blocks source = source.replace("<!-- block_content -->", "{% block natura2000_content %}{% endblock %}") source = source.replace("<!-- block_head -->", "{% block head %}{% endblock %}") # fix breadcrumb link source = source.replace('"%s"' % self.path, '"%s"' % flask.url_for('naturasites.index')) if self.cache_templates: self.cache.set(path, source) return source, path, lambda: False
class Settings: def __init__(self): self.cache = SimpleCache() def get_token(self): token = self.cache.get("token") if (type(token) == type(None)): return None return json.loads(token) def set_token(self, token): self.cache.set("token", token)
def get_classes_views_formats(): """ Caches the graph_classes JSON file in memory :return: a Python object parsed from the views_formats.json file """ cache = SimpleCache() cvf = cache.get('classes_views_formats') if cvf is None: cvf = json.load( open( join(dirname(dirname(__file__)), 'controller', 'views_formats.json'))) # times out never (i.e. on app startup/shutdown) cache.set('classes_views_formats', cvf) return cvf
class Cache(object): timeout = 604800 #week cache = None def __init__(self, timeout=None): self.timeout = timeout or self.timeout self.cache = SimpleCache() def __call__(self, f): @wraps(f) def decorator(*args, **kwargs): key = request.data + request.path response = self.cache.get(key) if response is None: response = f(*args, **kwargs) self.cache.set(key, response, self.timeout) return response return decorator def get(self, key): return self.cache.get(key) def set(self, key, val): return self.cache.set(key, val, self.timeout)
class Cache(object): def __init__(self): self.cache = SimpleCache(default_timeout=1800) def get(self, key): return self.cache.get(self._key(key)) def set(self, key, value): return self.cache.set(self._key(key), value) def delete(self, key): return self.cache.delte(self._key(key)) def _key(self, key): return '{}:{}'.format(g.id, key)
class BGAPI(object): # Timeout (in minutes) cache_timeout = 1440 def __init__(self): self.cache = SimpleCache() with open(app.app.config["BG_API_KEY"], "r") as f: self.auth_params = json.load(f) def get(self, url_path, params={}): """Build a simple cache of the requested data""" rv = self.cache.get(url_path) if rv is None: params.update(self.auth_params) url = "https://api.biblegateway.com/3/" + url_path response = requests.get(url, params=params) if response.status_code != 200: request = response.request raise RuntimeError("{} request {} returned {}".format(request.method, request.url, response.status_code)) rv = response.json() self.cache.set(url_path, rv, timeout=self.cache_timeout*60) return rv def list_translations(self): return self.get('bible')['data'] def get_translation(self, xlation): return self.get('bible/{}'.format(xlation))['data'][0] def get_book_info(self, xlation, book_osis): all_books = self.get_translation(xlation)['books'] for book in all_books: if book['osis'] == book_osis: return book raise RuntimeError("Invalid book {} in translation {}".format(book_osis, xlation)) def get_passage(self, xlation, passage_osis): verse_json = self.get("bible/{}/{}".format(passage_osis, xlation))['data'][0] passage_json = verse_json['passages'][0] return {'reference': passage_json['reference'], 'content': passage_json['content']}
class MemoryStorage(CircuitBreakerBaseStorage): def __init__(self): self._cache = SimpleCache() self._timeout = {} def get(self, key): timeout = self._timeout.get(key) if timeout and timeout < time.time(): return None return self._cache.get(key) def increment(self, key): return self._cache.inc(key) def set(self, key, value, timeout): self._cache.set(key, value, timeout) def expire(self, key, timeout): if timeout: self._timeout[key] = time.time() + timeout
class LocalSessionManager(SessionManager): """ Session manager that stores sessions in memory. Do not use it in a multi-threaded/multi-processes environment, and use RedisSessionManager instead. """ def __init__(self): # The default timeout is 0, meaning no expiration. # When creating a session, the timeout of the key will be set accordingly # to the expiration time of the access token. self.sessions = SimpleCache(default_timeout=0) def create_session(self, session_id, payload: dict, expire_seconds=None): self.sessions.add(session_id, payload, timeout=expire_seconds) def get(self, session_id): return self.sessions.get(session_id) def destroy_session(self, session_id): return self.sessions.delete(session_id)
class Exmail(object): def __init__(self, app=None): self.app = app if app is not None: self.init_app(app) def init_app(self, app): self._client_id = app.config['EXMAIL_ID'] self._client_secret = app.config['EXMAIL_SECRET'] self._cache_client = SimpleCache() self.access_token_cache_url = 'exmail:access_token' def _gen_access_token(self): url = 'https://exmail.qq.com/cgi-bin/token' payload = { 'client_id': self._client_id, 'client_secret': self._client_secret, 'grant_type': 'client_credentials', } r = requests.post(url, data=payload) r_dict = r.json() return (r_dict['access_token'], r_dict['expires_in']) @property def access_token(self): access_token = self._cache_client.get(self.access_token_cache_url) if not access_token: access_token, expired_secs = self._gen_access_token() self._cache_client.set(self.access_token_cache_url, access_token, expired_secs) return access_token def get_user(self, email): url = 'http://openapi.exmail.qq.com:12211/openapi/user/get' headers = { 'Authorization': 'Bearer %s' % self.access_token } payload = {'alias': email} r = requests.post(url=url, data=payload, headers=headers) r_dict = r.json() return r_dict def update_user(self, email, update_dict): print update_dict url = 'http://openapi.exmail.qq.com:12211/openapi/user/sync' update_dict['action'] = 3 update_dict['alias'] = email # update_dict['md5'] = 0 headers = { 'Authorization': 'Bearer %s' % self.access_token } r = requests.post(url=url, data=update_dict, headers=headers) print r return r def update_password(self, email, password): return self.update_user(email, {'password': password}) def add_user(self, email, name, password, org): url = 'http://openapi.exmail.qq.com:12211/openapi/user/sync' add_dict = { 'action': 2, 'alias': email, 'name': name, 'password': password, 'md5': 0, 'partypath': org, 'opentype': 1, } headers = { 'Authorization': 'Bearer %s' % self.access_token } r = requests.post(url=url, data=add_dict, headers=headers) print r.text print r.status_code def delete_user(self): pass
class ImageSimpleCache(ImageCache): """Simple image cache.""" def __init__(self): """Initialize the cache.""" super(ImageSimpleCache, self).__init__() self.cache = SimpleCache() def get(self, key): """Return the key value. :param key: the object's key :return: the stored object :rtype: `BytesIO` object """ return self.cache.get(key) def set(self, key, value, timeout=None): """Cache the object. :param key: the object's key :param value: the stored object :type value: `BytesIO` object :param timeout: the cache timeout in seconds """ timeout = timeout if timeout else self.timeout self.cache.set(key, value, timeout) self.set_last_modification(key, timeout=timeout) def get_last_modification(self, key): """Get last modification of cached file. :param key: the file object's key """ last = self.cache.get(self._last_modification_key_name(key)) return last def set_last_modification(self, key, last_modification=None, timeout=None): """Set last modification of cached file. :param key: the file object's key :param last_modification: Last modification date of file represented by the key :type last_modification: datetime :param timeout: the cache timeout in seconds """ if not key: return if not last_modification: last_modification = datetime.utcnow().replace(microsecond=0) timeout = timeout if timeout else self.timeout self.cache.set(self._last_modification_key_name(key), last_modification, timeout) def delete(self, key): """Delete the specific key.""" if key: self.cache.delete(key) self.cache.delete(self._last_modification_key_name(key)) def flush(self): """Flush the cache.""" self.cache.clear()
rst.headers['Access-Control-Allow-Origin'] = '*' rst.headers['Access-Control-Allow-Methods'] = 'PUT,GET,POST,DELETE' allow_headers = "Referer,Accept,Origin,User-Agent" rst.headers['Access-Control-Allow-Headers'] = allow_headers return rst return wrapper_fun ''' object to json ''' def to_json(model): """ Returns a JSON representation of an SQLAlchemy-backed object. """ json = {} # json['fields'] = {} # json['pk'] = getattr(model, 'id') for col in model._sa_class_manager.mapper.mapped_table.columns: # json['fields'][col.name] = getattr(model, col.name) json[col.name] = getattr(model, col.name) # return dumps([json]) return json if __name__ == '__main__': rv = cache.get('my-item') print rv
class Cache: TIMEOUT = 5 * 60 VERSION_MIN = 1 VERSION_MAX = 100 def __init__(self): self.store = SimpleCache() self.refresh_key = {} self.version = {} def get(self, key): if key in self.refresh_key: key += self.get_key_version(key) logging.info('GET: ' + key) res = self.store.get(key) return res def set(self, key, val, timeout = None, refresh_on = [], set_refresh = True): if set_refresh: self.set_refresh(key, refresh_on) if key in self.refresh_key: key += self.get_key_version(key) if not timeout: timeout = self.TIMEOUT logging.info('SET: ' + key) self.store.set(key, val, timeout) def set_refresh(self, key, refresh_on): if refresh_on: refresh_on = self.lower_list(refresh_on) self.refresh_key[key] = refresh_on elif key in self.refresh_key: self.refresh_key.pop(key, None) def get_key_version(self, key): version = [] for obj in self.refresh_key[key]: if not obj in self.version: self.version[obj] = self.VERSION_MIN version.append(obj + str(self.version[obj])) return '|v.' + '_'.join(version) def increment_version(self, objs): objs = self.lower_list(objs) for obj in objs: obj = obj.lower() # versions 1 - 100 then start back at 1 if obj in self.version: self.version[obj] += 1 if self.version[obj] > self.VERSION_MAX: self.version[obj] = self.VERSION_MIN else: self.version[obj] = self.VERSION_MIN logging.info('Increment version: ' + obj + ' to: ' + str(self.version[obj])) def make_key(self, route, role, org_id, dynamic = None, criteria = None): key = route + '|' + str(role) + '|' + str(org_id) if dynamic: key += '|' + dynamic if criteria: key += '|' + criteria return key def get_version(self, obj): if obj in self.version: return self.version[obj] else: return None def set_version(self, obj, version): if obj and version: self.version[obj] = version return True return False @staticmethod def lower_list(objs): for obj in objs: obj = obj.lower() return objs
class Server: service_pages = [ '\/api\/ping(\?data=[0-9]+)?$', '\/api\/getRequestPerSec$', '\/api\/get_table\?(table=.+)$' ] def __init__(self, ip, domain=None, seed=None): self.ip = ip self.domain = domain if domain != '' else None self.app = Flask(__name__) self.app.config['MAX_CONTENT_LENGTH'] = const.MAX_AVATAR_SIZE * 1024 self.sessions = dict() self.sessions_by_user_name = dict() for obj in DB.get_sessions(): s = Session(obj.name, obj.activated, obj.uid, obj.id, obj.admin) self.sessions[obj.id] = s self.sessions_by_user_name[obj.name] = s self.rooms = { 'PvE': {0: RoomPvE()}, 'PvP': {0: RoomPvP()}, 'Friends': {0: RoomFriend()} } # TODO(debug): При релизе убрать этот костыль для подсветки типов :) del self.rooms['PvE'][0] del self.rooms['PvP'][0] del self.rooms['Friends'][0] self.logger = Logger() self.logger.write_msg('==Server run at %s==' % ip) self.seed = seed self.cache = SimpleCache(default_timeout=60 * 60 * 24 * 30) def update_status(this): while True: delta = timedelta(minutes=5) now = datetime.now() for s in this.sessions.values(): if not s.last_request or (s.last_request + delta < now and s.status == Session.ONLINE): s.status = Session.OFFLINE gevent.sleep(5 * 60 + 1) gevent.spawn(update_status, self) # Error handling def error_handler_generator(e, data_): def func1(e): return render_template( 'error_page.html', text=data_['text'], description=data_['description'] ), int(error) func1.__name__ = "error" + e return func1 http_errors = loads(open('server/configs/http_errors.json').read()) for error, data in http_errors.items(): self.app.error_handler_spec[None][int(error)] = error_handler_generator(error, data) @self.app.route('/500') # static def error_500_generator(): return None # Error handling end @self.app.after_request def after_request(response): session = self.get_session(request) if session: session['ip'] = request.remote_addr session.last_request = datetime.now() if session.status == Session.OFFLINE: session.status = Session.ONLINE if const.FILTRATE_REQUEST_FOR_LOG: for item in self.service_pages: if search(item, request.url): break else: self.logger.write_record(request, response) else: self.logger.write_record(request, response) return response @self.app.route('/') # static def send_root(): session = self.get_session(request) if session: name = session.user return render_template( 'main_menu.html', page_name='Дурак online', page_title='Главная', user_name=name, admin=bool(session.admin)) elif session is None: return redirect(self.app.config["APPLICATION_ROOT"] + '/static/login.html') else: session = self.sessions[request.cookies['sessID']] session.update_activation_status() return redirect(self.app.config["APPLICATION_ROOT"] + '/static/errors/not_activated.html') @self.app.route('/account_settings.html') def account_settings(): session = self.get_session(request) if session: user_data = DB.check_user(session.user) return render_template( 'account_settings.html', header_mini=True, page_name='Настройки аккаунта', page_title='Настройки', u_name=session.user, email=user_data.email) else: return redirect(self.app.config["APPLICATION_ROOT"] + '/') @self.app.route('/static_/svg/<path:path>') # static def send_svg(path): data = self.cache.get(path) if data is None: data = open("./server/static/svg/" + path).read() self.cache.set(path, data) response = make_response(data) response.headers["Content-type"] = "image/svg+xml" response.headers["Cache-Control"] = "max-age=1000000, public" return response @self.app.route('/favicon.ico') # static def send_favicon(): return send_from_directory( os.path.join(self.app.root_path, 'static'), 'favicon.ico', mimetype='image/vnd.microsoft.icon' ) @self.app.route('/api') # static def send_api_methods(): # TODO: rewrite documentation return redirect(self.app.config["APPLICATION_ROOT"] + '/static/api_methods.html') @self.app.route('/arena') # static; need: get@mode; maybe: get@for(user) def send_arena(): url = self.app.config["APPLICATION_ROOT"] + '/static/arena.html?' for arg in request.args: url += arg + '=' + request.args[arg] + '&' return redirect(url) @self.app.route('/static/server_statistic.html') # static; need: session@admin def send_server_statistic_file(): session = self.get_session(request) if session: if session.admin: file = open(const.SERVER_FOLDER + const.STATIC_FOLDER + '/server_statistic.html', encoding='utf-8').read() response = make_response(file) response.headers["Content-type"] = "text/html" return response else: return 'Permission denied' else: return redirect(self.app.config["APPLICATION_ROOT"] + '/') @self.app.route('/api/activate_account') # need: get@token def activate_account(): token = request.args.get('token') if not search('^[a-zA-Z0-9]+$', token): return 'Bad token' result = DB.activate_account(token) if result is None: return 'Bad token' session = Session(result.name, result.activated, result.uid) self.sessions[session.get_id()] = session session['avatar'] = result.file DB.add_session(session, result.uid) response = redirect(self.app.config["APPLICATION_ROOT"] + '/') session.add_cookie_to_resp(response) return response @self.app.route('/api/resend_email') # need: session def resend_email(): if 'sessID' in request.cookies and request.cookies['sessID'] in self.sessions: session = self.sessions[request.cookies['sessID']] else: return 'Fail', 401 DB.update_email_token(session.user, 'email activation') (email, activation_token) = DB.get_email_adress(session.user) email_.send_email( "Для подтвеждения регистрации пожалуйста перейдите по ссылке " "http://{domain}/api/activate_account?token={token}".format( domain=(self.domain if self.domain is not None else self.ip), token=activation_token ), "Account activation", email ) return 'OK' @self.app.route("/api/subscribe_allow") # need: session def subscribe_allow(): session = self.get_session(request) if not session: return 'Fail', 401 return 'False' if 'cur_room' in session.data else 'True' @self.app.route("/api/subscribe") # need: session def subscribe(): # Create queue for updates from server def gen(sess_id): q = Queue() session = self.sessions[sess_id] session['msg_queue'] = q yield ServerSentEvent('init').encode() while True: # MainLoop for SSE, use threads result = q.get() if str(result) != 'stop': ev = ServerSentEvent(str(result)) yield ev.encode() else: break if q is session['msg_queue']: del session['msg_queue'] del q session = self.get_session(request) if not session: return 'Fail', 401 session.status = Session.PLAY return Response(gen(session.id), mimetype="text/event-stream") @self.app.route("/api/unsubscribe") # need: session@msg_queue def unsubscribe(): session = self.get_session(request) room = session['cur_room'] if not session: response = make_response('Fail', 401) response.headers["Cache-Control"] = "no-store" return response if 'msg_queue' in session.data: session['msg_queue'].put('stop') session.status = Session.ONLINE response = make_response('OK') response.headers["Cache-Control"] = "no-store" if room is None: return response room_id = room.id if room.type == const.MODE_PVE: del self.rooms['PvE'][room_id] del session['cur_room'] del room return response elif room.type == const.MODE_FRIEND: room.send_msg('exit') if room.is_ready(): room.remove_player(session['player_n']) else: del self.rooms['Friends'][room_id] del room del session['cur_room'] return response elif room.type == const.MODE_PVP: if room.is_ready(): room.remove_player(session['player_n']) self.merge_room(room) else: del self.rooms['PvP'][room_id] del room del session['cur_room'] return response @self.app.route("/api/join") # need: session@msg_queue, get@mode; maybe: get@for(user) def join_room(): session = self.get_session(request) if not self.get_session(request): return 'Fail', 401 mode = int(request.args.get('mode')) if mode == const.MODE_PVE: room = RoomPvE(session, seed=self.seed) self.rooms['PvE'][room.id] = session['cur_room'] = room session['player_n'] = const.PLAYER_HAND room.send_player_inf() room.send_changes() elif mode == const.MODE_FRIEND: for_ = request.args.get('for') if for_ is None or session.user == for_ or DB.check_user(for_) is None: return 'Bad request', 400 for id, room in self.rooms['Friends'].items(): if session.user in room.for_ and for_ in room.for_: if session in room.players: return 'Player is already invited' session['player_n'] = room.add_player(session) break else: room = RoomFriend(session, for_=for_, seed=self.seed) session['player_n'] = 0 self.rooms['Friends'][room.id] = room session['cur_room'] = room if room.is_ready(): room.send_player_inf() room.send_changes() room.send_msg(dumps({ 'data': [{ 'type': 'wait', 'player': room.game.turn, 'card': None, 'inf': None }] })) else: room.send_msg('wait') elif mode == const.MODE_PVP: for room in self.rooms['PvP'].values(): if room.type == const.MODE_PVP and not room.is_ready(): break else: room = RoomPvP(seed=self.seed) self.rooms['PvP'][room.id] = room session['player_n'] = room.add_player(session) session['cur_room'] = room if room.is_ready(): room.send_player_inf() room.send_changes() room.send_msg(dumps({ 'data': [{ 'type': 'wait', 'player': room.game.turn, 'card': None, 'inf': None }] })) else: room.send_msg('wait') return 'OK' @self.app.route("/api/attack", methods=['GET']) # need: session@cur_room, get@card def attack(): session = self.get_session(request) if not session: return 'Fail', 401 room = session['cur_room'] card = int(request.args.get('card')) result = room.attack(session['player_n'], card) if result == 'END': room_id = room.id if room.type == const.MODE_PVE: del self.rooms['PvE'][room_id] del session['cur_room'] return 'OK' return result @self.app.route("/api/defense", methods=['GET']) # need: session@cur_room, get@card def defense(): session = self.get_session(request) if not session: return 'Fail', 401 room = session['cur_room'] card = int(request.args.get('card')) result = room.defense(session['player_n'], card) if result == 'END': room_id = room.id if room.type == const.MODE_PVE: del self.rooms['PvE'][room_id] del session['cur_room'] return 'OK' return result @self.app.route("/api/chat", methods=['POST']) # need: session@cur_room, post@msg def send_msg_to_chat(): session = self.get_session(request) if not session: return 'Fail', 401 room = session['cur_room'] msg = request.form.get('msg') room.send_msg(dumps({ 'msg': msg, 'from': session.user, 'hand': session['player_n'] })) return 'OK' @self.app.route("/api/check_user", methods=['POST']) # -> bool or Error # (need: post@user_name; maybe: post@pass) XOR need: post@email def check_user(): password = request.form.get('pass') sha256 = hashlib.sha256(bytes(password, encoding='utf-8')).hexdigest() if password is not None else None email = request.form.get('email') name = request.form.get('name') if name is not None: result = DB.check_user(name, sha256) response = make_response((not result).__str__()) elif email is not None: result = DB.check_email(email) response = make_response((not result).__str__()) else: response = make_response('Bad request', 400) response.headers["Content-type"] = "text/plain" return response @self.app.route("/api/avatar", methods=['GET']) # need: get@user; maybe: get@type := menu | round | round_white any def get_avatar(): user = request.args.get('user') if user == 'AI' or user == 'root': if request.args.get('type') == 'menu' or request.args.get('type') == 'round': response = make_response("/static_/svg/ic_computer_24px.svg") else: response = make_response("/static_/svg/ic_computer_24px_white.svg") else: file_ext = DB.check_user(user).file if file_ext is not None and file_ext != 'None': response = make_response("/static/avatar/{user_name}{file_ext}". format(user_name=user, file_ext=file_ext)) else: if request.args.get('type') == 'menu': response = make_response("/static_/svg/ic_person_24px.svg") elif request.args.get('type') == 'round': response = make_response("/static_/svg/account-circle.svg") elif request.args.get('type') == 'round_white': response = make_response("/static_/svg/account-circle_white.svg") else: response = make_response("/static_/svg/ic_person_24px_white.svg") response.headers["Cache-Control"] = "no-store" return response @self.app.route("/api/add_user", methods=['POST']) # need: post@user_name, post@pass, post@email; maybe: post@file(image) def add_user(): sha256 = hashlib.sha256(bytes(request.form.get('pass'), encoding='utf-8')).hexdigest() name = request.form.get('name') email = request.form.get('email') result = not DB.check_user(name) and not DB.check_email(email) if not (search('^.+@.+\..+$', email) and search('^[a-zA-Z0-9_]+$', name) and result): return make_response('Wrong data', 400) if request.files: file = request.files['file'] if file.mimetype in const.IMAGES: file_ext = const.IMAGES[file.mimetype] file.save("./server/static/avatar/{}{}".format(name, file_ext)) else: return make_response('Wrong data', 400) else: file_ext = None (activation_token, result) = DB.add_user(name, sha256, file_ext, email) if result: response = make_response('OK') result2 = DB.check_user(name, sha256) if result2: session = Session(name, result2.activated, result2.uid) self.sessions[session.get_id()] = session self.sessions_by_user_name[name] = session session['avatar'] = result2.file DB.add_session(session, result2.uid) session.add_cookie_to_resp(response) email_.send_email( "Для подтвеждения регистрации пожалуйста перейдите по ссылке " "http://{domain}/api/activate_account?token={token}".format( domain=(self.domain if self.domain is not None else self.ip), token=activation_token ), "Account activation", email) else: self.logger.write_msg("Something wrong with registration ({})".format(name)) response.headers["Content-type"] = "text/plain" return response else: return 'Error', 500 @self.app.route("/api/change_avatar", methods=['POST']) # need: session, post@file(image) def change_avatar(): session = self.get_session(request) if not session: return 'Fail', 401 if request.files: file = request.files['file'] if file.mimetype in const.IMAGES: file_ext = const.IMAGES[file.mimetype] file.save("./server/static/avatar/{}{}".format(session.user, file_ext)) DB.set_avatar_ext(session.user, file_ext) else: return 'Wrong data', 400 else: return 'Wrong data', 400 return 'OK' @self.app.route("/api/change_pass", methods=['POST']) # need: session, post@old_pass, post@new_pass def change_pass(): session = self.get_session(request) if not session: return 'Fail', 401 sha256 = hashlib.sha256(bytes(request.form.get('old_pass'), encoding='utf-8')).hexdigest() if DB.check_user(session.user, sha256): sha256 = hashlib.sha256(bytes(request.form.get('new_pass'), encoding='utf-8')).hexdigest() DB.set_new_pass(session.user, sha256) return 'OK' else: return 'Wrong password' @self.app.route("/api/send_mail_for_auto_change_pass", methods=['GET']) # get@user def send_mail_for_auto_change_pass(): user = request.args.get('user') DB.update_email_token(user, 'email activation') (email, activation_token) = DB.get_email_adress(user) email_.send_email( "Для подтвеждения смены пароля пожалуйста перейдите по ссылке " "http://{domain}/api/auto_change_pass?user={user}&token={token}".format( domain=(self.domain if self.domain is not None else self.ip), user=user, token=activation_token ), "Password change confirmation", email) return 'OK' @self.app.route("/api/auto_change_pass", methods=['GET']) # get@user, get@token def auto_change_pass(): user = request.args.get('user') token = request.args.get('token') (email, activation_token) = DB.get_email_adress(user) if (token == activation_token): new_pass = DB.auto_set_new_pass(user, 'new password') email_.send_email( "Пользователь: {user}\n" "Новый пароль: {password} " "(Вы сможете изменить пароль на любой другой на странице пользователя)".format( domain=(self.domain if self.domain is not None else self.ip), user=user, password=new_pass ), "Password change", email) return render_template( "error_page.html", title="Password change", text="Парол изменен", description="Письмо с новым паролем отправлено на ваш e-mail" ) else: return redirect(self.app.config["APPLICATION_ROOT"] + '/404') @self.app.route("/api/init_session", methods=['POST']) # need: post@user_name, post@pass def init_session(): if self.get_session(request) is not None: response = make_response('OK') response.headers["Content-type"] = "text/plain" return response user_name = request.form.get('user_name') password = request.form.get('pass') if user_name is None or password is None: return 'Bad request', 400 sha256 = hashlib.sha256(bytes(password, encoding='utf-8')).hexdigest() result = DB.check_user(request.form.get('user_name'), sha256) if result: if user_name in self.sessions_by_user_name: session = self.sessions_by_user_name[user_name] else: session = Session(user_name, result.activated, result.uid, admin=result.admin) self.sessions[session.get_id()] = session self.sessions_by_user_name[user_name] = session session['avatar'] = result.file DB.add_session(session, result.uid) session['ip'] = request.remote_addr response = make_response('True') response.headers["Content-type"] = "text/plain" session.add_cookie_to_resp(response) return response else: response = make_response('False') response.headers["Content-type"] = "text/plain" return response @self.app.route("/api/destroy_session") # need: session def destroy_session(): session = self.get_session(request) if not session: return 'Fail', 401 response = make_response('OK') session.delete_cookie(response) DB.delete_session(session.id) del self.sessions_by_user_name[session.user] del self.sessions[session.id] return response @self.app.route('/api/ping') # maybe: get@data def ping(): data = request.args.get('data') return data if data is not None else 'Pong' @self.app.route('/api/getRequestPerSec') # need: session@admin def get_request_per_sec(): session = self.get_session(request) if session: if self.get_session(request).admin: return self.logger.time_log[-1].__str__() else: return 'Permission denied', 401 else: return 'Fail', 401 @self.app.route('/api/get_table', methods=['GET']) # need: session@admin, get@table def get_table(): session = self.get_session(request) if session: if session.admin: table_s = request.args.get('table') if table_s == 'sessions': table = self.sessions.values() result = list( map(lambda s: dict(s.to_json(), **{'status': self.get_user_status(s.user)}), table)) elif table_s == 'rooms': result = [] for table in [self.rooms['PvE'].values(), self.rooms['PvP'].values(), self.rooms['Friends'].values()]: result += list(map(lambda s: s.to_json(), table)) else: return 'Bad request', 400 return dumps(result) else: return 'Permission denied', 401 else: return 'Fail', 401 @self.app.route('/api/users/check_online', methods=['GET']) # need: session; get@name def check_online(): session = self.get_session(request) if not session: return 'Fail', 401 u_name = request.args.get('name') return self.get_user_status(u_name) @self.app.route('/api/users/get_friend_list') # need: session; maybe: get@invited(bool) def get_friend_list(): session = self.get_session(request) if not session: return 'Fail', 401 invited = 'invited' in request.args return dumps(list(map( lambda x: self.user_to_json(x, session), DB.get_friends(uid=session.uid, accepted=not invited) ))) @self.app.route('/api/users/find', methods=['GET']) # need: session; get@name def find_user(): session = self.get_session(request) if not session: return 'Fail', 401 name = request.args.get('name') if name and len(name) > 3: return dumps(list(map(lambda x: self.user_to_json(x, session), DB.find_user(name)))) else: return 'Bad request', 400 @self.app.route('/api/users/friend_invite', methods=['GET']) # need: session; get@user; maybe get@accept XOR get@reject def friend_invite(): session = self.get_session(request) if not session: return 'Fail', 401 friend = request.args.get('user') accept = 'accept' in request.args reject = 'reject' in request.args if friend: if accept and DB.is_friend(session.user, friend): DB.accept_invite(session.user, friend) return 'OK' elif reject and DB.is_friend(session.user, friend): DB.reject_invite(session.user, friend) return 'OK' elif not (accept or reject or DB.is_friend(session.user, friend)): DB.invite_friend(session.user, friend) return 'OK' return 'Fail' @self.app.route('/api/get_game_invites') # need: session; def get_game_invites(): session = self.get_session(request) if not session: return 'Fail', 401 rooms = self.get_friends_games_for_user(session.user) if rooms: return dumps(list(map( lambda room: room.for_[0] if room.for_[0] != session.user else room.for_[1], rooms ))) else: return '[]' def get_session(self, request_) -> Session: # -> Session | False | None if 'sessID' in request_.cookies and request_.cookies['sessID'] in self.sessions: session = self.sessions[request_.cookies['sessID']] if session.activated: return session else: return False else: return None def merge_room(self, room): if room.type == const.MODE_PVP: for thin_room in self.rooms['PvP'].values(): if thin_room.is_ready() or thin_room is room: continue session = room.players[0] if room.players[0] is not None else room.players[1] session['player_n'] = thin_room.add_player(session) session['cur_room'] = thin_room if thin_room.is_ready(): thin_room.send_player_inf() thin_room.send_changes() thin_room.send_msg(dumps({ 'data': [{ 'type': 'wait', 'player': thin_room.game.turn, 'card': None, 'inf': None }] })) del self.rooms['PvP'][room.id] return thin_room return None def user_to_json(self, user, session, color=""): (uid, u_name, u_avatar) = user status = self.get_user_status(u_name) if u_avatar is not None and u_avatar != 'None': u_avatar = "/static/avatar/{user_name}{file_ext}".format(user_name=u_name, file_ext=u_avatar) else: u_avatar = "/static_/svg/account-circle%s.svg" % (("_" + color) if color else "") c = DB.connect_() mutual_friends = list(map(lambda x: x[0], DB.get_mutual_friends(session.uid, uid, c))) c.close() return { 'name': u_name, 'status': status, 'avatar': u_avatar, 'mutual_friends': mutual_friends, 'is_friend': DB.is_friend(session.user, u_name) } def get_user_status(self, u_name): if u_name in self.sessions_by_user_name: tmp_session = self.sessions_by_user_name[u_name] if tmp_session.status == Session.ONLINE: return "Online" elif tmp_session.status == Session.PLAY: room = tmp_session['cur_room'] if room: if room.type == const.MODE_PVE: return "Играет с AI" else: player = room.players[1 - tmp_session['player_n']] return ("Играет с " + player.user) if player is not None else 'Ожидает оппонента' else: return "Online" else: return "Offline" else: return 'Offline' def get_friends_games_for_user(self, user) -> filter: return filter(lambda room: user in room.for_, self.rooms['Friends'].values())
class OpaqueValidator: def __init__(self, introspection_url, client_id, client_secret, verify_ssl_server=True): self.ctx = ssl.create_default_context() if not verify_ssl_server: self.ctx.check_hostname = False self.ctx.verify_mode = ssl.CERT_NONE self._introspection_url = introspection_url self._client_id = client_id self._client_secret = client_secret self._token_cache = SimpleCache() def introspect_token(self, token): params = {'token': token} headers = { 'content-type': 'application/x-www-form-urlencoded', 'accept': 'application/jwt, application/json;q=0.9, text/plain;q=0.8, text/html;q=0.7' } req = request("POST", self._introspection_url, allow_redirects=False, auth=(self._client_id, self._client_secret), verify=self.ctx.check_hostname, data=params, headers=headers) response_content_type = req.headers.get("Content-Type", "text/plain").split(";")[0] result = {} cache_duration_header_value = req.headers.get("Cache-Duration", None) if cache_duration_header_value: # Turn 'public, max-age=31536000' into {'public': None, 'max-age': '31536000'} cache_duration_parts = dict( (part_values[0], None if len(part_values) == 1 else part_values[1]) for part_values in [ part.strip().split("=") for part in cache_duration_header_value.split(",") ]) if "public" in cache_duration_parts: result["cache_timeout"] = int(cache_duration_parts["max-age"]) if req.status_code == 200: if response_content_type == "application/json": result.update(json.loads(req.text)) elif response_content_type == "application/jwt": jws = jwkest.jws.factory(req.text) if jws is not None and len(jws.jwt.part) >= 2: result["active"] = True result.update(json.loads(jws.jwt.part[1])) else: # Text or HTML presumably warnings.warn( "Response type from introspection endpoint was unsupported, response_type = " + response_content_type) raise Exception( "Response type is from introspect endpoint is " + response_content_type, req.text) elif req.status_code == 204: result.update(dict(active=False)) else: raise Exception("HTTP POST error from introspection: %s" % req.status_code) return result def validate(self, token): now = calendar.timegm(datetime.utcnow().utctimetuple()) # Lookup in cache: cached_response = self._token_cache.get(token) if cached_response is not None: if cached_response['active']: if cached_response['exp'] >= now: return { "subject": cached_response['sub'], "scope": cached_response['scope'], "active": True } else: return dict(active=False) introspect_response = self.introspect_token(token) cache_timeout = 0 if "cache_timeout" in introspect_response: cache_timeout = introspect_response["cache_timeout"] elif "exp" in introspect_response: cache_timeout = introspect_response["exp"] - now if "active" not in introspect_response: # The token isn't know to be active, so we'll never introspect it again introspect_response["active"] = False self._token_cache.set(token, introspect_response, timeout=cache_timeout) raise OpaqueValidatorException( "No active field in introspection response") self._token_cache.set(token, introspect_response, timeout=cache_timeout) if not introspect_response['active']: return {"active": False} if 'sub' not in introspect_response: raise OpaqueValidatorException( "Missing sub field in introspection response") if 'exp' not in introspect_response: raise OpaqueValidatorException( "Missing exp field in introspection response") if 'scope' not in introspect_response: raise OpaqueValidatorException( "Missing scope field in introspection response") return { "subject": introspect_response['sub'], "scope": introspect_response['scope'], "active": True }
def calcHipposcore(dictResult): logger.info('hipposcore.calcHipposcore launched') try: T = 182.625 n1 = float() n2 = float() n3 = float() #the final result hippodict = dict() #json object for hipposcore #{ # 'ioc': { # 'hipposcore': -89 # } #} scoredict = dict() now = strftime("%Y%m%dT%H%M%S%z") P = float() #a cache is used to speed up the source's score retrieve cache = SimpleCache() for ioc, listMatches in dictResult.items(): #if there is no match for the ioc #listmaches will be empt, then hipposcore will be 0 if not listMatches : hippodict[ioc] = dict() hippodict[ioc]['hipposcore'] = 0 #match case else: P = 0.0 for match in listMatches: try: #n1 retrieving the source according to its id idSource = match['idSource'] n1 = cache.get(idSource) if n1 is None: source = ExistingSource(idSource) source.forgeDocMatch() if source.search(): source.processMatchResponse() n1 = source.getScore() #the source's scor is between -100 and +100 #for convenience, #in the formula it is required to be within -1 and +1 n1 = n1 / 100.0 cache.set(idSource, n1, timeout=3 * 60) #n2 #rank is specific to alexa feed if 'rank' in match: n2 = match['rank'] else: n2 = 1.0 #n3 #last time hippocampe saw the ioc lastAppearance = match['lastAppearance'] #lastAppearance is a string, to calculate the time #difference, in other word the age of the ioc, #between lastAppearance and now, it is #needed to convert those strings to date lastAppearanceDate = dateutil.parser.parse(lastAppearance) nowDate = dateutil.parser.parse(now) #ioc's age in days age = (nowDate - lastAppearanceDate).days n3 = exp(-age / T) P = P + (n3 * n2 * n1) except Exception as e: logger.error('hipposcore.calcHipposcore.match failed to process', exc_info = True) logger.error(match) report = dict() report['error'] = str(e) score = (1 - exp(-2 * abs(P))) * (abs(P) / P) * 100 score = "%.2f" % score scoredict['hipposcore'] = score #scoredict['hipposcore'] = (1 - exp(-2 * abs(P))) * (abs(P) / P) * 100 hippodict[ioc] = scoredict logger.info('hipposcore.main end') return hippodict except Exception as e: logger.error('hipposcore.calcHipposcore failed, no idea where it came from...', exc_info = True) report = dict() report['error'] = str(e)
class JwtManager(object): ALGORITHMS = "RS256" def __init__(self, app=None): # These are all set in the init_app function, but are listed here for easy reference self.app = app self.well_known_config = None self.well_known_obj_cache = None self.algorithms = JwtManager.ALGORITHMS self.jwks_uri = None self.issuer = None self.audience = None self.cache = None self.caching_enabled = False self.jwt_oidc_test_mode = False self.jwt_oidc_test_keys = None if app is not None: self.init_app(app) def init_app(self, app): """initializze this extension if the config['JWT_OIDC_WELL_KNOWN_CONFIG'] is set, then try to load the JWKS_URI & ISSUER from that If it is not set attempt to load the JWKS_URI and ISSUE from the application config Required settings to function: WELL_KNOWN_CONFIG (optional) is this is set, the JWKS_URI & ISSUER will be loaded from there JWKS_URI: the endpoint defined for the jwks_keys ISSUER: the endpoint for the issuer of the tokens ALGORITHMS: only RS256 is supported AUDIENCE: the oidc audience (or API_IDENTIFIER) CLIENT_SECRET: the shared secret / key assigned to the client (audience) """ self.app = app self.jwt_oidc_test_mode = app.config.get('JWT_OIDC_TEST_MODE', None) # ## CHECK IF WE"RE RUNNING IN TEST_MODE!! # if self.jwt_oidc_test_mode: app.logger.debug( 'JWT MANAGER running in test mode, using locally defined certs & tokens' ) self.issuer = app.config.get('JWT_OIDC_TEST_ISSUER', 'localhost.localdomain') self.jwt_oidc_test_keys = app.config.get('JWT_OIDC_TEST_KEYS', None) self.audience = app.config.get('JWT_OIDC_TEST_AUDIENCE', None) self.jwt_oidc_test_private_key_pem = app.config.get( 'JWT_OIDC_TEST_PRIVATE_KEY_PEM', None) if self.jwt_oidc_test_keys: app.logger.debug('local key being used: {}'.format( self.jwt_oidc_test_keys)) else: app.logger.error( 'Attempting to run JWT Manager with no local key assigned') raise Exception( 'Attempting to run JWT Manager with no local key assigned') else: self.algorithms = [ app.config.get('JWT_OIDC_ALGORITHMS', JwtManager.ALGORITHMS) ] # If the WELL_KNOWN_CONFIG is set, then go fetch the JWKS & ISSUER self.well_known_config = app.config.get( 'JWT_OIDC_WELL_KNOWN_CONFIG', None) if self.well_known_config: # try to get the jwks & issuer from the well known config # jurl = urlopen(url=self.well_known_config, context=ssl.create_default_context()) jurl = urlopen(url=self.well_known_config) self.well_known_obj_cache = json.loads( jurl.read().decode("utf-8")) self.jwks_uri = self.well_known_obj_cache['jwks_uri'] self.issuer = self.well_known_obj_cache['issuer'] else: self.jwks_uri = app.config.get('JWT_OIDC_JWKS_URI', None) self.issuer = app.config.get('JWT_OIDC_ISSUER', None) # Setup JWKS caching self.caching_enabled = app.config.get('JWT_OIDC_CACHING_ENABLED', False) if self.caching_enabled: from werkzeug.contrib.cache import SimpleCache self.cache = SimpleCache(default_timeout=app.config.get( 'JWT_OIDC_JWKS_CACHE_TIMEOUT', 300)) self.audience = app.config.get('JWT_OIDC_AUDIENCE', None) app.logger.debug('JWKS_URI: {}'.format(self.jwks_uri)) app.logger.debug('ISSUER: {}'.format(self.issuer)) app.logger.debug('ALGORITHMS: {}'.format(self.algorithms)) app.logger.debug('AUDIENCE: {}'.format(self.audience)) app.logger.debug('JWT_OIDC_TEST_MODE: {}'.format( self.jwt_oidc_test_mode)) app.logger.debug('JWT_OIDC_TEST_KEYS: {}'.format( self.jwt_oidc_test_keys)) # set the auth error handler auth_err_handler = app.config.get('JWT_OIDC_AUTH_ERROR_HANDLER', JwtManager.handle_auth_error) app.register_error_handler(AuthError, auth_err_handler) app.teardown_appcontext(self.teardown) def teardown(self, exception): pass # ctx = _app_ctx_stack.top # if hasattr(ctx, 'cached object'): @staticmethod def handle_auth_error(ex): response = jsonify(ex.error) response.status_code = ex.status_code return response def get_token_auth_header(self): """Obtains the access token from the Authorization Header """ auth = request.headers.get("Authorization", None) if not auth: raise AuthError( { "code": "authorization_header_missing", "description": "Authorization header is expected" }, 401) parts = auth.split() if parts[0].lower() != "bearer": raise AuthError( { "code": "invalid_header", "description": "Authorization header must start with Bearer" }, 401) elif len(parts) < 2: raise AuthError( { "code": "invalid_header", "description": "Token not found after Bearer" }, 401) elif len(parts) > 2: raise AuthError( { "code": "invalid_header", "description": "Authorization header is an invalid token structure" }, 401) return parts[1] def contains_role(self, roles): """Checks that the listed roles are in the token using the registered callback Args: roles [str,]: Comma separated list of valid roles JWT_ROLE_CALLBACK (fn): The callback added to the Flask configuration """ token = self.get_token_auth_header() unverified_claims = jwt.get_unverified_claims(token) roles_in_token = current_app.config['JWT_ROLE_CALLBACK']( unverified_claims) if any(elem in roles_in_token for elem in roles): return True return False def has_one_of_roles(self, roles): """Checks that at least one of the roles are in the token using the registered callback Args: roles [str,]: Comma separated list of valid roles JWT_ROLE_CALLBACK (fn): The callback added to the Flask configuration """ def decorated(f): @wraps(f) def wrapper(*args, **kwargs): self._require_auth_validation(*args, **kwargs) if self.contains_role(roles): return f(*args, **kwargs) raise AuthError( { "code": "missing_a_valid_role", "description": "Missing a role required to access this endpoint" }, 401) return wrapper return decorated def validate_roles(self, required_roles): """Checks that the listed roles are in the token using the registered callback Args: required_roles [str,]: Comma separated list of required roles JWT_ROLE_CALLBACK (fn): The callback added to the Flask configuration """ token = self.get_token_auth_header() unverified_claims = jwt.get_unverified_claims(token) roles_in_token = current_app.config['JWT_ROLE_CALLBACK']( unverified_claims) if all(elem in roles_in_token for elem in required_roles): return True return False def requires_roles(self, required_roles): """Checks that the listed roles are in the token using the registered callback Args: required_roles [str,]: Comma separated list of required roles JWT_ROLE_CALLBACK (fn): The callback added to the Flask configuration """ def decorated(f): @wraps(f) def wrapper(*args, **kwargs): self._require_auth_validation(*args, **kwargs) if self.validate_roles(required_roles): return f(*args, **kwargs) raise AuthError( { "code": "missing_required_roles", "description": "Missing the role(s) required to access this endpoint" }, 401) return wrapper return decorated def requires_auth(self, f): """Validates the Bearer Token """ @wraps(f) def decorated(*args, **kwargs): self._require_auth_validation(*args, **kwargs) return f(*args, **kwargs) return decorated def _require_auth_validation(self, *args, **kwargs): token = self.get_token_auth_header() try: unverified_header = jwt.get_unverified_header(token) except jwt.JWTError: raise AuthError( { "code": "invalid_header", "description": "Invalid header. " "Use an RS256 signed JWT Access Token" }, 401) if unverified_header["alg"] == "HS256": raise AuthError( { "code": "invalid_header", "description": "Invalid header. " "Use an RS256 signed JWT Access Token" }, 401) if not "kid" in unverified_header: raise AuthError( { "code": "invalid_header", "description": "Invalid header. " "No KID in token header" }, 401) rsa_key = self.get_rsa_key(self.get_jwks(), unverified_header["kid"]) if not rsa_key and self.caching_enabled: # Could be key rotation, invalidate the cache and try again self.cache.delete('jwks') rsa_key = self.get_rsa_key(self.get_jwks(), unverified_header["kid"]) if not rsa_key: raise AuthError( { "code": "invalid_header", "description": "Unable to find jwks key referenced in token" }, 401) try: payload = jwt.decode(token, rsa_key, algorithms=self.algorithms, audience=self.audience, issuer=self.issuer) _request_ctx_stack.top.current_user = g.jwt_oidc_token_info = payload except jwt.ExpiredSignatureError: raise AuthError( { "code": "token_expired", "description": "token has expired" }, 401) except jwt.JWTClaimsError: raise AuthError( { "code": "invalid_claims", "description": "incorrect claims," " please check the audience and issuer" }, 401) except Exception: raise AuthError( { "code": "invalid_header", "description": "Unable to parse authentication" " token." }, 401) def get_jwks(self): if self.jwt_oidc_test_mode: return self.jwt_oidc_test_keys if self.caching_enabled: return self._get_jwks_from_cache() else: return self._fetch_jwks_from_url() def _get_jwks_from_cache(self): jwks = self.cache.get('jwks') if jwks is None: jwks = self._fetch_jwks_from_url() self.cache.set('jwks', jwks) return jwks def _fetch_jwks_from_url(self): jsonurl = urlopen(self.jwks_uri) return json.loads(jsonurl.read().decode("utf-8")) def create_jwt(self, claims, header): token = jwt.encode(claims, self.jwt_oidc_test_private_key_pem, headers=header, algorithm='RS256') return token def get_rsa_key(self, jwks, kid): rsa_key = {} for key in jwks["keys"]: if key["kid"] == kid: rsa_key = { "kty": key["kty"], "kid": key["kid"], "use": key["use"], "n": key["n"], "e": key["e"] } return rsa_key
class CacheService: def __init__(self): self.cache = SimpleCache() self.exchange_service = ExchangeService() def set_currency_count(self): currency_count = self.cache.get('currency_count') if currency_count is None: currency_count = self.exchange_service.get_currency_count() self.cache.set('currency_count', currency_count, timeout=60 * 60 * 24 * 30) return currency_count def set_markets_count(self): market_count = self.cache.get('market_count') if market_count is None: market_count = self.exchange_service.get_market_count() self.cache.set('market_count', market_count, timeout=60 * 60 * 24 * 30) return market_count @staticmethod def set_sellers(code, data): engine = create_engine(database.connect) Session = scoped_session( sessionmaker(autocommit=False, autoflush=False, bind=engine)) session = Session() sellers_cache = SellersCache.query.filter_by(code=code).first() if sellers_cache is None: sellers_cache = SellersCache() sellers_cache.code = code sellers_cache.data = data local_object = session.merge(sellers_cache) session.add(local_object) # db.session.add(sellers_cache) else: sellers_cache.data = data local_object = session.merge(sellers_cache) session.add(local_object) # db.session.add(sellers_cache) session.commit() session.close() # db.session.commit() return sellers_cache @staticmethod def get_sellers(code): return SellersCache.query.filter_by(code=code).first() @staticmethod def set_buyers(code, data): engine = create_engine(database.connect) Session = scoped_session( sessionmaker(autocommit=False, autoflush=False, bind=engine)) session = Session() buyers_cache = BuyersCache.query.filter_by(code=code).first() if buyers_cache is None: buyers_cache = BuyersCache() buyers_cache.code = code buyers_cache.data = data local_object = session.merge(buyers_cache) session.add(local_object) # db.session.add(buyers_cache) else: buyers_cache.data = data local_object = session.merge(buyers_cache) session.add(local_object) # db.session.add(buyers_cache) # db.session.commit() session.commit() session.close() return buyers_cache @staticmethod def get_buyers(code): return BuyersCache.query.filter_by(code=code).first()
logger = logging.getLogger(__name__) cache = SimpleCache() # Provides a simple layer of caching for the video list, based on Memcached. if "MEMCACHED" == CACHE_BACKEND: # Try to use memcached try: cache = MemcachedCache([MEMCACHED_HOST_PORT]) # testing the connection dummy_data = "The quick brown fox jumps over the lazy dog abcxyz" cache.set("cs2015_dummy_data", dummy_data) if dummy_data == cache.get("cs2015_dummy_data"): logger.info("MemcachedCache was started successfully.") else: logger.info("MemcachedCache is not working correctly. Using SimpleCache.") cache = SimpleCache() except RuntimeError as e: logger.warn("Error connecting to Memcached. Using SimpleCache. Error=[%r]." % e) except: logger.warn("Error connecting to Memcached. Error=[%r]." % sys.exc_info()[0]) ''' elif "REDIS" == CACHE_BACKEND: # Try to use redis try:
def get_status_for_user(): cache = SimpleCache() localization = cache.get("user_localization") return jsonify(value=localization)
class Elasticsearch(Datastore): engine = None index = 'aleph-samples' tracking_index = 'aleph-tracking' doc_type = 'sample' cache = None def __init__(self): self.engine = ES() self.cache = SimpleCache() def all(self, page=1, size=DEFAULT_PAGE_SIZE): body = { 'query': { 'match_all': {}, }, "sort": { "timestamp": { 'order': 'desc' }, } } start = ((page - 1) * size) res = self.raw_search(body, start=start, size=size) total = res['hits']['total'] entries = res['hits']['hits'] return (total, self.entries_to_samples(entries)) def entries_to_samples(self, entries): rv = [] if not entries: return rv entry_table = {} for entry in entries: sample_id = entry['_id'] entry_table[sample_id] = {'metadata': entry, 'tracking_data': {}} # Get tracking data for retrieved ids tracking_data = self._mget(list(entry_table.keys()), index=self.tracking_index) for td in tracking_data: sample_id = td['_id'] entry_table[sample_id]['tracking_data'] = td # Add return values for sample_id, sample_data in entry_table.items(): if '_source' in sample_data['metadata'] and '_source' in sample_data['tracking_data']: rv.append(Sample(sample_data['metadata'], sample_data['tracking_data'])) return rv def _mget(self, ids, index=None): if not index: index = self.index if not isinstance(ids, list): raise ValueError("ids is not a list") body = { 'ids': ids } result = self.engine.mget(index=index, doc_type=self.doc_type, body=body, ignore=404) if 'docs' not in result: return None return result['docs'] def _get(self, sample_id, index=None): if not index: index = self.index result = self.engine.get(index=index, doc_type=self.doc_type, id=sample_id, ignore=404) if result['found'] == False: return None return OrderedDict(sorted(result.items())) def mget(self, sample_ids): if not sample_ids: return None metadata = {s['_id']:s for s in self._mget(sample_ids)} tracking_data = {s['_id']:s for s in self._mget(sample_ids, index=self.tracking_index)} if not metadata or not tracking_data: return None entries = [] for sample_id, v in metadata.items(): if '_source' in metadata[sample_id].keys() and '_source' in tracking_data[sample_id].keys(): entries.append(Sample(metadata[sample_id], tracking_data[sample_id])) return entries def get(self, sample_id): metadata = self._get(sample_id) tracking_data = self._get(sample_id, index=self.tracking_index) if not metadata or not tracking_data: return None return Sample(metadata, tracking_data) def get_parents(self, sample_id): rv = self.cache.get('get-parents-%s' % sample_id) if not rv: tracking_data = self._get(sample_id, index=self.tracking_index) if not tracking_data: return [] rv = tracking_data['_source']['parents'] return rv def get_children(self, sample_id): rv = self.cache.get('get-children-%s' % sample_id) if not rv: search_body = { "query": { "bool": { "must": [ { "term": { "parents": sample_id } } ] } } } result = self.raw_search(search_body, index=self.tracking_index) rv = result['hits']['hits'] return rv def raw_search(self, body, q=None, start=0, size=DEFAULT_PAGE_SIZE, index=None): if not index: index = self.index result = [] try: hits = self.engine.search(index=index, doc_type=self.doc_type, q=q, from_=start, size=size, body=body) except NotFoundError: pass except Exception: raise return hits def search(self, query, page=1, size=DEFAULT_PAGE_SIZE): start = ((page - 1) * size) result = [] body = { "sort": { "timestamp": { 'order': 'desc' }, } } hits = self.raw_search(body, start=start, size=size, q=query) total = hits['hits']['total'] entries = hits['hits']['hits'] return (total, self.entries_to_samples(entries)) def count(self, body): return self.engine.count(index=self.tracking_index, doc_type=self.doc_type, body=body)['count'] # Aux Methods # Counters def count_all(self): body = { "query": { "match_all" : {} } } return self.count(body) def count_processing_samples(self): body = { "query": { "bool" : { "filter" : [ {"script" : {"script" : {"source": "!doc['processors_completed'].containsAll(doc['processors_dispatched'])", "lang": "painless"}}}, ] } } } return self.count(body) def count_analyzing_samples(self): body = { "query": { "bool" : { "filter" : [ {"script" : {"script" : {"source": "!doc['analyzers_completed'].containsAll(doc['analyzers_dispatched'])", "lang": "painless"}}}, ] } } } return self.count(body) # Graph Data def sample_histogram(self, size=24, interval="1h"): histogram = {} hist_body = { "aggs" : { "samples_over_time" : { "date_histogram" : { "field" : "timestamp", "interval" : interval, "min_doc_count": 0 } } } } hist_result = self.raw_search(hist_body)['aggregations'] for h in hist_result['samples_over_time']['buckets']: histogram[h['key_as_string']] = h['doc_count'] return histogram def sample_diversity(self): diversity = {} div_body = { "aggs" : { "genres" : { "terms" : { "field" : "filetype" } } } } div_result = self.raw_search(div_body)['aggregations'] for d in div_result['genres']['buckets']: diversity[d['key']] = d['doc_count'] return diversity
class HouseStateManager (object): def __init__(self): self.sc = SimpleCache() self.databaseService = database_service.DatabaseService() self.devicesControl = devices_control.DevicesControl() self.save_current_office_state(room_state.RoomState(0, self.format_current_time(), [], 0, 0, 0)) self.save_current_bedroom_state(room_state.RoomState(1, self.format_current_time(), [], 0, 0, 0)) print self.sc.get("current_office_state") def save_current_office_state(self, houseState): houseState.hour = self.format_current_time() self.sc.set("current_office_state", houseState) def save_current_bedroom_state(self, houseState): houseState.hour = self.format_current_time() self.sc.set("current_bedroom_state", houseState) def get_current_house_state(self): return self.sc.get("current_house_state") def get_current_office_state(self): return self.sc.get("current_office_state") def get_current_bedroom_state(self): return self.sc.get("current_bedroom_state") def save_house_state_in_db(self, houseState): houseState.hour = self.format_current_time() self.databaseService.save_house_state(houseState) def format_current_time(self): formatted_time = float(datetime.datetime.now().hour) if datetime.datetime.now().minute > 30: formatted_time = formatted_time + 0.5 return formatted_time def change_office_temperature(self, new_temperature): office_state = self.get_current_office_state() office_state.temperature = new_temperature self.save_current_office_state(office_state) self.devicesControl.change_device(new_temperature,"temperature","office") def change_bedroom_temperature(self, new_temperature): bedroom_state = self.get_current_bedroom_state() bedroom_state.temperature = new_temperature self.save_current_bedroom_state(bedroom_state) self.devicesControl.change_device(new_temperature,"temperature", "bedroom") def change_office_light(self, new_light): office_state = self.get_current_office_state() office_state.light = new_light self.save_current_office_state(office_state) self.devicesControl.change_device(new_light,"light","bedroom") def change_bedroom_light(self, new_light): bedroom_state = self.get_current_bedroom_state() bedroom_state.light = new_light self.save_current_bedroom_state(bedroom_state) self.devicesControl.change_device(new_light, "light","bedroom") def change_office_curtain(self, new_curtain): office_state = self.get_current_office_state() office_state.curtain = new_curtain self.save_current_office_state(office_state) self.devicesControl.change_device(new_curtain, "curtain","office") def change_bedroom_curtain(self, new_curtain): bedroom_state = self.get_current_bedroom_state() bedroom_state.curtain = new_curtain self.save_current_bedroom_state(bedroom_state) self.devicesControl.change_device(new_curtain, "curtain", "bedroom")
brightness = int('0'+request.args['brightness']) if 'brightness' in request.args else None if 'cam' in request.args: cam = request.args['cam'] if re.match(ur'^/dev/video\d$', cam): camera_id = cam elif recently_used_cam == cam: camera_id = recently_used_camera else: camera_id = get_camera_by_name(cam) if camera_id is not None: recently_used_cam = cam recently_used_camera = camera_id rotation=app.config['CAMERA_ROTATION'] if 'CAMERA_ROTATION' in app.config else None cam_slug = "%s-%s-%s" % (camera_id, rotation, brightness) filename = cache.get(cam_slug) status = 200 if filename is None: filename = os.path.join(app.config['TMP_IMAGES_PATH'], 'live-%s.jpeg' % (int(time.time()) % 10)) camera.get_webcam_image(filename, camera=camera_id, rotation=rotation, brightness=brightness) cache.set(cam_slug, filename, timeout=2) else: status = 203 return send_file(filename if os.path.exists(filename) else 'static/gdoor.jpg', mimetype='image/jpeg'), status @app.route('/action/gdoor') def action_gdoor(): gpio.pulse(1) # TODO: schedule door check here return jsonify(**get_app_state())
class JWTAuthenticationBackend(AuthenticationBackend): # pylint: disable=too-many-instance-attributes """JWT authentication backend for AuthenticationMiddleware.""" def __init__(self): """Initize app.""" self.algorithm = 'RS256' self.prefix = 'bearer' self.well_known_config = None self.well_known_obj_cache = None self.jwks_uri = None self.issuer = None self.audience = None self.client_secret = None self.cache = None self.caching_enabled = False self.jwt_oidc_test_mode = False self.jwt_oidc_test_public_key_pem = None self.jwt_oidc_test_private_key_pem = None def init_app(self, test_mode: bool = False): """Initize app.""" self.jwt_oidc_test_mode = test_mode # # CHECK IF WE'RE RUNNING IN TEST_MODE!! # if not self.jwt_oidc_test_mode: self.algorithm = get_api_settings().JWT_OIDC_ALGORITHMS # If the WELL_KNOWN_CONFIG is set, then go fetch the JWKS & ISSUER self.well_known_config = get_api_settings( ).JWT_OIDC_WELL_KNOWN_CONFIG if self.well_known_config: # try to get the jwks & issuer from the well known config with urlopen(url=self.well_known_config) as jurl: self.well_known_obj_cache = json.loads( jurl.read().decode('utf-8')) self.jwks_uri = self.well_known_obj_cache['jwks_uri'] self.issuer = self.well_known_obj_cache['issuer'] else: self.jwks_uri = get_api_settings().JWT_OIDC_JWKS_URI self.issuer = get_api_settings().JWT_OIDC_ISSUER # Setup JWKS caching self.caching_enabled = get_api_settings().JWT_OIDC_CACHING_ENABLED if self.caching_enabled: self.cache = SimpleCache(default_timeout=get_api_settings(). JWT_OIDC_JWKS_CACHE_TIMEOUT) self.audience = get_api_settings().JWT_OIDC_AUDIENCE self.client_secret = get_api_settings().JWT_OIDC_CLIENT_SECRET @classmethod def get_token_from_header(cls, authorization: str, prefix: str): """Get token from header.""" try: scheme, token = authorization.split() except ValueError: raise AuthenticationError( 'Could not separate Authorization scheme and token') if scheme.lower() != prefix.lower(): raise AuthenticationError( f'Authorization scheme {scheme} is not supported') return token async def authenticate(self, request): # pylint: disable=arguments-renamed """Authenticate the token.""" if 'Authorization' not in request.headers: return None auth = request.headers['Authorization'] token = self.get_token_from_header(authorization=auth, prefix=self.prefix) if self.jwt_oidc_test_mode: # in test mode, use a publick key to decode token directly. try: payload = jwt.decode(str.encode(token), self.jwt_oidc_test_public_key_pem, algorithms=self.algorithm) except jwt.InvalidTokenError as e: raise AuthenticationError(str(e)) else: # in production mod, get the public key from jwks_url try: unverified_header = jwt.get_unverified_header(token) except jwt.PyJWTError: raise AuthenticationError( 'Invalid header: Use an RS256 signed JWT Access Token') if unverified_header['alg'] == 'HS256': raise AuthenticationError( 'Invalid header: Use an RS256 signed JWT Access Token') if 'kid' not in unverified_header: raise AuthenticationError( 'Invalid header: No KID in token header') rsa_key = self.get_rsa_key(self.get_jwks(), unverified_header['kid']) if not rsa_key and self.caching_enabled: # Could be key rotation, invalidate the cache and try again self.cache.delete('jwks') rsa_key = self.get_rsa_key(self.get_jwks(), unverified_header['kid']) if not rsa_key: raise AuthenticationError( 'invalid_header: Unable to find jwks key referenced in token' ) public_key = RSAAlgorithm.from_jwk(json.dumps(rsa_key)) try: payload = jwt.decode(token, public_key, algorithms=self.algorithm, audience=self.audience) except jwt.InvalidTokenError as e: raise AuthenticationError(str(e)) return AuthCredentials(['authenticated' ]), JWTUser(username=payload['username'], token=token, payload=payload) def get_jwks(self): """Get jwks from well known config endpoint.""" if self.caching_enabled: return self._get_jwks_from_cache() return self._fetch_jwks_from_url() def _get_jwks_from_cache(self): jwks = self.cache.get('jwks') if jwks is None: jwks = self._fetch_jwks_from_url() self.cache.set('jwks', jwks) return jwks def _fetch_jwks_from_url(self): with urlopen(url=self.jwks_uri) as jsonurl: return json.loads(jsonurl.read().decode('utf-8')) def get_rsa_key(self, jwks, kid): # pylint: disable=no-self-use """Get a public key.""" rsa_key = {} for key in jwks['keys']: if key['kid'] == kid: rsa_key = { 'kty': key['kty'], 'kid': key['kid'], 'use': key['use'], 'n': key['n'], 'e': key['e'] } return rsa_key def create_testing_jwt(self, claims, header): """Create test jwt token.""" token = jwt.encode(claims, self.jwt_oidc_test_private_key_pem, headers=header, algorithm='RS256') return token.decode('utf-8') def set_testing_keys(self, private_key, public_key): """Set test keys.""" self.jwt_oidc_test_private_key_pem = private_key self.jwt_oidc_test_public_key_pem = public_key
class SearchService: # 初始化 def __init__(self): self.cache=SimpleCache() # 判断是否有筛选条件 def findFilter(self,params): if len(params["school"])>0: return True if len(params["field"]) >0: return True if len(params["h_index"]) > 0: return True def findFilter2(self,params): if "school"in params and len(params["school"])>0: return True if "code"in params and len(params["code"]) >0: return True if "name"in params and len(params["name"]) >0: return True return False # 找到搜索缓存的记录 def getKey2(self,params): key = ['keyword'] p={} for k in key: p[k]=params[k] return str(p) def getKey(self,params): key = ['keyword', 'name'] p={} for k in key: p[k]=params[k] return str(p) # 得到查询结果 def getSearchResult(self,params): re={} key=params['keyword'] temp=jeibaUitl.cut(params['keyword']) if len(temp)==0: if len(key)==0: params['keyword']=[] else: temp=[key] params['keyword']=temp else: params['keyword'] = temp temp=paperSearch.searchdao(params) re['num'] = temp["num"] key = self.getKey(params) if not self.findFilter(params): re['filter'] = self.getfilter(temp["filter"]) self.cache.set(key, re['filter'], timeout=5 * 60*60) else: value=self.cache.get(key) if value is None: re['filter'] = self.getfilter(temp["filter"]) self.cache.set(key, re['filter'], timeout=5 * 60 * 60) else: re['filter']=value re['allPage']=int(re['num']/params['pPageNum']) if re['num']%params['pPageNum']!=0: re['allPage']+=1 re['params']=params for r in temp['result']: r["link"]="/main/expert/"+str(r["author_id"]) # re['result']=temp['result'] re['result']=self.setLight(temp['result'],params['keyword']) if len(re['result'])>0 and "light_abstract" not in re['result'][0].keys(): for r in re['result']: r['light_abstract']=r['abstract'] return re def getSearchResult2(self,params): requrl ="http://"+ environment['se']["host"] + ":" + str(environment['se']["port"])+ environment['se']["url"] s = json.dumps(params) r = requests.post(requrl, data=s) res =r.text res=eval(res) t_id=[str(r[0]) for code in res for r in res[code]] if len(t_id)==0: teacher={} else: teacher=expertService.get_infosByIds(t_id) in_value={} t_value={str(r[0]):str(r[1]) for code in res for r in res[code]} for t in teacher: c=teacher[t] k=str(c["school_id"])+"_"+c["institution"] if k not in in_value: in_value[k]=0 in_value[k]+=float(t_value[str(c['id'])]) temp = sorted(in_value.items(), key=lambda item: item[1], reverse=True)[0:3] teacher_temp={} for t in temp: for ta in teacher: c=teacher[ta] k = str(c["school_id"]) + "_" + c["institution"] if k==t[0]: teacher_temp[ta]=c teacher=teacher_temp for id in teacher: if teacher[id]["fields"] is None or len(teacher[id]["fields"])==0: teacher[id]["fields"]=[] else: item=eval(teacher[id]["fields"]) te = sorted(item.items(), key=lambda item: item[1], reverse=True) teacher[id]["fields"]=[t[0] for t in te[0:5]] if teacher[id]["age"] is None: teacher[id]["age"]='' if teacher[id]["eduexp"] is None or len(teacher[id]["eduexp"])==0: teacher[id]["eduexp"]=[] else: teacher[id]["eduexp"] =teacher[id]["eduexp"].split('\n') school=list({str(teacher[t]['school_id']) for t in teacher}) if len(school)==0: school={} else: school = schoolService.get_infosByIds(school) for s in school: school[s]["important"]=schoolService.get_important_discipline_num(s) school[s]["main_lab"] = schoolService.get_main_lab_num(school[s]['name']) if school[s]["characteristic"]=="-": school[s]["characteristic"]=[] elif school[s]["characteristic"]=="-211": school[s]["characteristic"] = [211] elif school[s]["characteristic"]=="985-211": school[s]["characteristic"] = [985,211] result=[] temp2 = sorted(t_value.items(), key=lambda item: item[1], reverse=True) for t in temp: te=t[0].split('_') item={} item['school']=school[te[0]] item['institution']={"name":te[1],"main_lab":schoolService.get_main_lab(te)} teacher_list=[] for t2 in temp2: if str(t2[0]) in teacher: c=teacher[str(t2[0])] k = str(c["school_id"]) + "_" + c["institution"] if k==t[0] and len(teacher_list)<3: teacher_list.append(c) item['teacher']=teacher_list result.append(item) return result # 对查询的结果显示不同的样式 def setLight(self,result,keys): for r in result: for k in keys: r['abstract']=r['abstract'].replace(k,"<span class='light'>"+k+"</span>") return result # 对查询的结构生成筛选条件和数量 def getfilter(self, result): f = {} f['hindexs']={} f['hindexs']['>30']=0 f['hindexs']['20-29'] = 0 f['hindexs']['10-19'] = 0 f['hindexs']['<10'] = 0 f['fields']=[] f['schools'] = [] field=[] schools = [] for r in result: if r['h_index']>=30 : f['hindexs']['>30']+=1 elif r['h_index']>=20: f['hindexs']['20-29'] += 1 elif r['h_index'] >= 10: f['hindexs']['10-19'] += 1 else : f['hindexs']['<10'] += 1 field.extend(r['fields']) schools.append(r['school']) temp=[] temp.append({'value':">30",'num':f['hindexs']['>30']}) temp.append({'value': "20-29", 'num': f['hindexs']['20-29']}) temp.append({'value': "10-19", 'num': f['hindexs']['10-19']}) temp.append({'value': "<10", 'num': f['hindexs']['<10']}) f['hindexs']=temp c2 = Counter(field).items() c1=sorted(c2, key=lambda x: x[1], reverse=True) if len(c1)-5>=0: n=5 else: n=len(c1) for i in range(n): f['fields'].append({'value':c1[i][0],'num':c1[i][1]}) c3 = Counter(schools).items() c0=sorted(c3, key=lambda x: x[1], reverse=True) if len(c0)-5>=0: n=5 else: n=len(c0) for i in range(n): f['schools'].append({'value':c0[i][0],'num':c0[i][1]}) return f # 得到查询结果 # 对查询的结构生成筛选条件和数量 def getfilter2(self, result,teacher): f={} f["codes"]=[] f["schools"] = [] codes={} schools={} for code in result: codes[code]=len(result[code]) for t in result[code]: school_id=teacher[str(t[0])]["school_id"] if school_id in schools: schools[school_id]+=1 else: schools[school_id]= 1 if 0 in schools: del schools[0] school_id=[str(id) for id in schools] if len(school_id)==0: s={} else: s=schoolService.get_infosByIds(school_id) code_id=[c for c in codes] if len(code_id)==0: c={} else: c = schoolService.get_infosByCodes(code_id) c2 = Counter(codes).items() c1=sorted(c2, key=lambda x: x[1], reverse=True) if len(c1)-5>=0: n=5 else: n=len(c1) for i in range(n): f['codes'].append({'value':c1[i][0],'num':c1[i][1],"name":c[c1[i][0]]["name"]}) c2 = Counter(schools).items() c1=sorted(c2, key=lambda x: x[1], reverse=True) if len(c1)-5>=0: n=5 else: n=len(c1) for i in range(n): f['schools'].append({'value':c1[i][0],'num':c1[i][1],"name":s[str(c1[i][0])]["name"]}) return f # 得到查询结果 def getIndexSearchResult(self, params): key = params['keyword'] temp = jeibaUitl.cut(params['keyword']) if len(temp) == 0: if len(key) == 0: params['keyword'] = [] else: temp = [key] params['keyword'] = temp else: params['keyword'] = temp temp = paperSearch.IndexSearchdao(params) for r in temp['result']: r["link"] = "/main/expert/" + str(r["author_id"]) return temp def getHotSearch(self,params): if params["type"] is None: return [] r=taskService.getHotSearch(params) if params["type"]=="专家": ids = [t["value"] for t in r] expers=expertService.get_infosByIds(ids) result=[{"name":expers[t['value']]["name"],"url":"/main/expert/"+t["value"]} for t in r] return result else: result = [{"name":t["value"] , "url": "/main/school/" + t["value"]} for t in r] return result