def main(): # Setting pattern for redis pattern = 'factorial:' # Searching for parameters passed in the terminal ap = ArgumentParser() ap.add_argument("-a", "--address", required=True, help="address for connection with redis") ap.add_argument("-p", "--port", required=True, help="port for connection with redis") args = vars(ap.parse_args()) # Instantiating an object that connects to a redis database using the parameters passed db = Database(host=args['address'], port=int(args['port']), db=0) print("--- Welcome to Factorial Calculator ---") while True: number = input("Please enter a non negative integer number: ") # Checking if the entered string is alphanumeric or, if it is a number, less than 0 try: if number.isalpha() == True or int(number) < 0: print("You didn't write non negative integer!") else: # Checking if the number has already been calculated and is in the cache if(db.get(pattern + number) is None): print("--- Calculating factorial ---") result = factorial(int(number)) print(result) # Setting result in cache db[pattern + number] = result else: print("--- Fetching result in cache ---") print(db[pattern + number].decode()) except ValueError: print("You didn't write non negative integer!")
def query_cache(key=None, st=None, request=None, project_id=None): project_id = project_id if project_id \ else json.loads(request.session['project_id']) database = Database() ac_key_tag = namespace(project_id) ac_ids = database.autocomplete(namespace=ac_key_tag) return ac_ids.search(str(st))
def redis_connection(): if redis_pw == "NONE": return Database(host=redis_host, port=redis_port, db=0) else: return Database(host=redis_host, port=redis_port, db=0, password=redis_pw)
def get_consumer_group(self, create=False): db = Database(host=REDIS_HOST, port=REDIS_PORT) stream_keys = ['all_observations'] cg = db.time_series('cg-obs', stream_keys) if create: for stream in stream_keys: db.xadd(stream, {'data': ''}) if create: cg.create() cg.set_id('$') return cg.all_observations
def __init__(self, service_name, namespace, paths_dict, socket_path, hardfail, consumer_name='CANNON', **kwargs): self.db = Database(unix_socket_path=str(socket_path.resolve())) self.conf = ConfigManager(namespace, self.db) self.ledger = CannonLedger(self, service_name, consumer_name) self.paths_dict = paths_dict service = self.conf.service.get(service_name) if not service: print(f'No service named "{service_name}"') exit(1) if hardfail else None key_test = [i for i in service._data if i in service._keys] if not key_test: print(f'Service named "{service_name}" missing {key_test}') exit(1) if hardfail else None # check for run context class if service.shell not in self.paths_dict: print(f'No python file named {service.shell}.py') exit(1) if hardfail else None elif service.collector not in self.paths_dict: print(f'No python file named {service.collector}.py') exit(1) if hardfail else None self.service = service self.dataframes = BlindDataFrames(self)
def get_sim_log(db: Database, tournament_id, left_team, right_team) -> List[str]: key = get_sim_log_key(tournament_id, left_team, right_team) data = db.get(key) if data is not None: data = json.loads(data) return data
def polydmon(event, community, sql, redis, consumer_name, quiet): db = Database(redis) communities = community if 'all' not in community else polyd_communities if 'all' in event: streams = [f'polyd-{c}-all' for c in communities] else: streams = [] for c in communities: for e in events: streams.append(f'polyd-{c}-{e}') c = consumer.EventConsumer(streams, __name__, consumer_name, db) if not quiet: event_handlers = [print_event] else: event_handlers = [] if sql: from .models import EventHandler sql_handler = EventHandler(sql) event_handlers.append(sql_handler.handle_event) # this will quit if either thread exits for event in c.iter_events(): for handler in event_handlers: handler(event)
def __init__(self, db: Database): self.stream: Stream = db.Stream(EVENT_STREAM) self.sender_tag: str = fftbg.server.get_name() self.last_id = '$' messages = self.stream.revrange('+', '-', 1) if messages: self.last_id = messages[-1][0]
def mock_redis(): """Monkey patch service cache with mocked redis.""" from renku.service.cache.base import BaseCache from renku.service.cache.models.user import User from renku.service.cache.models.job import Job from renku.service.cache.models.file import File from renku.service.cache.models.project import Project from renku.service.jobs.queues import WorkerQueues monkey_patch = MonkeyPatch() with monkey_patch.context() as m: fake_redis = fakeredis.FakeRedis() fake_model_db = Database(connection_pool=fake_redis.connection_pool) m.setattr(WorkerQueues, 'connection', fake_redis) m.setattr(BaseCache, 'cache', fake_redis) m.setattr(BaseCache, 'model_db', fake_model_db) m.setattr(Job, '__database__', fake_model_db) m.setattr(User, '__database__', fake_model_db) m.setattr(File, '__database__', fake_model_db) m.setattr(Project, '__database__', fake_model_db) yield monkey_patch.undo()
def get_importance(db: Database, tournament_id, left_team, right_team) -> List[dict]: key = get_importance_key(tournament_id, left_team, right_team) data = db.get(key) if data is not None: data = json.loads(data) return data
def __init__(self, can_channel="can0"): self.logger = logging.getLogger() self.start_time = None self.rdb = redis.Redis(host='localhost', port=6379, db=0) self.idb = InfluxDBClient('localhost', 8086, 'USERNAME', 'PASSWORD', 'DATABASE') self.can_channel = can_channel self.describer = None self.init_prettyj1939(pgns=True, spns=True) # self.analyse_d = Dict(redis=self.rdb, key='cansnap') # self.simple_series = RedisSimpleTimeSeries(client=self.rdb) self.wdb = Database(host='localhost', port=6379, db=0) # Database() self.dbstream = None self.streamid = None self.streamlist = self.wdb.List(f'{self.can_channel}') self.sumcount = 0 # self.ids=Dict(redis=self.rdb, key='cansnap')#self.wdb.Hash('cansnap') self.ids = {}
def batch_get(self, query=None, cache_ttl=None): cache_on = bool(cache_ttl) if isinstance(obj, Query): request_params = obj.json(camelify=True) elif isinstance(obj, dict): request_params = _camelify(obj) else: raise InvalidJsonError() request_params['viewId'] = self.view_id self.request_query = {'reportRequests': request_params} execute = self.analytics(body=self.request_query).execute if cache_on: if not self.cache: self.cache = Database(**redis_args).cache() deco = self.cache.cached(self.key_fn, timeout=self.cache_ttl, metrics=True) execute = deco(execute) resp = execute() return resp
def __init__(self, topics, poll_interval=0.1, start=False, group="test", **kwargs): from walrus import Database self.consumer = None self.topics = topics self.group = group self.poll_interval = poll_interval self.db = Database() self.consumer = self.db.consumer_group(self.group, self.topics) self.consumer.create() # Create the consumer group. self.consumer.set_id('$') # 不会从头读 super(from_redis, self).__init__(ensure_io_loop=True, **kwargs) self.stopped = True if start: self.start()
class StreamHelperOps(): def __init__(self): self.stream_keys = ['all_observations'] self.db = Database(host=url.hostname, port=url.port) def create_push_cg(self): self.get_push_cg(create=True) def get_push_cg(self, create=False): cg = self.db.time_series('cg-push', self.stream_keys) if create: for stream in self.stream_keys: self.db.xadd(stream, {'data': ''}) cg.create() cg.set_id('$') return cg def create_pull_cg(self): self.get_pull_cg(create=True) def get_pull_cg(self, create=False): cg = self.db.time_series('cg-pull', self.stream_keys) if create: for stream in self.stream_keys: self.db.xadd(stream, {'data': ''}) cg.create() cg.set_id('$') return cg
class BaseCache: """Cache management.""" config_ = { 'host': REDIS_HOST, 'port': REDIS_PORT, 'db': REDIS_DATABASE, 'password': REDIS_PASSWORD, 'retry_on_timeout': True, 'health_check_interval': int(os.getenv('CACHE_HEALTH_CHECK_INTERVAL', 60)) } cache = redis.Redis(**config_) model_db = Database(**config_) def set_record(self, name, key, value): """Insert a record to hash set.""" if isinstance(value, dict): value = json.dumps(value) self.cache.hset(name, key, value) def invalidate_key(self, name, key): """Invalidate a cache `key` in users hash set.""" try: self.cache.hdel(name, key) except RedisError: pass def get_record(self, name, key): """Return record values from hash set.""" result = self.cache.hget(name, key) if result: return json.loads(result.decode('utf-8')) def get_all_records(self, name): """Return all record values from hash set.""" return [ json.loads(record.decode('utf-8')) for record in self.cache.hgetall(name).values() ] def scan_iter(self, pattern): """Scan keys to return all user cached elements.""" return self.cache.scan_iter(match=pattern) def hash_table(self, hash_table): """Return hash table.""" return self.cache.hgetall(hash_table)
def event_generator(event, community, redis, retry, quiet): db = Database(redis) communities = community if 'all' not in community else polyd_communities.keys( ) ws_q = queue.Queue(maxsize=1000) ws_threads = [ WSThread(c, polyd_communities[c], ws_q, retry) for c in communities ] # lazily create producers producers = { f'polyd-{c}-all': producer.EventProducer(f'polyd-{c}-all', db, max_len=20000) for c in communities } for t in ws_threads: t.start() events = event def handler(_, __): for t in ws_threads: t.stop_thread = True signal.signal(signal.SIGINT, handler) # this will quit if either thread exits for event in iter(ws_q.get, None): print(event) if event.event in events or 'all' in events: stream_name = f'polyd-{event.community}-{event.event}' community_stream = f'polyd-{event.community}-all' if stream_name not in producers: producers[stream_name] = producer.EventProducer(stream_name, db, max_len=20000) producers[stream_name].add_event(event) producers[community_stream].add_event(event) if not quiet: logger.info(str(event)) for t in ws_threads: t.join()
def __init__(self, baseurl, *args, headers=None, cache_ttl=0, logger=None, redis_host='localhost', redis_port=6379, redis_db=0, redis_password=None, redis_socket_timeout=None, **kw ): self.db = Database( host=redis_host, port=redis_port, db=redis_db, password=redis_password, socket_timeout=redis_socket_timeout ) super(PrefixedURLSession, self).__init__(*args, **kw) self.baseurl = baseurl self.cache_ttl = cache_ttl self.logger = logger or logging.getLogger() if headers: self.headers.update(headers) self.__post_init__()
def polyd_to_s3(community, redis, consumer_name, access_key, secret_key, bucket, endpoint, region, expires, psd_key, quiet): session = requests.Session() session.headers.update({'Authorization': psd_key}) db = Database(redis) communities = community if 'all' not in community else polyd_communities streams = [f'polyd-{c}-bounty' for c in communities] c = consumer.EventConsumer(streams, 'polyd_to_s3', consumer_name, db, consume_from_end=True) logger = logging.get_logger() if quiet: import logging as l logger.setLevel(l.WARN) # for now, we don't send these to 'all' which is really the websocket events producers = { c: producer.EventProducer(f'polyd-{c}-downloaded', db, max_len=20000) for c in communities } executor = thread.BoundedExecutor(100, 16) for event in c.iter_events(): logger.info('Processing: %s', event) # only process FILE artifacts if event.artifact_type != 'FILE': continue client = transfer.get_client(access_key, secret_key, endpoint, region) key = event.uri executor.submit(transfer.event_to_s3, event, bucket, key, client, producers[event.community], session, expires=expires) executor.shutdown()
class Client(object): def __init__(self, credentials, view_id, redis_args=None): credentials = service_account.Credentials.from_service_account_info(credentials) # Build the service object. self.redis_args = redis_args or {} self.view_id = view_id self.build(credentials) def build(self, credentials): self.analytics = build('analyticsreporting', 'v4', credentials=credentials).reports().batchGet def batch_get(self, query=None, cache_ttl=None): cache_on = bool(cache_ttl) if isinstance(query, Query): request_params = query.json(camelify=True) elif isinstance(query, dict): request_params = _camelify(query) else: raise InvalidJsonError() request_params['viewId'] = self.view_id self.request_query = { 'reportRequests': request_params } execute = self.analytics(body=self.request_query).execute if cache_on: if not self.cache: self.cache = Database(**redis_args).cache() deco = self.cache.cached(self.key_fn, timeout=self.cache_ttl, metrics=True) execute = deco(execute) resp = execute() return resp def key_fn(self, a, k): return hashlib.md5( pickle.dumps((self.request_query, a, k)) ).hexdigest() def __getattr__(self, name): if name == 'query': return Query()
def clamav_scan(community, redis, consumer_name, access_key, secret_key, endpoint, region, psd_key, eth_key, clamav_host, threads, quiet): eth_key = unhexlify(eth_key) session = requests.Session() session.headers.update({'Authorization': psd_key}) db = Database(redis) communities = community if 'all' not in community else polyd_communities streams = [f'polyd-{c}-downloaded' for c in communities] c = consumer.EventConsumer(streams, 'clamav_scan', consumer_name, db, consume_from_end=True) logger = logging.get_logger() psd_api = api.PolydAPI(psd_key) if quiet: import logging as l logger.setLevel(l.WARN) # for now, we don't produce anything on finish. # producers = {c: producer.EventProducer(f'polyd-{c}-downloaded', db) for c in communities} executor = thread.BoundedExecutor(10, threads) for event in c.iter_events(): logger.info('Processing: %s, %s', event, event.bounty) client = scan.get_client(access_key, secret_key, endpoint, region) executor.submit(scan.scan_event, event, client, clamav_host, psd_api, eth_key, c) executor.shutdown()
from walrus import Database REDIS_URL = 'redis://redis:6379' db = Database.from_url(REDIS_URL) cache = db.cache() def get_visited_links(): return cache.get('visited', None) or [] def append_visited_links(link): visited = cache.get('visited', None) or [] cache.set('visited', visited + [link]) processing = cache.get('processing', None) or [] cache.set('processing', processing + [link]) def set_item_data(data): if data: data = cache.get('data', None) or [] cache.set('data', data + [data]) def pop_processing(): pop_link = None processing = cache.get('processing', None) or [] if processing: pop_link = processing.pop() cache.set('processing', processing)
import settings from esl_events_config import ESL_EVENT_FAMILY_DICT from cache import RefreshCache esl_url_root = 'http://api.esl.tv/v1' # esl_url_root = 'http://cdn1.api.esl.tv/v1' facebook_graph_url_root = 'https://graph.facebook.com' esl_event_url = esl_url_root + '/event/bydomainurl?livedomain={esl_event_domain}&liveurl={esl_event_path}' esl_channel_url = esl_url_root + '/channel/eventchannels?pid={esl_event_id}&hideservice=web' facebook_graph_page_url = facebook_graph_url_root + '/{facebook_id}?fields=link,username&access_token={facebook_app_id}|{facebook_app_secret}' facebook_graph_page_live_videos_url = facebook_graph_url_root + '/{facebook_page_username}/live_videos?access_token={facebook_access_token}' facebook_stream_fetch_url = 'https://www.facebook.com/video/tahoe/async/{facebook_video_id}/?chain=true&isvideo=true&originalmediaid={facebook_video_id}&playerorigin=permalink&playersuborigin=tahoe&ispermalink=true&numcopyrightmatchedvideoplayedconsecutively=0&dpr=1' # dpr = device pixel ratio facebook_video_embed_url = 'https://www.facebook.com/embedvideo/video.php' db = Database(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.REDIS_DB) cache = RefreshCache(db, name='cache', default_timeout=3600) esl_event_family_dict = ESL_EVENT_FAMILY_DICT def set_esl_event_family_dict(): db['esl_event_family_dict'] = pickle.dumps(esl_event_family_dict, pickle.HIGHEST_PROTOCOL) def get_esl_event_family_dict(): global esl_event_family_dict esl_event_family_dict = pickle.loads(db['esl_event_family_dict']) return esl_event_family_dict
import hashlib import logging import string import requests try: import _pickle as pickle except: import pickle from urllib.parse import urljoin from walrus import Database db = Database() cache = db.cache() class BaseRequest: def __init__(self, session, type, path, cache_ttl=None): self.cache_ttl = cache_ttl if cache_ttl is not None else session.cache_ttl self.cache_on = bool(self.cache_ttl) self.session = session self.logger = session.logger self.type = type self.path = path self.send = self.method() if self.cache_on: deco = cache.cached(self.key_fn, timeout=self.cache_ttl, metrics=True)
import urllib2 import json import os import ssl from collections import OrderedDict from flask import Flask from walrus import Database import default_settings app = Flask(__name__) app.config.from_object(default_settings) if 'BUTLERCAM_SETTINGS_FILE' in os.environ: app.config.from_envvar('BUTLERCAM_SETTINGS_FILE') PIPELINES = app.config['PIPELINES'] db = Database(host=app.config['REDIS_HOST'], db=0) cache = db.cache(default_timeout=app.config['CACHE_TIMEOUT']) def get_all_builds(pipeline): """ Given the name of a pipeline, the saved builds for that pipeline are retrieved, Jenkins is queried, and the combined results are returned. """ pref_url = PIPELINES[pipeline][1] api_url = pref_url + '/api/json' context = ssl._create_unverified_context() data = urllib2.urlopen(api_url, context=context).read() parsed = json.loads(data) builds = get_saved_builds(pipeline)
def xstream(timeseries=True, name=None): db = Database() if timeseries: return (db.time_series('ExecutablePrice', ['xbid', 'xask'])) else: return (db.Stream(name))
# create an instance of User v1 user = UserModel(name=random.choice([ "Juan", "Peter", "Michael", "Moby", "Kim", ]), age=random.randint(1, 50)) msgid = consumer_group.my_stream.add({"message": user.serialize()}) print(f"Producing message {msgid}") print("Producer finished....") print("#" * 80) sleep(2) if __name__ == "__main__": db = Database() stream_name = 'my-stream' db.Stream(stream_name) # Create a new stream instance # create the consumer group consumer_group = db.consumer_group('my-consumer-group-1', [stream_name]) consumer_group.create() # Create the consumer group. consumer_group.set_id('$') produce(consumer_group) consume(consumer_group)
from walrus import Database, Model, ListField, SetField, HashField from config import REDIS_URL db = Database.from_url(REDIS_URL) LISTENER_TASK_KEY = 'listener:task_id' class RBase(Model): __database__ = db def to_dict(self): data = {} for name, field in self._fields.items(): if name in self._data: data[name] = field.db_value(self._data[name]) else: if isinstance(field, ListField): type_func = list elif isinstance(field, SetField): type_func = set elif isinstance(field, HashField): type_func = dict else: type_func = lambda x: x data[name] = type_func(getattr(self, name)) return data @classmethod def get(cls, id): try: return super().get(cls.id==id)
import urllib2 from walrus import Database db = Database() autocomplete = db.autocomplete(namespace='stocks') def load_data(): url = 'http://media.charlesleifer.com/blog/downloads/misc/NYSE.txt' contents = urllib2.urlopen(url).read() for row in contents.splitlines()[1:]: ticker, company = row.split('\t') autocomplete.store( ticker, company, {'ticker': ticker, 'company': company}) def search(p, **kwargs): return autocomplete.search(p, **kwargs) if __name__ == '__main__': autocomplete.flush() print 'Loading data (may take a few seconds...)' load_data() print 'Search stock data by typing a partial phrase.' print 'Examples: "uni sta", "micro", "food", "auto"' print 'Type "q" at any time to quit' while 1: cmd = raw_input('? ') if cmd == 'q':
import os import requests from walrus import Database from utils.parsers.package_parser import PackageParser base_url = "https://replicate.npmjs.com" db = Database(host=os.environ.get('REDIS_HOST'), port=os.environ.get('REDIS_PORT'), db=os.environ.get('REDIS_DB'), password=os.environ.get('REDIS_PASSWORD')) cache = db.cache() WEEKS_IN_SECONDS = 60 * 60 * 24 * 7 class NodeUtil: def parse_package(self, package_path): package_parser = PackageParser(package_path) package_data = package_parser.parse() return package_data def fetch_info(self, package_list: list) -> dict: info_dict = {} for req in package_list: info = self.fetch_npm(req) info_dict[req] = info return info_dict
import os import unittest from distutils.version import StrictVersion from walrus import Database HOST = os.environ.get('WALRUS_REDIS_HOST') or '127.0.0.1' PORT = os.environ.get('WALRUS_REDIS_PORT') or 6379 db = Database(host=HOST, port=PORT, db=15) REDIS_VERSION = None def requires_version(min_version): def decorator(fn): global REDIS_VERSION if REDIS_VERSION is None: REDIS_VERSION = db.info()['redis_version'] too_old = StrictVersion(REDIS_VERSION) < StrictVersion(min_version) return unittest.skipIf(too_old, 'redis too old, requires %s' % min_version)(fn) return decorator def stream_test(fn): test_stream = os.environ.get('TEST_STREAM') if not test_stream: return requires_version('4.9.101')(fn) else:
from walrus import Database, Model, TextField, IntegerField, JSONField db = Database(host="localhost", port=6379, charset="utf-8", decode_responses=True) MODES = (('public', 'public'), ('private', 'private')) class Lobby(Model): __database__ = db name = TextField(primary_key=True) timestamp = IntegerField() mode = TextField() password = TextField() game = IntegerField(index=True) setup = IntegerField() owner = IntegerField() class Player(Model): __database__ = db name = TextField( primary_key=True) ## combination of game, lobbyname and playername lobby = TextField(index=True) user = IntegerField() data = JSONField()
import unittest from walrus import Database db = Database(db=15) class WalrusTestCase(unittest.TestCase): def setUp(self): db.flushdb() db._transaction_local.pipes = [] def tearDown(self): db.flushdb() db._transaction_local.pipes = [] def assertList(self, values, expected): values = list(values) self.assertEqual(len(values), len(expected)) for value, item in zip(values, expected): self.assertEqual(value, item)