def get_redis_lite(*args, **kwargs): try: imp.find_module('redislite') import redislite if args: settings_file = args[0] + '.settings' try: remove_settings = False with open(settings_file, 'r') as f: settings = json.load(f) if not os.path.exists(settings['unixsocket']): remove_settings = True except Exception as e: pass if remove_settings: os.remove(settings_file) return redislite.StrictRedis(*args, **kwargs) except ImportError: raise EnvironmentError('Redislite module is not available')
def resolve_host(target): """ :param target: str The host:port pair or path :return: """ target = target.strip() if target.startswith('redis://') or target.startswith('unix://'): return redis.StrictRedis.from_url(target) try: hostname, port = target.split(':') return redis.StrictRedis(host=hostname, port=int(port)) except ValueError: start = time.time() while True: try: redislite.StrictRedis.start_timeout = \ REDISLITE_LOAD_WAIT_TIMEOUT conn = redislite.StrictRedis(target) except BusyLoadingError: logging.info('%s loading', target) elapsed = time.time() - start if elapsed > REDISLITE_LOAD_WAIT_TIMEOUT: raise BusyLoadingError('unable to load rdb %s' % target) time.sleep(REDISLITE_LOAD_WAIT_INTERVAL_SECS) continue if conn.info('persistence').get('loading', 0): logging.warn('%s loading', target) time.sleep(REDISLITE_LOAD_WAIT_INTERVAL_SECS) elapsed = time.time() - start if elapsed > REDISLITE_LOAD_WAIT_TIMEOUT: raise BusyLoadingError('unable to load rdb %s' % target) continue return conn
def test_shutdown_race_condition(self): """ Test that there is no race condition when a shutdown is run after a large data operation. """ if os.path.exists('bug.redis'): os.remove('bug.redis') db = redislite.StrictRedis('bug.redis') for key in range(10000): db.hset("h1", os.urandom(32), " " * 65536) db.save() db._cleanup() del db db = redislite.StrictRedis('bug.redis') db._cleanup() if os.path.exists('bug.redis'): os.remove('bug.redis')
def build_redis(port): if port is None: client = redislite.StrictRedis() else: client = redis.StrictRedis(port=int(port)) redpipe.reset() redpipe.connect_redis(client) return client
def work(identification, graph_name, step_to_do, redis_fn): global redis_connection, strict_redis_connection, sparql_server, step, step_graph step = step_to_do log('work ' + '[' + str(step) + ']') #for Collections step_graph = ConjunctiveGraph(sparqlstore.SPARQLStore(sparql_uri), graph_name) sparql_server = sparql.SPARQLServer(sparql_uri) redis_connection = redislite.Redis(redis_fn) strict_redis_connection = redislite.StrictRedis(redis_fn) gv_output_file_name = identification + '_' + str(step).zfill(7) + '.gv' if list(subjects(RDF.type, kbdbg.frame)) == []: log('no frames.' + '[' + str(step) + ']') put_last_bindings(step, []) return if (step == global_start - 1): gv_output_file_name = 'dummy' try: os.unlink(gv_output_file_name) except FileNotFoundError: pass gv_output_file = open(gv_output_file_name, 'w') e = Emitter(gv_output_file, step) e.generate_gv_image() gv_output_file.close() if (step == global_start - 1): return log('convert..' + '[' + str(step) + ']') #cmd, args = subprocess.check_output, ("convert", '-regard-warnings', "-extent", '6000x3000', gv_output_file_name, '-gravity', 'NorthWest', '-background', 'white', gv_output_file_name + '.svg') cmd, args = subprocess.check_output, ("dot", '-Tsvg', gv_output_file_name, '-O') try: r = cmd(args, stderr=subprocess.STDOUT) if r != b"": raise RuntimeError('[' + str(step) + '] ' + str(r)) except subprocess.CalledProcessError as e: log('[' + str(step) + ']' + e.output) log('convert done.' + '[' + str(step) + ']') if len(stats): print('stats:') for i in stats: print(i) #stats.clear() redis_connection._cleanup() strict_redis_connection._cleanup()
def _patch_up_redislite(): import redislite global _test_redis_client previous_redis_instance = _test_redis_client _test_redis_client = redislite.StrictRedis() try: yield _test_redis_client finally: _test_redis_client = previous_redis_instance
def redis_sync(self, db: int = 0) -> Redis: if config.get("redis.use_redislite"): REDIS_DB_PATH = os.path.join( config.get("redis.redislite.db_path", default_redislite_db_path)) return redislite.StrictRedis(REDIS_DB_PATH, decode_responses=True) self.red = ConsoleMeRedis( host=self.host, port=self.port, db=self.db, charset="utf-8", decode_responses=True, ) return self.red
def setUp(self): self.mockRedis = redislite.StrictRedis() self.agent = RandomPolicyAgent(self.mockRedis) self.actionSchema = { "type": "object", "properties": { "action": { "type": "string", "enum": [ "lookLeft", "lookUp", "lookRight", "lookDown", "moveLeft", "moveForward", "moveRight", "moveBack" ] }, "actuator": { "type": "string", "enum": ["look", "move"] }, "direction" : { "type": "string", "enum": ["left", "up", "right", "down", "back", "forward"] }, "factor": { "type" : "number" } }, "required": ["action", "actuator", "direction", "factor"] } self.actionSequenceSchema = { "type": "object", "properties": { "actions": { "type": "array", "items": self.actionSchema, "additionalItems": False } } }
def test_redis_log_small_chunks(self): r = redislite.StrictRedis() lines = r.redis_log_tail(4, width=20) self.assertIsInstance(lines, list) self.assertEqual(len(lines), 4)
def test_redis_log_many_lines(self): r = redislite.StrictRedis() lines = r.redis_log_tail(lines=99999) self.assertIsInstance(lines, list)
def test_redis_log_tail(self): r = redislite.StrictRedis() lines = r.redis_log_tail(2) self.assertIsInstance(lines, list) self.assertEqual(len(lines), 2)
def get_patcher(self): import redislite connection = redislite.StrictRedis() return patch("agir.api.redis._get_auth_redis_client", lambda: connection)
def work(serialized_graph, input_file_name, step, no_parallel, redis_fn): global redis_connection, strict_redis_connection strict_redis_connection = redis_fn log('work ' + '[' + str(step) + ']') redis_connection = redislite.Redis(redis_fn) strict_redis_connection = redislite.StrictRedis(redis_fn) gv_output_file_name = input_file_name + '_' + str(step).zfill(5) + '.gv' log('loads ' + '[' + str(step) + ']') g = pickle.loads(serialized_graph) #g = Graph(OrderedAndIndexedStore()) #for i in ujson.loads(serialized_graph): # g.add(i) #log('work' + str(id(g)) + ' ' + str(id(g.store)) + ' ' + str(id(g.store.indexes)) + ' ' + str(id(g.store.indexes['ttft'])) + ' ' + str(id(g.store.indexes['ttft'][rdflib.URIRef('http://kbd.bg/Rule1')]))) g.store.locked = True if list(g.subjects(RDF.type, kbdbg.frame)) == []: log('no frames.' + '[' + str(step) + ']') put_last_bindings(step, []) return if (step == global_start - 1): gv_output_file_name = 'dummy' try: os.unlink(gv_output_file_name) except FileNotFoundError: pass gv_output_file = open(gv_output_file_name, 'w') e = Emitter(g, gv_output_file, step) e.generate_gv_image() gv_output_file.close() if (step == global_start - 1): return log('convert..' + '[' + str(step) + ']') #cmd, args = subprocess.check_output, ("convert", '-regard-warnings', "-extent", '6000x3000', gv_output_file_name, '-gravity', 'NorthWest', '-background', 'white', gv_output_file_name + '.svg') cmd, args = subprocess.check_output, ("dot", '-Tsvg', gv_output_file_name, '-O') if True: try: r = cmd(args, stderr=subprocess.STDOUT) if r != b"": raise RuntimeError('[' + str(step) + '] ' + str(r)) except subprocess.CalledProcessError as e: log('[' + str(step) + ']' + e.output) log('convert done.' + '[' + str(step) + ']') else: def do_or_die(args): r = cmd(args, stderr=subprocess.STDOUT) if r != b"": log(r) raise RuntimeError(r) #exit() futures.append(graphviz_pool.submit(do_or_die, args)) redis_connection._cleanup() strict_redis_connection._cleanup()
#!/usr/bin/env python import unittest import string import random import webtest import os import shutil import redislite import napfs import falcon import hashlib from napfs.helpers import condense_byte_ranges, \ get_last_contiguous_byte redis_connection = redislite.StrictRedis(dbfilename='/tmp/test-napfs.db') NAPFS_DATA_DIR = '/tmp/test-napfs' NOT_FOUND_BODY = b'{"title": "404 Not Found"}' NOT_FOUND_BODY_LEN = str(len(NOT_FOUND_BODY)) def create_app(): if not os.path.exists(NAPFS_DATA_DIR): os.mkdir(NAPFS_DATA_DIR) return webtest.TestApp( napfs.create_app(data_dir=NAPFS_DATA_DIR, redis_connection=redis_connection)) def create_router_app(router): if not os.path.exists(NAPFS_DATA_DIR):
import napfs import redislite if __name__ == '__main__': parser = argparse.ArgumentParser(description='kick off the napfs server') parser.add_argument('-p', '--port', default=3035, type=int, help="specify the port to listen to") parser.add_argument('-d', '--data-dir', type=str, default="/tmp/napfs", help="specify the directory to use to write the files") args = parser.parse_args() httpd = make_server( '127.0.0.1', args.port, napfs.create_app(data_dir=args.data_dir, redis_connection=redislite.StrictRedis())) print("starting server on port %s" % args.port) print("data-dir %s" % args.data_dir) try: httpd.serve_forever() except KeyboardInterrupt: print("done")
#!/usr/bin/env python2 import os import redis from flask import Flask, render_template, redirect, request, url_for, make_response import uuid if 'VCAP_SERVICES' in os.environ: VCAP_SERVICES = json.loads(os.environ['VCAP_SERVICES']) CREDENTIALS = VCAP_SERVICES["rediscloud"][0]["credentials"] r = redis.Redis(host=CREDENTIALS["hostname"], port=CREDENTIALS["port"], password=CREDENTIALS["password"]) else: import redislite r = redislite.StrictRedis() app = Flask(__name__) my_uuid = str(uuid.uuid1()) @app.route('/') def mainmenu(): response = """ <HTML><BODY><h2> <a href="/agenda">Agenda</a><br> <a href="/survey">Take Survey</a><br> <a href="/dumpsurveys">Survey Results</a><br> </h2> </BODY> """
#!/usr/bin/env python import redpipe import redislite import unittest import redmab test_conn = redislite.StrictRedis() redpipe.connect_redis(test_conn, 'test') def clean(): test_conn.flushall() class BasicTestCase(unittest.TestCase): def setUp(self): clean() def tearDown(self): clean() def mab(self, arms, name='test', alpha=5, beta=5, klass=redmab.MultiArmedBandit, expires=3600): storage = redmab.create_storage('test', 'test') return klass( name=name, storage=storage, arms=arms, alpha=alpha, beta=beta,
def test_redis_log_attribute(self): r = redislite.StrictRedis() self.assertIn('The server is now ready to accept connections', r.redis_log)
parser = argparse.ArgumentParser(description="Speedtest quicktest") parser.add_argument('-t', '--host', help='Remote host address') parser.add_argument('-p', '--port', help='Remote port') args = parser.parse_args() if args.host: host = args.host if args.port: port = args.port slave = '{} {}'.format(host, port) print slave #rc = redislite.StrictRedis(serverconfig={'slaveof': slave}) rc = redislite.StrictRedis(host=host, port=port) lst = List(redis=rc, key='speed') print "Keys in redis: ", rc.keys() print "No of items in redis['speed']: ", len(lst) print "Weekly speed\n", processor.average_speed_weekly(lst) settings = Dict(redis=rc, key='settings') if settings.get('last_test'): print 'Last test: ', settings['last_test'] else: print 'No last run' settings['last_test'] = datetime.now() print "Last hour aggr: ", settings.get('last_hour') if settings.get(
# program. Many of the data structures Redis can store are # nearly identical to standard Python built in data types such # as lists and dictionaries. # We are using the RedisList object from the redis_collections # module to access Redis. This provides us with a python # object that works almost exactly like a normal Python list, # except the information is stored in Redis. # This means the information in the list can persist between # program runs or be stored to mulitple Redis servers using # Redis repliataion. # Unlike SQL databases, the default Redis configuration will # do a single disk write operation every 15 minutes instead # every time data is written. This significantly decreases # wear on flash/sd-card. # We are going to create two lists in Redis to store our temps # one for celsius and one for fahranheit. redis_connection = redislite.StrictRedis(redisrdb) temp_c_list = RedisList(redis=redis_connection, key='temp_c') temp_f_list = RedisList(redis=redis_connection, key='temp_f') del temp_c_list[:] del temp_f_list[:] # Now loop through while True: temp_c_list.append(read_temp_c()) temp_f_list.append(read_temp_f()) print('Temp F:', read_temp_f(), 'Hourly Average Temp F:', sum(temp_f_list[-3600:]) / len(temp_f_list[-3600:])) time.sleep(frequency)
#!/usr/bin/env python # std lib import os import unittest from io import StringIO # 3rd party import redislite # our package import redkeys # noqa TEST_DIR = os.path.dirname(__file__) SRC_RDB = os.path.join(TEST_DIR, '.redis_src.db') SRC = redislite.StrictRedis(SRC_RDB) def flush_redis_data(conn): if conn is None: return conn.flushdb() def clean(): flush_redis_data(SRC) class BasicTestCase(unittest.TestCase): data = [
def test_redis_log_tail_no_log(self): r = redislite.StrictRedis() if os.path.exists(r.logfile): os.remove(r.logfile) lines = r.redis_log_tail() self.assertEqual(lines, [])
def test_redis_log_tail_empty_log(self): r = redislite.StrictRedis() with open(r.logfile, 'w'): pass lines = r.redis_log_tail() self.assertEqual(lines, [])
from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker from redis_collections import Dict import redislite # database stored in a file (simplest way) # TODO: change engine type if needed db_uri = "sqlite:///database.sql" engine = create_engine(db_uri) Base = declarative_base() Session = sessionmaker(bind=engine) cache_uri = 'storage.rdb' redis_connection = redislite.StrictRedis(cache_uri) Cache = Dict(redis=redis_connection, key='storage')
from helpers import * from log import get_logger MB = 1024.0 * 1024.0 LOCAL_DIR = path.dirname(path.abspath(__file__)) PID_FILE = path.join(LOCAL_DIR, 'speedtest.pid') DB_FILE = path.join(LOCAL_DIR, 'speedtest.db') PORT = 8002 MIN_DATETIME = datetime(2016, 1, 1) logger = get_logger('speedtest') is_daemon = False redis = redislite.StrictRedis(DB_FILE, serverconfig={'port': PORT}) data = redis_collections.List(redis=redis, key='speed') settings = redis_collections.Dict(redis=redis, key='settings') def get_splitted_output(cmd): return subprocess.check_output(cmd).decode().replace('\n', '').replace( '\r', '').split(',') def get_speed(): headers = get_splitted_output(['speedtest', '--csv-header']) try: speed = get_splitted_output(['speedtest', '--csv']) except Exception as ex: if not is_daemon:
import os import unittest from six import StringIO # 3rd party import redis import redislite import rediscluster import redislite.patch # our package import redisimp # noqa TEST_DIR = os.path.dirname(__file__) SRC_RDB = os.path.join(TEST_DIR, '.redis_src.db') SRC = redislite.StrictRedis(SRC_RDB) SRC_ALT_RDB = os.path.join(TEST_DIR, '.redis_src_alt.db') SRC_ALT = redislite.StrictRedis(SRC_ALT_RDB) DST_RDB = os.path.join(TEST_DIR, '.redis_dst.db') DST = redislite.StrictRedis(DST_RDB) def flush_redis_data(conn): if conn is None: return if rediscluster and isinstance(conn, rediscluster.RedisCluster): conns = [ redis.StrictRedis(host=node['host'], port=node['port']) for node in conn.connection_pool.nodes.nodes.values() if node.get('server_type', None) == 'master'
def get_redis_connection(host): if redislite and host.startswith("redislite://"): return redislite.StrictRedis(host[12:]) else: return redis.StrictRedis.from_url(host)
def test_redis_log_attribute(self): r = redislite.StrictRedis() self.assertIn('Server started, Redis version', r.redis_log)
{'code': 'es'}, {'code': 'ru'}, {'code': 'uk'}, ), 'default': { 'fallback': 'en', 'hide_untranslated': False, } } LOGIN_REDIRECT_URL = 'dashboard' LOGIN_URL = 'login' LOGOUT_URL = 'logout' AUTHENTICATION_BACKENDS = [ 'django.contrib.auth.backends.ModelBackend', 'account.authentication.EmailAuthBackend', ] #from redislite import StrictRedis #Redis import redislite # Create a Redis instance using redislite REDIS_DB_PATH = os.path.join('/tmp/shop_redis.db') RDB = redislite.StrictRedis(REDIS_DB_PATH) REDIS_SOCKET_PATH = 'redis+socket://%s' % (RDB.socket_file, )
def work(identification, graph_name, _range_start, _range_end, redis_fn): global redis_connection, strict_redis_connection, sparql_server, current_step, range_start, range_end, ss, just_unbound_bindings, frames_done_count, frame_templates, bnode_strings range_start, range_end = _range_start, _range_end sparql_server = sparql.SPARQLServer(sparql_uri) redis_connection = redislite.Redis(redis_fn) strict_redis_connection = redislite.StrictRedis(redis_fn) frame_templates = redis_collections.Dict(key='frames', redis=strict_redis_connection, writeback=True) bnode_strings = redis_collections.Dict(key='bnodes', redis=strict_redis_connection, writeback=True) raw = defaultdict(list) #todo limit queries with range_start, then uncomment this #if range_start != 0: # raw = redis_load('checkpoint'+str(range_start - 1)) raw['frames'] += list( query(('frame', 'parent', 'is_for_rule', 'step_finished', 'step_created'), """WHERE { """ + frame_query() + """ OPTIONAL {?frame kbdbg:has_parent ?parent}. ?frame kbdbg:is_for_rule ?is_for_rule. }""")) raw['bnodes'] += list( query(('bnode', 'frame', 'items', 'step_created', 'step_finished'), """WHERE { ?bnode kbdbg:has_items ?items. ?bnode kbdbg:has_parent ?frame. GRAPH ?g_created {?bnode rdf:type kbdbg:bnode}. """ + step_magic('_created') + """ OPTIONAL { GRAPH ?g_finished{?frame kbdbg:is_finished true}. """ + step_bind('_finished') + """ } }""")) raw['results'] += list( query(('uri', 'value', 'step_unbound'), """WHERE {GRAPH ?g_created { ?uri rdf:type kbdbg:result. ?uri rdf:value ?value. }.""" + step_magic('_created') + """ OPTIONAL {GRAPH ?g_unbound {?uri kbdbg:was_ubound true}.}.""" + step_bind('_unbound') + '}')) raw['bindings'] += list( query(('x', 'source', 'target', 'source_frame', 'target_frame', 'source_is_bnode', 'target_is_bnode', 'source_term_idx', 'target_term_idx', 'source_is_in_head', 'target_is_in_head', 'source_arg_idx', 'target_arg_idx', 'stepbinding_unbound', 'stepbinding_failed', 'stepbinding_created'), """WHERE { GRAPH ?gbinding_created {?x rdf:type kbdbg:binding.}. """ + step_magic('binding_created') + """ OPTIONAL {GRAPH ?gbinding_unbound {?x kbdbg:was_unbound true}. """ + step_bind('binding_unbound') + """ }. OPTIONAL {GRAPH ?gbinding_failed {?x kbdbg:failed true}. """ + step_bind('binding_failed') + """ }. ?x kbdbg:has_source ?source. ?x kbdbg:has_target ?target. ?source kbdbg:has_frame ?source_frame. ?target kbdbg:has_frame ?target_frame. OPTIONAL {?source kbdbg:is_bnode ?source_is_bnode.}. OPTIONAL {?target kbdbg:is_bnode ?target_is_bnode.}. ?source kbdbg:term_idx ?source_term_idx. ?target kbdbg:term_idx ?target_term_idx. OPTIONAL {?source kbdbg:is_in_head ?source_is_in_head.}. OPTIONAL {?target kbdbg:is_in_head ?target_is_in_head.}. OPTIONAL {?source kbdbg:arg_idx ?source_arg_idx.}. OPTIONAL {?target kbdbg:arg_idx ?target_arg_idx.}. }""")) current_step = range_end redis_save('checkpoint' + str(range_end), filter_out_irrelevant_stuff(raw)) current_step = '666' last_bindings = raw['bindings'][:] for i in range(range_start, range_end + 1): current_step = i ss = '[' + str(current_step) + ']' info('work ' + ss) state = filter_out_irrelevant_stuff(raw) if len(state['frames']) == 0: info('no frames.' + ss) continue #wont work with euler-style steps #if last_bindings == state['bindings']: # return 'end' #todo make emitter save data to output, not to file e = Emitter() e.gv("digraph frame" + str(current_step) + "{ ") #splines=ortho;#gv("pack=true") e.do_frames(state['frames']) e.do_bnodes(state['bnodes']) e.do_results(state['results']) e.do_bindings(state['bindings']) e.gv("}") info('}..' + ss) args = (ss, identification + '_' + str(current_step).zfill(7) + '.gv', e.output) while len(graphviz_futures) > 10000: info('sleeping') time.sleep(10) check_futures2(graphviz_futures) graphviz_futures.append(graphviz_pool.submit(output, *args)) check_futures2(graphviz_futures) #print_stats() redis_connection._cleanup() strict_redis_connection._cleanup()