class RedisManager(NoSqlManager):
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
**params):
self.db = params.pop('db', None)
self.dbpass = params.pop('password', None)
self.connection_pool = params.get('redis_connection_pool', None)
self.expires = params.get('expires', params.get('expiretime', None))
NoSqlManager.__init__(self,
namespace,
url=url,
data_dir=data_dir,
lock_dir=lock_dir,
**params)
def open_connection(self, host, port, **params):
if not self.connection_pool:
self.connection_pool = ConnectionPool(host=host, port=port, db=self.db,
password=self.dbpass)
self.db_conn = StrictRedis(connection_pool=self.connection_pool, **params)
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
# beaker.container.Value.set_value calls NamespaceManager.set_value
# however it (until version 1.6.4) never sets expiretime param.
#
# Checking "type(value) is tuple" is a compromise
# because Manager class can be instantiated outside container.py (See: session.py)
if (expiretime is None) and (type(value) is tuple):
expiretime = value[1]
# If the machinery above fails, then pickup the expires time from the
# init params.
if not expiretime and self.expires is not None:
expiretime = self.expires
# Set or setex, according to whether we got an expires time or not.
if expiretime:
self.db_conn.setex(key, expiretime, pickle.dumps(value, 2))
else:
self.db_conn.set(key, pickle.dumps(value, 2))
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def _format_pool_key(self, host, port, db):
return '{0}:{1}:{2}'.format(host, port, self.db)
def do_remove(self):
self.db_conn.flush()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
def load(obj, namespace=None, silent=True, key=None):
"""
Reads and loads in to "settings" a single key or all keys from redis
:param obj: the settings instance
:param namespace: settings namespace default='DYNACONF'
:param silent: if errors should raise
:param key: if defined load a single key, else load all in namespace
:return: None
"""
redis = StrictRedis(**obj.REDIS_FOR_DYNACONF)
namespace = namespace or obj.DYNACONF_NAMESPACE
holder = "DYNACONF_%s" % namespace
try:
if key:
value = parse_conf_data(redis.hget(holder.upper(), key))
if value:
obj.set(key, value)
else:
data = {
key: parse_conf_data(value)
for key, value in redis.hgetall(holder.upper()).items()
}
if data:
obj.update(data, loader_identifier=IDENTIFIER)
except Exception as e:
e.message = 'Unable to load config from redis (%s)' % e.message
if silent:
obj.logger.error(e.message)
return False
raise
def feed_db(container_id, stats):
""" Store data to Redis.
args:
- constainer_id : (str) container's hash 12 first characters
- stats : a dictionary of stats
"""
if DEBUG:
print('feed db with container {} stats'.format(container_id))
# convert the time provided by stats to UTC format, parse it with strptime,
# and transform it again to the desired REDIS_KEY_TIMESTAMP format
instant_str = stats['read'][:-9]+stats['read'][-6:].replace(':', '')
instant = datetime.strptime(instant_str, '%Y-%m-%dT%H:%M:%S.%f%z')
timestamp = instant.strftime(REDIS_KEY_TIMESTAMP)
r = StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB)
for resource, value in stats.items():
if resource != 'read':
key = REDIS_KEY.format(timestamp=timestamp,
container_id=container_id,
resource=resource)
r.set(key, dumps(value))
r.expire(key, REDIS_EXPIRE_TIME)
if DEBUG:
print("Stored {} => {}".format(key, value))
def fetch_questions_ajax(request):
current_user = request.user
'''
query the database and find out whether questionpaper is made
or not for this user, if made then return the questions with
shuffled choices along with the answer of user, if not then
the same thing but answers will be empty or non existent
'''
questions = QuestionPaper.objects.all().filter(user=current_user)
question_list = []
if not questions:
# make new questions here
print "making new questions for the user:"******"now setting the key "+key+", which will expire in "+str(seconds)+" seconds.";
rd = StrictRedis()
rd.setex(key, seconds, "1")
else:
question_list = map(get_dictionary_from_question_paper, list(questions))
data = {"data":question_list}
return JsonResponse(data);
class RedisProducer(object):
def __init__(self, hostname = 'localhost', port = 6379):
log.debug("Initializing RedisProducer with hostname of %s and port %s" % (hostname, port))
self.r = StrictRedis(host = hostname, port = port)
def send(self, message):
tries = 0
next_index_key = get_next_index_for_topic_key(message.topic)
next_index = 1
result = None
log.debug("Sending message on topic %s" % message.topic)
while result is None and tries < TRIES_LIMIT:
if self.r.exists(next_index_key):
next_index = long(self.r.get(next_index_key)) + 1
message_key = get_message_key(message.topic, next_index)
try:
pl = self.r.pipeline()
pl.watch(next_index_key, message_key)
pl.multi()
pl.incr(next_index_key).set(message_key, message.payload)
result = pl.execute()
except WatchError:
# Should probably log something here, but all it means is we're
# retrying
pass
if result is None:
log.error("Could not send message, retry amount exceeded")
raise RuntimeError("Attempted to send message %s times and failed" % TRIES_LIMIT)
def load_data(ids, host, port):
"""from redis
uniq in index
"""
from redis import StrictRedis
from json import loads
db = StrictRedis(host, port, decode_responses=True)
logging.debug("all: %d", len(ids))
if 0 in ids:
raise KeyError(0)
from collections import Counter
checker = Counter(ids)
if len(checker) != len(ids): # not unique
raise ValueError(checker.most_common(3))
pipe = db.pipeline()
ids.insert(0, 0)
for i in ids:
pipe.hgetall(i)
properties = pipe.execute(True) # DO NOT allow error occurs in redis
raw = {}
for i, p in zip(ids, properties):
raw[i] = {k: loads(v) for k, v in p.items()}
return raw
def test_get_text(self):
r = StrictRedis()
r.set('test_get_2', "Open Mining")
DW = DataWarehouse()
self.assertEquals(
DW.get("test_get_2", content_type='application/text'),
"Open Mining")
def sync_get(self, identity, *args, **kwargs):
"""
For getting data from cache
:param identity: Unique Integer for the data
:param args: Args for the sync function. (Default: None)
"""
redis = StrictRedis(connection_pool=self.redis_pool)
key = key_generator(self.key, identity)
try:
if redis.exists(key):
data = self.get_func(redis.get(key))
else:
data = self.sync_func(identity, *args, **kwargs)
if self.expire:
self._setex(redis, key, self.set_func(data))
else:
redis.set(key, self.set_func(data))
if data is not None or data != "":
return data
return None
except RedisError as re:
self.log.error("[REDIS] %s", str(re))
data = self.sync_func(identity, args)
return data
finally:
del redis
def __admin_handler(self, endpoint: bytes):
"""
Handle Admin Request
:param bytes endpoint: Endpoint (in bytes!)
:return: jsonified answer data
"""
json_data = request.get_json()
if json_data is None:
raise BadRequest()
if 'admin_token' not in json_data:
raise BadRequest()
admin_token = json_data['admin_token']
if not isinstance(admin_token, str):
raise BadRequest()
if not RULE_TOKEN.match(admin_token):
raise BadRequest()
redis = StrictRedis(connection_pool=self.api_pool)
ep_key = 'ADMIN_TOKEN:{:s}'.format(admin_token)
should_endpoint = redis.get(ep_key)
if should_endpoint is None:
raise BadRequest()
redis.delete(ep_key)
if should_endpoint != endpoint:
raise BadRequest()
if 'data' not in json_data:
raise BadRequest()
data = json_data['data']
if not isinstance(data, dict):
raise BadRequest()
return jsonify(self.queue_dispatcher({
'_': 'admin:{:s}'.format(endpoint.decode('utf-8')),
'data': data,
}))
class SutroTestCase(TestCase):
def setUp(self):
redis_connection_pool = ConnectionPool(**settings.WS4REDIS_CONNECTION)
self.redis = StrictRedis(connection_pool=redis_connection_pool)
self.client = Client()
def tearDown(self):
self.redis.flushdb()
def random_string(self, length=None, str_type=None):
DEFAULT_LENGTH = 10
length = length if length else DEFAULT_LENGTH
if str_type == 'number':
string_type = string.digits
else:
string_type = string.lowercase
return ''.join(random.choice(string_type) for x in range(length))
def create_a_user(self, display_name, user_id=None, icon_url=None, user_url=None, rdio_key=None):
user = User()
user.display_name = display_name
user.id = user_id if user_id else uuid.uuid4().hex
user.icon_url = icon_url if icon_url else 'http://' + self.random_string() + '.jpg'
user.user_url = user_url if user_url else 'http://' + self.random_string() + '.com/' + self.random_string()
user.rdio_key = rdio_key if rdio_key else 's' + self.random_string(length=5, str_type='number')
user.save(self.redis)
return user
def create_a_party(self, party_id, name):
party = Party()
party.id = party_id
party.name = name
party.save(self.redis)
return party
def render_task(dburl, docpath, slug):
"""Render a document."""
oldcwd = os.getcwd()
try:
os.chdir(os.path.join(docpath, slug))
except:
db = StrictRedis.from_url(dburl)
job = get_current_job(db)
job.meta.update({'out': 'Document not found.', 'return': 127, 'status': False})
return 127
db = StrictRedis.from_url(dburl)
job = get_current_job(db)
job.meta.update({'out': '', 'milestone': 0, 'total': 1, 'return': None,
'status': None})
job.save()
p = subprocess.Popen(('lualatex', '--halt-on-error', slug + '.tex'),
stdout=subprocess.PIPE)
out = []
while p.poll() is None:
nl = p.stdout.readline()
out.append(nl)
job.meta.update({'out': ''.join(out), 'return': None,
'status': None})
job.save()
out = ''.join(out)
job.meta.update({'out': ''.join(out), 'return': p.returncode, 'status':
p.returncode == 0})
job.save()
os.chdir(oldcwd)
return p.returncode
def redis(self):
redis_config = self.config.get('redis')
redis = StrictRedis(
redis_config.get('host'), redis_config.get('port'), redis_config.get('db')
)
redis.ping()
return redis
def get_recent(**redis_kwargs):
""" Retrieve recent messages from Redis, in reverse chronological order.
Two lists are returned: one a single most-recent status message from
each process, the other a list of numerous messages from each process.
Each message is a tuple with floating point seconds elapsed, integer
process ID that created it, and an associated text message such as
"Got cache lock in 0.001 seconds" or "Started /osm/12/656/1582.png".
Keyword args are passed directly to redis.StrictRedis().
"""
pid = getpid()
red = StrictRedis(**redis_kwargs)
processes = []
messages = []
for key in red.keys('pid-*-statuses'):
try:
now = time()
pid = int(key.split('-')[1])
msgs = [msg.split(' ', 1) for msg in red.lrange(key, 0, _keep)]
msgs = [(now - float(t), pid, msg) for (t, msg) in msgs]
except:
continue
else:
messages += msgs
processes += msgs[:1]
messages.sort() # youngest-first
processes.sort() # youngest-first
return processes, messages
class RedisDataStore(DataStore):
"""Redis-backed datastore object."""
def __init__(self, number=0):
redis_host = os.environ.get('REDIS_PORT_6379_TCP_ADDR')
redis_port = os.environ.get('REDIS_PORT_6379_TCP_PORT')
self.redis_conn = StrictRedis(host=redis_host, port=redis_port,
db=number)
def __setitem__(self, k, v):
self.redis_conn.set(k, v)
def __getitem__(self, k):
return self.redis_conn.get(k)
def __delitem__(self, k):
self.redis_conn.delete(k)
def get(self, k):
return self.redis_conn.get(k)
def __contains__(self, k):
return self.redis_conn.exists(k)
def todict(self):
#TODO(tvoran): use paginate
#TODO(tvoran): do something besides multiple gets
data = {}
for key in self.redis_conn.keys():
data[key] = self.get(key)
return data
def clear_all(self):
self.redis_conn.flushdb()
def get_mail_from_redis(num=10):
r = StrictRedis('127.0.0.1', 6379)
key = "QQMAIL"
mail_list = r.lrange(key, 0, num)
for i in range(num):
r.lpop(key)
return set(mail_list)
def add_tweets_to_redis(tweet_file):
"""
'DVD: FBI WARNING Me: oh boy here we go DVD: The board advises you
to have lots of fun watching this Hollywood movie Me: Ah.. It's a
nice one'
"""
redis_client = StrictRedis(host='localhost', port=6379, db=0)
with open(tweet_file, 'r') as tweets:
for line in tweets:
# again, dealing with weird error here
try:
tweet = line.strip().split('|', 2)[2]
# need to investigate whether one-by-one inserting
# or building a list of tweets and doing a single insert
# would be more efficient
if not redis_client.sismember('tweets', tweet):
result = redis_client.sadd('tweets', tweet)
if not result:
print('error occurred adding tweet: "{}" to redis'
.format(tweet))
else:
# found all new tweets
break
except IndexError:
continue
redis_client.save()
def __init__(self, old_redis_url, new_redis_url, dry_run=True,
per_recording_list=False, s3_import=False, s3_root=None):
self.old_redis = StrictRedis.from_url(old_redis_url, decode_responses=True)
self.dry_run = dry_run
self.per_recording_list = per_recording_list
self.s3_import = s3_import
if s3_import:
assert(s3_root)
import boto3
self.s3_root = s3_root
self.s3 = boto3.client('s3')
else:
self.s3_root = None
self.s3 = None
if self.dry_run:
import redis
redis.StrictRedis = fakeredis.FakeStrictRedis
self.redis = FakeStrictRedis.from_url(new_redis_url, decode_responses=True)
else:
self.redis = StrictRedis.from_url(new_redis_url, decode_responses=True)
print('Redis Inited')
self.cli = CLIUserManager(new_redis_url)
def check_running(name: str):
socket_path = get_socket_path(name)
try:
r = StrictRedis(unix_socket_path=socket_path)
return r.ping()
except ConnectionError:
return False
def test_list_preparation_with_invalid_type(self) -> None:
"""
When a list key is already there with a wrong type
"""
redis = StrictRedis(connection_pool=self.__pool)
redis.set(self.__config['queue'], 'test')
self.assertRaises(ValueError, RedisQueueAccess, self.__config)
class JobStatsTest(TestCase):
def setUp(self):
self.redis = StrictRedis(**settings.REDISES['default'])
self.stats = JobStats(self.redis)
def tearDown(self):
self.redis.delete('job_stats.1')
def testIncr(self):
job_id = 1
field = 'success'
count = self.stats.get(job_id, field)
self.assertEqual(0, count)
self.stats.incr(job_id, field, 1)
count = self.stats.get(job_id, field)
self.assertEqual(1, count)
self.stats.incr(job_id, field, 2)
count = self.stats.get(job_id, field)
self.assertEqual(3, count)
self.stats.incr(job_id, 'failed')
self.stats.get(job_id, field)
self.assertEqual(3, count)
def testGet(self):
job_id = 1
self.stats.incr(job_id, 'failed', 1)
self.stats.incr(job_id, 'success', 2)
counts = self.stats.get(job_id)
self.assertEqual(1, counts['failed'])
self.assertEqual(2, counts['success'])
class RedisManager(NoSqlManager):
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
self.connection_pool = params.pop('connection_pool', None)
NoSqlManager.__init__(self, namespace, url=url, data_dir=data_dir, lock_dir=lock_dir, **params)
def open_connection(self, host, port, **params):
self.db_conn = StrictRedis(host=host, port=int(port), connection_pool=self.connection_pool, **params)
def __contains__(self, key):
log.debug('%s contained in redis cache (as %s) : %s'%(key, self._format_key(key), self.db_conn.exists(self._format_key(key))))
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#XXX: beaker.container.Value.set_value calls NamespaceManager.set_value
# however it(until version 1.6.3) never sets expiretime param. Why?
if expiretime:
self.db_conn.setex(key, expiretime, pickle.dumps(value))
else:
self.db_conn.set(key, pickle.dumps(value))
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def do_remove(self):
self.db_conn.flush()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
def test_token(redis_server):
conn = StrictRedis(unix_socket_path=UDS_PATH)
lock = Lock(conn, "foobar-tok")
tok = lock.id
assert conn.get(lock._name) is None
lock.acquire(blocking=False)
assert conn.get(lock._name) == tok
def app(environ, start_response):
ret = {}
qs = dict(parse_qsl(environ["QUERY_STRING"]))
red = StrictRedis(host="localhost", port=6379, db=0)
root_key = red.get("root")
if root_key is None: #Load trucks into redis if there is no tree already in it
inp_file = "Mobile_Food_Facility_Permit.csv"
load_trucks(inp_file, red)
root_key = red.get("root")
try:
lat = float(qs["latitude"])
lon = float(qs["longitude"])
rad = float(qs["radius"])
except KeyError: #Return error if required fields aren't present
start_response("400 Bad Request", [("Content-type", "text/plain")])
ret["error"] = "latitude, longitude, and radius query parameters are required"
return [dumps(ret)]
ret["latitude"] = lat
ret["longitude"] = lon
ret["radius"] = rad
food = qs.get("food", "").upper()
if food:
ret["food"] = food
ret["trucks"] = [str(t)
for t in get_trucks(lat, lon, rad, red, root_key) if food in t.food]
else:
trucks = []
foods = set()
for t in get_trucks(lat, lon, rad, red, root_key):
trucks.append(str(t))
foods |= set(t.food)
ret["trucks"] = trucks
ret["foods"] = list(foods)
start_response("200 OK", [("Content-type", "text/plain")])
return [dumps(ret)]
def __init_redis(self):
try:
redis = StrictRedis()
redis.ping() # raises an exception if it failes
self.redis = redis
except:
pass
def test_lshash_redis():
"""
Test external lshash module
"""
config = {"redis": {"host": 'localhost', "port": 6379, "db": 15}}
sr = StrictRedis(**config['redis'])
sr.flushdb()
lsh = LSHash(6, 8, 1, config)
for i in xrange(num_elements):
lsh.index(list(els[i]))
lsh.index(list(els[i])) # multiple insertions should be prevented by the library
hasht = lsh.hash_tables[0]
itms = [hasht.get_list(k) for k in hasht.keys()]
for itm in itms:
for el in itm:
assert itms.count(itm) == 1 # have multiple insertions been prevented?
assert el in els
for el in els:
res = lsh.query(list(el), num_results=1, distance_func='euclidean')[0]
el_v, el_dist = res
assert el_v in els
assert el_dist == 0
del lsh
sr.flushdb()
def test_lshash_redis_extra_val():
"""
Test external lshash module
"""
config = {"redis": {"host": 'localhost', "port": 6379, "db": 15}}
sr = StrictRedis(**config['redis'])
sr.flushdb()
lsh = LSHash(6, 8, 1, config)
for i in xrange(num_elements):
lsh.index(list(els[i]), el_names[i])
lsh.index(list(els[i]), el_names[i]) # multiple insertions
hasht = lsh.hash_tables[0]
itms = [hasht.get_list(k) for k in hasht.keys()]
for itm in itms:
assert itms.count(itm) == 1
for el in itm:
assert el[0] in els
assert el[1] in el_names
for el in els:
res = lsh.query(list(el), num_results=1, distance_func='euclidean')[0]
# vector an name are in the first element of the tuple res[0]
el_v, el_name = res[0]
# the distance is in the second element of the tuple
el_dist = res[1]
assert el_v in els
assert el_name in el_names
assert el_dist == 0
del lsh
sr.flushdb()
def show_search_results(query):
print("""
Search Result: {keyword}
==================================================
""".format(keyword=query))
p = Persistent("minamo")
r = StrictRedis(decode_responses=True)
resultset = None
for _query in query.split(" "):
qgram = ngram.ngram(_query, 2)
for bi in list(qgram)[:-1]:
if resultset is None:
resultset = set(r.lrange("minamo:bi:{}".format(bi), 0, -1))
else:
resultset = resultset & set(r.lrange("minamo:bi:{}".format(bi), 0, -1))
for page in (p.load(models.Page, x) for x in resultset):
if page.title is None:
continue
print("*", page.title)
print(" ", page.url)
def __init__(self, hosts, options):
sentinel_kwargs = self._get_sentinel_kwargs(options)
node_kwargs = self._get_node_kwargs(options)
masters = None
# Try to fetch a list of all masters from any sentinel.
hosts = list(hosts)
shuffle(hosts) # Randomly sort sentinels before trying to bootstrap.
for host, port in hosts:
client = StrictRedis(host=host, port=port, **sentinel_kwargs)
try:
masters = client.sentinel_masters().keys()
break
except RedisError:
pass
if masters is None:
# No Sentinel responded successfully?
raise errors.MastersListUnavailable
if not len(masters):
# The masters list was empty?
raise errors.NoMastersConfigured
sentinel_kwargs.update({
# Sentinels connected to fewer sentinels than `MIN_SENTINELS` will
# be ignored.
'min_other_sentinels': options.get('MIN_SENTINELS',
len(hosts) / 2),
})
self.sentinel = Sentinel(hosts, **sentinel_kwargs)
masters = [self.sentinel.master_for(name, **node_kwargs)
for name in masters]
super(SentinelBackedRingClient, self).__init__(masters, options)
def send_friend_request(request, recipient_id):
if request.is_ajax():
recipient = get_object_or_404(User, id=recipient_id)
redis_obj = StrictRedis(db=9)
try:
redis_obj.publish('notifications:%s' % recipient.username, 1)
except Exception, err:
print err
fr_obj = None
try:
fr_obj = FriendRequest.objects.get(
sender=request.user, recipient=recipient)
except:
fr_obj = FriendRequest.objects.create(
sender=request.user, recipient=recipient)
# Also create an entry in redis for this user.
# PubSub only works if the user is online and subscribed to the live
# stream
try:
# This is creating error;;
# hmset() takes exactly 3 arguments (10 given)
redis_obj.hmset(
'user:notify:%s' % request.user.id,
'obj_name', fr_obj._meta.object_name,
'obj_id', fr_obj.id,
'time_of', int(time()),
'was_viewed', 'false')
except:
pass
def get_redis_info():
"""Check Redis connection."""
try:
url = settings.BROKER_URL
_, host, port, _, password, db, _ = parse_redis_url(url)
except AttributeError:
log.error("No valid Redis connection info found in settings.")
return {"status": NO_CONFIG}
start = datetime.now()
try:
rdb = StrictRedis(
host=host, port=port, db=db,
password=password, socket_timeout=TIMEOUT_SECONDS,
)
info = rdb.info()
except (RedisConnectionError, TypeError) as ex:
log.error("Error making Redis connection: %s", ex.args)
return {"status": DOWN}
except RedisResponseError as ex:
log.error("Bad Redis response: %s", ex.args)
return {"status": DOWN, "message": "auth error"}
micro = (datetime.now() - start).microseconds
del rdb # the redis package does not support Redis's QUIT.
ret = {
"status": UP, "response_microseconds": micro,
}
fields = ("uptime_in_seconds", "used_memory", "used_memory_peak")
ret.update({x: info[x] for x in fields})
return ret
from flask_session import Session
from redis import StrictRedis
from flask import Flask, session
# from flask_session import Session
app = Flask(__name__)
app.config['SESSION_TYPE'] = 'redis' #设置存储系统
app.config['SESSION_REDIS'] = StrictRedis(host='192.168.59.128', port=6379)
app.config['SESSION_USE_SIGNER'] = True
app.config['SECRET_KEY'] = 'test' #设置应用的密钥
#初始化组件
Session(app)
@app.route('/index')
def home():
session['userid'] = 11
return 'index'
if __name__ == '__main__':
app.run(debug=True)
from telethon.sessions import StringSession
from .dB.core import *
from .dB.database import Var
from .misc import *
from .utils import *
from .version import __version__
LOGS = getLogger(__name__)
try:
redis_info = Var.REDIS_URI.split(":")
udB = StrictRedis(
host=redis_info[0],
port=redis_info[1],
password=Var.REDIS_PASSWORD,
charset="utf-8",
decode_responses=True,
)
except ConnectionError as ce:
wr(f"ERROR - {ce}")
exit(1)
except ResponseError as res:
wr(f"ERROR - {res}")
exit(1)
if not Var.API_ID or not Var.API_HASH:
wr("No API_ID or API_HASH found. Quiting...")
exit(1)
BOT_MODE = Var.BOT_MODE or udB.get("BOT_MODE")
# -*- coding: utf-8 -*-
import time
import random
from threading import Thread, active_count
from queue import Queue
import requests
from lxml import html
from debug.my_logging import create_logger
import csv
from redis import StrictRedis
my_logger = create_logger(__name__)
csv_writer = csv.writer(open("my_job.csv", "a", encoding="utf-8"))
redis = StrictRedis(host="127.0.0.1", port=63796)
# 页面抓取成功
start_url = "https://search.51job.com/list/000000,000000,0000,00,9,99,python,2,1.html" \
"?lang=c&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99" \
"&ord_field=0&dibiaoid=0&line=&welfare="
# 待爬取links队列
# 岗位搜索列表页面链接队列
list_links_queue = Queue()
list_links_queue.put(start_url)
# 岗位详情列表页面链接队列
job_links_queue = Queue()
# 已爬取links,已经爬取过的link添加到已爬取，防止重复爬取
crawled_links = set()
redis.delete("crawled_links")
def get_json_data(self):
q = {'event_type': self.event_type.value}
try:
# filter out old entries
date_from = int(self.request.GET['from']
) + 1 # offset because of timestamp rounding
q['date_created__gt'] = timezone.make_aware(
datetime.datetime.fromtimestamp(date_from), timezone.utc)
except (ValueError, KeyError):
pass
contestant = self.object
try:
latest_date = contestant.corrections.filter(
event_type=self.event_type.value).latest().date_created_utc
except contest.models.ContestantCorrection.DoesNotExist:
latest_date = None
# we order by date_created *asc* as it will be *prepended* on the frontend
corrections_qs = contestant.corrections.select_related(
'author').filter(**q).order_by('date_created').distinct()
corr_tpl = get_template('correction/chunk-correction.html')
corrections = [
corr_tpl.render({'corr': corr}) for corr in corrections_qs
]
changes = {}
latest_correction = corrections_qs.last()
if latest_correction and latest_correction.changes:
changes = latest_correction.changes
# presence, through redis
online_users = [self.request.user]
online_alive = False
try:
self_pk = self.request.user.pk
timeout = settings.CORRECTION_LIVE_UPDATE_TIMEOUT
now = timezone.make_naive(timezone.now(), timezone.utc).timestamp()
moments_ago = now - timeout
key = settings.CORRECTION_LIVE_UPDATE_REDIS_KEY.format(
key=contestant.pk)
client = StrictRedis(**settings.PROLOGIN_UTILITY_REDIS_STORE)
client.expire(
key, timeout * 2
) # garbage collect the whole set after a bit if no further updates
client.zadd(key, {self_pk: now}) # add self to the set
members = client.zrangebyscore(
key, moments_ago, '+inf') # list people that are recent enough
members = set(int(pk)
for pk in members) - {self_pk} # exclude self
online_users.extend(
User.objects.filter(pk__in=members).order_by('pk'))
online_alive = True
except Exception:
# it's not worth trying harder
pass
user_tpl = get_template('correction/chunk-online-user.html')
online_users = [(user.pk, user_tpl.render({'user': user}))
for user in online_users]
return {
'changes': changes,
'corrections': corrections,
'online': online_users,
'online_alive': online_alive,
'from': latest_date,
'delay': 1000 * settings.CORRECTION_LIVE_UPDATE_POLL_INTERVAL,
}
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from redis import StrictRedis
from flask_wtf.csrf import CSRFProtect
from flask_session import Session
from config import config_dict
# 1.创建app对象
app = Flask(__name__)
# 2.添加配置信息
app.config.from_object(config_dict["development"])
# 3.创建数据库对象
db = SQLAlchemy(app)
# 4.创建redis存储对象
redis_store = StrictRedis(host=config_dict["development"].REDIS_HOST,
port=config_dict["development"].REDIS_PORT)
# 5.开启CSRF
CSRFProtect(app)
# 6.创建session对象
Session(app)
import csv
from redis import StrictRedis
reader = csv.reader(open("./presidents.csv"))
header = reader.next()
client = StrictRedis()
for row in reader:
key = "president:%s" % (row[0], )
doc = dict(zip(header, row))
client.set(key, doc)
from flask import request
from redis import StrictRedis
from flask_oauthlib.provider import OAuth2Provider
from mongokit import Connection, Document
from bson.objectid import ObjectId
import logging
import settings
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger("oauth")
oauth = OAuth2Provider()
mongo = Connection(host=settings.MONGO_HOST,
port=settings.MONGO_PORT)
redisdb = StrictRedis(db=9, decode_responses=True)
# 认证数据结构
@mongo.register
class User(Document):
"""A user, or resource owner,
is usually the registered user on your site."""
__collection__ = 'users'
__database__ = 'crandom'
use_dot_notation = True
use_schemaless = True
structure = {
"username": str,
"password": str,
from flask_wtf import CSRFProtect
# 导入配置对象的字典
from config import config, Config
# 导入日志模块
import logging
# 导入日志模块中的文件处理
from logging.handlers import RotatingFileHandler
from redis import StrictRedis
# 导入wtf扩展生成csrf_token
from flask_wtf import csrf
# 实例化sqlalchemy对象
db = SQLAlchemy()
# 实例化redis数据库，用来临时缓存和业务逻辑相关的数据，比如说图片验证码、短信验证码、用户信息
redis_store = StrictRedis(host=Config.REDIS_HOST,
port=Config.REDIS_PORT,
decode_responses=True)
# 集成项目日志
# 设置日志的记录等级
logging.basicConfig(level=logging.DEBUG) # 调试debug级
# 创建日志记录器，指明日志保存的路径、每个日志文件的最大大小、保存的日志文件个数上限
file_log_handler = RotatingFileHandler("logs/log",
maxBytes=1024 * 1024 * 100,
backupCount=10)
# 创建日志记录的格式 日志等级 输入日志信息的文件名 行数 日志信息
formatter = logging.Formatter(
'%(levelname)s %(filename)s:%(lineno)d %(message)s')
# 为刚创建的日志记录器设置日志记录格式
file_log_handler.setFormatter(formatter)
# 为全局的日志工具对象（flask app使用的）添加日志记录器
def get_connection(self):
return StrictRedis(host=self.host, port=self.port, db=self.db)
def work_on_population(redis: StrictRedis,
start_time: int,
max_runtime_s: int,
kill_handler: KillHandler):
"""
Here the actual sampling happens.
"""
# set timers
population_start_time = time()
cumulative_simulation_time = 0
# read from pipeline
pipeline = redis.pipeline()
# extract bytes
ssa_b, batch_size_b, all_accepted_b, n_req_b, n_acc_b \
= (pipeline.get(SSA).get(BATCH_SIZE)
.get(ALL_ACCEPTED).get(N_REQ).get(N_ACC).execute())
if ssa_b is None:
return
kill_handler.exit = False
if n_acc_b is None:
return
# convert from bytes
simulate_one, sample_factory = pickle.loads(ssa_b)
batch_size = int(batch_size_b.decode())
all_accepted = bool(int(all_accepted_b.decode()))
n_req = int(n_req_b.decode())
# notify sign up as worker
n_worker = redis.incr(N_WORKER)
logger.info(
f"Begin population, batch size {batch_size}. "
f"I am worker {n_worker}")
# counter for number of simulations
internal_counter = 0
# create empty sample
sample = sample_factory()
# loop until no more particles required
while int(redis.get(N_ACC).decode()) < n_req \
and (not all_accepted or int(redis.get(N_EVAL).decode()) < n_req):
if kill_handler.killed:
logger.info(
f"Worker {n_worker} received stop signal. "
f"Terminating in the middle of a population "
f"after {internal_counter} samples.")
# notify quit
redis.decr(N_WORKER)
sys.exit(0)
# check whether time's up
current_runtime = time() - start_time
if current_runtime > max_runtime_s:
logger.info(
f"Worker {n_worker} stops during population because "
f"runtime {current_runtime} exceeds "
f"max runtime {max_runtime_s}")
# notify quit
redis.decr(N_WORKER)
return
# increase global number of evaluations counter
particle_max_id = redis.incr(N_EVAL, batch_size)
# timer for current simulation until batch_size acceptances
this_sim_start = time()
# collect accepted particles
accepted_samples = []
# make batch_size attempts
for n_batched in range(batch_size):
# increase evaluation counter
internal_counter += 1
try:
# simulate
new_sim = simulate_one()
# append to current sample
sample.append(new_sim)
# check for acceptance
if new_sim.accepted:
# the order of the IDs is reversed, but this does not
# matter. Important is only that the IDs are specified
# before the simulation starts
# append to accepted list
accepted_samples.append(
cloudpickle.dumps(
(particle_max_id - n_batched, sample)))
# initialize new sample
sample = sample_factory()
except Exception as e:
logger.warning(f"Redis worker number {n_worker} failed. "
f"Error message is: {e}")
# initialize new sample to be sure
sample = sample_factory()
# update total simulation-specific time
cumulative_simulation_time += time() - this_sim_start
# push to pipeline if at least one sample got accepted
if len(accepted_samples) > 0:
# new pipeline
pipeline = redis.pipeline()
# update particles counter
pipeline.incr(N_ACC, len(accepted_samples))
# note: samples are appended 1-by-1
pipeline.rpush(QUEUE, *accepted_samples)
# execute all commands
pipeline.execute()
# end of sampling loop
# notify quit
redis.decr(N_WORKER)
kill_handler.exit = True
population_total_time = time() - population_start_time
logger.info(
f"Finished population, did {internal_counter} samples. "
f"Simulation time: {cumulative_simulation_time:.2f}s, "
f"total time {population_total_time:.2f}.")
def run(self):
"""
Called when the process intializes.
"""
while 1:
now = time()
# Make sure Redis is up
try:
self.redis_conn.ping()
except:
logger.error('skyline can\'t connect to redis at socket path %s' % settings.REDIS_SOCKET_PATH)
sleep(10)
self.redis_conn = StrictRedis(unix_socket_path = settings.REDIS_SOCKET_PATH)
continue
# Discover unique metrics
unique_metrics = list(self.redis_conn.smembers(settings.FULL_NAMESPACE + 'unique_metrics'))
# Clear out the count of screen hits from holt-winters
self.redis_conn.set('holtfalse', int(0))
if len(unique_metrics) == 0:
logger.info('no metrics in redis. try adding some - see README')
sleep(10)
continue
# Spawn processes
pids = []
for i in range(1, settings.ANALYZER_PROCESSES + 1):
if i > len(unique_metrics):
logger.info('WARNING: skyline is set for more cores than needed.')
break
p = Process(target=self.spin_process, args=(i, unique_metrics))
pids.append(p)
p.start()
# Send wait signal to zombie processes
for p in pids:
p.join()
# Grab data from the queue and populate dictionaries
exceptions = dict()
anomaly_breakdown = dict()
while 1:
try:
key, value = self.anomaly_breakdown_q.get_nowait()
if key not in anomaly_breakdown.keys():
anomaly_breakdown[key] = value
else:
anomaly_breakdown[key] += value
except Empty:
break
while 1:
try:
key, value = self.exceptions_q.get_nowait()
if key not in exceptions.keys():
exceptions[key] = value
else:
exceptions[key] += value
except Empty:
break
# Send alerts
if settings.ENABLE_ALERTS:
for alert in settings.ALERTS:
for metric in self.anomalous_metrics:
if alert[0] in metric[1]:
cache_key = 'last_alert.%s.%s' % (alert[1], metric[1])
try:
last_alert = self.redis_conn.get(cache_key)
if not last_alert:
self.redis_conn.setex(cache_key, alert[2], packb(metric[0]))
trigger_alert(alert, metric)
except Exception as e:
logger.error("couldn't send alert: %s" % e)
# Write anomalous_metrics to static webapp directory
filename = path.abspath(path.join(path.dirname(__file__), '..', settings.ANOMALY_DUMP))
with open(filename, 'w') as fh:
# Make it JSONP with a handle_data() function
anomalous_metrics = list(self.anomalous_metrics)
anomalous_metrics.sort(key=operator.itemgetter(1))
fh.write('handle_data(%s)' % anomalous_metrics)
# Log progress
logger.info('seconds to run :: %.2f' % (time() - now))
logger.info('total metrics :: %d' % len(unique_metrics))
logger.info('total analyzed :: %d' % (len(unique_metrics) - sum(exceptions.values())))
logger.info('secondary screen :: %s' % self.redis_conn.get('holtfalse'))
logger.info('total anomalies :: %d' % len(self.anomalous_metrics))
logger.info('exception stats :: %s' % exceptions)
logger.info('anomaly breakdown :: %s' % anomaly_breakdown)
self.redis_conn.set('holtfalse', int(0))
# Log to Graphite
self.send_graphite_metric('skyline.analyzer.run_time', '%.2f' % (time() - now))
self.send_graphite_metric('skyline.analyzer.total_analyzed', '%.2f' % (len(unique_metrics) - sum(exceptions.values())))
# Check canary metric
raw_series = self.redis_conn.get(settings.FULL_NAMESPACE + settings.CANARY_METRIC)
if raw_series is not None:
unpacker = Unpacker(use_list = False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
time_human = (timeseries[-1][0] - timeseries[0][0]) / 3600
projected = 24 * (time() - now) / time_human
logger.info('canary duration :: %.2f' % time_human)
self.send_graphite_metric('skyline.analyzer.duration', '%.2f' % time_human)
self.send_graphite_metric('skyline.analyzer.projected', '%.2f' % projected)
# Reset counters
self.anomalous_metrics[:] = []
# Sleep if it went too fast
if time() - now < 5:
logger.info('sleeping due to low run time...')
sleep(10)
def get_cache(strict=False):
redis_url = current_app.config.get("REDIS_URL")
if strict:
return StrictRedis.from_url(redis_url)
return from_url(redis_url)
def init_db():
app.redis = StrictRedis.from_url(app.config['REDIS_URL'])
class Analyzer(Thread):
def __init__(self, parent_pid):
"""
Initialize the Analyzer
"""
super(Analyzer, self).__init__()
self.redis_conn = StrictRedis(unix_socket_path = settings.REDIS_SOCKET_PATH)
self.daemon = True
self.parent_pid = parent_pid
self.current_pid = getpid()
self.anomalous_metrics = Manager().list()
self.exceptions_q = Queue()
self.anomaly_breakdown_q = Queue()
def check_if_parent_is_alive(self):
"""
Self explanatory
"""
try:
kill(self.current_pid, 0)
kill(self.parent_pid, 0)
except:
exit(0)
def send_graphite_metric(self, name, value):
if settings.GRAPHITE_HOST != '':
sock = socket.socket()
sock.connect((settings.GRAPHITE_HOST, settings.CARBON_PORT))
sock.sendall('%s %s %i\n' % (name, value, time()))
sock.close()
return True
return False
def spin_process(self, i, unique_metrics):
"""
Assign a bunch of metrics for a process to analyze.
"""
# Discover assigned metrics
keys_per_processor = int(ceil(float(len(unique_metrics)) / float(settings.ANALYZER_PROCESSES)))
if i == settings.ANALYZER_PROCESSES:
assigned_max = len(unique_metrics)
else:
assigned_max = i * keys_per_processor
assigned_min = assigned_max - keys_per_processor
assigned_keys = range(assigned_min, assigned_max)
# Compile assigned metrics
assigned_metrics = [unique_metrics[index] for index in assigned_keys]
# Check if this process is unnecessary
if len(assigned_metrics) == 0:
return
# Multi get series
raw_assigned = self.redis_conn.mget(assigned_metrics)
# Make process-specific dicts
exceptions = defaultdict(int)
anomaly_breakdown = defaultdict(int)
# Distill timeseries strings into lists
for i, metric_name in enumerate(assigned_metrics):
self.check_if_parent_is_alive()
# Static exclusion
# TODO: Make this a configuration item
if (re.search('veritas', metric_name) and not re.search('yen', metric_name)):
exceptions['VeritasSkipped'] += 1
continue
try:
raw_series = raw_assigned[i]
unpacker = Unpacker(use_list = False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
anomalous, ensemble, datapoint = run_selected_algorithm(timeseries, metric_name)
# If it's anomalous, add it to list
if anomalous:
base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1)
metric = [datapoint, base_name]
self.anomalous_metrics.append(metric)
# Get the anomaly breakdown - who returned True?
for index, value in enumerate(ensemble):
if value:
algorithm = settings.ALGORITHMS[index]
anomaly_breakdown[algorithm] += 1
# It could have been deleted by the Roomba
except TypeError:
exceptions['DeletedByRoomba'] += 1
except TooShort:
exceptions['TooShort'] += 1
except Stale:
exceptions['Stale'] += 1
except Boring:
exceptions['Boring'] += 1
except:
exceptions['Other'] += 1
logger.info(traceback.format_exc())
# Add values to the queue so the parent process can collate
for key, value in anomaly_breakdown.items():
self.anomaly_breakdown_q.put((key, value))
for key, value in exceptions.items():
self.exceptions_q.put((key, value))
def run(self):
"""
Called when the process intializes.
"""
while 1:
now = time()
# Make sure Redis is up
try:
self.redis_conn.ping()
except:
logger.error('skyline can\'t connect to redis at socket path %s' % settings.REDIS_SOCKET_PATH)
sleep(10)
self.redis_conn = StrictRedis(unix_socket_path = settings.REDIS_SOCKET_PATH)
continue
# Discover unique metrics
unique_metrics = list(self.redis_conn.smembers(settings.FULL_NAMESPACE + 'unique_metrics'))
# Clear out the count of screen hits from holt-winters
self.redis_conn.set('holtfalse', int(0))
if len(unique_metrics) == 0:
logger.info('no metrics in redis. try adding some - see README')
sleep(10)
continue
# Spawn processes
pids = []
for i in range(1, settings.ANALYZER_PROCESSES + 1):
if i > len(unique_metrics):
logger.info('WARNING: skyline is set for more cores than needed.')
break
p = Process(target=self.spin_process, args=(i, unique_metrics))
pids.append(p)
p.start()
# Send wait signal to zombie processes
for p in pids:
p.join()
# Grab data from the queue and populate dictionaries
exceptions = dict()
anomaly_breakdown = dict()
while 1:
try:
key, value = self.anomaly_breakdown_q.get_nowait()
if key not in anomaly_breakdown.keys():
anomaly_breakdown[key] = value
else:
anomaly_breakdown[key] += value
except Empty:
break
while 1:
try:
key, value = self.exceptions_q.get_nowait()
if key not in exceptions.keys():
exceptions[key] = value
else:
exceptions[key] += value
except Empty:
break
# Send alerts
if settings.ENABLE_ALERTS:
for alert in settings.ALERTS:
for metric in self.anomalous_metrics:
if alert[0] in metric[1]:
cache_key = 'last_alert.%s.%s' % (alert[1], metric[1])
try:
last_alert = self.redis_conn.get(cache_key)
if not last_alert:
self.redis_conn.setex(cache_key, alert[2], packb(metric[0]))
trigger_alert(alert, metric)
except Exception as e:
logger.error("couldn't send alert: %s" % e)
# Write anomalous_metrics to static webapp directory
filename = path.abspath(path.join(path.dirname(__file__), '..', settings.ANOMALY_DUMP))
with open(filename, 'w') as fh:
# Make it JSONP with a handle_data() function
anomalous_metrics = list(self.anomalous_metrics)
anomalous_metrics.sort(key=operator.itemgetter(1))
fh.write('handle_data(%s)' % anomalous_metrics)
# Log progress
logger.info('seconds to run :: %.2f' % (time() - now))
logger.info('total metrics :: %d' % len(unique_metrics))
logger.info('total analyzed :: %d' % (len(unique_metrics) - sum(exceptions.values())))
logger.info('secondary screen :: %s' % self.redis_conn.get('holtfalse'))
logger.info('total anomalies :: %d' % len(self.anomalous_metrics))
logger.info('exception stats :: %s' % exceptions)
logger.info('anomaly breakdown :: %s' % anomaly_breakdown)
self.redis_conn.set('holtfalse', int(0))
# Log to Graphite
self.send_graphite_metric('skyline.analyzer.run_time', '%.2f' % (time() - now))
self.send_graphite_metric('skyline.analyzer.total_analyzed', '%.2f' % (len(unique_metrics) - sum(exceptions.values())))
# Check canary metric
raw_series = self.redis_conn.get(settings.FULL_NAMESPACE + settings.CANARY_METRIC)
if raw_series is not None:
unpacker = Unpacker(use_list = False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
time_human = (timeseries[-1][0] - timeseries[0][0]) / 3600
projected = 24 * (time() - now) / time_human
logger.info('canary duration :: %.2f' % time_human)
self.send_graphite_metric('skyline.analyzer.duration', '%.2f' % time_human)
self.send_graphite_metric('skyline.analyzer.projected', '%.2f' % projected)
# Reset counters
self.anomalous_metrics[:] = []
# Sleep if it went too fast
if time() - now < 5:
logger.info('sleeping due to low run time...')
sleep(10)
class BeaconProcessor(object):
redis = StrictRedis(**redisConfig)
rawQueueOGN = Queue()
rawQueueFLR = Queue()
rawQueueICA = Queue()
queues = (rawQueueOGN, rawQueueFLR, rawQueueICA)
queueKeys = ('rawQueueOGN', 'rawQueueFLR', 'rawQueueICA')
workers = list()
def __init__(self):
# restore unprocessed data from redis:
numRead = 0
for key, queue in zip(self.queueKeys, self.queues):
while True:
item = self.redis.lpop(key)
if not item:
break
queue.put(item)
numRead += 1
print(f"[INFO] Loaded {numRead} raw message(s) from redis.")
self.dbThread = DbThread(dbConnectionInfo)
self.dbThread.start()
for i, queue in enumerate(self.queues):
rawWorker = RawWorker(index=i,
dbThread=self.dbThread,
rawQueue=queue)
rawWorker.start()
self.workers.append(rawWorker)
def stop(self):
for worker in self.workers:
worker.stop()
# store all unprocessed data into redis:
n = 0
for key, queue in zip(self.queueKeys, self.queues):
n += queue.qsize()
for item in list(queue.queue):
self.redis.rpush(key, item)
print(f"[INFO] Flushed {n} rawQueueX items into redis.")
self.dbThread.stop()
print('[INFO] BeaconProcessor terminated.')
startTime = time.time()
numEnquedTasks = 0
def _printStats(self):
now = time.time()
tDiff = now - self.startTime
if tDiff >= 60:
numTasksPerMin = self.numEnquedTasks / tDiff * 60
numQueuedTasks = self.rawQueueOGN.qsize() + self.rawQueueFLR.qsize(
) + self.rawQueueICA.qsize()
print(
f"Beacon rate: {numTasksPerMin:.0f}/min. {numQueuedTasks} queued."
)
self.numEnquedTasks = 0
self.startTime = now
def enqueueForProcessing(self, raw_message: str):
self._printStats()
prefix = raw_message[:3]
if prefix == 'OGN':
self.rawQueueOGN.put(raw_message)
elif prefix == 'FLR':
self.rawQueueFLR.put(raw_message)
else: # 'ICA'
self.rawQueueICA.put(raw_message)
self.numEnquedTasks += 1
if logger_name not in loggers:
loggers[logger_name] = {'level': 'INFO'}
LOGGING = {
'version': 1,
'incremental': False,
'disable_existing_loggers': False,
'filters': {},
'formatters': {
'default': {
'format':
'[%(asctime)s %(levelname)-5.5s %(threadName)s %(name)s] %(message)s',
}
},
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout',
'formatter': 'default',
},
},
'loggers': loggers
}
SPINACH_BROKER = RedisBroker(
redis=StrictRedis.from_url(config('REDIS_SPINACH_URL', default='redis://'),
**recommended_socket_opts))
SPINACH_NAMESPACE = 'feedsubs'
SPINACH_CLEAR_SESSIONS_PERIODICITY = timedelta(weeks=1)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from redis import StrictRedis
r = StrictRedis(host='127.0.0.1', port=6379)
def test():
key = 'testkey'
r.zadd(key, 0, 0)
r.zadd(key, 1, '1')
r.zadd(key, 2, '2')
r.zadd(key, 3, '3')
print(r.zrange(key, -1, -1))[0]
print(type(r.zrange(key, -1, -1)[0]))
print(r.zrange(key, 0, 0))[0]
print(type(r.zrange(key, 0, 0)[0]))
print(r.zcard(key))
print(type(r.zcard(key)))
if __name__ == '__main__':
test()
class RawWorker(Thread):
redis = StrictRedis(**redisConfig)
def __init__(self, index: int, dbThread: DbThread, rawQueue: Queue):
super(RawWorker, self).__init__()
self.index = index
self.dbThread = dbThread
self.rawQueue = rawQueue
self.doRun = True
def __del__(self):
self.doRun = False
def stop(self):
self.doRun = False
def run(self):
print(f"[INFO] Starting worker #{self.index}")
while self.doRun:
try:
raw_message = self.rawQueue.get(block=False)
if raw_message:
self._processMessage(raw_message)
except Empty:
time.sleep(1) # ~ thread.yield()
print(f"[INFO] Worker #{self.index} terminated.")
def _saveToRedis(self, statusKey: str, status: Status):
self.redis.set(statusKey, str(status))
self.redis.expire(statusKey, REDIS_RECORD_EXPIRATION)
def _processMessage(self, raw_message: str):
beacon = None
try:
beacon = parse(raw_message)
if not beacon or 'beacon_type' not in beacon.keys(
) or beacon['beacon_type'] != 'aprs_aircraft':
return
except ParseError as e:
# print('[ERROR] when parsing a beacon: {}'.format(e.message))
# print("Failed BEACON:", raw_message)
return
except Exception as e:
# print('[ERROR] {}'.format(e))
# if beacon:
# print("Failed BEACON:", beacon)
return
# we are not interested in para, baloons, uavs, static stuff and others:
aircraftType = beacon['aircraft_type']
if aircraftType in [4, 6, 7, 13, 11, 15, 16]:
return
address = beacon['address']
groundSpeed = beacon['ground_speed']
ts = round(beacon['timestamp'].timestamp()) # [s]
# print(f"[INFO] {address} gs: {groundSpeed:.0f}")
# print('[DEBUG] Gps H:', beacon['gps_quality']['horizontal'])
currentStatus: Status = Status(ts=ts)
currentStatus.s = 0 if groundSpeed < SPEED_THRESHOLD else 1 # 0 = on ground, 1 = airborne, -1 = unknown
# TODO add AGL check (?)
# TODO threshold by aircraftType
prevStatus: Status = None
statusKey = f"{address}-status"
ps = self.redis.get(statusKey)
if ps:
try:
prevStatus = Status.parse(ps)
except ValueError as e:
print('[ERROR] when parsing prev. status: ', e)
if not prevStatus: # we have no prior information
self._saveToRedis(statusKey, currentStatus)
return
if currentStatus.s != prevStatus.s:
addressType = beacon['address_type']
aircraftType = beacon['aircraft_type']
lat = beacon['latitude']
lon = beacon['longitude']
icaoLocation = AirfieldManager().getNearest(lat, lon)
if not icaoLocation:
return
event = 'L' if currentStatus.s == 0 else 'T' # L = landing, T = take-off
flightTime = 0
if event == 'L':
flightTime = currentStatus.ts - prevStatus.ts # [s]
if flightTime < 60:
return
self._saveToRedis(statusKey, currentStatus)
dt = datetime.fromtimestamp(ts)
dtStr = dt.strftime('%H:%M:%S')
print(
f"[INFO] {dtStr}; {icaoLocation}; {address}; {event}; {flightTime}"
)
strSql = f"INSERT INTO logbook_events " \
f"(ts, address, address_type, aircraft_type, event, lat, lon, location_icao, flight_time) " \
f"VALUES " \
f"({ts}, '{address}', {addressType}, '{aircraftType}', " \
f"'{event}', {lat:.5f}, {lon:.5f}, '{icaoLocation}', {flightTime});"
# print('strSql:', strSql)
self.dbThread.addStatement(strSql)
@contextmanager
def auto_commit(self):
try:
yield
self.session.commit()
except Exception:
self.session.rollback()
raise SaveObjectException()
# mysql数据库
db = SQLAlchemy()
t_redis = StrictRedis(host=REDIS_HOST,
port=REDIS_PORT,
db=REDIS_DB,
password=REDIS_PASSWORD,
decode_responses=True)
# 缓存，依赖redis
cache = Cache(
config={
'CACHE_TYPE': 'redis',
'CACHE_REDIS_HOST': REDIS_HOST,
'CACHE_REDIS_PORT': REDIS_PORT,
'CACHE_REDIS_PASSWORD': REDIS_PASSWORD,
'CACHE_REDIS_DB': REDIS_DB
})
class EntityModel(db.Model):
#16 uint8_t secs;
#17 uint8_t sats;
#18 char[] padding;
fields = ['lat', 'lon', 'alt', 'i', 'hour', 'mins', 'secs', 'sats']
unpacked_payload = list(struct.unpack('<iiiHBBBB2s', packed))
payload = dict(zip(fields, unpacked_payload))
payload['lat'] /= 10000000.0
payload['lon'] /= 10000000.0
payload['alt'] /= 1000.0
return payload
if __name__ == '__main__':
r = StrictRedis(host=s.REDIS_HOST, port=s.REDIS_PORT, db=s.REDIS_DB)
try:
if r.ping():
print "Redis connected."
except ConnectionError:
"Error: Redis server not available."
msg = r.get('last_msg')
if not msg:
last_msg = None
else:
last_msg = json.loads(msg)
p = r.pubsub()
p.subscribe('dashboard_messages')
MONGOCLIENT = MongoClient(MONGO_URI, 27017, serverSelectionTimeoutMS=1)
MONGO = MONGOCLIENT.userbot
def is_mongo_alive():
try:
MONGOCLIENT.server_info()
except BaseException:
return False
return True
# Init Redis
# Redis will be hosted inside the docker container that hosts the bot
# We need redis for just caching, so we just leave it to non-persistent
REDIS = StrictRedis(host='localhost', port=6379, db=0)
def is_redis_alive():
try:
REDIS.ping()
return True
except BaseException:
return False
# Setting Up CloudMail.ru and MEGA.nz extractor binaries,
# and giving them correct perms to work properly.
if not os.path.exists('bin'):
os.mkdir('bin')
import dash_devices
import dash_html_components as html
import multiprocessing as mp
from dash_devices.dependencies import Input, Output
from redis import StrictRedis
import time
app = dash_devices.Dash(__name__)
storage = StrictRedis(host='localhost', port=6379)
pubsub = storage.pubsub()
def task(*args):
print('task - started')
storage['task-info'] = 'startTask'
time.sleep(3)
storage['task-info'] = 'firstStage'
time.sleep(3)
storage['task-info'] = 'secondStage'
time.sleep(3)
storage['task-info'] = 'Done'
print('task - ended')
class Example:
def __init__(self, app):
self.app = app
self.app.layout = html.Div([
def setUp(self):
super(TestWebHooks, self).setUp()
self.connection = StrictRedis()
self.connection.flushall()
# Redis Server Options
DOTENV_FILE = os.path.join(os.getcwd(), 'chatbot', '.env')
env_config = Config(RepositoryEnv(DOTENV_FILE))
HOST = env_config.get('REDIS_SERVER_HOST')
try:
PASSWORD = env_config.get('REDIS_SERVER_PASSWORD')
except UndefinedValueError:
PASSWORD = None
PORT = env_config.get('REDIS_SERVER_PORT')
if PASSWORD is None:
REDIS_CONNECTION = StrictRedis(host=HOST, port=PORT)
else:
REDIS_CONNECTION = StrictRedis(host=HOST, password=PASSWORD, port=PORT)
try:
CHATBOX_DEMO_APPLICATION = env_config.get('CHATBOX_DEMO_APPLICATION',
cast=bool)
except UndefinedValueError:
CHATBOX_DEMO_APPLICATION = False
try:
USE_CELERY = env_config.get('USE_CELERY', cast=bool)
except UndefinedValueError:
USE_CELERY = False
def make_redis():
return StrictRedis(host=config.REDIS_HOST, port=config.REDIS_PORT)
import json
import logging
from azure.eventhub import EventHubProducerClient
from flask import Flask
from flask_restful import Api
from redis import StrictRedis
with open('./config.json', 'r') as f:
config = json.load(f)
try:
redis_args = config["redis"]
event_hub_args = config["event_hub"]
except KeyError as error:
logging.error("Config error")
raise
# Main flask app
app = Flask(__name__)
# flask-restful api
api = Api(app)
# redis client
my_redis = StrictRedis(**redis_args)
# Azure Event hub client
event_hub_client = EventHubProducerClient.from_connection_string(**event_hub_args)
"""
args = docopt(__docopt__, version="0.3.0")
startup_nodes = [{"host": args["--host"], "port": args["--port"]}]
if not args["--nocluster"]:
from rediscluster import StrictRedisCluster
rc = StrictRedisCluster(startup_nodes=startup_nodes,
max_connections=32,
socket_timeout=0.1,
decode_responses=True)
else:
from redis import StrictRedis
rc = StrictRedis(host=args["--host"],
port=args["--port"],
socket_timeout=0.1,
decode_responses=True)
if args["--timeit"]:
test_itterstions = [
5000,
10000,
20000,
]
if args["--pipeline"]:
for itterations in test_itterstions:
timeit_pipeline(rc, itterations=itterations)
else:
for itterations in test_itterstions:
timeit(rc, itterations=itterations)
class TestWebHooks(Test):
def setUp(self):
super(TestWebHooks, self).setUp()
self.connection = StrictRedis()
self.connection.flushall()
@with_context
@patch('pybossa.jobs.requests')
def test_webhooks(self, mock):
"""Test WEBHOOK works."""
mock.post.return_value = True
err_msg = "The webhook should return True from patched method"
assert webhook('url'), err_msg
err_msg = "The post method should be called"
assert mock.post.called, err_msg
@with_context
@patch('pybossa.jobs.requests')
def test_webhooks_without_url(self, mock):
"""Test WEBHOOK without url works."""
mock.post.return_value = True
err_msg = "The webhook should return False"
assert webhook(None) is False, err_msg
@with_context
@patch('pybossa.model.event_listeners.webhook_queue', new=queue)
def test_trigger_webhook_without_url(self):
"""Test WEBHOOK is triggered without url."""
project = ProjectFactory.create()
task = TaskFactory.create(project=project, n_answers=1)
TaskRunFactory.create(project=project, task=task)
assert queue.enqueue.called is False, queue.enqueue.called
queue.reset_mock()
@with_context
@patch('pybossa.model.event_listeners.webhook_queue', new=queue)
def test_trigger_webhook_with_url_not_completed_task(self):
"""Test WEBHOOK is not triggered for uncompleted tasks."""
import random
project = ProjectFactory.create()
task = TaskFactory.create(project=project)
for i in range(1, random.randrange(2, 5)):
TaskRunFactory.create(project=project, task=task)
assert queue.enqueue.called is False, queue.enqueue.called
assert task.state != 'completed'
queue.reset_mock()
@with_context
@patch('pybossa.model.event_listeners.webhook_queue', new=queue)
def test_trigger_webhook_with_url(self):
"""Test WEBHOOK is triggered with url."""
url = 'http://server.com'
project = ProjectFactory.create(webhook=url, )
task = TaskFactory.create(project=project, n_answers=1)
TaskRunFactory.create(project=project, task=task)
payload = dict(
event='task_completed',
project_short_name=project.short_name,
project_id=project.id,
task_id=task.id,
fired_at=datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
assert queue.enqueue.called
assert queue.called_with(webhook, url, payload)
queue.reset_mock()
def register_redis(app):
redis_data = StrictRedis(db=app.config['CACHE_REDIS_DB'],
password=app.config['CACHE_REDIS_PASSWORD'],
decode_responses=True)
return redis_data
def get_redis_connection():
"""
Creates connection to redis, now we use default instance at localhost, no config needed
"""
return StrictRedis()
