def listen():
# 1. create connection to redis
redis = RedisClient()
try:
for msg in ssdp.listen():
# 2. Write JSON to redis pubsub channel
json_str = json.dumps(msg)
pub_ret = redis.publish('upnp-tools:ssdp', json_str.encode())
finally:
redis.close()
def connect(self):
return RedisClient(host=self.host,
port=self.port,
db=self.db,
socket_timeout=self.timeout,
password=self._password,
unix_socket_path=self.unix_socket_path)
def __init__(
self,
host: str = getenv("REDIS_MASTER_SERVICE_HOST", "0.0.0.0"),
port: str = getenv("REDIS_MASTER_SERVICE_PORT", "6379"),
hash: str = "default_hash",
):
self.db = RedisClient(host=host, port=int(port))
self.hash = hash
def __init__(self, key_prefix, key_sufix, db=0):
self.key_prefix = key_prefix
self.key_sufix = key_sufix
self.host = settings.OPPS_DB_HOST
self.port = settings.OPPS_DB_PORT
self.db = db
pool = ConnectionPool(host=self.host, port=self.port, db=self.db)
self.conn = RedisClient(connection_pool=pool)
def init(host='localhost', port=6379, db=0, password=None, r=None):
"""Configures module-level redis instance."""
global redis
if r is not None:
redis = r
else:
redis = RedisClient(host=host, port=port, db=db, password=password)
def check(self, **config):
timeout = config.get('timeout', self.timeout)
client = RedisClient(host=self.conn.hostname or self.default_host,
port=self.conn.port or self.default_port,
password=self.conn.password,
db=self.conn.path.replace('/', ''),
socket_connect_timeout=timeout,
socket_timeout=timeout)
client.ping()
return True
def __init__(self, DbName=0):
if self.db == False:
self.db = RedisClient(
host=self.getConfig()['host'],
port=self.getConfig()['port'],
password=self.getConfig()['auth'],
db=DbName,
decode_responses=True,
retry_on_timeout=10,
)
def getRedis(self):
if self.redis == False:
self.redis = RedisClient(host=self.redisConfig['host'],
port=self.redisConfig['port'],
password=self.redisConfig['auth'],
db=self.redisConfig['db'],
decode_responses=True)
return self.redis
else:
return self.redis
def __init_redis_dup(self, redis_dup: dict):
"""
初始化排重库
:param redis_dup: 任务排重库配置
:return:
"""
host, port, db = self.get_redis_params(redis_dup)
if redis_dup.get("pwd"):
pool = redis.ConnectionPool(host=host,
port=port,
db=db,
password=redis_dup.get("pwd"))
else:
pool = redis.ConnectionPool(host=host, port=port, db=db)
self.dup = RedisClient(connection_pool=pool)
def __init_redis_monitor(self, redis_task_monitor: dict):
"""
初始化任务监控库
:param redis_task_monitor: 任务监控库配置
:return:
"""
host, port, db = self.get_redis_params(redis_task_monitor)
if redis_task_monitor.get("pwd"):
pool = redis.ConnectionPool(host=host,
port=port,
db=db,
password=redis_task_monitor.get("pwd"))
else:
pool = redis.ConnectionPool(host=host, port=port, db=db)
self.monitor = RedisClient(connection_pool=pool)
self.expire = redis_task_monitor.get(
"expire") if redis_task_monitor.get("expire") else 10 * 60
def do_GET(self):
# want to split path into path?params
parts = urllib.parse.urlsplit(self.path)
path = parts.path
if path == '/':
self.sendfile('index.html', 'text/html; charset=utf-8')
elif path == '/main.js':
self.sendfile('main.js', 'application/javascript; charset=utf-8')
elif path == '/main.css':
self.sendfile('main.css', 'text/css; charset=utf-8')
elif path == '/api/discover':
# switch to 1.1 to get the browser to believe the chunked transfer
# encoding
old_proto = self.protocol_version
self.protocol_version = 'HTTP/1.1'
try:
self.send_response(HTTPStatus.OK)
self.send_header('Content-Type', 'application/x-json-stream')
self.send_header('Cache-Control', 'no-cache')
self.send_header('Transfer-Encoding', 'chunked')
self.end_headers()
if self.headers['ST']:
st = self.headers['ST']
else:
st = 'ssdp:all'
for svc in ssdp.discover_stream(st):
# convert to json
chunk = (json.dumps(svc) + '\n').encode('utf-8')
self.write_chunk(chunk)
# chunked encoding requires us to send an empty chunk at the
# end
self.write_chunk(b'')
finally:
self.protocol_version = old_proto
elif path == '/api/description' or path == '/api/scpd':
params = urllib.parse.parse_qs(parts.query)
if 'location' not in params:
self.respond_simple(
HTTPStatus.BAD_REQUEST,
'location query parameter is required but was missing',
)
return
location = params['location'][0]
print('location =', location)
parts = urllib.parse.urlsplit(location)
conn = http.client.HTTPConnection(parts.netloc)
print('parts.path =', parts.path)
rpath = self.path_and_query(location)
print('rpath =', rpath)
# Sky+ boxes reject our requests unless using this specific
# User-Agent string
conn.request(
'GET',
rpath,
headers={
'User-Agent': 'SKY_skyplus',
'Accept-Language': 'en'
},
)
try:
f = conn.getresponse()
self.send_response(HTTPStatus(f.status))
self.send_header('Content-Type', f.getheader('Content-Type'))
if f.getheader('Content-Length'):
self.send_header('Content-Length',
f.getheader('Content-Length'))
self.end_headers()
shutil.copyfileobj(f, self.wfile)
finally:
conn.close()
elif path == '/api/notifications':
redis = RedisClient()
try:
self.protocol_version = 'HTTP/1.1'
self.send_response(HTTPStatus.OK)
self.send_header('Content-Type', 'text/event-stream')
self.send_header('Cache-Control', 'no-cache')
self.end_headers()
for json_bytes in redis.subscribe('upnp-tools:ssdp'):
self.wfile.write(b'data: ' + json_bytes + b'\r\n\r\n')
finally:
redis.close()
else:
self.respond_simple(HTTPStatus.NOT_FOUND,
'Cannot find ' + self.path)
def client(cls):
if cls.CONNECTION_POOL is None:
cls.CONNECTION_POOL = ConnectionPool(host=settings.REDIS_HOST,
port=settings.REDIS_PORT,
db=settings.REDIS_DB)
return RedisClient(connection_pool=cls.CONNECTION_POOL)
#!/usr/bin/env python3
import time
# We need this for routing:
from celeryconfig import app # noqa
from celery import signature
from redis import StrictRedis as RedisClient
redis = RedisClient(host='localhost', port=6379, db=1)
# Here are the tasks imported using an agnostic way of queuing them
test_system = signature('tor_worker.role_anyone.tasks.test_system') # noqa
send_bot_message = signature(
'tor_worker.role_moderator.tasks.send_bot_message') # noqa
fake_task = signature('tor_worker.foo') # noqa
print('Pre-seed time: %d' % time.time())
for i in range(1, 500):
# fake_task.delay()
test_system.delay()
# send_bot_message.delay(f'test message {i}', to='spez',
# subject=f'Test {i}')
start = time.time()
print('Post-seed time: %d' % start)
count = redis.llen('celery')
while count > 0:
def connect(self):
return RedisClient(host=self.host,
port=self.port,
db=self.db,
socket_timeout=self.timeout)