def setUpClass(cls):
# Set up connection to Redis
testconn = find_empty_redis_database()
push_connection(testconn)
# Store the connection (for sanity checking)
cls.testconn = testconn
def setUpClass(cls):
# Set up connection to Redis
testconn = find_empty_redis_database()
push_connection(testconn)
# Store the connection (for sanity checking)
cls.testconn = testconn
def setUp(self):
self.app = create_app_with_blueprint()
self.app.testing = True
self.app.config['RQ_MONITOR_REDIS_URL'] = 'redis://127.0.0.1'
self.app.redis_conn = self.get_redis_client()
push_connection(self.get_redis_client())
self.client = self.app.test_client()
def setUp(self):
self.app = make_flask_app(None, None, None, '')
self.app.testing = True
self.app.config['RQ_DASHBOARD_REDIS_URL'] = 'redis://127.0.0.1'
self.app.redis_conn = self.get_redis_client()
push_connection(self.get_redis_client())
self.client = self.app.test_client()
def setUp(self):
queue_patcher = mock.patch('rq.Queue')
mock_queue = queue_patcher.start()
mock_queue.return_value = QueueMock()
self.addCleanup(queue_patcher.stop)
watch_patcher = mock.patch('oslo_utils.timeutils.StopWatch')
mock_watch = watch_patcher.start()
mock_watch.side_effect = StopWatchMock.make_instance
self.addCleanup(watch_patcher.stop)
world_patcher = mock.patch('act.engine.world.World')
self.world = WorldMock()
mock_world = world_patcher.start()
mock_world.return_value = self.world
self.addCleanup(world_patcher.stop)
rq.push_connection(StrictRedisMock())
self.addCleanup(rq.pop_connection)
choice_patcher = mock.patch('act.engine.utils.weighted_random_choice')
self.choice = RandomChoiceMock()
mock_choice = choice_patcher.start()
mock_choice.side_effect = self.choice.weighted_random_choice
self.addCleanup(choice_patcher.stop)
return super(TestEngine, self).setUp()
def set_up(cls, config):
for queue_name in ("default", "email", "kpi"):
queue = Queue(connection=Redis(
host=config["host"],
port=int(config["port"]),
password=config["password"],
),
**config["queues"][queue_name])
setattr(cls, queue_name, queue)
cls.scheduler = Scheduler(connection=Redis(
host=config["host"],
port=int(config["port"]),
password=config["password"],
))
cls.failed_queue = get_failed_queue(connection=Redis(
host=config["host"],
port=int(config["port"]),
password=config["password"],
))
push_connection(
Redis(
host=config["host"],
port=int(config["port"]),
password=config["password"],
))
def get_rqinfo(options):
"""获取rq队列信息
"""
redis_conn = Redis.from_url(options.connection)
push_connection(redis_conn)
# RQ队列信息获取操作
if options.status:
workers = Worker.all()
queues = Queue.all()
return workers, queues
if options.queue:
queue = Queue(options.queue)
return queue
if options.cancel_job:
cancel_job(options.cancel_job)
return 'OK'
if options.requeue_job:
requeue_job(options.requeue_job)
return 'OK'
if options.requeue_all:
return requeue_all()
if options.empty_queue:
empty_queue(options.empty_queue)
return 'OK'
if options.compact_queue:
compact_queue(options.compact_queue)
return 'OK'
if options.queues:
return list_queues()
if options.jobs:
return list_jobs(options.jobs)
if options.workers:
return list_workers()
pop_connection()
def push_rq_connection():
connection = current_app.extensions['rq-dashboard'].redis_conn
if connection is None:
current_app.redis_conn = Redis(host=current_app.config.get('REDIS_HOST', 'localhost'),
port=current_app.config.get('REDIS_PORT', 6379),
password=current_app.config.get('REDIS_PASSWORD', None),
db=current_app.config.get('REDIS_DB', 0))
push_connection(connection)
def setup_rq_connection():
redis_conn = Redis(
host=current_app.config.get("REDIS_HOST", "localhost"),
port=current_app.config.get("REDIS_PORT", 6379),
password=current_app.config.get("REDIS_PASSWORD", None),
db=current_app.config.get("REDIS_DB", 0),
)
push_connection(redis_conn)
def __init__(self, conn, async_mode=True, base_cache_path=None):
self.conn = conn
self.conn.flushdb()
push_connection(self.conn)
self.base_cache_path = base_cache_path
self.repositories = RepositoryManager()
self.scheduler = Scheduler(self.conn, async_mode=async_mode)
def setup_redis():
# Initialize Connection to Redis
push_connection(Redis.from_url('redis://127.0.0.1:6379/2'))
yield
# Close connection to Redis
pop_connection()
def __init__(self, conn, async_mode=True, base_cache_path=None):
self.conn = conn
self.conn.flushdb()
push_connection(self.conn)
self.base_cache_path = base_cache_path
self.repositories = RepositoryManager()
self.scheduler = Scheduler(self.conn, async_mode=async_mode)
def setup_rq_connection():
if current_app.config.get('REDIS_URL'):
redis_conn = from_url(current_app.config.get('REDIS_URL'))
else:
redis_conn = Redis(host=current_app.config.get('REDIS_HOST', 'localhost'),
port=current_app.config.get('REDIS_PORT', 6379),
password=current_app.config.get('REDIS_PASSWORD', None),
db=current_app.config.get('REDIS_DB', 0))
push_connection(redis_conn)
def __init__(self, conn, base_archive_path=None, async_mode=True):
self.conn = conn
self.conn.flushdb()
rq.push_connection(self.conn)
self.archive_path = base_archive_path
self._tasks = TaskRegistry()
self._scheduler = Scheduler(self.conn, self._tasks,
async_mode=async_mode)
def setup_rq_connection():
if current_app.config.get('REDIS_URL'):
redis_conn = from_url(current_app.config.get('REDIS_URL'))
else:
redis_conn = Redis(host=current_app.config.get('REDIS_HOST', 'localhost'),
port=current_app.config.get('REDIS_PORT', 6379),
password=current_app.config.get('REDIS_PASSWORD', None),
db=current_app.config.get('REDIS_DB', 0))
push_connection(redis_conn)
def dispatch(self, request, *args, **kwargs):
if not request.user.is_superuser:
raise PermissionDenied
opts = getattr(settings, 'RQ', {}).copy()
opts.pop('eager', None)
self.connection = redis.Redis(**opts)
push_connection(self.connection)
return super(SuperUserMixin, self).dispatch(request, *args, **kwargs)
def __init__(self, conn, base_archive_path=None, async_mode=True):
self.conn = conn
self.conn.flushdb()
rq.push_connection(self.conn)
self.archive_path = base_archive_path
self._tasks = TaskRegistry()
self._scheduler = Scheduler(self.conn,
self._tasks,
async_mode=async_mode)
def setUpClass(cls):
# Set up connection to Redis
testconn = find_empty_redis_database()
push_connection(testconn)
# Store the connection (for sanity checking)
cls.testconn = testconn
# Shut up logging
logging.disable("ERROR")
def setUpClass(cls):
# Set up connection to Redis
testconn = find_empty_redis_database()
push_connection(testconn)
# Store the connection (for sanity checking)
cls.testconn = testconn
# Shut up logging
logging.disable(logging.ERROR)
def dispatch(self, request, *args, **kwargs):
if not request.user.is_superuser:
raise PermissionDenied
opts = getattr(settings, 'RQ', {}).copy()
opts.pop('eager', None)
self.connection = redis.Redis(**opts)
push_connection(self.connection)
return super(SuperUserMixin, self).dispatch(request, *args, **kwargs)
def basic_http_auth(*args, **kwargs):
push_connection(current_app.redis_conn)
auth = request.authorization
if (
auth is None
or auth.password != current_app.config.get('RQ_DASHBOARD_PASSWORD')
or auth.username != current_app.config.get('RQ_DASHBOARD_USERNAME')):
return Response(
'Please login',
401,
{'WWW-Authenticate': 'Basic realm="RQ Dashboard"'})
def setUpClass(cls):
# Set up connection to Redis
testconn = find_empty_redis_database()
push_connection(testconn)
# Store the connection (for sanity checking)
cls.testconn = testconn
# Shut up logbook
cls.log_handler = NullHandler()
cls.log_handler.push_thread()
def setUp(self):
if not rq_version.VERSION.startswith('1.'):
self.skipTest('Skipping 1.x tests, because running with %s' %
rq_version.VERSION)
self.app = make_flask_app(None, None, None, '')
self.app.testing = True
self.client = self.app.test_client()
self.get_redis_client().flushdb()
push_connection(self.get_redis_client())
def setUpClass(cls):
# Set up connection to Redis
testconn = find_empty_redis_database()
push_connection(testconn)
# Store the connection (for sanity checking)
cls.testconn = testconn
# Shut up logbook
cls.log_handler = NullHandler()
cls.log_handler.push_thread()
def setup_redis_connection():
redis_conn = get_current_connection()
if not redis_conn:
if config.get('redis_url', False):
oorq_log('Connecting to redis using redis_url: %s'
% config['redis_url'])
redis_conn = from_url(config['redis_url'])
else:
oorq_log('Connecting to redis using defaults')
redis_conn = Redis()
push_connection(redis_conn)
return redis_conn
def setup_redis_connection():
redis_conn = get_current_connection()
if not redis_conn:
if config.get('redis_url', False):
oorq_log('Connecting to redis using redis_url: %s' %
config['redis_url'])
redis_conn = from_url(config['redis_url'])
else:
oorq_log('Connecting to redis using defaults')
redis_conn = Redis()
push_connection(redis_conn)
return redis_conn
def push_rq_connection():
new_instance_number = request.view_args.get("instance_number")
if new_instance_number is not None:
redis_url = current_app.config.get("RQ_DASHBOARD_REDIS_URL")
if new_instance_number < len(redis_url):
_, new_instance = from_url(redis_url[new_instance_number])
else:
raise LookupError("Index exceeds RQ list. Not Permitted.")
else:
new_instance = current_app.redis_conn
push_connection(new_instance)
current_app.redis_conn = new_instance
def setup_rq_connection():
# 'RQ_DASHBOARD_REDIS_URL' environmental variable takes priority;
# otherwise, we look at the Flask app's config for the redis information.
if os.environ.get('RQ_DASHBOARD_REDIS_URL', None):
redis_conn = from_url(os.environ.get('RQ_DASHBOARD_REDIS_URL'))
elif current_app.config.get('REDIS_URL'):
redis_conn = from_url(current_app.config.get('REDIS_URL'))
else:
redis_conn = Redis(host=current_app.config.get('REDIS_HOST', 'localhost'),
port=current_app.config.get('REDIS_PORT', 6379),
password=current_app.config.get('REDIS_PASSWORD', None),
db=current_app.config.get('REDIS_DB', 0))
push_connection(redis_conn)
def __init__(self,
conn,
base_archive_path=None,
async_mode=True,
pubsub_channel=CH_PUBSUB):
self.conn = conn
self.conn.flushdb()
rq.push_connection(self.conn)
self.archive_path = base_archive_path
self._tasks = TaskRegistry(self.conn)
self._scheduler = Scheduler(self.conn,
self._tasks,
pubsub_channel=pubsub_channel,
async_mode=async_mode)
def setUpClass(cls):
# Set up connection to Redis
dbnum, testconn = find_empty_redis_database()
push_connection(testconn)
# Store the connection (for sanity checking)
cls.testconn = testconn
# Shut up logging
logging.disable(logging.ERROR)
cls.app = create_app_with_blueprint()
cls.app.testing = True
cls.app.config[
'RQ_MONITOR_REDIS_URL'] = 'redis://127.0.0.1:6379/{0}'.format(
dbnum)
cls.client = cls.app.test_client()
def dispatch(self, request, *args, **kwargs):
if not request.user.is_authenticated():
return redirect('{}?{}'.format(
reverse('admin:login'),
urlencode({'next': reverse('rq_stats')})))
if not request.user.is_staff:
messages.error(
request,
'You lack the permission required to access the RQ Dashboard.')
return redirect('admin:login')
opts = getattr(settings, 'RQ', {}).copy()
opts.pop('eager', None)
self.connection = redis.Redis(**opts)
push_connection(self.connection)
return super(SuperUserMixin, self).dispatch(request, *args, **kwargs)
def export_failed_jobs():
"""
View function que retorna o CSV com os dados.
Caso a fila esteja vazia, faz um redirect para a view "home" com
uma mensagem flash, informativo.
"""
redis_host = current_app.config['REDIS_HOST']
redis_port = current_app.config['REDIS_PORT']
redis_pass = current_app.config['REDIS_PASSWORD']
con = Redis(host=redis_host, port=redis_port, password=redis_pass)
push_connection(con)
fq = get_failed_queue()
fq_jobs = [serialize_job(job) for job in fq.get_jobs()]
if fq.count > 0:
dest = io.BytesIO()
writer = csv.writer(dest)
headers = [
u'ID:',
u'FILA:',
u'PROCESSO:',
u'TRACEBACK:',
]
writer.writerow(headers)
for job_data in fq_jobs:
row_data = [
job_data['id'],
job_data['origin'],
job_data['description'],
job_data['exc_info'],
]
writer.writerow(row_data)
pop_connection()
output = make_response(dest.getvalue())
output.headers[
"Content-Disposition"] = "attachment; filename=export_failed.csv"
output.headers["Content-type"] = "text/csv"
return output
else:
pop_connection()
flash('A fila de falhas esta vazia!', 'warning')
return redirect(url_for('home'))
def export_failed_jobs():
"""
View function que retorna o CSV com os dados.
Caso a fila esteja vazia, faz um redirect para a view "home" com
uma mensagem flash, informativo.
"""
redis_host = current_app.config['REDIS_HOST']
redis_port = current_app.config['REDIS_PORT']
redis_pass = current_app.config['REDIS_PASSWORD']
con = Redis(host=redis_host, port=redis_port, password=redis_pass)
push_connection(con)
fq = get_failed_queue()
fq_jobs = [serialize_job(job) for job in fq.get_jobs()]
if fq.count > 0:
dest = io.BytesIO()
writer = csv.writer(dest)
headers = [
u'ID:',
u'FILA:',
u'PROCESSO:',
u'TRACEBACK:',
]
writer.writerow(headers)
for job_data in fq_jobs:
row_data = [
job_data['id'],
job_data['origin'],
job_data['description'],
job_data['exc_info'],
]
writer.writerow(row_data)
pop_connection()
output = make_response(dest.getvalue())
output.headers["Content-Disposition"] = "attachment; filename=export_failed.csv"
output.headers["Content-type"] = "text/csv"
return output
else:
pop_connection()
flash('A fila de falhas esta vazia!', 'warning')
return redirect(url_for('home'))
def get_workers(request):
from rq import Worker, push_connection
import redis
import json
connection = redis.Redis()
push_connection(connection)
workers = Worker.all(connection=connection)
workerdata = list()
# serialize workers
for w in workers:
cj = w.get_current_job()
if cj:
cjinfo = {'id' : cj.id, 'args' : cj.args, 'enqueued_at' : cj.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000"), 'description' : cj.description}
else:
cjinfo = None
worker_dict = {'pid': w.pid, 'name': w.name, 'state': w.get_state(), 'current_job': cjinfo}
workerdata.append(worker_dict)
data = json.dumps(workerdata)
return HttpResponse(data, content_type='application/json')
def get_workers(request):
from rq import Worker, push_connection
import redis
import json
connection = redis.Redis()
push_connection(connection)
workers = Worker.all(connection=connection)
workerdata = list()
# serialize workers
for w in workers:
cj = w.get_current_job()
if cj:
cjinfo = {
'id':
cj.id,
'args':
cj.args,
'enqueued_at':
cj.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
'description':
cj.description
}
else:
cjinfo = None
worker_dict = {
'pid': w.pid,
'name': w.name,
'state': w.get_state(),
'current_job': cjinfo
}
workerdata.append(worker_dict)
data = json.dumps(workerdata)
return HttpResponse(data, content_type='application/json')
def setUp(self):
self.app = make_flask_app(None, None, None, '')
self.app.testing = True
self.client = self.app.test_client()
push_connection(self.get_redis_client())
def setup_rq_connection():
redis_conn = Redis(host=current_app.config.get('REDIS_HOST', 'localhost'),
port=current_app.config.get('REDIS_PORT', 6379),
password=current_app.config.get('REDIS_PASSWORD', None))
push_connection(redis_conn)
def prepare_job_execution(self, job):
# Fixes the error #479 of RQ. Remove it as soon as it gets fixed.
# (https://github.com/nvie/rq/issues/479)
rq.push_connection(self.connection)
super().prepare_job_execution(job)
def setup_rq_connection():
redis_conn = Redis()
push_connection(redis_conn)
def push_rq_connection():
push_connection(current_app.redis_conn)
def setup_module(module):
logging.disable(logging.ERROR)
globals()['conn'] = find_empty_redis_database()
push_connection(conn)
__author__ = 'liorsabag'
import logging
import time
import rq
from rq.job import Job
from .constants import redis_conn
rq.push_connection(redis_conn)
def avg_processing_time_for_q(name, n=100):
fjr = rq.registry.FinishedJobRegistry(name)
last_n_finished_jobs = (Job.fetch(jid) for jid in fjr.get_job_ids(-n, -1))
processing_times = [(j.ended_at - j.started_at).total_seconds()
for j in last_n_finished_jobs
if j.ended_at and j.started_at]
lpt = len(processing_times)
if lpt != n:
logging.warn(
"Not all jobs have an 'ended_at' property, only {0} of {1}".format(
lpt, n))
return sum(processing_times) / float(lpt)
def monitor_q_throughput(name, n=100, interval=1):
while True:
avg = avg_processing_time_for_q(name, n)
print "Avg of last {0} processing times: {1}".format(n, avg)
time.sleep(interval)
def setUpClass(cls):
cls.conn = find_empty_redis_database()
push_connection(cls.conn)
cls.dir = os.path.dirname(os.path.realpath(__file__))
def __init__(self, redis_conn=None, poll_interval=2500):
self._poll_interval = poll_interval
if get_current_connection() is None:
if redis_conn is None:
redis_conn = Redis()
push_connection(redis_conn)
def prepare_job_execution(self, job):
# Fixes the error #479 of RQ. Remove it as soon as it gets fixed.
# (https://github.com/nvie/rq/issues/479)
rq.push_connection(self.connection)
super().prepare_job_execution(job)
def protect_admin_panel():
if not current_user.has_role("admin"):
return redirect(url_for("security.login"))
push_connection(connection)
def __init__(self, name="default", redis_conn=None):
if get_current_connection() is None:
if redis_conn is None:
redis_conn = Redis()
push_connection(redis_conn)
self.name = name
def connection(request):
push_connection(create_redis_connection('testing'))
yield
pop_connection()
def push_rq_connection():
push_connection(redis.from_url(current_app.config['REDIS_URL']))
def create_app(RQ_REDIS_URL="redis://localhost:6391/0",
RQ_REDIS_WORKERS=None,
WORKDIR=None,
DATADIR=None,
STATICDIR=None):
#TODO: join STATICDIR with flask app
cwd = os.getcwd()
if STATICDIR is None:
STATICDIR = os.path.join(cwd, "jobbedr", "static")
assert os.path.isdir(STATICDIR)
if WORKDIR is None:
WORKDIR = os.path.join(cwd, "jobbedr", "workdir")
assert os.path.isdir(WORKDIR)
if DATADIR is None:
DATADIR = os.path.join(cwd, "..")
assert os.path.isdir(DATADIR)
app = Flask(__name__)
# populate config for RQ2 queue
app.config.update((dict(RQ_REDIS_URL=RQ_REDIS_URL,
WORKDIR=WORKDIR,
DATADIR=DATADIR,
STATICDIR=STATICDIR)))
# init rq queue
rq.init_app(app)
if RQ_REDIS_WORKERS is None:
RQ_REDIS_WORKERS = int(subprocess.check_output("nproc"))
# initialize workers
for i in range(RQ_REDIS_WORKERS):
worker = rq.get_worker('default')
worker.log.setLevel(logging.DEBUG)
proc = multiprocessing.Process(target=worker.work,
kwargs={'burst': False})
rq_workers.append(proc)
proc.start()
# get an access to faled queue - this does not look like best Pythonnic API we can have ...
# see documentation: http://python-rq.org/docs/jobs/
# we solve it by adding new method to out rq object
pr = urlparse(RQ_REDIS_URL)
assert pr.scheme == "redis", "Support for protocols outside of redis:// are not implemented"
host = pr.netloc.split(':')[0]
port = int(pr.netloc.split(':')[1])
db = pr.path[1:]
con = StrictRedis(host=host, port=port, db=db)
push_connection(con)
rq.get_failed_queue = lambda: get_failed_queue()
rq.redis_connection = con
# load application blueprints
from .scripts import scripts
app.register_blueprint(scripts)
return app
def push_rq_connection():
push_connection(current_app.extensions['rq-dashboard'].redis_conn)
def setup_rq_connection():
redis_conn = Redis(
host=current_app.config.get("REDIS_HOST", "localhost"), port=current_app.config.get("REDIS_PORT", 6379)
)
push_connection(redis_conn)
def setUpClass(cls):
cls.conn = find_empty_redis_database()
push_connection(cls.conn)
def setUpClass(cls):
cls.conn = find_empty_redis_database()
push_connection(cls.conn)
cls.dir = os.path.dirname(os.path.realpath(__file__))
def push_rq_connection():
push_connection(get_redis_connection())
def push_rq_connection():
push_connection(current_app.redis_conn)
import gevent
from gevent import Greenlet, monkey
import datetime
import os
import bson
import time
from rq import push_connection, Queue
# ours
from .. import Utils
from .. import constants
from .. import background_removal
from .. import page_results
db = constants.db
push_connection(constants.redis_conn)
start_q = Queue('start_synced_pipeline', connection=constants.redis_conn)
add_results = Queue('add_results', connection=constants.redis_conn)
def check_if_exists(image_url, products):
# Temporarily remove whitelist for Recruit Test -- LS 22/06/2016.
# domain = tldextract.extract(page_url).registered_domain
# if not db.whitelist.find_one({'domain': domain}):
# return False
# start = time.time()
if image_url[:4] == "data":
return False
if check_db(image_url, 'images', products):
return True
def push_rq_connection():
push_connection(connections['redis_conn'])
