def setUpClass(cls): # Set up connection to Redis testconn = find_empty_redis_database() push_connection(testconn) # Store the connection (for sanity checking) cls.testconn = testconn
def setUp(self): self.app = create_app_with_blueprint() self.app.testing = True self.app.config['RQ_MONITOR_REDIS_URL'] = 'redis://127.0.0.1' self.app.redis_conn = self.get_redis_client() push_connection(self.get_redis_client()) self.client = self.app.test_client()
def setUp(self): self.app = make_flask_app(None, None, None, '') self.app.testing = True self.app.config['RQ_DASHBOARD_REDIS_URL'] = 'redis://127.0.0.1' self.app.redis_conn = self.get_redis_client() push_connection(self.get_redis_client()) self.client = self.app.test_client()
def setUp(self): queue_patcher = mock.patch('rq.Queue') mock_queue = queue_patcher.start() mock_queue.return_value = QueueMock() self.addCleanup(queue_patcher.stop) watch_patcher = mock.patch('oslo_utils.timeutils.StopWatch') mock_watch = watch_patcher.start() mock_watch.side_effect = StopWatchMock.make_instance self.addCleanup(watch_patcher.stop) world_patcher = mock.patch('act.engine.world.World') self.world = WorldMock() mock_world = world_patcher.start() mock_world.return_value = self.world self.addCleanup(world_patcher.stop) rq.push_connection(StrictRedisMock()) self.addCleanup(rq.pop_connection) choice_patcher = mock.patch('act.engine.utils.weighted_random_choice') self.choice = RandomChoiceMock() mock_choice = choice_patcher.start() mock_choice.side_effect = self.choice.weighted_random_choice self.addCleanup(choice_patcher.stop) return super(TestEngine, self).setUp()
def set_up(cls, config): for queue_name in ("default", "email", "kpi"): queue = Queue(connection=Redis( host=config["host"], port=int(config["port"]), password=config["password"], ), **config["queues"][queue_name]) setattr(cls, queue_name, queue) cls.scheduler = Scheduler(connection=Redis( host=config["host"], port=int(config["port"]), password=config["password"], )) cls.failed_queue = get_failed_queue(connection=Redis( host=config["host"], port=int(config["port"]), password=config["password"], )) push_connection( Redis( host=config["host"], port=int(config["port"]), password=config["password"], ))
def get_rqinfo(options): """获取rq队列信息 """ redis_conn = Redis.from_url(options.connection) push_connection(redis_conn) # RQ队列信息获取操作 if options.status: workers = Worker.all() queues = Queue.all() return workers, queues if options.queue: queue = Queue(options.queue) return queue if options.cancel_job: cancel_job(options.cancel_job) return 'OK' if options.requeue_job: requeue_job(options.requeue_job) return 'OK' if options.requeue_all: return requeue_all() if options.empty_queue: empty_queue(options.empty_queue) return 'OK' if options.compact_queue: compact_queue(options.compact_queue) return 'OK' if options.queues: return list_queues() if options.jobs: return list_jobs(options.jobs) if options.workers: return list_workers() pop_connection()
def push_rq_connection(): connection = current_app.extensions['rq-dashboard'].redis_conn if connection is None: current_app.redis_conn = Redis(host=current_app.config.get('REDIS_HOST', 'localhost'), port=current_app.config.get('REDIS_PORT', 6379), password=current_app.config.get('REDIS_PASSWORD', None), db=current_app.config.get('REDIS_DB', 0)) push_connection(connection)
def setup_rq_connection(): redis_conn = Redis( host=current_app.config.get("REDIS_HOST", "localhost"), port=current_app.config.get("REDIS_PORT", 6379), password=current_app.config.get("REDIS_PASSWORD", None), db=current_app.config.get("REDIS_DB", 0), ) push_connection(redis_conn)
def __init__(self, conn, async_mode=True, base_cache_path=None): self.conn = conn self.conn.flushdb() push_connection(self.conn) self.base_cache_path = base_cache_path self.repositories = RepositoryManager() self.scheduler = Scheduler(self.conn, async_mode=async_mode)
def setup_redis(): # Initialize Connection to Redis push_connection(Redis.from_url('redis://127.0.0.1:6379/2')) yield # Close connection to Redis pop_connection()
def setup_rq_connection(): if current_app.config.get('REDIS_URL'): redis_conn = from_url(current_app.config.get('REDIS_URL')) else: redis_conn = Redis(host=current_app.config.get('REDIS_HOST', 'localhost'), port=current_app.config.get('REDIS_PORT', 6379), password=current_app.config.get('REDIS_PASSWORD', None), db=current_app.config.get('REDIS_DB', 0)) push_connection(redis_conn)
def __init__(self, conn, base_archive_path=None, async_mode=True): self.conn = conn self.conn.flushdb() rq.push_connection(self.conn) self.archive_path = base_archive_path self._tasks = TaskRegistry() self._scheduler = Scheduler(self.conn, self._tasks, async_mode=async_mode)
def dispatch(self, request, *args, **kwargs): if not request.user.is_superuser: raise PermissionDenied opts = getattr(settings, 'RQ', {}).copy() opts.pop('eager', None) self.connection = redis.Redis(**opts) push_connection(self.connection) return super(SuperUserMixin, self).dispatch(request, *args, **kwargs)
def setUpClass(cls): # Set up connection to Redis testconn = find_empty_redis_database() push_connection(testconn) # Store the connection (for sanity checking) cls.testconn = testconn # Shut up logging logging.disable("ERROR")
def setUpClass(cls): # Set up connection to Redis testconn = find_empty_redis_database() push_connection(testconn) # Store the connection (for sanity checking) cls.testconn = testconn # Shut up logging logging.disable(logging.ERROR)
def basic_http_auth(*args, **kwargs): push_connection(current_app.redis_conn) auth = request.authorization if ( auth is None or auth.password != current_app.config.get('RQ_DASHBOARD_PASSWORD') or auth.username != current_app.config.get('RQ_DASHBOARD_USERNAME')): return Response( 'Please login', 401, {'WWW-Authenticate': 'Basic realm="RQ Dashboard"'})
def setUpClass(cls): # Set up connection to Redis testconn = find_empty_redis_database() push_connection(testconn) # Store the connection (for sanity checking) cls.testconn = testconn # Shut up logbook cls.log_handler = NullHandler() cls.log_handler.push_thread()
def setUp(self): if not rq_version.VERSION.startswith('1.'): self.skipTest('Skipping 1.x tests, because running with %s' % rq_version.VERSION) self.app = make_flask_app(None, None, None, '') self.app.testing = True self.client = self.app.test_client() self.get_redis_client().flushdb() push_connection(self.get_redis_client())
def setup_redis_connection(): redis_conn = get_current_connection() if not redis_conn: if config.get('redis_url', False): oorq_log('Connecting to redis using redis_url: %s' % config['redis_url']) redis_conn = from_url(config['redis_url']) else: oorq_log('Connecting to redis using defaults') redis_conn = Redis() push_connection(redis_conn) return redis_conn
def push_rq_connection(): new_instance_number = request.view_args.get("instance_number") if new_instance_number is not None: redis_url = current_app.config.get("RQ_DASHBOARD_REDIS_URL") if new_instance_number < len(redis_url): _, new_instance = from_url(redis_url[new_instance_number]) else: raise LookupError("Index exceeds RQ list. Not Permitted.") else: new_instance = current_app.redis_conn push_connection(new_instance) current_app.redis_conn = new_instance
def setup_rq_connection(): # 'RQ_DASHBOARD_REDIS_URL' environmental variable takes priority; # otherwise, we look at the Flask app's config for the redis information. if os.environ.get('RQ_DASHBOARD_REDIS_URL', None): redis_conn = from_url(os.environ.get('RQ_DASHBOARD_REDIS_URL')) elif current_app.config.get('REDIS_URL'): redis_conn = from_url(current_app.config.get('REDIS_URL')) else: redis_conn = Redis(host=current_app.config.get('REDIS_HOST', 'localhost'), port=current_app.config.get('REDIS_PORT', 6379), password=current_app.config.get('REDIS_PASSWORD', None), db=current_app.config.get('REDIS_DB', 0)) push_connection(redis_conn)
def __init__(self, conn, base_archive_path=None, async_mode=True, pubsub_channel=CH_PUBSUB): self.conn = conn self.conn.flushdb() rq.push_connection(self.conn) self.archive_path = base_archive_path self._tasks = TaskRegistry(self.conn) self._scheduler = Scheduler(self.conn, self._tasks, pubsub_channel=pubsub_channel, async_mode=async_mode)
def setUpClass(cls): # Set up connection to Redis dbnum, testconn = find_empty_redis_database() push_connection(testconn) # Store the connection (for sanity checking) cls.testconn = testconn # Shut up logging logging.disable(logging.ERROR) cls.app = create_app_with_blueprint() cls.app.testing = True cls.app.config[ 'RQ_MONITOR_REDIS_URL'] = 'redis://127.0.0.1:6379/{0}'.format( dbnum) cls.client = cls.app.test_client()
def dispatch(self, request, *args, **kwargs): if not request.user.is_authenticated(): return redirect('{}?{}'.format( reverse('admin:login'), urlencode({'next': reverse('rq_stats')}))) if not request.user.is_staff: messages.error( request, 'You lack the permission required to access the RQ Dashboard.') return redirect('admin:login') opts = getattr(settings, 'RQ', {}).copy() opts.pop('eager', None) self.connection = redis.Redis(**opts) push_connection(self.connection) return super(SuperUserMixin, self).dispatch(request, *args, **kwargs)
def export_failed_jobs(): """ View function que retorna o CSV com os dados. Caso a fila esteja vazia, faz um redirect para a view "home" com uma mensagem flash, informativo. """ redis_host = current_app.config['REDIS_HOST'] redis_port = current_app.config['REDIS_PORT'] redis_pass = current_app.config['REDIS_PASSWORD'] con = Redis(host=redis_host, port=redis_port, password=redis_pass) push_connection(con) fq = get_failed_queue() fq_jobs = [serialize_job(job) for job in fq.get_jobs()] if fq.count > 0: dest = io.BytesIO() writer = csv.writer(dest) headers = [ u'ID:', u'FILA:', u'PROCESSO:', u'TRACEBACK:', ] writer.writerow(headers) for job_data in fq_jobs: row_data = [ job_data['id'], job_data['origin'], job_data['description'], job_data['exc_info'], ] writer.writerow(row_data) pop_connection() output = make_response(dest.getvalue()) output.headers[ "Content-Disposition"] = "attachment; filename=export_failed.csv" output.headers["Content-type"] = "text/csv" return output else: pop_connection() flash('A fila de falhas esta vazia!', 'warning') return redirect(url_for('home'))
def export_failed_jobs(): """ View function que retorna o CSV com os dados. Caso a fila esteja vazia, faz um redirect para a view "home" com uma mensagem flash, informativo. """ redis_host = current_app.config['REDIS_HOST'] redis_port = current_app.config['REDIS_PORT'] redis_pass = current_app.config['REDIS_PASSWORD'] con = Redis(host=redis_host, port=redis_port, password=redis_pass) push_connection(con) fq = get_failed_queue() fq_jobs = [serialize_job(job) for job in fq.get_jobs()] if fq.count > 0: dest = io.BytesIO() writer = csv.writer(dest) headers = [ u'ID:', u'FILA:', u'PROCESSO:', u'TRACEBACK:', ] writer.writerow(headers) for job_data in fq_jobs: row_data = [ job_data['id'], job_data['origin'], job_data['description'], job_data['exc_info'], ] writer.writerow(row_data) pop_connection() output = make_response(dest.getvalue()) output.headers["Content-Disposition"] = "attachment; filename=export_failed.csv" output.headers["Content-type"] = "text/csv" return output else: pop_connection() flash('A fila de falhas esta vazia!', 'warning') return redirect(url_for('home'))
def get_workers(request): from rq import Worker, push_connection import redis import json connection = redis.Redis() push_connection(connection) workers = Worker.all(connection=connection) workerdata = list() # serialize workers for w in workers: cj = w.get_current_job() if cj: cjinfo = {'id' : cj.id, 'args' : cj.args, 'enqueued_at' : cj.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000"), 'description' : cj.description} else: cjinfo = None worker_dict = {'pid': w.pid, 'name': w.name, 'state': w.get_state(), 'current_job': cjinfo} workerdata.append(worker_dict) data = json.dumps(workerdata) return HttpResponse(data, content_type='application/json')
def get_workers(request): from rq import Worker, push_connection import redis import json connection = redis.Redis() push_connection(connection) workers = Worker.all(connection=connection) workerdata = list() # serialize workers for w in workers: cj = w.get_current_job() if cj: cjinfo = { 'id': cj.id, 'args': cj.args, 'enqueued_at': cj.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000"), 'description': cj.description } else: cjinfo = None worker_dict = { 'pid': w.pid, 'name': w.name, 'state': w.get_state(), 'current_job': cjinfo } workerdata.append(worker_dict) data = json.dumps(workerdata) return HttpResponse(data, content_type='application/json')
def setUp(self): self.app = make_flask_app(None, None, None, '') self.app.testing = True self.client = self.app.test_client() push_connection(self.get_redis_client())
def setup_rq_connection(): redis_conn = Redis(host=current_app.config.get('REDIS_HOST', 'localhost'), port=current_app.config.get('REDIS_PORT', 6379), password=current_app.config.get('REDIS_PASSWORD', None)) push_connection(redis_conn)
def prepare_job_execution(self, job): # Fixes the error #479 of RQ. Remove it as soon as it gets fixed. # (https://github.com/nvie/rq/issues/479) rq.push_connection(self.connection) super().prepare_job_execution(job)
def setup_rq_connection(): redis_conn = Redis() push_connection(redis_conn)
def push_rq_connection(): push_connection(current_app.redis_conn)
def setup_module(module): logging.disable(logging.ERROR) globals()['conn'] = find_empty_redis_database() push_connection(conn)
__author__ = 'liorsabag' import logging import time import rq from rq.job import Job from .constants import redis_conn rq.push_connection(redis_conn) def avg_processing_time_for_q(name, n=100): fjr = rq.registry.FinishedJobRegistry(name) last_n_finished_jobs = (Job.fetch(jid) for jid in fjr.get_job_ids(-n, -1)) processing_times = [(j.ended_at - j.started_at).total_seconds() for j in last_n_finished_jobs if j.ended_at and j.started_at] lpt = len(processing_times) if lpt != n: logging.warn( "Not all jobs have an 'ended_at' property, only {0} of {1}".format( lpt, n)) return sum(processing_times) / float(lpt) def monitor_q_throughput(name, n=100, interval=1): while True: avg = avg_processing_time_for_q(name, n) print "Avg of last {0} processing times: {1}".format(n, avg) time.sleep(interval)
def setUpClass(cls): cls.conn = find_empty_redis_database() push_connection(cls.conn) cls.dir = os.path.dirname(os.path.realpath(__file__))
def __init__(self, redis_conn=None, poll_interval=2500): self._poll_interval = poll_interval if get_current_connection() is None: if redis_conn is None: redis_conn = Redis() push_connection(redis_conn)
def protect_admin_panel(): if not current_user.has_role("admin"): return redirect(url_for("security.login")) push_connection(connection)
def __init__(self, name="default", redis_conn=None): if get_current_connection() is None: if redis_conn is None: redis_conn = Redis() push_connection(redis_conn) self.name = name
def connection(request): push_connection(create_redis_connection('testing')) yield pop_connection()
def push_rq_connection(): push_connection(redis.from_url(current_app.config['REDIS_URL']))
def create_app(RQ_REDIS_URL="redis://localhost:6391/0", RQ_REDIS_WORKERS=None, WORKDIR=None, DATADIR=None, STATICDIR=None): #TODO: join STATICDIR with flask app cwd = os.getcwd() if STATICDIR is None: STATICDIR = os.path.join(cwd, "jobbedr", "static") assert os.path.isdir(STATICDIR) if WORKDIR is None: WORKDIR = os.path.join(cwd, "jobbedr", "workdir") assert os.path.isdir(WORKDIR) if DATADIR is None: DATADIR = os.path.join(cwd, "..") assert os.path.isdir(DATADIR) app = Flask(__name__) # populate config for RQ2 queue app.config.update((dict(RQ_REDIS_URL=RQ_REDIS_URL, WORKDIR=WORKDIR, DATADIR=DATADIR, STATICDIR=STATICDIR))) # init rq queue rq.init_app(app) if RQ_REDIS_WORKERS is None: RQ_REDIS_WORKERS = int(subprocess.check_output("nproc")) # initialize workers for i in range(RQ_REDIS_WORKERS): worker = rq.get_worker('default') worker.log.setLevel(logging.DEBUG) proc = multiprocessing.Process(target=worker.work, kwargs={'burst': False}) rq_workers.append(proc) proc.start() # get an access to faled queue - this does not look like best Pythonnic API we can have ... # see documentation: http://python-rq.org/docs/jobs/ # we solve it by adding new method to out rq object pr = urlparse(RQ_REDIS_URL) assert pr.scheme == "redis", "Support for protocols outside of redis:// are not implemented" host = pr.netloc.split(':')[0] port = int(pr.netloc.split(':')[1]) db = pr.path[1:] con = StrictRedis(host=host, port=port, db=db) push_connection(con) rq.get_failed_queue = lambda: get_failed_queue() rq.redis_connection = con # load application blueprints from .scripts import scripts app.register_blueprint(scripts) return app
def push_rq_connection(): push_connection(current_app.extensions['rq-dashboard'].redis_conn)
def setup_rq_connection(): redis_conn = Redis( host=current_app.config.get("REDIS_HOST", "localhost"), port=current_app.config.get("REDIS_PORT", 6379) ) push_connection(redis_conn)
def setUpClass(cls): cls.conn = find_empty_redis_database() push_connection(cls.conn)
def push_rq_connection(): push_connection(get_redis_connection())
import gevent from gevent import Greenlet, monkey import datetime import os import bson import time from rq import push_connection, Queue # ours from .. import Utils from .. import constants from .. import background_removal from .. import page_results db = constants.db push_connection(constants.redis_conn) start_q = Queue('start_synced_pipeline', connection=constants.redis_conn) add_results = Queue('add_results', connection=constants.redis_conn) def check_if_exists(image_url, products): # Temporarily remove whitelist for Recruit Test -- LS 22/06/2016. # domain = tldextract.extract(page_url).registered_domain # if not db.whitelist.find_one({'domain': domain}): # return False # start = time.time() if image_url[:4] == "data": return False if check_db(image_url, 'images', products): return True
def push_rq_connection(): push_connection(connections['redis_conn'])