#!/usr/bin/env python #! coding: utf-8 # pylint: disable-msg=W0311 import sys reload(sys) sys.setdefaultencoding("utf-8") import api from rq import Queue, Worker, Connection with Connection(api.QUEUE): q = Queue(sys.argv[1]) Worker(q).work()
from store import create_app from rq import Connection, Worker from store.services import DownloadQueueFactory app = create_app() app.app_context().push() with Connection(DownloadQueueFactory.create()): worker = Worker(app.config['REDIS_QUEUES']) worker.work()
class CallableObject(object): def __call__(self): return u"I'm callable" class UnicodeStringObject(object): def __repr__(self): if PY2: return u'é'.encode('utf-8') else: return u'é' with Connection(): @job(queue='default') def decorated_job(x, y): return x + y def black_hole(job, *exc_info): # Don't fall through to default behaviour (moving to failed queue) return False def long_running_job(timeout=10): time.sleep(timeout) return 'Done sleeping...'
os.path.join(args.oil_change_data_root, 'annotations', args.ann_file)) cats = {cat['id']: cat for cat in coco.dataset['categories']} cameras = {camera['name']: camera for camera in coco.dataset['cameras']} camera_info = cameras[args.camera_name] camera_parameters = { 'width': camera_info['width'], 'height': camera_info['height'], 'f_x': camera_info['K'][0], 'f_y': camera_info['K'][4], 'p_x': camera_info['K'][2], 'p_y': camera_info['K'][5] } cat_ids = [int(c.strip()) for c in args.cat_ids.split(',')] with Connection(Redis(args.redis_host, args.redis_port)): q = Queue(args.queue_name) for i, subset_start_index in enumerate( tqdm(range(0, args.num_examples, args.subset_size))): for cat_id in cat_ids: cat = cats[cat_id] model_path = os.path.join(args.oil_change_data_root, 'meshes', cat['mesh']) completed_dir = os.path.join(args.output_cache_dir, args.camera_name, cat['name'], 'completed') if not os.path.exists(completed_dir): os.makedirs(completed_dir) output_dir = os.path.join(args.output_cache_dir, args.camera_name, cat['name'], 'subset_{:08}'.format(i + 1))
#!/usr/bin/env python import sys from rq import Connection, Worker import redis import os # Provide queue names to listen to as arguments to this script, # similar to rq worker r = redis.Redis(host=os.environ["REDIS_HOST"]) with Connection(connection=r): qs = sys.argv[1:] or [os.environ["RQ_QUEUE"]] w = Worker(qs) w.work()
def sender_task(message, interval, no_wa): url = BaseConfig.REDIS_URL with Connection(redis.from_url(url)): q = Queue() q.enqueue(create_task, message, interval, no_wa)
import os from dotenv import load_dotenv from rq import Connection, Queue load_dotenv(verbose=True) from recommender.db_config import primary_redis_conn # import correct worker if os.environ["IS_HEROKU"]: from rq.worker import HerokuWorker as Worker else: from rq.worker import Worker # optimization -> Import all libraries used in the consumer function from recommender.rcv.election_result_update_consumer import ElectionResultUpdateConsumer QUEUES_TO_WORK = [ElectionResultUpdateConsumer.QUEUE_NAME] if __name__ == "__main__": with Connection(primary_redis_conn): worker = Worker(map(Queue, QUEUES_TO_WORK)) worker.work()
def getQueuedJobs(cls): """Get a list of currently enqueued jobs""" with Connection(redis.from_url(current_app.config['REDIS_URL'])): q = Queue() return q.jobs # Gets a list of enqueued job instances
#!/usr/bin/env python import sys from redis import Redis from rq import Connection, Worker from utils import serializer redis = Redis(host = serializer.rq_host, port = serializer.rq_port) with Connection(connection=redis): qs = sys.argv[1:] or ['default'] w = Worker(qs) w.work()
from rq import Queue, Worker, Connection from redis import Redis import crawl_config if __name__ == '__main__': redis_conn = Redis(crawl_config.redis_server) with Connection(connection=redis_conn): q = Queue() Worker(q).work()
def getDeferredExperiments(cls): with Connection(redis.from_url(current_app.config['REDIS_URL'])) as conn: registry = DeferredJobRegistry('default', connection=conn) return [Job.fetch(id, connection=conn) for id in registry.get_job_ids()]
def trigger(): """ This endpoint is used to submit data to the queue so we can process it asynchronously to the web requests. The webrequests should be answered fast, while the processing might take more time The endpoint opens an API according to the ``--port`` and ``--host`` settings on launch. Thise API provides an endpoint on /trigger and consumes POST messages with JSON formatted body. The body is validated against the incident schema defined in bos-incidents .. note:: The trigger endpoint stores the incidents through (bos-incidents) already to allow later replaying. """ if request.method == "POST": # Don't bother wit requests from IPs that are not # whitelisted if request.remote_addr not in api_whitelist and "0.0.0.0" not in api_whitelist: return "Your IP address is not allowed to post here!", 403 # Obtain message from request body incident = request.get_json() # Ensure it is json try: validator.validate_incident(incident) except InvalidIncidentFormatException: log.error("Received invalid request: {}".format(str(incident))) return "Invalid data format", 400 # Only accept normalizable incidents # Normalize incident normalizer = IncidentsNormalizer(chain=config.get("network", "beatrice")) try: incident = normalizer.normalize(incident, True) except NotNormalizableException: log.warning( "Received not normalizable incident, discarding {}".format( str(incident) ) ) return "Not normalized incident", 400 try: # FIXME, remove copy() storage.insert_incident(incident.copy()) except exceptions.DuplicateIncidentException as e: # We merely pass here since we have the incident already # alerting anyone won't do anything # traceback.print_exc() pass # Send incident to redis with Connection(redis): q = Queue(connection=redis) job = q.enqueue( work.process, args=(incident,), kwargs=dict( proposer=app.config.get("BOOKIE_PROPOSER"), approver=app.config.get("BOOKIE_APPROVER"), ), ) log.info("Forwarded incident {} to worker via redis".format(str(incident))) # Return message with id return jsonify( dict( result="processing", message=incident, id=str(job.id), id_approve=str(job.id), ) ) return "", 503
import os import redis from rq import Worker, Queue, Connection listen = ['high', 'default', 'low'] # Redis URL setting in env REDISTOGO_URL or REDISTOGO_URL or default to localhost REDIS_URL = os.environ.get('REDISTOGO_URL', os.environ.get('REDIS_URL', 'localhost:6379')) redis_server = redis.from_url(REDIS_URL) if __name__ == '__main__': with Connection(redis_server): worker = Worker(map(Queue, listen)) worker.work()
def worker(queues='default'): if not queues: queues = ('default', ) with Connection(rq_redis_connection): w = Worker(queues) w.work()
get_repo_object, db, status, get_trans_object ) from utils import ( DockerUtils, set_server_status ) logger = status.logger doc_utils = DockerUtils(status) doc = doc_utils.doc with Connection(db): transaction_queue = Queue('transactions') repo_queue = Queue('update_repo') w1 = Worker([transaction_queue]) w2 = Worker([repo_queue]) def handle_hook(): saved_status = set_server_status(first=True) logger.debug('calling maybe build docker image') if not status.iso_flag: image = doc_utils.maybe_build_base_devel() else: status.iso_flag = False
Queue, ) from utils import (all_file_paths_exist, copy_or_symlink, try_run_command, DockerUtils, PacmanPackageCache, remove) from . import (RedisHash, get_build_object, get_pkg_object, status, get_repo_object) logger = status.logger doc_util = DockerUtils(status) doc = doc_util.doc pkg_cache_obj = PacmanPackageCache() with Connection(status.db): repo_queue = Queue('update_repo') class TransactionMeta(RedisHash): """ This is the base class for `Transaction`(s). It simply sets up the attributes which are stored in redis so they can be properly accessed. This class should not be used directly. Args: See `Transaction` docstring. Attributes: See `Transaction` docstring. """
import app.logging from app.rich_menu import RichMenu from app.scheduling import ReminderJob, ReminderWorker from rasa.lineagent import LineAgent from rasa.store import scheduler_store, tracker_store logger = logging.getLogger(__name__) logger.debug("Starting worker") line_access_token = os.getenv('LINE_CHANNEL_ACCESS_TOKEN', None) rich_menu = RichMenu(line_access_token) rich_menu.setup() agent = LineAgent.load("models/dialogue", interpreter=RasaNLUInterpreter("models/current/nlu"), tracker_store=tracker_store) workerKwargs = {"rich_menu": rich_menu, "agent": agent} listen = ['high', 'default', 'low'] scheduler = Scheduler(connection=scheduler_store, interval=60, job_class=ReminderJob) Process(target=scheduler.run).start() with Connection(scheduler_store): worker = ReminderWorker(map(Queue, listen), job_class=ReminderJob) logger.info("Worker is ready.") worker.work(workerKwargs=workerKwargs)
def report(self, processid): with Connection(Redis()): q = Queue() res = q.fetch_job(processid) print(res.return_value)
import os from Bio import pairwise2 from Bio import Align import redis from rq import Worker, Queue, Connection listen = ['high', 'default', 'low'] redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379') conn2 = redis.from_url(redis_url) if __name__ == '__main__': with Connection(conn2): worker = Worker(map(Queue, listen)) worker.work()
def get_workers(): with Connection(get_redis_conn()): workers = Worker.all() return workers
def run_worker(): redis_connection = redis.from_url(REDIS_URL) with Connection(redis_connection): worker = Worker(REDIS_QUEUES) worker.work()
def main(): with Connection(redis_connection): worker = Worker(map(Queue, listen)) worker.work()
def run_worker(): redis_url = app.config["REDIS_URL"] redis_connection = redis.from_url(redis_url) with Connection(redis_connection): worker = Worker(app.config["QUEUES"]) worker.work()
import os import redis from rq import Worker, Queue, Connection listen = ['high', 'default', 'low'] redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379') # redis_url = os.getenv('REDISTOGO_URL') r = redis.from_url(redis_url) # r = redis.Redis() if __name__ == '__main__': with Connection(r): worker = Worker(map(Queue, listen)) worker.work()
def worker_wrapper(self, worker): with silence_stdout(): with Connection(self.conn): worker.work()
def worker(): with Connection(Redis("jobqueue.local")): qs = sys.argv[1:] or ['default'] print("foo") w = Worker(qs) w.work()
from redis import Redis,ConnectionPool from rq import Worker, Queue, Connection import time listen = ['high', 'default', 'low'] pool = ConnectionPool(db=0, host='localhost', port=6379) redis_conn = Redis(connection_pool=pool) if __name__ == '__main__': start = time.time() with Connection(redis_conn): worker = Worker(map(Queue, listen)) worker.work() stop = time.time() print(stop-start)
def run_worker(redis_connection): with Connection(redis_connection): worker = Worker(app.config['QUEUES']) worker.work()
import os import redis from rq import Worker, Queue, Connection listen = ['default'] redis_url = os.getenv('REDIS', 'redis://localhost:6379') conn = redis.from_url(redis_url) if __name__ == '__main__': with Connection(conn): worker = Worker(list(map(Queue, listen))) worker.work()
# -*- coding: utf-8 -*- """ IncetOps.rq_worker ~~~~~~~~~~~~~~ The working process of the RQ queue. :copyright: (c) 2018 by staugur. :license: MIT, see LICENSE for more details. """ if __name__ == '__main__': import setproctitle from redis import from_url from config import GLOBAL, REDIS from rq import Worker, Queue, Connection listen = ['high', 'default', 'low'] setproctitle.setproctitle(GLOBAL['ProcessName'] + '.rq') with Connection(from_url(REDIS)): worker = Worker(map(Queue, listen)) worker.work()