def main(): parser = argparse.ArgumentParser(description='Start a MRQ worker') cfg = config.get_config(parser=parser, config_type="worker", sources=("file", "env", "args")) set_current_config(cfg) set_logger_config() # If we are launching with a --processes option and without MRQ_IS_SUBPROCESS, we are a manager process if cfg["processes"] > 0 and not os.environ.get("MRQ_IS_SUBPROCESS"): from mrq.supervisor import Supervisor command = " ".join(map(pipes.quote, sys.argv)) w = Supervisor(command, numprocs=cfg["processes"]) w.work() sys.exit(w.exitcode) # If not, start an actual worker else: worker_class = load_class_by_path(cfg["worker_class"]) w = worker_class() w.work() sys.exit(w.exitcode)
def main(): parser = argparse.ArgumentParser(description='Runs a task') cfg = config.get_config(parser=parser, config_type="run", sources=("file", "env", "args")) cfg["is_cli"] = True set_current_config(cfg) if len(cfg["taskargs"]) == 1: params = json.loads(cfg["taskargs"][0]) # pylint: disable=no-member else: params = {} # mrq-run taskpath a 1 b 2 => {"a": "1", "b": "2"} for group in utils.group_iter(cfg["taskargs"], n=2): if len(group) != 2: print("Number of arguments wasn't even") sys.exit(1) params[group[0]] = group[1] if cfg["queue"]: ret = queue_job(cfg["taskpath"], params, queue=cfg["queue"]) print(ret) else: worker_class = load_class_by_path(cfg["worker_class"]) job = worker_class.job_class(None) job.set_data({ "path": cfg["taskpath"], "params": params, "queue": cfg["queue"] }) job.datestarted = datetime.datetime.utcnow() set_current_job(job) ret = job.perform() print(json_stdlib.dumps(ret, cls=MongoJSONEncoder)) # pylint: disable=no-member
def main(): parser = argparse.ArgumentParser(description='Runs a task') cfg = config.get_config(parser=parser, config_type="run") cfg["is_cli"] = True set_current_config(cfg) log.info(cfg) if len(cfg["taskargs"]) == 1: params = json.loads(cfg["taskargs"][0]) else: params = {} # mrq-run taskpath a 1 b 2 => {"a": "1", "b": "2"} for group in utils.group_iter(cfg["taskargs"], n=2): if len(group) != 2: print "Number of arguments wasn't even" sys.exit(1) params[group[0]] = group[1] if cfg["async"]: ret = queue.send_task(cfg["taskpath"], params, sync=False, queue=cfg["queue"]) print ret else: worker_class = load_class_by_path(cfg["worker_class"]) job = worker_class.job_class(None) job.data = { "path": cfg["taskpath"], "params": params, "queue": cfg["queue"] } job.datestarted = datetime.datetime.utcnow() set_current_job(job) ret = job.perform() print json.dumps(ret)
def main(): set_current_config(get_config()) print "======== START =========" add_num = 0 print 'all:', add_num print "======== END ========="
def main(): parser = argparse.ArgumentParser(description='Start a MRQ agent') cfg = config.get_config(parser=parser, config_type="agent", sources=("file", "env", "args")) set_current_config(cfg) agent = Agent() agent.work() sys.exit(agent.exitcode)
import sys import psutil import time import re import json import urllib2 sys.path.append(os.getcwd()) from mrq.job import Job from mrq.queue import send_tasks, send_raw_tasks, Queue from mrq.config import get_config from mrq.utils import wait_for_net_service from mrq.context import connections, set_current_config set_current_config(get_config(sources=("env"))) os.system("rm -rf dump.rdb") class ProcessFixture(object): def __init__(self, request, cmdline=None, wait_port=None, quiet=False): self.request = request self.cmdline = cmdline self.process = None self.wait_port = wait_port self.quiet = quiet self.stopped = False self.request.addfinalizer(self.stop)
sys.path.insert(0, os.getcwd()) from mrq.queue import Queue from mrq.context import connections, set_current_config, get_current_config from mrq.job import queue_job from mrq.config import get_config from mrq.dashboard.utils import jsonify, requires_auth CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) parser = argparse.ArgumentParser(description='Start the MRQ dashboard') cfg = get_config(parser=parser, config_type="dashboard", sources=("file", "env", "args")) set_current_config(cfg) app = Flask("dashboard", static_folder=os.path.join(CURRENT_DIRECTORY, "static"), template_folder=os.path.join(CURRENT_DIRECTORY, "templates")) WHITELISTED_MRQ_CONFIG_KEYS = ["dashboard_autolink_repositories"] @app.route('/') @requires_auth def root(): return render_template("index.html", MRQ_CONFIG={
from werkzeug.serving import run_simple sys.path.insert(0, os.getcwd()) from mrq.queue import Queue from mrq.context import connections, set_current_config, get_current_config from mrq.job import queue_job from mrq.config import get_config from mrq.dashboard.utils import jsonify, requires_auth CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) parser = argparse.ArgumentParser(description='Start the MRQ dashboard') cfg = get_config(parser=parser, config_type="dashboard", sources=("file", "env", "args")) set_current_config(cfg) app = Flask( "dashboard", static_folder=os.path.join(CURRENT_DIRECTORY, "static"), template_folder=os.path.join(CURRENT_DIRECTORY, "templates") ) WHITELISTED_MRQ_CONFIG_KEYS = ["dashboard_autolink_repositories"] @app.route('/') @requires_auth def root(): return render_template("index.html", MRQ_CONFIG={
def main(): parser = argparse.ArgumentParser(description='Start a RQ worker') cfg = config.get_config(parser=parser, config_type="worker", sources=("file", "env", "args")) # If we are launching with a --processes option and without the SUPERVISOR_ENABLED env # then we should just call supervisord. if cfg["processes"] > 0 and not os.environ.get("SUPERVISOR_ENABLED"): # We wouldn't need to do all that if supervisord supported environment # variables in all its config fields! with open(cfg["supervisord_template"], "r") as f: conf = f.read() fh, path = tempfile.mkstemp(prefix="mrqsupervisordconfig") f = os.fdopen(fh, "w") # We basically relaunch ourselves, but the config will contain the # MRQ_SUPERVISORD_ISWORKER env. conf = conf.replace("{{ SUPERVISORD_COMMAND }}", " ".join(sys.argv)) conf = conf.replace("{{ SUPERVISORD_PROCESSES }}", str(cfg["processes"])) f.write(conf) f.close() try: # start_new_session=True avoids sending the current process' # signals to the child. process = subprocess.Popen(["supervisord", "-c", path], start_new_session=True) def sigint_handler(signum, frame): # pylint: disable=unused-argument # At this point we need to send SIGINT to all workers. Unfortunately supervisord # doesn't support this, so we have to find all the children pids and send them the # signal ourselves :-/ # https://github.com/Supervisor/supervisor/issues/179 # psutil_process = psutil.Process(process.pid) worker_processes = psutil_process.get_children(recursive=False) if len(worker_processes) == 0: return process.send_signal(signal.SIGTERM) for child_process in worker_processes: child_process.send_signal(signal.SIGINT) # Second time sigint is used, we should terminate supervisord itself which # will send SIGTERM to all the processes anyway. signal.signal(signal.SIGINT, sigterm_handler) # Wait for all the childs to finish for child_process in worker_processes: child_process.wait() # Then stop supervisord itself. process.send_signal(signal.SIGTERM) def sigterm_handler(signum, frame): # pylint: disable=unused-argument process.send_signal(signal.SIGTERM) signal.signal(signal.SIGINT, sigint_handler) signal.signal(signal.SIGTERM, sigterm_handler) process.wait() finally: os.remove(path) # If not, start the actual worker else: worker_class = load_class_by_path(cfg["worker_class"]) set_current_config(cfg) w = worker_class() exitcode = w.work_loop() sys.exit(exitcode)
import sys import psutil import time import re import json import urllib.request, urllib.error, urllib.parse sys.path.append(os.getcwd()) from mrq.job import Job, queue_raw_jobs, queue_jobs from mrq.queue import Queue from mrq.config import get_config from mrq.utils import wait_for_net_service from mrq.context import connections, set_current_config set_current_config(get_config(sources=("env"))) os.system("rm -rf dump.rdb") PYTHON_BIN = "python" if os.environ.get("PYTHON_BIN"): PYTHON_BIN = os.environ["PYTHON_BIN"] class ProcessFixture(object): def __init__(self, request, cmdline=None, wait_port=None, quiet=False): self.request = request self.cmdline = cmdline self.process = None self.wait_port = wait_port self.quiet = quiet
def main(): parser = argparse.ArgumentParser(description='Start a RQ worker') cfg = config.get_config(parser=parser, config_type="worker") # If we are launching with a --processes option and without the SUPERVISOR_ENABLED env # then we should just call supervisord. if cfg["processes"] > 0 and not os.environ.get("SUPERVISOR_ENABLED"): # We wouldn't need to do all that if supervisord supported environment variables in all its config fields! with open(cfg["supervisord_template"], "r") as f: conf = f.read() fh, path = tempfile.mkstemp(prefix="mrqsupervisordconfig") f = os.fdopen(fh, "w") # We basically relaunch ourselves, but the config will contain the MRQ_SUPERVISORD_ISWORKER env. conf = conf.replace("{{ SUPERVISORD_COMMAND }}", " ".join(sys.argv)) conf = conf.replace("{{ SUPERVISORD_PROCESSES }}", str(cfg["processes"])) f.write(conf) f.close() try: # start_new_session=True avoids sending the current process' signals to the child. process = subprocess.Popen(["supervisord", "-c", path], start_new_session=True) def sigint_handler(signum, frame): # At this point we need to send SIGINT to all workers. Unfortunately supervisord # doesn't support this, so we have to find all the children pids and send them the # signal ourselves :-/ # https://github.com/Supervisor/supervisor/issues/179 # psutil_process = psutil.Process(process.pid) worker_processes = psutil_process.get_children(recursive=False) if len(worker_processes) == 0: return process.send_signal(signal.SIGTERM) for child_process in worker_processes: child_process.send_signal(signal.SIGINT) # Second time sigint is used, we should terminate supervisord itself which # will send SIGTERM to all the processes anyway. signal.signal(signal.SIGINT, sigterm_handler) # Wait for all the childs to finish for child_process in worker_processes: child_process.wait() # Then stop supervisord itself. process.send_signal(signal.SIGTERM) def sigterm_handler(signum, frame): process.send_signal(signal.SIGTERM) signal.signal(signal.SIGINT, sigint_handler) signal.signal(signal.SIGTERM, sigterm_handler) process.wait() finally: os.remove(path) # If not, start the actual worker else: worker_class = load_class_by_path(cfg["worker_class"]) w = worker_class(cfg) exitcode = w.work_loop() sys.exit(exitcode)
import os import time import math import random from mrq.task import Task from mrq.job import queue_job from mrq.context import log, connections, get_current_job, retry_current_job, abort_current_job, set_current_config, get_current_config from mrq.config import get_config from pykl import pyfile, pyutils from utils import TaskSchemaWrapper, IPSchema, Regex, And, run_gdom_page from mrq.job import queue_raw_jobs, queue_job CONF = pyutils.Config(get_config()) CONF_CHECK_PROXY_FUNC = CONF.CONF_CHECK_PROXY_FUNC CONF_FETCHQL_PATH = CONF.CONF_FETCHQL_PATH CONF_DATA_OK_KEY = CONF.CONF_DATA_OK_KEY CONF_DATA_ALL_KEY = CONF.CONF_DATA_ALL_KEY CONF_DATA_RANK_KEY = CONF.CONF_DATA_RANK_KEY CONF_CHECK_INTERVAL = CONF.CONF_CHECK_INTERVAL _check_t = lambda p: p.get('ts', 0) * p.get('ti', 0) < int(time.time( )) - 2 * p.get('ts', 0) class CheckProxy(Task): @TaskSchemaWrapper( {
from gevent.pywsgi import WSGIServer from werkzeug.serving import run_with_reloader sys.path.insert(0, os.getcwd()) from mrq.queue import send_task, Queue from mrq.context import connections, set_current_config, get_current_config from mrq.config import get_config from mrq.dashboard.utils import jsonify, requires_auth CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) parser = argparse.ArgumentParser(description='Start the MRQ dashboard') cfg = get_config(parser=parser, config_type="dashboard") set_current_config(cfg) app = Flask("dashboard", static_folder=os.path.join(CURRENT_DIRECTORY, "static")) @app.route('/') @requires_auth def root(): return app.send_static_file("index.html") @app.route('/api/datatables/taskexceptions') @requires_auth def api_task_exceptions(): stats = list(connections.mongodb_jobs.mrq_jobs.aggregate([