def main():
parser = argparse.ArgumentParser(description='Start a MRQ worker')
cfg = config.get_config(parser=parser,
config_type="worker",
sources=("file", "env", "args"))
set_current_config(cfg)
set_logger_config()
# If we are launching with a --processes option and without MRQ_IS_SUBPROCESS, we are a manager process
if cfg["processes"] > 0 and not os.environ.get("MRQ_IS_SUBPROCESS"):
from mrq.supervisor import Supervisor
command = " ".join(map(pipes.quote, sys.argv))
w = Supervisor(command, numprocs=cfg["processes"])
w.work()
sys.exit(w.exitcode)
# If not, start an actual worker
else:
worker_class = load_class_by_path(cfg["worker_class"])
w = worker_class()
w.work()
sys.exit(w.exitcode)
def main():
parser = argparse.ArgumentParser(description='Runs a task')
cfg = config.get_config(parser=parser, config_type="run", sources=("file", "env", "args"))
cfg["is_cli"] = True
set_current_config(cfg)
if len(cfg["taskargs"]) == 1:
params = json.loads(cfg["taskargs"][0]) # pylint: disable=no-member
else:
params = {}
# mrq-run taskpath a 1 b 2 => {"a": "1", "b": "2"}
for group in utils.group_iter(cfg["taskargs"], n=2):
if len(group) != 2:
print("Number of arguments wasn't even")
sys.exit(1)
params[group[0]] = group[1]
if cfg["queue"]:
ret = queue_job(cfg["taskpath"], params, queue=cfg["queue"])
print(ret)
else:
worker_class = load_class_by_path(cfg["worker_class"])
job = worker_class.job_class(None)
job.set_data({
"path": cfg["taskpath"],
"params": params,
"queue": cfg["queue"]
})
job.datestarted = datetime.datetime.utcnow()
set_current_job(job)
ret = job.perform()
print(json_stdlib.dumps(ret, cls=MongoJSONEncoder)) # pylint: disable=no-member
def main():
parser = argparse.ArgumentParser(description='Runs a task')
cfg = config.get_config(parser=parser, config_type="run")
cfg["is_cli"] = True
set_current_config(cfg)
log.info(cfg)
if len(cfg["taskargs"]) == 1:
params = json.loads(cfg["taskargs"][0])
else:
params = {}
# mrq-run taskpath a 1 b 2 => {"a": "1", "b": "2"}
for group in utils.group_iter(cfg["taskargs"], n=2):
if len(group) != 2:
print "Number of arguments wasn't even"
sys.exit(1)
params[group[0]] = group[1]
if cfg["async"]:
ret = queue.send_task(cfg["taskpath"], params, sync=False, queue=cfg["queue"])
print ret
else:
worker_class = load_class_by_path(cfg["worker_class"])
job = worker_class.job_class(None)
job.data = {
"path": cfg["taskpath"],
"params": params,
"queue": cfg["queue"]
}
job.datestarted = datetime.datetime.utcnow()
set_current_job(job)
ret = job.perform()
print json.dumps(ret)
def main():
set_current_config(get_config())
print "======== START ========="
add_num = 0
print 'all:', add_num
print "======== END ========="
def main():
parser = argparse.ArgumentParser(description='Start a MRQ agent')
cfg = config.get_config(parser=parser, config_type="agent", sources=("file", "env", "args"))
set_current_config(cfg)
agent = Agent()
agent.work()
sys.exit(agent.exitcode)
import sys
import psutil
import time
import re
import json
import urllib2
sys.path.append(os.getcwd())
from mrq.job import Job
from mrq.queue import send_tasks, send_raw_tasks, Queue
from mrq.config import get_config
from mrq.utils import wait_for_net_service
from mrq.context import connections, set_current_config
set_current_config(get_config(sources=("env")))
os.system("rm -rf dump.rdb")
class ProcessFixture(object):
def __init__(self, request, cmdline=None, wait_port=None, quiet=False):
self.request = request
self.cmdline = cmdline
self.process = None
self.wait_port = wait_port
self.quiet = quiet
self.stopped = False
self.request.addfinalizer(self.stop)
sys.path.insert(0, os.getcwd())
from mrq.queue import Queue
from mrq.context import connections, set_current_config, get_current_config
from mrq.job import queue_job
from mrq.config import get_config
from mrq.dashboard.utils import jsonify, requires_auth
CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
parser = argparse.ArgumentParser(description='Start the MRQ dashboard')
cfg = get_config(parser=parser,
config_type="dashboard",
sources=("file", "env", "args"))
set_current_config(cfg)
app = Flask("dashboard",
static_folder=os.path.join(CURRENT_DIRECTORY, "static"),
template_folder=os.path.join(CURRENT_DIRECTORY, "templates"))
WHITELISTED_MRQ_CONFIG_KEYS = ["dashboard_autolink_repositories"]
@app.route('/')
@requires_auth
def root():
return render_template("index.html",
MRQ_CONFIG={
from werkzeug.serving import run_simple
sys.path.insert(0, os.getcwd())
from mrq.queue import Queue
from mrq.context import connections, set_current_config, get_current_config
from mrq.job import queue_job
from mrq.config import get_config
from mrq.dashboard.utils import jsonify, requires_auth
CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
parser = argparse.ArgumentParser(description='Start the MRQ dashboard')
cfg = get_config(parser=parser, config_type="dashboard", sources=("file", "env", "args"))
set_current_config(cfg)
app = Flask(
"dashboard",
static_folder=os.path.join(CURRENT_DIRECTORY, "static"),
template_folder=os.path.join(CURRENT_DIRECTORY, "templates")
)
WHITELISTED_MRQ_CONFIG_KEYS = ["dashboard_autolink_repositories"]
@app.route('/')
@requires_auth
def root():
return render_template("index.html", MRQ_CONFIG={
def main():
parser = argparse.ArgumentParser(description='Start a RQ worker')
cfg = config.get_config(parser=parser,
config_type="worker",
sources=("file", "env", "args"))
# If we are launching with a --processes option and without the SUPERVISOR_ENABLED env
# then we should just call supervisord.
if cfg["processes"] > 0 and not os.environ.get("SUPERVISOR_ENABLED"):
# We wouldn't need to do all that if supervisord supported environment
# variables in all its config fields!
with open(cfg["supervisord_template"], "r") as f:
conf = f.read()
fh, path = tempfile.mkstemp(prefix="mrqsupervisordconfig")
f = os.fdopen(fh, "w")
# We basically relaunch ourselves, but the config will contain the
# MRQ_SUPERVISORD_ISWORKER env.
conf = conf.replace("{{ SUPERVISORD_COMMAND }}", " ".join(sys.argv))
conf = conf.replace("{{ SUPERVISORD_PROCESSES }}",
str(cfg["processes"]))
f.write(conf)
f.close()
try:
# start_new_session=True avoids sending the current process'
# signals to the child.
process = subprocess.Popen(["supervisord", "-c", path],
start_new_session=True)
def sigint_handler(signum, frame): # pylint: disable=unused-argument
# At this point we need to send SIGINT to all workers. Unfortunately supervisord
# doesn't support this, so we have to find all the children pids and send them the
# signal ourselves :-/
# https://github.com/Supervisor/supervisor/issues/179
#
psutil_process = psutil.Process(process.pid)
worker_processes = psutil_process.get_children(recursive=False)
if len(worker_processes) == 0:
return process.send_signal(signal.SIGTERM)
for child_process in worker_processes:
child_process.send_signal(signal.SIGINT)
# Second time sigint is used, we should terminate supervisord itself which
# will send SIGTERM to all the processes anyway.
signal.signal(signal.SIGINT, sigterm_handler)
# Wait for all the childs to finish
for child_process in worker_processes:
child_process.wait()
# Then stop supervisord itself.
process.send_signal(signal.SIGTERM)
def sigterm_handler(signum, frame): # pylint: disable=unused-argument
process.send_signal(signal.SIGTERM)
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigterm_handler)
process.wait()
finally:
os.remove(path)
# If not, start the actual worker
else:
worker_class = load_class_by_path(cfg["worker_class"])
set_current_config(cfg)
w = worker_class()
exitcode = w.work_loop()
sys.exit(exitcode)
import sys
import psutil
import time
import re
import json
import urllib.request, urllib.error, urllib.parse
sys.path.append(os.getcwd())
from mrq.job import Job, queue_raw_jobs, queue_jobs
from mrq.queue import Queue
from mrq.config import get_config
from mrq.utils import wait_for_net_service
from mrq.context import connections, set_current_config
set_current_config(get_config(sources=("env")))
os.system("rm -rf dump.rdb")
PYTHON_BIN = "python"
if os.environ.get("PYTHON_BIN"):
PYTHON_BIN = os.environ["PYTHON_BIN"]
class ProcessFixture(object):
def __init__(self, request, cmdline=None, wait_port=None, quiet=False):
self.request = request
self.cmdline = cmdline
self.process = None
self.wait_port = wait_port
self.quiet = quiet
def main():
parser = argparse.ArgumentParser(description='Start a RQ worker')
cfg = config.get_config(parser=parser, config_type="worker")
# If we are launching with a --processes option and without the SUPERVISOR_ENABLED env
# then we should just call supervisord.
if cfg["processes"] > 0 and not os.environ.get("SUPERVISOR_ENABLED"):
# We wouldn't need to do all that if supervisord supported environment variables in all its config fields!
with open(cfg["supervisord_template"], "r") as f:
conf = f.read()
fh, path = tempfile.mkstemp(prefix="mrqsupervisordconfig")
f = os.fdopen(fh, "w")
# We basically relaunch ourselves, but the config will contain the MRQ_SUPERVISORD_ISWORKER env.
conf = conf.replace("{{ SUPERVISORD_COMMAND }}", " ".join(sys.argv))
conf = conf.replace("{{ SUPERVISORD_PROCESSES }}", str(cfg["processes"]))
f.write(conf)
f.close()
try:
# start_new_session=True avoids sending the current process' signals to the child.
process = subprocess.Popen(["supervisord", "-c", path], start_new_session=True)
def sigint_handler(signum, frame):
# At this point we need to send SIGINT to all workers. Unfortunately supervisord
# doesn't support this, so we have to find all the children pids and send them the
# signal ourselves :-/
# https://github.com/Supervisor/supervisor/issues/179
#
psutil_process = psutil.Process(process.pid)
worker_processes = psutil_process.get_children(recursive=False)
if len(worker_processes) == 0:
return process.send_signal(signal.SIGTERM)
for child_process in worker_processes:
child_process.send_signal(signal.SIGINT)
# Second time sigint is used, we should terminate supervisord itself which
# will send SIGTERM to all the processes anyway.
signal.signal(signal.SIGINT, sigterm_handler)
# Wait for all the childs to finish
for child_process in worker_processes:
child_process.wait()
# Then stop supervisord itself.
process.send_signal(signal.SIGTERM)
def sigterm_handler(signum, frame):
process.send_signal(signal.SIGTERM)
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigterm_handler)
process.wait()
finally:
os.remove(path)
# If not, start the actual worker
else:
worker_class = load_class_by_path(cfg["worker_class"])
w = worker_class(cfg)
exitcode = w.work_loop()
sys.exit(exitcode)
import os
import time
import math
import random
from mrq.task import Task
from mrq.job import queue_job
from mrq.context import log, connections, get_current_job, retry_current_job, abort_current_job, set_current_config, get_current_config
from mrq.config import get_config
from pykl import pyfile, pyutils
from utils import TaskSchemaWrapper, IPSchema, Regex, And, run_gdom_page
from mrq.job import queue_raw_jobs, queue_job
CONF = pyutils.Config(get_config())
CONF_CHECK_PROXY_FUNC = CONF.CONF_CHECK_PROXY_FUNC
CONF_FETCHQL_PATH = CONF.CONF_FETCHQL_PATH
CONF_DATA_OK_KEY = CONF.CONF_DATA_OK_KEY
CONF_DATA_ALL_KEY = CONF.CONF_DATA_ALL_KEY
CONF_DATA_RANK_KEY = CONF.CONF_DATA_RANK_KEY
CONF_CHECK_INTERVAL = CONF.CONF_CHECK_INTERVAL
_check_t = lambda p: p.get('ts', 0) * p.get('ti', 0) < int(time.time(
)) - 2 * p.get('ts', 0)
class CheckProxy(Task):
@TaskSchemaWrapper(
{
from gevent.pywsgi import WSGIServer
from werkzeug.serving import run_with_reloader
sys.path.insert(0, os.getcwd())
from mrq.queue import send_task, Queue
from mrq.context import connections, set_current_config, get_current_config
from mrq.config import get_config
from mrq.dashboard.utils import jsonify, requires_auth
CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
parser = argparse.ArgumentParser(description='Start the MRQ dashboard')
cfg = get_config(parser=parser, config_type="dashboard")
set_current_config(cfg)
app = Flask("dashboard", static_folder=os.path.join(CURRENT_DIRECTORY, "static"))
@app.route('/')
@requires_auth
def root():
return app.send_static_file("index.html")
@app.route('/api/datatables/taskexceptions')
@requires_auth
def api_task_exceptions():
stats = list(connections.mongodb_jobs.mrq_jobs.aggregate([
