コード例 #1
0
ファイル: tests.py プロジェクト: mlzboy/dreque
 def setUp(self):
     import logging
     logging.basicConfig(level=logging.DEBUG)
     self.tempfile = tempfile.mkstemp()[1]
     self.dreque = Dreque("127.0.0.1", db=self.db)
     self.dreque.redis.flushdb()
     self.queue = "test"
コード例 #2
0
"""
brief:
this script purpose is implement parallel generate matrix,reduce time
so we use dreque to make work more efficienat
"""
import redis
from dreque import Dreque
from datetime import datetime

s=datetime.now()
ip='192.168.1.7'
r2=redis.Redis(db=2,host=ip)
r3=redis.Redis(db=3,host=ip)
r5=redis.Redis(db=5,host=ip)
r5.flushdb()
dreque = Dreque((ip,6379),db=6)#储存任务队列
dreque.redis.flushdb()
r6=dreque.redis


def gen_zero_list(n):
    return [1 for x in xrange(0,n)]
    
def make_list(db,key,value_list):
    "在db这个数据库中向键为key的list type填充value_list这个list中的所有值"
    #for i in value_list:
    #    db.rpush(key,i)
    pipe=db.pipeline()
    for i in value_list:
        pipe.rpush(key,i)
    pipe.execute()
コード例 #3
0
ファイル: tests.py プロジェクト: mrslow/dreque
 def setUp(self):
     logging.basicConfig(level=logging.DEBUG)
     self.tempfile = tempfile.mkstemp()[1]
     self.dreque = Dreque(db=self.db)
     self.dreque.redis.flushdb()
     self.queue = "test"
コード例 #4
0
ファイル: tests.py プロジェクト: mrslow/dreque
class TestDreque(unittest.TestCase):

    db = None

    def setUp(self):
        logging.basicConfig(level=logging.DEBUG)
        self.tempfile = tempfile.mkstemp()[1]
        self.dreque = Dreque(db=self.db)
        self.dreque.redis.flushdb()
        self.queue = "test"

    def tearDown(self):
        os.unlink(self.tempfile)

    def _get_output(self):
        with open(self.tempfile, "rb") as fp:
            return fp.read()

    def testSimple(self):
        self.dreque.push("test", "foo")
        self.assertEqual(self.dreque.size("test"), 1)
        self.assertEqual(self.dreque.pop("test"), "foo")
        self.assertEqual(self.dreque.pop("test"), None)
        self.assertEqual(self.dreque.size("test"), 0)

    def testFunction(self):
        self.dreque.enqueue("test", tests.test_func, "positional")
        self.assertEqual(
            self.dreque.dequeue(["test"]),
            dict(queue="test",
                 func="tests.test_func",
                 args=["positional"],
                 kwargs={},
                 retries_left=5))
        self.dreque.enqueue("test", tests.test_func, keyword="argument")
        self.assertEqual(
            self.dreque.dequeue(["test"]),
            dict(queue="test",
                 func="tests.test_func",
                 args=[],
                 kwargs={'keyword': "argument"},
                 retries_left=5))

    def testPositionalWorker(self):
        self.dreque.enqueue("test", tests.test_func, self.tempfile,
                            "worker_test")
        worker = DrequeWorker(["test"], db=self.db)
        worker.work(0)
        self.assertEqual(self._get_output(), b"worker_test")
        self.assertEqual(self.dreque.stats.get("processed"), 1)

    def testKeywordWorker(self):
        import tests
        self.dreque.enqueue("test",
                            tests.test_func,
                            tempfile=self.tempfile,
                            text="worker_test")
        worker = DrequeWorker(["test"], db=self.db)
        worker.work(0)
        self.assertEqual(self._get_output(), b"worker_test")

    def testDelayedJob(self):
        self.dreque.enqueue("test",
                            tests.test_func,
                            val="worker_test",
                            _delay=1)
        self.assertEqual(self.dreque.dequeue("test"), None)
        time.sleep(1.5)
        self.assertEqual(
            self.dreque.dequeue(["test"]),
            dict(queue="test",
                 func="tests.test_func",
                 args=[],
                 kwargs={'val': "worker_test"},
                 retries_left=5))

    def testGracefulShutdown(self):
        def worker():
            w = DrequeWorker(["test"], db=self.db)
            w.work(0)

        self.dreque.enqueue("test",
                            tests.test_func,
                            tempfile=self.tempfile,
                            text="graceful",
                            delay=2)
        worker_child = Process(target=worker, args=())
        worker_child.start()
        time.sleep(0.5)  # Make sure the worker has spawned a child already
        # worker_child.terminate()
        os.kill(worker_child.pid, signal.SIGQUIT)
        worker_child.join()
        self.assertEqual(worker_child.exitcode, 0)
        self.assertEqual(self._get_output(), b"graceful")

    def testForcedShutdown(self):
        def worker():
            logging.getLogger("dreque.worker").setLevel(logging.CRITICAL)
            w = DrequeWorker(["test"], db=self.db)
            w.work(0)

        self.dreque.enqueue("test",
                            tests.test_func,
                            tempfile=self.tempfile,
                            text="graceful",
                            delay=2)
        worker_child = Process(target=worker, args=())
        worker_child.start()
        time.sleep(0.5)  # Make sure the worker has spawned a child already
        worker_child.terminate()
        worker_child.join()
        self.assertEqual(worker_child.exitcode, 0)
        self.assertEqual(self._get_output(), b"")

    def testRetries(self):
        def worker():
            logging.getLogger("dreque.worker").setLevel(logging.CRITICAL)
            w = DrequeWorker(["test"], db=self.db)
            w.work(0.1)

        self.dreque.enqueue("test",
                            tests.test_func,
                            tempfile=self.tempfile,
                            text=".",
                            fail=True)
        worker_child = Process(target=worker, args=())
        worker_child.start()
        time.sleep(
            3)  # Give enough time for worker to grab and execute the job
        os.kill(worker_child.pid, signal.SIGQUIT)
        worker_child.join()
        self.assertEqual(worker_child.exitcode, 0)
        self.assertEqual(self._get_output(), b"..")
コード例 #5
0
from groups import Groups
from git import Git
from builder import Builder
from build_info import BuildInfo
from config import BrickConfig

import cyclone.web
import cyclone.escape
from twisted.internet import reactor
from twisted.python import log
from twisted.application import service, internet
from dreque import Dreque

brickconfig = BrickConfig()

queue = Dreque(brickconfig.get('redis', 'redis-server'))


class Project(cyclone.web.RequestHandler):
    def post(self, *args):
        if len(args) >= 1:
            name = args[0]
            project = Projects(name)
            for key, value in self.request.arguments.iteritems():
                if key in ("git_url", "version", "build_cmd", "install_cmd"):
                    setattr(project, key, value[0])
            project.save()

        try:
            if not Projects(self.get_argument('name')).exists():
                raise
コード例 #6
0
ファイル: tests.py プロジェクト: kgaughan/dreque
class TestDreque(unittest.TestCase):

    db = None

    def setUp(self):
        logging.basicConfig(level=logging.DEBUG)
        self.tempfile = tempfile.mkstemp()[1]
        self.dreque = Dreque(db=self.db)
        self.dreque.redis.flushdb()
        self.queue = "test"

    def tearDown(self):
        os.unlink(self.tempfile)

    def _get_output(self):
        with open(self.tempfile, "rb") as fp:
            return fp.read()

    def testSimple(self):
        self.dreque.push("test", "foo")
        self.failUnlessEqual(self.dreque.size("test"), 1)
        self.failUnlessEqual(self.dreque.pop("test"), "foo")
        self.failUnlessEqual(self.dreque.pop("test"), None)
        self.failUnlessEqual(self.dreque.size("test"), 0)

    def testFunction(self):
        self.dreque.enqueue("test", tests.test_func, "positional")
        self.failUnlessEqual(self.dreque.dequeue(["test"]),
                             dict(queue="test", func="tests.test_func",
                                  args=["positional"], kwargs={}, retries_left=5))
        self.dreque.enqueue("test", tests.test_func, keyword="argument")
        self.failUnlessEqual(self.dreque.dequeue(["test"]),
                             dict(queue="test", func="tests.test_func",
                                  args=[], kwargs={'keyword':"argument"},
                                  retries_left=5))

    def testPositionalWorker(self):
        self.dreque.enqueue("test", tests.test_func, self.tempfile, "worker_test")
        worker = DrequeWorker(["test"], db=self.db)
        worker.work(0)
        self.failUnlessEqual(self._get_output(), "worker_test")
        self.failUnlessEqual(self.dreque.stats.get("processed"), 1)

    def testKeywordWorker(self):
        import tests
        self.dreque.enqueue("test", tests.test_func, tempfile=self.tempfile,
                            text="worker_test")
        worker = DrequeWorker(["test"], db=self.db)
        worker.work(0)
        self.failUnlessEqual(self._get_output(), "worker_test")

    def testDelayedJob(self):
        self.dreque.enqueue("test", tests.test_func, val="worker_test", _delay=1)
        self.failUnlessEqual(self.dreque.dequeue("test"), None)
        time.sleep(1.5)
        self.failUnlessEqual(self.dreque.dequeue(["test"]),
                             dict(queue="test", func="tests.test_func",
                                  args=[], kwargs={'val':"worker_test"},
                                  retries_left=5))

    def testGracefulShutdown(self):
        def worker():
            w = DrequeWorker(["test"], db=self.db)
            w.work(0)

        self.dreque.enqueue("test", tests.test_func,
                            tempfile=self.tempfile, text="graceful", delay=2)
        worker_child = Process(target=worker, args=())
        worker_child.start()
        time.sleep(0.5)  # Make sure the worker has spawned a child already
        # worker_child.terminate()
        os.kill(worker_child.pid, signal.SIGQUIT)
        worker_child.join()
        self.failUnlessEqual(worker_child.exitcode, 0)
        self.failUnlessEqual(self._get_output(), "graceful")

    def testForcedShutdown(self):
        def worker():
            logging.getLogger("dreque.worker").setLevel(logging.CRITICAL)
            w = DrequeWorker(["test"], db=self.db)
            w.work(0)

        self.dreque.enqueue("test", tests.test_func,
                            tempfile=self.tempfile, text="graceful", delay=2)
        worker_child = Process(target=worker, args=())
        worker_child.start()
        time.sleep(0.5)  # Make sure the worker has spawned a child already
        worker_child.terminate()
        worker_child.join()
        self.failUnlessEqual(worker_child.exitcode, 0)
        self.failUnlessEqual(self._get_output(), "")

    def testRetries(self):
        def worker():
            logging.getLogger("dreque.worker").setLevel(logging.CRITICAL)
            w = DrequeWorker(["test"], db=self.db)
            w.work(0.1)

        self.dreque.enqueue("test", tests.test_func,
                            tempfile=self.tempfile, text=".", fail=True)
        worker_child = Process(target=worker, args=())
        worker_child.start()
        time.sleep(3)  # Give enough time for worker to grab and execute the job
        os.kill(worker_child.pid, signal.SIGQUIT)
        worker_child.join()
        self.failUnlessEqual(worker_child.exitcode, 0)
        self.failUnlessEqual(self._get_output(), "..")
コード例 #7
0
ファイル: builder.py プロジェクト: deferraz/bricklayer
import git

from twisted.internet import threads, reactor, defer
from config import BrickConfig
from projects import Projects

from builder_rpm import BuilderRpm
from builder_deb import BuilderDeb
from build_options import BuildOptions
from dreque import Dreque

config = BrickConfig()
redis_server = config.get('redis', 'redis-server')
log_file = config.get('log', 'file')

queue = Dreque(redis_server)

logging.basicConfig(filename=log_file, level=logging.DEBUG)
log = logging.getLogger('builder')

@defer.inlineCallbacks
def build_project(kargs):
    builder = Builder(kargs['project'])
    kargs.pop('project')
    yield builder.build_project(**kargs)

class Builder(object):
    def __init__(self, project):
        self.project = Projects(project)
        self.templates_dir = BrickConfig().get('workspace', 'template_dir')
        self.git = git.Git(self.project)
コード例 #8
0
#encoding=utf8
"""
brief:
在执行完check_lost_tasks___.py后会将找回的丢失任务再插入到r1的queue中
这个脚本的作用是再将这些丢失的任务生成dreque的消息任务
"""
from __future__ import division#must occur at the beginning of the file
from math import sqrt,acos,log
import redis
from dreque import Dreque
from datetime import datetime

r1=redis.Redis(host='10.3.11.178',db=1)
r4=redis.Redis(host='10.3.11.178',db=4)#计算完成tf/idf后的矩阵
r6=redis.Redis(host='10.3.11.178',db=6)#储存执行结果
dreque = Dreque(("10.3.11.178",6379),db=5)#储存任务队列
#TODO:how to know dreque current has left tasks in it's queue,in therory,it should be 0
#print dreque.redis.dbsize()
start=datetime.now()

def sim(p1,p2):
    "余弦相似度"
    p1_mo=sqrt(sum([x**2 for x in p1]))
    p2_mo=sqrt(sum([x**2 for x in p2]))
    p1p2_dianji=0
    for i in xrange(len(p1)):
        p1p2_dianji+=p1[i]*p2[i] 
    try:
        return p1p2_dianji/(p1_mo*p2_mo)
    except:
        return 0.0
コード例 #9
0
ファイル: tests.py プロジェクト: henriklied/dreque
 def setUp(self):
     import logging
     logging.basicConfig(level=logging.DEBUG)
     self.dreque = Dreque("127.0.0.1")
     self.queue = "test"
     self.dreque.remove_queue(self.queue)
コード例 #10
0
ファイル: tests.py プロジェクト: henriklied/dreque
class TestDreque(unittest.TestCase):
    def setUp(self):
        import logging
        logging.basicConfig(level=logging.DEBUG)
        self.dreque = Dreque("127.0.0.1")
        self.queue = "test"
        self.dreque.remove_queue(self.queue)

    def tearDown(self):
        pass

    def testSimple(self):
        self.dreque.push("test", "foo")
        self.failUnlessEqual(self.dreque.pop("test"), "foo")
        self.failUnlessEqual(self.dreque.pop("test"), None)

    def testFunction(self):
        import tests
        self.dreque.enqueue("test", tests.set_something, "positional")
        self.failUnlessEqual(self.dreque.dequeue(["test"]), dict(queue="test", func="tests.set_something", args=["positional"], kwargs={}))
        self.dreque.enqueue("test", tests.set_something, keyword="argument")
        self.failUnlessEqual(self.dreque.dequeue(["test"]), dict(queue="test", func="tests.set_something", args=[], kwargs={'keyword':"argument"}))

    def testPositionalWorker(self):
        import tests
        self.dreque.enqueue("test", tests.set_something, "worker_test")
        worker = DrequeWorker(["test"], "127.0.0.1", nofork=True)
        worker.work(0)
        self.failUnlessEqual(tests.something, "worker_test")

    def testKeywordWorker(self):
        import tests
        self.dreque.enqueue("test", tests.set_something, val="worker_test")
        worker = DrequeWorker(["test"], "127.0.0.1", nofork=True)
        worker.work(0)
        self.failUnlessEqual(tests.something, "worker_test")

    def testDelayedJob(self):
        import tests
        self.dreque.enqueue("test", tests.set_something, val="worker_test", _delay=1)
        self.failUnlessEqual(self.dreque.dequeue("test"), None)
        time.sleep(1.5)
        self.failUnlessEqual(self.dreque.dequeue(["test"]), dict(queue="test", func="tests.set_something", args=[], kwargs={'val':"worker_test"}))