def setUp(self):
import logging
logging.basicConfig(level=logging.DEBUG)
self.tempfile = tempfile.mkstemp()[1]
self.dreque = Dreque("127.0.0.1", db=self.db)
self.dreque.redis.flushdb()
self.queue = "test"
"""
brief:
this script purpose is implement parallel generate matrix,reduce time
so we use dreque to make work more efficienat
"""
import redis
from dreque import Dreque
from datetime import datetime
s=datetime.now()
ip='192.168.1.7'
r2=redis.Redis(db=2,host=ip)
r3=redis.Redis(db=3,host=ip)
r5=redis.Redis(db=5,host=ip)
r5.flushdb()
dreque = Dreque((ip,6379),db=6)#储存任务队列
dreque.redis.flushdb()
r6=dreque.redis
def gen_zero_list(n):
return [1 for x in xrange(0,n)]
def make_list(db,key,value_list):
"在db这个数据库中向键为key的list type填充value_list这个list中的所有值"
#for i in value_list:
# db.rpush(key,i)
pipe=db.pipeline()
for i in value_list:
pipe.rpush(key,i)
pipe.execute()
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.tempfile = tempfile.mkstemp()[1]
self.dreque = Dreque(db=self.db)
self.dreque.redis.flushdb()
self.queue = "test"
class TestDreque(unittest.TestCase):
db = None
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.tempfile = tempfile.mkstemp()[1]
self.dreque = Dreque(db=self.db)
self.dreque.redis.flushdb()
self.queue = "test"
def tearDown(self):
os.unlink(self.tempfile)
def _get_output(self):
with open(self.tempfile, "rb") as fp:
return fp.read()
def testSimple(self):
self.dreque.push("test", "foo")
self.assertEqual(self.dreque.size("test"), 1)
self.assertEqual(self.dreque.pop("test"), "foo")
self.assertEqual(self.dreque.pop("test"), None)
self.assertEqual(self.dreque.size("test"), 0)
def testFunction(self):
self.dreque.enqueue("test", tests.test_func, "positional")
self.assertEqual(
self.dreque.dequeue(["test"]),
dict(queue="test",
func="tests.test_func",
args=["positional"],
kwargs={},
retries_left=5))
self.dreque.enqueue("test", tests.test_func, keyword="argument")
self.assertEqual(
self.dreque.dequeue(["test"]),
dict(queue="test",
func="tests.test_func",
args=[],
kwargs={'keyword': "argument"},
retries_left=5))
def testPositionalWorker(self):
self.dreque.enqueue("test", tests.test_func, self.tempfile,
"worker_test")
worker = DrequeWorker(["test"], db=self.db)
worker.work(0)
self.assertEqual(self._get_output(), b"worker_test")
self.assertEqual(self.dreque.stats.get("processed"), 1)
def testKeywordWorker(self):
import tests
self.dreque.enqueue("test",
tests.test_func,
tempfile=self.tempfile,
text="worker_test")
worker = DrequeWorker(["test"], db=self.db)
worker.work(0)
self.assertEqual(self._get_output(), b"worker_test")
def testDelayedJob(self):
self.dreque.enqueue("test",
tests.test_func,
val="worker_test",
_delay=1)
self.assertEqual(self.dreque.dequeue("test"), None)
time.sleep(1.5)
self.assertEqual(
self.dreque.dequeue(["test"]),
dict(queue="test",
func="tests.test_func",
args=[],
kwargs={'val': "worker_test"},
retries_left=5))
def testGracefulShutdown(self):
def worker():
w = DrequeWorker(["test"], db=self.db)
w.work(0)
self.dreque.enqueue("test",
tests.test_func,
tempfile=self.tempfile,
text="graceful",
delay=2)
worker_child = Process(target=worker, args=())
worker_child.start()
time.sleep(0.5) # Make sure the worker has spawned a child already
# worker_child.terminate()
os.kill(worker_child.pid, signal.SIGQUIT)
worker_child.join()
self.assertEqual(worker_child.exitcode, 0)
self.assertEqual(self._get_output(), b"graceful")
def testForcedShutdown(self):
def worker():
logging.getLogger("dreque.worker").setLevel(logging.CRITICAL)
w = DrequeWorker(["test"], db=self.db)
w.work(0)
self.dreque.enqueue("test",
tests.test_func,
tempfile=self.tempfile,
text="graceful",
delay=2)
worker_child = Process(target=worker, args=())
worker_child.start()
time.sleep(0.5) # Make sure the worker has spawned a child already
worker_child.terminate()
worker_child.join()
self.assertEqual(worker_child.exitcode, 0)
self.assertEqual(self._get_output(), b"")
def testRetries(self):
def worker():
logging.getLogger("dreque.worker").setLevel(logging.CRITICAL)
w = DrequeWorker(["test"], db=self.db)
w.work(0.1)
self.dreque.enqueue("test",
tests.test_func,
tempfile=self.tempfile,
text=".",
fail=True)
worker_child = Process(target=worker, args=())
worker_child.start()
time.sleep(
3) # Give enough time for worker to grab and execute the job
os.kill(worker_child.pid, signal.SIGQUIT)
worker_child.join()
self.assertEqual(worker_child.exitcode, 0)
self.assertEqual(self._get_output(), b"..")
from groups import Groups
from git import Git
from builder import Builder
from build_info import BuildInfo
from config import BrickConfig
import cyclone.web
import cyclone.escape
from twisted.internet import reactor
from twisted.python import log
from twisted.application import service, internet
from dreque import Dreque
brickconfig = BrickConfig()
queue = Dreque(brickconfig.get('redis', 'redis-server'))
class Project(cyclone.web.RequestHandler):
def post(self, *args):
if len(args) >= 1:
name = args[0]
project = Projects(name)
for key, value in self.request.arguments.iteritems():
if key in ("git_url", "version", "build_cmd", "install_cmd"):
setattr(project, key, value[0])
project.save()
try:
if not Projects(self.get_argument('name')).exists():
raise
class TestDreque(unittest.TestCase):
db = None
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.tempfile = tempfile.mkstemp()[1]
self.dreque = Dreque(db=self.db)
self.dreque.redis.flushdb()
self.queue = "test"
def tearDown(self):
os.unlink(self.tempfile)
def _get_output(self):
with open(self.tempfile, "rb") as fp:
return fp.read()
def testSimple(self):
self.dreque.push("test", "foo")
self.failUnlessEqual(self.dreque.size("test"), 1)
self.failUnlessEqual(self.dreque.pop("test"), "foo")
self.failUnlessEqual(self.dreque.pop("test"), None)
self.failUnlessEqual(self.dreque.size("test"), 0)
def testFunction(self):
self.dreque.enqueue("test", tests.test_func, "positional")
self.failUnlessEqual(self.dreque.dequeue(["test"]),
dict(queue="test", func="tests.test_func",
args=["positional"], kwargs={}, retries_left=5))
self.dreque.enqueue("test", tests.test_func, keyword="argument")
self.failUnlessEqual(self.dreque.dequeue(["test"]),
dict(queue="test", func="tests.test_func",
args=[], kwargs={'keyword':"argument"},
retries_left=5))
def testPositionalWorker(self):
self.dreque.enqueue("test", tests.test_func, self.tempfile, "worker_test")
worker = DrequeWorker(["test"], db=self.db)
worker.work(0)
self.failUnlessEqual(self._get_output(), "worker_test")
self.failUnlessEqual(self.dreque.stats.get("processed"), 1)
def testKeywordWorker(self):
import tests
self.dreque.enqueue("test", tests.test_func, tempfile=self.tempfile,
text="worker_test")
worker = DrequeWorker(["test"], db=self.db)
worker.work(0)
self.failUnlessEqual(self._get_output(), "worker_test")
def testDelayedJob(self):
self.dreque.enqueue("test", tests.test_func, val="worker_test", _delay=1)
self.failUnlessEqual(self.dreque.dequeue("test"), None)
time.sleep(1.5)
self.failUnlessEqual(self.dreque.dequeue(["test"]),
dict(queue="test", func="tests.test_func",
args=[], kwargs={'val':"worker_test"},
retries_left=5))
def testGracefulShutdown(self):
def worker():
w = DrequeWorker(["test"], db=self.db)
w.work(0)
self.dreque.enqueue("test", tests.test_func,
tempfile=self.tempfile, text="graceful", delay=2)
worker_child = Process(target=worker, args=())
worker_child.start()
time.sleep(0.5) # Make sure the worker has spawned a child already
# worker_child.terminate()
os.kill(worker_child.pid, signal.SIGQUIT)
worker_child.join()
self.failUnlessEqual(worker_child.exitcode, 0)
self.failUnlessEqual(self._get_output(), "graceful")
def testForcedShutdown(self):
def worker():
logging.getLogger("dreque.worker").setLevel(logging.CRITICAL)
w = DrequeWorker(["test"], db=self.db)
w.work(0)
self.dreque.enqueue("test", tests.test_func,
tempfile=self.tempfile, text="graceful", delay=2)
worker_child = Process(target=worker, args=())
worker_child.start()
time.sleep(0.5) # Make sure the worker has spawned a child already
worker_child.terminate()
worker_child.join()
self.failUnlessEqual(worker_child.exitcode, 0)
self.failUnlessEqual(self._get_output(), "")
def testRetries(self):
def worker():
logging.getLogger("dreque.worker").setLevel(logging.CRITICAL)
w = DrequeWorker(["test"], db=self.db)
w.work(0.1)
self.dreque.enqueue("test", tests.test_func,
tempfile=self.tempfile, text=".", fail=True)
worker_child = Process(target=worker, args=())
worker_child.start()
time.sleep(3) # Give enough time for worker to grab and execute the job
os.kill(worker_child.pid, signal.SIGQUIT)
worker_child.join()
self.failUnlessEqual(worker_child.exitcode, 0)
self.failUnlessEqual(self._get_output(), "..")
import git
from twisted.internet import threads, reactor, defer
from config import BrickConfig
from projects import Projects
from builder_rpm import BuilderRpm
from builder_deb import BuilderDeb
from build_options import BuildOptions
from dreque import Dreque
config = BrickConfig()
redis_server = config.get('redis', 'redis-server')
log_file = config.get('log', 'file')
queue = Dreque(redis_server)
logging.basicConfig(filename=log_file, level=logging.DEBUG)
log = logging.getLogger('builder')
@defer.inlineCallbacks
def build_project(kargs):
builder = Builder(kargs['project'])
kargs.pop('project')
yield builder.build_project(**kargs)
class Builder(object):
def __init__(self, project):
self.project = Projects(project)
self.templates_dir = BrickConfig().get('workspace', 'template_dir')
self.git = git.Git(self.project)
#encoding=utf8
"""
brief:
在执行完check_lost_tasks___.py后会将找回的丢失任务再插入到r1的queue中
这个脚本的作用是再将这些丢失的任务生成dreque的消息任务
"""
from __future__ import division#must occur at the beginning of the file
from math import sqrt,acos,log
import redis
from dreque import Dreque
from datetime import datetime
r1=redis.Redis(host='10.3.11.178',db=1)
r4=redis.Redis(host='10.3.11.178',db=4)#计算完成tf/idf后的矩阵
r6=redis.Redis(host='10.3.11.178',db=6)#储存执行结果
dreque = Dreque(("10.3.11.178",6379),db=5)#储存任务队列
#TODO:how to know dreque current has left tasks in it's queue,in therory,it should be 0
#print dreque.redis.dbsize()
start=datetime.now()
def sim(p1,p2):
"余弦相似度"
p1_mo=sqrt(sum([x**2 for x in p1]))
p2_mo=sqrt(sum([x**2 for x in p2]))
p1p2_dianji=0
for i in xrange(len(p1)):
p1p2_dianji+=p1[i]*p2[i]
try:
return p1p2_dianji/(p1_mo*p2_mo)
except:
return 0.0
def setUp(self):
import logging
logging.basicConfig(level=logging.DEBUG)
self.dreque = Dreque("127.0.0.1")
self.queue = "test"
self.dreque.remove_queue(self.queue)
class TestDreque(unittest.TestCase):
def setUp(self):
import logging
logging.basicConfig(level=logging.DEBUG)
self.dreque = Dreque("127.0.0.1")
self.queue = "test"
self.dreque.remove_queue(self.queue)
def tearDown(self):
pass
def testSimple(self):
self.dreque.push("test", "foo")
self.failUnlessEqual(self.dreque.pop("test"), "foo")
self.failUnlessEqual(self.dreque.pop("test"), None)
def testFunction(self):
import tests
self.dreque.enqueue("test", tests.set_something, "positional")
self.failUnlessEqual(self.dreque.dequeue(["test"]), dict(queue="test", func="tests.set_something", args=["positional"], kwargs={}))
self.dreque.enqueue("test", tests.set_something, keyword="argument")
self.failUnlessEqual(self.dreque.dequeue(["test"]), dict(queue="test", func="tests.set_something", args=[], kwargs={'keyword':"argument"}))
def testPositionalWorker(self):
import tests
self.dreque.enqueue("test", tests.set_something, "worker_test")
worker = DrequeWorker(["test"], "127.0.0.1", nofork=True)
worker.work(0)
self.failUnlessEqual(tests.something, "worker_test")
def testKeywordWorker(self):
import tests
self.dreque.enqueue("test", tests.set_something, val="worker_test")
worker = DrequeWorker(["test"], "127.0.0.1", nofork=True)
worker.work(0)
self.failUnlessEqual(tests.something, "worker_test")
def testDelayedJob(self):
import tests
self.dreque.enqueue("test", tests.set_something, val="worker_test", _delay=1)
self.failUnlessEqual(self.dreque.dequeue("test"), None)
time.sleep(1.5)
self.failUnlessEqual(self.dreque.dequeue(["test"]), dict(queue="test", func="tests.set_something", args=[], kwargs={'val':"worker_test"}))
