Exemple #1
0
def put_starts(master=None):
    if master is None:
        master = ['%s:%s' % (get_ip(), getattr(user_config.master, 'port'))]
    print('master:%s' % master)
    jobs = client_call(master, 'runnable_jobs')
    app_name = ''
    for a, j in jobs.items():
        if j == "douban movie":
            app_name = a
            break
    if not app_name:
        raise Exception('douban movie job has not upload')

    nodes = client_call(master, 'list_workers')
    addrs = []
    default_addr = master.split(':')[0]
    for ap, s in nodes:
        a, p = ap.split(':')
        if a.lower() == 'localhost':
            addrs.append('%s:%s' % (default_addr, p))
        else:
            addrs.append(ap)

    mq_client = MessageQueueClient(addrs, app_name)
    print('get:%s' % mq_client.get())
    urls = []
    size = 0
    for url in starts:
        urls.append(url)
        size += 1
        if size >= PUTSIZE:
            mq_client.put(urls)
            size = 0
            urls = []
    if len(urls) > 0:
        mq_client.put(urls)
Exemple #2
0
class Test(unittest.TestCase):


    def setUp(self):
        ports = (11111, 11211, 11311)
        self.nodes = ['localhost:%s'%port for port in ports]
        self.dirs = [tempfile.mkdtemp() for _ in range(2*len(ports))]
        self.size = len(ports)
        
        for i in range(self.size):
            setattr(self, 'rpc_server%s'%i, ColaRPCServer(('localhost', ports[i])))
            setattr(self, 'mq%s'%i, 
                MessageQueue(self.nodes[:], self.nodes[i], getattr(self, 'rpc_server%s'%i))
            )
            getattr(self, 'mq%s'%i).init_store(self.dirs[2*i], self.dirs[2*i+1])
            thd = threading.Thread(target=getattr(self, 'rpc_server%s'%i).serve_forever)
            thd.setDaemon(True)
            thd.start()
            
        self.client = MessageQueueClient(self.nodes)

    def tearDown(self):
        try:
            for i in range(self.size):
                getattr(self, 'rpc_server%s'%i).shutdown()
                getattr(self, 'mq%s'%i).shutdown()
        finally:
            for d in self.dirs:
                shutil.rmtree(d)


    def testMQ(self):
        mq = self.mq0
        data = [str(random.randint(10000, 50000)) for _ in range(20)]
              
        mq.put(data)
        gets = []
        while True:
            get = mq.get()
            if get is None:
                break
            gets.append(get)
              
        self.assertEqual(sorted(data), sorted(gets))
            
        # test mq client
        data = str(random.randint(10000, 50000))
        self.client.put(data)
            
        get = self.client.get()
                 
        self.assertEqual(data, get)
        
    def testAddOrRemoveNode(self):
        mq = self.mq0
        data = [str(i) for i in range(100)]
           
        mq.put(data)
        self.mq2.shutdown()
        self.assertEqual(len(self.nodes), 3)
        self.mq0.remove_node(self.nodes[2])
        self.assertEqual(len(self.nodes), 3)
        self.mq1.remove_node(self.nodes[2])
           
        gets = []
        while True:
            get = mq.get()
            if get is None:
                break
            gets.append(get)
             
        self.assertEqual(sorted(data), sorted(gets))
Exemple #3
0
class Test(unittest.TestCase):
    def setUp(self):
        ports = (11111, 11211, 11311)
        self.nodes = ['localhost:%s' % port for port in ports]
        self.dirs = [tempfile.mkdtemp() for _ in range(2 * len(ports))]
        self.size = len(ports)

        for i in range(self.size):
            setattr(self, 'rpc_server%s' % i,
                    ColaRPCServer(('localhost', ports[i])))
            setattr(
                self, 'mq%s' % i,
                MessageQueue(self.nodes[:], self.nodes[i],
                             getattr(self, 'rpc_server%s' % i)))
            getattr(self, 'mq%s' % i).init_store(self.dirs[2 * i],
                                                 self.dirs[2 * i + 1])
            thd = threading.Thread(target=getattr(self, 'rpc_server%s' %
                                                  i).serve_forever)
            thd.setDaemon(True)
            thd.start()

        self.client = MessageQueueClient(self.nodes)

    def tearDown(self):
        try:
            for i in range(self.size):
                getattr(self, 'rpc_server%s' % i).shutdown()
                getattr(self, 'mq%s' % i).shutdown()
        finally:
            for d in self.dirs:
                shutil.rmtree(d)

    def testMQ(self):
        mq = self.mq0
        data = [str(random.randint(10000, 50000)) for _ in range(20)]

        mq.put(data)
        gets = []
        while True:
            get = mq.get()
            if get is None:
                break
            gets.append(get)

        self.assertEqual(sorted(data), sorted(gets))

        # test mq client
        data = str(random.randint(10000, 50000))
        self.client.put(data)

        get = self.client.get()

        self.assertEqual(data, get)

    def testAddOrRemoveNode(self):
        mq = self.mq0
        data = [str(i) for i in range(100)]

        mq.put(data)
        self.mq2.shutdown()
        self.assertEqual(len(self.nodes), 3)
        self.mq0.remove_node(self.nodes[2])
        self.assertEqual(len(self.nodes), 3)
        self.mq1.remove_node(self.nodes[2])

        gets = []
        while True:
            get = mq.get()
            if get is None:
                break
            gets.append(get)

        self.assertEqual(sorted(data), sorted(gets))