def runTest(self): input = self.test_server.urls([''] * 5) a, b, c = WaitJob1(), WaitJob1(), WaitJob2() self.job = JobChain({a: input, b: input, c: input}) self.assertRaises(JobError, self.job.wait) valid = JobChain({a: input, b: input}) self.assertEquals(valid.wait(), valid)
def test_scheme_disco(self): a, b = SchemesJobA(), SchemesJobB() self.job = JobChain({a: self.test_server.urls([''] * 10), b: a}) self.job.wait() for key, value in self.results(b): self.assert_(key in self.animals) self.assertEquals(value, None)
def test_save(self): ducks = ['dewey', 'huey', 'louie'] a, b = SaveJob1(), SaveJob2() self.job = JobChain({a: self.test_server.urls(ducks), b: a}).wait() self.tag = self.disco.results(b)[1][0] self.assertAllEqual(sorted(self.results(b)), [('%s!?!?' % d, '') for d in ducks])
def runTest(self): N = self.num_workers self.job = JobChain((AsyncJob(), self.test_server.urls(self.sample(N))) for x in range(5)) self.job.wait() for job in self.job: self.assertEquals(sum(1 for result in self.results(job)), N * 20)
def runTest(self): a, b, c = DavinChainJobA(), DavinChainJobA(), DavinChainJobC() self.job = JobChain({a: ['raw://0', 'raw://1', 'raw://2'], b: ['raw://3', 'raw://4', 'raw://5'], c: [a, b]}) self.job.wait() self.assertAllEqual(sorted(self.results(c)), ((str(x), '') for x in range(6)))
def test_discodb(self): if self.settings['DISCO_TEST_DISCODB']: a, b = WordCountDDB(), Query() b.params = 'discover' self.job = JobChain({a: [chekhov], b: a}).wait() self.assertEquals(self.results(b).next()[1], ['2']) else: self.skipTest("DISCO_TEST_DISCODB not set")
def runTest(self): a, b = ChainJobA(), ChainJobB() self.job = JobChain({a: self.test_server.urls([''] * 100), b: a}) self.job.wait() for key, value in self.results(b): self.assert_(key[:5] in self.animals) self.assertEquals(key[5:], b'0-1-') self.assertEquals(value, 1)
def test_partitioned_reduce(self): beers = ['sam_adams', 'trader_jose', 'boont_esb'] input = ['raw://{0}'.format(beer) for beer in beers] a, b, c, d = MapJob(), MapJob(), ReduceJob(), MergeReduceJob() self.job = JobChain({a: input, b: input, c: [a, b], d: [a, b]}) self.job.wait() self.assertAllEqual(sorted(self.results(c)), sorted(self.results(d)))
def test_chain(self): a, b = OOBJob1(), OOBJob2() self.job = JobChain({a: [], b: []}) a.run(input=self.test_server.urls(list(string.ascii_lowercase))).wait() b.run(input=['raw://a', 'raw://b', 'raw://c'], params={'job': a.name}) self.assertResults(b, [('good', '')] * 3) self.assertEquals( sorted(a.oob_list()), sorted( list(string.ascii_lowercase) + ['reduce:{0}'.format(i) for i in range(a.partitions)]))
def test_chain(self): a, b = FirstJob(), ChainJob() self.job = JobChain({a: ['raw://0'], b: a}) self.job.wait() self.assertResults(b, [(2, '')])