def test_botgroups(self):
self.project.settings['botgroups'] = ['g1', 'g2']
self.project.settings.save()
pq = self.project.jobq
hq = self.hsclient.jobq
q1 = pq.push(self.spidername)
q2 = pq.push(self.spidername)
q3 = pq.push(self.spidername)
self.assertEqual(hq.start(), None)
self.assertEqual(hq.start(botgroup='g3'), None)
self.assertEqual(apipoll(hq.start, botgroup='g1')['key'], q1['key'])
self.assertEqual(apipoll(hq.start, botgroup='g2')['key'], q2['key'])
# cleanup project botgroups, q3 must be polled only by generic bots
del self.project.settings['botgroups']
self.project.settings.save()
q4 = pq.push(self.spidername)
self.assertEqual(hq.start(botgroup='g1'), None)
self.assertEqual(hq.start(botgroup='g2'), None)
self.assertEqual(hq.start(botgroup='g3'), None)
self.assertEqual(hq.start()['key'], q3['key'])
self.assertEqual(hq.start()['key'], q4['key'])
self.project.settings['botgroups'] = ['python-hubstorage-test']
self.project.settings.save()
def test_botgroups(self):
self.project.settings['botgroups'] = ['g1', 'g2']
self.project.settings.save()
pq = self.project.jobq
hq = self.hsclient.jobq
q1 = pq.push(self.spidername)
q2 = pq.push(self.spidername)
q3 = pq.push(self.spidername)
self.assertEqual(hq.start(), None)
self.assertEqual(hq.start(botgroup='g3'), None)
self.assertEqual(apipoll(hq.start, botgroup='g1')['key'], q1['key'])
self.assertEqual(apipoll(hq.start, botgroup='g2')['key'], q2['key'])
# cleanup project botgroups, q3 must be polled only by generic bots
del self.project.settings['botgroups']
self.project.settings.save()
q4 = pq.push(self.spidername)
self.assertEqual(hq.start(botgroup='g1'), None)
self.assertEqual(hq.start(botgroup='g2'), None)
self.assertEqual(hq.start(botgroup='g3'), None)
self.assertEqual(hq.start()['key'], q3['key'])
self.assertEqual(hq.start()['key'], q4['key'])
self.project.settings['botgroups'] = ['python-hubstorage-test']
self.project.settings.save()
def test_simple_botgroups(self):
self.project.settings['botgroups'] = ['g1', 'g2']
self.project.settings.save()
pq = self.project.jobq
hq = self.hsclient.jobq
q1 = pq.push(self.spidername)
q2 = pq.push(self.spidername)
self.assertEqual(hq.start(botgroup='g3'), None)
self.assertEqual(apipoll(hq.start, botgroup='g1')['key'], q1['key'])
self.assertEqual(apipoll(hq.start, botgroup='g2')['key'], q2['key'])
def test_simple_botgroups(self):
self.project.settings['botgroups'] = ['g1']
self.project.settings.save()
pq = self.project.jobq
hq = self.hsclient.jobq
q1 = pq.push(self.spidername)
self.assertEqual(hq.start(botgroup='g3'), None)
self.assertEqual(apipoll(hq.start, botgroup='g1')['key'], q1['key'])
def test_jobsummaries(self):
hsc = self.hsclient
# add at least one running or pending job to ensure summary is returned
hsc.push_job(self.projectid, self.spidername, state='running')
def _get_summary():
jss = hsc.projects.jobsummaries()
mjss = dict((str(js['project']), js) for js in jss)
return mjss.get(self.projectid)
summary = apipoll(_get_summary)
self.assertIsNotNone(summary)
def test_botgroup(self):
self.project.settings.update(botgroups=['foo'], created=millitime())
self.project.settings.save()
c = self.hsclient
q1 = c.push_job(self.project.projectid, self.spidername)
j1 = c.start_job()
self.assertEqual(j1, None, 'got %s, pushed job was %s' % (j1, q1))
j2 = c.start_job(botgroup='bar')
self.assertEqual(j2, None, 'got %s, pushed job was %s' % (j2, q1))
j3 = apipoll(self.hsclient.start_job, botgroup='foo')
self.assertEqual(j3.key, q1.key)
def test_jobsummaries(self):
hsc = self.hsclient
# add at least one running or pending job to ensure summary is returned
hsc.push_job(self.projectid, self.spidername, state='running')
def _get_summary():
jss = hsc.projects.jobsummaries()
mjss = dict((str(js['project']), js) for js in jss)
return mjss.get(self.projectid)
summary = apipoll(_get_summary)
self.assertIsNotNone(summary)
def test_start_job(self):
# Pending queue is empty
job = self.hsclient.start_job(botgroup=self.testbotgroup)
self.assertEqual(job, None)
# Push a new job into pending queue
j0 = self.hsclient.push_job(self.projectid, self.spidername)
# Assert it is pending
self.assertEqual(j0.metadata.get('state'), 'pending')
# Poll for the pending job
j1 = apipoll(self.hsclient.start_job, botgroup=self.testbotgroup)
# Assert started job does not need an extra request to fetch its metadata
self.assertTrue(j1.metadata._cached is not None)
# Assert started job is in running queue
self.assertEqual(j1.metadata.get('state'), 'running')
# Started job metadata must match metadata retrieved directly from /jobs/
j2 = self.hsclient.get_job(j1.key)
self.assertEqual(dict(j1.metadata), dict(j2.metadata))