class TestProcessor(unittest.TestCase):
projectdb_path = './test/data/project.db'
@classmethod
def setUpClass(self):
shutil.rmtree('./test/data/', ignore_errors=True)
os.makedirs('./test/data/')
def get_projectdb():
return projectdb.ProjectDB(self.projectdb_path)
self.projectdb = get_projectdb()
self.in_queue = Queue(10)
self.status_queue = Queue(10)
self.newtask_queue = Queue(10)
self.result_queue = Queue(10)
def run_processor():
self.processor = Processor(get_projectdb(), self.in_queue,
self.status_queue, self.newtask_queue,
self.result_queue)
self.processor.CHECK_PROJECTS_INTERVAL = 0.1
self.processor.run()
self.process = run_in_thread(run_processor)
time.sleep(1)
@classmethod
def tearDownClass(self):
if self.process.is_alive():
self.processor.quit()
self.process.join(2)
assert not self.process.is_alive()
shutil.rmtree('./test/data/', ignore_errors=True)
def test_10_update_project(self):
self.assertEqual(len(self.processor.projects), 0)
self.projectdb.insert(
'test_project', {
'name': 'test_project',
'group': 'group',
'status': 'TODO',
'script': open('libs/sample_handler.py', 'r').read(),
'comments': 'test project',
'rate': 1.0,
'burst': 10,
})
task = {
"process": {
"callback": "on_start"
},
"project": "not_exists",
"taskid": "data:,on_start",
"url": "data:,on_start"
}
self.in_queue.put((task, {}))
time.sleep(1)
self.assertTrue(self.status_queue.empty())
self.assertEqual(len(self.processor.projects), 1)
def test_30_new_task(self):
self.assertTrue(self.status_queue.empty())
self.assertTrue(self.newtask_queue.empty())
task = {
"process": {
"callback": "on_start"
},
"project": "test_project",
"taskid": "data:,on_start",
"url": "data:,on_start"
}
fetch_result = {
"orig_url": "data:,on_start",
"content": "on_start",
"headers": {},
"status_code": 200,
"url": "data:,on_start",
"time": 0,
}
self.in_queue.put((task, fetch_result))
time.sleep(1)
self.assertFalse(self.status_queue.empty())
while not self.status_queue.empty():
self.status_queue.get()
self.assertFalse(self.newtask_queue.empty())
def test_40_index_page(self):
task = None
while not self.newtask_queue.empty():
task = self.newtask_queue.get()
self.assertIsNotNone(task)
fetch_result = {
"orig_url": task['url'],
"content":
"<html><body><a href='http://binux.me'>binux</a></body></html>",
"headers": {},
"status_code": 200,
"url": task['url'],
"time": 0,
}
self.in_queue.put((task, fetch_result))
time.sleep(1)
self.assertFalse(self.status_queue.empty())
self.assertFalse(self.newtask_queue.empty())
task = self.newtask_queue.get()
self.assertEqual(task['url'], 'http://binux.me/')
class TestProcessor(unittest.TestCase):
projectdb_path = './test/data/project.db'
@classmethod
def setUpClass(self):
shutil.rmtree('./test/data/', ignore_errors=True)
os.makedirs('./test/data/')
def get_projectdb():
return projectdb.ProjectDB(self.projectdb_path)
self.projectdb = get_projectdb()
self.in_queue = Queue(10)
self.status_queue = Queue(10)
self.newtask_queue = Queue(10)
self.result_queue = Queue(10)
def run_processor():
self.processor = Processor(get_projectdb(), self.in_queue,
self.status_queue, self.newtask_queue, self.result_queue)
self.processor.CHECK_PROJECTS_INTERVAL = 0.1
self.processor.run()
self.process = run_in_thread(run_processor)
time.sleep(1)
@classmethod
def tearDownClass(self):
if self.process.is_alive():
self.processor.quit()
self.process.join(2)
assert not self.process.is_alive()
shutil.rmtree('./test/data/', ignore_errors=True)
def test_10_update_project(self):
self.assertEqual(len(self.processor.projects), 0)
self.projectdb.insert('test_project', {
'name': 'test_project',
'group': 'group',
'status': 'TODO',
'script': open('libs/sample_handler.py', 'r').read(),
'comments': 'test project',
'rate': 1.0,
'burst': 10,
})
task = {
"process": {
"callback": "on_start"
},
"project": "not_exists",
"taskid": "data:,on_start",
"url": "data:,on_start"
}
self.in_queue.put((task, {}))
time.sleep(1)
self.assertTrue(self.status_queue.empty())
self.assertEqual(len(self.processor.projects), 1)
def test_30_new_task(self):
self.assertTrue(self.status_queue.empty())
self.assertTrue(self.newtask_queue.empty())
task = {
"process": {
"callback": "on_start"
},
"project": "test_project",
"taskid": "data:,on_start",
"url": "data:,on_start"
}
fetch_result = {
"orig_url": "data:,on_start",
"content": "on_start",
"headers": {},
"status_code": 200,
"url": "data:,on_start",
"time": 0,
}
self.in_queue.put((task, fetch_result))
time.sleep(1)
self.assertFalse(self.status_queue.empty())
while not self.status_queue.empty():
self.status_queue.get()
self.assertFalse(self.newtask_queue.empty())
def test_40_index_page(self):
task = None
while not self.newtask_queue.empty():
task = self.newtask_queue.get()
self.assertIsNotNone(task)
fetch_result = {
"orig_url": task['url'],
"content": "<html><body><a href='http://binux.me'>binux</a></body></html>",
"headers": {},
"status_code": 200,
"url": task['url'],
"time": 0,
}
self.in_queue.put((task, fetch_result))
time.sleep(1)
self.assertFalse(self.status_queue.empty())
self.assertFalse(self.newtask_queue.empty())
task = self.newtask_queue.get()
self.assertEqual(task['url'], 'http://binux.me/')
