def download_async(uri, local, clear_local=False): subfolder = os.path.dirname(local) if subfolder and not os.path.exists(subfolder): os.makedirs(subfolder) command = {'uri': uri, 'local': local, 'clear_local': False} task = boss.Task(task_id=uri, command=command) boss.dedicate(task)
def create_download_task(uri, output_dir, clear_local=False): ''' @param uri Absolute URI ''' if not is_full_uri(uri): raise StreamBotError('{uri} is not full URI'.format(uri=uri)) local = _get_local(uri, output_dir) cmd = {'uri': uri, 'local': local, 'clear_local': clear_local} return boss.Task(uri, cmd, 'START')
def test_failed_tasks_saved(self): boss.start(action=failed_action) for i in range(10): task = boss.Task(i, {}) boss.assign_task(task) total_check = 0 while not boss.have_all_tasks_done(): time.sleep(1) total_check += 1 if total_check > 10: self.fail('workers have not stpped in time') boss.stop() for k, v in boss.tasks().items(): self.assertTrue(v.is_failed())
def test_boss_can_shut_down(self): boss.start(action=half_done_half_failed_action) for i in range(10): task = boss.Task(i, {}) boss.assign_task(task) total_check = 0 while not boss.have_all_tasks_done(): time.sleep(1) total_check += 1 if total_check > 10: self.fail('workers have not stpped in time') boss.stop() for t in boss._WORKING_THREADS: self.assertFalse(t.isAlive())
def test_duplicated_tasks_run_only_once(self): boss.start(action=half_done_half_failed_action) num_unique_tasks = 10 for i in range(num_unique_tasks): task = boss.Task(i, {}) # assign each task twice boss.assign_task(task) boss.assign_task(task) total_check = 0 while not boss.have_all_tasks_done(): time.sleep(1) total_check += 1 if total_check > 10: self.fail('workers have not stpped in time') boss.stop() self.assertEqual(len(boss.tasks()), num_unique_tasks)
def test_tasks_returns_all_assigned_tasks(self): boss.start(action=done_action) num_tasks = 10 for i in range(num_tasks): task = boss.Task(i, {}) boss.assign_task(task) total_check = 0 while not boss.have_all_tasks_done(): time.sleep(1) total_check += 1 if total_check > 10: self.fail('workers have not stpped in time') boss.stop() assigned_tasks = boss.tasks() self.assertTrue(len(assigned_tasks), num_tasks) for k, v in boss.tasks().items(): self.assertTrue(v.id in range(num_tasks))
def test_constructor_id(self): task_id = 1 task = boss.Task(task_id, {}) self.assertEqual(task.id, task_id)
def test_constructor_default_status(self): task = boss.Task(1, {}) self.assertEqual(task.status, 'START')
def teste_is_failed(self): task = boss.Task(1, {}, 'FAILED') self.assertTrue(task.is_failed())
def test_set_failed(self): task = boss.Task(1, {}) task.set_failed() self.assertEqual(task.status, 'FAILED')
def test_is_done(self): task = boss.Task(1, {}, 'DONE') self.assertTrue(task.is_done())
def test_set_done(self): task = boss.Task(1, {}) task.set_done() self.assertEqual(task.status, 'DONE')
def test_constructor_command(self): command = {'cmd', 'cmd'} task = boss.Task(1, command) self.assertIn('cmd', task.command)