def download_async(uri, local, clear_local=False):
subfolder = os.path.dirname(local)
if subfolder and not os.path.exists(subfolder):
os.makedirs(subfolder)
command = {'uri': uri, 'local': local, 'clear_local': False}
task = boss.Task(task_id=uri, command=command)
boss.dedicate(task)
def create_download_task(uri, output_dir, clear_local=False):
'''
@param uri Absolute URI
'''
if not is_full_uri(uri):
raise StreamBotError('{uri} is not full URI'.format(uri=uri))
local = _get_local(uri, output_dir)
cmd = {'uri': uri, 'local': local, 'clear_local': clear_local}
return boss.Task(uri, cmd, 'START')
def test_failed_tasks_saved(self):
boss.start(action=failed_action)
for i in range(10):
task = boss.Task(i, {})
boss.assign_task(task)
total_check = 0
while not boss.have_all_tasks_done():
time.sleep(1)
total_check += 1
if total_check > 10:
self.fail('workers have not stpped in time')
boss.stop()
for k, v in boss.tasks().items():
self.assertTrue(v.is_failed())
def test_boss_can_shut_down(self):
boss.start(action=half_done_half_failed_action)
for i in range(10):
task = boss.Task(i, {})
boss.assign_task(task)
total_check = 0
while not boss.have_all_tasks_done():
time.sleep(1)
total_check += 1
if total_check > 10:
self.fail('workers have not stpped in time')
boss.stop()
for t in boss._WORKING_THREADS:
self.assertFalse(t.isAlive())
def test_duplicated_tasks_run_only_once(self):
boss.start(action=half_done_half_failed_action)
num_unique_tasks = 10
for i in range(num_unique_tasks):
task = boss.Task(i, {})
# assign each task twice
boss.assign_task(task)
boss.assign_task(task)
total_check = 0
while not boss.have_all_tasks_done():
time.sleep(1)
total_check += 1
if total_check > 10:
self.fail('workers have not stpped in time')
boss.stop()
self.assertEqual(len(boss.tasks()), num_unique_tasks)
def test_tasks_returns_all_assigned_tasks(self):
boss.start(action=done_action)
num_tasks = 10
for i in range(num_tasks):
task = boss.Task(i, {})
boss.assign_task(task)
total_check = 0
while not boss.have_all_tasks_done():
time.sleep(1)
total_check += 1
if total_check > 10:
self.fail('workers have not stpped in time')
boss.stop()
assigned_tasks = boss.tasks()
self.assertTrue(len(assigned_tasks), num_tasks)
for k, v in boss.tasks().items():
self.assertTrue(v.id in range(num_tasks))
def test_constructor_id(self):
task_id = 1
task = boss.Task(task_id, {})
self.assertEqual(task.id, task_id)
def test_constructor_default_status(self):
task = boss.Task(1, {})
self.assertEqual(task.status, 'START')
def teste_is_failed(self):
task = boss.Task(1, {}, 'FAILED')
self.assertTrue(task.is_failed())
def test_set_failed(self):
task = boss.Task(1, {})
task.set_failed()
self.assertEqual(task.status, 'FAILED')
def test_is_done(self):
task = boss.Task(1, {}, 'DONE')
self.assertTrue(task.is_done())
def test_set_done(self):
task = boss.Task(1, {})
task.set_done()
self.assertEqual(task.status, 'DONE')
def test_constructor_command(self):
command = {'cmd', 'cmd'}
task = boss.Task(1, command)
self.assertIn('cmd', task.command)
