def start_worker(channel='tasks', max=None, resources=[]): """ channel: Queue from which tasks needs to be executed max: Maximum number of tasks to be exucuted, useful when worker is started from another process. None (default) means all and keep listening for more resources: List of available CPU and memory for the worker """ # initialize worker environment clear_logs() count = 0 q = TaskQueue(config_file='config.json') # start continuously checking queue for tasks while True: # check if max tasks are executed if max is not None: if count < max: count += 1 else: break task = q.pop() if task is not None: task = json.loads(task.decode('utf-8')) if len(resources) == 2 and \ resources[0] is not None and resources[1] is not None: req_cpu = int(task['args']['resources']['cpu']) req_mem = int(task['args']['resources']['mem']) # wait if enough resources are not available while resources[0] < req_cpu or resources[1] < req_mem: print('Waiting for resources ...') time.sleep(1) # grab them as soon as they are available resources[0] -= req_cpu resources[1] -= req_mem # execute the task if resources are available p = threading.Thread(target=execute_task, args=(task, resources)).start() else: # if task queue is empty, wait 1 second before checking again count -= 1 time.sleep(1) return 0
# test if task executes with exit status 0 print('Testing docker_task explicitly...') assert docker_task(args) == 0 print('Done') # test if task gets enqueued to the queue assert q.push('test_task', args) == 0 assert q.push('test_task', args) == 0 assert q.push('test_task', args) == 0 assert q.push('test_task', args) == 0 # test if task has enqueued assert q.q_size() == n + 4 # test if worker executes 3 tasks with given resources and exit with status 0 # since the required cpu of the tasks is 20% and available is 50% only 2 tasks # should be executed parallely and 3rd will be executed only when one of # the earlier two is completed. This can be verified through log timestamp. assert start_worker(max=3, resources=[50, 18000000]) == 0 # test if task has removed from queue assert q.q_size() == n + 1 # test if pop and front works assert not q.front() is None assert not q.pop() is None # test if pop reduced the size of queue assert q.q_size() == n