with redis.acquire_lock(task_id):
status = redis.hget('task:'+task_id, 'status')
if status == 'queued' or status == 'allocating' or status == 'allocated':
task.service_queue(redis, task_id, redis.hget('task:'+task_id, 'service'))
task.set_status(redis, 'task:'+task_id, 'queued')
else:
task.work_queue(redis, task_id, service)
# check integrity of tasks
if redis.hget(task_id, 'priority') is None:
redis.hset(task_id, 'priority', 0)
if redis.hget(task_id, 'queued_time') is None:
redis.hset(task_id, 'queued_time', time.time())
# Desallocate all resources that are not anymore associated to a running task
resources = services[service].list_resources()
for resource in resources:
keyr = 'resource:%s:%s' % (service, resource)
running_tasks = redis.hgetall(keyr)
for g, task_id in six.iteritems(running_tasks):
with redis.acquire_lock(task_id):
status = redis.hget('task:'+task_id, 'status')
if not(status == 'running' or status == 'terminating'):
redis.hdel(keyr, g)
# TODO: start multiple workers here?
worker = Worker(redis, services,
cfg.getint('default', 'refresh_counter'),
cfg.getint('default', 'quarantine_time'))
worker.run()
if redis.hget('task:' + task_id, 'queued_time') is None:
redis.hset('task:' + task_id, 'queued_time', time.time())
if services[service].valid:
# Desallocate all resources that are not anymore associated to a running task
resources = services[service].list_resources()
# TODO:
# if multiple workers are for same service with different configurations
# or storage definition change - restart all workers
redis.delete('admin:resources:' + service)
for resource in resources:
redis.lpush('admin:resources:' + service, resource)
keygr = 'gpu_resource:%s:%s' % (service, resource)
running_tasks = redis.hgetall(keygr)
for g, task_id in six.iteritems(running_tasks):
with redis.acquire_lock(task_id):
status = redis.hget('task:' + task_id, 'status')
if not (status == 'running' or status == 'terminating'):
redis.hdel(keygr, g)
keycr = 'cpu_resource:%s:%s' % (service, resource)
running_tasks = redis.hgetall(keycr)
for c, task_id in six.iteritems(running_tasks):
with redis.acquire_lock(task_id):
status = redis.hget('task:' + task_id, 'status')
if not (status == 'running' or status == 'terminating'):
redis.hdel(keycr, c)
# define ttl policy for a task