# remove queued tasks on service
for key in redis.keys('queued:%s' % service):
redis.delete(key)
# On startup, add all active tasks in the work queue or service queue
for task_id in task.list_active(redis, service):
with redis.acquire_lock(task_id):
status = redis.hget('task:'+task_id, 'status')
if status == 'queued' or status == 'allocating' or status == 'allocated':
task.service_queue(redis, task_id, redis.hget('task:'+task_id, 'service'))
task.set_status(redis, 'task:'+task_id, 'queued')
else:
task.work_queue(redis, task_id, service)
# check integrity of tasks
if redis.hget(task_id, 'priority') is None:
redis.hset(task_id, 'priority', 0)
if redis.hget(task_id, 'queued_time') is None:
redis.hset(task_id, 'queued_time', time.time())
# Desallocate all resources that are not anymore associated to a running task
resources = services[service].list_resources()
for resource in resources:
keyr = 'resource:%s:%s' % (service, resource)
running_tasks = redis.hgetall(keyr)
for g, task_id in six.iteritems(running_tasks):
with redis.acquire_lock(task_id):
status = redis.hget('task:'+task_id, 'status')
if not(status == 'running' or status == 'terminating'):
redis.hdel(keyr, g)
break
except ConnectionError as e:
retry += 1
logger.warn("cannot connect to redis DB - retrying (%d)" % retry)
time.sleep(1)
assert retry < 10, "Cannot connect to redis DB - aborting"
redis.set('admin:storages', json.dumps(base_config['storages']))
pid = os.getpid()
logger.info('Running worker for %s - PID = %d' % (service, pid))
keyw = 'admin:worker:%s:%d' % (service, pid)
redis.hset(keyw, "launch_time", time.time())
redis.hset(keyw, "beat_time", time.time())
redis.expire(keyw, 600)
keys = 'admin:service:%s' % service
redis.hset(keys, "current_configuration", current_configuration)
redis.hset(keys, "configurations", json.dumps(configurations))
redis.hset(keys, "def", pickle.dumps(services[service]))
# remove reserved state from resources
for key in redis.keys('reserved:%s:*' % service):
redis.delete(key)
# remove queued tasks on service
for key in redis.keys('queued:%s' % service):
redis.delete(key)
assert os.path.isdir(
taskfile_dir), "taskfile_dir (%s) must be a directory" % taskfile_dir
retry = 0
while retry < 10:
try:
current_default_config = redis_db.exists("default") \
and redis_db.hget("default", "configuration")
break
except (ConnectionError, AssertionError) as e:
retry += 1
time.sleep(1)
assert retry < 10, "Cannot connect to redis DB - aborting"
if current_default_config != default_config:
redis_db.hset("default", "configuration", default_config)
redis_db.hset("default", "timestamp", time.time())
def append_version(v):
global VERSION
VERSION += ":" + v
def get_version():
return VERSION
from app import routes
for filename in os.listdir("configurations"):
if filename.startswith(service + "_") and filename.endswith(".json"):
file_path = os.path.join("configurations", filename)
with open(file_path) as f:
configurations[filename[len(service) + 1:-5]] = (os.path.getmtime(file_path),
f.read())
config_service_md5[md5file(file_path)] = filename[len(service) + 1:-5]
current_configuration_md5 = md5file(args.config)
if current_configuration_md5 in config_service_md5:
current_configuration = config_service_md5[current_configuration_md5]
pid = os.getpid()
logger.info('Running worker for %s - PID = %d' % (service, pid))
instance_id = 'admin:worker:%s:%d' % (service, pid)
redis.hset(instance_id, "launch_time", time.time())
redis.hset(instance_id, "beat_time", time.time())
redis.expire(instance_id, 600)
keys = 'admin:service:%s' % service
redis.hset(keys, "current_configuration", current_configuration)
redis.hset(keys, "configurations", json.dumps(configurations))
redis2.hset(keys, "def", pickle.dumps(services[service]))
def graceful_exit(signum, frame):
logger.info('received interrupt - stopping')
redis.delete(instance_id)
sys.exit(0)