# remove queued tasks on service for key in redis.keys('queued:%s' % service): redis.delete(key) # On startup, add all active tasks in the work queue or service queue for task_id in task.list_active(redis, service): with redis.acquire_lock(task_id): status = redis.hget('task:'+task_id, 'status') if status == 'queued' or status == 'allocating' or status == 'allocated': task.service_queue(redis, task_id, redis.hget('task:'+task_id, 'service')) task.set_status(redis, 'task:'+task_id, 'queued') else: task.work_queue(redis, task_id, service) # check integrity of tasks if redis.hget(task_id, 'priority') is None: redis.hset(task_id, 'priority', 0) if redis.hget(task_id, 'queued_time') is None: redis.hset(task_id, 'queued_time', time.time()) # Desallocate all resources that are not anymore associated to a running task resources = services[service].list_resources() for resource in resources: keyr = 'resource:%s:%s' % (service, resource) running_tasks = redis.hgetall(keyr) for g, task_id in six.iteritems(running_tasks): with redis.acquire_lock(task_id): status = redis.hget('task:'+task_id, 'status') if not(status == 'running' or status == 'terminating'): redis.hdel(keyr, g)
break except ConnectionError as e: retry += 1 logger.warn("cannot connect to redis DB - retrying (%d)" % retry) time.sleep(1) assert retry < 10, "Cannot connect to redis DB - aborting" redis.set('admin:storages', json.dumps(base_config['storages'])) pid = os.getpid() logger.info('Running worker for %s - PID = %d' % (service, pid)) keyw = 'admin:worker:%s:%d' % (service, pid) redis.hset(keyw, "launch_time", time.time()) redis.hset(keyw, "beat_time", time.time()) redis.expire(keyw, 600) keys = 'admin:service:%s' % service redis.hset(keys, "current_configuration", current_configuration) redis.hset(keys, "configurations", json.dumps(configurations)) redis.hset(keys, "def", pickle.dumps(services[service])) # remove reserved state from resources for key in redis.keys('reserved:%s:*' % service): redis.delete(key) # remove queued tasks on service for key in redis.keys('queued:%s' % service): redis.delete(key)
assert os.path.isdir( taskfile_dir), "taskfile_dir (%s) must be a directory" % taskfile_dir retry = 0 while retry < 10: try: current_default_config = redis_db.exists("default") \ and redis_db.hget("default", "configuration") break except (ConnectionError, AssertionError) as e: retry += 1 time.sleep(1) assert retry < 10, "Cannot connect to redis DB - aborting" if current_default_config != default_config: redis_db.hset("default", "configuration", default_config) redis_db.hset("default", "timestamp", time.time()) def append_version(v): global VERSION VERSION += ":" + v def get_version(): return VERSION from app import routes
for filename in os.listdir("configurations"): if filename.startswith(service + "_") and filename.endswith(".json"): file_path = os.path.join("configurations", filename) with open(file_path) as f: configurations[filename[len(service) + 1:-5]] = (os.path.getmtime(file_path), f.read()) config_service_md5[md5file(file_path)] = filename[len(service) + 1:-5] current_configuration_md5 = md5file(args.config) if current_configuration_md5 in config_service_md5: current_configuration = config_service_md5[current_configuration_md5] pid = os.getpid() logger.info('Running worker for %s - PID = %d' % (service, pid)) instance_id = 'admin:worker:%s:%d' % (service, pid) redis.hset(instance_id, "launch_time", time.time()) redis.hset(instance_id, "beat_time", time.time()) redis.expire(instance_id, 600) keys = 'admin:service:%s' % service redis.hset(keys, "current_configuration", current_configuration) redis.hset(keys, "configurations", json.dumps(configurations)) redis2.hset(keys, "def", pickle.dumps(services[service])) def graceful_exit(signum, frame): logger.info('received interrupt - stopping') redis.delete(instance_id) sys.exit(0)