def get_pools():
"""Get all pools."""
try:
pools = pool_api.get_pools()
except AirflowException as e:
_log.error(e)
response = jsonify(error="{}".format(e))
response.status_code = getattr(e, 'status', 500)
return response
else:
return jsonify([p.to_json() for p in pools])
def get_pools():
"""Get all pools."""
try:
pools = pool_api.get_pools()
except AirflowException as err:
log.error(err)
response = jsonify(error="{}".format(err))
response.status_code = err.status_code
return response
else:
return jsonify([p.to_json() for p in pools])
def init_pools():
pools = pool_api.get_pools()
pools = [i.pool for i in pools]
if "MEMORY" not in pools:
print()
print("Get node resources...")
print()
node_memory = get_node_info(query=memory_query)
node_cpu = get_node_info(query=cpu_core_query)
node_gpu_mem = get_node_info(query=gpu_mem_available_query)
node_gpu_count = get_node_info(query=gpu_query)
try:
print("+++++++++++++++++++++++++++++++++++++++++++++ CREATING MEMORY POOL")
pool_api.create_pool(
name="MEMORY",
slots=abs(node_memory - 10000),
description="Memory of the node in MB"
)
# Variable.set("mem_alloc", NodeUtil.mem_alloc)
print("+++++++++++++++++++++++++++++++++++++++++++++ CREATING CPU POOL")
pool_api.create_pool(
name="CPU",
slots=node_cpu,
description="Count of CPU-cores of the node"
)
# Variable.set("cpu_alloc", NodeUtil.cpu_alloc)
print("+++++++++++++++++++++++++++++++++++++++++++++ CREATING GPU POOL")
pool_api.create_pool(
name="GPU_MEM",
slots=node_gpu_mem,
description="Memory of all GPUs of the node in MB"
)
pool_api.create_pool(
name="GPU_COUNT",
slots=node_gpu_count,
description="Count of GPUs of the node"
)
# Variable.set("gpu_alloc", NodeUtil.gpu_alloc)
# Variable.set("gpu_count", NodeUtil.gpu_count)
except Exception as e:
print("++++++++++++++++++++++++++++++++++++++ Error @ creating pools!")
print(e)
exit(1)
def create_spring_boot_jar_pools_for_dag(pool_configs, logger):
"""
iterates over pools configurations and initiates pools for them
in the future we might want to add the pools per dag
"""
logger.debug("creating pools")
for pool_config in pool_configs:
pool_name = pool_config.get("name")
if not any(p.pool == pool_name for p in pool.get_pools()):
pool.create_pool(name=pool_name,
slots=pool_config.get("slots"),
description=pool_config.get("description"))
logger.debug("pool_name=%s successfully initiated", pool_name)
else:
logger.debug("pool_name=%s already exist", pool_name)
def main():
print("Pools ********************")
created_pools = set()
for pool in pools:
name = pool["name"]
print(f"Create: {name}")
create_pool(**pool)
created_pools.add(name)
for pool in get_pools():
name = pool.pool
if name == default_pool_name or name in created_pools:
continue
if name not in created_pools:
print(f"Delete: {name}")
delete_pool(name)
return
def _get_pool_name_if_exist(self):
pool_name = self._get_conf(POOL_NAME_CONF_KEY)
if not any(p.pool == pool_name for p in pool.get_pools()):
pool_name = None
return pool_name
def test_get_pools(self):
pools = sorted(pool_api.get_pools(), key=lambda p: p.pool)
assert pools[0].pool == self.pools[0].pool
assert pools[1].pool == self.pools[1].pool
def get_pools(self):
return [(p.pool, p.slots, p.description) for p in pool.get_pools()]
def test_get_pools(self):
pools = sorted(pool_api.get_pools(), key=lambda p: p.pool)
self.assertEqual(pools[0].pool, self.pools[0].pool)
self.assertEqual(pools[1].pool, self.pools[1].pool)
def test_get_pools(self):
pools = sorted(pool_api.get_pools(session=self.session),
key=lambda p: p.pool)
self.assertEqual(pools[0].pool, self.pools[0].pool)
self.assertEqual(pools[1].pool, self.pools[1].pool)
def get_pools(self):
return [(p.pool, p.slots, p.description) for p in pool.get_pools()]
def resolve_pool(self, context, **kwargs):
print(get_pools())
return get_pools()
