def cleanup_ceph_nodes(osp_cred, pattern=None, timeout=300):
user = os.getlogin()
name = pattern if pattern else "-{user}-".format(user=user)
driver = get_openstack_driver(osp_cred)
timeout = datetime.timedelta(seconds=timeout)
with parallel() as p:
for node in driver.list_nodes():
if name in node.name:
for ip in node.public_ips:
log.info("removing ip %s from node %s", ip, node.name)
driver.ex_detach_floating_ip_from_node(node, ip)
starttime = datetime.datetime.now()
log.info(
"Destroying node {node_name} with {timeout} timeout".format(
node_name=node.name, timeout=timeout
)
)
while True:
try:
p.spawn(node.destroy)
break
except AttributeError:
if datetime.datetime.now() - starttime > timeout:
raise RuntimeError(
"Failed to destroy node {node_name} with {timeout} timeout:\n{stack_trace}".format(
node_name=node.name,
timeout=timeout,
stack_trace=traceback.format_exc(),
)
)
else:
sleep(1)
sleep(5)
with parallel() as p:
for fips in driver.ex_list_floating_ips():
if fips.node_id is None:
log.info("Releasing ip %s", fips.ip_address)
driver.ex_delete_floating_ip(fips)
with parallel() as p:
errors = {}
for volume in driver.list_volumes():
if volume.name is None:
log.info("Volume has no name, skipping")
elif name in volume.name:
log.info("Removing volume %s", volume.name)
sleep(10)
try:
volume.destroy()
except BaseHTTPError as e:
log.error(e, exc_info=True)
errors.update({volume.name: e.message})
if errors:
for vol, err in errors.items():
log.error("Error destroying {vol}: {err}".format(vol=vol, err=err))
raise RuntimeError(
"Encountered errors during volume deletion. Volume names and messages have been logged."
)
def setup_cdn_repos(ceph_nodes, build=None):
repos_13x = [
"rhel-7-server-rhceph-1.3-mon-rpms",
"rhel-7-server-rhceph-1.3-osd-rpms",
"rhel-7-server-rhceph-1.3-calamari-rpms",
"rhel-7-server-rhceph-1.3-installer-rpms",
"rhel-7-server-rhceph-1.3-tools-rpms",
]
repos_20 = [
"rhel-7-server-rhceph-2-mon-rpms",
"rhel-7-server-rhceph-2-osd-rpms",
"rhel-7-server-rhceph-2-tools-rpms",
"rhel-7-server-rhscon-2-agent-rpms",
"rhel-7-server-rhscon-2-installer-rpms",
"rhel-7-server-rhscon-2-main-rpms",
]
repos_30 = [
"rhel-7-server-rhceph-3-mon-rpms",
"rhel-7-server-rhceph-3-osd-rpms",
"rhel-7-server-rhceph-3-tools-rpms",
"rhel-7-server-extras-rpms",
]
repos = None
if build.startswith("1"):
repos = repos_13x
elif build.startswith("2"):
repos = repos_20
elif build.startswith("3"):
repos = repos_30
with parallel() as p:
for node in ceph_nodes:
p.spawn(set_cdn_repo, node, repos)
def create_ebs_volumes(
worker_pattern,
size=100,
region_name=None,
):
"""
Create volumes on workers
Args:
worker_pattern (string): Worker name pattern e.g.:
cluster-55jx2-worker*
size (int): Size in GB (default: 100)
region_name (str): Region name (default: config.ENV_DATA['region'])
"""
aws = AWS(region_name)
region_name = region_name or config.ENV_DATA['region']
worker_instances = aws.get_instances_by_name_pattern(worker_pattern)
with parallel() as p:
for worker in worker_instances:
log.info(
f"Creating and attaching {size} GB volume to {worker['name']}")
p.spawn(
aws.create_volume_and_attach,
availability_zone=worker['avz'],
instance_id=worker['id'],
name=f"{worker['name']}_extra_volume",
size=size,
)
def create_ebs_volumes(self, worker_pattern, size=100):
"""
Add new ebs volumes to the workers
Args:
worker_pattern (str): Worker name pattern e.g.:
cluster-55jx2-worker*
size (int): Size in GB (default: 100)
"""
worker_instances = self.aws.get_instances_by_name_pattern(
worker_pattern
)
with parallel() as p:
for worker in worker_instances:
logger.info(
f"Creating and attaching {size} GB "
f"volume to {worker['name']}"
)
p.spawn(
self.aws.create_volume_and_attach,
availability_zone=worker['avz'],
instance_id=worker['id'],
name=f"{worker['name']}_extra_volume",
size=size,
)
def create_ceph_nodes(cluster_conf, inventory, osp_cred, run_id, instances_name=None):
osp_glbs = osp_cred.get('globals')
os_cred = osp_glbs.get('openstack-credentials')
params = dict()
ceph_cluster = cluster_conf.get('ceph-cluster')
if ceph_cluster.get('inventory'):
inventory_path = os.path.abspath(ceph_cluster.get('inventory'))
with open(inventory_path, 'r') as inventory_stream:
inventory = yaml.safe_load(inventory_stream)
params['cloud-data'] = inventory.get('instance').get('setup')
params['username'] = os_cred['username']
params['password'] = os_cred['password']
params['auth-url'] = os_cred['auth-url']
params['auth-version'] = os_cred['auth-version']
params['tenant-name'] = os_cred['tenant-name']
params['service-region'] = os_cred['service-region']
params['keypair'] = os_cred.get('keypair', None)
ceph_nodes = dict()
if inventory.get('instance').get('create'):
if ceph_cluster.get('image-name'):
params['image-name'] = ceph_cluster.get('image-name')
else:
params['image-name'] = inventory.get('instance').get('create').get('image-name')
params['cluster-name'] = ceph_cluster.get('name')
params['vm-size'] = inventory.get('instance').get('create').get('vm-size')
if params.get('root-login') is False:
params['root-login'] = False
else:
params['root-login'] = True
with parallel() as p:
for node in range(1, 100):
node = "node" + str(node)
if not ceph_cluster.get(node):
break
node_dict = ceph_cluster.get(node)
node_params = params.copy()
node_params['role'] = RolesContainer(node_dict.get('role'))
role = node_params['role']
user = os.getlogin()
if instances_name:
node_params['node-name'] = "{}-{}-{}-{}-{}".format(
node_params.get('cluster-name', 'ceph'), instances_name, run_id, node, '+'.join(role))
else:
node_params['node-name'] = "{}-{}-{}-{}-{}".format(
node_params.get('cluster-name', 'ceph'), user, run_id, node, '+'.join(role))
if node_dict.get('no-of-volumes'):
node_params['no-of-volumes'] = node_dict.get('no-of-volumes')
node_params['size-of-disks'] = node_dict.get('disk-size')
if node_dict.get('image-name'):
node_params['image-name'] = node_dict.get('image-name')
if node_dict.get('cloud-data'):
node_params['cloud-data'] = node_dict.get('cloud-data')
p.spawn(setup_vm_node, node, ceph_nodes, **node_params)
log.info("Done creating nodes")
return ceph_nodes
def create_ceph_nodes(cluster_conf,
inventory,
osp_cred,
run_id,
instances_name=None):
osp_glbs = osp_cred.get("globals")
os_cred = osp_glbs.get("openstack-credentials")
params = dict()
ceph_cluster = cluster_conf.get("ceph-cluster")
if ceph_cluster.get("inventory"):
inventory_path = os.path.abspath(ceph_cluster.get("inventory"))
with open(inventory_path, "r") as inventory_stream:
inventory = yaml.safe_load(inventory_stream)
params["cloud-data"] = inventory.get("instance").get("setup")
params["username"] = os_cred["username"]
params["password"] = os_cred["password"]
params["auth-url"] = os_cred["auth-url"]
params["auth-version"] = os_cred["auth-version"]
params["tenant-name"] = os_cred["tenant-name"]
params["service-region"] = os_cred["service-region"]
params["keypair"] = os_cred.get("keypair", None)
ceph_nodes = dict()
if inventory.get("instance").get("create"):
if ceph_cluster.get("image-name"):
params["image-name"] = ceph_cluster.get("image-name")
else:
params["image-name"] = (
inventory.get("instance").get("create").get("image-name"))
params["cluster-name"] = ceph_cluster.get("name")
params["vm-size"] = inventory.get("instance").get("create").get(
"vm-size")
if params.get("root-login") is False:
params["root-login"] = False
else:
params["root-login"] = True
with parallel() as p:
for node in range(1, 100):
node = "node" + str(node)
if not ceph_cluster.get(node):
break
node_dict = ceph_cluster.get(node)
node_params = params.copy()
node_params["role"] = RolesContainer(node_dict.get("role"))
role = node_params["role"]
user = os.getlogin()
if instances_name:
node_params["node-name"] = "{}-{}-{}-{}-{}".format(
node_params.get("cluster-name", "ceph"),
instances_name,
run_id,
node,
"+".join(role),
)
else:
node_params["node-name"] = "{}-{}-{}-{}-{}".format(
node_params.get("cluster-name", "ceph"),
user,
run_id,
node,
"+".join(role),
)
if node_dict.get("no-of-volumes"):
node_params["no-of-volumes"] = node_dict.get(
"no-of-volumes")
node_params["size-of-disks"] = node_dict.get("disk-size")
if node_dict.get("image-name"):
node_params["image-name"] = node_dict.get("image-name")
if node_dict.get("cloud-data"):
node_params["cloud-data"] = node_dict.get("cloud-data")
p.spawn(setup_vm_node, node, ceph_nodes, **node_params)
log.info("Done creating nodes")
return ceph_nodes
