def main(profile, dry_run, skip_version, older_then, skip_zone, only_zone, restart_agent):
"""Destroy SVM per zone and waits for a new one"""
click_log.basic_config()
if dry_run:
logging.info('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run)
if older_then:
older_then = datetime.strptime(f"{older_then}T00:00:00+0200", '%Y%m%dT%H:%M:%S%z')
svms = co.get_all_systemvms()
zones = defaultdict(list)
for svm in svms:
if only_zone and co.get_host(name=svm['name']).get('zonename') != only_zone:
continue
if skip_zone and co.get_host(name=svm['name']).get('zonename') == skip_zone:
continue
if skip_version and co.get_host(name=svm['name']).get('version') == skip_version:
continue
if older_then and datetime.strptime(svm['created'], '%Y-%m-%dT%H:%M:%S%z') > older_then:
continue
zones[svm['zonename']].append(svm)
for zone in zones:
logging.info(f"Processing zone: {zone}")
for vm in zones[zone]:
if not vm.destroy():
sys.exit(1)
up = list()
down = list(zones[zone])
retries = 60
zone_id = vm['zoneid']
while len(up) < len(zones[zone]) or len(down) > 0:
if not dry_run:
time.sleep(5)
try:
systemvms = {x['name']: x for x in co.get_all_systemvms(zoneid=zone_id)}
host_status = {k: co.get_host(name=k) for k in systemvms}
up = list(filter(lambda x: x and x['state'] == 'Up' and x['resourcestate'] == 'Enabled', host_status.values()))
down = list(filter(lambda x: x and x['state'] != 'Up' and x['resourcestate'] == 'Enabled', host_status.values()))
retries -= 1
if retries == 0:
break
if down and restart_agent:
for d in down:
svm_object = list(filter(lambda x: x and x['name'].lower() == d['name'], systemvms.values()))[0]
svm_object.restart_agent()
except KeyError:
# Ignore keyerror, systemvm is still not available as host
pass
if retries == 0:
logging.error("Exceeded retry count waiting for new systemvm")
sys.exit(1)
def main(profile, destination_dc, dry_run, host, cluster):
"""Migrate all VMs on HOST to CLUSTER"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Live Migrate HV to new POD'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
cs = CosmicSQL(server=profile, dry_run=dry_run)
host = co.get_host(name=host)
if not host:
sys.exit(1)
for vm in host.get_all_vms() + host.get_all_project_vms():
live_migrate(co=co,
cs=cs,
cluster=cluster,
vm_name=vm['name'],
destination_dc=destination_dc,
add_affinity_group=None,
is_project_vm=None,
zwps_to_cwps=None,
log_to_slack=log_to_slack,
dry_run=dry_run)
def main(profile, dry_run, router):
"""Live migrate ROUTER"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Live Migrate HV to new POD'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
router = co.get_system_vm(name=router)
if not router:
sys.exit(1)
source_host = co.get_host(id=router['hostid'])
if not source_host:
sys.exit(1)
cluster = co.get_cluster(id=source_host['clusterid'])
if not cluster:
sys.exit(1)
destination_host = cluster.find_migration_host(router)
if not destination_host:
sys.exit(1)
if not router.migrate(destination_host):
sys.exit(1)
def main(profile, zwps_to_cwps, add_affinity_group, destination_dc, is_project_vm,
skip_within_cluster, dry_run, vm, cluster):
"""Live migrate VM to CLUSTER"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Live Migrate VM'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
cs = CosmicSQL(server=profile, dry_run=dry_run)
# Work around migration issue: first in the same pod to limit possible hiccup
vm_instance = co.get_vm(name=vm, is_project_vm=is_project_vm)
if not vm_instance:
logging.error(f"Cannot migrate, VM '{vm}' not found!")
sys.exit(1)
if not vm_instance['state'] == 'Running':
logging.error(f"Cannot migrate, VM has has state: '{vm_instance['state']}'")
sys.exit(1)
source_host = co.get_host(id=vm_instance['hostid'])
source_cluster = co.get_cluster(id=source_host['clusterid'])
if not skip_within_cluster:
if not vm_instance.migrate_within_cluster(vm=vm_instance, source_cluster=source_cluster,
source_host=source_host, instancename=vm_instance):
logging.info(f"VM Migration failed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n")
sys.exit(1)
if not live_migrate(co, cs, cluster, vm, destination_dc, add_affinity_group, is_project_vm, zwps_to_cwps,
log_to_slack, dry_run):
logging.info(f"VM Migration failed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n")
sys.exit(1)
logging.info(f"VM Migration completed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n")
def empty_host(profile, shutdown, skip_disable, dry_run, host):
co = CosmicOps(profile=profile, dry_run=dry_run)
host = co.get_host(name=host)
if not host:
raise RuntimeError(f"Host '{host['name']}' not found")
if not skip_disable and host['resourcestate'] != 'Disabled':
if not host.disable():
raise RuntimeError(f"Failed to disable host '{host['name']}'")
(total, success, failed) = host.empty()
result_message = f"Result: {success} successful, {failed} failed out of {total} total VMs"
if not failed and shutdown:
host.set_uid_led(True)
if not host.reboot(RebootAction.HALT):
raise RuntimeError(f"Failed to shutdown host '{host['name']}'")
host.wait_until_offline()
result_message = f"{result_message}\nHost '{host['name']}' has shutdown, UID led is turned on"
elif failed and shutdown:
result_message = f"{result_message}\nNot shutting down host '{host['name']}' because migration completed with failed VMs"
return result_message
def main(profile, is_project_router, only_when_required, cleanup, dry_run,
router):
"""Router restart and upgrade script"""
click_log.basic_config()
log_to_slack = True
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
router = co.get_router(name=router, is_project_router=is_project_router)
if not router:
sys.exit(1)
logging.instance_name = router['name']
logging.slack_title = 'Domain'
logging.slack_value = router['domain']
host = co.get_host(id=router['hostid'])
if not host:
sys.exit(1)
cluster = co.get_cluster(id=host['clusterid'])
if not cluster:
sys.exit(1)
logging.cluster = cluster['name']
if only_when_required and not router['requiresupgrade']:
logging.info(
f"Router '{router['name']}' does not need to be upgraded. Will not reboot because --only-when-required was specified."
)
sys.exit(0)
if cleanup:
if not router['vpcid']:
logging.error(
f"Cleanup specified but no VPC ID found for router '{router['name']}'"
)
sys.exit(1)
logging.task = 'Restart VPC with clean up'
vpc = co.get_vpc(id=router['vpcid'])
if not vpc:
sys.exit(1)
if not vpc.restart():
sys.exit(1)
logging.info(
f"Successfully restarted VPC '{vpc['name']}' with cleanup for router '{router['name']}'"
)
else:
logging.task = 'Reboot virtual router'
if not router.reboot():
sys.exit(1)
logging.info(f"Successfully rebooted router '{router['name']}'",
log_to_slack)
def main(dry_run, zwps_cluster, destination_cluster, virtual_machines,
force_end_hour):
"""Empty ZWPS by migrating VMs and/or it's volumes to the destination cluster."""
click_log.basic_config()
if force_end_hour:
try:
force_end_hour = int(force_end_hour)
except ValueError as e:
logging.error(
f"Specified time:'{force_end_hour}' is not a valid integer due to: '{e}'"
)
sys.exit(1)
if force_end_hour >= 24:
logging.error(f"Specified time:'{force_end_hour}' should be < 24")
sys.exit(1)
profile = 'nl2'
log_to_slack = True
logging.task = 'Live Migrate VM Volumes'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
if not dry_run:
cs = CosmicSQL(server=profile, dry_run=dry_run)
else:
cs = None
zwps_storage_pools = []
for storage_pool in co.get_all_storage_pools():
if zwps_cluster.upper() in storage_pool['name']:
zwps_storage_pools.append(storage_pool)
logging.info('ZWPS storage pools found:')
for zwps_storage_pool in zwps_storage_pools:
logging.info(f" - '{zwps_storage_pool['name']}'")
target_cluster = co.get_cluster(name=destination_cluster)
if not target_cluster:
logging.error(
f"Destination cluster not found:'{target_cluster['name']}'!")
sys.exit(1)
try:
destination_storage_pools = target_cluster.get_storage_pools(
scope='CLUSTER')
except IndexError:
logging.error(
f"No storage pools found for cluster '{target_cluster['name']}'")
sys.exit(1)
logging.info('Destination storage pools found:')
for target_storage_pool in destination_storage_pools:
logging.info(f" - '{target_storage_pool['name']}'")
target_storage_pool = random.choice(destination_storage_pools)
volumes = []
for zwps_storage_pool in zwps_storage_pools:
vols = co.get_all_volumes(list_all=True,
storageid=zwps_storage_pool['id'])
if vols:
volumes += vols
vm_ids = []
logging.info('Volumes found:')
for volume in volumes:
for virtual_machine in virtual_machines:
if re.search(virtual_machine, volume['vmname'], re.IGNORECASE):
logging.info(
f" - '{volume['name']}' on VM '{volume['vmname']}'")
if volume['virtualmachineid'] not in vm_ids:
vm_ids.append(volume['virtualmachineid'])
vms = []
for vm_id in vm_ids:
vm = co.get_vm(id=vm_id)
if vm['affinitygroup']:
for affinitygroup in vm['affinitygroup']:
if 'DedicatedGrp' in affinitygroup['name']:
logging.warning(
f"Skipping VM '{vm['name']}' because of 'DedicatedGrp' affinity group"
)
continue
vms.append(vm)
logging.info('Virtualmachines found:')
for vm in vms:
logging.info(f" - '{vm['name']}'")
logging.info(
f"Starting live migration of volumes and/or virtualmachines from the ZWPS storage pools to storage pool '{target_cluster['name']}'"
)
for vm in vms:
""" Can we start a new migration? """
if force_end_hour:
now = datetime.datetime.now(pytz.timezone('CET'))
if now.hour >= force_end_hour:
logging.info(
f"Stopping migration batch. We are not starting new migrations after '{force_end_hour}':00",
log_to_slack=log_to_slack)
sys.exit(0)
source_host = co.get_host(id=vm['hostid'])
source_cluster = co.get_cluster(zone='nl2',
id=source_host['clusterid'])
if source_cluster['name'] == target_cluster['name']:
""" VM is already on the destination cluster, so we only need to migrate the volumes to this storage pool """
logging.info(
f"Starting live migration of volumes of VM '{vm['name']}' to storage pool '{target_storage_pool['name']}' ({target_storage_pool['id']})",
log_to_slack=log_to_slack)
live_migrate_volumes(target_storage_pool['name'], co, cs, dry_run,
False, log_to_slack, 0, vm['name'], True)
else:
""" VM needs to be migrated live to the destination cluster, including volumes """
live_migrate(co=co,
cs=cs,
cluster=target_cluster['name'],
vm_name=vm['name'],
destination_dc=None,
add_affinity_group=None,
is_project_vm=None,
zwps_to_cwps=True,
log_to_slack=log_to_slack,
dry_run=dry_run)