def main(profile, zwps_to_cwps, add_affinity_group, destination_dc, is_project_vm, skip_within_cluster, dry_run, vm, cluster): """Live migrate VM to CLUSTER""" click_log.basic_config() log_to_slack = True logging.task = 'Live Migrate VM' logging.slack_title = 'Domain' if dry_run: log_to_slack = False logging.warning('Running in dry-run mode, will only show changes') co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack) cs = CosmicSQL(server=profile, dry_run=dry_run) # Work around migration issue: first in the same pod to limit possible hiccup vm_instance = co.get_vm(name=vm, is_project_vm=is_project_vm) if not vm_instance: logging.error(f"Cannot migrate, VM '{vm}' not found!") sys.exit(1) if not vm_instance['state'] == 'Running': logging.error(f"Cannot migrate, VM has has state: '{vm_instance['state']}'") sys.exit(1) source_host = co.get_host(id=vm_instance['hostid']) source_cluster = co.get_cluster(id=source_host['clusterid']) if not skip_within_cluster: if not vm_instance.migrate_within_cluster(vm=vm_instance, source_cluster=source_cluster, source_host=source_host, instancename=vm_instance): logging.info(f"VM Migration failed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n") sys.exit(1) if not live_migrate(co, cs, cluster, vm, destination_dc, add_affinity_group, is_project_vm, zwps_to_cwps, log_to_slack, dry_run): logging.info(f"VM Migration failed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n") sys.exit(1) logging.info(f"VM Migration completed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n")
def main(profile, is_project_vm, dry_run, vm, cluster, destination_dc, destination_so): """Offline migrate VM to CLUSTER""" click_log.basic_config() log_to_slack = True logging.task = 'Offline Migrate VM' logging.slack_title = 'Domain' if dry_run: log_to_slack = False logging.warning('Running in dry-run mode, will only show changes') co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack) cs = CosmicSQL(server=profile, dry_run=dry_run) target_cluster = co.get_cluster(name=cluster) if not target_cluster: sys.exit(1) vm = co.get_vm(name=vm, is_project_vm=is_project_vm) if not vm: sys.exit(1) if destination_dc and destination_dc not in DATACENTERS: logging.error(f"Unknown datacenter '{destination_dc}', should be one of {str(DATACENTERS)}") sys.exit(1) logging.instance_name = vm['instancename'] logging.slack_value = vm['domain'] logging.vm_name = vm['name'] logging.zone_name = vm['zonename'] target_storage_pool = None try: # Get CLUSTER scoped volume (no NVMe or ZONE-wide) while target_storage_pool is None or target_storage_pool['scope'] != 'CLUSTER': target_storage_pool = choice(target_cluster.get_storage_pools()) except IndexError: logging.error(f"No storage pools found for cluster '{target_cluster['name']}") sys.exit(1) if vm['state'] == 'Running': need_to_stop = True auto_start_vm = True else: need_to_stop = False auto_start_vm = False if destination_dc: for datacenter in DATACENTERS: if datacenter == destination_dc: continue if datacenter in vm['serviceofferingname']: new_offering = vm['serviceofferingname'].replace(datacenter, destination_dc) logging.info( f"Replacing '{vm['serviceofferingname']}' with '{new_offering}'") cs.update_service_offering_of_vm(vm['instancename'], new_offering) vm = co.get_vm(name=vm['instancename'], is_project_vm=is_project_vm) break if destination_so: logging.info( f"Replacing '{vm['serviceofferingname']}' with '{destination_so}'") cs.update_service_offering_of_vm(vm['instancename'], destination_so) vm = co.get_vm(name=vm['instancename'], is_project_vm=is_project_vm) vm_service_offering = co.get_service_offering(id=vm['serviceofferingid']) if vm_service_offering: storage_tags = vm_service_offering['tags'] if 'tags' in vm_service_offering else '' if not storage_tags: logging.warning('VM service offering has no storage tags') else: if storage_tags not in target_storage_pool['tags']: logging.error( f"Can't migrate '{vm['name']}', storage tags on target cluster ({target_storage_pool['tags']}) to not contain the tags on the VM's service offering ({storage_tags})'") sys.exit(1) if need_to_stop: if not vm.stop(): sys.exit(1) volumes = vm.get_volumes() for volume in volumes: if volume['storage'] == target_storage_pool['name']: logging.warning( f"Volume '{volume['name']}' ({volume['id']}) already on cluster '{target_cluster['name']}', skipping...") continue source_storage_pool = co.get_storage_pool(name=volume['storage']) if not source_storage_pool: sys.exit(1) if source_storage_pool['scope'] == 'ZONE': logging.warning(f"Scope of volume '{volume['name']}' ({volume['id']}) is ZONE, skipping...") continue if not volume.migrate(target_storage_pool): sys.exit(1) with click_spinner.spinner(): while True: volume.refresh() if volume['state'] == 'Ready': break logging.warning( f"Volume '{volume['name']}' ({volume['id']}) is in '{volume['state']}' state instead of 'Ready', sleeping...") time.sleep(60) if auto_start_vm: destination_host = target_cluster.find_migration_host(vm) if not destination_host: sys.exit(1) if not vm.start(destination_host): sys.exit(1) else: logging.info(f"Not starting VM '{vm['name']}' as it was not running", log_to_slack)
def main(dry_run, zwps_cluster, destination_cluster, virtual_machines, force_end_hour): """Empty ZWPS by migrating VMs and/or it's volumes to the destination cluster.""" click_log.basic_config() if force_end_hour: try: force_end_hour = int(force_end_hour) except ValueError as e: logging.error( f"Specified time:'{force_end_hour}' is not a valid integer due to: '{e}'" ) sys.exit(1) if force_end_hour >= 24: logging.error(f"Specified time:'{force_end_hour}' should be < 24") sys.exit(1) profile = 'nl2' log_to_slack = True logging.task = 'Live Migrate VM Volumes' logging.slack_title = 'Domain' if dry_run: log_to_slack = False logging.warning('Running in dry-run mode, will only show changes') co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack) if not dry_run: cs = CosmicSQL(server=profile, dry_run=dry_run) else: cs = None zwps_storage_pools = [] for storage_pool in co.get_all_storage_pools(): if zwps_cluster.upper() in storage_pool['name']: zwps_storage_pools.append(storage_pool) logging.info('ZWPS storage pools found:') for zwps_storage_pool in zwps_storage_pools: logging.info(f" - '{zwps_storage_pool['name']}'") target_cluster = co.get_cluster(name=destination_cluster) if not target_cluster: logging.error( f"Destination cluster not found:'{target_cluster['name']}'!") sys.exit(1) try: destination_storage_pools = target_cluster.get_storage_pools( scope='CLUSTER') except IndexError: logging.error( f"No storage pools found for cluster '{target_cluster['name']}'") sys.exit(1) logging.info('Destination storage pools found:') for target_storage_pool in destination_storage_pools: logging.info(f" - '{target_storage_pool['name']}'") target_storage_pool = random.choice(destination_storage_pools) volumes = [] for zwps_storage_pool in zwps_storage_pools: vols = co.get_all_volumes(list_all=True, storageid=zwps_storage_pool['id']) if vols: volumes += vols vm_ids = [] logging.info('Volumes found:') for volume in volumes: for virtual_machine in virtual_machines: if re.search(virtual_machine, volume['vmname'], re.IGNORECASE): logging.info( f" - '{volume['name']}' on VM '{volume['vmname']}'") if volume['virtualmachineid'] not in vm_ids: vm_ids.append(volume['virtualmachineid']) vms = [] for vm_id in vm_ids: vm = co.get_vm(id=vm_id) if vm['affinitygroup']: for affinitygroup in vm['affinitygroup']: if 'DedicatedGrp' in affinitygroup['name']: logging.warning( f"Skipping VM '{vm['name']}' because of 'DedicatedGrp' affinity group" ) continue vms.append(vm) logging.info('Virtualmachines found:') for vm in vms: logging.info(f" - '{vm['name']}'") logging.info( f"Starting live migration of volumes and/or virtualmachines from the ZWPS storage pools to storage pool '{target_cluster['name']}'" ) for vm in vms: """ Can we start a new migration? """ if force_end_hour: now = datetime.datetime.now(pytz.timezone('CET')) if now.hour >= force_end_hour: logging.info( f"Stopping migration batch. We are not starting new migrations after '{force_end_hour}':00", log_to_slack=log_to_slack) sys.exit(0) source_host = co.get_host(id=vm['hostid']) source_cluster = co.get_cluster(zone='nl2', id=source_host['clusterid']) if source_cluster['name'] == target_cluster['name']: """ VM is already on the destination cluster, so we only need to migrate the volumes to this storage pool """ logging.info( f"Starting live migration of volumes of VM '{vm['name']}' to storage pool '{target_storage_pool['name']}' ({target_storage_pool['id']})", log_to_slack=log_to_slack) live_migrate_volumes(target_storage_pool['name'], co, cs, dry_run, False, log_to_slack, 0, vm['name'], True) else: """ VM needs to be migrated live to the destination cluster, including volumes """ live_migrate(co=co, cs=cs, cluster=target_cluster['name'], vm_name=vm['name'], destination_dc=None, add_affinity_group=None, is_project_vm=None, zwps_to_cwps=True, log_to_slack=log_to_slack, dry_run=dry_run)
class TestCosmicOps(TestCase): def setUp(self): cs_patcher = patch('cosmicops.ops.CloudStack') self.mock_cs = cs_patcher.start() self.addCleanup(cs_patcher.stop) self.cs_instance = self.mock_cs.return_value slack_patcher = patch('cosmicops.log.Slack') self.mock_slack = slack_patcher.start() self.addCleanup(slack_patcher.stop) sleep_patcher = patch('time.sleep', return_value=None) self.mock_sleep = sleep_patcher.start() self.addCleanup(sleep_patcher.stop) self.co = CosmicOps(endpoint='https://localhost', key='key', secret='secret') @patch('cosmicops.ops._load_cloud_monkey_profile') def test_init_with_profile(self, mock_load): mock_load.return_value = ('profile_endpoint', 'profile_key', 'profile_secret') CosmicOps(profile='config') self.mock_cs.assert_called_with('profile_endpoint', 'profile_key', 'profile_secret', 60) @tempdir() def test_load_cloud_monkey_profile(self, tmp): config = (b"[testprofile]\n" b"url = http://localhost:8000/client/api\n" b"apikey = test_api_key\n" b"secretkey = test_secret_key\n") tmp.makedir('.cloudmonkey') tmp.write('.cloudmonkey/config', config) with patch('pathlib.Path.home') as path_home_mock: path_home_mock.return_value = Path(tmp.path) (endpoint, key, secret) = _load_cloud_monkey_profile('testprofile') self.assertEqual('http://localhost:8000/client/api', endpoint) self.assertEqual('test_api_key', key) self.assertEqual('test_secret_key', secret) @tempdir() def test_load_cloud_monkey_profile_with_default(self, tmp): config = (b"[core]\n" b"profile = profile2\n\n" b"[profile1]\n" b"url = http://localhost:8000/client/api/1\n" b"apikey = test_api_key_1\n" b"secretkey = test_secret_key_1\n\n" b"[profile2]\n" b"url = http://localhost:8000/client/api/2\n" b"apikey = test_api_key_2\n" b"secretkey = test_secret_key_2\n") tmp.makedir('.cloudmonkey') tmp.write('.cloudmonkey/config', config) with patch('pathlib.Path.home') as path_home_mock: path_home_mock.return_value = Path(tmp.path) (endpoint, key, secret) = _load_cloud_monkey_profile('config') self.assertEqual('http://localhost:8000/client/api/2', endpoint) self.assertEqual('test_api_key_2', key) self.assertEqual('test_secret_key_2', secret) def test_cs_get_single_result(self): self.cs_instance.listFunction.return_value = [{ 'id': 'id_field', 'name': 'name_field' }] result = self.co._cs_get_single_result('listFunction', {'name': 'name_field'}, CosmicObject, 'type') self.cs_instance.listFunction.assert_called_with(fetch_list=True, name='name_field') self.assertIsInstance(result, CosmicObject) self.assertDictEqual({ 'id': 'id_field', 'name': 'name_field' }, result._data) def test_get_get_single_result_failure(self): self.cs_instance.listFunction.return_value = [] self.assertIsNone( self.co._cs_get_single_result('listFunction', {}, CosmicObject, 'type')) self.cs_instance.listFunction.return_value = [{}, {}] self.assertIsNone( self.co._cs_get_single_result('listFunction', {}, CosmicObject, 'type')) def test_cs_get_all_results(self): self.cs_instance.listFunction.return_value = [{ 'id': 'id1', 'name': 'name1' }, { 'id': 'id2', 'name': 'name2' }] result = self.co._cs_get_all_results('listFunction', {}, CosmicObject, 'type') self.cs_instance.listFunction.assert_called_with(fetch_list=True) for i, item in enumerate(result): self.assertIsInstance(item, CosmicObject) self.assertDictEqual({ 'id': f'id{i + 1}', 'name': f'name{i + 1}' }, item._data) def test_get_vm(self): self.co._cs_get_single_result = Mock() self.co.get_vm(name='vm1') self.assertDictEqual({ 'name': 'vm1', 'listall': True }, self.co._cs_get_single_result.call_args[0][1]) self.co.get_vm(name='i-VM-1') self.assertDictEqual({ 'keyword': 'i-VM-1', 'listall': True }, self.co._cs_get_single_result.call_args[0][1]) self.co.get_vm(name='project_vm1', is_project_vm=True) self.assertDictEqual( { 'name': 'project_vm1', 'projectid': '-1', 'listall': True }, self.co._cs_get_single_result.call_args[0][1]) def test_get_project_vm(self): self.co._cs_get_single_result = Mock() self.co.get_project_vm(name='project_vm1') self.assertDictEqual( { 'name': 'project_vm1', 'projectid': '-1', 'listall': True }, self.co._cs_get_single_result.call_args[0][1]) def test_get_router(self): self.co._cs_get_single_result = Mock() self.co.get_router(name='router1') self.assertDictEqual({ 'name': 'router1', 'listall': True }, self.co._cs_get_single_result.call_args[0][1]) self.co.get_router(name='project_router1', is_project_router=True) self.assertDictEqual( { 'name': 'project_router1', 'projectid': '-1', 'listall': True }, self.co._cs_get_single_result.call_args[0][1]) def test_get_cluster(self): self.co._cs_get_single_result = Mock() self.co.get_zone = Mock( return_value=CosmicZone(Mock(), { 'id': 'z1', 'name': 'zone1' })) self.co.get_cluster(name='cluster1') self.assertDictEqual({'name': 'cluster1'}, self.co._cs_get_single_result.call_args[0][1]) self.co.get_cluster(name='cluster1', zone='zone1') self.assertDictEqual({ 'name': 'cluster1', 'zoneid': 'z1' }, self.co._cs_get_single_result.call_args[0][1]) self.co.get_zone.return_value = None self.assertIsNone(self.co.get_cluster(name='cluster1', zone='zone1')) def test_get_all_clusters(self): self.co._cs_get_all_results = Mock() self.co.get_all_clusters() self.assertDictEqual({}, self.co._cs_get_all_results.call_args[0][1]) self.co.get_all_clusters(zone=CosmicZone(Mock(), {'id': 'z1'})) self.assertDictEqual({'zoneid': 'z1'}, self.co._cs_get_all_results.call_args[0][1]) self.co.get_all_clusters(pod=CosmicPod(Mock(), {'id': 'p1'})) self.assertDictEqual({'podid': 'p1'}, self.co._cs_get_all_results.call_args[0][1]) def test_get_service_offering(self): self.co._cs_get_single_result = Mock() self.co.get_service_offering(name='so1') self.assertDictEqual({'name': 'so1'}, self.co._cs_get_single_result.call_args[0][1]) self.co.get_service_offering(name='so1', system=True) self.assertDictEqual({ 'name': 'so1', 'issystem': True }, self.co._cs_get_single_result.call_args[0][1]) def test_get_all_vms(self): self.co._cs_get_all_results = Mock() self.co.get_all_vms() self.assertDictEqual({'listall': True}, self.co._cs_get_all_results.call_args[0][1]) def test_get_all_project_vms(self): self.co._cs_get_all_results = Mock() self.co.get_all_project_vms() self.assertDictEqual({ 'projectid': '-1', 'listall': True }, self.co._cs_get_all_results.call_args[0][1]) def test_wait_for_job(self): self.cs_instance.queryAsyncJobResult.return_value = {'jobstatus': '1'} self.assertTrue(self.co.wait_for_job('job')) def test_wait_for_job_cloud_stack_exception(self): self.cs_instance.queryAsyncJobResult.side_effect = CloudStackException( response=Mock()) self.assertRaises(CloudStackException, self.co.wait_for_job, 'job') def test_wait_for_job_connection_error(self): self.cs_instance.queryAsyncJobResult.side_effect = ConnectionError self.assertRaises(ConnectionError, self.co.wait_for_job, 'job') def test_wait_for_job_retries(self): self.cs_instance.queryAsyncJobResult.side_effect = [ CloudStackException('multiple JSON fields named jobstatus', response=Mock()), ConnectionError('Connection aborted'), ConnectionError('Connection aborted') ] self.assertFalse(self.co.wait_for_job('job', 3)) def test_wait_for_job_failure(self): self.cs_instance.queryAsyncJobResult.return_value = {'jobstatus': '2'} self.assertFalse(self.co.wait_for_job('job'))