def main(profile, dry_run, router):
"""Live migrate ROUTER"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Live Migrate HV to new POD'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
router = co.get_system_vm(name=router)
if not router:
sys.exit(1)
source_host = co.get_host(id=router['hostid'])
if not source_host:
sys.exit(1)
cluster = co.get_cluster(id=source_host['clusterid'])
if not cluster:
sys.exit(1)
destination_host = cluster.find_migration_host(router)
if not destination_host:
sys.exit(1)
if not router.migrate(destination_host):
sys.exit(1)
def main(profile, destination_dc, dry_run, host, cluster):
"""Migrate all VMs on HOST to CLUSTER"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Live Migrate HV to new POD'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
cs = CosmicSQL(server=profile, dry_run=dry_run)
host = co.get_host(name=host)
if not host:
sys.exit(1)
for vm in host.get_all_vms() + host.get_all_project_vms():
live_migrate(co=co,
cs=cs,
cluster=cluster,
vm_name=vm['name'],
destination_dc=destination_dc,
add_affinity_group=None,
is_project_vm=None,
zwps_to_cwps=None,
log_to_slack=log_to_slack,
dry_run=dry_run)
def main(profile, dry_run, skip_version, older_then, skip_zone, only_zone, restart_agent):
"""Destroy SVM per zone and waits for a new one"""
click_log.basic_config()
if dry_run:
logging.info('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run)
if older_then:
older_then = datetime.strptime(f"{older_then}T00:00:00+0200", '%Y%m%dT%H:%M:%S%z')
svms = co.get_all_systemvms()
zones = defaultdict(list)
for svm in svms:
if only_zone and co.get_host(name=svm['name']).get('zonename') != only_zone:
continue
if skip_zone and co.get_host(name=svm['name']).get('zonename') == skip_zone:
continue
if skip_version and co.get_host(name=svm['name']).get('version') == skip_version:
continue
if older_then and datetime.strptime(svm['created'], '%Y-%m-%dT%H:%M:%S%z') > older_then:
continue
zones[svm['zonename']].append(svm)
for zone in zones:
logging.info(f"Processing zone: {zone}")
for vm in zones[zone]:
if not vm.destroy():
sys.exit(1)
up = list()
down = list(zones[zone])
retries = 60
zone_id = vm['zoneid']
while len(up) < len(zones[zone]) or len(down) > 0:
if not dry_run:
time.sleep(5)
try:
systemvms = {x['name']: x for x in co.get_all_systemvms(zoneid=zone_id)}
host_status = {k: co.get_host(name=k) for k in systemvms}
up = list(filter(lambda x: x and x['state'] == 'Up' and x['resourcestate'] == 'Enabled', host_status.values()))
down = list(filter(lambda x: x and x['state'] != 'Up' and x['resourcestate'] == 'Enabled', host_status.values()))
retries -= 1
if retries == 0:
break
if down and restart_agent:
for d in down:
svm_object = list(filter(lambda x: x and x['name'].lower() == d['name'], systemvms.values()))[0]
svm_object.restart_agent()
except KeyError:
# Ignore keyerror, systemvm is still not available as host
pass
if retries == 0:
logging.error("Exceeded retry count waiting for new systemvm")
sys.exit(1)
def setUp(self):
cs_patcher = patch('cosmicops.ops.CloudStack')
self.mock_cs = cs_patcher.start()
self.addCleanup(cs_patcher.stop)
self.cs_instance = self.mock_cs.return_value
slack_patcher = patch('cosmicops.log.Slack')
self.mock_slack = slack_patcher.start()
self.addCleanup(slack_patcher.stop)
sleep_patcher = patch('time.sleep', return_value=None)
self.mock_sleep = sleep_patcher.start()
self.addCleanup(sleep_patcher.stop)
self._vm_json = {
'id': 'v1',
'name': 'vm1',
'instancename': 'i-1-VM',
'hostname': 'host1',
'maintenancepolicy': 'LiveMigrate',
'isoid': 'iso1'
}
self.ops = CosmicOps(endpoint='https://localhost',
key='key',
secret='secret',
dry_run=False)
self.ops.wait_for_job = Mock(return_value=True)
self.vm = CosmicVM(self.ops, self._vm_json)
self.target_host = CosmicHost(self.ops, {'id': 'h1', 'name': 'host1'})
def setUp(self):
cs_patcher = patch('cosmicops.ops.CloudStack')
self.mock_cs = cs_patcher.start()
self.addCleanup(cs_patcher.stop)
self.cs_instance = self.mock_cs.return_value
slack_patcher = patch('cosmicops.log.Slack')
self.mock_slack = slack_patcher.start()
self.addCleanup(slack_patcher.stop)
sleep_patcher = patch('time.sleep', return_value=None)
self.mock_sleep = sleep_patcher.start()
self.addCleanup(sleep_patcher.stop)
self.co = CosmicOps(endpoint='https://localhost',
key='key',
secret='secret')
def main(profile, zwps_to_cwps, add_affinity_group, destination_dc, is_project_vm,
skip_within_cluster, dry_run, vm, cluster):
"""Live migrate VM to CLUSTER"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Live Migrate VM'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
cs = CosmicSQL(server=profile, dry_run=dry_run)
# Work around migration issue: first in the same pod to limit possible hiccup
vm_instance = co.get_vm(name=vm, is_project_vm=is_project_vm)
if not vm_instance:
logging.error(f"Cannot migrate, VM '{vm}' not found!")
sys.exit(1)
if not vm_instance['state'] == 'Running':
logging.error(f"Cannot migrate, VM has has state: '{vm_instance['state']}'")
sys.exit(1)
source_host = co.get_host(id=vm_instance['hostid'])
source_cluster = co.get_cluster(id=source_host['clusterid'])
if not skip_within_cluster:
if not vm_instance.migrate_within_cluster(vm=vm_instance, source_cluster=source_cluster,
source_host=source_host, instancename=vm_instance):
logging.info(f"VM Migration failed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n")
sys.exit(1)
if not live_migrate(co, cs, cluster, vm, destination_dc, add_affinity_group, is_project_vm, zwps_to_cwps,
log_to_slack, dry_run):
logging.info(f"VM Migration failed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n")
sys.exit(1)
logging.info(f"VM Migration completed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n")
def setUp(self):
cs_patcher = patch('cosmicops.ops.CloudStack')
self.mock_cs = cs_patcher.start()
self.addCleanup(cs_patcher.stop)
self.cs_instance = self.mock_cs.return_value
self.ops = CosmicOps(endpoint='https://localhost',
key='key',
secret='secret')
self.cluster = CosmicCluster(self.ops, {
'id': 'c1',
'name': 'cluster1'
})
def empty_host(profile, shutdown, skip_disable, dry_run, host):
co = CosmicOps(profile=profile, dry_run=dry_run)
host = co.get_host(name=host)
if not host:
raise RuntimeError(f"Host '{host['name']}' not found")
if not skip_disable and host['resourcestate'] != 'Disabled':
if not host.disable():
raise RuntimeError(f"Failed to disable host '{host['name']}'")
(total, success, failed) = host.empty()
result_message = f"Result: {success} successful, {failed} failed out of {total} total VMs"
if not failed and shutdown:
host.set_uid_led(True)
if not host.reboot(RebootAction.HALT):
raise RuntimeError(f"Failed to shutdown host '{host['name']}'")
host.wait_until_offline()
result_message = f"{result_message}\nHost '{host['name']}' has shutdown, UID led is turned on"
elif failed and shutdown:
result_message = f"{result_message}\nNot shutting down host '{host['name']}' because migration completed with failed VMs"
return result_message
def setUp(self):
cs_patcher = patch('cosmicops.ops.CloudStack')
self.mock_cs = cs_patcher.start()
self.addCleanup(cs_patcher.stop)
self.cs_instance = self.mock_cs.return_value
sleep_patcher = patch('time.sleep', return_value=None)
self.mock_sleep = sleep_patcher.start()
self.addCleanup(sleep_patcher.stop)
self.ops = CosmicOps(endpoint='https://localhost', key='key', secret='secret', dry_run=False)
self.ops.wait_for_job = Mock(return_value=True)
self.router = CosmicRouter(self.ops, {
'id': 'r1',
'name': 'router1'
})
def setUp(self):
cs_patcher = patch('cosmicops.ops.CloudStack')
self.mock_cs = cs_patcher.start()
self.addCleanup(cs_patcher.stop)
self.cs_instance = self.mock_cs.return_value
self.ops = CosmicOps(endpoint='https://localhost',
key='key',
secret='secret')
self.storage_pool = CosmicStoragePool(self.ops, {
'id': 'p1',
'name': 'pool1',
'ipaddress': 'ip_address',
'path': 'path'
})
def main(profile, max_iops, zwps_to_cwps, is_project_vm, dry_run, vm,
storage_pool):
"""Live migrate VM volumes to STORAGE_POOL"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Live Migrate VM Volumes'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
cs = CosmicSQL(server=profile, dry_run=dry_run)
if not live_migrate_volumes(storage_pool, co, cs, dry_run, is_project_vm,
log_to_slack, max_iops, vm, zwps_to_cwps):
sys.exit(1)
def setUp(self):
cs_patcher = patch('cosmicops.ops.CloudStack')
self.mock_cs = cs_patcher.start()
self.addCleanup(cs_patcher.stop)
self.cs_instance = self.mock_cs.return_value
sleep_patcher = patch('time.sleep', return_value=None)
self.mock_sleep = sleep_patcher.start()
self.addCleanup(sleep_patcher.stop)
self.ops = CosmicOps(endpoint='https://localhost',
key='key',
secret='secret',
dry_run=False)
self.ops.wait_for_volume_migration_job = Mock(return_value=True)
self.cs_instance.migrateVolume.return_value = {'jobid': 1}
self.volume = CosmicVolume(self.ops, {'id': 'vol1', 'name': 'volume1'})
self.storage_pool = CosmicStoragePool(self.ops, {
'id': 'p1',
'name': 'pool1'
})
def main(profile, uuid, network_uuid, dry_run, vpc):
"""VPC restart script"""
click_log.basic_config()
log_to_slack = True
if uuid and network_uuid:
logging.error('You can not specify --uuid and --network-uuid together')
sys.exit(1)
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
if uuid:
vpc = co.get_vpc(id=vpc)
elif network_uuid:
network = co.get_network(id=vpc)
if not network:
sys.exit(1)
vpc = co.get_vpc(id=network['vpcid'])
else:
vpc = co.get_vpc(name=vpc)
if not vpc:
sys.exit(1)
logging.slack_title = 'Domain'
logging.slack_value = vpc['domain']
logging.instance_name = vpc['name']
logging.zone_name = vpc['zonename']
if not vpc.restart():
sys.exit(1)
logging.info(
f"Successfully restarted VPC '{vpc['name']}' ({vpc['id']}) with clean up",
log_to_slack)
class TestCosmicOps(TestCase):
def setUp(self):
cs_patcher = patch('cosmicops.ops.CloudStack')
self.mock_cs = cs_patcher.start()
self.addCleanup(cs_patcher.stop)
self.cs_instance = self.mock_cs.return_value
slack_patcher = patch('cosmicops.log.Slack')
self.mock_slack = slack_patcher.start()
self.addCleanup(slack_patcher.stop)
sleep_patcher = patch('time.sleep', return_value=None)
self.mock_sleep = sleep_patcher.start()
self.addCleanup(sleep_patcher.stop)
self.co = CosmicOps(endpoint='https://localhost',
key='key',
secret='secret')
@patch('cosmicops.ops._load_cloud_monkey_profile')
def test_init_with_profile(self, mock_load):
mock_load.return_value = ('profile_endpoint', 'profile_key',
'profile_secret')
CosmicOps(profile='config')
self.mock_cs.assert_called_with('profile_endpoint', 'profile_key',
'profile_secret', 60)
@tempdir()
def test_load_cloud_monkey_profile(self, tmp):
config = (b"[testprofile]\n"
b"url = http://localhost:8000/client/api\n"
b"apikey = test_api_key\n"
b"secretkey = test_secret_key\n")
tmp.makedir('.cloudmonkey')
tmp.write('.cloudmonkey/config', config)
with patch('pathlib.Path.home') as path_home_mock:
path_home_mock.return_value = Path(tmp.path)
(endpoint, key, secret) = _load_cloud_monkey_profile('testprofile')
self.assertEqual('http://localhost:8000/client/api', endpoint)
self.assertEqual('test_api_key', key)
self.assertEqual('test_secret_key', secret)
@tempdir()
def test_load_cloud_monkey_profile_with_default(self, tmp):
config = (b"[core]\n"
b"profile = profile2\n\n"
b"[profile1]\n"
b"url = http://localhost:8000/client/api/1\n"
b"apikey = test_api_key_1\n"
b"secretkey = test_secret_key_1\n\n"
b"[profile2]\n"
b"url = http://localhost:8000/client/api/2\n"
b"apikey = test_api_key_2\n"
b"secretkey = test_secret_key_2\n")
tmp.makedir('.cloudmonkey')
tmp.write('.cloudmonkey/config', config)
with patch('pathlib.Path.home') as path_home_mock:
path_home_mock.return_value = Path(tmp.path)
(endpoint, key, secret) = _load_cloud_monkey_profile('config')
self.assertEqual('http://localhost:8000/client/api/2', endpoint)
self.assertEqual('test_api_key_2', key)
self.assertEqual('test_secret_key_2', secret)
def test_cs_get_single_result(self):
self.cs_instance.listFunction.return_value = [{
'id': 'id_field',
'name': 'name_field'
}]
result = self.co._cs_get_single_result('listFunction',
{'name': 'name_field'},
CosmicObject, 'type')
self.cs_instance.listFunction.assert_called_with(fetch_list=True,
name='name_field')
self.assertIsInstance(result, CosmicObject)
self.assertDictEqual({
'id': 'id_field',
'name': 'name_field'
}, result._data)
def test_get_get_single_result_failure(self):
self.cs_instance.listFunction.return_value = []
self.assertIsNone(
self.co._cs_get_single_result('listFunction', {}, CosmicObject,
'type'))
self.cs_instance.listFunction.return_value = [{}, {}]
self.assertIsNone(
self.co._cs_get_single_result('listFunction', {}, CosmicObject,
'type'))
def test_cs_get_all_results(self):
self.cs_instance.listFunction.return_value = [{
'id': 'id1',
'name': 'name1'
}, {
'id': 'id2',
'name': 'name2'
}]
result = self.co._cs_get_all_results('listFunction', {}, CosmicObject,
'type')
self.cs_instance.listFunction.assert_called_with(fetch_list=True)
for i, item in enumerate(result):
self.assertIsInstance(item, CosmicObject)
self.assertDictEqual({
'id': f'id{i + 1}',
'name': f'name{i + 1}'
}, item._data)
def test_get_vm(self):
self.co._cs_get_single_result = Mock()
self.co.get_vm(name='vm1')
self.assertDictEqual({
'name': 'vm1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
self.co.get_vm(name='i-VM-1')
self.assertDictEqual({
'keyword': 'i-VM-1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
self.co.get_vm(name='project_vm1', is_project_vm=True)
self.assertDictEqual(
{
'name': 'project_vm1',
'projectid': '-1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
def test_get_project_vm(self):
self.co._cs_get_single_result = Mock()
self.co.get_project_vm(name='project_vm1')
self.assertDictEqual(
{
'name': 'project_vm1',
'projectid': '-1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
def test_get_router(self):
self.co._cs_get_single_result = Mock()
self.co.get_router(name='router1')
self.assertDictEqual({
'name': 'router1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
self.co.get_router(name='project_router1', is_project_router=True)
self.assertDictEqual(
{
'name': 'project_router1',
'projectid': '-1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
def test_get_cluster(self):
self.co._cs_get_single_result = Mock()
self.co.get_zone = Mock(
return_value=CosmicZone(Mock(), {
'id': 'z1',
'name': 'zone1'
}))
self.co.get_cluster(name='cluster1')
self.assertDictEqual({'name': 'cluster1'},
self.co._cs_get_single_result.call_args[0][1])
self.co.get_cluster(name='cluster1', zone='zone1')
self.assertDictEqual({
'name': 'cluster1',
'zoneid': 'z1'
}, self.co._cs_get_single_result.call_args[0][1])
self.co.get_zone.return_value = None
self.assertIsNone(self.co.get_cluster(name='cluster1', zone='zone1'))
def test_get_all_clusters(self):
self.co._cs_get_all_results = Mock()
self.co.get_all_clusters()
self.assertDictEqual({}, self.co._cs_get_all_results.call_args[0][1])
self.co.get_all_clusters(zone=CosmicZone(Mock(), {'id': 'z1'}))
self.assertDictEqual({'zoneid': 'z1'},
self.co._cs_get_all_results.call_args[0][1])
self.co.get_all_clusters(pod=CosmicPod(Mock(), {'id': 'p1'}))
self.assertDictEqual({'podid': 'p1'},
self.co._cs_get_all_results.call_args[0][1])
def test_get_service_offering(self):
self.co._cs_get_single_result = Mock()
self.co.get_service_offering(name='so1')
self.assertDictEqual({'name': 'so1'},
self.co._cs_get_single_result.call_args[0][1])
self.co.get_service_offering(name='so1', system=True)
self.assertDictEqual({
'name': 'so1',
'issystem': True
}, self.co._cs_get_single_result.call_args[0][1])
def test_get_all_vms(self):
self.co._cs_get_all_results = Mock()
self.co.get_all_vms()
self.assertDictEqual({'listall': True},
self.co._cs_get_all_results.call_args[0][1])
def test_get_all_project_vms(self):
self.co._cs_get_all_results = Mock()
self.co.get_all_project_vms()
self.assertDictEqual({
'projectid': '-1',
'listall': True
}, self.co._cs_get_all_results.call_args[0][1])
def test_wait_for_job(self):
self.cs_instance.queryAsyncJobResult.return_value = {'jobstatus': '1'}
self.assertTrue(self.co.wait_for_job('job'))
def test_wait_for_job_cloud_stack_exception(self):
self.cs_instance.queryAsyncJobResult.side_effect = CloudStackException(
response=Mock())
self.assertRaises(CloudStackException, self.co.wait_for_job, 'job')
def test_wait_for_job_connection_error(self):
self.cs_instance.queryAsyncJobResult.side_effect = ConnectionError
self.assertRaises(ConnectionError, self.co.wait_for_job, 'job')
def test_wait_for_job_retries(self):
self.cs_instance.queryAsyncJobResult.side_effect = [
CloudStackException('multiple JSON fields named jobstatus',
response=Mock()),
ConnectionError('Connection aborted'),
ConnectionError('Connection aborted')
]
self.assertFalse(self.co.wait_for_job('job', 3))
def test_wait_for_job_failure(self):
self.cs_instance.queryAsyncJobResult.return_value = {'jobstatus': '2'}
self.assertFalse(self.co.wait_for_job('job'))
def setUp(self):
cs_patcher = patch('cosmicops.ops.CloudStack')
self.mock_cs = cs_patcher.start()
self.addCleanup(cs_patcher.stop)
self.cs_instance = self.mock_cs.return_value
connection_patcher = patch('cosmicops.objects.host.Connection')
self.mock_connection = connection_patcher.start()
self.addCleanup(connection_patcher.stop)
self.connection_instance = self.mock_connection.return_value
socket_patcher = patch('socket.socket')
self.mock_socket = socket_patcher.start()
self.addCleanup(socket_patcher.stop)
self.socket_context = self.mock_socket.return_value.__enter__.return_value
ilo_patcher = patch('hpilo.Ilo')
self.mock_ilo = ilo_patcher.start()
self.addCleanup(ilo_patcher.stop)
self.ilo_instance = self.mock_ilo.return_value
slack_patcher = patch('cosmicops.log.Slack')
self.mock_slack = slack_patcher.start()
self.addCleanup(slack_patcher.stop)
sleep_patcher = patch('time.sleep', return_value=None)
self.mock_sleep = sleep_patcher.start()
self.addCleanup(sleep_patcher.stop)
self.ops = CosmicOps(endpoint='https://localhost', key='key', secret='secret', dry_run=False)
self.user_vm = CosmicVM(self.ops, {
'id': 'v1',
'name': 'vm1',
'instancename': 'i-1-VM',
'hostname': 'host1',
'maintenancepolicy': 'LiveMigrate',
'isoid': 'iso1'
})
self.project_vm = CosmicVM(self.ops, {
'id': 'pv1',
'name': 'projectvm1',
'instancename': 'i-1-PROJECT-VM',
'hostname': 'host1',
'project': 'project1',
'projectid': 'p1'
})
self.router_vm = CosmicRouter(self.ops, {
'id': 'r1',
'name': 'r-1-VM',
'hostname': 'host1'
})
self.project_router_vm = CosmicRouter(self.ops, {
'id': 'pr1',
'name': 'r-1-PROJECT-VM',
'hostname': 'host1'
})
self.secondary_storage_vm = CosmicVM(self.ops, {
'id': 's1',
'name': 's-1-VM',
'hostname': 'host1',
'systemvmtype': 'secondarystoragevm'
})
self.console_proxy = CosmicVM(self.ops, {
'id': 's2',
'name': 'v-2-VM',
'hostname': 'host1',
'systemvmtype': 'consoleproxy'
})
self.all_vms = [self.user_vm, self.project_vm, self.router_vm, self.project_router_vm,
self.secondary_storage_vm, self.console_proxy]
self.vm_count = len(self.all_vms)
self.host = CosmicHost(self.ops, {
'id': 'h1',
'name': 'host1',
'clusterid': '1',
'resourcestate': 'Enabled'
})
def main(profile, is_project_vm, dry_run, vm, cluster, destination_dc, destination_so):
"""Offline migrate VM to CLUSTER"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Offline Migrate VM'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
cs = CosmicSQL(server=profile, dry_run=dry_run)
target_cluster = co.get_cluster(name=cluster)
if not target_cluster:
sys.exit(1)
vm = co.get_vm(name=vm, is_project_vm=is_project_vm)
if not vm:
sys.exit(1)
if destination_dc and destination_dc not in DATACENTERS:
logging.error(f"Unknown datacenter '{destination_dc}', should be one of {str(DATACENTERS)}")
sys.exit(1)
logging.instance_name = vm['instancename']
logging.slack_value = vm['domain']
logging.vm_name = vm['name']
logging.zone_name = vm['zonename']
target_storage_pool = None
try:
# Get CLUSTER scoped volume (no NVMe or ZONE-wide)
while target_storage_pool is None or target_storage_pool['scope'] != 'CLUSTER':
target_storage_pool = choice(target_cluster.get_storage_pools())
except IndexError:
logging.error(f"No storage pools found for cluster '{target_cluster['name']}")
sys.exit(1)
if vm['state'] == 'Running':
need_to_stop = True
auto_start_vm = True
else:
need_to_stop = False
auto_start_vm = False
if destination_dc:
for datacenter in DATACENTERS:
if datacenter == destination_dc:
continue
if datacenter in vm['serviceofferingname']:
new_offering = vm['serviceofferingname'].replace(datacenter, destination_dc)
logging.info(
f"Replacing '{vm['serviceofferingname']}' with '{new_offering}'")
cs.update_service_offering_of_vm(vm['instancename'], new_offering)
vm = co.get_vm(name=vm['instancename'], is_project_vm=is_project_vm)
break
if destination_so:
logging.info(
f"Replacing '{vm['serviceofferingname']}' with '{destination_so}'")
cs.update_service_offering_of_vm(vm['instancename'], destination_so)
vm = co.get_vm(name=vm['instancename'], is_project_vm=is_project_vm)
vm_service_offering = co.get_service_offering(id=vm['serviceofferingid'])
if vm_service_offering:
storage_tags = vm_service_offering['tags'] if 'tags' in vm_service_offering else ''
if not storage_tags:
logging.warning('VM service offering has no storage tags')
else:
if storage_tags not in target_storage_pool['tags']:
logging.error(
f"Can't migrate '{vm['name']}', storage tags on target cluster ({target_storage_pool['tags']}) to not contain the tags on the VM's service offering ({storage_tags})'")
sys.exit(1)
if need_to_stop:
if not vm.stop():
sys.exit(1)
volumes = vm.get_volumes()
for volume in volumes:
if volume['storage'] == target_storage_pool['name']:
logging.warning(
f"Volume '{volume['name']}' ({volume['id']}) already on cluster '{target_cluster['name']}', skipping...")
continue
source_storage_pool = co.get_storage_pool(name=volume['storage'])
if not source_storage_pool:
sys.exit(1)
if source_storage_pool['scope'] == 'ZONE':
logging.warning(f"Scope of volume '{volume['name']}' ({volume['id']}) is ZONE, skipping...")
continue
if not volume.migrate(target_storage_pool):
sys.exit(1)
with click_spinner.spinner():
while True:
volume.refresh()
if volume['state'] == 'Ready':
break
logging.warning(
f"Volume '{volume['name']}' ({volume['id']}) is in '{volume['state']}' state instead of 'Ready', sleeping...")
time.sleep(60)
if auto_start_vm:
destination_host = target_cluster.find_migration_host(vm)
if not destination_host:
sys.exit(1)
if not vm.start(destination_host):
sys.exit(1)
else:
logging.info(f"Not starting VM '{vm['name']}' as it was not running", log_to_slack)
def test_init_with_profile(self, mock_load):
mock_load.return_value = ('profile_endpoint', 'profile_key',
'profile_secret')
CosmicOps(profile='config')
self.mock_cs.assert_called_with('profile_endpoint', 'profile_key',
'profile_secret', 60)
def main(profile, domain_name, cluster_name, pod_name, zone_name, keyword_filter, only_routers,
only_routers_to_be_upgraded,
no_routers,
router_nic_count, nic_count_is_minimum, nic_count_is_maximum, router_max_version, router_min_version,
project_name, only_project, ignore_domains, calling_credentials, only_summary, no_summary, log_file):
"""List VMs"""
click_log.basic_config()
if log_file:
logger = logging.getLogger()
logger.addHandler(log_module.FileHandler(log_file))
if project_name and domain_name:
logging.error("The project and domain options can't be used together")
sys.exit(1)
co = CosmicOps(profile=profile, dry_run=False)
if ignore_domains:
ignore_domains = ignore_domains.replace(' ', '').split(',')
logging.info(f"Ignoring domains: {str(ignore_domains)}")
if calling_credentials:
table_headers = [
'VM',
'Storage',
'Template',
'Memory',
'Cores',
'Instance',
'Host',
'Domain',
'Account',
'Created',
'LastRebootVersion'
]
table_data = []
if only_project:
vms = co.get_all_project_vms(list_all=False)
else:
vms = co.get_all_vms(list_all=False)
with click_spinner.spinner():
for vm in vms:
if vm['domain'] in ignore_domains:
continue
storage_size = sum([volume['size'] for volume in vm.get_volumes()])
project_name = vm.get('project', None)
vm_account = f"Proj: {project_name}" if project_name else vm['account']
table_data.append([
vm['name'],
humanfriendly.format_size(storage_size, binary=True),
vm['templatedisplaytext'],
humanfriendly.format_size(vm['memory'] * 1024 ** 2, binary=True),
vm['cpunumber'],
vm['instancename'],
vm['hostname'],
vm['domain'],
vm_account,
vm['created'],
vm['laststartversion']
])
logging.info(tabulate(table_data, headers=table_headers, tablefmt='pretty'))
sys.exit(0)
if domain_name:
domain = co.get_domain(name=domain_name)
if domain is None or domain == []:
logging.error(f"The domain '{str(domain_name)}' could not be found!")
sys.exit(1)
else:
domain = None
if project_name:
project = co.get_project(name=project_name)
if project is None or project == []:
logging.error(f"The project '{str(project_name)}' could not be found!")
sys.exit(1)
else:
project = None
if pod_name:
pod = co.get_pod(name=pod_name)
if pod is None or pod == []:
logging.error(f"The pod '{str(pod_name)}' could not be found!")
sys.exit(1)
else:
pod = None
if zone_name:
zone = co.get_zone(name=zone_name)
if zone is None or zone == []:
logging.error(f"The zone '{str(zone_name)}' could not be found!")
sys.exit(1)
else:
zone = None
if cluster_name:
clusters = [co.get_cluster(name=cluster_name)]
if clusters[0] is None:
logging.error(f"The cluster '{str(cluster_name)}' could not be found!")
sys.exit(1)
elif pod:
clusters = co.get_all_clusters(pod=pod)
elif zone:
clusters = co.get_all_clusters(zone=zone)
else:
clusters = co.get_all_clusters()
total_host_counter = 0
total_vm_counter = 0
total_host_memory = 0
total_vm_memory = 0
total_storage = 0
total_cores = 0
for cluster in clusters:
hosts = cluster.get_all_hosts()
if not hosts:
logging.warning(f"No hosts found on cluster '{cluster['name']}'")
continue
cluster_host_counter = 0
cluster_vm_counter = 0
cluster_host_memory = 0
cluster_vm_memory = 0
cluster_storage = 0
cluster_cores = 0
cluster_table_headers = [
'VM',
'Storage',
'Template',
'Router nic count',
'Router version',
'Memory',
'Cores',
'Instance',
'Host',
'Domain',
'Account',
'Created',
'LastRebootVersion'
]
cluster_table_data = []
for host in hosts:
cluster_host_counter += 1
cluster_host_memory += host['memorytotal']
if not only_routers:
if project or only_project:
vms = host.get_all_project_vms(project=project)
else:
vms = host.get_all_vms(domain=domain, keyword_filter=keyword_filter)
for vm in vms:
if vm['domain'] in ignore_domains:
continue
cluster_vm_counter += 1
storage_size = sum([volume['size'] for volume in vm.get_volumes()])
cluster_storage += storage_size
cluster_vm_memory += vm['memory']
cluster_cores += vm['cpunumber']
vm_project_name = vm.get('project', None)
vm_account = f"Proj: {vm['project']}" if vm_project_name else vm['account']
cluster_table_data.append([
vm['name'],
humanfriendly.format_size(storage_size, binary=True),
vm['templatedisplaytext'],
'-',
'-',
humanfriendly.format_size(vm['memory'] * 1024 ** 2, binary=True),
vm['cpunumber'],
vm['instancename'],
vm['hostname'],
vm['domain'],
vm_account,
vm['created'],
vm['laststartversion']
])
if no_routers:
continue
if project or only_project:
routers = host.get_all_project_routers(project=project)
else:
routers = host.get_all_routers(domain=domain)
for router in routers:
if router['domain'] in ignore_domains:
continue
if router_min_version and LooseVersion(router['version']) < LooseVersion(router_min_version):
continue
if router_max_version and LooseVersion(router['version']) > LooseVersion(router_max_version):
continue
if router_nic_count and nic_count_is_minimum:
if router_nic_count > len(router['nic']):
continue
elif router_nic_count and nic_count_is_maximum:
if router_nic_count < len(router['nic']):
continue
elif router_nic_count:
if router_nic_count != len(router['nic']):
continue
if only_routers_to_be_upgraded and not router['requiresupgrade']:
continue
cluster_vm_counter += 1
service_offering = co.get_service_offering(id=router['serviceofferingid'], system=True)
if service_offering:
router['memory'] = service_offering['memory']
router['cpunumber'] = service_offering['cpunumber']
cluster_vm_memory += router['memory']
cluster_cores += router['cpunumber']
else:
router['memory'] = 'Unknown'
router['cpunumber'] = 'Unknown'
if router['isredundantrouter']:
redundant_state = router['redundantstate']
elif router['vpcid']:
redundant_state = 'VPC'
else:
redundant_state = 'SINGLE'
if router['vpcid']:
network = co.get_vpc(id=router['vpcid'])
else:
network = co.get_network(id=router['guestnetworkid'])
if network:
display_name = network['name']
else:
display_name = router['name']
display_name = f"{display_name} ({redundant_state.lower()})"
if router['requiresupgrade']:
display_name = f"{display_name} [ReqUpdate!]"
router_project_name = router.get('project', None)
router_account = f"Proj: {router['project']}" if router_project_name else router['account']
cluster_table_data.append([
display_name,
'-',
'-',
len(router['nic']),
router['version'],
humanfriendly.format_size(router['memory'] * 1024 ** 2, binary=True) if router[
'memory'] != 'Unknown' else
router['memory'],
router['cpunumber'],
router['name'],
router['hostname'],
router['domain'],
router_account,
router['created'],
router['laststartversion']
])
total_host_counter += cluster_host_counter
total_host_memory += cluster_host_memory
total_vm_memory += cluster_vm_memory
total_vm_counter += cluster_vm_counter
total_storage += cluster_storage
total_cores += cluster_cores
if not only_summary: # pragma: no cover
logging.info(tabulate(cluster_table_data, headers=cluster_table_headers, tablefmt='pretty'))
if not no_summary: # pragma: no cover
logging.info(f"\nSummary for '{cluster['name']}':")
logging.info(f"Number of VMs: {cluster_vm_counter}")
logging.info(f"Number of hosts: {cluster_host_counter}")
logging.info(
f"Allocated memory: {humanfriendly.format_size(cluster_vm_memory * 1024 ** 2, binary=True)} / {humanfriendly.format_size(cluster_host_memory, binary=True)}")
logging.info(f"Allocated cores: {cluster_cores}")
logging.info(f"Allocated storage: {humanfriendly.format_size(cluster_storage, binary=True)}")
if not no_summary: # pragma: no cover
logging.info('\n================== Grand Totals ===============')
logging.info(f"Total number of VMs: {total_vm_counter}")
logging.info(f"Total number of hosts: {total_host_counter}")
logging.info(
f"Total allocated memory: {humanfriendly.format_size(total_vm_memory * 1024 ** 2, binary=True)} / {humanfriendly.format_size(total_host_memory, binary=True)}")
logging.info(f"Total allocated cores: {total_cores}")
logging.info(f"Total allocated storage: {humanfriendly.format_size(total_storage, binary=True)}")
def main(profile, dry_run, ignore_volumes, zwps_to_cwps, skip_disk_offerings,
only_project, source_cluster_name, destination_cluster_name):
"""Migrate offline volumes from SOURCE_CLUSTER to DESTINATION_CLUSTER"""
click_log.basic_config()
if source_cluster_name == destination_cluster_name:
logging.error('Destination cluster cannot be the source cluster!')
sys.exit(1)
if dry_run:
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run)
cs = CosmicSQL(server=profile, dry_run=dry_run)
source_cluster = co.get_cluster(name=source_cluster_name)
source_storage_pools = co.get_all_storage_pools(name=source_cluster_name)
if not source_cluster and not source_storage_pools:
logging.error(f"Source cluster not found:'{source_cluster_name}'!")
sys.exit(1)
destination_cluster = co.get_cluster(name=destination_cluster_name)
if not destination_cluster:
logging.error(
f"Destination cluster not found:'{destination_cluster_name}'!")
sys.exit(1)
if source_cluster:
try:
source_storage_pools = source_cluster.get_storage_pools(
scope='CLUSTER')
except IndexError:
logging.error(
f"No storage pools found for cluster '{source_cluster['name']}'"
)
sys.exit(1)
logging.info('Source storage pools found:')
for source_storage_pool in source_storage_pools:
logging.info(f" - '{source_storage_pool['name']}'")
try:
destination_storage_pools = destination_cluster.get_storage_pools(
scope='CLUSTER')
except IndexError:
logging.error(
f"No storage pools found for cluster '{destination_cluster['name']}'"
)
sys.exit(1)
logging.info('Destination storage pools found:')
for destination_storage_pool in destination_storage_pools:
logging.info(f" - '{destination_storage_pool['name']}'")
if ignore_volumes:
ignore_volumes = ignore_volumes.replace(' ', '').split(',')
logging.info(f"Ignoring volumes: {str(ignore_volumes)}")
if skip_disk_offerings:
skip_disk_offerings = skip_disk_offerings.replace(' ', '').split(',')
logging.info(f"Skipping disk offerings: {str(skip_disk_offerings)}")
for source_storage_pool in source_storage_pools:
destination_storage_pool = choice(destination_storage_pools)
volumes = source_storage_pool.get_volumes(only_project)
for volume in volumes:
if volume['id'] in ignore_volumes:
continue
if skip_disk_offerings and volume.get(
'diskofferingname') in skip_disk_offerings:
logging.warning(
f"Volume '{volume['name']}' has offering '{volume['diskofferingname']}', skipping..."
)
continue
if 'storage' not in volume:
logging.warning(
f"No storage attribute found for volume '{volume['name']}' ({volume['id']}), skipping..."
)
continue
if volume['storage'] == destination_storage_pool['name']:
logging.warning(
f"Volume '{volume['name']}' ({volume['id']}) already on cluster '{destination_cluster['name']}', skipping..."
)
continue
if volume['state'] != 'Ready':
logging.warning(
f"Volume '{volume['name']}' ({volume['id']}) is in state '{volume['state']}', skipping..."
)
continue
if 'vmstate' in volume and volume['vmstate'] != 'Stopped':
logging.warning(
f"Volume '{volume['name']}' ({volume['id']}) is attached to {volume['vmstate']} VM '{volume['vmname']}', skipping..."
)
continue
if zwps_to_cwps:
if not dry_run:
logging.info(
f"Converting ZWPS volume '{volume['name']}' to CWPS before starting the migration"
)
if not cs.update_zwps_to_cwps('MCC_v1.CWPS',
volume_id=volume['id']):
logging.error(
f"Failed to apply CWPS disk offering to volume '{volume['name']}'"
)
return False
else:
logging.info(
f"Would have changed the diskoffering for volume '{volume['name']}' to CWPS before starting the migration"
)
if source_cluster:
logging.info(
f"Volume '{volume['name']}' will be migrated from cluster '{source_cluster['name']}' to '{destination_cluster['name']}'"
)
else:
logging.info(
f"Volume '{volume['name']}' will be migrated from storage pool '{source_storage_pool['name']}' to '{destination_cluster['name']}'"
)
if not volume.migrate(destination_storage_pool):
continue
def main(profile, is_project_router, only_when_required, cleanup, dry_run,
router):
"""Router restart and upgrade script"""
click_log.basic_config()
log_to_slack = True
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
router = co.get_router(name=router, is_project_router=is_project_router)
if not router:
sys.exit(1)
logging.instance_name = router['name']
logging.slack_title = 'Domain'
logging.slack_value = router['domain']
host = co.get_host(id=router['hostid'])
if not host:
sys.exit(1)
cluster = co.get_cluster(id=host['clusterid'])
if not cluster:
sys.exit(1)
logging.cluster = cluster['name']
if only_when_required and not router['requiresupgrade']:
logging.info(
f"Router '{router['name']}' does not need to be upgraded. Will not reboot because --only-when-required was specified."
)
sys.exit(0)
if cleanup:
if not router['vpcid']:
logging.error(
f"Cleanup specified but no VPC ID found for router '{router['name']}'"
)
sys.exit(1)
logging.task = 'Restart VPC with clean up'
vpc = co.get_vpc(id=router['vpcid'])
if not vpc:
sys.exit(1)
if not vpc.restart():
sys.exit(1)
logging.info(
f"Successfully restarted VPC '{vpc['name']}' with cleanup for router '{router['name']}'"
)
else:
logging.task = 'Reboot virtual router'
if not router.reboot():
sys.exit(1)
logging.info(f"Successfully rebooted router '{router['name']}'",
log_to_slack)
def main(profile, ignore_hosts, only_hosts, skip_os_version, reboot_action,
pre_empty_script, post_empty_script, post_reboot_script, dry_run,
cluster):
"""Perform rolling reboot of hosts in CLUSTER"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Rolling Reboot'
logging.slack_title = 'Hypervisor'
logging.instance_name = 'N/A'
logging.vm_name = 'N/A'
logging.cluster = cluster
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
cluster = co.get_cluster(name=cluster)
if not cluster:
sys.exit(1)
hosts = cluster.get_all_hosts()
logging.debug(f"Found hosts: {hosts}")
if ignore_hosts:
ignore_hosts = ignore_hosts.replace(' ', '').split(',')
logging.info(f"Ignoring hosts: {str(ignore_hosts)}")
hosts = [h for h in hosts if h['name'] not in ignore_hosts]
elif only_hosts:
only_hosts = only_hosts.replace(' ', '').split(',')
logging.info(f"Only processing hosts: {str(only_hosts)}")
hosts = [h for h in hosts if h['name'] in only_hosts]
if skip_os_version:
logging.info(f"Skipping hosts with OS: {skip_os_version}")
hosts = [
h for h in hosts if skip_os_version not in h['hypervisorversion']
]
hosts.sort(key=itemgetter('name'))
target_host = None
for host in hosts:
logging.slack_value = host['name']
logging.zone_name = host['zonename']
logging.info(f"Processing host {host['name']}", log_to_slack)
for script in filter(
None,
(pre_empty_script, post_empty_script, post_reboot_script)):
path = Path(script)
host.copy_file(str(path), f'/tmp/{path.name}', mode=0o755)
if pre_empty_script:
host.execute(f'/tmp/{Path(pre_empty_script).name}',
sudo=True,
hide_stdout=False,
pty=True)
if host['resourcestate'] != 'Disabled':
if not host.disable():
sys.exit(1)
if host['state'] != 'Up' and not dry_run:
logging.error(
f"Host '{host['name']} is not up (state: '{host['state']}'), aborting",
log_to_slack)
sys.exit(1)
running_vms = len(host.get_all_vms())
logging.info(
f"Found {running_vms} running on host '{host['name']}'. Will now start migrating them to other hosts in the same cluster",
log_to_slack)
while True:
(_, _, failed) = host.empty(target=target_host)
if failed == 0:
break
if target_host:
logging.warning(
f"Failed to empty host '{host['name']}' with target '{target_host['name']}', resetting target host and retrying...",
log_to_slack)
target_host = None
else:
logging.warning(
f"Failed to empty host '{host['name']}', retrying...",
log_to_slack)
time.sleep(5)
logging.info(f"Host {host['name']} is empty", log_to_slack)
if post_empty_script:
host.execute(f'/tmp/{Path(post_empty_script).name}',
sudo=True,
hide_stdout=False,
pty=True)
if not host.reboot(reboot_action):
sys.exit(1)
if reboot_action != RebootAction.SKIP:
host.wait_until_offline()
host.wait_until_online()
if post_reboot_script:
host.execute(f'/tmp/{Path(post_reboot_script).name}',
sudo=True,
hide_stdout=False,
pty=True)
if not host.enable():
sys.exit(1)
host.wait_for_agent()
host.restart_vms_with_shutdown_policy()
target_host = host
def main(dry_run, zwps_cluster, destination_cluster, virtual_machines,
force_end_hour):
"""Empty ZWPS by migrating VMs and/or it's volumes to the destination cluster."""
click_log.basic_config()
if force_end_hour:
try:
force_end_hour = int(force_end_hour)
except ValueError as e:
logging.error(
f"Specified time:'{force_end_hour}' is not a valid integer due to: '{e}'"
)
sys.exit(1)
if force_end_hour >= 24:
logging.error(f"Specified time:'{force_end_hour}' should be < 24")
sys.exit(1)
profile = 'nl2'
log_to_slack = True
logging.task = 'Live Migrate VM Volumes'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
if not dry_run:
cs = CosmicSQL(server=profile, dry_run=dry_run)
else:
cs = None
zwps_storage_pools = []
for storage_pool in co.get_all_storage_pools():
if zwps_cluster.upper() in storage_pool['name']:
zwps_storage_pools.append(storage_pool)
logging.info('ZWPS storage pools found:')
for zwps_storage_pool in zwps_storage_pools:
logging.info(f" - '{zwps_storage_pool['name']}'")
target_cluster = co.get_cluster(name=destination_cluster)
if not target_cluster:
logging.error(
f"Destination cluster not found:'{target_cluster['name']}'!")
sys.exit(1)
try:
destination_storage_pools = target_cluster.get_storage_pools(
scope='CLUSTER')
except IndexError:
logging.error(
f"No storage pools found for cluster '{target_cluster['name']}'")
sys.exit(1)
logging.info('Destination storage pools found:')
for target_storage_pool in destination_storage_pools:
logging.info(f" - '{target_storage_pool['name']}'")
target_storage_pool = random.choice(destination_storage_pools)
volumes = []
for zwps_storage_pool in zwps_storage_pools:
vols = co.get_all_volumes(list_all=True,
storageid=zwps_storage_pool['id'])
if vols:
volumes += vols
vm_ids = []
logging.info('Volumes found:')
for volume in volumes:
for virtual_machine in virtual_machines:
if re.search(virtual_machine, volume['vmname'], re.IGNORECASE):
logging.info(
f" - '{volume['name']}' on VM '{volume['vmname']}'")
if volume['virtualmachineid'] not in vm_ids:
vm_ids.append(volume['virtualmachineid'])
vms = []
for vm_id in vm_ids:
vm = co.get_vm(id=vm_id)
if vm['affinitygroup']:
for affinitygroup in vm['affinitygroup']:
if 'DedicatedGrp' in affinitygroup['name']:
logging.warning(
f"Skipping VM '{vm['name']}' because of 'DedicatedGrp' affinity group"
)
continue
vms.append(vm)
logging.info('Virtualmachines found:')
for vm in vms:
logging.info(f" - '{vm['name']}'")
logging.info(
f"Starting live migration of volumes and/or virtualmachines from the ZWPS storage pools to storage pool '{target_cluster['name']}'"
)
for vm in vms:
""" Can we start a new migration? """
if force_end_hour:
now = datetime.datetime.now(pytz.timezone('CET'))
if now.hour >= force_end_hour:
logging.info(
f"Stopping migration batch. We are not starting new migrations after '{force_end_hour}':00",
log_to_slack=log_to_slack)
sys.exit(0)
source_host = co.get_host(id=vm['hostid'])
source_cluster = co.get_cluster(zone='nl2',
id=source_host['clusterid'])
if source_cluster['name'] == target_cluster['name']:
""" VM is already on the destination cluster, so we only need to migrate the volumes to this storage pool """
logging.info(
f"Starting live migration of volumes of VM '{vm['name']}' to storage pool '{target_storage_pool['name']}' ({target_storage_pool['id']})",
log_to_slack=log_to_slack)
live_migrate_volumes(target_storage_pool['name'], co, cs, dry_run,
False, log_to_slack, 0, vm['name'], True)
else:
""" VM needs to be migrated live to the destination cluster, including volumes """
live_migrate(co=co,
cs=cs,
cluster=target_cluster['name'],
vm_name=vm['name'],
destination_dc=None,
add_affinity_group=None,
is_project_vm=None,
zwps_to_cwps=True,
log_to_slack=log_to_slack,
dry_run=dry_run)
