def test_update_history_failure(self):
config_locals = {}
config_globals = {}
execfile(self.config_file, config_globals, config_locals)
prepare_database(config_locals)
with database.session() as session:
history = LogProgressingHistory(
line_matcher_name="start",
progress=0.5,
message="dummy",
severity="INFO",
position=0,
partial_line="",
pathname="dummy",
)
session.add(history)
expected = {"progress": 0.8, "line_matcher_name": "start"}
reader = file_matcher.FileReader("dummy")
reader.update_history(
expected["line_matcher_name"], Progress(progress=expected["progress"], message="", severity="INFO")
)
res = {}
with database.session() as session:
history = session.query(LogProgressingHistory).first()
res.update({"line_matcher_name": history.line_matcher_name, "progress": history.progress})
self.assertNotEqual(expected, res)
def test_clusterHost_delete_subkey(self):
# 1. Try to delete an unqalified subkey of config
url = '/clusterhosts/1/config/gateway'
rv = self.app.delete(url)
self.assertEqual(400, rv.status_code)
# 2. Try to delete a subkey sucessfully
url = 'clusterhosts/1/config/ip'
rv = self.app.delete(url)
self.assertEqual(200, rv.status_code)
expected_config = deepcopy(self.test_config_data)
expected_config['networking']['interfaces']['management']['ip'] = ''
with database.session() as session:
config_db = session.query(ClusterHost.config_data).filter_by(id=1)\
.first()[0]
self.assertDictEqual(expected_config, json.loads(config_db))
# 3. Try to delete a subkey of a config belonged to an immtable host
with database.session() as session:
session.query(ClusterHost).filter_by(id=1)\
.update({'mutable': False})
url = 'clusterhosts/1/config/ip'
rv = self.app.delete(url)
self.assertEqual(400, rv.status_code)
def test_host_installing_progress(self):
# 1. Get progress of a non-existing host
url = '/clusterhosts/1000/progress'
rv = self.app.get(url)
self.assertEqual(404, rv.status_code)
# 2. Get progress of a host without state
url = '/clusterhosts/1/progress'
rv = self.app.get(url)
self.assertEqual(200, rv.status_code)
# 3. Get progress which is in UNINITIALIZED state
with database.session() as session:
host = session.query(ClusterHost).filter_by(id=1).first()
host.state = HostState()
rv = self.app.get(url)
self.assertEqual(200, rv.status_code)
data = json.loads(rv.get_data())
self.assertEqual('UNINITIALIZED', data['progress']['state'])
self.assertEqual(0, data['progress']['percentage'])
# 4. Get progress which is in INSTALLING state
with database.session() as session:
host = session.query(ClusterHost).filter_by(id=1).first()
host.state.state = 'INSTALLING'
session.query(HostState).filter_by(id=1)\
.update({'progress': 0.3,
'message': 'Configuring...',
'severity': 'INFO'})
rv = self.app.get(url)
self.assertEqual(200, rv.status_code)
data = json.loads(rv.get_data())
self.assertEqual('INSTALLING', data['progress']['state'])
self.assertEqual(0.3, data['progress']['percentage'])
def test_cluster_empty_name(self):
with database.session() as session:
session.add(model.Cluster(name=""))
with database.session() as session:
clusters = session.query(model.Cluster).all()
self.assertEqual(len(clusters), 1)
self.assertIsNotNone(clusters[0].name)
self.assertFalse(clusters[0].name == "")
def test_switch_config(self):
with database.session() as session:
session.add(model.SwitchConfig(ip="10.145.88.1", filter_port="123"))
with database.session() as session:
switch_configs = session.query(model.SwitchConfig).all()
self.assertEqual(len(switch_configs), 1)
self.assertEqual(switch_configs[0].ip, "10.145.88.1")
self.assertEqual(switch_configs[0].filter_port, "123")
def test_adapter(self):
with database.session() as session:
session.add(model.Adapter(name="CentOS_openstack", os="CentOS", target_system="Openstack"))
with database.session() as session:
adapters = session.query(model.Adapter).all()
self.assertEqual(len(adapters), 1)
self.assertEqual(adapters[0].name, "CentOS_openstack")
self.assertEqual(adapters[0].os, "CentOS")
self.assertEqual(adapters[0].target_system, "Openstack")
def test_cluster_adapter(self):
with database.session() as session:
adapter = model.Adapter(name="CentOS_openstack", os="CentOS", target_system="openstack")
cluster = model.Cluster(name="cluster1")
cluster.adapter = adapter
session.add(cluster)
with database.session() as session:
adapter = session.query(model.Adapter).first()
self.assertEqual(adapter.clusters.count(), 1)
def test_hostname_empty(self):
with database.session() as session:
cluster = model.Cluster()
cluster.hosts = [model.ClusterHost(hostname="")]
session.add(cluster)
with database.session() as session:
hosts = session.query(model.ClusterHost).all()
self.assertEqual(len(hosts), 1)
self.assertIsNotNone(hosts[0].hostname)
self.assertFalse(hosts[0].hostname == "")
def test_machine_no_switch(self):
with database.session() as session:
session.add(model.Machine(mac="00:00:00:01:02:03", port="123", vlan=100))
with database.session() as session:
machines = session.query(model.Machine).all()
self.assertEqual(len(machines), 1)
self.assertEqual(machines[0].mac, "00:00:00:01:02:03")
self.assertEqual(machines[0].port, "123")
self.assertEqual(machines[0].vlan, 100)
self.assertIsNone(machines[0].switch)
def test_clusterhost_delete_machine(self):
with database.session() as session:
host = model.ClusterHost(hostname="host1")
host.machine = model.Machine(mac="00:00:00:01:02:03", port="123", vlan=100)
session.add(host)
with database.session() as session:
session.query(model.Machine).delete()
with database.session() as session:
host = session.query(model.ClusterHost).first()
self.assertIsNone(host.machine)
def test_clusterhost_delete_cluster(self):
with database.session() as session:
cluster = model.Cluster(name="cluster1")
cluster.hosts = [model.ClusterHost(hostname="host1")]
session.add(cluster)
with database.session() as session:
session.query(model.Cluster).delete()
with database.session() as session:
host = session.query(model.ClusterHost).first()
self.assertIsNone(host.cluster)
def test_del_machine(self):
with database.session() as session:
switch = model.Switch(ip="192.68.1.1")
switch.machines = [model.Machine(mac="00:00:00:01:02:03", port="123", vlan=100)]
session.add(switch)
with database.session() as session:
session.query(model.Machine).delete()
with database.session() as session:
switch = session.query(model.Switch).first()
self.assertEqual(switch.machines.count(), 0)
def test_switch(self):
with database.session() as session:
switch = model.Switch(ip="10.145.88.1")
switch.credential = {"version": "v2c", "community": "public"}
switch.vendor = "huawei"
session.add(switch)
with database.session() as session:
switches = session.query(model.Switch).all()
self.assertEqual(len(switches), 1)
self.assertEqual(switches[0].ip, "10.145.88.1")
self.assertEqual(switches[0].credential, {"version": "v2c", "community": "public"})
self.assertEqual(switches[0].vendor, "huawei")
def test_adapter_del(self):
with database.session() as session:
adapter = model.Adapter(name="CentOS_openstack", os="CentOS", target_system="openstack")
cluster = model.Cluster(name="cluster1")
cluster.adapter = adapter
session.add(cluster)
with database.session() as session:
session.query(model.Adapter).delete()
with database.session() as session:
cluster = session.query(model.Cluster).first()
self.assertIsNone(cluster.adapter)
def test_clusterhost(self):
with database.session() as session:
host = model.ClusterHost(hostname="host1")
host.cluster = model.Cluster(name="cluster1")
host.machine = model.Machine(mac="00:00:00:01:02:03", port="123", vlan=100)
host.machine.switch = model.Switch(ip="192.168.1.1")
session.add(host)
with database.session() as session:
host = session.query(model.ClusterHost).first()
self.assertEqual(host.cluster.name, "cluster1")
self.assertEqual(host.machine.mac, "00:00:00:01:02:03")
self.assertEqual(host.machine.switch.ip, "192.168.1.1")
def test_cluster_del(self):
with database.session() as session:
adapter = model.Adapter(name="CentOS_openstack", os="CentOS", target_system="openstack")
cluster = model.Cluster(name="cluster1")
cluster.adapter = adapter
session.add(cluster)
with database.session() as session:
session.query(model.Cluster).delete()
with database.session() as session:
adapters = session.query(model.Adapter).all()
self.assertEqual(len(adapters), 1)
def test_machine_with_switch(self):
with database.session() as session:
switch = model.Switch(ip="192.168.1.1")
switch.machines.append(model.Machine(mac="00:00:00:01:02:03", port="123", vlan=100))
session.add(switch)
with database.session() as session:
machines = session.query(model.Machine).all()
self.assertEqual(len(machines), 1)
self.assertEqual(machines[0].mac, "00:00:00:01:02:03")
self.assertEqual(machines[0].port, "123")
self.assertEqual(machines[0].vlan, 100)
self.assertIsNotNone(machines[0].switch)
def test_machine_owned_by_one_switch(self):
with database.session() as session:
switch1 = model.Switch(ip="192.168.1.1")
switch2 = model.Switch(ip="192.168.1.2")
machine = model.Machine(mac="00:00:00:01:02:03", port="123", vlan=100)
switch1.machines = [machine]
switch2.machines = [machine]
session.add(switch1)
session.add(switch2)
with database.session() as session:
machine = session.query(model.Machine).first()
self.assertEqual(machine.switch.ip, "192.168.1.2")
def test_del_switch(self):
with database.session() as session:
switch = model.Switch(ip="192.68.1.1")
switch.machines = [model.Machine(mac="00:00:00:01:02:03", port="123", vlan=100)]
session.add(switch)
with database.session() as session:
session.query(model.Switch).delete()
with database.session() as session:
machines = session.query(model.Machine).all()
self.assertEqual(len(machines), 1)
self.assertEqual(machines[0].mac, "00:00:00:01:02:03")
self.assertIsNone(machines[0].switch)
def test_cluster_config_set(self):
with database.session() as session:
cluster = model.Cluster(name="cluster1")
cluster.config = {
"security": {"user": "******", "password": "******"},
"networking": {"interface": "eth0"},
"partition": "/tmp 20%",
}
session.add(cluster)
with database.session() as session:
cluster = session.query(model.Cluster).first()
self.assertEqual(cluster.security, {"user": "******", "password": "******"})
self.assertEqual(cluster.networking, {"interface": "eth0"})
self.assertEqual(cluster.partition, "/tmp 20%")
def test_poll_switch(self, mock_get_vendor, mock_learn):
# Incorrect IP address format
poll_switch.poll_switch("xxx")
with database.session() as session:
machines = session.query(Machine).filter_by(switch_id=1).all()
self.assertEqual([], machines)
# Switch is unreachable
mock_get_vendor.return_value = (None, 'unreachable', 'Timeout')
poll_switch.poll_switch('127.0.0.1')
with database.session() as session:
machines = session.query(Machine).filter_by(switch_id=1).all()
self.assertEqual([], machines)
switch = session.query(Switch).filter_by(id=1).first()
self.assertEqual(switch.state, 'unreachable')
# Successfully retrieve machines from the switch
mock_get_vendor.return_value = ('xxx', 'Found', "")
mock_learn.return_value = [
{'mac': '00:01:02:03:04:05', 'vlan': '1', 'port': '1'},
{'mac': '00:01:02:03:04:06', 'vlan': '1', 'port': '2'},
{'mac': '00:01:02:03:04:07', 'vlan': '2', 'port': '3'},
{'mac': '00:01:02:03:04:08', 'vlan': '2', 'port': '4'},
{'mac': '00:01:02:03:04:09', 'vlan': '3', 'port': '5'}
]
poll_switch.poll_switch('127.0.0.1')
with database.session() as session:
machines = session.query(Machine).filter_by(switch_id=1).all()
self.assertEqual(5, len(machines))
# The state and err_msg of the switch should be reset.
switch = session.query(Switch).filter_by(id=1).first()
self.assertEqual(switch.state, "under_monitoring")
self.assertEqual(switch.err_msg, "")
# Successfully retrieve and filter some machines
# In the following case, machines with port 6, 7 will be filtered.
mock_learn.return_value = [
{'mac': '00:01:02:03:04:10', 'vlan': '3', 'port': '6'},
{'mac': '00:01:02:03:04:0a', 'vlan': '4', 'port': '7'},
{'mac': '00:01:02:03:04:0b', 'vlan': '4', 'port': '8'},
{'mac': '00:01:02:03:04:0c', 'vlan': '5', 'port': '9'},
{'mac': '00:01:02:03:04:0d', 'vlan': '5', 'port': '10'}
]
poll_switch.poll_switch('127.0.0.1')
with database.session() as session:
machines = session.query(Machine).filter_by(switch_id=1).all()
self.assertEqual(8, len(machines))
def setUp(self):
super(TestClusterAPI, self).setUp()
#Prepare testing data
with database.session() as session:
cluster = Cluster(name='cluster_01')
session.add(cluster)
session.flush()
def setUp(self):
super(TestSwtichMachineAPI, self).setUp()
# Create one switch in database
with database.session() as session:
test_switch = Switch(ip=self.SWITCH_IP_ADDRESS1)
test_switch.credential = self.SWITCH_CREDENTIAL
session.add(test_switch)
def list_adapter_roles(adapter_id):
"""
Lists details of all roles of a specific adapter
:param adapter_id: the adaper ID in db
"""
roles_list = []
with database.session() as session:
adapter_q = session.query(Adapter)\
.filter_by(id=adapter_id).first()
if not adapter_q:
error_msg = "Adapter id=%s does not exist!" % adapter_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
roles = session.query(Role, Adapter)\
.filter(Adapter.id == adapter_id,
Adapter.target_system == Role.target_system)\
.all()
for role, adapter in roles:
role_res = {}
role_res['name'] = role.name
role_res['description'] = role.description
roles_list.append(role_res)
return util.make_json_response(200, {"status": "OK", "roles": roles_list})
def get(self, host_id):
"""
Lists progress details of a specific cluster host
:param host_id: cluster host ID in db
"""
progress_result = {}
with database.session() as session:
host = session.query(ModelClusterHost).filter_by(id=host_id)\
.first()
if not host:
error_msg = "The host id=%s does not exist!" % host_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
if not host.state:
progress_result = {
'id': host_id,
'state': 'UNINITIALIZED',
'percentage': 0,
'message': "Waiting..............",
'severity': "INFO",
}
else:
progress_result['id'] = host_id
progress_result['state'] = host.state.state
progress_result['percentage'] = host.state.progress
progress_result['message'] = host.state.message
progress_result['severity'] = host.state.severity
logging.info('progress result for %s: %s', host_id, progress_result)
return util.make_json_response(200, {
"status": "OK",
"progress": progress_result
})
def delete(self, host_id, subkey):
"""
Delete one attribute in configuration of a specific cluster host
:param host_id: the cluster host ID
:param subkey: the attribute name in configuration
"""
available_delete_keys = ['ip', 'role']
with database.session() as session:
host = session.query(ModelClusterHost).filter_by(id=host_id)\
.first()
if not host:
error_msg = "The host id=%s does not exist!" % host_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
if subkey not in available_delete_keys:
error_msg = "subkey %s is not supported!" % subkey
return errors.handle_invalid_usage(
errors.UserInvalidUsage(error_msg))
if not host.mutable:
error_msg = "The host 'id=%s' is not mutable!" % host_id
return errors.handle_invalid_usage(
errors.UserInvalidUsage(error_msg))
config = json.loads(host.config_data)
# Set the subkey's value to ""
util.update_dict_value(subkey, "", config)
host.config = config
return util.make_json_response(200, {"status": "OK"})
def list_adapter(adapter_id):
"""
Lists details of a specific adapter
:param adapter_id: the adapter ID in db
"""
endpoint = '/adapters'
adapter_res = {}
with database.session() as session:
adapter = session.query(Adapter).filter_by(id=adapter_id).first()
if not adapter:
error_msg = "Adapter id=%s does not exist!" % adapter_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
adapter_res['name'] = adapter.name
adapter_res['os'] = adapter.os
adapter_res['id'] = adapter.id
adapter_res['target_system'] = adapter.target_system,
adapter_res['link'] = {
"href": "/".join((endpoint, str(adapter.id))),
"rel": "self"
}
return util.make_json_response(200, {
"status": "OK",
"adapter": adapter_res
})
def _deploy(cluster_id):
"""Deploy the cluster"""
deploy_hosts_urls = []
with database.session() as session:
cluster_hosts_ids = session.query(ModelClusterHost.id)\
.filter_by(cluster_id=cluster_id).all()
if not cluster_hosts_ids:
# No host belongs to this cluster
error_msg = ('Cannot find any host in cluster id=%s' %
cluster_id)
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
for host_id in cluster_hosts_ids:
progress_url = '/cluster_hosts/%s/progress' % str(host_id)
deploy_hosts_urls.append(progress_url)
# Lock cluster hosts and its cluster
session.query(ModelClusterHost).filter_by(cluster_id=cluster_id)\
.update({'mutable': False})
session.query(ModelCluster).filter_by(id=cluster_id)\
.update({'mutable': False})
celery.send_task("compass.tasks.trigger_install", (cluster_id, ))
return util.make_json_response(202, {
"status": "OK",
"deployment": deploy_hosts_urls
})
def _remove_hosts(cluster_id, hosts):
"""Remove existing cluster host from the cluster"""
removed_hosts = []
with database.session() as session:
failed_hosts = []
for host_id in hosts:
host = session.query(ModelClusterHost).filter_by(id=host_id)\
.first()
if not host:
failed_hosts.append(host)
continue
host_res = {"id": host_id, "machine_id": host.machine_id}
removed_hosts.append(host_res)
if failed_hosts:
error_msg = 'Cluster hosts do not exist!'
value = {"failedHosts": failed_hosts}
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg), value)
filter_clause = []
for host_id in hosts:
filter_clause.append('id=%s' % host_id)
# Delete the requested hosts from database
session.query(ModelClusterHost).filter(or_(*filter_clause))\
.delete(synchronize_session='fetch')
return util.make_json_response(200, {
"status": "OK",
"clusterHosts": removed_hosts
})
def get_history(self):
"""Get log file read history from database.
:returns: (line_matcher_name progress)
.. note::
The function should be called out of database session.
It reads the log_progressing_history table to get the
position in the log file it has read in last run,
the partial line of the log, the line matcher name
in the last run, the progress, the message and the
severity it has got in the last run.
"""
with database.session() as session:
history = session.query(LogProgressingHistory).filter_by(
pathname=self.pathname_).first()
if history:
self.position_ = history.position
self.partial_line_ = history.partial_line
line_matcher_name = history.line_matcher_name
progress = Progress(history.progress, history.message,
history.severity)
else:
line_matcher_name = 'start'
progress = Progress(0.0, '', None)
return line_matcher_name, progress
def _reinstall_hosts(clusters):
logging.info('reinstall cluster hosts: %s', clusters)
manager = config_manager.ConfigManager()
with database.session() as session:
for clusterid, hostids in clusters.items():
cluster = session.query(Cluster).filter_by(id=clusterid).first()
if not cluster:
continue
all_hostids = [host.id for host in cluster.hosts]
logging.debug('all hosts in cluster %s is: %s', clusterid,
all_hostids)
logging.info('reinstall hosts %s in cluster %s', hostids,
clusterid)
adapter = cluster.adapter
for hostid in hostids:
host = session.query(ClusterHost).filter_by(id=hostid).first()
if not host:
continue
log_dir = os.path.join(setting.INSTALLATION_LOGDIR,
'%s.%s' % (host.hostname, clusterid))
logging.info('clean log dir %s', log_dir)
shutil.rmtree(log_dir, True)
session.query(LogProgressingHistory).filter(
LogProgressingHistory.pathname.startswith(
'%s/' % log_dir)).delete(synchronize_session='fetch')
logging.info('reinstall host %s', hostid)
manager.reinstall_host(hostid,
os_version=adapter.os,
target_system=adapter.target_system)
if host.state and host.state.state != 'UNINITIALIZED':
session.query(ClusterHost).filter_by(id=hostid).update(
{'mutable': False}, synchronize_session='fetch')
session.query(HostState).filter_by(id=hostid).update(
{
'state': 'INSTALLING',
'progress': 0.0,
'message': '',
'severity': 'INFO'
},
synchronize_session='fetch')
if set(all_hostids) == set(hostids):
logging.info('reinstall cluster %s', clusterid)
if cluster.state and cluster.state != 'UNINITIALIZED':
session.query(Cluster).filter_by(id=clusterid).update(
{'mutable': False}, synchronize_session='fetch')
session.query(ClusterState).filter_by(id=clusterid).update(
{
'state': 'INSTALLING',
'progress': 0.0,
'message': '',
'severity': 'INFO'
},
synchronize_session='fetch')
manager.sync()
def get(self):
"""
Return a list of dashboard links
"""
cluster_id = request.args.get('cluster_id', None)
logging.info('get cluster links with cluster_id=%s', cluster_id)
links = {}
with database.session() as session:
hosts = session.query(ModelClusterHost)\
.filter_by(cluster_id=cluster_id).all()
if not hosts:
error_msg = "Cannot find hosts in cluster id=%s" % cluster_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
for host in hosts:
config = host.config
if ('has_dashboard_roles' in config
and config['has_dashboard_roles']):
ip = config.get('networking',
{}).get('interfaces',
{}).get('management',
{}).get('ip', '')
roles = config.get('roles', [])
for role in roles:
links[role] = 'http://%s' % ip
return util.make_json_response(200, {
"status": "OK",
"dashboardlinks": links
})
def getHistory(self):
'''Get log file read history from database.
Args:
None
Returns:
line_matcher_name: str, line matcher name.
progress: Progress instance record the installing history.
The function should be called out of database session.
It reads the log_progressing_history table to get the
position in the log file it has read in last run,
the partial line of the log, the line matcher name
in the last run, the progress, the message and the
severity it has got in the last run.
'''
with database.session() as session:
history = session.query(LogProgressingHistory).filter_by(
pathname=self.pathname).first()
if history:
logging.log(logging.DEBUG - 1, 'get file %s history %s',
self.pathname, history)
self.position = history.position
self.partial_line = history.partial_line
line_matcher_name = history.line_matcher_name
progress = Progress(history.progress, history.message,
history.severity)
else:
line_matcher_name = 'start'
progress = Progress(0.0, '', None)
return line_matcher_name, progress
def setUp(self):
super(TestSwtichMachineAPI, self).setUp()
# Create one switch in database
with database.session() as session:
test_switch = Switch(ip=self.SWITCH_IP_ADDRESS1)
test_switch.credential = self.SWITCH_CREDENTIAL
session.add(test_switch)
def test_get_machineList(self):
#Prepare testing data
with database.session() as session:
machines = [Machine(mac='00:27:88:0c:01', port='1', vlan='1',
switch_id=1),
Machine(mac='00:27:88:0c:02', port='2', vlan='1',
switch_id=1),
Machine(mac='00:27:88:0c:03', port='3', vlan='1',
switch_id=1),
Machine(mac='00:27:88:0c:04', port='3', vlan='1',
switch_id=2),
Machine(mac='00:27:88:0c:05', port='4', vlan='2',
switch_id=2),
Machine(mac='00:27:88:0c:06', port='5', vlan='3',
switch_id=3)]
session.add_all(machines)
testList = [{'url': '/machines', 'expected': 6},
{'url': '/machines?limit=3', 'expected': 3},
{'url': '/machines?limit=50', 'expected': 6},
{'url': '/machines?switchId=1&vladId=1&port=2',
'expected': 1},
{'url': '/machines?switchId=1&vladId=1&limit=2',
'expected': 2},
{'url': '/machines?switchId=4', 'expected': 0}]
for test in testList:
url = test['url']
expected = test['expected']
rv = self.app.get(url)
data = json.loads(rv.get_data())
count = len(data['machines'])
self.assertEqual(rv.status_code, 200)
self.assertEqual(count, expected)
def setUp(self):
super(TestClusterAPI, self).setUp()
#Prepare testing data
with database.session() as session:
cluster = Cluster(name='cluster_01')
session.add(cluster)
session.flush()
def test_post_switchList(self):
# Test SwitchList POST method
url = '/switches'
# a. post a new switch
data = {'switch': {
'ip': '10.10.10.2',
'credential': self.SWITCH_CREDENTIAL}}
rv = self.app.post(url, data=json.dumps(data))
self.assertEqual(rv.status_code, 202)
with database.session() as session:
switch = session.query(Switch).filter_by(ip='10.10.10.2').first()
self.assertEqual(switch.ip, '10.10.10.2')
# b. Post Conflict switch Ip
rv = self.app.post(url, data=json.dumps(data))
self.assertEqual(rv.status_code, 409)
data = json.loads(rv.get_data())
self.assertEqual("IP address '10.10.10.2' already exists",
data['message'])
self.assertEqual(2, data['failedSwitch'])
# c. Invalid Ip format
data = {'switch': {
'ip': '192.543.1.1',
'credential': self.SWITCH_CREDENTIAL}}
rv = self.app.post(url, data=json.dumps(data))
self.assertEqual(rv.status_code, 400)
def delete(self, host_id, subkey):
"""
Delete one attribute in configuration of a specific cluster host
:param host_id: the cluster host ID
:param subkey: the attribute name in configuration
"""
available_delete_keys = ['ip', 'role']
with database.session() as session:
host = session.query(ModelClusterHost).filter_by(id=host_id)\
.first()
if not host:
error_msg = "The host id=%s does not exist!" % host_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
if subkey not in available_delete_keys:
error_msg = "subkey %s is not supported!" % subkey
return errors.handle_invalid_usage(
errors.UserInvalidUsage(error_msg))
if not host.mutable:
error_msg = "The host 'id=%s' is not mutable!" % host_id
return errors.handle_invalid_usage(
errors.UserInvalidUsage(error_msg))
config = json.loads(host.config_data)
# Set the subkey's value to ""
util.update_dict_value(subkey, "", config)
host.config = config
return util.make_json_response(
200, {"status": "OK"})
def get(self, host_id):
"""
Lists progress details of a specific cluster host
:param host_id: cluster host ID in db
"""
progress_result = {}
with database.session() as session:
host = session.query(ModelClusterHost).filter_by(id=host_id)\
.first()
if not host:
error_msg = "The host id=%s does not exist!" % host_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
if not host.state:
progress_result = {
'id': host_id,
'state': 'UNINITIALIZED',
'percentage': 0,
'message': "Waiting..............",
'severity': "INFO",
}
else:
progress_result['id'] = host_id
progress_result['state'] = host.state.state
progress_result['percentage'] = host.state.progress
progress_result['message'] = host.state.message
progress_result['severity'] = host.state.severity
logging.info('progress result for %s: %s', host_id, progress_result)
return util.make_json_response(
200, {"status": "OK",
"progress": progress_result})
def pollswitches(switch_ips):
"""poll switch."""
poll_switch_ips = []
with database.session():
poll_switch_ips = util.update_switch_ips(switch_ips)
if flags.OPTIONS.async:
for poll_switch_ip in poll_switch_ips:
celery.send_task(
'compass.tasks.pollswitch',
(poll_switch_ip,)
)
else:
try:
pool = Pool(processes=int(flags.OPTIONS.thread_pool_size))
for poll_switch_ip in poll_switch_ips:
pool.apply_async(
poll_switch.poll_switch,
(poll_switch_ip,)
)
pool.close()
pool.join()
except Exception as error:
logging.error('failed to poll switches %s',
poll_switch_ips)
logging.exception(error)
def test_del_switch(self):
with database.session() as session:
switch = model.Switch(ip='192.68.1.1')
switch.machines = [
model.Machine(mac='00:00:00:01:02:03', port='123', vlan=100)
]
session.add(switch)
with database.session() as session:
session.query(model.Switch).delete()
with database.session() as session:
machines = session.query(model.Machine).all()
self.assertEqual(len(machines), 1)
self.assertEqual(machines[0].mac, '00:00:00:01:02:03')
self.assertIsNone(machines[0].switch)
def list_adapter_roles(adapter_id):
"""
Lists details of all roles of a specific adapter
:param adapter_id: the adaper ID in db
"""
roles_list = []
with database.session() as session:
adapter_q = session.query(Adapter)\
.filter_by(id=adapter_id).first()
if not adapter_q:
error_msg = "Adapter id=%s does not exist!" % adapter_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
roles = session.query(Role, Adapter)\
.filter(Adapter.id == adapter_id,
Adapter.target_system == Role.target_system)\
.all()
for role, adapter in roles:
role_res = {}
role_res['name'] = role.name
role_res['description'] = role.description
roles_list.append(role_res)
return util.make_json_response(
200, {"status": "OK",
"roles": roles_list})
def sync_switch_configs():
"""Set switch configs in SwitchConfig table from setting.
.. note::
the switch config is stored in SWITCHES list in setting config.
for each entry in the SWITCHES, its type is dict and must contain
fields 'switch_ips' and 'filter_ports'.
The format of switch_ips is
<ip_blocks>.<ip_blocks>.<ip_blocks>.<ip_blocks>.
ip_blocks consists of ip_block separated by comma.
ip_block can be an integer and a range of integer like xx-xx.
The example of switch_ips is like: xxx.xxx.xxx-yyy,xxx-yyy.xxx,yyy
The format of filter_ports consists of list of
<port_prefix><port_range> separated by comma. port_range can be an
integer or a rnage of integer like xx-xx.
The example of filter_ports is like: ae1-5,20-40.
"""
switch_configs = _get_switch_config()
switch_config_tuples = set([])
with database.session() as session:
session.query(SwitchConfig).delete(synchronize_session='fetch')
for switch_config in switch_configs:
switch_config_tuple = tuple(switch_config.values())
if switch_config_tuple in switch_config_tuples:
logging.debug('ignore adding switch config: %s',
switch_config)
continue
else:
logging.debug('add switch config: %s', switch_config)
switch_config_tuples.add(switch_config_tuple)
session.add(SwitchConfig(**switch_config))
def get(self, switch_id):
"""Lists details of the specified switch.
:param switch_id: switch ID in db
"""
switch_res = {}
with database.session() as session:
switch = session.query(ModelSwitch).filter_by(id=switch_id).first()
logging.info('switch for id %s: %s', switch_id, switch)
if not switch:
error_msg = "Cannot find the switch with id=%s" % switch_id
logging.error("[/switches/{id}]error_msg: %s", error_msg)
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
switch_res['id'] = switch.id
switch_res['ip'] = switch.ip
switch_res['state'] = switch.state
switch_res['link'] = {
'rel': 'self',
'href': '/'.join((self.ENDPOINT, str(switch.id)))
}
logging.info('switch info for %s: %s', switch_id, switch_res)
return util.make_json_response(200, {
"status": "OK",
"switch": switch_res
})
def setUp(self):
super(ClusterHostAPITest, self).setUp()
self.test_config_data = {
"networking": {
"interfaces": {
"management": {
"ip": "192.168.1.1"
}
},
"global": {}
},
"roles": ""
}
# Insert a host into database for testing
with database.session() as session:
clusters_list = [
Cluster(name='cluster_01'),
Cluster(name='cluster_02')
]
session.add_all(clusters_list)
hosts_list = [
ClusterHost(hostname='host_02', cluster_id=1),
ClusterHost(hostname='host_03', cluster_id=1),
ClusterHost(hostname='host_04', cluster_id=2)
]
host = ClusterHost(hostname='host_01', cluster_id=1)
host.config_data = json.dumps(self.test_config_data)
session.add(host)
session.add_all(hosts_list)
def test_get_existing_history(self):
config_locals = {}
config_globals = {}
execfile(self.config_file, config_globals, config_locals)
prepare_database(config_locals)
with database.session() as session:
history = LogProgressingHistory(line_matcher_name='start',
progress=0.5,
message='dummy',
severity='INFO',
position=0,
partial_line='',
pathname='dummy')
session.add(history)
expected = {
'matcher_name': 'start',
'progress': 0.5,
'message': 'dummy',
'severity': 'INFO'
}
res = {}
reader = file_matcher.FileReader('dummy')
history = reader.get_history()
res.update({
'matcher_name': history[0],
'progress': history[1].progress,
'message': history[1].message,
'severity': history[1].severity
})
self.assertEqual(expected, res)
def get(self, cluster_id):
"""Lists progress details of a specific cluster
:param cluster_id: the unique identifier of the cluster
"""
progress_result = {}
with database.session() as session:
cluster = session.query(ModelCluster).filter_by(id=cluster_id)\
.first()
if not cluster:
error_msg = "The cluster id=%s does not exist!" % cluster_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
if not cluster.state:
progress_result = {
'id': cluster_id,
'state': 'UNINITIALIZED',
'percentage': 0,
'message': "Waiting..............",
'severity': "INFO"
}
else:
progress_result['id'] = cluster_id
progress_result['state'] = cluster.state.state
progress_result['percentage'] = cluster.state.progress
progress_result['message'] = cluster.state.message
progress_result['severity'] = cluster.state.severity
logging.info('progress result for cluster %s: %s', cluster_id,
progress_result)
return util.make_json_response(200, {
"status": "OK",
"progress": progress_result
})
def test_switch(self):
with database.session() as session:
switch = model.Switch(ip='10.145.88.1')
switch.credential = {'version': 'v2c', 'community': 'public'}
switch.vendor = 'huawei'
session.add(switch)
with database.session() as session:
switches = session.query(model.Switch).all()
self.assertEqual(len(switches), 1)
self.assertEqual(switches[0].ip, '10.145.88.1')
self.assertEqual(switches[0].credential, {
'version': 'v2c',
'community': 'public'
})
self.assertEqual(switches[0].vendor, 'huawei')
def triggerinstall(clusterid):
"""Deploy the given cluster.
:param clusterid: the id of the cluster to deploy.
:type clusterid: int
"""
with database.session():
trigger_install.trigger_install(clusterid)
def _call():
with database.session() as session:
cluster = model.Cluster(name='cluster1')
cluster.hosts = [
model.ClusterHost(hostname='host1'),
model.ClusterHost(hostname='host1')
]
session.add(cluster)
def put(self, switch_id):
"""Update an existing switch information.
:param switch_id: the unqiue identifier of the switch
"""
switch = None
with database.session() as session:
switch = session.query(ModelSwitch).filter_by(id=switch_id).first()
logging.info('PUT switch id is %s: %s', switch_id, switch)
if not switch:
# No switch is found.
error_msg = 'Cannot update a non-existing switch!'
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
credential = None
logging.debug('PUT a switch request from curl is %s', request.data)
json_data = json.loads(request.data)
credential = json_data['switch']['credential']
logging.info('PUT switch id=%s credential=%s(%s)', switch_id,
credential, type(credential))
ip_addr = None
switch_res = {}
with database.session() as session:
switch = session.query(ModelSwitch).filter_by(id=switch_id).first()
switch.credential = credential
switch.state = "not_reached"
ip_addr = switch.ip
switch_res['id'] = switch.id
switch_res['ip'] = switch.ip
switch_res['state'] = switch.state
link = {
'rel': 'self',
'href': '/'.join((self.ENDPOINT, str(switch.id)))
}
switch_res['link'] = link
celery.send_task("compass.tasks.pollswitch", (ip_addr, ))
return util.make_json_response(202, {
"status": "accepted",
"switch": switch_res
})
def _prepare_database(self, config_locals):
"""Prepare database."""
with database.session() as session:
adapters = {}
for adapter_config in config_locals['ADAPTERS']:
adapter = Adapter(**adapter_config)
session.add(adapter)
adapters[adapter_config['name']] = adapter
roles = {}
for role_config in config_locals['ROLES']:
role = Role(**role_config)
session.add(role)
roles[role_config['name']] = role
switches = {}
for switch_config in config_locals['SWITCHES']:
switch = Switch(**switch_config)
session.add(switch)
switches[switch_config['ip']] = switch
machines = {}
for switch_ip, machine_configs in (
config_locals['MACHINES_BY_SWITCH'].items()
):
for machine_config in machine_configs:
machine = Machine(**machine_config)
machines[machine_config['mac']] = machine
machine.switch = switches[switch_ip]
session.add(machine)
clusters = {}
for cluster_config in config_locals['CLUSTERS']:
adapter_name = cluster_config['adapter']
del cluster_config['adapter']
cluster = Cluster(**cluster_config)
clusters[cluster_config['name']] = cluster
cluster.adapter = adapters[adapter_name]
cluster.state = ClusterState(
state="INSTALLING", progress=0.0, message='')
session.add(cluster)
hosts = {}
for cluster_name, host_configs in (
config_locals['HOSTS_BY_CLUSTER'].items()
):
for host_config in host_configs:
mac = host_config['mac']
del host_config['mac']
host = ClusterHost(**host_config)
hosts['%s.%s' % (
host_config['hostname'], cluster_name)] = host
host.machine = machines[mac]
host.cluster = clusters[cluster_name]
host.state = HostState(
state="INSTALLING", progress=0.0, message='')
session.add(host)
def test_clusterhost_delete_host(self):
with database.session() as session:
cluster = model.Cluster(name='cluster1')
host = model.ClusterHost(hostname='host1')
cluster.hosts = [host]
host.machine = model.Machine(mac='00:00:00:01:02:03',
port='123',
vlan=100)
session.add(cluster)
with database.session() as session:
session.query(model.ClusterHost).delete()
with database.session() as session:
cluster = session.query(model.Cluster).first()
self.assertEqual(cluster.hosts.count(), 0)
machine = session.query(model.Machine).first()
self.assertIsNone(machine.host)
def _replace_all_hosts(cluster_id, hosts):
"""Remove all existing hosts from the cluster and add new ones"""
with database.session() as session:
# Delete all existing hosts of the cluster
session.query(ModelClusterHost)\
.filter_by(cluster_id=cluster_id).delete()
session.flush()
return _add_hosts(cluster_id, hosts)
def update_progress(self, clusterid, hostids):
"""Update cluster progress and hosts progresses.
:param clusterid: the id of the cluster to update.
:type clusterid: int.
:param hostids: the ids of the hosts to update.
:type hostids: list of int.
"""
host_progresses = {}
with database.session():
for hostid in hostids:
fullname, host_state, host_progress = (
self._get_host_progress(hostid))
if not fullname or not host_progress:
logging.error(
'nothing to update host %s => '
'state %s progress %s', fullname, host_state,
host_progress)
continue
logging.debug('got host %s state %s progress %s', fullname,
host_state, host_progress)
host_progresses[hostid] = (fullname, host_state, host_progress)
for hostid, host_value in host_progresses.items():
fullname, host_state, host_progress = host_value
if host_state == 'INSTALLING' and host_progress.progress < 1.0:
self.os_matcher_.update_progress(fullname, host_progress)
self.package_matcher_.update_progress(fullname, host_progress)
else:
logging.error(
'there is no need to update host %s '
'progress: state %s progress %s', fullname, host_state,
host_progress)
with database.session():
for hostid in hostids:
if hostid not in host_progresses:
continue
_, _, host_progress = host_progresses[hostid]
self._update_host_progress(hostid, host_progress)
self._update_cluster_progress(clusterid)
def _call():
with database.session() as session:
session.add(
model.Adapter(name='CentOS_openstack1',
os='CentOS',
target_system='Openstack'))
session.add(
model.Adapter(name='CentOS_openstack2',
os='CentOS',
target_system='Openstack'))
def pollswitch(ip_addr, req_obj='mac', oper="SCAN"):
""" Query switch and return expected result.
:param str ip_addr : switch ip address
:param str reqObj : the object requested to query from switch
:param str oper : the operation to query the switch
(SCAN, GET, SET)
"""
with database.session():
poll_switch.poll_switch(ip_addr, req_obj='mac', oper="SCAN")
def put(self, cluster_id, resource):
"""
Update the resource information of the specified cluster in database
:param cluster_id: the unique identifier of the cluster
:param resource: resource name(security, networking, partition)
"""
resources = {
self.SECURITY: {
'validator': 'is_valid_security_config',
'column': 'security_config'
},
self.NETWORKING: {
'validator': 'is_valid_networking_config',
'column': 'networking_config'
},
self.PARTITION: {
'validator': 'is_valid_partition_config',
'column': 'partition_config'
},
}
request_data = json.loads(request.data)
with database.session() as session:
cluster = session.query(ModelCluster).filter_by(id=cluster_id)\
.first()
if not cluster:
error_msg = 'You are trying to update a non-existing cluster!'
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
if resource not in request_data:
error_msg = "Invalid resource name '%s'" % resource
return errors.handle_invalid_usage(
errors.UserInvalidUsage(error_msg))
value = request_data[resource]
if resource not in resources.keys():
error_msg = "Invalid resource name '%s'" % resource
return errors.handle_invalid_usage(
errors.UserInvalidUsage(error_msg))
validate_func = resources[resource]['validator']
module = globals()['util']
is_valid, msg = getattr(module, validate_func)(value)
if is_valid:
column = resources[resource]['column']
session.query(ModelCluster).filter_by(id=cluster_id)\
.update({column: json.dumps(value)})
else:
return errors.handle_mssing_input(
errors.InputMissingError(msg))
return util.make_json_response(200, {"status": "OK"})
def _call():
with database.session() as session:
machine1 = model.Machine(mac='00:00:00:01:02:03',
port='123',
vlan=100)
machine2 = model.Machine(mac='00:00:00:01:02:03',
port='123',
vlan=100)
switch = model.Switch(ip='192.168.1.1')
switch.machines = [machine1, machine2]
session.add(switch)
def post(self):
"""
Insert switch IP and the credential to db. Invoke a task to poll
switch at the same time.
:param ip: switch IP address
:param credential: a dict for accessing the switch
"""
ip_addr = None
credential = None
logging.debug('post switch request from curl is %s', request.data)
json_data = json.loads(request.data)
ip_addr = json_data['switch']['ip']
credential = json_data['switch']['credential']
logging.info('post switch ip_addr=%s credential=%s(%s)', ip_addr,
credential, type(credential))
if not util.is_valid_ip(ip_addr):
error_msg = "Invalid IP address format!"
return errors.handle_invalid_usage(
errors.UserInvalidUsage(error_msg))
new_switch = {}
with database.session() as session:
switch = session.query(ModelSwitch).filter_by(ip=ip_addr).first()
logging.info('switch for ip %s: %s', ip_addr, switch)
if switch:
error_msg = "IP address '%s' already exists" % ip_addr
value = {'failedSwitch': switch.id}
return errors.handle_duplicate_object(
errors.ObjectDuplicateError(error_msg), value)
switch = ModelSwitch(ip=ip_addr)
switch.credential = credential
session.add(switch)
session.flush()
new_switch['id'] = switch.id
new_switch['ip'] = switch.ip
new_switch['state'] = switch.state
link = {
'rel': 'self',
'href': '/'.join((self.ENDPOINT, str(switch.id)))
}
new_switch['link'] = link
celery.send_task("compass.tasks.pollswitch", (ip_addr, ))
logging.info('new switch added: %s', new_switch)
return util.make_json_response(202, {
"status": "accepted",
"switch": new_switch
})
