def assert_check_objects(step, obj, serv_as):
serv = getattr(world, serv_as)
node = world.cloud.get_node(serv)
password = getattr(world, 'rabbitmq_password')
if obj == 'user':
LOG.info('Check user in rabbitmq')
out = node.run('rabbitmqctl list_users')[0]
world.assert_not_in('scalr', out, 'Not user scalr in list_users: %s' % out)
#if not 'scalr' in out[0]:
# raise AssertionError('Not user guest in list_users: %s' % out[0])
elif obj == 'vhost':
LOG.info('Check vhost in rabbitmq')
out = node.run('rabbitmqctl list_vhosts')[0]
world.assert_not_in('testvhost', out, 'Not vhost testvhost in list_vhosts: %s' % out)
#if not 'testvhost' in out[0]:
# raise AssertionError('Not vhost testvhost in list_vhosts: %s' % out[0])
elif obj == 'queue':
LOG.info('Check queue in rabbitmq')
out = node.run('rabbitmqctl list_queues')[0]
world.assert_not_in('test_queue', out, 'Not queue test_queue in list_queues: %s' % out)
#if not 'test_queue' in out[0]:
# raise AssertionError('Not queue test_queue in list_queues: %s' % out[0])
elif obj == 'message':
LOG.info('Check message in rabbitmq')
credentials = pika.PlainCredentials('scalr', password)
connection = pika.BlockingConnection(pika.ConnectionParameters(credentials=credentials, host=str(serv.public_ip)))
channel = connection.channel()
try:
m = channel.basic_get(queue='test_queue')
LOG.info('Give message in queue "test_queue"')
world.assert_not_equal(m[2], 'super test message', 'Message is not our, I\'m get: %s' % m[2])
#if not m[2] == 'super test message':
# raise AssertionError('Message is not our, I\'m get: %s' % m)
except pika.exceptions.AMQPChannelError:
raise AssertionError('Queue is not work')
def get_storage_id(step, action, db):
if not CONF.feature.storage == 'persistent':
LOG.debug(
'Verify the master storage id is only available with persistent system'
)
return True
get = True if action == 'get' else False
if get:
LOG.info(
'Get Master storage id for db %s before Slave -> Master promotion.'
% db)
storage_id = world.farm.db_info(db)['storage']['id']
if not storage_id:
raise AssertionError(
"Can't get Master storage id for db %s before Slave -> Master promotion."
% db)
setattr(world, 'storage_id', storage_id)
LOG.info(
'Master storage id for db %s before Slave -> Master promotion is %s'
% (db, storage_id))
else:
LOG.info(
'Get new Master storage id for db %s after Slave -> Master promotion.'
% db)
storage_id = world.farm.db_info(db)['storage']['id']
world.assert_not_equal(
world.storage_id, storage_id,
'New Master storage id %s not matched with id %s '
'saved before Slave -> Master promotion.' %
(storage_id, world.storage_id))
LOG.info(
'New Master storage id %s matched with id %s saved before Slave -> Master promotion.'
% (storage_id, world.storage_id))
def assert_node_count(step, node_count, serv_as):
serv = getattr(world, serv_as)
node = world.cloud.get_node(serv)
out = node.run('rabbitmqctl cluster_status')
co = len(re.findall(r'running_nodes,\[(.+)\]}', out[0])[0].split(','))
LOG.info('Nodes in rabbitmq cluster: %s' % co)
world.assert_not_equal(int(node_count), co, 'Node count is failure, in config %s, but must %s' % (co, node_count))
def assert_shard_status(step, serv_count):
serv_count = int(serv_count)
db_role = world.get_role()
world.farm.servers.reload()
# mongod Shard list command
command = {'listShards': 1}
# Set credentials
credentials = {'port': 27017, 'readPreference': 'primary'}
# Get random server from shard
for server in world.farm.servers:
if server.status == 'Running' and server.role_id == db_role.role_id:
if server.cluster_position == '0-0':
server = server
LOG.info('Found server %s with cluster position %s' %
(server.id, server.cluster_position))
break
else:
raise AssertionError('No servers found with index 0-0')
# Run command
res = db_role.db.run_admin_command(server,
command,
credentials=credentials)
LOG.info('Obtained Shards list from: %s\n%s' % (server.id, res))
# Check result
world.assert_not_equal(
serv_count, len(res['shards']),
'Cluster map has not %s shards. Found %s shard' %
(len(res['shards']), serv_count))
LOG.info('Cluster map has %s shards. Checked successfully.' %
len(res['shards']))
def assert_hostname(step, serv_as, hostname):
server = getattr(world, serv_as)
time.sleep(15)
host = world.get_hostname(server)
world.assert_not_equal(
hostname.strip(), host.strip(),
'Hostname in %s is invalid. Must be %s, but %s' %
(serv_as, hostname, host))
def check_processes(step, count, serv_as):
time.sleep(60)
server = getattr(world, serv_as)
node = world.cloud.get_node(server)
list_proc = node.run("pgrep -l scalarizr | awk {print'$1'}")[0]
LOG.info('Scalarizr count of processes %s' % len(list_proc.strip().splitlines()))
world.assert_not_equal(len(list_proc.strip().splitlines()), int(count),
'Scalarizr processes is: %s but processes \n%s' % (len(list_proc.strip().splitlines()), list_proc))
def assert_node_count(step, node_count, serv_as):
serv = getattr(world, serv_as)
node = world.cloud.get_node(serv)
out = node.run('rabbitmqctl cluster_status')
co = len(re.findall(r'running_nodes,\[(.+)\]}', out[0])[0].split(','))
LOG.info('Nodes in rabbitmq cluster: %s' % co)
world.assert_not_equal(
int(node_count), co,
'Node count is failure, in config %s, but must %s' % (co, node_count))
def check_processes(step, count, serv_as):
time.sleep(60)
server = getattr(world, serv_as)
node = world.cloud.get_node(server)
list_proc = node.run("pgrep -l scalarizr | awk {print'$1'}")[0]
LOG.info('Scalarizr count of processes %s' %
len(list_proc.strip().splitlines()))
world.assert_not_equal(
len(list_proc.strip().splitlines()), int(count),
'Scalarizr processes is: %s but processes \n%s' %
(len(list_proc.strip().splitlines()), list_proc))
def check_processes(step, count, serv_as):
time.sleep(60)
serv = getattr(world, serv_as)
cl = Cloud()
node = cl.get_node(serv)
list_proc = node.run("ps aux | grep scalarizr")[0]
c = 0
for pr in list_proc.splitlines():
if "bin/scalarizr" in pr:
c += 1
LOG.info("Scalarizr count of processes %s" % c)
world.assert_not_equal(c, int(count), "Scalarizr processes is: %s but processes \n%s" % (c, list_proc))
def assert_check_objects(step, obj, serv_as):
"""
Verify RabbitMQ object exists
"""
serv = getattr(world, serv_as)
node = world.cloud.get_node(serv)
password = getattr(world, 'rabbitmq_password')
port = 5672
if CONF.feature.driver.current_cloud in [
Platform.IDCF, Platform.CLOUDSTACK
]:
port = world.cloud.open_port(node, port)
if obj == 'user':
LOG.info('Check user in rabbitmq')
out = node.run('rabbitmqctl list_users')[0]
world.assert_not_in('scalr', out,
'Not user scalr in list_users: %s' % out)
#if not 'scalr' in out[0]:
# raise AssertionError('Not user guest in list_users: %s' % out[0])
elif obj == 'vhost':
LOG.info('Check vhost in rabbitmq')
out = node.run('rabbitmqctl list_vhosts')[0]
world.assert_not_in('testvhost', out,
'Not vhost testvhost in list_vhosts: %s' % out)
#if not 'testvhost' in out[0]:
# raise AssertionError('Not vhost testvhost in list_vhosts: %s' % out[0])
elif obj == 'queue':
LOG.info('Check queue in rabbitmq')
out = node.run('rabbitmqctl list_queues')[0]
world.assert_not_in('test_queue', out,
'Not queue test_queue in list_queues: %s' % out)
#if not 'test_queue' in out[0]:
# raise AssertionError('Not queue test_queue in list_queues: %s' % out[0])
elif obj == 'message':
LOG.info('Check message in rabbitmq')
credentials = pika.PlainCredentials('scalr', password)
connection = pika.BlockingConnection(
pika.ConnectionParameters(credentials=credentials,
host=str(serv.public_ip),
port=int(port)))
channel = connection.channel()
try:
m = channel.basic_get(queue='test_queue')
LOG.info('Give message in queue "test_queue"')
world.assert_not_equal(m[2], 'super test message',
'Message is not our, I\'m get: %s' % m[2])
#if not m[2] == 'super test message':
# raise AssertionError('Message is not our, I\'m get: %s' % m)
except pika.exceptions.AMQPChannelError:
raise AssertionError('Queue is not work')
def assert_port(step, port, state, serv_as):
if serv_as == 'all':
world.farm.servers.reload()
servers = [s for s in world.farm.servers if s.status == 'Running']
else:
servers = [getattr(world, serv_as), ]
state = state.strip()
state = False if state == 'not' else True
for serv in servers:
time.sleep(60)
if state:
res = wait_until(world.check_open_port, args=(serv, port.strip()), timeout=600,
error_text="Port %s is not open" % port)
else:
res = world.check_open_port(serv, port.strip())
world.assert_not_equal(state, res, 'Port %s is %s, but must %s in server %s' % (port, res, state, serv.id))
def check_database_in_new_server(step, serv_as, has_not, db_name, username=None):
has_not = has_not and True or False
time.sleep(5)
dbs = db_name.split(',')
if serv_as == 'all':
world.farm.servers.reload()
servers = filter(lambda s: s.status == ServerStatus.RUNNING, world.farm.servers)
else:
servers = [getattr(world, serv_as)]
credentials = (username, world.database_users[username]) if username else None
for server in servers:
for db in dbs:
LOG.info('Check database %s in server %s' % (db, server.id))
world.assert_not_equal(world.db.database_exist(db, server, credentials), not has_not,
(has_not and 'Database %s exist in server %s, but must be erased. All db: %s'
or 'Database %s not exist in server %s, all db: %s')
% (db_name, server.id, world.db.database_list(server)))
def assert_shard_status(step, serv_count):
serv_count = int(serv_count)
world.farm.servers.reload()
server = None
for serv in world.farm.servers:
if serv.status == 'Running' and serv.role_id == world.mongodb_role.role_id:
if serv.cluster_position == '0-0':
server = serv
LOG.info('Found server %s with cluster position %s' % (server.id, serv.cluster_position))
break
world.assert_not_exist(server, 'Not find server with index 0-0')
conn = world.db.get_connection(server, 27017)
LOG.info('Create mongo connection to %s' % server.id)
cur = conn.config.shards.find()
rs_list = set([rs['host'].split('/')[0][-1] for rs in cur])
LOG.info('Get replicaset status and it %s' % rs_list)
world.assert_not_equal(serv_count, len(rs_list), 'Replicaset count is not equal, see %s, but need %s' % (len(rs_list), serv_count))
def get_storage_id(step, action, db):
if not CONF.main.storage == 'persistent':
LOG.debug('Verify the master storage id is only available with persistent system')
return True
get = True if action == 'get' else False
if get:
LOG.info('Get Master storage id for db %s before Slave -> Master promotion.' % db)
storage_id = world.farm.db_info(db)['storage']['id']
if not storage_id:
raise AssertionError("Can't get Master storage id for db %s before Slave -> Master promotion." % db)
setattr(world, 'storage_id', storage_id)
LOG.info('Master storage id for db %s before Slave -> Master promotion is %s' % (db, storage_id))
else:
LOG.info('Get new Master storage id for db %s after Slave -> Master promotion.' % db)
storage_id = world.farm.db_info(db)['storage']['id']
world.assert_not_equal(world.storage_id, storage_id, 'New Master storage id %s not matched with id %s '
'saved before Slave -> Master promotion.' %
(storage_id, world.storage_id))
LOG.info('New Master storage id %s matched with id %s saved before Slave -> Master promotion.' % (storage_id, world.storage_id))
def assert_server_ratio(step, hdd_count, ram_count, serv_as):
serv = getattr(world, serv_as)
node = world.cloud.get_node(serv)
out = node.run('rabbitmqctl cluster_status')
disks = re.findall(r'disc,\[(.+)\]},', out[0])[0]
disks = re.findall("'((?:[a-z0-9@-]+)\@(?:[a-z0-9@-]+))+'", disks)
LOG.info('Disks nodes in rabbitmq cluster: %s' % disks)
rams = re.findall(r"{ram,\[(.+)\]}]},", out[0])
if not rams:
raise AssertionError(
'RAM nodes in rabbitmq is unavailable. All nodes: %s' % out[0])
rams = re.findall(r"'((?:[a-z0-9@-]+)\@(?:[a-z0-9@-]+))+'", rams[0])
LOG.info('RAMs nodes in rabbitmq cluster: %s' % rams)
runnings = re.findall(r'running_nodes,\[(.+)\]}', out[0])[0]
runnings = re.findall("'((?:[a-z0-9@-]+)\@(?:[a-z0-9@-]+))+'", runnings)
LOG.info('Running nodes in rabbitmq cluster: %s' % runnings)
all_count = len(disks) + len(rams)
world.assert_not_equal(
all_count, len(runnings),
"Count of runnings server in cluster is different, running %s in cluster but must: %s"
% (len(runnings), all_count))
world.assert_not_equal(
int(hdd_count), len(disks),
"Count of disk nodes is failure, disks %s must %s" %
(len(disks), hdd_count))
world.assert_not_equal(
int(ram_count), int(ram_count),
"Count of ram nodes is failure, disks %s must %s" %
(len(rams), hdd_count))
def check_database_in_new_server(step,
serv_as,
has_not,
db_name,
username=None):
has_not = has_not and True or False
time.sleep(5)
db_role = world.get_role()
dbs = db_name.split(',')
if serv_as == 'all':
world.farm.servers.reload()
servers = filter(lambda s: s.status == ServerStatus.RUNNING,
world.farm.servers)
else:
servers = [getattr(world, serv_as)]
credentials = (username,
db_role.db.credentials[username]) if username else None
if CONF.feature.driver.cloud_family == Platform.CLOUDSTACK:
for i in range(5):
try:
db_role.db.get_connection()
break
except DataBaseError:
time.sleep(5)
else:
raise
for server in servers:
for db in dbs:
LOG.info('Check database %s in server %s' % (db, server.id))
check = wait_until(target=db_role.db.database_exist,
args=(str(db), server, credentials),
timeout=60,
return_bool=True)
world.assert_not_equal(check, not has_not, (
has_not and
'Database %s exist in server %s, but must be erased. All db: %s'
or 'Database %s not exist in server %s, all db: %s') %
(db_name, server.id,
db_role.db.database_list(server)))
def assert_port(step, port, state, serv_as):
if serv_as == 'all':
world.farm.servers.reload()
servers = [s for s in world.farm.servers if s.status == 'Running']
else:
servers = [
getattr(world, serv_as),
]
state = state.strip()
state = False if state == 'not' else True
for serv in servers:
time.sleep(60)
if state:
res = wait_until(world.check_open_port,
args=(serv, port.strip()),
timeout=600,
error_text="Port %s is not open" % port)
else:
res = world.check_open_port(serv, port.strip())
world.assert_not_equal(
state, res, 'Port %s is %s, but must %s in server %s' %
(port, res, state, serv.id))
def assert_shard_status(step, serv_count):
serv_count = int(serv_count)
db_role = world.get_role()
world.farm.servers.reload()
# mongod Shard list command
command = {'listShards': 1}
# Set credentials
credentials = {'port': 27017, 'readPreference': 'primary'}
# Get random server from shard
for server in world.farm.servers:
if server.status == 'Running' and server.role_id == db_role.role_id:
if server.cluster_position == '0-0':
server = server
LOG.info('Found server %s with cluster position %s' % (server.id, server.cluster_position))
break
else:
raise AssertionError('No servers found with index 0-0')
# Run command
res = db_role.db.run_admin_command(server, command, credentials=credentials)
LOG.info('Obtained Shards list from: %s\n%s' % (server.id, res))
# Check result
world.assert_not_equal(serv_count, len(res['shards']), 'Cluster map has not %s shards. Found %s shard' % (len(res['shards']), serv_count))
LOG.info('Cluster map has %s shards. Checked successfully.' % len(res['shards']))
def assert_check_cert(step, domain_as1, domain_as2=None, serv_as=None):
domain1 = getattr(world, domain_as1)
domain2 = getattr(world, domain_as2) if domain_as2 else None
server = getattr(world, serv_as) if serv_as else None
#Get local certs
local_cert1 = resources('keys/httpd.crt').get()
if domain2 and server:
#Get local certs
local_cert2 = resources('keys/httpd2.crt').get()
#Get remote certs
LOG.info('Try get remote certificate for domain: {0}'.format(domain1.name))
remote_cert1 = OpenSSLSNI(domain1.name, 443).certificate
LOG.debug('Remote certificate is {0}: '.format(remote_cert1))
LOG.info('Try get remote certificate for domain: {0}'.format(domain2.name))
remote_cert2 = OpenSSLSNI(domain2.name, 443).certificate
LOG.debug('Remote certificate is: {0}'.format(remote_cert2))
LOG.info('Try get remote certificate by ip: {0}'.format(server.public_ip))
remote_cert_by_ip = OpenSSLSNI(server.public_ip, 443).certificate
LOG.debug('Remote certificate is: {0}'.format(remote_cert_by_ip))
#Assert Certificates
world.assert_equal(remote_cert1, remote_cert2, "Domains {0} and {1} are not unique certificates".format(domain1.name, domain2.name))
LOG.info('Domains {0} and {1} are unique certificates'.format(domain1.name, domain2.name))
world.assert_not_equal(remote_cert1, local_cert1, '{0} domain certificate does not match the local certificate'.format(domain1.name))
LOG.info('{0} domain certificate matched the local certificate'.format(domain1.name))
world.assert_not_equal(remote_cert2, local_cert2, '{0} domain certificate does not match the local certificate'.format(domain2.name))
LOG.info('{0} domain certificate matched the local certificate'.format(domain2.name))
world.assert_not_equal(remote_cert1, remote_cert_by_ip, 'Certificate obtained by the ip {0} does not match the certificate domain {1}'.format(server.public_ip, domain1.name))
LOG.info('Certificate obtained by the ip {0} matched the certificate domain {1}'.format(server.public_ip, domain1.name))
else:
#Get remote certs
cert = ssl.get_server_certificate((domain1.name, 443))
#Assert Certificates
world.assert_not_equal(cert, local_cert1, 'Cert not match local cert')
def assert_server_ratio(step, hdd_count, ram_count, serv_as):
serv = getattr(world, serv_as)
node = world.cloud.get_node(serv)
out = node.run('rabbitmqctl cluster_status')
disks = re.findall(r'disc,\[(.+)\]},', out[0])[0]
disks = re.findall("'((?:[a-z0-9@-]+)\@(?:[a-z0-9@-]+))+'", disks)
LOG.info('Disks nodes in rabbitmq cluster: %s' % disks)
rams = re.findall(r"{ram,\[(.+)\]}]},", out[0])
if not rams:
raise AssertionError('RAM nodes in rabbitmq is unavailable. All nodes: %s' % out[0])
rams = re.findall(r"'((?:[a-z0-9@-]+)\@(?:[a-z0-9@-]+))+'", rams[0])
LOG.info('RAMs nodes in rabbitmq cluster: %s' % rams)
runnings = re.findall(r'running_nodes,\[(.+)\]}', out[0])[0]
runnings = re.findall("'((?:[a-z0-9@-]+)\@(?:[a-z0-9@-]+))+'", runnings)
LOG.info('Running nodes in rabbitmq cluster: %s' % runnings)
all_count = len(disks) + len(rams)
world.assert_not_equal(all_count, len(runnings),
"Count of runnings server in cluster is different, running %s in cluster but must: %s" % (len(runnings), all_count))
world.assert_not_equal(int(hdd_count), len(disks),
"Count of disk nodes is failure, disks %s must %s" % (len(disks), hdd_count))
world.assert_not_equal(int(ram_count), int(ram_count),
"Count of ram nodes is failure, disks %s must %s" % (len(rams), hdd_count))
def having_server(step, state, serv_as):
server = getattr(world, serv_as)
world.assert_not_equal(server.status, ServerStatus.from_code(state),
"Server %s is not in state %s" % (server.id, state))
def having_server(step, state, serv_as):
server = getattr(world, serv_as)
world.assert_not_equal(server.status, ServerStatus.from_code(state), "Server %s is not in state %s" % (server.id, state))
def assert_check_resolv(step, vhost_name, serv_as, timeout=1200):
domain = getattr(world, vhost_name)
serv = getattr(world, serv_as)
domain_ip = wait_until(world.check_resolving, args=(domain,), timeout=timeout, error_text="Not see domain resolve")
world.assert_not_equal(domain_ip, serv.public_ip, 'Domain IP (%s) != server IP (%s)' % (domain_ip, serv.public_ip))
def assert_hostname(step, serv_as, hostname):
serv = getattr(world, serv_as)
time.sleep(15)
host = world.get_hostname(serv)
world.assert_not_equal(hostname.strip(), host.strip(), 'Hostname in %s is invalid. Must be %s, but %s' % (serv_as, hostname, host))
def check_deploy_status(step):
time.sleep(30)
LOG.info("Check task status")
LOG.debug("All tasks %s" % IMPL.deploy.tasks_list())
task = filter(lambda x: x["id"] == world.task_id, IMPL.deploy.tasks_list())[0]
world.assert_not_equal(task["status"], "deployed", "Task not deployed, status: %s" % task["status"])
