def get_mount_table(cloud: Cloud, server: Server) -> tp.Dict[str, str]:
LOG.info('Save mount table from server "%s"' % server.id)
node = cloud.get_node(server)
mount_table = node.run('mount').std_out.splitlines()
mount_table = {x.split()[2]: x.split()[0] for x in mount_table if x}
LOG.debug('Mount table:\n %s' % mount_table)
return mount_table
def test_webhooks(self, context: dict, cloud: Cloud, farm: Farm, servers: dict, testenv, ssl_verify: bool, webhooks: list, expected_results: list):
server = servers.get('F1')
params = {
"scalr.system.webhooks.scalr_labs_workflow_engine": True,
"scalr.system.webhooks.ssl_verify": ssl_verify,
"scalr.system.webhooks.retry_interval": 5,
"scalr.system.webhooks.use_proxy": False}
lib_scalr.update_scalr_config(testenv, params)
testenv.restart_service("workflow-engine")
testenv.restart_service("zmq_service")
node = cloud.get_node(server)
node.put_file("/tmp/default",
resources('configs/nginx_to_flask_proxy.conf').get().decode("utf-8"))
node.put_file("/tmp/prepare_flask.sh",
resources('scripts/prepare_flask.sh').get().decode("utf-8"))
node.put_file('webhooks.py',
resources('scripts/webhooks.py').get().decode("utf-8")) # Put flask script
with node.remote_connection() as conn:
# Run preparation script
conn.run("sudo bash /tmp/prepare_flask.sh")
# Run flask in background process
conn.run("gunicorn -D -w 1 --bind localhost:5000 webhooks:app")
lib_apache.assert_check_http_get_answer(server)
lib_webhooks.configure_webhooks(webhooks, server, farm, context)
result = lib_node.execute_command(cloud, server, 'szradm --fire-event AccountEvent')
assert not result.std_err, "Command szradm --fire-event AccountEvent failed with %s" % result.std_err
lib_webhooks.assert_webhooks(context['test_webhooks'], expected_results, server_id=server.id)
assert not testenv.check_service_log("workflow-engine", "Traceback"), "Found Traceback in workflow-engine service log!"
def check_redis_instances(step, instances_count, instance_type, serv_as):
server = getattr(world, serv_as)
instances = getattr(world, 'redis_instances', {})
count = 0
for instance in instances:
if CONF.feature.platform in ['cloudstack', 'idcf']:
cloud = Cloud()
node = cloud.get_node(server)
ip = filter(lambda x: x.address == server.public_ip, node.driver.ex_list_public_ip())[0]
try:
rule = node.driver.ex_add_port_forwarding_rule(node, ip, 'TCP', instance, instance)
except:
rules = node.driver.ex_list_port_forwarding_rule()
rule = filter(lambda x: x.public_port == instance and x.address == ip, rules)[0]
LOG.info('Rule for open port add: %s %s - %s' % (ip, instance, instance))
info = {}
try:
LOG.debug('Try connect to redis instance: %s:%s:%s' % (server.public_ip, instance, instances[instance]))
r = redis.Redis(host=server.public_ip, port=instance, password=instances[instance], socket_timeout=5)
info = r.info()
except redis.ConnectionError, e:
LOG.error('Connection to redis: %s:%s with password %s is FAILED' % (server.public_ip, instance,
instances[instance]))
raise redis.ConnectionError('Connection to redis: %s:%s with password %s is FAILED' % (server.public_ip, instance,
instances[instance]))
finally:
def action_on_redis(step, action, instance_number, serv_as):
server = getattr(world, serv_as)
instances = getattr(world, 'redis_instances', {})
instance = sorted(instances.items())[int(instance_number)-1]
if CONF.feature.platform in ['cloudstack', 'idcf']:
cloud = Cloud()
node = cloud.get_node(server)
ip = filter(lambda x: x.address == server.public_ip, node.driver.ex_list_public_ip())[0]
try:
rule = node.driver.ex_add_port_forwarding_rule(node, ip, 'TCP', instance[0], instance[0])
except:
rules = node.driver.ex_list_port_forwarding_rule()
rule = filter(lambda x: x.public_port == instance[0] and x.address == ip, rules)[0]
LOG.info('Rule for open port add: %s %s - %s' % (ip, instance[0], instance[0]))
r = redis.Redis(host=server.public_ip, port=instance[0], password=instance[1], socket_timeout=5, db=0)
if action == 'write':
LOG.info('Insert test key to %s:%s' % (server.public_ip, instance[0]))
r.set('test_key', 'test_value')
elif action == 'read':
LOG.info('Read test key from %s:%s' % (server.public_ip, instance[0]))
data = r.get('test_key')
if not data == 'test_value':
LOG.error('Receive bad key value from redis instance: %s:%s' % (server.public_ip, instance[0]))
raise AssertionError('Receive bad key value from redis instance: %s:%s' % (server.public_ip, instance[0]))
if CONF.feature.platform in ['cloudstack', 'idcf']:
node.driver.ex_delete_port_forwarding_rule(node, rule)
LOG.info('Rule for open port was delete')
def assert_at_user_on_server(cloud: Cloud, server: api.Server, expected_user: str):
node = cloud.get_node(server)
cmd = 'net user' if CONF.feature.dist.is_windows else 'cut -d : -f 1 /etc/passwd'
with node.remote_connection() as conn:
user_list = conn.run(cmd).std_out.split()
assert expected_user in user_list, \
f'User {expected_user} was not found on the server! User list output: {user_list}'
def test_checking_config_changes(self, cloud: Cloud, servers: dict):
"""Checking config changes: INTERVAL"""
server = servers['M1']
node = cloud.get_node(server)
interval = 15
provision.change_chef_client_interval_value(node, interval)
provision.assert_chef_client_interval_value(node, interval)
provision.assert_chef_runs_time(node, interval)
def test_chef_deployment_linux(self, context: dict, cloud: Cloud, servers: dict):
"""Verify chef executed fine"""
server = servers['M1']
node = cloud.get_node(server)
context['chef_deployment_time'] = provision.get_chef_bootstrap_stat(node)
lib_node.assert_process_has_options(cloud, server, process='memcached', options='-m 1024')
lib_node.assert_process_has_options(cloud, server, process='chef-client', options='--daemonize')
provision.assert_chef_node_name_equal_hostname(cloud, server)
def check_count_redis_instances(step, instances_count, serv_as):
server = getattr(world, serv_as)
c = Cloud()
node = c.get_node(server)
out = node.run('ps -A | grep redis-server')
if not len(out[0].splitlines()) == int(instances_count):
LOG.error('Redis processes: %s' % out[0])
raise AssertionError('Invalid redis processes count, must be: %s but %s' % (instances_count, len(out[0].splitlines())))
def assert_scalarizr_log_errors(cloud: Cloud, server: Server, log_type: str = None):
"""Check scalarizr log for errors"""
log_type = log_type or 'debug'
node = cloud.get_node(server)
if CONF.feature.dist.is_windows:
validate_scalarizr_log_errors(cloud, node, windows=True, server=server, log_type=log_type)
else:
validate_scalarizr_log_errors(cloud, node, log_type=log_type)
def test_chef_after_resume_linux(self, context: dict, cloud: Cloud, servers: dict, farm: Farm):
"""Verify chef not executed after resume"""
server = servers['M1']
node = cloud.get_node(server)
assert context['chef_deployment_time'] == provision.get_chef_bootstrap_stat(node), \
'Chef was started after resume!'
provision.check_process_status(node, 'memcached', False)
lib_node.assert_process_has_options(cloud, server, process='chef-client', options='--daemonize')
provision.assert_chef_node_name_equal_hostname(cloud, server)
def validate_string_in_file(cloud: Cloud, server: Server, file_path: str, value: str, invert: bool = False):
LOG.info('Verify file "%s" in %s %s "%s"' % (file_path, server.id,
'does not contain' if invert else 'contains',
value))
node = cloud.get_node(server)
out = node.run('cat %s | grep %s' % (file_path, value)).std_out.strip()
assert bool(out) ^ invert, \
'File %s %s: %s. Result of grep: %s' % (file_path,
'contains' if invert else 'does not contain',
value, out)
def assert_szr_terminated_in_log(cloud: Cloud, server: Server):
# TODO: PP > consolidate win/linux methods
node = cloud.get_node(server)
if CONF.feature.ci_repo == 'buildbot':
out = node.run("findstr /c:\"Scalarizr terminated\" \"C:\Program Files\Scalarizr\\var\log\scalarizr_debug.log\"")
else:
out = node.run("findstr /c:\"Scalarizr terminated\" \"C:\opt\scalarizr\\var\log\scalarizr_debug.log\"")
if 'Scalarizr terminated' in out.std_out:
return True
raise AssertionError("Not see 'Scalarizr terminated' in debug log")
def test_chef_solo_bootstrapping(self, context: dict, cloud: Cloud, farm: Farm, role_options: str):
"""Bootstrapping role with chef-solo"""
lib_farm.clear(farm)
farm.terminate()
lib_farm.add_role_to_farm(context, farm, role_options=[role_options])
farm.launch()
server = lib_server.wait_server_status(context, cloud, farm, status=ServerStatus.RUNNING)
node = cloud.get_node(server)
lib_server.assert_file_exist(node, f'C:\{role_options}')
provision.assert_script_data_deleted(cloud, server)
def szradm_execute_command(command: str, cloud: Cloud, server: Server, format_output: bool=True):
command = format_output and f"{command} --format=json" or command
if CONF.feature.dist.id == 'coreos':
command = f'PATH=$PATH:/opt/bin; {command}'
LOG.info(f'Execute the command: {command} a remote host: {server.id}')
node = cloud.get_node(server)
with node.remote_connection() as conn:
result = conn.run(command)
if result.status_code:
raise AssertionError(f"An error has occurred while execute szradm:\n {out.std_err}")
return format_output and json.loads(result.std_out) or result.std_out
def assert_chef_node_name_equal_hostname(cloud: Cloud, server: api.Server):
hostname = lib_server.get_hostname_by_server_format(server)
node = cloud.get_node(server)
command = CONF.feature.dist.is_windows \
and 'findstr node_name c:\\chef\\client.rb' \
or 'cat /etc/chef/client.rb | grep node_name'
with node.remote_connection() as conn:
result = conn.run(command).std_out
node_name = result.strip().split()[1][1:-1]
if not node_name == hostname:
raise AssertionError(f'Chef node_name "{node_name}" != hostname on server "{hostname}"')
def check_processes(step, count, serv_as):
time.sleep(60)
serv = getattr(world, serv_as)
cl = Cloud()
node = cl.get_node(serv)
list_proc = node.run("ps aux | grep scalarizr")[0]
c = 0
for pr in list_proc.splitlines():
if "bin/scalarizr" in pr:
c += 1
LOG.info("Scalarizr count of processes %s" % c)
world.assert_not_equal(c, int(count), "Scalarizr processes is: %s but processes \n%s" % (c, list_proc))
def set_iptables_rule(cloud: Cloud, server: Server, port: tp.Union[int, tuple, list]):
"""Insert iptables rule in the top of the list"""
LOG.info('Insert iptables rule to server %s for opening port %s' % (server, port))
node = cloud.get_node(server)
my_ip = get_external_local_ip()
LOG.info('My IP address: %s' % my_ip)
if isinstance(port, (tuple, list)):
if len(port) == 2:
port = ':'.join(str(x) for x in port)
else:
port = ','.join(str(x) for x in port)
node.run('iptables -I INPUT -p tcp -s %s --dport %s -j ACCEPT' % (my_ip, port))
def agent_restart(cloud: Cloud, server: Server):
# TODO: PP > consolidate win/linux methods
node = cloud.get_node(server)
with node.remote_connection() as conn:
LOG.info('Restart scalarizr via winrm')
LOG.debug('Stop scalarizr')
out = conn.run('net stop Scalarizr')
time.sleep(3)
LOG.debug(out)
out = conn.run('net start Scalarizr')
LOG.debug(out)
time.sleep(15)
def test_bootstrapping_form_chef_role(self, context: dict, cloud: Cloud, farm: Farm):
"""Bootstrapping from chef role"""
lib_farm.clear(farm)
farm.terminate()
lib_farm.add_role_to_farm(context, farm, role_options=['winchef-role'])
farm.launch()
server = lib_server.wait_server_status(context, cloud, farm, status=ServerStatus.RUNNING)
node = cloud.get_node(server)
lib_server.assert_file_exist(node, 'C:\chef_result_file')
lib_server.assert_scalarizr_log_errors(cloud, server)
lifecycle.assert_szr_version_last(server)
lifecycle.assert_hostname(server)
def assert_path_exist(cloud: Cloud, server: Server, path: str):
"""Validate path exist in server"""
LOG.info(f'Verify path {path} exist in server {server.id}')
node = cloud.get_node(server)
with node.remote_connection() as conn:
for attempt in range(5):
out = conn.run('/bin/ls %s' % path)
if out.status_code == 0:
break
time.sleep(15)
else:
LOG.error(f'Path {path} does not exist in server {server.id}')
raise AssertionError(f'Path {path} does not exist in server {server.id}')
def get_config_from_message(cloud: Cloud, server: Server, config_group: str, message: str) -> dict:
node = cloud.get_node(server)
LOG.info(f'Get messages from server {server.id}')
messages = lib_server.get_szr_messages(node)
msg_id = next(filter(lambda x: x['name'] == message, messages))['id']
LOG.info(f'Message id for {message} is {msg_id}')
cmd = f'szradm message-details {msg_id} --json'
if CONF.feature.dist.id == 'coreos':
cmd = "/opt/bin/" + cmd
message_details = json.loads(node.run(cmd).std_out)['body']
LOG.info(f'Message details is {message_details}')
LOG.info(f'Returning message part {config_group}')
return message_details[config_group]
def assert_chef_bootstrap_failed(cloud: Cloud, server: api.Server):
node = cloud.get_node(server)
assertion_msg = "Chef bootstrap marker not found in scalarizr_debug.log"
if CONF.feature.dist.is_windows:
failure_marker = 'chef-client" exited with code 1'
cmd = 'findstr /C:"Command \\"C:\opscode\chef\\bin\chef-client\\" exited with code 1"' \
' "C:\opt\scalarizr\\var\log\scalarizr_debug.log"'
assert failure_marker in node.run(cmd).std_out, assertion_msg
else:
failure_markers = [
'Command "/usr/bin/chef-client" exited with code 1',
'Command /usr/bin/chef-client exited with code 1']
assert any(node.run(f'grep {m} /var/log/scalarizr_debug.log').std_out.strip() for m in failure_markers), \
assertion_msg
def test_bootstrapping_with_chef(self, context: dict, cloud: Cloud, farm: Farm, servers: dict):
"""Bootstrapping with chef"""
lib_farm.add_role_to_farm(context, farm, role_options=['winchef'])
farm.launch()
server = lib_server.wait_server_status(context, cloud, farm, status=ServerStatus.RUNNING)
servers['M1'] = server
lib_server.assert_scalarizr_log_errors(cloud, server)
lifecycle.assert_szr_version_last(server)
provision.assert_node_exists_on_chef_server(server)
orchestration.assert_recipes_in_runlist(server, ['windows_file_create', 'revizorenv', 'revizor_chef_multi'])
node = cloud.get_node(server)
lib_server.assert_file_exist(node, 'C:\chef_result_file')
lib_server.assert_file_exist(node, 'C:\changed_result')
provision.assert_chef_node_name_equal_hostname(cloud, server)
provision.assert_chef_log_contains_text(server, "revizor_chef_variable=REVIZOR_CHEF_VARIABLE_VALUE_WORK")
def create_partitions_on_volume(cloud: Cloud, server: Server, mnt_point: str):
script_name = 'create_partitions.sh'
script_src = resources(Path('scripts', script_name)).get().decode()
path = Path('/tmp', script_name)
node = cloud.get_node(server)
LOG.info('Creating partitions table for volume on %s' % mnt_point)
node.put_file(str(path), script_src % mnt_point)
out = node.run(f'source {path}')
partition_table = out.std_out.strip('\n').splitlines()[-4:]
LOG.debug('Created partitions table for volume:\n%s' % '\n'.join(partition_table))
assert all(line.startswith('/dev/') for line in partition_table), \
'Create volume partitions failed: %s' % out.std_err
LOG.info('Partitions table for volume was successfully created')
def assert_script_data_deleted(cloud: Cloud, server: api.Server):
node = cloud.get_node(server)
server.scriptlogs.reload()
LOG.info('Check script executed data was deleted')
if not server.scriptlogs:
raise AssertionError("No orchestration logs found on %s" % server.id)
task_dir = server.scriptlogs[0].execution_id.replace('-', '')
if CONF.feature.dist.is_windows:
cmd = f'dir c:\\opt\\scalarizr\\var\\lib\\tasks\\{task_dir} /b /s /ad | findstr /e "\\bin \\data"'
else:
cmd = f'find /var/lib/scalarizr/tasks/{task_dir} -type d -regex ".*/\\(bin\\|data\\)"'
with node.remote_connection() as conn:
result = conn.run(cmd)
assert not result.std_err, f"Command: {cmd} execution error:\n {result.std_err}"
folders = [l for l in result.std_out.splitlines() if l.strip()]
assert not folders, f"Find script data {folders} on {server.id}"
def assert_attached_disks_size(cloud: Cloud, server: Server, disks: tp.List[tp.Tuple[str, str, int]]):
node = cloud.get_node(server)
out = node.run('wmic volume get Caption,Capacity,Label').std_out
server_disks = [line.split() for line in out.splitlines() if line.strip()][1:]
for d, label, size in disks:
for disk in server_disks:
if disk[1] == d:
server_size = int(disk[0]) // 1000000000
if server_size != size:
raise AssertionError("Disk %s is of wrong size - %s " % (disk[1], server_size))
if len(disk) > 2 and disk[2] not in label:
raise AssertionError("Disk %s has incorrect or no label '%s'. Should be '%s'." % (
disk[1], disk[2], label))
break
else:
raise AssertionError("Disk not found! All server disks %s" % server_disks)
def assert_errors_in_szr_logs(cloud: Cloud, server: Server):
node = cloud.get_node(server)
out = node.run("findstr /c:\"ERROR\" \"C:\\opt\\scalarizr\\var\\log\\scalarizr_debug.log\"").std_out
errors = []
if 'ERROR' in out:
log = out.splitlines()
for l in log:
try:
d = datetime.strptime(l.split()[0], '%Y-%m-%d')
log_level = l.strip().split()[3]
except ValueError:
continue
now = datetime.now()
if not d.year == now.year or not d.month == now.month or not d.day == now.day or not log_level == 'ERROR':
continue
errors.append(l)
if errors:
raise AssertionError('ERROR in log: %s' % errors)
def wait_unstored_message(cloud: Cloud, servers: tp.Union[Server, tp.List[Server]],
message_name: str, message_type: str = 'out',
find_in_all: bool = False,
timeout: int = 1000):
if not isinstance(servers, (list, tuple)):
servers = [servers]
delivered_to = []
server_messages = {}
message_type = 'in' if message_type.strip() not in ('sends', 'out') else 'out'
start_time = time.time()
while time.time() - start_time < timeout:
if delivered_to == servers:
LOG.info('All servers has message: %s / %s' % (message_type, message_name))
break
for server in servers:
if server in delivered_to:
continue
LOG.info('Searching message "%s/%s" on %s node' % (message_type, message_name, server.id))
node = cloud.get_node(server)
lookup_messages = server_messages.setdefault(server.id, [])
node_messages = reversed(get_szr_messages(node, convert=True))
message = list(filter(lambda m:
m.name == message_name
and m.direction == message_type
and m.id not in lookup_messages
and strtobool(m.handled), node_messages))
if message:
LOG.info('Message found: %s' % message[0].id)
lookup_messages.append(message[0].id)
if find_in_all:
LOG.info('Message %s delivered to the server %s' % (message_name, server.id))
delivered_to.append(server)
continue
return server
time.sleep(30)
else:
raise MessageNotFounded('%s/%s was not finding' % (message_type, message_name))
def assert_mount_point_in_fstab(cloud: Cloud, server: Server, mount_table: tp.Dict[str, str], mount_point: str):
LOG.info('Verify disk from mount point "%s" exist in fstab on server "%s"' %
(mount_point, server.id))
node = cloud.get_node(server)
with node.remote_connection() as conn:
for i in range(3):
fstab = conn.run('cat /etc/fstab').std_out
if not fstab: # FIXME: on openstack this trouble was, fix this
LOG.warning('cat /etc/fstab return nothing')
time.sleep(15)
continue
break
fstab = fstab.splitlines()
fstab = {x.split()[1]: x.split()[0] for x in fstab if x if re.match(r"^[\/\d]", x)}
LOG.debug('Fstab on server "%s" contains:\n %s' % (server.id, fstab))
if mount_point not in mount_table:
raise AssertionError('Mount point "%s" not exist in mount table:\n%s' %
(mount_point, mount_table))
if mount_point not in fstab:
raise AssertionError('Mount point "%s" not exist in fstab:\n%s' %
(mount_point, fstab))
fstab_real_path_disk = fstab[mount_point]
mount_real_path_disk = mount_table[mount_point]
LOG.debug(f'Fstab and mount table state for {mount_point}: {fstab[mount_point]} {mount_table[mount_point]}')
if 'by-uuid' in fstab[mount_point]:
LOG.debug(f'Fstab mount point has by-uuid, get real path for {fstab[mount_point]}')
fstab_real_path_disk = conn.run(f'readlink -f {fstab[mount_point]}').std_out.strip()
LOG.debug(f'Fstab real path for disk: {fstab_real_path_disk}')
elif 'by-uuid' in mount_table[mount_point]:
LOG.debug(f'Mount point has by-uuid, get real path for {fstab[mount_point]}')
mount_real_path_disk = conn.run(f'readlink -f {mount_table[mount_point]}').std_out.strip()
LOG.debug(f'Mount point real path for disk: {mount_real_path_disk}')
assert fstab_real_path_disk == mount_real_path_disk, (
'Disk from mount != disk in fstab: "%s" != "%s"' % (mount_real_path_disk, fstab_real_path_disk))
def install_docker(cloud: Cloud, server: Server) -> docker.APIClient:
node = cloud.get_node(server)
conf_folder = ''
restart_cmd = 'service docker restart'
if CONF.feature.dist.is_systemd:
conf_file = '/etc/systemd/system/docker.service.d/docker.conf'
echo_line = '''"[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H unix:///var/run/docker.sock -H tcp://0.0.0.0:3389"'''
conf_folder = 'mkdir /etc/systemd/system/docker.service.d;'
restart_cmd = 'systemctl daemon-reload; systemctl restart docker'
else:
conf_file = '/etc/default/docker'
echo_line = """'DOCKER_OPTS="-H unix:///var/run/docker.sock -H tcp://0.0.0.0:3389"'"""
command = '''curl -fsSL https://get.docker.com/ | sh; {}\
echo -e {} >> {};\
{}; \
docker pull ubuntu; \
docker pull nginx; \
docker pull alpine'''.format(conf_folder, echo_line, conf_file, restart_cmd)
with node.remote_connection() as conn:
conn.run(command)
conn.run("iptables -I INPUT 1 -p tcp --dport 3389 -j ACCEPT")
conn.run('echo "sleep 1d" >> /home/scalr/{}'.format(SCRIPT))
assert conn.run('docker --version')
return docker.APIClient(base_url='http://%s:3389' % server.public_ip, version='auto')
