def test_webhooks(self, context: dict, cloud: Cloud, farm: Farm, servers: dict, testenv, ssl_verify: bool, webhooks: list, expected_results: list): server = servers.get('F1') params = { "scalr.system.webhooks.scalr_labs_workflow_engine": True, "scalr.system.webhooks.ssl_verify": ssl_verify, "scalr.system.webhooks.retry_interval": 5, "scalr.system.webhooks.use_proxy": False} lib_scalr.update_scalr_config(testenv, params) testenv.restart_service("workflow-engine") testenv.restart_service("zmq_service") node = cloud.get_node(server) node.put_file("/tmp/default", resources('configs/nginx_to_flask_proxy.conf').get().decode("utf-8")) node.put_file("/tmp/prepare_flask.sh", resources('scripts/prepare_flask.sh').get().decode("utf-8")) node.put_file('webhooks.py', resources('scripts/webhooks.py').get().decode("utf-8")) # Put flask script with node.remote_connection() as conn: # Run preparation script conn.run("sudo bash /tmp/prepare_flask.sh") # Run flask in background process conn.run("gunicorn -D -w 1 --bind localhost:5000 webhooks:app") lib_apache.assert_check_http_get_answer(server) lib_webhooks.configure_webhooks(webhooks, server, farm, context) result = lib_node.execute_command(cloud, server, 'szradm --fire-event AccountEvent') assert not result.std_err, "Command szradm --fire-event AccountEvent failed with %s" % result.std_err lib_webhooks.assert_webhooks(context['test_webhooks'], expected_results, server_id=server.id) assert not testenv.check_service_log("workflow-engine", "Traceback"), "Found Traceback in workflow-engine service log!"
def configure_flask(step, serv_as, tls): server = getattr(world, serv_as) node = world.cloud.get_node(server) node.put_file("/tmp/default", resources('configs/nginx_to_flask_proxy.conf').get()) # Put custom nginx config in server # if tls: # node.run("ex -sc '13i| ssl_protocols TLSv1.2;' -cx /tmp/default") node.put_file("/tmp/prepare_flask.sh", resources('scripts/prepare_flask.sh').get()) # Put flask preparation script in server node.put_file('webhooks.py', resources('scripts/webhooks.py').get()) # Put flask script with node.remote_connection() as conn: conn.run("sudo bash /tmp/prepare_flask.sh") # Run preparation script conn.run("gunicorn -D -w 1 --bind localhost:5000 webhooks:app") # Run flask in background process
def prepare_environment(step, env, serv_as): server = getattr(world, serv_as) node = world.cloud.get_node(server) LOG.info('Upload database test script') node.put(path='/root/check_db.py', content=resources('scripts/check_db.py').get()) LOG.info('Launch database test script for update environment') node.run('python /root/check_db.py --db=%s' % env)
def mongodb_wait_data2(node, data): #TODO: rewrite it and use only python! node.put_file(path='/root/mongoslave.js', content=resources('scripts/mongoslave.js').get()) res = node.run('mongo localhost:27018 < /root/mongoslave.js') node.run('rm /root/mongoslave.js') if not str(data['id']) in res[0]: return False return True
def add_test_app(step, serv_as): server = getattr(world, serv_as) cloud = Cloud() node = cloud.node_from_server(server) node.run('mkdir env') cont = resources('scripts/env.rb') node.put(path='/root/env/env.rb', content=cont.get()) out = node.run("/bin/bash -c 'source /usr/local/rvm/scripts/rvm; vmc login --email [email protected] --passwd password; echo Y | vmc push testapp --url %s --mem 64 --path /root/env/'" % world.D1)[0] world.assert_not_in('Starting Application: OK', out, 'Application is not starting: %s' % out)
def assert_check_cert(step, domain_as1, domain_as2=None, serv_as=None): domain1 = getattr(world, domain_as1) domain2 = getattr(world, domain_as2) if domain_as2 else None server = getattr(world, serv_as) if serv_as else None #Get local certs local_cert1 = resources('keys/httpd.crt').get() if domain2 and server: #Get local certs local_cert2 = resources('keys/httpd2.crt').get() #Get remote certs LOG.info('Try get remote certificate for domain: {0}'.format(domain1.name)) remote_cert1 = OpenSSLSNI(domain1.name, 443).certificate LOG.debug('Remote certificate is {0}: '.format(remote_cert1)) LOG.info('Try get remote certificate for domain: {0}'.format(domain2.name)) remote_cert2 = OpenSSLSNI(domain2.name, 443).certificate LOG.debug('Remote certificate is: {0}'.format(remote_cert2)) LOG.info('Try get remote certificate by ip: {0}'.format(server.public_ip)) remote_cert_by_ip = OpenSSLSNI(server.public_ip, 443).certificate LOG.debug('Remote certificate is: {0}'.format(remote_cert_by_ip)) #Assert Certificates world.assert_equal(remote_cert1, remote_cert2, "Domains {0} and {1} are not unique certificates".format(domain1.name, domain2.name)) LOG.info('Domains {0} and {1} are unique certificates'.format(domain1.name, domain2.name)) world.assert_not_equal(remote_cert1, local_cert1, '{0} domain certificate does not match the local certificate'.format(domain1.name)) LOG.info('{0} domain certificate matched the local certificate'.format(domain1.name)) world.assert_not_equal(remote_cert2, local_cert2, '{0} domain certificate does not match the local certificate'.format(domain2.name)) LOG.info('{0} domain certificate matched the local certificate'.format(domain2.name)) world.assert_not_equal(remote_cert1, remote_cert_by_ip, 'Certificate obtained by the ip {0} does not match the certificate domain {1}'.format(server.public_ip, domain1.name)) LOG.info('Certificate obtained by the ip {0} matched the certificate domain {1}'.format(server.public_ip, domain1.name)) else: #Get remote certs cert = ssl.get_server_certificate((domain1.name, 443)) #Assert Certificates world.assert_not_equal(cert, local_cert1, 'Cert not match local cert')
def having_branch_copy(step, branch=None, is_patched=False): # NOTE: migrated if branch == 'system': # Use environ because CONF.feature replace '/' to '-' branch = os.environ.get('RV_BRANCH') elif branch == 'new': branch = world.test_branch_copy elif not branch: # Use environ because CONF.feature replace '/' to '-' branch = os.environ.get('RV_TO_BRANCH') else: branch = branch.strip() world.test_branch_copy = getattr(world, 'test_branch_copy', 'test-{}'.format(int(time.time()))) if is_patched: fixture_path = 'scripts/scalarizr_app.py' script_path = 'src/scalarizr/app.py' content = resources(fixture_path).get() commit_msg = 'Patch app.py, corrupt windows start' else: script_path = 'README.md' commit_msg = 'Tested build for %s at %s ' % (branch, time.strftime('%-H:%M:%S')) content = 'Scalarizr\n=========\n%s' % commit_msg LOG.info('Cloning branch: %s to %s' % (branch, world.test_branch_copy)) git = GH.repos(ORG)(SCALARIZR_REPO).git # Get the SHA the current test branch points to base_sha = git.refs('heads/%s' % branch).get().object.sha # Create a new blob with the content of the file blob = git.blobs.post( content=base64.b64encode(content), encoding='base64') # Fetch the tree this base SHA belongs to base_commit = git.commits(base_sha).get() # Create a new tree object with the new blob, based on the old tree tree = git.trees.post( base_tree=base_commit.tree.sha, tree=[{'path': script_path, 'mode': '100644', 'type': 'blob', 'sha': blob.sha}]) # Create a new commit object using the new tree and point its parent to the current master commit = git.commits.post( message=commit_msg, parents=[base_sha], tree=tree.sha) base_sha = commit.sha LOG.debug('Scalarizr service was patched. GitHub api res: %s' % commit) # Finally update the heads/master reference to point to the new commit try: res = git.refs.post(ref='refs/heads/%s' % world.test_branch_copy, sha=base_sha) LOG.debug('New branch was created. %s' % res) except github.ApiError: res = git.refs('heads/%s' % world.test_branch_copy).patch(sha=base_sha) LOG.debug('New created branch %s was updated.' % res.get('ref')) world.build_commit_sha = base_sha
def having_branch_copy(step, branch=None, is_patched=False): if branch == 'system': # Use environ because CONF.feature replace '/' to '-' branch = os.environ.get('RV_BRANCH') elif branch == 'new': branch = world.test_branch_copy elif not branch: # Use environ because CONF.feature replace '/' to '-' branch = os.environ.get('RV_TO_BRANCH') else: branch = branch.strip() world.test_branch_copy = getattr(world, 'test_branch_copy', 'test-{}'.format(int(time.time()))) if is_patched: fixture_path = 'scripts/scalarizr_app.py' script_path = 'src/scalarizr/app.py' content = resources(fixture_path).get() commit_msg = 'Patch app.py, corrupt windows start' else: script_path = 'README.md' commit_msg = 'Tested build for %s at %s ' % (branch, time.strftime('%-H:%M:%S')) content = 'Scalarizr\n=========\n%s' % commit_msg LOG.info('Cloning branch: %s to %s' % (branch, world.test_branch_copy)) git = GH.repos(ORG)(SCALARIZR_REPO).git # Get the SHA the current test branch points to base_sha = git.refs('heads/%s' % branch).get().object.sha # Create a new blob with the content of the file blob = git.blobs.post( content=base64.b64encode(content), encoding='base64') # Fetch the tree this base SHA belongs to base_commit = git.commits(base_sha).get() # Create a new tree object with the new blob, based on the old tree tree = git.trees.post( base_tree=base_commit.tree.sha, tree=[{'path': script_path, 'mode': '100644', 'type': 'blob', 'sha': blob.sha}]) # Create a new commit object using the new tree and point its parent to the current master commit = git.commits.post( message=commit_msg, parents=[base_sha], tree=tree.sha) base_sha = commit.sha LOG.debug('Scalarizr service was patched. GitHub api res: %s' % commit) # Finally update the heads/master reference to point to the new commit try: res = git.refs.post(ref='refs/heads/%s' % world.test_branch_copy, sha=base_sha) LOG.debug('New branch was created. %s' % res) except github.ApiError: res = git.refs('heads/%s' % world.test_branch_copy).patch(sha=base_sha) LOG.debug('New created branch %s was updated.' % res.get('ref')) world.build_commit_sha = base_sha
def mock_ssmtp(request): if hasattr(request.instance, 'container'): container = request.instance.container else: raise AttributeError("Test instance has no TestEnv associated with it!") ssh = container.get_ssh() ssh.run("rm -f /opt/scalr-server/libexec/mail/ssmtp") ssmtp_script = resources('scripts/ssmtp') container.put_file( ssmtp_script.fp.name, '/opt/scalr-server/libexec/mail/ssmtp') ssh.run('chmod 777 /opt/scalr-server/libexec/mail/ssmtp')
def download_dump(step, serv_as): server = getattr(world, serv_as) node = world.cloud.get_node(server) node.put_file('/tmp/download_backup.py', resources('scripts/download_backup.py').get()) if CONF.main.driver == Platform.EC2: if node.os[0] == 'redhat' and node.os[1].startswith('5'): node.run('python26 /tmp/download_backup.py --platform=ec2 --key=%s --secret=%s --url=%s' % ( world.cloud.config.libcloud.key, world.cloud.config.libcloud.secret, world.last_backup_url )) else: node.run('python /tmp/download_backup.py --platform=ec2 --key=%s --secret=%s --url=%s' % ( world.cloud.config.libcloud.key, world.cloud.config.libcloud.secret, world.last_backup_url ))
def start_basehttpserver(step, port, serv_as): server = getattr(world, serv_as) LOG.info('Run BaseHttpServer in server %s' % server.id) node = world.cloud.get_node(server) LOG.debug('Put base_server.py script') node.put_file('/tmp/base_server.py', resources('scripts/base_server.py').get()) LOG.debug('Run BaseHttpServer script') if node.os[0] in ['ubuntu', 'debian']: node.run('apt-get install screen -y') elif node.os[0] in ['centos', 'redhat', 'oel']: node.run('yum install screen -y') node.run('iptables -A INPUT -p tcp --dport %s -j ACCEPT' % port) node.run('screen -d -m python /tmp/base_server.py %s' % port)
def create_partitions_on_volume(cloud: Cloud, server: Server, mnt_point: str): script_name = 'create_partitions.sh' script_src = resources(Path('scripts', script_name)).get().decode() path = Path('/tmp', script_name) node = cloud.get_node(server) LOG.info('Creating partitions table for volume on %s' % mnt_point) node.put_file(str(path), script_src % mnt_point) out = node.run(f'source {path}') partition_table = out.std_out.strip('\n').splitlines()[-4:] LOG.debug('Created partitions table for volume:\n%s' % '\n'.join(partition_table)) assert all(line.startswith('/dev/') for line in partition_table), \ 'Create volume partitions failed: %s' % out.std_err LOG.info('Partitions table for volume was successfully created')
def download_dump(step, serv_as): #TODO: Add support for gce and openstack if Scalr support server = getattr(world, serv_as) node = world.cloud.get_node(server) node.put_file('/tmp/download_backup.py', resources('scripts/download_backup.py').get()) if CONF.feature.driver.current_cloud == Platform.EC2: interpretator = 'python' check_omnibus = node.run('ls /opt/scalarizr/embedded/bin/python') if not check_omnibus[1].strip(): interpretator = '/opt/scalarizr/embedded/bin/python' out = node.run( '%s /tmp/download_backup.py --platform=ec2 --key=%s --secret=%s --url=%s' % (interpretator, world.cloud.config.libcloud.key, world.cloud.config.libcloud.secret, world.last_backup_url)) if out[2] == '1': raise AssertionError('Backup download failed. Error: %s' % out[1])
def start_basehttpserver(step, port, serv_as): server = getattr(world, serv_as) LOG.info('Run BaseHttpServer in server %s' % server.id) node = world.cloud.get_node(server) with node.remote_connection() as conn: LOG.debug('Put base_server.py script') node.put_file('/tmp/base_server.py', resources('scripts/base_server.py').get()) LOG.debug('Run BaseHttpServer script') if CONF.feature.dist.is_debian: conn.run('apt-get install screen -y') elif CONF.feature.dist.is_centos: conn.run('yum install screen -y') conn.run('iptables -I INPUT 1 -p tcp --dport %s -j ACCEPT' % port) if node.run('which python3').status_code == 0: python_alias = 'python3' else: python_alias = 'python' conn.run('screen -d -m %s /tmp/base_server.py %s' % (python_alias, port))
def create_partitions_on_volume(step, mnt_point, serv_as): # NOTE: migrated script_name = "create_partitions.sh" script_src = resources(os.path.join("scripts", script_name)).get() path = os.path.join('/tmp', script_name) server = getattr(world, serv_as) node = world.cloud.get_node(server) LOG.info('Creating partitions table for volume on %s' % mnt_point) node.put_file(path, script_src % mnt_point) out = node.run('source %s' % path) partition_table = out.std_out.strip("\n").splitlines()[-4:] LOG.debug('Created partitions table for volume:\n%s' % "\n".join(partition_table)) assert all(line.startswith('/dev/') for line in partition_table), \ "Create volume partitions failed: %s" % out.std_err LOG.info('Partitions table for volume was successfully created')
def check_index_page(node, proto, revert, domain_name, name): #TODO: Rewrite this index = resources('html/index_test.php') index = index.get() % {'id': name} url = '%s://%s/' % (proto, domain_name) nodes = node if isinstance(node, (list, tuple)) else [node] for n in nodes: LOG.debug('Upload index page %s to server %s' % (name, n.id)) n.run('mkdir -p /var/www/%s' % name) n.put_file(path='/var/www/%s/index.php' % name, content=index) for i in range(10): LOG.info('Try get index from URL: %s, attempt %s ' % (url, i + 1)) try: resp = requests.get(url, timeout=30, verify=False) break except Exception, e: LOG.warning("Error in openning page '%s': %s" % (url, e)) time.sleep(15)
def start_basehttpserver(step, port, serv_as): server = getattr(world, serv_as) LOG.info('Run BaseHttpServer in server %s' % server.id) node = world.cloud.get_node(server) LOG.debug('Put base_server.py script') node.put_file('/tmp/base_server.py', resources('scripts/base_server.py').get()) LOG.debug('Run BaseHttpServer script') if CONF.feature.dist.is_debian: node.run('apt-get install screen -y') elif CONF.feature.dist.is_centos: node.run('yum install screen -y') node.run('iptables -I INPUT 1 -p tcp --dport %s -j ACCEPT' % port) if node.run('which python3')[2] == 0: python_alias = 'python3' else: python_alias = 'python' node.run('screen -d -m %s /tmp/base_server.py %s' % (python_alias, port))
def check_index_page(node, proto, revert, domain_name, name): index = resources('html/index_test.php') index = index.get() % {'id': name} if proto.isdigit(): url = 'http://%s:%s/' % (domain_name, proto) else: url = '%s://%s/' % (proto, domain_name) nodes = node if isinstance(node, (list, tuple)) else [node] for n in nodes: LOG.debug('Upload index page %s to server %s' % (name, n.public_ip)) n.run('mkdir /var/www/%s' % name) n.put_file(path='/var/www/%s/index.php' % name, content=index) for i in range(5): LOG.info('Try get index from URL: %s, attempt %s ' % (url, i+1)) try: resp = requests.get(url, timeout=30, verify=False) break except Exception, e: LOG.warning("Error in openning page '%s': %s" % (url, e)) time.sleep(10)
def check_index_page(node, proto, revert, domain_name, name): #TODO: Rewrite this index = resources('html/index_test.php') index = index.get() % {'id': name} url = '%s://%s/' % (proto, domain_name) nodes = node if isinstance(node, (list, tuple)) else [node] for n in nodes: LOG.debug('Upload index page %s to server %s' % (name, n.id)) n.run('mkdir -p /var/www/{0} && chmod 777 /var/www/{0}'.format(name)) n.put_file('/var/www/%s/index.php' % name, index) for i in range(10): LOG.info('Try get index from URL: %s, attempt %s ' % (url, i+1)) try: resp = requests.get(url, timeout=30, verify=False) break except Exception as e: LOG.warning("Error in openning page '%s': %s" % (url, e)) time.sleep(15) else: raise AssertionError("Can't get index page: %s" % url) if ('VHost %s added' % name in resp.text) or (revert and resp.status_code == 200): return True raise AssertionError('Index page not valid: %s. Status code: %s' % (resp.text, resp.status_code))
def create_vhost(step, key_name, domain_as, serv_as): domain = getattr(world, domain_as) # Get server server = getattr(world, serv_as) # Set vhost args args = dict( hostname=domain.name, port=443, template=templates['ApacheApi']['ssl-name-based-template'].replace( 'www.secure.example.com', domain.name), ssl=True, ssl_certificate_id=Certificate.get_by_name(key_name).id, reload=True) res = run_api(server, 'create_vhost', args) LOG.info('Add new virtual hosts to domain {0} with key {1}:\n{2}'.format( domain.name, key_name, res)) index = resources('html/index_test.php') index = index.get() % {'id': domain.name} # Get node node = world.cloud.get_node(server) node.run('rm -f /tmp/example/index.html') node.put_file(path='/tmp/example/index.php', content=index) LOG.debug('Uploaded index page %s to server %s' % (domain.name, node.id))