def test_event_hub(self, ssh_con, test_vars): # noqa: F811 log = logging.getLogger("test_event_hub") atd = test_vars["atd_obj"] atd.template = requests.get( url= 'https://raw.githubusercontent.com/Azure/azure-quickstart-templates/master/301-eventHub-create-authrule-namespace-and-eventHub/azuredeploy.json' ).json() eh_name = "eh-" + atd.deploy_id atd.deploy_params = { "namespaceName": eh_name + "-ns", "namespaceAuthorizationRuleName": eh_name + "-nsar", "eventHubName": eh_name, "eventhubAuthorizationRuleName": eh_name + "-ehar", "eventhubAuthorizationRuleName1": eh_name + "-ehar1", "consumerGroupName": eh_name + "-cgn", } atd.deploy_name = "test_event_hub" log.debug('Generated deploy parameters: \n{}'.format( json.dumps(atd.deploy_params, indent=4))) deploy_result = wait_for_op(atd.deploy()) test_vars["deploy_eh_outputs"] = deploy_result.properties.outputs log.debug(test_vars["deploy_eh_outputs"]) policy_primary_key = test_vars["deploy_eh_outputs"][ "eventHubSharedAccessPolicyPrimaryKey"]["value"] log.debug("policy_primary_key = {}".format(policy_primary_key)) commands = """ export AZURE_EVENTHUB_SENDERKEYNAME="RootManageSharedAccessKey" export AZURE_EVENTHUB_SENDERKEY={0} export AZURE_EVENTHUB_NAMESPACENAME="edasimeventhub2" """.format(policy_primary_key).split("\n") run_ssh_commands(ssh_con, commands) test_vars[ "cmd2"] = "AZURE_EVENTHUB_SENDERKEYNAME=\"RootManageSharedAccessKey\" AZURE_EVENTHUB_SENDERKEY=\"{}\" AZURE_EVENTHUB_NAMESPACENAME=\"edasimeventhub2\"".format( policy_primary_key)
def test_vdbench_run(self, test_vars): # noqa: F811 log = logging.getLogger("test_vdbench_run") node_ip = test_vars["deploy_vd_outputs"]["node_0_ip_address"]["value"] with SSHTunnelForwarder( test_vars["public_ip"], ssh_username=test_vars["controller_user"], ssh_pkey=test_vars["ssh_priv_key"], remote_bind_address=(node_ip, 22), ) as ssh_tunnel: sleep(1) try: ssh_client = create_ssh_client( test_vars["controller_user"], "127.0.0.1", ssh_tunnel.local_bind_port, key_filename=test_vars["ssh_priv_key"]) scp_client = SCPClient(ssh_client.get_transport()) try: scp_client.put(test_vars["ssh_priv_key"], r"~/.ssh/id_rsa") finally: scp_client.close() commands = """ ~/copy_idrsa.sh cd ./run_vdbench.sh inmem32node3.conf uniquestring1 """.split("\n") run_ssh_commands(ssh_client, commands) finally: ssh_client.close()
def mnt_nodes(ssh_con, test_vars): if ("storage_account" not in test_vars) or (not test_vars["storage_account"]): return check = run_ssh_command(ssh_con, "ls ~/STATUS.NODES_MOUNTED", ignore_nonzero_rc=True, timeout=30) if check['rc']: # nodes were not already mounted # Update needed packages. commands = ["sudo apt-get update", "sudo apt-get install nfs-common"] run_ssh_commands(ssh_con, commands, timeout=600) # Set up mount points and /etc/fstab. commands = [] for i, vs_ip in enumerate(test_vars["cluster_vs_ips"]): commands.append("sudo mkdir -p /nfs/node{}".format(i)) commands.append("sudo chown nobody:nogroup /nfs/node{}".format(i)) fstab_line = "{}:/msazure /nfs/node{} nfs ".format(vs_ip, i) + \ "hard,nointr,proto=tcp,mountproto=tcp,retry=30 0 0" commands.append( "sudo sh -c 'echo \"{}\" >> /etc/fstab'".format(fstab_line)) run_ssh_commands(ssh_con, commands, timeout=30) # Mount the nodes. run_ssh_command(ssh_con, "sudo mount -a", timeout=300) run_ssh_command(ssh_con, "touch ~/STATUS.NODES_MOUNTED", timeout=30)
def test_download_go(self, ssh_con): # noqa: F811 commands = """ sudo apt -y install golang-go mkdir -p ~/gopath echo "export GOPATH=$HOME/gopath" >> ~/.profile echo "export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin" >> ~/.profile source ~/.profile && cd $GOPATH && go get -v github.com/Azure/Avere/src/go/... """.split("\n") run_ssh_commands(ssh_con, commands)
def test_vdbench_setup(self, mnt_nodes, ssh_con): # noqa: F811 log = logging.getLogger("test_vdbench_setup") commands = """ sudo mkdir -p /nfs/node0/bootstrap cd /nfs/node0/bootstrap sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/bootstrap.vdbench.sh https://raw.githubusercontent.com/Azure/Avere/master/src/clientapps/vdbench/bootstrap.vdbench.sh sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/vdbench50407.zip https://avereimageswestus.blob.core.windows.net/vdbench/vdbench50407.zip sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/vdbenchVerify.sh https://raw.githubusercontent.com/Azure/Avere/master/src/clientapps/vdbench/vdbenchVerify.sh sudo chmod +x /nfs/node0/bootstrap/vdbenchVerify.sh /nfs/node0/bootstrap/vdbenchVerify.sh """.split("\n") run_ssh_commands(ssh_con, commands)
def test_vdbench_setup(self, mnt_nodes, ssh_con): # noqa: F811 log = logging.getLogger("test_vdbench_setup") vdbench_url = os.environ.get("VDBENCH_URL", "http://localhost") commands = """ sudo mkdir -p /nfs/node0/bootstrap cd /nfs/node0/bootstrap sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/bootstrap.vdbench.sh https://raw.githubusercontent.com/Azure/Avere/main/src/clientapps/vdbench/bootstrap.vdbench.sh sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/vdbench50407.zip '{0}' sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/vdbenchVerify.sh https://raw.githubusercontent.com/Azure/Avere/main/src/clientapps/vdbench/vdbenchVerify.sh sudo chmod +x /nfs/node0/bootstrap/vdbenchVerify.sh /nfs/node0/bootstrap/vdbenchVerify.sh """.format(vdbench_url).split("\n") run_ssh_commands(ssh_con, commands)
def test_storage_account(self, resource_group, ssh_con, test_vars): # noqa: F811, E501 log = logging.getLogger("test_storage_account") atd = test_vars["atd_obj"] storage_account = test_vars["storage_account"] storage_keys = atd.st_client.storage_accounts.list_keys( resource_group.name, storage_account) storage_keys = {v.key_name: v.value for v in storage_keys.keys} key = storage_keys['key1'] log.debug("storage_account = {}".format(storage_account)) log.debug("key = {}".format(key)) commands = """ export AZURE_STORAGE_ACCOUNT= {0} export AZURE_STORAGE_ACCOUNT_KEY={1} """.format(storage_account, key).split("\n") run_ssh_commands(ssh_con, commands) test_vars[ "cmd1"] = "AZURE_STORAGE_ACCOUNT=\"{}\" AZURE_STORAGE_ACCOUNT_KEY=\"{}\" ".format( storage_account, key)
def test_basic_fileops(self, mnt_nodes, scp_con, ssh_con, test_vars): # noqa: E501, F811 """ Quick check of file operations. See check_node_basic_fileops.sh for more information. """ if ("storage_account" not in test_vars) or (not test_vars["storage_account"]): pytest.skip("no storage account") script_name = "check_node_basic_fileops.sh" scp_con.put( "{0}/test/{1}".format(test_vars["build_root"], script_name), r"~/.", ) commands = """ chmod +x {0} ./{0} """.format(script_name).split("\n") run_ssh_commands(ssh_con, commands)
def test_edasim_setup(self, mnt_nodes, ssh_con): # noqa: F811 commands = """ sudo mkdir -p /nfs/node0/bootstrap sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/bootstrap.jobsubmitter.sh https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/bootstrap.jobsubmitter.sh sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/bootstrap.orchestrator.sh https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/bootstrap.orchestrator.sh sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/bootstrap.onpremjobuploader.sh https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/bootstrap.onpremjobuploader.sh sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/bootstrap.worker.sh https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/bootstrap.worker.sh sudo mkdir -p /nfs/node0/bootstrap/edasim source ~/.profile && sudo cp $GOPATH/bin/* /nfs/node0/bootstrap/edasim sudo mkdir -p /nfs/node0/bootstrap/rsyslog sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/rsyslog/33-jobsubmitter.conf https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/rsyslog/33-jobsubmitter.conf sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/rsyslog/30-orchestrator.conf https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/rsyslog/30-orchestrator.conf sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/rsyslog/31-worker.conf https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/rsyslog/31-worker.conf sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/rsyslog/32-onpremjobuploader.conf https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/rsyslog/32-onpremjobuploader.conf sudo mkdir -p /nfs/node0/bootstrap/systemd sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/systemd/jobsubmitter.service https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/systemd/jobsubmitter.service sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/systemd/onpremjobuploader.service https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/systemd/onpremjobuploader.service sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/systemd/orchestrator.service https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/systemd/orchestrator.service sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/systemd/worker.service https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/edasim/deploymentartifacts/bootstrap/systemd/worker.service """.split("\n") run_ssh_commands(ssh_con, commands)
def test_client_docker_run(self, test_vars): # noqa: F811 log = logging.getLogger("test_client_docker_run") node_ip = test_vars["deploy_client_docker_outputs"]["node_0_ip_address"]["value"] atd = test_vars["atd_obj"] cluster_mgmt_ip = test_vars["cluster_mgmt_ip"] with SSHTunnelForwarder( test_vars["public_ip"], ssh_username=test_vars["controller_user"], ssh_pkey=test_vars["ssh_priv_key"], remote_bind_address=(node_ip, 22), ) as ssh_tunnel: sleep(1) try: ssh_client = create_ssh_client( test_vars["controller_user"], "127.0.0.1", ssh_tunnel.local_bind_port, key_filename=test_vars["ssh_priv_key"] ) scp_client = SCPClient(ssh_client.get_transport()) try: scp_client.put(test_vars["ssh_priv_key"], r"~/.ssh/id_rsa") finally: scp_client.close() commands = """ cd curl -fsSL https://get.docker.com -o get-docker.sh sudo sh get-docker.sh sudo docker login https://{0} -u {1} -p {2} sudo docker pull {0}/test echo "export STORAGEACT='{3}'" >> ~/.bashrc echo "export MGMIP='{4}'" >> ~/.bashrc echo "export SA_KEY='{5}'" >> ~/.bashrc echo "export CLUSTER_MGMT_IP='{6}'" >> ~/.bashrc echo "export ADMIN_PW='{7}'" >> ~/.bashrc """.format(os.environ["dockerRegistry"], os.environ["dockerUsername"], os.environ["dockerPassword"], atd.deploy_id + "sa", test_vars["public_ip"], os.environ["SA_KEY"], cluster_mgmt_ip, os.environ["AVERE_ADMIN_PW"]).split("\n") run_ssh_commands(ssh_client, commands) finally: ssh_client.close()
def mnt_nodes(ssh_con, test_vars): if ("storage_account" not in test_vars) or (not test_vars["storage_account"]): return check = run_ssh_command(ssh_con, "ls ~/STATUS.NODES_MOUNTED", ignore_nonzero_rc=True) if check['rc']: # nodes were not already mounted commands = """ sudo apt-get update sudo apt-get install nfs-common """.split("\n") for i, vs_ip in enumerate(test_vars["cluster_vs_ips"]): commands.append("sudo mkdir -p /nfs/node{}".format(i)) commands.append("sudo chown nobody:nogroup /nfs/node{}".format(i)) fstab_line = "{}:/msazure /nfs/node{} nfs ".format(vs_ip, i) + \ "hard,nointr,proto=tcp,mountproto=tcp,retry=30 0 0" commands.append( "sudo sh -c 'echo \"{}\" >> /etc/fstab'".format(fstab_line)) commands.append("sudo mount -a") commands.append("touch ~/STATUS.NODES_MOUNTED") run_ssh_commands(ssh_con, commands)
def test_ping_vservers(self, ssh_con, test_vars): # noqa: F811 """Ping all of the vserver IPs from the controller.""" commands = [] for vs_ip in test_vars["cluster_vs_ips"]: commands.append("ping -c 3 {}".format(vs_ip)) run_ssh_commands(ssh_con, commands)
def mnt_nodes(ssh_con, test_vars): if ("storage_account" not in test_vars) or (not test_vars["storage_account"]): return log = logging.getLogger("mnt_nodes") check = run_ssh_command(ssh_con, "ls ~/STATUS.NODES_MOUNTED", ignore_nonzero_rc=True, timeout=30) if check['rc']: # nodes were not already mounted # Update needed packages. commands = ["sudo apt-get update", "sudo apt-get install nfs-common"] run_ssh_commands(ssh_con, commands, timeout=600) # Set up mount points and /etc/fstab. commands = [] for i, vs_ip in enumerate(test_vars["cluster_vs_ips"]): commands.append("sudo mkdir -p /nfs/node{}".format(i)) commands.append("sudo chown nobody:nogroup /nfs/node{}".format(i)) fstab_line = "{}:/msazure /nfs/node{} nfs ".format(vs_ip, i) + \ "hard,nointr,proto=tcp,mountproto=tcp,retry=5 0 0" commands.append( "sudo sh -c 'echo \"{}\" >> /etc/fstab'".format(fstab_line)) run_ssh_commands(ssh_con, commands, timeout=30) # Mount the nodes. def _log_diag(in_str): log.info(json.dumps(in_str, indent=4).replace("\\n", "\n")) try: commands = """ sudo mount -av touch ~/STATUS.NODES_MOUNTED """.split("\n") _log_diag(run_ssh_commands(ssh_con, commands, timeout=300)) except Exception as e: # Show some diag info. log.info("Exception caught when attempting to mount. Diag info:") diag_commands = """ cat /etc/mtab nfsstat sudo ufw status service portmap status sudo iptables -n -L -v netstat -rn """.split("\n") _log_diag( run_ssh_commands(ssh_con, diag_commands, ignore_nonzero_rc=True)) for vs_ip in test_vars["cluster_vs_ips"]: _log_diag( run_ssh_command(ssh_con, "rpcinfo -p " + vs_ip, ignore_nonzero_rc=True)) run_ssh_command(ssh_con, "sudo apt -y install traceroute nmap", ignore_nonzero_rc=True) _log_diag( run_ssh_command(ssh_con, "traceroute " + vs_ip, ignore_nonzero_rc=True)) _log_diag( run_ssh_command(ssh_con, "sudo nmap -sS " + vs_ip, ignore_nonzero_rc=True)) raise
def test_ping_node_ips(self, node_ips, ssh_con, test_vars): # noqa: F811 """Ping the node IPs from the controller.""" commands = [] for node_ip in node_ips: commands.append("ping -c 3 {}".format(node_ip)) run_ssh_commands(ssh_con, commands)
def test_update_reg_clients_hosts(self, test_vars): """ Updates /etc/hosts on the STAF clients so they can contact the STAF server. """ log = logging.getLogger("test_update_reg_clients_hosts") atd = test_vars["atd_obj"] commands = """ cp /etc/hosts . echo ' ' >> hosts echo '# STAF server IP' >> hosts echo '{0} staf' >> hosts sudo mv hosts /etc/hosts echo '#!/bin/bash' > ~/hostdb_entries.sh chmod 755 ~/hostdb_entries.sh echo "cd ~/Avere-sv" >> ~/hostdb_entries.sh echo "source /usr/sv/env/bin/activate" >> ~/hostdb_entries.sh echo "export PYTHONPATH=~/Avere-sv:~/Avere-sv/averesv:$PYTHONPATH:$PATH" >> ~/hostdb_entries.sh echo "averesv/hostdb.py -a vfxt -m {1} -p '{2}'" >> ~/hostdb_entries.sh """.format(test_vars["staf_server_priv_ip"], test_vars["cluster_mgmt_ip"], os.environ["AVERE_ADMIN_PW"]).split("\n") # Add hostdb entry calls for each regression client. for i, staf_client_ip in enumerate(test_vars["staf_client_priv_ips"]): commands.append( "echo 'averesv/hostdb.py -L regclient{0} -m {1}' >> ~/hostdb_entries.sh" .format(i, staf_client_ip)) # Get the storage account's access key and add that hostdb entry, too. sa_key = atd.st_client.storage_accounts.list_keys( atd.resource_group, test_vars["storage_account"]).keys[0].value commands.append( "echo 'averesv/hostdb.py -s {0}.blob.core.windows.net -m {0}.blob.core.windows.net -M az --cloudCreds \"{0}::{1}\"' >> ~/hostdb_entries.sh" .format(test_vars["storage_account"], sa_key)) last_error = None for staf_client_ip in test_vars["staf_client_priv_ips"]: for port_attempt in range(1, 11): tunnel_local_port = get_unused_local_port() with Connection(test_vars["public_ip"], user=test_vars["controller_user"], connect_kwargs={ "key_filename": test_vars["ssh_priv_key"], }).forward_local(local_port=tunnel_local_port, remote_port=22, remote_host=staf_client_ip): node_c = Connection("127.0.0.1", user=test_vars["controller_user"], port=tunnel_local_port, connect_kwargs={ "key_filename": test_vars["ssh_priv_key"], }) try: node_c.open() # If port_attempt > 1, last_error had the exception # from the last iteration. Clear it. last_error = None except NoValidConnectionsError as ex: last_error = ex exp_err = "Unable to connect to port {} on 127.0.0.1".format( tunnel_local_port) if exp_err not in str(ex): raise else: log.warning("{0} (attempt #{1}, retrying)".format( exp_err, str(port_attempt))) continue # iterate run_ssh_commands(node_c.client, commands) # Copy SSH keys to the client. scp_cli = SCPClient(node_c.transport) scp_cli.put(test_vars["ssh_priv_key"], "~/.ssh/id_rsa") scp_cli.put(test_vars["ssh_pub_key"], "~/.ssh/id_rsa.pub") scp_cli.close() log.debug("Connection to {} closed".format(staf_client_ip)) break # no need to iterate again if last_error: log.error( "See previous error(s) above. Raising last exception.") raise last_error