def start(self, unique_id, configs=None): import subprocess runtime.set_deployer("unittest", self) self._proc = subprocess.Popen(["sleep","150"]) if self._proc is not None: self._pid = self._proc.pid self.processes[unique_id] = Process(unique_id=unique_id, servicename=unique_id + "-srv", install_path=None, hostname="localhost" )
def setup_suite(self): print "Starting zookeeper" env_dict = {} if "localhost" not in runtime.get_active_config('zookeeper_host'): env_dict = {'JAVA_HOME': '/export/apps/jdk/current'} zookeeper_deployer = adhoc_deployer.SSHDeployer( "zookeeper", { 'pid_keyword': "zookeeper", 'executable': runtime.get_active_config('zookeeper_exec_location'), 'env': env_dict, 'extract': True, 'post_install_cmds': runtime.get_active_config('zookeeper_post_install_cmds'), 'stop_command': runtime.get_active_config('zookeeper_stop_command'), 'start_command': runtime.get_active_config('zookeeper_start_command') }) runtime.set_deployer("zookeeper", zookeeper_deployer) zookeeper_deployer.install( "zookeeper", { "hostname": runtime.get_active_config('zookeeper_host'), "install_path": "/tmp/zookeeper_test" }) zookeeper_deployer.start("zookeeper", configs={"sync": True})
def setup_suite(): client_exec_location = runtime.get_active_config('client_exec_path') #Clients and server exec can also be in remote location. #if so specify the client_exec as <Remoteserver>:<path> #In that case you can also path a temp_scratch config for the sftp #else /tmp folder used as default if (":" in client_exec_location): client_exec = client_exec_location else: client_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)), client_exec_location) global client_deployer client_deployer = adhoc_deployer.SSHDeployer("AdditionClient", {'pid_keyword': "AdditionClient", 'executable': client_exec, 'start_command': runtime.get_active_config('client_start_command')}) runtime.set_deployer("AdditionClient", client_deployer) client_deployer.install("client1", {"hostname": "localhost", "install_path": runtime.get_active_config('client_install_path') + 'client1'}) client_deployer.install("client2", {"hostname": "localhost", "install_path": runtime.get_active_config('client_install_path') + 'client2'}) server_exec_location = runtime.get_active_config('server_exec_path') if (":" in server_exec_location): server_exec = server_exec_location else: server_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)), runtime.get_active_config('server_exec_path')) global server_deployer server_deployer = adhoc_deployer.SSHDeployer("AdditionServer", {'pid_keyword': "AdditionServer", 'executable': server_exec, 'start_command': runtime.get_active_config('server_start_command')}) runtime.set_deployer("AdditionServer", server_deployer) server_deployer.deploy("server1", {"hostname": "localhost", "install_path": runtime.get_active_config('server_install_path') + 'server1', "args": "localhost 8000".split()}) server_deployer.deploy("server2", {"hostname": "localhost", "install_path": runtime.get_active_config('server_install_path') + 'server2', "args": "localhost 8001".split()}) server_deployer.deploy("server3", {"hostname": "localhost", "install_path": runtime.get_active_config('server_install_path') + 'server3', "args": "localhost 8002".split()})
def setup_suite(): client_exec = os.path.join( os.path.dirname(os.path.abspath(__file__)), "AdditionClient/out/artifacts/AdditionClient_jar/AdditionClient.jar") global client_deployer client_deployer = adhoc_deployer.SSHDeployer( "AdditionClient", { 'pid_keyword': "AdditionClient", 'executable': client_exec, 'start_command': "java -jar AdditionClient.jar" }) runtime.set_deployer("AdditionClient", client_deployer) client_deployer.install( "client1", { "hostname": "localhost", "install_path": "/tmp/server_client/AdditionClients/client1" }) client_deployer.install( "client2", { "hostname": "localhost", "install_path": "/tmp/server_client/AdditionClients/client2" }) server_exec = os.path.join( os.path.dirname(os.path.abspath(__file__)), "AdditionServer/out/artifacts/AdditionServer_jar/AdditionServer.jar") global server_deployer server_deployer = adhoc_deployer.SSHDeployer( "AdditionServer", { 'pid_keyword': "AdditionServer", 'executable': server_exec, 'start_command': "java -jar AdditionServer.jar" }) runtime.set_deployer("AdditionServer", server_deployer) server_deployer.deploy( "server1", { "hostname": "localhost", "install_path": "/tmp/server_client/AdditionServers/server1", "args": "localhost 8000".split() }) server_deployer.deploy( "server2", { "hostname": "localhost", "install_path": "/tmp/server_client/AdditionServers/server2", "args": "localhost 8001".split() }) server_deployer.deploy( "server3", { "hostname": "localhost", "install_path": "/tmp/server_client/AdditionServers/server3", "args": "localhost 8002".split() })
def start(self, unique_id, configs=None): import subprocess runtime.set_deployer("unittest", self) self._proc = subprocess.Popen(["sleep", "150"]) if self._proc is not None: self._pid = self._proc.pid self.processes[unique_id] = Process(unique_id=unique_id, servicename=unique_id + "-srv", install_path=None, hostname="localhost")
def setup_suite(self): if self._deployer is not None: runtime.set_deployer("ztestsuite.unittest.deployer", self._deployer ) if os.path.isdir("/tmp/ztestsute"): shutil.rmtree("/tmp/ztestsuite") if not os.path.isdir(runtime.get_active_config("LOGS_DIRECTORY")): os.makedirs(runtime.get_active_config("LOGS_DIRECTORY")) if not os.path.isdir(runtime.get_active_config("OUTPUT_DIRECTORY")): os.makedirs(runtime.get_active_config("OUTPUT_DIRECTORY")) sample_code = os.path.join(TEST_DIRECTORY, "samples","trivial_program_with_timing") self.ssh.connect("localhost") self.sample_code_in, stdout, stderr = self.ssh.exec_command("python {0}".format(sample_code)) if self._deployer is not None: self._deployer.start("ztestsuite.unittest")
def setup_suite(): global deployers, samza_job_deployer, samza_install_path logger.info('Current working directory: {0}'.format(os.getcwd())) samza_install_path = os.path.join(c('remote_install_path'), c('samza_install_path')) _download_packages() deployers = { 'zookeeper': _new_ssh_deployer('zookeeper'), 'yarn_rm': _new_ssh_deployer('yarn_rm'), 'yarn_nm': _new_ssh_deployer('yarn_nm'), 'kafka': _new_ssh_deployer('kafka'), } # Enforce install order through list. for name in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']: deployer = deployers[name] runtime.set_deployer(name, deployer) for instance, host in c(name + '_hosts').iteritems(): logger.info('Deploying {0} on host: {1}'.format(instance, host)) deployer.deploy(instance, {'hostname': host}) # Start the Samza jobs. samza_job_deployer = SamzaJobYarnDeployer({ 'yarn_site_template': c('yarn_site_template'), 'yarn_driver_configs': c('yarn_driver_configs'), 'yarn_nm_hosts': c('yarn_nm_hosts').values(), 'install_path': samza_install_path, }) samza_job_deployer.install('smoke_tests', { 'executable': c('samza_executable'), }) samza_job_deployer.start( 'negate_number', { 'package_id': 'smoke_tests', 'config_factory': c('samza_config_factory'), 'config_file': c('samza_config_file'), 'install_path': samza_install_path, })
def setup_suite(): print "Starting zookeeper quorum" global zookeper_deployer env_dict = {} if "localhost" not in runtime.get_active_config('zookeeper_host'): env_dict = {'JAVA_HOME':'/export/apps/jdk/current'} zookeper_deployer = adhoc_deployer.SSHDeployer("zookeeper", {'pid_keyword': "zookeeper", 'executable': runtime.get_active_config('zookeeper_exec_location'), 'env':env_dict, 'extract': True, 'stop_command':runtime.get_active_config('zookeeper_stop_command'), 'start_command': runtime.get_active_config('zookeeper_start_command')}) runtime.set_deployer("zookeeper", zookeper_deployer) # Deploy Zookeeper1 print "Deploy Zookeeper1" zookeper_deployer.install("zookeeper1", {"hostname": "localhost", "install_path": runtime.get_active_config('zookeeper1_install_path'), "pid_file": runtime.get_active_config('zookeeper1_pid_file'), 'post_install_cmds':runtime.get_active_config('zookeeper1_post_install_cmds')}) zookeper_deployer.start("zookeeper1",configs={"sync": True}) # Deploy Zookeeper2 print "Deploy Zookeeper2" zookeper_deployer.install("zookeeper2", {"hostname": "localhost", "install_path": runtime.get_active_config('zookeeper2_install_path'), "pid_file": runtime.get_active_config('zookeeper2_pid_file'), 'post_install_cmds':runtime.get_active_config('zookeeper2_post_install_cmds')}) zookeper_deployer.start("zookeeper2",configs={"sync": True}) # Deploy Zookeeper3 print "Deploy Zookeeper3" zookeper_deployer.install("zookeeper3", {"hostname": "localhost", "install_path": runtime.get_active_config('zookeeper3_install_path'), "pid_file": runtime.get_active_config('zookeeper3_pid_file'), 'post_install_cmds':runtime.get_active_config('zookeeper3_post_install_cmds')}) zookeper_deployer.start("zookeeper3",configs={"sync": True})
def setup_suite(): client_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)), "AdditionClient/out/artifacts/AdditionClient_jar/AdditionClient.jar") global client_deployer client_deployer = adhoc_deployer.SSHDeployer("AdditionClient", {'pid_keyword': "AdditionClient", 'executable': client_exec, 'start_command': "java -jar AdditionClient.jar"}) runtime.set_deployer("AdditionClient", client_deployer) client_deployer.install("client1", {"hostname": "localhost", "install_path": "/tmp/server_client/AdditionClients/client1"}) client_deployer.install("client2", {"hostname": "localhost", "install_path": "/tmp/server_client/AdditionClients/client2"}) server_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)), "AdditionServer/out/artifacts/AdditionServer_jar/AdditionServer.jar") global server_deployer server_deployer = adhoc_deployer.SSHDeployer("AdditionServer", {'pid_keyword': "AdditionServer", 'executable': server_exec, 'start_command': "java -jar AdditionServer.jar"}) runtime.set_deployer("AdditionServer", server_deployer) server_deployer.deploy("server1", {"hostname": "localhost", "install_path": "/tmp/server_client/AdditionServers/server1", "args": "localhost 8000".split()}) server_deployer.deploy("server2", {"hostname": "localhost", "install_path": "/tmp/server_client/AdditionServers/server2", "args": "localhost 8001".split()}) server_deployer.deploy("server3", {"hostname": "localhost", "install_path": "/tmp/server_client/AdditionServers/server3", "args": "localhost 8002".split()})
def setup_suite(self): print "Starting zookeeper" env_dict = {} if "localhost" not in runtime.get_active_config('zookeeper_host'): env_dict = {'JAVA_HOME':'/export/apps/jdk/current'} zookeeper_deployer = adhoc_deployer.SSHDeployer("zookeeper", {'pid_keyword': "zookeeper", 'executable': runtime.get_active_config('zookeeper_exec_location'), 'env':env_dict, 'extract': True, 'post_install_cmds':runtime.get_active_config('zookeeper_post_install_cmds'), 'stop_command':runtime.get_active_config('zookeeper_stop_command'), 'start_command': runtime.get_active_config('zookeeper_start_command')}) runtime.set_deployer("zookeeper", zookeeper_deployer) zookeeper_deployer.install("zookeeper", {"hostname": runtime.get_active_config('zookeeper_host'), "install_path": "/tmp/zookeeper_test"}) zookeeper_deployer.start("zookeeper",configs={"sync": True})
def setup_suite(): global deployers, samza_job_deployer, samza_install_path logger.info('Current working directory: {0}'.format(os.getcwd())) samza_install_path = os.path.join(c('remote_install_path'), c('samza_install_path')) _download_packages() deployers = { 'zookeeper': _new_ssh_deployer('zookeeper'), 'yarn_rm': _new_ssh_deployer('yarn_rm'), 'yarn_nm': _new_ssh_deployer('yarn_nm'), 'kafka': _new_ssh_deployer('kafka'), } # Enforce install order through list. for name in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']: deployer = deployers[name] runtime.set_deployer(name, deployer) for instance, host in c(name + '_hosts').iteritems(): logger.info('Deploying {0} on host: {1}'.format(instance, host)) deployer.deploy(instance, { 'hostname': host }) # Setup Samza job deployer. samza_job_deployer = SamzaJobYarnDeployer({ 'config_factory': c('samza_config_factory'), 'yarn_site_template': c('yarn_site_template'), 'yarn_driver_configs': c('yarn_driver_configs'), 'yarn_nm_hosts': c('yarn_nm_hosts').values(), 'install_path': samza_install_path, }) samza_job_deployer.install('tests', { 'executable': c('samza_executable'), }) runtime.set_deployer('samza_job_deployer', samza_job_deployer)
def setup_suite(): print "==> Starting tests for PYHK." runtime.set_user('root', '') global pyhk_deployer global tcp_server # Set the server deployer. tcp_server = adhoc_deployer.SSHDeployer( "server", {'executable': runtime.get_active_config('pyhk_exec'), 'extract': True, 'start_command': runtime.get_active_config('tcp_server_cmd'), 'stop_command': "ps ax | grep '[p]ython server' | awk '{print $1}' | xargs kill -9"}) runtime.set_deployer("server", tcp_server) # Provisioning the server. tcp_server.install("server1", {"hostname": "10.0.1.23", "install_path": runtime.get_active_config('pyhk_install')}) # Set the client deployer tcp_client = adhoc_deployer.SSHDeployer( "client", {'executable': runtime.get_active_config('pyhk_exec'), 'extract': True, 'start_command': runtime.get_active_config('tcp_client_cmd')}) runtime.set_deployer("client", tcp_client) # Provisioning the client. tcp_client.install("client1", {"hostname": "10.0.1.24", "install_path": runtime.get_active_config('pyhk_install')}) # Set general deployer. pyhk_deployer = adhoc_deployer.SSHDeployer( "pyhk", {'executable': runtime.get_active_config('pyhk_exec'), 'extract': True, 'start_command': runtime.get_active_config('pyhk_cmd')}) runtime.set_deployer("pyhk", pyhk_deployer) # Hostname 1 pyhk_deployer.install( "machine1", {"hostname": runtime.get_active_config('pyhk_hostname1'), "install_path": runtime.get_active_config('pyhk_install')}) # Hostname 2 pyhk_deployer.install( "machine2", {"hostname": runtime.get_active_config('pyhk_hostname2'), "install_path": runtime.get_active_config('pyhk_install')})
def setup_suite(): # Set up authentication username = runtime.get_active_config("username") password = runtime.get_active_config("password") runtime.set_user(username, password) # Set up the target directories and properties userdir = os.path.join("/home", username) riaps_app_path = os.path.join(userdir, runtime.get_active_config("riaps_apps_path")) test_app_path = os.path.join(riaps_app_path, runtime.get_active_config('app_dir')) start_actor_path = "/opt/riaps/armhf/bin/" env = { "PATH": "/usr/local/bin/:" + start_actor_path + ":$PATH", "RIAPSHOME": "/usr/local/riaps", "RIAPSAPPS": "$HOME/riaps_apps", "LD_LIBRARY_PATH": "/opt/riaps/armhf/lib:" + test_app_path } #start_riaps_lang = "riaps_lang " + runtime.get_active_config('model_file') # Set up the sources model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), runtime.get_active_config('app_dir'), runtime.get_active_config('model_file')) # Script to check discovery service discoCheckScript = "checkDiscoveryService.py" discoCheckScriptPath = "../../test_common" # Script to start the discovery discoStartScript = "startDiscovery.py" discoStartScriptPath = "../../test_common" # Script to stop the discovery discoStopScript = "stopDiscovery.py" discoStopScriptPath = "../../test_common" killRiapsScript = "killRiaps.py" killRiapsScriptPath = "../../test_common" # Deploy the riaps killer script for target in runtime.get_active_config('targets'): deployerId = "killer_" + target["host"] killscriptpath = os.path.abspath( os.path.join(os.path.dirname(os.path.abspath(__file__)), killRiapsScriptPath, killRiapsScript)) killDeployer = adhoc_deployer.SSHDeployer( deployerId, { 'executable': killscriptpath, 'install_path': riaps_app_path, 'env': env, 'hostname': target["host"], "start_command": "python3 " + os.path.join(riaps_app_path, killRiapsScript) }) runtime.set_deployer(deployerId, killDeployer) killDeployer.install(deployerId) # Deploy the discovery starter script for target in runtime.get_active_config('targets'): deployerId = "discostart_" + target["host"] startscriptpath = os.path.abspath( os.path.join(os.path.dirname(os.path.abspath(__file__)), discoStartScriptPath, discoStartScript)) startDiscoveryDeployer = adhoc_deployer.SSHDeployer( deployerId, { 'executable': startscriptpath, 'install_path': riaps_app_path, 'env': env, 'hostname': target["host"], "start_command": "python3 " + os.path.join(riaps_app_path, discoStartScript) }) runtime.set_deployer(deployerId, startDiscoveryDeployer) startDiscoveryDeployer.install(deployerId) # Deploy the discovery stop script for target in runtime.get_active_config('targets'): deployerId = "discostop_" + target["host"] stopscriptpath = os.path.abspath( os.path.join(os.path.dirname(os.path.abspath(__file__)), discoStopScriptPath, discoStopScript)) stopDiscoveryDeployer = adhoc_deployer.SSHDeployer( deployerId, { 'executable': stopscriptpath, 'install_path': riaps_app_path, 'env': env, 'hostname': target["host"], "start_command": "python3 " + os.path.join(riaps_app_path, discoStopScript) }) runtime.set_deployer(deployerId, stopDiscoveryDeployer) stopDiscoveryDeployer.install(deployerId) # Deploy the riaps-disco checker script for target in runtime.get_active_config('targets'): deployerId = "disco" + target["host"] checkscriptpath = os.path.abspath( os.path.join(os.path.dirname(os.path.abspath(__file__)), discoCheckScriptPath, discoCheckScript)) checkDiscoDeployer = adhoc_deployer.SSHDeployer( deployerId, { 'executable': checkscriptpath, 'install_path': riaps_app_path, 'env': env, 'hostname': target["host"], "start_command": "python3 " + os.path.join(riaps_app_path, "checkDiscoveryService.py") }) runtime.set_deployer(deployerId, checkDiscoDeployer) checkDiscoDeployer.install(deployerId) # Deploy the riaps-components/model file local_riaps_lang = "riaps_lang " + model_path local_test_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), runtime.get_active_config('app_dir')) subprocess.call(local_riaps_lang, shell=True) move_cmd = "mv " + runtime.get_active_config( 'app_dir') + '.json ' + local_test_dir subprocess.call(move_cmd, shell=True) local_model_json = os.path.join( local_test_dir, runtime.get_active_config('app_dir') + '.json') for target in runtime.get_active_config('targets'): model_deployer = adhoc_deployer.SSHDeployer( target["actor"], { 'executable': local_model_json, 'install_path': test_app_path, 'hostname': target["host"], #'start_command': os.path.join(start_actor_path, "start_actor"), 'start_command': "start_actor", #'args': [os.path.join(test_app_path, runtime.get_active_config('app_dir') + '.json'), # target["actor"]], 'env': env #'terminate_only': False, #'pid_keyword': model_path, }) runtime.set_deployer(target["actor"], model_deployer) # Add test cases testcases = [ "pubfirst_" + target["actor"], "subfirst_" + target["actor"] ] for testcase in testcases: model_deployer.install( testcase, { 'args': [ os.path.join( test_app_path, runtime.get_active_config('app_dir') + '.json'), target["actor"], '--logfile="' + testcase + '.log"' ] }) for component in runtime.get_active_config('components_so'): localPath = os.path.join( os.path.dirname(os.path.abspath(__file__)), runtime.get_active_config('app_dir'), component) #localPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), # runtime.get_active_config('app_dir'), # "lib", # component) #targetPath = target_path#os.path.join(riaps_app_path, runtime.get_active_config('app_dir')) component_deployer = adhoc_deployer.SSHDeployer( component, { 'executable': localPath, 'install_path': test_app_path, 'hostname': target["host"], #'start_command': "start_actor", #os.path.join(target_path, "start_actor"), #'args': [runtime.get_active_config('app_dir'), # runtime.get_active_config('app_dir') + '.json', # target["actor"]], 'env': env, #'terminate_only': True, #'pid_keyword': component, }) runtime.set_deployer(component, component_deployer) component_deployer.install(component) print("Deployment done.")
def setup_suite(): client_exec_location = runtime.get_active_config('client_exec_path') #Clients and server exec can also be in remote location. #if so specify the client_exec as <Remoteserver>:<path> #In that case you can also path a temp_scratch config for the sftp #else /tmp folder used as default if (":" in client_exec_location): client_exec = client_exec_location else: client_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)), client_exec_location) global client_deployer client_deployer = adhoc_deployer.SSHDeployer( "AdditionClient", { 'pid_keyword': "AdditionClient", 'executable': client_exec, 'start_command': runtime.get_active_config('client_start_command') }) runtime.set_deployer("AdditionClient", client_deployer) client_deployer.install( "client1", { "hostname": "localhost", "install_path": runtime.get_active_config('client_install_path') + 'client1' }) client_deployer.install( "client2", { "hostname": "localhost", "install_path": runtime.get_active_config('client_install_path') + 'client2' }) server_exec_location = runtime.get_active_config('server_exec_path') if (":" in server_exec_location): server_exec = server_exec_location else: server_exec = os.path.join( os.path.dirname(os.path.abspath(__file__)), runtime.get_active_config('server_exec_path')) global server_deployer server_deployer = adhoc_deployer.SSHDeployer( "AdditionServer", { 'pid_keyword': "AdditionServer", 'executable': server_exec, 'start_command': runtime.get_active_config('server_start_command') }) runtime.set_deployer("AdditionServer", server_deployer) server_deployer.deploy( "server1", { "hostname": "localhost", "install_path": runtime.get_active_config('server_install_path') + 'server1', "args": "localhost 8000".split() }) server_deployer.deploy( "server2", { "hostname": "localhost", "install_path": runtime.get_active_config('server_install_path') + 'server2', "args": "localhost 8001".split() }) server_deployer.deploy( "server3", { "hostname": "localhost", "install_path": runtime.get_active_config('server_install_path') + 'server3', "args": "localhost 8002".split() })