def start(self, unique_id, configs=None):
import subprocess
runtime.set_deployer("unittest", self)
self._proc = subprocess.Popen(["sleep","150"])
if self._proc is not None:
self._pid = self._proc.pid
self.processes[unique_id] = Process(unique_id=unique_id, servicename=unique_id + "-srv", install_path=None, hostname="localhost" )
def setup_suite(self):
print "Starting zookeeper"
env_dict = {}
if "localhost" not in runtime.get_active_config('zookeeper_host'):
env_dict = {'JAVA_HOME': '/export/apps/jdk/current'}
zookeeper_deployer = adhoc_deployer.SSHDeployer(
"zookeeper", {
'pid_keyword':
"zookeeper",
'executable':
runtime.get_active_config('zookeeper_exec_location'),
'env':
env_dict,
'extract':
True,
'post_install_cmds':
runtime.get_active_config('zookeeper_post_install_cmds'),
'stop_command':
runtime.get_active_config('zookeeper_stop_command'),
'start_command':
runtime.get_active_config('zookeeper_start_command')
})
runtime.set_deployer("zookeeper", zookeeper_deployer)
zookeeper_deployer.install(
"zookeeper", {
"hostname": runtime.get_active_config('zookeeper_host'),
"install_path": "/tmp/zookeeper_test"
})
zookeeper_deployer.start("zookeeper", configs={"sync": True})
def setup_suite():
client_exec_location = runtime.get_active_config('client_exec_path')
#Clients and server exec can also be in remote location.
#if so specify the client_exec as <Remoteserver>:<path>
#In that case you can also path a temp_scratch config for the sftp
#else /tmp folder used as default
if (":" in client_exec_location):
client_exec = client_exec_location
else:
client_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)),
client_exec_location)
global client_deployer
client_deployer = adhoc_deployer.SSHDeployer("AdditionClient",
{'pid_keyword': "AdditionClient",
'executable': client_exec,
'start_command': runtime.get_active_config('client_start_command')})
runtime.set_deployer("AdditionClient", client_deployer)
client_deployer.install("client1",
{"hostname": "localhost",
"install_path": runtime.get_active_config('client_install_path') + 'client1'})
client_deployer.install("client2",
{"hostname": "localhost",
"install_path": runtime.get_active_config('client_install_path') + 'client2'})
server_exec_location = runtime.get_active_config('server_exec_path')
if (":" in server_exec_location):
server_exec = server_exec_location
else:
server_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)),
runtime.get_active_config('server_exec_path'))
global server_deployer
server_deployer = adhoc_deployer.SSHDeployer("AdditionServer",
{'pid_keyword': "AdditionServer",
'executable': server_exec,
'start_command': runtime.get_active_config('server_start_command')})
runtime.set_deployer("AdditionServer", server_deployer)
server_deployer.deploy("server1",
{"hostname": "localhost",
"install_path": runtime.get_active_config('server_install_path') + 'server1',
"args": "localhost 8000".split()})
server_deployer.deploy("server2",
{"hostname": "localhost",
"install_path": runtime.get_active_config('server_install_path') + 'server2',
"args": "localhost 8001".split()})
server_deployer.deploy("server3",
{"hostname": "localhost",
"install_path": runtime.get_active_config('server_install_path') + 'server3',
"args": "localhost 8002".split()})
def setup_suite():
client_exec = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"AdditionClient/out/artifacts/AdditionClient_jar/AdditionClient.jar")
global client_deployer
client_deployer = adhoc_deployer.SSHDeployer(
"AdditionClient", {
'pid_keyword': "AdditionClient",
'executable': client_exec,
'start_command': "java -jar AdditionClient.jar"
})
runtime.set_deployer("AdditionClient", client_deployer)
client_deployer.install(
"client1", {
"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionClients/client1"
})
client_deployer.install(
"client2", {
"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionClients/client2"
})
server_exec = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"AdditionServer/out/artifacts/AdditionServer_jar/AdditionServer.jar")
global server_deployer
server_deployer = adhoc_deployer.SSHDeployer(
"AdditionServer", {
'pid_keyword': "AdditionServer",
'executable': server_exec,
'start_command': "java -jar AdditionServer.jar"
})
runtime.set_deployer("AdditionServer", server_deployer)
server_deployer.deploy(
"server1", {
"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionServers/server1",
"args": "localhost 8000".split()
})
server_deployer.deploy(
"server2", {
"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionServers/server2",
"args": "localhost 8001".split()
})
server_deployer.deploy(
"server3", {
"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionServers/server3",
"args": "localhost 8002".split()
})
def start(self, unique_id, configs=None):
import subprocess
runtime.set_deployer("unittest", self)
self._proc = subprocess.Popen(["sleep", "150"])
if self._proc is not None:
self._pid = self._proc.pid
self.processes[unique_id] = Process(unique_id=unique_id,
servicename=unique_id + "-srv",
install_path=None,
hostname="localhost")
def setup_suite(self):
if self._deployer is not None:
runtime.set_deployer("ztestsuite.unittest.deployer", self._deployer )
if os.path.isdir("/tmp/ztestsute"):
shutil.rmtree("/tmp/ztestsuite")
if not os.path.isdir(runtime.get_active_config("LOGS_DIRECTORY")):
os.makedirs(runtime.get_active_config("LOGS_DIRECTORY"))
if not os.path.isdir(runtime.get_active_config("OUTPUT_DIRECTORY")):
os.makedirs(runtime.get_active_config("OUTPUT_DIRECTORY"))
sample_code = os.path.join(TEST_DIRECTORY, "samples","trivial_program_with_timing")
self.ssh.connect("localhost")
self.sample_code_in, stdout, stderr = self.ssh.exec_command("python {0}".format(sample_code))
if self._deployer is not None:
self._deployer.start("ztestsuite.unittest")
def setup_suite():
global deployers, samza_job_deployer, samza_install_path
logger.info('Current working directory: {0}'.format(os.getcwd()))
samza_install_path = os.path.join(c('remote_install_path'),
c('samza_install_path'))
_download_packages()
deployers = {
'zookeeper': _new_ssh_deployer('zookeeper'),
'yarn_rm': _new_ssh_deployer('yarn_rm'),
'yarn_nm': _new_ssh_deployer('yarn_nm'),
'kafka': _new_ssh_deployer('kafka'),
}
# Enforce install order through list.
for name in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']:
deployer = deployers[name]
runtime.set_deployer(name, deployer)
for instance, host in c(name + '_hosts').iteritems():
logger.info('Deploying {0} on host: {1}'.format(instance, host))
deployer.deploy(instance, {'hostname': host})
# Start the Samza jobs.
samza_job_deployer = SamzaJobYarnDeployer({
'yarn_site_template':
c('yarn_site_template'),
'yarn_driver_configs':
c('yarn_driver_configs'),
'yarn_nm_hosts':
c('yarn_nm_hosts').values(),
'install_path':
samza_install_path,
})
samza_job_deployer.install('smoke_tests', {
'executable': c('samza_executable'),
})
samza_job_deployer.start(
'negate_number', {
'package_id': 'smoke_tests',
'config_factory': c('samza_config_factory'),
'config_file': c('samza_config_file'),
'install_path': samza_install_path,
})
def setup_suite():
print "Starting zookeeper quorum"
global zookeper_deployer
env_dict = {}
if "localhost" not in runtime.get_active_config('zookeeper_host'):
env_dict = {'JAVA_HOME':'/export/apps/jdk/current'}
zookeper_deployer = adhoc_deployer.SSHDeployer("zookeeper",
{'pid_keyword': "zookeeper",
'executable': runtime.get_active_config('zookeeper_exec_location'),
'env':env_dict,
'extract': True,
'stop_command':runtime.get_active_config('zookeeper_stop_command'),
'start_command': runtime.get_active_config('zookeeper_start_command')})
runtime.set_deployer("zookeeper", zookeper_deployer)
# Deploy Zookeeper1
print "Deploy Zookeeper1"
zookeper_deployer.install("zookeeper1",
{"hostname": "localhost",
"install_path": runtime.get_active_config('zookeeper1_install_path'),
"pid_file": runtime.get_active_config('zookeeper1_pid_file'),
'post_install_cmds':runtime.get_active_config('zookeeper1_post_install_cmds')})
zookeper_deployer.start("zookeeper1",configs={"sync": True})
# Deploy Zookeeper2
print "Deploy Zookeeper2"
zookeper_deployer.install("zookeeper2",
{"hostname": "localhost",
"install_path": runtime.get_active_config('zookeeper2_install_path'),
"pid_file": runtime.get_active_config('zookeeper2_pid_file'),
'post_install_cmds':runtime.get_active_config('zookeeper2_post_install_cmds')})
zookeper_deployer.start("zookeeper2",configs={"sync": True})
# Deploy Zookeeper3
print "Deploy Zookeeper3"
zookeper_deployer.install("zookeeper3",
{"hostname": "localhost",
"install_path": runtime.get_active_config('zookeeper3_install_path'),
"pid_file": runtime.get_active_config('zookeeper3_pid_file'),
'post_install_cmds':runtime.get_active_config('zookeeper3_post_install_cmds')})
zookeper_deployer.start("zookeeper3",configs={"sync": True})
def setup_suite():
client_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"AdditionClient/out/artifacts/AdditionClient_jar/AdditionClient.jar")
global client_deployer
client_deployer = adhoc_deployer.SSHDeployer("AdditionClient",
{'pid_keyword': "AdditionClient",
'executable': client_exec,
'start_command': "java -jar AdditionClient.jar"})
runtime.set_deployer("AdditionClient", client_deployer)
client_deployer.install("client1",
{"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionClients/client1"})
client_deployer.install("client2",
{"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionClients/client2"})
server_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"AdditionServer/out/artifacts/AdditionServer_jar/AdditionServer.jar")
global server_deployer
server_deployer = adhoc_deployer.SSHDeployer("AdditionServer",
{'pid_keyword': "AdditionServer",
'executable': server_exec,
'start_command': "java -jar AdditionServer.jar"})
runtime.set_deployer("AdditionServer", server_deployer)
server_deployer.deploy("server1",
{"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionServers/server1",
"args": "localhost 8000".split()})
server_deployer.deploy("server2",
{"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionServers/server2",
"args": "localhost 8001".split()})
server_deployer.deploy("server3",
{"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionServers/server3",
"args": "localhost 8002".split()})
def setup_suite(self):
print "Starting zookeeper"
env_dict = {}
if "localhost" not in runtime.get_active_config('zookeeper_host'):
env_dict = {'JAVA_HOME':'/export/apps/jdk/current'}
zookeeper_deployer = adhoc_deployer.SSHDeployer("zookeeper",
{'pid_keyword': "zookeeper",
'executable': runtime.get_active_config('zookeeper_exec_location'),
'env':env_dict,
'extract': True,
'post_install_cmds':runtime.get_active_config('zookeeper_post_install_cmds'),
'stop_command':runtime.get_active_config('zookeeper_stop_command'),
'start_command': runtime.get_active_config('zookeeper_start_command')})
runtime.set_deployer("zookeeper", zookeeper_deployer)
zookeeper_deployer.install("zookeeper",
{"hostname": runtime.get_active_config('zookeeper_host'),
"install_path": "/tmp/zookeeper_test"})
zookeeper_deployer.start("zookeeper",configs={"sync": True})
def setup_suite():
global deployers, samza_job_deployer, samza_install_path
logger.info('Current working directory: {0}'.format(os.getcwd()))
samza_install_path = os.path.join(c('remote_install_path'), c('samza_install_path'))
_download_packages()
deployers = {
'zookeeper': _new_ssh_deployer('zookeeper'),
'yarn_rm': _new_ssh_deployer('yarn_rm'),
'yarn_nm': _new_ssh_deployer('yarn_nm'),
'kafka': _new_ssh_deployer('kafka'),
}
# Enforce install order through list.
for name in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']:
deployer = deployers[name]
runtime.set_deployer(name, deployer)
for instance, host in c(name + '_hosts').iteritems():
logger.info('Deploying {0} on host: {1}'.format(instance, host))
deployer.deploy(instance, {
'hostname': host
})
# Setup Samza job deployer.
samza_job_deployer = SamzaJobYarnDeployer({
'config_factory': c('samza_config_factory'),
'yarn_site_template': c('yarn_site_template'),
'yarn_driver_configs': c('yarn_driver_configs'),
'yarn_nm_hosts': c('yarn_nm_hosts').values(),
'install_path': samza_install_path,
})
samza_job_deployer.install('tests', {
'executable': c('samza_executable'),
})
runtime.set_deployer('samza_job_deployer', samza_job_deployer)
def setup_suite():
print "==> Starting tests for PYHK."
runtime.set_user('root', '')
global pyhk_deployer
global tcp_server
# Set the server deployer.
tcp_server = adhoc_deployer.SSHDeployer(
"server",
{'executable': runtime.get_active_config('pyhk_exec'),
'extract': True,
'start_command': runtime.get_active_config('tcp_server_cmd'),
'stop_command': "ps ax | grep '[p]ython server' | awk '{print $1}' | xargs kill -9"})
runtime.set_deployer("server", tcp_server)
# Provisioning the server.
tcp_server.install("server1",
{"hostname": "10.0.1.23",
"install_path": runtime.get_active_config('pyhk_install')})
# Set the client deployer
tcp_client = adhoc_deployer.SSHDeployer(
"client",
{'executable': runtime.get_active_config('pyhk_exec'),
'extract': True,
'start_command': runtime.get_active_config('tcp_client_cmd')})
runtime.set_deployer("client", tcp_client)
# Provisioning the client.
tcp_client.install("client1",
{"hostname": "10.0.1.24",
"install_path": runtime.get_active_config('pyhk_install')})
# Set general deployer.
pyhk_deployer = adhoc_deployer.SSHDeployer(
"pyhk",
{'executable': runtime.get_active_config('pyhk_exec'),
'extract': True,
'start_command': runtime.get_active_config('pyhk_cmd')})
runtime.set_deployer("pyhk", pyhk_deployer)
# Hostname 1
pyhk_deployer.install(
"machine1",
{"hostname": runtime.get_active_config('pyhk_hostname1'),
"install_path": runtime.get_active_config('pyhk_install')})
# Hostname 2
pyhk_deployer.install(
"machine2",
{"hostname": runtime.get_active_config('pyhk_hostname2'),
"install_path": runtime.get_active_config('pyhk_install')})
def setup_suite():
# Set up authentication
username = runtime.get_active_config("username")
password = runtime.get_active_config("password")
runtime.set_user(username, password)
# Set up the target directories and properties
userdir = os.path.join("/home", username)
riaps_app_path = os.path.join(userdir,
runtime.get_active_config("riaps_apps_path"))
test_app_path = os.path.join(riaps_app_path,
runtime.get_active_config('app_dir'))
start_actor_path = "/opt/riaps/armhf/bin/"
env = {
"PATH": "/usr/local/bin/:" + start_actor_path + ":$PATH",
"RIAPSHOME": "/usr/local/riaps",
"RIAPSAPPS": "$HOME/riaps_apps",
"LD_LIBRARY_PATH": "/opt/riaps/armhf/lib:" + test_app_path
}
#start_riaps_lang = "riaps_lang " + runtime.get_active_config('model_file')
# Set up the sources
model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
runtime.get_active_config('app_dir'),
runtime.get_active_config('model_file'))
# Script to check discovery service
discoCheckScript = "checkDiscoveryService.py"
discoCheckScriptPath = "../../test_common"
# Script to start the discovery
discoStartScript = "startDiscovery.py"
discoStartScriptPath = "../../test_common"
# Script to stop the discovery
discoStopScript = "stopDiscovery.py"
discoStopScriptPath = "../../test_common"
killRiapsScript = "killRiaps.py"
killRiapsScriptPath = "../../test_common"
# Deploy the riaps killer script
for target in runtime.get_active_config('targets'):
deployerId = "killer_" + target["host"]
killscriptpath = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
killRiapsScriptPath, killRiapsScript))
killDeployer = adhoc_deployer.SSHDeployer(
deployerId, {
'executable':
killscriptpath,
'install_path':
riaps_app_path,
'env':
env,
'hostname':
target["host"],
"start_command":
"python3 " + os.path.join(riaps_app_path, killRiapsScript)
})
runtime.set_deployer(deployerId, killDeployer)
killDeployer.install(deployerId)
# Deploy the discovery starter script
for target in runtime.get_active_config('targets'):
deployerId = "discostart_" + target["host"]
startscriptpath = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
discoStartScriptPath, discoStartScript))
startDiscoveryDeployer = adhoc_deployer.SSHDeployer(
deployerId, {
'executable':
startscriptpath,
'install_path':
riaps_app_path,
'env':
env,
'hostname':
target["host"],
"start_command":
"python3 " + os.path.join(riaps_app_path, discoStartScript)
})
runtime.set_deployer(deployerId, startDiscoveryDeployer)
startDiscoveryDeployer.install(deployerId)
# Deploy the discovery stop script
for target in runtime.get_active_config('targets'):
deployerId = "discostop_" + target["host"]
stopscriptpath = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
discoStopScriptPath, discoStopScript))
stopDiscoveryDeployer = adhoc_deployer.SSHDeployer(
deployerId, {
'executable':
stopscriptpath,
'install_path':
riaps_app_path,
'env':
env,
'hostname':
target["host"],
"start_command":
"python3 " + os.path.join(riaps_app_path, discoStopScript)
})
runtime.set_deployer(deployerId, stopDiscoveryDeployer)
stopDiscoveryDeployer.install(deployerId)
# Deploy the riaps-disco checker script
for target in runtime.get_active_config('targets'):
deployerId = "disco" + target["host"]
checkscriptpath = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
discoCheckScriptPath, discoCheckScript))
checkDiscoDeployer = adhoc_deployer.SSHDeployer(
deployerId, {
'executable':
checkscriptpath,
'install_path':
riaps_app_path,
'env':
env,
'hostname':
target["host"],
"start_command":
"python3 " +
os.path.join(riaps_app_path, "checkDiscoveryService.py")
})
runtime.set_deployer(deployerId, checkDiscoDeployer)
checkDiscoDeployer.install(deployerId)
# Deploy the riaps-components/model file
local_riaps_lang = "riaps_lang " + model_path
local_test_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
runtime.get_active_config('app_dir'))
subprocess.call(local_riaps_lang, shell=True)
move_cmd = "mv " + runtime.get_active_config(
'app_dir') + '.json ' + local_test_dir
subprocess.call(move_cmd, shell=True)
local_model_json = os.path.join(
local_test_dir,
runtime.get_active_config('app_dir') + '.json')
for target in runtime.get_active_config('targets'):
model_deployer = adhoc_deployer.SSHDeployer(
target["actor"],
{
'executable': local_model_json,
'install_path': test_app_path,
'hostname': target["host"],
#'start_command': os.path.join(start_actor_path, "start_actor"),
'start_command': "start_actor",
#'args': [os.path.join(test_app_path, runtime.get_active_config('app_dir') + '.json'),
# target["actor"]],
'env': env
#'terminate_only': False,
#'pid_keyword': model_path,
})
runtime.set_deployer(target["actor"], model_deployer)
# Add test cases
testcases = [
"pubfirst_" + target["actor"], "subfirst_" + target["actor"]
]
for testcase in testcases:
model_deployer.install(
testcase, {
'args': [
os.path.join(
test_app_path,
runtime.get_active_config('app_dir') + '.json'),
target["actor"], '--logfile="' + testcase + '.log"'
]
})
for component in runtime.get_active_config('components_so'):
localPath = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
runtime.get_active_config('app_dir'), component)
#localPath = os.path.join(os.path.dirname(os.path.abspath(__file__)),
# runtime.get_active_config('app_dir'),
# "lib",
# component)
#targetPath = target_path#os.path.join(riaps_app_path, runtime.get_active_config('app_dir'))
component_deployer = adhoc_deployer.SSHDeployer(
component,
{
'executable': localPath,
'install_path': test_app_path,
'hostname': target["host"],
#'start_command': "start_actor", #os.path.join(target_path, "start_actor"),
#'args': [runtime.get_active_config('app_dir'),
# runtime.get_active_config('app_dir') + '.json',
# target["actor"]],
'env': env,
#'terminate_only': True,
#'pid_keyword': component,
})
runtime.set_deployer(component, component_deployer)
component_deployer.install(component)
print("Deployment done.")
def setup_suite():
client_exec_location = runtime.get_active_config('client_exec_path')
#Clients and server exec can also be in remote location.
#if so specify the client_exec as <Remoteserver>:<path>
#In that case you can also path a temp_scratch config for the sftp
#else /tmp folder used as default
if (":" in client_exec_location):
client_exec = client_exec_location
else:
client_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)),
client_exec_location)
global client_deployer
client_deployer = adhoc_deployer.SSHDeployer(
"AdditionClient", {
'pid_keyword': "AdditionClient",
'executable': client_exec,
'start_command': runtime.get_active_config('client_start_command')
})
runtime.set_deployer("AdditionClient", client_deployer)
client_deployer.install(
"client1", {
"hostname":
"localhost",
"install_path":
runtime.get_active_config('client_install_path') + 'client1'
})
client_deployer.install(
"client2", {
"hostname":
"localhost",
"install_path":
runtime.get_active_config('client_install_path') + 'client2'
})
server_exec_location = runtime.get_active_config('server_exec_path')
if (":" in server_exec_location):
server_exec = server_exec_location
else:
server_exec = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
runtime.get_active_config('server_exec_path'))
global server_deployer
server_deployer = adhoc_deployer.SSHDeployer(
"AdditionServer", {
'pid_keyword': "AdditionServer",
'executable': server_exec,
'start_command': runtime.get_active_config('server_start_command')
})
runtime.set_deployer("AdditionServer", server_deployer)
server_deployer.deploy(
"server1", {
"hostname": "localhost",
"install_path":
runtime.get_active_config('server_install_path') + 'server1',
"args": "localhost 8000".split()
})
server_deployer.deploy(
"server2", {
"hostname": "localhost",
"install_path":
runtime.get_active_config('server_install_path') + 'server2',
"args": "localhost 8001".split()
})
server_deployer.deploy(
"server3", {
"hostname": "localhost",
"install_path":
runtime.get_active_config('server_install_path') + 'server3',
"args": "localhost 8002".split()
})