def setup_suite():
client_exec = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"AdditionClient/out/artifacts/AdditionClient_jar/AdditionClient.jar")
global client_deployer
client_deployer = adhoc_deployer.SSHDeployer(
"AdditionClient", {
'pid_keyword': "AdditionClient",
'executable': client_exec,
'start_command': "java -jar AdditionClient.jar"
})
runtime.set_deployer("AdditionClient", client_deployer)
client_deployer.install(
"client1", {
"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionClients/client1"
})
client_deployer.install(
"client2", {
"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionClients/client2"
})
server_exec = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"AdditionServer/out/artifacts/AdditionServer_jar/AdditionServer.jar")
global server_deployer
server_deployer = adhoc_deployer.SSHDeployer(
"AdditionServer", {
'pid_keyword': "AdditionServer",
'executable': server_exec,
'start_command': "java -jar AdditionServer.jar"
})
runtime.set_deployer("AdditionServer", server_deployer)
server_deployer.deploy(
"server1", {
"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionServers/server1",
"args": "localhost 8000".split()
})
server_deployer.deploy(
"server2", {
"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionServers/server2",
"args": "localhost 8001".split()
})
server_deployer.deploy(
"server3", {
"hostname": "localhost",
"install_path": "/tmp/server_client/AdditionServers/server3",
"args": "localhost 8002".split()
})
def __init__(self, host_name, processor_id):
"""
:param host_name: Represents the host name in which this StreamProcessor will run.
:param processor_id: Represents the processor_id of StreamProcessor.
"""
start_cmd = 'export SAMZA_LOG_DIR=\"deploy/{0}\"; export JAVA_OPTS=\"$JAVA_OPTS -Xmx2G\"; ./bin/run-class.sh org.apache.samza.test.integration.LocalApplicationRunnerMain --config-path ./config/standalone.failure.test.properties --operation run --config processor.id={0} >> /tmp/{0}.log &'
self.username = runtime.get_username()
self.password = runtime.get_password()
self.processor_id = processor_id
self.host_name = host_name
self.processor_start_command = start_cmd.format(self.processor_id)
logger.info('Running processor start command: {0}'.format(
self.processor_start_command))
self.deployment_config = {
'install_path':
os.path.join(runtime.get_active_config('remote_install_path'),
'deploy/{0}'.format(self.processor_id)),
'executable':
'samza-test_2.11-1.0.0.tgz',
'post_install_cmds': [],
'start_command':
self.processor_start_command,
'stop_command':
'',
'extract':
True,
'sync':
True,
}
self.deployer = adhoc_deployer.SSHDeployer(self.processor_id,
self.deployment_config)
def test_zip_executable(self):
test_dir = os.path.dirname(os.path.abspath(__file__))
executable = os.path.join(test_dir, "samples/trivial_program.zip")
zf = zipfile.ZipFile(executable, "w")
try:
zf.write(os.path.join(test_dir, "samples/trivial_program"),
"samples/trivial_program", zipfile.ZIP_DEFLATED)
finally:
zf.close()
install_path = "/tmp/zip_test/"
start_cmd = "chmod a+x samples/trivial_program; ./samples/trivial_program"
files_to_clean = ["/tmp/trivial_output"]
zip_deployer = adhoc_deployer.SSHDeployer(
"samples/trivial_program", {
'executable': executable,
'extract': True,
'install_path': install_path,
'start_command': start_cmd,
'directories_to_remove': files_to_clean
})
zip_deployer.deploy("samples/trivial_program",
{'hostname': "localhost"})
self.assertNotEqual(
zip_deployer.get_pid("samples/trivial_program",
{'pid_keyword': 'samples/trivial_program'}),
constants.PROCESS_NOT_RUNNING_PID)
zip_deployer.undeploy("samples/trivial_program",
{'pid_keyword': 'samples/trivial_program'})
os.remove(executable)
def setup_suite(self):
print "Starting zookeeper"
env_dict = {}
if "localhost" not in runtime.get_active_config('zookeeper_host'):
env_dict = {'JAVA_HOME': '/export/apps/jdk/current'}
zookeeper_deployer = adhoc_deployer.SSHDeployer(
"zookeeper", {
'pid_keyword':
"zookeeper",
'executable':
runtime.get_active_config('zookeeper_exec_location'),
'env':
env_dict,
'extract':
True,
'post_install_cmds':
runtime.get_active_config('zookeeper_post_install_cmds'),
'stop_command':
runtime.get_active_config('zookeeper_stop_command'),
'start_command':
runtime.get_active_config('zookeeper_start_command')
})
runtime.set_deployer("zookeeper", zookeeper_deployer)
zookeeper_deployer.install(
"zookeeper", {
"hostname": runtime.get_active_config('zookeeper_host'),
"install_path": "/tmp/zookeeper_test"
})
zookeeper_deployer.start("zookeeper", configs={"sync": True})
def _deploy_components(components):
"""
Install and start all the :param components through binaries in deployment directory.
"""
global deployers
for component in components:
config = {
'install_path':
os.path.join(c('remote_install_path'),
c(component + '_install_path')),
'executable':
c(component + '_executable'),
'post_install_cmds':
c(component + '_post_install_cmds', []),
'start_command':
c(component + '_start_cmd'),
'stop_command':
c(component + '_stop_cmd'),
'extract':
True,
'sync':
True,
}
deployer = adhoc_deployer.SSHDeployer(component, config)
deployers[component] = deployer
for instance, host in c(component + '_hosts').iteritems():
logger.info('Deploying {0} on host: {1}'.format(instance, host))
deployer.start(instance, {'hostname': host})
time.sleep(5)
def setup_suite():
print "Starting zookeeper quorum"
global zookeper_deployer
env_dict = {}
if "localhost" not in runtime.get_active_config('zookeeper_host'):
env_dict = {'JAVA_HOME':'/export/apps/jdk/current'}
zookeper_deployer = adhoc_deployer.SSHDeployer("zookeeper",
{'pid_keyword': "zookeeper",
'executable': runtime.get_active_config('zookeeper_exec_location'),
'env':env_dict,
'extract': True,
'stop_command':runtime.get_active_config('zookeeper_stop_command'),
'start_command': runtime.get_active_config('zookeeper_start_command')})
runtime.set_deployer("zookeeper", zookeper_deployer)
# Deploy Zookeeper1
print "Deploy Zookeeper1"
zookeper_deployer.install("zookeeper1",
{"hostname": "localhost",
"install_path": runtime.get_active_config('zookeeper1_install_path'),
"pid_file": runtime.get_active_config('zookeeper1_pid_file'),
'post_install_cmds':runtime.get_active_config('zookeeper1_post_install_cmds')})
zookeper_deployer.start("zookeeper1",configs={"sync": True})
# Deploy Zookeeper2
print "Deploy Zookeeper2"
zookeper_deployer.install("zookeeper2",
{"hostname": "localhost",
"install_path": runtime.get_active_config('zookeeper2_install_path'),
"pid_file": runtime.get_active_config('zookeeper2_pid_file'),
'post_install_cmds':runtime.get_active_config('zookeeper2_post_install_cmds')})
zookeper_deployer.start("zookeeper2",configs={"sync": True})
# Deploy Zookeeper3
print "Deploy Zookeeper3"
zookeper_deployer.install("zookeeper3",
{"hostname": "localhost",
"install_path": runtime.get_active_config('zookeeper3_install_path'),
"pid_file": runtime.get_active_config('zookeeper3_pid_file'),
'post_install_cmds':runtime.get_active_config('zookeeper3_post_install_cmds')})
zookeper_deployer.start("zookeeper3",configs={"sync": True})
def _new_ssh_deployer(config_prefix, name=None):
deployer_name = config_prefix if name == None else name
return adhoc_deployer.SSHDeployer(
deployer_name, {
'install_path':
os.path.join(c('remote_install_path'),
c(config_prefix + '_install_path')),
'executable':
c(config_prefix + '_executable'),
'post_install_cmds':
c(config_prefix + '_post_install_cmds', []),
'start_command':
c(config_prefix + '_start_cmd'),
'stop_command':
c(config_prefix + '_stop_cmd'),
'extract':
True,
'sync':
True,
})
def setUp(self):
test_dir = os.path.dirname(os.path.abspath(__file__))
executable = os.path.join(test_dir, "samples/trivial_program")
install_path = "/tmp/ssh_deployer_test/"
start_cmd = "chmod a+x trivial_program; ./trivial_program"
files_to_clean = ["/tmp/trivial_output"]
self.trivial_deployer = adhoc_deployer.SSHDeployer(
"sample_program", {
'executable':
executable,
'install_path':
install_path,
'start_command':
start_cmd,
'directories_to_clean':
files_to_clean,
'terminate_only':
True,
'pid_keyword':
'trivial_program',
'post_install_cmds':
['touch {0}/trivial-post-cmd'.format(install_path)]
})
def setup_suite():
# Set up authentication
username = runtime.get_active_config("username")
password = runtime.get_active_config("password")
runtime.set_user(username, password)
# Set up the target directories and properties
userdir = os.path.join("/home", username)
riaps_app_path = os.path.join(userdir,
runtime.get_active_config("riaps_apps_path"))
test_app_path = os.path.join(riaps_app_path,
runtime.get_active_config('app_dir'))
start_actor_path = "/opt/riaps/armhf/bin/"
env = {
"PATH": "/usr/local/bin/:" + start_actor_path + ":$PATH",
"RIAPSHOME": "/usr/local/riaps",
"RIAPSAPPS": "$HOME/riaps_apps",
"LD_LIBRARY_PATH": "/opt/riaps/armhf/lib:" + test_app_path
}
#start_riaps_lang = "riaps_lang " + runtime.get_active_config('model_file')
# Set up the sources
model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
runtime.get_active_config('app_dir'),
runtime.get_active_config('model_file'))
# Script to check discovery service
discoCheckScript = "checkDiscoveryService.py"
discoCheckScriptPath = "../../test_common"
# Script to start the discovery
discoStartScript = "startDiscovery.py"
discoStartScriptPath = "../../test_common"
# Script to stop the discovery
discoStopScript = "stopDiscovery.py"
discoStopScriptPath = "../../test_common"
killRiapsScript = "killRiaps.py"
killRiapsScriptPath = "../../test_common"
# Deploy the riaps killer script
for target in runtime.get_active_config('targets'):
deployerId = "killer_" + target["host"]
killscriptpath = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
killRiapsScriptPath, killRiapsScript))
killDeployer = adhoc_deployer.SSHDeployer(
deployerId, {
'executable':
killscriptpath,
'install_path':
riaps_app_path,
'env':
env,
'hostname':
target["host"],
"start_command":
"python3 " + os.path.join(riaps_app_path, killRiapsScript)
})
runtime.set_deployer(deployerId, killDeployer)
killDeployer.install(deployerId)
# Deploy the discovery starter script
for target in runtime.get_active_config('targets'):
deployerId = "discostart_" + target["host"]
startscriptpath = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
discoStartScriptPath, discoStartScript))
startDiscoveryDeployer = adhoc_deployer.SSHDeployer(
deployerId, {
'executable':
startscriptpath,
'install_path':
riaps_app_path,
'env':
env,
'hostname':
target["host"],
"start_command":
"python3 " + os.path.join(riaps_app_path, discoStartScript)
})
runtime.set_deployer(deployerId, startDiscoveryDeployer)
startDiscoveryDeployer.install(deployerId)
# Deploy the discovery stop script
for target in runtime.get_active_config('targets'):
deployerId = "discostop_" + target["host"]
stopscriptpath = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
discoStopScriptPath, discoStopScript))
stopDiscoveryDeployer = adhoc_deployer.SSHDeployer(
deployerId, {
'executable':
stopscriptpath,
'install_path':
riaps_app_path,
'env':
env,
'hostname':
target["host"],
"start_command":
"python3 " + os.path.join(riaps_app_path, discoStopScript)
})
runtime.set_deployer(deployerId, stopDiscoveryDeployer)
stopDiscoveryDeployer.install(deployerId)
# Deploy the riaps-disco checker script
for target in runtime.get_active_config('targets'):
deployerId = "disco" + target["host"]
checkscriptpath = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
discoCheckScriptPath, discoCheckScript))
checkDiscoDeployer = adhoc_deployer.SSHDeployer(
deployerId, {
'executable':
checkscriptpath,
'install_path':
riaps_app_path,
'env':
env,
'hostname':
target["host"],
"start_command":
"python3 " +
os.path.join(riaps_app_path, "checkDiscoveryService.py")
})
runtime.set_deployer(deployerId, checkDiscoDeployer)
checkDiscoDeployer.install(deployerId)
# Deploy the riaps-components/model file
local_riaps_lang = "riaps_lang " + model_path
local_test_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
runtime.get_active_config('app_dir'))
subprocess.call(local_riaps_lang, shell=True)
move_cmd = "mv " + runtime.get_active_config(
'app_dir') + '.json ' + local_test_dir
subprocess.call(move_cmd, shell=True)
local_model_json = os.path.join(
local_test_dir,
runtime.get_active_config('app_dir') + '.json')
for target in runtime.get_active_config('targets'):
model_deployer = adhoc_deployer.SSHDeployer(
target["actor"],
{
'executable': local_model_json,
'install_path': test_app_path,
'hostname': target["host"],
#'start_command': os.path.join(start_actor_path, "start_actor"),
'start_command': "start_actor",
#'args': [os.path.join(test_app_path, runtime.get_active_config('app_dir') + '.json'),
# target["actor"]],
'env': env
#'terminate_only': False,
#'pid_keyword': model_path,
})
runtime.set_deployer(target["actor"], model_deployer)
# Add test cases
testcases = [
"pubfirst_" + target["actor"], "subfirst_" + target["actor"]
]
for testcase in testcases:
model_deployer.install(
testcase, {
'args': [
os.path.join(
test_app_path,
runtime.get_active_config('app_dir') + '.json'),
target["actor"], '--logfile="' + testcase + '.log"'
]
})
for component in runtime.get_active_config('components_so'):
localPath = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
runtime.get_active_config('app_dir'), component)
#localPath = os.path.join(os.path.dirname(os.path.abspath(__file__)),
# runtime.get_active_config('app_dir'),
# "lib",
# component)
#targetPath = target_path#os.path.join(riaps_app_path, runtime.get_active_config('app_dir'))
component_deployer = adhoc_deployer.SSHDeployer(
component,
{
'executable': localPath,
'install_path': test_app_path,
'hostname': target["host"],
#'start_command': "start_actor", #os.path.join(target_path, "start_actor"),
#'args': [runtime.get_active_config('app_dir'),
# runtime.get_active_config('app_dir') + '.json',
# target["actor"]],
'env': env,
#'terminate_only': True,
#'pid_keyword': component,
})
runtime.set_deployer(component, component_deployer)
component_deployer.install(component)
print("Deployment done.")
def setup_suite():
client_exec_location = runtime.get_active_config('client_exec_path')
#Clients and server exec can also be in remote location.
#if so specify the client_exec as <Remoteserver>:<path>
#In that case you can also path a temp_scratch config for the sftp
#else /tmp folder used as default
if (":" in client_exec_location):
client_exec = client_exec_location
else:
client_exec = os.path.join(os.path.dirname(os.path.abspath(__file__)),
client_exec_location)
global client_deployer
client_deployer = adhoc_deployer.SSHDeployer(
"AdditionClient", {
'pid_keyword': "AdditionClient",
'executable': client_exec,
'start_command': runtime.get_active_config('client_start_command')
})
runtime.set_deployer("AdditionClient", client_deployer)
client_deployer.install(
"client1", {
"hostname":
"localhost",
"install_path":
runtime.get_active_config('client_install_path') + 'client1'
})
client_deployer.install(
"client2", {
"hostname":
"localhost",
"install_path":
runtime.get_active_config('client_install_path') + 'client2'
})
server_exec_location = runtime.get_active_config('server_exec_path')
if (":" in server_exec_location):
server_exec = server_exec_location
else:
server_exec = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
runtime.get_active_config('server_exec_path'))
global server_deployer
server_deployer = adhoc_deployer.SSHDeployer(
"AdditionServer", {
'pid_keyword': "AdditionServer",
'executable': server_exec,
'start_command': runtime.get_active_config('server_start_command')
})
runtime.set_deployer("AdditionServer", server_deployer)
server_deployer.deploy(
"server1", {
"hostname": "localhost",
"install_path":
runtime.get_active_config('server_install_path') + 'server1',
"args": "localhost 8000".split()
})
server_deployer.deploy(
"server2", {
"hostname": "localhost",
"install_path":
runtime.get_active_config('server_install_path') + 'server2',
"args": "localhost 8001".split()
})
server_deployer.deploy(
"server3", {
"hostname": "localhost",
"install_path":
runtime.get_active_config('server_install_path') + 'server3',
"args": "localhost 8002".split()
})
