def machine_logs():
log_paths = {}
# Attach proper path to all logs.
for config_prefix in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']:
deployed_path = os.path.join(c('remote_install_path'), c(config_prefix + '_install_path'))
relative_log_paths = c(config_prefix + '_logs')
log_paths[config_prefix] = map(lambda l: os.path.join(deployed_path, l), relative_log_paths)
return {
'zookeeper_instance_0': log_paths['zookeeper'],
'kafka_instance_0': log_paths['kafka'],
'yarn_rm_instance_0': log_paths['yarn_rm'],
'yarn_nm_instance_0': log_paths['yarn_nm'],
}
def teardown_suite():
samza_job_deployer.uninstall('tests')
# Undeploy everything.
for name, deployer in deployers.iteritems():
for instance, host in c(name + '_hosts').iteritems():
deployer.undeploy(instance)
def teardown_suite():
samza_job_deployer.uninstall('tests')
# Undeploy everything.
for name, deployer in deployers.iteritems():
for instance, host in c(name + '_hosts').iteritems():
deployer.undeploy(instance)
def teardown_suite():
"""
Teardown method that will be run once by zopkio test_runner after all the integration tests.
"""
for component in ['kafka', 'zookeeper']:
deployer = deployers[component]
for instance, host in c(component + '_hosts').iteritems():
deployer.undeploy(instance)
def machine_logs():
log_paths = {}
# Attach proper path to all logs.
for config_prefix in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']:
deployed_path = os.path.join(c('remote_install_path'),
c(config_prefix + '_install_path'))
relative_log_paths = c(config_prefix + '_logs')
log_paths[config_prefix] = map(
lambda l: os.path.join(deployed_path, l), relative_log_paths)
return {
'zookeeper_instance_0': log_paths['zookeeper'],
'kafka_instance_0': log_paths['kafka'],
'yarn_rm_instance_0': log_paths['yarn_rm'],
'yarn_nm_instance_0': log_paths['yarn_nm'],
}
def _deploy_components(components):
"""
Install and start all the :param components through binaries in deployment directory.
"""
global deployers
for component in components:
config = {
'install_path':
os.path.join(c('remote_install_path'),
c(component + '_install_path')),
'executable':
c(component + '_executable'),
'post_install_cmds':
c(component + '_post_install_cmds', []),
'start_command':
c(component + '_start_cmd'),
'stop_command':
c(component + '_stop_cmd'),
'extract':
True,
'sync':
True,
}
deployer = adhoc_deployer.SSHDeployer(component, config)
deployers[component] = deployer
for instance, host in c(component + '_hosts').iteritems():
logger.info('Deploying {0} on host: {1}'.format(instance, host))
deployer.start(instance, {'hostname': host})
time.sleep(5)
def _download_packages():
for url_key in ['url_hadoop', 'url_kafka', 'url_zookeeper']:
logger.debug('Getting download URL for: {0}'.format(url_key))
url = c(url_key)
filename = os.path.basename(url)
if os.path.exists(filename):
logger.debug('Using cached file: {0}'.format(filename))
else:
logger.info('Downloading: {0}'.format(url))
urllib.urlretrieve(url, filename)
def _download_packages():
for url_key in ['url_hadoop', 'url_kafka', 'url_zookeeper']:
logger.debug('Getting download URL for: {0}'.format(url_key))
url = c(url_key)
filename = os.path.basename(url)
if os.path.exists(filename):
logger.debug('Using cached file: {0}'.format(filename))
else:
logger.info('Downloading: {0}'.format(url))
urllib.urlretrieve(url, filename)
def setup_suite():
global deployers, samza_job_deployer, samza_install_path
logger.info('Current working directory: {0}'.format(os.getcwd()))
samza_install_path = os.path.join(c('remote_install_path'),
c('samza_install_path'))
_download_packages()
deployers = {
'zookeeper': _new_ssh_deployer('zookeeper'),
'yarn_rm': _new_ssh_deployer('yarn_rm'),
'yarn_nm': _new_ssh_deployer('yarn_nm'),
'kafka': _new_ssh_deployer('kafka'),
}
# Enforce install order through list.
for name in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']:
deployer = deployers[name]
runtime.set_deployer(name, deployer)
for instance, host in c(name + '_hosts').iteritems():
logger.info('Deploying {0} on host: {1}'.format(instance, host))
deployer.deploy(instance, {'hostname': host})
# Start the Samza jobs.
samza_job_deployer = SamzaJobYarnDeployer({
'yarn_site_template':
c('yarn_site_template'),
'yarn_driver_configs':
c('yarn_driver_configs'),
'yarn_nm_hosts':
c('yarn_nm_hosts').values(),
'install_path':
samza_install_path,
})
samza_job_deployer.install('smoke_tests', {
'executable': c('samza_executable'),
})
samza_job_deployer.start(
'negate_number', {
'package_id': 'smoke_tests',
'config_factory': c('samza_config_factory'),
'config_file': c('samza_config_file'),
'install_path': samza_install_path,
})
def teardown_suite():
# Stop the samza jobs.
samza_job_deployer.stop('negate_number', {
'package_id': 'smoke_tests',
'install_path': samza_install_path,
})
samza_job_deployer.uninstall('smoke_tests')
# Undeploy everything.
for name, deployer in deployers.iteritems():
for instance, host in c(name + '_hosts').iteritems():
deployer.undeploy(instance)
def _download_components(components):
"""
Download the :param components if unavailable in deployment directory using url defined in config.
"""
for component in components:
url_key = 'url_{0}'.format(component)
url = c(url_key)
filename = os.path.basename(url)
if os.path.exists(filename):
logger.debug('Using cached file: {0}.'.format(filename))
else:
logger.info('Downloading {0} from {1}.'.format(component, url))
urllib.urlretrieve(url, filename)
def _new_ssh_deployer(config_prefix, name=None):
deployer_name = config_prefix if name == None else name
return adhoc_deployer.SSHDeployer(deployer_name, {
'install_path': os.path.join(c('remote_install_path'), c(config_prefix + '_install_path')),
'executable': c(config_prefix + '_executable'),
'post_install_cmds': c(config_prefix + '_post_install_cmds', []),
'start_command': c(config_prefix + '_start_cmd'),
'stop_command': c(config_prefix + '_stop_cmd'),
'extract': True,
'sync': True,
})
def setup_suite():
global deployers, samza_job_deployer, samza_install_path
logger.info('Current working directory: {0}'.format(os.getcwd()))
samza_install_path = os.path.join(c('remote_install_path'), c('samza_install_path'))
_download_packages()
deployers = {
'zookeeper': _new_ssh_deployer('zookeeper'),
'yarn_rm': _new_ssh_deployer('yarn_rm'),
'yarn_nm': _new_ssh_deployer('yarn_nm'),
'kafka': _new_ssh_deployer('kafka'),
}
# Enforce install order through list.
for name in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']:
deployer = deployers[name]
runtime.set_deployer(name, deployer)
for instance, host in c(name + '_hosts').iteritems():
logger.info('Deploying {0} on host: {1}'.format(instance, host))
deployer.deploy(instance, {
'hostname': host
})
# Setup Samza job deployer.
samza_job_deployer = SamzaJobYarnDeployer({
'config_factory': c('samza_config_factory'),
'yarn_site_template': c('yarn_site_template'),
'yarn_driver_configs': c('yarn_driver_configs'),
'yarn_nm_hosts': c('yarn_nm_hosts').values(),
'install_path': samza_install_path,
})
samza_job_deployer.install('tests', {
'executable': c('samza_executable'),
})
runtime.set_deployer('samza_job_deployer', samza_job_deployer)
def _new_ssh_deployer(config_prefix, name=None):
deployer_name = config_prefix if name == None else name
return adhoc_deployer.SSHDeployer(
deployer_name, {
'install_path':
os.path.join(c('remote_install_path'),
c(config_prefix + '_install_path')),
'executable':
c(config_prefix + '_executable'),
'post_install_cmds':
c(config_prefix + '_post_install_cmds', []),
'start_command':
c(config_prefix + '_start_cmd'),
'stop_command':
c(config_prefix + '_stop_cmd'),
'extract':
True,
'sync':
True,
})
