Exemple #1
0
def machine_logs():
  log_paths = {}

  # Attach proper path to all logs.
  for config_prefix in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']:
    deployed_path = os.path.join(c('remote_install_path'), c(config_prefix + '_install_path'))
    relative_log_paths = c(config_prefix + '_logs')
    log_paths[config_prefix] = map(lambda l: os.path.join(deployed_path, l), relative_log_paths)

  return {
    'zookeeper_instance_0': log_paths['zookeeper'],
    'kafka_instance_0': log_paths['kafka'],
    'yarn_rm_instance_0': log_paths['yarn_rm'],
    'yarn_nm_instance_0': log_paths['yarn_nm'],
  }
def teardown_suite():
    samza_job_deployer.uninstall('tests')

    # Undeploy everything.
    for name, deployer in deployers.iteritems():
        for instance, host in c(name + '_hosts').iteritems():
            deployer.undeploy(instance)
Exemple #3
0
def teardown_suite():
  samza_job_deployer.uninstall('tests')

  # Undeploy everything.
  for name, deployer in deployers.iteritems():
    for instance, host in c(name + '_hosts').iteritems():
      deployer.undeploy(instance)
def teardown_suite():
    """
    Teardown method that will be run once by zopkio test_runner after all the integration tests.
    """
    for component in ['kafka', 'zookeeper']:
        deployer = deployers[component]
        for instance, host in c(component + '_hosts').iteritems():
            deployer.undeploy(instance)
Exemple #5
0
def machine_logs():
    log_paths = {}

    # Attach proper path to all logs.
    for config_prefix in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']:
        deployed_path = os.path.join(c('remote_install_path'),
                                     c(config_prefix + '_install_path'))
        relative_log_paths = c(config_prefix + '_logs')
        log_paths[config_prefix] = map(
            lambda l: os.path.join(deployed_path, l), relative_log_paths)

    return {
        'zookeeper_instance_0': log_paths['zookeeper'],
        'kafka_instance_0': log_paths['kafka'],
        'yarn_rm_instance_0': log_paths['yarn_rm'],
        'yarn_nm_instance_0': log_paths['yarn_nm'],
    }
def _deploy_components(components):
    """
    Install and start all the :param components through binaries in deployment directory.
    """

    global deployers

    for component in components:
        config = {
            'install_path':
            os.path.join(c('remote_install_path'),
                         c(component + '_install_path')),
            'executable':
            c(component + '_executable'),
            'post_install_cmds':
            c(component + '_post_install_cmds', []),
            'start_command':
            c(component + '_start_cmd'),
            'stop_command':
            c(component + '_stop_cmd'),
            'extract':
            True,
            'sync':
            True,
        }
        deployer = adhoc_deployer.SSHDeployer(component, config)
        deployers[component] = deployer
        for instance, host in c(component + '_hosts').iteritems():
            logger.info('Deploying {0} on host: {1}'.format(instance, host))
            deployer.start(instance, {'hostname': host})
            time.sleep(5)
def _download_packages():
    for url_key in ['url_hadoop', 'url_kafka', 'url_zookeeper']:
        logger.debug('Getting download URL for: {0}'.format(url_key))
        url = c(url_key)
        filename = os.path.basename(url)
        if os.path.exists(filename):
            logger.debug('Using cached file: {0}'.format(filename))
        else:
            logger.info('Downloading: {0}'.format(url))
            urllib.urlretrieve(url, filename)
Exemple #8
0
def _download_packages():
  for url_key in ['url_hadoop', 'url_kafka', 'url_zookeeper']:
    logger.debug('Getting download URL for: {0}'.format(url_key))
    url = c(url_key)
    filename = os.path.basename(url)
    if os.path.exists(filename):
      logger.debug('Using cached file: {0}'.format(filename))
    else:
      logger.info('Downloading: {0}'.format(url))
      urllib.urlretrieve(url, filename)
Exemple #9
0
def setup_suite():
    global deployers, samza_job_deployer, samza_install_path
    logger.info('Current working directory: {0}'.format(os.getcwd()))
    samza_install_path = os.path.join(c('remote_install_path'),
                                      c('samza_install_path'))

    _download_packages()

    deployers = {
        'zookeeper': _new_ssh_deployer('zookeeper'),
        'yarn_rm': _new_ssh_deployer('yarn_rm'),
        'yarn_nm': _new_ssh_deployer('yarn_nm'),
        'kafka': _new_ssh_deployer('kafka'),
    }

    # Enforce install order through list.
    for name in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']:
        deployer = deployers[name]
        runtime.set_deployer(name, deployer)
        for instance, host in c(name + '_hosts').iteritems():
            logger.info('Deploying {0} on host: {1}'.format(instance, host))
            deployer.deploy(instance, {'hostname': host})

    # Start the Samza jobs.
    samza_job_deployer = SamzaJobYarnDeployer({
        'yarn_site_template':
        c('yarn_site_template'),
        'yarn_driver_configs':
        c('yarn_driver_configs'),
        'yarn_nm_hosts':
        c('yarn_nm_hosts').values(),
        'install_path':
        samza_install_path,
    })

    samza_job_deployer.install('smoke_tests', {
        'executable': c('samza_executable'),
    })

    samza_job_deployer.start(
        'negate_number', {
            'package_id': 'smoke_tests',
            'config_factory': c('samza_config_factory'),
            'config_file': c('samza_config_file'),
            'install_path': samza_install_path,
        })
Exemple #10
0
def teardown_suite():
    # Stop the samza jobs.
    samza_job_deployer.stop('negate_number', {
        'package_id': 'smoke_tests',
        'install_path': samza_install_path,
    })

    samza_job_deployer.uninstall('smoke_tests')

    # Undeploy everything.
    for name, deployer in deployers.iteritems():
        for instance, host in c(name + '_hosts').iteritems():
            deployer.undeploy(instance)
def _download_components(components):
    """
    Download the :param components if unavailable in deployment directory using url defined in config.
    """

    for component in components:
        url_key = 'url_{0}'.format(component)
        url = c(url_key)
        filename = os.path.basename(url)
        if os.path.exists(filename):
            logger.debug('Using cached file: {0}.'.format(filename))
        else:
            logger.info('Downloading {0} from {1}.'.format(component, url))
            urllib.urlretrieve(url, filename)
Exemple #12
0
def _new_ssh_deployer(config_prefix, name=None):
  deployer_name = config_prefix if name == None else name
  return adhoc_deployer.SSHDeployer(deployer_name, {
    'install_path': os.path.join(c('remote_install_path'), c(config_prefix + '_install_path')),
    'executable': c(config_prefix + '_executable'),
    'post_install_cmds': c(config_prefix + '_post_install_cmds', []),
    'start_command': c(config_prefix + '_start_cmd'),
    'stop_command': c(config_prefix + '_stop_cmd'),
    'extract': True,
    'sync': True,
  })
Exemple #13
0
def setup_suite():
  global deployers, samza_job_deployer, samza_install_path
  logger.info('Current working directory: {0}'.format(os.getcwd()))
  samza_install_path = os.path.join(c('remote_install_path'), c('samza_install_path'))

  _download_packages()

  deployers = {
    'zookeeper': _new_ssh_deployer('zookeeper'),
    'yarn_rm': _new_ssh_deployer('yarn_rm'),
    'yarn_nm': _new_ssh_deployer('yarn_nm'),
    'kafka': _new_ssh_deployer('kafka'),
  }

  # Enforce install order through list.
  for name in ['zookeeper', 'yarn_rm', 'yarn_nm', 'kafka']:
    deployer = deployers[name]
    runtime.set_deployer(name, deployer)
    for instance, host in c(name + '_hosts').iteritems():
      logger.info('Deploying {0} on host: {1}'.format(instance, host))
      deployer.deploy(instance, {
        'hostname': host
      })

  # Setup Samza job deployer.
  samza_job_deployer = SamzaJobYarnDeployer({
    'config_factory': c('samza_config_factory'),
    'yarn_site_template': c('yarn_site_template'),
    'yarn_driver_configs': c('yarn_driver_configs'),
    'yarn_nm_hosts': c('yarn_nm_hosts').values(),
    'install_path': samza_install_path,
  })

  samza_job_deployer.install('tests', {
    'executable': c('samza_executable'),
  })

  runtime.set_deployer('samza_job_deployer', samza_job_deployer)
def _new_ssh_deployer(config_prefix, name=None):
    deployer_name = config_prefix if name == None else name
    return adhoc_deployer.SSHDeployer(
        deployer_name, {
            'install_path':
            os.path.join(c('remote_install_path'),
                         c(config_prefix + '_install_path')),
            'executable':
            c(config_prefix + '_executable'),
            'post_install_cmds':
            c(config_prefix + '_post_install_cmds', []),
            'start_command':
            c(config_prefix + '_start_cmd'),
            'stop_command':
            c(config_prefix + '_stop_cmd'),
            'extract':
            True,
            'sync':
            True,
        })