def _install_influxdb():

    influxdb_source_url = ctx.node.properties['influxdb_rpm_source_url']

    influxdb_user = '******'
    influxdb_group = 'influxdb'
    influxdb_home = '/opt/influxdb'
    influxdb_log_path = '/var/log/cloudify/influxdb'

    ctx.logger.info('Installing InfluxDB...')
    utils.set_selinux_permissive()

    utils.copy_notice('influxdb')
    utils.mkdir(influxdb_home)
    utils.mkdir(influxdb_log_path)

    utils.yum_install(influxdb_source_url)
    utils.sudo(['rm', '-rf', '/etc/init.d/influxdb'])

    ctx.logger.info('Deploying InfluxDB config.toml...')
    utils.deploy_blueprint_resource(
        '{0}/config.toml'.format(CONFIG_PATH),
        '{0}/shared/config.toml'.format(influxdb_home))

    ctx.logger.info('Fixing user permissions...')
    utils.chown(influxdb_user, influxdb_group, influxdb_home)
    utils.chown(influxdb_user, influxdb_group, influxdb_log_path)

    utils.logrotate('influxdb')
    utils.systemd.configure('influxdb')
示例#2
0
def dump_database():
    dumpfile = os.path.join(dconf.DB_DUMP_DIR, dconf.DB_NAME + '.dump')
    if dconf.DB_TYPE == 'oracle':
        if not dconf.ORACLE_FLASH_BACK and file_exists(dumpfile):
            LOG.info('%s already exists ! ', dumpfile)
            return False
    else:
        if file_exists(dumpfile):
            LOG.info('%s already exists ! ', dumpfile)
            return False

    if dconf.DB_TYPE == 'oracle' and dconf.ORACLE_FLASH_BACK:
        LOG.info('create restore point %s for database %s in %s', dconf.RESTORE_POINT,
                 dconf.DB_NAME, dconf.RECOVERY_FILE_DEST)
    else:
        LOG.info('Dump database %s to %s', dconf.DB_NAME, dumpfile)

    if dconf.DB_TYPE == 'oracle':
        if dconf.ORACLE_FLASH_BACK:
            run_sql_script('createRestore.sh', dconf.RESTORE_POINT,
                           dconf.RECOVERY_FILE_DEST_SIZE, dconf.RECOVERY_FILE_DEST)
        else:
            run_sql_script('dumpOracle.sh', dconf.DB_USER, dconf.DB_PASSWORD,
                           dconf.DB_NAME, dconf.DB_DUMP_DIR)

    elif dconf.DB_TYPE == 'postgres':
        run('PGPASSWORD={} pg_dump -U {} -h {} -F c -d {} > {}'.format(
            dconf.DB_PASSWORD, dconf.DB_USER, dconf.DB_HOST, dconf.DB_NAME,
            dumpfile))
    elif dconf.DB_TYPE == 'mysql':
        sudo('mysqldump --user={} --password={} --databases {} > {}'.format(
            dconf.DB_USER, dconf.DB_PASSWORD, dconf.DB_NAME, dumpfile))
    else:
        raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
    return True
def _install_influxdb():

    influxdb_source_url = ctx_properties['influxdb_rpm_source_url']

    influxdb_user = '******'
    influxdb_group = 'influxdb'
    influxdb_home = '/opt/influxdb'
    influxdb_log_path = '/var/log/cloudify/influxdb'

    ctx.logger.info('Installing InfluxDB...')
    utils.set_selinux_permissive()

    utils.copy_notice(INFLUX_SERVICE_NAME)
    utils.mkdir(influxdb_home)
    utils.mkdir(influxdb_log_path)

    utils.yum_install(influxdb_source_url, service_name=INFLUX_SERVICE_NAME)
    utils.sudo(['rm', '-rf', '/etc/init.d/influxdb'])

    ctx.logger.info('Deploying InfluxDB config.toml...')
    utils.deploy_blueprint_resource(
        '{0}/config.toml'.format(CONFIG_PATH),
        '{0}/shared/config.toml'.format(influxdb_home),
        INFLUX_SERVICE_NAME)

    ctx.logger.info('Fixing user permissions...')
    utils.chown(influxdb_user, influxdb_group, influxdb_home)
    utils.chown(influxdb_user, influxdb_group, influxdb_log_path)

    utils.systemd.configure(INFLUX_SERVICE_NAME)
    # Provided with InfluxDB's package. Will be removed if it exists.
    utils.remove('/etc/init.d/influxdb')
    utils.logrotate(INFLUX_SERVICE_NAME)
示例#4
0
def _install_influxdb():

    influxdb_source_url = ctx_properties['influxdb_rpm_source_url']

    influxdb_user = '******'
    influxdb_group = 'influxdb'
    influxdb_home = '/opt/influxdb'
    influxdb_log_path = '/var/log/cloudify/influxdb'

    ctx.logger.info('Installing InfluxDB...')
    utils.set_selinux_permissive()

    utils.copy_notice(INFLUX_SERVICE_NAME)
    utils.mkdir(influxdb_home)
    utils.mkdir(influxdb_log_path)

    utils.yum_install(influxdb_source_url, service_name=INFLUX_SERVICE_NAME)
    utils.sudo(['rm', '-rf', '/etc/init.d/influxdb'])

    ctx.logger.info('Deploying InfluxDB config.toml...')
    utils.deploy_blueprint_resource(
        '{0}/config.toml'.format(CONFIG_PATH),
        '{0}/shared/config.toml'.format(influxdb_home), INFLUX_SERVICE_NAME)

    ctx.logger.info('Fixing user permissions...')
    utils.chown(influxdb_user, influxdb_group, influxdb_home)
    utils.chown(influxdb_user, influxdb_group, influxdb_log_path)

    utils.systemd.configure(INFLUX_SERVICE_NAME)
    utils.logrotate(INFLUX_SERVICE_NAME)
示例#5
0
def install_system_packages():
    """
    Perform the installation of system-level packages needed by NGAS to work.
    """

    # Install required packages
    linux_flavor = get_linux_flavor()
    if (linux_flavor in ['CentOS','Amazon Linux']):
        # Update the machine completely
        errmsg = sudo('yum --assumeyes --quiet update', combine_stderr=True, warn_only=True)
        processCentOSErrMsg(errmsg)
        install_yum(YUM_PACKAGES)
        if linux_flavor == 'CentOS':
            sudo('/etc/init.d/iptables stop') # CentOS firewall blocks NGAS port!
    elif (linux_flavor in ['Ubuntu', 'Debian']):
        errmsg = sudo('apt-get -qq -y update', combine_stderr=True, warn_only=True)
        install_apt(APT_PACKAGES)
    elif linux_flavor in ['SUSE','SLES-SP2', 'SLES-SP3', 'SLES', 'openSUSE']:
        errmsg = sudo('zypper -n -q patch', combine_stderr=True, warn_only=True)
        install_zypper(SLES_PACKAGES)
    elif linux_flavor == 'Darwin':
        pkg_mgr = check_brew_port()
        if pkg_mgr == None:
            install_homebrew()
            pkg_mgr = 'brew'
        if pkg_mgr == 'brew':
            for package in BREW_PACKAGES:
                install_brew(package)
        elif pkg_mgr == 'port':
            for package in PORT_PACKAGES:
                install_port(package)
    else:
        abort("Unsupported linux flavor detected: {0}".format(linux_flavor))
示例#6
0
def _install_stage():
    nodejs_source_url = ctx_properties['nodejs_tar_source_url']
    stage_source_url = ctx_properties['stage_tar_source_url']

    if not utils.resource_factory.local_resource_exists(stage_source_url):
        ctx.logger.info('Stage package not found in manager resources '
                        'package. Stage will not be installed.')
        ctx.instance.runtime_properties['skip_installation'] = 'true'
        return

    # injected as an input to the script
    ctx.instance.runtime_properties['influxdb_endpoint_ip'] = \
        os.environ.get('INFLUXDB_ENDPOINT_IP')

    utils.set_selinux_permissive()
    utils.copy_notice(SERVICE_NAME)

    utils.mkdir(NODEJS_DIR)
    utils.mkdir(HOME_DIR)
    utils.mkdir(LOG_DIR)

    utils.create_service_user(STAGE_USER, STAGE_GROUP, HOME_DIR)

    ctx.logger.info('Installing NodeJS...')
    nodejs = utils.download_cloudify_resource(nodejs_source_url, SERVICE_NAME)
    utils.untar(nodejs, NODEJS_DIR)
    utils.remove(nodejs)

    ctx.logger.info('Installing Cloudify Stage (UI)...')
    stage_tar = utils.download_cloudify_resource(stage_source_url,
                                                 SERVICE_NAME)
    if 'community' in stage_tar:
        ctx.logger.info('Community edition')
        ctx.instance.runtime_properties['community_mode'] = '-mode community'
    else:
        ctx.instance.runtime_properties['community_mode'] = ''

    utils.untar(stage_tar, HOME_DIR)
    utils.remove(stage_tar)

    ctx.logger.info('Fixing permissions...')
    utils.chown(STAGE_USER, STAGE_GROUP, HOME_DIR)
    utils.chown(STAGE_USER, STAGE_GROUP, NODEJS_DIR)
    utils.chown(STAGE_USER, STAGE_GROUP, LOG_DIR)
    utils.deploy_sudo_command_script(
        'restore-snapshot.py',
        'Restore stage directories from a snapshot path',
        component=SERVICE_NAME,
        allow_as=STAGE_USER)
    utils.chmod('a+rx', '/opt/cloudify/stage/restore-snapshot.py')
    utils.sudo(['usermod', '-aG', utils.CLOUDIFY_GROUP, STAGE_USER])

    utils.logrotate(SERVICE_NAME)
    utils.systemd.configure(SERVICE_NAME)

    backend_dir = join(HOME_DIR, 'backend')
    npm_path = join(NODEJS_DIR, 'bin', 'npm')
    subprocess.check_call('cd {0}; {1} run db-migrate'.format(
        backend_dir, npm_path),
                          shell=True)
示例#7
0
def _install_rabbitmq():
    erlang_rpm_source_url = ctx.node.properties['erlang_rpm_source_url']
    rabbitmq_rpm_source_url = ctx.node.properties['rabbitmq_rpm_source_url']
    # TODO: maybe we don't need this env var
    os.putenv('RABBITMQ_FD_LIMIT',
              str(ctx.node.properties['rabbitmq_fd_limit']))
    rabbitmq_log_path = '/var/log/cloudify/rabbitmq'
    rabbitmq_username = ctx.node.properties['rabbitmq_username']
    rabbitmq_password = ctx.node.properties['rabbitmq_password']
    rabbitmq_cert_public = ctx.node.properties['rabbitmq_cert_public']
    rabbitmq_ssl_enabled = ctx.node.properties['rabbitmq_ssl_enabled']
    rabbitmq_cert_private = ctx.node.properties['rabbitmq_cert_private']

    ctx.logger.info('Installing RabbitMQ...')
    utils.set_selinux_permissive()

    utils.copy_notice('rabbitmq')
    utils.mkdir(rabbitmq_log_path)

    utils.yum_install(erlang_rpm_source_url)
    utils.yum_install(rabbitmq_rpm_source_url)

    utils.logrotate('rabbitmq')

    utils.deploy_blueprint_resource(
        '{0}/kill-rabbit'.format(CONFIG_PATH),
        '/usr/local/bin/kill-rabbit')
    utils.chmod('500', '/usr/local/bin/kill-rabbit')

    utils.systemd.configure('rabbitmq')

    ctx.logger.info('Configuring File Descriptors Limit...')
    utils.deploy_blueprint_resource(
        '{0}/rabbitmq_ulimit.conf'.format(CONFIG_PATH),
        '/etc/security/limits.d/rabbitmq.conf')

    utils.systemd.systemctl('daemon-reload')

    utils.chown('rabbitmq', 'rabbitmq', rabbitmq_log_path)

    utils.systemd.start('cloudify-rabbitmq')

    time.sleep(10)
    utils.wait_for_port(5672)

    ctx.logger.info('Enabling RabbitMQ Plugins...')
    # Occasional timing issues with rabbitmq starting have resulted in
    # failures when first trying to enable plugins
    utils.sudo(['rabbitmq-plugins', 'enable', 'rabbitmq_management'],
               retries=5)
    utils.sudo(['rabbitmq-plugins', 'enable', 'rabbitmq_tracing'], retries=5)

    _clear_guest_permissions_if_guest_exists()
    _create_user_and_set_permissions(rabbitmq_username, rabbitmq_password)
    _set_security(
        rabbitmq_ssl_enabled,
        rabbitmq_cert_private,
        rabbitmq_cert_public)

    utils.systemd.stop('cloudify-rabbitmq', retries=5)
def _deploy_security_configuration():
    ctx.logger.info('Deploying REST Security configuration file...')

    # Generating random hash salt and secret key
    security_configuration = {
        'hash_salt': base64.b64encode(os.urandom(32)),
        'secret_key': base64.b64encode(os.urandom(32)),
        'encoding_alphabet': _random_alphanumeric(),
        'encoding_block_size': 24,
        'encoding_min_length': 5
    }

    # Pre-creating paths so permissions fix can work correctly
    # in mgmtworker
    for path in utils.MANAGER_RESOURCES_SNAPSHOT_PATHS:
        utils.mkdir(path)
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, utils.MANAGER_RESOURCES_HOME)
    utils.sudo(['ls', '-la', '/opt/manager'])

    current_props = runtime_props['security_configuration']
    current_props.update(security_configuration)
    runtime_props['security_configuration'] = current_props

    for key in ['admin_username', 'admin_password']:
        security_configuration[key] = current_props[key]

    fd, path = tempfile.mkstemp()
    os.close(fd)
    with open(path, 'w') as f:
        json.dump(security_configuration, f)
    rest_security_path = join(runtime_props['home_dir'], 'rest-security.conf')
    utils.move(path, rest_security_path)
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, rest_security_path)
    utils.chmod('g+r', rest_security_path)
def _deploy_security_configuration():
    ctx.logger.info('Deploying REST Security configuration file...')

    # Generating random hash salt and secret key
    security_configuration = {
        'hash_salt': base64.b64encode(os.urandom(32)),
        'secret_key': base64.b64encode(os.urandom(32)),
        'encoding_alphabet': _random_alphanumeric(),
        'encoding_block_size': 24,
        'encoding_min_length': 5
    }

    # Pre-creating paths so permissions fix can work correctly
    # in mgmtworker
    for path in utils.MANAGER_RESOURCES_SNAPSHOT_PATHS:
        utils.mkdir(path)
    utils.chown(
        CLOUDIFY_USER, CLOUDIFY_GROUP,
        utils.MANAGER_RESOURCES_HOME)
    utils.sudo(['ls', '-la', '/opt/manager'])

    current_props = runtime_props['security_configuration']
    current_props.update(security_configuration)
    runtime_props['security_configuration'] = current_props

    fd, path = tempfile.mkstemp()
    os.close(fd)
    with open(path, 'w') as f:
        json.dump(security_configuration, f)
    rest_security_path = join(runtime_props['home_dir'], 'rest-security.conf')
    utils.move(path, rest_security_path)
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, rest_security_path)
示例#10
0
def restart_database():
    if dconf.DB_TYPE == 'postgres':
        if dconf.HOST_CONN == 'docker':
            # Restarting the docker container here is the cleanest way to do it
            # becaues there's no init system running and the only process running
            # in the container is postgres itself
            local('docker restart {}'.format(dconf.CONTAINER_NAME))
        else:
            sudo('pg_ctl -D {} -w -t 600 restart -m fast'.format(
                dconf.PG_DATADIR),
                 user=dconf.ADMIN_USER,
                 capture=False)
    elif dconf.DB_TYPE == 'oracle':
        db_log_path = os.path.join(
            os.path.split(dconf.DB_CONF)[0], 'startup.log')
        local_log_path = os.path.join(dconf.LOG_DIR, 'startup.log')
        local_logs_path = os.path.join(dconf.LOG_DIR, 'startups.log')
        run_sql_script('restartOracle.sh', db_log_path)
        get(db_log_path, local_log_path)
        with open(local_log_path, 'r') as fin, open(local_logs_path,
                                                    'a') as fout:
            lines = fin.readlines()
            for line in lines:
                if line.startswith('ORACLE instance started.'):
                    return True
                if not line.startswith('SQL>'):
                    fout.write(line)
            fout.write('\n')
        return False
    else:
        raise Exception("Database Type {} Not Implemented !".format(
            dconf.DB_TYPE))
    return True
示例#11
0
def set_rabbitmq_policy(name, expression, policy):
    policy = json.dumps(policy)
    ctx.logger.info('Setting policy {0} on queues {1} to {2}'.format(
        name, expression, policy))
    # shlex screws this up because we need to pass json and shlex
    # strips quotes so we explicitly pass it as a list.
    utils.sudo(['rabbitmqctl', 'set_policy', name,
               expression, policy, '--apply-to', 'queues'])
示例#12
0
def free_cache():
    if dconf.HOST_CONN not in ['docker', 'remote_docker']:
        with show('everything'), settings(warn_only=True):  # pylint: disable=not-context-manager
            res = sudo("sh -c \"echo 3 > /proc/sys/vm/drop_caches\"")
            if res.failed:
                LOG.error('%s (return code %s)', res.stderr.strip(), res.return_code)
    else:
        res = sudo("sh -c \"echo 3 > /proc/sys/vm/drop_caches\"", remote_only=True)
def _disable_requiretty():
    script_dest = '/tmp/configure_manager.sh'
    utils.deploy_blueprint_resource(
        'components/manager/scripts'
        '/configure_manager.sh', script_dest, NODE_NAME)

    utils.sudo('chmod +x {0}'.format(script_dest))
    utils.sudo(script_dest)
示例#14
0
def install_zypper(packages):
    """
    Install packages using zypper (SLES)
    """
    sudo('zypper --non-interactive install {0}'.format(
        ' '.join(packages + extra_packages())),
         combine_stderr=True,
         warn_only=True)
def set_rabbitmq_policy(name, expression, policy):
    policy = json.dumps(policy)
    ctx.logger.debug('Setting policy {0} on queues {1} to {2}'.format(
        name, expression, policy))
    # shlex screws this up because we need to pass json and shlex
    # strips quotes so we explicitly pass it as a list.
    utils.sudo(['rabbitmqctl', 'set_policy', name,
               expression, policy, '--apply-to', 'queues'])
示例#16
0
def install_logstash():

    logstash_unit_override = '/etc/systemd/system/logstash.service.d'

    logstash_source_url = ctx_properties['logstash_rpm_source_url']

    rabbitmq_username = ctx_properties['rabbitmq_username']
    rabbitmq_password = ctx_properties['rabbitmq_password']

    logstash_log_path = '/var/log/cloudify/logstash'
    logstash_conf_path = '/etc/logstash/conf.d'

    # injected as an input to the script
    ctx.instance.runtime_properties['es_endpoint_ip'] = \
        os.environ.get('ES_ENDPOINT_IP')
    ctx.instance.runtime_properties['rabbitmq_endpoint_ip'] = \
        utils.get_rabbitmq_endpoint_ip(
                ctx_properties.get('rabbitmq_endpoint_ip'))

    # Confirm username and password have been supplied for broker before
    # continuing.
    # Components other than logstash and riemann have this handled in code.
    # Note that these are not directly used in this script, but are used by the
    # deployed resources, hence the check here.
    if not rabbitmq_username or not rabbitmq_password:
        utils.error_exit(
            'Both rabbitmq_username and rabbitmq_password must be supplied '
            'and at least 1 character long in the manager blueprint inputs.')

    ctx.logger.info('Installing Logstash...')
    utils.set_selinux_permissive()
    utils.copy_notice(LOGSTASH_SERVICE_NAME)

    utils.yum_install(logstash_source_url, service_name=LOGSTASH_SERVICE_NAME)

    utils.mkdir(logstash_log_path)
    utils.chown('logstash', 'logstash', logstash_log_path)

    ctx.logger.info('Creating systemd unit override...')
    utils.mkdir(logstash_unit_override)
    utils.deploy_blueprint_resource(
        '{0}/restart.conf'.format(CONFIG_PATH),
        '{0}/restart.conf'.format(logstash_unit_override),
        LOGSTASH_SERVICE_NAME)
    ctx.logger.info('Deploying Logstash conf...')
    utils.deploy_blueprint_resource(
        '{0}/logstash.conf'.format(CONFIG_PATH),
        '{0}/logstash.conf'.format(logstash_conf_path), LOGSTASH_SERVICE_NAME)

    ctx.logger.info('Deploying Logstash sysconfig...')
    utils.deploy_blueprint_resource('{0}/logstash'.format(CONFIG_PATH),
                                    '/etc/sysconfig/logstash',
                                    LOGSTASH_SERVICE_NAME)

    utils.logrotate(LOGSTASH_SERVICE_NAME)
    utils.sudo(['/sbin/chkconfig', 'logstash', 'on'])
    utils.clean_var_log_dir(LOGSTASH_SERVICE_NAME)
def configure_script(script_name, description):
    utils.deploy_sudo_command_script(
        script_name,
        description,
        component=SERVICE_NAME,
        allow_as=STAGE_USER,
    )
    utils.chmod('a+rx', '/opt/cloudify/stage/' + script_name)
    utils.sudo(['usermod', '-aG', utils.CLOUDIFY_GROUP, STAGE_USER])
def _configure_index_rotation():
    ctx.logger.info('Configuring Elasticsearch Index Rotation cronjob for '
                    'logstash-YYYY.mm.dd index patterns...')
    utils.deploy_blueprint_resource(
        'components/elasticsearch/scripts/rotate_es_indices',
        '/etc/cron.daily/rotate_es_indices', ES_SERVICE_NAME)
    utils.chown('root', 'root', '/etc/cron.daily/rotate_es_indices')
    # VALIDATE!
    utils.sudo('chmod +x /etc/cron.daily/rotate_es_indices')
def _disable_requiretty():
    script_dest = '/tmp/configure_manager.sh'
    utils.deploy_blueprint_resource('components/manager/scripts'
                                    '/configure_manager.sh',
                                    script_dest,
                                    NODE_NAME)

    utils.sudo('chmod +x {0}'.format(script_dest))
    utils.sudo(script_dest)
def _configure_index_rotation():
    ctx.logger.info('Configuring Elasticsearch Index Rotation cronjob for '
                    'logstash-YYYY.mm.dd index patterns...')
    utils.deploy_blueprint_resource(
        'components/elasticsearch/scripts/rotate_es_indices',
        '/etc/cron.daily/rotate_es_indices', ES_SERVICE_NAME)
    utils.chown('root', 'root', '/etc/cron.daily/rotate_es_indices')
    # VALIDATE!
    utils.sudo('chmod +x /etc/cron.daily/rotate_es_indices')
示例#21
0
def install_riemann():
    langohr_source_url = ctx_properties['langohr_jar_source_url']
    daemonize_source_url = ctx_properties['daemonize_rpm_source_url']
    riemann_source_url = ctx_properties['riemann_rpm_source_url']
    # Needed for Riemann's config
    cloudify_resources_url = ctx_properties['cloudify_resources_url']
    rabbitmq_username = ctx_properties['rabbitmq_username']
    rabbitmq_password = ctx_properties['rabbitmq_password']

    riemann_config_path = '/etc/riemann'
    riemann_log_path = '/var/log/cloudify/riemann'
    langohr_home = '/opt/lib'
    extra_classpath = '{0}/langohr.jar'.format(langohr_home)

    # Confirm username and password have been supplied for broker before
    # continuing.
    # Components other than logstash and riemann have this handled in code.
    # Note that these are not directly used in this script, but are used by the
    # deployed resources, hence the check here.
    if not rabbitmq_username or not rabbitmq_password:
        ctx.abort_operation(
            'Both rabbitmq_username and rabbitmq_password must be supplied '
            'and at least 1 character long in the manager blueprint inputs.')

    rabbit_props = utils.ctx_factory.get('rabbitmq')
    ctx.instance.runtime_properties['rabbitmq_endpoint_ip'] = \
        utils.get_rabbitmq_endpoint_ip(
                rabbit_props.get('rabbitmq_endpoint_ip'))
    ctx.instance.runtime_properties['rabbitmq_username'] = \
        rabbit_props.get('rabbitmq_username')
    ctx.instance.runtime_properties['rabbitmq_password'] = \
        rabbit_props.get('rabbitmq_password')

    ctx.logger.info('Installing Riemann...')
    utils.set_selinux_permissive()

    utils.copy_notice(RIEMANN_SERVICE_NAME)
    utils.mkdir(riemann_log_path)
    utils.mkdir(langohr_home)
    utils.mkdir(riemann_config_path)
    utils.mkdir('{0}/conf.d'.format(riemann_config_path))

    langohr = utils.download_cloudify_resource(langohr_source_url,
                                               RIEMANN_SERVICE_NAME)
    utils.sudo(['cp', langohr, extra_classpath])
    ctx.logger.info('Applying Langohr permissions...')
    utils.sudo(['chmod', '644', extra_classpath])
    utils.yum_install(daemonize_source_url, service_name=RIEMANN_SERVICE_NAME)
    utils.yum_install(riemann_source_url, service_name=RIEMANN_SERVICE_NAME)

    utils.logrotate(RIEMANN_SERVICE_NAME)

    ctx.logger.info('Downloading cloudify-manager Repository...')
    manager_repo = utils.download_cloudify_resource(cloudify_resources_url,
                                                    RIEMANN_SERVICE_NAME)
    ctx.logger.info('Extracting Manager Repository...')
    utils.untar(manager_repo, '/tmp')
def install_riemann():
    langohr_source_url = ctx_properties['langohr_jar_source_url']
    daemonize_source_url = ctx_properties['daemonize_rpm_source_url']
    riemann_source_url = ctx_properties['riemann_rpm_source_url']
    # Needed for Riemann's config
    cloudify_resources_url = ctx_properties['cloudify_resources_url']
    rabbitmq_username = ctx_properties['rabbitmq_username']
    rabbitmq_password = ctx_properties['rabbitmq_password']

    riemann_config_path = '/etc/riemann'
    riemann_log_path = '/var/log/cloudify/riemann'
    langohr_home = '/opt/lib'
    extra_classpath = '{0}/langohr.jar'.format(langohr_home)

    # Confirm username and password have been supplied for broker before
    # continuing.
    # Components other than logstash and riemann have this handled in code.
    # Note that these are not directly used in this script, but are used by the
    # deployed resources, hence the check here.
    if not rabbitmq_username or not rabbitmq_password:
        ctx.abort_operation(
            'Both rabbitmq_username and rabbitmq_password must be supplied '
            'and at least 1 character long in the manager blueprint inputs.')

    rabbit_props = utils.ctx_factory.get('rabbitmq')
    ctx.instance.runtime_properties['rabbitmq_endpoint_ip'] = \
        utils.get_rabbitmq_endpoint_ip(
                rabbit_props.get('rabbitmq_endpoint_ip'))
    ctx.instance.runtime_properties['rabbitmq_username'] = \
        rabbit_props.get('rabbitmq_username')
    ctx.instance.runtime_properties['rabbitmq_password'] = \
        rabbit_props.get('rabbitmq_password')

    ctx.logger.info('Installing Riemann...')
    utils.set_selinux_permissive()

    utils.copy_notice(RIEMANN_SERVICE_NAME)
    utils.mkdir(riemann_log_path)
    utils.mkdir(langohr_home)
    utils.mkdir(riemann_config_path)
    utils.mkdir('{0}/conf.d'.format(riemann_config_path))

    langohr = utils.download_cloudify_resource(langohr_source_url,
                                               RIEMANN_SERVICE_NAME)
    utils.sudo(['cp', langohr, extra_classpath])
    ctx.logger.info('Applying Langohr permissions...')
    utils.sudo(['chmod', '644', extra_classpath])
    utils.yum_install(daemonize_source_url, service_name=RIEMANN_SERVICE_NAME)
    utils.yum_install(riemann_source_url, service_name=RIEMANN_SERVICE_NAME)

    utils.logrotate(RIEMANN_SERVICE_NAME)

    ctx.logger.info('Downloading cloudify-manager Repository...')
    manager_repo = utils.download_cloudify_resource(cloudify_resources_url,
                                                    RIEMANN_SERVICE_NAME)
    ctx.logger.info('Extracting Manager Repository...')
    utils.untar(manager_repo, '/tmp')
示例#23
0
def change_conf(next_conf=None):
    signal = "# configurations recommended by ottertune:\n"
    next_conf = next_conf or {}

    tmp_conf_in = os.path.join(dconf.TEMP_DIR,
                               os.path.basename(dconf.DB_CONF) + '.in')
    get(dconf.DB_CONF, tmp_conf_in)
    with open(tmp_conf_in, 'r') as f:
        lines = f.readlines()

    if signal not in lines:
        lines += ['\n', signal]

    signal_idx = lines.index(signal)
    lines = lines[0:signal_idx + 1]

    if dconf.DB_TYPE == 'mysql':
        lines.append('[mysqld]\n')

    if dconf.BASE_DB_CONF:
        assert isinstance(dconf.BASE_DB_CONF, dict), \
            (type(dconf.BASE_DB_CONF), dconf.BASE_DB_CONF)
        for name, value in sorted(dconf.BASE_DB_CONF.items()):
            if value is None:
                lines.append('{}\n'.format(name))
            else:
                lines.append('{} = {}\n'.format(name, value))

    if isinstance(next_conf, str):
        with open(next_conf, 'r') as f:
            recommendation = json.load(
                f, encoding="UTF-8",
                object_pairs_hook=OrderedDict)['recommendation']
    else:
        recommendation = next_conf

    assert isinstance(recommendation, dict)

    for name, value in recommendation.items():
        if dconf.DB_TYPE == 'oracle' and isinstance(value, str):
            value = value.strip('B')
        # If innodb_flush_method is set to NULL on a Unix-like system,
        # the fsync option is used by default.
        if name == 'innodb_flush_method' and value == '':
            value = "fsync"
        lines.append('{} = {}\n'.format(name, value))
    lines.append('\n')

    tmp_conf_out = os.path.join(dconf.TEMP_DIR,
                                os.path.basename(dconf.DB_CONF) + '.out')
    with open(tmp_conf_out, 'w') as f:
        f.write(''.join(lines))

    sudo('cp {0} {0}.ottertune.bak'.format(dconf.DB_CONF), remote_only=True)
    put(tmp_conf_out, dconf.DB_CONF, use_sudo=True)
    local('rm -f {} {}'.format(tmp_conf_in, tmp_conf_out))
def _create_user_and_set_permissions(rabbitmq_username,
                                     rabbitmq_password):
    if not check_if_user_exists(rabbitmq_username):
        ctx.logger.info('Creating new user {0}:{1} and setting '
                        'permissions...'.format(
                            rabbitmq_username, rabbitmq_password))
        utils.sudo(['rabbitmqctl', 'add_user',
                    rabbitmq_username, rabbitmq_password])
        utils.sudo(['rabbitmqctl', 'set_permissions',
                    rabbitmq_username, '.*', '.*', '.*'], retries=5)
示例#25
0
def _create_user_and_set_permissions(rabbitmq_username,
                                     rabbitmq_password):
    if not check_if_user_exists(rabbitmq_username):
        ctx.logger.info('Creating new user {0}:{1} and setting '
                        'permissions...'.format(
                            rabbitmq_username, rabbitmq_password))
        utils.sudo(['rabbitmqctl', 'add_user',
                    rabbitmq_username, rabbitmq_password])
        utils.sudo(['rabbitmqctl', 'set_permissions',
                    rabbitmq_username, '.*', '.*', '.*'], retries=5)
def install_logstash():

    logstash_unit_override = '/etc/systemd/system/logstash.service.d'

    logstash_source_url = ctx.node.properties['logstash_rpm_source_url']

    rabbitmq_username = ctx.node.properties['rabbitmq_username']
    rabbitmq_password = ctx.node.properties['rabbitmq_password']

    logstash_log_path = '/var/log/cloudify/logstash'
    logstash_conf_path = '/etc/logstash/conf.d'

    # injected as an input to the script
    ctx.instance.runtime_properties['es_endpoint_ip'] = \
        os.environ.get('ES_ENDPOINT_IP')
    ctx.instance.runtime_properties['rabbitmq_endpoint_ip'] = \
        utils.get_rabbitmq_endpoint_ip()

    # Confirm username and password have been supplied for broker before
    # continuing.
    # Components other than logstash and riemann have this handled in code.
    # Note that these are not directly used in this script, but are used by the
    # deployed resources, hence the check here.
    if not rabbitmq_username or not rabbitmq_password:
        utils.error_exit(
            'Both rabbitmq_username and rabbitmq_password must be supplied '
            'and at least 1 character long in the manager blueprint inputs.')

    ctx.logger.info('Installing Logstash...')
    utils.set_selinux_permissive()
    utils.copy_notice('logstash')

    utils.yum_install(logstash_source_url)

    utils.mkdir(logstash_log_path)
    utils.chown('logstash', 'logstash', logstash_log_path)

    ctx.logger.info('Creating systemd unit override...')
    utils.mkdir(logstash_unit_override)
    utils.deploy_blueprint_resource(
        '{0}/restart.conf'.format(CONFIG_PATH),
        '{0}/restart.conf'.format(logstash_unit_override))
    ctx.logger.info('Deploying Logstash conf...')
    utils.deploy_blueprint_resource(
        '{0}/logstash.conf'.format(CONFIG_PATH),
        '{0}/logstash.conf'.format(logstash_conf_path))

    ctx.logger.info('Deploying Logstash sysconfig...')
    utils.deploy_blueprint_resource(
        '{0}/logstash'.format(CONFIG_PATH),
        '/etc/sysconfig/logstash')

    utils.logrotate('logstash')
    utils.sudo(['/sbin/chkconfig', 'logstash', 'on'])
    utils.clean_var_log_dir('logstash')
示例#27
0
def install_apt(packages):
    """
    Install packages using APT
    """
    # We need to iterate over each one because if at least one of them
    # is actually not a package (misspelled, doesn't exist anymore, debian-
    # or ubuntu-specific, etc) the whole install process would fail
    # On the other hand there appears to be no flag to ignore these errors
    # on apt-get (tested on Ubuntu 12.04)
    for pkg in packages + extra_packages():
        sudo('apt-get -qq -y install {0}'.format(pkg))
def _configure_index_rotation():
    ctx.logger.info('Configurating index rotation...')
    ctx.logger.debug(
        'Setting up curator rotation cronjob for logstash-YYYY.mm.dd '
        'index patterns...')
    utils.deploy_blueprint_resource(
        'components/elasticsearch/scripts/rotate_es_indices',
        '/etc/cron.daily/rotate_es_indices', ES_SERVICE_NAME)
    utils.chown('root', 'root', '/etc/cron.daily/rotate_es_indices')
    # TODO: VALIDATE!
    # TODO: use utils.chmod
    utils.sudo('chmod +x /etc/cron.daily/rotate_es_indices')
def _create_default_db(db_name, username, password):
    ctx.logger.info('Creating default postgresql database: {0}...'.format(
        db_name))
    ps_config_source = 'components/postgresql/config/create_default_db.sh'
    ps_config_destination = join(tempfile.gettempdir(),
                                 'create_default_db.sh')
    ctx.download_resource(source=ps_config_source,
                          destination=ps_config_destination)
    utils.chmod('+x', ps_config_destination)
    # TODO: Can't we use a rest call here? Is there such a thing?
    utils.sudo('su - postgres -c "{cmd} {db} {user} {password}"'
               .format(cmd=ps_config_destination, db=db_name,
                       user=username, password=password))
def _deploy_db_cleanup_script():
    """ Copy the script that deletes logs and events from the
    Cloudify DB to /etc/cloudify, so it will be available to the users."""
    try:
        script_name = 'delete_logs_and_events_from_db.py'
        script_destination = join(utils.get_exec_tempdir(), script_name)
        ctx.download_resource(join(CONFIG_PATH, script_name),
                              script_destination)
        utils.sudo(['mv', script_destination,
                    join(utils.CLOUDIFY_HOME_DIR, script_name)])
    except Exception as ex:
        ctx.logger.info('Failed to deploy delete_logs script. Error: {0}'
                        ''.format(ex))
示例#31
0
def configure_mgmtworker():
    celery_work_dir = '{0}/work'.format(runtime_props['home_dir'])
    runtime_props['file_server_root'] = utils.MANAGER_RESOURCES_HOME

    ctx.logger.info('Configuring Management worker...')
    broker_conf_path = join(celery_work_dir, 'broker_config.json')
    utils.deploy_blueprint_resource(
        '{0}/broker_config.json'.format(CONFIG_PATH), broker_conf_path,
        SERVICE_NAME)
    # The config contains credentials, do not let the world read it
    utils.sudo(['chmod', '440', broker_conf_path])
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, broker_conf_path)
    utils.systemd.configure(SERVICE_NAME)
    utils.logrotate(SERVICE_NAME)
def _init_postgresql():
    ctx.logger.info('Init PostreSQL DATA folder...')
    postgresql95_setup = '/usr/pgsql-9.5/bin/postgresql95-setup'
    utils.sudo(command=[postgresql95_setup, 'initdb'])

    ctx.logger.info('Starting PostgreSQL server...')
    utils.systemd.enable(service_name=PS_SERVICE_NAME, append_prefix=False)
    utils.systemd.start(service_name=PS_SERVICE_NAME, append_prefix=False)

    ctx.logger.info('Setting PostgreSQL logs path...')
    ps_95_logs_path = "/var/lib/pgsql/9.5/data/pg_log"
    ps_logs_path = "/var/log/cloudify/postgresql"
    utils.mkdir(ps_logs_path)
    utils.ln(source=ps_95_logs_path, target=ps_logs_path, params='-s')
def configure_mgmtworker():
    celery_work_dir = '{0}/work'.format(runtime_props['home_dir'])
    runtime_props['file_server_root'] = utils.MANAGER_RESOURCES_HOME

    ctx.logger.info('Configuring Management worker...')
    broker_conf_path = join(celery_work_dir, 'broker_config.json')
    utils.deploy_blueprint_resource(
        '{0}/broker_config.json'.format(CONFIG_PATH), broker_conf_path,
        SERVICE_NAME)
    # The config contains credentials, do not let the world read it
    utils.sudo(['chmod', '440', broker_conf_path])
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, broker_conf_path)
    utils.systemd.configure(SERVICE_NAME)
    utils.logrotate(SERVICE_NAME)
示例#34
0
def _create_default_db(db_name, username, password):
    ctx.logger.info(
        'Creating default postgresql database: {0}...'.format(db_name))
    ps_config_source = 'components/postgresql/config/create_default_db.sh'
    ps_config_destination = join(tempfile.gettempdir(), 'create_default_db.sh')
    ctx.download_resource(source=ps_config_source,
                          destination=ps_config_destination)
    utils.chmod('+x', ps_config_destination)
    # TODO: Can't we use a rest call here? Is there such a thing?
    utils.sudo('su - postgres -c "{cmd} {db} {user} {password}"'.format(
        cmd=ps_config_destination,
        db=db_name,
        user=username,
        password=password))
示例#35
0
def install_docker():
    ctx.logger.info('Installing the latest version of docker...')
    script_name = 'get_docker.sh'
    utils.run('curl -o {0} https://get.docker.com/'.format(script_name),
              workdir=utils.WORKDIR)
    utils.run('chmod +x {0}'.format(script_name), workdir=utils.WORKDIR)
    utils.sudo('bash {0}'.format(script_name), workdir=utils.WORKDIR)

    utils.sudo('usermod -aG docker {0}'.format(
        ctx.node.properties['ssh_user']))
    utils.sudo('service docker stop')
    _create_cloudify_bridge()
    utils.sudo("sed -i '$ a DOCKER_OPTS=\"--bridge cfy0 "
               "--host 172.20.0.1\"' /etc/default/docker")
    utils.sudo('service docker start')
示例#36
0
def download(url, target=None, root=False):
    if target is None:
        parts = urlparse.urlparse(url)
        target = parts.path.split('/')[-1]
    if check_command('wget'):
        cmd = 'wget --no-check-certificate -q -O {0} {1}'.format(target, url)
    elif check_command('curl'):
        cmd = 'curl -o {0} {1}'.format(target, url)
    else:
        raise Exception("Neither wget nor curl are installed")
    if root:
        sudo(cmd)
    else:
        run(cmd)
    return target
示例#37
0
def install_docker():
    ctx.logger.info('Installing the latest version of docker...')
    script_name = 'get_docker.sh'
    utils.run('curl -o {0} https://get.docker.com/'.format(script_name),
              workdir=utils.WORKDIR)
    utils.run('chmod +x {0}'.format(script_name), workdir=utils.WORKDIR)
    utils.sudo('bash {0}'.format(script_name), workdir=utils.WORKDIR)

    utils.sudo('usermod -aG docker {0}'
               .format(ctx.node.properties['ssh_user']))
    utils.sudo('service docker stop')
    _create_cloudify_bridge()
    utils.sudo("sed -i '$ a DOCKER_OPTS=\"--bridge cfy0 "
               "--host 172.20.0.1\"' /etc/default/docker")
    utils.sudo('service docker start')
def _create_db_tables_and_add_users():
    ctx.logger.info('Creating SQL tables and adding admin users...')
    create_script_path = 'components/restservice/config' \
                         '/create_tables_and_add_users.py'
    create_script_destination = join(tempfile.gettempdir(),
                                     'create_tables_and_add_users.py')
    ctx.download_resource(source=create_script_path,
                          destination=create_script_destination)
    # Directly calling with this python bin, in order to make sure it's run
    # in the correct venv
    python_path = '{0}/env/bin/python'.format(REST_SERVICE_HOME)
    runtime_props = ctx.instance.runtime_properties

    args_dict = json.loads(runtime_props['security_configuration'])
    args_dict['postgresql_host'] = runtime_props['postgresql_host']

    # The script won't have access to the ctx, so we dump the relevant args
    # to a JSON file, and pass its path to the script
    args_file_location = join(tempfile.gettempdir(), 'security_config.json')
    with open(args_file_location, 'w') as f:
        json.dump(args_dict, f)

    result = utils.sudo(
        [python_path, create_script_destination, args_file_location]
    )

    _log_results(result)
    utils.remove(args_file_location)
示例#39
0
def install_yum(packages):
    """
    Install packages using YUM
    """
    errmsg = sudo('yum --assumeyes --quiet install {0}'.format(' '.join(packages + extra_packages())),\
                   combine_stderr=True, warn_only=True)
    processCentOSErrMsg(errmsg)
def _create_db_tables_and_add_defaults():
    ctx.logger.info('Creating SQL tables and adding default values...')
    script_name = 'create_tables_and_add_defaults.py'
    source_script_path = join('components/restservice/config', script_name)
    destination_script_path = join(tempfile.gettempdir(), script_name)
    ctx.download_resource(source_script_path, destination_script_path)

    args_dict = runtime_props['security_configuration']
    args_dict['amqp_host'] = runtime_props['rabbitmq_endpoint_ip']
    args_dict['amqp_username'] = runtime_props['rabbitmq_username']
    args_dict['amqp_password'] = runtime_props['rabbitmq_password']
    args_dict['postgresql_host'] = runtime_props['postgresql_host']
    args_dict['db_migrate_dir'] = join(
        utils.MANAGER_RESOURCES_HOME,
        'cloudify',
        'migrations'
    )

    # The script won't have access to the ctx, so we dump the relevant args
    # to a JSON file, and pass its path to the script
    args_file_location = join(tempfile.gettempdir(), 'security_config.json')
    with open(args_file_location, 'w') as f:
        json.dump(args_dict, f)

    # Directly calling with this python bin, in order to make sure it's run
    # in the correct venv
    python_path = join(runtime_props['home_dir'], 'env', 'bin', 'python')
    result = utils.sudo(
        [python_path, destination_script_path, args_file_location]
    )

    _log_results(result)
    utils.remove(args_file_location)
    utils.remove(destination_script_path)
def _create_db_tables_and_add_defaults():
    ctx.logger.info('Creating SQL tables and adding default values...')
    script_name = 'create_tables_and_add_defaults.py'
    source_script_path = join('components/restservice/config', script_name)
    destination_script_path = join(tempfile.gettempdir(), script_name)
    ctx.download_resource(source_script_path, destination_script_path)

    args_dict = runtime_props['security_configuration']
    args_dict['amqp_host'] = runtime_props['rabbitmq_endpoint_ip']
    args_dict['amqp_username'] = ctx_properties['rabbitmq_username']
    args_dict['amqp_password'] = ctx_properties['rabbitmq_password']
    args_dict['postgresql_host'] = ctx_properties['postgresql_host']
    args_dict['authorization_file_path'] = \
        runtime_props['authorization_file_path']
    args_dict['db_migrate_dir'] = join(utils.MANAGER_RESOURCES_HOME,
                                       'cloudify', 'migrations')

    # The script won't have access to the ctx, so we dump the relevant args
    # to a JSON file, and pass its path to the script
    args_file_location = join(tempfile.gettempdir(), 'security_config.json')
    with open(args_file_location, 'w') as f:
        json.dump(args_dict, f)

    # Directly calling with this python bin, in order to make sure it's run
    # in the correct venv
    python_path = join(runtime_props['home_dir'], 'env', 'bin', 'python')
    result = utils.sudo(
        [python_path, destination_script_path, args_file_location])

    _log_results(result)
    utils.remove(args_file_location)
    utils.remove(destination_script_path)
def status(job, status='start/running'):
	result = sudo('status %s' % job)
	
	if result.succeeded:
		return result.find(status) > -1
	else:
		return False
def configure_logstash():

    logstash_conf_path = '/etc/logstash/conf.d'

    runtime_properties = ctx.instance.runtime_properties
    rabbitmq_username = runtime_properties.get('rabbitmq_username')
    rabbitmq_password = runtime_properties.get('rabbitmq_password')

    # Confirm username and password have been supplied for broker before
    # continuing.
    # Components other than logstash and riemann have this handled in code.
    # Note that these are not directly used in this script, but are used by the
    # deployed resources, hence the check here.
    if not rabbitmq_username or not rabbitmq_password:
        ctx.abort_operation(
            'Both rabbitmq_username and rabbitmq_password must be supplied '
            'and at least 1 character long in the manager blueprint inputs.')

    ctx.logger.info('Deploying Logstash configuration...')
    utils.deploy_blueprint_resource(
        '{0}/logstash.conf'.format(CONFIG_PATH),
        '{0}/logstash.conf'.format(logstash_conf_path),
        LOGSTASH_SERVICE_NAME)

    # Due to a bug in the handling of configuration files,
    # configuration files with the same name cannot be deployed.
    # Since the logrotate config file is called `logstash`,
    # we change the name of the logstash env vars config file
    # from logstash to cloudify-logstash to be consistent with
    # other service env var files.
    init_file = '/etc/init.d/logstash'
    utils.replace_in_file(
        'sysconfig/\$name',
        'sysconfig/cloudify-$name',
        init_file)
    utils.chmod('755', init_file)
    utils.chown('root', 'root', init_file)

    ctx.logger.debug('Deploying Logstash sysconfig...')
    utils.deploy_blueprint_resource(
        '{0}/cloudify-logstash'.format(CONFIG_PATH),
        '/etc/sysconfig/cloudify-logstash',
        LOGSTASH_SERVICE_NAME)

    utils.logrotate(LOGSTASH_SERVICE_NAME)
    utils.sudo(['/sbin/chkconfig', 'logstash', 'on'])
    utils.clean_var_log_dir(LOGSTASH_SERVICE_NAME)
示例#44
0
def change_conf(next_conf=None):
    signal = "# configurations recommended by ottertune:\n"
    next_conf = next_conf or {}

    tmp_conf_in = os.path.join(dconf.TEMP_DIR,
                               os.path.basename(dconf.DB_CONF) + '.in')
    get(dconf.DB_CONF, tmp_conf_in)
    with open(tmp_conf_in, 'r') as f:
        lines = f.readlines()

    if signal not in lines:
        lines += ['\n', signal]

    signal_idx = lines.index(signal)
    lines = lines[0:signal_idx + 1]
    if dconf.BASE_DB_CONF:
        assert isinstance(dconf.BASE_DB_CONF, dict), \
            (type(dconf.BASE_DB_CONF), dconf.BASE_DB_CONF)
        base_conf = [
            '{} = {}\n'.format(*c) for c in sorted(dconf.BASE_DB_CONF.items())
        ]
        lines.extend(base_conf)

    if isinstance(next_conf, str):
        with open(next_conf, 'r') as f:
            recommendation = json.load(
                f, encoding="UTF-8",
                object_pairs_hook=OrderedDict)['recommendation']
    else:
        recommendation = next_conf

    assert isinstance(recommendation, dict)

    for name, value in recommendation.items():
        if dconf.DB_TYPE == 'oracle' and isinstance(value, str):
            value = value.strip('B')
        lines.append('{} = {}\n'.format(name, value))
    lines.append('\n')

    tmp_conf_out = os.path.join(dconf.TEMP_DIR,
                                os.path.basename(dconf.DB_CONF) + '.out')
    with open(tmp_conf_out, 'w') as f:
        f.write(''.join(lines))

    sudo('cp {0} {0}.ottertune.bak'.format(dconf.DB_CONF))
    put(tmp_conf_out, dconf.DB_CONF, use_sudo=False)
    local('rm -f {} {}'.format(tmp_conf_in, tmp_conf_out))
def install_java():
    java_source_url = ctx.node.properties['java_rpm_source_url']

    ctx.logger.info('Installing Java...')
    utils.set_selinux_permissive()
    utils.copy_notice('java')

    utils.yum_install(java_source_url)

    # Make sure the cloudify logs dir exists before we try moving the java log
    # there -p will cause it not to error if the dir already exists
    utils.mkdir('/var/log/cloudify')

    # Java install log is dropped in /var/log.
    # Move it to live with the rest of the cloudify logs
    if os.path.isfile('/var/log/java_install.log'):
        utils.sudo('mv /var/log/java_install.log /var/log/cloudify')
示例#46
0
def install_java():
    java_source_url = ctx_properties['java_rpm_source_url']

    ctx.logger.info('Installing Java...')
    utils.set_selinux_permissive()
    utils.copy_notice('java')

    utils.yum_install(java_source_url, service_name='java')

    # Make sure the cloudify logs dir exists before we try moving the java log
    # there -p will cause it not to error if the dir already exists
    utils.mkdir('/var/log/cloudify')

    # Java install log is dropped in /var/log.
    # Move it to live with the rest of the cloudify logs
    if os.path.isfile('/var/log/java_install.log'):
        utils.sudo('mv /var/log/java_install.log /var/log/cloudify')
示例#47
0
def _create_cloudify_bridge():
    proc, stdout, stderr = utils.sudo('brctl show')
    if 'cfy0' not in stdout:
        ctx.logger.info('creating cfy0 network bridge...')
        utils.sudo('brctl addbr cfy0')
        utils.sudo('ip addr add 172.20.0.1/24 dev cfy0')
        utils.sudo('ip link set dev cfy0 up')
def _init_postgresql():
    ctx.logger.info('Initializing PostreSQL DATA folder...')
    postgresql95_setup = join(PGSQL_USR_DIR, 'bin', 'postgresql95-setup')
    try:
        utils.sudo(command=[postgresql95_setup, 'initdb'])
    except Exception:
        ctx.logger.debug('PostreSQL DATA folder already been init...')
        pass

    ctx.logger.info('Starting PostgreSQL server...')
    utils.systemd.enable(service_name=SERVICE_NAME, append_prefix=False)
    utils.systemd.start(service_name=SERVICE_NAME, append_prefix=False)

    ctx.logger.info('Setting PostgreSQL logs path...')
    ps_95_logs_path = join(PGSQL_LIB_DIR, '9.5', 'data', 'pg_log')
    utils.mkdir(PGSQL_LOGS_DIR)
    if not os.path.isdir(ps_95_logs_path):
        utils.ln(source=ps_95_logs_path, target=PGSQL_LOGS_DIR, params='-s')
def check_rabbit_running():
    """Use rabbitmqctl status to check if RabbitMQ is working.

    Sometimes rabbit takes a while to start, so this is retried several times.
    Note that this is currently impossible to do on a remote host, so
    this check only runs when rabbitmq is installed locally.
    """
    result = utils.sudo(['rabbitmqctl', 'status'], ignore_failures=True)
    if result.returncode != 0:
        raise ValueError('rabbitmqctl status: rabbitmq not running')
示例#50
0
def add(grpname, gid=False):
	cmd = 'groupadd --system %s' % grpname
	if gid:
		cmd = 'groupadd --system --gid %i %s' % (gid, grpname)
	
	result = sudo(cmd)
	if not result.failed:
		return result.stderr.endswith('already exists')
	else:
		return False
def check_worker_running():
    """Use `celery status` to check if the worker is running."""
    result = utils.sudo([
        'CELERY_WORK_DIR=/opt/mgmtworker/work',
        CELERY_PATH,
        '--config=cloudify.broker_config',
        'status'
    ], ignore_failures=True)
    if result.returncode != 0:
        raise ValueError('celery status: worker not running')
def check_worker_running():
    """Use `celery status` to check if the worker is running."""
    work_dir = join(HOME_DIR, 'work')
    celery_path = join(HOME_DIR, 'env', 'bin', 'celery')
    result = utils.sudo([
        'CELERY_WORK_DIR={0}'.format(work_dir),
        celery_path,
        '--config=cloudify.broker_config',
        'status'
    ], ignore_failures=True)
    if result.returncode != 0:
        raise ValueError('celery status: worker not running')
def uninstall(pkg, warn=True, silent=False):
	status = installed(pkg)
	
	if status == True:
		_msg('Uninstalling %s...' % pkg, silent)
		cmd = 'apt-get remove -y %s' % pkg
	
		if warn:
			if confirm('Are you sure you want to uninstall %s from: %s?' % (pkg, env.host_string)):
				result = sudo(cmd)
		else:
			result = sudo(cmd)
		
		if result.failed:
			about(result.stderr)
		else:
			_msg('Uninstalled %s' % pkg, silent)
			return 
	elif status == False:
		_msg('Not Already Installed %s: Skipped' % pkg, silent)
	else:
		abort(result.stderr)
示例#54
0
def add(username, uid=False, gid=False, verbose=False):
	cmd = 'useradd --no-create-home --system '
	if uid:
		cmd += '--uid %s ' % uid
	if gid:
		cmd += '--gid %s ' % uid
	
	cmd += username
	
	result = sudo(cmd)
	if not result.failed:
		return result.stderr.endswith('already exists')
	else:
		return False
def configure_mgmtworker():
    # these must all be exported as part of the start operation.
    # they will not persist, so we should use the new agent
    # don't forget to change all localhosts to the relevant ips
    mgmtworker_home = '/opt/mgmtworker'
    mgmtworker_venv = '{0}/env'.format(mgmtworker_home)
    celery_work_dir = '{0}/work'.format(mgmtworker_home)

    ctx.logger.info('Configuring Management worker...')
    # Deploy the broker configuration
    # TODO: This will break interestingly if mgmtworker_venv is empty.
    # Some sort of check for that would be sensible.
    # To sandy: I don't quite understand this check...
    # there is no else here..
    # for python_path in ${mgmtworker_venv}/lib/python*; do
    if os.path.isfile(os.path.join(mgmtworker_venv, 'bin/python')):
        broker_conf_path = os.path.join(celery_work_dir, 'broker_config.json')
        utils.deploy_blueprint_resource(
            '{0}/broker_config.json'.format(CONFIG_PATH), broker_conf_path,
            MGMT_WORKER_SERVICE_NAME)
        # The config contains credentials, do not let the world read it
        utils.sudo(['chmod', '440', broker_conf_path])
    utils.systemd.configure(MGMT_WORKER_SERVICE_NAME)
    utils.logrotate(MGMT_WORKER_SERVICE_NAME)
def install(pkg, silent=False):
	status = installed(pkg)
	
	if status == True:
		_msg('Already installed: %s' % pkg, silent)
		return True
	elif status == False:
		_msg('Installing %s...' % pkg, silent)
		# Update result, as generally you want to know 
		# if something did actually install
		result = sudo('apt-get install -y %s' % pkg)
		if result.failed:
			abort(result.stderr)
		else:
			_msg('Installed %s' % pkg, silent)
			return result
	else:
		abort(result.stderr)
def install_mgmtworker():

    management_worker_rpm_source_url = \
        ctx_properties['management_worker_rpm_source_url']

    # these must all be exported as part of the start operation.
    # they will not persist, so we should use the new agent
    # don't forget to change all localhosts to the relevant ips
    mgmtworker_home = '/opt/mgmtworker'
    mgmtworker_venv = '{0}/env'.format(mgmtworker_home)
    celery_work_dir = '{0}/work'.format(mgmtworker_home)
    celery_log_dir = "/var/log/cloudify/mgmtworker"

    broker_port_ssl = '5671'
    broker_port_no_ssl = '5672'
    rabbit_props = utils.ctx_factory.get('rabbitmq')
    rabbitmq_ssl_enabled = rabbit_props['rabbitmq_ssl_enabled']
    ctx.logger.info("rabbitmq_ssl_enabled: {0}".format(rabbitmq_ssl_enabled))
    rabbitmq_cert_public = rabbit_props['rabbitmq_cert_public']

    ctx.instance.runtime_properties['rabbitmq_endpoint_ip'] = \
        utils.get_rabbitmq_endpoint_ip(
                rabbit_props.get('rabbitmq_endpoint_ip'))

    # Fix possible injections in json of rabbit credentials
    # See json.org for string spec
    for key in ['rabbitmq_username', 'rabbitmq_password']:
        # We will not escape newlines or other control characters,
        # we will accept them breaking
        # things noisily, e.g. on newlines and backspaces.
        # TODO: add:
        # sed 's/"/\\"/' | sed 's/\\/\\\\/' | sed s-/-\\/- | sed 's/\t/\\t/'
        ctx.instance.runtime_properties[key] = ctx_properties[key]

    # Make the ssl enabled flag work with json (boolean in lower case)
    # TODO: check if still needed:
    # broker_ssl_enabled = "$(echo ${rabbitmq_ssl_enabled} | tr '[:upper:]' '[:lower:]')"  # NOQA
    ctx.instance.runtime_properties['rabbitmq_ssl_enabled'] = \
        rabbitmq_ssl_enabled

    ctx.logger.info('Installing Management Worker...')
    utils.set_selinux_permissive()

    utils.copy_notice(MGMT_WORKER_SERVICE_NAME)
    utils.mkdir(mgmtworker_home)
    utils.mkdir('{0}/config'.format(mgmtworker_home))
    utils.mkdir(celery_log_dir)
    utils.mkdir(celery_work_dir)

    # this create the mgmtworker_venv and installs the relevant
    # modules into it.
    utils.yum_install(management_worker_rpm_source_url,
                      service_name=MGMT_WORKER_SERVICE_NAME)
    _install_optional(mgmtworker_venv)

    # Add certificate and select port, as applicable
    if rabbitmq_ssl_enabled:
        broker_cert_path = '{0}/amqp_pub.pem'.format(mgmtworker_home)
        utils.deploy_ssl_certificate(
            'public', broker_cert_path, 'root', rabbitmq_cert_public)
        ctx.instance.runtime_properties['broker_cert_path'] = broker_cert_path
        # Use SSL port
        ctx.instance.runtime_properties['broker_port'] = broker_port_ssl
    else:
        # No SSL, don't use SSL port
        ctx.instance.runtime_properties['broker_port'] = broker_port_no_ssl
        if rabbitmq_cert_public is not None:
            ctx.logger.warn('Broker SSL cert supplied but SSL not enabled '
                            '(broker_ssl_enabled is False).')

    ctx.logger.info("broker_port: {0}".format(
        ctx.instance.runtime_properties['broker_port']))
    ctx.logger.info('Configuring Management worker...')
    # Deploy the broker configuration
    # TODO: This will break interestingly if mgmtworker_venv is empty.
    # Some sort of check for that would be sensible.
    # To sandy: I don't quite understand this check...
    # there is no else here..
    # for python_path in ${mgmtworker_venv}/lib/python*; do
    if os.path.isfile(os.path.join(mgmtworker_venv, 'bin/python')):
        broker_conf_path = os.path.join(celery_work_dir, 'broker_config.json')
        utils.deploy_blueprint_resource(
            '{0}/broker_config.json'.format(CONFIG_PATH), broker_conf_path,
            MGMT_WORKER_SERVICE_NAME)
        # The config contains credentials, do not let the world read it
        utils.sudo(['chmod', '440', broker_conf_path])
    utils.systemd.configure(MGMT_WORKER_SERVICE_NAME)
    utils.logrotate(MGMT_WORKER_SERVICE_NAME)