def install_deps(config, messages): deps = ["puppet", "openssh-clients", "tar", "nc"] modules_pkg = 'openstack-puppet-modules' local = utils.ScriptRunner() local.append('rpm -q --requires %s | egrep -v "^(rpmlib|\/|perl)"' % modules_pkg) # This can fail if there are no dependencies other than those # filtered out by the egrep expression. rc, modules_deps = local.execute(can_fail=False) # Modules package might not be installed if we are running from source. # In this case we assume user knows what (s)he's doing and we don't # install modules dependencies if ('%s is not installed' % modules_pkg) not in modules_deps: modules_deps = [i.strip() for i in modules_deps.split() if i.strip()] deps.extend(modules_deps) for hostname in filtered_hosts(config): server = utils.ScriptRunner(hostname) for package in deps: server.append("rpm -q --whatprovides %s || yum install -y %s" % (package, package)) server.execute()
def create_manifest(config, messages): if config['CONFIG_MARIADB_INSTALL'] == 'y': suffix = 'install' host = config['CONFIG_MARIADB_HOST'] else: suffix = 'noinstall' host = config['CONFIG_CONTROLLER_HOST'] manifestfile = "%s_mariadb.pp" % host manifestdata = [getManifestTemplate('mariadb_%s.pp' % suffix)] def append_for(module, suffix): # Modules have to be appended to the existing mysql.pp # otherwise pp will fail for some of them saying that # Mysql::Config definition is missing. template = "mariadb_%s_%s.pp" % (module, suffix) manifestdata.append(getManifestTemplate(template)) append_for("keystone", suffix) for mod in ['nova', 'cinder', 'glance', 'neutron', 'heat']: if config['CONFIG_%s_INSTALL' % mod.upper()] == 'y': append_for(mod, suffix) hosts = filtered_hosts(config, exclude=False, dbhost=True) config['FIREWALL_SERVICE_NAME'] = "mariadb" config['FIREWALL_PORTS'] = "'3306'" config['FIREWALL_CHAIN'] = "INPUT" config['FIREWALL_PROTOCOL'] = 'tcp' for host in hosts: config['FIREWALL_ALLOWED'] = "'%s'" % host config['FIREWALL_SERVICE_ID'] = "mariadb_%s" % host manifestdata.append(getManifestTemplate("firewall.pp")) appendManifestFile(manifestfile, "\n".join(manifestdata), 'pre')
def remove_remote_var_dirs(options, config, messages): """ Removes the temp directories on remote hosts, doesn't remove data on localhost """ for host in filtered_hosts(config): try: host_dir = config['HOST_DETAILS'][host]['tmpdir'] except KeyError: # Nothing was added to this host yet, so we have nothing to delete continue if options.debug: # we keep temporary directories on hosts in debug mode messages.append( 'Note temporary directory {host_dir} on host {host} was ' 'not deleted for debugging purposes.'.format(**locals()) ) continue logging.debug(output_messages.INFO_REMOVE_REMOTE_VAR % (host_dir, host)) server = utils.ScriptRunner(host) server.append('rm -rf %s' % host_dir) try: server.execute() except Exception as e: msg = output_messages.ERR_REMOVE_REMOTE_VAR % (host_dir, host) logging.error(msg) logging.exception(e) messages.append(utils.color_text(msg, 'red'))
def check_nm_status(config, messages): hosts_with_nm = [] for host in filtered_hosts(config): server = utils.ScriptRunner(host) server.append("systemctl") rc, out = server.execute(can_fail=False) server.clear() if rc < 1: server.append("systemctl is-enabled NetworkManager") rc, is_enabled = server.execute(can_fail=False) is_enabled = is_enabled.strip("\n ") server.clear() server.append("systemctl is-active NetworkManager") rc, is_active = server.execute(can_fail=False) is_active = is_active.strip("\n ") if is_enabled == "enabled" or is_active == "active": hosts_with_nm.append(host) else: server.clear() server.append("service NetworkManager status") rc, out = server.execute(can_fail=False) if rc < 1: hosts_with_nm.append(host) server.clear() if hosts_with_nm: hosts_list = ', '.join("%s" % x for x in hosts_with_nm) msg = output_messages.WARN_NM_ENABLED messages.append(utils.color_text(msg % hosts_list, 'yellow'))
def create_ntp_manifest(config, messages): srvlist = [i.strip() for i in config['CONFIG_NTP_SERVERS'].split(',') if i.strip()] config['CONFIG_NTP_SERVERS'] = ' '.join(srvlist) definiton = '\n'.join(['server %s' % i for i in srvlist]) config['CONFIG_NTP_SERVER_DEF'] = '%s\n' % definiton marker = uuid.uuid4().hex[:16] for hostname in filtered_hosts(config): releaseos = config['HOST_DETAILS'][hostname]['os'] releasever = config['HOST_DETAILS'][hostname]['release'].split('.')[0] # Configure chrony for Fedora or RHEL/CentOS 7 if releaseos == 'Fedora' or releasever == '7': manifestdata = getManifestTemplate('chrony.pp') appendManifestFile('%s_chrony.pp' % hostname, manifestdata, marker=marker) # For previous versions, configure ntpd else: manifestdata = getManifestTemplate('ntpd.pp') appendManifestFile('%s_ntpd.pp' % hostname, manifestdata, marker=marker)
def disable_nm(config): """ Sets NM_CONTROLLED="no" in existing network scripts on all nodes. """ for hostname in filtered_hosts(config): server = utils.ScriptRunner(hostname) server.append('ip a | grep -e "^[0-9]\: [a-zA-Z0-9\-]*\:" | ' 'sed -e "s/.*: \\(.*\\):.*/\\1/g"') rc, out = server.execute() devices = [i.strip() for i in out.split('\n') if i.strip()] devre = '\|'.join(devices) path = '/etc/sysconfig/network-scripts/' server.clear() server.append('ls -1 %(path)s | grep -e "ifcfg-\(%(devre)s\)"' % locals()) rc, out = server.execute() netscripts = [i.strip() for i in out.split('\n') if i.strip()] opt = 'NM_CONTROLLED' server.clear() for script in netscripts: server.append('sed -i \'s/^%(opt)s=.*/%(opt)s="no"/g\' ' '%(path)s%(script)s' % locals()) server.execute()
def createmanifest(config): manifestfile = "%s_qpid.pp"%config['CONFIG_QPID_HOST'] manifestdata = "" ssl_manifestdata = "" server = utils.ScriptRunner(config['CONFIG_QPID_HOST']) ports = set(["'5672'"]) if config['CONFIG_QPID_ENABLE_SSL'] == 'y': ports.add("'%s'" % (config['CONFIG_QPID_SSL_PORT'])) config['CONFIG_QPID_ENABLE_SSL'] = 'true' if config['CONFIG_QPID_SSL_SELF_SIGNED'] == 'y': server.append( "openssl req -batch -new -x509 -nodes -keyout %s -out %s -days 1095" % (config['CONFIG_QPID_SSL_KEY_FILE'], config['CONFIG_QPID_SSL_CERT_FILE']) ) server.execute() ssl_manifestdata = getManifestTemplate('qpid_ssl.pp') else: #Set default values config['CONFIG_QPID_SSL_PORT'] = "5671" config['CONFIG_QPID_SSL_CERT_FILE'] = "" config['CONFIG_QPID_SSL_KEY_FILE'] = "" config['CONFIG_QPID_NSS_CERTDB_PW'] = "" config['CONFIG_QPID_ENABLE_SSL'] = 'false' manifestdata = getManifestTemplate('qpid.pp') manifestdata += ssl_manifestdata #All hosts should be able to talk to qpid hosts = ["'%s'" % i for i in filtered_hosts(config, exclude=False)] config['FIREWALL_ALLOWED'] = ','.join(hosts) config['FIREWALL_SERVICE_NAME'] = "qpid" config['FIREWALL_PORTS'] = ','.join(ports) manifestdata += getManifestTemplate("firewall.pp") appendManifestFile(manifestfile, manifestdata, 'pre')
def create_manifest(config): key = "CONFIG_DEBUG_MODE" config[key] = config[key] == "y" and "true" or "false" for hostname in filtered_hosts(config): manifestfile = "%s_prescript.pp" % hostname manifestdata = getManifestTemplate("prescript.pp") appendManifestFile(manifestfile, manifestdata)
def create_manifest(config, messages): key = 'CONFIG_DEBUG_MODE' config[key] = config[key] == 'y' and 'true' or 'false' for hostname in filtered_hosts(config): manifestfile = "%s_prescript.pp" % hostname manifestdata = getManifestTemplate("prescript.pp") appendManifestFile(manifestfile, manifestdata)
def installdeps(config): for hostname in filtered_hosts(config): server = utils.ScriptRunner(hostname) # cylee : 2014-05-13 : change install package name for Ubuntu for package in ("puppet", "openssh-client", "tar", "netcat"): server.append("sudo -u root apt-get install -y %s" % (package)) server.execute()
def createmanifest(config): for hostname in filtered_hosts(config): manifestfile = "%s_postscript.pp" % hostname manifestdata = getManifestTemplate("postscript.pp") appendManifestFile(manifestfile, manifestdata, 'postscript') if config.get("CONFIG_PROVISION_ALL_IN_ONE_OVS_BRIDGE") != 'n': config['EXT_BRIDGE_VAR'] = config['CONFIG_NEUTRON_L3_EXT_BRIDGE'].replace('-','_') manifestdata = getManifestTemplate("persist_ovs_bridge.pp") appendManifestFile(manifestfile, manifestdata, 'postscript')
def finalize(config): for hostname in filtered_hosts(config): server = utils.ScriptRunner(hostname) server.append("installed=$(rpm -q kernel --last | head -n1 | " "sed 's/kernel-\([a-z0-9\.\_\-]*\).*/\\1/g')") server.append("loaded=$(uname -r | head -n1)") server.append('[ "$loaded" == "$installed" ]') try: rc, out = server.execute() except ScriptRuntimeError: controller.MESSAGES.append("Because of the kernel update the host " "%s requires reboot." % hostname)
def create_ntp_manifest(config): srvlist = [i.strip() for i in config["CONFIG_NTP_SERVERS"].split(",") if i.strip()] config["CONFIG_NTP_SERVERS"] = " ".join(srvlist) definiton = "\n".join(["server %s" % i for i in srvlist]) config["CONFIG_NTP_SERVER_DEF"] = "%s\n" % definiton marker = uuid.uuid4().hex[:16] for hostname in filtered_hosts(config): manifestdata = getManifestTemplate("ntpd.pp") appendManifestFile("%s_ntpd.pp" % hostname, manifestdata, marker=marker)
def create_manifest(config, messages): for hostname in filtered_hosts(config): manifestfile = "%s_postscript.pp" % hostname manifestdata = getManifestTemplate("postscript") appendManifestFile(manifestfile, manifestdata, 'postscript') # TO-DO: remove this temporary fix for nova-network/neutron # nondeterministic behavior provision = ( config.get("CONFIG_PROVISION_ALL_IN_ONE_OVS_BRIDGE") not in set(['false', 'n', None]) )
def install_keys(config): with open(config["CONFIG_SSH_KEY"]) as fp: sshkeydata = fp.read().strip() for hostname in filtered_hosts(config): if '/' in hostname: hostname = hostname.split('/')[0] server = utils.ScriptRunner(hostname) server.append("sshpass -p stack ssh-copy-id -i %s " "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null stack@%s" % (config['CONFIG_SSH_KEY'], hostname)) server.execute()
def installKeys(config): with open(config["CONFIG_SSH_KEY"]) as fp: sshkeydata = fp.read().strip() for hostname in filtered_hosts(config): server = utils.ScriptRunner(hostname) # TODO replace all that with ssh-copy-id server.append("mkdir -p ~/.ssh") server.append("chmod 500 ~/.ssh") server.append("grep '%s' ~/.ssh/authorized_keys > /dev/null 2>&1 || echo %s >> ~/.ssh/authorized_keys" % (sshkeydata, sshkeydata)) server.append("chmod 400 ~/.ssh/authorized_keys") server.append("restorecon -r ~/.ssh") server.execute()
def applyPuppetManifest(config): if config.get("DRY_RUN"): return currently_running = [] lastmarker = None loglevel = '' logcmd = False if logging.root.level <= logging.DEBUG: loglevel = '--debug' logcmd = True for manifest, marker in manifestfiles.getFiles(): # if the marker has changed then we don't want to proceed until # all of the previous puppet runs have finished if lastmarker != None and lastmarker != marker: waitforpuppet(currently_running) lastmarker = marker for hostname in filtered_hosts(config): if "%s_" % hostname not in manifest: continue host_dir = config['HOST_DETAILS'][hostname]['tmpdir'] print "Applying %s" % manifest server = utils.ScriptRunner(hostname) man_path = os.path.join(config['HOST_DETAILS'][hostname]['tmpdir'], basedefs.PUPPET_MANIFEST_RELATIVE, manifest) running_logfile = "%s.running" % man_path finished_logfile = "%s.finished" % man_path currently_running.append((hostname, finished_logfile)) # The apache puppet module doesn't work if we set FACTERLIB # https://github.com/puppetlabs/puppetlabs-apache/pull/138 if not (manifest.endswith('_horizon.pp') or manifest.endswith('_nagios.pp')): server.append("export FACTERLIB=$FACTERLIB:%s/facts" % host_dir) server.append("touch %s" % running_logfile) server.append("chmod 600 %s" % running_logfile) server.append("export PACKSTACK_VAR_DIR=%s" % host_dir) # cylee : Apply patch should run with sudo command = ("( flock %s/ps.lock sudo puppet apply" " --profile --debug --verbose " "%s --modulepath %s/modules %s > %s 2>&1 < /dev/null ;" " mv %s %s ) > /dev/null 2>&1 < /dev/null &")\ % (host_dir, loglevel, host_dir, man_path, running_logfile, running_logfile, finished_logfile) server.append(command) server.execute(log=logcmd) # wait for outstanding puppet runs befor exiting waitforpuppet(currently_running)
def copy_puppet_modules(config, messages): os_modules = ' '.join(('apache', 'ceilometer', 'certmonger', 'cinder', 'concat', 'firewall', 'glance', 'heat', 'horizon', 'inifile', 'ironic', 'keystone', 'memcached', 'mongodb', 'mysql', 'neutron', 'nova', 'nssdb', 'openstack', 'packstack', 'qpid', 'rabbitmq', 'redis', 'remote', 'rsync', 'sahara', 'ssh', 'stdlib', 'swift', 'sysctl', 'tempest', 'trove', 'vcsrepo', 'vlan', 'vswitch', 'xinetd', 'openstacklib')) # write puppet manifest to disk manifestfiles.writeManifests() # write hieradata file to disk generateHieraDataFile() server = utils.ScriptRunner() for hostname in filtered_hosts(config): host_dir = config['HOST_DETAILS'][hostname]['tmpdir'] # copy hiera defaults.yaml file server.append("cd %s" % basedefs.HIERADATA_DIR) server.append("tar --dereference -cpzf - ../hieradata | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "root@%s tar -C %s -xpzf -" % (hostname, host_dir)) # copy Packstack manifests server.append("cd %s/puppet" % basedefs.DIR_PROJECT_DIR) server.append("cd %s" % basedefs.PUPPET_MANIFEST_DIR) server.append("tar --dereference -cpzf - ../manifests | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "root@%s tar -C %s -xpzf -" % (hostname, host_dir)) # copy resources resources = config.get('RESOURCES', {}) for path, localname in resources.get(hostname, []): server.append("scp -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "%s root@%s:%s/resources/%s" % (path, hostname, host_dir, localname)) # copy Puppet modules required by Packstack server.append("cd %s" % MODULE_DIR) server.append("tar --dereference -cpzf - %s | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "root@%s tar -C %s -xpzf -" % (os_modules, hostname, os.path.join(host_dir, 'modules'))) server.execute()
def create_common_manifest(config, messages): global compute_hosts, network_hosts network_type = (config['CONFIG_NEUTRON_INSTALL'] == "y" and 'neutron' or 'nova') network_multi = len(network_hosts) > 1 dbacces_hosts = set([config.get('CONFIG_CONTROLLER_HOST')]) dbacces_hosts |= network_hosts for host in filtered_hosts(config): pw_in_sqlconn = False host = host.strip() if host in compute_hosts and host not in dbacces_hosts: # we should omit password in case we are installing only # nova-compute to the host perms = "nova" pw_in_sqlconn = False else: perms = "nova:%s" % config['CONFIG_NOVA_DB_PW'] pw_in_sqlconn = True mariadb_host_url = config['CONFIG_MARIADB_HOST_URL'] sqlconn = "mysql+pymysql://%s@%s/nova" % (perms, mariadb_host_url) if pw_in_sqlconn: config['CONFIG_NOVA_SQL_CONN_PW'] = sqlconn else: config['CONFIG_NOVA_SQL_CONN_NOPW'] = sqlconn # for nova-network in multihost mode each compute host is metadata # host otherwise we use api host if (network_type == 'nova' and network_multi and host in compute_hosts): metadata = host else: metadata = config['CONFIG_CONTROLLER_HOST'] config['CONFIG_NOVA_METADATA_HOST'] = metadata if config['CONFIG_AMQP_ENABLE_SSL'] == 'y': nova_hosts = compute_hosts nova_hosts |= set([config.get('CONFIG_CONTROLLER_HOST')]) ssl_cert_file = config['CONFIG_NOVA_SSL_CERT'] = ( '/etc/pki/tls/certs/ssl_amqp_nova.crt' ) ssl_key_file = config['CONFIG_NOVA_SSL_KEY'] = ( '/etc/pki/tls/private/ssl_amqp_nova.key' ) service = 'nova' for host in nova_hosts: generate_ssl_cert(config, host, service, ssl_key_file, ssl_cert_file)
def install_keys(config, messages): with open(config["CONFIG_SSH_KEY"]) as fp: sshkeydata = fp.read().strip() # If this is a --allinone install *and* we are running as root, # we can configure the authorized_keys file locally, avoid problems # if PasswordAuthentication is disabled. if is_all_in_one(config) and os.getuid() == 0: install_keys_on_host(None, sshkeydata) else: for hostname in filtered_hosts(config): if '/' in hostname: hostname = hostname.split('/')[0] install_keys_on_host(hostname, sshkeydata)
def create_manifest(config, messages): server = utils.ScriptRunner(config['CONFIG_AMQP_HOST']) if config['CONFIG_AMQP_ENABLE_SSL'] == 'y': config['CONFIG_AMQP_ENABLE_SSL'] = True config['CONFIG_AMQP_PROTOCOL'] = 'ssl' config['CONFIG_AMQP_CLIENTS_PORT'] = "5671" if config['CONFIG_AMQP_SSL_SELF_SIGNED'] == 'y': server.append( "openssl req -batch -new -x509 -nodes -keyout %s " "-out %s -days 1095" % (config['CONFIG_AMQP_SSL_KEY_FILE'], config['CONFIG_AMQP_SSL_CERT_FILE']) ) server.execute() else: # Set default values config['CONFIG_AMQP_CLIENTS_PORT'] = "5672" config['CONFIG_AMQP_SSL_PORT'] = "5671" config['CONFIG_AMQP_SSL_CERT_FILE'] = '' config['CONFIG_AMQP_SSL_KEY_FILE'] = '' config['CONFIG_AMQP_NSS_CERTDB_PW'] = '' config['CONFIG_AMQP_ENABLE_SSL'] = False config['CONFIG_AMQP_PROTOCOL'] = 'tcp' if config['CONFIG_AMQP_ENABLE_AUTH'] == 'n': config['CONFIG_AMQP_AUTH_PASSWORD'] = '******' config['CONFIG_AMQP_AUTH_USER'] = '******' manifestfile = "%s_amqp.pp" % config['CONFIG_AMQP_HOST'] manifestdata = getManifestTemplate('amqp') if config['CONFIG_IP_VERSION'] == 'ipv6': config['CONFIG_AMQP_HOST_URL'] = "[%s]" % config['CONFIG_AMQP_HOST'] else: config['CONFIG_AMQP_HOST_URL'] = config['CONFIG_AMQP_HOST'] fw_details = dict() # All hosts should be able to talk to amqp for host in filtered_hosts(config, exclude=False): key = "amqp_%s" % host fw_details.setdefault(key, {}) fw_details[key]['host'] = "%s" % host fw_details[key]['service_name'] = "amqp" fw_details[key]['chain'] = "INPUT" fw_details[key]['ports'] = ['5671', '5672'] fw_details[key]['proto'] = "tcp" config['FIREWALL_AMQP_RULES'] = fw_details manifestdata += createFirewallResources('FIREWALL_AMQP_RULES') appendManifestFile(manifestfile, manifestdata, 'pre')
def create_ntp_manifest(config): srvlist = [i.strip() for i in config['CONFIG_NTP_SERVERS'].split(',') if i.strip()] config['CONFIG_NTP_SERVERS'] = ' '.join(srvlist) definiton = '\n'.join(['server %s' % i for i in srvlist]) config['CONFIG_NTP_SERVER_DEF'] = '%s\n' % definiton marker = uuid.uuid4().hex[:16] for hostname in filtered_hosts(config): manifestdata = getManifestTemplate('ntpd.pp') appendManifestFile('%s_ntpd.pp' % hostname, manifestdata, marker=marker)
def createmanifest(config): manifestfile = "%s_qpid.pp"%config['CONFIG_QPID_HOST'] manifestdata = "" ssl_manifestdata = "" server = utils.ScriptRunner(config['CONFIG_QPID_HOST']) if config['CONFIG_QPID_ENABLE_SSL'] == 'y': config['CONFIG_QPID_ENABLE_SSL'] = 'true' config['CONFIG_QPID_PROTOCOL'] = 'ssl' config['CONFIG_QPID_CLIENTS_PORT'] = "5671" if config['CONFIG_QPID_SSL_SELF_SIGNED'] == 'y': server.append( "openssl req -batch -new -x509 -nodes -keyout %s -out %s -days 1095" % (config['CONFIG_QPID_SSL_KEY_FILE'], config['CONFIG_QPID_SSL_CERT_FILE']) ) server.execute() ssl_manifestdata = getManifestTemplate('qpid_ssl.pp') else: #Set default values config['CONFIG_QPID_CLIENTS_PORT'] = "5672" config['CONFIG_QPID_SSL_PORT'] = "5671" config['CONFIG_QPID_SSL_CERT_FILE'] = "" config['CONFIG_QPID_SSL_KEY_FILE'] = "" config['CONFIG_QPID_NSS_CERTDB_PW'] = "" config['CONFIG_QPID_ENABLE_SSL'] = 'false' config['CONFIG_QPID_PROTOCOL'] = 'tcp' manifestdata = getManifestTemplate('qpid.pp') manifestdata += ssl_manifestdata if config['CONFIG_QPID_ENABLE_AUTH'] == 'y': manifestdata += getManifestTemplate('qpid_auth.pp') else: config['CONFIG_QPID_AUTH_PASSWORD'] = '******' config['CONFIG_QPID_AUTH_USER'] = '******' #All hosts should be able to talk to qpid hosts = ["'%s'" % i for i in filtered_hosts(config, exclude=False)] # if the rule already exists for one port puppet will fail # so i had to add always both qpid ports (plain and SSL) in order # to avoid rule changes, this is due some problematic behaviour of # the puppet firewall module # this is a temporary solution, as soon as the firewall module is # updated we'll go back to previous state in which we open just # the needed ports config['FIREWALL_ALLOWED'] = ','.join(hosts) config['FIREWALL_SERVICE_NAME'] = "qpid" config['FIREWALL_PORTS'] = "'5671', '5672'" manifestdata += getManifestTemplate("firewall.pp") appendManifestFile(manifestfile, manifestdata, 'pre')
def create_manifest(config, messages): for hostname in filtered_hosts(config): manifestfile = "%s_postscript.pp" % hostname manifestdata = getManifestTemplate("postscript.pp") appendManifestFile(manifestfile, manifestdata, 'postscript') # TO-DO: remove this temporary fix for nova-network/neutron # undeterministic behavior provision = ( config.get("CONFIG_PROVISION_ALL_IN_ONE_OVS_BRIDGE") not in set(['false', 'n', None]) ) if config.get('CONFIG_NEUTRON_INSTALL', 'n') == 'y' and provision: fmted = config['CONFIG_NEUTRON_L3_EXT_BRIDGE'].replace('-', '_') config['EXT_BRIDGE_VAR'] = fmted manifestdata = getManifestTemplate("persist_ovs_bridge.pp") appendManifestFile(manifestfile, manifestdata, 'postscript')
def apply_puppet_manifest(config, messages): if config.get("DRY_RUN"): return currently_running = [] lastmarker = None loglevel = '' logcmd = False if logging.root.level <= logging.DEBUG: loglevel = '--debug' logcmd = True for manifest, marker in manifestfiles.getFiles(): # if the marker has changed then we don't want to proceed until # all of the previous puppet runs have finished if lastmarker is not None and lastmarker != marker: wait_for_puppet(currently_running, messages) lastmarker = marker for hostname in filtered_hosts(config): if "%s_" % hostname not in manifest: continue host_dir = config['HOST_DETAILS'][hostname]['tmpdir'] print "Applying %s" % manifest server = utils.ScriptRunner(hostname) man_path = os.path.join(config['HOST_DETAILS'][hostname]['tmpdir'], basedefs.PUPPET_MANIFEST_RELATIVE, manifest) running_logfile = "%s.running" % man_path finished_logfile = "%s.finished" % man_path currently_running.append((hostname, finished_logfile)) server.append("touch %s" % running_logfile) server.append("chmod 600 %s" % running_logfile) server.append("export PACKSTACK_VAR_DIR=%s" % host_dir) cmd = ("( flock %s/ps.lock " "puppet apply %s --modulepath %s/modules %s > %s " "2>&1 < /dev/null ; " "mv %s %s ) > /dev/null 2>&1 < /dev/null &" % (host_dir, loglevel, host_dir, man_path, running_logfile, running_logfile, finished_logfile)) server.append(cmd) server.execute(log=logcmd) # wait for outstanding puppet runs befor exiting wait_for_puppet(currently_running, messages)
def copyPuppetModules(config): os_modules = ' '.join(('apache', 'ceilometer', 'cinder', 'concat', 'create_resources', 'firewall', 'glance', 'heat', 'horizon', 'inifile', 'keystone', 'memcached', 'mongodb', 'mysql', 'neutron', 'nova', 'openstack', 'packstack', 'qpid', 'rsync', 'ssh', 'stdlib', 'swift', 'sysctl', 'tempest', 'vcsrepo', 'vlan', 'vswitch', 'xinetd')) # write puppet manifest to disk manifestfiles.writeManifests() server = utils.ScriptRunner() tar_opts = "" if platform.linux_distribution()[0] == "Fedora": tar_opts += "--exclude create_resources " for hostname in filtered_hosts(config): host_dir = controller.temp_map[hostname] server.append("cd %s/puppet" % basedefs.DIR_PROJECT_DIR) # copy Packstack facts server.append("tar %s --dereference -cpzf - facts | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "root@%s tar -C %s -xpzf -" % (tar_opts, hostname, host_dir)) # copy Packstack manifests server.append("cd %s" % basedefs.PUPPET_MANIFEST_DIR) server.append("tar %s --dereference -cpzf - ../manifests | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "root@%s tar -C %s -xpzf -" % (tar_opts, hostname, host_dir)) # copy resources for path, localname in controller.resources.get(hostname, []): server.append("scp -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null %s root@%s:%s/resources/%s" % (path, hostname, host_dir, localname)) # copy Puppet modules required by Packstack server.append("cd %s/puppet/modules" % basedefs.DIR_PROJECT_DIR) server.append("tar %s --dereference -cpzf - %s | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "root@%s tar -C %s -xpzf -" % (tar_opts, os_modules, hostname, os.path.join(host_dir, 'modules'))) server.execute()
def create_manifest(config, messages): server = utils.ScriptRunner(config['CONFIG_AMQP_HOST']) if config['CONFIG_AMQP_ENABLE_SSL'] == 'y': config['CONFIG_AMQP_SSL_ENABLED'] = True config['CONFIG_AMQP_PROTOCOL'] = 'ssl' config['CONFIG_AMQP_CLIENTS_PORT'] = "5671" amqp_host = config['CONFIG_AMQP_HOST'] service = 'AMQP' ssl_key_file = '/etc/pki/tls/private/ssl_amqp.key' ssl_cert_file = '/etc/pki/tls/certs/ssl_amqp.crt' cacert = config['CONFIG_AMQP_SSL_CACERT_FILE'] = ( config['CONFIG_SSL_CACERT'] ) generate_ssl_cert(config, amqp_host, service, ssl_key_file, ssl_cert_file) else: # Set default values config['CONFIG_AMQP_CLIENTS_PORT'] = "5672" config['CONFIG_AMQP_SSL_ENABLED'] = False config['CONFIG_AMQP_PROTOCOL'] = 'tcp' if config['CONFIG_AMQP_ENABLE_AUTH'] == 'n': config['CONFIG_AMQP_AUTH_PASSWORD'] = '******' config['CONFIG_AMQP_AUTH_USER'] = '******' manifestfile = "%s_amqp.pp" % config['CONFIG_AMQP_HOST'] manifestdata = getManifestTemplate('amqp') if config['CONFIG_IP_VERSION'] == 'ipv6': config['CONFIG_AMQP_HOST_URL'] = "[%s]" % config['CONFIG_AMQP_HOST'] else: config['CONFIG_AMQP_HOST_URL'] = config['CONFIG_AMQP_HOST'] fw_details = dict() # All hosts should be able to talk to amqp for host in filtered_hosts(config, exclude=False): key = "amqp_%s" % host fw_details.setdefault(key, {}) fw_details[key]['host'] = "%s" % host fw_details[key]['service_name'] = "amqp" fw_details[key]['chain'] = "INPUT" fw_details[key]['ports'] = ['5671', '5672'] fw_details[key]['proto'] = "tcp" config['FIREWALL_AMQP_RULES'] = fw_details manifestdata += createFirewallResources('FIREWALL_AMQP_RULES') appendManifestFile(manifestfile, manifestdata, 'pre')
def copyPuppetModules(config): os_modules = ' '.join(('apache', 'ceilometer', 'certmonger', 'cinder', 'concat', 'firewall', 'glance', 'heat', 'horizon', 'inifile', 'keystone', 'memcached', 'mongodb', 'mysql', 'neutron', 'nova', 'nssdb', 'openstack', 'packstack', 'qpid', 'rabbitmq', 'rsync', 'ssh', 'stdlib', 'swift', 'sysctl', 'tempest', 'vcsrepo', 'vlan', 'vswitch', 'xinetd', 'postgresql', 'staging', 'apt')) # write puppet manifest to disk manifestfiles.writeManifests() server = utils.ScriptRunner() # cylee : Each system must create an sudoer called 'stack', # it don't need password while sudo. for hostname in filtered_hosts(config): host_dir = config['HOST_DETAILS'][hostname]['tmpdir'] # copy Packstack manifests server.append("cd %s/puppet" % basedefs.DIR_PROJECT_DIR) server.append("cd %s" % basedefs.PUPPET_MANIFEST_DIR) server.append("tar --dereference -cpzf - ../manifests | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "stack@%s tar -C %s -xpzf -" % (hostname, host_dir)) server.append("cd %s/" % basedefs.DIR_PROJECT_DIR) server.append("tar --dereference -cpzf - other | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "stack@%s tar -C %s -xpzf -" % (hostname, host_dir)) # copy resources for path, localname in controller.resources.get(hostname, []): server.append("scp -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null %s stack@%s:%s/resources/%s" % (path, hostname, host_dir, localname)) # copy Puppet modules required by Packstack server.append("cd %s" % MODULE_DIR) server.append("tar --dereference -cpzf - %s | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "stack@%s tar -C %s -xpzf -" % (os_modules, hostname, os.path.join(host_dir, 'modules'))) server.execute()
def discover(config): """ Discovers details about hosts. """ # TODO: Once Controller is refactored, move this function to it (facter can # be used for that too). details = {} release_regexp = re.compile(r'^(?P<OS>.*) release (?P<release>[\d\.]*)') for host in filtered_hosts(config): details.setdefault(host, {}) server = utils.ScriptRunner(host) # discover OS and release #server.append('cat /etc/redhat-release') # Bevis : change to ubuntu release file server.append('cat /etc/issue.net') try: rc, out = server.execute() match = release_regexp.search(out) if not match: raise exceptions.ScriptRuntimeError() except exceptions.ScriptRuntimeError: details[host]['os'] = 'Unknown' details[host]['release'] = 'Unknown' else: opsys = match.group('OS') for pattern, surr in [('^Red Hat Enterprise Linux.*', 'RHEL'), ('^Fedora.*', 'Fedora'), ('^CentOS.*', 'CentOS'), ('^Scientific Linux.*', 'SL'), ('^Ubuntu*', '^Debain*')]: opsys = re.sub(pattern, surr, opsys) details[host]['os'] = opsys details[host]['release'] = match.group('release') # Create the packstack tmp directory server.clear() server.append("mkdir -p %s" % basedefs.PACKSTACK_VAR_DIR) # Separately create the tmp directory for this packstack run, this will # fail if the directory already exists host_dir = os.path.join(basedefs.PACKSTACK_VAR_DIR, uuid.uuid4().hex) server.append("mkdir --mode 0700 %s" % host_dir) for i in ('modules', 'resources'): server.append("mkdir --mode 0700 %s" % os.path.join(host_dir, i)) server.execute() details[host]['tmpdir'] = host_dir config['HOST_DETAILS'] = details
def createmanifest(config): manifestfile = "%s_qpid.pp"%config['CONFIG_QPID_HOST'] manifestdata = "" ssl_manifestdata = "" server = utils.ScriptRunner(config['CONFIG_QPID_HOST']) ports = set(["'5672'"]) if config['CONFIG_QPID_ENABLE_SSL'] == 'y': ports.add("'%s'" % (config['CONFIG_QPID_SSL_PORT'])) config['CONFIG_QPID_ENABLE_SSL'] = 'true' config['CONFIG_QPID_PROTOCOL'] = 'ssl' config['CONFIG_QPID_CLIENTS_PORT'] = "5671" if config['CONFIG_QPID_SSL_SELF_SIGNED'] == 'y': server.append( "openssl req -batch -new -x509 -nodes -keyout %s -out %s -days 1095" % (config['CONFIG_QPID_SSL_KEY_FILE'], config['CONFIG_QPID_SSL_CERT_FILE']) ) server.execute() ssl_manifestdata = getManifestTemplate('qpid_ssl.pp') else: #Set default values config['CONFIG_QPID_CLIENTS_PORT'] = "5672" config['CONFIG_QPID_SSL_PORT'] = "5671" config['CONFIG_QPID_SSL_CERT_FILE'] = "" config['CONFIG_QPID_SSL_KEY_FILE'] = "" config['CONFIG_QPID_NSS_CERTDB_PW'] = "" config['CONFIG_QPID_ENABLE_SSL'] = 'false' config['CONFIG_QPID_PROTOCOL'] = 'tcp' manifestdata = getManifestTemplate('qpid.pp') manifestdata += ssl_manifestdata if config['CONFIG_QPID_ENABLE_AUTH'] == 'y': manifestdata += getManifestTemplate('qpid_auth.pp') else: config['CONFIG_QPID_AUTH_PASSWORD'] = '******' config['CONFIG_QPID_AUTH_USER'] = '******' #All hosts should be able to talk to qpid hosts = ["'%s'" % i for i in filtered_hosts(config, exclude=False)] # if the rule already exists for one port puppet will fail # we have to add them by separate for port in ports: config['FIREWALL_ALLOWED'] = ','.join(hosts) config['FIREWALL_SERVICE_NAME'] = "qpid - %s" % (port) config['FIREWALL_PORTS'] = port manifestdata += getManifestTemplate("firewall.pp") appendManifestFile(manifestfile, manifestdata, 'pre')
def copy_puppet_modules(config, messages): os_modules = ' '.join(('apache', 'ceilometer', 'certmonger', 'cinder', 'concat', 'firewall', 'glance', 'heat', 'horizon', 'inifile', 'keystone', 'memcached', 'mongodb', 'mysql', 'neutron', 'nova', 'nssdb', 'openstack', 'packstack', 'qpid', 'rabbitmq', 'rsync', 'ssh', 'stdlib', 'swift', 'sysctl', 'tempest', 'vcsrepo', 'vlan', 'vswitch', 'xinetd')) # write puppet manifest to disk manifestfiles.writeManifests() server = utils.ScriptRunner() for hostname in filtered_hosts(config): host_dir = config['HOST_DETAILS'][hostname]['tmpdir'] # copy Packstack manifests server.append("cd %s/puppet" % basedefs.DIR_PROJECT_DIR) server.append("cd %s" % basedefs.PUPPET_MANIFEST_DIR) server.append("tar --dereference -cpzf - ../manifests | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "root@%s tar -C %s -xpzf -" % (hostname, host_dir)) # copy resources resources = config.get('RESOURCES', {}) for path, localname in resources.get(hostname, []): server.append("scp -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "%s root@%s:%s/resources/%s" % (path, hostname, host_dir, localname)) # copy Puppet modules required by Packstack server.append("cd %s" % MODULE_DIR) server.append("tar --dereference -cpzf - %s | " "ssh -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "root@%s tar -C %s -xpzf -" % (os_modules, hostname, os.path.join(host_dir, 'modules'))) server.execute()
def install_deps(config, messages): deps = ["puppet", "openssh-clients", "tar", "nc"] modules_pkg = 'openstack-puppet-modules' local = utils.ScriptRunner() local.append('rpm -q --requires %s | egrep -v "^(rpmlib|\/|perl)"' % modules_pkg) rc, modules_deps = local.execute() # Modules package might not be installed if we are running from source. # In this case we assume user knows what (s)he's doing and we don't # install modules dependencies if ('%s is not installed' % modules_pkg) not in modules_deps: modules_deps = [i.strip() for i in modules_deps.split() if i.strip()] deps.extend(modules_deps) for hostname in filtered_hosts(config): server = utils.ScriptRunner(hostname) for package in deps: server.append("rpm -q --whatprovides %s || yum install -y %s" % (package, package)) server.execute()
def create_manifest(config, messages): config['CONFIG_NAGIOS_NODES'] = list(filtered_hosts(config)) openstack_services = [] openstack_services.append('keystone-user-list') if config['CONFIG_GLANCE_INSTALL'] == 'y': openstack_services.append('glance-index') if config['CONFIG_NOVA_INSTALL'] == 'y': openstack_services.append('nova-list') if config['CONFIG_CINDER_INSTALL'] == 'y': openstack_services.append('cinder-list') if config['CONFIG_SWIFT_INSTALL'] == 'y': openstack_services.append('swift-list') config['CONFIG_NAGIOS_SERVICES'] = openstack_services manifestfile = "%s_nagios.pp" % config['CONFIG_CONTROLLER_HOST'] manifestdata = getManifestTemplate("nagios_server") appendManifestFile(manifestfile, manifestdata)
def create_manifest(config, messages): if config['CONFIG_MARIADB_INSTALL'] == 'y': suffix = 'install' host = config['CONFIG_MARIADB_HOST'] else: suffix = 'noinstall' host = config['CONFIG_CONTROLLER_HOST'] manifestfile = "%s_mariadb.pp" % host manifestdata = [getManifestTemplate('mariadb_%s' % suffix)] def append_for(module, suffix): # Modules have to be appended to the existing mysql.pp # otherwise pp will fail for some of them saying that # Mysql::Config definition is missing. template = "mariadb_%s_%s" % (module, suffix) manifestdata.append(getManifestTemplate(template)) append_for("keystone", suffix) for mod in ['nova', 'cinder', 'glance', 'neutron', 'heat', 'sahara']: if config['CONFIG_%s_INSTALL' % mod.upper()] == 'y': append_for(mod, suffix) hosts = filtered_hosts(config, exclude=False, dbhost=True) fw_details = dict() for host in hosts: key = "mariadb_%s" % host fw_details.setdefault(key, {}) fw_details[key]['host'] = "%s" % host fw_details[key]['service_name'] = "mariadb" fw_details[key]['chain'] = "INPUT" fw_details[key]['ports'] = ['3306'] fw_details[key]['proto'] = "tcp" config['FIREWALL_MARIADB_RULES'] = fw_details manifestdata.append(createFirewallResources('FIREWALL_MARIADB_RULES')) appendManifestFile(manifestfile, "\n".join(manifestdata), 'pre')
def create_nrpe_manifests(config, messages): for hostname in filtered_hosts(config): config['CONFIG_NRPE_HOST'] = hostname manifestfile = "%s_nagios_nrpe.pp" % hostname manifestdata = getManifestTemplate("nagios_nrpe") # Only the Nagios host is allowed to talk to nrpe fw_details = dict() key = "nagios_nrpe" fw_details.setdefault(key, {}) fw_details[key]['host'] = "%s" % config['CONFIG_CONTROLLER_HOST'] fw_details[key]['service_name'] = "nagios-nrpe" fw_details[key]['chain'] = "INPUT" fw_details[key]['ports'] = ['5666'] fw_details[key]['proto'] = "tcp" config['FIREWALL_NAGIOS_NRPE_RULES'] = fw_details manifestdata += createFirewallResources('FIREWALL_NAGIOS_NRPE_RULES') appendManifestFile(manifestfile, manifestdata) messages.append("To use Nagios, browse to " "http://%(CONFIG_CONTROLLER_HOST)s/nagios " "username: nagiosadmin, password: %(CONFIG_NAGIOS_PW)s" % config)
def create_manifest(config): for hostname in filtered_hosts(config): manifestfile = "%s_prescript.pp" % hostname manifestdata = getManifestTemplate("prescript.pp") appendManifestFile(manifestfile, manifestdata)
def create_manifest(config, messages): manifest_entries = '' # I should be adding service entries with nagios_service # but it appears to be broken http://projects.puppetlabs.com/issues/3420 service_entries = '' for hostname in filtered_hosts(config): manifest_entries += nagios_host(hostname, address=hostname, use='linux-server') service_entries += _serviceentry( name='load5-%s' % hostname, service_description='5 minute load average', host_name=hostname, check_command="check_nrpe!load5", use="generic-service", normal_check_interval='5' ) service_entries += _serviceentry( name='df_var-%s' % hostname, service_description='Percent disk space used on /var', host_name=hostname, check_command="check_nrpe!df_var", use="generic-service" ) manifest_entries += _copy_script(name="keystone-user-list") service_entries += _serviceentry( name='keystone-user-list', service_description='number of keystone users', host_name=config['CONFIG_CONTROLLER_HOST'], check_command="keystone-user-list", use="generic-service", normal_check_interval='5' ) if config['CONFIG_GLANCE_INSTALL'] == 'y': manifest_entries += _copy_script(name="glance-index") service_entries += _serviceentry( name='glance-index', service_description='number of glance images', host_name=config['CONFIG_CONTROLLER_HOST'], check_command="glance-index", use="generic-service", normal_check_interval='5' ) if config['CONFIG_NOVA_INSTALL'] == 'y': manifest_entries += _copy_script(name="nova-list") service_entries += _serviceentry( name='nova-list', service_description='number of nova vm instances', host_name=config['CONFIG_CONTROLLER_HOST'], check_command="nova-list", use="generic-service", normal_check_interval='5' ) if config['CONFIG_CINDER_INSTALL'] == 'y': manifest_entries += _copy_script(name="cinder-list") service_entries += _serviceentry( name='cinder-list', service_description='number of cinder volumes', host_name=config['CONFIG_CONTROLLER_HOST'], check_command="cinder-list", use="generic-service", normal_check_interval='5' ) if config['CONFIG_SWIFT_INSTALL'] == 'y': manifest_entries += _copy_script(name="swift-list") service_entries += _serviceentry( name='swift-list', service_description='number of swift containers', host_name=config['CONFIG_CONTROLLER_HOST'], check_command="swift-list", use="generic-service", normal_check_interval='5' ) manifest_entries += ("file { '/etc/nagios/nagios_service.cfg': \n" "ensure => present, mode => 644,\n" "owner => 'nagios', group => 'nagios',\n" "before => Service['nagios'],\n" "content => '%s'}" % service_entries) config['CONFIG_NAGIOS_MANIFEST_CONFIG'] = manifest_entries manifestfile = "%s_nagios.pp" % config['CONFIG_CONTROLLER_HOST'] manifestdata = getManifestTemplate("nagios_server") appendManifestFile(manifestfile, manifestdata)
def server_prep(config, messages): rh_username = None sat_url = None if is_rhel(): rh_username = config["CONFIG_RH_USER"].strip() rh_password = config["CONFIG_RH_PW"].strip() sat_registered = set() sat_url = config["CONFIG_SATELLITE_URL"].strip() if sat_url: flag_list = config["CONFIG_SATELLITE_FLAGS"].split(',') sat_flags = [i.strip() for i in flag_list if i.strip()] sat_proxy_user = config.get("CONFIG_SATELLITE_PROXY_USER", '') sat_proxy_pass = config.get("CONFIG_SATELLITE_PROXY_PW", '') sat_args = { 'username': config["CONFIG_SATELLITE_USER"].strip(), 'password': config["CONFIG_SATELLITE_PW"].strip(), 'cacert': config["CONFIG_SATELLITE_CACERT"].strip(), 'activation_key': config["CONFIG_SATELLITE_AKEY"].strip(), 'profile_name': config["CONFIG_SATELLITE_PROFILE"].strip(), 'proxy_host': config["CONFIG_SATELLITE_PROXY"].strip(), 'proxy_user': sat_proxy_user.strip(), 'proxy_pass': sat_proxy_pass.strip(), 'flags': sat_flags } for hostname in filtered_hosts(config): # Subscribe to Red Hat Repositories if configured if rh_username: run_rhsm_reg(config, hostname, rh_username, rh_password) # Subscribe to RHN Satellite if configured if sat_url and hostname not in sat_registered: run_rhn_reg(hostname, sat_url, **sat_args) sat_registered.add(hostname) server = utils.ScriptRunner(hostname) server.append('rpm -q --whatprovides yum-utils || ' 'yum install -y yum-utils') # Installing rhos-log-collector and sos-plugins-openstack if # these rpms are available from yum. sos_rpms = ' '.join( ('rhos-log-collector', 'sos', 'sos-plugins-openstack')) server.append('yum list available rhos-log-collector && ' 'yum -y install %s || ' 'echo "no rhos-log-collector available"' % sos_rpms) server.execute() # enable or disable EPEL according to configuration manage_epel(hostname, config) # enable RDO if it is installed locally manage_rdo(hostname, config) reponame = 'rhel-server-ost-6-4-rpms' server.clear() server.append('yum install -y yum-plugin-priorities || true') server.append('rpm -q epel-release && yum-config-manager ' '--setopt="%(reponame)s.priority=1" ' '--save %(reponame)s' % locals()) # Add yum repositories if configured CONFIG_REPO = config["CONFIG_REPO"].strip() if CONFIG_REPO: for i, repourl in enumerate(CONFIG_REPO.split(',')): reponame = 'packstack_%d' % i server.append('echo "[%(reponame)s]\nname=%(reponame)s\n' 'baseurl=%(repourl)s\nenabled=1\n' 'priority=1\ngpgcheck=0"' ' > /etc/yum.repos.d/%(reponame)s.repo' % locals()) server.append("yum clean metadata") server.execute()