def on_host_init_response(self, message): with bus.initialization_op as op: with op.phase(self._phase_rabbitmq): with op.step(self._step_accept_scalr_conf): if not message.body.has_key("rabbitmq"): raise HandlerError("HostInitResponse message for RabbitMQ behaviour must have 'rabbitmq' property") rabbitmq_data = message.rabbitmq.copy() if not rabbitmq_data['password']: rabbitmq_data['password'] = cryptotool.pwgen(10) hostname = RABBIT_HOSTNAME_TPL % int(message.server_index) rabbitmq_data['server_index'] = message.server_index rabbitmq_data['hostname'] = hostname dns.ScalrHosts.set('127.0.0.1', hostname) with open('/etc/hostname', 'w') as f: f.write(hostname) system2(('hostname', '-F', '/etc/hostname')) volume_config = rabbitmq_data.pop('volume_config') volume_config['mpoint'] = DEFAULT_STORAGE_PATH rabbitmq_data['volume'] = storage2.volume(volume_config) rabbitmq_data['volume'].tags = self.rabbitmq_tags __rabbitmq__.update(rabbitmq_data)
def on_init(self): bus.on(host_init_response=self.on_host_init_response, block_device_mounted=self.on_block_device_mounted) # Add internal messages to scripting skip list try: for m in (Messages.INT_SERVER_REBOOT, Messages.INT_SERVER_HALT, Messages.HOST_INIT_RESPONSE): scalarizr.handlers.script_executor.skip_events.add(m) except AttributeError: pass # Mount all filesystems if os_dist["family"] != "Windows": system2(("mount", "-a"), raise_exc=False) # cloud-init scripts may disable root ssh login for path in ("/etc/ec2-init/ec2-config.cfg", "/etc/cloud/cloud.cfg"): if os.path.exists(path): c = None with open(path, "r") as fp: c = fp.read() c = re.sub(re.compile(r"^disable_root[^:=]*([:=]).*", re.M), r"disable_root\1 0", c) with open(path, "w") as fp: fp.write(c) # Add firewall rules # if self._cnf.state in (ScalarizrState.BOOTSTRAPPING, ScalarizrState.IMPORTING): self._insert_iptables_rules() # if __node__['state'] != ScalarizrState.IMPORTING: if __node__["state"] == "running": scalarizr.handlers.sync_globals()
def _format_image(self): LOG.info("Formatting image") vol_entry = [v for v in self._mtab if v.device.startswith('/dev')][0] if vol_entry.device == '/dev/root' and not os.path.exists(vol_entry.device): vol_entry = [v for v in mount.mounts('/etc/mtab') if v.device.startswith('/dev')][0] fs = filesystem(vol_entry.fstype) # create filesystem fs.mkfs(self.devname) # set EXT3/4 options if fs.type.startswith('ext'): # max mounts before check (-1 = disable) system2(('/sbin/tune2fs', '-c', '1', self.devname)) # time based (3m = 3 month) system2(('/sbin/tune2fs', '-i', '3m', self.devname)) # set label label = fs.get_label(vol_entry.device) if label: fs.set_label(self.devname, label) LOG.debug('Image %s formatted', self.devname)
def on_host_init_response(self, message): with bus.initialization_op as op: with op.phase(self._phase_rabbitmq): with op.step(self._step_accept_scalr_conf): if not message.body.has_key("rabbitmq"): raise HandlerError("HostInitResponse message for RabbitMQ behaviour must have 'rabbitmq' property") path = os.path.dirname(self._volume_config_path) if not os.path.exists(path): os.makedirs(path) rabbitmq_data = message.rabbitmq.copy() if not rabbitmq_data['password']: rabbitmq_data['password'] = cryptotool.pwgen(10) if os.path.exists(self._volume_config_path): os.remove(self._volume_config_path) hostname = RABBIT_HOSTNAME_TPL % int(message.server_index) rabbitmq_data['server_index'] = message.server_index rabbitmq_data['hostname'] = hostname dns.ScalrHosts.set('127.0.0.1', hostname) with open('/etc/hostname', 'w') as f: f.write(hostname) system2(('hostname', '-F', '/etc/hostname')) if OPT_VOLUME_CNF in rabbitmq_data: if rabbitmq_data[OPT_VOLUME_CNF]: storage.Storage.backup_config(rabbitmq_data[OPT_VOLUME_CNF], self._volume_config_path) del rabbitmq_data[OPT_VOLUME_CNF] self._update_config(rabbitmq_data)
def make_snapshot(self, volume): prepared_image_path = os.path.join(self.destination, self.image_name) LOG.debug('sgp_dd image into volume %s' % volume.device) system2(('sgp_dd', 'if='+prepared_image_path, 'of='+volume.device, 'bs=8k', 'count=%s' % (self.image_size*1024*1024/8))) # coreutils.dd(**{'if': prepared_image_path, 'of': volume.device, 'bs': '8M'}) volume.mount() self.clean_snapshot(volume) LOG.debug('detaching volume') volume.detach() LOG.debug('Making snapshot of volume %s' % volume.device) snapshot = volume.snapshot() util.wait_until( lambda: snapshot.status() == 'completed', logger=LOG, error_text='EBS snapshot %s wasnt completed' % snapshot.id) LOG.debug('Snapshot is made') volume.ensure(mount=True) return snapshot.id
def __init__(self): if not os.path.exists(MDADM_EXEC): if linux.os.redhat_family: system2(('/usr/bin/yum', '-d0', '-y', 'install', 'mdadm', '-x', 'exim'), raise_exc=False) else: pkgmgr.installed('mdadm') if not os.path.exists('/proc/mdstat'): coreutils.modprobe('md_mod') for location in ['/etc ', '/lib']: path = os.path.join(location, 'udev/rules.d/85-mdadm.rules') if os.path.exists(path): rule = None with open(path, 'r') as fp: rule = fp.read() if rule: rule = re.sub(re.compile('^([^#])', re.M), '#\\1', rule) with open(path, 'w') as fp: fp.write(rule) self._raid_devices_re = re.compile('Raid\s+Devices\s+:\s+(?P<count>\d+)') self._total_devices_re = re.compile('Total\s+Devices\s+:\s+(?P<count>\d+)') self._state_re = re.compile('State\s+:\s+(?P<state>.+)') self._rebuild_re = re.compile('Rebuild\s+Status\s+:\s+(?P<percent>\d+)%') self._level_re = re.compile('Raid Level : (?P<level>.+)')
def cluster_with(self, hostnames, do_reset=True): nodes = ['rabbit@%s' % host for host in hostnames] cmd = [RABBITMQCTL, 'cluster'] + nodes clustered = False while not clustered: self.stop_app() if do_reset: self.reset() system2(cmd, logger=self._logger) p = subprocess.Popen((RABBITMQCTL, 'start_app')) for i in range(15): if p.poll() is None: time.sleep(1) continue if p.returncode: raise Exception(p.stderr.read()) else: clustered = True break else: p.kill() self.service.restart(force=True)
def create_snapshot(ec2_conn, volume_id, description=None, logger=None, timeout=SNAPSHOT_TIMEOUT, wait_completion=False, tags=None): if isinstance(volume_id, Volume): volume_id = volume_id.id logger = logger or logging.getLogger(__name__) # Create snapshot logger.debug('Creating snapshot of EBS volume %s', volume_id) system2('sync', shell=True) snap = ec2_conn.create_snapshot(volume_id, description) logger.debug('Snapshot %s created for EBS volume %s', snap.id, volume_id) # Apply tags if not tags: logger.debug('No tags to apply to EBS snapshot %s' % snap.id) else: try: logger.debug('Applying tags to EBS snapshot %s : %s' % (snap.id, tags)) ec2_conn.create_tags((snap.id, ), tags) except: logger.warn('Cannot apply tags to EBS snapshot %s', snap.id) if wait_completion: wait_snapshot(ec2_conn, snap, logger, timeout) return snap
def __init__(self): if 'gce' == node.__node__['platform'].name: self.ensure_pid_directory() self.mysql_cli = MySQLClient() service_exec = '/usr/sbin/service' if linux.os.debian_family else '/sbin/service' service_name = 'mysql' if linux.os.redhat_family: if linux.os['release'] >= (7, 0) and \ system2('systemctl list-unit-files | grep mariadb', raise_exc=False, shell=True)[2] == 0: service_name = 'mariadb' elif 'percona' not in node.__node__['behavior']: service_name = 'mysqld' if linux.os.redhat_family or linux.os.ubuntu: initd_script = (service_exec, service_name) else: initd_script = '/etc/init.d/{0}'.format(service_name) pid_file = None try: out = system2("my_print_defaults mysqld", shell=True, silent=True) m = re.search("--pid[-_]file=(.*)", out[0], re.MULTILINE) if m: pid_file = m.group(1) m = re.search("--socket=(.*)", out[0], re.MULTILINE) if m: self.socket_file = m.group(1) except: pass initdv2.ParametrizedInitScript.__init__(self, SERVICE_NAME, initd_script, pid_file, socks=[initdv2.SockParam(MYSQL_DEFAULT_PORT, timeout=3600)])
def cluster_with(self, self_hostname, hostnames, disk_node=True, do_reset=True): if RABBITMQ_VERSION >= (3, 0, 0): # New way of clustering was introduced in rabbit 3.0.0 one_node = NODE_HOSTNAME_TPL % hostnames[0] cmd = [RABBITMQCTL, 'join_cluster', one_node] if not disk_node: cmd.append('--ram') else: nodes = [NODE_HOSTNAME_TPL % host for host in hostnames] if disk_node: nodes.append(NODE_HOSTNAME_TPL % self_hostname) cmd = [RABBITMQCTL, 'cluster'] + nodes clustered = False while not clustered: self.stop_app() if do_reset: self.reset() system2(cmd, logger=self._logger) p = subprocess.Popen((RABBITMQCTL, 'start_app')) for i in range(15): if p.poll() is None: time.sleep(1) continue if p.returncode: raise Exception(p.stderr.read()) else: clustered = True break else: p.kill() self.service.restart(force=True)
def change_node_type(self, self_hostname, hostnames, disk_node): if RABBITMQ_VERSION >= (3, 0, 0): type = disk_node and 'disk' or 'ram' cmd = [RABBITMQCTL, 'change_cluster_node_type', type] system2(cmd, logger=self._logger) else: self.cluster_with(self_hostname, hostnames, disk_node, do_reset=False)
def move_mysqldir_to(self, storage_path): LOG.info('Moving mysql dir to %s' % storage_path) for directive, dirname in ( ('mysqld/log_bin', os.path.join(storage_path,STORAGE_BINLOG)), ('mysqld/datadir', os.path.join(storage_path,STORAGE_DATA_DIR) + '/') ): dest = os.path.dirname(dirname) if os.path.isdir(dest): LOG.info('No need to move %s to %s: already in place.' % (directive, dest)) else: os.makedirs(dest) raw_value = self.my_cnf.get(directive) LOG.debug('directive %s:%s' % (directive, raw_value)) if raw_value: src_dir = os.path.dirname(raw_value + "/") + "/" LOG.debug('source path: %s' % src_dir) if os.path.isdir(src_dir) and src_dir != dest: selinuxenabled = software.which('selinuxenabled') if selinuxenabled: if not system2((selinuxenabled, ), raise_exc=False)[2]: if not system2((software.which('getsebool'), 'mysqld_disable_trans'), raise_exc=False)[2]: LOG.debug('Make SELinux rule for rsync') system2((software.which('setsebool'), '-P', 'mysqld_disable_trans', '1')) LOG.info('Copying mysql directory \'%s\' to \'%s\'', src_dir, dest) rsync(src_dir, dest, archive=True, exclude=['ib_logfile*', '*.sock']) self.my_cnf.set(directive, dirname) chown_r(dest, "mysql", "mysql") # Adding rules to apparmor config if disttool.is_debian_based(): _add_apparmor_rules(dest)
def execute(self, query, silent=False): if not self.password: full_query = query else: full_query = 'AUTH %s\n%s' % (self.password, query) try: out = system2([self.path, '-p', self.port], stdin=full_query,silent=True, warn_stderr=False)[0] #fix for redis 2.4 AUTH if 'Client sent AUTH, but no password is set' in out: out = system2([self.path], stdin=query,silent=True)[0] if out.startswith('ERR') or out.startswith('LOADING'): raise PopenError(out) elif out.startswith('OK\n'): out = out[3:] if out.endswith('\n'): out = out[:-1] return out except PopenError, e: if 'LOADING' in str(e): LOG.debug('Unable to execute query %s: Redis is loading the dataset in memory' % query) elif not silent: LOG.error('Unable to execute query %s with redis-cli: %s' % (query, e)) raise
def start(cls): if not cls.is_running(): cls._logger.info("Starting %s process" % MONGOS) args = [ "sudo", "-u", DEFAULT_USER, MONGOS, "--fork", "--logpath", ROUTER_LOG_PATH, "--configdb", "mongo-0-0:%s" % CONFIG_SERVER_DEFAULT_PORT, ] if cls.keyfile and os.path.exists(cls.keyfile): chown_r(cls.keyfile, DEFAULT_USER) args.append("--keyFile=%s" % cls.keyfile) if cls.verbose and isinstance(cls.verbose, int) and 0 < cls.verbose < 6: args.append("-" + "v" * cls.verbose) if os.path.exists(ROUTER_LOG_PATH): chown_r(ROUTER_LOG_PATH, DEFAULT_USER) system2(args, close_fds=True, preexec_fn=mongo_preexec_fn) wait_until(lambda: cls.is_running, timeout=MAX_START_TIMEOUT) wait_until(lambda: cls.get_cli().has_connection, timeout=MAX_START_TIMEOUT) cls._logger.debug("%s process has been started." % MONGOS)
def start(self): try: if not self.running: #TODO: think about moving this code elsewhere if self.port == DEFAULT_PORT: base_dir = self.redis_conf.dir snap_src = os.path.join(base_dir, DB_FILENAME) snap_dst = os.path.join(base_dir, get_snap_db_filename(DEFAULT_PORT)) if os.path.exists(snap_src) and not os.path.exists(snap_dst): shutil.move(snap_src, snap_dst) self.redis_conf.dbfilename = snap_dst aof_src = os.path.join(base_dir, AOF_FILENAME) aof_dst = os.path.join(base_dir, get_aof_db_filename(DEFAULT_PORT)) if os.path.exists(aof_src) and not os.path.exists(aof_dst): shutil.move(aof_src, aof_dst) self.redis_conf.appendfilename = aof_dst LOG.debug('Starting %s on port %s' % (BIN_PATH, self.port)) system2('%s %s -s %s -c "%s %s"'%(SU_EXEC, DEFAULT_USER, BASH, BIN_PATH, self.config_path), shell=True, close_fds=True, preexec_fn=os.setsid) wait_until(lambda: self.running, timeout=MAX_START_TIMEOUT) wait_until(lambda: self.cli.test_connection(), timeout=MAX_START_TIMEOUT) LOG.debug('%s process has been started.' % SERVICE_NAME) except PopenError, e: LOG.error('Unable to start redis process: %s' % e) raise initdv2.InitdError(e)
def __init__(self): if not os.path.exists(MDADM_EXEC): if disttool.is_redhat_based(): system2(('/usr/bin/yum', '-d0', '-y', 'install', 'mdadm', '-x', 'exim'), raise_exc=False) else: mgr = dynimp.package_mgr() mgr.install('mdadm', mgr.candidates('mdadm')[-1]) for location in ['/etc ', '/lib']: path = os.path.join(location, 'udev/rules.d/85-mdadm.rules') if os.path.exists(path): rule = None with open(path, 'r') as fp: rule = fp.read() if rule: rule = re.sub(re.compile('^([^#])', re.M), '#\\1', rule) with open(path, 'w') as fp: fp.write(rule) self._raid_devices_re = re.compile('Raid\s+Devices\s+:\s+(?P<count>\d+)') self._total_devices_re = re.compile('Total\s+Devices\s+:\s+(?P<count>\d+)') self._state_re = re.compile('State\s+:\s+(?P<state>.+)') self._rebuild_re = re.compile('Rebuild\s+Status\s+:\s+(?P<percent>\d+)%') self._level_re = re.compile('Raid Level : (?P<level>.+)')
def on_init(self, *args, **kwargs): bus.on("before_hello", self.on_before_hello) bus.on("before_host_init", self.on_before_host_init) bus.on("before_restart", self.on_before_restart) bus.on("before_reboot_finish", self.on_before_reboot_finish) try: system(('ntpdate', '-u', '0.amazon.pool.ntp.org')) except: pass msg_service = bus.messaging_service producer = msg_service.get_producer() producer.on("before_send", self.on_before_message_send) if not os_dist.windows_family and not __node__.get('hostname'): # Set the hostname to this instance's public hostname try: hostname_as_pubdns = int(__ec2__['hostname_as_pubdns']) except: hostname_as_pubdns = True if hostname_as_pubdns: pub_hostname = self._platform.get_public_hostname() self._logger.debug('Setting hostname to %s' % pub_hostname) system2("hostname " + pub_hostname, shell=True) if disttool.is_ubuntu(): # Ubuntu cloud-init scripts may disable root ssh login for path in ('/etc/ec2-init/ec2-config.cfg', '/etc/cloud/cloud.cfg'): if os.path.exists(path): c = None with open(path, 'r') as fp: c = fp.read() c = re.sub(re.compile(r'^disable_root[^:=]*([:=]).*', re.M), r'disable_root\1 0', c) with open(path, 'w') as fp: fp.write(c) if not linux.os.windows_family: # Add server ssh public key to authorized_keys ssh_key = self._platform.get_ssh_pub_key() if ssh_key: add_authorized_key(ssh_key) # Mount ephemeral devices # Seen on eucalyptus: # - fstab contains invalid fstype and `mount -a` fails if self._platform.name == 'eucalyptus': mtab = mount.mounts() fstab = mount.fstab() for device in self._platform.instance_store_devices: if os.path.exists(device) and device in fstab and device not in mtab: entry = fstab[device] try: mount.mount(device, entry.mpoint, '-o', entry.options) except: self._logger.warn(sys.exc_info()[1]) else: if not os_dist.windows_family: system2('mount -a', shell=True, raise_exc=False)
def _change_selinux_ctx(self): chcon = software.whereis('chcon') if disttool.is_rhel() and chcon: LOG.debug('Changing SELinux file security context for new mysql datadir') system2((chcon[0], '-R', '-u', 'system_u', '-r', 'object_r', '-t', 'mysqld_db_t', os.path.dirname(__mysql__['storage_dir'])), raise_exc=False)
def execute(self, query, silent=False): if not self.password: full_query = query else: full_query = 'AUTH %s\n%s' % (self.password, query) execute_query = lambda: system2([self.path, '-p', self.port], stdin=full_query,silent=True, warn_stderr=False) try: out = execute_query()[0] #fix for redis 2.4 AUTH if 'Client sent AUTH, but no password is set' in out: execute_query = lambda: system2([self.path], stdin=query, silent=True) out = execute_query()[0] if "Redis is loading the dataset in memory" in out: #[SCALARIZR-1604] #test until service becomes available: wait_until(lambda: "LOADING" not in system2([self.path], stdin='ping', silent=True)[0]) #run query again: out = execute_query()[0] elif out.startswith('ERR'): raise PopenError(out) elif out.startswith('OK\n'): out = out[3:] if out.endswith('\n'): out = out[:-1] return out except PopenError, e: if not silent: LOG.error('Unable to execute query %s with redis-cli: %s' % (query, e)) raise
def move_to(self, dst, move_files=True): new_cluster_dir = os.path.join(dst, STORAGE_DATA_DIR) if not os.path.exists(dst): LOG.debug('Creating directory structure for postgresql cluster: %s' % dst) os.makedirs(dst) if move_files: source = self.path if not os.path.exists(self.path): source = self.default_path LOG.debug('data_directory in postgresql.conf points to non-existing location, using %s instead' % source) if source != new_cluster_dir: LOG.debug("copying cluster files from %s into %s" % (source, new_cluster_dir)) shutil.copytree(source, new_cluster_dir) LOG.debug("changing directory owner to %s" % self.user) chown_r(dst, self.user) LOG.debug("Changing postgres user`s home directory") if disttool.is_redhat_based(): #looks like ubuntu doesn`t need this system2([USERMOD, '-d', new_cluster_dir, self.user]) self.path = new_cluster_dir return new_cluster_dir
def rebundle(self): image_name = self._role_name + "-" + time.strftime("%Y%m%d%H%M%S") nova = __node__['openstack']['new_nova_connection'] nova.connect() server_id = __node__['openstack']['server_id'] system2("sync", shell=True) LOG.info('Creating server image (server_id: %s)', server_id) image_id = nova.servers.create_image(server_id, image_name) LOG.info('Server image %s created', image_id) result = [None] def image_completed(): try: result[0] = nova.images.get(image_id) return result[0].status in ('ACTIVE', 'FAILED') except: e = sys.exc_info()[1] if 'Unhandled exception occurred during processing' in str(e): return raise wait_until(image_completed, start_text='Polling image status', sleep=30) image_id = result[0].id if result[0].status == 'FAILED': raise handlers.HandlerError('Image %s becomes FAILED', image_id) LOG.info('Image %s completed and available for use!', image_id) return image_id
def _cleanup_after_rebundle(): cnf = bus.cnf pl = bus.platform logger = logging.getLogger(__name__) if 'volumes' not in pl.features: # Destory mysql storages if os.path.exists(cnf.private_path('storage/mysql.json')) and pl.name == 'rackspace': logger.info('Cleanuping old MySQL storage') mysql_bhv = firstmatched(lambda x: x in node.__node__['behavior'], ('mysql', 'mysql2', 'percona')) vol = node.__node__[mysql_bhv]['volume'] vol.destroy(force=True) if os.path.exists('/etc/chef/client.pem'): os.remove('/etc/chef/client.pem') if os.path.exists('/etc/chef/client.rb'): os.remove('/etc/chef/client.rb') # Reset private configuration priv_path = cnf.private_path() for file in os.listdir(priv_path): if file in ('.user-data', '.update'): continue path = os.path.join(priv_path, file) coreutils.chmod_r(path, 0700) os.remove(path) if (os.path.isfile(path) or os.path.islink(path)) else shutil.rmtree(path) if not linux.os.windows_family: system2('sync', shell=True)
def _run_chef_client(self): system2(self.get_cmd(), close_fds=not linux.os.windows_family, log_level=logging.INFO, preexec_fn=not linux.os.windows_family and os.setsid or None, env=self.environment_variables )
def start(self): try: if not self.running: #TODO: think about moving this code elsewhere if self.port == __redis__['defaults']['port']: base_dir = self.redis_conf.dir snap_src = os.path.join(base_dir, __redis__['db_filename']) snap_dst = os.path.join(base_dir, get_snap_db_filename(__redis__['defaults']['port'])) if os.path.exists(snap_src) and not os.path.exists(snap_dst): shutil.move(snap_src, snap_dst) if 'snapshotting' == __redis__["persistence_type"]: self.redis_conf.dbfilename = snap_dst aof_src = os.path.join(base_dir, __redis__['aof_filename']) aof_dst = os.path.join(base_dir, get_aof_db_filename(__redis__['defaults']['port'])) if os.path.exists(aof_src) and not os.path.exists(aof_dst): shutil.move(aof_src, aof_dst) if 'aof' == __redis__["persistence_type"]: self.redis_conf.appendfilename = aof_dst LOG.debug('Starting %s on port %s' % (__redis__['redis-server'], self.port)) system2('%s %s -s %s -c "%s %s"' % ( __redis__['su'], __redis__['defaults']['user'], __redis__['bash'], __redis__['redis-server'], self.config_path), shell=True, close_fds=True, preexec_fn=os.setsid) wait_until(lambda: self.running) wait_until(lambda: self.cli.test_connection()) LOG.debug('%s process has been started.' % SERVICE_NAME) except PopenError, e: LOG.error('Unable to start redis process: %s' % e) raise initdv2.InitdError(e)
def create(self, dbname, filename, opts=None, mysql_upgrade=True): _opts = opts and list(opts) or [] LOG.debug("Dumping database %s to %s" % (dbname, filename)) _opts = [MYSQLDUMP_PATH, "-u", self.root_user, "--password="******"--databases"] with open(filename, "w") as fp: system2(_opts + [dbname], stdout=fp) # commented cause mysql_upgrade hanged forever on devel roles """
def make_local_files(self, suffix=None): # Create some 1Mb files files = [] #for file in ('chocolate', 'caramel', 'fluids'): for file in ('chocolate', 'caramel'): file = '%s/%s%s' % (self.dst, file, suffix or '') system2('dd if=/dev/urandom of=%s bs=100K count=1' % file, shell=True) files.append(file) return files
def make(self): self.devname = self._create_image() self._format_image() system2("sync", shell=True) # Flush so newly formatted filesystem is ready to mount. self._mount_image() self._make_special_dirs() self._copy_rec(self._volume, self.mpoint) system2("sync", shell=True) # Flush buffers return self.mpoint
def _change_selinux_ctx(self): try: chcon = software.which('chcon') except LookupError: return if disttool.is_redhat_based(): LOG.debug('Changing SELinux file security context for new mysql datadir') system2((chcon, '-R', '-u', 'system_u', '-r', 'object_r', '-t', 'mysqld_db_t', os.path.dirname(__mysql__['storage_dir'])), raise_exc=False)
def test_fqdn(self): (out, err, rc) = system2(('hostname')) self.assertEqual(out.strip(), self.info.fqdn()) old_name = out.strip() self.info.fqdn('Scalr-Role-12345') (out, err, rc) = system2(('hostname')) self.assertEqual(out.strip(), 'Scalr-Role-12345') self.info.fqdn(old_name)
def on_Rebundle(self, message): try: self._role_name = message.role_name.encode("ascii") self._excludes = message.excludes.encode("ascii").split(":") \ if message.body.has_key("excludes") and message.excludes else [] self._rebundle_message = message # Preparing... self.before_rebundle() bus.fire("before_rebundle", role_name=self._role_name) # Send wall message before rebundling. So console users can run away if not system2(('which', 'wall'), raise_exc=False)[2]: system2(('wall'), stdin=WALL_MESSAGE, raise_exc=False) # Do actual rebundle work cnf = bus.cnf saved_state = cnf.state try: cnf.state = ScalarizrState.REBUNDLING image_id = self.rebundle() finally: cnf.state = saved_state # Creating message result = dict( status = "ok", snapshot_id = image_id, bundle_task_id = message.bundle_task_id ) # Updating message with OS, software and modules info result.update(software.system_info()) # Fire 'rebundle'diss bus.fire("rebundle", role_name=self._role_name, snapshot_id=image_id, rebundle_result=result) # Notify Scalr self.send_message(Messages.REBUNDLE_RESULT, result) LOG.info('Rebundle complete! If you imported this server to Scalr, ' 'you can terminate Scalarizr now.') except (Exception, BaseException), e: LOG.exception(e) last_error = hasattr(e, "error_message") and e.error_message or str(e) # Send message to Scalr self.send_message(Messages.REBUNDLE_RESULT, dict( status = "error", last_error = last_error, bundle_task_id = message.bundle_task_id )) # Fire 'rebundle_error' bus.fire("rebundle_error", role_name=self._role_name, last_error=last_error)
def change_system_password(self, new_pass): LOG.debug('Changing password of system user %s to %s' % (self.name, new_pass)) out, err, retcode = system2( [OPENSSL, 'passwd', '-1', '"%s"' % new_pass]) shadow_password = out.strip() if retcode != 0: LOG.error('Error creating hash for ' + self.name) if err: LOG.error(err) r = system2([USERMOD, '-p', '"%s"' % shadow_password, self.name])[2] if r != 0: LOG.error('Error changing password for ' + self.name) #change password in privated/pgsql.ini self.password = new_pass return new_pass
def extract_public_ssh_key(self): if not os.path.exists(self.private_key_path): raise Exception('Private key file %s does not exist.' % self.private_key_path) args = shlex.split('%s -y -f' % SSH_KEYGEN) args.append(self.private_key_path) out, err, retcode = system2(args) if err: LOG.error('Failed to extract public key from %s : %s' % (self.private_key_path, err)) if retcode != 0: raise Exception("Error handling would be nice, eh?") return out.strip()
def _create_pg_role(self, password=None, super=True): if self._is_role_exist: LOG.debug('Cannot create role: role %s already exists' % self.name) else: LOG.debug('Creating role %s' % self.name) try: out = system2([SU_EXEC, '-', self.group, '-c', '%s -s %s' % (CREATEUSER, self.name)])[0] LOG.debug(out or 'Role %s has been successfully created.' % self.name) except PopenError, e: LOG.error('Unable to create role %s: %s' % (self.name, e)) raise
def _create_pg_database(self): if self._is_pg_database_exist: LOG.debug('Cannot create db: database %s already exists' % self.name) else: LOG.debug('Creating db %s' % self.name) try: out = system2([SU_EXEC, '-', self.group, '-c', '%s %s' % (CREATEDB,self.name)])[0] LOG.debug(out or 'DB %s has been successfully created.' % self.name) except PopenError, e: LOG.error('Unable to create db %s: %s' % (self.name, e)) raise
def install(self): """" Download and install MMS agent """ if not os.path.exists('%s/mms-agent' % _MMSAgent.install_dir): _MMSAgent._download() out, err, returncode = util.system2([ 'tar', '-xf', '/tmp/10gen-mms-agent.tar.gz', '-C', _MMSAgent.install_dir ])
def rebundle(self): image_name = self._role_name + "-" + time.strftime("%Y%m%d%H%M%S") nova = __node__['openstack'].connect_nova() server_id = __node__['openstack']['server_id'] system2("sync", shell=True) LOG.info('Creating server image (server_id: %s)', server_id) image_id = nova.servers.create_image(server_id, image_name) LOG.info('Server image %s created', image_id) result = [None] def image_completed(): try: result[0] = nova.images.get(image_id) return result[0].status in ('ACTIVE', 'FAILED', 'DELETED') except: e = sys.exc_info()[1] if 'Unhandled exception occurred during processing' in str(e): return raise def delete_image(): if not result[0]: return image_id = result[0].id try: LOG.info('Cleanup: deleting image %s', image_id) nova.images.delete(image_id) except: LOG.warn('Cleanup failed: %s', exc_info=sys.exc_info()) try: wait_until(image_completed, start_text='Polling image status', sleep=30) image = result[0] if image.status != 'ACTIVE': raise handlers.HandlerError('Image %s becomes %s', image.id, image.status) LOG.info('Image %s completed and available for use!', image.id) return image.id except: exc = sys.exc_info() delete_image() raise exc[0], exc[1], exc[2]
def start(self): try: if not self.running: #TODO: think about moving this code elsewhere if self.port == __redis__['defaults']['port']: base_dir = self.redis_conf.dir snap_src = os.path.join(base_dir, __redis__['db_filename']) snap_dst = os.path.join( base_dir, get_snap_db_filename(__redis__['defaults']['port'])) if os.path.exists( snap_src) and not os.path.exists(snap_dst): shutil.move(snap_src, snap_dst) if 'snapshotting' == __redis__["persistence_type"]: self.redis_conf.dbfilename = snap_dst aof_src = os.path.join(base_dir, __redis__['aof_filename']) aof_dst = os.path.join( base_dir, get_aof_db_filename(__redis__['defaults']['port'])) if os.path.exists(aof_src) and not os.path.exists(aof_dst): shutil.move(aof_src, aof_dst) if 'aof' == __redis__["persistence_type"]: self.redis_conf.appendfilename = aof_dst LOG.debug('Starting %s on port %s' % (__redis__['redis-server'], self.port)) system2('%s %s -s %s -c "%s %s"' % (__redis__['su'], __redis__['defaults']['user'], __redis__['bash'], __redis__['redis-server'], self.config_path), shell=True, close_fds=True, preexec_fn=os.setsid) wait_until(lambda: self.running) wait_until(lambda: self.cli.test_connection()) LOG.debug('%s process has been started.' % SERVICE_NAME) except PopenError, e: LOG.error('Unable to start redis process: %s' % e) raise initdv2.InitdError(e)
def system(*args, **kwds): args = list(args) kwds['exc_class'] = LinuxError kwds['close_fds'] = system.close_fds if not kwds.get('shell') and not osmod.access(args[0][0], osmod.X_OK): executable = which(args[0][0]) if not executable: msg = "Executable '%s' not found" % args[0][0] raise LinuxError(msg) args[0] = list(args[0]) args[0][0] = executable return util.system2(*args, **kwds)
def snapshot(self, op, name): rebundle_dir = tempfile.mkdtemp() archive_path = '' try: pl = __node__['platform'] proj_id = pl.get_numeric_project_id() proj_name = pl.get_project_id() cloudstorage = pl.new_storage_client() root_part_path = os.path.realpath('/dev/root') root_part_sysblock_path = glob.glob('/sys/block/*/%s' % os.path.basename(root_part_path))[0] root_device = '/dev/%s' % os.path.basename(os.path.dirname(root_part_sysblock_path)) archive_name = '%s.tar.gz' % name.lower() archive_path = os.path.join(rebundle_dir, archive_name) self._prepare_software() gcimagebundle_bin = software.which('gcimagebundle') out, err, code = util.system2((gcimagebundle_bin, '-d', root_device, '-e', ','.join(self.exclude_dirs), '-o', rebundle_dir, '--output_file_name', archive_name), raise_exc=False) if code: raise ImageAPIError('Gcimagebundle util returned non-zero code %s. Stderr: %s' % (code, err)) LOG.info('Uploading compressed image to cloud storage') tmp_bucket_name = 'scalr-images-%s-%s' % (random.randint(1, 1000000), int(time.time())) remote_path = 'gcs://%s/%s' % (tmp_bucket_name, archive_name) arch_size = os.stat(archive_path).st_size uploader = FileTransfer(src=archive_path, dst=remote_path) try: upload_result = uploader.run() if upload_result['failed']: errors = [str(failed['exc_info'][1]) for failed in upload_result['failed']] raise ImageAPIError('Image upload failed. Errors:\n%s' % '\n'.join(errors)) assert arch_size == upload_result['completed'][0]['size'] except: self._remove_bucket(tmp_bucket_name, archive_name, cloudstorage) raise finally: shutil.rmtree(rebundle_dir) if os.path.exists(archive_path): os.remove(archive_path) image_name = name.lower().replace('_', '-') + '-' + str(int(time.time())) self._register_image(image_name, tmp_bucket_name, archive_name, cloudstorage) return '%s/images/%s' % (proj_name, image_name)
def _get_netiface_ip(self, iface=None): if not iface: raise PlatformError( 'You must specify interface name to retrieve ip address') if not hasattr(self, '_ip_re'): self._ip_re = re.compile('inet\s*addr:(?P<ip>[\d\.]+)', re.M) out = system2('/sbin/ifconfig ' + iface, shell=True)[0] result = re.search(self._ip_re, out) if not result: return None return result.group('ip')
def save(): ''' on RHEL call 'service iptables save' on Ubuntu: - touch or create /etc/network/if-pre-up.d/iptables.sh $ cat /etc/network/if-pre-up.d/iptables.sh #!/bin/bash iptables-restore < /etc/iptables.rules - iptables-save > /etc/iptables.rules ''' if linux.os["family"] in ("RedHat", "Oracle"): if linux.os["name"] == "CentOS" and linux.os["release"] > (7, 0): system2(("/usr/libexec/iptables/iptables.init", "save")) else: linux.system(linux.build_cmd_args(executable="service", short=['iptables', 'save'])) elif linux.os["family"] == "Debian": with open('/etc/network/if-pre-up.d/iptables.sh', 'w') as fp: fp.write('#!/bin/bash\n' 'iptables-restore < /etc/iptables.rules') iptables_save('/etc/iptables.rules')
def on_init(self): bus.on("host_init_response", self.on_host_init_response) self._producer.on("before_send", self.on_before_message_send) # Add internal messages to scripting skip list try: map(scalarizr.handlers.script_executor.skip_events.add, (Messages.INT_SERVER_REBOOT, Messages.INT_SERVER_HALT, Messages.HOST_INIT_RESPONSE)) except AttributeError: pass # Mount all filesystems if os_dist['family'] != 'Windows': system2(('mount', '-a'), raise_exc=False) # Add firewall rules #if self._cnf.state in (ScalarizrState.BOOTSTRAPPING, ScalarizrState.IMPORTING): self._insert_iptables_rules() if __node__['state'] != ScalarizrState.IMPORTING: self._fetch_globals()
def _start_stop_reload(self, action): try: args = [self.initd_script] \ if isinstance(self.initd_script, basestring) \ else list(self.initd_script) args.append(action) out, err, returncode = system2(args, close_fds=True, preexec_fn=os.setsid) except PopenError, e: raise InitdError("Popen failed with error %s" % (e, ))
def is_running(name): if not _services.has_key(name): raise InitdError("Unknown service '%s'" % (name, )) cmd = [_services[name]["initd_script"], "status"] out, err = system2(cmd)[0:2] out += err if name == "mysql" and disttool.is_ubuntu() and disttool.linux_dist( )[1] == '8.04': return out.lower().find("Uptime:") != -1 else: return out.lower().find("running") != -1 or out.lower().find( "[ ok ]") != -1 or out.lower().find("done.") != -1
def get_redis_processes(): config_files = list() try: out = system2(('ps', '-G', 'redis', '-o', 'command', '--no-headers'))[0] except: out = '' if out: for line in out.split('\n'): words = line.split() if len(words) == 2 and words[0] == BIN_PATH: config_files.append(words[1]) return config_files
def start(self): if self.status() == 0: raise initdv2.InitdError( "Cannot start HAProxy. It already running.") util.system2([ self.haproxy_exec, '-f', self.config_path, '-p', self.pid_file, '-D' ], ) if self.pid_file: try: util.wait_until( lambda: os.path.exists(self.pid_file), timeout=self.timeout, sleep=0.2, error_text="HAProxy pid file %s does'not exist" % self.pid_file) except: err = "Cannot start HAProxy: pid file %s hasn't been created. " \ "Details: %s" % (self.pid_file, sys.exc_info()[1]) raise initdv2.InitdError, err, sys.exc_info()[2]
def _manifest(self): f_manifest = CnfController._manifest base_manifest = f_manifest.fget(self) path = self._manifest_path s = {} out = None if not self._merged_manifest: cmd = '%s --no-defaults --verbose SU_EXEC' % mysql_svc.MYSQLD_PATH out = system2('%s - mysql -s %s -c "%s"' % (SU_EXEC, BASH, cmd), shell=True, raise_exc=False, silent=True)[0] if out: raw = out.split(49*'-'+' '+24*'-') if raw: a = raw[-1].split('\n') if len(a) > 5: b = a[1:-5] for item in b: c = item.split() if len(c) > 1: key = c[0] val = ' '.join(c[1:]) s[key.strip()] = val.strip() if s: m_config = Configuration('ini') if os.path.exists(path): m_config.read(path) for variable in base_manifest: name = variable.name dv_path = './%s/default-value' % name try: old_value = m_config.get(dv_path) if name in s: new_value = s[name] else: name = name.replace('_','-') if name in s: new_value = self.definitions[s[name]] if s[name] in self.definitions else s[name] if old_value != new_value and new_value != '(No default value)': LOG.debug('Replacing %s default value %s with precompiled value %s', name, old_value, new_value) m_config.set(path=dv_path, value=new_value, force=True) except NoPathError: pass m_config.write(path) self._merged_manifest = _CnfManifest(path) return self._merged_manifest
def apache_software_info(): binary_name = "httpd" if linux.os.redhat_family else "apache2ctl" binary = which(binary_name) if not binary: raise SoftwareError("Can't find executable for apache http server") try: out = system2((binary, '-V'))[0] except PopenError, e: pkg_mgr = pkgmgr.package_mgr() version_string = pkg_mgr.info('apache2')['installed']
def mongodb_software_info(): try: mongod = which('mongod') except: raise SoftwareError("Can't find mongodb server executable") else: out = system2((mongod, '--version'))[0] version_string = out.splitlines()[0] m = re.search('[\d\.]+', version_string) if m: return SoftwareInfo('mongodb', m.group(0), out) raise SoftwareError("Can't parse `mongod --version` output")
def _start_stop_reload(self, action): ''' XXX: Temporary ugly hack (Ubuntu 1004 upstart problem - Job is already running)''' try: args = [self.initd_script] \ if isinstance(self.initd_script, basestring) \ else list(self.initd_script) args.append(action) out, err, returncode = system2(args, close_fds=True, preexec_fn=os.setsid) except PopenError, e: if 'Job is already running' not in str(e): raise InitdError("Popen failed with error %s" % (e, ))
def move_mysqldir_to(self, storage_path): LOG.info('Moving mysql dir to %s' % storage_path) for directive, dirname in ( ('mysqld/log_bin', os.path.join(storage_path, STORAGE_BINLOG)), ('mysqld/datadir', os.path.join(storage_path, STORAGE_DATA_DIR) + '/')): dest = os.path.dirname(dirname) if os.path.isdir(dest): LOG.info('No need to move %s to %s: already in place.' % (directive, dest)) else: os.makedirs(dest) raw_value = self.my_cnf.get(directive) LOG.debug('directive %s:%s' % (directive, raw_value)) if raw_value and node.__node__['platform'] != 'openstack': src_dir = os.path.dirname(raw_value + "/") + "/" LOG.debug('source path: %s' % src_dir) if os.path.isdir(src_dir) and src_dir != dest: selinuxenabled = software.which('selinuxenabled') if selinuxenabled: if not system2( (selinuxenabled, ), raise_exc=False)[2]: if not system2((software.which('getsebool'), 'mysqld_disable_trans'), raise_exc=False)[2]: LOG.debug('Make SELinux rule for rsync') system2((software.which('setsebool'), '-P', 'mysqld_disable_trans', '1')) else: semanage = get_semanage() system2((semanage, 'fcontext', '-a', '-t', 'bin_t', '/usr/bin/rsync')) system2((software.which('restorecon'), '-v', '/usr/bin/rsync')) LOG.info('Copying mysql directory \'%s\' to \'%s\'', src_dir, dest) rsync(src_dir, dest, archive=True, exclude=['ib_logfile*', '*.sock']) self.my_cnf.set(directive, dirname) chown_r(dest, "mysql", "mysql") # Adding rules to apparmor config if disttool.is_debian_based(): _add_apparmor_rules(dest)
def _create_linux_user(self, password): if self._is_linux_user_exist: LOG.debug('Cannot create system user: user %s already exists' % self.name) #TODO: check password else: try: out = system2([USERADD, '-m', '-g', self.group, '-p', password, self.name])[0] if out: LOG.debug(out) LOG.debug('Creating system user %s' % self.name) except PopenError, e: LOG.error('Unable to create system user %s: %s' % (self.name, e)) raise
def cluster_with(self, self_hostname, hostnames, disk_node=True, do_reset=True): if self.version >= (3, 0, 0): # New way of clustering was introduced in rabbit 3.0.0 one_node = NODE_HOSTNAME_TPL % hostnames[0] cmd = [RABBITMQCTL, 'join_cluster', one_node] if not disk_node: cmd.append('--ram') else: nodes = [NODE_HOSTNAME_TPL % host for host in hostnames] if disk_node: nodes.append(NODE_HOSTNAME_TPL % self_hostname) cmd = [RABBITMQCTL, 'cluster'] + nodes clustered = False while not clustered: self.stop_app() if do_reset: self.reset() system2(cmd, logger=self._logger) p = subprocess.Popen((RABBITMQCTL, 'start_app')) for i in range(15): if p.poll() is None: time.sleep(1) continue if p.returncode: raise Exception(p.stderr.read()) else: clustered = True break else: p.kill() self.service.restart(force=True)
def _set_overcommit_option(self): try: with open('/proc/sys/vm/overcommit_memory', 'r') as f: proc = f.read().strip() with open('/etc/sysctl.conf', 'r') as f: match = re.search(r'^\s*vm.overcommit_memory\s*=\s*(\d+)', f.read(), re.M) sysctl = match.group(1) if match else None if (proc == '2') or (proc == sysctl == '0'): LOG.info( 'Kernel option vm.overcommit_memory is set to %s by user. ' 'Consider changing it to 1 for optimal Redis functioning. ' 'More information here: http://redis.io/topics/admin', proc) else: LOG.debug('Setting vm.overcommit_memory to 1') system2((which('sysctl'), 'vm.overcommit_memory=1')) except: LOG.debug("Failed to set vm.overcommit_memory option", exc_info=sys.exc_info())
def make_snapshot(self, volume): prepared_image_path = os.path.join(self.destination, self.image_name) LOG.debug('sgp_dd image into volume %s' % volume.device) system2(('sgp_dd', 'if=' + prepared_image_path, 'of=' + volume.device, 'bs=8k', 'count=%s' % (self.image_size * 1024 * 1024 / 8))) # coreutils.dd(**{'if': prepared_image_path, 'of': volume.device, 'bs': '8M'}) volume.mount() self.clean_snapshot(volume) LOG.debug('detaching volume') volume.detach() LOG.debug('Making snapshot of volume %s' % volume.device) snapshot = volume.snapshot() util.wait_until(lambda: snapshot.status() == 'completed', logger=LOG, error_text='EBS snapshot %s wasnt completed' % snapshot.id) LOG.debug('Snapshot is made') volume.ensure(mount=True) return snapshot.id
def _is_aof_rewrite_running(self): """ Redis 2.8 seems to be unable to shutdown when it has child processes running """ try: out = system2(('ps', '-G', 'redis', '-o', 'command', '--no-headers'))[0] except: out = '' if out: for line in out.split('\n'): if "redis-aof-rewrite" in line and str(self.port) in line: return True return False
def execute(self, query, silent=False): try: out = system2([ SU_EXEC, '-', self.user, '-c', 'export LANG=en_US; %s -c "%s"' % (self.path, query) ], silent=True)[0] return out except PopenError, e: if not silent: LOG.error('Unable to execute query %s from user %s: %s' % (query, self.user, e)) raise
def postgresql_software_info(): versions_dirs = glob.glob('/usr/lib/p*sql/*') versions_dirs.extend(glob.glob('/usr/p*sql*/')) versions_dirs.sort() versions_dirs.reverse() for version in versions_dirs: bin_path = os.path.join(version, 'bin/postgres') if os.path.isfile(bin_path): version_string = system2((bin_path, '--version'))[0].strip() version = version_string.split()[-1] return SoftwareInfo('postgresql', version, version_string) else: raise SoftwareError
def __init__(self): if 'gce' == node.__node__['platform'].name: self.ensure_pid_directory() self.mysql_cli = MySQLClient() service_exec = '/usr/sbin/service' if linux.os.debian_family else '/sbin/service' service_name = 'mysql' if linux.os.redhat_family: if linux.os['release'] >= (7, 0) and \ system2('systemctl list-unit-files | grep mariadb', raise_exc=False, shell=True)[2] == 0: service_name = 'mariadb' elif 'percona' not in node.__node__['behavior']: service_name = 'mysqld' if linux.os.redhat_family or linux.os.ubuntu: initd_script = (service_exec, service_name) else: initd_script = '/etc/init.d/{0}'.format(service_name) pid_file = None try: out = system2("my_print_defaults mysqld", shell=True, silent=True) m = re.search("--pid[-_]file=(.*)", out[0], re.MULTILINE) if m: pid_file = m.group(1) m = re.search("--socket=(.*)", out[0], re.MULTILINE) if m: self.socket_file = m.group(1) except: pass initdv2.ParametrizedInitScript.__init__( self, SERVICE_NAME, initd_script, pid_file, socks=[initdv2.SockParam(MYSQL_DEFAULT_PORT, timeout=3600)])
def _cleanup_after_rebundle(): cnf = bus.cnf pl = bus.platform logger = logging.getLogger(__name__) if 'volumes' not in pl.features: # Destory mysql storages if os.path.exists(cnf.private_path( 'storage/mysql.json')) and pl.name == 'rackspace': logger.info('Cleanuping old MySQL storage') vol = Storage.create( Storage.restore_config(cnf.private_path('storage/mysql.json'))) vol.destroy(force=True) # Reset private configuration priv_path = cnf.private_path() for file in os.listdir(priv_path): if file in ('.user-data', '.update'): continue path = os.path.join(priv_path, file) os.remove(path) if (os.path.isfile(path) or os.path.islink(path)) else shutil.rmtree(path) system2('sync', shell=True)