Beispiel #1
0
    def check(self):
        if linux.os['family'] in ('RedHat', 'Oracle') and linux.os['version'] >= (6, 0):
            # Avoid "Can't locate Time/HiRes.pm in @INC"
            # with InnoDB Backup Utility v1.5.1-xtrabackup
            pkgmgr.installed('perl-Time-HiRes')         

        mgr = pkgmgr.package_mgr()
        if not 'percona' in mgr.repos():
            if linux.os['family'] in ('RedHat', 'Oracle'):
                url = 'http://www.percona.com/downloads/percona-release/percona-release-0.0-1.%s.rpm' % linux.os['arch']
                pkgmgr.YumPackageMgr().localinstall(url)
                if linux.os.amazon:
                    linux.system(("sed -i 's/\$releasever/latest/g' "
                                    "/etc/yum.repos.d/Percona.repo"), shell=True)
            else:
                try:
                    codename = linux.os['lsb_codename']
                except KeyError:
                    codename = linux.ubuntu_release_to_codename[linux.os['lsb_release']]
                pkgmgr.apt_source(
                        'percona.list', 
                        ['deb http://repo.percona.com/apt %s main' % codename],
                        gpg_keyserver='hkp://keys.gnupg.net',
                        gpg_keyid='CD2EFD2A')
            mgr.updatedb()
        if software.mysql_software_info().version < (5, 5):
            self.package = 'percona-xtrabackup-21'
        else:
            self.package = 'percona-xtrabackup'

        return super(PerconaExec, self).check()
Beispiel #2
0
 def wrapper(*args, **kwds):
     try:
         return fn(*args, **kwds)
     finally:
         linux.system('service udev restart',
                      shell=True,
                      raise_exc=False)
Beispiel #3
0
    def on_init(self, *args, **kwargs):
        bus.on("before_hello", self.on_before_hello)
        bus.on("before_host_init", self.on_before_host_init)
        bus.on("before_restart", self.on_before_restart)
        bus.on("before_reboot_finish", self.on_before_reboot_finish)

        try:
            system(('ntpdate', '-u', '0.amazon.pool.ntp.org'))
        except:
            pass

        msg_service = bus.messaging_service
        producer = msg_service.get_producer()
        producer.on("before_send", self.on_before_message_send)

        if not os_dist.windows_family and not __node__.get('hostname'):
            # Set the hostname to this instance's public hostname
            try:
                hostname_as_pubdns = int(__ec2__['hostname_as_pubdns'])
            except:
                hostname_as_pubdns = True

            if hostname_as_pubdns:
                pub_hostname = self._platform.get_public_hostname()
                self._logger.debug('Setting hostname to %s' % pub_hostname)
                system2("hostname " + pub_hostname, shell=True)

        if disttool.is_ubuntu():
            # Ubuntu cloud-init scripts may disable root ssh login
            for path in ('/etc/ec2-init/ec2-config.cfg', '/etc/cloud/cloud.cfg'):
                if os.path.exists(path):
                    c = None
                    with open(path, 'r') as fp:
                        c = fp.read()
                    c = re.sub(re.compile(r'^disable_root[^:=]*([:=]).*', re.M), r'disable_root\1 0', c)
                    with open(path, 'w') as fp:
                        fp.write(c)

        if not linux.os.windows_family:
            # Add server ssh public key to authorized_keys
            ssh_key = self._platform.get_ssh_pub_key()
            if ssh_key:
                add_authorized_key(ssh_key)

        # Mount ephemeral devices
        # Seen on eucalyptus:
        #       - fstab contains invalid fstype and `mount -a` fails
        if self._platform.name == 'eucalyptus':
            mtab = mount.mounts()
            fstab = mount.fstab()
            for device in self._platform.instance_store_devices:
                if os.path.exists(device) and device in fstab and device not in mtab:
                    entry = fstab[device]
                    try:
                        mount.mount(device, entry.mpoint, '-o', entry.options)
                    except:
                        self._logger.warn(sys.exc_info()[1])
        else:
            if not os_dist.windows_family:
                system2('mount -a', shell=True, raise_exc=False)
Beispiel #4
0
def apt_source(name, sources, gpg_keyserver=None, gpg_keyid=None):
    '''
    @param sources: list of apt sources.list entries.
    Example:
            ['deb http://repo.percona.com/apt ${codename} main',
            'deb-src http://repo.percona.com/apt ${codename} main']
            All ${var} templates should be replaced with
            scalarizr.linux.os['var'] substitution
    if gpg_keyserver:
            apt-key adv --keyserver ${gpg_keyserver} --recv ${gpg_keyid}
    Creates file /etc/apt/sources.list.d/${name}
    '''
    if linux.os['family'] in ('RedHat', 'Oracle'):
        return

    def _vars(s):
        vars_ = re.findall('\$\{.+?\}', s)
        return map((lambda (name): name[2:-1]), vars_)  #2 is len of '${'

    def _substitude(s):
        for var in _vars(s):
            s = s.replace('${' + var + '}', linux.os[var])
        return s

    prepared_sources = map(_substitude, sources)
    with open('/etc/apt/sources.list.d/' + name, 'w+') as fp:
        fp.write('\n'.join(prepared_sources))

    if gpg_keyserver and gpg_keyid:
        if gpg_keyid not in linux.system(('apt-key', 'list'))[0]:
            linux.system(('apt-key', 'adv', '--keyserver', gpg_keyserver,
                          '--recv', gpg_keyid),
                         raise_exc=False)
Beispiel #5
0
def apt_source(name, sources, gpg_keyserver=None, gpg_keyid=None):
    '''
    @param sources: list of apt sources.list entries.
    Example:
            ['deb http://repo.percona.com/apt ${codename} main',
            'deb-src http://repo.percona.com/apt ${codename} main']
            All ${var} templates should be replaced with
            scalarizr.linux.os['var'] substitution
    if gpg_keyserver:
            apt-key adv --keyserver ${gpg_keyserver} --recv ${gpg_keyid}
    Creates file /etc/apt/sources.list.d/${name}
    '''
    if linux.os['family'] in ('RedHat', 'Oracle'):
        return

    def _vars(s):
        vars_ = re.findall('\$\{.+?\}', s)
        return map((lambda(name): name[2:-1]), vars_) #2 is len of '${'

    def _substitude(s):
        for var in _vars(s):
            s = s.replace('${'+var+'}', linux.os[var])
        return s

    prepared_sources = map(_substitude, sources)
    with open('/etc/apt/sources.list.d/' + name, 'w+') as fp:
        fp.write('\n'.join(prepared_sources))

    if gpg_keyserver and gpg_keyid:
        if gpg_keyid not in linux.system(('apt-key', 'list'))[0]:
            linux.system(('apt-key', 'adv',
                                      '--keyserver', gpg_keyserver,
                                      '--recv', gpg_keyid),
                                     raise_exc=False)
Beispiel #6
0
    def check(self):
        if linux.os['family'] in ('RedHat',
                                  'Oracle') and linux.os['version'] >= (6, 0):
            # Avoid "Can't locate Time/HiRes.pm in @INC"
            # with InnoDB Backup Utility v1.5.1-xtrabackup
            pkgmgr.installed('perl-Time-HiRes')

        mgr = pkgmgr.package_mgr()
        if not 'percona' in mgr.repos():
            if linux.os['family'] in ('RedHat', 'Oracle'):
                url = 'http://www.percona.com/downloads/percona-release/percona-release-0.0-1.%s.rpm' % linux.os[
                    'arch']
                pkgmgr.YumPackageMgr().localinstall(url)
                if linux.os.amazon:
                    linux.system(("sed -i 's/\$releasever/latest/g' "
                                  "/etc/yum.repos.d/Percona.repo"),
                                 shell=True)
            else:
                try:
                    codename = linux.os['lsb_codename']
                except KeyError:
                    codename = linux.ubuntu_release_to_codename[
                        linux.os['lsb_release']]
                pkgmgr.apt_source(
                    'percona.list',
                    ['deb http://repo.percona.com/apt %s main' % codename],
                    gpg_keyserver='hkp://keys.gnupg.net',
                    gpg_keyid='CD2EFD2A')
            mgr.updatedb()
        if software.mysql_software_info().version < (5, 5):
            self.package = 'percona-xtrabackup-21'
        else:
            self.package = 'percona-xtrabackup'

        return super(PerconaExec, self).check()
Beispiel #7
0
    def create_snapshot(self, volume_id):
        assert volume_id in self.volumes, 'Volume "%s" not found' % volume_id
        volume = self.volumes[volume_id]

        snapshot_id = str(uuid.uuid4())[:7]
        if not os.path.isdir(snapshot_dir):
            os.makedirs(snapshot_dir)
        snap_path = self._get_snapshot_path(snapshot_id)

        lvm2.lvcreate(os.path.join('/dev', vg_name, volume['id']),
                      snapshot=True,
                      name=snapshot_id,
                      size=snap_size)
        lv_info = None
        try:
            lv_info = lvm2.lvs(lvm2.lvpath(vg_name, snapshot_id)).values()[0]
            system('dd if=%s | cp --sparse=always /dev/stdin %s' %
                   (lv_info.lv_path, snap_path),
                   shell=True)
        finally:
            if lv_info:
                lvm2.lvremove(lv_info.lv_path)
            else:
                lvm2.lvremove(os.path.join('/dev', vg_name, snapshot_id))

        snapshot = dict(id=snapshot_id, size=volume['size'])
        self.snapshots[snapshot_id] = snapshot
        return snapshot
Beispiel #8
0
def upload(step):
        key_path = os.environ.get('GCS_KEY')
        with open(key_path) as f:
            key = base64.b64encode(f.read())
        access_data = dict(service_account_name='*****@*****.**', key=key)
 
        gcs.bus = mock.MagicMock()
        gcs.bus.platform.get_access_data = lambda k: access_data[k]
 
        gsm = GoogleServiceManager(gcs.bus.platform,
                "storage", "v1beta2", *STORAGE_FULL_SCOPE)
 
        gcs.bus.platform.get_numeric_project_id.return_value = '876103924605'
        gcs.bus.platform.new_storage_client = lambda: gsm.get_service()
 
        world.gcs = gcs.GCSFileSystem()
        world.tmpdir = tempfile.mkdtemp()
        # make random file
        tmp_file = os.path.join(world.tmpdir, 'test_file')
        system("dd if=/dev/urandom of=%s bs=1M count=1" % tmp_file, shell=True)
        world.src_md5 = system('md5sum %s' % tmp_file, shell=True)[0]
        LOG.info('MD5 : %s' % world.src_md5)
        world.bucket_name = 'scalr-tests-%s' % str(uuid.uuid4())[:8]
        LOG.info('Bucket name: %s' % world.bucket_name)
 
        world.dst_path = 'gcs://%s/test_file' % world.bucket_name
 
        try:
            world.gcs.ls(world.dst_path)
        except:
            pass
        else:
            raise Exception('Destination path already exist')
        world.gcs.put(tmp_file, world.dst_path)
        os.unlink(tmp_file)
Beispiel #9
0
def iptables_restore(filename, *short_args, **long_kwds):
    if isinstance(filename, basestring):
        filename = open(filename)
    linux.system(linux.build_cmd_args(executable=IPTABLES_RESTORE,
                                      short=short_args,
                                      long=long_kwds),
                 stdin=filename)
Beispiel #10
0
def then_i_see_recipe_scalarizr_proxy_applied(step):
    assert 'is running' in linux.system('/sbin/service nginx status',
                                        shell=True)[0], 'Nginx is running'
    for port in (8008, 8010, 8013):
        assert 'nginx' in linux.system(
            '/sbin/fuser -n tcp %d -v' % port,
            shell=True)[1], 'Nginx is listening port %d' % port
Beispiel #11
0
def upload(step):
        key_path = os.environ.get('GCS_KEY')
        with open(key_path) as f:
            key = base64.b64encode(f.read())
        access_data = dict(service_account_name='*****@*****.**', key=key)

        gcs.bus = mock.MagicMock()
        gcs.bus.platform.get_access_data = lambda k: access_data[k]

        gsm = GoogleServiceManager(gcs.bus.platform,
                "storage", "v1beta2", *STORAGE_FULL_SCOPE)

        gcs.bus.platform.get_numeric_project_id.return_value = '876103924605'
        gcs.bus.platform.new_storage_client = lambda: gsm.get_service()

        world.gcs = gcs.GCSFileSystem()
        world.tmpdir = tempfile.mkdtemp()
        # make random file
        tmp_file = os.path.join(world.tmpdir, 'test_file')
        system("dd if=/dev/urandom of=%s bs=1M count=1" % tmp_file, shell=True)
        world.src_md5 = system('md5sum %s' % tmp_file, shell=True)[0]
        LOG.info('MD5 : %s' % world.src_md5)
        world.bucket_name = 'scalr-tests-%s' % str(uuid.uuid4())[:8]
        LOG.info('Bucket name: %s' % world.bucket_name)

        world.dst_path = 'gcs://%s/test_file' % world.bucket_name

        try:
            world.gcs.ls(world.dst_path)
        except:
            pass
        else:
            raise Exception('Destination path already exist')
        world.gcs.put(tmp_file, world.dst_path)
        os.unlink(tmp_file)
Beispiel #12
0
    def create_volume(self, **params):
        size = params.get('size')
        snapshot_id = params.get('snapshot')

        assert size or snapshot_id, 'Not enough params to create volume'
        if snapshot_id:
            snapshot = self.describe_snapshot(snapshot_id)
            if size:
                if int(size) < int(snapshot['size']):
                    raise StorageError('Size you specified is smaller than snapshot')
        else:
            # Size in Gigabytes
            size = int(size)

        id = 'vol-%s' % str(uuid.uuid4())[:7]

        lvm2.lvcreate(vg_name, name=id, size='%sG' % size)
        lvinfo = lvm2.lvs(lvm2.lvpath(vg_name, id)).values()[0]
        device = os.path.realpath(lvinfo.lv_path)
        if snapshot_id:
            # Apply snapshot
            system('dd if=%s of=%s' % (self._get_snapshot_path(snapshot_id), device), shell=True)

        stat = os.stat(device)
        maj, min = (os.major(stat.st_rdev), os.minor(stat.st_rdev))

        self.volumes[id] = dict(id=id, attached_to=None, maj=maj, min=min,
                      host_path=device, size=str(size), source_snapshot=snapshot_id)
        return self.volumes[id]
Beispiel #13
0
 def on_start(self):
     if __node__['state'] == 'running':
         lfrp = bus.queryenv_service.list_farm_role_params(farm_role_id=__node__['farm_role_id'])['params']
         self._data = lfrp.get('router', {})
         if self._data:
             self._configure()
         else:
             linux.system('/etc/init.d/nginx start', shell=True, raise_exc=False)
Beispiel #14
0
def _cleanup_after_rebundle():
    cnf = bus.cnf
    pl = bus.platform
    logger = logging.getLogger(__name__)

    if os.path.exists('/etc/chef/client.pem'):
        os.remove('/etc/chef/client.pem')
    if os.path.exists('/etc/chef/client.rb'):
        os.remove('/etc/chef/client.rb')

    # remove storage devices from fstab
    if not linux.os.windows_family:

        def remove_systemd_generated_mount_unit(mpoint):
            unit_name = '{}.mount'.format(mpoint.replace('/', '-')[1:])
            logger.debug('Removing systemd runtime unit %s', unit_name)
            coreutils.remove(
                '/run/systemd/generator/local-fs.target.wants/{}'.format(
                    unit_name))
            coreutils.remove('/run/systemd/generator/{}'.format(unit_name))

        fstab = mount.fstab()
        should_reload_systemd = False
        for entry in fstab:
            if 'comment=scalr' in entry.options:
                logger.debug('Removing %s from fstab', entry.device)
                fstab.remove(entry.device)

                if linux.os['family'] == 'RedHat' and \
                    linux.os['name'] != 'Amazon' and \
                    linux.os['release'] >= (7, 0):

                    remove_systemd_generated_mount_unit(entry.mpoint)
                    should_reload_systemd = True

        if should_reload_systemd:
            linux.system('systemctl daemon-reload', shell=True)

    # Reset private configuration
    priv_path = cnf.private_path()
    for file in os.listdir(priv_path):
        if file in ('.user-data', 'update.status', 'keys'):
            # protect user-data and UpdateClient status
            # keys/default maybe already refreshed by UpdateClient
            continue
        path = os.path.join(priv_path, file)
        coreutils.chmod_r(path, 0700)
        try:
            os.remove(path) if (os.path.isfile(path) or
                                os.path.islink(path)) else shutil.rmtree(path)
        except:
            if linux.os.windows and sys.exc_info()[0] == WindowsError:
                # ScalrUpdClient locks db.sqlite
                logger.debug(sys.exc_info()[1])
            else:
                raise
    if not linux.os.windows_family:
        system2('sync', shell=True)
Beispiel #15
0
def rmloop(device):
    try:
        system((LOSETUP_EXEC, '-d', device))
    except LinuxError, e:
        if 'No such device or address' in e.err:
            ''' Silently pass non-existed loop removal '''
            pass
        else:
            raise
Beispiel #16
0
def setup(feat):
    if feat.name == FEATURE:
        p = mock.patch.multiple(s3.S3FileSystem,
                        _bucket_location=mock.Mock(return_value=''),
                        _get_connection=mock.Mock(side_effect=lambda: boto.connect_s3()))
        p.start()
        world.s3_patcher = p
        world.mysql_init = mysql_svc.MysqlInitScript()
        linux.system('mysqladmin create xtrabackup', shell=True, raise_exc=False)
Beispiel #17
0
def rsync(src, dst, **long_kwds):
    linux.system(['sync'])
    output = linux.system(linux.build_cmd_args(
            executable=os.path.join(__node__['embedded_bin_dir'], 'rsync'),
            long=long_kwds,
            params=[src, dst],
            duplicate_keys=True))
    linux.system(['sync'])
    return output
Beispiel #18
0
 def _patch_selinux(self):
     try:
         pkgmgr.installed('policycoreutils-python')
         system(['semanage', 'permissive', '-a', 'rabbitmq_beam_t'])
     except:
         e = sys.exc_info()[1]
         self._logger.warning(
             'Setting permissive selinux policy for rabbitmq context failed: {0}'
             .format(e))
Beispiel #19
0
def rmloop(device):
    try:
        system((LOSETUP_EXEC, '-d', device))
    except LinuxError, e:
        if 'No such device or address' in e.err:
            ''' Silently pass non-existed loop removal '''
            pass
        else:
            raise
Beispiel #20
0
def rsync(src, dst, **long_kwds):
    if not os.path.exists('/usr/bin/rsync'):
        pkgmgr.installed('rsync')
    system(['sync'])
    output = system(build_cmd_args(executable='/usr/bin/rsync',
                                   long=long_kwds,
                                   params=[src, dst],
                                   duplicate_keys=True))
    system(['sync'])
    return output
Beispiel #21
0
def umount(device_or_mpoint, **long_kwds):
    args = linux.build_cmd_args(executable='/bin/umount',
                                short=('-f', device_or_mpoint),
                                long=long_kwds)
    try:
        linux.system(args, error_text='Cannot umount %s' % device_or_mpoint)
    except linux.LinuxError, e:
        if 'not mounted' in e.err or 'not found' in e.err:
            return
        raise
Beispiel #22
0
def rsync(src, dst, **long_kwds):
    if not os.path.exists('/usr/bin/rsync'):
        pkgmgr.installed('rsync')
    system(['sync'])
    output = system(
        build_cmd_args(executable='/usr/bin/rsync',
                       long=long_kwds,
                       params=[src, dst],
                       duplicate_keys=True))
    system(['sync'])
    return output
Beispiel #23
0
 def on_start(self):
     if __node__['state'] == 'running':
         lfrp = bus.queryenv_service.list_farm_role_params(
             farm_role_id=__node__['farm_role_id'])['params']
         self._data = lfrp.get('router', {})
         if self._data:
             self._configure()
         else:
             linux.system('/etc/init.d/nginx start',
                          shell=True,
                          raise_exc=False)
Beispiel #24
0
def umount(device_or_mpoint, **long_kwds):
    args = linux.build_cmd_args(
            executable='/bin/umount',
            short=('-f', device_or_mpoint),
            long=long_kwds)
    try:
        linux.system(args, error_text='Cannot umount %s' % device_or_mpoint)
    except linux.LinuxError, e:
        if 'not mounted' in e.err or 'not found' in e.err:
            return
        raise
Beispiel #25
0
    def _create_spec_devices(self, root):
        nodes = (
            'console c 5 1',
            'null c 1 3',
            'zero c 1 5',
            'tty c 5 0',
        )

        for node in nodes:
            args = node.split()
            args[0] = os.path.join(root, 'dev', args[0])
            system(' '.join(['mknod'] + args), shell=True)
Beispiel #26
0
    def _create_spec_devices(self, root):
        nodes = (
                'console c 5 1',
                'null c 1 3',
                'zero c 1 5',
                'tty c 5 0',
        )

        for node in nodes:
            args = node.split()
            args[0] = os.path.join(root, 'dev', args[0])
            system(' '.join(['mknod'] + args), shell=True)
Beispiel #27
0
def mount(device, mpoint, *short_args, **long_kwds):
	args = linux.build_cmd_args(
		executable='/bin/mount',
		short=short_args, 
		long=long_kwds, 
		params=(device, mpoint)
	)
	try:
		msg = 'Cannot mount %s -> %s' % (device, mpoint)
		linux.system(args, error_text=msg)
	except linux.LinuxError, e:
		if 'you must specify the filesystem type' in e.err:
			raise NoFileSystem(device)
		raise
Beispiel #28
0
    def uninstall(self):
        pid = None
        if not linux.os.windows:
            # Prevent scalr-upd-client restart when updating from old versions
            # package 'scalr-upd-client' replaced with 'scalarizr'
            pid_file = "/var/run/scalr-upd-client.pid"
            if os.path.exists(pid_file):
                with open(pid_file) as fp:
                    pid = fp.read().strip()
                with open(pid_file, "w") as fp:
                    fp.write("0")
        try:
            self.pkgmgr.removed(self.package)
            if not linux.os.windows:
                if linux.os.redhat_family:
                    installed_ver = self.pkgmgr.info("scalarizr")["installed"]
                    cmd = "rpm -e scalarizr"
                    if installed_ver and distutils.version.LooseVersion(installed_ver) < "0.7":
                        # On CentOS 5 there is a case when scalarizr-0.6.24-5 has error
                        # in preun scriplet and cannot be uninstalled
                        cmd += " --noscripts"
                    linux.system(cmd, shell=True, raise_exc=False)
                else:
                    self.pkgmgr.removed("scalarizr", purge=True)
                self.pkgmgr.removed("scalarizr-base", purge=True)  # Compatibility with BuildBot packaging
                updclient_pkginfo = self.pkgmgr.info("scalr-upd-client")
                if updclient_pkginfo["installed"]:
                    # Only latest package don't stop scalr-upd-client in postrm script
                    if updclient_pkginfo["candidate"]:
                        if linux.os.debian_family:
                            cmd = ("-o Dpkg::Options::=--force-confmiss " "install scalr-upd-client={0}").format(
                                updclient_pkginfo["candidate"]
                            )
                            self.pkgmgr.apt_get_command(cmd)
                        else:
                            self.pkgmgr.install("scalr-upd-client", updclient_pkginfo["candidate"])
                    try:
                        self.pkgmgr.removed("scalr-upd-client", purge=True)
                    except:
                        if linux.os.redhat_family:
                            linux.system("rpm -e --noscripts scalr-upd-client", shell=True, raise_exc=False)
                        else:
                            raise

        finally:
            if pid:
                with open(pid_file, "w+") as fp:
                    fp.write(pid)
Beispiel #29
0
    def on_init(self, *args, **kwargs):
        bus.on("before_hello", self.on_before_hello)
        bus.on("before_host_init", self.on_before_host_init)
        bus.on("before_restart", self.on_before_restart)
        bus.on("before_reboot_finish", self.on_before_reboot_finish)

        try:
            system(('ntpdate', '-u', '0.amazon.pool.ntp.org'))
        except:
            pass

        msg_service = bus.messaging_service
        producer = msg_service.get_producer()
        producer.on("before_send", self.on_before_message_send)

        if not os_dist.windows_family and not __node__['base'].get('hostname'):
            # Set the hostname to this instance's public hostname
            try:
                hostname_as_pubdns = int(__ec2__['hostname_as_pubdns'])
            except:
                hostname_as_pubdns = True

            if hostname_as_pubdns:
                pub_hostname = self._platform.get_public_hostname()
                self._logger.debug('Setting hostname to %s' % pub_hostname)
                system2("hostname " + pub_hostname, shell=True)

        if linux.os.ubuntu:
            # Ubuntu cloud-init scripts may disable root ssh login
            for path in ('/etc/ec2-init/ec2-config.cfg',
                         '/etc/cloud/cloud.cfg'):
                if os.path.exists(path):
                    c = None
                    with open(path, 'r') as fp:
                        c = fp.read()
                    c = re.sub(
                        re.compile(r'^disable_root[^:=]*([:=]).*', re.M),
                        r'disable_root\1 0', c)
                    with open(path, 'w') as fp:
                        fp.write(c)

        if not linux.os.windows_family:
            # Add server ssh public key to authorized_keys
            ssh_key = self._platform.get_ssh_pub_key()
            if ssh_key:
                add_authorized_key(ssh_key)

            system2('mount -a', shell=True, raise_exc=False)
Beispiel #30
0
def df():
    out = linux.system(('df', '-Pk'))[0]
    return [
        _dfrow(line[0], _parse_int(line[1]), line[-1])
        for line in map(str.split,
                        out.splitlines()[1:])
    ]
Beispiel #31
0
def blkid(device_path, **kwargs):
    if not os.path.exists(device_path):
        raise Exception("Device %s doesn't exist")

    ret = dict()

    args = ['/sbin/blkid']
    for k, v in kwargs.items():
        if type(v) == bool:
            args.append('-%s' % k)
        else:
            args.extend(['-%s' % k, str(v)])

    args.append(device_path)

    out = linux.system(args, raise_exc=False)[0]
    if out.strip():
        pairs = out.split()[1:]
        for line in pairs:
            line = line.strip()
            if line:
                k, v = line.split('=', 1)
                ret[k.lower()] = v[1:-1]

    return ret
Beispiel #32
0
 def execute(self, command=None):
     out, err, ret = linux.system(command, shell=True)
     return {
         'stdout': out,
         'stderr': err,
         'return_code': ret
     }
Beispiel #33
0
 def prepare_image(self):
     # prepares image with ec2-bundle-vol command
     if not os.path.exists(self.destination):
         os.mkdir(self.destination)
     cmd = (
         self.bundle_vol_cmd,
         '--cert',
         self.credentials['cert'],
         '--privatekey',
         self.credentials['key'],
         '--user',
         self.credentials['user'],
         '--arch',
         linux.os['arch'],
         '--size',
         str(self.image_size),
         '--destination',
         self.destination,
         # '--exclude', ','.join(self.excludes),
         # '--block-device-mapping', ,  # TODO:
         '--prefix',
         self.image_name,
         '--volume',
         '/',
         '--debug')
     LOG.debug('Image prepare command: ' + ' '.join(cmd))
     out = linux.system(cmd,
                        env=self.environ,
                        stdout=subprocess.PIPE,
                        stderr=subprocess.STDOUT)[0]
     LOG.debug('Image prepare command out: %s' % out)
Beispiel #34
0
def blkid(device_path, **kwargs):
    if not os.path.exists(device_path):
        raise Exception("Device %s doesn't exist")

    ret = dict()

    args = ['/sbin/blkid']
    for k, v in kwargs.items():
        if type(v) == bool:
            args.append('-%s' % k)
        else:
            args.extend(['-%s' % k, str(v)])

    args.append(device_path)

    out = linux.system(args, raise_exc=False)[0]
    if out.strip():
        pairs = out.split()[1:]
        for line in pairs:
            line = line.strip()
            if line:
                k, v = line.split('=', 1)
                ret[k.lower()] = v[1:-1]

    return ret
Beispiel #35
0
def listloop():
    ret = {}
    loop_lines = system((LOSETUP_EXEC, '-a'))[0].strip().splitlines()
    for loop_line in loop_lines:
        words = loop_line.split()
        ret[words[0][:-1]] = words[-1][1:-1]
    return ret
Beispiel #36
0
 def apt_get_command(self, command, **kwds):
     kwds.update(env={
         'DEBIAN_FRONTEND':
         'noninteractive',
         'DEBIAN_PRIORITY':
         'critical',
         'PATH':
         '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games'
     },
                 error_text='Package manager command failed')
     max_attempt = 17
     for attempt in range(max_attempt + 1):  # timeout approx 3 minutes
         try:
             return linux.system(('/usr/bin/apt-get',
                                 '-q', '-y', '--force-yes',
                                 '-o Dpkg::Options::=--force-confold') + \
                                 tuple(filter(None, command.split())), **kwds)
         except linux.LinuxError, e:
             if attempt < max_attempt and 'is another process using it?' in e.err \
                     or 'Could not get lock' in e.err:
                 LOG.debug(
                     'Could not get dpkg lock (perhaps, another process is using it.)'
                 )
                 time.sleep(10)
                 continue
             raise
Beispiel #37
0
 def version_cmp(self, name_1, name_2):
     if name_1 == name_2:
         return 0
     return_code = linux.system(('/usr/bin/dpkg', '--compare-versions',
                                 str(name_1), '>', str(name_2)),
                                raise_exc=False)[2]
     return 1 if not return_code else -1
Beispiel #38
0
def mysqldump(*databases, **long_kwds):
    output = long_kwds.pop("output", None)
    cmd = linux.build_cmd_args(executable="/usr/bin/mysqldump", long=long_kwds, params=databases)
    kwds = {}
    if output:
        kwds["stdout"] = open(output, "w+")
    return linux.system(cmd, **kwds)
Beispiel #39
0
def innobackupex(*params, **long_kwds):
	if not os.path.exists('/usr/bin/innobackupex'):
		pkgmgr.installed('percona-xtrabackup')
	return linux.system(linux.build_cmd_args(
			executable='/usr/bin/innobackupex', 
			long=long_kwds, 
			params=params))
Beispiel #40
0
def dd(**kwds):
    short = []
    for k, v in kwds.items():
        short.append('%s=%s' % (k, v))
    return linux.system(linux.build_cmd_args(
                executable='/bin/dd',
                short=short))
Beispiel #41
0
def listloop():
    ret = {}
    loop_lines = system((LOSETUP_EXEC, '-a'))[0].strip().splitlines()
    for loop_line in loop_lines:
        words = loop_line.split()
        ret[words[0][:-1]] = words[-1][1:-1]
    return ret
Beispiel #42
0
    def apt_get_command(self, command, **kwds):
        kwds.update(env={
            'DEBIAN_FRONTEND':
            'noninteractive',
            'DEBIAN_PRIORITY':
            'critical',
            'PATH':
            '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games'
        },
                    raise_exc=False)
        for _ in range(18):  # timeout approx 3 minutes
            out, err, code = linux.system(('/usr/bin/apt-get',
                                            '-q', '-y', '--force-yes',
                                            '-o Dpkg::Options::=--force-confold') + \
                                            tuple(filter(None, command.split())), **kwds)
            if code:
                if 'is another process using it?' in err \
                    or 'Could not get lock' in err:
                    LOG.debug(
                        'Could not get dpkg lock (perhaps, another process is using it.)'
                    )
                    time.sleep(10)
                    continue
                else:
                    raise LinuxError(
                        'Apt-get command failed. Out: %s \nErrors: %s' %
                        (out, err))

            else:
                return out, err, code

        raise Exception(
            'Apt-get command failed: dpkg is being used by another process')
Beispiel #43
0
def dd(**kwds):
    short = []
    for k, v in kwds.items():
        short.append('%s=%s' % (k, v))
    return linux.system(linux.build_cmd_args(
                executable='/bin/dd',
                short=short))
Beispiel #44
0
    def uninstall(self):
        pid = None
        if not linux.os.windows:
            # Prevent scalr-upd-client restart when updating from old versions
            # package 'scalr-upd-client' replaced with 'scalarizr'
            pid_file = '/var/run/scalr-upd-client.pid'
            if os.path.exists(pid_file):
                with open(pid_file) as fp:
                    pid = fp.read().strip()
                with open(pid_file, 'w') as fp:
                    fp.write('0')
        try:
            self.pkgmgr.removed(self.package)
            if not linux.os.windows:
                if linux.os.redhat_family:
                    installed_ver = self.pkgmgr.info('scalarizr')['installed']
                    cmd = 'rpm -e scalarizr'
                    if installed_ver and distutils.version.LooseVersion(installed_ver) < '0.7':      
                        # On CentOS 5 there is a case when scalarizr-0.6.24-5 has error 
                        # in preun scriplet and cannot be uninstalled
                        cmd += ' --noscripts'
                    linux.system(cmd, shell=True, raise_exc=False)
                else:
                    self.pkgmgr.removed('scalarizr', purge=True)
                self.pkgmgr.removed('scalarizr-base', purge=True)  # Compatibility with BuildBot packaging
                updclient_pkginfo = self.pkgmgr.info('scalr-upd-client')
                if updclient_pkginfo['installed']:
                    # Only latest package don't stop scalr-upd-client in postrm script
                    if updclient_pkginfo['candidate']:
                        if linux.os.debian_family:
                            cmd = ('-o Dpkg::Options::=--force-confmiss '
                                    'install scalr-upd-client={0}').format(updclient_pkginfo['candidate'])
                            self.pkgmgr.apt_get_command(cmd)
                        else:
                            self.pkgmgr.install('scalr-upd-client', updclient_pkginfo['candidate'])
                    try:
                        self.pkgmgr.removed('scalr-upd-client', purge=True)
                    except:
                        if linux.os.redhat_family:
                            linux.system('rpm -e --noscripts scalr-upd-client', shell=True, raise_exc=False)
                        else:
                            raise

        finally:
            if pid:
                with open(pid_file, 'w+') as fp:
                    fp.write(pid)
Beispiel #45
0
    def on_init(self, *args, **kwargs):
        bus.on("before_hello", self.on_before_hello)
        bus.on("before_host_init", self.on_before_host_init)
        bus.on("before_restart", self.on_before_restart)
        bus.on("before_reboot_finish", self.on_before_reboot_finish)

        try:
            system(('ntpdate', '-u', '0.amazon.pool.ntp.org'))
        except:
            pass

        msg_service = bus.messaging_service
        producer = msg_service.get_producer()
        producer.on("before_send", self.on_before_message_send)

        if not os_dist.windows_family and not __node__['base'].get('hostname'):
            # Set the hostname to this instance's public hostname
            try:
                hostname_as_pubdns = int(__ec2__['hostname_as_pubdns'])
            except:
                hostname_as_pubdns = True

            if hostname_as_pubdns:
                pub_hostname = self._platform.get_public_hostname()
                self._logger.debug('Setting hostname to %s' % pub_hostname)
                system2("hostname " + pub_hostname, shell=True)

        if linux.os.ubuntu:
            # Ubuntu cloud-init scripts may disable root ssh login
            for path in ('/etc/ec2-init/ec2-config.cfg', '/etc/cloud/cloud.cfg'):
                if os.path.exists(path):
                    c = None
                    with open(path, 'r') as fp:
                        c = fp.read()
                    c = re.sub(re.compile(r'^disable_root[^:=]*([:=]).*', re.M), r'disable_root\1 0', c)
                    with open(path, 'w') as fp:
                        fp.write(c)

        if not linux.os.windows_family:
            # Add server ssh public key to authorized_keys
            ssh_key = self._platform.get_ssh_pub_key()
            if ssh_key:
                add_authorized_key(ssh_key)

            system2('mount -a', shell=True, raise_exc=False)
Beispiel #46
0
def modprobe(module_name, **long_kwds):
    if not os_info['mods_enabled']:
        return (None, None, 0)

    return linux.system(linux.build_cmd_args(executable='/sbin/modprobe',
                                             long=long_kwds,
                                             params=[module_name]),
                        error_text='Kernel module %s is not available' %
                        module_name)
Beispiel #47
0
def system(*args, **kwargs):
	kwargs['logger'] = LOG
	kwargs['close_fds'] = True
	'''
	To prevent this garbage in stderr (Fedora/CentOS):
	File descriptor 6 (/tmp/ffik4yjng (deleted)) leaked on lv* invocation. 
	Parent PID 29542: /usr/bin/python
	'''
	return linux.system(*args, **kwargs)
Beispiel #48
0
 def repos(self):
     ret = []
     repo_re = re.compile(r'Repo-id\s+:\s(.*)')
     out = linux.system(('/usr/bin/yum', 'repolist', '--verbose'))[0]
     for line in out.splitlines():
         m = repo_re.search(line)
         if m:
             ret.append(m.group(1))
     return map(string.lower, ret)
Beispiel #49
0
 def repos(self):
     ret = []
     repo_re = re.compile(r'Repo-id\s+:\s(.*)')
     out = linux.system(('/usr/bin/yum', 'repolist', '--verbose'))[0]
     for line in out.splitlines():
         m = repo_re.search(line)
         if m:
             ret.append(m.group(1))
     return map(string.lower, ret)
Beispiel #50
0
def mysqldump(*databases, **long_kwds):
    output = long_kwds.pop('output', None)
    cmd = linux.build_cmd_args(executable='/usr/bin/mysqldump',
                               long=long_kwds,
                               params=databases)
    kwds = {}
    if output:
        kwds['stdout'] = open(output, 'w+')
    return linux.system(cmd, **kwds)
Beispiel #51
0
def modprobe(module_name, **long_kwds):
    if not os_info['mods_enabled']:
        return (None, None, 0)

    return linux.system(linux.build_cmd_args(
                executable='/sbin/modprobe', 
                long=long_kwds,
                params=[module_name]), 
            error_text='Kernel module %s is not available' % module_name)
Beispiel #52
0
def mysqldump(*databases, **long_kwds):
	output = long_kwds.pop('output', None)
	cmd = linux.build_cmd_args(
			executable='/usr/bin/mysqldump',
			long=long_kwds,
			params=databases)
	kwds = {}
	if output:
		kwds['stdout'] = open(output, 'w+')
	return linux.system(cmd, **kwds)
Beispiel #53
0
	def apt_get_command(self, command, **kwds):
		kwds.update(env={
			'DEBIAN_FRONTEND': 'noninteractive', 
			'DEBIAN_PRIORITY': 'critical',
			'PATH': '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games'
		})
		return linux.system(('/usr/bin/apt-get',
						'-q', '-y', '--force-yes',
						'-o Dpkg::Options::=--force-confold') + \
						tuple(filter(None, command.split())), **kwds)
Beispiel #54
0
 def forcerestart(self):
     LOG.info('Forcefully restarting %s', self.name)
     self.ctl('stop')
     try:
         out = linux.system('ps -C %s --noheaders -o pid' % self.name)[0]
         for pid in out.strip().splitlines():
             LOG.debug('Killing process %s', pid)
             os.kill(pid, 9)
     finally:
         self.ctl('start')
Beispiel #55
0
def iptables(**long_kwds):

	ordered_long = OrderedDict()
	for key in ("protocol", "match"):
		if key in long_kwds:
			ordered_long[key] = long_kwds.pop(key)
	ordered_long.update(long_kwds)

	return linux.system(linux.build_cmd_args(executable=IPTABLES_BIN,
		long=ordered_long))
Beispiel #56
0
def pvchange(*physical_volume_paths, **long_kwds):
	try:
		return linux.system(linux.build_cmd_args(
				executable='/sbin/pvchange',
				long=long_kwds,
				params=physical_volume_paths))
	except linux.LinuxError, e:
		if e.returncode == 5:
			raise NotFound()
		raise
Beispiel #57
0
def vgchange(*volume_group_names, **long_kwds):
	try:
		return linux.system(linux.build_cmd_args(
				executable='/sbin/vgchange',
				long=long_kwds,
				params=volume_group_names))
	except linux.LinuxError, e:
		if e.returncode == 5:
			raise NotFound()
		raise
Beispiel #58
0
def save():
    '''
    on RHEL call 'service iptables save'
    on Ubuntu:
            - touch or create /etc/network/if-pre-up.d/iptables.sh
                    $ cat /etc/network/if-pre-up.d/iptables.sh
                    #!/bin/bash
                    iptables-restore < /etc/iptables.rules
            - iptables-save > /etc/iptables.rules
    '''
    if linux.os["family"] in ("RedHat", "Oracle"):
        linux.system(linux.build_cmd_args(executable="service", short=['iptables',
                                                                                                                                   'save']))
    elif linux.os["family"] == "Debian":
        with open('/etc/network/if-pre-up.d/iptables.sh', 'w') as fp:
            fp.write('#!/bin/bash\n'
                             'iptables-restore < /etc/iptables.rules')

        iptables_save('/etc/iptables.rules')
Beispiel #59
0
def my_print_defaults(*option_groups):
    out = linux.system(linux.build_cmd_args(executable="/usr/bin/my_print_defaults", params=option_groups))[0]
    ret = {}
    for line in out.splitlines():
        cols = line.split("=")
        ret[cols[0][2:]] = cols[1] if len(cols) > 1 else True
    for key in __mysql__["defaults"]:
        if key not in ret:
            ret[key] = __mysql__["defaults"][key]
    return ret