def _crawl_packages(self, dbpath=None, root_dir='/'):

        assert(self.crawl_mode is not Modes.OUTCONTAINER)

        # package attributes: ["installed", "name", "size", "version"]

        (installtime, name, version, size) = (None, None, None, None)

        if self.crawl_mode == Modes.INVM:

            logger.debug('Using in-VM state information (crawl mode: ' +
                         self.crawl_mode + ')')
            system_type = platform.system().lower()
            distro = platform.linux_distribution()[0].lower()
        elif self.crawl_mode == Modes.MOUNTPOINT:
            logger.debug('Using disk image information (crawl mode: ' +
                         self.crawl_mode + ')')
            system_type = \
                platform_outofband.system(prefix=root_dir).lower()
            distro = platform_outofband.linux_distribution(prefix=root_dir)[
                0].lower()
        else:
            logger.error('Unsupported crawl mode: ' + self.crawl_mode +
                         '. Skipping package crawl.')
            system_type = 'unknown'
            distro = 'unknown'

        installed_since = self.feature_epoch
        if system_type != 'linux':

            # Package feature is only valid for Linux platforms.

            raise StopIteration()
        logger.debug('Crawling Packages')

        pkg_manager = 'unknown'
        if distro in ['ubuntu', 'debian']:
            pkg_manager = 'dpkg'
        elif distro.startswith('red hat') or distro in ['redhat',
                                                        'fedora', 'centos']:

            pkg_manager = 'rpm'
        elif os.path.exists(os.path.join(root_dir, 'var/lib/dpkg')):
            pkg_manager = 'dpkg'
        elif os.path.exists(os.path.join(root_dir, 'var/lib/rpm')):
            pkg_manager = 'rpm'

        try:
            if pkg_manager == 'dpkg':
                if not dbpath:
                    dbpath = 'var/lib/dpkg'
                if os.path.isabs(dbpath):
                    logger.warning(
                        'dbpath: ' +
                        dbpath +
                        ' is defined absolute. Ignoring prefix: ' +
                        root_dir +
                        '.')

                # Update for a different route.

                dbpath = os.path.join(root_dir, dbpath)
                if installed_since > 0:
                    logger.warning(
                        'dpkg does not provide install-time, defaulting to '
                        'all packages installed since epoch')
                try:
                    dpkg = subprocess.Popen(['dpkg-query', '-W',
                                             '--admindir={0}'.format(dbpath),
                                             '-f=${Package}|${Version}'
                                             '|${Installed-Size}\n'
                                             ], stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)
                    dpkglist = dpkg.stdout.read().strip('\n')
                except OSError as e:
                    logger.error(
                        'Failed to launch dpkg query for packages. Check if '
                        'dpkg-query is installed: [Errno: %d] ' %
                        e.errno + e.strerror + ' [Exception: ' +
                        type(e).__name__ + ']')
                    dpkglist = None
                if dpkglist:
                    for dpkginfo in dpkglist.split('\n'):
                        (name, version, size) = dpkginfo.split(r'|')

            # dpkg does not provide any installtime field
            # feature_key = '{0}/{1}'.format(name, version) -->
            # changed to below per Suriya's request

                        feature_key = '{0}'.format(name, version)
                        yield (feature_key, PackageFeature(None, name,
                                                           size, version))
            elif pkg_manager == 'rpm':
                if not dbpath:
                    dbpath = 'var/lib/rpm'
                if os.path.isabs(dbpath):
                    logger.warning(
                        'dbpath: ' +
                        dbpath +
                        ' is defined absolute. Ignoring prefix: ' +
                        root_dir +
                        '.')
                # update for a different route
                dbpath = os.path.join(root_dir, dbpath)
                try:
                    rpm = subprocess.Popen([
                        'rpm',
                        '--dbpath',
                        dbpath,
                        '-qa',
                        '--queryformat',
                        '%{installtime}|%{name}|%{version}'
                        '-%{release}|%{size}\n',
                    ], stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE)
                    rpmlist = rpm.stdout.read().strip('\n')
                except OSError as e:
                    logger.error(
                        'Failed to launch rpm query for packages. Check if '
                        'rpm is installed: [Errno: %d] ' %
                        e.errno + e.strerror + ' [Exception: ' +
                        type(e).__name__ + ']')
                    rpmlist = None
                if rpmlist:
                    for rpminfo in rpmlist.split('\n'):
                        (installtime, name, version, size) = \
                            rpminfo.split(r'|')

            # if int(installtime) <= installed_since: --> this
            # barfs for sth like: 1376416422. Consider try: xxx
            # except ValueError: pass

                        if installtime <= installed_since:
                            continue

            # feature_key = '{0}/{1}'.format(name, version) -->
            # changed to below per Suriya's request

                        feature_key = '{0}'.format(name, version)
                        yield (feature_key,
                               PackageFeature(installtime,
                                              name, size, version))
            else:
                raise CrawlError(
                    Exception(
                        'Unsupported package manager for Linux distro %s' %
                        distro))
        except Exception as e:
            logger.error('Error crawling package %s'
                         % ((name if name else 'Unknown')),
                         exc_info=True)
            raise CrawlError(e)
    def _crawl_os(self, mountpoint=None):

        assert(self.crawl_mode is not Modes.OUTCONTAINER)

        logger.debug('Crawling OS')
        if self.crawl_mode == Modes.INVM:
            logger.debug('Using in-VM state information (crawl mode: ' +
                         self.crawl_mode + ')')
            feature_key = platform.system().lower()

            try:
                ips = misc.get_host_ip4_addresses()
            except Exception as e:
                ips = 'unknown'
            try:
                distro = platform.linux_distribution()[0]
            except Exception as e:
                distro = 'unknown'
            try:
                osname = platform.platform()
            except Exception as e:
                osname = 'unknown'

            boot_time = (
                psutil.boot_time() if hasattr(
                    psutil, 'boot_time') else psutil.BOOT_TIME)
            uptime = int(time.time()) - boot_time
            feature_attributes = OSFeature(
                boot_time,
                uptime,
                ips,
                distro,
                osname,
                platform.machine(),
                platform.release(),
                platform.system().lower(),
                platform.version(),
            )
        elif self.crawl_mode == Modes.MOUNTPOINT:
            logger.debug('Using disk image information (crawl mode: ' +
                         self.crawl_mode + ')')
            feature_key = \
                platform_outofband.system(prefix=mountpoint).lower()
            feature_attributes = OSFeature(  # boot time unknown for img
                                             # live IP unknown for img
                'unsupported',
                '0.0.0.0',
                platform_outofband.linux_distribution(
                    prefix=mountpoint)[0],
                platform_outofband.platform(prefix=mountpoint),
                platform_outofband.machine(prefix=mountpoint),
                platform_outofband.release(prefix=mountpoint),
                platform_outofband.system(prefix=mountpoint).lower(),
                platform_outofband.version(prefix=mountpoint),
            )
        elif self.crawl_mode == Modes.OUTVM:

            (domain_name, kernel_version, distro, arch) = self.vm
            from psvmi import system_info
            sys = system_info(domain_name, kernel_version, distro, arch)
            uptime = int(time.time()) - sys.boottime
            feature_attributes = OSFeature(
                sys.boottime,
                sys.ipaddr,
                sys.osdistro,
                sys.osname,
                sys.osplatform,
                sys.osrelease,
                sys.ostype,
                sys.osversion,
            )
            feature_key = sys.ostype
        else:
            raise NotImplementedError()
        try:
            yield (feature_key, feature_attributes)
        except Exception as e:
            logger.error('Error crawling OS', exc_info=True)
            raise CrawlError(e)
    def _crawl_packages(self, dbpath=None, root_dir='/'):

        # package attributes: ["installed", "name", "size", "version"]

        (installtime, name, version, size) = (None, None, None, None)

        if self.crawl_mode == Modes.INVM:

            logger.debug('Using in-VM state information (crawl mode: ' +
                         self.crawl_mode + ')')
            system_type = platform.system().lower()
            distro = platform.linux_distribution()[0].lower()
            reload_needed = False
        elif self.crawl_mode == Modes.OUTCONTAINER:

            logger.debug('Using outcontainer state information (crawl mode: ' +
                         self.crawl_mode + ')')

            system_type = 'linux'
            distro = ''
            reload_needed = True
        elif self.crawl_mode == Modes.MOUNTPOINT:
            logger.debug('Using disk image information (crawl mode: ' +
                         self.crawl_mode + ')')
            system_type = \
                platform_outofband.system(prefix=root_dir).lower()
            distro = platform_outofband.linux_distribution(prefix=root_dir)[
                0].lower()
            reload_needed = False
        else:
            logger.error('Unsupported crawl mode: ' + self.crawl_mode +
                         '. Skipping package crawl.')
            system_type = 'unknown'
            distro = 'unknown'
            reload_needed = True

        installed_since = self.feature_epoch
        if system_type != 'linux':
            # Package feature is only valid for Linux platforms.

            raise StopIteration()
        logger.debug('Crawling Packages')

        pkg_manager = 'unknown'
        if distro in ['ubuntu', 'debian']:
            pkg_manager = 'dpkg'
        elif distro.startswith('red hat') or distro in ['redhat',
                                                        'fedora', 'centos']:
            pkg_manager = 'rpm'
        elif os.path.exists(os.path.join(root_dir, 'var/lib/dpkg')):
            pkg_manager = 'dpkg'
        elif os.path.exists(os.path.join(root_dir, 'var/lib/rpm')):
            pkg_manager = 'rpm'

        try:
            if pkg_manager == 'dpkg':
                if not dbpath:
                    dbpath = 'var/lib/dpkg'
                for (key, feature) in get_dpkg_packages(
                        root_dir, dbpath, installed_since):
                    yield (key, feature)
            elif pkg_manager == 'rpm':
                if not dbpath:
                    dbpath = 'var/lib/rpm'
                for (key, feature) in get_rpm_packages(
                        root_dir, dbpath, installed_since, reload_needed):
                    yield (key, feature)
            else:
                logger.warning('Unsupported package manager for Linux distro')
        except Exception as e:
            logger.error('Error crawling package %s'
                         % ((name if name else 'Unknown')),
                         exc_info=True)
            raise CrawlError(e)
Exemplo n.º 4
0
    def _crawl_packages(self, dbpath=None, root_dir='/'):

        assert(self.crawl_mode is not Modes.OUTCONTAINER)

        # package attributes: ["installed", "name", "size", "version"]

        (installtime, name, version, size) = (None, None, None, None)

        if self.crawl_mode == Modes.INVM:

            logger.debug('Using in-VM state information (crawl mode: ' +
                         self.crawl_mode + ')')
            system_type = platform.system().lower()
            distro = platform.linux_distribution()[0].lower()
        elif self.crawl_mode == Modes.MOUNTPOINT:
            logger.debug('Using disk image information (crawl mode: ' +
                         self.crawl_mode + ')')
            system_type = \
                platform_outofband.system(prefix=root_dir).lower()
            distro = platform_outofband.linux_distribution(prefix=root_dir)[
                0].lower()
        else:
            logger.error('Unsupported crawl mode: ' + self.crawl_mode +
                         '. Skipping package crawl.')
            system_type = 'unknown'
            distro = 'unknown'

        installed_since = self.feature_epoch
        if system_type != 'linux' or (system_type == 'linux' and distro == ''):
            # Distro is blank for FROM scratch images
            # Package feature is only valid for Linux platforms.

            raise StopIteration()
        logger.debug('Crawling Packages')

        pkg_manager = 'unknown'
        if distro in ['ubuntu', 'debian']:
            pkg_manager = 'dpkg'
        elif distro.startswith('red hat') or distro in ['redhat',
                                                        'fedora', 'centos']:

            pkg_manager = 'rpm'
        elif os.path.exists(os.path.join(root_dir, 'var/lib/dpkg')):
            pkg_manager = 'dpkg'
        elif os.path.exists(os.path.join(root_dir, 'var/lib/rpm')):
            pkg_manager = 'rpm'

        try:
            if pkg_manager == 'dpkg':
                if not dbpath:
                    dbpath = 'var/lib/dpkg'
                if os.path.isabs(dbpath):
                    logger.warning(
                        'dbpath: ' +
                        dbpath +
                        ' is defined absolute. Ignoring prefix: ' +
                        root_dir +
                        '.')

                # Update for a different route.

                dbpath = os.path.join(root_dir, dbpath)
                if installed_since > 0:
                    logger.warning(
                        'dpkg does not provide install-time, defaulting to '
                        'all packages installed since epoch')
                try:
                    dpkg = subprocess.Popen(['dpkg-query', '-W',
                                             '--admindir={0}'.format(dbpath),
                                             '-f=${Package}|${Version}'
                                             '|${Installed-Size}\n'
                                             ], stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)
                    dpkglist = dpkg.stdout.read().strip('\n')
                except OSError as e:
                    logger.error(
                        'Failed to launch dpkg query for packages. Check if '
                        'dpkg-query is installed: [Errno: %d] ' %
                        e.errno + e.strerror + ' [Exception: ' +
                        type(e).__name__ + ']')
                    dpkglist = None
                if dpkglist:
                    for dpkginfo in dpkglist.split('\n'):
                        (name, version, size) = dpkginfo.split(r'|')

            # dpkg does not provide any installtime field
            # feature_key = '{0}/{1}'.format(name, version) -->
            # changed to below per Suriya's request

                        feature_key = '{0}'.format(name, version)
                        yield (feature_key, PackageFeature(None, name,
                                                           size, version))
            elif pkg_manager == 'rpm':
                if not dbpath:
                    dbpath = 'var/lib/rpm'
                if os.path.isabs(dbpath):
                    logger.warning(
                        'dbpath: ' +
                        dbpath +
                        ' is defined absolute. Ignoring prefix: ' +
                        root_dir +
                        '.')
                # update for a different route
                dbpath = os.path.join(root_dir, dbpath)
                try:
                    rpm = subprocess.Popen([
                        'rpm',
                        '--dbpath',
                        dbpath,
                        '-qa',
                        '--queryformat',
                        '%{installtime}|%{name}|%{version}'
                        '-%{release}|%{size}\n',
                    ], stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE)
                    rpmlist = rpm.stdout.read().strip('\n')
                except OSError as e:
                    logger.error(
                        'Failed to launch rpm query for packages. Check if '
                        'rpm is installed: [Errno: %d] ' %
                        e.errno + e.strerror + ' [Exception: ' +
                        type(e).__name__ + ']')
                    rpmlist = None
                if rpmlist:
                    for rpminfo in rpmlist.split('\n'):
                        (installtime, name, version, size) = \
                            rpminfo.split(r'|')

            # if int(installtime) <= installed_since: --> this
            # barfs for sth like: 1376416422. Consider try: xxx
            # except ValueError: pass

                        if installtime <= installed_since:
                            continue

            # feature_key = '{0}/{1}'.format(name, version) -->
            # changed to below per Suriya's request

                        feature_key = '{0}'.format(name, version)
                        yield (feature_key,
                               PackageFeature(installtime,
                                              name, size, version))
            else:
                raise CrawlError(
                    Exception(
                        'Unsupported package manager for Linux distro %s' %
                        distro))
        except Exception as e:
            logger.error('Error crawling package %s'
                         % ((name if name else 'Unknown')),
                         exc_info=True)
            raise CrawlError(e)
Exemplo n.º 5
0
    def _crawl_os(self, mountpoint=None):

        assert(self.crawl_mode is not Modes.OUTCONTAINER)

        logger.debug('Crawling OS')
        if self.crawl_mode == Modes.INVM:
            logger.debug('Using in-VM state information (crawl mode: ' +
                         self.crawl_mode + ')')
            feature_key = platform.system().lower()

            try:
                ips = misc.get_host_ip4_addresses()
            except Exception as e:
                ips = 'unknown'
            try:
                distro = platform.linux_distribution()[0]
            except Exception as e:
                distro = 'unknown'
            try:
                osname = platform.platform()
            except Exception as e:
                osname = 'unknown'

            boot_time = (
                psutil.boot_time() if hasattr(
                    psutil, 'boot_time') else psutil.BOOT_TIME)
            uptime = int(time.time()) - boot_time
            feature_attributes = OSFeature(
                boot_time,
                uptime,
                ips,
                distro,
                osname,
                platform.machine(),
                platform.release(),
                platform.system().lower(),
                platform.version(),
            )
        elif self.crawl_mode == Modes.MOUNTPOINT:
            logger.debug('Using disk image information (crawl mode: ' +
                         self.crawl_mode + ')')
            feature_key = \
                platform_outofband.system(prefix=mountpoint).lower()
            feature_attributes = OSFeature(  # boot time unknown for img
                                             # live IP unknown for img
                'unsupported',
                '0.0.0.0',
                platform_outofband.linux_distribution(
                    prefix=mountpoint)[0],
                platform_outofband.platform(prefix=mountpoint),
                platform_outofband.machine(prefix=mountpoint),
                platform_outofband.release(prefix=mountpoint),
                platform_outofband.system(prefix=mountpoint).lower(),
                platform_outofband.version(prefix=mountpoint),
            )
        elif self.crawl_mode == Modes.OUTVM:

            (domain_name, kernel_version, distro, arch) = self.vm
            from psvmi import system_info
            sys = system_info(domain_name, kernel_version, distro, arch)
            uptime = int(time.time()) - sys.boottime
            feature_attributes = OSFeature(
                sys.boottime,
                sys.ipaddr,
                sys.osdistro,
                sys.osname,
                sys.osplatform,
                sys.osrelease,
                sys.ostype,
                sys.osversion,
            )
            feature_key = sys.ostype
        else:
            raise NotImplementedError()
        try:
            yield (feature_key, feature_attributes)
        except Exception as e:
            logger.error('Error crawling OS', exc_info=True)
            raise CrawlError(e)
Exemplo n.º 6
0
    def _crawl_packages(self, dbpath=None, root_dir='/'):

        # package attributes: ["installed", "name", "size", "version"]

        (installtime, name, version, size) = (None, None, None, None)

        if self.crawl_mode == Modes.INVM:

            logger.debug('Using in-VM state information (crawl mode: ' +
                         self.crawl_mode + ')')
            system_type = platform.system().lower()
            distro = platform.linux_distribution()[0].lower()
            reload_needed = False
        elif self.crawl_mode == Modes.OUTCONTAINER:

            logger.debug('Using outcontainer state information (crawl mode: ' +
                         self.crawl_mode + ')')

            # XXX assuming containers will always run in linux

            system_type = 'linux'

            # The package manager will be discovered after checking for the
            # existence of /var/lib/dpkg or /ar/lib/rpm

            distro = ''

            reload_needed = True
        elif self.crawl_mode == Modes.MOUNTPOINT:
            logger.debug('Using disk image information (crawl mode: ' +
                         self.crawl_mode + ')')
            system_type = \
                platform_outofband.system(prefix=root_dir).lower()
            distro = platform_outofband.linux_distribution(prefix=root_dir)[
                0].lower()
            reload_needed = False
        else:
            raise NotImplementedError('Unsupported crawl mode')

        installed_since = self.feature_epoch
        if system_type != 'linux':
            # Package feature is only valid for Linux platforms.

            raise StopIteration()
        logger.debug('Crawling Packages')

        pkg_manager = 'unknown'
        if distro in ['ubuntu', 'debian']:
            pkg_manager = 'dpkg'
        elif distro.startswith('red hat') or distro in ['redhat',
                                                        'fedora', 'centos']:
            pkg_manager = 'rpm'
        elif os.path.exists(os.path.join(root_dir, 'var/lib/dpkg')):
            pkg_manager = 'dpkg'
        elif os.path.exists(os.path.join(root_dir, 'var/lib/rpm')):
            pkg_manager = 'rpm'

        try:
            if pkg_manager == 'dpkg':
                if not dbpath:
                    dbpath = 'var/lib/dpkg'
                for (key, feature) in get_dpkg_packages(
                        root_dir, dbpath, installed_since):
                    yield (key, feature)
            elif pkg_manager == 'rpm':
                if not dbpath:
                    dbpath = 'var/lib/rpm'
                for (key, feature) in get_rpm_packages(
                        root_dir, dbpath, installed_since, reload_needed):
                    yield (key, feature)
            else:
                logger.warning('Unsupported package manager for Linux distro')
        except Exception as e:
            logger.error('Error crawling package %s'
                         % ((name if name else 'Unknown')),
                         exc_info=True)
            raise CrawlError(e)