def _crawl_packages(self, dbpath=None, root_dir='/'):
assert(self.crawl_mode is not Modes.OUTCONTAINER)
# package attributes: ["installed", "name", "size", "version"]
(installtime, name, version, size) = (None, None, None, None)
if self.crawl_mode == Modes.INVM:
logger.debug('Using in-VM state information (crawl mode: ' +
self.crawl_mode + ')')
system_type = platform.system().lower()
distro = platform.linux_distribution()[0].lower()
elif self.crawl_mode == Modes.MOUNTPOINT:
logger.debug('Using disk image information (crawl mode: ' +
self.crawl_mode + ')')
system_type = \
platform_outofband.system(prefix=root_dir).lower()
distro = platform_outofband.linux_distribution(prefix=root_dir)[
0].lower()
else:
logger.error('Unsupported crawl mode: ' + self.crawl_mode +
'. Skipping package crawl.')
system_type = 'unknown'
distro = 'unknown'
installed_since = self.feature_epoch
if system_type != 'linux':
# Package feature is only valid for Linux platforms.
raise StopIteration()
logger.debug('Crawling Packages')
pkg_manager = 'unknown'
if distro in ['ubuntu', 'debian']:
pkg_manager = 'dpkg'
elif distro.startswith('red hat') or distro in ['redhat',
'fedora', 'centos']:
pkg_manager = 'rpm'
elif os.path.exists(os.path.join(root_dir, 'var/lib/dpkg')):
pkg_manager = 'dpkg'
elif os.path.exists(os.path.join(root_dir, 'var/lib/rpm')):
pkg_manager = 'rpm'
try:
if pkg_manager == 'dpkg':
if not dbpath:
dbpath = 'var/lib/dpkg'
if os.path.isabs(dbpath):
logger.warning(
'dbpath: ' +
dbpath +
' is defined absolute. Ignoring prefix: ' +
root_dir +
'.')
# Update for a different route.
dbpath = os.path.join(root_dir, dbpath)
if installed_since > 0:
logger.warning(
'dpkg does not provide install-time, defaulting to '
'all packages installed since epoch')
try:
dpkg = subprocess.Popen(['dpkg-query', '-W',
'--admindir={0}'.format(dbpath),
'-f=${Package}|${Version}'
'|${Installed-Size}\n'
], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
dpkglist = dpkg.stdout.read().strip('\n')
except OSError as e:
logger.error(
'Failed to launch dpkg query for packages. Check if '
'dpkg-query is installed: [Errno: %d] ' %
e.errno + e.strerror + ' [Exception: ' +
type(e).__name__ + ']')
dpkglist = None
if dpkglist:
for dpkginfo in dpkglist.split('\n'):
(name, version, size) = dpkginfo.split(r'|')
# dpkg does not provide any installtime field
# feature_key = '{0}/{1}'.format(name, version) -->
# changed to below per Suriya's request
feature_key = '{0}'.format(name, version)
yield (feature_key, PackageFeature(None, name,
size, version))
elif pkg_manager == 'rpm':
if not dbpath:
dbpath = 'var/lib/rpm'
if os.path.isabs(dbpath):
logger.warning(
'dbpath: ' +
dbpath +
' is defined absolute. Ignoring prefix: ' +
root_dir +
'.')
# update for a different route
dbpath = os.path.join(root_dir, dbpath)
try:
rpm = subprocess.Popen([
'rpm',
'--dbpath',
dbpath,
'-qa',
'--queryformat',
'%{installtime}|%{name}|%{version}'
'-%{release}|%{size}\n',
], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
rpmlist = rpm.stdout.read().strip('\n')
except OSError as e:
logger.error(
'Failed to launch rpm query for packages. Check if '
'rpm is installed: [Errno: %d] ' %
e.errno + e.strerror + ' [Exception: ' +
type(e).__name__ + ']')
rpmlist = None
if rpmlist:
for rpminfo in rpmlist.split('\n'):
(installtime, name, version, size) = \
rpminfo.split(r'|')
# if int(installtime) <= installed_since: --> this
# barfs for sth like: 1376416422. Consider try: xxx
# except ValueError: pass
if installtime <= installed_since:
continue
# feature_key = '{0}/{1}'.format(name, version) -->
# changed to below per Suriya's request
feature_key = '{0}'.format(name, version)
yield (feature_key,
PackageFeature(installtime,
name, size, version))
else:
raise CrawlError(
Exception(
'Unsupported package manager for Linux distro %s' %
distro))
except Exception as e:
logger.error('Error crawling package %s'
% ((name if name else 'Unknown')),
exc_info=True)
raise CrawlError(e)
def _crawl_os(self, mountpoint=None):
assert(self.crawl_mode is not Modes.OUTCONTAINER)
logger.debug('Crawling OS')
if self.crawl_mode == Modes.INVM:
logger.debug('Using in-VM state information (crawl mode: ' +
self.crawl_mode + ')')
feature_key = platform.system().lower()
try:
ips = misc.get_host_ip4_addresses()
except Exception as e:
ips = 'unknown'
try:
distro = platform.linux_distribution()[0]
except Exception as e:
distro = 'unknown'
try:
osname = platform.platform()
except Exception as e:
osname = 'unknown'
boot_time = (
psutil.boot_time() if hasattr(
psutil, 'boot_time') else psutil.BOOT_TIME)
uptime = int(time.time()) - boot_time
feature_attributes = OSFeature(
boot_time,
uptime,
ips,
distro,
osname,
platform.machine(),
platform.release(),
platform.system().lower(),
platform.version(),
)
elif self.crawl_mode == Modes.MOUNTPOINT:
logger.debug('Using disk image information (crawl mode: ' +
self.crawl_mode + ')')
feature_key = \
platform_outofband.system(prefix=mountpoint).lower()
feature_attributes = OSFeature( # boot time unknown for img
# live IP unknown for img
'unsupported',
'0.0.0.0',
platform_outofband.linux_distribution(
prefix=mountpoint)[0],
platform_outofband.platform(prefix=mountpoint),
platform_outofband.machine(prefix=mountpoint),
platform_outofband.release(prefix=mountpoint),
platform_outofband.system(prefix=mountpoint).lower(),
platform_outofband.version(prefix=mountpoint),
)
elif self.crawl_mode == Modes.OUTVM:
(domain_name, kernel_version, distro, arch) = self.vm
from psvmi import system_info
sys = system_info(domain_name, kernel_version, distro, arch)
uptime = int(time.time()) - sys.boottime
feature_attributes = OSFeature(
sys.boottime,
sys.ipaddr,
sys.osdistro,
sys.osname,
sys.osplatform,
sys.osrelease,
sys.ostype,
sys.osversion,
)
feature_key = sys.ostype
else:
raise NotImplementedError()
try:
yield (feature_key, feature_attributes)
except Exception as e:
logger.error('Error crawling OS', exc_info=True)
raise CrawlError(e)
def _crawl_packages(self, dbpath=None, root_dir='/'):
# package attributes: ["installed", "name", "size", "version"]
(installtime, name, version, size) = (None, None, None, None)
if self.crawl_mode == Modes.INVM:
logger.debug('Using in-VM state information (crawl mode: ' +
self.crawl_mode + ')')
system_type = platform.system().lower()
distro = platform.linux_distribution()[0].lower()
reload_needed = False
elif self.crawl_mode == Modes.OUTCONTAINER:
logger.debug('Using outcontainer state information (crawl mode: ' +
self.crawl_mode + ')')
system_type = 'linux'
distro = ''
reload_needed = True
elif self.crawl_mode == Modes.MOUNTPOINT:
logger.debug('Using disk image information (crawl mode: ' +
self.crawl_mode + ')')
system_type = \
platform_outofband.system(prefix=root_dir).lower()
distro = platform_outofband.linux_distribution(prefix=root_dir)[
0].lower()
reload_needed = False
else:
logger.error('Unsupported crawl mode: ' + self.crawl_mode +
'. Skipping package crawl.')
system_type = 'unknown'
distro = 'unknown'
reload_needed = True
installed_since = self.feature_epoch
if system_type != 'linux':
# Package feature is only valid for Linux platforms.
raise StopIteration()
logger.debug('Crawling Packages')
pkg_manager = 'unknown'
if distro in ['ubuntu', 'debian']:
pkg_manager = 'dpkg'
elif distro.startswith('red hat') or distro in ['redhat',
'fedora', 'centos']:
pkg_manager = 'rpm'
elif os.path.exists(os.path.join(root_dir, 'var/lib/dpkg')):
pkg_manager = 'dpkg'
elif os.path.exists(os.path.join(root_dir, 'var/lib/rpm')):
pkg_manager = 'rpm'
try:
if pkg_manager == 'dpkg':
if not dbpath:
dbpath = 'var/lib/dpkg'
for (key, feature) in get_dpkg_packages(
root_dir, dbpath, installed_since):
yield (key, feature)
elif pkg_manager == 'rpm':
if not dbpath:
dbpath = 'var/lib/rpm'
for (key, feature) in get_rpm_packages(
root_dir, dbpath, installed_since, reload_needed):
yield (key, feature)
else:
logger.warning('Unsupported package manager for Linux distro')
except Exception as e:
logger.error('Error crawling package %s'
% ((name if name else 'Unknown')),
exc_info=True)
raise CrawlError(e)
def _crawl_packages(self, dbpath=None, root_dir='/'):
assert(self.crawl_mode is not Modes.OUTCONTAINER)
# package attributes: ["installed", "name", "size", "version"]
(installtime, name, version, size) = (None, None, None, None)
if self.crawl_mode == Modes.INVM:
logger.debug('Using in-VM state information (crawl mode: ' +
self.crawl_mode + ')')
system_type = platform.system().lower()
distro = platform.linux_distribution()[0].lower()
elif self.crawl_mode == Modes.MOUNTPOINT:
logger.debug('Using disk image information (crawl mode: ' +
self.crawl_mode + ')')
system_type = \
platform_outofband.system(prefix=root_dir).lower()
distro = platform_outofband.linux_distribution(prefix=root_dir)[
0].lower()
else:
logger.error('Unsupported crawl mode: ' + self.crawl_mode +
'. Skipping package crawl.')
system_type = 'unknown'
distro = 'unknown'
installed_since = self.feature_epoch
if system_type != 'linux' or (system_type == 'linux' and distro == ''):
# Distro is blank for FROM scratch images
# Package feature is only valid for Linux platforms.
raise StopIteration()
logger.debug('Crawling Packages')
pkg_manager = 'unknown'
if distro in ['ubuntu', 'debian']:
pkg_manager = 'dpkg'
elif distro.startswith('red hat') or distro in ['redhat',
'fedora', 'centos']:
pkg_manager = 'rpm'
elif os.path.exists(os.path.join(root_dir, 'var/lib/dpkg')):
pkg_manager = 'dpkg'
elif os.path.exists(os.path.join(root_dir, 'var/lib/rpm')):
pkg_manager = 'rpm'
try:
if pkg_manager == 'dpkg':
if not dbpath:
dbpath = 'var/lib/dpkg'
if os.path.isabs(dbpath):
logger.warning(
'dbpath: ' +
dbpath +
' is defined absolute. Ignoring prefix: ' +
root_dir +
'.')
# Update for a different route.
dbpath = os.path.join(root_dir, dbpath)
if installed_since > 0:
logger.warning(
'dpkg does not provide install-time, defaulting to '
'all packages installed since epoch')
try:
dpkg = subprocess.Popen(['dpkg-query', '-W',
'--admindir={0}'.format(dbpath),
'-f=${Package}|${Version}'
'|${Installed-Size}\n'
], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
dpkglist = dpkg.stdout.read().strip('\n')
except OSError as e:
logger.error(
'Failed to launch dpkg query for packages. Check if '
'dpkg-query is installed: [Errno: %d] ' %
e.errno + e.strerror + ' [Exception: ' +
type(e).__name__ + ']')
dpkglist = None
if dpkglist:
for dpkginfo in dpkglist.split('\n'):
(name, version, size) = dpkginfo.split(r'|')
# dpkg does not provide any installtime field
# feature_key = '{0}/{1}'.format(name, version) -->
# changed to below per Suriya's request
feature_key = '{0}'.format(name, version)
yield (feature_key, PackageFeature(None, name,
size, version))
elif pkg_manager == 'rpm':
if not dbpath:
dbpath = 'var/lib/rpm'
if os.path.isabs(dbpath):
logger.warning(
'dbpath: ' +
dbpath +
' is defined absolute. Ignoring prefix: ' +
root_dir +
'.')
# update for a different route
dbpath = os.path.join(root_dir, dbpath)
try:
rpm = subprocess.Popen([
'rpm',
'--dbpath',
dbpath,
'-qa',
'--queryformat',
'%{installtime}|%{name}|%{version}'
'-%{release}|%{size}\n',
], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
rpmlist = rpm.stdout.read().strip('\n')
except OSError as e:
logger.error(
'Failed to launch rpm query for packages. Check if '
'rpm is installed: [Errno: %d] ' %
e.errno + e.strerror + ' [Exception: ' +
type(e).__name__ + ']')
rpmlist = None
if rpmlist:
for rpminfo in rpmlist.split('\n'):
(installtime, name, version, size) = \
rpminfo.split(r'|')
# if int(installtime) <= installed_since: --> this
# barfs for sth like: 1376416422. Consider try: xxx
# except ValueError: pass
if installtime <= installed_since:
continue
# feature_key = '{0}/{1}'.format(name, version) -->
# changed to below per Suriya's request
feature_key = '{0}'.format(name, version)
yield (feature_key,
PackageFeature(installtime,
name, size, version))
else:
raise CrawlError(
Exception(
'Unsupported package manager for Linux distro %s' %
distro))
except Exception as e:
logger.error('Error crawling package %s'
% ((name if name else 'Unknown')),
exc_info=True)
raise CrawlError(e)
def _crawl_os(self, mountpoint=None):
assert(self.crawl_mode is not Modes.OUTCONTAINER)
logger.debug('Crawling OS')
if self.crawl_mode == Modes.INVM:
logger.debug('Using in-VM state information (crawl mode: ' +
self.crawl_mode + ')')
feature_key = platform.system().lower()
try:
ips = misc.get_host_ip4_addresses()
except Exception as e:
ips = 'unknown'
try:
distro = platform.linux_distribution()[0]
except Exception as e:
distro = 'unknown'
try:
osname = platform.platform()
except Exception as e:
osname = 'unknown'
boot_time = (
psutil.boot_time() if hasattr(
psutil, 'boot_time') else psutil.BOOT_TIME)
uptime = int(time.time()) - boot_time
feature_attributes = OSFeature(
boot_time,
uptime,
ips,
distro,
osname,
platform.machine(),
platform.release(),
platform.system().lower(),
platform.version(),
)
elif self.crawl_mode == Modes.MOUNTPOINT:
logger.debug('Using disk image information (crawl mode: ' +
self.crawl_mode + ')')
feature_key = \
platform_outofband.system(prefix=mountpoint).lower()
feature_attributes = OSFeature( # boot time unknown for img
# live IP unknown for img
'unsupported',
'0.0.0.0',
platform_outofband.linux_distribution(
prefix=mountpoint)[0],
platform_outofband.platform(prefix=mountpoint),
platform_outofband.machine(prefix=mountpoint),
platform_outofband.release(prefix=mountpoint),
platform_outofband.system(prefix=mountpoint).lower(),
platform_outofband.version(prefix=mountpoint),
)
elif self.crawl_mode == Modes.OUTVM:
(domain_name, kernel_version, distro, arch) = self.vm
from psvmi import system_info
sys = system_info(domain_name, kernel_version, distro, arch)
uptime = int(time.time()) - sys.boottime
feature_attributes = OSFeature(
sys.boottime,
sys.ipaddr,
sys.osdistro,
sys.osname,
sys.osplatform,
sys.osrelease,
sys.ostype,
sys.osversion,
)
feature_key = sys.ostype
else:
raise NotImplementedError()
try:
yield (feature_key, feature_attributes)
except Exception as e:
logger.error('Error crawling OS', exc_info=True)
raise CrawlError(e)
def _crawl_packages(self, dbpath=None, root_dir='/'):
# package attributes: ["installed", "name", "size", "version"]
(installtime, name, version, size) = (None, None, None, None)
if self.crawl_mode == Modes.INVM:
logger.debug('Using in-VM state information (crawl mode: ' +
self.crawl_mode + ')')
system_type = platform.system().lower()
distro = platform.linux_distribution()[0].lower()
reload_needed = False
elif self.crawl_mode == Modes.OUTCONTAINER:
logger.debug('Using outcontainer state information (crawl mode: ' +
self.crawl_mode + ')')
# XXX assuming containers will always run in linux
system_type = 'linux'
# The package manager will be discovered after checking for the
# existence of /var/lib/dpkg or /ar/lib/rpm
distro = ''
reload_needed = True
elif self.crawl_mode == Modes.MOUNTPOINT:
logger.debug('Using disk image information (crawl mode: ' +
self.crawl_mode + ')')
system_type = \
platform_outofband.system(prefix=root_dir).lower()
distro = platform_outofband.linux_distribution(prefix=root_dir)[
0].lower()
reload_needed = False
else:
raise NotImplementedError('Unsupported crawl mode')
installed_since = self.feature_epoch
if system_type != 'linux':
# Package feature is only valid for Linux platforms.
raise StopIteration()
logger.debug('Crawling Packages')
pkg_manager = 'unknown'
if distro in ['ubuntu', 'debian']:
pkg_manager = 'dpkg'
elif distro.startswith('red hat') or distro in ['redhat',
'fedora', 'centos']:
pkg_manager = 'rpm'
elif os.path.exists(os.path.join(root_dir, 'var/lib/dpkg')):
pkg_manager = 'dpkg'
elif os.path.exists(os.path.join(root_dir, 'var/lib/rpm')):
pkg_manager = 'rpm'
try:
if pkg_manager == 'dpkg':
if not dbpath:
dbpath = 'var/lib/dpkg'
for (key, feature) in get_dpkg_packages(
root_dir, dbpath, installed_since):
yield (key, feature)
elif pkg_manager == 'rpm':
if not dbpath:
dbpath = 'var/lib/rpm'
for (key, feature) in get_rpm_packages(
root_dir, dbpath, installed_since, reload_needed):
yield (key, feature)
else:
logger.warning('Unsupported package manager for Linux distro')
except Exception as e:
logger.error('Error crawling package %s'
% ((name if name else 'Unknown')),
exc_info=True)
raise CrawlError(e)
