def apt(b, r):
logging.info('searching for APT packages')
# Try for the full list of packages. If this fails, don't even
# bother with the rest because this is probably a Yum/RPM-based
# system.
try:
p = subprocess.Popen(['dpkg-query',
'-f=${Status}\x1E${Package}\x1E${Version}\n',
'-W'],
close_fds=True, stdout=subprocess.PIPE)
except OSError:
return
for line in p.stdout:
status, package, version = line.strip().split('\x1E')
if 'install ok installed' != status:
continue
if r.ignore_package('apt', package):
continue
b.add_package('apt', package, version)
# Create service resources for each service init script or config
# found in this package.
p = subprocess.Popen(['dpkg-query', '-L', package],
close_fds=True, stdout=subprocess.PIPE)
for line in p.stdout:
try:
manager, service = util.parse_service(line.rstrip())
if not r.ignore_service(manager, service):
b.add_service(manager, service)
b.add_service_package(manager, service, 'apt', package)
except ValueError:
pass
def apt(b, r):
logging.info('searching for APT packages')
# Define a default output format string for dpkg-query.
output_format = '${Status}\x1E${binary:Package}\x1E${Version}\n'
# Try running dpkg --print-foreign-architectures to see if dpkg is
# multi-arch aware. If not, revert to old style output_format.
try:
with open(os.devnull, 'w') as fnull:
rv = subprocess.call(['dpkg', '--print-foreign-architectures'],
stdout=fnull,
stderr=fnull)
if rv != 0:
output_format = '${Status}\x1E${Package}\x1E${Version}\n'
except OSError:
return
# Try for the full list of packages. If this fails, don't even
# bother with the rest because this is probably a Yum/RPM-based
# system.
try:
p = subprocess.Popen(['dpkg-query', '-Wf', output_format],
close_fds=True,
stdout=subprocess.PIPE)
except OSError:
return
for line in p.stdout:
status, package, version = line.strip().split('\x1E')
if 'install ok installed' != status:
continue
if r.ignore_package('apt', package):
continue
b.add_package('apt', package, version)
# Create service resources for each service init script or config
# found in this package.
p = subprocess.Popen(['dpkg-query', '-L', package],
close_fds=True,
stdout=subprocess.PIPE)
for line in p.stdout:
try:
manager, service = util.parse_service(line.rstrip())
if not r.ignore_service(manager, service):
b.add_service(manager, service)
b.add_service_package(manager, service, 'apt', package)
except ValueError:
pass
def apt(b, r):
logging.info('searching for APT packages')
# Define a default output format string for dpkg-query.
output_format = '${Status}\x1E${binary:Package}\x1E${Version}\n'
# Try running dpkg --print-foreign-architectures to see if dpkg is
# multi-arch aware. If not, revert to old style output_format.
try:
with open(os.devnull, 'w') as fnull:
rv = subprocess.call(['dpkg', '--print-foreign-architectures'],
stdout = fnull, stderr = fnull)
if rv != 0:
output_format = '${Status}\x1E${Package}\x1E${Version}\n'
except OSError:
return
# Try for the full list of packages. If this fails, don't even
# bother with the rest because this is probably a Yum/RPM-based
# system.
try:
p = subprocess.Popen(['dpkg-query','-Wf', output_format],
close_fds=True, stdout=subprocess.PIPE)
except OSError:
return
for line in p.stdout:
status, package, version = line.strip().split('\x1E')
if 'install ok installed' != status:
continue
if r.ignore_package('apt', package):
continue
b.add_package('apt', package, version)
# Create service resources for each service init script or config
# found in this package.
p = subprocess.Popen(['dpkg-query', '-L', package],
close_fds=True, stdout=subprocess.PIPE)
for line in p.stdout:
try:
manager, service = util.parse_service(line.rstrip())
if not r.ignore_service(manager, service):
b.add_service(manager, service)
b.add_service_package(manager, service, 'apt', package)
except ValueError:
pass
def yum(b, r):
logging.info('searching for Yum packages')
# Try for the full list of packages. If this fails, don't even
# bother with the rest because this is probably a Debian-based
# system.
try:
p = subprocess.Popen(
[
'rpm',
'--qf=%{NAME}\x1E%{GROUP}\x1E%{EPOCH}' # No ,
'\x1E%{VERSION}-%{RELEASE}\x1E%{ARCH}\n',
'-qa'
],
close_fds=True,
stdout=subprocess.PIPE)
except OSError:
return
for line in p.stdout:
package, group, epoch, version, arch = line.strip().split('\x1E')
if r.ignore_package('yum', package):
continue
if '(none)' != epoch:
version = '{0}:{1}'.format(epoch, version)
if '(none)' != arch:
version = '{0}.{1}'.format(version, arch)
b.add_package('yum', package, version)
# Create service resources for each service init script or config
# in this package.
p = subprocess.Popen(['rpm', '-ql', package],
close_fds=True,
stdout=subprocess.PIPE)
for line in p.stdout:
try:
manager, service = util.parse_service(line.rstrip())
if not r.ignore_service(manager, service):
b.add_service(manager, service)
b.add_service_package(manager, service, 'yum', package)
except ValueError:
pass
def yum(b, r):
logging.info('searching for Yum packages')
# Try for the full list of packages. If this fails, don't even
# bother with the rest because this is probably a Debian-based
# system.
try:
p = subprocess.Popen(['rpm',
'--qf=%{NAME}\x1E%{GROUP}\x1E%{EPOCH}' # No ,
'\x1E%{VERSION}-%{RELEASE}\x1E%{ARCH}\n',
'-qa'],
close_fds=True, stdout=subprocess.PIPE)
except OSError:
return
for line in p.stdout:
package, group, epoch, version, arch = line.strip().split('\x1E')
if r.ignore_package('yum', package):
continue
if '(none)' != epoch:
version = '{0}:{1}'.format(epoch, version)
if '(none)' != arch:
version = '{0}.{1}'.format(version, arch)
b.add_package('yum', package, version)
# Create service resources for each service init script or config
# in this package.
p = subprocess.Popen(['rpm', '-ql', package],
close_fds=True, stdout=subprocess.PIPE)
for line in p.stdout:
try:
manager, service = util.parse_service(line.rstrip())
if not r.ignore_service(manager, service):
b.add_service(manager, service)
b.add_service_package(manager, service, 'yum', package)
except ValueError:
pass
def apt(b, r):
logging.info('searching for APT packages')
# Try for the full list of packages. If this fails, don't even
# bother with the rest because this is probably a Yum/RPM-based
# system.
try:
p = subprocess.Popen(
['dpkg-query', '-f=${Status}\x1E${Package}\x1E${Version}\n', '-W'],
close_fds=True,
stdout=subprocess.PIPE)
except OSError:
return
for line in p.stdout:
status, package, version = line.strip().split('\x1E')
if 'install ok installed' != status:
continue
if r.ignore_package('apt', package):
continue
b.add_package('apt', package, version)
# Create service resources for each service init script or config
# found in this package.
p = subprocess.Popen(['dpkg-query', '-L', package],
close_fds=True,
stdout=subprocess.PIPE)
for line in p.stdout:
try:
manager, service = util.parse_service(line.rstrip())
if not r.ignore_service(manager, service):
b.add_service(manager, service)
b.add_service_package(manager, service, 'apt', package)
except ValueError:
pass
def files(b):
logging.info('searching for configuration files')
# Visit every file in `/etc` except those on the exclusion list above.
for dirpath, dirnames, filenames in os.walk('/etc'):
# Determine if this entire directory should be ignored by default.
ignored = ignore.file(dirpath)
# Collect up the full pathname to each file, `lstat` them all, and
# note which ones will probably be ignored.
files = []
for filename in filenames:
pathname = os.path.join(dirpath, filename)
try:
files.append((pathname, os.lstat(pathname),
ignore.file(pathname, ignored)))
except OSError as e:
logging.warning('{0} caused {1} - try running as root'.format(
pathname, errno.errorcode[e.errno]))
# Track the ctime of each file in this directory. Weed out false
# positives by ignoring files with common ctimes.
ctimes = defaultdict(lambda: 0)
# Map the ctimes of each directory entry that isn't being ignored.
for pathname, s, ignored in files:
if not ignored:
ctimes[s.st_ctime] += 1
for dirname in dirnames:
try:
ctimes[os.lstat(os.path.join(dirpath, dirname)).st_ctime] += 1
except OSError:
pass
for pathname, s, ignored in files:
# Always ignore block special files, character special files,
# pipes, and sockets. They end up looking like deadlocks.
if stat.S_ISBLK(s.st_mode) \
or stat.S_ISCHR(s.st_mode) \
or stat.S_ISFIFO(s.st_mode) \
or stat.S_ISSOCK(s.st_mode):
continue
# Make sure this pathname will actually be able to be included
# in the blueprint. This is a bit of a cop-out since the file
# could be important but at least it's not a crashing bug.
try:
pathname = unicode(pathname)
except UnicodeDecodeError:
logging.warning('{0} not UTF-8 - skipping it'.format(
repr(pathname)[1:-1]))
continue
# Ignore ignored files and files that share their ctime with other
# files in the directory. This is a very strong indication that
# the file is original to the system and should be ignored.
if ignored or 1 < ctimes[s.st_ctime] and ignore.file(
pathname, True):
continue
# Check for a Mustache template and an optional shell script
# that templatize this file.
try:
template = open(
'{0}.blueprint-template.mustache'.format(pathname)).read()
except IOError:
template = None
try:
data = open(
'{0}.blueprint-template.sh'.format(pathname)).read()
except IOError:
data = None
# The content is used even for symbolic links to determine whether
# it has changed from the packaged version.
try:
content = open(pathname).read()
except IOError:
#logging.warning('{0} not readable'.format(pathname))
continue
# Ignore files that are unchanged from their packaged version.
if _unchanged(pathname, content):
continue
# Resolve the rest of the file's metadata from the
# `/etc/passwd` and `/etc/group` databases.
try:
pw = pwd.getpwuid(s.st_uid)
owner = pw.pw_name
except KeyError:
owner = s.st_uid
try:
gr = grp.getgrgid(s.st_gid)
group = gr.gr_name
except KeyError:
group = s.st_gid
mode = '{0:o}'.format(s.st_mode)
# A symbolic link's content is the link target.
if stat.S_ISLNK(s.st_mode):
content = os.readlink(pathname)
# Ignore symbolic links providing backwards compatibility
# between SystemV init and Upstart.
if '/lib/init/upstart-job' == content:
continue
# Ignore symbolic links into the Debian alternatives system.
# These are almost certainly managed by packages.
if content.startswith('/etc/alternatives/'):
continue
b.add_file(pathname,
content=content,
encoding='plain',
group=group,
mode=mode,
owner=owner)
# A regular file is stored as plain text only if it is valid
# UTF-8, which is required for JSON serialization.
else:
kwargs = dict(group=group, mode=mode, owner=owner)
try:
if template:
if data:
kwargs['data'] = data.decode('utf_8')
kwargs['template'] = template.decode('utf_8')
else:
kwargs['content'] = content.decode('utf_8')
kwargs['encoding'] = 'plain'
except UnicodeDecodeError:
if template:
if data:
kwargs['data'] = base64.b64encode(data)
kwargs['template'] = base64.b64encode(template)
else:
kwargs['content'] = base64.b64encode(content)
kwargs['encoding'] = 'base64'
b.add_file(pathname, **kwargs)
# If this file is a service init script or config , create a
# service resource.
try:
manager, service = util.parse_service(pathname)
if not ignore.service(manager, service):
b.add_service(manager, service)
b.add_service_package(manager, service, 'apt',
*_dpkg_query_S(pathname))
b.add_service_package(manager, service, 'yum',
*_rpm_qf(pathname))
except ValueError:
pass
def files(b):
logging.info('searching for configuration files')
# Visit every file in `/etc` except those on the exclusion list above.
for dirpath, dirnames, filenames in os.walk('/etc'):
# Determine if this entire directory should be ignored by default.
ignored = ignore.file(dirpath)
# Collect up the full pathname to each file, `lstat` them all, and
# note which ones will probably be ignored.
files = []
for filename in filenames:
pathname = os.path.join(dirpath, filename)
try:
files.append((pathname,
os.lstat(pathname),
ignore.file(pathname, ignored)))
except OSError as e:
logging.warning('{0} caused {1} - try running as root'.
format(pathname, errno.errorcode[e.errno]))
# Track the ctime of each file in this directory. Weed out false
# positives by ignoring files with common ctimes.
ctimes = defaultdict(lambda: 0)
# Map the ctimes of each directory entry that isn't being ignored.
for pathname, s, ignored in files:
if not ignored:
ctimes[s.st_ctime] += 1
for dirname in dirnames:
try:
ctimes[os.lstat(os.path.join(dirpath, dirname)).st_ctime] += 1
except OSError:
pass
for pathname, s, ignored in files:
# Make sure this pathname will actually be able to be included
# in the blueprint. This is a bit of a cop-out since the file
# could be important but at least it's not a crashing bug.
try:
pathname = unicode(pathname)
except UnicodeDecodeError:
logging.warning('{0} not UTF-8 - skipping it'.
format(repr(pathname)[1:-1]))
continue
# Ignore ignored files and files that share their ctime with other
# files in the directory. This is a very strong indication that
# the file is original to the system and should be ignored.
if ignored or 1 < ctimes[s.st_ctime] and ignore.file(pathname,
True):
continue
# The content is used even for symbolic links to determine whether
# it has changed from the packaged version.
try:
content = open(pathname).read()
except IOError:
#logging.warning('{0} not readable'.format(pathname))
continue
# Ignore files that are from the `base-files` package (which
# doesn't include MD5 sums for every file for some reason).
apt_packages = _dpkg_query_S(pathname)
if 'base-files' in apt_packages:
continue
# Ignore files that are unchanged from their packaged version,
# or match in MD5SUMS.
md5sums = MD5SUMS.get(pathname, [])
md5sums.extend([_dpkg_md5sum(package, pathname)
for package in apt_packages])
md5sum = _rpm_md5sum(pathname)
if md5sum is not None:
md5sums.append(md5sum)
if (hashlib.md5(content).hexdigest() in md5sums \
or 64 in [len(md5sum or '') for md5sum in md5sums] \
and hashlib.sha256(content).hexdigest() in md5sums) \
and ignore.file(pathname, True):
continue
# A symbolic link's content is the link target.
if stat.S_ISLNK(s.st_mode):
content = os.readlink(pathname)
# Ignore symbolic links providing backwards compatibility
# between SystemV init and Upstart.
if '/lib/init/upstart-job' == content:
continue
# Ignore symbolic links into the Debian alternatives system.
# These are almost certainly managed by packages.
if content.startswith('/etc/alternatives/'):
continue
encoding = 'plain'
# A regular file is stored as plain text only if it is valid
# UTF-8, which is required for JSON serialization.
elif stat.S_ISREG(s.st_mode):
try:
content = content.decode('utf_8')
encoding = 'plain'
except UnicodeDecodeError:
content = base64.b64encode(content)
encoding = 'base64'
# Other types, like FIFOs and sockets are not supported within
# a blueprint and really shouldn't appear in `/etc` at all.
else:
logging.warning('{0} is not a regular file or symbolic link'.
format(pathname))
continue
try:
pw = pwd.getpwuid(s.st_uid)
owner = pw.pw_name
except KeyError:
owner = s.st_uid
try:
gr = grp.getgrgid(s.st_gid)
group = gr.gr_name
except KeyError:
group = s.st_gid
b.add_file(pathname,
content=content,
encoding=encoding,
group=group,
mode='{0:o}'.format(s.st_mode),
owner=owner)
# If this file is a service init script or config , create a
# service resource.
try:
manager, service = util.parse_service(pathname)
if not ignore.service(manager, service):
b.add_service(manager, service)
b.add_service_package(manager,
service,
'apt',
*apt_packages)
b.add_service_package(manager,
service,
'yum',
*_rpm_qf(pathname))
except ValueError:
pass
def files(b):
logging.info('searching for configuration files')
# Visit every file in `/etc` except those on the exclusion list above.
for dirpath, dirnames, filenames in os.walk('/etc'):
# Determine if this entire directory should be ignored by default.
ignored = ignore.file(dirpath)
# Collect up the full pathname to each file, `lstat` them all, and
# note which ones will probably be ignored.
files = []
for filename in filenames:
pathname = os.path.join(dirpath, filename)
try:
files.append((pathname,
os.lstat(pathname),
ignore.file(pathname, ignored)))
except OSError as e:
logging.warning('{0} caused {1} - try running as root'.
format(pathname, errno.errorcode[e.errno]))
# Track the ctime of each file in this directory. Weed out false
# positives by ignoring files with common ctimes.
ctimes = defaultdict(lambda: 0)
# Map the ctimes of each directory entry that isn't being ignored.
for pathname, s, ignored in files:
if not ignored:
ctimes[s.st_ctime] += 1
for dirname in dirnames:
try:
ctimes[os.lstat(os.path.join(dirpath, dirname)).st_ctime] += 1
except OSError:
pass
for pathname, s, ignored in files:
# Always ignore block special files, character special files,
# pipes, and sockets. They end up looking like deadlocks.
if stat.S_ISBLK(s.st_mode) \
or stat.S_ISCHR(s.st_mode) \
or stat.S_ISFIFO(s.st_mode) \
or stat.S_ISSOCK(s.st_mode):
continue
# Make sure this pathname will actually be able to be included
# in the blueprint. This is a bit of a cop-out since the file
# could be important but at least it's not a crashing bug.
try:
pathname = unicode(pathname)
except UnicodeDecodeError:
logging.warning('{0} not UTF-8 - skipping it'.
format(repr(pathname)[1:-1]))
continue
# Ignore ignored files and files that share their ctime with other
# files in the directory. This is a very strong indication that
# the file is original to the system and should be ignored.
if ignored or 1 < ctimes[s.st_ctime] and ignore.file(pathname,
True):
continue
# Check for a Mustache template and an optional shell script
# that templatize this file.
try:
template = open(
'{0}.blueprint-template.mustache'.format(pathname)).read()
except IOError:
template = None
try:
data = open(
'{0}.blueprint-template.sh'.format(pathname)).read()
except IOError:
data = None
# The content is used even for symbolic links to determine whether
# it has changed from the packaged version.
try:
content = open(pathname).read()
except IOError:
#logging.warning('{0} not readable'.format(pathname))
continue
# Ignore files that are unchanged from their packaged version.
if _unchanged(pathname, content):
continue
# Resolve the rest of the file's metadata from the
# `/etc/passwd` and `/etc/group` databases.
try:
pw = pwd.getpwuid(s.st_uid)
owner = pw.pw_name
except KeyError:
owner = s.st_uid
try:
gr = grp.getgrgid(s.st_gid)
group = gr.gr_name
except KeyError:
group = s.st_gid
mode = '{0:o}'.format(s.st_mode)
# A symbolic link's content is the link target.
if stat.S_ISLNK(s.st_mode):
content = os.readlink(pathname)
# Ignore symbolic links providing backwards compatibility
# between SystemV init and Upstart.
if '/lib/init/upstart-job' == content:
continue
# Ignore symbolic links into the Debian alternatives system.
# These are almost certainly managed by packages.
if content.startswith('/etc/alternatives/'):
continue
b.add_file(pathname,
content=content,
encoding='plain',
group=group,
mode=mode,
owner=owner)
# A regular file is stored as plain text only if it is valid
# UTF-8, which is required for JSON serialization.
else:
kwargs = dict(group=group,
mode=mode,
owner=owner)
try:
if template:
if data:
kwargs['data'] = data.decode('utf_8')
kwargs['template'] = template.decode('utf_8')
else:
kwargs['content'] = content.decode('utf_8')
kwargs['encoding'] = 'plain'
except UnicodeDecodeError:
if template:
if data:
kwargs['data'] = base64.b64encode(data)
kwargs['template'] = base64.b64encode(template)
else:
kwargs['content'] = base64.b64encode(content)
kwargs['encoding'] = 'base64'
b.add_file(pathname, **kwargs)
# If this file is a service init script or config , create a
# service resource.
try:
manager, service = util.parse_service(pathname)
if not ignore.service(manager, service):
b.add_service(manager, service)
b.add_service_package(manager,
service,
'apt',
*_dpkg_query_S(pathname))
b.add_service_package(manager,
service,
'yum',
*_rpm_qf(pathname))
except ValueError:
pass