Exemple #1
0
def apt(b):
    logging.info('searching for APT packages')

    # Try for the full list of packages.  If this fails, don't even
    # bother with the rest because this is probably a Yum/RPM-based
    # system.
    try:
        p = subprocess.Popen(['dpkg-query',
                              '-f=${Status}\x1E${Package}\x1E${Version}\n',
                              '-W'],
                             close_fds=True, stdout=subprocess.PIPE)
    except OSError:
        return

    for line in p.stdout:
        status, package, version = line.strip().split('\x1E')
        if 'install ok installed' != status:
            continue
        if ignore.package('apt', package):
            continue

        b.add_package('apt', package, version)

        # Create service resources for each service init script or config
        # found in this package.
        p = subprocess.Popen(['dpkg-query', '-L', package],
                             close_fds=True, stdout=subprocess.PIPE)
        for line in p.stdout:
            try:
                manager, service = util.parse_service(line.rstrip())
                if not ignore.service(manager, service):
                    b.add_service(manager, service)
                    b.add_service_package(manager, service, 'apt', package)
            except ValueError:
                pass
Exemple #2
0
def yum(b):
    logging.info('searching for Yum packages')

    # Try for the full list of packages.  If this fails, don't even
    # bother with the rest because this is probably a Debian-based
    # system.
    try:
        p = subprocess.Popen(
            [
                'rpm',
                '--qf=%{NAME}\x1E%{GROUP}\x1E%{EPOCH}'  # No ,
                '\x1E%{VERSION}-%{RELEASE}\x1E%{ARCH}\n',
                '-qa'
            ],
            close_fds=True,
            stdout=subprocess.PIPE)
    except OSError:
        return

    for line in p.stdout:
        package, group, epoch, version, arch = line.strip().split('\x1E')
        if ignore.package('yum', package):
            continue

        if '(none)' != epoch:
            version = '{0}:{1}'.format(epoch, version)
        if '(none)' != arch:
            version = '{0}.{1}'.format(version, arch)
        b.add_package('yum', package, version)

        # Create service resources for each service init script or config
        # in this package.
        p = subprocess.Popen(['rpm', '-ql', package],
                             close_fds=True,
                             stdout=subprocess.PIPE)
        for line in p.stdout:
            try:
                manager, service = util.parse_service(line.rstrip())
                if not ignore.service(manager, service):
                    b.add_service(manager, service)
                    b.add_service_package(manager, service, 'yum', package)
            except ValueError:
                pass
Exemple #3
0
def yum(b):
    logging.info('searching for Yum packages')

    # Try for the full list of packages.  If this fails, don't even
    # bother with the rest because this is probably a Debian-based
    # system.
    try:
        p = subprocess.Popen(['rpm',
                              '--qf=%{NAME}\x1E%{GROUP}\x1E%{EPOCH}' # No ,
                              '\x1E%{VERSION}-%{RELEASE}\x1E%{ARCH}\n',
                              '-qa'],
                             close_fds=True, stdout=subprocess.PIPE)
    except OSError:
        return

    for line in p.stdout:
        package, group, epoch, version, arch = line.strip().split('\x1E')
        if ignore.package('yum', package):
            continue

        if '(none)' != epoch:
            version = '{0}:{1}'.format(epoch, version)
        if '(none)' != arch:
            version = '{0}.{1}'.format(version, arch)
        b.add_package('yum', package, version)

        # Create service resources for each service init script or config
        # in this package.
        p = subprocess.Popen(['rpm', '-ql', package],
                             close_fds=True, stdout=subprocess.PIPE)
        for line in p.stdout:
            try:
                manager, service = util.parse_service(line.rstrip())
                if not ignore.service(manager, service):
                    b.add_service(manager, service)
                    b.add_service_package(manager, service, 'yum', package)
            except ValueError:
                pass
Exemple #4
0
def apt(b):
    logging.info('searching for APT packages')

    # Try for the full list of packages.  If this fails, don't even
    # bother with the rest because this is probably a Yum/RPM-based
    # system.
    try:
        p = subprocess.Popen(
            ['dpkg-query', '-f=${Status}\x1E${Package}\x1E${Version}\n', '-W'],
            close_fds=True,
            stdout=subprocess.PIPE)
    except OSError:
        return

    for line in p.stdout:
        status, package, version = line.strip().split('\x1E')
        if 'install ok installed' != status:
            continue
        if ignore.package('apt', package):
            continue

        b.add_package('apt', package, version)

        # Create service resources for each service init script or config
        # found in this package.
        p = subprocess.Popen(['dpkg-query', '-L', package],
                             close_fds=True,
                             stdout=subprocess.PIPE)
        for line in p.stdout:
            try:
                manager, service = util.parse_service(line.rstrip())
                if not ignore.service(manager, service):
                    b.add_service(manager, service)
                    b.add_service_package(manager, service, 'apt', package)
            except ValueError:
                pass
Exemple #5
0
def files(b):
    logging.info('searching for configuration files')

    # Visit every file in `/etc` except those on the exclusion list above.
    for dirpath, dirnames, filenames in os.walk('/etc'):

        # Determine if this entire directory should be ignored by default.
        ignored = ignore.file(dirpath)

        # Collect up the full pathname to each file, `lstat` them all, and
        # note which ones will probably be ignored.
        files = []
        for filename in filenames:
            pathname = os.path.join(dirpath, filename)
            try:
                files.append((pathname, os.lstat(pathname),
                              ignore.file(pathname, ignored)))
            except OSError as e:
                logging.warning('{0} caused {1} - try running as root'.format(
                    pathname, errno.errorcode[e.errno]))

        # Track the ctime of each file in this directory.  Weed out false
        # positives by ignoring files with common ctimes.
        ctimes = defaultdict(lambda: 0)

        # Map the ctimes of each directory entry that isn't being ignored.
        for pathname, s, ignored in files:
            if not ignored:
                ctimes[s.st_ctime] += 1
        for dirname in dirnames:
            try:
                ctimes[os.lstat(os.path.join(dirpath, dirname)).st_ctime] += 1
            except OSError:
                pass

        for pathname, s, ignored in files:

            # Always ignore block special files, character special files,
            # pipes, and sockets.  They end up looking like deadlocks.
            if stat.S_ISBLK(s.st_mode) \
            or stat.S_ISCHR(s.st_mode) \
            or stat.S_ISFIFO(s.st_mode) \
            or stat.S_ISSOCK(s.st_mode):
                continue

            # Make sure this pathname will actually be able to be included
            # in the blueprint.  This is a bit of a cop-out since the file
            # could be important but at least it's not a crashing bug.
            try:
                pathname = unicode(pathname)
            except UnicodeDecodeError:
                logging.warning('{0} not UTF-8 - skipping it'.format(
                    repr(pathname)[1:-1]))
                continue

            # Ignore ignored files and files that share their ctime with other
            # files in the directory.  This is a very strong indication that
            # the file is original to the system and should be ignored.
            if ignored or 1 < ctimes[s.st_ctime] and ignore.file(
                    pathname, True):
                continue

            # Check for a Mustache template and an optional shell script
            # that templatize this file.
            try:
                template = open(
                    '{0}.blueprint-template.mustache'.format(pathname)).read()
            except IOError:
                template = None
            try:
                data = open(
                    '{0}.blueprint-template.sh'.format(pathname)).read()
            except IOError:
                data = None

            # The content is used even for symbolic links to determine whether
            # it has changed from the packaged version.
            try:
                content = open(pathname).read()
            except IOError:
                #logging.warning('{0} not readable'.format(pathname))
                continue

            # Ignore files that are unchanged from their packaged version.
            if _unchanged(pathname, content):
                continue

            # Resolve the rest of the file's metadata from the
            # `/etc/passwd` and `/etc/group` databases.
            try:
                pw = pwd.getpwuid(s.st_uid)
                owner = pw.pw_name
            except KeyError:
                owner = s.st_uid
            try:
                gr = grp.getgrgid(s.st_gid)
                group = gr.gr_name
            except KeyError:
                group = s.st_gid
            mode = '{0:o}'.format(s.st_mode)

            # A symbolic link's content is the link target.
            if stat.S_ISLNK(s.st_mode):
                content = os.readlink(pathname)

                # Ignore symbolic links providing backwards compatibility
                # between SystemV init and Upstart.
                if '/lib/init/upstart-job' == content:
                    continue

                # Ignore symbolic links into the Debian alternatives system.
                # These are almost certainly managed by packages.
                if content.startswith('/etc/alternatives/'):
                    continue

                b.add_file(pathname,
                           content=content,
                           encoding='plain',
                           group=group,
                           mode=mode,
                           owner=owner)

            # A regular file is stored as plain text only if it is valid
            # UTF-8, which is required for JSON serialization.
            else:
                kwargs = dict(group=group, mode=mode, owner=owner)
                try:
                    if template:
                        if data:
                            kwargs['data'] = data.decode('utf_8')
                        kwargs['template'] = template.decode('utf_8')
                    else:
                        kwargs['content'] = content.decode('utf_8')
                    kwargs['encoding'] = 'plain'
                except UnicodeDecodeError:
                    if template:
                        if data:
                            kwargs['data'] = base64.b64encode(data)
                        kwargs['template'] = base64.b64encode(template)
                    else:
                        kwargs['content'] = base64.b64encode(content)
                    kwargs['encoding'] = 'base64'
                b.add_file(pathname, **kwargs)

            # If this file is a service init script or config , create a
            # service resource.
            try:
                manager, service = util.parse_service(pathname)
                if not ignore.service(manager, service):
                    b.add_service(manager, service)
                    b.add_service_package(manager, service, 'apt',
                                          *_dpkg_query_S(pathname))
                    b.add_service_package(manager, service, 'yum',
                                          *_rpm_qf(pathname))
            except ValueError:
                pass
Exemple #6
0
def files(b):
    logging.info('searching for configuration files')

    # Visit every file in `/etc` except those on the exclusion list above.
    for dirpath, dirnames, filenames in os.walk('/etc'):

        # Determine if this entire directory should be ignored by default.
        ignored = ignore.file(dirpath)

        # Collect up the full pathname to each file, `lstat` them all, and
        # note which ones will probably be ignored.
        files = []
        for filename in filenames:
            pathname = os.path.join(dirpath, filename)
            try:
                files.append((pathname,
                              os.lstat(pathname),
                              ignore.file(pathname, ignored)))
            except OSError as e:
                logging.warning('{0} caused {1} - try running as root'.
                                format(pathname, errno.errorcode[e.errno]))

        # Track the ctime of each file in this directory.  Weed out false
        # positives by ignoring files with common ctimes.
        ctimes = defaultdict(lambda: 0)

        # Map the ctimes of each directory entry that isn't being ignored.
        for pathname, s, ignored in files:
            if not ignored:
                ctimes[s.st_ctime] += 1
        for dirname in dirnames:
            try:
                ctimes[os.lstat(os.path.join(dirpath, dirname)).st_ctime] += 1
            except OSError:
                pass

        for pathname, s, ignored in files:

            # Make sure this pathname will actually be able to be included
            # in the blueprint.  This is a bit of a cop-out since the file
            # could be important but at least it's not a crashing bug.
            try:
                pathname = unicode(pathname)
            except UnicodeDecodeError:
                logging.warning('{0} not UTF-8 - skipping it'.
                                format(repr(pathname)[1:-1]))
                continue

            # Ignore ignored files and files that share their ctime with other
            # files in the directory.  This is a very strong indication that
            # the file is original to the system and should be ignored.
            if ignored or 1 < ctimes[s.st_ctime] and ignore.file(pathname,
                                                                 True):
                continue

            # The content is used even for symbolic links to determine whether
            # it has changed from the packaged version.
            try:
                content = open(pathname).read()
            except IOError:
                #logging.warning('{0} not readable'.format(pathname))
                continue

            # Ignore files that are from the `base-files` package (which
            # doesn't include MD5 sums for every file for some reason).
            apt_packages = _dpkg_query_S(pathname)
            if 'base-files' in apt_packages:
                continue

            # Ignore files that are unchanged from their packaged version,
            # or match in MD5SUMS.
            md5sums = MD5SUMS.get(pathname, [])
            md5sums.extend([_dpkg_md5sum(package, pathname)
                            for package in apt_packages])
            md5sum = _rpm_md5sum(pathname)
            if md5sum is not None:
                md5sums.append(md5sum)
            if (hashlib.md5(content).hexdigest() in md5sums \
                or 64 in [len(md5sum or '') for md5sum in md5sums] \
                   and hashlib.sha256(content).hexdigest() in md5sums) \
               and ignore.file(pathname, True):
                continue

            # A symbolic link's content is the link target.
            if stat.S_ISLNK(s.st_mode):
                content = os.readlink(pathname)

                # Ignore symbolic links providing backwards compatibility
                # between SystemV init and Upstart.
                if '/lib/init/upstart-job' == content:
                    continue

                # Ignore symbolic links into the Debian alternatives system.
                # These are almost certainly managed by packages.
                if content.startswith('/etc/alternatives/'):
                    continue

                encoding = 'plain'

            # A regular file is stored as plain text only if it is valid
            # UTF-8, which is required for JSON serialization.
            elif stat.S_ISREG(s.st_mode):
                try:
                    content = content.decode('utf_8')
                    encoding = 'plain'
                except UnicodeDecodeError:
                    content = base64.b64encode(content)
                    encoding = 'base64'

            # Other types, like FIFOs and sockets are not supported within
            # a blueprint and really shouldn't appear in `/etc` at all.
            else:
                logging.warning('{0} is not a regular file or symbolic link'.
                                format(pathname))
                continue

            try:
                pw = pwd.getpwuid(s.st_uid)
                owner = pw.pw_name
            except KeyError:
                owner = s.st_uid
            try:
                gr = grp.getgrgid(s.st_gid)
                group = gr.gr_name
            except KeyError:
                group = s.st_gid
            b.add_file(pathname,
                       content=content,
                       encoding=encoding,
                       group=group,
                       mode='{0:o}'.format(s.st_mode),
                       owner=owner)

            # If this file is a service init script or config , create a
            # service resource.
            try:
                manager, service = util.parse_service(pathname)
                if not ignore.service(manager, service):
                    b.add_service(manager, service)
                    b.add_service_package(manager,
                                          service,
                                          'apt',
                                          *apt_packages)
                    b.add_service_package(manager,
                                          service,
                                          'yum',
                                          *_rpm_qf(pathname))
            except ValueError:
                pass
Exemple #7
0
def files(b):
    logging.info('searching for configuration files')

    # Visit every file in `/etc` except those on the exclusion list above.
    for dirpath, dirnames, filenames in os.walk('/etc'):

        # Determine if this entire directory should be ignored by default.
        ignored = ignore.file(dirpath)

        # Collect up the full pathname to each file, `lstat` them all, and
        # note which ones will probably be ignored.
        files = []
        for filename in filenames:
            pathname = os.path.join(dirpath, filename)
            try:
                files.append((pathname,
                              os.lstat(pathname),
                              ignore.file(pathname, ignored)))
            except OSError as e:
                logging.warning('{0} caused {1} - try running as root'.
                                format(pathname, errno.errorcode[e.errno]))

        # Track the ctime of each file in this directory.  Weed out false
        # positives by ignoring files with common ctimes.
        ctimes = defaultdict(lambda: 0)

        # Map the ctimes of each directory entry that isn't being ignored.
        for pathname, s, ignored in files:
            if not ignored:
                ctimes[s.st_ctime] += 1
        for dirname in dirnames:
            try:
                ctimes[os.lstat(os.path.join(dirpath, dirname)).st_ctime] += 1
            except OSError:
                pass

        for pathname, s, ignored in files:

            # Always ignore block special files, character special files,
            # pipes, and sockets.  They end up looking like deadlocks.
            if stat.S_ISBLK(s.st_mode) \
            or stat.S_ISCHR(s.st_mode) \
            or stat.S_ISFIFO(s.st_mode) \
            or stat.S_ISSOCK(s.st_mode):
                continue

            # Make sure this pathname will actually be able to be included
            # in the blueprint.  This is a bit of a cop-out since the file
            # could be important but at least it's not a crashing bug.
            try:
                pathname = unicode(pathname)
            except UnicodeDecodeError:
                logging.warning('{0} not UTF-8 - skipping it'.
                                format(repr(pathname)[1:-1]))
                continue

            # Ignore ignored files and files that share their ctime with other
            # files in the directory.  This is a very strong indication that
            # the file is original to the system and should be ignored.
            if ignored or 1 < ctimes[s.st_ctime] and ignore.file(pathname,
                                                                 True):
                continue

            # Check for a Mustache template and an optional shell script
            # that templatize this file.
            try:
                template = open(
                    '{0}.blueprint-template.mustache'.format(pathname)).read()
            except IOError:
                template = None
            try:
                data = open(
                    '{0}.blueprint-template.sh'.format(pathname)).read()
            except IOError:
                data = None

            # The content is used even for symbolic links to determine whether
            # it has changed from the packaged version.
            try:
                content = open(pathname).read()
            except IOError:
                #logging.warning('{0} not readable'.format(pathname))
                continue

            # Ignore files that are unchanged from their packaged version.
            if _unchanged(pathname, content):
                continue

            # Resolve the rest of the file's metadata from the
            # `/etc/passwd` and `/etc/group` databases.
            try:
                pw = pwd.getpwuid(s.st_uid)
                owner = pw.pw_name
            except KeyError:
                owner = s.st_uid
            try:
                gr = grp.getgrgid(s.st_gid)
                group = gr.gr_name
            except KeyError:
                group = s.st_gid
            mode = '{0:o}'.format(s.st_mode)

            # A symbolic link's content is the link target.
            if stat.S_ISLNK(s.st_mode):
                content = os.readlink(pathname)

                # Ignore symbolic links providing backwards compatibility
                # between SystemV init and Upstart.
                if '/lib/init/upstart-job' == content:
                    continue

                # Ignore symbolic links into the Debian alternatives system.
                # These are almost certainly managed by packages.
                if content.startswith('/etc/alternatives/'):
                    continue

                b.add_file(pathname,
                           content=content,
                           encoding='plain',
                           group=group,
                           mode=mode,
                           owner=owner)

            # A regular file is stored as plain text only if it is valid
            # UTF-8, which is required for JSON serialization.
            else:
                kwargs = dict(group=group,
                              mode=mode,
                              owner=owner)
                try:
                    if template:
                        if data:
                            kwargs['data'] = data.decode('utf_8')
                        kwargs['template'] = template.decode('utf_8')
                    else:
                        kwargs['content'] = content.decode('utf_8')
                    kwargs['encoding'] = 'plain'
                except UnicodeDecodeError:
                    if template:
                        if data:
                            kwargs['data'] = base64.b64encode(data)
                        kwargs['template'] = base64.b64encode(template)
                    else:
                        kwargs['content'] = base64.b64encode(content)
                    kwargs['encoding'] = 'base64'
                b.add_file(pathname, **kwargs)

            # If this file is a service init script or config , create a
            # service resource.
            try:
                manager, service = util.parse_service(pathname)
                if not ignore.service(manager, service):
                    b.add_service(manager, service)
                    b.add_service_package(manager,
                                          service,
                                          'apt',
                                          *_dpkg_query_S(pathname))
                    b.add_service_package(manager,
                                          service,
                                          'yum',
                                          *_rpm_qf(pathname))
            except ValueError:
                pass