def __init__(self, outputdir, overwrite=False, crcs={}, **options):
        self.outputdir = outputdir
        self.overwrite = overwrite
        self.parseopts(**options)

        # Read MD5 sums for testing file changes.
        self.md5file = os.path.join(self.outputdir, '.md5sums')
        self.md5sums = {}
        if os.path.exists(self.md5file):
            for line in open(self.md5file, 'r'):
                try:
                    digest, filename = line.rstrip().split(None, 1)
                    self.md5sums[filename] = digest
                except:
                    pass

        # Migrate legacy CRCs to MD5 sums. If the file is unchanged, calculate
        # the MD5; otherwise, continue to check the old CRC32.
        self.crcs = {}
        for filename in crcs:
            if filename in self.md5sums:
                # Assume MD5 sum is more recent
                continue
            # Check if the file has changed since the last CRC32.
            pathname = os.path.join(self.outputdir, filename)
            if not os.path.exists(pathname):
                # The file has been removed, and can be regenerated.
                continue
            if crcs[filename] == str(utils.fileCRC(pathname, stripnewlines=True)):
                # File is unchanged, calculate MD5 sum.
                self.md5sums[filename] = utils.fileMD5(pathname)
            else:
                self.crcs[filename] = crcs[filename]
 def fileChanged(self, filename):
     pathname = os.path.join(self.outputdir, filename)
     if not os.path.exists(pathname):
         return False
     lastHash = self.md5sums.get(filename, None)
     if lastHash is None and filename in self.crcs:
         lastHash = self.crcs[filename]
         currentHash = str(utils.fileCRC(pathname, stripnewlines=True))
     else:
         currentHash = utils.fileMD5(pathname)
     return lastHash != currentHash
Exemple #3
0
 def fileChanged(self, filename, userfile=False):
     if userfile:
         return True
     pathname = os.path.join(self.outputdir, filename)
     if not os.path.exists(pathname):
         return False
     lastHash = self.md5sums.get(filename, None)
     if lastHash is None and filename in self.crcs:
         lastHash = self.crcs[filename]
         currentHash = str(utils.fileCRC(pathname, stripnewlines=True))
     else:
         currentHash = utils.fileMD5(pathname)
     return lastHash != currentHash
Exemple #4
0
    def __init__(self,
                 outputdir,
                 overwrite=False,
                 crcs={},
                 variant="",
                 header=None,
                 **options):

        self.outputdir = outputdir
        self.overwrite = overwrite
        if variant != "":
            variant = "_" + variant
        self.variant = variant
        self.parseopts(**options)

        # Read MD5 sums for testing file changes.
        self.md5file = os.path.join(self.outputdir, '.md5sums')
        self.md5sums = {}
        if os.path.exists(self.md5file):
            for line in open(self.md5file, 'r'):
                try:
                    digest, filename = line.rstrip().split(None, 1)
                    self.md5sums[filename] = digest
                except:
                    pass

        # Migrate legacy CRCs to MD5 sums. If the file is unchanged, calculate
        # the MD5; otherwise, continue to check the old CRC32.
        self.crcs = {}
        for filename in crcs:
            if filename in self.md5sums:
                # Assume MD5 sum is more recent
                continue
            # Check if the file has changed since the last CRC32.
            pathname = os.path.join(self.outputdir, filename)
            if not os.path.exists(pathname):
                # The file has been removed, and can be regenerated.
                continue
            if crcs[filename] == str(
                    utils.fileCRC(pathname, stripnewlines=True)):
                # File is unchanged, calculate MD5 sum.
                self.md5sums[filename] = utils.fileMD5(pathname)
            else:
                self.crcs[filename] = crcs[filename]

        # Save the header (if given)
        self.header = header
Exemple #5
0
    def generate(self, softpkg, *filenames):
        loader = self.loader(softpkg)

        # Map the component model into a language-specific version
        component = self.map(softpkg)

        generated = []
        skipped = []

        # Note all tracked files, for cleaning up stale files; if a file list
        # was given, only consider those files for deletion.
        stale = set(self.trackedFiles())
        if filenames:
            stale.intersection_update(filenames)

        for template in self.templates(component):
            # Mark the current template as seen so it will not be deleted,
            # regardless of whether it gets generated
            if template.filename in stale:
                stale.remove(template.filename)

            # If a file list was given, skip files not explicitly listed.
            if filenames and template.filename not in filenames:
                continue

            filename = os.path.join(self.outputdir, template.filename)

            if os.path.exists(filename):
                # Check if the file has been modified since last generation.
                if self.fileChanged(template.filename,
                                    template.userfile) and not self.overwrite:
                    skipped.append((template.filename, 'overwrite'))
                    continue
                action = ''
            else:
                action = '(added)'

            # Attempt to ensure that the full required path exists
            parentdir = os.path.dirname(filename)
            if parentdir and not os.path.isdir(parentdir):
                os.makedirs(parentdir)

            env = CodegenEnvironment(loader=loader, **template.options())
            env.filters.update(template.filters())
            tmpl = env.get_template(template.template)

            # Initially, write the output to a temporary file to avoid trashing
            # the original file if the template is malformed
            with tempfile.NamedTemporaryFile() as outfile:
                # Start with the template-specific context, then add the mapped
                # component and a reference to this generator with known names.
                context = template.context()
                context['component'] = component
                context['generator'] = self
                context['versions'] = versions

                # Evaluate the template in streaming mode (rather than all at
                # once)
                gen = tmpl.generate(**context)
                if self.header:
                    # Define a generator function to insert the header at the
                    # top of the file
                    def generate(gen, header):
                        first = True
                        for chunk in gen:
                            if first:
                                # Take "shebang" into account for executable
                                # scripts
                                if chunk.startswith('#!'):
                                    line, chunk = chunk.split('\n', 1)
                                    yield line + '\n'
                                yield header
                            first = False
                            yield chunk

                    # Wrap the template's generator with the header insertion
                    # generator
                    gen = generate(gen, template.comment(self.header))

                # Write the stream to the output file
                for chunk in gen:
                    outfile.write(chunk)

                # Add a trailing newline to work around a Jinja bug.
                outfile.write('\n')

                # Now that generation has succeeded, flush the temporary file
                # to ensure the contents are completer, and copy to the target
                # location
                outfile.file.flush()
                shutil.copy(outfile.name, filename)

                # Set the executable bit, if requested by the template.
                if template.executable:
                    st = os.stat(filename)
                    os.chmod(filename, st.st_mode | stat.S_IEXEC)

            generated.append((template.filename, action))

            # Update the MD5 digest
            self.md5sums[template.filename] = utils.fileMD5(filename)

        # Remove old files that were not (and would not have been) generated on
        # this pass, and are unchanged.
        for existing in stale:
            filename = os.path.join(self.outputdir, existing)
            if not os.path.exists(filename):
                continue

            # Check for changes, and require explicit action to remove.
            if self.fileChanged(existing) and not self.overwrite:
                skipped.append((existing, 'delete'))
                continue

            # Delete the file, and remove its MD5 sum (if it has one).
            os.unlink(filename)
            generated.append((existing, '(deleted)'))
            if existing in self.md5sums:
                del self.md5sums[existing]

        # Save updated MD5 digests
        # NB: To work with "md5sum -c", there must be two spaces between the
        #     MD5 digest and the filename.
        md5out = open(self.md5file, 'w')
        for name, digest in self.md5sums.items():
            print >> md5out, "%s  %s" % (digest, name)
        md5out.close()

        return generated, skipped
    def generate(self, softpkg, *filenames):
        if not os.path.exists(self.outputdir):
            os.mkdir(self.outputdir)

        loader = self.loader(softpkg)

        # Map the component model into a language-specific version
        component = self.map(softpkg)

        generated = []
        skipped = []

        # Note all tracked files, for cleaning up stale files; if a file list
        # was given, only consider those files for deletion.
        stale = set(self.trackedFiles())
        if filenames:
            stale.intersection_update(filenames)

        for template in self.templates(component):
            # Mark the current template as seen so it will not be deleted,
            # regardless of whether it gets generated
            if template.filename in stale:
                stale.remove(template.filename)

            # If a file list was given, skip files not explicitly listed.
            if filenames and template.filename not in filenames:
                continue

            filename = os.path.join(self.outputdir, template.filename)

            if os.path.exists(filename):
                # Check if the file has been modified since last generation.
                if self.fileChanged(template.filename) and not self.overwrite:
                    skipped.append((template.filename, 'overwrite'))
                    continue
                action = ''
            else:
                action = '(added)'

            # Attempt to ensure that the full required path exists for files
            # that are more deeply nested.
            if not os.path.isdir(os.path.dirname(filename)):
                os.makedirs(os.path.dirname(filename))

            env = CodegenEnvironment(loader=loader, **template.options())
            env.filters.update(template.filters())
            tmpl = env.get_template(template.template)
            outfile = open(filename, 'w')
            try:
                # Start with the template-specific context, then add the mapped
                # component and a reference to this generator with known names.
                context = template.context()
                context['component'] = component
                context['generator'] = self

                # Evaluate the template in streaming mode (rather than all at
                # once), dumping to the output file.
                tmpl.stream(**context).dump(outfile)
                # Add a trailing newline to work around a Jinja bug.
                outfile.write('\n')

                # Set the executable bit, if requested by the template.
                if template.executable:
                    fd = outfile.fileno()
                    st = os.fstat(fd)
                    os.chmod(filename, st.st_mode|stat.S_IEXEC)
            finally:
                outfile.close()

            generated.append((template.filename, action))

            # Update the MD5 digest
            self.md5sums[template.filename] = utils.fileMD5(filename)

        # Remove old files that were not (and would not have been) generated on
        # this pass, and are unchanged.
        for existing in stale:
            filename = os.path.join(self.outputdir, existing)
            if not os.path.exists(filename):
                continue

            # Check for changes, and require explicit action to remove.
            if self.fileChanged(existing) and not self.overwrite:
                skipped.append((existing, 'delete'))
                continue

            # Delete the file, and remove its MD5 sum (if it has one).
            os.unlink(filename)
            generated.append((existing, '(deleted)'))
            if existing in self.md5sums:
                del self.md5sums[existing]

        # Save updated MD5 digests
        md5out = open(self.md5file, 'w')
        for name, digest in self.md5sums.items():
            print >>md5out, "%s %s" % (digest, name)
        md5out.close()

        return generated, skipped
    def generate(self, softpkg, *filenames):
        loader = self.loader(softpkg)

        # Map the component model into a language-specific version
        component = self.map(softpkg)

        generated = []
        skipped = []

        # Note all tracked files, for cleaning up stale files; if a file list
        # was given, only consider those files for deletion.
        stale = set(self.trackedFiles())
        if filenames:
            stale.intersection_update(filenames)

        for template in self.templates(component):
            # Mark the current template as seen so it will not be deleted,
            # regardless of whether it gets generated
            if template.filename in stale:
                stale.remove(template.filename)

            # If a file list was given, skip files not explicitly listed.
            if filenames and template.filename not in filenames:
                continue

            filename = os.path.join(self.outputdir, template.filename)

            if os.path.exists(filename):
                # Check if the file has been modified since last generation.
                if self.fileChanged(template.filename, template.userfile) and not self.overwrite:
                    skipped.append((template.filename, 'overwrite'))
                    continue
                action = ''
            else:
                action = '(added)'

            # Attempt to ensure that the full required path exists
            parentdir = os.path.dirname(filename)
            if parentdir and not os.path.isdir(parentdir):
                os.makedirs(parentdir)

            env = CodegenEnvironment(loader=loader, **template.options())
            env.filters.update(template.filters())
            tmpl = env.get_template(template.template)
            outfile = open(filename, 'w')
            try:
                # Start with the template-specific context, then add the mapped
                # component and a reference to this generator with known names.
                context = template.context()
                context['component'] = component
                context['generator'] = self
                context['versions'] = versions

                # Evaluate the template in streaming mode (rather than all at
                # once), dumping to the output file.
                tmpl.stream(**context).dump(outfile)
                # Add a trailing newline to work around a Jinja bug.
                outfile.write('\n')

                # Set the executable bit, if requested by the template.
                if template.executable:
                    fd = outfile.fileno()
                    st = os.fstat(fd)
                    os.chmod(filename, st.st_mode|stat.S_IEXEC)
            finally:
                outfile.close()

            generated.append((template.filename, action))

            # Update the MD5 digest
            self.md5sums[template.filename] = utils.fileMD5(filename)

        # Remove old files that were not (and would not have been) generated on
        # this pass, and are unchanged.
        for existing in stale:
            filename = os.path.join(self.outputdir, existing)
            if not os.path.exists(filename):
                continue

            # Check for changes, and require explicit action to remove.
            if self.fileChanged(existing) and not self.overwrite:
                skipped.append((existing, 'delete'))
                continue

            # Delete the file, and remove its MD5 sum (if it has one).
            os.unlink(filename)
            generated.append((existing, '(deleted)'))
            if existing in self.md5sums:
                del self.md5sums[existing]

        # Save updated MD5 digests
        # NB: To work with "md5sum -c", there must be two spaces between the
        #     MD5 digest and the filename.
        md5out = open(self.md5file, 'w')
        for name, digest in self.md5sums.items():
            print >>md5out, "%s  %s" % (digest, name)
        md5out.close()

        return generated, skipped