def __init__(self, name=None, commit=None, create=False): """ Construct a blueprint in the new format in a backwards-compatible manner. """ self.name = name self._commit = commit # Create a new blueprint object and populate it based on this server. if create: super(Blueprint, self).__init__() import backend for funcname in backend.__all__: getattr(backend, funcname)(self) import services services.services(self) # Create a blueprint from a Git repository. elif name is not None: git.init() if self._commit is None: self._commit = git.rev_parse('refs/heads/{0}'.format(name)) if self._commit is None: raise NotFoundError(name) tree = git.tree(self._commit) blob = git.blob(tree, 'blueprint.json') content = git.content(blob) super(Blueprint, self).__init__(**json.loads(content)) # Create an empty blueprint object to be filled in later. else: super(Blueprint, self).__init__()
def sh(self): """ Generate shell code. """ s = sh.Script(self.name, comment=self.DISCLAIMER) # Extract source tarballs. tree = git.tree(self._commit) for dirname, filename in sorted(self.sources.iteritems()): blob = git.blob(tree, filename) content = git.content(blob) s.add('tar xf "{0}" -C "{1}"', filename, dirname, sources={filename: content}) # Place files. for pathname, f in sorted(self.files.iteritems()): s.add('mkdir -p "{0}"', os.path.dirname(pathname)) if '120000' == f['mode'] or '120777' == f['mode']: s.add('ln -s "{0}" "{1}"', f['content'], pathname) continue command = 'cat' if 'base64' == f['encoding']: command = 'base64 --decode' eof = 'EOF' while re.search(r'{0}'.format(eof), f['content']): eof += 'EOF' s.add('{0} >"{1}" <<{2}', command, pathname, eof) s.add(raw=f['content']) if 0 < len(f['content']) and '\n' != f['content'][-1]: eof = '\n{0}'.format(eof) s.add(eof) if 'root' != f['owner']: s.add('chown {0} "{1}"', f['owner'], pathname) if 'root' != f['group']: s.add('chgrp {0} "{1}"', f['group'], pathname) if '000644' != f['mode']: s.add('chmod {0} "{1}"', f['mode'][-4:], pathname) # Install packages. def before(manager): if 'apt' == manager.name: s.add('apt-get -q update') def package(manager, package, version): if manager.name == package: return s.add(manager(package, version)) match = re.match(r'^rubygems(\d+\.\d+(?:\.\d+)?)$', package) if 'apt' != manager.name: return if match is not None and rubygems_update(): s.add('/usr/bin/gem{0} install --no-rdoc --no-ri ' 'rubygems-update', match.group(1)) s.add('/usr/bin/ruby{0} $(PATH=$PATH:/var/lib/gems/{0}/bin ' 'which update_rubygems)', match.group(1)) self.walk(before=before, package=package) return s
def list(self, show_types=['open', 'test'], releases_filter=[]): self.require_itdb() releasedirs = filter(lambda x: x[1] == 'tree', git.tree(it.ITDB_BRANCH + \ ':' + it.TICKET_DIR)) # Filter releases if releases_filter: filtered = [] for dir in releasedirs: _, _, _, name = dir if name in releases_filter: filtered.append(dir) releasedirs = filtered # Show message if no tickets there if len(releasedirs) == 0: print 'no tickets yet. use \'it new\' to add new tickets.' return # Collect tickets assigned to self on the way inbox = [] print_count = 0 releasedirs.sort(cmp_by_release_dir) for _, _, sha, rel in releasedirs: reldir = os.path.join(it.TICKET_DIR, rel) ticketfiles = git.tree(it.ITDB_BRANCH + ':' + reldir) tickets = [ ticket.create_from_lines(git.cat_file(sha), ticket_id, rel, True) \ for _, type, sha, ticket_id in ticketfiles \ if type == 'blob' and ticket_id != it.HOLD_FILE \ ] # Store the tickets in the inbox if neccessary inbox += filter(lambda t: t.is_mine(), tickets) print_count += self.__print_ticket_rows(rel, tickets, show_types, True, True) print_count += self.__print_ticket_rows( 'INBOX', inbox, (show_types == ['open', 'test']) and ['open'] or show_types, False, False) if print_count == 0: print 'use the -a flag to show all tickets'
def gen_content(): # It's a good thing `gen_content` is never called by the # `Blueprint.__init__` callbacks, since this would always # raise `AttributeError` on the fake blueprint structure # used to initialize a real `Blueprint` object. tree = git.tree(b._commit) blob = git.blob(tree, filename) return git.content(blob)
def checkout(cls, name, commit=None): git.init() if commit is None: commit = git.rev_parse('refs/heads/{0}'.format(name)) if commit is None: raise NotFoundError(name) tree = git.tree(commit) blob = git.blob(tree, 'blueprint.json') content = git.content(blob) return cls(name, commit, **json.loads(content))
def show_all(self): # code adapted from self.list() self.require_itdb() releasedirs = filter(lambda x: x[1] == 'tree', git.tree(it.ITDB_BRANCH + \ ':' + it.TICKET_DIR)) # Show message if no tickets there if len(releasedirs) == 0: print 'no tickets yet. use \'it new\' to add new tickets.' return releasedirs.sort(cmp_by_release_dir) for _, _, sha, rel in releasedirs: reldir = os.path.join(it.TICKET_DIR, rel) ticketfiles = git.tree(it.ITDB_BRANCH + ':' + reldir) for _, _, sha, fullsha in ticketfiles: if fullsha == it.HOLD_FILE: continue self.show(fullsha) print ''
def list(self, show_types = ['open', 'test'], releases_filter = []): self.require_itdb() releasedirs = filter(lambda x: x[1] == 'tree', git.tree(it.ITDB_BRANCH + \ ':' + it.TICKET_DIR)) # Filter releases if releases_filter: filtered = [] for dir in releasedirs: _, _, _, name = dir if name in releases_filter: filtered.append(dir) releasedirs = filtered # Show message if no tickets there if len(releasedirs) == 0: print 'no tickets yet. use \'it new\' to add new tickets.' return # Collect tickets assigned to self on the way inbox = [] print_count = 0 releasedirs.sort(cmp_by_release_dir) for _, _, sha, rel in releasedirs: reldir = os.path.join(it.TICKET_DIR, rel) ticketfiles = git.tree(it.ITDB_BRANCH + ':' + reldir) tickets = [ ticket.create_from_lines(git.cat_file(sha), ticket_id, rel, True) \ for _, type, sha, ticket_id in ticketfiles \ if type == 'blob' and ticket_id != it.HOLD_FILE \ ] # Store the tickets in the inbox if neccessary inbox += filter(lambda t: t.is_mine(), tickets) print_count += self.__print_ticket_rows(rel, tickets, show_types, True, True) print_count += self.__print_ticket_rows('INBOX', inbox, (show_types == ['open','test']) and ['open'] or show_types, False, False) if print_count == 0: print 'use the -a flag to show all tickets'
def blueprintignore(self): """ Return the blueprint's ~/.blueprintignore file. Prior to v3.0.4 this file was stored as .gitignore in the repository. """ tree = git.tree(self._commit) blob = git.blob(tree, '.blueprintignore') if blob is None: blob = git.blob(tree, '.gitignore') import ignore if blob is None: return ignore.Rules('') content = git.content(blob) if content is None: return ignore.Rules('') return ignore.Rules(content)
def blueprintignore(self): """ Return an open file pointer to the blueprint's blueprintignore file, which is suitable for passing back to `blueprint.rules.Rules.parse`. Prior to v3.0.9 this file was stored as .blueprintignore in the repository. Prior to v3.0.4 this file was stored as .gitignore in the repository. """ tree = git.tree(self._commit) blob = git.blob(tree, 'blueprintignore') if blob is None: blob = git.blob(tree, '.blueprintignore') if blob is None: blob = git.blob(tree, '.gitignore') if blob is None: return [] return git.cat_file(blob)
def commit(self, message=''): """ Create a new revision of this blueprint in the local Git repository. Include the blueprint JSON and any source archives referenced by the JSON. """ git.init() refname = 'refs/heads/{0}'.format(self.name) parent = git.rev_parse(refname) # Start with an empty index every time. Specifically, clear out # source tarballs from the parent commit. if parent is not None: for mode, type, sha, pathname in git.ls_tree(git.tree(parent)): git.git('update-index', '--force-remove', pathname) # Add `blueprint.json` to the index. f = open('blueprint.json', 'w') f.write(self.dumps()) f.close() git.git('update-index', '--add', os.path.abspath('blueprint.json')) # Add source tarballs to the index. for filename in self.sources.itervalues(): git.git('update-index', '--add', os.path.abspath(filename)) # Add `/etc/blueprintignore` and `~/.blueprintignore` to the index. # Since adding extra syntax to this file, it no longer makes sense # to store it as `.gitignore`. f = open('blueprintignore', 'w') for pathname in ('/etc/blueprintignore', os.path.expanduser('~/.blueprintignore')): try: f.write(open(pathname).read()) except IOError: pass f.close() git.git('update-index', '--add', os.path.abspath('blueprintignore')) # Write the index to Git's object store. tree = git.write_tree() # Write the commit and update the tip of the branch. self._commit = git.commit_tree(tree, message, parent) git.git('update-ref', refname, self._commit)
def gen_content(): tree = git.tree(self._commit) blob = git.blob(tree, filename) return git.content(blob)
def chef(self): """ Generate Chef code. """ c = chef.Cookbook(self.name, comment=self.DISCLAIMER) # Extract source tarballs. tree = git.tree(self._commit) for dirname, filename in sorted(self.sources.iteritems()): blob = git.blob(tree, filename) content = git.content(blob) pathname = os.path.join('/tmp', filename) c.file(pathname, content, owner='root', group='root', mode='0644', backup=False, source=pathname[1:]) c.execute('tar xf {0}'.format(pathname), cwd=dirname) # Place files. for pathname, f in sorted(self.files.iteritems()): c.directory(os.path.dirname(pathname), group='root', mode='755', owner='root', recursive=True) if '120000' == f['mode'] or '120777' == f['mode']: c.link(pathname, owner=f['owner'], group=f['group'], to=f['content']) continue content = f['content'] if 'base64' == f['encoding']: content = base64.b64decode(content) c.file(pathname, content, owner=f['owner'], group=f['group'], mode=f['mode'][-4:], backup=False, source=pathname[1:]) # Install packages. def before(manager): if 'apt' == manager.name: c.execute('apt-get -q update') def package(manager, package, version): if manager.name == package: return if 'apt' == manager.name: c.apt_package(package, version=version) match = re.match(r'^rubygems(\d+\.\d+(?:\.\d+)?)$', package) if match is not None and rubygems_update(): c.execute('/usr/bin/gem{0} install --no-rdoc --no-ri ' 'rubygems-update'.format(match.group(1))) c.execute('/usr/bin/ruby{0} ' '$(PATH=$PATH:/var/lib/gems/{0}/bin ' 'which update_rubygems)"'.format(match.group(1))) # All types of gems get to have package resources. elif re.search(r'ruby', manager.name) is not None: match = re.match(r'^ruby(?:gems)?(\d+\.\d+(?:\.\d+)?)', manager.name) c.gem_package(package, gem_binary='/usr/bin/gem{0}'.format(match.group(1)), version=version) # Everything else is an execute resource. else: c.execute(manager(package, version)) self.walk(before=before, package=package) return c
def puppet(self): """ Generate Puppet code. """ m = puppet.Manifest(self.name, comment=self.DISCLAIMER) # Set the default `PATH` for exec resources. m.add(puppet.Exec.defaults(path=os.environ['PATH'])) # Extract source tarballs. tree = git.tree(self._commit) for dirname, filename in sorted(self.sources.iteritems()): blob = git.blob(tree, filename) content = git.content(blob) pathname = os.path.join('/tmp', filename) m['sources'].add(puppet.File( pathname, self.name, content, owner='root', group='root', mode='0644', source='puppet:///{0}/{1}'.format(self.name, pathname[1:]))) m['sources'].add(puppet.Exec( 'tar xf {0}'.format(pathname), cwd=dirname, require=puppet.File.ref(pathname))) # Place files. if 0 < len(self.files): for pathname, f in sorted(self.files.iteritems()): # Create resources for parent directories and let the # autorequire mechanism work out dependencies. dirnames = os.path.dirname(pathname).split('/')[1:] for i in xrange(len(dirnames)): m['files'].add(puppet.File( os.path.join('/', *dirnames[0:i+1]), ensure='directory')) # Create the actual file resource. if '120000' == f['mode'] or '120777' == f['mode']: m['files'].add(puppet.File(pathname, None, None, owner=f['owner'], group=f['group'], ensure=f['content'])) continue content = f['content'] if 'base64' == f['encoding']: content = base64.b64decode(content) m['files'].add(puppet.File(pathname, self.name, content, owner=f['owner'], group=f['group'], mode=f['mode'][-4:], ensure='file')) # Install packages. deps = [] def before(manager): deps.append(manager) if 'apt' != manager.name: return if 0 == len(manager): return if 1 == len(manager) and manager.name in manager: return m['packages'].add(puppet.Exec('apt-get -q update', before=puppet.Class.ref('apt'))) def package(manager, package, version): # `apt` is easy since it's the default. if 'apt' == manager.name: m['packages'][manager].add(puppet.Package(package, ensure=version)) # If APT is installing RubyGems, get complicated. match = re.match(r'^rubygems(\d+\.\d+(?:\.\d+)?)$', package) if match is not None and rubygems_update(): m['packages'][manager].add(puppet.Exec('/bin/sh -c "' '/usr/bin/gem{0} install --no-rdoc --no-ri ' 'rubygems-update; ' '/usr/bin/ruby{0} $(PATH=$PATH:/var/lib/gems/{0}/bin ' 'which update_rubygems)"'.format(match.group(1)), require=puppet.Package.ref(package))) # RubyGems for Ruby 1.8 is easy, too, because Puppet has a # built in provider. elif 'rubygems1.8' == manager.name: m['packages'][manager].add(puppet.Package(package, ensure=version, provider='gem')) # Other versions of RubyGems are slightly more complicated. elif re.search(r'ruby', manager.name) is not None: match = re.match(r'^ruby(?:gems)?(\d+\.\d+(?:\.\d+)?)', manager.name) m['packages'][manager].add(puppet.Exec( manager(package, version), creates='{0}/{1}/gems/{2}-{3}'.format(rubygems_path(), match.group(1), package, version))) # Python works basically like alternative versions of Ruby # but follows a less predictable directory structure so the # directory is not known ahead of time. This just so happens # to be the way everything else works, too. else: m['packages'][manager].add(puppet.Exec( manager(package, version))) self.walk(before=before, package=package) m['packages'].dep(*[puppet.Class.ref(dep) for dep in deps]) # Strict ordering of classes. deps = [] if 0 < len(self.sources): deps.append('sources') if 0 < len(self.files): deps.append('files') if 0 < len(self.packages): deps.append('packages') m.dep(*[puppet.Class.ref(dep) for dep in deps]) return m
def sh(self, server='https://devstructure.com', secret=None): """ Generate shell code. """ import sh s = sh.Script(self.name, comment=self.DISCLAIMER) # Extract source tarballs. if secret is not None: for dirname, filename in sorted(self.sources.iteritems()): s.add('wget "{0}/{1}/{2}/{3}"', server, secret, self.name, filename) s.add('tar xf "{0}" -C "{1}"', filename, dirname) else: tree = git.tree(self._commit) for dirname, filename in sorted(self.sources.iteritems()): blob = git.blob(tree, filename) content = git.content(blob) s.add('tar xf "{0}" -C "{1}"', filename, dirname, sources={filename: content}) # Place files. for pathname, f in sorted(self.files.iteritems()): s.add('mkdir -p "{0}"', os.path.dirname(pathname)) if '120000' == f['mode'] or '120777' == f['mode']: s.add('ln -s "{0}" "{1}"', f['content'], pathname) continue command = 'cat' if 'base64' == f['encoding']: command = 'base64 --decode' eof = 'EOF' while re.search(r'{0}'.format(eof), f['content']): eof += 'EOF' s.add('{0} >"{1}" <<{2}', command, pathname, eof) s.add(raw=f['content']) if 0 < len(f['content']) and '\n' != f['content'][-1]: eof = '\n{0}'.format(eof) s.add(eof) if 'root' != f['owner']: s.add('chown {0} "{1}"', f['owner'], pathname) if 'root' != f['group']: s.add('chgrp {0} "{1}"', f['group'], pathname) if '000644' != f['mode']: s.add('chmod {0} "{1}"', f['mode'][-4:], pathname) # Install packages. def before(manager): if 0 == len(manager): return if 'apt' == manager.name: s.add('export APT_LISTBUGS_FRONTEND="none"') s.add('export APT_LISTCHANGES_FRONTEND="none"') s.add('export DEBIAN_FRONTEND="noninteractive"') s.add('apt-get -q update') elif 'yum' == manager.name: s.add('yum makecache') def package(manager, package, version): if manager.name == package: return s.add(manager(package, version)) if manager.name not in ('apt', 'yum'): return # See comments on this section in `puppet` above. match = re.match(r'^rubygems(\d+\.\d+(?:\.\d+)?)$', package) if match is not None and util.rubygems_update(): s.add('/usr/bin/gem{0} install --no-rdoc --no-ri ' 'rubygems-update', match.group(1)) s.add('/usr/bin/ruby{0} $(PATH=$PATH:/var/lib/gems/{0}/bin ' 'which update_rubygems)', match.group(1)) self.walk(before=before, package=package) return s
def walk(b, choose): """ Given a function for choosing a `Blueprint` object (based typically on the result of a `raw_input` call within the `choose` function), populate one or more `Blueprint`s closed into `choose`. """ def file(pathname, f): print(pathname) b_chosen = choose() if b_chosen is None: return b_chosen.add_file(pathname, **f) def package(manager, package, version): print('{0} {1} {2}'.format(manager, package, version)) b_chosen = choose() if b_chosen is None: return b_chosen.add_package(manager, package, version) def service(manager, service): print('{0} {1}'.format(manager, service)) b_chosen = choose() if b_chosen is None: return b_chosen.add_service(manager, service) def service_file(manager, service, pathname): b_chosen.add_service_file(manager, service, pathname) walklib.walk_service_files(b_chosen, manager, service, service_file=service_file) def service_package(manager, service, package_manager, package): b_chosen.add_service_package(manager, service, package_manager, package) walklib.walk_service_packages(b_chosen, manager, service, service_package=service_package) def service_source(manager, service, dirname): b_chosen.add_service_source(manager, service, dirname) walklib.walk_service_sources(b_chosen, manager, service, service_source=service_source) commit = git.rev_parse(b.name) tree = git.tree(commit) def source(dirname, filename, gen_content, url): if url is not None: print('{0} {1}'.format(dirname, url)) elif gen_content is not None: blob = git.blob(tree, filename) git.cat_file(blob, filename) print('{0} {1}'.format(dirname, filename)) b_chosen = choose() if b_chosen is None: return b_chosen.add_source(dirname, filename) b.walk(file=file, package=package, service=service, source=source)