def __init__(self, name=None, commit=None, create=False):
"""
Construct a blueprint in the new format in a backwards-compatible
manner.
"""
self.name = name
self._commit = commit
# Create a new blueprint object and populate it based on this server.
if create:
super(Blueprint, self).__init__()
import backend
for funcname in backend.__all__:
getattr(backend, funcname)(self)
import services
services.services(self)
# Create a blueprint from a Git repository.
elif name is not None:
git.init()
if self._commit is None:
self._commit = git.rev_parse('refs/heads/{0}'.format(name))
if self._commit is None:
raise NotFoundError(name)
tree = git.tree(self._commit)
blob = git.blob(tree, 'blueprint.json')
content = git.content(blob)
super(Blueprint, self).__init__(**json.loads(content))
# Create an empty blueprint object to be filled in later.
else:
super(Blueprint, self).__init__()
def sh(self):
"""
Generate shell code.
"""
s = sh.Script(self.name, comment=self.DISCLAIMER)
# Extract source tarballs.
tree = git.tree(self._commit)
for dirname, filename in sorted(self.sources.iteritems()):
blob = git.blob(tree, filename)
content = git.content(blob)
s.add('tar xf "{0}" -C "{1}"',
filename,
dirname,
sources={filename: content})
# Place files.
for pathname, f in sorted(self.files.iteritems()):
s.add('mkdir -p "{0}"', os.path.dirname(pathname))
if '120000' == f['mode'] or '120777' == f['mode']:
s.add('ln -s "{0}" "{1}"', f['content'], pathname)
continue
command = 'cat'
if 'base64' == f['encoding']:
command = 'base64 --decode'
eof = 'EOF'
while re.search(r'{0}'.format(eof), f['content']):
eof += 'EOF'
s.add('{0} >"{1}" <<{2}', command, pathname, eof)
s.add(raw=f['content'])
if 0 < len(f['content']) and '\n' != f['content'][-1]:
eof = '\n{0}'.format(eof)
s.add(eof)
if 'root' != f['owner']:
s.add('chown {0} "{1}"', f['owner'], pathname)
if 'root' != f['group']:
s.add('chgrp {0} "{1}"', f['group'], pathname)
if '000644' != f['mode']:
s.add('chmod {0} "{1}"', f['mode'][-4:], pathname)
# Install packages.
def before(manager):
if 'apt' == manager.name:
s.add('apt-get -q update')
def package(manager, package, version):
if manager.name == package:
return
s.add(manager(package, version))
match = re.match(r'^rubygems(\d+\.\d+(?:\.\d+)?)$', package)
if 'apt' != manager.name:
return
if match is not None and rubygems_update():
s.add('/usr/bin/gem{0} install --no-rdoc --no-ri '
'rubygems-update', match.group(1))
s.add('/usr/bin/ruby{0} $(PATH=$PATH:/var/lib/gems/{0}/bin '
'which update_rubygems)', match.group(1))
self.walk(before=before, package=package)
return s
def list(self, show_types=['open', 'test'], releases_filter=[]):
self.require_itdb()
releasedirs = filter(lambda x: x[1] == 'tree', git.tree(it.ITDB_BRANCH + \
':' + it.TICKET_DIR))
# Filter releases
if releases_filter:
filtered = []
for dir in releasedirs:
_, _, _, name = dir
if name in releases_filter:
filtered.append(dir)
releasedirs = filtered
# Show message if no tickets there
if len(releasedirs) == 0:
print 'no tickets yet. use \'it new\' to add new tickets.'
return
# Collect tickets assigned to self on the way
inbox = []
print_count = 0
releasedirs.sort(cmp_by_release_dir)
for _, _, sha, rel in releasedirs:
reldir = os.path.join(it.TICKET_DIR, rel)
ticketfiles = git.tree(it.ITDB_BRANCH + ':' + reldir)
tickets = [ ticket.create_from_lines(git.cat_file(sha), ticket_id, rel, True) \
for _, type, sha, ticket_id in ticketfiles \
if type == 'blob' and ticket_id != it.HOLD_FILE \
]
# Store the tickets in the inbox if neccessary
inbox += filter(lambda t: t.is_mine(), tickets)
print_count += self.__print_ticket_rows(rel, tickets, show_types,
True, True)
print_count += self.__print_ticket_rows(
'INBOX', inbox, (show_types == ['open', 'test']) and ['open']
or show_types, False, False)
if print_count == 0:
print 'use the -a flag to show all tickets'
def gen_content():
# It's a good thing `gen_content` is never called by the
# `Blueprint.__init__` callbacks, since this would always
# raise `AttributeError` on the fake blueprint structure
# used to initialize a real `Blueprint` object.
tree = git.tree(b._commit)
blob = git.blob(tree, filename)
return git.content(blob)
def checkout(cls, name, commit=None):
git.init()
if commit is None:
commit = git.rev_parse('refs/heads/{0}'.format(name))
if commit is None:
raise NotFoundError(name)
tree = git.tree(commit)
blob = git.blob(tree, 'blueprint.json')
content = git.content(blob)
return cls(name, commit, **json.loads(content))
def show_all(self):
# code adapted from self.list()
self.require_itdb()
releasedirs = filter(lambda x: x[1] == 'tree', git.tree(it.ITDB_BRANCH + \
':' + it.TICKET_DIR))
# Show message if no tickets there
if len(releasedirs) == 0:
print 'no tickets yet. use \'it new\' to add new tickets.'
return
releasedirs.sort(cmp_by_release_dir)
for _, _, sha, rel in releasedirs:
reldir = os.path.join(it.TICKET_DIR, rel)
ticketfiles = git.tree(it.ITDB_BRANCH + ':' + reldir)
for _, _, sha, fullsha in ticketfiles:
if fullsha == it.HOLD_FILE:
continue
self.show(fullsha)
print ''
def show_all(self):
# code adapted from self.list()
self.require_itdb()
releasedirs = filter(lambda x: x[1] == 'tree', git.tree(it.ITDB_BRANCH + \
':' + it.TICKET_DIR))
# Show message if no tickets there
if len(releasedirs) == 0:
print 'no tickets yet. use \'it new\' to add new tickets.'
return
releasedirs.sort(cmp_by_release_dir)
for _, _, sha, rel in releasedirs:
reldir = os.path.join(it.TICKET_DIR, rel)
ticketfiles = git.tree(it.ITDB_BRANCH + ':' + reldir)
for _, _, sha, fullsha in ticketfiles:
if fullsha == it.HOLD_FILE:
continue
self.show(fullsha)
print ''
def list(self, show_types = ['open', 'test'], releases_filter = []):
self.require_itdb()
releasedirs = filter(lambda x: x[1] == 'tree', git.tree(it.ITDB_BRANCH + \
':' + it.TICKET_DIR))
# Filter releases
if releases_filter:
filtered = []
for dir in releasedirs:
_, _, _, name = dir
if name in releases_filter:
filtered.append(dir)
releasedirs = filtered
# Show message if no tickets there
if len(releasedirs) == 0:
print 'no tickets yet. use \'it new\' to add new tickets.'
return
# Collect tickets assigned to self on the way
inbox = []
print_count = 0
releasedirs.sort(cmp_by_release_dir)
for _, _, sha, rel in releasedirs:
reldir = os.path.join(it.TICKET_DIR, rel)
ticketfiles = git.tree(it.ITDB_BRANCH + ':' + reldir)
tickets = [ ticket.create_from_lines(git.cat_file(sha), ticket_id, rel, True) \
for _, type, sha, ticket_id in ticketfiles \
if type == 'blob' and ticket_id != it.HOLD_FILE \
]
# Store the tickets in the inbox if neccessary
inbox += filter(lambda t: t.is_mine(), tickets)
print_count += self.__print_ticket_rows(rel, tickets, show_types, True, True)
print_count += self.__print_ticket_rows('INBOX', inbox, (show_types == ['open','test']) and ['open'] or show_types, False, False)
if print_count == 0:
print 'use the -a flag to show all tickets'
def blueprintignore(self):
"""
Return the blueprint's ~/.blueprintignore file. Prior to v3.0.4
this file was stored as .gitignore in the repository.
"""
tree = git.tree(self._commit)
blob = git.blob(tree, '.blueprintignore')
if blob is None:
blob = git.blob(tree, '.gitignore')
import ignore
if blob is None:
return ignore.Rules('')
content = git.content(blob)
if content is None:
return ignore.Rules('')
return ignore.Rules(content)
def blueprintignore(self):
"""
Return an open file pointer to the blueprint's blueprintignore file,
which is suitable for passing back to `blueprint.rules.Rules.parse`.
Prior to v3.0.9 this file was stored as .blueprintignore in the
repository. Prior to v3.0.4 this file was stored as .gitignore in
the repository.
"""
tree = git.tree(self._commit)
blob = git.blob(tree, 'blueprintignore')
if blob is None:
blob = git.blob(tree, '.blueprintignore')
if blob is None:
blob = git.blob(tree, '.gitignore')
if blob is None:
return []
return git.cat_file(blob)
def commit(self, message=''):
"""
Create a new revision of this blueprint in the local Git repository.
Include the blueprint JSON and any source archives referenced by
the JSON.
"""
git.init()
refname = 'refs/heads/{0}'.format(self.name)
parent = git.rev_parse(refname)
# Start with an empty index every time. Specifically, clear out
# source tarballs from the parent commit.
if parent is not None:
for mode, type, sha, pathname in git.ls_tree(git.tree(parent)):
git.git('update-index', '--force-remove', pathname)
# Add `blueprint.json` to the index.
f = open('blueprint.json', 'w')
f.write(self.dumps())
f.close()
git.git('update-index', '--add', os.path.abspath('blueprint.json'))
# Add source tarballs to the index.
for filename in self.sources.itervalues():
git.git('update-index', '--add', os.path.abspath(filename))
# Add `/etc/blueprintignore` and `~/.blueprintignore` to the index.
# Since adding extra syntax to this file, it no longer makes sense
# to store it as `.gitignore`.
f = open('blueprintignore', 'w')
for pathname in ('/etc/blueprintignore',
os.path.expanduser('~/.blueprintignore')):
try:
f.write(open(pathname).read())
except IOError:
pass
f.close()
git.git('update-index', '--add', os.path.abspath('blueprintignore'))
# Write the index to Git's object store.
tree = git.write_tree()
# Write the commit and update the tip of the branch.
self._commit = git.commit_tree(tree, message, parent)
git.git('update-ref', refname, self._commit)
def gen_content():
tree = git.tree(self._commit)
blob = git.blob(tree, filename)
return git.content(blob)
def chef(self):
"""
Generate Chef code.
"""
c = chef.Cookbook(self.name, comment=self.DISCLAIMER)
# Extract source tarballs.
tree = git.tree(self._commit)
for dirname, filename in sorted(self.sources.iteritems()):
blob = git.blob(tree, filename)
content = git.content(blob)
pathname = os.path.join('/tmp', filename)
c.file(pathname,
content,
owner='root',
group='root',
mode='0644',
backup=False,
source=pathname[1:])
c.execute('tar xf {0}'.format(pathname), cwd=dirname)
# Place files.
for pathname, f in sorted(self.files.iteritems()):
c.directory(os.path.dirname(pathname),
group='root',
mode='755',
owner='root',
recursive=True)
if '120000' == f['mode'] or '120777' == f['mode']:
c.link(pathname,
owner=f['owner'],
group=f['group'],
to=f['content'])
continue
content = f['content']
if 'base64' == f['encoding']:
content = base64.b64decode(content)
c.file(pathname, content,
owner=f['owner'],
group=f['group'],
mode=f['mode'][-4:],
backup=False,
source=pathname[1:])
# Install packages.
def before(manager):
if 'apt' == manager.name:
c.execute('apt-get -q update')
def package(manager, package, version):
if manager.name == package:
return
if 'apt' == manager.name:
c.apt_package(package, version=version)
match = re.match(r'^rubygems(\d+\.\d+(?:\.\d+)?)$', package)
if match is not None and rubygems_update():
c.execute('/usr/bin/gem{0} install --no-rdoc --no-ri '
'rubygems-update'.format(match.group(1)))
c.execute('/usr/bin/ruby{0} '
'$(PATH=$PATH:/var/lib/gems/{0}/bin '
'which update_rubygems)"'.format(match.group(1)))
# All types of gems get to have package resources.
elif re.search(r'ruby', manager.name) is not None:
match = re.match(r'^ruby(?:gems)?(\d+\.\d+(?:\.\d+)?)',
manager.name)
c.gem_package(package,
gem_binary='/usr/bin/gem{0}'.format(match.group(1)),
version=version)
# Everything else is an execute resource.
else:
c.execute(manager(package, version))
self.walk(before=before, package=package)
return c
def puppet(self):
"""
Generate Puppet code.
"""
m = puppet.Manifest(self.name, comment=self.DISCLAIMER)
# Set the default `PATH` for exec resources.
m.add(puppet.Exec.defaults(path=os.environ['PATH']))
# Extract source tarballs.
tree = git.tree(self._commit)
for dirname, filename in sorted(self.sources.iteritems()):
blob = git.blob(tree, filename)
content = git.content(blob)
pathname = os.path.join('/tmp', filename)
m['sources'].add(puppet.File(
pathname,
self.name,
content,
owner='root',
group='root',
mode='0644',
source='puppet:///{0}/{1}'.format(self.name,
pathname[1:])))
m['sources'].add(puppet.Exec(
'tar xf {0}'.format(pathname),
cwd=dirname,
require=puppet.File.ref(pathname)))
# Place files.
if 0 < len(self.files):
for pathname, f in sorted(self.files.iteritems()):
# Create resources for parent directories and let the
# autorequire mechanism work out dependencies.
dirnames = os.path.dirname(pathname).split('/')[1:]
for i in xrange(len(dirnames)):
m['files'].add(puppet.File(
os.path.join('/', *dirnames[0:i+1]),
ensure='directory'))
# Create the actual file resource.
if '120000' == f['mode'] or '120777' == f['mode']:
m['files'].add(puppet.File(pathname,
None,
None,
owner=f['owner'],
group=f['group'],
ensure=f['content']))
continue
content = f['content']
if 'base64' == f['encoding']:
content = base64.b64decode(content)
m['files'].add(puppet.File(pathname,
self.name,
content,
owner=f['owner'],
group=f['group'],
mode=f['mode'][-4:],
ensure='file'))
# Install packages.
deps = []
def before(manager):
deps.append(manager)
if 'apt' != manager.name:
return
if 0 == len(manager):
return
if 1 == len(manager) and manager.name in manager:
return
m['packages'].add(puppet.Exec('apt-get -q update',
before=puppet.Class.ref('apt')))
def package(manager, package, version):
# `apt` is easy since it's the default.
if 'apt' == manager.name:
m['packages'][manager].add(puppet.Package(package,
ensure=version))
# If APT is installing RubyGems, get complicated.
match = re.match(r'^rubygems(\d+\.\d+(?:\.\d+)?)$', package)
if match is not None and rubygems_update():
m['packages'][manager].add(puppet.Exec('/bin/sh -c "'
'/usr/bin/gem{0} install --no-rdoc --no-ri '
'rubygems-update; '
'/usr/bin/ruby{0} $(PATH=$PATH:/var/lib/gems/{0}/bin '
'which update_rubygems)"'.format(match.group(1)),
require=puppet.Package.ref(package)))
# RubyGems for Ruby 1.8 is easy, too, because Puppet has a
# built in provider.
elif 'rubygems1.8' == manager.name:
m['packages'][manager].add(puppet.Package(package,
ensure=version,
provider='gem'))
# Other versions of RubyGems are slightly more complicated.
elif re.search(r'ruby', manager.name) is not None:
match = re.match(r'^ruby(?:gems)?(\d+\.\d+(?:\.\d+)?)',
manager.name)
m['packages'][manager].add(puppet.Exec(
manager(package, version),
creates='{0}/{1}/gems/{2}-{3}'.format(rubygems_path(),
match.group(1),
package,
version)))
# Python works basically like alternative versions of Ruby
# but follows a less predictable directory structure so the
# directory is not known ahead of time. This just so happens
# to be the way everything else works, too.
else:
m['packages'][manager].add(puppet.Exec(
manager(package, version)))
self.walk(before=before, package=package)
m['packages'].dep(*[puppet.Class.ref(dep) for dep in deps])
# Strict ordering of classes.
deps = []
if 0 < len(self.sources):
deps.append('sources')
if 0 < len(self.files):
deps.append('files')
if 0 < len(self.packages):
deps.append('packages')
m.dep(*[puppet.Class.ref(dep) for dep in deps])
return m
def sh(self, server='https://devstructure.com', secret=None):
"""
Generate shell code.
"""
import sh
s = sh.Script(self.name, comment=self.DISCLAIMER)
# Extract source tarballs.
if secret is not None:
for dirname, filename in sorted(self.sources.iteritems()):
s.add('wget "{0}/{1}/{2}/{3}"',
server,
secret,
self.name,
filename)
s.add('tar xf "{0}" -C "{1}"',
filename,
dirname)
else:
tree = git.tree(self._commit)
for dirname, filename in sorted(self.sources.iteritems()):
blob = git.blob(tree, filename)
content = git.content(blob)
s.add('tar xf "{0}" -C "{1}"',
filename,
dirname,
sources={filename: content})
# Place files.
for pathname, f in sorted(self.files.iteritems()):
s.add('mkdir -p "{0}"', os.path.dirname(pathname))
if '120000' == f['mode'] or '120777' == f['mode']:
s.add('ln -s "{0}" "{1}"', f['content'], pathname)
continue
command = 'cat'
if 'base64' == f['encoding']:
command = 'base64 --decode'
eof = 'EOF'
while re.search(r'{0}'.format(eof), f['content']):
eof += 'EOF'
s.add('{0} >"{1}" <<{2}', command, pathname, eof)
s.add(raw=f['content'])
if 0 < len(f['content']) and '\n' != f['content'][-1]:
eof = '\n{0}'.format(eof)
s.add(eof)
if 'root' != f['owner']:
s.add('chown {0} "{1}"', f['owner'], pathname)
if 'root' != f['group']:
s.add('chgrp {0} "{1}"', f['group'], pathname)
if '000644' != f['mode']:
s.add('chmod {0} "{1}"', f['mode'][-4:], pathname)
# Install packages.
def before(manager):
if 0 == len(manager):
return
if 'apt' == manager.name:
s.add('export APT_LISTBUGS_FRONTEND="none"')
s.add('export APT_LISTCHANGES_FRONTEND="none"')
s.add('export DEBIAN_FRONTEND="noninteractive"')
s.add('apt-get -q update')
elif 'yum' == manager.name:
s.add('yum makecache')
def package(manager, package, version):
if manager.name == package:
return
s.add(manager(package, version))
if manager.name not in ('apt', 'yum'):
return
# See comments on this section in `puppet` above.
match = re.match(r'^rubygems(\d+\.\d+(?:\.\d+)?)$', package)
if match is not None and util.rubygems_update():
s.add('/usr/bin/gem{0} install --no-rdoc --no-ri '
'rubygems-update', match.group(1))
s.add('/usr/bin/ruby{0} $(PATH=$PATH:/var/lib/gems/{0}/bin '
'which update_rubygems)', match.group(1))
self.walk(before=before, package=package)
return s
def walk(b, choose):
"""
Given a function for choosing a `Blueprint` object (based typically on
the result of a `raw_input` call within the `choose` function), populate
one or more `Blueprint`s closed into `choose`.
"""
def file(pathname, f):
print(pathname)
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_file(pathname, **f)
def package(manager, package, version):
print('{0} {1} {2}'.format(manager, package, version))
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_package(manager, package, version)
def service(manager, service):
print('{0} {1}'.format(manager, service))
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_service(manager, service)
def service_file(manager, service, pathname):
b_chosen.add_service_file(manager, service, pathname)
walklib.walk_service_files(b_chosen,
manager,
service,
service_file=service_file)
def service_package(manager, service, package_manager, package):
b_chosen.add_service_package(manager,
service,
package_manager,
package)
walklib.walk_service_packages(b_chosen,
manager,
service,
service_package=service_package)
def service_source(manager, service, dirname):
b_chosen.add_service_source(manager, service, dirname)
walklib.walk_service_sources(b_chosen,
manager,
service,
service_source=service_source)
commit = git.rev_parse(b.name)
tree = git.tree(commit)
def source(dirname, filename, gen_content, url):
if url is not None:
print('{0} {1}'.format(dirname, url))
elif gen_content is not None:
blob = git.blob(tree, filename)
git.cat_file(blob, filename)
print('{0} {1}'.format(dirname, filename))
b_chosen = choose()
if b_chosen is None:
return
b_chosen.add_source(dirname, filename)
b.walk(file=file, package=package, service=service, source=source)
