def push(server, secret, b):
"""
Push a blueprint to the secret and its name on the configured server.
"""
r = http.put('/{0}/{1}'.format(secret, b.name),
b.dumps(),
{'Content-Type': 'application/json'},
server=server)
if 202 == r.status:
pass
elif 400 == r.status:
logging.error('malformed blueprint')
return None
elif 502 == r.status:
logging.error('upstream storage service failed')
return None
else:
logging.error('unexpected {0} storing blueprint'.format(r.status))
return None
if b._commit is None and 0 < len(b.sources):
logging.warning('blueprint came from standard input - '
'source tarballs will not be pushed')
elif b._commit is not None:
tree = git.tree(b._commit)
for dirname, filename in sorted(b.sources.iteritems()):
blob = git.blob(tree, filename)
content = git.content(blob)
logging.info('storing source tarballs - this may take a while')
r = http.put('/{0}/{1}/{2}'.format(secret, b.name, filename),
content,
{'Content-Type': 'application/x-tar'},
server=server)
if 202 == r.status:
pass
elif 400 == r.status:
logging.error('tarball content or name not expected')
return None
elif 404 == r.status:
logging.error('blueprint not found')
return None
elif 413 == r.status:
logging.error('tarballs can\'t exceed 64MB')
return None
elif 502 == r.status:
logging.error('upstream storage service failed')
return None
else:
logging.error('unexpected {0} storing tarball'.
format(r.status))
return None
return '{0}/{1}/{2}'.format(server, secret, b.name)
def push(server, secret, b):
"""
Push a blueprint to the secret and its name on the configured server.
"""
r = http.put("/{0}/{1}".format(secret, b.name), b.dumps(), {"Content-Type": "application/json"}, server=server)
if 202 == r.status:
pass
elif 400 == r.status:
logging.error("malformed blueprint")
return None
elif 502 == r.status:
logging.error("upstream storage service failed")
return None
else:
logging.error("unexpected {0} storing blueprint".format(r.status))
return None
tree = git.tree(b._commit)
for dirname, filename in sorted(b.sources.iteritems()):
blob = git.blob(tree, filename)
content = git.content(blob)
logging.info("storing source tarballs - this may take a while")
r = http.put(
"/{0}/{1}/{2}".format(secret, b.name, filename),
content,
{"Content-Type": "application/x-tar"},
server=server,
)
if 202 == r.status:
pass
elif 400 == r.status:
logging.error("tarball content or name not expected")
return None
elif 404 == r.status:
logging.error("blueprint not found")
return None
elif 413 == r.status:
logging.error("tarballs can't exceed 64MB")
return None
elif 502 == r.status:
logging.error("upstream storage service failed")
return None
else:
logging.error("unexpected {0} storing tarball".format(r.status))
return None
return "{0}/{1}/{2}".format(server, secret, b.name)
def sh(b, relaxed=False, server='https://devstructure.com', secret=None):
"""
Generate shell code.
"""
s = Script(b.name, comment=b.DISCLAIMER)
# Build an inverted index (lookup table, like in hardware, hence the name)
# of service dependencies to services.
lut = {'files': defaultdict(set),
'packages': defaultdict(lambda: defaultdict(set)),
'sources': defaultdict(set)}
def service_file(manager, service, pathname):
lut['files'][pathname].add((manager, service))
def service_package(manager, service, package_manager, package):
lut['packages'][package_manager][package].add((manager, service))
def service_source(manager, service, dirname):
lut['sources'][dirname].add((manager, service))
b.walk(service_file=service_file,
service_package=service_package,
service_source=service_source)
commit = git.rev_parse(b.name)
tree = None if commit is None else git.tree(commit)
def source(dirname, filename, gen_content, url):
"""
Extract a source tarball.
"""
if dirname in lut['sources']:
s.add('MD5SUM="$(find "{0}" -printf %T@\\\\n | md5sum)"',
args=(dirname,))
if url is not None:
s.add_list(('curl -o "{0}" "{1}"',),
('wget -O "{0}" "{1}"',),
args=(filename, url),
operator='||')
if '.zip' == pathname[-4:]:
s.add('unzip "{0}" -d "{1}"', args=(filename, dirname))
else:
s.add('mkdir -p "{1}" && tar xf "{0}" -C "{1}"', args=(filename, dirname))
elif secret is not None:
s.add_list(('curl -O "{0}/{1}/{2}/{3}"',),
('wget "{0}/{1}/{2}/{3}"',),
args=(server, secret, b.name, filename),
operator='||')
s.add('mkdir -p "{1}" && tar xf "{0}" -C "{1}"', args=(filename, dirname))
elif gen_content is not None:
s.add('mkdir -p "{1}" && tar xf "{0}" -C "{1}"', args=(filename, dirname))
s.add_source(filename, git.blob(tree, filename))
for manager, service in lut['sources'][dirname]:
s.add_list(('[ "$MD5SUM" != "$(find "{0}" -printf %T@\\\\n '
'| md5sum)" ]',),
('{1}=1',),
args=(dirname, manager.env_var(service)),
operator='&&')
def file(pathname, f):
"""
Place a file.
"""
if pathname in lut['files']:
s.add('MD5SUM="$(md5sum "{0}" 2>/dev/null)"', args=(pathname,))
s.add('mkdir -p "{0}"', args=(os.path.dirname(pathname),))
if '120000' == f['mode'] or '120777' == f['mode']:
s.add('ln -s "{0}" "{1}"', args=(f['content'], pathname))
else:
if 'source' in f:
s.add_list(('curl -o "{0}" "{1}"',),
('wget -O "{0}" "{1}"',),
args=(pathname, f['source']),
operator='||')
else:
if 'template' in f:
s.templates = True
if 'base64' == f['encoding']:
commands = ('base64 --decode', 'mustache')
else:
commands = ('mustache',)
s.add_list(('set +x',),
('. "lib/mustache.sh"',),
('for F in */blueprint-template.d/*.sh',),
('do',),
('\t. "$F"',),
('done',),
(f.get('data', '').rstrip(),),
(command(*commands,
escape_stdin=True,
stdin=f['template'],
stdout=pathname),),
operator='\n',
wrapper='()')
else:
if 'base64' == f['encoding']:
commands = ('base64 --decode',)
else:
commands = ('cat',)
s.add(*commands, stdin=f['content'], stdout=pathname)
if 'root' != f['owner']:
s.add('chown {0} "{1}"', args=(f['owner'], pathname))
if 'root' != f['group']:
s.add('chgrp {0} "{1}"', args=(f['group'], pathname))
if '100644' != f['mode']:
s.add('chmod {0} "{1}"', args=(f['mode'][-4:], pathname))
for manager, service in lut['files'][pathname]:
s.add('[ "$MD5SUM" != "$(md5sum "{0}")" ] && {1}=1',
args=(pathname, manager.env_var(service)))
def before_packages(manager):
"""
Configure the package managers.
"""
if manager not in b.packages:
return
if 'apt' == manager:
s.add('export APT_LISTBUGS_FRONTEND="none"')
s.add('export APT_LISTCHANGES_FRONTEND="none"')
s.add('export DEBIAN_FRONTEND="noninteractive"')
s.add('apt-get -q update')
elif 'yum' == manager:
s.add('yum makecache')
def package(manager, package, version):
"""
Install a package.
"""
if manager == package:
return
if manager in lut['packages'] and package in lut['packages'][manager]:
s.add_list((manager.gate(package, version, relaxed),),
(command_list((manager.install(package,
version,
relaxed),),
*[('{0}=1'.format(m.env_var(service)),)
for m, service in
lut['packages'][manager][package]],
wrapper='{{}}'),),
operator='||')
else:
s.add(manager(package, version, relaxed))
if manager not in ('apt', 'rpm', 'yum'):
return
# See comments on this section in `blueprint.frontend.puppet`.
match = re.match(r'^rubygems(\d+\.\d+(?:\.\d+)?)$', package)
if match is not None and util.rubygems_update():
s.add('/usr/bin/gem{0} install --no-rdoc --no-ri rubygems-update',
args=(match.group(1),))
s.add('/usr/bin/ruby{0} $(PATH=$PATH:/var/lib/gems/{0}/bin '
'which update_rubygems)',
args=(match.group(1),))
if 'nodejs' == package:
s.add_list(('which npm',),
(command_list(('curl http://npmjs.org/install.sh',),
('wget -O- http://npmjs.org/install.sh',),
operator='||',
wrapper='{{}}'),
'sh'),
operator='||')
def service(manager, service):
s.add(manager(service))
b.walk(source=source,
file=file,
before_packages=before_packages,
package=package,
service=service)
return s
def sh(b, relaxed=False, server="https://devstructure.com", secret=None):
"""
Generate shell code.
"""
s = Script(b.name, comment=b.DISCLAIMER)
# Build an inverted index (lookup table, like in hardware, hence the name)
# of service dependencies to services.
lut = {"files": defaultdict(set), "packages": defaultdict(lambda: defaultdict(set)), "sources": defaultdict(set)}
def service_file(manager, service, pathname):
lut["files"][pathname].add((manager, service))
def service_package(manager, service, package_manager, package):
lut["packages"][package_manager][package].add((manager, service))
def service_source(manager, service, dirname):
lut["sources"][dirname].add((manager, service))
b.walk(service_file=service_file, service_package=service_package, service_source=service_source)
commit = git.rev_parse(b.name)
tree = git.tree(commit)
def source(dirname, filename, gen_content, url):
"""
Extract a source tarball.
"""
if dirname in lut["sources"]:
s.add('MD5SUM="$(find "{0}" -printf %T@\\\\n | md5sum)"', args=(dirname,))
if url is not None:
s.add_list(('curl -o "{0}" "{1}"',), ('wget -O "{0}" "{1}"',), args=(filename, url), operator="||")
if ".zip" == pathname[-4:]:
s.add('unzip "{0}" -d "{1}"', args=(filename, dirname))
else:
s.add('tar xf "{0}" -C "{1}"', args=(filename, dirname))
elif secret is not None:
s.add_list(
('curl -O "{0}/{1}/{2}/{3}"',)('wget "{0}/{1}/{2}/{3}"'),
args=(server, secret, b.name, filename),
operator="||",
)
s.add('tar xf "{0}" -C "{1}"', args=(filename, dirname))
elif gen_content is not None:
s.add('tar xf "{0}" -C "{1}"', args=(filename, dirname))
s.add_source(filename, git.blob(tree, filename))
for manager, service in lut["sources"][dirname]:
s.add_list(
('[ "$MD5SUM" != "$(find "{0}" -printf %T@\\\\n ' '| md5sum)" ]',),
("{1}=1",),
args=(dirname, manager.env_var(service)),
operator="&&",
)
def file(pathname, f):
"""
Place a file.
"""
if pathname in lut["files"]:
s.add('MD5SUM="$(md5sum "{0}" 2>/dev/null)"', args=(pathname,))
s.add('mkdir -p "{0}"', args=(os.path.dirname(pathname),))
if "120000" == f["mode"] or "120777" == f["mode"]:
s.add('ln -s "{0}" "{1}"', args=(f["content"], pathname))
else:
if "source" in f:
s.add_list(
('curl -o "{0}" "{1}"',), ('wget -O "{0}" "{1}"',), args=(pathname, f["source"]), operator="||"
)
else:
if "template" in f:
s.templates = True
if "base64" == f["encoding"]:
commands = ("base64 --decode", "mustache")
else:
commands = ("mustache",)
s.add_list(
("set +x",),
('. "lib/mustache.sh"',),
("for F in */blueprint-template.d/*.sh",),
("do",),
('\t. "$F"',),
("done",),
(f["data"].rstrip(),),
(command(*commands, escape_stdin=True, stdin=f["template"], stdout=pathname),),
operator="\n",
wrapper="()",
)
else:
if "base64" == f["encoding"]:
commands = ("base64 --decode",)
else:
commands = ("cat",)
s.add(*commands, stdin=f["content"], stdout=pathname)
if "root" != f["owner"]:
s.add('chown {0} "{1}"', args=(f["owner"], pathname))
if "root" != f["group"]:
s.add('chgrp {0} "{1}"', args=(f["group"], pathname))
if "100644" != f["mode"]:
s.add('chmod {0} "{1}"', args=(f["mode"][-4:], pathname))
for manager, service in lut["files"][pathname]:
s.add('[ "$MD5SUM" != "$(md5sum "{0}")" ] && {1}=1', args=(pathname, manager.env_var(service)))
def before_packages(manager):
"""
Configure the package managers.
"""
if manager not in b.packages:
return
if "apt" == manager:
s.add('export APT_LISTBUGS_FRONTEND="none"')
s.add('export APT_LISTCHANGES_FRONTEND="none"')
s.add('export DEBIAN_FRONTEND="noninteractive"')
s.add("apt-get -q update")
elif "yum" == manager:
s.add("yum makecache")
def package(manager, package, version):
"""
Install a package.
"""
if manager == package:
return
if manager in lut["packages"] and package in lut["packages"][manager]:
s.add_list(
(manager.gate(package, version, relaxed),),
(
command_list(
(manager.install(package, version, relaxed),),
*[("{0}=1".format(m.env_var(service)),) for m, service in lut["packages"][manager][package]],
wrapper="{{}}"
),
),
operator="||",
)
else:
s.add(manager(package, version, relaxed))
if manager not in ("apt", "rpm", "yum"):
return
# See comments on this section in `blueprint.frontend.puppet`.
match = re.match(r"^rubygems(\d+\.\d+(?:\.\d+)?)$", package)
if match is not None and util.rubygems_update():
s.add("/usr/bin/gem{0} install --no-rdoc --no-ri rubygems-update", args=(match.group(1),))
s.add(
"/usr/bin/ruby{0} $(PATH=$PATH:/var/lib/gems/{0}/bin " "which update_rubygems)", args=(match.group(1),)
)
if "nodejs" == package:
s.add_list(
("which npm",),
(
command_list(
("curl http://npmjs.org/install.sh",),
("wget -O- http://npmjs.org/install.sh",),
operator="||",
wrapper="{{}}",
),
"sh",
),
operator="||",
)
def service(manager, service):
s.add(manager(service))
b.walk(source=source, file=file, before_packages=before_packages, package=package, service=service)
return s
