def sh(b, relaxed=False, server='https://devstructure.com', secret=None): """ Generate shell code. """ s = Script(b.name, comment=b.DISCLAIMER) # Build an inverted index (lookup table, like in hardware, hence the name) # of service dependencies to services. lut = {'files': defaultdict(set), 'packages': defaultdict(lambda: defaultdict(set)), 'sources': defaultdict(set)} def service_file(manager, service, pathname): lut['files'][pathname].add((manager, service)) def service_package(manager, service, package_manager, package): lut['packages'][package_manager][package].add((manager, service)) def service_source(manager, service, dirname): lut['sources'][dirname].add((manager, service)) b.walk(service_file=service_file, service_package=service_package, service_source=service_source) commit = git.rev_parse(b.name) tree = None if commit is None else git.tree(commit) def source(dirname, filename, gen_content, url): """ Extract a source tarball. """ if dirname in lut['sources']: s.add('MD5SUM="$(find "{0}" -printf %T@\\\\n | md5sum)"', args=(dirname,)) if url is not None: s.add_list(('curl -o "{0}" "{1}"',), ('wget -O "{0}" "{1}"',), args=(filename, url), operator='||') if '.zip' == pathname[-4:]: s.add('unzip "{0}" -d "{1}"', args=(filename, dirname)) else: s.add('mkdir -p "{1}" && tar xf "{0}" -C "{1}"', args=(filename, dirname)) elif secret is not None: s.add_list(('curl -O "{0}/{1}/{2}/{3}"',), ('wget "{0}/{1}/{2}/{3}"',), args=(server, secret, b.name, filename), operator='||') s.add('mkdir -p "{1}" && tar xf "{0}" -C "{1}"', args=(filename, dirname)) elif gen_content is not None: s.add('mkdir -p "{1}" && tar xf "{0}" -C "{1}"', args=(filename, dirname)) s.add_source(filename, git.blob(tree, filename)) for manager, service in lut['sources'][dirname]: s.add_list(('[ "$MD5SUM" != "$(find "{0}" -printf %T@\\\\n ' '| md5sum)" ]',), ('{1}=1',), args=(dirname, manager.env_var(service)), operator='&&') def file(pathname, f): """ Place a file. """ if pathname in lut['files']: s.add('MD5SUM="$(md5sum "{0}" 2>/dev/null)"', args=(pathname,)) s.add('mkdir -p "{0}"', args=(os.path.dirname(pathname),)) if '120000' == f['mode'] or '120777' == f['mode']: s.add('ln -s "{0}" "{1}"', args=(f['content'], pathname)) else: if 'source' in f: s.add_list(('curl -o "{0}" "{1}"',), ('wget -O "{0}" "{1}"',), args=(pathname, f['source']), operator='||') else: if 'template' in f: s.templates = True if 'base64' == f['encoding']: commands = ('base64 --decode', 'mustache') else: commands = ('mustache',) s.add_list(('set +x',), ('. "lib/mustache.sh"',), ('for F in */blueprint-template.d/*.sh',), ('do',), ('\t. "$F"',), ('done',), (f.get('data', '').rstrip(),), (command(*commands, escape_stdin=True, stdin=f['template'], stdout=pathname),), operator='\n', wrapper='()') else: if 'base64' == f['encoding']: commands = ('base64 --decode',) else: commands = ('cat',) s.add(*commands, stdin=f['content'], stdout=pathname) if 'root' != f['owner']: s.add('chown {0} "{1}"', args=(f['owner'], pathname)) if 'root' != f['group']: s.add('chgrp {0} "{1}"', args=(f['group'], pathname)) if '100644' != f['mode']: s.add('chmod {0} "{1}"', args=(f['mode'][-4:], pathname)) for manager, service in lut['files'][pathname]: s.add('[ "$MD5SUM" != "$(md5sum "{0}")" ] && {1}=1', args=(pathname, manager.env_var(service))) def before_packages(manager): """ Configure the package managers. """ if manager not in b.packages: return if 'apt' == manager: s.add('export APT_LISTBUGS_FRONTEND="none"') s.add('export APT_LISTCHANGES_FRONTEND="none"') s.add('export DEBIAN_FRONTEND="noninteractive"') s.add('apt-get -q update') elif 'yum' == manager: s.add('yum makecache') def package(manager, package, version): """ Install a package. """ if manager == package: return if manager in lut['packages'] and package in lut['packages'][manager]: s.add_list((manager.gate(package, version, relaxed),), (command_list((manager.install(package, version, relaxed),), *[('{0}=1'.format(m.env_var(service)),) for m, service in lut['packages'][manager][package]], wrapper='{{}}'),), operator='||') else: s.add(manager(package, version, relaxed)) if manager not in ('apt', 'rpm', 'yum'): return # See comments on this section in `blueprint.frontend.puppet`. match = re.match(r'^rubygems(\d+\.\d+(?:\.\d+)?)$', package) if match is not None and util.rubygems_update(): s.add('/usr/bin/gem{0} install --no-rdoc --no-ri rubygems-update', args=(match.group(1),)) s.add('/usr/bin/ruby{0} $(PATH=$PATH:/var/lib/gems/{0}/bin ' 'which update_rubygems)', args=(match.group(1),)) if 'nodejs' == package: s.add_list(('which npm',), (command_list(('curl http://npmjs.org/install.sh',), ('wget -O- http://npmjs.org/install.sh',), operator='||', wrapper='{{}}'), 'sh'), operator='||') def service(manager, service): s.add(manager(service)) b.walk(source=source, file=file, before_packages=before_packages, package=package, service=service) return s
def sh(b, relaxed=False, server="https://devstructure.com", secret=None): """ Generate shell code. """ s = Script(b.name, comment=b.DISCLAIMER) # Build an inverted index (lookup table, like in hardware, hence the name) # of service dependencies to services. lut = {"files": defaultdict(set), "packages": defaultdict(lambda: defaultdict(set)), "sources": defaultdict(set)} def service_file(manager, service, pathname): lut["files"][pathname].add((manager, service)) def service_package(manager, service, package_manager, package): lut["packages"][package_manager][package].add((manager, service)) def service_source(manager, service, dirname): lut["sources"][dirname].add((manager, service)) b.walk(service_file=service_file, service_package=service_package, service_source=service_source) commit = git.rev_parse(b.name) tree = git.tree(commit) def source(dirname, filename, gen_content, url): """ Extract a source tarball. """ if dirname in lut["sources"]: s.add('MD5SUM="$(find "{0}" -printf %T@\\\\n | md5sum)"', args=(dirname,)) if url is not None: s.add_list(('curl -o "{0}" "{1}"',), ('wget -O "{0}" "{1}"',), args=(filename, url), operator="||") if ".zip" == pathname[-4:]: s.add('unzip "{0}" -d "{1}"', args=(filename, dirname)) else: s.add('tar xf "{0}" -C "{1}"', args=(filename, dirname)) elif secret is not None: s.add_list( ('curl -O "{0}/{1}/{2}/{3}"',)('wget "{0}/{1}/{2}/{3}"'), args=(server, secret, b.name, filename), operator="||", ) s.add('tar xf "{0}" -C "{1}"', args=(filename, dirname)) elif gen_content is not None: s.add('tar xf "{0}" -C "{1}"', args=(filename, dirname)) s.add_source(filename, git.blob(tree, filename)) for manager, service in lut["sources"][dirname]: s.add_list( ('[ "$MD5SUM" != "$(find "{0}" -printf %T@\\\\n ' '| md5sum)" ]',), ("{1}=1",), args=(dirname, manager.env_var(service)), operator="&&", ) def file(pathname, f): """ Place a file. """ if pathname in lut["files"]: s.add('MD5SUM="$(md5sum "{0}" 2>/dev/null)"', args=(pathname,)) s.add('mkdir -p "{0}"', args=(os.path.dirname(pathname),)) if "120000" == f["mode"] or "120777" == f["mode"]: s.add('ln -s "{0}" "{1}"', args=(f["content"], pathname)) else: if "source" in f: s.add_list( ('curl -o "{0}" "{1}"',), ('wget -O "{0}" "{1}"',), args=(pathname, f["source"]), operator="||" ) else: if "template" in f: s.templates = True if "base64" == f["encoding"]: commands = ("base64 --decode", "mustache") else: commands = ("mustache",) s.add_list( ("set +x",), ('. "lib/mustache.sh"',), ("for F in */blueprint-template.d/*.sh",), ("do",), ('\t. "$F"',), ("done",), (f["data"].rstrip(),), (command(*commands, escape_stdin=True, stdin=f["template"], stdout=pathname),), operator="\n", wrapper="()", ) else: if "base64" == f["encoding"]: commands = ("base64 --decode",) else: commands = ("cat",) s.add(*commands, stdin=f["content"], stdout=pathname) if "root" != f["owner"]: s.add('chown {0} "{1}"', args=(f["owner"], pathname)) if "root" != f["group"]: s.add('chgrp {0} "{1}"', args=(f["group"], pathname)) if "100644" != f["mode"]: s.add('chmod {0} "{1}"', args=(f["mode"][-4:], pathname)) for manager, service in lut["files"][pathname]: s.add('[ "$MD5SUM" != "$(md5sum "{0}")" ] && {1}=1', args=(pathname, manager.env_var(service))) def before_packages(manager): """ Configure the package managers. """ if manager not in b.packages: return if "apt" == manager: s.add('export APT_LISTBUGS_FRONTEND="none"') s.add('export APT_LISTCHANGES_FRONTEND="none"') s.add('export DEBIAN_FRONTEND="noninteractive"') s.add("apt-get -q update") elif "yum" == manager: s.add("yum makecache") def package(manager, package, version): """ Install a package. """ if manager == package: return if manager in lut["packages"] and package in lut["packages"][manager]: s.add_list( (manager.gate(package, version, relaxed),), ( command_list( (manager.install(package, version, relaxed),), *[("{0}=1".format(m.env_var(service)),) for m, service in lut["packages"][manager][package]], wrapper="{{}}" ), ), operator="||", ) else: s.add(manager(package, version, relaxed)) if manager not in ("apt", "rpm", "yum"): return # See comments on this section in `blueprint.frontend.puppet`. match = re.match(r"^rubygems(\d+\.\d+(?:\.\d+)?)$", package) if match is not None and util.rubygems_update(): s.add("/usr/bin/gem{0} install --no-rdoc --no-ri rubygems-update", args=(match.group(1),)) s.add( "/usr/bin/ruby{0} $(PATH=$PATH:/var/lib/gems/{0}/bin " "which update_rubygems)", args=(match.group(1),) ) if "nodejs" == package: s.add_list( ("which npm",), ( command_list( ("curl http://npmjs.org/install.sh",), ("wget -O- http://npmjs.org/install.sh",), operator="||", wrapper="{{}}", ), "sh", ), operator="||", ) def service(manager, service): s.add(manager(service)) b.walk(source=source, file=file, before_packages=before_packages, package=package, service=service) return s