def replace(filename, old, new, variable='version'): if variable is None: regex = "s/%s/%s/"%(old, new) else: regex = "s,%s[ =].*['\"]%s['\"],%s = '%s',"%(variable, old, variable, new) command = [ 'sed', '-e', regex, '-i', filename, ] stacks.process(command)
def run(cmd): # Check that there isn't already a key if cmd.repo.key_exists( cmd.parent.opts.key_dir, cmd.parent.opts.base_repo_dir ): cmd.err( 'Cannot install a new key because ' 'packages may have already been signed with the existing one' ) return 1 cmd_ = '. /usr/lib/buildkit/buildkit_common.sh\nbuildkit_ensure_public_key "%s" "%s" "%s" "%s" "%s"\nchown -R buildkit:buildkit /var/lib/buildkit/key\nchown buildkit:www-data /var/lib/buildkit/repo/packages_public.key\nexit'%( cmd.opts.name, cmd.opts.email, cmd.opts.comment, #cmd.opts.password, cmd.parent.opts.key_dir, cmd.parent.opts.base_repo_dir, ) cmd.out(cmd_) result = stacks.process( cmd_, # cwd=os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]), shell=True, echo=True, ) cmd.out(result.stdout)
def run(scm, cmd, path, error_msg): res = stacks.process(cmd, cwd=path, merge=True) if res.retcode != 0: scm.log.error(str(cmd)) scm.log.error(res.stdout) raise Exception(error_msg) return res
def run(cmd): cmd_ = 'find . | grep .deb' result = stacks.process( cmd_, cwd=os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]), shell=True, merge=True, ) for line in result.stdout.strip().split(): src = os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0], line.strip()) dst = os.path.join(cmd.opts.output_dir, line.split('/')[-1]) cmd.out(src) shutil.copyfile(src, dst)
def get_pkg_info(path): # Cope with capitalisation variations stacks.process(['python', 'setup.py', 'egg_info'], cwd=path) files = os.listdir(path) name = None for filename in files: if os.path.isdir(os.path.join(path, filename)) and filename.endswith('.egg-info'): name = filename[:-len('.egg-info')] if name is None: raise Exception('Couldn\'t determine the package name for package in %r, is it really a setuptools-enabled package?'%path) if os.path.exists(os.path.join(path, '%s.egg-info'%name.replace('-','_'), 'PKG-INFO')): pkg_info_path = os.path.join(path, '%s.egg-info'%name.replace('-','_'), 'PKG-INFO') elif os.path.exists(os.path.join(path, 'PKG-INFO')): pkg_info_path = os.path.join(path, 'PKG-INFO') else: log.error('No PKG-INFO found for %s in %s', name, path) return {} fp = open( pkg_info_path, 'rb', ) data = fp.read() fp.close() pkg_info = {} last = None for line in data.split('\n'): if line.startswith(' '): pkg_info[last][-1] += '\n'+line[8:] else: parts = line.split(': ') last = parts[0] if not pkg_info.has_key(last): pkg_info[last] = [] value = ': '.join(parts[1:]) if value.startswith('Development Status :: '): pkg_info['Status'] = [value.split('-')[-1].strip()] pkg_info[last].append(value) return pkg_info
def run(cmd): # Check that a key is installed if not cmd.repo.key_exists( cmd.parent.opts.key_dir, cmd.parent.opts.base_repo_dir ): cmd.err( 'No repo key has ben set up, please run `buildkit repo ' 'installkey\' to install one' ) return 1 package_names = cmd.args[1:] if cmd.opts.all: cmd_ = 'reprepro list lucid' result = stacks.process( cmd_, cwd=os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]), shell=True, ) for line in result.stdout.split('\n'): if line.strip(): info, name = line.split(':') name, version = name.strip().split(' ') if name not in package_names: package_names.append(name) if not package_names: cmd.err('ERROR: No packages to remove') return 1 cmd_ = 'reprepro --gnupghome "%s" remove lucid %s' % (cmd.parent.opts.key_dir, ' '.join(package_names)) cmd.out(cmd_) result = stacks.process( cmd_, cwd=os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]), shell=True, ) cmd.out(result.stdout)
def run(cmd): # Check that a key is installed if not cmd.repo.key_exists( cmd.parent.opts.key_dir, cmd.parent.opts.base_repo_dir ): cmd.err( 'No repo key has ben set up, please run `buildkit repo ' 'installkey\' to install one' ) return 1 cmd_ = 'reprepro list lucid' result = stacks.process( cmd_, cwd=os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]), shell=True, merge=True, ) cmd.out(result.stdout.strip())
def run(cmd): # Check that a key is installed if not cmd.repo.key_exists( cmd.parent.opts.key_dir, cmd.parent.opts.base_repo_dir ): cmd.err( 'No repo key has ben set up, please run `buildkit repo ' 'installkey\' to install one' ) return 1 packages = cmd.args[1:] abs_paths = [] for package in packages: abs_paths.append(stacks.uniform_path(package)) if cmd.opts.find_in: cmd_ = 'find "%s" | grep .deb$ | xargs --no-run-if-empty reprepro --gnupghome "%s" includedeb lucid %s' % ( stacks.uniform_path(cmd.opts.find_in), cmd.parent.opts.key_dir, abs_paths and '"' + '" "'.join(abs_paths) + '"' or '', ) else: cmd_ = ' '.join([ 'reprepro', '--gnupghome "%s"' % cmd.parent.opts.key_dir, 'includedeb', 'lucid', ] + abs_paths) cmd.out(cmd_) result = stacks.process( cmd_, cwd = os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]), merge=True, shell=True, ) cmd.out(result.stdout)
def test_02_start(self): expected = """\ running bdist_egg running egg_info creating generated_package.egg-info writing generated_package.egg-info/PKG-INFO writing top-level names to generated_package.egg-info/top_level.txt writing dependency_links to generated_package.egg-info/dependency_links.txt writing entry points to generated_package.egg-info/entry_points.txt writing manifest file 'generated_package.egg-info/SOURCES.txt' reading manifest file 'generated_package.egg-info/SOURCES.txt' reading manifest template 'MANIFEST.in' warning: no files found matching 'example/*.py' warning: no files found matching 'ez_setup.py' writing manifest file 'generated_package.egg-info/SOURCES.txt' installing library code to build/bdist.linux-x86_64/egg running install_lib running build_py creating build creating build/lib.linux-x86_64-2.6 creating build/lib.linux-x86_64-2.6/generated_package copying generated_package/__init__.py -> build/lib.linux-x86_64-2.6/generated_package creating build/bdist.linux-x86_64 creating build/bdist.linux-x86_64/egg creating build/bdist.linux-x86_64/egg/generated_package copying build/lib.linux-x86_64-2.6/generated_package/__init__.py -> build/bdist.linux-x86_64/egg/generated_package byte-compiling build/bdist.linux-x86_64/egg/generated_package/__init__.py to __init__.pyc creating build/bdist.linux-x86_64/egg/EGG-INFO copying generated_package.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO copying generated_package.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO copying generated_package.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO copying generated_package.egg-info/entry_points.txt -> build/bdist.linux-x86_64/egg/EGG-INFO copying generated_package.egg-info/not-zip-safe -> build/bdist.linux-x86_64/egg/EGG-INFO copying generated_package.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO creating dist creating 'dist/generated_package-0.1.2-py2.6.egg' and adding 'build/bdist.linux-x86_64/egg' to it removing 'build/bdist.linux-x86_64/egg' (and everything under it) """ here = stacks.uniform_path(os.path.dirname(__file__)) if os.path.exists(os.path.join(here, 'generate_package')): shutil.rmtree(os.path.join(here, 'generate_package')) output = [] def out(*k, **p): output.append(stacks.format_string(*k, **p)) result = stacks.run( argv=[ 'start', '--git', '--author-name', 'James Gardner', '--author-email', '*****@*****.**', '--version', '0.1.2', 'generated_package', 'James\'s test package', os.path.join(here, 'generate_package'), ], out=out, err=out, **run_args ) self.assertEqual(result, 0) self.assertEqual(len(output), 0) # Now check that the directory structure is identical apart from the ignored file cmd = "python setup.py bdist_egg" result = stacks.process( cmd.split(' '), cwd=os.path.join(here, 'generate_package') ) for line in difflib.unified_diff( expected.split('\n'), result.stdout.split('\n'), ): print line self.assertEqual(result.stdout, expected)
def test_01_generate(self): expected = r"""import base64 import os import sys templates={ 'run.py': '''from b import message print \"%(LOG_LEVEL)s: %%s\" %% message ''', 'a1.png': '''iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAAXNSR0IArs4c6QAAAAZiS0dEAP8A/wD/oL2nkwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9sEGAkQLzrCi3sAAAAZdEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIEdJTVBXgQ4XAAADgElEQVR42u2Zv0vrUBTHv3lRqwi1mKIUBQUdFKlFEQc76OyQQbHgooI4dlAQnPwLXJwFuzkIgqGLIFYQrINVCIilIlbBUsWfUDQSbc6bLC/Key/pD17yeg/c4YTcnHs/N+eec8/liIhQxvIDZS4MAAPAADAADAADwAAwAAwAA8AAMAAMAAPAAFhbTk5OsLy8DFEUUVdXB47jdC0fqbDyhJPJJCKRCHZ2dhCJRHB7e1t0G5yVS2JmVzWfqdjKBVpaWjA5OYlQKIRkMlmUb1ZYfcJDQ0O51traWl4uYNZF/nsXKIWYcoF0Oo2joyPIsoyzszNcXFzg5uYG9/f3eH19haZpcLlcEAQB3d3dGBwcxMjICDwej3UJkAkBYLpVV1dTMBikTCZDxZavtvL6RqkBfDav10tXV1eWA2BqE/zcdNxuN0RRhN/vh8/nQ3NzM1wuF7LZLO7u7hCLxbC2tobNzU1ompbr39vbi4ODA1RVVVlmEzSFbWBggCRJoo+PD0Pv7+3tUWNjo26VFhcX7esC+cjh4SFVVlbmBikIAimKYhkAJQ+DfX19mJqayukPDw+QJKm8ToOBQECn7+/vlxcAn8+n02OxWHkBEARBp6dSKXtmgr/K8fExwuEwZFlGPB7H4+MjMpkMFEX5a9/n52d7ZoJERJIkUWdnZ0FJEcdx9guD2WyWZmZmCpp4oYMtBQDDLjA/P4+VlRXdM4/Hg7GxMfj9frS3t6OpqQm1tbWoqakBz/MFVXcs5QKyLBPP8zraCwsLpKqqIcovLy+W/QMM9ZqdndUZCgQCpoxcX19bFoChMLi9va3Tg8Ggqb/s9PTU3vcCX+O21+s1ZcRKmV9eNUGHwwFVVXP629sbHA6HIQOapqGtrQ2Xl5eFH13/VU3Q7Xbr9EQiYdhAKBT6NnnbuUBPT8+3SRmRRCKBubk5a1dFjeyUq6urut2W53na2Nj4Y59oNPqtGGLbMPj+/k4dHR3f0tmJiQna3d2lp6cnUlWVUqkUhcNhGh8fJ47jcu9OT0/bGwARUTwep/r6etMp7+joKKmqmvdgS516m8J2fn5O/f39hgw6nU5aWloiTdMKWq1SA8jramxrawvr6+uIRqNIp9NQFAWCIKChoQFdXV0QRRHDw8NwOp0Fh6xinSF+Z8/Wd4O2qQgxAAwAA8AAMAAMAAPAADAADAADwABYTX4C3JAjt3gTqkQAAAAASUVORK5CYII=''', 'B/__init__.py': '''from .b import message ''', 'B/b.py': '''from .%(C)s import message ''', 'B/B.py': '''message = \'\'\'\'hello, world! * & %% \"\"\" \" \'\'\'\' ''', } def render(replacements=None, base_path=None): template_vars = {'LOG_LEVEL': 'OUTPUT', 'C': 'B'} if replacements is not None: template_vars.update(replacements) if base_path is None: base_path = os.getcwd() elif not os.path.exists(base_path): os.mkdir(base_path) if os.path.exists(os.path.join(base_path, "%(C)s"%dict(template_vars))): raise Exception("Directory already exists: %s" % (os.path.join(base_path, "%(C)s"%dict(template_vars)))) os.mkdir(os.path.join(base_path, "%(C)s"%dict(template_vars))) if os.path.exists(os.path.join(base_path, 'run.py'%dict(template_vars))): raise Exception('File already exists: %s' % os.path.join(base_path, 'run.py'%dict(template_vars))) fp = open(os.path.join(base_path, 'run.py'%dict(template_vars)), 'wb') fp.write(templates['run.py']%dict(template_vars)) fp.close() if os.path.exists(os.path.join(base_path, 'a1.png'%dict(template_vars))): raise Exception('File already exists: %s' % os.path.join(base_path, 'a1.png'%dict(template_vars))) fp = open(os.path.join(base_path, 'a1.png'%dict(template_vars)), 'wb') fp.write(base64.standard_b64decode(templates['a1.png'])) fp.close() if os.path.exists(os.path.join(base_path, '%(C)s/__init__.py'%dict(template_vars))): raise Exception('File already exists: %s' % os.path.join(base_path, '%(C)s/__init__.py'%dict(template_vars))) fp = open(os.path.join(base_path, '%(C)s/__init__.py'%dict(template_vars)), 'wb') fp.write(templates['B/__init__.py']%dict(template_vars)) fp.close() if os.path.exists(os.path.join(base_path, '%(C)s/b.py'%dict(template_vars))): raise Exception('File already exists: %s' % os.path.join(base_path, '%(C)s/b.py'%dict(template_vars))) fp = open(os.path.join(base_path, '%(C)s/b.py'%dict(template_vars)), 'wb') fp.write(templates['B/b.py']%dict(template_vars)) fp.close() if os.path.exists(os.path.join(base_path, '%(C)s/%(C)s.py'%dict(template_vars))): raise Exception('File already exists: %s' % os.path.join(base_path, '%(C)s/%(C)s.py'%dict(template_vars))) fp = open(os.path.join(base_path, '%(C)s/%(C)s.py'%dict(template_vars)), 'wb') fp.write(templates['B/B.py']%dict(template_vars)) fp.close() return template_vars if __name__ == "__main__" and len(sys.argv)>1 and not os.path.exists(sys.argv[1]): print "Creating template ..." render(base_path=sys.argv[1]) print "done." """ here = stacks.uniform_path(os.path.dirname(__file__)) output = [] def out(*k, **p): output.append(stacks.format_string(*k, **p)) result = stacks.run( argv=['generate', '--binary', 'png', '--ignore', 'ignore', 'generate_src', 'OUTPUT', 'LOG_LEVEL', 'B', 'C'], out=out, err=out, **run_args ) self.assertEqual(result, 0) self.assertEqual(len(output), 1) for line in difflib.unified_diff( expected.split('\n'), output[0].split('\n'), ): print line self.assertEqual(''.join(output), expected) # Now test that when we execute the file it generates the correct directory structure if os.path.exists(os.path.join(here, 'generate.py')): os.remove(os.path.join(here, 'generate.py')) if os.path.exists(os.path.join(here, 'generate_dst')): shutil.rmtree(os.path.join(here, 'generate_dst')) fp = open(os.path.join(here, 'generate.py'), 'wb') fp.write(''.join(output)) fp.close() cmd = [ sys.executable, os.path.join(here, 'generate.py'), 'generate_dst', ] stacks.process(cmd) # Now check that the directory structure is identical apart from the ignored file cmd = "diff -ru generate_src/ generate_dst/" result = stacks.process(cmd.split(' ')) self.assertEqual(result.stdout, "Only in generate_src/: a1.ignore\n")
def build(cmd, vm_ip_part, base_ip, arch, output_dir, proxy_ip=None, packages=None, base_name='buildkit'): if packages is None: packages = [] # Need to check apt-proxy is set up on port 3142 # http://127.0.0.1:3142/ubuntu/ gives "ERROR 403 - too few slashes in URI /ubuntu" args = dict( vm_ip_part=vm_ip_part, base_ip=base_ip, proxy_ip=proxy_ip, base_name=base_name, arch=arch, ) cmd_ = [ 'vmbuilder', 'kvm', 'ubuntu', '--mem', '512', '--cpus', '1', '--domain', '%(base_name)s%(vm_ip_part)s'%args, '--dest', '%(base_name)s%(vm_ip_part)s'%args, '--flavour', 'virtual', '--suite', 'lucid', '--arch', '%(arch)s'%args, '--hostname', '%(base_name)s%(vm_ip_part)s'%args, '--user', 'ubuntu', '--pass', 'ubuntu', '--rootpass', 'ubuntu' '--debug', '-v', '--ip', '%(base_ip)s.%(vm_ip_part)s'%args, '--mask', '255.255.255.0', '--net', '%(base_ip)s.0'%args, '--bcast', '%(base_ip)s.255'%args, '--gw', '%(base_ip)s.254'%args, '--dns', '%(base_ip)s.254'%args, '--proxy', 'http://%(proxy_ip)s:3142/ubuntu'%args, '--components', 'main,universe', '--addpkg', 'openssh-server', '--addpkg', 'acpid', ] for package in packages: cmd_.append('--addpkg') cmd_.append(package) cmd.out(cmd_) result = stacks.process( cmd_, cwd=output_dir, merge=True, echo=True, ) qcow2 = None new_output_dir = os.path.join(output_dir, "%(base_name)s%(vm_ip_part)s"%args) for filename in os.listdir(new_output_dir): if filename.endswith('.qcow2'): qcow2 = filename if not qcow2: cmd.err('Could not find a generated .qcow2 file') return 1 dst = '%s/disk.raw'%(new_output_dir,) if not os.path.exists(dst): cmd.out('Generated %s', qcow2) cmd.out('Converting to raw ...') cmd_ = 'qemu-img convert -f qcow2 -O raw %s %s/disk.raw'%(qcow2, new_output_dir) cmd.out(cmd_) result = stacks.process( cmd_, shell=True, echo=True, cwd=new_output_dir, ) else: cmd.err('Path %s already exists, not overwriting', dst) return result