def replace(filename, old, new, variable='version'):
if variable is None:
regex = "s/%s/%s/"%(old, new)
else:
regex = "s,%s[ =].*['\"]%s['\"],%s = '%s',"%(variable, old, variable, new)
command = [
'sed',
'-e',
regex,
'-i',
filename,
]
stacks.process(command)
def run(cmd):
# Check that there isn't already a key
if cmd.repo.key_exists(
cmd.parent.opts.key_dir,
cmd.parent.opts.base_repo_dir
):
cmd.err(
'Cannot install a new key because '
'packages may have already been signed with the existing one'
)
return 1
cmd_ = '. /usr/lib/buildkit/buildkit_common.sh\nbuildkit_ensure_public_key "%s" "%s" "%s" "%s" "%s"\nchown -R buildkit:buildkit /var/lib/buildkit/key\nchown buildkit:www-data /var/lib/buildkit/repo/packages_public.key\nexit'%(
cmd.opts.name,
cmd.opts.email,
cmd.opts.comment,
#cmd.opts.password,
cmd.parent.opts.key_dir,
cmd.parent.opts.base_repo_dir,
)
cmd.out(cmd_)
result = stacks.process(
cmd_,
# cwd=os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]),
shell=True,
echo=True,
)
cmd.out(result.stdout)
def run(scm, cmd, path, error_msg):
res = stacks.process(cmd, cwd=path, merge=True)
if res.retcode != 0:
scm.log.error(str(cmd))
scm.log.error(res.stdout)
raise Exception(error_msg)
return res
def run(cmd):
cmd_ = 'find . | grep .deb'
result = stacks.process(
cmd_,
cwd=os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]),
shell=True,
merge=True,
)
for line in result.stdout.strip().split():
src = os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0], line.strip())
dst = os.path.join(cmd.opts.output_dir, line.split('/')[-1])
cmd.out(src)
shutil.copyfile(src, dst)
def get_pkg_info(path):
# Cope with capitalisation variations
stacks.process(['python', 'setup.py', 'egg_info'], cwd=path)
files = os.listdir(path)
name = None
for filename in files:
if os.path.isdir(os.path.join(path, filename)) and filename.endswith('.egg-info'):
name = filename[:-len('.egg-info')]
if name is None:
raise Exception('Couldn\'t determine the package name for package in %r, is it really a setuptools-enabled package?'%path)
if os.path.exists(os.path.join(path, '%s.egg-info'%name.replace('-','_'), 'PKG-INFO')):
pkg_info_path = os.path.join(path, '%s.egg-info'%name.replace('-','_'), 'PKG-INFO')
elif os.path.exists(os.path.join(path, 'PKG-INFO')):
pkg_info_path = os.path.join(path, 'PKG-INFO')
else:
log.error('No PKG-INFO found for %s in %s', name, path)
return {}
fp = open(
pkg_info_path,
'rb',
)
data = fp.read()
fp.close()
pkg_info = {}
last = None
for line in data.split('\n'):
if line.startswith(' '):
pkg_info[last][-1] += '\n'+line[8:]
else:
parts = line.split(': ')
last = parts[0]
if not pkg_info.has_key(last):
pkg_info[last] = []
value = ': '.join(parts[1:])
if value.startswith('Development Status :: '):
pkg_info['Status'] = [value.split('-')[-1].strip()]
pkg_info[last].append(value)
return pkg_info
def run(cmd):
# Check that a key is installed
if not cmd.repo.key_exists(
cmd.parent.opts.key_dir,
cmd.parent.opts.base_repo_dir
):
cmd.err(
'No repo key has ben set up, please run `buildkit repo '
'installkey\' to install one'
)
return 1
package_names = cmd.args[1:]
if cmd.opts.all:
cmd_ = 'reprepro list lucid'
result = stacks.process(
cmd_,
cwd=os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]),
shell=True,
)
for line in result.stdout.split('\n'):
if line.strip():
info, name = line.split(':')
name, version = name.strip().split(' ')
if name not in package_names:
package_names.append(name)
if not package_names:
cmd.err('ERROR: No packages to remove')
return 1
cmd_ = 'reprepro --gnupghome "%s" remove lucid %s' % (cmd.parent.opts.key_dir, ' '.join(package_names))
cmd.out(cmd_)
result = stacks.process(
cmd_,
cwd=os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]),
shell=True,
)
cmd.out(result.stdout)
def run(cmd):
# Check that a key is installed
if not cmd.repo.key_exists(
cmd.parent.opts.key_dir,
cmd.parent.opts.base_repo_dir
):
cmd.err(
'No repo key has ben set up, please run `buildkit repo '
'installkey\' to install one'
)
return 1
cmd_ = 'reprepro list lucid'
result = stacks.process(
cmd_,
cwd=os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]),
shell=True,
merge=True,
)
cmd.out(result.stdout.strip())
def run(cmd):
# Check that a key is installed
if not cmd.repo.key_exists(
cmd.parent.opts.key_dir,
cmd.parent.opts.base_repo_dir
):
cmd.err(
'No repo key has ben set up, please run `buildkit repo '
'installkey\' to install one'
)
return 1
packages = cmd.args[1:]
abs_paths = []
for package in packages:
abs_paths.append(stacks.uniform_path(package))
if cmd.opts.find_in:
cmd_ = 'find "%s" | grep .deb$ | xargs --no-run-if-empty reprepro --gnupghome "%s" includedeb lucid %s' % (
stacks.uniform_path(cmd.opts.find_in),
cmd.parent.opts.key_dir,
abs_paths and '"' + '" "'.join(abs_paths) + '"' or '',
)
else:
cmd_ = ' '.join([
'reprepro',
'--gnupghome "%s"' % cmd.parent.opts.key_dir,
'includedeb',
'lucid',
] + abs_paths)
cmd.out(cmd_)
result = stacks.process(
cmd_,
cwd = os.path.join(cmd.parent.opts.base_repo_dir, cmd.args[0]),
merge=True,
shell=True,
)
cmd.out(result.stdout)
def test_02_start(self):
expected = """\
running bdist_egg
running egg_info
creating generated_package.egg-info
writing generated_package.egg-info/PKG-INFO
writing top-level names to generated_package.egg-info/top_level.txt
writing dependency_links to generated_package.egg-info/dependency_links.txt
writing entry points to generated_package.egg-info/entry_points.txt
writing manifest file 'generated_package.egg-info/SOURCES.txt'
reading manifest file 'generated_package.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
warning: no files found matching 'example/*.py'
warning: no files found matching 'ez_setup.py'
writing manifest file 'generated_package.egg-info/SOURCES.txt'
installing library code to build/bdist.linux-x86_64/egg
running install_lib
running build_py
creating build
creating build/lib.linux-x86_64-2.6
creating build/lib.linux-x86_64-2.6/generated_package
copying generated_package/__init__.py -> build/lib.linux-x86_64-2.6/generated_package
creating build/bdist.linux-x86_64
creating build/bdist.linux-x86_64/egg
creating build/bdist.linux-x86_64/egg/generated_package
copying build/lib.linux-x86_64-2.6/generated_package/__init__.py -> build/bdist.linux-x86_64/egg/generated_package
byte-compiling build/bdist.linux-x86_64/egg/generated_package/__init__.py to __init__.pyc
creating build/bdist.linux-x86_64/egg/EGG-INFO
copying generated_package.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO
copying generated_package.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO
copying generated_package.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO
copying generated_package.egg-info/entry_points.txt -> build/bdist.linux-x86_64/egg/EGG-INFO
copying generated_package.egg-info/not-zip-safe -> build/bdist.linux-x86_64/egg/EGG-INFO
copying generated_package.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO
creating dist
creating 'dist/generated_package-0.1.2-py2.6.egg' and adding 'build/bdist.linux-x86_64/egg' to it
removing 'build/bdist.linux-x86_64/egg' (and everything under it)
"""
here = stacks.uniform_path(os.path.dirname(__file__))
if os.path.exists(os.path.join(here, 'generate_package')):
shutil.rmtree(os.path.join(here, 'generate_package'))
output = []
def out(*k, **p):
output.append(stacks.format_string(*k, **p))
result = stacks.run(
argv=[
'start',
'--git',
'--author-name', 'James Gardner',
'--author-email', '*****@*****.**',
'--version', '0.1.2',
'generated_package',
'James\'s test package',
os.path.join(here, 'generate_package'),
],
out=out,
err=out,
**run_args
)
self.assertEqual(result, 0)
self.assertEqual(len(output), 0)
# Now check that the directory structure is identical apart from the ignored file
cmd = "python setup.py bdist_egg"
result = stacks.process(
cmd.split(' '),
cwd=os.path.join(here, 'generate_package')
)
for line in difflib.unified_diff(
expected.split('\n'),
result.stdout.split('\n'),
):
print line
self.assertEqual(result.stdout, expected)
def test_01_generate(self):
expected = r"""import base64
import os
import sys
templates={
'run.py': '''from b import message
print \"%(LOG_LEVEL)s: %%s\" %% message
''',
'a1.png': '''iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAAXNSR0IArs4c6QAAAAZiS0dEAP8A/wD/oL2nkwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9sEGAkQLzrCi3sAAAAZdEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIEdJTVBXgQ4XAAADgElEQVR42u2Zv0vrUBTHv3lRqwi1mKIUBQUdFKlFEQc76OyQQbHgooI4dlAQnPwLXJwFuzkIgqGLIFYQrINVCIilIlbBUsWfUDQSbc6bLC/Key/pD17yeg/c4YTcnHs/N+eec8/liIhQxvIDZS4MAAPAADAADAADwAAwAAwAA8AAMAAMAAPAAFhbTk5OsLy8DFEUUVdXB47jdC0fqbDyhJPJJCKRCHZ2dhCJRHB7e1t0G5yVS2JmVzWfqdjKBVpaWjA5OYlQKIRkMlmUb1ZYfcJDQ0O51traWl4uYNZF/nsXKIWYcoF0Oo2joyPIsoyzszNcXFzg5uYG9/f3eH19haZpcLlcEAQB3d3dGBwcxMjICDwej3UJkAkBYLpVV1dTMBikTCZDxZavtvL6RqkBfDav10tXV1eWA2BqE/zcdNxuN0RRhN/vh8/nQ3NzM1wuF7LZLO7u7hCLxbC2tobNzU1ompbr39vbi4ODA1RVVVlmEzSFbWBggCRJoo+PD0Pv7+3tUWNjo26VFhcX7esC+cjh4SFVVlbmBikIAimKYhkAJQ+DfX19mJqayukPDw+QJKm8ToOBQECn7+/vlxcAn8+n02OxWHkBEARBp6dSKXtmgr/K8fExwuEwZFlGPB7H4+MjMpkMFEX5a9/n52d7ZoJERJIkUWdnZ0FJEcdx9guD2WyWZmZmCpp4oYMtBQDDLjA/P4+VlRXdM4/Hg7GxMfj9frS3t6OpqQm1tbWoqakBz/MFVXcs5QKyLBPP8zraCwsLpKqqIcovLy+W/QMM9ZqdndUZCgQCpoxcX19bFoChMLi9va3Tg8Ggqb/s9PTU3vcCX+O21+s1ZcRKmV9eNUGHwwFVVXP629sbHA6HIQOapqGtrQ2Xl5eFH13/VU3Q7Xbr9EQiYdhAKBT6NnnbuUBPT8+3SRmRRCKBubk5a1dFjeyUq6urut2W53na2Nj4Y59oNPqtGGLbMPj+/k4dHR3f0tmJiQna3d2lp6cnUlWVUqkUhcNhGh8fJ47jcu9OT0/bGwARUTwep/r6etMp7+joKKmqmvdgS516m8J2fn5O/f39hgw6nU5aWloiTdMKWq1SA8jramxrawvr6+uIRqNIp9NQFAWCIKChoQFdXV0QRRHDw8NwOp0Fh6xinSF+Z8/Wd4O2qQgxAAwAA8AAMAAMAAPAADAADAADwABYTX4C3JAjt3gTqkQAAAAASUVORK5CYII=''',
'B/__init__.py': '''from .b import message
''',
'B/b.py': '''from .%(C)s import message
''',
'B/B.py': '''message = \'\'\'\'hello, world! * & %% \"\"\" \" \'\'\'\'
''',
}
def render(replacements=None, base_path=None):
template_vars = {'LOG_LEVEL': 'OUTPUT', 'C': 'B'}
if replacements is not None:
template_vars.update(replacements)
if base_path is None:
base_path = os.getcwd()
elif not os.path.exists(base_path):
os.mkdir(base_path)
if os.path.exists(os.path.join(base_path, "%(C)s"%dict(template_vars))):
raise Exception("Directory already exists: %s" % (os.path.join(base_path, "%(C)s"%dict(template_vars))))
os.mkdir(os.path.join(base_path, "%(C)s"%dict(template_vars)))
if os.path.exists(os.path.join(base_path, 'run.py'%dict(template_vars))):
raise Exception('File already exists: %s' % os.path.join(base_path, 'run.py'%dict(template_vars)))
fp = open(os.path.join(base_path, 'run.py'%dict(template_vars)), 'wb')
fp.write(templates['run.py']%dict(template_vars))
fp.close()
if os.path.exists(os.path.join(base_path, 'a1.png'%dict(template_vars))):
raise Exception('File already exists: %s' % os.path.join(base_path, 'a1.png'%dict(template_vars)))
fp = open(os.path.join(base_path, 'a1.png'%dict(template_vars)), 'wb')
fp.write(base64.standard_b64decode(templates['a1.png']))
fp.close()
if os.path.exists(os.path.join(base_path, '%(C)s/__init__.py'%dict(template_vars))):
raise Exception('File already exists: %s' % os.path.join(base_path, '%(C)s/__init__.py'%dict(template_vars)))
fp = open(os.path.join(base_path, '%(C)s/__init__.py'%dict(template_vars)), 'wb')
fp.write(templates['B/__init__.py']%dict(template_vars))
fp.close()
if os.path.exists(os.path.join(base_path, '%(C)s/b.py'%dict(template_vars))):
raise Exception('File already exists: %s' % os.path.join(base_path, '%(C)s/b.py'%dict(template_vars)))
fp = open(os.path.join(base_path, '%(C)s/b.py'%dict(template_vars)), 'wb')
fp.write(templates['B/b.py']%dict(template_vars))
fp.close()
if os.path.exists(os.path.join(base_path, '%(C)s/%(C)s.py'%dict(template_vars))):
raise Exception('File already exists: %s' % os.path.join(base_path, '%(C)s/%(C)s.py'%dict(template_vars)))
fp = open(os.path.join(base_path, '%(C)s/%(C)s.py'%dict(template_vars)), 'wb')
fp.write(templates['B/B.py']%dict(template_vars))
fp.close()
return template_vars
if __name__ == "__main__" and len(sys.argv)>1 and not os.path.exists(sys.argv[1]):
print "Creating template ..."
render(base_path=sys.argv[1])
print "done."
"""
here = stacks.uniform_path(os.path.dirname(__file__))
output = []
def out(*k, **p):
output.append(stacks.format_string(*k, **p))
result = stacks.run(
argv=['generate', '--binary', 'png', '--ignore', 'ignore', 'generate_src', 'OUTPUT', 'LOG_LEVEL', 'B', 'C'],
out=out,
err=out,
**run_args
)
self.assertEqual(result, 0)
self.assertEqual(len(output), 1)
for line in difflib.unified_diff(
expected.split('\n'),
output[0].split('\n'),
):
print line
self.assertEqual(''.join(output), expected)
# Now test that when we execute the file it generates the correct directory structure
if os.path.exists(os.path.join(here, 'generate.py')):
os.remove(os.path.join(here, 'generate.py'))
if os.path.exists(os.path.join(here, 'generate_dst')):
shutil.rmtree(os.path.join(here, 'generate_dst'))
fp = open(os.path.join(here, 'generate.py'), 'wb')
fp.write(''.join(output))
fp.close()
cmd = [
sys.executable,
os.path.join(here, 'generate.py'),
'generate_dst',
]
stacks.process(cmd)
# Now check that the directory structure is identical apart from the ignored file
cmd = "diff -ru generate_src/ generate_dst/"
result = stacks.process(cmd.split(' '))
self.assertEqual(result.stdout, "Only in generate_src/: a1.ignore\n")
def build(cmd, vm_ip_part, base_ip, arch, output_dir, proxy_ip=None, packages=None, base_name='buildkit'):
if packages is None:
packages = []
# Need to check apt-proxy is set up on port 3142
# http://127.0.0.1:3142/ubuntu/ gives "ERROR 403 - too few slashes in URI /ubuntu"
args = dict(
vm_ip_part=vm_ip_part,
base_ip=base_ip,
proxy_ip=proxy_ip,
base_name=base_name,
arch=arch,
)
cmd_ = [
'vmbuilder', 'kvm', 'ubuntu',
'--mem', '512',
'--cpus', '1',
'--domain', '%(base_name)s%(vm_ip_part)s'%args,
'--dest', '%(base_name)s%(vm_ip_part)s'%args,
'--flavour', 'virtual',
'--suite', 'lucid',
'--arch', '%(arch)s'%args,
'--hostname', '%(base_name)s%(vm_ip_part)s'%args,
'--user', 'ubuntu',
'--pass', 'ubuntu',
'--rootpass', 'ubuntu'
'--debug', '-v',
'--ip', '%(base_ip)s.%(vm_ip_part)s'%args,
'--mask', '255.255.255.0',
'--net', '%(base_ip)s.0'%args,
'--bcast', '%(base_ip)s.255'%args,
'--gw', '%(base_ip)s.254'%args,
'--dns', '%(base_ip)s.254'%args,
'--proxy', 'http://%(proxy_ip)s:3142/ubuntu'%args,
'--components', 'main,universe',
'--addpkg', 'openssh-server',
'--addpkg', 'acpid',
]
for package in packages:
cmd_.append('--addpkg')
cmd_.append(package)
cmd.out(cmd_)
result = stacks.process(
cmd_,
cwd=output_dir,
merge=True,
echo=True,
)
qcow2 = None
new_output_dir = os.path.join(output_dir, "%(base_name)s%(vm_ip_part)s"%args)
for filename in os.listdir(new_output_dir):
if filename.endswith('.qcow2'):
qcow2 = filename
if not qcow2:
cmd.err('Could not find a generated .qcow2 file')
return 1
dst = '%s/disk.raw'%(new_output_dir,)
if not os.path.exists(dst):
cmd.out('Generated %s', qcow2)
cmd.out('Converting to raw ...')
cmd_ = 'qemu-img convert -f qcow2 -O raw %s %s/disk.raw'%(qcow2, new_output_dir)
cmd.out(cmd_)
result = stacks.process(
cmd_,
shell=True,
echo=True,
cwd=new_output_dir,
)
else:
cmd.err('Path %s already exists, not overwriting', dst)
return result
