def install_packages(packages_to_install, build_dir, module):
if not packages_to_install:
return
control = PackageControlParser()
control.read(os.path.join(build_dir, 'debian/control'))
# Make a list of packages
packages = [x['Package'] for x in control]
# Make a list of dependencies we didn't build ourselves
# FIXME: we skip the ${} for now.
dependencies = []
for package_info in control:
if package_info['Package'] in packages:
for dep in package_info.get('Depends', '').split(','):
dep = dep.strip().split(None, 1)[0]
if dep and not dep.startswith('$') and dep not in packages:
dependencies.append(dep)
# Install said dependencies if there are any
if dependencies:
print blue("Installing dependencies for packages '%s'" % "', '".join(packages_to_install))
code = system('apt-get install %s' % ' '.join(dependencies), root=True)
check_code(code, module)
# Install built packages
print blue("Installing packages '%s'" % "', '".join(packages_to_install))
dpkg_args = []
for filename in glob.glob(os.path.join(os.path.dirname(build_dir), '*.deb')):
if os.path.basename(filename).split('_', 1)[0] in packages_to_install:
dpkg_args.append(filename)
code = system("dpkg -i %s" % ' '.join(dpkg_args), root=True)
check_code(code, module)
def build(module, args):
method = get_build_method(module, args)
if not args.force and method.module_is_built(module):
print blue("Module '%s' has already been built" % module.name)
return False
if args.recursive:
dependencies = [x for x in [i.strip() for i in module['Depends'].split(',')] if x]
if dependencies:
print blue("Building dependencies for '%s'" % module.name)
for i in dependencies:
build(args.moduleset[i], args)
TMP_BUILD_DIR = os.path.expanduser('~/tmp-build-dir')
BUILD_DIR = os.path.join(TMP_BUILD_DIR, '%s-%s' % (module.name, module['Version']))
code = system('rm -rf ' + os.path.expanduser('~/tmp-build-dir'), root=True)
check_code(code, module)
code = system('mkdir -p %s' % os.path.dirname(BUILD_DIR))
check_code(code, module)
method.build_module(module, BUILD_DIR)
if module['Install']:
install_packages([i.strip() for i in module['Install'].split(',')], BUILD_DIR, module)
repo_dir = 'repository-%s' % args.moduleset.name
if not os.path.exists(repo_dir):
os.mkdir(repo_dir)
code = system('mv -f %s/*.deb %s/' % (os.path.dirname(BUILD_DIR), repo_dir))
return True
def get_build_method(module, args, quiet=False):
if not quiet:
print blue("Using build method '%s'" % module['Build-Method'])
pymodule = __import__("novabuild.commands.build.%s" % module['Build-Method'],
globals(), locals(), ["BuildMethod"])
return pymodule.BuildMethod(args)
def setup_build_env(self, module, build_dir):
self.uncompress_tarball(module, build_dir)
print blue("Configuring the kernel")
code = system('cp debian/config-%s-%s %s/.config' %
(module['Version'], self.args.arch, build_dir))
check_code(code, module)
def uncompress_tarball(self, module, destination):
filename = os.path.join('tarballs', module['Basename'])
if not os.path.exists(filename):
fetch(module)
dest_parent = os.path.dirname(destination)
print blue("Uncompressing '%s'" % filename)
if filename.endswith('.tar.bz2') or filename.endswith('.tbz'):
check_code(system('tar xjf %s -C %s' % (filename, dest_parent)),
module)
elif filename.endswith('.tar.gz') or filename.endswith('.tgz'):
check_code(system('tar xzf %s -C %s' % (filename, dest_parent)),
module)
elif filename.endswith('.zip'):
check_code(system('unzip %s -d %s' % (filename, dest_parent)),
module)
else:
raise Exception("Cannot find the type of the archive.")
# Put the directory in the tarball in the right directory
if not os.path.exists(destination):
contents = os.listdir(dest_parent)
if len(contents) == 1 and os.path.isdir(
'%s/%s' % (dest_parent, contents[0])):
check_code(
system('mv "%s/%s" %s' %
(dest_parent, contents[0], destination)), module)
else:
check_code(system('mkdir %s' % destination), module)
files = [os.path.join(dest_parent, file) for file in contents]
check_code(system('mv %s %s' % (' '.join(files), destination)),
module)
def uncompress_tarball(self, module, destination):
filename = os.path.join('tarballs', module['Basename'])
if not os.path.exists(filename):
fetch(module)
dest_parent = os.path.dirname(destination)
print blue("Uncompressing '%s'" % filename)
if filename.endswith('.tar.bz2') or filename.endswith('.tbz'):
check_code(system('tar xjf %s -C %s' % (filename, dest_parent)), module)
elif filename.endswith('.tar.gz') or filename.endswith('.tgz'):
check_code(system('tar xzf %s -C %s' % (filename, dest_parent)), module)
elif filename.endswith('.zip'):
check_code(system('unzip %s -d %s' % (filename, dest_parent)), module)
else:
raise Exception("Cannot find the type of the archive.")
# Put the directory in the tarball in the right directory
if not os.path.exists(destination):
contents = os.listdir(dest_parent)
if len(contents) == 1 and os.path.isdir('%s/%s' % (dest_parent, contents[0])):
check_code(system('mv "%s/%s" %s' % (dest_parent, contents[0], destination)), module)
else:
check_code(system('mkdir %s' % destination), module)
files = [os.path.join(dest_parent, file) for file in contents]
check_code(system('mv %s %s' % (' '.join(files), destination)), module)
def get_build_method(module, args, quiet=False):
if not quiet:
print blue("Using build method '%s'" % module['Build-Method'])
pymodule = __import__(
"novabuild.commands.build.%s" % module['Build-Method'], globals(),
locals(), ["BuildMethod"])
return pymodule.BuildMethod(args)
def build_module(self, module, build_dir):
self.setup_build_env(module, build_dir)
print blue("Building linux kernel '%s'" % module['Version'])
cwd = os.path.expanduser('~/tmp-build-dir/linux-%s' % module['Version'])
code = system('make-kpkg --revision=%s kernel_image kernel_headers kernel_source' % self.get_rev_tag(module),
cwd=cwd, root=True)
check_code(code, module)
def main(args):
for mod_name in sorted(args.moduleset):
try:
module = args.moduleset[mod_name]
print blue("Fetching '%s' from '%s'" % (module.name, module['Source']))
fetch(module)
except Exception, e:
print red(e)
status = 1
def main(args):
for mod_name in args.moduleset:
try:
module = args.moduleset[mod_name]
print blue("Building '%s'" % module.name)
build(module, args)
except Exception, e:
print red("Exception %s: %s" % (e.__class__.__name__, e))
return 1
def main(args):
for mod_name in sorted(args.moduleset):
try:
module = args.moduleset[mod_name]
print blue("Fetching '%s' from '%s'" %
(module.name, module['Source']))
fetch(module)
except Exception, e:
print red(e)
status = 1
def build_module(self, module, build_dir):
self.setup_build_env(module, build_dir)
print blue("Building linux kernel '%s'" % module['Version'])
cwd = os.path.expanduser('~/tmp-build-dir/linux-%s' %
module['Version'])
code = system(
'make-kpkg --revision=%s kernel_image kernel_headers kernel_source'
% self.get_rev_tag(module),
cwd=cwd,
root=True)
check_code(code, module)
def update_changelog(self, module, build_dir, debian_dir):
filename = os.path.join(debian_dir, "changelog")
build_filename = os.path.join(build_dir, "debian", "changelog")
version = self.get_version(module)
if not changelog_is_up_to_date(filename, version):
print blue("Update ChangeLog for version %s" % version)
prepend_changelog_entry(filename, module.name, version)
else:
print blue("No ChangeLog update is needed")
copy_changelog_with_build_tag(filename, build_filename, self.args.build_tag)
def update_changelog(self, module, build_dir, debian_dir):
filename = os.path.join(debian_dir, 'changelog')
build_filename = os.path.join(build_dir, 'debian', 'changelog')
version = self.get_version(module)
if not changelog_is_up_to_date(filename, version):
print blue("Update ChangeLog for version %s" % version)
prepend_changelog_entry(filename, module.name, version)
else:
print blue("No ChangeLog update is needed")
copy_changelog_with_build_tag(filename, build_filename, self.args.build_tag)
def setup_build_env(self, debian_dir, build_dir, module):
orig_dir = build_dir + ".original"
self.uncompress_tarball(module, orig_dir)
print blue("Encoding PHP files")
try:
php_version = int(module["PHP-Version"])
except:
php_version = 5
ioncube = php_version == 5 and IONCUBE5 or IONCUBE4
code = system("%s %s %s -o %s" % (ioncube, IONCUBE_ARGS, orig_dir, build_dir))
check_code(code, module)
code = system("rm -rf %s" % orig_dir)
check_code(code, module)
code = system("cp -r %s %s/debian" % (debian_dir, build_dir))
check_code(code, module)
def setup_build_env(self, debian_dir, build_dir, module):
orig_dir = build_dir + ".original"
self.uncompress_tarball(module, orig_dir)
print blue("Encoding PHP files")
try:
php_version = int(module['PHP-Version'])
except:
php_version = 5
ioncube = php_version == 5 and IONCUBE5 or IONCUBE4
code = system('%s %s %s -o %s' %
(ioncube, IONCUBE_ARGS, orig_dir, build_dir))
check_code(code, module)
code = system('rm -rf %s' % orig_dir)
check_code(code, module)
code = system('cp -r %s %s/debian' % (debian_dir, build_dir))
check_code(code, module)
def build(module, args):
method = get_build_method(module, args)
if not args.force and method.module_is_built(module):
print blue("Module '%s' has already been built" % module.name)
return False
if args.recursive:
dependencies = [
x for x in [i.strip() for i in module['Depends'].split(',')] if x
]
if dependencies:
print blue("Building dependencies for '%s'" % module.name)
for i in dependencies:
build(args.moduleset[i], args)
TMP_BUILD_DIR = os.path.expanduser('~/tmp-build-dir')
BUILD_DIR = os.path.join(TMP_BUILD_DIR,
'%s-%s' % (module.name, module['Version']))
code = system('rm -rf ' + os.path.expanduser('~/tmp-build-dir'), root=True)
check_code(code, module)
code = system('mkdir -p %s' % os.path.dirname(BUILD_DIR))
check_code(code, module)
method.build_module(module, BUILD_DIR)
if module['Install']:
install_packages([i.strip() for i in module['Install'].split(',')],
BUILD_DIR, module)
repo_dir = 'repository-%s' % args.moduleset.name
if not os.path.exists(repo_dir):
os.mkdir(repo_dir)
code = system('mv -f %s/*.deb %s/' %
(os.path.dirname(BUILD_DIR), repo_dir))
return True
def install_packages(packages_to_install, build_dir, module):
if not packages_to_install:
return
control = PackageControlParser()
control.read(os.path.join(build_dir, 'debian/control'))
# Make a list of packages
packages = [x['Package'] for x in control]
# Make a list of dependencies we didn't build ourselves
# FIXME: we skip the ${} for now.
dependencies = []
for package_info in control:
if package_info['Package'] in packages:
for dep in package_info.get('Depends', '').split(','):
dep = dep.strip().split(None, 1)[0]
if dep and not dep.startswith('$') and dep not in packages:
dependencies.append(dep)
# Install said dependencies if there are any
if dependencies:
print blue("Installing dependencies for packages '%s'" %
"', '".join(packages_to_install))
code = system('apt-get install %s' % ' '.join(dependencies), root=True)
check_code(code, module)
# Install built packages
print blue("Installing packages '%s'" % "', '".join(packages_to_install))
dpkg_args = []
for filename in glob.glob(os.path.join(os.path.dirname(build_dir),
'*.deb')):
if os.path.basename(filename).split('_', 1)[0] in packages_to_install:
dpkg_args.append(filename)
code = system("dpkg -i %s" % ' '.join(dpkg_args), root=True)
check_code(code, module)
def build_module(self, module, build_dir):
debian_dir = self.get_debian_dir(module)
print blue('Set up dpkg build environment')
self.setup_build_env(debian_dir, build_dir, module)
print blue('Installing dependencies')
self.install_dependencies(module, build_dir)
self.update_changelog(module, build_dir, debian_dir)
print blue("Building '%s' '%s'" % (module.name, module['Version']))
self.build(module)
def build_module(self, module, build_dir):
debian_dir = self.get_debian_dir(module)
print blue("Set up dpkg build environment")
self.setup_build_env(debian_dir, build_dir, module)
print blue("Installing dependencies")
self.install_dependencies(module, build_dir)
self.update_changelog(module, build_dir, debian_dir)
print blue("Building '%s' '%s'" % (module.name, module["Version"]))
self.build(module)
def fetch(module):
if (os.path.exists('tarballs/%s' % module['Basename'])):
print blue("File already exists: '%s'" % module['Basename'])
return False
if not os.path.exists('tarballs'):
os.mkdir('tarballs')
source_type = module['Source-Type']
if source_type == 'wget':
code = system('wget -nc %s -Otarballs/%s' % (module['Source'], module['Basename']))
check_code(code, module)
elif source_type == 'git':
fullname = '%s-%s' % (module.name, module['Version'])
branch = module.get('Branch', 'master')
print blue("Cloning the last revision of the remote repository")
code = system('git clone --depth=1 --branch=%s -- %s tarballs/%s' % (branch, module['Source'], fullname))
check_code(code, module)
print blue("Generating tarball")
code = system('( cd tarballs/%s && git archive --format=tar --prefix=%s/ origin/%s ) | gzip -9 > tarballs/%s' \
% (fullname, fullname, branch, module['Basename']))
check_code(code, module)
print blue("Removing temporary directory")
code = system('rm -rf tarballs/%s' % fullname)
check_code(code, module)
elif source_type == 'svn':
tmpdir = '%s-%s' % (module.name, module['Version'])
print blue("Exporting the SVN snapshot")
code = system('svn export --force %s tarballs/%s' % (module['Source'], tmpdir))
check_code(code, module)
print blue("Generating tarball")
code = system('cd tarballs && tar czvf %s %s' % (module['Basename'], tmpdir))
check_code(code, module)
print blue("Removing temporary directory")
code = system('rm -rf tarballs/%s' % tmpdir)
check_code(code, module)
elif source_type == 'local':
filename = module['Source']
if filename.startswith('file://'):
filename = filename[7:]
code = system('cp %s tarballs/' % filename)
check_code(code, module)
else:
raise Exception("The '%s' source type is not handled yet" % source_type)
return True
def fetch(module):
if (os.path.exists('tarballs/%s' % module['Basename'])):
print blue("File already exists: '%s'" % module['Basename'])
return False
if not os.path.exists('tarballs'):
os.mkdir('tarballs')
source_type = module['Source-Type']
if source_type == 'wget':
code = system('wget -nc %s -Otarballs/%s' %
(module['Source'], module['Basename']))
check_code(code, module)
elif source_type == 'git':
fullname = '%s-%s' % (module.name, module['Version'])
branch = module.get('Branch', 'master')
print blue("Cloning the last revision of the remote repository")
code = system('git clone --depth=1 --branch=%s -- %s tarballs/%s' %
(branch, module['Source'], fullname))
check_code(code, module)
print blue("Generating tarball")
code = system('( cd tarballs/%s && git archive --format=tar --prefix=%s/ origin/%s ) | gzip -9 > tarballs/%s' \
% (fullname, fullname, branch, module['Basename']))
check_code(code, module)
print blue("Removing temporary directory")
code = system('rm -rf tarballs/%s' % fullname)
check_code(code, module)
elif source_type == 'svn':
tmpdir = '%s-%s' % (module.name, module['Version'])
print blue("Exporting the SVN snapshot")
code = system('svn export --force %s tarballs/%s' %
(module['Source'], tmpdir))
check_code(code, module)
print blue("Generating tarball")
code = system('cd tarballs && tar czvf %s %s' %
(module['Basename'], tmpdir))
check_code(code, module)
print blue("Removing temporary directory")
code = system('rm -rf tarballs/%s' % tmpdir)
check_code(code, module)
elif source_type == 'local':
filename = module['Source']
if filename.startswith('file://'):
filename = filename[7:]
code = system('cp %s tarballs/' % filename)
check_code(code, module)
else:
raise Exception("The '%s' source type is not handled yet" %
source_type)
return True
def main(args):
# Get the user name.
# Remember we are supposed to be ran with sudo
user = env.get_user_name()
uid = env.get_uid()
if not user or not uid:
print "You should run 'novabuild prepare' using sudo."
return 1
########################################################################
print blue("Configuring APT")
# Here we save a custom APT configuration, for a space-saving non-interactive
# apt-get.
f = file('/etc/apt/apt.conf.d/95novabuild', 'w')
f.write("""
APT::Get::Assume-Yes "true";
APT::Get::AllowUnauthenticated "true";
APT::Get::Clean "always";
APT::Install-Recommends "false";
""")
f.close()
########################################################################
print blue("Configuring sudo")
f = file('/etc/sudoers.d/95novabuild', 'w')
f.write("%s ALL = NOPASSWD: ALL" % user)
f.close()
os.chmod('/etc/sudoers.d/95novabuild', 0440)
########################################################################
print blue("Installing software required for packaging")
# What packages should we install?
to_install = (
'dpkg-dev',
'debhelper',
'dpatch',
'kernel-package',
'fakeroot',
'bzip2',
'dialog',
'vim',
)
code = system('apt-get install ' + ' '.join(to_install), root=True)
if code != 0:
print red("Could not install additional packages")
return 1
########################################################################
print blue("Bootstraping done")
return 0
def setup_build_env(self, module, build_dir):
self.uncompress_tarball(module, build_dir)
print blue("Configuring the kernel")
code = system('cp debian/config-%s-%s %s/.config' % (module['Version'], self.args.arch, build_dir))
check_code(code, module)