class Map(BuildStep):
'''
Maps a value to another one.
Supports regex matching and priority matching by order.
'''
parameters = {
'input': Parameter(type=str, description='Input to map'),
'mapping': Parameter(type=OrderedDict,
description='Mapping ordered dict'),
}
outparameters = {
'result': Parameter(type=str, description='Result of the calculation')
}
def run(self):
for pattern, subst in self.mapping.items():
match = re.match(pattern, self.input)
if match is not None:
self.result = subst.format(*match.groups())
self.log.debug('Map %r via %r to %r' %
(self.input, pattern, self.result))
return
raise RuntimeError('No suitable mapping entry found')
class SystemCall(BuildStep):
'''
Build step to execute given shell command.
'''
parameters = {
'command':
Parameter(type=str, description='command to execute'),
'workingdir':
Parameter(type=str,
description='Working directory for command execution',
default='.'),
}
outparameters = {
'commandoutput':
Parameter(type=none_or(str),
description='Command output (if captured)',
default=None)
}
def run(self):
cwd = os.getcwd()
try:
os.chdir(self.workingdir)
self.commandoutput = systemCall(self.command, log=self.log)
finally:
os.chdir(cwd)
class MakeDirs(BuildStep):
'''
This build step create the desired directories.
'''
parameters = {
'dirs':
Parameter(type=listof(str), description='List of directories'),
'removeoncleanup':
Parameter(type=bool,
description='Remove the directories on clenup',
default=True),
}
def run(self):
for entry in self.dirs:
# TODO: Referencer support for nested types
entry = Referencer(entry).evaluate(self.chain)
self.log.debug('Create directory: %s ...' % entry)
ensureDirectory(entry)
def cleanup(self):
if self.removeoncleanup:
for entry in self.dirs:
entry = Referencer(entry).evaluate(self.chain)
shutil.rmtree(entry)
class TmpDir(BuildStep):
parameters = {
'parentdir':
Parameter(type=str,
description='Path to parent directory',
default='/tmp'),
}
outparameters = {
'tmpdir': Parameter(
type=str,
description='Created temporary directory',
)
}
def run(self):
timehash = hashlib.sha1(str(time.time())).hexdigest()
dirhash = hashlib.sha1(self.parentdir).hexdigest()
dest = hashlib.sha1(timehash + dirhash).hexdigest()
dest = path.join(self.parentdir, dest)
self.log.info('Create temporary dir: %s' % dest)
ensureDirectory(dest)
self.tmpdir = dest
def cleanup(self):
shutil.rmtree(self.tmpdir)
class MovePath(BuildStep):
'''
This build step moves/renames a path to a given destination path.
Shell wildcards are supported!
'''
parameters = {
'source': Parameter(type=str, description='Source path'),
'destination': Parameter(
type=str,
description='Destination path',
),
}
def run(self):
self.log.info('Move %s to %s' % (self.source, self.destination))
for entry in glob.glob(self.source):
self.log.debug('Copy %s to %s' % (entry, self.destination))
if os.path.isfile(entry):
shutil.copy(entry, self.destination)
else:
shutil.copytree(entry, self.destination)
self.log.debug('Remove %s ...' % entry)
if os.path.isfile(entry):
os.remove(entry)
else:
shutil.rmtree(entry)
class GitClone(BuildStep):
'''
Clones a git project.
'''
parameters = {
'url' : Parameter(type=str,
description='Url to clone from'),
'destdir' : Parameter(type=str,
description='Destination directory to clone '
'to (will be completely removed during '
'cleanup!'),
'uselastversion' : Parameter(type=bool,
description='Try to determine the last release '
'version (by analyzing the tags) and use it '
'as target',
default=False),
'target' : Parameter(type=str,
description='Checkout target (tag or branch)',
default=''),
'asbranch' : Parameter(type=str,
description='Checkout the desired target as '
'given branch',
default=''),
}
def run(self):
systemCall('git clone %s %s' % (self.url, self.destdir), log=self.log)
if self.uselastversion:
self.target = self._determineLastVersion()
if self.target:
checkoutCmd = 'git --git-dir=%s/.git --work-tree=%s checkout %s' % (
self.destdir, self.destdir, self.target
)
if self.asbranch:
checkoutCmd += ' -b %s' % self.asbranch
systemCall(checkoutCmd, log=self.log)
def cleanup(self):
shutil.rmtree(self.destdir)
def _determineLastVersion(self):
self.log.debug('Determine last version ...')
# find last tag
tags = systemCall('git --git-dir=%s/.git --work-tree=%s tag -l' % (
self.destdir,
self.destdir),
log=self.log).splitlines()
tags = sorted([LooseVersion(entry.strip()) for entry in tags])
result = tags[-1].vstring
self.log.debug('Last version: %s' % result)
return result
class PBuilderExecCmds(BuildStep):
'''
Execute commands inside a pbuilder jail.
'''
parameters = {
'cmds':
Parameter(type=listof(str), description='List of commands to execute'),
'save':
Parameter(type=bool,
description='Save jail after execution',
default=False),
'config':
Parameter(type=str,
description='Pbuilder config file '
'(pbuilderrc)',
default='/etc/pbuilderrc'),
}
def __init__(self, name, paramValues, chain=None):
BuildStep.__init__(self, name, paramValues, chain)
self._scriptfile = None
def run(self):
# create tmp script file name
self._scriptfile = '/tmp/conduct.PBuilderExecCmds.%s.sh' \
% hashlib.md5(str(time.time)).hexdigest()
# create script with commands
script = '#!/bin/sh\n'
self.cmds.append('exit') # exit the jail at the end
for cmd in self.cmds:
script += '''echo '%s' \n''' % cmd
self.log.debug('Generated script:\n%s' % script)
with open(self._scriptfile, 'w') as f:
f.write(script)
# pipe commands to pbuilder
cmd = 'sh %s | pbuilder --login ' % self._scriptfile
if self.save:
cmd += '--save-after-login '
cmd += '--configfile %s' % self.config
systemCall(cmd, log=self.log)
def cleanup(self):
if self._scriptfile and path.exists(self._scriptfile):
os.remove(self._scriptfile)
class Debootstrap(BuildStep):
'''
This build step bootstraps a basic debian system to the given directory.
'''
parameters = {
'distribution':
Parameter(type=str, description='Desired distribution'),
# TODO: map archs? => system analyzer?
'arch':
Parameter(type=str, description='Desired architecture'),
'destdir':
Parameter(type=str, description='Destination directory'),
'includes':
Parameter(type=listof(str),
description='Packages to include',
default=[]),
}
def run(self):
cmd = 'debootstrap --verbose --arch=%s ' % self.arch
if self.includes:
cmd += '--include %s ' % ','.join(self.includes)
if self._isForeignArch():
# separate first and second stage
cmd += '--foreign '
cmd += '%s %s' % (self.distribution, self.destdir)
self.log.info('Bootstrapping ...')
systemCall(cmd, log=self.log)
if self._isForeignArch():
self._strapSecondStage()
def _isForeignArch(self):
return self.arch != conduct.app.sysinfo['arch']
def _strapSecondStage(self):
self.log.info('Boostrap second stage ...')
qemuStatic = '/usr/bin/qemu-%s-static' % self.arch
chrootQemuStaticPlace = path.join(self.destdir, 'usr', 'bin')
self.log.debug('Copy qemu static to chroot ...')
shutil.copy(qemuStatic, chrootQemuStaticPlace)
chrootedSystemCall(self.destdir,
'debootstrap/debootstrap --second-stage',
mountPseudoFs=False,
log=self.log)
class Calculation(BuildStep):
'''
Build step to do some calculation.
'''
parameters = {
'formula': Parameter(type=str, description='Formula to calculate'),
}
outparameters = {
'result': Parameter(type=float,
description='Result of the calculation')
}
def run(self):
self.result = float(eval(self.formula))
class Mount(BuildStep):
'''
This build step mounts given device to given mount point.
'''
parameters = {
'dev': Parameter(type=str, description='Path to the device file'),
'mountpoint': Parameter(type=str,
description='Path to the mount point'),
}
def run(self):
mount(self.dev, self.mountpoint, log=self.log)
def cleanup(self):
umount(self.mountpoint, log=self.log)
class CreateFileSystem(BuildStep):
'''
This build step creates the desired file system on the given device.
Used tool: mkfs
'''
parameters = {
'dev':
Parameter(type=str, description='Path to the device file'),
'fstype':
Parameter(type=oneof('bfs', 'cramfs', 'ext2', 'ext3', 'ext4', 'fat',
'ntfs', 'vfat'),
description='Desired file system'),
}
def run(self):
systemCall('mkfs -t %s %s' % (self.fstype, self.dev), log=self.log)
class InstallDebPkg(BuildStep):
parameters = {
'pkg':
Parameter(type=str, description='Package to install'),
'chrootdir':
Parameter(type=str,
description='Chroot directory (if desired)',
default=''),
'depsonly':
Parameter(type=bool,
description='Install dependencies only',
default=False),
}
def run(self):
self._syscall = lambda cmd: (chrootedSystemCall(self.chrootdir, cmd, log=self.log) \
if self.chrootdir else systemCall(cmd, log=self.log))
self._installCmd = 'env DEBIAN_FRONTEND=noninteractive ' \
'apt-get install --yes --force-yes ' \
'--no-install-recommends ' \
'-o Dpkg::Options::="--force-overwrite" ' \
'-o Dpkg::Options::="--force-confnew" ' \
'%s'
if self.depsonly:
deps = self._determineDependencies()
for pkg in deps:
self._syscall(self._installCmd % pkg)
else:
# install actual pkg
self._syscall(self._installCmd % self.pkg)
def _determineDependencies(self):
try:
out = self._syscall('apt-cache show %s | grep Depends:' % self.pkg)
except RuntimeError as e:
self.log.exception(e)
self.log.warn('Therefore: Assume that there are no dependencies!')
return [] # no deps
out = out[9:] # strip 'Depends: '
depStrs = [entry.strip() for entry in out.split(',')]
return [entry.split(' ')[0] for entry in depStrs]
class CopyPath(BuildStep):
'''
This build step copies a path to a given destination path.
Shell wildcards are supported!
'''
parameters = {
'source': Parameter(type=str, description='Source path'),
'destination': Parameter(
type=str,
description='Destination path',
),
}
def run(self):
self.log.info('Copy %s to %s' % (self.source, self.destination))
for entry in glob.glob(self.source):
shutil.copytree(entry, self.destination)
class Partitioning(BuildStep):
parameters = {
'dev' : Parameter(type=str,
description='Path to the device file'),
'partitions' : Parameter(type=listof(referencer_or(int)),
description='List of partition sizes (in MB)')
}
def run(self):
cmds = []
for i in range(len(self.partitions)):
cmds += self._createPartitionCmds(i+1, self.partitions[i])
cmds.append('p') # print partition table
cmds.append('w') # write partition table
cmds.append('') # confirm
shCmd = '(%s)' % ''.join([ 'echo %s;' % entry for entry in cmds])
shCmd += '| fdisk %s 2>&1' % self.dev
systemCall(shCmd, log=self.log)
def _createPartitionCmds(self, index, size):
# TODO: better referencer handling: give step instead of chain
if isinstance(size, Referencer):
size = size.evaluate(self.chain, float)
cmds = [
'n' # new partition
]
if index < 4:
cmds.append('p') # primary
else:
cmds.append('e') # extended
cmds.append(str(index)) # partition number
cmds.append('') # confirm
cmds.append('+%dM' % size) # partition size
return cmds
class RmPath(BuildStep):
parameters = {
'path':
Parameter(type=str, description='Path to remove'),
'recursive':
Parameter(type=bool, description='Remove recursive', default=True),
}
def run(self):
self.log.info('Remove path: %s' % self.path)
if path.isfile(self.path):
os.remove(self.path)
elif path.isdir(self.path):
if self.recursive:
shutil.rmtree(self.path)
else:
os.rmdir(self.path)
if path.exists(self.path):
raise RuntimeError('Could not remove path')
class WriteFile(BuildStep):
parameters = {
'path':
Parameter(type=str, description='Path to target file'),
'content':
Parameter(type=str, description='Content wo write'),
'append':
Parameter(type=bool,
description='Append to the file '
'(if existing)',
default=False),
}
def run(self):
ensureDirectory(path.dirname(self.path))
openMode = 'a' if self.append else 'w'
self.log.info('Write to file %s ...' % self.path)
with open(self.path, openMode) as f:
f.write(self.content)
class Pdebuild(BuildStep):
'''
Build debian package via pdebuild.
'''
parameters = {
'sourcedir':
Parameter(type=str, description='Source directory'),
'config':
Parameter(type=str,
description='Pbuilder config file '
'(pbuilderrc)',
default='/etc/pbuilderrc'),
}
def run(self):
cwd = os.getcwd()
try:
os.chdir(self.sourcedir)
cmd = 'pdebuild --configfile %s' % self.config
systemCall(cmd, log=self.log)
finally:
os.chdir(cwd)
class ChrootedSystemCall(BuildStep):
'''
Build step to execute given shell command in a chroot environment.
'''
parameters = {
'command':
Parameter(type=str, description='command to execute'),
'chrootdir':
Parameter(type=str, description='Chroot directory', default='.'),
}
outparameters = {
'commandoutput':
Parameter(type=none_or(str),
description='Command output (if captured)',
default=None)
}
def run(self):
self.commandoutput = chrootedSystemCall(self.chrootdir,
self.command,
log=self.log)
class DevMapper(BuildStep):
'''
This build step uses kpartx (devmapper) to map the partitions of the given
device to own device files.
'''
parameters = {
'dev' : Parameter(type=str,
description='Path to the device file'),
}
outparameters = {
'mapped' : Parameter(type=list,
description='Created device files',),
'loopdev' : Parameter(type=str,
description='Used loop device',)
}
def run(self):
# request a proper formated list of created devs
out = systemCall('kpartx -v -l -s %s' % self.dev, log=self.log)
# create device files
systemCall('kpartx -v -a -s %s' % self.dev, log=self.log)
# store loop dev
self.loopdev = re.findall('(/dev/.*?) ', out)[0]
# store created device file paths
self.mapped = []
for line in out.splitlines():
devFile = line.rpartition(':')[0].strip()
self.mapped.append(path.join('/dev/mapper', devFile))
self.log.info('Loop device: %s' % self.loopdev)
self.log.info('Mapped devices: %s' % ', '.join(self.mapped))
def cleanup(self):
systemCall('kpartx -v -d -s %s' % self.dev,
log=self.log)
class TriggerCleanup(BuildStep):
'''
Build step that triggers the cleanup of another step.
'''
parameters = {
'step': Parameter(type=str, description='Step to clean up'),
}
def run(self):
self.log.info('Trigger cleanup of %s ...' % self.step)
if (self.step not in self.chain.steps):
self.error('Given step (%s) not found!' % self.step)
return
step = self.chain.steps[self.step]
step.cleanupBuild()
class Config(BuildStep):
'''
Build step to read given configuration file.
'''
parameters = {
'path':
Parameter(type=str,
description='Path to the configuration file',
default='.'),
'format':
Parameter(type=oneof('ini', 'py', 'auto'),
description='Format of config file',
default='auto'),
}
outparameters = {
'config':
Parameter(type=dict,
description='Command output (if captured)',
default={})
}
def run(self):
parseFuncs = {'ini': self._parseIni, 'py': self._parsePy}
self.log.info('Parse config: %s' % self.path)
configFormat = self.format
if configFormat == 'auto':
configFormat = path.splitext(self.path)[1][1:]
if configFormat not in parseFuncs.keys():
raise RuntimeError('Unsupported configuration format: %s' %
configFormat)
self.log.debug('Used format: %s' % configFormat)
self.config = parseFuncs[configFormat](self.path)
self.log.debug('Parsed config: %r' % self.config)
def _parseIni(self, path):
cfg = {}
parser = ConfigParser.SafeConfigParser()
parser.readfp(open(path))
for section in parser.sections():
cfg[section] = {}
for name, value in parser.items(section):
cfg[section][name] = value
return cfg
def _parsePy(self, path):
cfg = {}
content = open(path).read()
exec content in cfg
del cfg['__builtins__']
return cfg
class BuildStep(object):
__metaclass__ = BuildStepMeta
parameters = {
'description':
Parameter(type=str,
description='Build step description',
default='Undescribed'),
'loglevel':
Parameter(type=oneof(LOGLEVELS.keys()),
description='Log level',
default='info'),
'retries':
Parameter(type=int,
description='Number of retries to execute the '
'build step',
default=0),
}
outparameters = {}
def __init__(self, name, paramValues, chain=None):
self.name = name
self.chain = chain # Maintain a reference to the chain (for refs)
self.wasRun = False
self._params = {}
self._initLogger()
self._applyParams(paramValues)
def build(self):
'''
Build the build step.
This method is a wrapper around run() that does some logging and
exception handling.
'''
# log some bs stuff
self.log.info('=' * 80)
self.log.info('Build: %s' % self.name)
self.log.info(self.description)
self.log.info('-' * 80)
success = False
for i in range(0, self.retries + 1):
try:
# execute actual build actions
self.run()
self.wasRun = True
success = True
break
except Exception as exc:
self.log.exception(exc)
# handle retries
if self.retries > i:
self.log.warn('Failed; Retry %s/%s' %
(i + 1, self.retries))
# log some bs stuff
self.log.info('')
self.log.info('%s' % 'SUCCESS' if success else 'FAILED')
self.log.info('')
if not success:
raise RuntimeError('Build step failed')
def cleanupBuild(self):
'''
Cleanup all the build step's leftovers.
his method is a wrapper around cleanup() that does some logging and
exception handling.
'''
if not self.wasRun:
self.log.info('Cleanup: Step was not run; Skip')
return
if self.cleanup.im_func == BuildStep.cleanup.im_func:
self.log.info('Cleanup: No custom cleanup; Skip')
return
self.log.info('=' * 80)
self.log.info('Cleanup: %s' % self.name)
self.log.info(self.description)
self.log.info('-' * 80)
resultStr = 'SUCCESS'
try:
self.cleanup()
self.wasRun = False
except Exception as exc:
resultStr = 'FAILED'
self.log.exception(exc)
raise
finally:
self.log.info('')
self.log.info('%s' % resultStr)
self.log.info('')
def cleanup(self):
'''
This function shall be overwritten by the specific build steps
and should do everything that's necessary for cleaning up after
the build.
'''
pass
def run(self):
'''
This function shall be overwritten by the specific build steps
and should do everything that's necessary build this step.
'''
raise NotImplemented('Buildstep not implemented (may be abstract)')
def doWriteLoglevel(self, value):
self.log.setLevel(LOGLEVELS[value])
def doReadLoglevel(self):
level = self.log.getEffectiveLevel()
return INVLOGLEVELS[level]
def _initLogger(self):
if self.chain is not None:
self.log = self.chain.log.getChild(self.name)
else:
self.log = conduct.app.log.getChild(self.name)
self.log.setLevel(LOGLEVELS[self.loglevel])
def _applyParams(self, paramValues):
for name, paramDef in self.parameters.items():
if name in paramValues:
setattr(self, name, paramValues[name])
elif paramDef.default is None:
raise RuntimeError('%s: Mandatory parameter %s is missing' %
(self.name, name))