def _update(self, buildscript, copydir=None):
# sanity check the existing working tree:
if copydir:
outputdir = os.path.join(copydir, os.path.basename(self.srcdir))
else:
outputdir = self.srcdir
try:
wc_root = check_root(outputdir)
except IOError:
raise BuildStateError(
_('"%s" does not appear to be a CVS working copy') %
os.path.abspath(outputdir))
if wc_root != self.repository.cvsroot:
raise BuildStateError(
_('working copy points at the wrong repository (expected %(root1)s but got %(root2)s). '
) % {
'root1': self.repository.cvsroot,
'root2': wc_root
} +
_('Consider using the changecvsroot.py script to fix this.'))
# update the working tree
cmd = [
'cvs', '-z3', '-q', '-d', self.repository.cvsroot, 'update', '-P'
]
if self.update_new_dirs:
cmd.append('-d')
if self.revision:
cmd.extend(['-r', self.revision])
if self.config.sticky_date:
cmd.extend(['-D', self.config.sticky_date])
if not (self.revision or self.config.sticky_date):
cmd.append('-A')
cmd.append('.')
buildscript.execute(cmd, 'cvs', cwd=outputdir)
def _do_patches(self, buildscript):
# now patch the working tree
for (patch, patchstrip) in self.patches:
patchfile = ''
if urlparse.urlparse(patch)[0]:
# patch name has scheme, get patch from network
try:
patchfile = httpcache.load(
patch, nonetwork=buildscript.config.nonetwork)
except urllib2.HTTPError as e:
raise BuildStateError(
_('could not download patch (error: %s)') % e.code)
except urllib2.URLError as e:
raise BuildStateError(_('could not download patch'))
elif self.repository.moduleset_uri:
# get it relative to the moduleset uri, either in the same
# directory or a patches/ subdirectory
for patch_prefix in ('.', 'patches', '../patches'):
uri = urlparse.urljoin(self.repository.moduleset_uri,
os.path.join(patch_prefix, patch))
try:
patchfile = httpcache.load(
uri, nonetwork=buildscript.config.nonetwork)
except Exception as e:
continue
if not os.path.isfile(patchfile):
continue
break
else:
patchfile = ''
if not patchfile:
# nothing else, use jhbuild provided patches
possible_locations = []
if self.config.modulesets_dir:
possible_locations.append(
os.path.join(self.config.modulesets_dir, 'patches'))
possible_locations.append(
os.path.join(self.config.modulesets_dir, '../patches'))
if PKGDATADIR:
possible_locations.append(
os.path.join(PKGDATADIR, 'patches'))
if SRCDIR:
possible_locations.append(os.path.join(SRCDIR, 'patches'))
for dirname in possible_locations:
patchfile = os.path.join(dirname, patch)
if os.path.exists(patchfile):
break
else:
raise CommandError(_('Failed to find patch: %s') % patch)
buildscript.set_action(_('Applying patch'),
self,
action_target=patch)
# patchfile can be a relative file
buildscript.execute('patch -p%d < "%s"' %
(patchstrip, os.path.abspath(patchfile)),
cwd=self.raw_srcdir)
def _get_patch_files(self, buildscript):
patch_files = []
# now patch the working tree
for (patch, patchstrip) in self.patches:
patchfile = ''
if urlutils.urlparse(patch)[0]:
# patch name has scheme, get patch from network
try:
patchfile = httpcache.load(
patch, nonetwork=buildscript.config.nonetwork)
except urlutils.HTTPError as e:
raise BuildStateError(
_('could not download patch (error: %s)') % e.code)
except urlutils.URLError:
raise BuildStateError(_('could not download patch'))
elif self.repository.moduleset_uri:
# get it relative to the moduleset uri, either in the same
# directory or a patches/ subdirectory
for patch_prefix in ('.', 'patches', '../patches'):
uri = urlutils.urljoin(self.repository.moduleset_uri,
os.path.join(patch_prefix, patch))
try:
patchfile = httpcache.load(
uri, nonetwork=buildscript.config.nonetwork)
except Exception:
continue
if not os.path.isfile(patchfile):
continue
break
else:
patchfile = ''
if not patchfile:
# nothing else, use jhbuild provided patches
possible_locations = []
if self.config.modulesets_dir:
possible_locations.append(
os.path.join(self.config.modulesets_dir, 'patches'))
possible_locations.append(
os.path.join(self.config.modulesets_dir, '../patches'))
if PKGDATADIR:
possible_locations.append(
os.path.join(PKGDATADIR, 'patches'))
if SRCDIR:
possible_locations.append(os.path.join(SRCDIR, 'patches'))
for dirname in possible_locations:
patchfile = os.path.join(dirname, patch)
if os.path.exists(patchfile):
break
else:
raise CommandError(_('Failed to find patch: %s') % patch)
patch_files.append((patchfile, patch, patchstrip))
return patch_files
def get_uri(filename):
try:
info = get_info(filename)
except CommandError:
raise BuildStateError(_('could not get Subversion URI for %s')
% filename)
if 'url' not in info:
raise BuildStateError(_('could not parse "svn info" output for %s')
% filename)
return info['url']
def _do_patches(self, buildscript):
# now patch the working tree
for (patch, patchstrip) in self.patches:
patchfile = ''
if urlparse.urlparse(patch)[0]:
# patch name has scheme, get patch from network
try:
patchfile = httpcache.load(
patch, nonetwork=buildscript.config.nonetwork)
except urllib2.HTTPError, e:
raise BuildStateError(
_('could not download patch (error: %s)') % e.code)
except urllib2.URLError, e:
raise BuildStateError(_('could not download patch'))
def do_test(self, buildscript):
buildscript.set_action('Testing', self)
if not buildscript.config.noxvfb:
# start Xvfb
old_display = os.environ.get('DISPLAY')
old_xauth = os.environ.get('XAUTHORITY')
xvfb_pid = self._start_xvfb(buildscript.config.xvfbargs)
if xvfb_pid == -1:
raise BuildStateError('Unable to start Xvfb')
# either do_ldtp_test or do_dogtail_test
method = getattr(self, 'do_' + self.test_type + '_test')
try:
method(buildscript)
finally:
if not buildscript.config.noxvfb:
# kill Xvfb if it has been started
self._stop_xvfb(xvfb_pid)
if old_display:
os.environ['DISPLAY'] = old_display
else:
os.unsetenv('DISPLAY')
if old_xauth:
os.environ['XAUTHORITY'] = old_xauth
else:
os.unsetenv('XAUTHORITY')
def _download_and_unpack(self, buildscript):
localfile = self._local_tarball
if not os.path.exists(self.config.tarballdir):
try:
os.makedirs(self.config.tarballdir)
except OSError:
raise FatalError(
_('tarball dir (%s) can not be created') %
self.config.tarballdir)
try:
self._check_tarball()
except BuildStateError:
# don't have the tarball, try downloading it and check again
self._download_tarball(buildscript, localfile)
self._check_tarball()
# now to unpack it
try:
unpack_archive(buildscript, localfile, self.checkoutroot,
self.checkoutdir)
except CommandError:
raise FatalError(_('failed to unpack %s') % localfile)
if not os.path.exists(self.srcdir):
raise BuildStateError(
_('could not unpack tarball (expected %s dir)') %
os.path.basename(self.srcdir))
if self.patches:
self._do_patches(buildscript)
def _download_and_unpack(self, buildscript):
localfile = self._local_tarball
if not os.path.exists(self.config.tarballdir):
try:
os.makedirs(self.config.tarballdir)
except OSError:
raise FatalError(
_('tarball dir (%s) can not be created') %
self.config.tarballdir)
if not os.access(self.config.tarballdir, os.R_OK | os.W_OK | os.X_OK):
raise FatalError(
_('tarball dir (%s) must be writable') %
self.config.tarballdir)
try:
self._check_tarball()
except BuildStateError:
# don't have the tarball, try downloading it and check again
if has_command('wget'):
res = buildscript.execute(
['wget', '--continue', self.module, '-O', localfile],
extra_env={
'LD_LIBRARY_PATH':
os.environ.get('UNMANGLED_LD_LIBRARY_PATH'),
'PATH':
os.environ.get('UNMANGLED_PATH')
})
elif has_command('curl'):
res = buildscript.execute(
[
'curl', '--continue-at', '-', '-L', self.module, '-o',
localfile
],
extra_env={
'LD_LIBRARY_PATH':
os.environ.get('UNMANGLED_LD_LIBRARY_PATH'),
'PATH':
os.environ.get('UNMANGLED_PATH')
})
else:
raise FatalError(_("unable to find wget or curl"))
self._check_tarball()
# now to unpack it
try:
unpack_archive(buildscript, localfile, self.checkoutroot,
self.checkoutdir)
except CommandError:
raise FatalError(_('failed to unpack %s') % localfile)
if not os.path.exists(self.srcdir):
raise BuildStateError(
_('could not unpack tarball (expected %s dir)') %
os.path.basename(self.srcdir))
if self.patches:
self._do_patches(buildscript)
def checkout(self, buildscript):
srcdir = self.get_srcdir(buildscript)
buildscript.set_action(_('Checking out'), self)
self.branch.checkout(buildscript)
# did the checkout succeed?
if not os.path.exists(srcdir):
raise BuildStateError(_('source directory %s was not created') % srcdir)
if self.check_build_policy(buildscript) == self.PHASE_DONE:
raise SkipToEnd()
def _check_tarball(self):
"""Check whether the tarball has been downloaded correctly."""
localfile = self._local_tarball
if not os.path.exists(localfile):
raise BuildStateError(_('file not downloaded'))
if self.source_size is not None:
local_size = os.stat(localfile).st_size
if local_size != self.source_size:
raise BuildStateError(
_('downloaded file size is incorrect (expected %(size1)d, got %(size2)d)'
) % {
'size1': self.source_size,
'size2': local_size
})
if self.source_hash is not None:
try:
algo, hash = self.source_hash.split(':')
except ValueError:
logging.warning(
_('invalid hash attribute on module %s') % self.module)
return
if hasattr(hashlib, algo):
local_hash = getattr(hashlib, algo)()
fp = open(localfile, 'rb')
data = fp.read(32768)
while data:
local_hash.update(data)
data = fp.read(32768)
fp.close()
if local_hash.hexdigest() != hash:
raise BuildStateError(
_('file hash is incorrect (expected %(sum1)s, got %(sum2)s)'
) % {
'sum1': hash,
'sum2': local_hash.hexdigest()
})
else:
logging.warning(
_('skipped hash check (missing support for %s)') % algo)
def do_ldtp_test(self, buildscript):
src_dir = self.get_srcdir(buildscript)
old_debug = os.getenv('LDTP_DEBUG')
if old_debug != None:
del os.environ['LDTP_DEBUG']
ldtp_pid = self._start_ldtp()
if ldtp_pid == -1:
raise BuildStateError('Unable to start ldtp server')
try:
if buildscript.config.noxvfb:
buildscript.execute('ldtprunner run.xml', cwd=src_dir)
else:
buildscript.execute(
'ldtprunner run.xml',
cwd=src_dir,
extra_env={'DISPLAY': ':%s' % self.screennum})
except CommandError, e:
os.kill(ldtp_pid, signal.SIGINT)
if e.returncode == 32512: # ldtprunner not installed
raise BuildStateError('ldtprunner not available')
raise BuildStateError('error %s during test' % e.returncode)
def ninja(self, buildscript, target='', ninjaargs=None, env=None):
ninjacmd = os.environ.get('NINJA', self.get_ninjacmd(buildscript.config))
if ninjacmd is None:
raise BuildStateError(_('ninja not found; use NINJA to point to a specific ninja binary'))
if ninjaargs is None:
ninjaargs = self.get_ninjaargs(buildscript)
extra_env = (self.extra_env or {}).copy()
for k in (env or {}):
extra_env[k] = env[k]
cmd = '{ninja} {ninjaargs} {target}'.format(ninja=ninjacmd,
ninjaargs=ninjaargs,
target=target)
buildscript.execute(cmd, cwd=self.get_builddir(buildscript), extra_env=extra_env)
def do_dogtail_test(self, buildscript):
src_dir = self.get_srcdir(buildscript)
test_cases = []
all_files = os.listdir(src_dir)
for file in all_files:
if file[-3:] == '.py':
test_cases.append(file)
if buildscript.config.noxvfb:
extra_env = {}
else:
extra_env = {'DISPLAY': ':%s' % self.screennum}
for test_case in test_cases:
try:
buildscript.execute('python %s' % test_case,
cwd=src_dir,
extra_env=extra_env)
except CommandError as e:
if e.returncode != 0:
raise BuildStateError('%s failed' % test_case)
def do_ldtp_test(self, buildscript):
src_dir = self.get_srcdir(buildscript)
old_debug = os.getenv('LDTP_DEBUG')
if old_debug is not None:
del os.environ['LDTP_DEBUG']
ldtp_pid = self._start_ldtp()
if ldtp_pid == -1:
raise BuildStateError('Unable to start ldtp server')
try:
if buildscript.config.noxvfb:
buildscript.execute('ldtprunner run.xml', cwd=src_dir)
else:
buildscript.execute(
'ldtprunner run.xml',
cwd=src_dir,
extra_env={'DISPLAY': ':%s' % self.screennum})
except CommandError as e:
os.kill(ldtp_pid, signal.SIGINT)
if e.returncode == 32512: # ldtprunner not installed
raise BuildStateError('ldtprunner not available')
raise BuildStateError('error %s during test' % e.returncode)
os.kill(ldtp_pid, signal.SIGINT)
if old_debug is not None:
os.environ['LDTP_DEBUG'] = old_debug
log_file = self.get_ldtp_log_file(os.path.join(src_dir, 'run.xml'))
if not log_file:
raise BuildStateError('missing log file')
try:
groups = self._check_ldtp_log_file(log_file)
flag, status = self.check_groups(groups)
if flag:
raise BuildStateError(status)
except Exception:
raise BuildStateError('malformed log file')
def checkout(self, buildscript):
if self.branch:
self.branch.checkout(buildscript)
if not os.path.exists(self.path):
raise BuildStateError(
_('kconfig file %s was not created') % self.path)
class TestModule(Package, DownloadableModule):
type = 'test'
PHASE_CHECKOUT = DownloadableModule.PHASE_CHECKOUT
PHASE_FORCE_CHECKOUT = DownloadableModule.PHASE_FORCE_CHECKOUT
PHASE_TEST = 'test'
def __init__(self, name, branch=None, test_type=None, tested_pkgs=[]):
Package.__init__(self, name, branch=branch)
self.test_type = test_type
self.tested_pkgs = tested_pkgs
### modify environ for tests to be working
if os.environ.has_key('LDTP_DEBUG'):
del os.environ['LDTP_DEBUG'] # get rid of verbose LDTP output
if not os.environ.has_key('GNOME_ACCESSIBILITY'
) or os.environ['GNOME_ACCESSIBILITY'] != 1:
os.environ['GNOME_ACCESSIBILITY'] = '1'
def get_srcdir(self, buildscript):
return self.branch.srcdir
def _get_display(self):
# get free display
servernum = 99
while True:
if not os.path.exists('/tmp/.X%s-lock' % servernum):
break
servernum += 1
return str(servernum)
def _set_xauth(self, servernum):
# create auth file
paths = os.environ.get('PATH').split(':')
flag = False
for path in paths:
if os.path.exists(os.path.join(path, 'xauth')):
flag = True
break
tmpdir = tempfile.gettempdir()
if not flag or os.path.exists(
os.path.join(tmpdir, 'jhbuild.%s' % os.getpid())):
return ''
try:
os.mkdir(os.path.join(tmpdir, 'jhbuild.%s' % os.getpid()))
new_xauth = os.path.join(tmpdir, 'jhbuild.%s' % os.getpid(),
'Xauthority')
open(new_xauth, 'w').close()
hexdigest = hashlib.md5(str(random.random())).hexdigest()
os.system('xauth -f "%s" add ":%s" "." "%s"' %
(new_xauth, servernum, hexdigest))
except OSError:
return ''
return new_xauth
def do_test(self, buildscript):
buildscript.set_action('Testing', self)
if not buildscript.config.noxvfb:
# start Xvfb
old_display = os.environ.get('DISPLAY')
old_xauth = os.environ.get('XAUTHORITY')
xvfb_pid = self._start_xvfb(buildscript.config.xvfbargs)
if xvfb_pid == -1:
raise BuildStateError('Unable to start Xvfb')
# either do_ldtp_test or do_dogtail_test
method = getattr(self, 'do_' + self.test_type + '_test')
try:
method(buildscript)
finally:
if not buildscript.config.noxvfb:
# kill Xvfb if it has been started
self._stop_xvfb(xvfb_pid)
if old_display:
os.environ['DISPLAY'] = old_display
else:
os.unsetenv('DISPLAY')
if old_xauth:
os.environ['XAUTHORITY'] = old_xauth
else:
os.unsetenv('XAUTHORITY')
do_test.depends = [PHASE_CHECKOUT]
def get_ldtp_log_file(self, filename):
# <ldtp>
# |
# -- <logfile>filename</logfile>
run_file = xml.dom.minidom.parse(filename)
try:
return run_file.getElementsByTagName('ldtp')[
0].getElementsByTagName('logfile')[0].childNodes[0].data
except IndexError:
return None
def _get_ldtp_info(self, node):
infos = []
errors = []
warnings = []
causes = []
for info in node.getElementsByTagName('info'):
for child in info.childNodes:
infos.append(child.data)
for cause in node.getElementsByTagName('cause'):
for child in cause.childNodes:
causes.append(child.data)
for error in node.getElementsByTagName('error'):
for child in error.childNodes:
errors.append(child.data)
for warning in node.getElementsByTagName('warning'):
for child in warning.childNodes:
warnings.append(child.data)
return infos, errors, causes, warnings
def _check_ldtp_log_file(self, logfile):
log_file = xml.dom.minidom.parse(logfile)
ldtp_node = log_file.getElementsByTagName('ldtp')[0]
groups = []
for group in ldtp_node.getElementsByTagName('group'):
scr = []
for script in group.getElementsByTagName('script'):
tests = {}
for test in script.getElementsByTagName('test'):
test_name = test.getAttribute('name')
pass_status = test.getElementsByTagName(
'pass')[0].childNodes[0].data
infos, errors, causes, warnings = self._get_ldtp_info(test)
tests[test_name] = {
'pass': pass_status,
'info': infos,
'error': errors,
'cause': causes,
'warning': warnings
}
infos, errors, causes, warnings = self._get_ldtp_info(script)
scr.append({
'tests': tests,
'info': infos,
'error': errors,
'cause': causes,
'warning': warnings
})
groupstatus = group.getElementsByTagName(
'groupstatus')[0].childNodes[0].data
groups.append({'script': scr, 'groupstatus': groupstatus})
return groups
def check_groups(self, groups):
group_num = 1
flag = False
status = ''
for group in groups:
status += 'In Group #%s (%s)\n' % (group_num, group['groupstatus'])
for script in group['script']:
for test in script['tests'].keys():
status += 'Test \'%s\' ' % test
if script['tests'][test]['pass'] == '0': # failed
status += 'failed\n\tErrors'
for error in script['tests'][test]['error']:
status += ', '
status += error
status += '\n\tCauses'
for cause in script['tests'][test]['cause']:
status += ', '
status += cause
status += '\n\tWarnings'
for warning in script['tests'][test]['warning']:
status += ', '
status += warning
status += '\n\tInfos'
for info in script['tests'][test]['info']:
status += ', '
status += info
else:
status += 'passed'
status += '\n'
group_num += 1
if self._check_ldtp_group_status(group['groupstatus']):
flag = True
return flag, status
def _check_ldtp_group_status(self, status):
status = status.split()
if status[0] != status[-1]:
return True
return False
def _start_xvfb(self, xvfbargs):
new_display = self._get_display()
new_xauth = self._set_xauth(new_display)
if new_xauth == '':
return -1
os.environ['DISPLAY'] = ':' + new_display
os.environ['XAUTHORITY'] = new_xauth
try:
xvfb = subprocess.Popen(['Xvfb', ':' + new_display] +
xvfbargs.split(),
shell=False)
self.screennum = new_display
self.xauth = new_xauth
except OSError:
return -1
time.sleep(2) #allow Xvfb to start
if xvfb.poll() != None:
return -1
return xvfb.pid
def _stop_xvfb(self, xvfb_pid):
os.kill(xvfb_pid, signal.SIGINT)
os.system('xauth remove ":%s"' % self.screennum)
os.system('rm -r %s' % os.path.split(self.xauth)[0])
def _start_ldtp(self):
try:
ldtp = subprocess.Popen('ldtp', shell=False)
except OSError:
return -1
time.sleep(1)
if ldtp.poll() != None:
return -1
return ldtp.pid
def do_ldtp_test(self, buildscript):
src_dir = self.get_srcdir(buildscript)
old_debug = os.getenv('LDTP_DEBUG')
if old_debug != None:
del os.environ['LDTP_DEBUG']
ldtp_pid = self._start_ldtp()
if ldtp_pid == -1:
raise BuildStateError('Unable to start ldtp server')
try:
if buildscript.config.noxvfb:
buildscript.execute('ldtprunner run.xml', cwd=src_dir)
else:
buildscript.execute(
'ldtprunner run.xml',
cwd=src_dir,
extra_env={'DISPLAY': ':%s' % self.screennum})
except CommandError, e:
os.kill(ldtp_pid, signal.SIGINT)
if e.returncode == 32512: # ldtprunner not installed
raise BuildStateError('ldtprunner not available')
raise BuildStateError('error %s during test' % e.returncode)
os.kill(ldtp_pid, signal.SIGINT)
if old_debug != None:
os.environ['LDTP_DEBUG'] = old_debug
log_file = self.get_ldtp_log_file(os.path.join(src_dir, 'run.xml'))
if not log_file:
raise BuildStateError('missing log file')
try:
groups = self._check_ldtp_log_file(log_file)
flag, status = self.check_groups(groups)
if flag:
raise BuildStateError(status)
except:
raise BuildStateError('malformed log file')