def _update(self, buildscript, copydir=None): # sanity check the existing working tree: if copydir: outputdir = os.path.join(copydir, os.path.basename(self.srcdir)) else: outputdir = self.srcdir try: wc_root = check_root(outputdir) except IOError: raise BuildStateError( _('"%s" does not appear to be a CVS working copy') % os.path.abspath(outputdir)) if wc_root != self.repository.cvsroot: raise BuildStateError( _('working copy points at the wrong repository (expected %(root1)s but got %(root2)s). ' ) % { 'root1': self.repository.cvsroot, 'root2': wc_root } + _('Consider using the changecvsroot.py script to fix this.')) # update the working tree cmd = [ 'cvs', '-z3', '-q', '-d', self.repository.cvsroot, 'update', '-P' ] if self.update_new_dirs: cmd.append('-d') if self.revision: cmd.extend(['-r', self.revision]) if self.config.sticky_date: cmd.extend(['-D', self.config.sticky_date]) if not (self.revision or self.config.sticky_date): cmd.append('-A') cmd.append('.') buildscript.execute(cmd, 'cvs', cwd=outputdir)
def _do_patches(self, buildscript): # now patch the working tree for (patch, patchstrip) in self.patches: patchfile = '' if urlparse.urlparse(patch)[0]: # patch name has scheme, get patch from network try: patchfile = httpcache.load( patch, nonetwork=buildscript.config.nonetwork) except urllib2.HTTPError as e: raise BuildStateError( _('could not download patch (error: %s)') % e.code) except urllib2.URLError as e: raise BuildStateError(_('could not download patch')) elif self.repository.moduleset_uri: # get it relative to the moduleset uri, either in the same # directory or a patches/ subdirectory for patch_prefix in ('.', 'patches', '../patches'): uri = urlparse.urljoin(self.repository.moduleset_uri, os.path.join(patch_prefix, patch)) try: patchfile = httpcache.load( uri, nonetwork=buildscript.config.nonetwork) except Exception as e: continue if not os.path.isfile(patchfile): continue break else: patchfile = '' if not patchfile: # nothing else, use jhbuild provided patches possible_locations = [] if self.config.modulesets_dir: possible_locations.append( os.path.join(self.config.modulesets_dir, 'patches')) possible_locations.append( os.path.join(self.config.modulesets_dir, '../patches')) if PKGDATADIR: possible_locations.append( os.path.join(PKGDATADIR, 'patches')) if SRCDIR: possible_locations.append(os.path.join(SRCDIR, 'patches')) for dirname in possible_locations: patchfile = os.path.join(dirname, patch) if os.path.exists(patchfile): break else: raise CommandError(_('Failed to find patch: %s') % patch) buildscript.set_action(_('Applying patch'), self, action_target=patch) # patchfile can be a relative file buildscript.execute('patch -p%d < "%s"' % (patchstrip, os.path.abspath(patchfile)), cwd=self.raw_srcdir)
def _get_patch_files(self, buildscript): patch_files = [] # now patch the working tree for (patch, patchstrip) in self.patches: patchfile = '' if urlutils.urlparse(patch)[0]: # patch name has scheme, get patch from network try: patchfile = httpcache.load( patch, nonetwork=buildscript.config.nonetwork) except urlutils.HTTPError as e: raise BuildStateError( _('could not download patch (error: %s)') % e.code) except urlutils.URLError: raise BuildStateError(_('could not download patch')) elif self.repository.moduleset_uri: # get it relative to the moduleset uri, either in the same # directory or a patches/ subdirectory for patch_prefix in ('.', 'patches', '../patches'): uri = urlutils.urljoin(self.repository.moduleset_uri, os.path.join(patch_prefix, patch)) try: patchfile = httpcache.load( uri, nonetwork=buildscript.config.nonetwork) except Exception: continue if not os.path.isfile(patchfile): continue break else: patchfile = '' if not patchfile: # nothing else, use jhbuild provided patches possible_locations = [] if self.config.modulesets_dir: possible_locations.append( os.path.join(self.config.modulesets_dir, 'patches')) possible_locations.append( os.path.join(self.config.modulesets_dir, '../patches')) if PKGDATADIR: possible_locations.append( os.path.join(PKGDATADIR, 'patches')) if SRCDIR: possible_locations.append(os.path.join(SRCDIR, 'patches')) for dirname in possible_locations: patchfile = os.path.join(dirname, patch) if os.path.exists(patchfile): break else: raise CommandError(_('Failed to find patch: %s') % patch) patch_files.append((patchfile, patch, patchstrip)) return patch_files
def get_uri(filename): try: info = get_info(filename) except CommandError: raise BuildStateError(_('could not get Subversion URI for %s') % filename) if 'url' not in info: raise BuildStateError(_('could not parse "svn info" output for %s') % filename) return info['url']
def _do_patches(self, buildscript): # now patch the working tree for (patch, patchstrip) in self.patches: patchfile = '' if urlparse.urlparse(patch)[0]: # patch name has scheme, get patch from network try: patchfile = httpcache.load( patch, nonetwork=buildscript.config.nonetwork) except urllib2.HTTPError, e: raise BuildStateError( _('could not download patch (error: %s)') % e.code) except urllib2.URLError, e: raise BuildStateError(_('could not download patch'))
def do_test(self, buildscript): buildscript.set_action('Testing', self) if not buildscript.config.noxvfb: # start Xvfb old_display = os.environ.get('DISPLAY') old_xauth = os.environ.get('XAUTHORITY') xvfb_pid = self._start_xvfb(buildscript.config.xvfbargs) if xvfb_pid == -1: raise BuildStateError('Unable to start Xvfb') # either do_ldtp_test or do_dogtail_test method = getattr(self, 'do_' + self.test_type + '_test') try: method(buildscript) finally: if not buildscript.config.noxvfb: # kill Xvfb if it has been started self._stop_xvfb(xvfb_pid) if old_display: os.environ['DISPLAY'] = old_display else: os.unsetenv('DISPLAY') if old_xauth: os.environ['XAUTHORITY'] = old_xauth else: os.unsetenv('XAUTHORITY')
def _download_and_unpack(self, buildscript): localfile = self._local_tarball if not os.path.exists(self.config.tarballdir): try: os.makedirs(self.config.tarballdir) except OSError: raise FatalError( _('tarball dir (%s) can not be created') % self.config.tarballdir) try: self._check_tarball() except BuildStateError: # don't have the tarball, try downloading it and check again self._download_tarball(buildscript, localfile) self._check_tarball() # now to unpack it try: unpack_archive(buildscript, localfile, self.checkoutroot, self.checkoutdir) except CommandError: raise FatalError(_('failed to unpack %s') % localfile) if not os.path.exists(self.srcdir): raise BuildStateError( _('could not unpack tarball (expected %s dir)') % os.path.basename(self.srcdir)) if self.patches: self._do_patches(buildscript)
def _download_and_unpack(self, buildscript): localfile = self._local_tarball if not os.path.exists(self.config.tarballdir): try: os.makedirs(self.config.tarballdir) except OSError: raise FatalError( _('tarball dir (%s) can not be created') % self.config.tarballdir) if not os.access(self.config.tarballdir, os.R_OK | os.W_OK | os.X_OK): raise FatalError( _('tarball dir (%s) must be writable') % self.config.tarballdir) try: self._check_tarball() except BuildStateError: # don't have the tarball, try downloading it and check again if has_command('wget'): res = buildscript.execute( ['wget', '--continue', self.module, '-O', localfile], extra_env={ 'LD_LIBRARY_PATH': os.environ.get('UNMANGLED_LD_LIBRARY_PATH'), 'PATH': os.environ.get('UNMANGLED_PATH') }) elif has_command('curl'): res = buildscript.execute( [ 'curl', '--continue-at', '-', '-L', self.module, '-o', localfile ], extra_env={ 'LD_LIBRARY_PATH': os.environ.get('UNMANGLED_LD_LIBRARY_PATH'), 'PATH': os.environ.get('UNMANGLED_PATH') }) else: raise FatalError(_("unable to find wget or curl")) self._check_tarball() # now to unpack it try: unpack_archive(buildscript, localfile, self.checkoutroot, self.checkoutdir) except CommandError: raise FatalError(_('failed to unpack %s') % localfile) if not os.path.exists(self.srcdir): raise BuildStateError( _('could not unpack tarball (expected %s dir)') % os.path.basename(self.srcdir)) if self.patches: self._do_patches(buildscript)
def checkout(self, buildscript): srcdir = self.get_srcdir(buildscript) buildscript.set_action(_('Checking out'), self) self.branch.checkout(buildscript) # did the checkout succeed? if not os.path.exists(srcdir): raise BuildStateError(_('source directory %s was not created') % srcdir) if self.check_build_policy(buildscript) == self.PHASE_DONE: raise SkipToEnd()
def _check_tarball(self): """Check whether the tarball has been downloaded correctly.""" localfile = self._local_tarball if not os.path.exists(localfile): raise BuildStateError(_('file not downloaded')) if self.source_size is not None: local_size = os.stat(localfile).st_size if local_size != self.source_size: raise BuildStateError( _('downloaded file size is incorrect (expected %(size1)d, got %(size2)d)' ) % { 'size1': self.source_size, 'size2': local_size }) if self.source_hash is not None: try: algo, hash = self.source_hash.split(':') except ValueError: logging.warning( _('invalid hash attribute on module %s') % self.module) return if hasattr(hashlib, algo): local_hash = getattr(hashlib, algo)() fp = open(localfile, 'rb') data = fp.read(32768) while data: local_hash.update(data) data = fp.read(32768) fp.close() if local_hash.hexdigest() != hash: raise BuildStateError( _('file hash is incorrect (expected %(sum1)s, got %(sum2)s)' ) % { 'sum1': hash, 'sum2': local_hash.hexdigest() }) else: logging.warning( _('skipped hash check (missing support for %s)') % algo)
def do_ldtp_test(self, buildscript): src_dir = self.get_srcdir(buildscript) old_debug = os.getenv('LDTP_DEBUG') if old_debug != None: del os.environ['LDTP_DEBUG'] ldtp_pid = self._start_ldtp() if ldtp_pid == -1: raise BuildStateError('Unable to start ldtp server') try: if buildscript.config.noxvfb: buildscript.execute('ldtprunner run.xml', cwd=src_dir) else: buildscript.execute( 'ldtprunner run.xml', cwd=src_dir, extra_env={'DISPLAY': ':%s' % self.screennum}) except CommandError, e: os.kill(ldtp_pid, signal.SIGINT) if e.returncode == 32512: # ldtprunner not installed raise BuildStateError('ldtprunner not available') raise BuildStateError('error %s during test' % e.returncode)
def ninja(self, buildscript, target='', ninjaargs=None, env=None): ninjacmd = os.environ.get('NINJA', self.get_ninjacmd(buildscript.config)) if ninjacmd is None: raise BuildStateError(_('ninja not found; use NINJA to point to a specific ninja binary')) if ninjaargs is None: ninjaargs = self.get_ninjaargs(buildscript) extra_env = (self.extra_env or {}).copy() for k in (env or {}): extra_env[k] = env[k] cmd = '{ninja} {ninjaargs} {target}'.format(ninja=ninjacmd, ninjaargs=ninjaargs, target=target) buildscript.execute(cmd, cwd=self.get_builddir(buildscript), extra_env=extra_env)
def do_dogtail_test(self, buildscript): src_dir = self.get_srcdir(buildscript) test_cases = [] all_files = os.listdir(src_dir) for file in all_files: if file[-3:] == '.py': test_cases.append(file) if buildscript.config.noxvfb: extra_env = {} else: extra_env = {'DISPLAY': ':%s' % self.screennum} for test_case in test_cases: try: buildscript.execute('python %s' % test_case, cwd=src_dir, extra_env=extra_env) except CommandError as e: if e.returncode != 0: raise BuildStateError('%s failed' % test_case)
def do_ldtp_test(self, buildscript): src_dir = self.get_srcdir(buildscript) old_debug = os.getenv('LDTP_DEBUG') if old_debug is not None: del os.environ['LDTP_DEBUG'] ldtp_pid = self._start_ldtp() if ldtp_pid == -1: raise BuildStateError('Unable to start ldtp server') try: if buildscript.config.noxvfb: buildscript.execute('ldtprunner run.xml', cwd=src_dir) else: buildscript.execute( 'ldtprunner run.xml', cwd=src_dir, extra_env={'DISPLAY': ':%s' % self.screennum}) except CommandError as e: os.kill(ldtp_pid, signal.SIGINT) if e.returncode == 32512: # ldtprunner not installed raise BuildStateError('ldtprunner not available') raise BuildStateError('error %s during test' % e.returncode) os.kill(ldtp_pid, signal.SIGINT) if old_debug is not None: os.environ['LDTP_DEBUG'] = old_debug log_file = self.get_ldtp_log_file(os.path.join(src_dir, 'run.xml')) if not log_file: raise BuildStateError('missing log file') try: groups = self._check_ldtp_log_file(log_file) flag, status = self.check_groups(groups) if flag: raise BuildStateError(status) except Exception: raise BuildStateError('malformed log file')
def checkout(self, buildscript): if self.branch: self.branch.checkout(buildscript) if not os.path.exists(self.path): raise BuildStateError( _('kconfig file %s was not created') % self.path)
class TestModule(Package, DownloadableModule): type = 'test' PHASE_CHECKOUT = DownloadableModule.PHASE_CHECKOUT PHASE_FORCE_CHECKOUT = DownloadableModule.PHASE_FORCE_CHECKOUT PHASE_TEST = 'test' def __init__(self, name, branch=None, test_type=None, tested_pkgs=[]): Package.__init__(self, name, branch=branch) self.test_type = test_type self.tested_pkgs = tested_pkgs ### modify environ for tests to be working if os.environ.has_key('LDTP_DEBUG'): del os.environ['LDTP_DEBUG'] # get rid of verbose LDTP output if not os.environ.has_key('GNOME_ACCESSIBILITY' ) or os.environ['GNOME_ACCESSIBILITY'] != 1: os.environ['GNOME_ACCESSIBILITY'] = '1' def get_srcdir(self, buildscript): return self.branch.srcdir def _get_display(self): # get free display servernum = 99 while True: if not os.path.exists('/tmp/.X%s-lock' % servernum): break servernum += 1 return str(servernum) def _set_xauth(self, servernum): # create auth file paths = os.environ.get('PATH').split(':') flag = False for path in paths: if os.path.exists(os.path.join(path, 'xauth')): flag = True break tmpdir = tempfile.gettempdir() if not flag or os.path.exists( os.path.join(tmpdir, 'jhbuild.%s' % os.getpid())): return '' try: os.mkdir(os.path.join(tmpdir, 'jhbuild.%s' % os.getpid())) new_xauth = os.path.join(tmpdir, 'jhbuild.%s' % os.getpid(), 'Xauthority') open(new_xauth, 'w').close() hexdigest = hashlib.md5(str(random.random())).hexdigest() os.system('xauth -f "%s" add ":%s" "." "%s"' % (new_xauth, servernum, hexdigest)) except OSError: return '' return new_xauth def do_test(self, buildscript): buildscript.set_action('Testing', self) if not buildscript.config.noxvfb: # start Xvfb old_display = os.environ.get('DISPLAY') old_xauth = os.environ.get('XAUTHORITY') xvfb_pid = self._start_xvfb(buildscript.config.xvfbargs) if xvfb_pid == -1: raise BuildStateError('Unable to start Xvfb') # either do_ldtp_test or do_dogtail_test method = getattr(self, 'do_' + self.test_type + '_test') try: method(buildscript) finally: if not buildscript.config.noxvfb: # kill Xvfb if it has been started self._stop_xvfb(xvfb_pid) if old_display: os.environ['DISPLAY'] = old_display else: os.unsetenv('DISPLAY') if old_xauth: os.environ['XAUTHORITY'] = old_xauth else: os.unsetenv('XAUTHORITY') do_test.depends = [PHASE_CHECKOUT] def get_ldtp_log_file(self, filename): # <ldtp> # | # -- <logfile>filename</logfile> run_file = xml.dom.minidom.parse(filename) try: return run_file.getElementsByTagName('ldtp')[ 0].getElementsByTagName('logfile')[0].childNodes[0].data except IndexError: return None def _get_ldtp_info(self, node): infos = [] errors = [] warnings = [] causes = [] for info in node.getElementsByTagName('info'): for child in info.childNodes: infos.append(child.data) for cause in node.getElementsByTagName('cause'): for child in cause.childNodes: causes.append(child.data) for error in node.getElementsByTagName('error'): for child in error.childNodes: errors.append(child.data) for warning in node.getElementsByTagName('warning'): for child in warning.childNodes: warnings.append(child.data) return infos, errors, causes, warnings def _check_ldtp_log_file(self, logfile): log_file = xml.dom.minidom.parse(logfile) ldtp_node = log_file.getElementsByTagName('ldtp')[0] groups = [] for group in ldtp_node.getElementsByTagName('group'): scr = [] for script in group.getElementsByTagName('script'): tests = {} for test in script.getElementsByTagName('test'): test_name = test.getAttribute('name') pass_status = test.getElementsByTagName( 'pass')[0].childNodes[0].data infos, errors, causes, warnings = self._get_ldtp_info(test) tests[test_name] = { 'pass': pass_status, 'info': infos, 'error': errors, 'cause': causes, 'warning': warnings } infos, errors, causes, warnings = self._get_ldtp_info(script) scr.append({ 'tests': tests, 'info': infos, 'error': errors, 'cause': causes, 'warning': warnings }) groupstatus = group.getElementsByTagName( 'groupstatus')[0].childNodes[0].data groups.append({'script': scr, 'groupstatus': groupstatus}) return groups def check_groups(self, groups): group_num = 1 flag = False status = '' for group in groups: status += 'In Group #%s (%s)\n' % (group_num, group['groupstatus']) for script in group['script']: for test in script['tests'].keys(): status += 'Test \'%s\' ' % test if script['tests'][test]['pass'] == '0': # failed status += 'failed\n\tErrors' for error in script['tests'][test]['error']: status += ', ' status += error status += '\n\tCauses' for cause in script['tests'][test]['cause']: status += ', ' status += cause status += '\n\tWarnings' for warning in script['tests'][test]['warning']: status += ', ' status += warning status += '\n\tInfos' for info in script['tests'][test]['info']: status += ', ' status += info else: status += 'passed' status += '\n' group_num += 1 if self._check_ldtp_group_status(group['groupstatus']): flag = True return flag, status def _check_ldtp_group_status(self, status): status = status.split() if status[0] != status[-1]: return True return False def _start_xvfb(self, xvfbargs): new_display = self._get_display() new_xauth = self._set_xauth(new_display) if new_xauth == '': return -1 os.environ['DISPLAY'] = ':' + new_display os.environ['XAUTHORITY'] = new_xauth try: xvfb = subprocess.Popen(['Xvfb', ':' + new_display] + xvfbargs.split(), shell=False) self.screennum = new_display self.xauth = new_xauth except OSError: return -1 time.sleep(2) #allow Xvfb to start if xvfb.poll() != None: return -1 return xvfb.pid def _stop_xvfb(self, xvfb_pid): os.kill(xvfb_pid, signal.SIGINT) os.system('xauth remove ":%s"' % self.screennum) os.system('rm -r %s' % os.path.split(self.xauth)[0]) def _start_ldtp(self): try: ldtp = subprocess.Popen('ldtp', shell=False) except OSError: return -1 time.sleep(1) if ldtp.poll() != None: return -1 return ldtp.pid def do_ldtp_test(self, buildscript): src_dir = self.get_srcdir(buildscript) old_debug = os.getenv('LDTP_DEBUG') if old_debug != None: del os.environ['LDTP_DEBUG'] ldtp_pid = self._start_ldtp() if ldtp_pid == -1: raise BuildStateError('Unable to start ldtp server') try: if buildscript.config.noxvfb: buildscript.execute('ldtprunner run.xml', cwd=src_dir) else: buildscript.execute( 'ldtprunner run.xml', cwd=src_dir, extra_env={'DISPLAY': ':%s' % self.screennum}) except CommandError, e: os.kill(ldtp_pid, signal.SIGINT) if e.returncode == 32512: # ldtprunner not installed raise BuildStateError('ldtprunner not available') raise BuildStateError('error %s during test' % e.returncode) os.kill(ldtp_pid, signal.SIGINT) if old_debug != None: os.environ['LDTP_DEBUG'] = old_debug log_file = self.get_ldtp_log_file(os.path.join(src_dir, 'run.xml')) if not log_file: raise BuildStateError('missing log file') try: groups = self._check_ldtp_log_file(log_file) flag, status = self.check_groups(groups) if flag: raise BuildStateError(status) except: raise BuildStateError('malformed log file')