def fetch_next_commit(self, server_config, last_fetched): if not last_fetched: # FIXME: This is a problematic if dashboard can get results for revisions older than oldest_revision # in the future because we never refetch older revisions. last_fetched = self.fetch_revision_from_dasbhoard( server_config, 'oldest') revision_to_fetch = int(last_fetched) + 1 args = [ 'svn', 'log', '--revision', str(revision_to_fetch), '--xml', self._svn_url, '--non-interactive' ] if self._use_server_auth and 'auth' in server_config['server']: server_auth = server_config['server']['auth'] args += [ '--no-auth-cache', '--username', server_auth['username'], '--password', server_auth['password'] ] if self._should_trust_certificate: args += ['--trust-server-cert'] try: output = subprocess.check_output(args, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as error: if (': No such revision ' + str(revision_to_fetch)) in error.output: return None raise error xml = parseXmlString(output) time = text_content(xml.getElementsByTagName("date")[0]) author_elements = xml.getElementsByTagName("author") author_account = text_content( author_elements[0]) if author_elements.length else None message = text_content(xml.getElementsByTagName("msg")[0]) name = self._resolve_author_name( author_account ) if author_account and self._account_name_script_path else None result = { 'repository': self._name, 'revision': revision_to_fetch, 'time': time, 'message': message, } if author_account: result['author'] = {'account': author_account, 'name': name} return result
def fetch_commit(repository_name, repository_url, revision): args = ['svn', 'log', '--revision', str(revision), '--xml', repository_url] try: output = subprocess.check_output(args, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as error: if (': No such revision ' + str(revision)) in error.output: return None raise error xml = parseXmlString(output) time = text_content(xml.getElementsByTagName("date")[0]) author_account = text_content(xml.getElementsByTagName("author")[0]) message = text_content(xml.getElementsByTagName("msg")[0]) return { 'repository': repository_name, 'revision': revision, 'time': time, 'author': {'account': author_account}, 'message': message, }
def fetch_commit(repository_name, repository_url, revision): args = ["svn", "log", "--revision", str(revision), "--xml", repository_url] try: output = subprocess.check_output(args, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as error: if (": No such revision " + str(revision)) in error.output: return None raise error xml = parseXmlString(output) time = textContent(xml.getElementsByTagName("date")[0]) author_account = textContent(xml.getElementsByTagName("author")[0]) message = textContent(xml.getElementsByTagName("msg")[0]) return { "repository": repository_name, "revision": revision, "time": time, "author": {"account": author_account}, "message": message, }
def test_init(self): self.maxDiff = None actual = convertXmlToDictionary(self.writer._doc) target = 'ide_' + self.makefileProject.name output = os.path.join('..', attributes.OUTPUT_DIRECTORY, self.makefileProject.outputName, target) expected = convertXmlToDictionary( parseXmlString( self.PROJECT_TEMPLATE.format( version=TEST_VERSION, name=self.makefileProject.name, guid=self.writer._PROJECT_GUID, output=output, build=self.formatCommandLine("build"), rebuild=self.formatCommandLine("rebuild"), clean=self.formatCommandLine("clean"), executable="{0}.exe".format( self.makefileProject.outputName), definitions=";".join(self.makefileProject.definitions)))) self.assertSameItems(expected, actual)
def fetch_commit(repository, revision): args = ['svn', 'log', '--revision', str(revision), '--xml', repository['url'], '--non-interactive'] if 'username' in repository and 'password' in repository: args += ['--no-auth-cache', '--username', repository['username'], '--password', repository['password']] if repository.get('trustCertificate', False): args += ['--trust-server-cert'] try: output = subprocess.check_output(args, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as error: if (': No such revision ' + str(revision)) in error.output: return None raise error xml = parseXmlString(output) time = text_content(xml.getElementsByTagName("date")[0]) author_account = text_content(xml.getElementsByTagName("author")[0]) message = text_content(xml.getElementsByTagName("msg")[0]) return { 'repository': repository['name'], 'revision': revision, 'time': time, 'author': {'account': author_account}, 'message': message, }
def fetch_commit(self, server_config, last_fetched): if not last_fetched: # FIXME: This is a problematic if dashboard can get results for revisions older than oldest_revision # in the future because we never refetch older revisions. last_fetched = self.fetch_revision_from_dasbhoard(server_config, 'oldest') revision_to_fetch = int(last_fetched) + 1 args = ['svn', 'log', '--revision', str(revision_to_fetch), '--xml', self._svn_url, '--non-interactive'] if self._use_server_auth and 'auth' in server_config['server']: server_auth = server_config['server']['auth'] args += ['--no-auth-cache', '--username', server_auth['username'], '--password', server_auth['password']] if self._should_trust_certificate: args += ['--trust-server-cert'] try: output = subprocess.check_output(args, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as error: if (': No such revision ' + str(revision_to_fetch)) in error.output: return None raise error xml = parseXmlString(output) time = text_content(xml.getElementsByTagName("date")[0]) author_account = text_content(xml.getElementsByTagName("author")[0]) message = text_content(xml.getElementsByTagName("msg")[0]) name = self._resolve_author_name(author_account) if self._account_name_script_path else None return { 'repository': self._name, 'revision': revision_to_fetch, 'time': time, 'author': {'account': author_account, 'name': name}, 'message': message, }
def fetch_available_builds(repository_name, url, train_version_map): output = urllib2.urlopen(url).read() try: xml = parseXmlString(output) except Exception, error: raise Exception(error, output)
def run(self): timestamp = datetime.today() baseDir = self.testDirectory.getDirectory() pldir = os.path.join(baseDir, '.pageload') if not os.path.isdir(pldir): os.makedirs(pldir) testDir = os.path.normpath( os.path.join(baseDir, timestamp.strftime('%Y%m%d%H%M%S')) ) assets = { 'parameters.json': os.path.join(testDir, 'parameters.json'), 'request.xml': os.path.join(testDir, 'request.xml'), 'response.xml': os.path.join(testDir, 'response.xml'), 'summary.csv': os.path.join(testDir, 'summary.csv'), 'detail.csv': os.path.join(testDir, 'detail.csv'), 'run': os.path.join(testDir, 'run') } try: os.makedirs(testDir) self.logger.info('Directory %s created' % (testDir)) except OSError as error: self.logger.error('Cannot create directory %s: %s' % (testDir, error)) sys.exit(-1) try: os.mkdir( assets['run'] ) self.logger.info('Directory %s created' % (assets['run'])) except OSError as error: self.logger.error('Cannot create directory ./%s: %s' % (assets['run'], error)) sys.exit(-1) testParameters = { 'url': self.url, 'config': self.config } parameters = open( assets['parameters.json'], 'w' ) parameters.write( json.dumps(testParameters, sort_keys=True, indent=4) ) parameters.close() (response, data) = self._makeHttpRequest('POST', self.config['wptserver'], '/runtest.php', urllib.urlencode(self.config), self.defaultHeaders) fp = open( assets['request.xml'], 'w' ) fp.write(data) fp.close() self.logger.debug("Retrieved data\n %s" % data) tree = self._getXmlTree(data) xmlUrl = urlparse(tree.find('data/xmlUrl').text).path self.logger.info('Waiting for results') slept = 0 cont = False while True: (response, statusData) = self._makeHttpRequest('GET', self.config['wptserver'], xmlUrl) statusTree = self._getXmlTree(statusData) statusCode = int(statusTree.find('statusCode').text) if 100 <= statusCode < 200: sleep(10) slept += 10 if slept > 900: self.logger.error('Timed out waiting for response (900 seconds)') cont = True break continue elif 400 <= statusCode < 500: self.logger.error('Could not retrieve response (HTTP status code %d)' % (statusCode)) cont = True break break if cont: shutil.rmtree(testDir) return None self.logger.info('Results finished processing') self.logger.info('Downloading reports') (response, data) = self._makeHttpRequest('GET', self.config['wptserver'], urlparse(tree.find('data/xmlUrl').text).path) resultTree = self._getXmlTree(data) tData = data fp = open( assets['response.xml'], 'w' ) fp.write( parseXmlString(data).toprettyxml() ) fp.close() for asset, xmlElement in {'summary.csv': 'data/summaryCSV', 'detail.csv': 'data/detailCSV'}.items(): urlParts = urlparse( tree.find( xmlElement ).text ) (response, data) = self._makeHttpRequest('GET', self.config['wptserver'], urlParts.path) fp = open( assets[asset], 'w' ) fp.write( data ) fp.close() images = ['waterfall', 'checklist', 'screenShot'] rawData = ['headers', 'pageData', 'requestsData', 'utilization', 'PageSpeedData'] self.logger.info('Downloading run data') for run in resultTree.findall('data/run'): id = run.find('id').text runDir = os.path.join(assets['run'], id) try: os.mkdir(runDir) self.logger.info('Directory %s created' % (runDir)) except OSError as error: self.logger.error('Cannot create directory ./%s: %s' % (runDir, error)) sys.exit(-1) for view in [run.find('firstView'), run.find('repeatView')]: if view is None: self.logger.error('Malformed response from server:') self.logger.error(tData) continue viewDir = os.path.join(runDir, view.tag) imageDir = os.path.join(viewDir, 'images') dataDir = os.path.join(viewDir, 'data') for Dir in [viewDir, imageDir, dataDir]: try: os.mkdir( Dir ) self.logger.info('Directory %s created' % ( Dir )) except OSError as error: self.logger.error('Cannot create directory ./%s: %s' % ( Dir, error )) sys.exit(-1) for imageName in images: imageUrl = view.find('images/' + imageName).text localFileName = os.path.join(imageDir, imageName + '.' + imageUrl[-3:]) self._downloadFile(imageUrl, localFileName) for rawDataName in rawData: rawDataUrl = view.find('rawData/' + rawDataName).text localFileName = os.path.join(dataDir, rawDataName) self._downloadFile(rawDataUrl, localFileName) result = self.testResultsFactory.create( testDir ) self.testDirectory.addResult( result ) return result