def fetch_next_commit(self, server_config, last_fetched):
if not last_fetched:
# FIXME: This is a problematic if dashboard can get results for revisions older than oldest_revision
# in the future because we never refetch older revisions.
last_fetched = self.fetch_revision_from_dasbhoard(
server_config, 'oldest')
revision_to_fetch = int(last_fetched) + 1
args = [
'svn', 'log', '--revision',
str(revision_to_fetch), '--xml', self._svn_url, '--non-interactive'
]
if self._use_server_auth and 'auth' in server_config['server']:
server_auth = server_config['server']['auth']
args += [
'--no-auth-cache', '--username', server_auth['username'],
'--password', server_auth['password']
]
if self._should_trust_certificate:
args += ['--trust-server-cert']
try:
output = subprocess.check_output(args, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
if (': No such revision ' +
str(revision_to_fetch)) in error.output:
return None
raise error
xml = parseXmlString(output)
time = text_content(xml.getElementsByTagName("date")[0])
author_elements = xml.getElementsByTagName("author")
author_account = text_content(
author_elements[0]) if author_elements.length else None
message = text_content(xml.getElementsByTagName("msg")[0])
name = self._resolve_author_name(
author_account
) if author_account and self._account_name_script_path else None
result = {
'repository': self._name,
'revision': revision_to_fetch,
'time': time,
'message': message,
}
if author_account:
result['author'] = {'account': author_account, 'name': name}
return result
def fetch_commit(repository_name, repository_url, revision):
args = ['svn', 'log', '--revision', str(revision), '--xml', repository_url]
try:
output = subprocess.check_output(args, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
if (': No such revision ' + str(revision)) in error.output:
return None
raise error
xml = parseXmlString(output)
time = text_content(xml.getElementsByTagName("date")[0])
author_account = text_content(xml.getElementsByTagName("author")[0])
message = text_content(xml.getElementsByTagName("msg")[0])
return {
'repository': repository_name,
'revision': revision,
'time': time,
'author': {'account': author_account},
'message': message,
}
def fetch_commit(repository_name, repository_url, revision):
args = ["svn", "log", "--revision", str(revision), "--xml", repository_url]
try:
output = subprocess.check_output(args, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
if (": No such revision " + str(revision)) in error.output:
return None
raise error
xml = parseXmlString(output)
time = textContent(xml.getElementsByTagName("date")[0])
author_account = textContent(xml.getElementsByTagName("author")[0])
message = textContent(xml.getElementsByTagName("msg")[0])
return {
"repository": repository_name,
"revision": revision,
"time": time,
"author": {"account": author_account},
"message": message,
}
def test_init(self):
self.maxDiff = None
actual = convertXmlToDictionary(self.writer._doc)
target = 'ide_' + self.makefileProject.name
output = os.path.join('..', attributes.OUTPUT_DIRECTORY,
self.makefileProject.outputName, target)
expected = convertXmlToDictionary(
parseXmlString(
self.PROJECT_TEMPLATE.format(
version=TEST_VERSION,
name=self.makefileProject.name,
guid=self.writer._PROJECT_GUID,
output=output,
build=self.formatCommandLine("build"),
rebuild=self.formatCommandLine("rebuild"),
clean=self.formatCommandLine("clean"),
executable="{0}.exe".format(
self.makefileProject.outputName),
definitions=";".join(self.makefileProject.definitions))))
self.assertSameItems(expected, actual)
def fetch_commit(repository, revision):
args = ['svn', 'log', '--revision', str(revision), '--xml', repository['url'], '--non-interactive']
if 'username' in repository and 'password' in repository:
args += ['--no-auth-cache', '--username', repository['username'], '--password', repository['password']]
if repository.get('trustCertificate', False):
args += ['--trust-server-cert']
try:
output = subprocess.check_output(args, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
if (': No such revision ' + str(revision)) in error.output:
return None
raise error
xml = parseXmlString(output)
time = text_content(xml.getElementsByTagName("date")[0])
author_account = text_content(xml.getElementsByTagName("author")[0])
message = text_content(xml.getElementsByTagName("msg")[0])
return {
'repository': repository['name'],
'revision': revision,
'time': time,
'author': {'account': author_account},
'message': message,
}
def fetch_commit(self, server_config, last_fetched):
if not last_fetched:
# FIXME: This is a problematic if dashboard can get results for revisions older than oldest_revision
# in the future because we never refetch older revisions.
last_fetched = self.fetch_revision_from_dasbhoard(server_config, 'oldest')
revision_to_fetch = int(last_fetched) + 1
args = ['svn', 'log', '--revision', str(revision_to_fetch), '--xml', self._svn_url, '--non-interactive']
if self._use_server_auth and 'auth' in server_config['server']:
server_auth = server_config['server']['auth']
args += ['--no-auth-cache', '--username', server_auth['username'], '--password', server_auth['password']]
if self._should_trust_certificate:
args += ['--trust-server-cert']
try:
output = subprocess.check_output(args, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
if (': No such revision ' + str(revision_to_fetch)) in error.output:
return None
raise error
xml = parseXmlString(output)
time = text_content(xml.getElementsByTagName("date")[0])
author_account = text_content(xml.getElementsByTagName("author")[0])
message = text_content(xml.getElementsByTagName("msg")[0])
name = self._resolve_author_name(author_account) if self._account_name_script_path else None
return {
'repository': self._name,
'revision': revision_to_fetch,
'time': time,
'author': {'account': author_account, 'name': name},
'message': message,
}
def fetch_available_builds(repository_name, url, train_version_map):
output = urllib2.urlopen(url).read()
try:
xml = parseXmlString(output)
except Exception, error:
raise Exception(error, output)
def fetch_available_builds(repository_name, url, train_version_map):
output = urllib2.urlopen(url).read()
try:
xml = parseXmlString(output)
except Exception, error:
raise Exception(error, output)
def run(self):
timestamp = datetime.today()
baseDir = self.testDirectory.getDirectory()
pldir = os.path.join(baseDir, '.pageload')
if not os.path.isdir(pldir):
os.makedirs(pldir)
testDir = os.path.normpath( os.path.join(baseDir, timestamp.strftime('%Y%m%d%H%M%S')) )
assets = {
'parameters.json': os.path.join(testDir, 'parameters.json'),
'request.xml': os.path.join(testDir, 'request.xml'),
'response.xml': os.path.join(testDir, 'response.xml'),
'summary.csv': os.path.join(testDir, 'summary.csv'),
'detail.csv': os.path.join(testDir, 'detail.csv'),
'run': os.path.join(testDir, 'run')
}
try:
os.makedirs(testDir)
self.logger.info('Directory %s created' % (testDir))
except OSError as error:
self.logger.error('Cannot create directory %s: %s' % (testDir, error))
sys.exit(-1)
try:
os.mkdir( assets['run'] )
self.logger.info('Directory %s created' % (assets['run']))
except OSError as error:
self.logger.error('Cannot create directory ./%s: %s' % (assets['run'], error))
sys.exit(-1)
testParameters = {
'url': self.url,
'config': self.config
}
parameters = open( assets['parameters.json'], 'w' )
parameters.write( json.dumps(testParameters, sort_keys=True, indent=4) )
parameters.close()
(response, data) = self._makeHttpRequest('POST', self.config['wptserver'], '/runtest.php', urllib.urlencode(self.config), self.defaultHeaders)
fp = open( assets['request.xml'], 'w' )
fp.write(data)
fp.close()
self.logger.debug("Retrieved data\n %s" % data)
tree = self._getXmlTree(data)
xmlUrl = urlparse(tree.find('data/xmlUrl').text).path
self.logger.info('Waiting for results')
slept = 0
cont = False
while True:
(response, statusData) = self._makeHttpRequest('GET', self.config['wptserver'], xmlUrl)
statusTree = self._getXmlTree(statusData)
statusCode = int(statusTree.find('statusCode').text)
if 100 <= statusCode < 200:
sleep(10)
slept += 10
if slept > 900:
self.logger.error('Timed out waiting for response (900 seconds)')
cont = True
break
continue
elif 400 <= statusCode < 500:
self.logger.error('Could not retrieve response (HTTP status code %d)' % (statusCode))
cont = True
break
break
if cont:
shutil.rmtree(testDir)
return None
self.logger.info('Results finished processing')
self.logger.info('Downloading reports')
(response, data) = self._makeHttpRequest('GET', self.config['wptserver'], urlparse(tree.find('data/xmlUrl').text).path)
resultTree = self._getXmlTree(data)
tData = data
fp = open( assets['response.xml'], 'w' )
fp.write( parseXmlString(data).toprettyxml() )
fp.close()
for asset, xmlElement in {'summary.csv': 'data/summaryCSV', 'detail.csv': 'data/detailCSV'}.items():
urlParts = urlparse( tree.find( xmlElement ).text )
(response, data) = self._makeHttpRequest('GET', self.config['wptserver'], urlParts.path)
fp = open( assets[asset], 'w' )
fp.write( data )
fp.close()
images = ['waterfall', 'checklist', 'screenShot']
rawData = ['headers', 'pageData', 'requestsData', 'utilization', 'PageSpeedData']
self.logger.info('Downloading run data')
for run in resultTree.findall('data/run'):
id = run.find('id').text
runDir = os.path.join(assets['run'], id)
try:
os.mkdir(runDir)
self.logger.info('Directory %s created' % (runDir))
except OSError as error:
self.logger.error('Cannot create directory ./%s: %s' % (runDir, error))
sys.exit(-1)
for view in [run.find('firstView'), run.find('repeatView')]:
if view is None:
self.logger.error('Malformed response from server:')
self.logger.error(tData)
continue
viewDir = os.path.join(runDir, view.tag)
imageDir = os.path.join(viewDir, 'images')
dataDir = os.path.join(viewDir, 'data')
for Dir in [viewDir, imageDir, dataDir]:
try:
os.mkdir( Dir )
self.logger.info('Directory %s created' % ( Dir ))
except OSError as error:
self.logger.error('Cannot create directory ./%s: %s' % ( Dir, error ))
sys.exit(-1)
for imageName in images:
imageUrl = view.find('images/' + imageName).text
localFileName = os.path.join(imageDir, imageName + '.' + imageUrl[-3:])
self._downloadFile(imageUrl, localFileName)
for rawDataName in rawData:
rawDataUrl = view.find('rawData/' + rawDataName).text
localFileName = os.path.join(dataDir, rawDataName)
self._downloadFile(rawDataUrl, localFileName)
result = self.testResultsFactory.create( testDir )
self.testDirectory.addResult( result )
return result
