def extract_metric(metric, line, output, prefix='', lower_is_better=True):
try:
name, value_part = [part.strip() for part in line.split(':')]
if name != metric:
message = 'Name mismatch: expected "{}", got "{}"'
raise WorkloadError(message.format(metric, name.strip()))
if not value_part or not value_part[0].isdigit():
raise ValueError(
'value part does not start with a digit: {}'.format(
value_part))
idx = -1
if not value_part[idx].isdigit(
): # units detected at the end of the line
while not value_part[idx - 1].isdigit():
idx -= 1
value = numeric(value_part[:idx])
units = value_part[idx:]
else:
value = numeric(value_part)
units = None
output.add_metric(prefix + metric,
value,
units,
lower_is_better=lower_is_better)
except Exception as e:
message = 'Could not extract sysbench metric "{}"; got "{}"'
raise WorkloadError(message.format(prefix + metric, e))
def initialize(self, context):
ab = which('ab')
if not ab:
msg = 'ab not found on host; make sure apache2-utils (or you distro equivalent) package is installed.'
raise WorkloadError(msg)
response = urlopen('http://{}:{}{}'.format(self.target.conn.host,
self.port, self.path))
code = response.getcode()
if code != 200:
msg = 'HTTP request failed with status {}; is Apache running on target?'
raise WorkloadError(msg.format(code))
def extract_threads_fairness_metric(metric, line, output):
try:
name_part, value_part = [part.strip() for part in line.split(':')]
name = name_part.split('(')[0].strip()
if name != metric:
message = 'Name mismatch: expected "{}", got "{}"'
raise WorkloadError(message.format(metric, name))
avg, stddev = [numeric(v) for v in value_part.split('/')]
output.add_metric('thread fairness {} avg'.format(metric), avg)
output.add_metric('thread fairness {} stddev'.format(metric),
stddev, lower_is_better=True)
except Exception as e:
message = 'Could not extract sysbench metric "{}"; got "{}"'
raise WorkloadError(message.format(metric, e))
def update_output(self, context):
super(Gfxbench, self).update_output(context)
expected_results = len(self.regex_matches)
logcat_file = context.get_artifact_path('logcat')
with open(logcat_file) as fh:
for line in fh:
for regex in self.regex_matches:
match = regex.search(line)
# Check if we have matched the score string in logcat
if match:
score_match = self.score_regex.search(match.group(1))
# Check if there is valid number found for the score.
if score_match:
result = float(score_match.group(1))
else:
result = 'NaN'
entry = regex.pattern.rsplit(None, 1)[0]
context.add_metric(entry,
result,
'FPS',
lower_is_better=False)
expected_results -= 1
if expected_results > 0:
msg = "The GFXBench workload has failed. Expected {} scores, Detected {} scores."
raise WorkloadError(
msg.format(len(self.regex_matches), expected_results))
def run(self, context):
result = self.target.execute(self.command, timeout=self.timeout)
if 'FAILURE' in result:
raise WorkloadError(result)
else:
self.logger.debug(result)
self.target.sleep(DELAY)
self.monitor.wait_for_run_end(self.timeout)
def run(self, context):
self.output = self.target.execute(self.command)
# You see 'FAILURES' if an exception is thrown.
# You see 'Process crashed' if it doesn't recognise the class for some
# reason.
# But neither reports an error in the exit code, so check explicitly.
if 'FAILURES' in self.output or 'Process crashed' in self.output:
raise WorkloadError('Failed to run workload: {}'.format(
self.output))
def initialize(self, context):
self.old_screen_state = None
if self.target.os == 'android':
if self.stop_android and not self.target.is_rooted:
msg = 'Idle workload requires the device to be rooted in order to stop Android.'
raise WorkloadError(msg)
else:
if self.stop_android or self.screen_off:
msg = 'stop_android/screen_off can only be set for Android devices'
raise ConfigError(msg)
def check_for_crash(self, context, fps, frames, exec_time):
if not self.crash_threshold:
return
self.logger.debug('Checking for crashed content.')
if all([exec_time, fps, frames]):
expected_frames = fps * exec_time
ratio = frames / expected_frames
self.logger.debug('actual/expected frames: {:.2}'.format(ratio))
if ratio < self.crash_threshold:
msg = 'Content for {} appears to have crashed.\n'.format(
context.current_job.spec.label)
msg += 'Content crash detected (actual/expected frames: {:.2}).'.format(
ratio)
raise WorkloadError(msg)
def update_output(self, context):
super(Gfxbench, self).update_output(context)
regex_matches = [
re.compile(self.regex_template.format(t)) for t in self.tests
]
logcat_file = context.get_artifact_path('logcat')
found = []
detected_results = 0
failed = False
with open(logcat_file, errors='replace') as fh:
for line in fh:
for regex in regex_matches:
match = regex.search(line)
# Check if we have matched the score string in logcat and not already found.
if match and match.group('test_name') not in found:
found.append(match.group('test_name'))
# Set Default values
result = 'NaN'
unit = 'FPS'
# For most tests we usually want the `sub_result`
# as this is our FPS value
try:
result = float(
match.group('sub_result').split()[0].replace(
',', ''))
except (ValueError, TypeError):
# However for some tests the value is stored in `result`
# and the unit is saved in the `sub_result`.
try:
result = float(
match.group('result').replace(',', ''))
if match.group('sub_result'):
unit = match.group('sub_result').upper()
except (ValueError, TypeError):
failed = True
entry = match.group('test_name')
context.add_metric(entry,
result,
unit,
lower_is_better=False)
detected_results += 1
if failed or detected_results < len(regex_matches):
msg = "The workload has failed to process all scores. Expected >={} scores, Detected {} scores."
raise WorkloadError(
msg.format(len(regex_matches), detected_results))
def initialize(self, context):
if self.use_system_binary:
try:
cmd = '{0} md5sum < $({0} which openssl)'
output = self.target.execute(cmd.format(self.target.busybox))
md5hash = output.split()[0]
version = self.target.execute('openssl version').strip()
context.update_metadata('hashes', 'openssl', md5hash)
context.update_metadata('versions', 'openssl', version)
except TargetError:
msg = 'Openssl does not appear to be installed on target.'
raise WorkloadError(msg)
Openssl.target_exe = 'openssl'
else:
resource = Executable(self, self.target.abi, 'openssl')
host_exe = context.get_resource(resource)
Openssl.target_exe = self.target.install(host_exe)
def _find_host_video_file(self):
"""Pick the video file we're going to use, download it if necessary"""
if self.filename:
if self.filename[0] in './' or len(
self.filename) > 1 and self.filename[1] == ':':
filepath = os.path.abspath(self.filename)
else:
filepath = os.path.join(self.video_directory, self.filename)
if not os.path.isfile(filepath):
raise WorkloadError('{} does not exist.'.format(filepath))
return filepath
else:
# Search for files we've already downloaded
files = []
format_ext, format_resolution = self.format.split('_')
for filename in os.listdir(self.video_directory):
_, file_ext = os.path.splitext(filename)
if file_ext == '.' + format_ext and format_resolution in filename:
files.append(os.path.join(self.video_directory, filename))
if not files:
# Download a file with the requested format
url = DOWNLOAD_URLS[self.format]
filename = '{}_{}'.format(format_resolution,
os.path.basename(url))
filepath = os.path.join(self.video_directory, filename)
self.logger.info('Downloading {} to {}...'.format(
url, filepath))
urllib.request.urlretrieve(url, filepath)
return filepath
else:
if len(files) > 1:
self.logger.warning(
'Multiple files found for {} format. Using {}.'.format(
self.format, files[0]))
self.logger.warning(
'Use "filename"parameter instead of '
'"format" to specify a different file.')
return files[0]
def update_output(self, context):
super(Antutu, self).update_output(context)
expected_results = len(self.regex_matches)
logcat_file = context.get_artifact_path('logcat')
with open(logcat_file) as fh:
for line in fh:
for regex in self.regex_matches:
match = regex.search(line)
if match:
try:
result = float(match.group(1))
except ValueError:
result = float('NaN')
entry = regex.pattern.rsplit(None, 1)[0]
context.add_metric(entry,
result,
lower_is_better=False)
expected_results -= 1
if expected_results > 0:
msg = "The Antutu workload has failed. Expected {} scores, Detected {} scores."
raise WorkloadError(
msg.format(len(self.regex_matches), expected_results))
def extract_scores(self, context, regex_version):
#pylint: disable=no-self-use
expected_results = len(regex_version)
logcat_file = context.get_artifact_path('logcat')
with open(logcat_file, errors='replace') as fh:
for line in fh:
for regex in regex_version:
match = regex.search(line)
if match:
try:
result = float(match.group(1))
except ValueError:
result = float('NaN')
entry = regex.pattern.rsplit(None, 1)[0]
context.add_metric(entry,
result,
lower_is_better=False)
expected_results -= 1
if expected_results > 0:
msg = "The Antutu workload has failed. Expected {} scores, Detected {} scores."
raise WorkloadError(
msg.format(len(regex_version), expected_results))
def update_output(self, context):
host_file = context.get_artifact_path('mongoperf-output')
results = defaultdict(list)
threads = None
with open(host_file) as fh:
for line in fh:
if 'new thread,' in line:
threads = int(line.split()[-1])
elif 'ops/sec' in line:
results[threads].append(int(line.split()[0]))
if not results:
raise WorkloadError('No mongoperf results found in the output.')
for threads, values in results.items():
rs = pd.Series(values)
context.add_metric('ops_per_sec',
rs.mean(),
classifiers={'threads': threads})
context.add_metric('ops_per_sec_std',
rs.std(),
lower_is_better=True,
classifiers={'threads': threads})
def update_output(self, context):
if self.output is None:
self.logger.warning('Did not collect output')
return
outfile = os.path.join(context.output_directory, 'meabo-output.txt')
with open(outfile, 'wb') as wfh:
if sys.version_info[0] == 3:
wfh.write(self.output.encode('utf-8'))
else:
wfh.write(self.output)
context.add_artifact('meabo-output', outfile, kind='raw')
cur_phase = 0
for line in self.output.split('\n'):
line = line.strip()
match = phase_start_regex.search(line)
if match:
cur_phase = match.group('phase')
match = counter_value_regex.search(line)
if match:
if cur_phase == 0:
msg = 'Matched thread performance counters outside of phase!'
raise WorkloadError(msg)
name = 'phase_{}_thread_{}_{}'.format(cur_phase,
match.group('thread'),
match.group('name'))
context.add_metric(name, int(match.group('value')))
match = duration_regex.search(line)
if match:
context.add_metric("phase_{}_duration".format(
match.group('phase')),
int(match.group('duration')),
units="ns")
def initialize(self, context):
try:
self.target.execute('mongoperf -h')
except TargetError:
raise WorkloadError(
'Mongoperf must be installed an in $PATH on the target.')
def find_line_with(text, fh):
for line in fh:
if text in line:
return
message = 'Could not extract sysbench results from {}; did not see "{}"'
raise WorkloadError(message.format(fh.name, text))