def ReduceMapResults(job_results, key, file_handle, job):
project = perf_insights_project.PerfInsightsProject()
all_source_paths = list(project.source_paths)
all_source_paths.append(project.perf_insights_root_path)
with file_handle.PrepareFileForProcessing() as prepared_file_handle:
js_args = [
key,
json.dumps(prepared_file_handle.AsDict()),
json.dumps(job.AsDict()),
]
res = vinn.RunFile(_REDUCE_MAP_RESULTS_CMDLINE_PATH,
source_paths=all_source_paths,
js_args=js_args)
if res.returncode != 0:
try:
sys.stderr.write(res.stdout)
except Exception:
pass
job_results.AddFailure(
failure.Failure(job, job.map_function_handle, None, 'Error',
'vinn runtime error while reducing results.',
'Unknown stack'))
return
for line in res.stdout.split('\n'):
m = re.match('^JOB_(RESULTS|FAILURE): (.+)', line, re.DOTALL)
if m:
found_type = m.group(1)
found_dict = json.loads(m.group(2))
if found_type == 'FAILURE':
try:
sys.stderr.write(res.stdout)
except Exception:
pass
job_results.AddFailure(
failure.Failure(
job, job.map_function_handle, None, 'Error',
'vinn runtime error while reducing results.',
'Unknown stack'))
elif found_type == 'RESULTS':
job_results.AddPair(key, found_dict[key])
else:
if len(line) > 0:
sys.stderr.write(line)
sys.stderr.write('\n')
if len(job_results.pairs) == 0 and len(job_results.failures) == 0:
raise map_single_trace.InternalMapError(
'Internal error: No results were produced!')
def MapSingleTrace(trace_handle, job, extra_import_options=None):
assert (type(extra_import_options) is types.NoneType
or type(extra_import_options) is types.DictType), (
'extra_import_options should be a dict or None.')
project = perf_insights_project.PerfInsightsProject()
all_source_paths = list(project.source_paths)
all_source_paths.append(project.perf_insights_root_path)
result = mre_result.MreResult()
with trace_handle.PrepareFileForProcessing() as prepared_trace_handle:
js_args = [
json.dumps(prepared_trace_handle.AsDict()),
json.dumps(job.AsDict()),
]
if extra_import_options:
js_args.append(json.dumps(extra_import_options))
res = vinn.RunFile(_MAP_SINGLE_TRACE_CMDLINE_PATH,
source_paths=all_source_paths,
js_args=js_args)
if res.returncode != 0:
try:
sys.stderr.write(res.stdout)
except Exception:
pass
result.AddFailure(
failure.Failure(job, trace_handle.canonical_url, 'Error',
'vinn runtime error while mapping trace.',
'vinn runtime error while mapping trace.',
'Unknown stack'))
return result
for line in res.stdout.split('\n'):
m = re.match('^MRE_RESULT: (.+)', line, re.DOTALL)
if m:
found_dict = json.loads(m.group(1))
failures = [
failure.Failure.FromDict(f, job,
_FAILURE_NAME_TO_FAILURE_CONSTRUCTOR)
for f in found_dict['failures']
]
for f in failures:
result.AddFailure(f)
for k, v in found_dict['pairs'].iteritems():
result.AddPair(k, v)
else:
if len(line) > 0:
sys.stderr.write(line)
sys.stderr.write('\n')
if not (len(result.pairs) or len(result.failures)):
raise InternalMapError('Internal error: No results were produced!')
return result
def testAsDict(self):
result = mre_result.MreResult()
with map_single_trace.TemporaryMapScript("""
pi.FunctionRegistry.register(
function MyMapFunction(result, model) {
var canonicalUrl = model.canonicalUrl;
result.addPair('result', {
numProcesses: model.getAllProcesses().length
});
});
""") as map_script:
module = function_handle.ModuleToLoad(filename=map_script.filename)
map_handle = function_handle.FunctionHandle(
modules_to_load=[module], function_name='MyMapFunction')
job = job_module.Job(map_handle, None)
failure = failure_module.Failure(job, '2', '3', 'err', 'desc',
'stack')
result.AddFailure(failure)
result.AddPair('foo', 'bar')
result_dict = result.AsDict()
self.assertEquals(result_dict['failures'], [failure.AsDict()])
self.assertEquals(result_dict['pairs'], {'foo': 'bar'})
def MapSingleTrace(trace_handle, map_function_handle):
project = perf_insights_project.PerfInsightsProject()
all_source_paths = list(project.source_paths)
pi_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..'))
all_source_paths.append(pi_path)
result = mre_result.MreResult()
with trace_handle.PrepareFileForProcessing() as prepared_trace_handle:
js_args = [
json.dumps(prepared_trace_handle.AsDict()),
json.dumps(map_function_handle.AsDict())
]
res = vinn.RunFile(
os.path.join(pi_path, 'perf_insights', 'map_single_trace_cmdline.html'),
source_paths=all_source_paths,
js_args=js_args)
if res.returncode != 0:
try:
sys.stderr.write(res.stdout)
except Exception:
pass
result.AddFailure(failure.Failure(
map_function_handle.AsUserFriendlyString(), trace_handle.canonical_url,
'Error', 'vinn runtime error while mapping trace.',
'vinn runtime error while mapping trace.', 'Unknown stack'))
return
for line in res.stdout.split('\n'):
m = re.match('^MRE_RESULT: (.+)', line, re.DOTALL)
if m:
found_dict = json.loads(m.group(1))
failures = [failure.Failure.FromDict(
f, _FAILURE_NAME_TO_FAILURE_CONSTRUCTOR)
for f in found_dict['failures']]
for f in failures:
result.AddFailure(f)
for k, v in found_dict['pairs'].iteritems():
result.AddPair(k, v)
else:
if len(line) > 0:
sys.stderr.write(line)
sys.stderr.write('\n')
if not (len(result.pairs) or len(result.failures)):
raise InternalMapError('Internal error: No results were produced!')
return result
def testAsDict(self):
failure = failure_module.Failure(None, 'foo.html:Foo', 'file://foo.html',
'err', 'desc', 'stack')
self.assertEquals(failure.AsDict(), {
'function_handle_string': 'foo.html:Foo',
'trace_canonical_url': 'file://foo.html',
'type': 'err',
'description': 'desc',
'stack': 'stack'
})
def testTranslateMreFailure(self):
map_function_handle = _SingleFileFunctionHandle('foo.html', 'Foo', '2')
job = job_module.Job(map_function_handle, '1')
story_set = story.StorySet(base_dir=os.path.dirname(__file__))
p = page.Page('http://www.foo.com/', story_set, story_set.base_dir)
f = failure.Failure(job, 'foo', '/a.json', 'MyFailure', 'failure',
'stack')
fv = common_value_helpers.TranslateMreFailure(f, p)
self.assertIn('stack', str(fv))
def testAsDict(self):
failure = failure_module.Failure('1', '2', '3', 'err', 'desc', 'stack')
self.assertEquals(
failure.AsDict(), {
'job_guid': '1',
'function_handle_guid': '2',
'trace_guid': '3',
'failure_type_name': 'err',
'description': 'desc',
'stack': 'stack'
})
def testAsDict(self):
map_function_handle = _SingleFileFunctionHandle('foo.html', 'Foo', '2')
reduce_function_handle = _SingleFileFunctionHandle('bar.html', 'Bar', '3')
job = job_module.Job(map_function_handle, reduce_function_handle, '1')
failure = failure_module.Failure(job, 'foo.html:Foo',
'file://foo.html',
'err', 'desc', 'stack')
self.assertEquals(failure.AsDict(), {
'job_guid': '1',
'function_handle_string': 'foo.html:Foo',
'trace_canonical_url': 'file://foo.html',
'type': 'err',
'description': 'desc',
'stack': 'stack'
})
