def testAsDict(self):
mtl0 = function_handle.ModuleToLoad(href='/foo')
mtl1 = function_handle.ModuleToLoad(filename='foo.html')
self.assertEquals(mtl0.AsDict(), {'href': '/foo'})
self.assertEquals(mtl1.AsDict(), {'filename': 'foo.html'})
def testAsDict(self):
result = mre_result.MreResult()
with map_single_trace.TemporaryMapScript("""
pi.FunctionRegistry.register(
function MyMapFunction(result, model) {
var canonicalUrl = model.canonicalUrl;
result.addPair('result', {
numProcesses: model.getAllProcesses().length
});
});
""") as map_script:
module = function_handle.ModuleToLoad(filename=map_script.filename)
map_handle = function_handle.FunctionHandle(
modules_to_load=[module], function_name='MyMapFunction')
job = job_module.Job(map_handle, None)
failure = failure_module.Failure(job, '2', '3', 'err', 'desc',
'stack')
result.AddFailure(failure)
result.AddPair('foo', 'bar')
result_dict = result.AsDict()
self.assertEquals(result_dict['failures'], [failure.AsDict()])
self.assertEquals(result_dict['pairs'], {'foo': 'bar'})
def PiReportToHTML(ofile,
corpus_driver,
pi_report_file,
query,
json_output=False,
stop_on_error=False,
jobs=1,
quiet=False):
project = perf_insights_project.PerfInsightsProject()
with open(pi_report_file, 'r') as f:
pi_report_file_contents = f.read()
map_function_href, map_function_name, pi_report_element_name = (
_GetMapFunctionHrefFromPiReport(pi_report_file_contents))
map_file = project.GetAbsPathFromHRef(map_function_href)
module = function_handle.ModuleToLoad(filename=map_file)
map_function_handle = function_handle.FunctionHandle([module],
map_function_name)
job = job_module.Job(map_function_handle, None)
if map_file == None:
raise Exception('Could not find %s' % map_function_href)
results = _MapTraces(corpus_driver, job, query, stop_on_error, jobs, quiet)
if stop_on_error and results.had_failures:
sys.stderr.write('There were mapping errors. Aborting.')
return 255
if json_output:
json.dump([result.AsDict() for result in results], ofile, indent=2)
else:
WriteResultsToFile(ofile, project, pi_report_file,
pi_report_element_name, results)
return 0
def _CreateDefaultReduceHandle(self):
module = function_handle.ModuleToLoad(
filename=_DEFAULT_REDUCE_FILE_PATH)
handle = function_handle.FunctionHandle(
modules_to_load=[module],
function_name=_DEFAULT_REDUCE_FUNCTION_NAME)
return handle
def testRepr(self):
module = function_handle.ModuleToLoad(href='/foo')
handle = function_handle.FunctionHandle([module], 'Bar')
self.assertEquals(
str(handle),
'FunctionHandle(modules_to_load=[ModuleToLoad(href="/foo")], '
'function_name="Bar")')
def testAsDict(self):
module = function_handle.ModuleToLoad(href='/foo')
handle = function_handle.FunctionHandle([module], 'Bar')
self.assertEquals(
handle.AsDict(), {
'modules_to_load' : [{'href': '/foo'}],
'function_name': 'Bar'
})
def _GetMetricRunnerHandle(metric):
assert isinstance(metric, basestring)
metrics_dir = _GetMetricsDir()
metric_mapper_path = os.path.join(metrics_dir,
_METRIC_MAP_FUNCTION_FILENAME)
modules_to_load = [
function_handle.ModuleToLoad(filename=metric_mapper_path)
]
map_function_handle = function_handle.FunctionHandle(
modules_to_load, _METRIC_MAP_FUNCTION_NAME, {'metric': metric})
return job_module.Job(map_function_handle, None)
def _GetMetricRunnerHandle(metrics):
assert isinstance(metrics, list)
metrics_dir = _GetMetricsDir()
metric_paths = [os.path.join(metrics_dir, metric) for metric in metrics]
metric_mapper_path = os.path.join(metrics_dir,
_METRIC_MAP_FUNCTION_FILENAME)
metric_paths.append(metric_mapper_path)
modules_to_load = [
function_handle.ModuleToLoad(filename=path) for path in metric_paths
]
return function_handle.FunctionHandle(modules_to_load,
_METRIC_MAP_FUNCTION_NAME)
def Main(argv):
parser = argparse.ArgumentParser(description=_DEFAULT_DESCRIPTION)
parser.add_argument('map_file_url')
parser.add_argument('map_function_name')
parser.add_argument('input_url')
parser.add_argument('output_url')
parser.add_argument('--jobs', type=int, default=1)
args = parser.parse_args(argv[1:])
map_file = _ReadMapperGCSFile(args.map_file_url)
if not map_file:
parser.error('Map does not exist.')
if not args.map_function_name:
parser.error('Must provide map function name.')
temp_directory = tempfile.mkdtemp()
_, file_name = tempfile.mkstemp()
ofile = open(file_name, 'w')
try:
output_formatter = json_output_formatter.JSONOutputFormatter(ofile)
map_function_module = function_handle.ModuleToLoad(
filename=os.path.abspath(map_file))
map_function_handle = function_handle.FunctionHandle(
modules_to_load=[map_function_module],
function_name=args.map_function_name)
trace_handles = _DownloadTraceHandles(args.input_url, temp_directory)
runner = map_runner.MapRunner(trace_handles,
map_function_handle,
jobs=args.jobs,
output_formatters=[output_formatter])
results = runner.Run()
if args.map_function_handle:
results = runner.RunMapper()
elif args.reduce_function_handle:
results = runner.RunReducer(trace_handles)
output_formatter.Format(results)
return results
finally:
ofile.close()
os.unlink(map_file)
shutil.rmtree(temp_directory)
def testRepr(self):
mtl0 = function_handle.ModuleToLoad(href='/foo')
mtl1 = function_handle.ModuleToLoad(filename='foo.html')
self.assertEquals(str(mtl0), 'ModuleToLoad(href="/foo")')
self.assertEquals(str(mtl1), 'ModuleToLoad(filename="foo.html")')
def testExactlyOneHrefOrFilename(self):
with self.assertRaises(Exception):
function_handle.ModuleToLoad()
with self.assertRaises(Exception):
function_handle.ModuleToLoad('foo', 'foo')
def _SingleFileFunctionHandle(filename, function_name, guid):
return function_handle.FunctionHandle(
modules_to_load=[function_handle.ModuleToLoad(filename=filename)],
function_name=function_name, guid=guid)
def _Handle(filename):
module = function_handle.ModuleToLoad(filename=filename)
map_handle = function_handle.FunctionHandle(modules_to_load=[module],
function_name='MyMapFunction')
return job_module.Job(map_handle, None)
def _Handle(filename):
module = function_handle.ModuleToLoad(filename=filename)
return function_handle.FunctionHandle(modules_to_load=[module],
function_name='MyMapFunction')
