def testAsDict(self): mtl0 = function_handle.ModuleToLoad(href='/foo') mtl1 = function_handle.ModuleToLoad(filename='foo.html') self.assertEquals(mtl0.AsDict(), {'href': '/foo'}) self.assertEquals(mtl1.AsDict(), {'filename': 'foo.html'})
def testAsDict(self): result = mre_result.MreResult() with map_single_trace.TemporaryMapScript(""" pi.FunctionRegistry.register( function MyMapFunction(result, model) { var canonicalUrl = model.canonicalUrl; result.addPair('result', { numProcesses: model.getAllProcesses().length }); }); """) as map_script: module = function_handle.ModuleToLoad(filename=map_script.filename) map_handle = function_handle.FunctionHandle( modules_to_load=[module], function_name='MyMapFunction') job = job_module.Job(map_handle, None) failure = failure_module.Failure(job, '2', '3', 'err', 'desc', 'stack') result.AddFailure(failure) result.AddPair('foo', 'bar') result_dict = result.AsDict() self.assertEquals(result_dict['failures'], [failure.AsDict()]) self.assertEquals(result_dict['pairs'], {'foo': 'bar'})
def PiReportToHTML(ofile, corpus_driver, pi_report_file, query, json_output=False, stop_on_error=False, jobs=1, quiet=False): project = perf_insights_project.PerfInsightsProject() with open(pi_report_file, 'r') as f: pi_report_file_contents = f.read() map_function_href, map_function_name, pi_report_element_name = ( _GetMapFunctionHrefFromPiReport(pi_report_file_contents)) map_file = project.GetAbsPathFromHRef(map_function_href) module = function_handle.ModuleToLoad(filename=map_file) map_function_handle = function_handle.FunctionHandle([module], map_function_name) job = job_module.Job(map_function_handle, None) if map_file == None: raise Exception('Could not find %s' % map_function_href) results = _MapTraces(corpus_driver, job, query, stop_on_error, jobs, quiet) if stop_on_error and results.had_failures: sys.stderr.write('There were mapping errors. Aborting.') return 255 if json_output: json.dump([result.AsDict() for result in results], ofile, indent=2) else: WriteResultsToFile(ofile, project, pi_report_file, pi_report_element_name, results) return 0
def _CreateDefaultReduceHandle(self): module = function_handle.ModuleToLoad( filename=_DEFAULT_REDUCE_FILE_PATH) handle = function_handle.FunctionHandle( modules_to_load=[module], function_name=_DEFAULT_REDUCE_FUNCTION_NAME) return handle
def testRepr(self): module = function_handle.ModuleToLoad(href='/foo') handle = function_handle.FunctionHandle([module], 'Bar') self.assertEquals( str(handle), 'FunctionHandle(modules_to_load=[ModuleToLoad(href="/foo")], ' 'function_name="Bar")')
def testAsDict(self): module = function_handle.ModuleToLoad(href='/foo') handle = function_handle.FunctionHandle([module], 'Bar') self.assertEquals( handle.AsDict(), { 'modules_to_load' : [{'href': '/foo'}], 'function_name': 'Bar' })
def _GetMetricRunnerHandle(metric): assert isinstance(metric, basestring) metrics_dir = _GetMetricsDir() metric_mapper_path = os.path.join(metrics_dir, _METRIC_MAP_FUNCTION_FILENAME) modules_to_load = [ function_handle.ModuleToLoad(filename=metric_mapper_path) ] map_function_handle = function_handle.FunctionHandle( modules_to_load, _METRIC_MAP_FUNCTION_NAME, {'metric': metric}) return job_module.Job(map_function_handle, None)
def _GetMetricRunnerHandle(metrics): assert isinstance(metrics, list) metrics_dir = _GetMetricsDir() metric_paths = [os.path.join(metrics_dir, metric) for metric in metrics] metric_mapper_path = os.path.join(metrics_dir, _METRIC_MAP_FUNCTION_FILENAME) metric_paths.append(metric_mapper_path) modules_to_load = [ function_handle.ModuleToLoad(filename=path) for path in metric_paths ] return function_handle.FunctionHandle(modules_to_load, _METRIC_MAP_FUNCTION_NAME)
def Main(argv): parser = argparse.ArgumentParser(description=_DEFAULT_DESCRIPTION) parser.add_argument('map_file_url') parser.add_argument('map_function_name') parser.add_argument('input_url') parser.add_argument('output_url') parser.add_argument('--jobs', type=int, default=1) args = parser.parse_args(argv[1:]) map_file = _ReadMapperGCSFile(args.map_file_url) if not map_file: parser.error('Map does not exist.') if not args.map_function_name: parser.error('Must provide map function name.') temp_directory = tempfile.mkdtemp() _, file_name = tempfile.mkstemp() ofile = open(file_name, 'w') try: output_formatter = json_output_formatter.JSONOutputFormatter(ofile) map_function_module = function_handle.ModuleToLoad( filename=os.path.abspath(map_file)) map_function_handle = function_handle.FunctionHandle( modules_to_load=[map_function_module], function_name=args.map_function_name) trace_handles = _DownloadTraceHandles(args.input_url, temp_directory) runner = map_runner.MapRunner(trace_handles, map_function_handle, jobs=args.jobs, output_formatters=[output_formatter]) results = runner.Run() if args.map_function_handle: results = runner.RunMapper() elif args.reduce_function_handle: results = runner.RunReducer(trace_handles) output_formatter.Format(results) return results finally: ofile.close() os.unlink(map_file) shutil.rmtree(temp_directory)
def testRepr(self): mtl0 = function_handle.ModuleToLoad(href='/foo') mtl1 = function_handle.ModuleToLoad(filename='foo.html') self.assertEquals(str(mtl0), 'ModuleToLoad(href="/foo")') self.assertEquals(str(mtl1), 'ModuleToLoad(filename="foo.html")')
def testExactlyOneHrefOrFilename(self): with self.assertRaises(Exception): function_handle.ModuleToLoad() with self.assertRaises(Exception): function_handle.ModuleToLoad('foo', 'foo')
def _SingleFileFunctionHandle(filename, function_name, guid): return function_handle.FunctionHandle( modules_to_load=[function_handle.ModuleToLoad(filename=filename)], function_name=function_name, guid=guid)
def _Handle(filename): module = function_handle.ModuleToLoad(filename=filename) map_handle = function_handle.FunctionHandle(modules_to_load=[module], function_name='MyMapFunction') return job_module.Job(map_handle, None)
def _Handle(filename): module = function_handle.ModuleToLoad(filename=filename) return function_handle.FunctionHandle(modules_to_load=[module], function_name='MyMapFunction')