def Main(argv): parser = argparse.ArgumentParser( description=_DEFAULT_DESCRIPTION) corpus_driver_cmdline.AddArguments(parser) parser.add_argument('--map_function_handle') parser.add_argument('--reduce_function_handle') parser.add_argument('-o', '--output-file') args = parser.parse_args(argv[1:]) corpus_driver = corpus_driver_cmdline.GetCorpusDriver(parser, args) query = corpus_query.CorpusQuery.FromString('True') if not args.output_file: parser.exit("No output file specified.") ofile = open(args.output_file, 'w') output_formatter = json_output_formatter.JSONOutputFormatter(ofile) try: job = None if args.map_function_handle: map_handle = function_handle.FunctionHandle.FromUserFriendlyString( args.map_function_handle) job = job_module.Job(map_handle, None) elif args.reduce_function_handle: reduce_handle = function_handle.FunctionHandle.FromUserFriendlyString( args.reduce_function_handle) job = job_module.Job(None, reduce_handle) except function_handle.UserFriendlyStringInvalidError: parser.error('Either mapper or reducer not specified.') try: trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query) runner = map_runner.MapRunner(trace_handles, job, stop_on_error=False, jobs=map_runner.AUTO_JOB_COUNT, output_formatters=[output_formatter]) if args.map_function_handle: results = runner.RunMapper() elif args.reduce_function_handle: results = runner.RunReducer(trace_handles) output_formatter.Format(results) return results finally: if ofile != sys.stdout: ofile.close()
def PiReportToHTML(ofile, corpus_driver, pi_report_file, query, json_output=False, stop_on_error=False, jobs=1, quiet=False): project = perf_insights_project.PerfInsightsProject() with open(pi_report_file, 'r') as f: pi_report_file_contents = f.read() map_function_href, map_function_name, pi_report_element_name = ( _GetMapFunctionHrefFromPiReport(pi_report_file_contents)) map_file = project.GetAbsPathFromHRef(map_function_href) module = function_handle.ModuleToLoad(filename=map_file) map_function_handle = function_handle.FunctionHandle([module], map_function_name) job = job_module.Job(map_function_handle, None) if map_file == None: raise Exception('Could not find %s' % map_function_href) results = _MapTraces(corpus_driver, job, query, stop_on_error, jobs, quiet) if stop_on_error and results.had_failures: sys.stderr.write('There were mapping errors. Aborting.') return 255 if json_output: json.dump([result.AsDict() for result in results], ofile, indent=2) else: WriteResultsToFile(ofile, project, pi_report_file, pi_report_element_name, results) return 0
def testAsDict(self): result = mre_result.MreResult() with map_single_trace.TemporaryMapScript(""" pi.FunctionRegistry.register( function MyMapFunction(result, model) { var canonicalUrl = model.canonicalUrl; result.addPair('result', { numProcesses: model.getAllProcesses().length }); }); """) as map_script: module = function_handle.ModuleToLoad(filename=map_script.filename) map_handle = function_handle.FunctionHandle( modules_to_load=[module], function_name='MyMapFunction') job = job_module.Job(map_handle, None) failure = failure_module.Failure(job, '2', '3', 'err', 'desc', 'stack') result.AddFailure(failure) result.AddPair('foo', 'bar') result_dict = result.AsDict() self.assertEquals(result_dict['failures'], [failure.AsDict()]) self.assertEquals(result_dict['pairs'], {'foo': 'bar'})
def Main(argv): parser = argparse.ArgumentParser(description='Bulk trace processing') corpus_driver_cmdline.AddArguments(parser) parser.add_argument('--query') parser.add_argument('--map_function_handle') parser.add_argument('-j', '--jobs', type=int, default=map_runner.AUTO_JOB_COUNT) parser.add_argument('-o', '--output-file') parser.add_argument('-s', '--stop-on-error', action='store_true') args = parser.parse_args(argv[1:]) corpus_driver = corpus_driver_cmdline.GetCorpusDriver(parser, args) if args.query == 'help': parser.exit(_CORPUS_QUERY_HELP) elif args.query is None: query = corpus_query.CorpusQuery.FromString('True') else: query = corpus_query.CorpusQuery.FromString(args.query) if args.output_file: ofile = open(args.output_file, 'w') else: ofile = sys.stdout output_formatter = json_output_formatter.JSONOutputFormatter(ofile) try: map_handle = None if args.map_function_handle: map_handle = function_handle.FunctionHandle.FromUserFriendlyString( args.map_function_handle) job = job_module.Job(map_handle) except function_handle.UserFriendlyStringInvalidError: error_lines = [ 'The map_traces command-line API has changed! You must now specify the', 'filenames to load and the map function name, separated by :. For ' 'example, a mapper in', 'foo.html called Foo would be written as foo.html:Foo .' ] parser.error('\n'.join(error_lines)) try: trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query) runner = map_runner.MapRunner(trace_handles, job, stop_on_error=args.stop_on_error, jobs=args.jobs, output_formatters=[output_formatter]) results = runner.Run() if not any(result.failures for result in results): return 0 else: return 255 finally: if ofile != sys.stdout: ofile.close()
def testTranslateMreFailure(self): map_function_handle = _SingleFileFunctionHandle('foo.html', 'Foo', '2') job = job_module.Job(map_function_handle, '1') story_set = story.StorySet(base_dir=os.path.dirname(__file__)) p = page.Page('http://www.foo.com/', story_set, story_set.base_dir) f = failure.Failure(job, 'foo', '/a.json', 'MyFailure', 'failure', 'stack') fv = common_value_helpers.TranslateMreFailure(f, p) self.assertIn('stack', str(fv))
def _GetMetricRunnerHandle(metric): assert isinstance(metric, basestring) metrics_dir = _GetMetricsDir() metric_mapper_path = os.path.join(metrics_dir, _METRIC_MAP_FUNCTION_FILENAME) modules_to_load = [ function_handle.ModuleToLoad(filename=metric_mapper_path) ] map_function_handle = function_handle.FunctionHandle( modules_to_load, _METRIC_MAP_FUNCTION_NAME, {'metric': metric}) return job_module.Job(map_function_handle, None)
def testAsDict(self): map_function_handle = _SingleFileFunctionHandle('foo.html', 'Foo', '2') reduce_function_handle = _SingleFileFunctionHandle('bar.html', 'Bar', '3') job = job_module.Job(map_function_handle, reduce_function_handle, '1') failure = failure_module.Failure(job, 'foo.html:Foo', 'file://foo.html', 'err', 'desc', 'stack') self.assertEquals(failure.AsDict(), { 'job_guid': '1', 'function_handle_string': 'foo.html:Foo', 'trace_canonical_url': 'file://foo.html', 'type': 'err', 'description': 'desc', 'stack': 'stack' })
def _ProcessTracesWithMapper(mapper_handle_str, traces, output_file): map_handle = function_handle.FunctionHandle.FromUserFriendlyString( mapper_handle_str) job = job_module.Job(map_handle) trace_handles = [ file_handle.URLFileHandle(f, 'file://%s' % f) for f in traces ] runner = map_runner.MapRunner( trace_handles, job, progress_reporter=progress_reporter.ProgressReporter()) results = runner.RunMapper() results_dict = {k: v.AsDict() for k, v in results.iteritems()} _DumpToOutputJson(results_dict, output_file) return _GetExitCodeForResults(results)
def post(self, *args, **kwargs): # pylint: disable=unused-argument job_dict = json.loads(self.request.body) job = job_module.Job.FromDict(job_dict) job_with_filenames = job_module.Job( job.map_function_handle.ConvertHrefsToAbsFilenames(self.app)) corpus_driver = local_directory_corpus_driver.LocalDirectoryCorpusDriver( trace_directory=kwargs.pop('_pi_data_dir'), url_resolver=self.app.GetURLForAbsFilename) # TODO(nduca): pass self.request.params to the map function [maybe]. query_string = self.request.get('corpus_query', 'True') query = corpus_query.CorpusQuery.FromString(query_string) trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query) self._RunMapper(trace_handles, job_with_filenames)
def testFromDict(self): map_function_handle = _SingleFileFunctionHandle('foo.html', 'Foo', '2') job = job_module.Job(map_function_handle, '1') failure_dict = { 'job_guid': '1', 'function_handle_string': 'foo.html:Foo', 'trace_canonical_url': 'file://foo.html', 'type': 'err', 'description': 'desc', 'stack': 'stack' } failure = failure_module.Failure.FromDict(failure_dict, job) self.assertEquals(failure.job.guid, '1') self.assertEquals(failure.function_handle_string, 'foo.html:Foo') self.assertEquals(failure.trace_canonical_url, 'file://foo.html') self.assertEquals(failure.failure_type_name, 'err') self.assertEquals(failure.description, 'desc') self.assertEquals(failure.stack, 'stack')
def _Handle(filename): module = function_handle.ModuleToLoad(filename=filename) map_handle = function_handle.FunctionHandle(modules_to_load=[module], function_name='MyMapFunction') return job_module.Job(map_handle, None)
def post(self, *args, **kwargs): # pylint: disable=unused-argument job_dict = json.loads(self.request.body) job = job_module.Job.FromDict(job_dict) job_with_filenames = job_module.Job( job.map_function_handle.ConvertHrefsToAbsFilenames(self.app)) mapper_handle = job_with_filenames.map_function_handle with open(mapper_handle.modules_to_load[0].filename, 'r') as f: mapper = f.read() mapper_name = job_with_filenames.map_function_handle.function_name query_string = self.request.get('corpus_query', 'True') query = corpus_query.CorpusQuery.FromString(query_string) if query.max_trace_handles > MAX_TRACES: print 'Capping query at %d' % MAX_TRACES query.max_trace_handles = MAX_TRACES query_string = query.AsQueryString() params = urllib.urlencode({ 'query': query_string, 'mapper': mapper, 'mapper_function': mapper_name, 'revision': 'HEAD', 'corpus': 'https://performance-insights.appspot.com', 'timeout': 240, 'function_timeout': 120 }) cloud_mapper_url = 'https://performance-insights.appspot.com' if self.request.get('local') == 'true': cloud_mapper_url = 'http://localhost:8080' create_url = '%s/cloud_mapper/create' % cloud_mapper_url response = urllib2.urlopen(create_url, data=params) response_data = response.read() print response_data results = json.loads(response_data) if results['status']: jobid = results['jobid'] status_url = '%s/cloud_mapper/status?jobid=%s' % (cloud_mapper_url, jobid) start_time = datetime.datetime.now() while datetime.datetime.now() - start_time < datetime.timedelta( seconds=300): time.sleep(1) print 'Waiting for results.' response = urllib2.urlopen(status_url) results = json.loads(response.read()) if results['status'] == 'COMPLETE': print 'Mapping complete. Downloading results.' output_handle, output_name = tempfile.mkstemp() try: print ' -> %s' % results['data'] cloud_storage.Copy(results['data'], output_name) except cloud_storage.CloudStorageError as e: print 'Cloud storage error: %s' % str(e) return map_results = '' with open(output_name, 'r') as f: map_results = f.read() os.close(output_handle) self.response.write(map_results) total_time = datetime.datetime.now() - start_time print 'Time taken: %ss' % total_time.total_seconds() print map_results[:128] return