def _RunMapper(self, trace_handles, map_function_handle): self.response.content_type = 'application/json' output_formatter = json_output_formatter.JSONOutputFormatter( self.response.out) runner = map_runner.MapRunner(trace_handles, map_function_handle) runner.Run(jobs=map_runner.AUTO_JOB_COUNT, output_formatters=[output_formatter])
def Main(argv): parser = argparse.ArgumentParser(description='Bulk trace processing') corpus_driver_cmdline.AddArguments(parser) parser.add_argument('--query') parser.add_argument('map_function_handle') parser.add_argument('-j', '--jobs', type=int, default=map_runner.AUTO_JOB_COUNT) parser.add_argument('-o', '--output-file') parser.add_argument('-s', '--stop-on-error', action='store_true') args = parser.parse_args(argv[1:]) corpus_driver = corpus_driver_cmdline.GetCorpusDriver(parser, args) if args.query == 'help': parser.exit(_CORPUS_QUERY_HELP) elif args.query is None: query = corpus_query.CorpusQuery.FromString('True') else: query = corpus_query.CorpusQuery.FromString(args.query) if args.output_file: ofile = open(args.output_file, 'w') else: ofile = sys.stdout output_formatter = json_output_formatter.JSONOutputFormatter(ofile) try: map_function_handle = function_handle.FunctionHandle.FromUserFriendlyString( args.map_function_handle) except function_handle.UserFriendlyStringInvalidError: error_lines = [ 'The map_traces command-line API has changed! You must now specify the', 'filenames to load and the map function name, separated by :. For ' 'example, a mapper in', 'foo.html called Foo would be written as foo.html:Foo .' ] parser.error('\n'.join(error_lines)) try: trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query) runner = map_runner.MapRunner(trace_handles, map_function_handle, stop_on_error=args.stop_on_error, jobs=args.jobs, output_formatters=[output_formatter]) results = runner.Run() if not results.had_failures: return 0 else: return 255 finally: if ofile != sys.stdout: ofile.close()
def _MapTraces(corpus_driver, map_function_handle, query, stop_on_error=False, jobs=1, quiet=False): trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query) if quiet: alt_progress_reporter = progress_reporter_module.ProgressReporter() else: alt_progress_reporter = None runner = map_runner.MapRunner(trace_handles, map_function_handle, stop_on_error=stop_on_error, progress_reporter=alt_progress_reporter, jobs=jobs) return runner.Run()
def Main(argv): parser = argparse.ArgumentParser( description=_DEFAULT_DESCRIPTION) corpus_driver_cmdline.AddArguments(parser) parser.add_argument('--map_function_handle') parser.add_argument('--reduce_function_handle') parser.add_argument('-o', '--output-file') args = parser.parse_args(argv[1:]) corpus_driver = corpus_driver_cmdline.GetCorpusDriver(parser, args) query = corpus_query.CorpusQuery.FromString('True') if not args.output_file: parser.exit("No output file specified.") ofile = open(args.output_file, 'w') output_formatter = json_output_formatter.JSONOutputFormatter(ofile) try: job = None if args.map_function_handle: map_handle = function_handle.FunctionHandle.FromUserFriendlyString( args.map_function_handle) job = job_module.Job(map_handle, None) elif args.reduce_function_handle: reduce_handle = function_handle.FunctionHandle.FromUserFriendlyString( args.reduce_function_handle) job = job_module.Job(None, reduce_handle) except function_handle.UserFriendlyStringInvalidError: parser.error('Either mapper or reducer not specified.') try: trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query) runner = map_runner.MapRunner(trace_handles, job, stop_on_error=False, jobs=map_runner.AUTO_JOB_COUNT, output_formatters=[output_formatter]) if args.map_function_handle: results = runner.RunMapper() elif args.reduce_function_handle: results = runner.RunReducer(trace_handles) output_formatter.Format(results) return results finally: if ofile != sys.stdout: ofile.close()
def Main(argv): parser = argparse.ArgumentParser(description=_DEFAULT_DESCRIPTION) parser.add_argument('map_file_url') parser.add_argument('map_function_name') parser.add_argument('input_url') parser.add_argument('output_url') parser.add_argument('--jobs', type=int, default=1) args = parser.parse_args(argv[1:]) map_file = _ReadMapperGCSFile(args.map_file_url) if not map_file: parser.error('Map does not exist.') if not args.map_function_name: parser.error('Must provide map function name.') temp_directory = tempfile.mkdtemp() _, file_name = tempfile.mkstemp() ofile = open(file_name, 'w') try: output_formatter = json_output_formatter.JSONOutputFormatter(ofile) map_function_module = function_handle.ModuleToLoad( filename=os.path.abspath(map_file)) map_function_handle = function_handle.FunctionHandle( modules_to_load=[map_function_module], function_name=args.map_function_name) trace_handles = _DownloadTraceHandles(args.input_url, temp_directory) runner = map_runner.MapRunner(trace_handles, map_function_handle, jobs=args.jobs, output_formatters=[output_formatter]) results = runner.Run() if args.map_function_handle: results = runner.RunMapper() elif args.reduce_function_handle: results = runner.RunReducer(trace_handles) output_formatter.Format(results) return results finally: ofile.close() os.unlink(map_file) shutil.rmtree(temp_directory)
def MapTracesWithWeatherReport(project, trace_directory, stop_on_error=False, jobs=1): map_file = os.path.join(project.perf_insights_src_path, 'wr', 'weather_report_map_function.html') corpus_driver = local_directory_corpus_driver.LocalDirectoryCorpusDriver( os.path.abspath(os.path.expanduser(trace_directory))) query = get_trace_handles_query.GetTraceHandlesQuery.FromString('True') trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query) runner = map_runner.MapRunner(trace_handles, map_file, stop_on_error=stop_on_error) return runner.Run(jobs=jobs)
def Main(argv): parser = argparse.ArgumentParser(description='Local bulk trace processing') parser.add_argument('trace_directory') parser.add_argument('--query') parser.add_argument('map_file') parser.add_argument('-j', '--jobs', type=int, default=1) parser.add_argument('-o', '--output-file') parser.add_argument('-s', '--stop-on-error', action='store_true') args = parser.parse_args(argv[1:]) if not os.path.exists(args.trace_directory): parser.error('trace_directory does not exist') if not os.path.exists(args.map_file): parser.error('map does not exist') corpus_driver = local_directory_corpus_driver.LocalDirectoryCorpusDriver( os.path.abspath(os.path.expanduser(args.trace_directory))) if args.query is None: query = corpus_query.CorpusQuery.FromString('True') else: query = corpus_query.CorpusQuery.FromString(args.query) if args.output_file: ofile = open(args.output_file, 'w') else: ofile = sys.stdout output_formatter = json_output_formatter.JSONOutputFormatter(ofile) map_function_handle = map_function_handle_module.MapFunctionHandle( filename=os.path.abspath(args.map_file)) try: trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query) runner = map_runner.MapRunner(trace_handles, map_function_handle, stop_on_error=args.stop_on_error) results = runner.Run(jobs=args.jobs, output_formatters=[output_formatter]) if not results.had_failures: return 0 else: return 255 finally: if ofile != sys.stdout: ofile.close()
def _MapTraces(trace_directory, map_function_handle, query, stop_on_error=False, jobs=1, quiet=False): corpus_driver = local_directory_corpus_driver.LocalDirectoryCorpusDriver( os.path.abspath(os.path.expanduser(trace_directory))) trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query) if quiet: alt_progress_reporter = progress_reporter_module.ProgressReporter() else: alt_progress_reporter = None runner = map_runner.MapRunner(trace_handles, map_function_handle, stop_on_error=stop_on_error, progress_reporter=alt_progress_reporter) return runner.Run(jobs=jobs)