def _RunMapper(self, trace_handles, map_function_handle):
self.response.content_type = 'application/json'
output_formatter = json_output_formatter.JSONOutputFormatter(
self.response.out)
runner = map_runner.MapRunner(trace_handles, map_function_handle)
runner.Run(jobs=map_runner.AUTO_JOB_COUNT,
output_formatters=[output_formatter])
def Main(argv):
parser = argparse.ArgumentParser(description='Bulk trace processing')
corpus_driver_cmdline.AddArguments(parser)
parser.add_argument('--query')
parser.add_argument('map_function_handle')
parser.add_argument('-j',
'--jobs',
type=int,
default=map_runner.AUTO_JOB_COUNT)
parser.add_argument('-o', '--output-file')
parser.add_argument('-s', '--stop-on-error', action='store_true')
args = parser.parse_args(argv[1:])
corpus_driver = corpus_driver_cmdline.GetCorpusDriver(parser, args)
if args.query == 'help':
parser.exit(_CORPUS_QUERY_HELP)
elif args.query is None:
query = corpus_query.CorpusQuery.FromString('True')
else:
query = corpus_query.CorpusQuery.FromString(args.query)
if args.output_file:
ofile = open(args.output_file, 'w')
else:
ofile = sys.stdout
output_formatter = json_output_formatter.JSONOutputFormatter(ofile)
try:
map_function_handle = function_handle.FunctionHandle.FromUserFriendlyString(
args.map_function_handle)
except function_handle.UserFriendlyStringInvalidError:
error_lines = [
'The map_traces command-line API has changed! You must now specify the',
'filenames to load and the map function name, separated by :. For '
'example, a mapper in',
'foo.html called Foo would be written as foo.html:Foo .'
]
parser.error('\n'.join(error_lines))
try:
trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query)
runner = map_runner.MapRunner(trace_handles,
map_function_handle,
stop_on_error=args.stop_on_error,
jobs=args.jobs,
output_formatters=[output_formatter])
results = runner.Run()
if not results.had_failures:
return 0
else:
return 255
finally:
if ofile != sys.stdout:
ofile.close()
def Main(argv):
parser = argparse.ArgumentParser(
description=_DEFAULT_DESCRIPTION)
corpus_driver_cmdline.AddArguments(parser)
parser.add_argument('--map_function_handle')
parser.add_argument('--reduce_function_handle')
parser.add_argument('-o', '--output-file')
args = parser.parse_args(argv[1:])
corpus_driver = corpus_driver_cmdline.GetCorpusDriver(parser, args)
query = corpus_query.CorpusQuery.FromString('True')
if not args.output_file:
parser.exit("No output file specified.")
ofile = open(args.output_file, 'w')
output_formatter = json_output_formatter.JSONOutputFormatter(ofile)
try:
job = None
if args.map_function_handle:
map_handle = function_handle.FunctionHandle.FromUserFriendlyString(
args.map_function_handle)
job = job_module.Job(map_handle, None)
elif args.reduce_function_handle:
reduce_handle = function_handle.FunctionHandle.FromUserFriendlyString(
args.reduce_function_handle)
job = job_module.Job(None, reduce_handle)
except function_handle.UserFriendlyStringInvalidError:
parser.error('Either mapper or reducer not specified.')
try:
trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query)
runner = map_runner.MapRunner(trace_handles, job,
stop_on_error=False,
jobs=map_runner.AUTO_JOB_COUNT,
output_formatters=[output_formatter])
if args.map_function_handle:
results = runner.RunMapper()
elif args.reduce_function_handle:
results = runner.RunReducer(trace_handles)
output_formatter.Format(results)
return results
finally:
if ofile != sys.stdout:
ofile.close()
def Main(argv):
parser = argparse.ArgumentParser(description=_DEFAULT_DESCRIPTION)
parser.add_argument('map_file_url')
parser.add_argument('map_function_name')
parser.add_argument('input_url')
parser.add_argument('output_url')
parser.add_argument('--jobs', type=int, default=1)
args = parser.parse_args(argv[1:])
map_file = _ReadMapperGCSFile(args.map_file_url)
if not map_file:
parser.error('Map does not exist.')
if not args.map_function_name:
parser.error('Must provide map function name.')
temp_directory = tempfile.mkdtemp()
_, file_name = tempfile.mkstemp()
ofile = open(file_name, 'w')
try:
output_formatter = json_output_formatter.JSONOutputFormatter(ofile)
map_function_module = function_handle.ModuleToLoad(
filename=os.path.abspath(map_file))
map_function_handle = function_handle.FunctionHandle(
modules_to_load=[map_function_module],
function_name=args.map_function_name)
trace_handles = _DownloadTraceHandles(args.input_url, temp_directory)
runner = map_runner.MapRunner(trace_handles,
map_function_handle,
jobs=args.jobs,
output_formatters=[output_formatter])
results = runner.Run()
if args.map_function_handle:
results = runner.RunMapper()
elif args.reduce_function_handle:
results = runner.RunReducer(trace_handles)
output_formatter.Format(results)
return results
finally:
ofile.close()
os.unlink(map_file)
shutil.rmtree(temp_directory)
def Main(argv):
parser = argparse.ArgumentParser(description='Local bulk trace processing')
parser.add_argument('trace_directory')
parser.add_argument('--query')
parser.add_argument('map_file')
parser.add_argument('-j', '--jobs', type=int, default=1)
parser.add_argument('-o', '--output-file')
parser.add_argument('-s', '--stop-on-error', action='store_true')
args = parser.parse_args(argv[1:])
if not os.path.exists(args.trace_directory):
parser.error('trace_directory does not exist')
if not os.path.exists(args.map_file):
parser.error('map does not exist')
corpus_driver = local_directory_corpus_driver.LocalDirectoryCorpusDriver(
os.path.abspath(os.path.expanduser(args.trace_directory)))
if args.query is None:
query = corpus_query.CorpusQuery.FromString('True')
else:
query = corpus_query.CorpusQuery.FromString(args.query)
if args.output_file:
ofile = open(args.output_file, 'w')
else:
ofile = sys.stdout
output_formatter = json_output_formatter.JSONOutputFormatter(ofile)
map_function_handle = map_function_handle_module.MapFunctionHandle(
filename=os.path.abspath(args.map_file))
try:
trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query)
runner = map_runner.MapRunner(trace_handles,
map_function_handle,
stop_on_error=args.stop_on_error)
results = runner.Run(jobs=args.jobs,
output_formatters=[output_formatter])
if not results.had_failures:
return 0
else:
return 255
finally:
if ofile != sys.stdout:
ofile.close()