def Main(argv):
parser = argparse.ArgumentParser(
description=_DEFAULT_DESCRIPTION)
corpus_driver_cmdline.AddArguments(parser)
parser.add_argument('--map_function_handle')
parser.add_argument('--reduce_function_handle')
parser.add_argument('-o', '--output-file')
args = parser.parse_args(argv[1:])
corpus_driver = corpus_driver_cmdline.GetCorpusDriver(parser, args)
query = corpus_query.CorpusQuery.FromString('True')
if not args.output_file:
parser.exit("No output file specified.")
ofile = open(args.output_file, 'w')
output_formatter = json_output_formatter.JSONOutputFormatter(ofile)
try:
job = None
if args.map_function_handle:
map_handle = function_handle.FunctionHandle.FromUserFriendlyString(
args.map_function_handle)
job = job_module.Job(map_handle, None)
elif args.reduce_function_handle:
reduce_handle = function_handle.FunctionHandle.FromUserFriendlyString(
args.reduce_function_handle)
job = job_module.Job(None, reduce_handle)
except function_handle.UserFriendlyStringInvalidError:
parser.error('Either mapper or reducer not specified.')
try:
trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query)
runner = map_runner.MapRunner(trace_handles, job,
stop_on_error=False,
jobs=map_runner.AUTO_JOB_COUNT,
output_formatters=[output_formatter])
if args.map_function_handle:
results = runner.RunMapper()
elif args.reduce_function_handle:
results = runner.RunReducer(trace_handles)
output_formatter.Format(results)
return results
finally:
if ofile != sys.stdout:
ofile.close()
def PiReportToHTML(ofile, corpus_driver, pi_report_file, query,
json_output=False, stop_on_error=False, jobs=1, quiet=False):
project = perf_insights_project.PerfInsightsProject()
with open(pi_report_file, 'r') as f:
pi_report_file_contents = f.read()
map_function_href, map_function_name, pi_report_element_name = (
_GetMapFunctionHrefFromPiReport(pi_report_file_contents))
map_file = project.GetAbsPathFromHRef(map_function_href)
module = function_handle.ModuleToLoad(filename=map_file)
map_function_handle = function_handle.FunctionHandle([module],
map_function_name)
job = job_module.Job(map_function_handle, None)
if map_file == None:
raise Exception('Could not find %s' % map_function_href)
results = _MapTraces(corpus_driver, job, query, stop_on_error,
jobs, quiet)
if stop_on_error and results.had_failures:
sys.stderr.write('There were mapping errors. Aborting.')
return 255
if json_output:
json.dump([result.AsDict() for result in results], ofile, indent=2)
else:
WriteResultsToFile(ofile, project,
pi_report_file, pi_report_element_name,
results)
return 0
def testAsDict(self):
result = mre_result.MreResult()
with map_single_trace.TemporaryMapScript("""
pi.FunctionRegistry.register(
function MyMapFunction(result, model) {
var canonicalUrl = model.canonicalUrl;
result.addPair('result', {
numProcesses: model.getAllProcesses().length
});
});
""") as map_script:
module = function_handle.ModuleToLoad(filename=map_script.filename)
map_handle = function_handle.FunctionHandle(
modules_to_load=[module], function_name='MyMapFunction')
job = job_module.Job(map_handle, None)
failure = failure_module.Failure(job, '2', '3', 'err', 'desc',
'stack')
result.AddFailure(failure)
result.AddPair('foo', 'bar')
result_dict = result.AsDict()
self.assertEquals(result_dict['failures'], [failure.AsDict()])
self.assertEquals(result_dict['pairs'], {'foo': 'bar'})
def Main(argv):
parser = argparse.ArgumentParser(description='Bulk trace processing')
corpus_driver_cmdline.AddArguments(parser)
parser.add_argument('--query')
parser.add_argument('--map_function_handle')
parser.add_argument('-j',
'--jobs',
type=int,
default=map_runner.AUTO_JOB_COUNT)
parser.add_argument('-o', '--output-file')
parser.add_argument('-s', '--stop-on-error', action='store_true')
args = parser.parse_args(argv[1:])
corpus_driver = corpus_driver_cmdline.GetCorpusDriver(parser, args)
if args.query == 'help':
parser.exit(_CORPUS_QUERY_HELP)
elif args.query is None:
query = corpus_query.CorpusQuery.FromString('True')
else:
query = corpus_query.CorpusQuery.FromString(args.query)
if args.output_file:
ofile = open(args.output_file, 'w')
else:
ofile = sys.stdout
output_formatter = json_output_formatter.JSONOutputFormatter(ofile)
try:
map_handle = None
if args.map_function_handle:
map_handle = function_handle.FunctionHandle.FromUserFriendlyString(
args.map_function_handle)
job = job_module.Job(map_handle)
except function_handle.UserFriendlyStringInvalidError:
error_lines = [
'The map_traces command-line API has changed! You must now specify the',
'filenames to load and the map function name, separated by :. For '
'example, a mapper in',
'foo.html called Foo would be written as foo.html:Foo .'
]
parser.error('\n'.join(error_lines))
try:
trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query)
runner = map_runner.MapRunner(trace_handles,
job,
stop_on_error=args.stop_on_error,
jobs=args.jobs,
output_formatters=[output_formatter])
results = runner.Run()
if not any(result.failures for result in results):
return 0
else:
return 255
finally:
if ofile != sys.stdout:
ofile.close()
def testTranslateMreFailure(self):
map_function_handle = _SingleFileFunctionHandle('foo.html', 'Foo', '2')
job = job_module.Job(map_function_handle, '1')
story_set = story.StorySet(base_dir=os.path.dirname(__file__))
p = page.Page('http://www.foo.com/', story_set, story_set.base_dir)
f = failure.Failure(job, 'foo', '/a.json', 'MyFailure', 'failure',
'stack')
fv = common_value_helpers.TranslateMreFailure(f, p)
self.assertIn('stack', str(fv))
def _GetMetricRunnerHandle(metric):
assert isinstance(metric, basestring)
metrics_dir = _GetMetricsDir()
metric_mapper_path = os.path.join(metrics_dir,
_METRIC_MAP_FUNCTION_FILENAME)
modules_to_load = [
function_handle.ModuleToLoad(filename=metric_mapper_path)
]
map_function_handle = function_handle.FunctionHandle(
modules_to_load, _METRIC_MAP_FUNCTION_NAME, {'metric': metric})
return job_module.Job(map_function_handle, None)
def testAsDict(self):
map_function_handle = _SingleFileFunctionHandle('foo.html', 'Foo', '2')
reduce_function_handle = _SingleFileFunctionHandle('bar.html', 'Bar', '3')
job = job_module.Job(map_function_handle, reduce_function_handle, '1')
failure = failure_module.Failure(job, 'foo.html:Foo',
'file://foo.html',
'err', 'desc', 'stack')
self.assertEquals(failure.AsDict(), {
'job_guid': '1',
'function_handle_string': 'foo.html:Foo',
'trace_canonical_url': 'file://foo.html',
'type': 'err',
'description': 'desc',
'stack': 'stack'
})
def _ProcessTracesWithMapper(mapper_handle_str, traces, output_file):
map_handle = function_handle.FunctionHandle.FromUserFriendlyString(
mapper_handle_str)
job = job_module.Job(map_handle)
trace_handles = [
file_handle.URLFileHandle(f, 'file://%s' % f) for f in traces
]
runner = map_runner.MapRunner(
trace_handles,
job,
progress_reporter=progress_reporter.ProgressReporter())
results = runner.RunMapper()
results_dict = {k: v.AsDict() for k, v in results.iteritems()}
_DumpToOutputJson(results_dict, output_file)
return _GetExitCodeForResults(results)
def post(self, *args, **kwargs): # pylint: disable=unused-argument
job_dict = json.loads(self.request.body)
job = job_module.Job.FromDict(job_dict)
job_with_filenames = job_module.Job(
job.map_function_handle.ConvertHrefsToAbsFilenames(self.app))
corpus_driver = local_directory_corpus_driver.LocalDirectoryCorpusDriver(
trace_directory=kwargs.pop('_pi_data_dir'),
url_resolver=self.app.GetURLForAbsFilename)
# TODO(nduca): pass self.request.params to the map function [maybe].
query_string = self.request.get('corpus_query', 'True')
query = corpus_query.CorpusQuery.FromString(query_string)
trace_handles = corpus_driver.GetTraceHandlesMatchingQuery(query)
self._RunMapper(trace_handles, job_with_filenames)
def testFromDict(self):
map_function_handle = _SingleFileFunctionHandle('foo.html', 'Foo', '2')
job = job_module.Job(map_function_handle, '1')
failure_dict = {
'job_guid': '1',
'function_handle_string': 'foo.html:Foo',
'trace_canonical_url': 'file://foo.html',
'type': 'err',
'description': 'desc',
'stack': 'stack'
}
failure = failure_module.Failure.FromDict(failure_dict, job)
self.assertEquals(failure.job.guid, '1')
self.assertEquals(failure.function_handle_string, 'foo.html:Foo')
self.assertEquals(failure.trace_canonical_url, 'file://foo.html')
self.assertEquals(failure.failure_type_name, 'err')
self.assertEquals(failure.description, 'desc')
self.assertEquals(failure.stack, 'stack')
def _Handle(filename):
module = function_handle.ModuleToLoad(filename=filename)
map_handle = function_handle.FunctionHandle(modules_to_load=[module],
function_name='MyMapFunction')
return job_module.Job(map_handle, None)
def post(self, *args, **kwargs): # pylint: disable=unused-argument
job_dict = json.loads(self.request.body)
job = job_module.Job.FromDict(job_dict)
job_with_filenames = job_module.Job(
job.map_function_handle.ConvertHrefsToAbsFilenames(self.app))
mapper_handle = job_with_filenames.map_function_handle
with open(mapper_handle.modules_to_load[0].filename, 'r') as f:
mapper = f.read()
mapper_name = job_with_filenames.map_function_handle.function_name
query_string = self.request.get('corpus_query', 'True')
query = corpus_query.CorpusQuery.FromString(query_string)
if query.max_trace_handles > MAX_TRACES:
print 'Capping query at %d' % MAX_TRACES
query.max_trace_handles = MAX_TRACES
query_string = query.AsQueryString()
params = urllib.urlencode({
'query': query_string,
'mapper': mapper,
'mapper_function': mapper_name,
'revision': 'HEAD',
'corpus': 'https://performance-insights.appspot.com',
'timeout': 240,
'function_timeout': 120
})
cloud_mapper_url = 'https://performance-insights.appspot.com'
if self.request.get('local') == 'true':
cloud_mapper_url = 'http://localhost:8080'
create_url = '%s/cloud_mapper/create' % cloud_mapper_url
response = urllib2.urlopen(create_url, data=params)
response_data = response.read()
print response_data
results = json.loads(response_data)
if results['status']:
jobid = results['jobid']
status_url = '%s/cloud_mapper/status?jobid=%s' % (cloud_mapper_url,
jobid)
start_time = datetime.datetime.now()
while datetime.datetime.now() - start_time < datetime.timedelta(
seconds=300):
time.sleep(1)
print 'Waiting for results.'
response = urllib2.urlopen(status_url)
results = json.loads(response.read())
if results['status'] == 'COMPLETE':
print 'Mapping complete. Downloading results.'
output_handle, output_name = tempfile.mkstemp()
try:
print ' -> %s' % results['data']
cloud_storage.Copy(results['data'], output_name)
except cloud_storage.CloudStorageError as e:
print 'Cloud storage error: %s' % str(e)
return
map_results = ''
with open(output_name, 'r') as f:
map_results = f.read()
os.close(output_handle)
self.response.write(map_results)
total_time = datetime.datetime.now() - start_time
print 'Time taken: %ss' % total_time.total_seconds()
print map_results[:128]
return
