def _WillProcess(self):
if not os.path.exists(self.cache_file):
try:
cloud_storage.Copy(self.canonical_url, self.cache_file)
except cloud_storage.CloudStorageError:
return None
return URLFileHandle(self.canonical_url, 'file://' + self.cache_file)
def post(self, *args, **kwargs): # pylint: disable=unused-argument
self.response.content_type = 'application/json'
url = self.request.get('url', 'True')
# Doesn't need to be downloaded since it's not a cloud file, just return
# true and the path to the file.
if not 'gs://' in url:
self.response.write(json.dumps({'success': True, 'file': url}))
return
output_name = os.path.join(kwargs.pop('_pi_data_dir'),
url.split('/')[-1])
try:
print 'Downloading: %s' % url
cloud_storage.Copy(url, output_name)
except cloud_storage.CloudStorageError:
print ' -> Failed to download: %s' % url
self.response.write(json.dumps({'success': False}))
return
output_name = os.path.join('/perf_insights/test_data',
url.split('/')[-1])
self.response.write(json.dumps({'success': True, 'file': output_name}))
def _ReadMapperGCSFile(url):
file_handle, file_name = tempfile.mkstemp()
try:
cloud_storage.Copy(url, file_name)
except cloud_storage.CloudStorageError as e:
logging.info("Failed to copy: %s" % e)
os.close(file_handle)
os.unlink(file_name)
file_name = None
return file_name
def _ReadTracesGCSFile(url):
file_handle, file_name = tempfile.mkstemp()
file_urls = []
try:
cloud_storage.Copy(url, file_name)
with open(file_name, 'r') as f:
file_urls = json.loads(f.read())
except cloud_storage.CloudStorageError as e:
logging.info("Failed to copy: %s" % e)
finally:
os.close(file_handle)
os.unlink(file_name)
return file_urls
def post(self, *args, **kwargs): # pylint: disable=unused-argument
job_dict = json.loads(self.request.body)
job = job_module.Job.FromDict(job_dict)
job_with_filenames = job_module.Job(
job.map_function_handle.ConvertHrefsToAbsFilenames(self.app))
mapper_handle = job_with_filenames.map_function_handle
with open(mapper_handle.modules_to_load[0].filename, 'r') as f:
mapper = f.read()
mapper_name = job_with_filenames.map_function_handle.function_name
query_string = self.request.get('corpus_query', 'True')
query = corpus_query.CorpusQuery.FromString(query_string)
if query.max_trace_handles > MAX_TRACES:
print 'Capping query at %d' % MAX_TRACES
query.max_trace_handles = MAX_TRACES
query_string = query.AsQueryString()
params = urllib.urlencode({
'query': query_string,
'mapper': mapper,
'mapper_function': mapper_name,
'revision': 'HEAD',
'corpus': 'https://performance-insights.appspot.com',
'timeout': 240,
'function_timeout': 120
})
cloud_mapper_url = 'https://performance-insights.appspot.com'
if self.request.get('local') == 'true':
cloud_mapper_url = 'http://localhost:8080'
create_url = '%s/cloud_mapper/create' % cloud_mapper_url
response = urllib2.urlopen(create_url, data=params)
response_data = response.read()
print response_data
results = json.loads(response_data)
if results['status']:
jobid = results['jobid']
status_url = '%s/cloud_mapper/status?jobid=%s' % (cloud_mapper_url,
jobid)
start_time = datetime.datetime.now()
while datetime.datetime.now() - start_time < datetime.timedelta(
seconds=300):
time.sleep(1)
print 'Waiting for results.'
response = urllib2.urlopen(status_url)
results = json.loads(response.read())
if results['status'] == 'COMPLETE':
print 'Mapping complete. Downloading results.'
output_handle, output_name = tempfile.mkstemp()
try:
print ' -> %s' % results['data']
cloud_storage.Copy(results['data'], output_name)
except cloud_storage.CloudStorageError as e:
print 'Cloud storage error: %s' % str(e)
return
map_results = ''
with open(output_name, 'r') as f:
map_results = f.read()
os.close(output_handle)
self.response.write(map_results)
total_time = datetime.datetime.now() - start_time
print 'Time taken: %ss' % total_time.total_seconds()
print map_results[:128]
return