def StoreObjectInGCS(
content, mime_type, project_id, thumb_width=DEFAULT_THUMB_WIDTH,
thumb_height=DEFAULT_THUMB_HEIGHT, filename=None):
bucket_name = app_identity.get_default_gcs_bucket_name()
guid = uuid.uuid4()
object_id = '/%s/attachments/%s' % (project_id, guid)
object_path = '/' + bucket_name + object_id
options = {}
if filename:
if not framework_constants.FILENAME_RE.match(filename):
logging.info('bad file name: %s' % filename)
filename = 'attachment.dat'
options['Content-Disposition'] = 'inline; filename="%s"' % filename
logging.info('Writing with options %r', options)
with cloudstorage.open(object_path, 'w', mime_type, options=options) as f:
f.write(content)
if mime_type in RESIZABLE_MIME_TYPES:
# Create and save a thumbnail too.
thumb_content = None
try:
thumb_content = images.resize(content, thumb_width, thumb_height)
except images.LargeImageError:
# Don't log the whole exception because we don't need to see
# this on the Cloud Error Reporting page.
logging.info('Got LargeImageError on image with %d bytes', len(content))
except Exception, e:
# Do not raise exception for incorrectly formed images.
# See https://bugs.chromium.org/p/monorail/issues/detail?id=597 for more
# detail.
logging.exception(e)
if thumb_content:
thumb_path = '%s-thumbnail' % object_path
with cloudstorage.open(thumb_path, 'w', 'image/png') as f:
f.write(thumb_content)
def testStoreObjectInGCS_NotResizableMimeType(self):
guid = 'aaaaa'
project_id = 100
object_id = '/%s/attachments/%s' % (project_id, guid)
bucket_name = 'test_bucket'
object_path = '/' + bucket_name + object_id
mime_type = 'not_resizable_mime_type'
content = 'content'
self.mox.StubOutWithMock(app_identity, 'get_default_gcs_bucket_name')
app_identity.get_default_gcs_bucket_name().AndReturn(bucket_name)
self.mox.StubOutWithMock(uuid, 'uuid4')
uuid.uuid4().AndReturn(guid)
self.mox.StubOutWithMock(cloudstorage, 'open')
cloudstorage.open(object_path, 'w',
mime_type).AndReturn(fake.FakeFile())
self.mox.ReplayAll()
ret_id = gcs_helpers.StoreObjectInGCS(content, mime_type, project_id,
gcs_helpers.DEFAULT_THUMB_WIDTH,
gcs_helpers.DEFAULT_THUMB_HEIGHT)
self.mox.VerifyAll()
self.assertEquals(object_id, ret_id)
def post(self):
trace_uuid = str(uuid.uuid4())
if 'GCS_BUCKET_NAME' not in os.environ:
bucket_name = app_identity.get_default_gcs_bucket_name()
else:
bucket_name = os.environ['GCS_BUCKET_NAME']
gcs_path = ('/' + bucket_name + '/' + trace_uuid + '.gz')
gcs_file = gcs.open(gcs_path,
'w',
content_type='application/octet-stream',
options={},
retry_params=default_retry_params)
gcs_file.write(self.request.get('trace'))
gcs_file.close()
trace_object = trace_info.TraceInfo(id=trace_uuid)
trace_object.prod = self.request.get('prod')
trace_object.network_type = self.request.get('network-type')
trace_object.remote_addr = os.environ["REMOTE_ADDR"]
tags_string = self.request.get('tags')
if tags_string:
# Tags are comma separated and should only include alphanumeric + '-'.
if re.match('^[a-zA-Z0-9-,]+$', tags_string):
trace_object.tags = tags_string.split(',')
else:
logging.warning('The provided tags string includes one or more invalid'
' characters and will be ignored')
trace_object.user_agent = self.request.headers.get('User-Agent')
trace_object.ver = self.request.get('product-version')
trace_object.config = self.request.get('config')
trace_object.put()
self.response.write(trace_uuid)
def post(self):
trace_uuid = str(uuid.uuid4())
if 'GCS_BUCKET_NAME' not in os.environ:
bucket_name = app_identity.get_default_gcs_bucket_name()
else:
bucket_name = os.environ['GCS_BUCKET_NAME']
gcs_path = ('/' + bucket_name + '/' + trace_uuid + '.gz')
gcs_file = gcs.open(gcs_path,
'w',
content_type='application/octet-stream',
options={},
retry_params=default_retry_params)
gcs_file.write(self.request.get('trace'))
gcs_file.close()
trace_object = trace_info.TraceInfo(id=trace_uuid)
trace_object.prod = self.request.get('prod')
trace_object.network_type = self.request.get('network_type')
trace_object.remote_addr = os.environ["REMOTE_ADDR"]
tags_string = self.request.get('tags')
if re.match('^[a-zA-Z0-9,]+$', tags_string): # ignore non alpha-numeric tags
trace_object.tags = tags_string.split(',')
trace_object.user_agent = self.request.headers.get('User-Agent')
trace_object.ver = self.request.get('product_version')
trace_object.put()
self.response.write(trace_uuid)
def WriteGCS(fullurl, data):
gcs_file = gcs.open(_remove_gcs_prefix(fullurl),
'w',
content_type='text/plain',
options={},
retry_params=default_retry_params)
gcs_file.write(data)
gcs_file.close()
def WriteGCS(fullurl, data):
gcs_file = gcs.open(_remove_gcs_prefix(fullurl),
'w',
content_type='text/plain',
options={},
retry_params=default_retry_params)
gcs_file.write(data)
gcs_file.close()
def ReadGCS(fullurl):
gcs_file = gcs.open(_remove_gcs_prefix(fullurl),
'r',
retry_params=default_retry_params)
contents = gcs_file.read()
gcs_file.close()
return contents
def ReadGCS(fullurl):
gcs_file = gcs.open(_remove_gcs_prefix(fullurl),
'r',
retry_params=default_retry_params)
contents = gcs_file.read()
gcs_file.close()
return contents
def post(self):
self.response.headers['Content-Type'] = 'text/plain'
jobid = self.request.get('jobid')
job = job_info.JobInfo.get_by_id(jobid)
if not job:
return
payload = urllib.urlencode({'q': 'MAX_TRACE_HANDLES=10'})
query_url = '%s/query?%s' % (_PERFORMANCE_INSIGHTS_URL, payload)
result = urlfetch.fetch(url=query_url,
payload=payload,
method=urlfetch.GET,
follow_redirects=False,
deadline=10)
logging.info(result.content)
taskid = str(uuid.uuid4())
traces = json.loads(result.content)
default_retry_params = gcs.RetryParams(initial_delay=0.2,
max_delay=5.0,
backoff_factor=2,
max_retry_period=15)
gcs_file = gcs.open(_DEFAULT_BUCKET.format(name=taskid),
'w',
content_type='text/plain',
options={},
retry_params=default_retry_params)
gcs_file.write(json.dumps(traces))
gcs_file.close()
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/compute')
http = credentials.authorize(httplib2.Http(memcache))
compute = build("compute", "v1", http=http)
startup_script = _STARTUP_SCRIPT.format(
revision=job.revision)
result = self._CreateGCEInstace(
compute, 'mr-%s' % jobid, startup_script)
logging.info('Call to instances().insert response:\n')
for k, v in sorted(result.iteritems()):
logging.info(' %s: %s' % (k, v))
job.status = 'COMPLETE'
job.put()
response = {'success': False}
self.response.out.write(json.dumps(response))
def MaybeCreateDownload(bucket_name, object_id, filename):
"""If the obj is not huge, and no download version exists, create it."""
src = '/%s%s' % (bucket_name, object_id)
dst = '/%s%s-download' % (bucket_name, object_id)
cloudstorage.validate_file_path(src)
cloudstorage.validate_file_path(dst)
logging.info('Maybe create %r from %r', dst, src)
if IS_DEV_APPSERVER:
logging.info('dev environment never makes download copies.')
return False
# If "Download" object already exists, we are done.
try:
cloudstorage.stat(dst)
logging.info('Download version of attachment already exists')
return True
except errors.NotFoundError:
pass
# If "View" object is huge, give up.
src_stat = cloudstorage.stat(src)
if src_stat.st_size > MAX_ATTACH_SIZE_TO_COPY:
logging.info('Download version of attachment would be too big')
return False
with cloudstorage.open(src, 'r') as infile:
content = infile.read()
logging.info('opened GCS object and read %r bytes', len(content))
content_type = src_stat.content_type
options = {
'Content-Disposition': 'attachment; filename="%s"' % filename,
}
logging.info('Writing with options %r', options)
with cloudstorage.open(dst, 'w', content_type, options=options) as outfile:
outfile.write(content)
logging.info('done writing')
return True
def StoreObjectInGCS(
content, mime_type, project_id, thumb_width=DEFAULT_THUMB_WIDTH,
thumb_height=DEFAULT_THUMB_HEIGHT):
bucket_name = app_identity.get_default_gcs_bucket_name()
guid = uuid.uuid4()
object_id = '/%s/attachments/%s' % (project_id, guid)
object_path = '/' + bucket_name + object_id
with cloudstorage.open(object_path, 'w', mime_type) as f:
f.write(content)
if mime_type in RESIZABLE_MIME_TYPES:
# Create and save a thumbnail too.
thumb_content = None
try:
thumb_content = images.resize(content, thumb_width, thumb_height)
except Exception, e:
# Do not raise exception for incorrectly formed images.
# See https://bugs.chromium.org/p/monorail/issues/detail?id=597 for more
# detail.
logging.exception(e)
if thumb_content:
thumb_path = '%s-thumbnail' % object_path
with cloudstorage.open(thumb_path, 'w', 'image/png') as f:
f.write(thumb_content)
def post(self):
trace_uuid = str(uuid.uuid4())
gcs_path = '/%s/%s.gz' % (cloud_config.Get().trace_upload_bucket,
trace_uuid)
gcs_file = gcs.open(gcs_path,
'w',
content_type='application/octet-stream',
options={},
retry_params=default_retry_params)
gcs_file.write(self.request.get('trace'))
gcs_file.close()
trace_object = trace_info.TraceInfo(id=trace_uuid)
trace_object.remote_addr = os.environ["REMOTE_ADDR"]
for arg in self.request.arguments():
arg_key = arg.replace('-', '_').lower()
if arg_key in trace_object._properties:
try:
setattr(trace_object, arg_key, self.request.get(arg))
except datastore_errors.BadValueError:
pass
scenario_config = self.request.get('config')
if scenario_config:
config_json = json.loads(scenario_config)
if 'scenario_name' in config_json:
trace_object.scenario_name = config_json['scenario_name']
tags_string = self.request.get('tags')
if tags_string:
# Tags are comma separated and should only include alphanumeric + '-'.
if re.match('^[a-zA-Z0-9-,]+$', tags_string):
trace_object.tags = tags_string.split(',')
else:
logging.warning(
'The provided tags string includes one or more invalid'
' characters and will be ignored')
trace_object.ver = self.request.get('product-version')
trace_object.put()
self.response.write(trace_uuid)
def post(self):
trace_uuid = str(uuid.uuid4())
gcs_path = '/%s/%s.gz' % (
cloud_config.Get().trace_upload_bucket, trace_uuid)
gcs_file = gcs.open(gcs_path,
'w',
content_type='application/octet-stream',
options={},
retry_params=default_retry_params)
gcs_file.write(self.request.get('trace'))
gcs_file.close()
trace_object = trace_info.TraceInfo(id=trace_uuid)
trace_object.remote_addr = os.environ["REMOTE_ADDR"]
for arg in self.request.arguments():
arg_key = arg.replace('-', '_').lower()
if arg_key in trace_object._properties:
try:
setattr(trace_object, arg_key, self.request.get(arg))
except datastore_errors.BadValueError:
pass
scenario_config = self.request.get('config')
if scenario_config:
config_json = json.loads(scenario_config)
if 'scenario_name' in config_json:
trace_object.scenario_name = config_json['scenario_name']
tags_string = self.request.get('tags')
if tags_string:
# Tags are comma separated and should only include alphanumeric + '-'.
if re.match('^[a-zA-Z0-9-,]+$', tags_string):
trace_object.tags = tags_string.split(',')
else:
logging.warning('The provided tags string includes one or more invalid'
' characters and will be ignored')
trace_object.ver = self.request.get('product-version')
trace_object.put()
self.response.write(trace_uuid)
def update_annotations_from_csv(filename="anotace.csv"):
logging.info("Starting to update from CSV")
my_default_retry_params = gcs.RetryParams(initial_delay=0.2,
max_delay=5.0,
backoff_factor=2,
max_retry_period=15)
gcs.set_default_retry_params(my_default_retry_params)
# bucket_name = os.environ.get('BUCKET_NAME',
# app_identity.get_default_gcs_bucket_name())
bucket_name = "tisic-knih.appspot.com"
bucket = '/' + bucket_name
filename = bucket + '/' + filename
try:
if is_dev_server():
gcs_file = open("anotace.csv")
else:
gcs_file = gcs.open(filename)
# gcs_file.seek(1000)
r = ucsv.reader(gcs_file, encoding='utf-8')
r.next() # Skip first line (header).
annotation_data = {}
for row_number, row in enumerate(r):
if row_number % 10000 == 0:
logging.info("Processing row number {} of CSV file. "
.format(row_number))
item_id = row[0]
short_text = row[1]
long_text = row[2]
annotation_data[item_id] = (short_text, long_text)
if len(annotation_data) > MAX_OBJECTS_PER_BATCH:
deferred.defer(_put_annotations_batch, annotation_data)
annotation_data = {}
deferred.defer(_put_annotations_batch, annotation_data)
gcs_file.close()
except Exception as e:
logging.error(e)
raise deferred.PermanentTaskFailure()
def update_annotations_from_csv(filename="anotace.csv"):
logging.info("Starting to update from CSV")
my_default_retry_params = gcs.RetryParams(initial_delay=0.2,
max_delay=5.0,
backoff_factor=2,
max_retry_period=15)
gcs.set_default_retry_params(my_default_retry_params)
# bucket_name = os.environ.get('BUCKET_NAME',
# app_identity.get_default_gcs_bucket_name())
bucket_name = "tisic-knih.appspot.com"
bucket = '/' + bucket_name
filename = bucket + '/' + filename
try:
if is_dev_server():
gcs_file = open("anotace.csv")
else:
gcs_file = gcs.open(filename)
# gcs_file.seek(1000)
r = ucsv.reader(gcs_file, encoding='utf-8')
r.next() # Skip first line (header).
annotation_data = {}
for row_number, row in enumerate(r):
if row_number % 10000 == 0:
logging.info("Processing row number {} of CSV file. ".format(
row_number))
item_id = row[0]
short_text = row[1]
long_text = row[2]
annotation_data[item_id] = (short_text, long_text)
if len(annotation_data) > MAX_OBJECTS_PER_BATCH:
deferred.defer(_put_annotations_batch, annotation_data)
annotation_data = {}
deferred.defer(_put_annotations_batch, annotation_data)
gcs_file.close()
except Exception as e:
logging.error(e)
raise deferred.PermanentTaskFailure()
def post(self):
trace_uuid = str(uuid.uuid4())
bucket_name = ('/performance-insights/' + trace_uuid)
gcs_file = gcs.open(bucket_name,
'w',
content_type='application/octet-stream',
options={},
retry_params=default_retry_params)
gcs_file.write(self.request.get('trace'))
gcs_file.close()
trace_object = trace_info.TraceInfo(id=trace_uuid)
trace_object.prod = self.request.get('prod')
trace_object.network_type = self.request.get('network_type')
trace_object.remote_addr = os.environ["REMOTE_ADDR"]
tags_string = self.request.get('tags')
if re.match('^[a-zA-Z0-9,]+$', tags_string): # ignore non alpha-numeric tags
trace_object.tags = tags_string.split(',')
trace_object.user_agent = self.request.headers.get('User-Agent')
trace_object.ver = self.request.get('product_version')
trace_object.put()
self.response.write(trace_uuid)
def post(self):
trace_uuid = str(uuid.uuid4())
bucket_name = ('/performance-insights/' + trace_uuid)
gcs_file = gcs.open(bucket_name,
'w',
content_type='application/octet-stream',
options={},
retry_params=default_retry_params)
gcs_file.write(self.request.get('trace'))
gcs_file.close()
trace_object = trace_info.TraceInfo(id=trace_uuid)
trace_object.prod = self.request.get('prod')
trace_object.network_type = self.request.get('network_type')
trace_object.remote_addr = os.environ["REMOTE_ADDR"]
tags_string = self.request.get('tags')
if re.match('^[a-zA-Z0-9,]+$',
tags_string): # ignore non alpha-numeric tags
trace_object.tags = tags_string.split(',')
trace_object.user_agent = self.request.headers.get('User-Agent')
trace_object.ver = self.request.get('product_version')
trace_object.put()
self.response.write(trace_uuid)
def GatherPageData(self, mr):
"""Parse the attachment ID from the request and serve its content.
Args:
mr: commonly used info parsed from the request.
Returns:
Dict of values used by EZT for rendering almost the page.
"""
with mr.profiler.Phase('get issue, comment, and attachment'):
try:
attachment, issue = tracker_helpers.GetAttachmentIfAllowed(
mr, self.services)
except exceptions.NoSuchIssueException:
webapp2.abort(404, 'issue not found')
except exceptions.NoSuchAttachmentException:
webapp2.abort(404, 'attachment not found')
except exceptions.NoSuchCommentException:
webapp2.abort(404, 'comment not found')
content = []
if attachment.gcs_object_id:
bucket_name = app_identity.get_default_gcs_bucket_name()
full_path = '/' + bucket_name + attachment.gcs_object_id
logging.info("reading gcs: %s" % full_path)
with cloudstorage.open(full_path, 'r') as f:
content = f.read()
filesize = len(content)
# This servlet only displays safe textual attachments. The user should
# not have been given a link to this servlet for any other kind.
if not attachment_helpers.IsViewableText(attachment.mimetype,
filesize):
self.abort(400, 'not a text file')
u_text, is_binary, too_large = filecontent.DecodeFileContents(content)
lines = prettify.PrepareSourceLinesForHighlighting(
u_text.encode('utf8'))
config = self.services.config.GetProjectConfig(mr.cnxn, mr.project_id)
granted_perms = tracker_bizobj.GetGrantedPerms(issue,
mr.auth.effective_ids,
config)
page_perms = self.MakePagePerms(mr,
issue,
permissions.DELETE_ISSUE,
permissions.CREATE_ISSUE,
granted_perms=granted_perms)
page_data = {
'issue_tab_mode': 'issueDetail',
'local_id': issue.local_id,
'filename': attachment.filename,
'filesize': template_helpers.BytesKbOrMb(filesize),
'file_lines': lines,
'is_binary': ezt.boolean(is_binary),
'too_large': ezt.boolean(too_large),
'code_reviews': None,
'page_perms': page_perms,
}
if is_binary or too_large:
page_data['should_prettify'] = ezt.boolean(False)
else:
page_data.update(
prettify.BuildPrettifyData(len(lines), attachment.filename))
return page_data