def GetGoogleStorage(self): """ Used for App Engine HTTP requests. """ version_key = blobstore.create_gs_key(GS_VERSION) blob_reader = blobstore.BlobReader(version_key) version = blob_reader.read() path = GS_PATH + version + self.request.path[len('docs/'):] gs_key = blobstore.create_gs_key(path) self.HandleCacheAge(path) self.send_blob(gs_key)
def post(self): # get the info about the uploaded images upload_files_front = self.get_file_infos('front_image') upload_files_ing = self.get_file_infos('ingredients_image') if not len(upload_files_front) or not len(upload_files_ing): self.response.out.write('Images not uploaded') return front_image = upload_files_front[0] ingredients_image = upload_files_ing[0] # check whether the image has been uploaded if not front_image.gs_object_name or not ingredients_image.gs_object_name: self.response.out.write( 'Image info not found. Seems to be upload error!') return ingredients = self.request.get('ingredients') if not ingredients: self.response.out.write( 'You must provide a list of ingredients for the product to be created.' ) return user = User.load(users.get_current_user()) product = Product.create_by_barcode(self.PRODUCT_BARCODE, user) product.name = self.PRODUCT_NAME product.put() Image.create(creator=user, product=product, blob=blobstore.create_gs_key(front_image.gs_object_name), image_type=ImageType.Front, ocr_result='', featured=True) Image.create(creator=user, product=product, blob=blobstore.create_gs_key( ingredients_image.gs_object_name), image_type=ImageType.Ingredient, ocr_result=ingredients, featured=False) for ingredient_name in ingredients.splitlines(): ingredient = Ingredient.find_by_name(ingredient_name) if ingredient: ProductIngredient.add_entry(product, ingredient, user) self.redirect('/utils')
def ler_arquivo_dict(filename): blobstore_filename = '/gs/{0}/{1}'.format(bucket_identity, filename) blob_key = blobstore.create_gs_key(blobstore_filename) blob_reader = blobstore.BlobReader(blob_key) conteudo = blob_reader.read() dict = xmltodict.parse(conteudo, process_namespaces=True) return dict
def update_response_headers(self, request_headers, response_headers): try: # cloudstorage.stat doesn't use "/gs" prefix. gs_object_name = self.gs_object_name[3:] stat = cloudstorage.stat(gs_object_name) except cloudstorage.errors.NotFoundError as e: raise FileNotFoundError(str(e)) headers = {} time_obj = datetime.fromtimestamp(stat.st_ctime).timetuple() headers['Last-Modified'] = time.strftime('%a, %d %b %Y %H:%M:%S GMT', time_obj) headers['ETag'] = '"{}"'.format(stat.etag) if stat.content_type: headers['Content-Type'] = stat.content_type # The presence of "X-AppEngine-BlobKey" tells App Engine that we want to # serve the GCS blob directly to the user. This avoids reading the blob data # into the App Engine application. If the user has the file cached already, # omit the X-AppEngine-BlobKey header since we want to serve an empty response # with a 304 status code. request_etag = request_headers.get('If-None-Match') if request_etag != headers['ETag']: key = blobstore.create_gs_key(self.gs_object_name) headers['X-AppEngine-BlobKey'] = key response_headers.update(headers)
def post(self): image = self.request.get('image') name = self.request.get('name') content_type = self.request.get('contentType') bucket_name = generate_bucket_filename(name) with gcs.open(bucket_name, 'w', content_type=content_type) as f: f.write(image) blobstore_filename = generate_blobstore_filename(name) blob_key = blobstore.create_gs_key(blobstore_filename) restaurant_image_url = images.get_serving_url(blob_key) image_model = ImageModel(name=name, bucket_name=bucket_name, blob_key=blob_key) image_key = image_model.put() self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ 'url': restaurant_image_url, 'name': name, 'mimetype': content_type, 'id': image_key.id() }))
def post(self): image_file = self.request.POST.get("pic", None) file_obj = self.request.get("pic", None) if not isinstance(image_file, cgi.FieldStorage): return json_response(self.response, {}, ERROR, 'Select image file') key = self.request.get('key') e = ndb.Key(urlsafe=key).get() file_name = image_file.filename bucket_path = '/designer_frames/%s' % (file_name) bucket_path = bucket_path.lower() serving_url = '' upload_file(file_obj, bucket_path) try: logging.info('create_gs_key') bucket_key = blobstore.create_gs_key('/gs' + bucket_path) logging.info(bucket_key) logging.info('serving_url') if self.DEV: serving_url = images.get_serving_url(bucket_key) else: file_name = file_name.replace(' ', '%20').lower() serving_url = 'https://storage.googleapis.com/designer_frames/%s' % ( file_name) logging.info(serving_url) except Exception, msg: logging.error(msg) return json_response(self.response, {}, WARNING, 'Try again')
def save_to_gcs(file_obj): serving_url = ''#just assign it adn reassign later time_stamp = int(time.time()) app_id = app_identity.get_application_id() fname = '/%s.appspot.com/post_%s.jpg' % (app_id, time_stamp) # logging.error(fname) # Content Types # audio/mpeg # image/jpeg gcs_file = gcs.open(fname, 'w', content_type="image/jpeg") gcs_file.write(file_obj) gcs_file.close() height = images.Image(image_data=file_obj).height width = images.Image(image_data=file_obj).width gcs_filename = "/gs%s" % fname serving_url = images.get_serving_url(blobstore.create_gs_key(gcs_filename)) media_obj = save_gcs_to_media(gcs_filename, serving_url) return media_obj
def get(self): uuid = self.request.get('uuid') size = self.request.get('size') size = int(size) if size else config.PROFILE_ICON_SIZE if not uuid: api.write_error(self.response, 400, 'Unknown or missing user') return filename = config.PROFILE_BUCKET + uuid image = images.Image(blob_key=blobstore.create_gs_key('/gs' + filename)) image.resize(width=size, height=size, crop_to_fit=True, allow_stretch=False) try: png_data = StringIO.StringIO(image.execute_transforms(output_encoding=images.PNG)) except: self.redirect(config.DEFAULT_PHOTO_URL) return im = Image.open(png_data) bigsize = (im.size[0] * 3, im.size[1] * 3) mask = Image.new('L', bigsize, 0) draw = ImageDraw.Draw(mask) draw.ellipse((0, 0) + bigsize, fill=255) mask = mask.resize(im.size, Image.ANTIALIAS) im.putalpha(mask) output = StringIO.StringIO() im.save(output, 'PNG') self.response.headers['Content-Type'] = 'image/png' self.response.headers['Cache-Control'] = 'public,max-age=%d' % (config.MEDIA_MAX_AGE) self.response.out.write(output.getvalue()) output.close()
def saveImageInGCS(image_data): # ====================== # Save file in GCS # ====================== image_data = base64.b64decode(image_data) #image_data.encode('utf-8') bucket_name = os.environ.get('BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) bucket = '/' + bucket_name filename = bucket + '/' + getImageHash(image_data) + '.png' write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open(filename, 'w', content_type='image/png', options={ 'x-goog-meta-foo': 'foo', 'x-goog-meta-bar': 'bar' }, retry_params=write_retry_params) gcs_file.write(image_data) gcs_file.close() gcs_object_name = '/gs' + filename # logging.info(gcs_object_name) blob_key = blobstore.create_gs_key(gcs_object_name) image_url = images.get_serving_url(blob_key) return image_url
def FAQCategoryUpdate(self, faqcat): """ update faqcat""" if self.is_current_user_admin(): print(faqcat.id) if faqcat.id != None: item = ndb.Key(urlsafe=faqcat.id).get() if item is None: raise endpoints.BadRequestException("FAQCategory not found") else: if faqcat.image != None: image = faqcat.image if len(image) > 6*1024*1024: raise endpoints.BadRequestException("Max. image size is 6*1024*1024 bytes") write_retry_params = gcs.RetryParams(backoff_factor=1.1) filename = "/" + BUCKET_NAME + "/" +str(uuid.uuid4()) png = images.rotate(image, 0, output_encoding=images.PNG) gcs_file = gcs.open(filename,'w',retry_params=write_retry_params,content_type='image/png',) gcs_file.write(png) gcs_file.close() blobkey = blobstore.create_gs_key("/gs" + filename) url = images.get_serving_url(blobkey) item.image_url = url item.blobkey = filename del faqcat.image item.put() return item else: raise endpoints.BadRequestException("ID missing") else: raise endpoints.UnauthorizedException("Only Volunteers users can update FAQCategory. \ Contact [email protected] for more information.")
def OfferInsert(self, offer): """ Created create offer""" user = self.get_current_user() offer.owner_key = user.key urls = [] blobkeys = [] for image in offer.images: if len(image) > 6*1024*1024: for blobkey in blobkeys: gcs.delete(blobkey) raise endpoints.BadRequestException("Max. image size is 6*1024*1024 bytes") write_retry_params = gcs.RetryParams(backoff_factor=1.1) filename = "/" + BUCKET_NAME + "/" +str(uuid.uuid4()) png = images.rotate(image, 0, output_encoding=images.PNG) gcs_file = gcs.open(filename,'w',retry_params=write_retry_params,content_type='image/png',) gcs_file.write(image) gcs_file.close() blobkey = blobstore.create_gs_key("/gs" + filename) blobkeys.append(filename) #url = images.get_serving_url("gs" + filename) url = images.get_serving_url(blobkey) urls.append(url) offer.image_urls = urls offer.blobkeys = blobkeys del offer.images offer.put() return offer
def post(self): try: #Retrieve the image from http submit. It will be a blob object. image = self.get_uploads()[0] #Get info from blobfile uploaded file_info = self.get_file_infos()[0] #Create the serving url for the blob uploaded into GS (Google cloud storage) serving_url = images.get_serving_url(blobstore.create_gs_key(file_info.gs_object_name)) #Fetch image dimensions. The last integer specifies how much to fetch 50000 is plenty. data = blobstore.fetch_data(image.key(), 0, 50000) img = images.Image(image_data = data) width_to_height_ratio = float(img.width)/img.height #Get name and description name = self.request.get('name') caption = self.request.get('caption') #Now generate the database object and store it. image_model = ImageModel( name = name, caption = caption, blob_key = image.key(), gs_object_name = file_info.gs_object_name, serving_url = serving_url, width_to_height_ratio = width_to_height_ratio) image_model.put() #Redirect to the upload site and tell it that it succeeded. self.redirect('/admin/upload_image#success') except: #Redirect to upload site and tell that something went wrong. self.redirect('/admin/upload_image#fail')
def migrate(image): """Copies blobs stored in Blobstore over to a GCS bucket. Args: image: main.Image instance representing a single entity in the Datastore. This does not delete migrated (old) blobs so it is safe to run the job multiple times. """ if image.blob_key and not image.gs_key: blob_info = blobstore.get(image.blob_key) if not blob_info: image.blob_key = None else: gs_key = '/'.join(['', BUCKET, blob_info.filename]) try: gcs.stat(gs_key) except gcs.NotFoundError: reader = blobstore.BlobReader(blob_info) with gcs.open(gs_key, 'w', content_type=blob_info.content_type) as f: while True: data = reader.read(1024**2) if not data: break f.write(data) blob_gs_key = blobstore.create_gs_key('/gs'+gs_key) image.url = images_api.get_serving_url(blob_gs_key, secure_url=True) image.gs_key = gs_key yield op.db.Put(image) if image.gs_key: yield op.counters.Increment('Migrated')
def _create_note(self, user, title, content, attachments): note = Note(parent=ndb.Key("User", user.nickname()), title=title, content=content) note.put() if attachments: bucket_name = app_identity.get_default_gcs_bucket_name() for file_name, file_content in attachments: content_t = mimetypes.guess_type(file_name)[0] real_path = os.path.join('/', bucket_name, user.user_id(), file_name) with cloudstorage.open(real_path, 'w', content_type=content_t, options={'x-goog-acl': 'public-read'}) as f: f.write(file_content.decode()) key = blobstore.create_gs_key('/gs' + real_path) try: url = images.get_serving_url(key, size=0) thumbnail_url = images.get_serving_url(key, size=150, crop=True) except images.TransformationError, images.NotImageError: url = "http://storage.googleapis.com{}".format(real_path) thumbnail_url = None f = NoteFile(parent=note.key, name=file_name, url=url, thumbnail_url=thumbnail_url, full_path=real_path) f.put() note.files.append(f.key) note.put()
def url(self, filename): if not is_hosted(): key = blobstore.create_gs_key('/gs' + self._real_path(filename)) return images.get_serving_url(key) return 'https://storage.googleapis.com{path}'.format( path=self._real_path(filename) )
def serveurl(): config = ConfigParser.RawConfigParser(allow_no_value=True) config.read('config.ini') try: expectedKey = config.get("auth", "key") receivedKey = request.form['key'] except: return json.dumps({'error': 'Key error'}), 401, { 'ContentType': 'application/json' } if expectedKey == receivedKey: image = request.form['image'] bucket = request.form['bucket'] logging.info('Create Serving URL for ' + image) filename = (bucket + "/" + image) logging.info('Filename is ' + filename) gskey = blobstore.create_gs_key("/gs/" + filename) logging.info('gskey is ' + gskey) servingImage = images.get_serving_url(gskey, secure_url=True) logging.info('Serving url: ' + servingImage) return (servingImage) else: return json.dumps({'error': 'No valid key provided'}), 401, { 'ContentType': 'application/json' }
def CreateFile(self, nombre, datos): my_default_retry_params = gcs.RetryParams(initial_delay=0.2, max_delay=5.0, backoff_factor=2, max_retry_period=15) gcs.set_default_retry_params(my_default_retry_params) bucket_name = os.environ.get('BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) #bucket_name = os.environ.get('BUCKET_NAME', 'prueba') #print bucket_name #bucket_name = 'prueba' bucket = '/' + bucket_name filename = bucket + '/' + nombre print 'filename: '+filename #https://cloud.google.com/appengine/docs/python/googlecloudstorageclient/functions write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open(filename, 'w', content_type='image/jpeg', options={'x-goog-meta-foo': 'foo', 'x-goog-meta-bar': 'bar', 'x-goog-acl': 'public-read'}, retry_params=write_retry_params) gcs_file.write(datos) gcs_file.close() blobstore_filename = '/gs' + filename key = blobstore.create_gs_key(blobstore_filename) #Si se encuentra en el servidor de Google if os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/'): return 'http://storage.googleapis.com'+filename #Si está en entorno de desarrollo local: else: return get_serving_url(key)
def CreateFile(my_file,tienda): """Create a GCS file with GCS client lib. Args: filename: GCS filename. Returns: The corresponding string blobkey for this GCS file. """ # Create a GCS file with GCS client. #with gcs.open(filename, 'w') as f: # f.write(my_file.file.read()) nombre ='/gs/'+BUCKET +'/'+str(tienda)+"/"+my_file.filename #write_retry_params = gcs.RetryParams(backoff_factor=1.1) #with gcs.open(my_file.filename, 'w') as f: # f.write(my_file.file.read()) gcs_file =files.gs.create(nombre,mime_type=my_file.type,acl='public-read') #, retry_params=write_retry_paramsretry_params=write_retry_params with files.open(gcs_file, 'a') as f: f.write(my_file.file.read()) files.finalize(gcs_file) # Blobstore API requires extra /gs to distinguish against blobstore files. #blobstore_filename = '/gs' + nombre # This blob_key works with blobstore APIs that do not expect a # corresponding BlobInfo in datastore. return blobstore.create_gs_key(nombre)
def get(self): """Loads the data file located at |filename|. Args: filename: The filename for the data file to be loaded. """ yesterday = datetime.date.today() - datetime.timedelta(1) yesterday_formatted = yesterday.strftime("%Y.%m.%d") filename = 'histograms/daily/%s/Everything' % (yesterday_formatted) if settings.PROD: try: with files.open(BIGSTORE_BUCKET + filename, 'r') as unused_f: pass except files.file.ExistenceError, e: self.response.write(e) return # The file exists; serve it. blob_key = blobstore.create_gs_key(BIGSTORE_BUCKET + filename) blob_reader = blobstore.BlobReader(blob_key, buffer_size=3510000) try: result = blob_reader.read() finally: blob_reader.close()
def gcs_upload(file_to_upload, bucket=settings.GCS_BUCKET): """Uploads a file to google cloud storage.""" try: filename = create_random_gcs_filename() bucket_and_filename = {'bucket': bucket, 'filename': filename} gcs_filename = '/%(bucket)s/%(filename)s' % bucket_and_filename public_url = 'https://storage.googleapis.com/%(bucket)s/%(filename)s' % bucket_and_filename gcs_file = gcs.open( gcs_filename, mode='w', content_type=file_to_upload.type, options={'x-goog-acl': 'public-read'}) gcs_file.write(file_to_upload.value) gcs_file.close() gs_key = blobstore.create_gs_key('/gs' + gcs_filename) try: serving_url = images.get_serving_url(gs_key) except: serving_url = public_url return { 'gs_key': gs_key, 'gcs_filename': gcs_filename, 'serving_url': serving_url, 'public_url': public_url, 'filename': filename, 'filetype': file_to_upload.type } except Exception, e: logging.exception(e) return None
def editormd_image_upload(): mimetypes.init() if 'editormd-image-file' not in request.files: return jsonify({"success": 0, "message": u"No file part"}) file = request.files['editormd-image-file'] if file.filename == '': return jsonify({"success": 0, "message": u"No selected file"}) if file and allowed_file(file.filename): directory = "upload/{0}".format(datetime.now().strftime("%Y%m%d/%H")) bucket_name = os.environ.get( 'BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) bucket = '/' + bucket_name filename = "{0}/{3}/{1}.{2}".format(bucket, slugify(file.filename.rsplit('.', 1)[0]).replace("-", "_"), file.filename.rsplit('.', 1)[1], directory) content_type = mimetypes.guess_type(filename)[0] or "application/octet-stream" write_retry_params = cloudstorage.RetryParams(backoff_factor=1.1) gcs_file = cloudstorage.open(filename, 'w', content_type=content_type, options={'x-goog-acl': 'public-read'}, retry_params=write_retry_params) gcs_file.write(file.read()) gcs_file.close() gs = "/gs{0}".format(filename) blob_key = blobstore.create_gs_key(gs) url = images.get_serving_url(blob_key, size=app.config["SITE_POST_IMG_WIDTH"], crop=False, secure_url=True) return jsonify({"success": 1, "message": u"No allowed_file", "url": url}) return jsonify({"success": 0, "message": u"No allowed_file"})
def process_images_create(self, request, context, *args, **kwargs): """ Upload a new image to the Artwork Gallery """ from merkabah.core.files.api.cloudstorage import Cloudstorage from google.appengine.ext import blobstore from plugins.artwork.internal.api.images import create_media # Get the file upload url fs = Cloudstorage(DEFAULT_GS_BUCKET_NAME) form = ImageUploadForm() context['form'] = form has_files = fs.get_uploads(request, 'the_file', True) if has_files: file_info = has_files[0] original_filename = file_info.filename content_type = file_info.content_type size = file_info.size gs_object_name = file_info.gs_object_name # Using this we could urlfetch, but the file isn't public... blob_key = blobstore.create_gs_key(gs_object_name) logging.warning(blob_key) data = fs.read(gs_object_name.replace('/gs', '')) slug = original_filename.split('.')[0] # I dislike this.. media = create_media(slug, data) # What we want to do now is create a copy of the file with our own info dest_filename = '%s' % original_filename new_gcs_filename = fs.write(dest_filename, data, content_type) logging.warning(new_gcs_filename) # Finally delete the tmp file data = fs.delete(gs_object_name.replace('/gs', '')) if not kwargs.get('give_me_json_back'): return HttpResponseRedirect(urlresolvers.reverse('admin_plugin_action', args=(context['plugin_slug'], 'images'))) # Else... we're hacky emulating an upload rest endpoint - return json info about the image response_dict = { 'cool': True, 'keystr': media.key.urlsafe(), 'filename': media.filename, 'thumbnail_url': media.get_thumbnail_url() } return HttpResponse(json.dumps(response_dict)) upload_url = fs.create_upload_url('/madmin/plugin/artwork/images_create/') return FormResponse(form, id='images_create_form', title="Upload a file", target_url=upload_url, target_action='images_create', is_upload=True)
def finish(self): """ Called when all shards have finished processing """ if self.get_meta().generate_error_csv: self.error_csv_filename = self._error_csv_filename() with cloudstorage.open(self.error_csv_filename, 'w') as f: # Concat all error csvs from shards into 1 file has_written = False for shard in ImportShard.objects.filter(task_id=self.pk, task_model_path=self.model_path): if not shard.error_csv_filename: continue # If this is the first row, write the column headers if not has_written: data = json.loads(shard.source_data_json)[0] cols = getattr(self, "detected_columns", sorted(data.keys())) + [ "errors" ] csvwriter = csv.writer(f) csvwriter.writerow(cols) has_written = True # Write the shard's error file into the master file f.write(cloudstorage.open(shard.error_csv_filename).read()) cloudstorage.delete(shard.error_csv_filename) if has_written: # Create a blobstore key for the GCS file blob_key = create_gs_key('/gs%s' % self.error_csv_filename) self.error_csv = '%s/errors.csv' % blob_key self.save() else: cloudstorage.delete(self.error_csv_filename)
def CreateFile(my_file,tienda): """Create a GCS file with GCS client lib. Args: filename: GCS filename. Returns: The corresponding string blobkey for this GCS file. """ # Create a GCS file with GCS client. #with gcs.open(filename, 'w') as f: # f.write(my_file.file.read()) nombre = '/'+BUCKET +'/'+str(tienda)+"/"+my_file.filename #write_retry_params = gcs.RetryParams(backoff_factor=1.1) #with gcs.open(my_file.filename, 'w') as f: # f.write(my_file.file.read()) gcs_file = gcs.open(nombre,'w',content_type=my_file.type,options={'x-goog-acl': 'public-read'}) #, retry_params=write_retry_paramsretry_params=write_retry_params gcs_file.write(my_file.file.read()) gcs_file.close() # Blobstore API requires extra /gs to distinguish against blobstore files. blobstore_filename = '/gs' + nombre # This blob_key works with blobstore APIs that do not expect a # corresponding BlobInfo in datastore. return blobstore.create_gs_key(blobstore_filename)
def delete(self): """Delete the original file and dynamic serving url if it exists """ filepath = request.args.get('filepath') if not filepath: return make_response_validation_error('filepath', message='Parameter filepath is required') try: cloudstorage.delete(filename) except cloudstorage.AuthorizationError: abort_json(401, "Unauthorized request has been received by GCS.") except cloudstorage.ForbiddenError: abort_json(403, "Cloud Storage Forbidden Error. GCS replies with a 403 error for many reasons, the most common one is due to bucket permission not correctly setup for your app to access.") except cloudstorage.NotFoundError: abort_json(404, filepath + " not found on GCS in bucket " + self.bucket) except cloudstorage.TimeoutError: abort_json(408, 'Remote timed out') # TODO get the query string and delete file if asked to blobstore_filename = u'/gs/{}/{}'.format(bucket_name, filepath) blob_key = blobstore.create_gs_key(blobstore_filename) try: images.delete_serving_url(blob_key) except images.AccessDeniedError: abort_json(403, "App Engine Images API Access Denied Error. Files has already been deleted from Cloud Storage") except images.ObjectNotFoundError: pass return '', 204
def fetch_cached_thumbnail(cls, cached_key): blob_info = blobstore.get( blobstore.create_gs_key("/gs{0}".format(cached_key))) if blob_info: with blob_info.open() as f_blob: return f_blob.read()
def post(self, category, name): """Saves a resource in the cloud storage Multiple files are possible, if multiple files are uploaded the 'name' needs to be 'multiple'. For multiple files the file name is take as name. If multiple fils are uploaded without 'multiple' only the last file is saved. The function can also gerenate a serving link, this is either public or private (not guessable). If category or name (without extension) is a user key it needs to belong to the loged in user. """ link = request.form.get( 'link', default='private') # either public, private or False gcs_links = [] api_links = [] private_links = [] links = [] print "Category: " + str(category) try: category_key = ndb.Key(urlsafe=category) except: category_key = False for k, f in request.files.iteritems(multi=False): if name == 'multiple': name = f.filename try: name_key = ndb.Key( urlsafe=os.path.splitext(os.path.basename(name))[0]) except: name_key = False if category_key or name_key: user_key = category_key or name_key if not auth.is_authorized(user_key): return abort(403) write_retry_params = gcs.RetryParams(backoff_factor=1.1) adr = "{}/{}/{}".format(GCS_BUCKET, category, name) gcs_file = gcs.open(adr, 'w', content_type=f.mimetype, options={'x-goog-meta-name': f.filename}, retry_params=write_retry_params) f.save(gcs_file) # saves file to cloud storage gcs_file.close() f.close() gcs_links.append("/_ah/gcs" + adr) api_links.append("/api/v1/upload/" + category + '/' + name) links.append("/resource/" + '/' + category + '/' + name) if link == 'private': #TODO implement public links blob_key = blobstore.create_gs_key('/gs' + adr) img_url = images.get_serving_url(blob_key=blob_key) private_links.append(img_url) return { 'links': links, 'private_links': private_links, 'gcs_links': gcs_links, 'api_links': api_links }
def get(self, category, name): """Returns a link to an uploaded resource""" # TODO permissions, how, what? adr = GCS_BUCKET + '/' + category + '/' + name blob_key = blobstore.create_gs_key('/gs' + adr) img_url = images.get_serving_url(blob_key=blob_key) return img_url
def upload_bucket_images(): if dict(request.files) == {}: return jsonify( user_message = 'no image' ), 401 files = request.files bucket_id = request.args['bucketId'] bucket_images = [] for image_file in files: image = request.files[image_file] image_params = upload_image(image.read(), 'bucket', image.mimetype) storage_key = image_params[0] storage_url = image_params[1] gs_key = blobstore.create_gs_key('/gs' + storage_key) serving_url = images.get_serving_url(gs_key) created_image = BucketImage(serving_url=serving_url, storage_url=storage_url, bucket_id=bucket_id) db.session.add(created_image) db.session.commit() bucket_images.append(model_to_dict(created_image)) return jsonify( data=bucket_images )
def save_file(file_hash, file_content): blob_filename = '/%s/%s/%s' % (app_identity.get_default_gcs_bucket_name(), 'ics', file_hash) with gcs.open(blob_filename, 'w') as f: f.write(file_content) blob_store_filename = '/gs' + blob_filename return blobstore.create_gs_key(blob_store_filename)
def _create_branding(hash_, zip_content, description, service_user, branding_type, meta_properties, pokes): if pokes and not service_user: raise BrandingValidationException('Cannot create branding with one or more poke tags without a service user') filename = get_branding_cloudstorage_path(hash_, service_user) with cloudstorage.open(filename, 'w') as f: f.write(zip_content) blobstore_filename = '/gs' + filename blobstore_key = blobstore.create_gs_key(blobstore_filename) b = Branding(key_name=hash_) b.blob_key = blobstore_key.decode('utf-8') b.description = description b.timestamp = now() b.user = service_user b.pokes = list() b.type = branding_type color_scheme = meta_properties.get('color-scheme') or Branding.DEFAULT_COLOR_SCHEME b.menu_item_color = meta_properties.get('menu-item-color') or Branding.DEFAULT_MENU_ITEM_COLORS[color_scheme] b.content_type = meta_properties.get('content-type') or Branding.CONTENT_TYPE_HTML b.orientation = meta_properties.get('orientation') or Branding.DEFAULT_ORIENTATION puts = [b] for poke_hash, unicode_tag in pokes: ptm = PokeTagMap(key_name=poke_hash, parent=parent_key(service_user)) ptm.tag = unicode_tag b.pokes.append(ptm.key()) puts.append(ptm) db.put(puts) return b
def get(self): # Get the default Cloud Storage Bucket name and create a file name for # the object in Cloud Storage. bucket = app_identity.get_default_gcs_bucket_name() # Cloud Storage file names are in the format /bucket/object. filename = '/{}/blobstore_demo'.format(bucket) # Create a file in Google Cloud Storage and write something to it. with cloudstorage.open(filename, 'w') as filehandle: filehandle.write('abcde\n') # In order to read the contents of the file using the Blobstore API, # you must create a blob_key from the Cloud Storage file name. # Blobstore expects the filename to be in the format of: # /gs/bucket/object blobstore_filename = '/gs{}'.format(filename) blob_key = blobstore.create_gs_key(blobstore_filename) # Read the file's contents using the Blobstore API. # The last two parameters specify the start and end index of bytes we # want to read. data = blobstore.fetch_data(blob_key, 0, 6) # Write the contents to the response. self.response.headers['Content-Type'] = 'text/plain' self.response.write(data) # Delete the file from Google Cloud Storage using the blob_key. blobstore.delete(blob_key)
def upload_image(self, blob, filename): mime_type = 'image/png' if filename.split('.')[-1] == 'jpg' or filename.split( '.')[-1] == 'jpeg': mime_type = 'image/jpeg' bucket_name = os.environ.get( 'BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) bucket = '/' + bucket_name filename_final = bucket + '/' + str(uuid.uuid4()) # Create a GCS file with GCS client. with gcs.open(filename_final, 'w') as f: f.write(blob) f.close() # Blobstore API requires extra /gs to distinguish against blobstore files. blobstore_filename = '/gs' + filename_final # Get the file's blob key blob_key = blobstore.create_gs_key(blobstore_filename) # Store it self.image = blob_key self.mime_type = mime_type self.url = get_serving_url(blob_key)
def get(self): path = self.resolve_doc_path() gs_key = blobstore.create_gs_key(path) age = self.get_cache_age(path) self.response.headers['Cache-Control'] = 'max-age=' + \ str(age) + ',s-maxage=' + str(age) # is there a better way to check if a file exists in cloud storage? # AE will serve a 500 if the file doesn't exist, but that should # be a 404 path_exists = memcache.get(path) if path_exists is not None: if path_exists == "1": self.send_blob(gs_key) else: self.error(404) else: try: # just check for existance files.open(path, 'r').close() memcache.add(key=path, value="1", time=ONE_DAY) self.send_blob(gs_key) except files.file.ExistenceError: memcache.add(key=path, value="0", time=ONE_DAY) self.error(404)
def get(self): """Loads the data file located at |filename|. Args: filename: The filename for the data file to be loaded. """ yesterday = datetime.date.today() - datetime.timedelta(1) yesterday_formatted = yesterday.strftime("%Y.%m.%d") filename = 'histograms/daily/%s/Everything' % (yesterday_formatted) if settings.PROD: try: with cloudstorage.open(BIGSTORE_BUCKET + filename, 'r') as unused_f: pass except cloudstorage.errors.Error, e: logging.error(e) self.response.write(e) return # The file exists; serve it. blob_key = blobstore.create_gs_key('/gs' + BIGSTORE_BUCKET + filename) blob_reader = blobstore.BlobReader(blob_key, buffer_size=3510000) try: result = blob_reader.read() finally: blob_reader.close()
def get(self, user_id, practice_id, gcs_object): # Build the GCS path for this object, which is based on bucket, user, # and object. gs_object_name = '/gs/{}/{}/{}'.format(util.get_upload_bucket(), user_id, gcs_object) # Although this inherits from webapp's "Blobstore" handler, the files # actually reside in Google Cloud Storage. That's why we convert from # the gcs file name. blob_key = blobstore.create_gs_key(gs_object_name) # Look up the human-readable file name in the file info data of the # practice. See the UploadFiles handlers for details on what else is in # the file_info dictionary. practice = Practice.get_by_id(practice_id) filename = None for file_info in practice.json_properties['files']: if gs_object_name == file_info['gs_object_name']: filename = file_info['filename'] if filename is None: raise Exception("Could not find file in practice: {} {}".format( practice_id, gcs_object)) # Attach headers that make the file 1) immediately download rather than # opening in the browser and 2) have a pretty file name. self.response.headers['Content-Disposition'] = ( "attachment; filename=" + str(filename)) self.send_blob(blob_key)
def get(self, key): server = self.get_server_by_key(key) blobstore_filename = "/gs/{0}/{1}".format( gcs.get_default_bucket_name(), gcs.get_gcs_archive_name(server.key.urlsafe()) ) blob_key = blobstore.create_gs_key(blobstore_filename) self.send_blob(blob_key, save_as="{0}.zip".format(server.short_name or server.name or server.key.urlsafe()))
def upload_watermarked_image(image, job_id, new_url): # gets the information of the new url filename, content_type, extension = get_file_info(new_url) gcs_filename = 'watermarked/%s/%s' % (job_id, filename) watermarked_file = '/%s/%s' % (BUCKET_NAME, gcs_filename) new_url = PHOTOS_ROOT_URL + '/' + gcs_filename # render the image as a string output = StringIO.StringIO() image.save(output, format="jpeg") image_output = output.getvalue() output.close() # upload the file to google cloud storage retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open(watermarked_file, 'w', content_type=content_type, options={'x-goog-acl': 'public-read'}) gcs_file.write(image_output) gcs_file.close() query_file = '/gs' + watermarked_file pprint.pprint('looking for the key: %s' % query_file) key = blobstore.create_gs_key(query_file) new_url = images.get_serving_url(key) pprint.pprint('key:') pprint.pprint(key) return new_url
def store_mapping_entity(old_blob_info_or_key, gcs_filename): """Store the mapping in Datastore. Args: old_blob_info_or_key: The old blob's BlobInfo, BlobKey, or BlobKey's encrypted string. gcs_filename: the GCS filenames where the blob was copied. Returns: The datastore mapping entity that was written. """ if not old_blob_info_or_key: raise ValueError('old_blob_info_or_key is required.') if not gcs_filename: raise ValueError('gcs_filename is required.') if not gcs_filename.startswith('/'): gcs_filename = '/' + gcs_filename old_blob_key_str = _get_blob_key_str(old_blob_info_or_key) new_blob_key_str = blobstore.create_gs_key('/gs' + gcs_filename) kwargs = { 'key': models.BlobKeyMapping.build_key(old_blob_key_str), 'gcs_filename': gcs_filename, 'new_blob_key': new_blob_key_str, } entity = models.BlobKeyMapping(**kwargs) entity.put() logging.info('Migrated blob_key "%s" to "%s" (GCS file "%s").' % (old_blob_key_str, new_blob_key_str, gcs_filename)) return entity
def CreateFile(filename, imageFile): with gcs.open(filename, 'w', content_type='image/jpeg') as f: f.write(imageFile) f.close() blobstore_filename = '/gs' + filename return blobstore.create_gs_key(blobstore_filename)
def unserveurl(): config = ConfigParser.RawConfigParser(allow_no_value=True) config.read('config.ini') expectedKey = config.get("auth", "key") receivedKey = request.form['key'] if expectedKey == receivedKey: image = request.form['image'] bucket = request.form['bucket'] logging.info('Remove Serving URL for ' + image) filename = (bucket + "/" + image) logging.info('Filename is ' + filename) gskey = blobstore.create_gs_key("/gs/" + filename) logging.info('gskey is ' + gskey) removal = images.delete_serving_url(gskey) logging.info('URL is removed') return ("OK") else: return json.dumps({'error': 'No valid key provided'}), 401, { 'ContentType': 'application/json' }
def get_resource(collection,category,name): # TODO auth.is_admin and collectio permission does not work !!! -> used True or if True or auth.is_admin() or model.Collection.has_permission(collection,auth.current_user_key().urlsafe(),'read',urlsafe=True): adr = BUCKET + '/' + collection + '/' + category + '/' + name blob_key = blobstore.create_gs_key('/gs' + adr) img_url = images.get_serving_url(blob_key=blob_key) print img_url return flask.redirect(img_url)
def img_in_cache(self): """ Returns True if the thumbnail image is in the cache """ return blobstore.get( blobstore.create_gs_key( "/gs/greenday-project-v02-local.appspot.com/gd-yt-thumbs/{yt_id}/ytthumb-{yt_id}-{ms}" .format(yt_id=self.yt_id, ms=self.at_milliseconds)))
def make_blob(nid, data): filename = '/{0}/{1}'.format(get_default_gcs_bucket_name(), nid) with gcs.open( filename, 'w', content_type='text/plain; charset=UTF-8' ) as fh: fh.write(data.encode('utf-8')) blobstore_filename = '/gs{0}'.format(filename) return blobstore.create_gs_key(blobstore_filename)
def create_file(project_db, name, data): filename = '/%s/%s/%s.dmp' % (config.CONFIG_DB.bucket_name, project_db.key.id(), name) with cloudstorage.open(filename, 'w', content_type='text/plain') as f: f.write(data) # Blobstore API requires extra /gs to distinguish against blobstore files. blobstore_filename = '/gs' + filename return ndb.BlobKey(blobstore.create_gs_key(blobstore_filename))
def get_gcs_image_serving_url(filename): """ Gets a serving URL from the images API for a file in GCS. This does not work locally """ gskey = blobstore.create_gs_key("/gs/" + filename) return images.get_serving_url(gskey, secure_url=True)
def get(self, resource): if DEBUG: filename = "{}/{}".format(_config.LOCAL, resource) self.response.write(open(filename).read()) return filename = "/gs/{}/{}".format(_config.BUCKET, resource) key = blobstore.create_gs_key(filename) self.send_blob(key)
def get(self, image_name): # pylint: disable=C0111 if not image_name: self.error(404) return blob_key = blobstore.create_gs_key('/gs' + config.BUCKET + '/' + image_name) img_url = images.get_serving_url(blob_key=blob_key) self.redirect(img_url)
def get(self, siteName, imageFile): filename = '/gs/' + siteName + '/webhook-uploads/' + imageFile key = blobstore.create_gs_key(filename) img = images.Image(blob_key=key) url = images.get_serving_url(key) self.response.out.write(url)
def get(self, siteName, imageFile): filename = '/gs/' + siteName + '/webhook-uploads/' + imageFile; key = blobstore.create_gs_key(filename) img = images.Image(blob_key=key) url = images.get_serving_url(key) self.response.out.write(url);
def post(self): # pylint: disable=g-bad-name """Handles Object Change Notifications.""" logging.debug( '%s\n\n%s', '\n'.join(['%s: %s' % x for x in self.request.headers.iteritems()]), self.request.body) resource_state = self.request.headers['X-Goog-Resource-State'] if resource_state == 'sync': logging.info('Sync OCN message received.') elif resource_state == 'exists': logging.info('New file upload OCN message received.') data = json.loads(self.request.body) bucket = data['bucket'] object_name = data['name'] # Get Image location in GCS. gcs_image_location = '/gs/%s/%s' % (bucket, object_name) blob_key = blobstore.create_gs_key(gcs_image_location) # Try and get username from metadata. if data.has_key('metadata') and data['metadata'].has_key('owner'): owner = data['metadata']['owner'] else: owner = data['owner']['entity'] # Try to get public image URL for entry creation. image_link = None try: image_link = images.get_serving_url(blob_key, secure_url=True) except images.ObjectNotFoundError: logging.error('Could not find image link for %s.', gcs_image_location) except images.TransformationError: logging.error('Could not convert link to image: %s.', gcs_image_location) if image_link: bitdoc = Bitdoc(user=owner, image_link=image_link, file_name=object_name) logging.info('Creating Entry... %s - %s', bitdoc.user, bitdoc.image_link) # timestamp auto. bitdoc.put() # Add Task to pull queue. info = {'key': unicode(bitdoc.key()), 'image_link': unicode(image_link)} processing_task_queue.add(taskqueue.Task(payload=json.dumps(info), method='PULL'))