def __init__(self, image_data=None, blob_key=None, filename=None): """Constructor. Only one of image_data, blob_key or filename can be specified. Args: image_data: str, image data in string form. blob_key: BlobKey, BlobInfo, str, or unicode representation of BlobKey of blob containing the image data. filename: str, the filename of a Google Storage file containing the image data. Must be in the format '/gs/bucket_name/object_name'. Raises: NotImageError if the given data is empty. """ if not image_data and not blob_key and not filename: raise NotImageError("Empty image data.") if image_data and (blob_key or filename): raise NotImageError("Can only take one of image, blob key or filename.") if blob_key and filename: raise NotImageError("Can only take one of image, blob key or filename.") self._image_data = image_data if filename: self._blob_key = blobstore.create_gs_key(filename) else: self._blob_key = _extract_blob_key(blob_key) self._transforms = [] self._width = None self._height = None self._format = None self._correct_orientation = UNCHANGED_ORIENTATION self._original_metadata = None
def url(self, name): if settings.DEBUG: # we need this in order to display images, links to files, etc from the local appengine server filename = "/gs"+self.location+"/"+name key = create_gs_key(filename) return "http://localhost:8000/blobstore/blob/"+key+"?display=inline" return self.base_url+"/"+name
def post(self): # or we updating or creating a new one item_key = self.request.get('item', None) is_new = (item_key is None) # get the info about the uploaded image upload_files = self.get_file_infos('image') if is_new and not len(upload_files): self.response.out.write( '<p>Image not uploaded, please try again!</p>') self.response.out.write('<p><a href="/guide">Return</a></p>') return # check whether the image has been uploaded if is_new and not upload_files[0].gs_object_name: self.response.out.write( '<p>Image info not found. Seems to be upload error! Please report this to an administrator.</p>' ) self.response.out.write('<p><a href="/guide">Return</a></p>') return text = self.request.get('text') if not text: self.response.out.write( 'You must provide the text for the guide. Please try again.') self.response.out.write('<p><a href="/guide">Return</a></p>') return user = User.load(users.get_current_user()) if is_new: # create the guide item AppGuide.create(user=user, image=blobstore.create_gs_key( upload_files[0].gs_object_name), text=text) else: image = None if upload_files and upload_files[0].gs_object_name: image = blobstore.create_gs_key(upload_files[0].gs_object_name) item = AppGuide.get_by_urlsafe_key(item_key) if item: item.update(editor=user, text=text, image=image) self.redirect('/guide')
def url(self, name): server_software = os.getenv("SERVER_SOFTWARE", "") if not server_software.startswith("Google App Engine"): # we need this in order to display images, links to files, etc # from the local appengine server filename = "/gs" + self.location + "/" + name key = create_gs_key(filename) local_base_url = getattr(settings, "GOOGLE_CLOUD_STORAGE_DEV_URL", "http://localhost:8001/blobstore/blob/") return local_base_url + key + "?display=inline" return self.base_url + "/" + name
def create_file(filename, contents, mime_type): filename = "/" + GCS_BUCKET + "/" + filename # Create a GCS file with GCS client. with cloudstorage.open(filename, 'w', content_type=mime_type) as f: f.write(contents.read()) # Blobstore API requires extra /gs to distinguish against blobstore files. blobstore_filename = '/gs' + filename # This blob_key works with blobstore APIs that do not expect a # corresponding BlobInfo in datastore. return blobstore.create_gs_key(blobstore_filename)
def delete(self): if self.request.get('imgPath'): try: blobKey = blobstore.create_gs_key(self.request.get('imgPath')) print(blobKey) images.delete_serving_url(blobKey) except images.Error as e: self.response.set_status(404, e) except image.InvalidBlobKeyError as e: self.response.set_status(404, e) except blobstore.Error as e: self.response.set_status(404, e)
def file_complete(self, file_size): self.file_size = file_size self.gcs_file.close() return GoogleCloudStorageUploadedFile( file=self.gcs_file, name=self.file_name, blob_key=create_gs_key('/gs' + self.bucket + '/' + self.file_name), content_type=self.content_type, size=self.file_size, charset=self.charset )
def url(self, name): """ Returns the publically accessible URL for the file If in DEBUG mode then we server it via the google sdk blobstore using the google storage key """ if settings.DEBUG: filename = "/gs" + self.location + "/" + name key = create_gs_key(filename) return "http://localhost:8000/blobstore/blob/" + \ key + "?display=inline" return self.base_url + "/" + name
def url(self, name): if self.force_use_gcs or is_gae_server(): return '{}/{}'.format(self.base_url, name) # we need this in order to display images, links to files, etc # from the local appengine server from google.appengine.api.blobstore import create_gs_key filename = '/gs{}/{}'.format(self.location, name) key = create_gs_key(filename) local_base_url = getattr(settings, 'GOOGLE_CLOUD_STORAGE_DEV_URL', 'http://localhost:8001/blobstore/blob/') return '{}{}?display=inline'.format(local_base_url, key)
def download_images(self, images_for_download): """ @param images_for_download: dict by url of {'url':..., 'caption':..., '...':...} @return: dict by url of {'id':..., 'url':..., etc...} The filename is arbitrarily based on the url """ d = {} for url in images_for_download: try: d[url] = images_for_download[url] filename = images_for_download[url]['filename'] result = urlfetch.fetch(url) if result.status_code != 200: raise DownloadException("status code != 200") content_type = result.headers['content-type'] if content_type.find("image/") == -1: raise DownloadException("content type not containing 'image/'") file_extension = mimetypes.guess_extension(content_type) gcs_file_name = '%s/%s/%s%s' % (bucket, self._path, filename, file_extension) logging.debug("download_images() downloaded {}".format(url)) with gcs.open(gcs_file_name, 'w', content_type=content_type, options={b'x-goog-acl': b'public-read'}) as f: f.write(result.content) logging.debug("download_images() uploaded {}".format(gcs_file_name)) public_url = images.get_serving_url(blobstore.create_gs_key('/gs' + gcs_file_name)) logging.debug("download_images() published {}".format(public_url)) d[url].update({ 'id': gcs_file_name, 'url': public_url, 'fileExtension': file_extension, 'filename': '%s%s' % (filename, file_extension) }) except TypeError: d[url].update({ 'wasted': 'TRUE' }) logging.error("Could not compose filename successfully due to TypeError (image is uploaded!): %s" % url) except images.TransformationError: d[url].update({ 'wasted': 'TRUE' }) logging.error("Could not upload to Google Cloud Storage because of TransformationError: %s" % url) except DownloadException as e: d[url].update({ 'wasted': 'TRUE' }) logging.error("Could not fetch because {} [{}]".format(e.message, url)) except httplib.HTTPException as e: logging.info("Upload failed: " + e.message) return d
def _fetch_image(url_base64, cache_info): url = base64.urlsafe_b64decode(url_base64) logging.info("image url: %s", url) headers = {} if cache_info: logging.info("image was cached") headers['If-Modified-Since'] = cache_info.last_modified response = urlfetch.fetch(url, headers=headers) logging.info("image fetched, status is %s", response.status_code) if response.status_code == 200: # new or updated image logging.info("image is new or modified") image_data = response.content # resize image = Image.open(StringIO.StringIO(image_data)) width, height = image.size logging.info("image size is %sx%s", width, height) if width > MAX_SIZE or height > MAX_SIZE: ratio = min(MAX_SIZE / width, MAX_SIZE / height) new_size = int(width * ratio), int(height * ratio) logging.info("resizing to %sx%s", *new_size) image = image.resize(new_size, Image.ANTIALIAS) # save to GCS filename = "/%s/%s" % (_BUCKET, url_base64) image_file = cloudstorage.open(filename, "w", "image/png") image.save(image_file, "PNG") image_file.close() # get serving url blob_key = blobstore.create_gs_key("/gs" + filename) serving_url = images.get_serving_url(blob_key, size=max(image.size)) if serving_url.startswith('https'): serving_url = 'http' + serving_url[5:] # save cache info cache_info = ImageCache( id=url_base64, last_modified=response.headers["Last-Modified"], serving_url=serving_url) cache_info.put() elif response.status_code == 304: logging.info("image not modified") cache_info.put() # refresh cache_info.updated else: return None return cache_info
def save_image(cls, photo, user_key=None): img_title = photo.filename img_content = photo.file.read() img_type = photo.type cloud_storage_path = "/gs/fooddonation/%s/%s" % (user_key.id(), img_title) blobstore_key = blobstore.create_gs_key(cloud_storage_path) print blobstore_key cloud_storage_file = cloudstorage_api.open(filename=cloud_storage_path[3:], mode="w", content_type=img_type) cloud_storage_file.write(img_content) cloud_storage_file.close() blobstore_key = blobstore.BlobKey(blobstore_key) serving_url = images.get_serving_url(blobstore_key) print serving_url return {"serving_url": serving_url, "blobstore_key": blobstore_key}
def url(self, name): """ Returns the public URL to a cloudstorage-website bucket. """ if settings.DEBUG: # we need this in order to display images, links to files, etc from the local appengine server filename = "/gs"+self.location+"/"+name key = create_gs_key(filename) hostport = settings.get('GOOGLE_CLOUD_STORAGE_SDK_HOST', 'localhost:8000') url = "http://"+hostport+"/blobstore/blob/"+key+"?display=inline" url = self.base_url+"/"+name if settings.GOOGLE_CLOUD_STORAGE_LOGGING: logging.info("GoogleCloudStorage-url %s", url) return url
def _write_to_gcs(content, content_hash, mime_type): """Write the given data to cloud storage. Args: content: the file data to write. content_hash: a hash identifying the content. mime_type: the mime type of the content. Returns: the blob key for the content. """ gcs_filename = '/content_images/%s' % content_hash with gcs.open(gcs_filename, 'w', content_type=mime_type) as gcs_file: gcs_file.write(content) return blobstore.create_gs_key('/gs%s' % gcs_filename)
def save_image(cls, photo, user_key=None): img_title = photo.filename img_content = photo.file.read() img_type = photo.type cloud_storage_path = '/gs/fooddonation/%s/%s' % (user_key.id(), img_title) blobstore_key = blobstore.create_gs_key(cloud_storage_path) print blobstore_key cloud_storage_file = cloudstorage_api.open( filename=cloud_storage_path[3:], mode='w', content_type=img_type) cloud_storage_file.write(img_content) cloud_storage_file.close() blobstore_key = blobstore.BlobKey(blobstore_key) serving_url = images.get_serving_url(blobstore_key) print serving_url return {'serving_url': serving_url, 'blobstore_key': blobstore_key}
def save_image(cls, photo, user_key): img_title = photo.filename img_content = photo.file.read() img_type = photo.type #todd-search-images is a google cloud bucket cloud_storage_path = '/gs/todd-search-images/%s/%s' % (user_key.id(), img_title) blobstore_key = blobstore.create_gs_key(cloud_storage_path) cloud_storage_file = cloudstorage_api.open( filename=cloud_storage_path[3:], mode='w', content_type=img_type) cloud_storage_file.write(img_content) cloud_storage_file.close() blobstore_key = blobstore.BlobKey(blobstore_key) serving_url = images.get_serving_url(blobstore_key) return {'serving_url': serving_url, 'blobstore_key': blobstore_key}
def CreateFile(filename): """Create a GCS file with GCS client lib. Args: filename: GCS filename. Returns: The corresponding string blobkey for this GCS file. """ # Create a GCS file with GCS client. with gcs.open(filename, 'w') as f: f.write('abcde\n') # Blobstore API requires extra /gs to distinguish against blobstore files. blobstore_filename = '/gs' + filename # This blob_key works with blobstore APIs that do not expect a # corresponding BlobInfo in datastore. return blobstore.create_gs_key(blobstore_filename)
def upload_image(image_binary, content_type): """ :param image_binary: :param content_type: :return: (profile_url, profile_serving_url) """ filename = str(time()).replace('.', '') + "." \ + content_type.split('/')[1] directory = '/bd-profile-image/temp/' filepath = directory + filename _save_file(image_binary, filepath) # generate profile serving url bkey = blobstore.create_gs_key('/gs' + filepath) profile_serving_url = images.get_serving_url(bkey) # pulic profile url profile_url = 'http://storage.googleapis.com' + filepath return profile_url, profile_serving_url
def CreateFile(filename, image): """Create a GCS file with GCS client lib. Args: filename: GCS filename. Returns: The corresponding string blobkey for this GCS file. """ # Create a GCS file with GCS client. with gcs.open(filename, "w", content_type=image.mimetype) as f: f.write(image.stream.read()) # Blobstore API requires extra /gs to distinguish against blobstore files. blobstore_filename = "/gs" + filename # This blob_key works with blobstore APIs that do not expect a # corresponding BlobInfo in datastore. key = blobstore.create_gs_key(blobstore_filename) return key
def get(self): if self.request.get("imgPath"): try: blobKey = blobstore.create_gs_key(self.request.get('imgPath')) photoURL = images.get_serving_url(blobKey, secure_url=True) except images.ObjectNotFoundError as e: self.response.set_status(404, e) except images.AccessDeniedError as e: self.response.set_status(404, e) except images.UnsupportedSizeError as e: self.response.set_status(404, e) if photoURL: self.response.headers['Content-Type'] = 'application/json' self.response.out.write(json.encode(photoURL)) self.response.set_status(200) return # Either "id" wasn't provided, or there was no image with that ID # in the datastore. self.error(404)
def save_image(cls, photo, user_key): img_title = photo.filename img_content = photo.file.read() img_type = photo.type #todd-search-images is a google cloud bucket cloud_storage_path = '/gs/todd-search-images/%s/%s' % (user_key.id(), img_title) blobstore_key = blobstore.create_gs_key(cloud_storage_path) cloud_storage_file = cloudstorage_api.open( filename=cloud_storage_path[3:], mode='w', content_type=img_type ) cloud_storage_file.write(img_content) cloud_storage_file.close() blobstore_key = blobstore.BlobKey(blobstore_key) serving_url = images.get_serving_url(blobstore_key) return { 'serving_url': serving_url, 'blobstore_key': blobstore_key }
def create_blob(self): """Create a GS object in the datastore and on disk. Overrides the superclass create_blob method. Returns: The BlobKey of the new object." """ data = 'a blob' filename = '/gs/some_bucket/some_object' blob_store_key = base64.urlsafe_b64encode(filename) self.blob_storage.StoreBlob(blob_store_key, cStringIO.StringIO(data)) blob_key = blobstore.create_gs_key(filename) entity = datastore.Entity(file_service_stub.GS_INFO_KIND, name=blob_key, namespace='') entity['content_type'] = 'image/png' entity['filename'] = 'largeblob.png' entity['size'] = len(data) entity['storage_key'] = blob_store_key datastore.Put(entity) return blob_key
def get(self): verifier = self.request.get("oauth_verifier") if verifier: # Get access token handler = auth.OAuthHandler(config.CONSUMER_KEY, config.CONSUMER_SECRET) handler.set_request_token(self.session.get("request_token_key"), self.session.get("request_token_secret")) access_token = handler.get_access_token(verifier) if access_token: # Get user logging.info("Access token: %s" %(access_token)) user = User.all().filter("twitter_access_token_key", access_token.key).get() if((not user) or (user and user.updated < datetime.now() - timedelta(0,86400))): logging.info("Connecting to the Twitter API") api = API(handler) temp_user = api.verify_credentials() temp_image = urlfetch.Fetch(str(temp_user.profile_image_url).replace("_normal", "")).content # Transform image into .PNG image_manager = images.Image(image_data=temp_image) image_manager.rotate(360) temp_png = image_manager.execute_transforms() logging.info("Encoded into .PNG") # Save or update image in Cloud storage filename = config.FOLDER + "/" + str(temp_user.id) gcs_file = gcs.open(filename,'w',content_type="image/png",options={"x-goog-acl":"public-read"}) gcs_file.write(temp_png) gcs_file.close() logging.info("Image saved to Google Cloud Storage") # Get avatar blob_filename = "/gs" + filename blobkey = blobstore.create_gs_key(blob_filename) temp_avatar = str(images.get_serving_url(blobkey)) if not user: logging.info("User did not exist") user = User( twitter_id = str(temp_user.id), twitter_access_token_key = str(access_token.key), twitter_access_token_secret = str(access_token.secret), username = str(temp_user.screen_name).lower(), name = temp_user.name, bio = temp_user.description, avatar = temp_avatar, ) else: logging.info("User had to be updated") user.twitter_id = str(temp_user.id) user.twitter_access_token_key = str(access_token.key) user.twitter_access_token_secret = str(access_token.secret) user.username = str(temp_user.screen_name).lower() user.name = temp_user.name user.bio = temp_user.description user.avatar = temp_avatar user.put() logging.info("User @%s saved in datastore"%(user.username)) # Save user in session self.session["id"] = user.key().id() else: logging.error("No access token from Twitter") print "Error" else: logging.error("No verifier") print "Error" # Redirect users to the page they came from or the page they're supposed to head to next = self.session.get("next") redirect = self.session.get("referer") if next: redirect = next self.redirect(str(redirect))
def get_serving_url(blob_key, size=None, crop=False, secure_url=None, filename=None): """Obtain a url that will serve the underlying image. This URL is served by a high-performance dynamic image serving infrastructure. This URL format also allows dynamic resizing and crop with certain restrictions. To get dynamic resizing and cropping, specify size and crop arguments, or simply append options to the end of the default url obtained via this call. Here is an example: get_serving_url -> "http://lh3.ggpht.com/SomeCharactersGoesHere" To get a 32 pixel sized version (aspect-ratio preserved) simply append "=s32" to the url: "http://lh3.ggpht.com/SomeCharactersGoesHere=s32" To get a 32 pixel cropped version simply append "=s32-c": "http://lh3.ggpht.com/SomeCharactersGoesHere=s32-c" Available sizes are any interger in the range [0, 1600] and is available as IMG_SERVING_SIZES_LIMIT. Args: blob_key: BlobKey, BlobInfo, str, or unicode representation of BlobKey of blob to get URL of. size: int, size of resulting images crop: bool, True requests a cropped image, False a resized one. secure_url: bool, True requests a https url, False requests a http url. filename: The filename of a Google Storage object to get the URL of. Returns: str, a url Raises: BlobKeyRequiredError: when no blobkey was specified in the ctor. UnsupportedSizeError: when size parameters uses unsupported sizes. BadRequestError: when crop/size are present in wrong combination, or a blob_key and a filename have been specified. TypeError: when secure_url is not a boolean type. AccessDeniedError: when the blobkey refers to a Google Storage object, and the application does not have permission to access the object. """ if not blob_key and not filename: raise BlobKeyRequiredError( "A Blobkey or a filename is required for this operation.") if crop and not size: raise BadRequestError("Size should be set for crop operation") if size is not None and (size > IMG_SERVING_SIZES_LIMIT or size < 0): raise UnsupportedSizeError("Unsupported size") if secure_url and not isinstance(secure_url, bool): raise TypeError("secure_url must be boolean.") if filename and blob_key: raise BadRequestError("Cannot specify a blob_key and a filename."); if filename: _blob_key = blobstore.create_gs_key(filename) else: _blob_key = _extract_blob_key(blob_key) request = images_service_pb.ImagesGetUrlBaseRequest() response = images_service_pb.ImagesGetUrlBaseResponse() request.set_blob_key(_blob_key) if secure_url: request.set_create_secure_url(secure_url) try: apiproxy_stub_map.MakeSyncCall("images", "GetUrlBase", request, response) except apiproxy_errors.ApplicationError, e: if (e.application_error == images_service_pb.ImagesServiceError.NOT_IMAGE): raise NotImageError() elif (e.application_error == images_service_pb.ImagesServiceError.BAD_IMAGE_DATA): raise BadImageError() elif (e.application_error == images_service_pb.ImagesServiceError.IMAGE_TOO_LARGE): raise LargeImageError() elif (e.application_error == images_service_pb.ImagesServiceError.INVALID_BLOB_KEY): raise InvalidBlobKeyError() elif (e.application_error == images_service_pb.ImagesServiceError.ACCESS_DENIED): raise AccessDeniedError() else: raise Error()
def post(self): cursor = self.request.get("cursor") q = User.all() q.order("created") # Is there a cursor? if(cursor): logging.info("Cursor found") q.with_cursor(start_cursor = cursor) else: logging.info("No cursor") batch_size = 5 users = q.fetch(batch_size) new_users = [] for user in users: logging.info("Copying @%s avatar" % (user.username)) original_filename = "/gs" +config.FOLDER + "/" + str(user.twitter_id) image_manager = images.Image(filename=original_filename) image_manager.rotate(360) png = image_manager.execute_transforms() logging.info("Encoded into .PNG") destination_filename = config.NEW_FOLDER + "/" + str(user.twitter_id) gcs_file = gcs.open(destination_filename,'w',content_type="image/png",options={"x-goog-acl":"public-read"}) gcs_file.write(png) gcs_file.close() logging.info("Saved into the new bucket") # Save blobstore URL in user blob_filename = "/gs" + destination_filename blobkey = blobstore.create_gs_key(blob_filename) user.avatar = images.get_serving_url(blobkey) new_users.append(user) logging.info("Blobstore URL updated in user info") db.put(new_users) logging.info("Users saved") """ batch_size = 5 ideas = q.fetch(batch_size) new_ideas = [] for idea in ideas: problem = idea.answers[0] solution = idea.answers[1] competitors = idea.answers[3] business_model = idea.answers[4] acquisition = idea.answers[5] milestones = idea.answers[6] new_answers = [ problem, solution, acquisition, business_model, competitors, milestones, ] idea.answers = new_answers idea.version = "1" new_ideas.append(idea) db.put(new_ideas) ######### for user in users: try: del user.image except: logging.info("there was a pb") try: del user.img except: logging.info("there was a pb 2") db.put(users) ########## if(len(users)>0): user = users[0] logging.info(user.username) filename = config.FOLDER + "/" + str(user.twitter_id) gcs_file = gcs.open(filename,'w',content_type="image",options={"x-goog-acl":"public-read"}) gcs_file.write(user.image) gcs_file.close() """ if(len(users)==batch_size): new_cursor = q.cursor() task = Task( url = "/queue/update", params = { 'cursor': new_cursor, }, countdown = 2, ) task.add(queue_name="update-queue") logging.info("New task started") else: logging.info("No more task to create")