def post(self): uploaded_file = self.request.POST.get("file") uploaded_file_content = uploaded_file.file.read() uploaded_file_filename = uploaded_file.filename uploaded_file_type = uploaded_file.type bucket_name = 'pick-up-sports-images' # This write_retry_params params bit isn't essential, but Google's examples recommend it write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open("/" + bucket_name + "/" + uploaded_file_filename, "w", content_type=uploaded_file_type, retry_params=write_retry_params) gcs_file.write(uploaded_file_content) gcs_file.close() image_url = 'https://storage.googleapis.com' + "/" + bucket_name + "/" + uploaded_file_filename location_name = self.request.get('location_name') address = self.request.get('address') sport_theme_name = self.request.get('sport_theme_name') tag_names_string = self.request.get('tags') tag_name_list = tag_names_string.split() current_user = users.get_current_user() if current_user: creator = create_or_get_creator(current_user.email()) creator_email = creator.email else: creator_email = None # create location report create_location_report(location_name, address, sport_theme_name, creator_email, tag_name_list, image_url) self.redirect('/')
def post(self): image = self.request.get('image') name = self.request.get('name') content_type = self.request.get('contentType') bucket_name = generate_bucket_filename(name) with gcs.open(bucket_name, 'w', content_type=content_type) as f: f.write(image) blobstore_filename = generate_blobstore_filename(name) blob_key = blobstore.create_gs_key(blobstore_filename) restaurant_image_url = images.get_serving_url(blob_key) image_model = ImageModel(name=name, bucket_name=bucket_name, blob_key=blob_key) image_key = image_model.put() self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps({ 'url': restaurant_image_url, 'name': name, 'mimetype': content_type, 'id': image_key.id() }))
def initialize_ds_names(local_dir=None, size_limit=None): for name,file in NamingGenerator.FILES.iteritems(): valid_items = [] delim = "" if local_dir is None: gcs_file = cloudstorage.open(filename=NamingGenerator.PATH_BASE + file, mode='r') else: gcs_file = open(local_dir + file, mode='r') for line in gcs_file: line = line.strip() if not line or\ len(line.split(" ")) > 1 or\ line.find("-") > 0 or\ len(line) > 8: continue valid_items.append(line.capitalize()) entry = models.Naming(id=name) random.shuffle(valid_items) if size_limit is None: entry.items = valid_items else: entry.items = valid_items[0:size_limit] entry.put()
def post(self): payload = self.request.get('data').encode(constants.UTF8_ENCODING_LABEL) hash = binascii.crc32(payload) & 0xffffffff write_retry_params = gcs.RetryParams(backoff_factor=1.1) filename = constants.CLOUD_FILE_PATTERN % (constants.CLOUD_STORAGE_BUCKET, hash) gcs_file = gcs.open(filename, 'w', content_type=constants.OUTPUT_CONTENT_TYPE_TEXT, retry_params=write_retry_params) gcs_file.write(payload) gcs_file.close() # Now let's train the model! result = service.trainedmodels().insert( project=constants.PROJECT_ID, body={ 'id': str(hash), 'storageDataLocation': filename[1:], 'modelType': 'REGRESSION', }).execute() self.response.headers['Content-Type'] = constants.OUTPUT_CONTENT_TYPE_TEXT # self.response.out.write('Result: ' + repr(result)) self.response.write(hash)
def CreateFile(filename, imageFile): with gcs.open(filename, 'w', content_type='image/jpeg') as f: f.write(imageFile) f.close() blobstore_filename = '/gs' + filename return blobstore.create_gs_key(blobstore_filename)
def get_logs(self, release_id, task_name, log_file='1.log'): """Gets the logs for a task from GCS. Args: release_id: str task_name: str log_file: str Returns: Structured log text """ # !!! Note that because of overlapping datetimes and dag_ids, some logs WILL be broken. # This cannot be helped, because of the way which Airflow defines a release !!! dag_id, execution_date = release_id_parser(release_id) for i, config in enumerate(self._configs): dag_run = config.db.query(to_sql_tasks(dag_id, float(time.mktime(execution_date.timetuple())))) if len(dag_run): execution_date = str(execution_date).replace(' ', 'T') # put into same format as gcs bucket logging.debug('Bucket name: ' + config.bucket_name) filename = os.path.join(os.path.sep, config.bucket_name, 'logs', dag_id, task_name, execution_date, log_file) logging.info('Retrieving from GCS: ' + str(filename)) try: gcs_file = gcs.open(filename) contents = gcs_file.read() gcs_file.close() return contents except: continue
def getFromStorage(bookUrl): gcs_file = gcs.open(bookUrl) data = gcs_file.read() gcs_file.close() return data
def _write_local_config(self): """Writes a local config file to GCS.""" if not os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/'): with open(self.local_path, 'r') as stream: content = stream.read() gcs_file = gcs.open(self.file_path, 'w', content_type='text/plain') gcs_file.write(content) gcs_file.close()
def read_file(self, filename): self.response.write('Abbreviated file content (first line and last 1K):\n') gcs_file = gcs.open(filename) self.response.write(gcs_file.readline()) gcs_file.seek(-1024, os.SEEK_END) self.response.write(gcs_file.read()) gcs_file.close()
def post(self): reportFile = self.request.POST['file_input'] folder_name = self.request.get('folderName') user_name = self.request.get('userName') user_id = self.request.get('userId') if not reportFile.filename or not folder_name or not user_name or not user_id: self.response.set_status(400) self.render_json({'reason': 'Not all required parameters found', 'status': 'error'}) return bucket_name = os.environ.get('BUCKET_NAME', 'deepspace9-1134.appspot.com') bucket = '/' + bucket_name + '/' + folder_name filename = bucket + '/' + reportFile.filename # Check if the file already exists in google cloud storage exists = True try: gcs.stat(filename) except gcs.NotFoundError: exists = False if exists: # find it in the database and get its key files = GCSFile.get_by_gcs_file_name(filename) if len(files) == 1: db_gcs_file = files[0] db_gcs_file.user_id = user_id db_gcs_file.user_name = user_name db_gcs_file.original_file_name = reportFile.filename db_gcs_file.gcs_file_name = filename db_gcs_file.timestamp = datetime.now() logging.error(db_gcs_file.key.id()) else: self.response.set_status(500) self.render_json({'status': 'error', 'key': '', 'reason': 'Unexpected number of files found in the database'}) return else: # write a new entry in the DB db_gcs_file = GCSFile.save_new(user_id=user_id, user_name=user_name, original_file_name=reportFile.filename, gcs_file_name=filename) # save to database db_gcs_file.put() # write the file to GCS write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open(filename, 'w', content_type=reportFile.type, options={'x-goog-meta-user-name': user_name, 'x-goog-meta-user-id': user_id, 'x-goog-meta-original-name': str(reportFile.filename)}, retry_params=write_retry_params) gcs_file.write(reportFile.value) gcs_file.close() # # reply to the app with success and the key self.render_json({'status': 'success', 'id': db_gcs_file.key.id()})
def _upload_image(self, img_name, img_fh, plaque): """ Upload pic into GCS The blobstore.create_gs_key and images.get_serving_url calls are outside of the with block; I think this is correct. The blobstore.create_gs_key call was erroring out on production when it was inside the with block. If gcs_fn is specified, overwrite that gcs filename. This is used for updating the picture. """ # Turn this off while Tony Bonomolo is editing: # # # Kill old image and URL, if they exist. Tolerate failure in case # # this is a redo: # if plaque.pic is not None: # try: # gcs.delete(plaque.pic) # except: # pass # if plaque.img_url is not None: # try: # images.delete_serving_url(plaque.img_url) # except: # pass # Make GCS filename date_slash_time = datetime.datetime.now().strftime("%Y%m%d/%H%M%S") gcs_filename = '%s/%s/%s' % (GCS_BUCKET, date_slash_time, img_name) plaque.pic = gcs_filename # TODO openbenches: set plaque.pic to None? # TODO openbenches: skip this try block, set plaque.img_url and return immediately? # TODO openbenches: set img_url to the hotlink URL? # Write image to GCS try: ct = 'image/jpeg' op = {b'x-goog-acl': b'public-read'} with gcs.open(gcs_filename, 'w', content_type=ct, options=op) as fh: img_contents = img_fh.read() fh.write(img_contents) # Make serving_url for image: blobstore_gs_key = blobstore.create_gs_key('/gs' + gcs_filename) plaque.img_url = images.get_serving_url(blobstore_gs_key) except AttributeError: submit_err = SubmitError( "The image for the plaque was not " "specified-- please click the back button " "and resubmit.") logging.error(submit_err) raise submit_err
def blob_read(self): """ read binary blob from google cloud storage """ try: with gcs.open(self.gcs_filename) as f: return f.read() except gcs.NotFoundError, e: logging.warning('GCS file %s NOT FOUND : %s' % (self.gcs_filename, e)) return None
def get(self, lang='ru'): import shutil key = ndb.Key(Dictionary, lang).get() if not key: self.abort(404) key = key.gcs_key if key in self.request.if_none_match: self.response.status = 304 return dictionary_file = gcs.open(get_gcs_filename(key)) shutil.copyfileobj(dictionary_file, self.response) dictionary_file.close() self.response.etag = key
def post(self): reportFile = self.request.POST['file_input'] folder_name = self.request.get('folderName') bucket_name = os.environ.get('BUCKET_NAME', 'deepspace9-1134.appspot.com') # self.response.headers['Content-Type'] = 'text/plain' # self.response.write('Demo GCS Application running from Version: ' # + os.environ['CURRENT_VERSION_ID'] + '\n') # self.response.write('Using bucket name: ' + bucket_name + '\n\n') bucket = '/' + bucket_name + '/' + folder_name filename = bucket + '/' + reportFile.filename """Create a file. The retry_params specified in the open call will override the default retry params for this particular file handle. Args: filename: filename. """ # self.response.write('Creating file %s\n' % filename) write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open(filename, 'w', content_type=reportFile.type, options={'x-goog-meta-foo': 'foo', 'x-goog-meta-bar': 'bar'}, retry_params=write_retry_params) gcs_file.write(reportFile.value) gcs_file.close() # # echo the file back gcs_file = gcs.open(filename) self.response.headers['Content-Type'] = reportFile.type self.response.headers['Content-Disposition'] = "attachment; filename=" + str(reportFile.filename) self.response.write(gcs_file.read()) gcs_file.close()
def get(self, lang="ru"): import shutil key = ndb.Key(Dictionary, lang).get() if not key: self.abort(404) key = key.gcs_key if key in self.request.if_none_match: self.response.status = 304 return dictionary_file = gcs.open(get_gcs_filename(key)) shutil.copyfileobj(dictionary_file, self.response) dictionary_file.close() self.response.etag = key
def post(self): try: gurl = self.request.get('imageURL') # print("urlColor: " +self.request.get('color')) gcolor = float(self.request.get('color')) gbrightness = float(self.request.get('brightness')) gcontrast = float(self.request.get('contrast')) gsharpness = float(self.request.get('sharpness')) grotate = int(self.request.get('rotate')) gImFLSelected = bool(self.request.get('lucky')) c = urlfetch.fetch(gurl, deadline=10).content im = Image.open(StringIO(c)) mimeType = im.format try: if gImFLSelected: img = images.Image(c) img.im_feeling_lucky() data = img.execute_transforms(output_encoding=images.JPEG) else: enh = ImageEnhance.Color(im) # 0 - 2 to be considered out = enh.enhance(gcolor) enh = ImageEnhance.Brightness(out) # 0 - black image, 1 - original image; Can give more than 1.0 out = enh.enhance(gbrightness) enh = ImageEnhance.Contrast(out) # 0 - solid grey image, 1 - original image out = enh.enhance(gcontrast) enh = ImageEnhance.Sharpness(out) # 0 - blurred image, 1 - original image, 2 - sharpened image out = enh.enhance(gsharpness) out = out.rotate(grotate, resample=Image.BICUBIC, expand=True) buf = cStringIO.StringIO() out.save(buf, mimeType) data = buf.getvalue() bucket_name = 'dem-ode' bucket = '/' + bucket_name filename = bucket + '/' + urllib.quote(u"{0}".format(time.time()).encode('utf8')) with gcs.open(filename, 'w') as f: f.write(data) blobstore_filename = "/gs"+filename # this is needed if you want to continue using blob_keys. ieurl = images.get_serving_url(blobstore.BlobKey(blobstore.create_gs_key(str(blobstore_filename)))) # self.response.out.write('<img width="100%" height="100%" src="' + ieurl +'"/>') self.response.out.write('<a style="float:right;position:absolute;top:8px;right:8px;width:48px;height:48px" href="' + ieurl + '" download>'+'<img style="width: 48px;height: 48px;" src="https://upload.wikimedia.org/wikipedia/commons/thumb/1/1e/Download-Icon.png/480px-Download-Icon.png"></a>') except: self.response.headers[b'Content-Type'] = b'text/plain' self.response.out.write('Image size is too large. Can\'t handle') except: self.response.headers[b'Content-Type'] = b'text/plain' self.response.out.write('Image size is too large. Can\'t handle')
def _upload_image(self, img_name, img_fh, plaque): """ Upload pic into GCS The blobstore.create_gs_key and images.get_serving_url calls are outside of the with block; I think this is correct. The blobstore.create_gs_key call was erroring out on production when it was inside the with block. If gcs_fn is specified, overwrite that gcs filename. This is used for updating the picture. """ # Turn this off while Tony Bonomolo is editing: # # # Kill old image and URL, if they exist. Tolerate failure in case # # this is a redo: # if plaque.pic is not None: # try: # gcs.delete(plaque.pic) # except: # pass # if plaque.img_url is not None: # try: # images.delete_serving_url(plaque.img_url) # except: # pass # Make GCS filename date_slash_time = datetime.datetime.now().strftime("%Y%m%d/%H%M%S") gcs_filename = '%s/%s/%s' % (GCS_BUCKET, date_slash_time, img_name) plaque.pic = gcs_filename # Write image to GCS try: ct, op = self._gcs_extras(img_name) with gcs.open(gcs_filename, 'w', content_type=ct, options=op) as fh: img_contents = img_fh.read() fh.write(img_contents) except AttributeError: submit_err = SubmitError("The image for the plaque was not " "specified-- please click the back button " "and resubmit.") logging.error(submit_err) raise submit_err # Make serving_url for image: blobstore_gs_key = blobstore.create_gs_key('/gs' + gcs_filename) plaque.img_url = images.get_serving_url(blobstore_gs_key)
def _fetch_image(url_base64, cache_info): url = base64.urlsafe_b64decode(url_base64) logging.info("image url: %s", url) headers = {} if cache_info: logging.info("image was cached") headers['If-Modified-Since'] = cache_info.last_modified response = urlfetch.fetch(url, headers=headers) logging.info("image fetched, status is %s", response.status_code) if response.status_code == 200: # new or updated image logging.info("image is new or modified") image_data = response.content # resize image = Image.open(StringIO.StringIO(image_data)) width, height = image.size logging.info("image size is %sx%s", width, height) if width > MAX_SIZE or height > MAX_SIZE: ratio = min(MAX_SIZE / width, MAX_SIZE / height) new_size = int(width * ratio), int(height * ratio) logging.info("resizing to %sx%s", *new_size) image = image.resize(new_size, Image.ANTIALIAS) # save to GCS filename = "/%s/%s" % (_BUCKET, url_base64) image_file = cloudstorage.open(filename, "w", "image/png") image.save(image_file, "PNG") image_file.close() # get serving url blob_key = blobstore.create_gs_key("/gs" + filename) serving_url = images.get_serving_url(blob_key, size=max(image.size)) if serving_url.startswith('https'): serving_url = 'http' + serving_url[5:] # save cache info cache_info = ImageCache( id=url_base64, last_modified=response.headers["Last-Modified"], serving_url=serving_url) cache_info.put() elif response.status_code == 304: logging.info("image not modified") cache_info.put() # refresh cache_info.updated else: return None return cache_info
def blob_write(self, blob): """ update google cloud storage bf entity """ content_type = mimetypes.guess_type(self.filename)[0] if not content_type: logging.warning('Mimetype not guessed for: %s', self.filename) if content_type and self.extension in config.UTF_8_FILE_EXTENSIONS: content_type += b'; charset=utf-8' try: with gcs.open(self.gcs_filename, 'w', content_type=content_type or b'binary/octet-stream', options={b'x-goog-acl': b'public-read'}) as f: f.write(blob) return self.gcs_filename except Exception, e: raise Exception('Blob write failed for %s, exception: %s. Additional info was logged' % (self.filename, str(e)))
def blob_archive(new_bf=None): """ bonus: save all GCSFile in a zip archive """ @ndb.tasklet def callback(bf_key): """ key_only query and get() lookup for entity consistency """ bf = yield bf_key.get_async() raise ndb.Return(bf) def blobfiles(insert, archive_key): """ We do not use ancestor queries. This Generator takes care of index and entity inconsistencies https://cloud.google.com/developers/articles/balancing-strong-and-eventual-consistency-with-google-cloud-datastore/ """ for bf in GCSFile.query().filter(GCSFile.key != archive_key).map(callback, keys_only=True): if insert and new_bf.key == bf.key: insert = False # no index inconsistency yield bf # if the new_bf entity is not yet present in GCSFile (due to index inconsistencies), it will be inserted here if insert: yield new_bf # add all files to archive, except the archive zipfile itself which has a reserved name (GCSFile key) (archive_folder, _, archive_file) = config.ARCHIVE_PATH.rpartition('/') if new_bf and new_bf.filename != archive_file: new_zf = GCSFile.new(archive_file, folder=archive_folder) with gcs.open(new_zf.gcs_filename, 'w', content_type=b'multipart/x-zip', options={b'x-goog-acl': b'public-read', b'cache-control': b'private, max-age=0, no-cache'}) as nzf: # nzf is a cloudstorage.storage_api.StreamingBuffer, which can be pickled to append data in a chained task with zipfile.ZipFile(nzf, 'w') as zf: for each in blobfiles(new_bf is not None, new_zf.key): # We also could have used : each.blob_read() logging.info(each.filename) blob = each.blob_reader().read() zf.writestr(each.filename.encode('utf-8'), blob) new_zf.put_async() else: new_zf = new_bf return new_zf
def post(self): unicorn = self.request.get('img') img_name = self.request.get('img_name') stream_id = self.request.get('stream_id') stream_tags = self.request.get('tags') tag_list = re.findall(r'#\w+', stream_tags) lon = float(self.request.get('lon')) lat = float(self.request.get('lat')) img_comment = "comment" unicorn = images.resize(unicorn, 500, 500) # find the right bucket-stream path b = "/pigeonhole-apt.appspot.com/" + str(stream_id) # much be post! img_real_name = self.request.POST['img'].filename pat = "(.+)\.(.+)" img_real_type = re.match(pat, str(img_real_name)).group(2) # construct a new content_type content_type_value = "image/" + str(img_real_type).lower() # create such file and write to it gcs_file = gcs.open(b + "/" + str(img_name) + "." + str(img_real_type), 'w', content_type=content_type_value) gcs_file.write(unicorn) gcs_file.close() # generate the public url # not test yet TODO: test the url unicorn_url = "https://storage.googleapis.com/pigeonhole-apt.appspot.com/" \ + str(stream_id) + "/" + str(img_name) + "." + str(img_real_type).lower() # back to ndb server ops.create_image(img_comment, img_name, unicorn_url, stream_id, lon, lat) ops.update_stream_tag(stream_id, tag_list) # redirect the user to the view single page self.redirect( str("https://pigeonhole-apt.appspot.com/view_single?stream_id=" + str(stream_id)))
def get(self, photo_key): if not blobstore.get(photo_key): self.error(404) else: try: blob_info = blobstore.BlobInfo.get(photo_key) im = Image.open(blob_info.open()) iR = ndb.Key(ImageData, '123').get() mimeType = im.format try: if iR.gImFLSelected: img = images.Image(blob_key = iR.imageKey) img.im_feeling_lucky() data = img.execute_transforms(output_encoding=images.JPEG) else: enh = ImageEnhance.Color(im) # 0 - 2 to be considered out = enh.enhance(iR.gColor) enh = ImageEnhance.Brightness(out) # 0 - black image, 1 - original image; Can give more than 1.0 out = enh.enhance(iR.gBrightness) enh = ImageEnhance.Contrast(out) # 0 - solid grey image, 1 - original image out = enh.enhance(iR.gContrast) enh = ImageEnhance.Sharpness(out) # 0 - blurred image, 1 - original image, 2 - sharpened image out = enh.enhance(iR.gSharpness) out = out.rotate(iR.gRotate, resample=Image.BICUBIC, expand=True) buf = cStringIO.StringIO() out.save(buf, mimeType) data = buf.getvalue() bucket_name = 'dem-ode' bucket = '/' + bucket_name filename = bucket + '/' + urllib.quote(u"{0}".format(time.time()).encode('utf8')) with gcs.open(filename, 'w') as f: f.write(data) blobstore_filename = "/gs"+filename #this is needed if you want to continue using blob_keys. ieurl = images.get_serving_url(blobstore.BlobKey(blobstore.create_gs_key(str(blobstore_filename)))) self.response.out.write('<img width="100%" height="100%" src="' + ieurl +'"/>') self.response.out.write('<a style="float:right;position:absolute;top:8px;right:8px;width:48px;height:48px" href="' + ieurl + '" download>'+'<img style="width: 48px;height: 48px;" src="https://upload.wikimedia.org/wikipedia/commons/thumb/1/1e/Download-Icon.png/480px-Download-Icon.png"></a>') except: self.response.headers[b'Content-Type'] = b'text/plain' self.response.out.write('Image size is too large. Can\'t handle') except: self.response.headers[b'Content-Type'] = b'text/plain' self.response.out.write('Image size is too large. Can\'t handle')
def post(self): for dictionary in Dictionary.query(): data_object = [] key = str(int(time.time())) words = GlobalDictionaryWord.query(GlobalDictionaryWord.deleted == False, GlobalDictionaryWord.lang == dictionary.key.id()).order(GlobalDictionaryWord.E).fetch() chunk_size = len(words) // 100 for i, word in enumerate(words): data_object.append({"word": word.word, "diff": i // chunk_size, "used": word.used_times, "tags": word.tags}) output_file = gcs.open(get_gcs_filename(key), "w", "application/json") json.dump(data_object, output_file) output_file.close() old_key = dictionary.gcs_key dictionary.gcs_key = key dictionary.put() if old_key: gcs.delete(get_gcs_filename(old_key))
def uploadToStorage(bookName, bookText): bucket_name = os.environ.get('BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) bucket = '/' + bucket_name filename = bucket + '/books/' + str(datetime.now()) + '/' + bookName write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open(filename, 'w', content_type='text/plain', retry_params=write_retry_params, options={'x-goog-acl': 'public-read'}) gcs_file.write(bookText) gcs_file.close() return filename
def get(self, _id, img_type): self.response.headers['Access-Control-Allow-Origin'] = "*" try: bucket_name = os.environ.get( 'BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) if img_type == 'p': filename = '/%s/%s/%s' % (bucket_name, FOLDER_PHOTOS, _id) if img_type == 'm': filename = '/%s/%s/%s' % (bucket_name, FOLDER_MASKS, _id) stat = gcs.stat(filename) self.response.headers['Content-Type'] = stat.content_type self.response.headers['Content-Length'] = stat.st_size self.response.headers['etag'] = stat.etag gcs_file = gcs.open(filename) self.response.write(gcs_file.read()) gcs_file.close() except Exception, e: self.response.set_status('400') self.response.write(e.message)
def post(self): # ファイル名とファイルデータを得る file_name = self.request.POST["file_data"].filename file_data = self.request.get("file_data") # このアプリケーションのGCSバケット名を得る bucket_name = app_identity.get_default_gcs_bucket_name() # 保存パスを作成 filepath = "/" + bucket_name + "/file_save/" + file_name # ファイル作成 gcs_file = gcs.open(filepath, "w") gcs_file.write(file_data) gcs_file.close() gcs_key = blobstore.create_gs_key("/gs" + filepath) self.set_template_value("message", gcs_key) self.draw_template("front/file_save/done.html")
def post(self): # ファイル名とファイルデータを得る file_name = self.request.POST['file_data'].filename file_data = self.request.get('file_data') # このアプリケーションのGCSバケット名を得る bucket_name = app_identity.get_default_gcs_bucket_name() # 保存パスを作成 filepath = '/' + bucket_name + '/file_save/' + file_name # ファイル作成 gcs_file = gcs.open(filepath, 'w') gcs_file.write(file_data) gcs_file.close() gcs_key = blobstore.create_gs_key('/gs' + filepath) self.set_template_value('message', gcs_key) self.draw_template('front/file_save/done.html')
def post (self): user_id = self.request.get ('user') print 'task queue recevied user id: %s' % user_id album_key = self.request.get ('album') fb_albums = json.loads (self.request.get ('fb_albums')) images = [] user = User.get_user_by_id (user_id) album = ndb.Key (Album, int(album_key), parent=user.key).get () graph = facebook.GraphAPI (user.access_token) albums_count = len(fb_albums) for fb_album in fb_albums: images_list = graph.get_object ("%s/photos" % fb_album["id"]) if images_list: for image in images_list["data"]: imgdata = cStringIO.StringIO(urllib2.urlopen(image["images"][0]["source"]).read ()) filename = "/photos/%s_photo" % image["id"] with gcs.open (filename, 'w') as photo: photo.write (imgdata.getvalue ()) blobkey = blobstore.create_gs_key ("/gs%s" % filename) images.append ( Photo ( id=image["id"], payload = blobstore.BlobKey (blobkey), width=image["images"][0]["width"], height=image["images"][0]["height"] ) ) album.images = images album.put ()
def get(self): file_id = self.request.get('fileId') if file_id and file_id.isdigit(): # get the gcs_file_name from the database db_gcs_file = GCSFile.get(int(file_id)) if db_gcs_file: # check if it exists in gcs try: stat = gcs.stat(db_gcs_file.gcs_file_name) except gcs.NotFoundError: self.response.set_status(404) self.write('404: This file does not exist') else: # return it if it does gcs_file = gcs.open(db_gcs_file.gcs_file_name) self.response.headers['Content-Type'] = stat.content_type self.response.headers['Content-Disposition'] = "attachment; filename=" + stat.metadata[ 'x-goog-meta-original-name'] self.response.write(gcs_file.read()) gcs_file.close() else: self.response.set_status(404) self.write('404: This file does not exist')
def create_file(self, filename): """Create a file. The retry_params specified in the open call will override the default retry params for this particular file handle. Args: filename: filename. """ self.response.write('Creating file %s\n' % filename) write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open(filename, 'w', content_type='text/plain', options={'x-goog-meta-foo': 'foo', 'x-goog-meta-bar': 'bar'}, retry_params=write_retry_params) gcs_file.write('abcde\n') gcs_file.write('f' * 1024 * 4 + '\n') gcs_file.close() self.tmp_filenames_to_clean_up.append(filename)
def _upload_to_gcs(self, bucket_url, file_data, check_box): """Create a file. The retry_params specified in the open call will override the default retry params for this particular file handle. Args: bucket_url: filename on GCS. file_data: raw file data. check_box: boolean showing status of the checkbox in the form. """ self.response.write('Creating file %s\n' % bucket_url) write_retry_params = gcs.RetryParams(backoff_factor=1.1) options = { 'x-goog-acl': 'public-read', 'x-goog-meta-public': str(check_box) } gcs_file = gcs.open(bucket_url, 'w', options=options, retry_params=write_retry_params) gcs_file.write(file_data) gcs_file.close()
def post(self): for dictionary in Dictionary.query(): data_object = [] key = str(int(time.time())) words = ( GlobalDictionaryWord.query( GlobalDictionaryWord.deleted == False, GlobalDictionaryWord.lang == dictionary.key.id() ) .order(GlobalDictionaryWord.E) .fetch() ) chunk_size = len(words) // 100 for i, word in enumerate(words): data_object.append( {"word": word.word, "diff": i // chunk_size, "used": word.used_times, "tags": word.tags} ) output_file = gcs.open(get_gcs_filename(key), "w", "application/json") json.dump(data_object, output_file) output_file.close() old_key = dictionary.gcs_key dictionary.gcs_key = key dictionary.put() if old_key: gcs.delete(get_gcs_filename(old_key))
def get_tree(self, request): filename = '/' + bucket_name + '/tree.json' gcs_file = gcs.open(filename) tree = gcs_file.read() gcs_file.close() return TreeMessage(tree=tree.decode('utf-8').encode('unicode-escape'))
def post(self): try: #image_file = self.request.get('image', default_value=None) data = json.loads(self.request.body) ext = data.get('type') # mime type must be present and must be png or gif if ext is None or ext not in ['png', 'gif']: raise Exception('invalid format') # image must be present if data.get('image', None) is None: raise Exception('missing image') image = base64.b64decode( data.get('image').replace('data:image/%s;base64,' % ext, '')) ip = self.request.remote_addr credits = data.get('credits', '') tags = data.get('tags', []) lang = data.get('lang', 'en') audience = int(data.get('audience', 1)) is_safe = True if audience <= 2 else False email = data.get('email', None) # flip image horizontally # image_object = images.Image(image) # image_object.horizontal_flip() # image = image_object.execute_transforms() # caclulate mask id mask_id = hashlib.sha1(image).hexdigest() print mask_id # check if mask exists if Mask.get_by_id(mask_id) is not None: self.response.headers['Access-Control-Allow-Origin'] = "*" self.response.set_status('304') return # if len(tags) == 0: # raise Exception('missing tags') tags_list = [] # if there are new tags create them for tag in tags: tag = Tag.get_or_create(tag, source='MASK', lang=lang) tags_list.append(str(tag.key.id())) # save it on cloud storage bucket_name = os.environ.get( 'BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) filename = '%s.%s' % (mask_id, ext) gs_filename = '/%s/%s/%s' % (bucket_name, FOLDER_MASKS, filename) gcs_file = gcs.open(gs_filename, 'w', content_type='image/%s' % ext, options={ 'x-goog-meta-foo': 'foo', 'x-goog-meta-bar': 'bar' }) gcs_file.write(image) gcs_file.close() mask = Mask(id=mask_id) mask.populate(filename=filename, ip=ip, audience=audience, email=email, tags=tags_list, credits=credits, is_safe=is_safe) mask.put() # memcache memcache.set(mask_id, Mask.to_json_object(mask)) response_data = { "id": mask_id, "mask": "/masks/%s" % mask_id, "url": "/m/%s" % filename } self.response.headers['Access-Control-Allow-Origin'] = "*" self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(response_data)) except Exception, e: self.response.headers['Access-Control-Allow-Origin'] = "*" self.response.set_status('400') self.response.write(e.message)
def read_metrics(metric): gcs_file = gcs.open(METRICS_DIR + metric + '.json') data = gcs_file.read() gcs_file.close() return data
def read_file(self, filename): gcs_file = gcs.open(filename) the_file = gcs_file.read() gcs_file.close() return the_file, self.get_content_type(filename)
def write_metrics(metric, data): gcs_file = gcs.open(METRICS_DIR + metric + '.json', 'w') gcs_file.write(json.dumps(data)) gcs_file.close()
def post(self): try: # get all parameters data = json.loads(self.request.body) ext = data.get('type') # mime type must be present and must be png or gif if ext is None or ext not in ['png', 'gif']: raise Exception('invalid format') # image must be present image = data.get('image', None) if image is None: raise Exception('missing image') # parse image image = base64.b64decode( image.replace('data:image/%s;base64,' % ext, '')) # source ip ip = self.request.remote_addr email = data.get('email', '') tags = data.get('tags', []) lang = data.get('lang', 'en') audience = int(data.get('audience', 1)) masks = [str(x) for x in data.get('masks', [])] # mask must be present if len(masks) == 0: raise Exception('masks is empty') # caclulate photo id photo_id = hashlib.sha1(image).hexdigest() photo = Photo.get_by_id(photo_id) # check if phot exists if photo is not None: self.response.headers['Access-Control-Allow-Origin'] = "*" self.response.set_status('304') return tags_list = [] # if there are new tags create them for tag in tags: tag = Tag.get_or_create(tag, source='USER', lang=lang) tags_list.append(tag.key.id()) # find the other tags in the masks for mask_id in masks: if mask_id == 0: # test mask continue mask = Mask.get_by_id(mask_id) if mask is not None: tags_list.extend(mask.tags) mask.photo_count += 1 mask.put() # set the minumum audience to the maximum of the mask audience if mask.audience > audience: audience = mask.audience # now understand if it is safe or not is_safe = True if int(audience) <= 2 else False tags_list = list(set(tags_list)) # save it on cloud storage bucket_name = os.environ.get( 'BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) filename = '%s.%s' % (photo_id, ext) gs_filename = '/%s/%s/%s' % (bucket_name, FOLDER_PHOTOS, filename) gcs_file = gcs.open(gs_filename, 'w', content_type='image/%s' % ext, options={ 'x-goog-acl': 'public-read', 'x-goog-meta-foo': 'foo' }) gcs_file.write(image) gcs_file.close() # create a key for the stored file gs_key = blobstore.create_gs_key('/gs' + gs_filename) # get the cache url for the image serving_url = images.get_serving_url(gs_key) photo = Photo(id=photo_id) photo.populate(filename=filename, ip=ip, audience=audience, masks=masks, tags=tags_list, email=email, ext=ext, thumb=serving_url, is_safe=is_safe) photo.put() response_data = { "id": photo_id, "blowout": "/blowout/%s" % photo_id, "image": "/p/%s" % photo.filename, "thumb": photo.thumb } # memcache json_photo = Photo.to_json_object(photo) memcache.set(photo.key.id(), json_photo) self.response.headers['Access-Control-Allow-Origin'] = "*" self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(response_data)) except Exception, e: logging.info(e) self.response.headers['Access-Control-Allow-Origin'] = "*" self.response.set_status('400') self.response.write(e.message)
def create_stream_in_storage(stream_name): bucket = "/pigeonhole-apt.appspot.com/" filename = bucket + str(stream_name) + "/" gcs_file = gcs.open(filename, 'w') gcs_file.close()
def get(self, photo_key): if not blobstore.get(photo_key): self.error(404) else: try: blob_info = blobstore.BlobInfo.get(photo_key) im = Image.open(blob_info.open()) iR = ndb.Key(ImageData, '123').get() mimeType = im.format try: if iR.gImFLSelected: img = images.Image(blob_key=iR.imageKey) img.im_feeling_lucky() data = img.execute_transforms( output_encoding=images.JPEG) else: enh = ImageEnhance.Color(im) # 0 - 2 to be considered out = enh.enhance(iR.gColor) enh = ImageEnhance.Brightness( out ) # 0 - black image, 1 - original image; Can give more than 1.0 out = enh.enhance(iR.gBrightness) enh = ImageEnhance.Contrast( out) # 0 - solid grey image, 1 - original image out = enh.enhance(iR.gContrast) enh = ImageEnhance.Sharpness( out ) # 0 - blurred image, 1 - original image, 2 - sharpened image out = enh.enhance(iR.gSharpness) out = out.rotate(iR.gRotate, resample=Image.BICUBIC, expand=True) buf = cStringIO.StringIO() out.save(buf, mimeType) data = buf.getvalue() bucket_name = 'dem-ode' bucket = '/' + bucket_name filename = bucket + '/' + urllib.quote(u"{0}".format( time.time()).encode('utf8')) with gcs.open(filename, 'w') as f: f.write(data) blobstore_filename = "/gs" + filename #this is needed if you want to continue using blob_keys. ieurl = images.get_serving_url( blobstore.BlobKey( blobstore.create_gs_key(str(blobstore_filename)))) self.response.out.write( '<img width="100%" height="100%" src="' + ieurl + '"/>') self.response.out.write( '<a style="float:right;position:absolute;top:8px;right:8px;width:48px;height:48px" href="' + ieurl + '" download>' + '<img style="width: 48px;height: 48px;" src="https://upload.wikimedia.org/wikipedia/commons/thumb/1/1e/Download-Icon.png/480px-Download-Icon.png"></a>' ) except: self.response.headers[b'Content-Type'] = b'text/plain' self.response.out.write( 'Image size is too large. Can\'t handle') except: self.response.headers[b'Content-Type'] = b'text/plain' self.response.out.write( 'Image size is too large. Can\'t handle')
def post (self): user_id = self.request.get ('user') print 'Create cover for user id: %s' % user_id album_key = self.request.get ('album') fb_album = self.request.get ('fb_album') images = [] user = User.get_user_by_id (user_id) album = ndb.Key (Album, int(album_key), parent=user.key).get () graph = facebook.GraphAPI (user.access_token) pictures = graph.get_object ("%s/photos" % fb_album, fields="picture", limit=4) if pictures: bg = imaging.new ('RGBA', (200, 200)) position = ( (0, 0, 100, 100), (0, 100, 100, 200), (100, 0, 200, 100), (100, 100, 200, 200) ) try: index = 0 for picture in pictures["data"]: data = cStringIO.StringIO(urllib2.urlopen(picture["picture"]).read ()) img = imaging.open (data) ratio = float (max (img.size)) / float (min (img.size)) size = int (img.size[0] * ratio), int (img.size[1] * ratio) oldsize = img.size diff = tuple (x - y for x, y in zip (size, oldsize)) img = img.resize (size) thumbnail = img.crop ((diff[0] / 2, diff[1] / 2, 100 + diff[0] / 2, 100 + diff[1] /2 )) padding = [(100 - thumbnail.size[0]) / 2, (100 - thumbnail.size[1]) / 2] bg.paste (thumbnail, (position[index][0] + padding[0], position[index][1] + padding[1], position[index][2] - padding[0], position[index][3] - padding[1])) index += 1 if index > 4: break filename = "/covers/%s_cover" % album.key with gcs.open (filename, 'w') as cover: bg.save (cover, 'PNG') blobkey = blobstore.create_gs_key ("/gs%s" % filename) album.cover = blobstore.BlobKey (blobkey) album.put () except ValueError as e: raise e
def post(self): try: gurl = self.request.get('imageURL') # print("urlColor: " +self.request.get('color')) gcolor = float(self.request.get('color')) gbrightness = float(self.request.get('brightness')) gcontrast = float(self.request.get('contrast')) gsharpness = float(self.request.get('sharpness')) grotate = int(self.request.get('rotate')) gImFLSelected = bool(self.request.get('lucky')) c = urlfetch.fetch(gurl, deadline=10).content im = Image.open(StringIO(c)) mimeType = im.format try: if gImFLSelected: img = images.Image(c) img.im_feeling_lucky() data = img.execute_transforms(output_encoding=images.JPEG) else: enh = ImageEnhance.Color(im) # 0 - 2 to be considered out = enh.enhance(gcolor) enh = ImageEnhance.Brightness( out ) # 0 - black image, 1 - original image; Can give more than 1.0 out = enh.enhance(gbrightness) enh = ImageEnhance.Contrast( out) # 0 - solid grey image, 1 - original image out = enh.enhance(gcontrast) enh = ImageEnhance.Sharpness( out ) # 0 - blurred image, 1 - original image, 2 - sharpened image out = enh.enhance(gsharpness) out = out.rotate(grotate, resample=Image.BICUBIC, expand=True) buf = cStringIO.StringIO() out.save(buf, mimeType) data = buf.getvalue() bucket_name = 'dem-ode' bucket = '/' + bucket_name filename = bucket + '/' + urllib.quote(u"{0}".format( time.time()).encode('utf8')) with gcs.open(filename, 'w') as f: f.write(data) blobstore_filename = "/gs" + filename # this is needed if you want to continue using blob_keys. ieurl = images.get_serving_url( blobstore.BlobKey( blobstore.create_gs_key(str(blobstore_filename)))) # self.response.out.write( '<img width="100%" height="100%" src="' + ieurl + '"/>') self.response.out.write( '<a style="float:right;position:absolute;top:8px;right:8px;width:48px;height:48px" href="' + ieurl + '" download>' + '<img style="width: 48px;height: 48px;" src="https://upload.wikimedia.org/wikipedia/commons/thumb/1/1e/Download-Icon.png/480px-Download-Icon.png"></a>' ) except: self.response.headers[b'Content-Type'] = b'text/plain' self.response.out.write( 'Image size is too large. Can\'t handle') except: self.response.headers[b'Content-Type'] = b'text/plain' self.response.out.write('Image size is too large. Can\'t handle')
def store_thumbnail_in_gcs(self, thumbnail_key, thumbnail): write_retry_params = gcs.RetryParams(backoff_factor=1.1) filename = '/' + THUMBNAIL_BUCKET + '/' + thumbnail_key with gcs.open(filename, 'w') as filehandle: filehandle.write(thumbnail)
def _load_config(self): """Loads the contents of the config file from GCS.""" gcs_file = gcs.open(self.file_path) contents = gcs_file.read() gcs_file.close() self.contents = yaml.load(contents)