def testSortFile(self): """Test sorting a file.""" input_file = files.blobstore.create() input_data = [ (str(i), "_" + str(i)) for i in range(100)] with files.open(input_file, "a") as f: with records.RecordsWriter(f) as w: for (k, v) in input_data: proto = file_service_pb.KeyValue() proto.set_key(k) proto.set_value(v) w.write(proto.Encode()) files.finalize(input_file) input_file = files.blobstore.get_file_name( files.blobstore.get_blob_key(input_file)) p = shuffler._SortChunksPipeline("testjob", [input_file]) p.start() test_support.execute_until_empty(self.taskqueue) p = shuffler._SortChunksPipeline.from_id(p.pipeline_id) input_data.sort() output_files = p.outputs.default.value[0] output_data = [] for output_file in output_files: with files.open(output_file, "r") as f: for binary_record in records.RecordsReader(f): proto = file_service_pb.KeyValue() proto.ParseFromString(binary_record) output_data.append((proto.key(), proto.value())) self.assertEquals(input_data, output_data)
def moveCoverToBlobstore(album): if not album.small_filetype: return from slughifi import slughifi fn = "%s_%s"%(slughifi(album.artist), slughifi(album.title)) small_file = files.blobstore.create(mime_type=album.small_filetype, _blobinfo_uploaded_filename="%s_small.png"%fn) large_file = files.blobstore.create(mime_type=album.large_filetype, _blobinfo_uploaded_filename="%s_big.png"%fn) with files.open(small_file, 'a') as small: small.write(album.small_cover) with files.open(large_file, 'a') as large: large.write(album.large_cover) files.finalize(small_file) files.finalize(large_file) album.cover_small = files.blobstore.get_blob_key(small_file) album.cover_large = files.blobstore.get_blob_key(large_file) del album.small_cover del album.large_cover del album.large_filetype del album.small_filetype album.put()
def get(self, firmid, process): self.json_content() firm_key = ndb.Key('Firm', firmid) if process == 'process_hier': deferred.defer( process_hier ) self.w( json.dumps("ok") ) if process == 'get_hier': self.w(self.request.get('callback') + '(') fname = '/gs/frl-arch/' + firmid + '/json/proj-detailed.json' with files.open(fname, 'r') as f: data = f.read(60 * 1000) while data: self.w( data ) data = f.read(60 * 1000) self.w(');') elif process == 'mail': mail.send_mail( sender = 'Uri London <*****@*****.**>', to = 'Live Person <*****@*****.**>', subject = 'your test has succeeded', body = 'Dear Live Person,\n\n I\'m happy to tell you that your test is good' ) self.w( json.dumps('ok') ) elif process == 'test_file': fname = '/gs/frl-arch/test.html' wfname = files.gs.create( fname, mime_type='text/html', acl='public-read') with files.open(wfname, 'a') as f: f.write( 'hello world' ) f.write( 'bye bye' ) files.finalize(wfname)
def testMergeFiles(self): """Test merging multiple files.""" input_data = [(str(i), "_" + str(i)) for i in range(100)] input_data.sort() input_file = files.blobstore.create() with files.open(input_file, "a") as f: with records.RecordsWriter(f) as w: for (k, v) in input_data: proto = file_service_pb.KeyValue() proto.set_key(k) proto.set_value(v) w.write(proto.Encode()) files.finalize(input_file) input_file = files.blobstore.get_file_name( files.blobstore.get_blob_key(input_file)) p = TestMergePipeline([input_file, input_file, input_file]) p.start() test_support.execute_until_empty(self.taskqueue) p = TestMergePipeline.from_id(p.pipeline_id) output_file = p.outputs.default.value[0] output_data = [] with files.open(output_file, "r") as f: for record in records.RecordsReader(f): output_data.append(record) expected_data = [ str((k, [v, v, v], False)) for (k, v) in input_data] self.assertEquals(expected_data, output_data)
def storeImageFromBGG(bgg_image_url): '''Returns a blob_key for an image fetched from BGG and stored to the BlobStore. ''' _trace = TRACE+'storeImageFromBGG():: ' logging.info(_trace) image_blob_key = None if bgg_image_url is not None: image_name = bgg_image_url.split('/')[-1] file_name = files.blobstore.create(mime_type='image/jpeg', _blobinfo_uploaded_filename=image_name) # Sometimes the image from BGG is too large for GAE ... try: image = urllib2.urlopen(bgg_image_url) with files.open(file_name, 'a') as f: f.write(image.read()) # Get a smaller version ... except apiproxy_errors.RequestTooLargeError: bgg_md_image_url = bgg_image_url.replace('.jpg', '_md.jpg') image = urllib2.urlopen(bgg_md_image_url) with files.open(file_name, 'a') as f: f.write(image.read()) finally: files.finalize(file_name) image_blob_key = files.blobstore.get_blob_key(file_name) else: return None return image_blob_key
def process(owner, content): im_data = io.BytesIO(content) im_copy = io.BytesIO(content) im = Image.open(im_data) cf = imaging.ColorFinder(im) top = imaging.ColorUtil.generate_color_panes(tuple(cf.strategy_enhanced_complements())) output = StringIO.StringIO() top.save(output, format="JPEG") palette = io.BytesIO(output.getvalue()) palette_copy = io.BytesIO(output.getvalue()) output.close() upload = Picture(parent=Picture.picture_key(owner)) upload.owner = owner file_name = files.blobstore.create(mime_type='image/jpeg') with files.open(file_name, 'a') as f: f.write(im_copy.read()) files.finalize(file_name) picture_blob_key = files.blobstore.get_blob_key(file_name) upload.picture = str(picture_blob_key) palette_name = files.blobstore.create(mime_type='image/jpeg') with files.open(palette_name, 'a') as f: f.write(palette_copy.read()) files.finalize(palette_name) palette_blob_key = files.blobstore.get_blob_key(palette_name) upload.palette = str(palette_blob_key) upload.put() return palette
def encoded_video_receiver(mp4_url, webm_url, video_id): logging.info("starting the encoded_video_receiver") if mp4_url and webm_url: video = Video.get_by_id(int(video_id)) try: # adding the mp4 video ref mp4_file_name = files.blobstore.create(mime_type="video/mp4") mp4_video = urllib2.urlopen(mp4_url) with files.open(mp4_file_name, "a") as mp4_f: mp4_f.write(mp4_video.read()) files.finalize(mp4_file_name) mp4_blob_key = files.blobstore.get_blob_key(mp4_file_name) video.mp4_video_ref = mp4_blob_key # adding the webm video ref webm_file_name = files.blobstore.create(mime_type="video/webm") webm_video = urllib2.urlopen(webm_url) with files.open(webm_file_name, "a") as webm_f: webm_f.write(webm_video.read()) files.finalize(webm_file_name) webm_blob_key = files.blobstore.get_blob_key(webm_file_name) video.webm_video_ref = webm_blob_key video.put() logging.info("just saved the video in the datastore") except urllib2.HTTPError, e: logging.error("HTTPError: Failed somehow with a HTTPerror: " + str(e)) # time.sleep(10) except Exception, e: logging.error("Exception: Failed somehow: " + str(e))
def get(self): path = self.resolve_doc_path() gs_key = blobstore.create_gs_key(path) age = self.get_cache_age(path) self.response.headers['Cache-Control'] = 'max-age=' + \ str(age) + ',s-maxage=' + str(age) # is there a better way to check if a file exists in cloud storage? # AE will serve a 500 if the file doesn't exist, but that should # be a 404 path_exists = memcache.get(path) if path_exists is not None: if path_exists == "1": self.send_blob(gs_key) else: self.error(404) else: try: # just check for existance files.open(path, 'r').close() memcache.add(key=path, value="1", time=ONE_DAY) self.send_blob(gs_key) except files.file.ExistenceError: memcache.add(key=path, value="0", time=ONE_DAY) self.error(404)
def testPartialRecords(self): """Test merging into partial key values.""" try: self._prev_max_values_count = shuffler._MergePipeline._MAX_VALUES_COUNT # force max values count to extremely low value. shuffler._MergePipeline._MAX_VALUES_COUNT = 1 input_data = [('1', 'a'), ('2', 'b'), ('3', 'c')] input_data.sort() input_file = files.blobstore.create() with files.open(input_file, "a") as f: with records.RecordsWriter(f) as w: for (k, v) in input_data: proto = file_service_pb.KeyValue() proto.set_key(k) proto.set_value(v) w.write(proto.Encode()) files.finalize(input_file) input_file = files.blobstore.get_file_name( files.blobstore.get_blob_key(input_file)) p = TestMergePipeline([input_file, input_file, input_file]) p.start() test_support.execute_until_empty(self.taskqueue) p = TestMergePipeline.from_id(p.pipeline_id) output_file = p.outputs.default.value[0] output_data = [] with files.open(output_file, "r") as f: for record in records.RecordsReader(f): output_data.append(record) expected_data = [ ('1', ['a'], True), ('1', ['a'], True), ('1', ['a'], False), ('2', ['b'], True), ('2', ['b'], True), ('2', ['b'], False), ('3', ['c'], True), ('3', ['c'], True), ('3', ['c'], False), ] self.assertEquals([str(e) for e in expected_data], output_data) finally: shuffler._MergePipeline._MAX_VALUES_COUNT = self._prev_max_values_count
def save_blob(self, blobdata='blobdata'): file_name = files.blobstore.create(mime_type='application/octet-stream') with files.open(file_name, 'a') as f: f.write(blobdata) files.finalize(file_name) blob_key = files.blobstore.get_blob_key(file_name) return blob_key
def uploadXmlsToBlobstore(): """Writing Files to the Blobstore (Experimental) via remote api to production server not working! https://developers.google.com/appengine/docs/python/blobstore/overview#Writing_Files_to_the_Blobstore http://stackoverflow.com/questions/8201283/google-app-engine-how-to-write-large-files-to-google-cloud-storage http://stackoverflow.com/questions/3530990/upload-data-to-blobstore-using-remote-api http://stackoverflow.com/questions/6545247/erratic-problem-with-app-engine-when-writing-files-directly-to-the-blobstore """ romn_dir = getRomnDir() for dirpath, dirnames, filenames in os.walk(romn_dir): for filename in filenames: path = os.path.join(dirpath, filename) key = path[len(romn_dir):] print('uploading %s ...' % key) with open(path, 'rb') as f: # Create the file file_name = files.blobstore.create( mime_type='application/octet-stream') # Open the file and write to it with files.open(file_name, 'a') as f2: f2.write(f.read()) # Finalize the file. Do this before attempting to read it. files.finalize(file_name) XmlBlobKey( id=key, blob_key=files.blobstore.get_blob_key(file_name)).put()
def _save(self, name, content): name = name.replace('\\', '/') if hasattr(content, 'file') and \ hasattr(content.file, 'blobstore_info'): data = content.file.blobstore_info elif hasattr(content, 'blobstore_info'): data = content.blobstore_info elif isinstance(content, File): guessed_type = mimetypes.guess_type(name)[0] file_name = files.blobstore.create( mime_type=guessed_type or 'application/octet-stream', _blobinfo_uploaded_filename=name) with files.open(file_name, 'a') as f: for chunk in content.chunks(): f.write(chunk) files.finalize(file_name) data = files.blobstore.get_blob_key(file_name) else: raise ValueError("The App Engine storage backend only supports " "BlobstoreFile instances or File instances.") if isinstance(data, (BlobInfo, BlobKey)): # We change the file name to the BlobKey's str() value. if isinstance(data, BlobInfo): data = data.key() return '%s/%s' % (data, name.lstrip('/')) else: raise ValueError("The App Engine Blobstore only supports " "BlobInfo values. Data can't be uploaded " "directly. You have to use the file upload " "handler.")
def write_to_blobstore(data, mime_type, name=None): '''Writes a file to the App Engine blobstore and returns an instance of a BlobKey if successful. :param data: Blob data. :param mime_type: String, mime type of the blob. :param name: String, name of the blob. :returns: Instance of a `BlobKey`. ''' if not name: name = ''.join(random.choice(string.letters) for x in range(DEFAULT_NAME_LEN)) blob = files.blobstore.create( mime_type=mime_type, _blobinfo_uploaded_filename=name) with files.open(blob, 'a', exclusive_lock=True) as f: f.write(data) files.finalize(blob) result = files.blobstore.get_blob_key(blob) # issue with the local development SDK. we can only write to the blobstore # so fast, so set a retry_count and delay the execution thread between # each attempt.. for i in range(1, WRITE_MAX_RETRIES): if result: break else: logging.debug( 'blob still None.. will retry to write to blobstore..') time.sleep(WRITE_SLEEP_SECONDS) result = files.blobstore.get_blob_key(blob) logging.debug('File written to blobstore: key: "%s"', result) return result
def write_blob(self, data, info): ''' This is where the image is chucked into the blobstore. The approach being used is not the one that's recommened in the google docs. Their method is to use the direct upload url. However direct upload urls do not seem to work with CORS (and this is code That I have copy and pasted from a project that needed CORS ''' (root, ext) = os.path.splitext(info['name']) img = images.Image(data) if img: blob = files.blobstore.create( _blobinfo_uploaded_filename=info['name'], mime_type=IMAGE_TYPES[ext.lower()] ) with files.open(blob, 'a') as f: f.write(data) files.finalize(blob) key = files.blobstore.get_blob_key(blob) return key, img return None, None
def testLotsOfValuesForSingleKey(self): TestEntity(data=str(1)).put() # Run Mapreduce p = mapreduce_pipeline.MapreducePipeline( "test", __name__ + ".map_yield_lots_of_values", __name__ + ".reduce_length", input_reader_spec=input_readers.__name__ + ".DatastoreInputReader", output_writer_spec= output_writers.__name__ + ".BlobstoreRecordsOutputWriter", mapper_params= { "entity_kind": __name__ + "." + TestEntity.__name__, }, shards=16) p.start() test_support.execute_until_empty(self.taskqueue) self.assertEquals(1, len(self.emails)) self.assertTrue(self.emails[0][1].startswith( "Pipeline successful:")) # Verify reduce output. p = mapreduce_pipeline.MapreducePipeline.from_id(p.pipeline_id) output_data = [] for output_file in p.outputs.default.value: with files.open(output_file, "r") as f: for record in records.RecordsReader(f): output_data.append(record) expected_data = ["('1', 50000)"] expected_data.sort() output_data.sort() self.assertEquals(expected_data, output_data)
def testSingleShard(self): entity_count = 1000 for _ in range(entity_count): TestEntity().put() mapreduce_id = control.start_map( "test_map", __name__ + ".test_handler_yield_key_str", DATASTORE_READER_NAME, { "entity_kind": __name__ + "." + TestEntity.__name__, "filesystem": "gs", "gs_bucket_name": "bucket" }, shard_count=4, base_path="/mapreduce_base_path", output_writer_spec=FILE_WRITER_NAME) test_support.execute_until_empty(self.taskqueue) mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id) filenames = output_writers.FileOutputWriter.get_filenames( mapreduce_state) self.assertEqual(1, len(filenames)) self.assertTrue(filenames[0].startswith("/gs/bucket/")) with files.open(filenames[0], "r") as f: data = f.read(10000000) self.assertEquals(1000, len(data.strip().split("\n")))
def write_wav(samples): '''Samples are assumed to be in -1 to 1 range and mono, 44100 Hz. Returns key of newly created blob.''' return from google.appengine.api import files file_name = files.blobstore.create(mime_type='application/octet-stream') bytes = [struct.pack('h', v * 32767) for v in samples] byte_str = ''.join(bytes) # Write wav data into a stringIO import StringIO buffer = StringIO.StringIO() wav = wave.open(buffer, 'wb') wav.setnchannels(1) wav.setsampwidth(2) wav.setframerate(44100) wav.writeframes(byte_str) wav.close() with files.open(file_name, 'a') as f: f.write(buffer) files.finalize(file_name) blob_key = files.blobstore.get_blob_key(file_name) return blob_key
def testRecordsReader(self): """End-to-end test for records reader.""" input_file = files.blobstore.create() input_data = [str(i) for i in range(100)] with files.open(input_file, "a") as f: with records.RecordsWriter(f) as w: for record in input_data: w.write(record) files.finalize(input_file) input_file = files.blobstore.get_file_name( files.blobstore.get_blob_key(input_file)) mapreduce_id = control.start_map( "test_map", __name__ + ".TestHandler", "mapreduce.input_readers.RecordsReader", { "file": input_file }, shard_count=4, base_path="/mapreduce_base_path") test_support.execute_until_empty(self.taskqueue) self.assertEquals(100, len(TestHandler.processed_entites))
def POST(self): """Receive HTTP POST (json-format data), and store data to blobstore. """ data = json.loads(web.data()) # Create the file # set filename property in BlobStore # http://stackoverflow.com/questions/5697844/how-to-set-filename-property-in-blobstore file_name = files.blobstore.create( mime_type='application/octet-stream', _blobinfo_uploaded_filename=data['key']) # Open the file and write to it with files.open(file_name, 'a') as f: f.write(base64.b64decode(data['payload'])) # Finalize the file. Do this before attempting to read it. files.finalize(file_name) # Get the file's blob key blob_key = files.blobstore.get_blob_key(file_name) # store blob_key in datastore XmlBlobKey(id=data['key'], blob_key=blob_key).put() return 'OK<br />%s<br />%s' % (data['key'], blob_key)
def _save(self, name, content): name = name.replace('\\', '/') if hasattr(content, 'file') and \ hasattr(content.file, 'blobstore_info'): data = content.file.blobstore_info elif hasattr(content, 'blobstore_info'): data = content.blobstore_info elif isinstance(content, File): guessed_type = mimetypes.guess_type(name)[0] file_name = files.blobstore.create(mime_type=guessed_type or 'application/octet-stream', _blobinfo_uploaded_filename=name) with files.open(file_name, 'a') as f: for chunk in content.chunks(): f.write(chunk) files.finalize(file_name) data = files.blobstore.get_blob_key(file_name) else: raise ValueError("The App Engine storage backend only supports " "BlobstoreFile instances or File instances.") if isinstance(data, (BlobInfo, BlobKey)): # We change the file name to the BlobKey's str() value. if isinstance(data, BlobInfo): data = data.key() return '%s/%s' % (data, name.lstrip('/')) else: raise ValueError("The App Engine Blobstore only supports " "BlobInfo values. Data can't be uploaded " "directly. You have to use the file upload " "handler.")
def flush(self): """Flush pool contents.""" buf = _StringWriter() with records.RecordsWriter(buf) as w: for record in self._buffer: w.write(record) str_buf = buf.to_string() if not self._exclusive and len(str_buf) > _FILES_API_MAX_SIZE: raise errors.Error( "Buffer too big. Can't write more than %s bytes in one request: " "risk of writes interleaving. Got: %s" % (_FILES_API_MAX_SIZE, len(str_buf))) start_time = time.time() with files.open(self._filename, "a", exclusive_lock=self._exclusive) as f: f.write(str_buf) if self._ctx: operation.counters.Increment(COUNTER_IO_WRITE_BYTES, len(str_buf))(self._ctx) if self._ctx: operation.counters.Increment( COUNTER_IO_WRITE_MSEC, int( (time.time() - start_time) * 1000))(self._ctx) self._buffer = [] self._size = 0 gc.collect()
def run(self, job_name, gs_bucket_name, shards=8): file_names = [] try: with files.open("/gs/"+gs_bucket_name+TARGETS_FILE_PATH, 'r') as fp: targets = fp.read() file_names = targets.splitlines() except Exception: logging.warning("Can't find file:" + TARGETS_FILE_PATH) if len(file_names)>0: yield mapreduce_pipeline.MapperPipeline( job_name, __name__ + "._to_datastore_map", "lakshmi.cooperate.input_readers.GoogleStorageLineInputReader", output_writer_spec="mapreduce.output_writers.BlobstoreOutputWriter", params={ "input_reader":{ "file_paths": file_names, "gs_bucket_name": gs_bucket_name }, "output_writer":{ "mime_type": "text/plain" } }, shards=shards)
def generate_all(): q = MyFile.all() mfiles = q.fetch(100) # delete all old files and data for mf in mfiles: bk = mf.blob_key blobstore.delete(bk) # delete data entity mf.delete() # dict of zipfiles being generated and their relevant generators generators = { 'openvpn.zip': generate_ovpn, 'windows.zip': generate_win, 'linux.zip': generate_linux, 'mac.zip': generate_mac, 'android.zip': generate_android } ip_data = fetch_ip_data() for fn, g in generators.iteritems(): data = g(ip_data) z = zipit(data) #compress the data blob_file = files.blobstore.create('application/zip', _blobinfo_uploaded_filename=fn) with files.open(blob_file, 'a') as f: f.write(z) files.finalize(blob_file) blob_key = files.blobstore.get_blob_key(blob_file) mf = MyFile(name=fn, blob_key=str(blob_key)) mf.update_date = datetime.datetime.now().date() mf.put()
def WriteTempImageFile(stream): blob = files.blobstore.create(_blobinfo_uploaded_filename='ImageTemp') with files.open(blob, 'a') as f: f.write(stream.getvalue()) files.finalize(blob) return files.blobstore.get_blob_key(blob)
def store_raw_data_as_blob(data,name,content_type): logging.info('store (bin) raw_data as %s (%s)'%(name,content_type)) file_name = files.blobstore.create(mime_type=content_type, _blobinfo_uploaded_filename = name) with files.open(file_name, 'a') as out: out.write(data) files.finalize(file_name) blob_key = files.blobstore.get_blob_key(file_name) #http://code.google.com/p/googleappengine/issues/detail?id=4872 #FIXME if not blob_key: logging.info('again....1') time.sleep(1) blob_key = files.blobstore.get_blob_key(file_name) if not blob_key: logging.info('again....2') time.sleep(1) blob_key = files.blobstore.get_blob_key(file_name) #endofhack logging.info('file key:%s'%blob_key) return blob_key
def write_blob(self, data, info): blob = files.blobstore.create(mime_type=info['type'], _blobinfo_uploaded_filename=info['name']) with files.open(blob, 'a') as f: f.write(data) files.finalize(blob) return files.blobstore.get_blob_key(blob)
def post(self): fkstring = self.request.get("filekey") filekey = FileKey(key_name =fkstring, parent=filelist_key()) self.response.out.write("File key: "+ filekey.key().id_or_name()) upload_files = self.get_uploads('file') blob_file = upload_files[0] filekey.blobinfokey = str(blob_file.key()) if MEMCACHE and blob_file.size <= 102400: self.response.out.write("</br> Inserted in MEMCACHE") memcache.add(fkstring, blob_file) filekey.filelocation = "memcache" self.response.out.write("""<br><b><a href="/">RETURN TO HOME</a></b><br>""") else: self.response.out.write("</br> Inserted in GOOGLE CLOUD STORAGE") wr_path = files.gs.create(BUCKET_PATH+"/"+filekey.key().id_or_name(), mime_type='text/plain', acl='public-read') with files.open(wr_path, 'a') as filepath: start = 0 fetchsz = blobstore.MAX_BLOB_FETCH_SIZE - 1 filesize=blob_file.size while start < filesize: filepath.write( blobstore.fetch_data(blob_file, start, start+fetchsz)) start = start + fetchsz files.finalize(wr_path) filekey.filelocation = "cloudstorage" self.response.out.write("""<br><b><a href="/">RETURN TO HOME</a></b><br>""") filekey.put()
def get(self): """Loads the data file located at |filename|. Args: filename: The filename for the data file to be loaded. """ yesterday = datetime.date.today() - datetime.timedelta(1) yesterday_formatted = yesterday.strftime("%Y.%m.%d") filename = 'histograms/daily/%s/Everything' % (yesterday_formatted) if settings.PROD: try: with files.open(BIGSTORE_BUCKET + filename, 'r') as unused_f: pass except files.file.ExistenceError, e: self.response.write(e) return # The file exists; serve it. blob_key = blobstore.create_gs_key(BIGSTORE_BUCKET + filename) blob_reader = blobstore.BlobReader(blob_key, buffer_size=3510000) try: result = blob_reader.read() finally: blob_reader.close()
def post(self): fkeystr = self.request.get("filekey") regexpstr = self.request.get("regexp") filekeys = FileKey.all() filekeys.filter('__key__ =', db.Key.from_path("FileKey", fkeystr, parent=filelist_key())) if filekeys.count() == 0: self.response.out.write("Key(%s) does NOT exists." % fkeystr) else: for ifile in filekeys: self.response.out.write("Matching contents") if ifile.filelocation == "memcache": blob_info = memcache.get(ifile.key().id_or_name()) #elf.send_blob(blob_info) blob_reader = blob_info.open() for line in blob_reader: if regexpstr in line: self.response.out.write(line) else: #self.response.out.write("File is in Cloud Storage") with files.open(BUCKET_PATH+"/"+ifile.key().id_or_name(), 'r') as fp: buf = fp.read(1000) while buf: if regexpstr in buf: self.response.out.write(buf) buf = fp.read(1000000)
def testHugeTaskUseDatastore(self): """Test map job with huge parameter values.""" input_file = files.blobstore.create() input_data = [str(i) for i in range(100)] with files.open(input_file, "a") as f: with records.RecordsWriter(f) as w: for record in input_data: w.write(record) files.finalize(input_file) input_file = files.blobstore.get_file_name( files.blobstore.get_blob_key(input_file)) mapreduce_id = control.start_map( "test_map", __name__ + ".TestHandler", "mapreduce.input_readers.RecordsReader", { "file": input_file, # the parameter can't be compressed and wouldn't fit into # taskqueue payload "huge_parameter": random_string(900000) }, shard_count=4, base_path="/mapreduce_base_path") test_support.execute_until_empty(self.taskqueue) self.assertEquals(100, len(TestHandler.processed_entites)) self.assertEquals([], model._HugeTaskPayload.all().fetch(100))
def output(self, filename, file_out): content = file_out.read() base, ext = os.path.splitext(filename) if ext == ".css": mimetype = "text/css" elif ext == ".js": mimetype = "text/javascript" else: mimetype = "application/octet-stream" already_exists = False for info in BlobInfo.all().filter('content_type = ', mimetype): if info.filename == filename: already_exists = True continue #Clear out old blobs if info.filename.split(".")[0] == filename.split(".")[0]: logging.debug("Deleting: %s", info.filename) info.delete() if not already_exists: logging.info("Creating: %s", filename) result = files.blobstore.create(mime_type=mimetype, _blobinfo_uploaded_filename=filename) with files.open(result, "a") as f: f.write(content) files.finalize(result) blob_key = files.blobstore.get_blob_key(result) while not blob_key: blob_key = files.blobstore.get_blob_key(result)
def post(self): countries = self.all_countries tourist_id = int(self.get_cookie("query")[0]) tourist = Tourist.Tourist.get_by_id(tourist_id) new_email = self.request.get("email") first_name = self.request.get("first_name") last_name = self.request.get("last_name") country = self.request.get("country") state = self.request.get("state") picture = self.request.POST.get("photo") # self.write(picture.filename[picture.filename.find(".") + 1 : ]) profile_args = {"email" : new_email, "country" : country, "first_name" : first_name, "last_name" : last_name, "state" : state, "picture" : picture} if self.validate_email(new_email) and self.validate_name(first_name) and self.validate_name(last_name): Tourist.Tourist.updateTourist(tourist, new_email, first_name, last_name, country, state) if picture != None and picture != "": picture_extension = picture.filename[picture.filename.rfind(".") : ] picture_url = "/tourbly/profile_pictures/" + str(tourist.key().id()) + self.rand_salt("picture") + picture_extension picture_name = "/gs" + picture_url writable_file_name = files.gs.create(picture_name, mime_type='image/jpeg', acl='public-read') with files.open(writable_file_name, 'a') as f: f.write(picture.file.read()) files.finalize(writable_file_name) tourist.picture = "http://storage.googleapis.com" + picture_url tourist.put() self.render("profile.html", isLoggedIn = self.check_session("query"), profile_args = profile_args, success_message = "Your profile has been updated successfully", tourist = tourist, countries = countries) else: self.render("profile.html", email_error = self.profile_email_error_prompt(tourist.email, new_email), profile_args = profile_args, success_message = "there is something wrong", countries = countries)
def write_to_blobstore(content, old_blobinfo=None): """Write content to blobstore. Args: content: A byte-string. old_blobinfo: An optional BlobInfo object used for comparison. If the md5_hash of the old blob and the new content are equal, the old blobkey is returned and nothing new is written. Returns: A BlobKey. """ # Require pre-encoded content. assert not isinstance(content, unicode) # Blob de-duping: if the blob content is the same as the old file, # just use the old blob key to save space and RPCs. if (old_blobinfo and old_blobinfo.md5_hash == hashlib.md5(content).hexdigest()): return old_blobinfo.key() # write new blob. filename = blobstore_files.blobstore.create() content_file = cStringIO.StringIO(content) blobstore_file = blobstore_files.open(filename, 'a') # Blobstore writes cannot exceed an RPC size limit, so chunk the writes. while True: content_chunk = content_file.read(BLOBSTORE_APPEND_CHUNK_SIZE) if not content_chunk: break blobstore_file.write(content_chunk) blobstore_file.close() blobstore_files.finalize(filename) blob_key = blobstore_files.blobstore.get_blob_key(filename) return blob_key
def post(self, post): form = PostForm(data=self.request.POST, instance=post, initial={'draft': post and post.published is None}) if form.is_valid(): post = form.save(commit=False) image_url = form.clean_data['image_url'] if image_url: try: u = urllib2.urlopen(image_url) file_name = files.blobstore.create(mime_type='image/jpeg') with files.open(file_name, 'a') as f: f.write(u.read()) files.finalize(file_name) post.image_id = str(files.blobstore.get_blob_key(file_name)) except: # Not sure how to use ErrorList in Django 0.96 # form._errors['image'] = '' self.render_form(form) return if form.clean_data['draft']:# Draft post post.published = datetime.datetime.max post.put() else: if not post.path: # Publish post post.updated = post.published = datetime.datetime.now() else:# Edit post post.updated = datetime.datetime.now() post.publish() self.render_to_response("published.html", { 'post': post, 'draft': form.clean_data['draft']}) else: self.render_form(form)
def ArchiveToGoogleStorage(self, **unused_args): """Posts data in compressed JSON format to Google Storage for Developers. Allows a file containing the requested data to be stored in Google Storage for developers for later download. Please see _Archive for details on how arguments are handled and data is packaged. In preparation for use of this method it is important that the bucket and account that will be used be properly prepared by following these instructions (http://goo.gl/S0LRl) paying careful attention to the ACLs. This information should be used to adjust the pertinent parts of this application's config file. Raises: DeadlineExceededError: Handled in the case where a request exceeds the response time limit. A error is displayed in the VERY short time for reporting errors. No new exceptions generated here. """ #TODO(mdw) Unit test needed. try: archive_dir, archive_data = self._Archive() anonymize = self.request.get('anonymize') anonymize = not not anonymize # Create the file if not anonymize: bucket = config_private.ARCHIVE_GS_BUCKET acl = config_private.ARCHIVE_GS_ACL else: bucket = config.ARCHIVE_GS_BUCKET_PUBLIC acl = config.ARCHIVE_GS_ACL_PUBLIC gs_archive_name = '/gs/%s/%s.zip' % (bucket, archive_dir) gs_archive = files.gs.create( gs_archive_name, mime_type=config.ARCHIVE_CONTENT_TYPE, acl=acl, content_disposition=(config.ARCHIVE_CONTENT_DISPOSITION_BASE % archive_dir)) # Open the file and write the data. with files.open(gs_archive, 'a') as f: f.write(archive_data) # Finalize (a special close) the file. files.finalize(gs_archive) self.response.headers['Content-Type'] = 'application/json' self.response.out.write( '{\'status\':200, \'archive_name\':\'%s\'}' % gs_archive_name) except DeadlineExceededError, e: logging.exception(e) self.response.clear() self.response.set_status(500) #NOTE: if you see this error make sure it is run on a backend instance. self.response.headers['Content-Type'] = 'application/json' self.response.out.write( ('{\'status\':500, \'error_name\':\'%s\', ' '\'error_value\':\'%s\'}' % ('DeadlineExceededError', e)))
def flush(self): """Flush pool contents.""" # Write data to in-memory buffer first. buf = cStringIO.StringIO() with records.RecordsWriter(buf) as w: for record in self._buffer: w.write(record) w._pad_block() str_buf = buf.getvalue() buf.close() if not self._exclusive and len(str_buf) > _FILES_API_MAX_SIZE: # Shouldn't really happen because of flush size. raise errors.Error( "Buffer too big. Can't write more than %s bytes in one request: " "risk of writes interleaving. Got: %s" % (_FILES_API_MAX_SIZE, len(str_buf))) # Write data to file. start_time = time.time() with files.open(self._filename, "a", exclusive_lock=self._exclusive) as f: f.write(str_buf) if self._ctx: operation.counters.Increment(COUNTER_IO_WRITE_BYTES, len(str_buf))(self._ctx) if self._ctx: operation.counters.Increment( COUNTER_IO_WRITE_MSEC, int( (time.time() - start_time) * 1000))(self._ctx) # reset buffer self._buffer = [] self._size = 0 gc.collect()
def get(self, key): file_name = files.blobstore.create(mime_type='text/csv') ref = Ref.get(db.Key(key)) cols = get_all_label(ref) header = ['Name', 'Parents'] for col in cols: header.append('%s - Owner' % col) header.append('%s - Type' % col) header.append('%s - Value' % col) with files.open(file_name, 'a') as f: writer = UnicodeWriter(f, delimiter=',', quoting=csv.QUOTE_MINIMAL) writer.writerow(header) p_props = parent_props(ref) writerow(ref, writer, cols, p_props) # Finalize the file. Do this before attempting to read it. files.finalize(file_name) # Get the file's blob key blob_key = files.blobstore.get_blob_key(file_name) self.response.headers[ 'Content-Disposition'] = 'attachment; filename=%s_.csv' % ref.name self.send_blob(blob_key)
def testMultipleShards(self): entity_count = 1000 for _ in range(entity_count): TestEntity().put() mapreduce_id = control.start_map( "test_map", __name__ + ".test_handler_yield_key_str", DATASTORE_READER_NAME, { "entity_kind": __name__ + "." + TestEntity.__name__, "output_sharding": "input", }, shard_count=4, base_path="/mapreduce_base_path", output_writer_spec=BLOBSTORE_WRITER_NAME) test_support.execute_until_empty(self.taskqueue) mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id) filenames = output_writers.BlobstoreOutputWriter.get_filenames( mapreduce_state) self.assertEqual(4, len(filenames)) file_lengths = [] for filename in filenames: self.assertTrue(filename.startswith("/blobstore/")) self.assertFalse(filename.startswith("/blobstore/writable:")) with files.open(filename, "r") as f: data = f.read(10000000) file_lengths.append(len(data.strip().split("\n"))) self.assertEqual(1000, sum(file_lengths))
def post(self): upload_files = self.get_uploads('file') blob_info = upload_files[0] # Resize the image image = images.Image(blob_key=blob_info.key()) image.resize(width=WOOF_FEED_ITEM_IMAGE_MAX_WIDTH, height=WOOF_FEED_ITEM_IMAGE_MAX_HEIGHT) thumbnail = image.execute_transforms(output_encoding=images.JPEG) # Save Resized Image back to blobstore file_name = files.blobstore.create(mime_type='image/jpeg') with files.open(file_name, 'a') as f: f.write(thumbnail) files.finalize(file_name) # Remove the original image blobstore.delete(blob_info.key()) blob_key = files.blobstore.get_blob_key(file_name) # New FeedImage feed_image = FeedImage() feed_image.data = BlobInfo.get(blob_key) feed_image.width = image.width feed_image.height = image.height feed_image.put() # Create new FeedItem feed_item = FeedItem() feed_item.text = self.request.get("text") feed_item.image = feed_image feed_item.put() self.redirect('/')
def flush(self): """Flush pool contents.""" # Write data to in-memory buffer first. buf = cStringIO.StringIO() with records.RecordsWriter(buf) as w: for record in self._buffer: w.write(record) w._pad_block() str_buf = buf.getvalue() buf.close() if not self._exclusive and len(str_buf) > _FILES_API_MAX_SIZE: # Shouldn't really happen because of flush size. raise errors.Error( "Buffer too big. Can't write more than %s bytes in one request: " "risk of writes interleaving. Got: %s" % (_FILES_API_MAX_SIZE, len(str_buf))) # Write data to file. start_time = time.time() with files.open(self._filename, "a", exclusive_lock=self._exclusive) as f: f.write(str_buf) if self._ctx: operation.counters.Increment( COUNTER_IO_WRITE_BYTES, len(str_buf))(self._ctx) if self._ctx: operation.counters.Increment( COUNTER_IO_WRITE_MSEC, int((time.time() - start_time) * 1000))(self._ctx) # reset buffer self._buffer = [] self._size = 0 gc.collect()
def upload(self, file, key, acl=None, policy=None, signature=None, success_action_redirect=None, **kwargs): """A development-only action for uploading a package archive. In production, package archives are uploaded directly to cloud storage, using a signed form for authentication. The signed form doesn't work for the development server, since it uses a local database in place of cloud storage, so this action emulates it by manually saving the file to the development database. """ if handlers.is_production(): raise handlers.http_error(404) if PrivateKey.sign(policy) != signature: raise handlers.http_error(403) write_path = files.gs.create('/gs/' + key, acl=acl) with files.open(write_path, 'a') as f: f.write(file.file.read()) files.finalize(write_path) if success_action_redirect: raise cherrypy.HTTPRedirect(success_action_redirect) cherrypy.response.status = 204 return ""
def finish_upload(file_id): file_item = File.get_by_id(file_id) size = file_item.size current_size = file_item.current_size write_path = file_item.write_path files.finalize(write_path) if size != current_size: file_item.delete() raise Exception('file size error') # check sha1 sh = hashlib.sha1() with files.open(file_item.read_path, 'r') as fp: buf = fp.read(1000000) while buf: sh.update(buf) buf = fp.read(1000000) sha1_value = sh.hexdigest() if sha1_value != file_item.sha1: file_item.delete() raise Exception('file size error') file_item.upload_finished = True file_item.put()
def testMultipleShards(self): entity_count = 1000 for _ in range(entity_count): TestEntity().put() mapreduce_id = control.start_map( "test_map", __name__ + ".test_handler_yield_key_str", DATASTORE_READER_NAME, { "entity_kind": __name__ + "." + TestEntity.__name__, "output_sharding": "input", }, shard_count=4, base_path="/mapreduce_base_path", output_writer_spec=BLOBSTORE_WRITER_NAME) test_support.execute_until_empty(self.taskqueue) mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id) filenames = output_writers.BlobstoreOutputWriter.get_filenames( mapreduce_state) self.assertEqual(4, len(set(filenames))) file_lengths = [] for filename in filenames: self.assertTrue(filename.startswith("/blobstore/")) self.assertFalse(filename.startswith("/blobstore/writable:")) with files.open(filename, "r") as f: data = f.read(10000000) file_lengths.append(len(data.strip().split("\n"))) self.assertEqual(1000, sum(file_lengths))
def _make_backup_info_blobstore(self, num): file_num = 3 dt = datetime.datetime.utcnow() for i in range(num): # _AE_Backup_Information info = AeBackupInformation( name=config.BACKUP_NAME, filesystem='blobstore', start_time=dt, complete_time=dt + datetime.timedelta(hours=1) ) info.put() dt -= datetime.timedelta(days=1) # _AE_Backup_Information_Kind_Files info_files = AeBackupInformationKindFiles() info_files.files = [] if i == (num - 1): file_num = 120 # 100 over at last for j in range(file_num): # make blobstore file file_name = files.blobstore.create( mime_type='application/octet-stream') with files.open(file_name, 'a') as f: f.write('asdfghjkl') files.finalize(file_name) info_files.files.append(file_name) info_files.key = ndb.Key( AeBackupInformationKindFiles._get_kind(), AeBackupInformationKindFiles.allocate_ids(1)[0], parent=info.key ) info_files.put()
def post(self): """Store a file after encripting it. """ rows = self.request.POST.get('file').value file_name = files.blobstore.create(mime_type='text/plain') with files.open(file_name, 'a') as raw_file: writer = csv.writer(raw_file, delimiter=',') for row in csv.reader(StringIO(rows), delimiter=','): if len(row) > 1: row[1] = self.crypt(row[1]) writer.writerow(row) files.finalize(file_name) # TODO: with high replication, the newly created file will not # show in the list. # # It show redirect to a page listing the files instead blobs = blobstore.BlobInfo.all() blob_links = [ '<a href="/serve/%s">File %s</a><br/>' % (blob.key(), index+1) for index, blob in enumerate(blobs) ] self.response.out.write("""<html> <body> <form action="/upload" enctype="multipart/form-data" method="post"> <input type="file" name="file"/><input type="submit" /> </form> <br> %s </body> </html> """ % "".join(blob_links) )
def write_to_blobstore(data, mime_type, name=None): """Writes a file to the App Engine blobstore and returns an instance of a BlobKey if successful. :param data: Blob data. :param mime_type: String, mime type of the blob. :param name: String, name of the blob. :returns: Instance of a `BlobKey`. """ if not name: name = "".join(random.choice(string.letters) for x in range(DEFAULT_NAME_LEN)) blob = files.blobstore.create(mime_type=mime_type, _blobinfo_uploaded_filename=name) with files.open(blob, "a", exclusive_lock=True) as f: f.write(data) files.finalize(blob) result = files.blobstore.get_blob_key(blob) # issue with the local development SDK. we can only write to the blobstore # so fast, so set a retry_count and delay the execution thread between # each attempt.. for i in range(1, WRITE_MAX_RETRIES): if result: break else: logging.debug("blob still None.. will retry to write to blobstore..") time.sleep(WRITE_SLEEP_SECONDS) result = files.blobstore.get_blob_key(blob) logging.debug('File written to blobstore: key: "%s"', result) return result
def WriteTempImageFile(stream): blob = files.blobstore.create(_blobinfo_uploaded_filename="ImageTemp") with files.open(blob, "a") as f: f.write(stream.getvalue()) files.finalize(blob) return files.blobstore.get_blob_key(blob)
def store_image(self, link): file_name = files.blobstore.create( mime_type='application/octet-stream') with files.open(file_name, 'a') as f: f.write(urlfetch.Fetch(link, deadline=60).content) files.finalize(file_name) self.image_name = file_name
def testDedicatedParams(self): entity_count = 1000 for _ in range(entity_count): TestEntity().put() mapreduce_id = control.start_map( "test_map", __name__ + ".test_handler_yield_key_str", "mapreduce.input_readers.DatastoreInputReader", { "input_reader": { "entity_kind": __name__ + "." + TestEntity.__name__, }, "output_writer": { "filesystem": "gs", "gs_bucket_name": "bucket", }, }, shard_count=4, base_path="/mapreduce_base_path", output_writer_spec=FILE_WRITER_NAME) test_support.execute_until_empty(self.taskqueue) mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id) filenames = output_writers.FileOutputWriter.get_filenames(mapreduce_state) self.assertEqual(1, len(filenames)) self.assertTrue(filenames[0].startswith("/gs/bucket/")) with files.open(filenames[0], "r") as f: data = f.read(10000000) self.assertEquals(1000, len(data.strip().split("\n")))
def testSingleShard(self): entity_count = 1000 for _ in range(entity_count): TestEntity().put() mapreduce_id = control.start_map( "test_map", __name__ + ".test_handler_yield_key_str", "mapreduce.input_readers.DatastoreInputReader", { "entity_kind": __name__ + "." + TestEntity.__name__, }, shard_count=4, base_path="/mapreduce_base_path", output_writer_spec=BLOBSTORE_WRITER_NAME) test_support.execute_until_empty(self.taskqueue) mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id) filenames = output_writers.BlobstoreOutputWriter.get_filenames( mapreduce_state) self.assertEqual(1, len(filenames)) blob_name = filenames[0] self.assertTrue(blob_name.startswith("/blobstore/")) self.assertFalse(blob_name.startswith("/blobstore/writable:")) with files.open(blob_name, "r") as f: data = f.read(10000000) self.assertEquals(1000, len(data.strip().split("\n")))
def execute(self, context, obj): """ Writes the CSV file """ # if already wrote a file, don't do anything semaphore = RunOnceSemaphore(context.instanceName, context) if not semaphore.readRunOnceSemaphore(payload='payload'): # fetch the CsvCounter, since it is the parent of all the other Models counter = CsvProgressCounter.get_by_key_name(context.instanceName) # fetch the single aggregated results Model aggResults = CsvIntermediateResults.get_by_key_name( context.instanceName, counter) # open the file fileName = files.blobstore.create( mime_type='application/octet-stream') with files.open(fileName, 'a') as f: # the csv module has a convenient row writing interface writer = csv.writer(f) # this queries for all the intermediate results query = CsvIntermediateResults.all().ancestor(counter) for results in query: # the aggregated results may also be in the results, so skip them if aggResults and results.key() == aggResults.key(): continue # for all the intermediate data, write the rows data = results.data for item in data: rows = self.getRows(context, obj, item, aggResults.data) if rows: for row in rows: writer.writerow(row) if aggResults: # now also write down any specific aggregated data rows rows = self.getAggregatedRows(context, obj, aggResults.data) if rows: for row in rows: writer.writerow(row) # finalize the file files.finalize(fileName) # FIXME: what to do with this? blobKey = files.blobstore.get_blob_key(fileName) # at this point we have successfully written the file, lets make sure we don't do it again # if a retry occurs downstream semaphore.writeRunOnceSemaphore(payload='payload') # store the key of the counter (ie. parent of intermediate results) for cleanup context[COUNTER_KEY_PARAM] = counter.key() return OK_EVENT
def _CreateInputFile(self): input_file = files.blobstore.create() with files.open(input_file, "a") as f: f.write("foo") files.finalize(input_file) input_file = files.blobstore.get_file_name( files.blobstore.get_blob_key(input_file)) return input_file
def create_blob_file(mime_type, binary, filename="blob_filename"): from google.appengine.api import files blob_file_name = files.blobstore.create(mime_type=mime_type, _blobinfo_uploaded_filename=filename) with files.open(blob_file_name, 'a') as f: f.write(binary) files.finalize(blob_file_name) blob_key = files.blobstore.get_blob_key(blob_file_name) return blob_key
def test_blobstore_sanity_check(self): filename = files.blobstore.create(mime_type='application/octet-stream') self.assertNotEquals(None, filename) with files.open(filename, 'a') as f: f.write('test blob data..') files.finalize(filename) blobkey = files.blobstore.get_blob_key(filename) self.assertNotEquals(None, blobkey)
def generate_pledges_csv(file_name, pledge_type, pledge_time, full_data=False): """ Generates the pledges.csv file in a deferred way """ PAGE_SIZE = 500 csv_buffer = StringIO.StringIO() w = csv.writer(csv_buffer) if not pledge_time and pledge_type == 'WpPledge': # First time through, add the column headers if full_data: headers = [ 'SOURCE', 'donationTime', 'Amount ($)', 'url_nonce', 'stripeCustomer', 'Email', 'First Name', 'Last Name', 'Address', 'Address 2', 'City', 'State', 'Zip', 'Phone', 'Country', 'Occupation', 'Employer', 'Targeting', 'Type', 'Recurring', 'Source' ] else: headers = [ 'zip', 'dollars', 'timestamp', 'date', 'city', 'state', 'latitude', 'longitude' ] w.writerow(headers) zg = Zipgun('zipgun/zipcodes') # Get the next PAGE_SIZE pledges query = getattr(model, pledge_type).all().order('donationTime') if pledge_time: # Filter instead of using 'offset' because offset is very inefficient, # according to https://developers.google.com/appengine/articles/paging query = query.filter('donationTime >= ', pledge_time) pledges = query.fetch(PAGE_SIZE + 1) next_pledge_time = None if len(pledges) == PAGE_SIZE + 1: next_pledge_time = pledges[-1].donationTime pledges = pledges[:PAGE_SIZE] # Loop through the current pledges and write them to the csv for pledge in pledges: w.writerow(pledge_row(pledge, zg, full_data)) with files.open(file_name, 'a') as f: f.write(csv_buffer.getvalue()) csv_buffer.close() if not next_pledge_time and pledge_type == 'Pledge': # Last time through, finalize the file files.finalize(file_name) else: # More to process, recursively run again next_pledge_type = pledge_type if pledge_type == 'WpPledge' and not next_pledge_time: next_pledge_type = 'Pledge' deferred.defer(generate_pledges_csv, file_name, next_pledge_type, next_pledge_time, full_data, _queue='generateCSV')
def action_edit(self): if self.section.handler.request.get('submit'): self.SITE_HEADER = self.section.handler.request.get('SITE_HEADER') self.SITE_SUB_HEADER = self.section.handler.request.get( 'SITE_SUB_HEADER') self.DEFAULT_THEME = self.section.handler.request.get( 'DEFAULT_THEME') self.GOOGLE_ANALYTICS_UA = self.section.handler.request.get( 'GOOGLE_ANALYTICS_UA') self.ROBOTS_TXT = self.section.handler.request.get('ROBOTS_TXT') if self.section.handler.request.get('FAVICON_ICO'): if self.FAVICON_ICO: BlobInfo.get(self.FAVICON_ICO).delete() data = self.section.handler.request.get('FAVICON_ICO') handle = files.blobstore.create( mime_type='image/x-icon', _blobinfo_uploaded_filename='favicon.ico') with files.open(handle, 'a') as f: f.write(data) files.finalize(handle) self.FAVICON_ICO = files.blobstore.get_blob_key(handle) self.ENABLE_THEME_PREVIEW = self.section.handler.request.get( 'ENABLE_THEME_PREVIEW') != '' self.DEBUG_MODE = self.section.handler.request.get( 'DEBUG_MODE') != '' cache.delete(CACHE_KEY) self.update() raise Exception('Redirect', self.section.action_redirect_path) f = form(self.section, self.section.full_path) f.add_control( control(self.section, 'text', 'SITE_HEADER', self.SITE_HEADER, 'Site header', 50)) f.add_control( control(self.section, 'text', 'SITE_SUB_HEADER', self.SITE_SUB_HEADER, 'Site sub-header', 50)) combined_themes = get_local_theme_namespaces( ) + get_custom_theme_namespaces() f.add_control( selectcontrol( self.section, 'DEFAULT_THEME', combined_themes, self.DEFAULT_THEME if self.DEFAULT_THEME else DEFAULT_LOCAL_THEME_TEMPLATE, 'Default theme')) f.add_control( control(self.section, 'text', 'GOOGLE_ANALYTICS_UA', self.GOOGLE_ANALYTICS_UA, 'Google analytics UA')) f.add_control( control(self.section, 'file', 'FAVICON_ICO', label='favicon.ico')) f.add_control( textareacontrol(self.section, 'ROBOTS_TXT', self.ROBOTS_TXT, 'robots.txt', 90, 5)) f.add_control( checkboxcontrol(self.section, 'ENABLE_THEME_PREVIEW', self.ENABLE_THEME_PREVIEW, 'Enable theme preview')) f.add_control( checkboxcontrol(self.section, 'DEBUG_MODE', self.DEBUG_MODE, 'Debug mode')) f.add_control(control(self.section, 'submit', 'submit', 'Submit')) return '<h2>Edit configuration</h2>%s' % unicode(f)