def testSortFile(self):
"""Test sorting a file."""
input_file = files.blobstore.create()
input_data = [
(str(i), "_" + str(i)) for i in range(100)]
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
for (k, v) in input_data:
proto = file_service_pb.KeyValue()
proto.set_key(k)
proto.set_value(v)
w.write(proto.Encode())
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
p = shuffler._SortChunksPipeline("testjob", [input_file])
p.start()
test_support.execute_until_empty(self.taskqueue)
p = shuffler._SortChunksPipeline.from_id(p.pipeline_id)
input_data.sort()
output_files = p.outputs.default.value[0]
output_data = []
for output_file in output_files:
with files.open(output_file, "r") as f:
for binary_record in records.RecordsReader(f):
proto = file_service_pb.KeyValue()
proto.ParseFromString(binary_record)
output_data.append((proto.key(), proto.value()))
self.assertEquals(input_data, output_data)
def moveCoverToBlobstore(album):
if not album.small_filetype:
return
from slughifi import slughifi
fn = "%s_%s"%(slughifi(album.artist), slughifi(album.title))
small_file = files.blobstore.create(mime_type=album.small_filetype,
_blobinfo_uploaded_filename="%s_small.png"%fn)
large_file = files.blobstore.create(mime_type=album.large_filetype,
_blobinfo_uploaded_filename="%s_big.png"%fn)
with files.open(small_file, 'a') as small:
small.write(album.small_cover)
with files.open(large_file, 'a') as large:
large.write(album.large_cover)
files.finalize(small_file)
files.finalize(large_file)
album.cover_small = files.blobstore.get_blob_key(small_file)
album.cover_large = files.blobstore.get_blob_key(large_file)
del album.small_cover
del album.large_cover
del album.large_filetype
del album.small_filetype
album.put()
def get(self, firmid, process):
self.json_content()
firm_key = ndb.Key('Firm', firmid)
if process == 'process_hier':
deferred.defer( process_hier )
self.w( json.dumps("ok") )
if process == 'get_hier':
self.w(self.request.get('callback') + '(')
fname = '/gs/frl-arch/' + firmid + '/json/proj-detailed.json'
with files.open(fname, 'r') as f:
data = f.read(60 * 1000)
while data:
self.w( data )
data = f.read(60 * 1000)
self.w(');')
elif process == 'mail':
mail.send_mail( sender = 'Uri London <*****@*****.**>',
to = 'Live Person <*****@*****.**>',
subject = 'your test has succeeded',
body = 'Dear Live Person,\n\n I\'m happy to tell you that your test is good'
)
self.w( json.dumps('ok') )
elif process == 'test_file':
fname = '/gs/frl-arch/test.html'
wfname = files.gs.create( fname, mime_type='text/html', acl='public-read')
with files.open(wfname, 'a') as f:
f.write( 'hello world' )
f.write( 'bye bye' )
files.finalize(wfname)
def testMergeFiles(self):
"""Test merging multiple files."""
input_data = [(str(i), "_" + str(i)) for i in range(100)]
input_data.sort()
input_file = files.blobstore.create()
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
for (k, v) in input_data:
proto = file_service_pb.KeyValue()
proto.set_key(k)
proto.set_value(v)
w.write(proto.Encode())
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
p = TestMergePipeline([input_file, input_file, input_file])
p.start()
test_support.execute_until_empty(self.taskqueue)
p = TestMergePipeline.from_id(p.pipeline_id)
output_file = p.outputs.default.value[0]
output_data = []
with files.open(output_file, "r") as f:
for record in records.RecordsReader(f):
output_data.append(record)
expected_data = [
str((k, [v, v, v], False)) for (k, v) in input_data]
self.assertEquals(expected_data, output_data)
def storeImageFromBGG(bgg_image_url):
'''Returns a blob_key for an image fetched from BGG and stored to the
BlobStore.
'''
_trace = TRACE+'storeImageFromBGG():: '
logging.info(_trace)
image_blob_key = None
if bgg_image_url is not None:
image_name = bgg_image_url.split('/')[-1]
file_name = files.blobstore.create(mime_type='image/jpeg',
_blobinfo_uploaded_filename=image_name)
# Sometimes the image from BGG is too large for GAE ...
try:
image = urllib2.urlopen(bgg_image_url)
with files.open(file_name, 'a') as f:
f.write(image.read())
# Get a smaller version ...
except apiproxy_errors.RequestTooLargeError:
bgg_md_image_url = bgg_image_url.replace('.jpg', '_md.jpg')
image = urllib2.urlopen(bgg_md_image_url)
with files.open(file_name, 'a') as f:
f.write(image.read())
finally:
files.finalize(file_name)
image_blob_key = files.blobstore.get_blob_key(file_name)
else:
return None
return image_blob_key
def process(owner, content):
im_data = io.BytesIO(content)
im_copy = io.BytesIO(content)
im = Image.open(im_data)
cf = imaging.ColorFinder(im)
top = imaging.ColorUtil.generate_color_panes(tuple(cf.strategy_enhanced_complements()))
output = StringIO.StringIO()
top.save(output, format="JPEG")
palette = io.BytesIO(output.getvalue())
palette_copy = io.BytesIO(output.getvalue())
output.close()
upload = Picture(parent=Picture.picture_key(owner))
upload.owner = owner
file_name = files.blobstore.create(mime_type='image/jpeg')
with files.open(file_name, 'a') as f:
f.write(im_copy.read())
files.finalize(file_name)
picture_blob_key = files.blobstore.get_blob_key(file_name)
upload.picture = str(picture_blob_key)
palette_name = files.blobstore.create(mime_type='image/jpeg')
with files.open(palette_name, 'a') as f:
f.write(palette_copy.read())
files.finalize(palette_name)
palette_blob_key = files.blobstore.get_blob_key(palette_name)
upload.palette = str(palette_blob_key)
upload.put()
return palette
def encoded_video_receiver(mp4_url, webm_url, video_id):
logging.info("starting the encoded_video_receiver")
if mp4_url and webm_url:
video = Video.get_by_id(int(video_id))
try:
# adding the mp4 video ref
mp4_file_name = files.blobstore.create(mime_type="video/mp4")
mp4_video = urllib2.urlopen(mp4_url)
with files.open(mp4_file_name, "a") as mp4_f:
mp4_f.write(mp4_video.read())
files.finalize(mp4_file_name)
mp4_blob_key = files.blobstore.get_blob_key(mp4_file_name)
video.mp4_video_ref = mp4_blob_key
# adding the webm video ref
webm_file_name = files.blobstore.create(mime_type="video/webm")
webm_video = urllib2.urlopen(webm_url)
with files.open(webm_file_name, "a") as webm_f:
webm_f.write(webm_video.read())
files.finalize(webm_file_name)
webm_blob_key = files.blobstore.get_blob_key(webm_file_name)
video.webm_video_ref = webm_blob_key
video.put()
logging.info("just saved the video in the datastore")
except urllib2.HTTPError, e:
logging.error("HTTPError: Failed somehow with a HTTPerror: " + str(e))
# time.sleep(10)
except Exception, e:
logging.error("Exception: Failed somehow: " + str(e))
def get(self):
path = self.resolve_doc_path()
gs_key = blobstore.create_gs_key(path)
age = self.get_cache_age(path)
self.response.headers['Cache-Control'] = 'max-age=' + \
str(age) + ',s-maxage=' + str(age)
# is there a better way to check if a file exists in cloud storage?
# AE will serve a 500 if the file doesn't exist, but that should
# be a 404
path_exists = memcache.get(path)
if path_exists is not None:
if path_exists == "1":
self.send_blob(gs_key)
else:
self.error(404)
else:
try:
# just check for existance
files.open(path, 'r').close()
memcache.add(key=path, value="1", time=ONE_DAY)
self.send_blob(gs_key)
except files.file.ExistenceError:
memcache.add(key=path, value="0", time=ONE_DAY)
self.error(404)
def testPartialRecords(self):
"""Test merging into partial key values."""
try:
self._prev_max_values_count = shuffler._MergePipeline._MAX_VALUES_COUNT
# force max values count to extremely low value.
shuffler._MergePipeline._MAX_VALUES_COUNT = 1
input_data = [('1', 'a'), ('2', 'b'), ('3', 'c')]
input_data.sort()
input_file = files.blobstore.create()
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
for (k, v) in input_data:
proto = file_service_pb.KeyValue()
proto.set_key(k)
proto.set_value(v)
w.write(proto.Encode())
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
p = TestMergePipeline([input_file, input_file, input_file])
p.start()
test_support.execute_until_empty(self.taskqueue)
p = TestMergePipeline.from_id(p.pipeline_id)
output_file = p.outputs.default.value[0]
output_data = []
with files.open(output_file, "r") as f:
for record in records.RecordsReader(f):
output_data.append(record)
expected_data = [
('1', ['a'], True),
('1', ['a'], True),
('1', ['a'], False),
('2', ['b'], True),
('2', ['b'], True),
('2', ['b'], False),
('3', ['c'], True),
('3', ['c'], True),
('3', ['c'], False),
]
self.assertEquals([str(e) for e in expected_data], output_data)
finally:
shuffler._MergePipeline._MAX_VALUES_COUNT = self._prev_max_values_count
def save_blob(self, blobdata='blobdata'):
file_name = files.blobstore.create(mime_type='application/octet-stream')
with files.open(file_name, 'a') as f:
f.write(blobdata)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
return blob_key
def uploadXmlsToBlobstore():
"""Writing Files to the Blobstore (Experimental) via remote api to
production server not working!
https://developers.google.com/appengine/docs/python/blobstore/overview#Writing_Files_to_the_Blobstore
http://stackoverflow.com/questions/8201283/google-app-engine-how-to-write-large-files-to-google-cloud-storage
http://stackoverflow.com/questions/3530990/upload-data-to-blobstore-using-remote-api
http://stackoverflow.com/questions/6545247/erratic-problem-with-app-engine-when-writing-files-directly-to-the-blobstore
"""
romn_dir = getRomnDir()
for dirpath, dirnames, filenames in os.walk(romn_dir):
for filename in filenames:
path = os.path.join(dirpath, filename)
key = path[len(romn_dir):]
print('uploading %s ...' % key)
with open(path, 'rb') as f:
# Create the file
file_name = files.blobstore.create(
mime_type='application/octet-stream')
# Open the file and write to it
with files.open(file_name, 'a') as f2:
f2.write(f.read())
# Finalize the file. Do this before attempting to read it.
files.finalize(file_name)
XmlBlobKey(
id=key,
blob_key=files.blobstore.get_blob_key(file_name)).put()
def _save(self, name, content):
name = name.replace('\\', '/')
if hasattr(content, 'file') and \
hasattr(content.file, 'blobstore_info'):
data = content.file.blobstore_info
elif hasattr(content, 'blobstore_info'):
data = content.blobstore_info
elif isinstance(content, File):
guessed_type = mimetypes.guess_type(name)[0]
file_name = files.blobstore.create(
mime_type=guessed_type or 'application/octet-stream',
_blobinfo_uploaded_filename=name)
with files.open(file_name, 'a') as f:
for chunk in content.chunks():
f.write(chunk)
files.finalize(file_name)
data = files.blobstore.get_blob_key(file_name)
else:
raise ValueError("The App Engine storage backend only supports "
"BlobstoreFile instances or File instances.")
if isinstance(data, (BlobInfo, BlobKey)):
# We change the file name to the BlobKey's str() value.
if isinstance(data, BlobInfo):
data = data.key()
return '%s/%s' % (data, name.lstrip('/'))
else:
raise ValueError("The App Engine Blobstore only supports "
"BlobInfo values. Data can't be uploaded "
"directly. You have to use the file upload "
"handler.")
def write_to_blobstore(data, mime_type, name=None):
'''Writes a file to the App Engine blobstore and returns an instance of a
BlobKey if successful.
:param data: Blob data.
:param mime_type: String, mime type of the blob.
:param name: String, name of the blob.
:returns: Instance of a `BlobKey`.
'''
if not name:
name = ''.join(random.choice(string.letters)
for x in range(DEFAULT_NAME_LEN))
blob = files.blobstore.create(
mime_type=mime_type,
_blobinfo_uploaded_filename=name)
with files.open(blob, 'a', exclusive_lock=True) as f:
f.write(data)
files.finalize(blob)
result = files.blobstore.get_blob_key(blob)
# issue with the local development SDK. we can only write to the blobstore
# so fast, so set a retry_count and delay the execution thread between
# each attempt..
for i in range(1, WRITE_MAX_RETRIES):
if result:
break
else:
logging.debug(
'blob still None.. will retry to write to blobstore..')
time.sleep(WRITE_SLEEP_SECONDS)
result = files.blobstore.get_blob_key(blob)
logging.debug('File written to blobstore: key: "%s"', result)
return result
def write_blob(self, data, info):
'''
This is where the image is chucked into the blobstore. The approach
being used is not the one that's recommened in the google docs. Their
method is to use the direct upload url. However direct upload urls
do not seem to work with CORS (and this is code That I have copy and
pasted from a project that needed CORS
'''
(root, ext) = os.path.splitext(info['name'])
img = images.Image(data)
if img:
blob = files.blobstore.create(
_blobinfo_uploaded_filename=info['name'],
mime_type=IMAGE_TYPES[ext.lower()]
)
with files.open(blob, 'a') as f:
f.write(data)
files.finalize(blob)
key = files.blobstore.get_blob_key(blob)
return key, img
return None, None
def testLotsOfValuesForSingleKey(self):
TestEntity(data=str(1)).put()
# Run Mapreduce
p = mapreduce_pipeline.MapreducePipeline(
"test",
__name__ + ".map_yield_lots_of_values",
__name__ + ".reduce_length",
input_reader_spec=input_readers.__name__ + ".DatastoreInputReader",
output_writer_spec=
output_writers.__name__ + ".BlobstoreRecordsOutputWriter",
mapper_params= {
"entity_kind": __name__ + "." + TestEntity.__name__,
},
shards=16)
p.start()
test_support.execute_until_empty(self.taskqueue)
self.assertEquals(1, len(self.emails))
self.assertTrue(self.emails[0][1].startswith(
"Pipeline successful:"))
# Verify reduce output.
p = mapreduce_pipeline.MapreducePipeline.from_id(p.pipeline_id)
output_data = []
for output_file in p.outputs.default.value:
with files.open(output_file, "r") as f:
for record in records.RecordsReader(f):
output_data.append(record)
expected_data = ["('1', 50000)"]
expected_data.sort()
output_data.sort()
self.assertEquals(expected_data, output_data)
def testSingleShard(self):
entity_count = 1000
for _ in range(entity_count):
TestEntity().put()
mapreduce_id = control.start_map(
"test_map",
__name__ + ".test_handler_yield_key_str",
DATASTORE_READER_NAME, {
"entity_kind": __name__ + "." + TestEntity.__name__,
"filesystem": "gs",
"gs_bucket_name": "bucket"
},
shard_count=4,
base_path="/mapreduce_base_path",
output_writer_spec=FILE_WRITER_NAME)
test_support.execute_until_empty(self.taskqueue)
mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
filenames = output_writers.FileOutputWriter.get_filenames(
mapreduce_state)
self.assertEqual(1, len(filenames))
self.assertTrue(filenames[0].startswith("/gs/bucket/"))
with files.open(filenames[0], "r") as f:
data = f.read(10000000)
self.assertEquals(1000, len(data.strip().split("\n")))
def write_wav(samples):
'''Samples are assumed to be in -1 to 1 range and mono, 44100 Hz.
Returns key of newly created blob.'''
return
from google.appengine.api import files
file_name = files.blobstore.create(mime_type='application/octet-stream')
bytes = [struct.pack('h', v * 32767) for v in samples]
byte_str = ''.join(bytes)
# Write wav data into a stringIO
import StringIO
buffer = StringIO.StringIO()
wav = wave.open(buffer, 'wb')
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(44100)
wav.writeframes(byte_str)
wav.close()
with files.open(file_name, 'a') as f:
f.write(buffer)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
return blob_key
def testRecordsReader(self):
"""End-to-end test for records reader."""
input_file = files.blobstore.create()
input_data = [str(i) for i in range(100)]
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
for record in input_data:
w.write(record)
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
mapreduce_id = control.start_map(
"test_map",
__name__ + ".TestHandler",
"mapreduce.input_readers.RecordsReader",
{
"file": input_file
},
shard_count=4,
base_path="/mapreduce_base_path")
test_support.execute_until_empty(self.taskqueue)
self.assertEquals(100, len(TestHandler.processed_entites))
def POST(self):
"""Receive HTTP POST (json-format data), and store data to blobstore.
"""
data = json.loads(web.data())
# Create the file
# set filename property in BlobStore
# http://stackoverflow.com/questions/5697844/how-to-set-filename-property-in-blobstore
file_name = files.blobstore.create(
mime_type='application/octet-stream',
_blobinfo_uploaded_filename=data['key'])
# Open the file and write to it
with files.open(file_name, 'a') as f:
f.write(base64.b64decode(data['payload']))
# Finalize the file. Do this before attempting to read it.
files.finalize(file_name)
# Get the file's blob key
blob_key = files.blobstore.get_blob_key(file_name)
# store blob_key in datastore
XmlBlobKey(id=data['key'], blob_key=blob_key).put()
return 'OK<br />%s<br />%s' % (data['key'], blob_key)
def _save(self, name, content):
name = name.replace('\\', '/')
if hasattr(content, 'file') and \
hasattr(content.file, 'blobstore_info'):
data = content.file.blobstore_info
elif hasattr(content, 'blobstore_info'):
data = content.blobstore_info
elif isinstance(content, File):
guessed_type = mimetypes.guess_type(name)[0]
file_name = files.blobstore.create(mime_type=guessed_type or 'application/octet-stream',
_blobinfo_uploaded_filename=name)
with files.open(file_name, 'a') as f:
for chunk in content.chunks():
f.write(chunk)
files.finalize(file_name)
data = files.blobstore.get_blob_key(file_name)
else:
raise ValueError("The App Engine storage backend only supports "
"BlobstoreFile instances or File instances.")
if isinstance(data, (BlobInfo, BlobKey)):
# We change the file name to the BlobKey's str() value.
if isinstance(data, BlobInfo):
data = data.key()
return '%s/%s' % (data, name.lstrip('/'))
else:
raise ValueError("The App Engine Blobstore only supports "
"BlobInfo values. Data can't be uploaded "
"directly. You have to use the file upload "
"handler.")
def flush(self):
"""Flush pool contents."""
buf = _StringWriter()
with records.RecordsWriter(buf) as w:
for record in self._buffer:
w.write(record)
str_buf = buf.to_string()
if not self._exclusive and len(str_buf) > _FILES_API_MAX_SIZE:
raise errors.Error(
"Buffer too big. Can't write more than %s bytes in one request: "
"risk of writes interleaving. Got: %s" %
(_FILES_API_MAX_SIZE, len(str_buf)))
start_time = time.time()
with files.open(self._filename, "a",
exclusive_lock=self._exclusive) as f:
f.write(str_buf)
if self._ctx:
operation.counters.Increment(COUNTER_IO_WRITE_BYTES,
len(str_buf))(self._ctx)
if self._ctx:
operation.counters.Increment(
COUNTER_IO_WRITE_MSEC, int(
(time.time() - start_time) * 1000))(self._ctx)
self._buffer = []
self._size = 0
gc.collect()
def run(self,
job_name,
gs_bucket_name,
shards=8):
file_names = []
try:
with files.open("/gs/"+gs_bucket_name+TARGETS_FILE_PATH, 'r') as fp:
targets = fp.read()
file_names = targets.splitlines()
except Exception:
logging.warning("Can't find file:" + TARGETS_FILE_PATH)
if len(file_names)>0:
yield mapreduce_pipeline.MapperPipeline(
job_name,
__name__ + "._to_datastore_map",
"lakshmi.cooperate.input_readers.GoogleStorageLineInputReader",
output_writer_spec="mapreduce.output_writers.BlobstoreOutputWriter",
params={
"input_reader":{
"file_paths": file_names,
"gs_bucket_name": gs_bucket_name
},
"output_writer":{
"mime_type": "text/plain"
}
},
shards=shards)
def generate_all():
q = MyFile.all()
mfiles = q.fetch(100)
# delete all old files and data
for mf in mfiles:
bk = mf.blob_key
blobstore.delete(bk)
# delete data entity
mf.delete()
# dict of zipfiles being generated and their relevant generators
generators = {
'openvpn.zip': generate_ovpn,
'windows.zip': generate_win,
'linux.zip': generate_linux,
'mac.zip': generate_mac,
'android.zip': generate_android
}
ip_data = fetch_ip_data()
for fn, g in generators.iteritems():
data = g(ip_data)
z = zipit(data) #compress the data
blob_file = files.blobstore.create('application/zip',
_blobinfo_uploaded_filename=fn)
with files.open(blob_file, 'a') as f:
f.write(z)
files.finalize(blob_file)
blob_key = files.blobstore.get_blob_key(blob_file)
mf = MyFile(name=fn, blob_key=str(blob_key))
mf.update_date = datetime.datetime.now().date()
mf.put()
def WriteTempImageFile(stream):
blob = files.blobstore.create(_blobinfo_uploaded_filename='ImageTemp')
with files.open(blob, 'a') as f:
f.write(stream.getvalue())
files.finalize(blob)
return files.blobstore.get_blob_key(blob)
def store_raw_data_as_blob(data,name,content_type):
logging.info('store (bin) raw_data as %s (%s)'%(name,content_type))
file_name = files.blobstore.create(mime_type=content_type, _blobinfo_uploaded_filename = name)
with files.open(file_name, 'a') as out:
out.write(data)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
#http://code.google.com/p/googleappengine/issues/detail?id=4872
#FIXME
if not blob_key:
logging.info('again....1')
time.sleep(1)
blob_key = files.blobstore.get_blob_key(file_name)
if not blob_key:
logging.info('again....2')
time.sleep(1)
blob_key = files.blobstore.get_blob_key(file_name)
#endofhack
logging.info('file key:%s'%blob_key)
return blob_key
def write_blob(self, data, info):
blob = files.blobstore.create(mime_type=info['type'],
_blobinfo_uploaded_filename=info['name'])
with files.open(blob, 'a') as f:
f.write(data)
files.finalize(blob)
return files.blobstore.get_blob_key(blob)
def post(self):
fkstring = self.request.get("filekey")
filekey = FileKey(key_name =fkstring, parent=filelist_key())
self.response.out.write("File key: "+ filekey.key().id_or_name())
upload_files = self.get_uploads('file')
blob_file = upload_files[0]
filekey.blobinfokey = str(blob_file.key())
if MEMCACHE and blob_file.size <= 102400:
self.response.out.write("</br> Inserted in MEMCACHE")
memcache.add(fkstring, blob_file)
filekey.filelocation = "memcache"
self.response.out.write("""<br><b><a href="/">RETURN TO HOME</a></b><br>""")
else:
self.response.out.write("</br> Inserted in GOOGLE CLOUD STORAGE")
wr_path = files.gs.create(BUCKET_PATH+"/"+filekey.key().id_or_name(), mime_type='text/plain', acl='public-read')
with files.open(wr_path, 'a') as filepath:
start = 0
fetchsz = blobstore.MAX_BLOB_FETCH_SIZE - 1
filesize=blob_file.size
while start < filesize:
filepath.write( blobstore.fetch_data(blob_file, start, start+fetchsz))
start = start + fetchsz
files.finalize(wr_path)
filekey.filelocation = "cloudstorage"
self.response.out.write("""<br><b><a href="/">RETURN TO HOME</a></b><br>""")
filekey.put()
def get(self):
"""Loads the data file located at |filename|.
Args:
filename: The filename for the data file to be loaded.
"""
yesterday = datetime.date.today() - datetime.timedelta(1)
yesterday_formatted = yesterday.strftime("%Y.%m.%d")
filename = 'histograms/daily/%s/Everything' % (yesterday_formatted)
if settings.PROD:
try:
with files.open(BIGSTORE_BUCKET + filename, 'r') as unused_f:
pass
except files.file.ExistenceError, e:
self.response.write(e)
return
# The file exists; serve it.
blob_key = blobstore.create_gs_key(BIGSTORE_BUCKET + filename)
blob_reader = blobstore.BlobReader(blob_key, buffer_size=3510000)
try:
result = blob_reader.read()
finally:
blob_reader.close()
def post(self):
fkeystr = self.request.get("filekey")
regexpstr = self.request.get("regexp")
filekeys = FileKey.all()
filekeys.filter('__key__ =', db.Key.from_path("FileKey", fkeystr, parent=filelist_key()))
if filekeys.count() == 0:
self.response.out.write("Key(%s) does NOT exists." % fkeystr)
else:
for ifile in filekeys:
self.response.out.write("Matching contents")
if ifile.filelocation == "memcache":
blob_info = memcache.get(ifile.key().id_or_name())
#elf.send_blob(blob_info)
blob_reader = blob_info.open()
for line in blob_reader:
if regexpstr in line:
self.response.out.write(line)
else:
#self.response.out.write("File is in Cloud Storage")
with files.open(BUCKET_PATH+"/"+ifile.key().id_or_name(), 'r') as fp:
buf = fp.read(1000)
while buf:
if regexpstr in buf:
self.response.out.write(buf)
buf = fp.read(1000000)
def testHugeTaskUseDatastore(self):
"""Test map job with huge parameter values."""
input_file = files.blobstore.create()
input_data = [str(i) for i in range(100)]
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
for record in input_data:
w.write(record)
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
mapreduce_id = control.start_map(
"test_map",
__name__ + ".TestHandler",
"mapreduce.input_readers.RecordsReader",
{
"file": input_file,
# the parameter can't be compressed and wouldn't fit into
# taskqueue payload
"huge_parameter": random_string(900000)
},
shard_count=4,
base_path="/mapreduce_base_path")
test_support.execute_until_empty(self.taskqueue)
self.assertEquals(100, len(TestHandler.processed_entites))
self.assertEquals([], model._HugeTaskPayload.all().fetch(100))
def output(self, filename, file_out):
content = file_out.read()
base, ext = os.path.splitext(filename)
if ext == ".css":
mimetype = "text/css"
elif ext == ".js":
mimetype = "text/javascript"
else:
mimetype = "application/octet-stream"
already_exists = False
for info in BlobInfo.all().filter('content_type = ', mimetype):
if info.filename == filename:
already_exists = True
continue
#Clear out old blobs
if info.filename.split(".")[0] == filename.split(".")[0]:
logging.debug("Deleting: %s", info.filename)
info.delete()
if not already_exists:
logging.info("Creating: %s", filename)
result = files.blobstore.create(mime_type=mimetype, _blobinfo_uploaded_filename=filename)
with files.open(result, "a") as f:
f.write(content)
files.finalize(result)
blob_key = files.blobstore.get_blob_key(result)
while not blob_key:
blob_key = files.blobstore.get_blob_key(result)
def post(self):
countries = self.all_countries
tourist_id = int(self.get_cookie("query")[0])
tourist = Tourist.Tourist.get_by_id(tourist_id)
new_email = self.request.get("email")
first_name = self.request.get("first_name")
last_name = self.request.get("last_name")
country = self.request.get("country")
state = self.request.get("state")
picture = self.request.POST.get("photo")
# self.write(picture.filename[picture.filename.find(".") + 1 : ])
profile_args = {"email" : new_email, "country" : country, "first_name" : first_name,
"last_name" : last_name, "state" : state, "picture" : picture}
if self.validate_email(new_email) and self.validate_name(first_name) and self.validate_name(last_name):
Tourist.Tourist.updateTourist(tourist, new_email, first_name, last_name, country, state)
if picture != None and picture != "":
picture_extension = picture.filename[picture.filename.rfind(".") : ]
picture_url = "/tourbly/profile_pictures/" + str(tourist.key().id()) + self.rand_salt("picture") + picture_extension
picture_name = "/gs" + picture_url
writable_file_name = files.gs.create(picture_name, mime_type='image/jpeg', acl='public-read')
with files.open(writable_file_name, 'a') as f:
f.write(picture.file.read())
files.finalize(writable_file_name)
tourist.picture = "http://storage.googleapis.com" + picture_url
tourist.put()
self.render("profile.html", isLoggedIn = self.check_session("query"), profile_args = profile_args,
success_message = "Your profile has been updated successfully", tourist = tourist,
countries = countries)
else:
self.render("profile.html", email_error = self.profile_email_error_prompt(tourist.email, new_email),
profile_args = profile_args, success_message = "there is something wrong", countries = countries)
def write_to_blobstore(content, old_blobinfo=None):
"""Write content to blobstore.
Args:
content: A byte-string.
old_blobinfo: An optional BlobInfo object used for comparison. If the
md5_hash of the old blob and the new content are equal, the
old blobkey is returned and nothing new is written.
Returns:
A BlobKey.
"""
# Require pre-encoded content.
assert not isinstance(content, unicode)
# Blob de-duping: if the blob content is the same as the old file,
# just use the old blob key to save space and RPCs.
if (old_blobinfo
and old_blobinfo.md5_hash == hashlib.md5(content).hexdigest()):
return old_blobinfo.key()
# write new blob.
filename = blobstore_files.blobstore.create()
content_file = cStringIO.StringIO(content)
blobstore_file = blobstore_files.open(filename, 'a')
# Blobstore writes cannot exceed an RPC size limit, so chunk the writes.
while True:
content_chunk = content_file.read(BLOBSTORE_APPEND_CHUNK_SIZE)
if not content_chunk:
break
blobstore_file.write(content_chunk)
blobstore_file.close()
blobstore_files.finalize(filename)
blob_key = blobstore_files.blobstore.get_blob_key(filename)
return blob_key
def post(self, post):
form = PostForm(data=self.request.POST, instance=post,
initial={'draft': post and post.published is None})
if form.is_valid():
post = form.save(commit=False)
image_url = form.clean_data['image_url']
if image_url:
try:
u = urllib2.urlopen(image_url)
file_name = files.blobstore.create(mime_type='image/jpeg')
with files.open(file_name, 'a') as f:
f.write(u.read())
files.finalize(file_name)
post.image_id = str(files.blobstore.get_blob_key(file_name))
except:
# Not sure how to use ErrorList in Django 0.96
# form._errors['image'] = ''
self.render_form(form)
return
if form.clean_data['draft']:# Draft post
post.published = datetime.datetime.max
post.put()
else:
if not post.path: # Publish post
post.updated = post.published = datetime.datetime.now()
else:# Edit post
post.updated = datetime.datetime.now()
post.publish()
self.render_to_response("published.html", {
'post': post,
'draft': form.clean_data['draft']})
else:
self.render_form(form)
def ArchiveToGoogleStorage(self, **unused_args):
"""Posts data in compressed JSON format to Google Storage for Developers.
Allows a file containing the requested data to be stored in Google Storage
for developers for later download. Please see _Archive for details on
how arguments are handled and data is packaged.
In preparation for use of this method it is important that the bucket and
account that will be used be properly prepared by following these
instructions (http://goo.gl/S0LRl) paying careful attention to the ACLs.
This information should be used to adjust the pertinent parts of this
application's config file.
Raises:
DeadlineExceededError: Handled in the case where a request exceeds the
response time limit. A error is displayed in the VERY short time for
reporting errors.
No new exceptions generated here.
"""
#TODO(mdw) Unit test needed.
try:
archive_dir, archive_data = self._Archive()
anonymize = self.request.get('anonymize')
anonymize = not not anonymize
# Create the file
if not anonymize:
bucket = config_private.ARCHIVE_GS_BUCKET
acl = config_private.ARCHIVE_GS_ACL
else:
bucket = config.ARCHIVE_GS_BUCKET_PUBLIC
acl = config.ARCHIVE_GS_ACL_PUBLIC
gs_archive_name = '/gs/%s/%s.zip' % (bucket, archive_dir)
gs_archive = files.gs.create(
gs_archive_name,
mime_type=config.ARCHIVE_CONTENT_TYPE,
acl=acl,
content_disposition=(config.ARCHIVE_CONTENT_DISPOSITION_BASE %
archive_dir))
# Open the file and write the data.
with files.open(gs_archive, 'a') as f:
f.write(archive_data)
# Finalize (a special close) the file.
files.finalize(gs_archive)
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(
'{\'status\':200, \'archive_name\':\'%s\'}' % gs_archive_name)
except DeadlineExceededError, e:
logging.exception(e)
self.response.clear()
self.response.set_status(500)
#NOTE: if you see this error make sure it is run on a backend instance.
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(
('{\'status\':500, \'error_name\':\'%s\', '
'\'error_value\':\'%s\'}' % ('DeadlineExceededError', e)))
def testHugeTaskUseDatastore(self):
"""Test map job with huge parameter values."""
input_file = files.blobstore.create()
input_data = [str(i) for i in range(100)]
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
for record in input_data:
w.write(record)
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
mapreduce_id = control.start_map(
"test_map",
__name__ + ".TestHandler",
"mapreduce.input_readers.RecordsReader",
{
"file": input_file,
# the parameter can't be compressed and wouldn't fit into
# taskqueue payload
"huge_parameter": random_string(900000)
},
shard_count=4,
base_path="/mapreduce_base_path")
test_support.execute_until_empty(self.taskqueue)
self.assertEquals(100, len(TestHandler.processed_entites))
self.assertEquals([], model._HugeTaskPayload.all().fetch(100))
def get(self):
"""Loads the data file located at |filename|.
Args:
filename: The filename for the data file to be loaded.
"""
yesterday = datetime.date.today() - datetime.timedelta(1)
yesterday_formatted = yesterday.strftime("%Y.%m.%d")
filename = 'histograms/daily/%s/Everything' % (yesterday_formatted)
if settings.PROD:
try:
with files.open(BIGSTORE_BUCKET + filename, 'r') as unused_f:
pass
except files.file.ExistenceError, e:
self.response.write(e)
return
# The file exists; serve it.
blob_key = blobstore.create_gs_key(BIGSTORE_BUCKET + filename)
blob_reader = blobstore.BlobReader(blob_key, buffer_size=3510000)
try:
result = blob_reader.read()
finally:
blob_reader.close()
def flush(self):
"""Flush pool contents."""
# Write data to in-memory buffer first.
buf = cStringIO.StringIO()
with records.RecordsWriter(buf) as w:
for record in self._buffer:
w.write(record)
w._pad_block()
str_buf = buf.getvalue()
buf.close()
if not self._exclusive and len(str_buf) > _FILES_API_MAX_SIZE:
# Shouldn't really happen because of flush size.
raise errors.Error(
"Buffer too big. Can't write more than %s bytes in one request: "
"risk of writes interleaving. Got: %s" %
(_FILES_API_MAX_SIZE, len(str_buf)))
# Write data to file.
start_time = time.time()
with files.open(self._filename, "a",
exclusive_lock=self._exclusive) as f:
f.write(str_buf)
if self._ctx:
operation.counters.Increment(COUNTER_IO_WRITE_BYTES,
len(str_buf))(self._ctx)
if self._ctx:
operation.counters.Increment(
COUNTER_IO_WRITE_MSEC, int(
(time.time() - start_time) * 1000))(self._ctx)
# reset buffer
self._buffer = []
self._size = 0
gc.collect()
def get(self, key):
file_name = files.blobstore.create(mime_type='text/csv')
ref = Ref.get(db.Key(key))
cols = get_all_label(ref)
header = ['Name', 'Parents']
for col in cols:
header.append('%s - Owner' % col)
header.append('%s - Type' % col)
header.append('%s - Value' % col)
with files.open(file_name, 'a') as f:
writer = UnicodeWriter(f, delimiter=',', quoting=csv.QUOTE_MINIMAL)
writer.writerow(header)
p_props = parent_props(ref)
writerow(ref, writer, cols, p_props)
# Finalize the file. Do this before attempting to read it.
files.finalize(file_name)
# Get the file's blob key
blob_key = files.blobstore.get_blob_key(file_name)
self.response.headers[
'Content-Disposition'] = 'attachment; filename=%s_.csv' % ref.name
self.send_blob(blob_key)
def testMultipleShards(self):
entity_count = 1000
for _ in range(entity_count):
TestEntity().put()
mapreduce_id = control.start_map(
"test_map",
__name__ + ".test_handler_yield_key_str",
DATASTORE_READER_NAME,
{
"entity_kind": __name__ + "." + TestEntity.__name__,
"output_sharding": "input",
},
shard_count=4,
base_path="/mapreduce_base_path",
output_writer_spec=BLOBSTORE_WRITER_NAME)
test_support.execute_until_empty(self.taskqueue)
mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
filenames = output_writers.BlobstoreOutputWriter.get_filenames(
mapreduce_state)
self.assertEqual(4, len(filenames))
file_lengths = []
for filename in filenames:
self.assertTrue(filename.startswith("/blobstore/"))
self.assertFalse(filename.startswith("/blobstore/writable:"))
with files.open(filename, "r") as f:
data = f.read(10000000)
file_lengths.append(len(data.strip().split("\n")))
self.assertEqual(1000, sum(file_lengths))
def post(self):
upload_files = self.get_uploads('file')
blob_info = upload_files[0]
# Resize the image
image = images.Image(blob_key=blob_info.key())
image.resize(width=WOOF_FEED_ITEM_IMAGE_MAX_WIDTH,
height=WOOF_FEED_ITEM_IMAGE_MAX_HEIGHT)
thumbnail = image.execute_transforms(output_encoding=images.JPEG)
# Save Resized Image back to blobstore
file_name = files.blobstore.create(mime_type='image/jpeg')
with files.open(file_name, 'a') as f:
f.write(thumbnail)
files.finalize(file_name)
# Remove the original image
blobstore.delete(blob_info.key())
blob_key = files.blobstore.get_blob_key(file_name)
# New FeedImage
feed_image = FeedImage()
feed_image.data = BlobInfo.get(blob_key)
feed_image.width = image.width
feed_image.height = image.height
feed_image.put()
# Create new FeedItem
feed_item = FeedItem()
feed_item.text = self.request.get("text")
feed_item.image = feed_image
feed_item.put()
self.redirect('/')
def flush(self):
"""Flush pool contents."""
# Write data to in-memory buffer first.
buf = cStringIO.StringIO()
with records.RecordsWriter(buf) as w:
for record in self._buffer:
w.write(record)
w._pad_block()
str_buf = buf.getvalue()
buf.close()
if not self._exclusive and len(str_buf) > _FILES_API_MAX_SIZE:
# Shouldn't really happen because of flush size.
raise errors.Error(
"Buffer too big. Can't write more than %s bytes in one request: "
"risk of writes interleaving. Got: %s" %
(_FILES_API_MAX_SIZE, len(str_buf)))
# Write data to file.
start_time = time.time()
with files.open(self._filename, "a", exclusive_lock=self._exclusive) as f:
f.write(str_buf)
if self._ctx:
operation.counters.Increment(
COUNTER_IO_WRITE_BYTES, len(str_buf))(self._ctx)
if self._ctx:
operation.counters.Increment(
COUNTER_IO_WRITE_MSEC,
int((time.time() - start_time) * 1000))(self._ctx)
# reset buffer
self._buffer = []
self._size = 0
gc.collect()
def upload(self,
file,
key,
acl=None,
policy=None,
signature=None,
success_action_redirect=None,
**kwargs):
"""A development-only action for uploading a package archive.
In production, package archives are uploaded directly to cloud storage,
using a signed form for authentication. The signed form doesn't work for
the development server, since it uses a local database in place of cloud
storage, so this action emulates it by manually saving the file to the
development database.
"""
if handlers.is_production(): raise handlers.http_error(404)
if PrivateKey.sign(policy) != signature: raise handlers.http_error(403)
write_path = files.gs.create('/gs/' + key, acl=acl)
with files.open(write_path, 'a') as f:
f.write(file.file.read())
files.finalize(write_path)
if success_action_redirect:
raise cherrypy.HTTPRedirect(success_action_redirect)
cherrypy.response.status = 204
return ""
def finish_upload(file_id):
file_item = File.get_by_id(file_id)
size = file_item.size
current_size = file_item.current_size
write_path = file_item.write_path
files.finalize(write_path)
if size != current_size:
file_item.delete()
raise Exception('file size error')
# check sha1
sh = hashlib.sha1()
with files.open(file_item.read_path, 'r') as fp:
buf = fp.read(1000000)
while buf:
sh.update(buf)
buf = fp.read(1000000)
sha1_value = sh.hexdigest()
if sha1_value != file_item.sha1:
file_item.delete()
raise Exception('file size error')
file_item.upload_finished = True
file_item.put()
def testMultipleShards(self):
entity_count = 1000
for _ in range(entity_count):
TestEntity().put()
mapreduce_id = control.start_map(
"test_map",
__name__ + ".test_handler_yield_key_str",
DATASTORE_READER_NAME, {
"entity_kind": __name__ + "." + TestEntity.__name__,
"output_sharding": "input",
},
shard_count=4,
base_path="/mapreduce_base_path",
output_writer_spec=BLOBSTORE_WRITER_NAME)
test_support.execute_until_empty(self.taskqueue)
mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
filenames = output_writers.BlobstoreOutputWriter.get_filenames(
mapreduce_state)
self.assertEqual(4, len(set(filenames)))
file_lengths = []
for filename in filenames:
self.assertTrue(filename.startswith("/blobstore/"))
self.assertFalse(filename.startswith("/blobstore/writable:"))
with files.open(filename, "r") as f:
data = f.read(10000000)
file_lengths.append(len(data.strip().split("\n")))
self.assertEqual(1000, sum(file_lengths))
def _make_backup_info_blobstore(self, num):
file_num = 3
dt = datetime.datetime.utcnow()
for i in range(num):
# _AE_Backup_Information
info = AeBackupInformation(
name=config.BACKUP_NAME,
filesystem='blobstore',
start_time=dt,
complete_time=dt + datetime.timedelta(hours=1)
)
info.put()
dt -= datetime.timedelta(days=1)
# _AE_Backup_Information_Kind_Files
info_files = AeBackupInformationKindFiles()
info_files.files = []
if i == (num - 1):
file_num = 120 # 100 over at last
for j in range(file_num):
# make blobstore file
file_name = files.blobstore.create(
mime_type='application/octet-stream')
with files.open(file_name, 'a') as f:
f.write('asdfghjkl')
files.finalize(file_name)
info_files.files.append(file_name)
info_files.key = ndb.Key(
AeBackupInformationKindFiles._get_kind(),
AeBackupInformationKindFiles.allocate_ids(1)[0],
parent=info.key
)
info_files.put()
def post(self):
"""Store a file after encripting it.
"""
rows = self.request.POST.get('file').value
file_name = files.blobstore.create(mime_type='text/plain')
with files.open(file_name, 'a') as raw_file:
writer = csv.writer(raw_file, delimiter=',')
for row in csv.reader(StringIO(rows), delimiter=','):
if len(row) > 1:
row[1] = self.crypt(row[1])
writer.writerow(row)
files.finalize(file_name)
# TODO: with high replication, the newly created file will not
# show in the list.
#
# It show redirect to a page listing the files instead
blobs = blobstore.BlobInfo.all()
blob_links = [
'<a href="/serve/%s">File %s</a><br/>' % (blob.key(), index+1)
for index, blob in enumerate(blobs)
]
self.response.out.write("""<html>
<body>
<form action="/upload" enctype="multipart/form-data" method="post">
<input type="file" name="file"/><input type="submit" />
</form>
<br>
%s
</body>
</html>
""" % "".join(blob_links)
)
def write_to_blobstore(data, mime_type, name=None):
"""Writes a file to the App Engine blobstore and returns an instance of a
BlobKey if successful.
:param data: Blob data.
:param mime_type: String, mime type of the blob.
:param name: String, name of the blob.
:returns: Instance of a `BlobKey`.
"""
if not name:
name = "".join(random.choice(string.letters) for x in range(DEFAULT_NAME_LEN))
blob = files.blobstore.create(mime_type=mime_type, _blobinfo_uploaded_filename=name)
with files.open(blob, "a", exclusive_lock=True) as f:
f.write(data)
files.finalize(blob)
result = files.blobstore.get_blob_key(blob)
# issue with the local development SDK. we can only write to the blobstore
# so fast, so set a retry_count and delay the execution thread between
# each attempt..
for i in range(1, WRITE_MAX_RETRIES):
if result:
break
else:
logging.debug("blob still None.. will retry to write to blobstore..")
time.sleep(WRITE_SLEEP_SECONDS)
result = files.blobstore.get_blob_key(blob)
logging.debug('File written to blobstore: key: "%s"', result)
return result
def WriteTempImageFile(stream):
blob = files.blobstore.create(_blobinfo_uploaded_filename="ImageTemp")
with files.open(blob, "a") as f:
f.write(stream.getvalue())
files.finalize(blob)
return files.blobstore.get_blob_key(blob)
def save_blob(self, blobdata='blobdata'):
file_name = files.blobstore.create(mime_type='application/octet-stream')
with files.open(file_name, 'a') as f:
f.write(blobdata)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
return blob_key
def store_image(self, link):
file_name = files.blobstore.create(
mime_type='application/octet-stream')
with files.open(file_name, 'a') as f:
f.write(urlfetch.Fetch(link, deadline=60).content)
files.finalize(file_name)
self.image_name = file_name
def testLotsOfValuesForSingleKey(self):
TestEntity(data=str(1)).put()
# Run Mapreduce
p = mapreduce_pipeline.MapreducePipeline(
"test",
__name__ + ".map_yield_lots_of_values",
__name__ + ".reduce_length",
input_reader_spec=input_readers.__name__ + ".DatastoreInputReader",
output_writer_spec=
output_writers.__name__ + ".BlobstoreRecordsOutputWriter",
mapper_params= {
"entity_kind": __name__ + "." + TestEntity.__name__,
},
shards=16)
p.start()
test_support.execute_until_empty(self.taskqueue)
self.assertEquals(1, len(self.emails))
self.assertTrue(self.emails[0][1].startswith(
"Pipeline successful:"))
# Verify reduce output.
p = mapreduce_pipeline.MapreducePipeline.from_id(p.pipeline_id)
output_data = []
for output_file in p.outputs.default.value:
with files.open(output_file, "r") as f:
for record in records.RecordsReader(f):
output_data.append(record)
expected_data = ["('1', 50000)"]
expected_data.sort()
output_data.sort()
self.assertEquals(expected_data, output_data)
def testDedicatedParams(self):
entity_count = 1000
for _ in range(entity_count):
TestEntity().put()
mapreduce_id = control.start_map(
"test_map",
__name__ + ".test_handler_yield_key_str",
"mapreduce.input_readers.DatastoreInputReader",
{
"input_reader": {
"entity_kind": __name__ + "." + TestEntity.__name__,
},
"output_writer": {
"filesystem": "gs",
"gs_bucket_name": "bucket",
},
},
shard_count=4,
base_path="/mapreduce_base_path",
output_writer_spec=FILE_WRITER_NAME)
test_support.execute_until_empty(self.taskqueue)
mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
filenames = output_writers.FileOutputWriter.get_filenames(mapreduce_state)
self.assertEqual(1, len(filenames))
self.assertTrue(filenames[0].startswith("/gs/bucket/"))
with files.open(filenames[0], "r") as f:
data = f.read(10000000)
self.assertEquals(1000, len(data.strip().split("\n")))
def testSingleShard(self):
entity_count = 1000
for _ in range(entity_count):
TestEntity().put()
mapreduce_id = control.start_map(
"test_map",
__name__ + ".test_handler_yield_key_str",
"mapreduce.input_readers.DatastoreInputReader",
{
"entity_kind": __name__ + "." + TestEntity.__name__,
},
shard_count=4,
base_path="/mapreduce_base_path",
output_writer_spec=BLOBSTORE_WRITER_NAME)
test_support.execute_until_empty(self.taskqueue)
mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
filenames = output_writers.BlobstoreOutputWriter.get_filenames(
mapreduce_state)
self.assertEqual(1, len(filenames))
blob_name = filenames[0]
self.assertTrue(blob_name.startswith("/blobstore/"))
self.assertFalse(blob_name.startswith("/blobstore/writable:"))
with files.open(blob_name, "r") as f:
data = f.read(10000000)
self.assertEquals(1000, len(data.strip().split("\n")))
def execute(self, context, obj):
""" Writes the CSV file """
# if already wrote a file, don't do anything
semaphore = RunOnceSemaphore(context.instanceName, context)
if not semaphore.readRunOnceSemaphore(payload='payload'):
# fetch the CsvCounter, since it is the parent of all the other Models
counter = CsvProgressCounter.get_by_key_name(context.instanceName)
# fetch the single aggregated results Model
aggResults = CsvIntermediateResults.get_by_key_name(
context.instanceName, counter)
# open the file
fileName = files.blobstore.create(
mime_type='application/octet-stream')
with files.open(fileName, 'a') as f:
# the csv module has a convenient row writing interface
writer = csv.writer(f)
# this queries for all the intermediate results
query = CsvIntermediateResults.all().ancestor(counter)
for results in query:
# the aggregated results may also be in the results, so skip them
if aggResults and results.key() == aggResults.key():
continue
# for all the intermediate data, write the rows
data = results.data
for item in data:
rows = self.getRows(context, obj, item,
aggResults.data)
if rows:
for row in rows:
writer.writerow(row)
if aggResults:
# now also write down any specific aggregated data rows
rows = self.getAggregatedRows(context, obj,
aggResults.data)
if rows:
for row in rows:
writer.writerow(row)
# finalize the file
files.finalize(fileName)
# FIXME: what to do with this?
blobKey = files.blobstore.get_blob_key(fileName)
# at this point we have successfully written the file, lets make sure we don't do it again
# if a retry occurs downstream
semaphore.writeRunOnceSemaphore(payload='payload')
# store the key of the counter (ie. parent of intermediate results) for cleanup
context[COUNTER_KEY_PARAM] = counter.key()
return OK_EVENT
def _CreateInputFile(self):
input_file = files.blobstore.create()
with files.open(input_file, "a") as f:
f.write("foo")
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
return input_file
def create_blob_file(mime_type, binary, filename="blob_filename"):
from google.appengine.api import files
blob_file_name = files.blobstore.create(mime_type=mime_type, _blobinfo_uploaded_filename=filename)
with files.open(blob_file_name, 'a') as f:
f.write(binary)
files.finalize(blob_file_name)
blob_key = files.blobstore.get_blob_key(blob_file_name)
return blob_key
def test_blobstore_sanity_check(self):
filename = files.blobstore.create(mime_type='application/octet-stream')
self.assertNotEquals(None, filename)
with files.open(filename, 'a') as f:
f.write('test blob data..')
files.finalize(filename)
blobkey = files.blobstore.get_blob_key(filename)
self.assertNotEquals(None, blobkey)
def generate_pledges_csv(file_name, pledge_type, pledge_time, full_data=False):
""" Generates the pledges.csv file in a deferred way """
PAGE_SIZE = 500
csv_buffer = StringIO.StringIO()
w = csv.writer(csv_buffer)
if not pledge_time and pledge_type == 'WpPledge':
# First time through, add the column headers
if full_data:
headers = [
'SOURCE', 'donationTime', 'Amount ($)', 'url_nonce',
'stripeCustomer', 'Email', 'First Name', 'Last Name',
'Address', 'Address 2', 'City', 'State', 'Zip', 'Phone',
'Country', 'Occupation', 'Employer', 'Targeting', 'Type',
'Recurring', 'Source'
]
else:
headers = [
'zip', 'dollars', 'timestamp', 'date', 'city', 'state',
'latitude', 'longitude'
]
w.writerow(headers)
zg = Zipgun('zipgun/zipcodes')
# Get the next PAGE_SIZE pledges
query = getattr(model, pledge_type).all().order('donationTime')
if pledge_time:
# Filter instead of using 'offset' because offset is very inefficient,
# according to https://developers.google.com/appengine/articles/paging
query = query.filter('donationTime >= ', pledge_time)
pledges = query.fetch(PAGE_SIZE + 1)
next_pledge_time = None
if len(pledges) == PAGE_SIZE + 1:
next_pledge_time = pledges[-1].donationTime
pledges = pledges[:PAGE_SIZE]
# Loop through the current pledges and write them to the csv
for pledge in pledges:
w.writerow(pledge_row(pledge, zg, full_data))
with files.open(file_name, 'a') as f:
f.write(csv_buffer.getvalue())
csv_buffer.close()
if not next_pledge_time and pledge_type == 'Pledge':
# Last time through, finalize the file
files.finalize(file_name)
else:
# More to process, recursively run again
next_pledge_type = pledge_type
if pledge_type == 'WpPledge' and not next_pledge_time:
next_pledge_type = 'Pledge'
deferred.defer(generate_pledges_csv,
file_name,
next_pledge_type,
next_pledge_time,
full_data,
_queue='generateCSV')
def action_edit(self):
if self.section.handler.request.get('submit'):
self.SITE_HEADER = self.section.handler.request.get('SITE_HEADER')
self.SITE_SUB_HEADER = self.section.handler.request.get(
'SITE_SUB_HEADER')
self.DEFAULT_THEME = self.section.handler.request.get(
'DEFAULT_THEME')
self.GOOGLE_ANALYTICS_UA = self.section.handler.request.get(
'GOOGLE_ANALYTICS_UA')
self.ROBOTS_TXT = self.section.handler.request.get('ROBOTS_TXT')
if self.section.handler.request.get('FAVICON_ICO'):
if self.FAVICON_ICO:
BlobInfo.get(self.FAVICON_ICO).delete()
data = self.section.handler.request.get('FAVICON_ICO')
handle = files.blobstore.create(
mime_type='image/x-icon',
_blobinfo_uploaded_filename='favicon.ico')
with files.open(handle, 'a') as f:
f.write(data)
files.finalize(handle)
self.FAVICON_ICO = files.blobstore.get_blob_key(handle)
self.ENABLE_THEME_PREVIEW = self.section.handler.request.get(
'ENABLE_THEME_PREVIEW') != ''
self.DEBUG_MODE = self.section.handler.request.get(
'DEBUG_MODE') != ''
cache.delete(CACHE_KEY)
self.update()
raise Exception('Redirect', self.section.action_redirect_path)
f = form(self.section, self.section.full_path)
f.add_control(
control(self.section, 'text', 'SITE_HEADER', self.SITE_HEADER,
'Site header', 50))
f.add_control(
control(self.section, 'text', 'SITE_SUB_HEADER',
self.SITE_SUB_HEADER, 'Site sub-header', 50))
combined_themes = get_local_theme_namespaces(
) + get_custom_theme_namespaces()
f.add_control(
selectcontrol(
self.section, 'DEFAULT_THEME', combined_themes,
self.DEFAULT_THEME if self.DEFAULT_THEME else
DEFAULT_LOCAL_THEME_TEMPLATE, 'Default theme'))
f.add_control(
control(self.section, 'text', 'GOOGLE_ANALYTICS_UA',
self.GOOGLE_ANALYTICS_UA, 'Google analytics UA'))
f.add_control(
control(self.section, 'file', 'FAVICON_ICO', label='favicon.ico'))
f.add_control(
textareacontrol(self.section, 'ROBOTS_TXT', self.ROBOTS_TXT,
'robots.txt', 90, 5))
f.add_control(
checkboxcontrol(self.section, 'ENABLE_THEME_PREVIEW',
self.ENABLE_THEME_PREVIEW, 'Enable theme preview'))
f.add_control(
checkboxcontrol(self.section, 'DEBUG_MODE', self.DEBUG_MODE,
'Debug mode'))
f.add_control(control(self.section, 'submit', 'submit', 'Submit'))
return '<h2>Edit configuration</h2>%s' % unicode(f)
