def testMultipleRequests(self):
"""Tests restoring the reader state across multiple requests."""
input_file = files.blobstore.create()
# Create a file with two records.
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
proto = file_service_pb.KeyValues()
proto.set_key("key2")
proto.value_list().extend(["a", "b"])
proto.set_partial(True)
w.write(proto.Encode())
proto = file_service_pb.KeyValues()
proto.set_key("key2")
proto.value_list().extend(["c", "d"])
w.write(proto.Encode())
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
# Now read the records in two attempts, serializing and recreating the
# input reader as if it's a separate request.
reader = mapreduce_pipeline._ReducerReader([input_file], 0)
it = iter(reader)
self.assertEquals(input_readers.ALLOW_CHECKPOINT, it.next())
reader_state = reader.to_json()
other_reader = mapreduce_pipeline._ReducerReader.from_json(reader_state)
it = iter(reader)
self.assertEquals(("key2", ["a", "b", "c", "d"]), it.next())
def store_image(self, link):
file_name = files.blobstore.create(
mime_type='application/octet-stream')
with files.open(file_name, 'a') as f:
f.write(urlfetch.Fetch(link, deadline=60).content)
files.finalize(file_name)
self.image_name = file_name
def testMergeFiles(self):
"""Test merging multiple files."""
input_data = [(str(i), "_" + str(i)) for i in range(100)]
input_data.sort()
input_file = files.blobstore.create()
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
for (k, v) in input_data:
proto = file_service_pb.KeyValue()
proto.set_key(k)
proto.set_value(v)
w.write(proto.Encode())
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
p = TestMergePipeline([input_file, input_file, input_file])
p.start()
test_support.execute_until_empty(self.taskqueue)
p = TestMergePipeline.from_id(p.pipeline_id)
output_file = p.outputs.default.value[0]
output_data = []
with files.open(output_file, "r") as f:
for record in records.RecordsReader(f):
output_data.append(record)
expected_data = [
str((k, [v, v, v], False)) for (k, v) in input_data]
self.assertEquals(expected_data, output_data)
def _save(self, name, content):
name = name.replace('\\', '/')
if hasattr(content, 'file') and \
hasattr(content.file, 'blobstore_info'):
data = content.file.blobstore_info
elif hasattr(content, 'blobstore_info'):
data = content.blobstore_info
elif isinstance(content, File):
guessed_type = mimetypes.guess_type(name)[0]
file_name = files.blobstore.create(mime_type=guessed_type or 'application/octet-stream',
_blobinfo_uploaded_filename=name)
with files.open(file_name, 'a') as f:
for chunk in content.chunks():
f.write(chunk)
files.finalize(file_name)
data = files.blobstore.get_blob_key(file_name)
else:
raise ValueError("The App Engine storage backend only supports "
"BlobstoreFile instances or File instances.")
if isinstance(data, (BlobInfo, BlobKey)):
# We change the file name to the BlobKey's str() value.
if isinstance(data, BlobInfo):
data = data.key()
return '%s/%s' % (data, name.lstrip('/'))
else:
raise ValueError("The App Engine Blobstore only supports "
"BlobInfo values. Data can't be uploaded "
"directly. You have to use the file upload "
"handler.")
def testSortFile(self):
"""Test sorting a file."""
input_file = files.blobstore.create()
input_data = [
(str(i), "_" + str(i)) for i in range(100)]
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
for (k, v) in input_data:
proto = file_service_pb.KeyValue()
proto.set_key(k)
proto.set_value(v)
w.write(proto.Encode())
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
p = shuffler._SortChunksPipeline("testjob", [input_file])
p.start()
test_support.execute_until_empty(self.taskqueue)
p = shuffler._SortChunksPipeline.from_id(p.pipeline_id)
input_data.sort()
output_files = p.outputs.default.value[0]
output_data = []
for output_file in output_files:
with files.open(output_file, "r") as f:
for binary_record in records.RecordsReader(f):
proto = file_service_pb.KeyValue()
proto.ParseFromString(binary_record)
output_data.append((proto.key(), proto.value()))
self.assertEquals(input_data, output_data)
def save_blob(self, blobdata='blobdata'):
file_name = files.blobstore.create(mime_type='application/octet-stream')
with files.open(file_name, 'a') as f:
f.write(blobdata)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
return blob_key
def finish_upload(file_id):
file_item = File.get_by_id(file_id)
size = file_item.size
current_size = file_item.current_size
write_path = file_item.write_path
files.finalize(write_path)
if size != current_size:
file_item.delete()
raise Exception('file size error')
# check sha1
sh = hashlib.sha1()
with files.open(file_item.read_path, 'r') as fp:
buf = fp.read(1000000)
while buf:
sh.update(buf)
buf = fp.read(1000000)
sha1_value = sh.hexdigest()
if sha1_value != file_item.sha1:
file_item.delete()
raise Exception('file size error')
file_item.upload_finished = True
file_item.put()
def set(key, result, time=DEFAULT_LAYER_CACHE_EXPIRATION_SECONDS, namespace=""):
old_blob_infos = BlobCache.get_blob_infos(key, namespace)
value = pickle.dumps(result)
# Create the file
file_name = files.blobstore.create(mime_type='application/octet-stream', _blobinfo_uploaded_filename=BlobCache.get_filename(key, namespace))
# might need to wrap it in an object to handle expiration time here
# write the pickled result to the file
pos = 0
chunkSize = 65536
with files.open(file_name, 'a') as f:
while pos < len(value):
chunk = value[pos:pos+chunkSize]
pos += chunkSize
f.write(chunk)
# Finalize the file. Do this before attempting to read it.
files.finalize(file_name)
# Get the file's blob key
blob_key = files.blobstore.get_blob_key(file_name)
for info in old_blob_infos:
try:
info.delete()
except Exception, e:
# If deleting blob times out, don't crash the request. Just log the error.
logging.error("Failed to delete old blob from layer_cache: %s" % e)
def moveCoverToBlobstore(album):
if not album.small_filetype:
return
from slughifi import slughifi
fn = "%s_%s"%(slughifi(album.artist), slughifi(album.title))
small_file = files.blobstore.create(mime_type=album.small_filetype,
_blobinfo_uploaded_filename="%s_small.png"%fn)
large_file = files.blobstore.create(mime_type=album.large_filetype,
_blobinfo_uploaded_filename="%s_big.png"%fn)
with files.open(small_file, 'a') as small:
small.write(album.small_cover)
with files.open(large_file, 'a') as large:
large.write(album.large_cover)
files.finalize(small_file)
files.finalize(large_file)
album.cover_small = files.blobstore.get_blob_key(small_file)
album.cover_large = files.blobstore.get_blob_key(large_file)
del album.small_cover
del album.large_cover
del album.large_filetype
del album.small_filetype
album.put()
def post(self):
rows=self.request.POST.get('file').value
key = self.request.POST.get('key')
file_name = files.blobstore.create(mime_type='text/plain')
with files.open(file_name, 'a') as f:
writer = csv.writer(f , delimiter=',')
for row in csv.reader(StringIO(rows), delimiter=','):
if len(row) > 1:
row[1] = self.crypt(row[1], key)
writer.writerow(row)
files.finalize(file_name)
blobs = blobstore.BlobInfo.all()
blob_links = [
'<a href="/serve/%s">File %s</a><br/>' % (blob.key(), index+1)
for index, blob in enumerate(blobs)
]
self.response.out.write(
'''
<html>
<body>
%s
</body>
</html>
''' % "".join(blob_links)
)
def post(self):
"""Store a file after encripting it.
"""
rows = self.request.POST.get('file').value
file_name = files.blobstore.create(mime_type='text/plain')
with files.open(file_name, 'a') as raw_file:
writer = csv.writer(raw_file, delimiter=',')
for row in csv.reader(StringIO(rows), delimiter=','):
if len(row) > 1:
row[1] = self.crypt(row[1])
writer.writerow(row)
files.finalize(file_name)
# TODO: with high replication, the newly created file will not
# show in the list.
#
# It show redirect to a page listing the files instead
blobs = blobstore.BlobInfo.all()
blob_links = [
'<a href="/serve/%s">File %s</a><br/>' % (blob.key(), index+1)
for index, blob in enumerate(blobs)
]
self.response.out.write("""<html>
<body>
<form action="/upload" enctype="multipart/form-data" method="post">
<input type="file" name="file"/><input type="submit" />
</form>
<br>
%s
</body>
</html>
""" % "".join(blob_links)
)
def get(self, firmid, process):
self.json_content()
firm_key = ndb.Key('Firm', firmid)
if process == 'process_hier':
deferred.defer( process_hier )
self.w( json.dumps("ok") )
if process == 'get_hier':
self.w(self.request.get('callback') + '(')
fname = '/gs/frl-arch/' + firmid + '/json/proj-detailed.json'
with files.open(fname, 'r') as f:
data = f.read(60 * 1000)
while data:
self.w( data )
data = f.read(60 * 1000)
self.w(');')
elif process == 'mail':
mail.send_mail( sender = 'Uri London <*****@*****.**>',
to = 'Live Person <*****@*****.**>',
subject = 'your test has succeeded',
body = 'Dear Live Person,\n\n I\'m happy to tell you that your test is good'
)
self.w( json.dumps('ok') )
elif process == 'test_file':
fname = '/gs/frl-arch/test.html'
wfname = files.gs.create( fname, mime_type='text/html', acl='public-read')
with files.open(wfname, 'a') as f:
f.write( 'hello world' )
f.write( 'bye bye' )
files.finalize(wfname)
def finalize(self, ctx, shard_state):
"""Finalize writer shard-level state.
Args:
ctx: an instance of context.Context.
shard_state: shard state.
"""
mapreduce_spec = ctx.mapreduce_spec
output_sharding = self.__class__._get_output_sharding(
mapper_spec=mapreduce_spec.mapper)
if self._request_filename is None or hasattr(self, "_183_test"):
writer_state = self._State.from_json(shard_state.writer_state)
self._request_filename = writer_state.request_filenames[0]
if output_sharding == self.OUTPUT_SHARDING_INPUT_SHARDS:
filesystem = self._get_filesystem(mapreduce_spec.mapper)
files.finalize(self._filename)
finalized_filenames = [self._get_finalized_filename(
filesystem, self._filename, self._request_filename)]
shard_state.writer_state = self._State(
finalized_filenames, []).to_json()
if filesystem == "blobstore":
logging.info(
"Shard %s-%s finalized blobstore file %s.",
mapreduce_spec.mapreduce_id,
shard_state.shard_number,
self._filename)
logging.info("Finalized name is %s.", finalized_filenames[0])
def store_raw_data_as_blob(data,name,content_type):
logging.info('store (bin) raw_data as %s (%s)'%(name,content_type))
file_name = files.blobstore.create(mime_type=content_type, _blobinfo_uploaded_filename = name)
with files.open(file_name, 'a') as out:
out.write(data)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
#http://code.google.com/p/googleappengine/issues/detail?id=4872
#FIXME
if not blob_key:
logging.info('again....1')
time.sleep(1)
blob_key = files.blobstore.get_blob_key(file_name)
if not blob_key:
logging.info('again....2')
time.sleep(1)
blob_key = files.blobstore.get_blob_key(file_name)
#endofhack
logging.info('file key:%s'%blob_key)
return blob_key
def POST(self):
"""Receive HTTP POST (json-format data), and store data to blobstore.
"""
data = json.loads(web.data())
# Create the file
# set filename property in BlobStore
# http://stackoverflow.com/questions/5697844/how-to-set-filename-property-in-blobstore
file_name = files.blobstore.create(mime_type='application/octet-stream',
_blobinfo_uploaded_filename=data['key'])
# Open the file and write to it
with files.open(file_name, 'a') as f:
f.write(base64.b64decode(data['payload']))
# Finalize the file. Do this before attempting to read it.
files.finalize(file_name)
# Get the file's blob key
blob_key = files.blobstore.get_blob_key(file_name)
# store blob_key in datastore
XmlBlobKey(id=data['key'], blob_key=blob_key).put()
return 'OK<br />%s<br />%s' % (data['key'], blob_key)
def bitcointalk_plot(bt_data=None, outputtype='html'):
dtf, (dt1, dt2) = plt.subplots(2, 1, sharex=True, figsize=(8, 4))
plt.subplots_adjust(hspace = .000)
dtf.suptitle('Bitcointalk')
if bt_data == None:
bt_data = bitcointalk_data()
bitcointalk(dt1, bt_data, keys=['new_topics','new_members'])
bitcointalk(dt2, bt_data, keys=['most_members_online'])
dt1.xaxis.set_major_formatter(matplotlib.dates.DateFormatter("%b\n%d"))
dt2.yaxis.set_ticks(dt2.get_yticks()[:-1])
dtf.set_size_inches(8,4)
if outputtype == 'html':
dt = StringIO.StringIO()
plt.savefig(dt, format="png")
return """<img src="data:image/png;base64,%s"/>""" % dt.getvalue().encode("base64").strip()
plt.clf()
elif outputtype == 'blobstore':
dt = StringIO.StringIO()
plt.savefig(dt, format="png")
file_name = files.blobstore.create(mime_type='image/png',
_blobinfo_uploaded_filename='bitcointalk.png')
with files.open(file_name, 'a') as f:
f.write(dt.getvalue())
files.finalize(file_name)
plt.clf()
else:
plt.show()
def finalize_job(cls, mapreduce_state):
"""Finalize job-level writer state.
Collect from model.ShardState if this job has output per shard.
Args:
mapreduce_state: an instance of model.MapreduceState describing current
job.
"""
state = cls._State.from_json(mapreduce_state.writer_state)
output_sharding = cls._get_output_sharding(mapreduce_state=mapreduce_state)
filesystem = cls._get_filesystem(mapreduce_state.mapreduce_spec.mapper)
if output_sharding != cls.OUTPUT_SHARDING_INPUT_SHARDS:
files.finalize(state.filenames[0])
finalized_filenames = [
cls._get_finalized_filename(filesystem, state.filenames[0], state.request_filenames[0])
]
else:
shards = model.ShardState.find_by_mapreduce_state(mapreduce_state)
finalized_filenames = []
for shard in shards:
state = cls._State.from_json(shard.writer_state)
finalized_filenames.append(state.filenames[0])
state.filenames = finalized_filenames
state.request_filenames = []
mapreduce_state.writer_state = state.to_json()
def post(self, post):
form = PostForm(data=self.request.POST, instance=post,
initial={'draft': post and post.published is None})
if form.is_valid():
post = form.save(commit=False)
image_url = form.clean_data['image_url']
if image_url:
try:
u = urllib2.urlopen(image_url)
file_name = files.blobstore.create(mime_type='image/jpeg')
with files.open(file_name, 'a') as f:
f.write(u.read())
files.finalize(file_name)
post.image_id = str(files.blobstore.get_blob_key(file_name))
except:
# Not sure how to use ErrorList in Django 0.96
# form._errors['image'] = ''
self.render_form(form)
return
if form.clean_data['draft']:# Draft post
post.published = datetime.datetime.max
post.put()
else:
if not post.path: # Publish post
post.updated = post.published = datetime.datetime.now()
else:# Edit post
post.updated = datetime.datetime.now()
post.publish()
self.render_to_response("published.html", {
'post': post,
'draft': form.clean_data['draft']})
else:
self.render_form(form)
def testRecordsReader(self):
"""End-to-end test for records reader."""
input_file = files.blobstore.create()
input_data = [str(i) for i in range(100)]
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
for record in input_data:
w.write(record)
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
mapreduce_id = control.start_map(
"test_map",
__name__ + ".TestHandler",
"mapreduce.input_readers.RecordsReader",
{
"file": input_file
},
shard_count=4,
base_path="/mapreduce_base_path")
test_support.execute_until_empty(self.taskqueue)
self.assertEquals(100, len(TestHandler.processed_entites))
def finalize(self, ctx, shard_state):
"""Finalize writer shard-level state.
Args:
ctx: an instance of context.Context.
shard_state: shard state.
"""
mapreduce_spec = ctx.mapreduce_spec
output_sharding = self.__class__._get_output_sharding(
mapper_spec=mapreduce_spec.mapper)
if output_sharding == self.OUTPUT_SHARDING_INPUT_SHARDS:
filesystem = self._get_filesystem(mapreduce_spec.mapper)
files.finalize(self._filename)
finalized_filenames = [self._get_finalized_filename(
filesystem, self._filename, self._request_filename)]
shard_state.writer_state = self._State(
finalized_filenames, []).to_json()
# Log to help debug empty blobstore key.
# b/8302363
if filesystem == "blobstore":
logging.info(
"Shard %s-%s finalized blobstore file %s.",
mapreduce_spec.mapreduce_id,
shard_state.shard_number,
self._filename)
logging.info("Finalized name is %s.", finalized_filenames[0])
def output(self, filename, file_out):
content = file_out.read()
base, ext = os.path.splitext(filename)
if ext == ".css":
mimetype = "text/css"
elif ext == ".js":
mimetype = "text/javascript"
else:
mimetype = "application/octet-stream"
already_exists = False
for info in BlobInfo.all().filter('content_type = ', mimetype):
if info.filename == filename:
already_exists = True
continue
#Clear out old blobs
if info.filename.split(".")[0] == filename.split(".")[0]:
logging.debug("Deleting: %s", info.filename)
info.delete()
if not already_exists:
logging.info("Creating: %s", filename)
result = files.blobstore.create(mime_type=mimetype, _blobinfo_uploaded_filename=filename)
with files.open(result, "a") as f:
f.write(content)
files.finalize(result)
blob_key = files.blobstore.get_blob_key(result)
while not blob_key:
blob_key = files.blobstore.get_blob_key(result)
def run(self, pid, config, *l):
r, g, b = l
w, h = config['width'], config['height']
r = Counter(tuple(x) for x in r if x)
g = Counter(tuple(x) for x in g if x)
b = Counter(tuple(x) for x in b if x)
ra, rb = minmax(r)
ga, gb = minmax(g)
ba, bb = minmax(b)
img = bmp.BitMap(w, h)
for y in range(h):
for x in range(w):
rc = int(round(255.0 * (r[(x, y)] - ra) / (rb - ra)))
gc = int(round(255.0 * (g[(x, y)] - ga) / (gb - ga)))
bc = int(round(255.0 * (b[(x, y)] - ba) / (bb - ba)))
img.setPenColor(bmp.Color(rc, gc, bc))
img.plotPoint(x, y)
filename = files.blobstore.create(mime_type='image/bmp')
with files.open(filename, 'a') as f:
f.write(img.getBitmap())
files.finalize(filename)
imr = ImgResult()
imr.pid = pid
imr.blob_key = files.blobstore.get_blob_key(filename)
imr.put()
def testHugeTaskUseDatastore(self):
"""Test map job with huge parameter values."""
input_file = files.blobstore.create()
input_data = [str(i) for i in range(100)]
with files.open(input_file, "a") as f:
with records.RecordsWriter(f) as w:
for record in input_data:
w.write(record)
files.finalize(input_file)
input_file = files.blobstore.get_file_name(
files.blobstore.get_blob_key(input_file))
mapreduce_id = control.start_map(
"test_map",
__name__ + ".TestHandler",
"mapreduce.input_readers.RecordsReader",
{
"file": input_file,
# the parameter can't be compressed and wouldn't fit into
# taskqueue payload
"huge_parameter": random_string(900000)
},
shard_count=4,
base_path="/mapreduce_base_path")
test_support.execute_until_empty(self.taskqueue)
self.assertEquals(100, len(TestHandler.processed_entites))
self.assertEquals([], model._HugeTaskPayload.all().fetch(100))
def WriteTempImageFile(stream):
blob = files.blobstore.create(_blobinfo_uploaded_filename="ImageTemp")
with files.open(blob, "a") as f:
f.write(stream.getvalue())
files.finalize(blob)
return files.blobstore.get_blob_key(blob)
def encoded_video_receiver(mp4_url, webm_url, video_id):
logging.info("starting the encoded_video_receiver")
if mp4_url and webm_url:
video = Video.get_by_id(int(video_id))
try:
# adding the mp4 video ref
mp4_file_name = files.blobstore.create(mime_type="video/mp4")
mp4_video = urllib2.urlopen(mp4_url)
with files.open(mp4_file_name, "a") as mp4_f:
mp4_f.write(mp4_video.read())
files.finalize(mp4_file_name)
mp4_blob_key = files.blobstore.get_blob_key(mp4_file_name)
video.mp4_video_ref = mp4_blob_key
# adding the webm video ref
webm_file_name = files.blobstore.create(mime_type="video/webm")
webm_video = urllib2.urlopen(webm_url)
with files.open(webm_file_name, "a") as webm_f:
webm_f.write(webm_video.read())
files.finalize(webm_file_name)
webm_blob_key = files.blobstore.get_blob_key(webm_file_name)
video.webm_video_ref = webm_blob_key
video.put()
logging.info("just saved the video in the datastore")
except urllib2.HTTPError, e:
logging.error("HTTPError: Failed somehow with a HTTPerror: " + str(e))
# time.sleep(10)
except Exception, e:
logging.error("Exception: Failed somehow: " + str(e))
def _make_backup_info_blobstore(self, num):
file_num = 3
dt = datetime.datetime.utcnow()
for i in range(num):
# _AE_Backup_Information
info = AeBackupInformation(
name=config.BACKUP_NAME,
filesystem='blobstore',
start_time=dt,
complete_time=dt + datetime.timedelta(hours=1)
)
info.put()
dt -= datetime.timedelta(days=1)
# _AE_Backup_Information_Kind_Files
info_files = AeBackupInformationKindFiles()
info_files.files = []
if i == (num - 1):
file_num = 120 # 100 over at last
for j in range(file_num):
# make blobstore file
file_name = files.blobstore.create(
mime_type='application/octet-stream')
with files.open(file_name, 'a') as f:
f.write('asdfghjkl')
files.finalize(file_name)
info_files.files.append(file_name)
info_files.key = ndb.Key(
AeBackupInformationKindFiles._get_kind(),
AeBackupInformationKindFiles.allocate_ids(1)[0],
parent=info.key
)
info_files.put()
def add_file_as_blob(filename, mime_type):
blob_name = files.blobstore.create(mime_type='image/jpeg',
_blobinfo_uploaded_filename=filename.split("/")[-1])
with files.open(blob_name, 'a') as f:
f.write(open(filename).read())
files.finalize(blob_name)
return files.blobstore.get_blob_key(blob_name)
def write_to_blobstore(data, mime_type, name=None):
"""Writes a file to the App Engine blobstore and returns an instance of a
BlobKey if successful.
:param data: Blob data.
:param mime_type: String, mime type of the blob.
:param name: String, name of the blob.
:returns: Instance of a `BlobKey`.
"""
if not name:
name = "".join(random.choice(string.letters) for x in range(DEFAULT_NAME_LEN))
blob = files.blobstore.create(mime_type=mime_type, _blobinfo_uploaded_filename=name)
with files.open(blob, "a", exclusive_lock=True) as f:
f.write(data)
files.finalize(blob)
result = files.blobstore.get_blob_key(blob)
# issue with the local development SDK. we can only write to the blobstore
# so fast, so set a retry_count and delay the execution thread between
# each attempt..
for i in range(1, WRITE_MAX_RETRIES):
if result:
break
else:
logging.debug("blob still None.. will retry to write to blobstore..")
time.sleep(WRITE_SLEEP_SECONDS)
result = files.blobstore.get_blob_key(blob)
logging.debug('File written to blobstore: key: "%s"', result)
return result
def uploadXmlsToBlobstore():
"""Writing Files to the Blobstore (Experimental) via remote api to
production server not working!
https://developers.google.com/appengine/docs/python/blobstore/overview#Writing_Files_to_the_Blobstore
http://stackoverflow.com/questions/8201283/google-app-engine-how-to-write-large-files-to-google-cloud-storage
http://stackoverflow.com/questions/3530990/upload-data-to-blobstore-using-remote-api
http://stackoverflow.com/questions/6545247/erratic-problem-with-app-engine-when-writing-files-directly-to-the-blobstore
"""
romn_dir = getRomnDir()
for dirpath, dirnames, filenames in os.walk(romn_dir):
for filename in filenames:
path = os.path.join(dirpath, filename)
key = path[len(romn_dir)+1:]
print('uploading %s ...' % key)
with open(path, 'rb') as f:
# Create the file
file_name = files.blobstore.create(mime_type='application/octet-stream')
# Open the file and write to it
with files.open(file_name, 'a') as f2:
f2.write(f.read())
# Finalize the file. Do this before attempting to read it.
files.finalize(file_name)
XmlBlobKey(id=key, blob_key=files.blobstore.get_blob_key(file_name)).put()
def post(self):
countries = self.all_countries
tourist_id = int(self.get_cookie("query")[0])
tourist = Tourist.Tourist.get_by_id(tourist_id)
new_email = self.request.get("email")
first_name = self.request.get("first_name")
last_name = self.request.get("last_name")
country = self.request.get("country")
state = self.request.get("state")
picture = self.request.POST.get("photo")
# self.write(picture.filename[picture.filename.find(".") + 1 : ])
profile_args = {"email" : new_email, "country" : country, "first_name" : first_name,
"last_name" : last_name, "state" : state, "picture" : picture}
if self.validate_email(new_email) and self.validate_name(first_name) and self.validate_name(last_name):
Tourist.Tourist.updateTourist(tourist, new_email, first_name, last_name, country, state)
if picture != None and picture != "":
picture_extension = picture.filename[picture.filename.rfind(".") : ]
picture_url = "/tourbly/profile_pictures/" + str(tourist.key().id()) + self.rand_salt("picture") + picture_extension
picture_name = "/gs" + picture_url
writable_file_name = files.gs.create(picture_name, mime_type='image/jpeg', acl='public-read')
with files.open(writable_file_name, 'a') as f:
f.write(picture.file.read())
files.finalize(writable_file_name)
tourist.picture = "http://storage.googleapis.com" + picture_url
tourist.put()
self.render("profile.html", isLoggedIn = self.check_session("query"), profile_args = profile_args,
success_message = "Your profile has been updated successfully", tourist = tourist,
countries = countries)
else:
self.render("profile.html", email_error = self.profile_email_error_prompt(tourist.email, new_email),
profile_args = profile_args, success_message = "there is something wrong", countries = countries)
def generate_pledges_csv(file_name, pledge_type, pledge_time, full_data=False):
""" Generates the pledges.csv file in a deferred way """
PAGE_SIZE = 500
csv_buffer = StringIO.StringIO()
w = csv.writer(csv_buffer)
if not pledge_time and pledge_type == 'WpPledge':
# First time through, add the column headers
if full_data:
headers = [
'SOURCE', 'donationTime', 'Amount ($)', 'url_nonce',
'stripeCustomer', 'Email', 'First Name', 'Last Name',
'Address', 'Address 2', 'City', 'State', 'Zip', 'Phone',
'Country', 'Occupation', 'Employer', 'Targeting', 'Type',
'Recurring', 'Source'
]
else:
headers = [
'zip', 'dollars', 'timestamp', 'date', 'city', 'state',
'latitude', 'longitude'
]
w.writerow(headers)
zg = Zipgun('zipgun/zipcodes')
# Get the next PAGE_SIZE pledges
query = getattr(model, pledge_type).all().order('donationTime')
if pledge_time:
# Filter instead of using 'offset' because offset is very inefficient,
# according to https://developers.google.com/appengine/articles/paging
query = query.filter('donationTime >= ', pledge_time)
pledges = query.fetch(PAGE_SIZE + 1)
next_pledge_time = None
if len(pledges) == PAGE_SIZE + 1:
next_pledge_time = pledges[-1].donationTime
pledges = pledges[:PAGE_SIZE]
# Loop through the current pledges and write them to the csv
for pledge in pledges:
w.writerow(pledge_row(pledge, zg, full_data))
with files.open(file_name, 'a') as f:
f.write(csv_buffer.getvalue())
csv_buffer.close()
if not next_pledge_time and pledge_type == 'Pledge':
# Last time through, finalize the file
files.finalize(file_name)
else:
# More to process, recursively run again
next_pledge_type = pledge_type
if pledge_type == 'WpPledge' and not next_pledge_time:
next_pledge_type = 'Pledge'
deferred.defer(generate_pledges_csv,
file_name,
next_pledge_type,
next_pledge_time,
full_data,
_queue='generateCSV')
def action_edit(self):
if self.section.handler.request.get('submit'):
self.SITE_HEADER = self.section.handler.request.get('SITE_HEADER')
self.SITE_SUB_HEADER = self.section.handler.request.get(
'SITE_SUB_HEADER')
self.DEFAULT_THEME = self.section.handler.request.get(
'DEFAULT_THEME')
self.GOOGLE_ANALYTICS_UA = self.section.handler.request.get(
'GOOGLE_ANALYTICS_UA')
self.ROBOTS_TXT = self.section.handler.request.get('ROBOTS_TXT')
if self.section.handler.request.get('FAVICON_ICO'):
if self.FAVICON_ICO:
BlobInfo.get(self.FAVICON_ICO).delete()
data = self.section.handler.request.get('FAVICON_ICO')
handle = files.blobstore.create(
mime_type='image/x-icon',
_blobinfo_uploaded_filename='favicon.ico')
with files.open(handle, 'a') as f:
f.write(data)
files.finalize(handle)
self.FAVICON_ICO = files.blobstore.get_blob_key(handle)
self.ENABLE_THEME_PREVIEW = self.section.handler.request.get(
'ENABLE_THEME_PREVIEW') != ''
self.DEBUG_MODE = self.section.handler.request.get(
'DEBUG_MODE') != ''
cache.delete(CACHE_KEY)
self.update()
raise Exception('Redirect', self.section.action_redirect_path)
f = form(self.section, self.section.full_path)
f.add_control(
control(self.section, 'text', 'SITE_HEADER', self.SITE_HEADER,
'Site header', 50))
f.add_control(
control(self.section, 'text', 'SITE_SUB_HEADER',
self.SITE_SUB_HEADER, 'Site sub-header', 50))
combined_themes = get_local_theme_namespaces(
) + get_custom_theme_namespaces()
f.add_control(
selectcontrol(
self.section, 'DEFAULT_THEME', combined_themes,
self.DEFAULT_THEME if self.DEFAULT_THEME else
DEFAULT_LOCAL_THEME_TEMPLATE, 'Default theme'))
f.add_control(
control(self.section, 'text', 'GOOGLE_ANALYTICS_UA',
self.GOOGLE_ANALYTICS_UA, 'Google analytics UA'))
f.add_control(
control(self.section, 'file', 'FAVICON_ICO', label='favicon.ico'))
f.add_control(
textareacontrol(self.section, 'ROBOTS_TXT', self.ROBOTS_TXT,
'robots.txt', 90, 5))
f.add_control(
checkboxcontrol(self.section, 'ENABLE_THEME_PREVIEW',
self.ENABLE_THEME_PREVIEW, 'Enable theme preview'))
f.add_control(
checkboxcontrol(self.section, 'DEBUG_MODE', self.DEBUG_MODE,
'Debug mode'))
f.add_control(control(self.section, 'submit', 'submit', 'Submit'))
return '<h2>Edit configuration</h2>%s' % unicode(f)
def _fetchContentMap(binary_record):
"""Map function of fetch content.
Fetched content will store to blobstore.
Arg:
binary_record: key value data, that key is url of target page,
value is url of target of fetch.
Returns:
url: fetched url.
"""
proto = file_service_pb.KeyValue()
proto.ParseFromString(binary_record)
page_url = proto.key()
target_url = proto.value()
#Fetch to CrawlDbDatum
try:
query = CrawlDbDatum.query(CrawlDbDatum.url == page_url)
crawl_db_datum_future = query.fetch_async()
except Exception as e:
logging.warning("Failed create key, caused by invalid url:" +
page_url + ":" + e.message)
#start fetch
fetcher = fetchers.SimpleHttpFetcher(1, fetcher_policy_yaml.fetcher_policy)
stored_url = None
if re.match("^/", target_url):
crawl_db_datum = _getCrawlDatum(crawl_db_datum_future)
target_url = "%s%s" % (crawl_db_datum.extract_domain_url, target_url)
try:
fetch_result = fetcher.get(target_url)
if fetch_result:
#Storing to blobstore
blob_io = files.blobstore.create(
mime_type=fetch_result.get("mime_type"),
_blobinfo_uploaded_filename=fetch_result.get("fetched_url"))
with files.open(blob_io, 'a') as f:
f.write(fetch_result.get("content"))
files.finalize(blob_io)
blob_key = files.blobstore.get_blob_key(blob_io)
stored_url = images.get_serving_url(str(blob_key))
except Exception as e:
logging.warning("Fetch Error Occurs:" + e.message)
#Put content to datastore.
crawl_db_datum = _getCrawlDatum(crawl_db_datum_future)
if crawl_db_datum and stored_url is not None:
entity = ContentDbDatum(
parent=crawl_db_datum.key,
fetched_url=fetch_result.get("fetched_url"),
stored_url=stored_url,
content_type=fetch_result.get("mime_type"),
content_size=fetch_result.get("content_length"),
http_headers=str(fetch_result.get("headers")))
entity.put()
yield "%s:%s" % (target_url, stored_url)
def close(self):
fn = files.blobstore.create(mime_type='application/octet-stream')
f = files.open(fn, 'a')
# TODO: grouping into chunks would probably help optimize RPC's
for b in self.bs:
f.write(b)
f.close()
files.finalize(fn)
return blobstore.get(files.blobstore.get_blob_key(fn))
def write_blob(self, data, info):
blob = files.blobstore.create(
mime_type=info['type'],
_blobinfo_uploaded_filename=info['name']
)
with files.open(blob, 'a') as f:
f.write(data)
files.finalize(blob)
return files.blobstore.get_blob_key(blob)
def post(self, id):
self.response.content_type = "application/json"
if self._content is None:
self.response.status = 400
self.response.out.write(
utils.createError(
400, "Couldn't decode content or invalid content-type"))
# Trying to access card to see if user is allowed to
request = self._service.timeline().get(id=id)
try:
card = request.execute()
except HttpError as e:
self.response.status = e.resp.status
self.response.out.write(e.content)
return
# 2) Insert data into blob store
file_name = files.blobstore.create(mime_type=self._content_type)
with files.open(file_name, 'a') as f:
f.write(self._content)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
# 3) Update card with attachment info
if not "attachments" in card:
card["attachments"] = []
attachment = {
"id":
str(blob_key),
"contentType":
self._content_type,
"contentUrl":
"%s/upload/mirror/v1/timeline/%s/attachments/%s" %
(utils.base_url, card["id"], str(blob_key)),
"isProcessing":
False
}
card["attachments"].append(attachment)
request = self._service.internal().timeline().update(id=card["id"],
body=card)
try:
result = request.execute()
except HttpError as e:
self.response.status = e.resp.status
self.response.out.write(e.content)
return
self.response.status = 200
self.response.out.write(json.dumps(result))
def store_file_as_blob(self, dropbox_link):
response = urllib2.urlopen(dropbox_link)
data = response.read(
) # TODO In future, read 10Mb at a time rather than entire file
blob_file_name = files.blobstore.create(
mime_type='application/octet-stream')
with files.open(blob_file_name, 'a') as f:
f.write(data)
files.finalize(blob_file_name)
return files.blobstore.get_blob_key(blob_file_name)
def finalize(self, ctx, shard_number):
"""Finalize writer shard-level state.
Args:
ctx: an instance of context.Context.
shard_number: shard number as integer.
"""
finalized_filenames = []
for filename in self._filenames:
files.finalize(filename)
def post(self):
"""
POST
'number' and 'maxsize' are required.
"""
number = self.request.POST.get('number', '')
maxsize = self.request.POST.get('maxsize', '')
context = {
'number': number,
'maxsize': maxsize,
}
errors = []
try:
number = int(number.strip())
except ValueError:
errors.append('Number must be an integer.')
try:
maxsize = int(maxsize.strip())
except ValueError:
errors.append('Max Size must be an integer.')
if number < 1:
errors.append('Number must be at least 1.')
if maxsize < 100:
errors.append('Max Size must be at least 100 bytes.')
if maxsize > 10000000:
errors.append('Max Size must be less than 10000000 bytes.')
if errors:
context['errors'] = errors
self.render_response('create.html', **context)
return
for index in range(number):
content_type, extension = random.choice(self.SAMPLE_CONTENT_TYPES)
kwargs = {'mime_type': content_type}
# put an uploaded filename on some random set of files
if random.random() < 0.5:
random_chars = (''.join(
random.sample(string.letters + string.digits, 12)))
kwargs['_blobinfo_uploaded_filename'] = (
'testfile-%s-%s.%s' % (index, random_chars, extension))
output_filename = files.blobstore.create(**kwargs)
with files.open(output_filename, 'a') as outfile:
input_bytes = '1' * (random.randint(100, maxsize))
outfile.write(input_bytes)
files.finalize(output_filename)
context['message'] = ('%s blob%s created. ' %
(number, number > 1 and 's' or '') +
'<a href="/view-blobs">View blobs</a>')
self.render_response('create.html', **context)
def finalize_job(cls, mapreduce_state):
"""See parent class."""
output_sharding = cls._get_output_sharding(
mapreduce_state=mapreduce_state)
if output_sharding != cls.OUTPUT_SHARDING_INPUT_SHARDS:
state = cls._State.from_json(mapreduce_state.writer_state)
files.finalize(state.filenames[0])
finalized_filenames = cls.get_filenames(mapreduce_state)
state = cls._State(finalized_filenames, [])
mapreduce_state.writer_state = state.to_json()
def generate_attachments(self, project):
file_string = self.generate_attachments_zip(project)
blob_file_name = files.blobstore.create(mime_type=u'application/octet-stream')
with files.open(blob_file_name, u'a') as f:
f.write(file_string)
files.finalize(blob_file_name)
if project.blob_key:
blobstore.delete(project.blob_key)
project.blob_key = files.blobstore.get_blob_key(blob_file_name)
project.filename = u'{}.zip'.format(project.internal_name)
project.put()
def _save(self, name, content):
objectname = self._gcs_path(name)
writable_name = files.gs.create(
objectname,
mime_type='application/vnd.google-earth.kml+xml',
acl="public-read")
with files.open(writable_name, 'a') as fp:
fp.write(content.read())
files.finalize(writable_name)
logging.debug("_save wrote %s " % objectname)
return name
def _save(self, name, content):
file_name = files.blobstore.create(
mime_type='application/octet-stream',
_blobinfo_uploaded_filename=name)
with files.open(file_name, 'a') as f:
content.seek(0)
f.write(content.read())
files.finalize(file_name)
return name
def uploadImage(b64image):
file_name = files.blobstore.create(mime_type='image/png')
with files.open(file_name, 'a') as f:
f.write(b64image)
files.finalize(file_name)
key = files.blobstore.get_blob_key(file_name)
return key
def setUp(self):
super(FileTestCase, self).setUp()
# Create a blob and blob_reader for testing.
filename = blobstore_files.blobstore.create(
mime_type='application/octet-stream')
with blobstore_files.open(filename, 'a') as fp:
fp.write('Blobstore!')
blobstore_files.finalize(filename)
self.blob_key = blobstore_files.blobstore.get_blob_key(filename)
self.blob_reader = blobstore.BlobReader(self.blob_key)
def createCloudStorageFiles(files_num, content):
"""Create cloudstorage file for test"""
file_names = []
for num in range(files_num):
file_path = "/gs/" + TEST_BUCKET_NAME + "/contents_" + str(num)
file_name = files.gs.create(file_path)
for text in content:
with files.open(file_name, 'a') as fp:
fp.write(text + "\n")
files.finalize(file_name)
file_names.append(file_path)
return file_names
def create():
from google.appengine.ext import blobstore
from google.appengine.api import files
file_name = files.blobstore.create(
mime_type='text/javascript',
_blobinfo_uploaded_filename=simplr.random())
with files.open(file_name, 'a') as f:
f.write(content)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
logging.info('Backup Success: blob key is %s' % blob_key)
return 'Backup Was Successful'
def write_blob(upload_file):
file_name = files.blobstore.create(mime_type=upload_file.type)
# Open the file and write to it
with files.open(file_name, 'a') as f:
f.write(upload_file.value)
# Finalize the file. Do this before attempting to read it.
files.finalize(file_name)
# Get the file's blob key
return str(files.blobstore.get_blob_key(file_name))
def run(self,
file_names,
bucket_name):
#Save filenames to cloud storage
#If already exists same filename, will override it.
print("/gs/"+bucket_name+TARGETS_FILE_PATH)
target_file = files.gs.create("/gs/"+bucket_name+TARGETS_FILE_PATH)
#Append the filename to the Google Cloud Storage object.
for file_name in file_names:
with files.open(target_file, 'a') as fp:
fp.write(file_name+"\n")
files.finalize(target_file)
def post(self):
QL.Results = []
for key, value in self.request.params.items():
if isinstance(value, cgi.FieldStorage):
upload = value
if upload.type:
mimetype = upload.type
else:
mimetype = 'application/octet-stream'
if upload.filename:
pass
filename = urllib.quote(str(upload.filename.encode('utf8')))
else:
filename = "no-name-" + str(time())[-6:]
try:
title = strftime("%Y%m%d",gmtime()) + '/' + filename
filepath = '/gs/%s/%s' % (BUCKET,title)
real_filepath = files.gs.create(filename=filepath,
acl='bucket-owner-full-control',
mime_type= mimetype,
cache_control='no-cache')
with files.open(real_filepath,'a') as fp:
pass
fp.write(upload.value)
files.finalize(real_filepath)
# template_values = {
# 'message':'Upload Success!',
# 'origin_url':"http://commondatastorage.googleapis.com/" + filepath[4:]
# }
# origin_url = "http://commondatastorage.googleapis.com/" + filepath[4:]
origin_url = 'http://'+self.request.host + '/view/' + filepath[4:]
QL.add(str(upload.filename.encode('utf8')), 'Success', origin_url)
#print str(QL.Results)
#self.response.write(str(QL.Results))
#self.out(template_values)
except BaseException, e:
pass
#raise(e)
# template_values = {
# 'message':'Upload Failed!'
# }
#self.out(template_values)
QL.add(str(upload.filename.encode('utf8')), 'Failed', 'None')
def finalize_job(cls, mapreduce_state):
"""See parent class."""
output_sharding = cls._get_output_sharding(
mapreduce_state=mapreduce_state)
if output_sharding != cls.OUTPUT_SHARDING_INPUT_SHARDS:
state = cls._State.from_json(mapreduce_state.writer_state)
files.finalize(state.filenames[0])
# Keep writing for 183 clients.
# TODO(user): Remove after 184 is out.
finalized_filenames = cls.get_filenames(mapreduce_state)
state = cls._State(finalized_filenames, [])
mapreduce_state.writer_state = state.to_json()
def post(self, unused_key):
"""Creates a file from the posted data."""
filename = files.blobstore.create()
f = files.open(filename, "a")
f.write(self.request.get('text'))
f.close()
files.finalize(filename)
self.redirect('/')
def post(self):
# user = users.get_current_user()
# userQuery = User.gql("WHERE name='{}'".format(user.nickname())).get()
patient = Patient()
patient.firstName = cgi.escape(self.request.get('firstName'))
patient.lastName = cgi.escape(self.request.get('lastName'))
patient.middleName = cgi.escape(self.request.get('middleName'))
patient.age = cgi.escape(self.request.get('age'))
patient.leftEyeRating = cgi.escape(self.request.get('leftEyeRating'))
patient.rightEyeRating = cgi.escape(self.request.get('rightEyeRating'))
patient.note = cgi.escape(self.request.get('note'))
patient.join_date = datetime.datetime.now().date()
patient.sex = cgi.escape(self.request.get('sex'))
patient.leftEyeNote = cgi.escape(self.request.get('leftEyeNote'))
patient.rightEyeNote = cgi.escape(self.request.get('rightEyeNote'))
patient.clinicNumber = 1
patient.otherNotes = cgi.escape(self.request.get('otherNotes'))
upload_files = self.get_uploads('inputLeftEye')
if (len(upload_files)):
# logging.ERROR('Start guestbook signing request')
blob_info = upload_files[0]
patient.leftEyePhoto = str(upload_files[0].key())
patient.leftEyePhotoURL = str(blob_info.key())
else:
leftimg = cgi.escape(self.request.get('inputLeftEyeHidden'))
if (len(leftimg)):
#reconstruct and store image
leftimg = base64.b64decode(leftimg)
# image/jpeg
file_name = files.blobstore.create(mime_type='image/jpeg')
# Open the file and write to it
with files.open(file_name, 'a') as f:
f.write('leftimg')
# Finalize the file. Do this before attempting to read it.
files.finalize(file_name)
# Get the file's blob key
blob_key = files.blobstore.get_blob_key(file_name)
patient.leftEyePhotoURL = str(blob_key)
upload_files2 = self.get_uploads('inputRightEye')
if (len(upload_files2)):
blob_info2 = upload_files2[0]
patient.rightEyePhotoURL = str(blob_info2.key())
patient.rightEyePhoto = upload_files2[0].key()
patient.put()
def render_text_into_blob(text):
#text = realestate.email if realestate.email else ''
from pybmp.bmpfont_arial_12 import font_data
from pybmp.bmp import BitMap, Color
bmp = BitMap(250, 22, Color.WHITE)
bmp.setFont(font_data)
bmp.setPenColor(Color.BLACK.darken())
bmp.drawText(text, 4, 4)
#bmp.drawLine( 0, 0, 10, 10)
f = StringIO()
f.write(bmp.getBitmap())
f.flush()
file_name = files.blobstore.create(mime_type='image/bmp',
_blobinfo_uploaded_filename=str(
random.randint(0, 100000)))
img = images.Image(image_data=f.getvalue())
# img = images.Image.open(f)
img.resize(width=250, height=22)
myfile = files.open(file_name, 'a')
myfile.write(
img.execute_transforms(output_encoding=images.JPEG, quality=100))
myfile.close()
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
for i in range(1, 10):
if not blob_key:
time.sleep(0.05)
blob_key = files.blobstore.get_blob_key(file_name)
else:
break
if not blob_key:
logging.error(
"no pude obtener el blob_key, hay un leak en el blobstore!")
abort(500)
return blob_key
# realestate.email_image = blob_key
# realestate.email_image_url = get_serving_url(blob_key)
# realestate.save()
return 0
def write_blob(self, data, info):
blob = files.blobstore.create(mime_type=info['type'],
_blobinfo_uploaded_filename=info['name'])
# image = images.Image(image_data=data)
# image.im_feeling_lucky()
# image.resize(width=980)
# image.execute_transforms(output_encoding=images.JPEG)
# blob = files.blobstore.create(mime_type='image/jpeg')
with files.open(blob, 'a') as f:
f.write(data)
files.finalize(blob)
return files.blobstore.get_blob_key(blob)
def post(self):
lang = self.request.get('lang')
extracted_data = ''
file_name = files.blobstore.create(mime_type='text/json', _blobinfo_uploaded_filename='extract_' + lang + time.strftime("%y%m%d") + '.json')
with files.open(file_name, 'a') as f:
ancestor = ndb.Key('Language', lang)
for candidate in SuspectedEncoding.query(ancestor=ancestor):
f.write(serializer.encode({
'key': candidate.key.id(),
'approved': candidate.approved,
'rejected': candidate.rejected}) + '\n')
files.finalize(file_name)
logging.info('finished writing file ' + file_name)
def create(cls, team, datafile):
file_name = files.blobstore.create(
mime_type='application/octet-stream')
with files.open(file_name, 'a') as f:
for chunk in datafile.chunks():
f.write(chunk)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
braindata = cls(blob=blobstore.BlobInfo.get(blob_key),
team=team,
filename=datafile.name,
parent=team.group)
braindata.put()
def gs_write(name, mime, content, cache='no-cache'):
if os.environ.get('SERVER_SOFTWARE').startswith('Google App Engine'):
fn = files.gs.create(
'/gs/' + name,
mime_type=mime,
acl='public-read',
cache_control=cache,
)
with files.open(fn, 'a') as f:
f.write(content)
files.finalize(fn)
else:
logging.info('gs write: %s, %s, %s', mime, name, len(content))
def finalize(self, ctx, shard_number):
"""Finalize writer shard-level state.
Args:
ctx: an instance of context.Context.
shard_number: shard number as integer.
"""
mapreduce_spec = ctx.mapreduce_spec
output_sharding = _get_output_sharding(
mapper_spec=mapreduce_spec.mapper)
if output_sharding == self.OUTPUT_SHARDING_INPUT_SHARDS:
files.finalize(self._filename)
def finalize_job(cls, mapreduce_state):
"""Finalize job-level writer state.
Args:
mapreduce_state: an instance of model.MapreduceState describing current
job.
"""
state = BlobstoreOutputWriter._State.from_json(
mapreduce_state.writer_state)
files.finalize(state.filename)
state.filename = files.blobstore.get_file_name(
files.blobstore.get_blob_key(state.filename))
mapreduce_state.writer_state = state.to_json()
