def blobfiles(insert, archive_key): """ We do not use ancestor queries. This Generator takes care of index and entity inconsistencies https://cloud.google.com/developers/articles/balancing-strong-and-eventual-consistency-with-google-cloud-datastore/ """ for bf in BlobFiles.query().filter(BlobFiles.key != archive_key).map(callback, keys_only=True): if insert and new_bf.key == bf.key: insert = False # no index inconsistency yield bf # if the new_bf entity is not yet present in BlobFiles (due to index inconsistencies), it will be inserted here if insert: yield new_bf
def blob_archive(new_bf=None, archive_path='/'): """ bonus: save all BlobFiles in a zip archive """ @ndb.tasklet def callback(bf_key): """ key_only query and get() lookup for entity consistency """ bf = yield bf_key.get_async() raise ndb.Return(bf) def blobfiles(insert, archive_key): """ We do not use ancestor queries. This Generator takes care of index and entity inconsistencies https://cloud.google.com/developers/articles/balancing-strong-and-eventual-consistency-with-google-cloud-datastore/ """ for bf in BlobFiles.query().filter(BlobFiles.key != archive_key).map(callback, keys_only=True): if insert and new_bf.key == bf.key: insert = False # no index inconsistency yield bf # if the new_bf entity is not yet present in BlobFiles (due to index inconsistencies), it will be inserted here if insert: yield new_bf # add all files to archive, except the archive zipfile itself which has a reserved name (BlobFiles key) (archive_folder, _, archive_file) = archive_path if new_bf and new_bf.filename != archive_file: new_zf = BlobFiles.new(archive_file, folder=archive_folder) with gcs.open(new_zf.gcs_filename, 'w', content_type=b'multipart/x-zip', options={b'x-goog-acl': b'public-read', b'cache-control': b'private, max-age=0, no-cache'}) as nzf: # nzf is a cloudstorage.storage_api.StreamingBuffer, which can be pickled to append data in a chained task with zipfile.ZipFile(nzf, 'w') as zf: for each in blobfiles(new_bf is not None, new_zf.key): # We also could have used : each.blob_read() logging.info(each.filename) blob = each.blob_reader().read() zf.writestr(each.filename.encode('utf-8'), blob) new_zf.put_async() else: new_zf = new_bf return new_zf