def testWriteToBlobstore(self): # There are stronger tests for the behavior of this in files_test. old_blob_key = utils.write_to_blobstore('Blobstore!') self.assertTrue(old_blob_key) old_blobinfo = blobstore.BlobInfo.get(old_blob_key) new_blob_key = utils.write_to_blobstore('Blobstore!', old_blobinfo=old_blobinfo) self.assertEqual(new_blob_key, old_blob_key) self.stubs.SmartUnsetAll()
def _maybe_write_to_blobstore(self, content, blob, force_blobstore=False): if content and blob: raise TypeError('Exactly one of "content" or "blob" must be given.') if force_blobstore or content and len(content) > MAX_CONTENT_SIZE: if not force_blobstore: logging.debug( 'Content size %s exceeds %s bytes, uploading to blobstore.', len(content), MAX_CONTENT_SIZE) old_blobinfo = self.blob if self.exists else None blob = utils.write_to_blobstore(content, old_blobinfo=old_blobinfo) _store_blob_cache(self.real_path, content) content = None return content, blob
def testWrite(self): expected_file = files._TitanFile( id='/foo/bar.html', name='bar.html', content='Test', dir_path='/foo', paths=[u'/', u'/foo'], depth=1, mime_type=u'text/html', created_by=users.TitanUser('*****@*****.**'), modified_by=users.TitanUser('*****@*****.**'), # Arbitrary meta data for expando: color=u'blue', flag=False, md5_hash=hashlib.md5('Test').hexdigest(), ) original_expected_file = copy.deepcopy(expected_file) meta = {'color': 'blue', 'flag': False} new_meta = {'color': 'blue', 'flag': True} dates = ['modified', 'created'] # Synchronous write of a new file. actual_file = files.File('/foo/bar.html').write('Test', meta=meta) self.assertNdbEntityEqual(expected_file, actual_file._file, ignore=dates) self.assertNotEqual(None, actual_file.modified, 'modified is not being set') # Synchronous update without changes. actual_file = files.File('/foo/bar.html').write(meta=meta) self.assertNdbEntityEqual(expected_file, actual_file._file, ignore=dates) # Synchronous update with changes. old_modified = actual_file.modified actual_file = files.File('/foo/bar.html') actual_file.write('New content', meta=new_meta, mime_type='fake/type') expected_file.content = 'New content' expected_file.md5_hash = hashlib.md5('New content').hexdigest() expected_file.flag = True expected_file.mime_type = 'fake/type' self.assertNdbEntityEqual(expected_file, actual_file._file, ignore=dates) self.assertNotEqual(old_modified, actual_file.modified) # Allow writing blank files. actual_file = files.File('/foo/bar.html').write('') self.assertEqual(actual_file.content, '') # Allow overwriting mime_type and meta without touching content. files.File('/foo/bar.html').write(content='Test') actual_file = files.File('/foo/bar.html').write( mime_type='fake/mimetype') self.assertEqual('fake/mimetype', actual_file.mime_type) self.assertEqual('Test', actual_file.content) actual_file = files.File('/foo/bar.html').write(meta=new_meta) self.assertEqual(True, actual_file.meta.flag) self.assertEqual('Test', actual_file.content) # Allow overwriting created and modified without touching content. files.File('/foo/bar.html').write(content='Test') now = datetime.datetime.now() + datetime.timedelta(days=1) actual_file = files.File('/foo/bar.html').write(created=now, modified=now) self.assertEqual(now, actual_file.created) self.assertEqual(now, actual_file.modified) # Verify the same behavior for the file creation codepath. files.File('/foo/bar.html').delete().write('Test', created=now, modified=now) self.assertEqual(now, actual_file.created) self.assertEqual(now, actual_file.modified) # Error handling. self.assertRaises(ValueError, files.File('/a').write, '', created='foo') self.assertRaises(ValueError, files.File('/a').write, '', modified='foo') # Allow overwriting created_by and modified_by without touching content. files.File('/foo/bar.html').write(content='Test') user = users.TitanUser( '*****@*****.**') # Not the current logged in user. actual_file = files.File('/foo/bar.html').write(created_by=user, modified_by=user) self.assertEqual(user, actual_file.created_by) self.assertEqual(user, actual_file.modified_by) # Verify the same behavior for the file creation codepath. files.File('/foo/bar.html').delete().write('Test', created_by=user, modified_by=user) self.assertEqual(user, actual_file.created_by) self.assertEqual(user, actual_file.modified_by) # Error handling. self.assertRaises(ValueError, files.File('/a').write, '', created_by='foo') self.assertRaises(ValueError, files.File('/a').write, '', modified_by='foo') # Cleanup. expected_file = original_expected_file files.File('/foo/bar.html').delete() # write large content to blobstore. titan_file = files.File('/foo/bar.html').write( content=LARGE_FILE_CONTENT) blob_key = titan_file.blob.key() self.assertTrue(blob_key) self.assertEqual(LARGE_FILE_CONTENT, titan_file.content) self.assertIsNone(titan_file._file_ent.content) self.assertEqual(LARGE_FILE_CONTENT, files.File('/foo/bar.html').content) self.assertEqual( hashlib.md5(LARGE_FILE_CONTENT).hexdigest(), titan_file.md5_hash) # De-duping check: verify the blob key doesn't change if the content # doesn't change. old_blob_key = blob_key titan_file = files.File('/foo/bar.html').write( content=LARGE_FILE_CONTENT) blob_key = titan_file.blob.key() self.assertEqual(old_blob_key, blob_key) self.assertEqual(LARGE_FILE_CONTENT, titan_file.content) self.assertIsNone(titan_file._file_ent.content) self.assertEqual(LARGE_FILE_CONTENT, files.File('/foo/bar.html').content) self.stubs.SmartUnsetAll() # write with a blob key and encoding; verify proper decoding. encoded_foo = u'f♥♥'.encode('utf-8') blob_key = utils.write_to_blobstore(encoded_foo) titan_file = files.File('/foo/bar.html') # Verify that without encoding, the encoded bytestring is returned. titan_file.write(blob=blob_key) self.assertEqual(encoded_foo, titan_file.content) # Verify that with encoding, a unicode string is returned. titan_file.write(blob=blob_key, encoding='utf-8') self.assertEqual(u'f♥♥', titan_file.content) # Argument error handling for mixing encoding and unicode content: self.assertRaises(TypeError, titan_file.write, content=u'Test', encoding='utf-8') # Make sure the blob is deleted with the file: titan_file.delete() self.assertIsNone(blobstore.get(blob_key)) self.assertRaises(files.BadFileError, lambda: titan_file.blob) # Make sure the blob is deleted if the file gets smaller: titan_file = files.File('/foo/bar.html').write( content=LARGE_FILE_CONTENT) blob_key = titan_file.blob.key() titan_file.write(content='Test') self.assertIsNone(blobstore.get(blob_key)) # Test the current object and a new instance: self.assertEqual('Test', titan_file.content) self.assertEqual('Test', files.File('/foo/bar.html').content) self.assertIsNone(titan_file.blob) self.assertIsNone(files.File('/foo/bar.html').blob) # write with a BlobKey: titan_file = files.File('/foo/bar.html').write(blob=self.blob_key) blob_content = self.blob_reader.read() # Test the current object and a new instance: self.assertEqual(blob_content, files.File('/foo/bar.html').content) self.assertEqual(blob_content, titan_file.content) self.assertEqual( hashlib.md5('Blobstore!').hexdigest(), titan_file.md5_hash) # Cleanup. expected_file = original_expected_file files.File('/foo/bar.html').delete() # Error handling: # Updating mime_type or meta when entity doesn't exist. titan_file = files.File('/fake/file') self.assertRaises(files.BadFileError, titan_file.write, meta=meta) self.assertRaises(files.BadFileError, titan_file.write, mime_type='fake/mimetype') # Bad arguments: self.assertRaises(TypeError, titan_file.write) self.assertRaises(TypeError, titan_file.write, content=None, blob=None) self.assertRaises(TypeError, titan_file.write, content='Test', blob=self.blob_key) self.assertRaises(TypeError, titan_file.write, encoding='utf-8') # There are some reserved words that cannot be used in meta properties. invalid_meta_keys = [ # Titan reserved: 'name', 'path', 'dir_path', 'paths', 'depth', 'mime_type', 'encoding', 'created', 'modified', 'content', 'blob', 'blobs', 'created_by', 'modified_by', 'md5_hash', # NDB reserved: 'key', 'app', 'id', 'parent', 'namespace', 'projection', ] for key in invalid_meta_keys: try: titan_file.write(content='', meta={key: ''}) except files.InvalidMetaError: pass else: self.fail( 'Invalid meta key should have failed: {!r}'.format(key))
def testWrite(self): expected_file = files._TitanFile( id='/foo/bar.html', name='bar.html', content='Test', dir_path='/foo', paths=[u'/', u'/foo'], depth=1, mime_type=u'text/html', created_by=users.TitanUser('*****@*****.**'), modified_by=users.TitanUser('*****@*****.**'), # Arbitrary meta data for expando: color=u'blue', flag=False, md5_hash=hashlib.md5('Test').hexdigest(), ) original_expected_file = copy.deepcopy(expected_file) meta = {'color': 'blue', 'flag': False} new_meta = {'color': 'blue', 'flag': True} dates = ['modified', 'created'] # Synchronous write of a new file. actual_file = files.File('/foo/bar.html').write('Test', meta=meta) self.assertNdbEntityEqual(expected_file, actual_file._file, ignore=dates) self.assertNotEqual(None, actual_file.modified, 'modified is not being set') # Synchronous update without changes. actual_file = files.File('/foo/bar.html').write(meta=meta) self.assertNdbEntityEqual(expected_file, actual_file._file, ignore=dates) # Synchronous update with changes. old_modified = actual_file.modified actual_file = files.File('/foo/bar.html') actual_file.write('New content', meta=new_meta, mime_type='fake/type') expected_file.content = 'New content' expected_file.md5_hash = hashlib.md5('New content').hexdigest() expected_file.flag = True expected_file.mime_type = 'fake/type' self.assertNdbEntityEqual(expected_file, actual_file._file, ignore=dates) self.assertNotEqual(old_modified, actual_file.modified) # Allow writing blank files. actual_file = files.File('/foo/bar.html').write('') self.assertEqual(actual_file.content, '') # Allow overwriting mime_type and meta without touching content. files.File('/foo/bar.html').write(content='Test') actual_file = files.File('/foo/bar.html').write(mime_type='fake/mimetype') self.assertEqual('fake/mimetype', actual_file.mime_type) self.assertEqual('Test', actual_file.content) actual_file = files.File('/foo/bar.html').write(meta=new_meta) self.assertEqual(True, actual_file.meta.flag) self.assertEqual('Test', actual_file.content) # Allow overwriting created and modified without touching content. files.File('/foo/bar.html').write(content='Test') now = datetime.datetime.now() + datetime.timedelta(days=1) actual_file = files.File('/foo/bar.html').write(created=now, modified=now) self.assertEqual(now, actual_file.created) self.assertEqual(now, actual_file.modified) # Verify the same behavior for the file creation codepath. files.File('/foo/bar.html').delete().write( 'Test', created=now, modified=now) self.assertEqual(now, actual_file.created) self.assertEqual(now, actual_file.modified) # Error handling. self.assertRaises(ValueError, files.File('/a').write, '', created='foo') self.assertRaises(ValueError, files.File('/a').write, '', modified='foo') # Allow overwriting created_by and modified_by without touching content. files.File('/foo/bar.html').write(content='Test') user = users.TitanUser('*****@*****.**') # Not the current logged in user. actual_file = files.File('/foo/bar.html').write( created_by=user, modified_by=user) self.assertEqual(user, actual_file.created_by) self.assertEqual(user, actual_file.modified_by) # Verify the same behavior for the file creation codepath. files.File('/foo/bar.html').delete().write( 'Test', created_by=user, modified_by=user) self.assertEqual(user, actual_file.created_by) self.assertEqual(user, actual_file.modified_by) # Error handling. self.assertRaises(ValueError, files.File('/a').write, '', created_by='foo') self.assertRaises(ValueError, files.File('/a').write, '', modified_by='foo') # Cleanup. expected_file = original_expected_file files.File('/foo/bar.html').delete() # write large content to blobstore. titan_file = files.File('/foo/bar.html').write(content=LARGE_FILE_CONTENT) blob_key = titan_file.blob.key() self.assertTrue(blob_key) self.assertEqual(LARGE_FILE_CONTENT, titan_file.content) self.assertIsNone(titan_file._file_ent.content) self.assertEqual(LARGE_FILE_CONTENT, files.File('/foo/bar.html').content) self.assertEqual(hashlib.md5(LARGE_FILE_CONTENT).hexdigest(), titan_file.md5_hash) # De-duping check: verify the blob key doesn't change if the content # doesn't change. old_blob_key = blob_key titan_file = files.File('/foo/bar.html').write(content=LARGE_FILE_CONTENT) blob_key = titan_file.blob.key() self.assertEqual(old_blob_key, blob_key) self.assertEqual(LARGE_FILE_CONTENT, titan_file.content) self.assertIsNone(titan_file._file_ent.content) self.assertEqual(LARGE_FILE_CONTENT, files.File('/foo/bar.html').content) self.stubs.SmartUnsetAll() # write with a blob key and encoding; verify proper decoding. encoded_foo = u'f♥♥'.encode('utf-8') blob_key = utils.write_to_blobstore(encoded_foo) titan_file = files.File('/foo/bar.html') # Verify that without encoding, the encoded bytestring is returned. titan_file.write(blob=blob_key) self.assertEqual(encoded_foo, titan_file.content) # Verify that with encoding, a unicode string is returned. titan_file.write(blob=blob_key, encoding='utf-8') self.assertEqual(u'f♥♥', titan_file.content) # Argument error handling for mixing encoding and unicode content: self.assertRaises(TypeError, titan_file.write, content=u'Test', encoding='utf-8') # Make sure the blob is deleted with the file: titan_file.delete() self.assertIsNone(blobstore.get(blob_key)) self.assertRaises(files.BadFileError, lambda: titan_file.blob) # Make sure the blob is deleted if the file gets smaller: titan_file = files.File('/foo/bar.html').write(content=LARGE_FILE_CONTENT) blob_key = titan_file.blob.key() titan_file.write(content='Test') self.assertIsNone(blobstore.get(blob_key)) # Test the current object and a new instance: self.assertEqual('Test', titan_file.content) self.assertEqual('Test', files.File('/foo/bar.html').content) self.assertIsNone(titan_file.blob) self.assertIsNone(files.File('/foo/bar.html').blob) # write with a BlobKey: titan_file = files.File('/foo/bar.html').write(blob=self.blob_key) blob_content = self.blob_reader.read() # Test the current object and a new instance: self.assertEqual(blob_content, files.File('/foo/bar.html').content) self.assertEqual(blob_content, titan_file.content) self.assertEqual(hashlib.md5('Blobstore!').hexdigest(), titan_file.md5_hash) # Cleanup. expected_file = original_expected_file files.File('/foo/bar.html').delete() # Error handling: # Updating mime_type or meta when entity doesn't exist. titan_file = files.File('/fake/file') self.assertRaises(files.BadFileError, titan_file.write, meta=meta) self.assertRaises(files.BadFileError, titan_file.write, mime_type='fake/mimetype') # Bad arguments: self.assertRaises(TypeError, titan_file.write) self.assertRaises(TypeError, titan_file.write, content=None, blob=None) self.assertRaises(TypeError, titan_file.write, content='Test', blob=self.blob_key) self.assertRaises(TypeError, titan_file.write, encoding='utf-8') # There are some reserved words that cannot be used in meta properties. invalid_meta_keys = [ # Titan reserved: 'name', 'path', 'dir_path', 'paths', 'depth', 'mime_type', 'encoding', 'created', 'modified', 'content', 'blob', 'blobs', 'created_by', 'modified_by', 'md5_hash', # NDB reserved: 'key', 'app', 'id', 'parent', 'namespace', 'projection', ] for key in invalid_meta_keys: try: titan_file.write(content='', meta={key: ''}) except files.InvalidMetaError: pass else: self.fail( 'Invalid meta key should have failed: {!r}'.format(key))