def test_fetch(self):
service = impl.CASService(
'/bucket/real', '/bucket/temp',
auth.ServiceAccountKey('*****@*****.**', 'PEM private key', 'id'))
# Actual _rsa_sign implementation depends on PyCrypto, that for some reason
# is not importable in unit tests. _rsa_sign is small enough to be "tested"
# manually on the dev server.
calls = []
def fake_sign(pkey, data):
calls.append((pkey, data))
return '+signature+'
self.mock(service, '_rsa_sign', fake_sign)
self.mock_now(utils.timestamp_to_datetime(1416444987 * 1000000.))
# Signature and email should be urlencoded.
url = service.generate_fetch_url('SHA1', 'a' * 40)
self.assertEqual(
'https://storage.googleapis.com/bucket/real/SHA1/'
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?'
'GoogleAccessId=account%40email.com&'
'Expires=1416448587&'
'Signature=%2Bsignature%2B', url)
# Since _rsa_sign is mocked out, at least verify it is called as expected.
self.assertEqual([(
'PEM private key',
'GET\n\n\n1416448587\n/bucket/real/SHA1/' + 'a'*40
)], calls)
def test_is_object_present(self):
service = impl.CASService('/bucket/real', '/bucket/temp')
self.mock_cloudstorage_stat(['/bucket/real/SHA1/' + 'a' * 40])
self.assertTrue(service.is_object_present('SHA1', 'a' * 40))
self.assertFalse(service.is_object_present('SHA1', 'b' * 40))
with self.assertRaises(AssertionError):
service.is_object_present('SHA1', 'wrong')
def test_open_not_found(self):
service = impl.CASService('/bucket/real', '/bucket/temp')
def mocked_cloudstorage_open(**kwargs):
raise impl.cloudstorage.NotFoundError()
self.mock(impl.cloudstorage, 'open', mocked_cloudstorage_open)
with self.assertRaises(impl.NotFoundError):
service.open('SHA1', 'a'*40, 1234)
def test_verify_pending_upload_good_hash(self):
fake_file = StringIO.StringIO('test buffer')
fake_file._etag = 'fake_etag'
obj = common.make_fake_session(
status=impl.UploadSession.STATUS_VERIFYING,
hash_algo='SHA1',
hash_digest='9682248358c830bcb5f8cb867186022acfe6eeb3',
final_gs_location=(
'/bucket/real/SHA1/9682248358c830bcb5f8cb867186022acfe6eeb3'),
temp_gs_location='/bucket/temp/temp_crap')
obj.put()
def mocked_open(filename, mode, read_buffer_size, retry_params):
self.assertEqual(filename, '/bucket/temp/temp_crap')
self.assertEqual(mode, 'r')
self.assertEqual(read_buffer_size, impl.READ_BUFFER_SIZE)
return fake_file
self.mock(impl.cloudstorage, 'open', mocked_open)
service = impl.CASService('/bucket/real', '/bucket/temp')
def mocked_copy(src, dst, src_etag):
self.assertEqual(src, '/bucket/temp/temp_crap')
self.assertEqual(
dst, '/bucket/real/SHA1/9682248358c830bcb5f8cb867186022acfe6eeb3')
self.assertEqual(src_etag, 'fake_etag')
self.mock(service, '_gs_copy', mocked_copy)
self.assertTrue(service.verify_pending_upload(obj.key.id()))
# Moved to PUBLISHED.
obj = obj.key.get()
self.assertEqual(obj.status, impl.UploadSession.STATUS_PUBLISHED)
def test_verify_pending_upload_bad_hash(self):
fake_file = StringIO.StringIO('test buffer')
fake_file._etag = 'fake_etag'
obj = common.make_fake_session(
status=impl.UploadSession.STATUS_VERIFYING,
hash_algo='SHA1',
hash_digest='a' * 40,
final_gs_location='/bucket/real/SHA1/' + 'a' * 40,
temp_gs_location='/bucket/temp/temp_crap')
obj.put()
def mocked_open(filename, mode, read_buffer_size, retry_params):
self.assertEqual(filename, '/bucket/temp/temp_crap')
self.assertEqual(mode, 'r')
self.assertEqual(read_buffer_size, impl.READ_BUFFER_SIZE)
return fake_file
self.mock(impl.cloudstorage, 'open', mocked_open)
service = impl.CASService('/bucket/real', '/bucket/temp')
self.assertTrue(service.verify_pending_upload(obj.key.id()))
# Moved to ERROR.
obj = obj.key.get()
self.assertEqual(obj.status, impl.UploadSession.STATUS_ERROR)
self.assertEqual(
obj.error_message,
'Invalid SHA1 hash: expected aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, '
'got 9682248358c830bcb5f8cb867186022acfe6eeb3.')
def test_direct_upload(self):
service = impl.CASService('/bucket/real', '/bucket/temp')
calls = []
def mocked_cloudstorage_open(filename, **_kwargs):
calls.append(('open', filename))
return StringIO.StringIO()
self.mock(impl.cloudstorage, 'open', mocked_cloudstorage_open)
self.mock(service, '_gs_copy', lambda *a: calls.append(('copy',) + a))
self.mock(service, '_gs_delete', lambda *a: calls.append(('delete',) + a))
self.mock_now(datetime.datetime(2014, 1, 1))
self.mock(impl.random, 'choice', lambda x: x[0])
with service.start_direct_upload('SHA1') as f:
f.write('abc')
f.write('def')
self.assertEqual(f.hash_digest, '1f8ac10f23c5b5bc1167bda84b833e5c057a77d2')
self.assertEqual(f.length, 6)
self.assertEqual([
(
'open',
'/bucket/temp/1388534400_direct_aaaaaaaaaaaaaaaaaaaa',
),
(
'copy',
'/bucket/temp/1388534400_direct_aaaaaaaaaaaaaaaaaaaa',
'/bucket/real/SHA1/1f8ac10f23c5b5bc1167bda84b833e5c057a77d2',
),
(
'delete',
'/bucket/temp/1388534400_direct_aaaaaaaaaaaaaaaaaaaa'
),
], calls)
# Code coverage for second noop close.
f.close()
# Code coverage for commit=False code path.
del calls[:]
with self.assertRaises(ValueError):
with service.start_direct_upload('SHA1') as f:
f.write('abc')
raise ValueError()
self.assertEqual([
(
'open',
'/bucket/temp/1388534400_direct_aaaaaaaaaaaaaaaaaaaa',
),
(
'delete',
'/bucket/temp/1388534400_direct_aaaaaaaaaaaaaaaaaaaa'
),
], calls)
def test_maybe_finish_upload_non_uploading(self):
service = impl.CASService('/bucket/real', '/bucket/temp')
def die(**_kwargs): # pragma: no cover
self.fail('Should not be called')
self.mock(impl.utils, 'enqueue_task', die)
obj1 = common.make_fake_session(status=impl.UploadSession.STATUS_ERROR)
obj1.put()
# Left in the same state.
obj2 = service.maybe_finish_upload(obj1)
self.assertEqual(obj2.status, impl.UploadSession.STATUS_ERROR)
def test_open_ok(self):
service = impl.CASService('/bucket/real', '/bucket/temp')
calls = []
def mocked_cloudstorage_open(**kwargs):
calls.append(kwargs)
return object()
self.mock(impl.cloudstorage, 'open', mocked_cloudstorage_open)
service.open('SHA1', 'a'*40, 1234)
self.assertEqual(calls, [{
'filename': '/bucket/real/SHA1/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'mode': 'r',
'read_buffer_size': 1234,
'retry_params': service._retry_params,
}])
def test_verify_pending_upload_when_file_exists(self):
obj = common.make_fake_session(
status=impl.UploadSession.STATUS_VERIFYING,
final_gs_location='/bucket/real/SHA1/' + 'a' * 40,
temp_gs_location='/bucket/temp/temp_crap')
obj.put()
self.mock_cloudstorage_stat([obj.final_gs_location])
deleted_files = self.mock_cloudstorage_delete()
service = impl.CASService('/bucket/real', '/bucket/temp')
self.assertTrue(service.verify_pending_upload(obj.key.id()))
# Moved to PUBLISHED.
obj = obj.key.get()
self.assertEqual(obj.status, impl.UploadSession.STATUS_PUBLISHED)
# Temp clean up called.
self.assertEqual(deleted_files, set(['/bucket/temp/temp_crap']))
def test_create_upload_session_and_fetch_upload_session(self):
service = impl.CASService('/bucket/real', '/bucket/temp')
mocked_time = utils.timestamp_to_datetime(1416444987 * 1000000.)
self.mock_now(mocked_time)
def mocked_open(filename, mode, retry_params):
self.assertEqual(filename, '/bucket/temp/1416444987_1')
self.assertEqual(mode, 'w')
self.assertEqual(retry_params, service._retry_params)
# Mock guts of ReadingBuffer :(
return common.Mock(
_path_with_token='/bucket/temp/1416444987_1?upload_id=abc',
_api=common.Mock(api_url='https://fake.com'))
self.mock(impl.cloudstorage, 'open', mocked_open)
obj, signed_id = service.create_upload_session(
'SHA1', 'a' * 40, auth_testing.DEFAULT_MOCKED_IDENTITY)
self.assertEqual(obj.key.id(), 1)
self.assertEqual(obj.to_dict(), {
'created_by': auth_testing.DEFAULT_MOCKED_IDENTITY,
'created_ts': mocked_time,
'error_message': None,
'final_gs_location': '/bucket/real/SHA1/' + 'a' * 40,
'hash_algo': 'SHA1',
'hash_digest': 'a' * 40,
'status': impl.UploadSession.STATUS_UPLOADING,
'temp_gs_location': '/bucket/temp/1416444987_1',
'upload_url': 'https://fake.com/bucket/temp/1416444987_1?upload_id=abc',
})
# Token should be readable.
embedded = impl.UploadIdSignature.validate(
signed_id, [auth_testing.DEFAULT_MOCKED_IDENTITY.to_bytes()])
self.assertEqual(embedded, {'id': '1'})
# Verify fetch_upload_session can use it too.
fetched = service.fetch_upload_session(
signed_id, auth_testing.DEFAULT_MOCKED_IDENTITY)
self.assertIsNotNone(fetched)
self.assertEqual(fetched.to_dict(), obj.to_dict())
def test_verify_pending_upload_unfinalized(self):
obj = common.make_fake_session(
status=impl.UploadSession.STATUS_VERIFYING,
final_gs_location='/bucket/real/SHA1/' + 'a' * 40,
temp_gs_location='/bucket/temp/temp_crap')
obj.put()
def mocked_open(filename, mode, read_buffer_size, retry_params):
self.assertEqual(filename, '/bucket/temp/temp_crap')
self.assertEqual(mode, 'r')
self.assertEqual(read_buffer_size, impl.READ_BUFFER_SIZE)
raise cloudstorage.NotFoundError()
self.mock(impl.cloudstorage, 'open', mocked_open)
service = impl.CASService('/bucket/real', '/bucket/temp')
self.assertTrue(service.verify_pending_upload(obj.key.id()))
# Moved to ERROR.
obj = obj.key.get()
self.assertEqual(obj.status, impl.UploadSession.STATUS_ERROR)
self.assertEqual(
obj.error_message, 'Google Storage upload wasn\'t finalized.')
def test_maybe_finish_upload(self):
service = impl.CASService('/bucket/real', '/bucket/temp')
calls = []
def mocked_enqueue_task(**kwargs):
calls.append(kwargs)
return True
self.mock(impl.utils, 'enqueue_task', mocked_enqueue_task)
obj1 = common.make_fake_session(status=impl.UploadSession.STATUS_UPLOADING)
obj1.put()
# Changed state.
obj2 = service.maybe_finish_upload(obj1)
self.assertEqual(obj2.status, impl.UploadSession.STATUS_VERIFYING)
# Task enqueued.
self.assertEqual(calls, [{
'queue_name': 'cas-verify',
'transactional': True,
'url': '/internal/taskqueue/cas-verify/666',
}])
def test_verify_pending_upload_bad_state(self):
obj = common.make_fake_session(status=impl.UploadSession.STATUS_ERROR)
obj.put()
service = impl.CASService('/bucket/real', '/bucket/temp')
self.assertTrue(service.verify_pending_upload(obj.key.id()))
def test_verify_pending_upload_bad_session(self):
service = impl.CASService('/bucket/real', '/bucket/temp')
self.assertTrue(service.verify_pending_upload(1234))
def test_fetch_upload_session_bad_token(self):
service = impl.CASService('/bucket/real', '/bucket/temp')
obj = service.fetch_upload_session(
'blah', auth_testing.DEFAULT_MOCKED_IDENTITY)
self.assertIsNone(obj)
