class SqlManifestShould(unittest.TestCase): def setUp(self): self.schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)')) self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, Mock()) self.bucket.save = Mock() self.database = create_autospec(Database) self.key_names = [ 'object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a', 'object_path/19440481-7766-4061-bd42-4a54fa0aac7c', 'object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c', 'object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd', 'object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268', 'object_path/80536e83-6bbe-4a42-ade1-533d99321a6c', 'object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40', 'object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1'] self.bucket.list = Mock( return_value=[self.mock_key(key) for key in self.key_names]) self.manifest = SqlManifest(metadata='', source='', schema=self.schema, bucket=self.bucket, db_connection=self.database) self.expected_manifest = {'entries': [ { 'url': 's3://{0}/object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a'.format( BUCKET_NAME), 'mandatory': True}, { 'url': 's3://{0}/object_path/19440481-7766-4061-bd42-4a54fa0aac7c'.format( BUCKET_NAME), 'mandatory': True}, { 'url': 's3://{0}/object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c'.format( BUCKET_NAME), 'mandatory': True}, { 'url': 's3://{0}/object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd'.format( BUCKET_NAME), 'mandatory': True}, { 'url': 's3://{0}/object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268'.format( BUCKET_NAME), 'mandatory': True}, { 'url': 's3://{0}/object_path/80536e83-6bbe-4a42-ade1-533d99321a6c'.format( BUCKET_NAME), 'mandatory': True}, { 'url': 's3://{0}/object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40'.format( BUCKET_NAME), 'mandatory': True}, { 'url': 's3://{0}/object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1'.format( BUCKET_NAME), 'mandatory': True} ]} def mock_key(self, name): return Key(Mock(), name) def mock_key_exists(self, key_name, exists): key = self.mock_key(key_name) key.exists = Mock(return_value=exists) key.get_contents_to_filename = Mock() self.bucket.get = Mock(return_value=key) return key def mock_journal(self, exists, key_names=None): self.mock_key_exists(self.manifest.journal_key, exists) if exists: self.database.fetchall = Mock( return_value=((key,) for key in key_names)) def test_have_all_keys(self): self.assertEqual(self.key_names, list(self.manifest.all_keys)) def test_have_manifest_key(self): self.assertEqual('/event_created_manifest.json', self.manifest.manifest_key) def test_have_journal_key(self): self.assertEqual('/event_created_journal.db', self.manifest.journal_key) def test_have_manifest_url(self): self.assertEqual( 's3://{0}/event_created_manifest.json'.format(BUCKET_NAME), self.manifest.manifest_url) def test_have_empty_journal(self): self.mock_journal(False) self.assertEqual([], list(self.manifest.journal())) self.database.open.assert_called_once_with() sql = 'CREATE TABLE IF NOT EXISTS journal (key TEXT)' self.database.execute.assert_called_once_with(sql) self.database.commit.assert_called_once_with() self.database.close.assert_called_once_with() def test_have_journal(self): self.mock_journal(True, self.key_names) self.assertEqual(self.key_names, list(self.manifest.journal())) self.database.open.assert_called_once_with() sql = 'SELECT key FROM journal' self.database.execute.assert_called_once_with(sql) def test_have_manifest_when_journal_is_empty(self): self.mock_journal(False) expected_entries = set( [entry['url'] for entry in self.expected_manifest['entries']]) actual_entries = set( [entry['url'] for entry in self.manifest.get()['manifest']['entries']]) self.assertEqual(expected_entries, actual_entries) def test_update_existing_manifest_when_key_not_in_journal(self): self.mock_journal(True, list(self.key_names)) self.key_names.append( 'object_path/5acd5fb0-be96-451a-be32-b65c4461b3f4') self.bucket.list = Mock( return_value=[self.mock_key(key) for key in self.key_names]) self.assertEqual([{ 'url': 's3://{0}/object_path/5acd5fb0-be96-451a-be32-b65c4461b3f4'.format( BUCKET_NAME), 'mandatory': True}], list(self.manifest.get()['manifest']['entries'])) def test_save(self): f = mock_open() self.mock_journal(False) key = self.mock_key(self.manifest.manifest_key) key.set_contents_from_filename = Mock() key.get_contents_to_filename = Mock() self.bucket.get = Mock(return_value=key) with patch('__builtin__.open', f, create=True): self.manifest.save() f.assert_called_once_with(self.manifest.file_name, 'wb') handle = f() self.assertEqual(call(0), handle.seek.call_args_list[0]) self.assertEqual(2, handle.truncate.call_count) self.assertEqual([call('{\n'), call('"entries": [\n'), call( '{"url": "s3://bucket/object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268", "mandatory": true},\n'), call( '{"url": "s3://bucket/object_path/80536e83-6bbe-4a42-ade1-533d99321a6c", "mandatory": true},\n'), call( '{"url": "s3://bucket/object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1", "mandatory": true},\n'), call( '{"url": "s3://bucket/object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40", "mandatory": true},\n'), call( '{"url": "s3://bucket/object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c", "mandatory": true},\n'), call( '{"url": "s3://bucket/object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd", "mandatory": true},\n'), call( '{"url": "s3://bucket/object_path/19440481-7766-4061-bd42-4a54fa0aac7c", "mandatory": true},\n'), call( '{"url": "s3://bucket/object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a", "mandatory": true},\n'), call( '{"url": "s3://bucket/object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a", "mandatory": true}\n'), call(']}')], handle.write.call_args_list) key.set_contents_from_filename.assert_called_once_with( self.manifest.file_name) def test_commit(self): key = self.mock_key(self.manifest.journal_key) key.set_contents_from_filename = Mock() self.bucket.get = Mock(return_value=key) self.manifest.commit(self.key_names) self.database.open.assert_called_once_with() inserts = [call('DELETE FROM journal'), call('INSERT INTO journal VALUES (?)', ( 'object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a',)), call('INSERT INTO journal VALUES (?)', ( 'object_path/19440481-7766-4061-bd42-4a54fa0aac7c',)), call('INSERT INTO journal VALUES (?)', ( 'object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c',)), call('INSERT INTO journal VALUES (?)', ( 'object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd',)), call('INSERT INTO journal VALUES (?)', ( 'object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268',)), call('INSERT INTO journal VALUES (?)', ( 'object_path/80536e83-6bbe-4a42-ade1-533d99321a6c',)), call('INSERT INTO journal VALUES (?)', ( 'object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40',)), call('INSERT INTO journal VALUES (?)', ( 'object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1',))] self.database.execute.assert_has_calls(inserts) self.database.commit.assert_called_once_with() self.database.close.assert_called_once_with() key.set_contents_from_filename.assert_called_once_with( self.manifest.journal_file_name) def test_exist(self): exists = True self.mock_key_exists(self.manifest.manifest_key, exists) self.assertEqual(exists, self.manifest.exists()) def test_not_exist(self): exists = False self.mock_key_exists(self.manifest.manifest_key, exists) self.assertEqual(exists, self.manifest.exists()) def test_have_journal_existence(self): exists = True self.mock_key_exists(self.manifest.journal_key, exists) self.assertEqual(exists, self.manifest.journal_exists()) def test_not_have_journal_existence(self): exists = False self.mock_key_exists(self.manifest.journal_key, exists) self.assertEqual(exists, self.manifest.journal_exists())
class SqlManifestShould(unittest.TestCase): def setUp(self): self.schema = JsonObject(TABLE_NAME, Property('id', 'VARCHAR(36)')) self.bucket = Bucket(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, Mock()) self.bucket.save = Mock() self.database = create_autospec(Database) self.key_names = [ 'object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a', 'object_path/19440481-7766-4061-bd42-4a54fa0aac7c', 'object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c', 'object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd', 'object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268', 'object_path/80536e83-6bbe-4a42-ade1-533d99321a6c', 'object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40', 'object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1' ] self.bucket.list = Mock( return_value=[self.mock_key(key) for key in self.key_names]) self.manifest = SqlManifest(metadata='', source='', schema=self.schema, bucket=self.bucket, db_connection=self.database) self.expected_manifest = { 'entries': [{ 'url': 's3://{0}/object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/19440481-7766-4061-bd42-4a54fa0aac7c'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c' .format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/80536e83-6bbe-4a42-ade1-533d99321a6c'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40'. format(BUCKET_NAME), 'mandatory': True }, { 'url': 's3://{0}/object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1'. format(BUCKET_NAME), 'mandatory': True }] } def mock_key(self, name): return Key(Mock(), name) def mock_key_exists(self, key_name, exists): key = self.mock_key(key_name) key.exists = Mock(return_value=exists) key.get_contents_to_filename = Mock() self.bucket.get = Mock(return_value=key) return key def mock_journal(self, exists, key_names=None): self.mock_key_exists(self.manifest.journal_key, exists) if exists: self.database.fetchall = Mock(return_value=((key, ) for key in key_names)) def test_have_all_keys(self): self.assertEqual(self.key_names, list(self.manifest.all_keys)) def test_have_manifest_key(self): self.assertEqual('/event_created_manifest.json', self.manifest.manifest_key) def test_have_journal_key(self): self.assertEqual('/event_created_journal.db', self.manifest.journal_key) def test_have_manifest_url(self): self.assertEqual( 's3://{0}/event_created_manifest.json'.format(BUCKET_NAME), self.manifest.manifest_url) def test_have_empty_journal(self): self.mock_journal(False) self.assertEqual([], list(self.manifest.journal())) self.database.open.assert_called_once_with() sql = 'CREATE TABLE IF NOT EXISTS journal (key TEXT)' self.database.execute.assert_called_once_with(sql) self.database.commit.assert_called_once_with() self.database.close.assert_called_once_with() def test_have_journal(self): self.mock_journal(True, self.key_names) self.assertEqual(self.key_names, list(self.manifest.journal())) self.database.open.assert_called_once_with() sql = 'SELECT key FROM journal' self.database.execute.assert_called_once_with(sql) def test_have_manifest_when_journal_is_empty(self): self.mock_journal(False) expected_entries = set( [entry['url'] for entry in self.expected_manifest['entries']]) actual_entries = set([ entry['url'] for entry in self.manifest.get()['manifest']['entries'] ]) self.assertEqual(expected_entries, actual_entries) def test_update_existing_manifest_when_key_not_in_journal(self): self.mock_journal(True, list(self.key_names)) self.key_names.append( 'object_path/5acd5fb0-be96-451a-be32-b65c4461b3f4') self.bucket.list = Mock( return_value=[self.mock_key(key) for key in self.key_names]) self.assertEqual([{ 'url': 's3://{0}/object_path/5acd5fb0-be96-451a-be32-b65c4461b3f4'.format( BUCKET_NAME), 'mandatory': True }], list(self.manifest.get()['manifest']['entries'])) def test_save(self): f = mock_open() self.mock_journal(False) key = self.mock_key(self.manifest.manifest_key) key.set_contents_from_filename = Mock() key.get_contents_to_filename = Mock() self.bucket.get = Mock(return_value=key) with patch('__builtin__.open', f, create=True): self.manifest.save() f.assert_called_once_with(self.manifest.file_name, 'wb') handle = f() self.assertEqual(call(0), handle.seek.call_args_list[0]) self.assertEqual(2, handle.truncate.call_count) self.assertEqual([ call('{\n'), call('"entries": [\n'), call( '{"url": "s3://bucket/object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268", "mandatory": true},\n' ), call( '{"url": "s3://bucket/object_path/80536e83-6bbe-4a42-ade1-533d99321a6c", "mandatory": true},\n' ), call( '{"url": "s3://bucket/object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1", "mandatory": true},\n' ), call( '{"url": "s3://bucket/object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40", "mandatory": true},\n' ), call( '{"url": "s3://bucket/object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c", "mandatory": true},\n' ), call( '{"url": "s3://bucket/object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd", "mandatory": true},\n' ), call( '{"url": "s3://bucket/object_path/19440481-7766-4061-bd42-4a54fa0aac7c", "mandatory": true},\n' ), call( '{"url": "s3://bucket/object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a", "mandatory": true},\n' ), call( '{"url": "s3://bucket/object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a", "mandatory": true}\n' ), call(']}') ], handle.write.call_args_list) key.set_contents_from_filename.assert_called_once_with( self.manifest.file_name) def test_commit(self): key = self.mock_key(self.manifest.journal_key) key.set_contents_from_filename = Mock() self.bucket.get = Mock(return_value=key) self.manifest.commit(self.key_names) self.database.open.assert_called_once_with() inserts = [ call('DELETE FROM journal'), call('INSERT INTO journal VALUES (?)', ('object_path/00c68a1e-85f2-49e5-9d07-6922046dbc5a', )), call('INSERT INTO journal VALUES (?)', ('object_path/19440481-7766-4061-bd42-4a54fa0aac7c', )), call( 'INSERT INTO journal VALUES (?)', ('object_path/2014-09-02/19440481-7766-4061-bd42-4a54fa0aac7c', )), call('INSERT INTO journal VALUES (?)', ('object_path/282e6063-ecef-4e45-bdfb-9fdfb39840cd', )), call('INSERT INTO journal VALUES (?)', ('object_path/35cbf09a-b2dc-43f2-96f6-7d7573906268', )), call('INSERT INTO journal VALUES (?)', ('object_path/80536e83-6bbe-4a42-ade1-533d99321a6c', )), call('INSERT INTO journal VALUES (?)', ('object_path/cf00b394-3ff3-4418-b244-2ccf104fcc40', )), call('INSERT INTO journal VALUES (?)', ('object_path/e822e2ae-61f5-4be0-aacd-ca6de70faad1', )) ] self.database.execute.assert_has_calls(inserts) self.database.commit.assert_called_once_with() self.database.close.assert_called_once_with() key.set_contents_from_filename.assert_called_once_with( self.manifest.journal_file_name) def test_exist(self): exists = True self.mock_key_exists(self.manifest.manifest_key, exists) self.assertEqual(exists, self.manifest.exists()) def test_not_exist(self): exists = False self.mock_key_exists(self.manifest.manifest_key, exists) self.assertEqual(exists, self.manifest.exists()) def test_have_journal_existence(self): exists = True self.mock_key_exists(self.manifest.journal_key, exists) self.assertEqual(exists, self.manifest.journal_exists()) def test_not_have_journal_existence(self): exists = False self.mock_key_exists(self.manifest.journal_key, exists) self.assertEqual(exists, self.manifest.journal_exists())