Example #1
0
 def test_meet_supported_options(self):
     exporter_options = ExporterConfig(
         valid_config_with_updates({
             'reader': {'name': 'exporters.readers.s3_reader.S3Reader',
                        'options': {'prefix': 'mock', 'bucket': 'mock'}},
             'writer': {'name': 'exporters.writers.s3_writer.S3Writer',
                        'options': {'bucket': 'mock', 'filebase': 'mock'}},
             'exporter_options': {'formatter': JSON_FORMATTER}
         })
     )
     S3Bypass.meets_conditions(exporter_options)
Example #2
0
    def test_copy_bypass_s3_with_env(self):
        # given
        self.s3_conn.create_bucket('dest_bucket')
        reader = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'source_bucket',
                'prefix': 'some_prefix/'
            }
        }
        options = create_s3_bypass_simple_config(reader=reader)
        env = {
            'EXPORTERS_S3READER_AWS_KEY': 'a',
            'EXPORTERS_S3READER_AWS_SECRET': 'b'
        }

        # when:
        with closing(S3Bypass(options, meta())) as bypass:
            with environment(env):
                bypass.execute()

        # then:
        bucket = self.s3_conn.get_bucket('dest_bucket')
        key = next(iter(bucket.list('some_prefix/')))
        self.assertEquals('some_prefix/test_key', key.name)
        self.assertEqual(self.data, json.loads(key.get_contents_as_string()))
        self.assertEqual(bypass.total_items, len(self.data),
                         'Bypass got an incorrect number of total items')
Example #3
0
    def test_write_pointer(self):
        # given:
        writer = {
            'name': 'exporters.writers.s3_writer.S3Writer',
            'options': {
                'bucket': 'pointer_fake_bucket',
                'aws_access_key_id': 'a',
                'aws_secret_access_key': 'a',
                'filebase': 'tests/',
                'save_pointer': 'pointer/LAST'
            }
        }

        options = create_s3_bypass_simple_config(writer=writer)
        self.s3_conn.create_bucket('pointer_fake_bucket')

        # when:
        with closing(S3Bypass(options, meta())) as bypass:
            bypass.execute()

        # then:
        bucket = self.s3_conn.get_bucket('pointer_fake_bucket')
        saved_keys = [k for k in bucket.list('pointer/')]
        self.assertEquals(1, len(saved_keys))
        key = saved_keys[0]
        self.assertEqual('tests/', key.get_contents_as_string())
    def test_items_limit_should_not_meet_conditions(self):
        # given:
        config = create_s3_bypass_simple_config()
        config.writer_options['options']['items_limit'] = 10

        # when:

        # then:
        self.assertFalse(S3Bypass.meets_conditions(config))
Example #5
0
    def test_items_limit_should_not_meet_conditions(self):
        # given:
        config = create_s3_bypass_simple_config()
        config.writer_options['options']['items_limit'] = 10

        # when:

        # then:
        self.assertFalse(S3Bypass.meets_conditions(config))
Example #6
0
    def test_custom_grouper_should_not_meet_conditions(self):
        # given:
        config = create_s3_bypass_simple_config(grouper={
            'name': 'whatever.Grouper',
        })

        # when:

        # then:
        self.assertFalse(S3Bypass.meets_conditions(config))
    def test_custom_grouper_should_not_meet_conditions(self):
        # given:
        config = create_s3_bypass_simple_config(grouper={
            'name': 'whatever.Grouper',
        })

        # when:

        # then:
        self.assertFalse(S3Bypass.meets_conditions(config))
    def test_custom_filter_should_not_meet_conditions(self):
        # given:
        config = create_s3_bypass_simple_config(filter={
            'name': 'exporters.filters.PythonexpFilter',
            'options': {'python_expression': 'None'}
        })

        # when:

        # then:
        self.assertFalse(S3Bypass.meets_conditions(config))
Example #9
0
 def test_meet_supported_options(self):
     exporter_options = ExporterConfig(
         valid_config_with_updates({
             'reader': {
                 'name': 'exporters.readers.s3_reader.S3Reader',
                 'options': {
                     'prefix': 'mock',
                     'bucket': 'mock'
                 }
             },
             'writer': {
                 'name': 'exporters.writers.s3_writer.S3Writer',
                 'options': {
                     'bucket': 'mock',
                     'filebase': 'mock'
                 }
             },
             'exporter_options': {
                 'formatter': JSON_FORMATTER
             }
         }))
     S3Bypass.meets_conditions(exporter_options)
Example #10
0
    def test_custom_filter_should_not_meet_conditions(self):
        # given:
        config = create_s3_bypass_simple_config(
            filter={
                'name': 'exporters.filters.PythonexpFilter',
                'options': {
                    'python_expression': 'None'
                }
            })

        # when:

        # then:
        self.assertFalse(S3Bypass.meets_conditions(config))
Example #11
0
    def test_filebase_naming(self):
        self.s3_conn.create_bucket('dest_bucket')
        options = create_s3_bypass_simple_config()
        options.writer_options['options'][
            'filebase'] = 'other_prefix/%Y-%m-%d/'

        with closing(S3Bypass(options, meta())) as bypass:
            bypass.execute()

        # then:
        bucket = self.s3_conn.get_bucket('dest_bucket')
        key = next(iter(bucket.list('other_prefix/2010-01-01/')))
        file_name = key.name.split('/')[-1]
        self.assertEquals('test_key', file_name)
Example #12
0
    def test_copy_mode_bypass_when_cant_get_user_id(self, get_user_id_mock):
        get_user_id_mock.side_effect = S3ResponseError(
            'Fake 403 Forbidden Error', meta())
        # given
        bucket = self.s3_conn.create_bucket('dest_bucket')
        options = create_s3_bypass_simple_config()

        # when:
        with closing(S3Bypass(options, meta())) as bypass:
            bypass.execute()

        # then:
        key = next(iter(bucket.list('some_prefix/')))
        self.assertEqual(self.data, json.loads(key.get_contents_as_string()))
Example #13
0
    def test_copy_bypass_s3(self):
        # given
        self.s3_conn.create_bucket('dest_bucket')
        options = create_s3_bypass_simple_config()

        # when:
        with closing(S3Bypass(options, meta())) as bypass:
            bypass.execute()

        # then:
        bucket = self.s3_conn.get_bucket('dest_bucket')
        key = next(iter(bucket.list('some_prefix/')))
        self.assertEquals('some_prefix/test_key', key.name)
        self.assertEqual(self.data, json.loads(key.get_contents_as_string()))
        self.assertEqual(bypass.total_items, 2,
                         'Bypass got an incorrect number of total items')
Example #14
0
    def test_get_md5(self):
        # given
        self.s3_conn.create_bucket('dest_bucket')
        options = create_s3_bypass_simple_config()
        options.writer_options['options']['save_metadata'] = True

        # when:
        bypass = S3Bypass(options, meta())
        bucket = self.s3_conn.get_bucket('source_bucket')
        key = bucket.get_key('some_prefix/test_key')

        with TmpFile() as tmp_filename:
            key.get_contents_to_filename(tmp_filename)
            metadata_md5 = bypass._get_md5(key, tmp_filename)

        # then:
        self.assertEqual(metadata_md5, self.key_md5)
Example #15
0
    def test_prefix_pointer_list_keys(self):
        # given
        reader = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'source_pointer_bucket',
                'aws_access_key_id': 'a',
                'aws_secret_access_key': 'a',
                'prefix_pointer': 'test_pointer/LAST'
            }
        }

        writer = {
            'name': 'exporters.writers.s3_writer.S3Writer',
            'options': {
                'bucket': 'dest_pointer_bucket',
                'aws_access_key_id': 'b',
                'aws_secret_access_key': 'b',
                'filebase': 'some_prefix/'
            }
        }

        self._create_and_populate_bucket('source_pointer_bucket')
        bucket = self.s3_conn.get_bucket('source_pointer_bucket')

        expected_prefix_pointer = 'some_prefix/'
        with closing(bucket.new_key('test_pointer/LAST')) as key:
            key.set_contents_from_string(expected_prefix_pointer)

        self.s3_conn.create_bucket('dest_pointer_bucket')
        options = create_s3_bypass_simple_config(reader=reader, writer=writer)
        expected_keys = [
            'some_prefix/key1', 'some_prefix/key2', 'some_prefix/key3'
        ]

        # when
        with closing(S3Bypass(options, meta())) as bypass:
            bypass.execute()

        # then
        dest_bucket = self.s3_conn.get_bucket('dest_pointer_bucket')
        keys = dest_bucket.list(prefix='some_prefix/')
        keys_list = []
        for key in keys:
            keys_list.append(key.name)
        self.assertEqual(expected_keys, keys_list)
Example #16
0
    def test_load_from_env(self):
        # given
        reader = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'source_bucket',
                'prefix': 'some_prefix/'
            }
        }
        options = create_s3_bypass_simple_config(reader=reader)
        # when:
        bypass = S3Bypass(options, meta())

        # then
        expected = '123'
        with environment({'EXPORTERS_S3READER_AWS_KEY': expected}):
            self.assertEqual(bypass.read_option('reader', 'aws_access_key_id'),
                             expected)
        self.assertIsNone(bypass.read_option('reader', 'aws_access_key_id'))
Example #17
0
    def test_filebase_format_bypass(self):
        # given
        writer = {
            'name': 'exporters.writers.s3_writer.S3Writer',
            'options': {
                'bucket': 'a',
                'aws_access_key_id': 'a',
                'aws_secret_access_key': 'a',
                'filebase': 'some_path/%Y-%m-%d/'
            }
        }

        expected = 'some_path/%Y-%m-%d/'.format(datetime.datetime.now())
        expected = datetime.datetime.now().strftime(expected)
        options = create_s3_bypass_simple_config(writer=writer)

        # when:
        with closing(S3Bypass(options, meta())) as bypass:
            # then:
            filebase = bypass._get_filebase(options.writer_options['options'])
            self.assertEqual(expected, filebase)
Example #18
0
    def test_resume_bypass(self):
        # given
        options = create_s3_bypass_simple_config()
        options.reader_options['options']['bucket'] = 'resume_bucket'
        options.writer_options['options']['bucket'] = 'resume_dest_bucket'
        options.persistence_options.update(
            resume=True,
            persistence_state_id='tmp_s3_bypass_resume_persistence.pickle')
        options.persistence_options['options']['file_path'] = 'tests/data/'
        # Initial state is:
        # copied = ['some_prefix/key1']
        # pending = ['some_prefix/key2', 'some_prefix/key3']
        self._create_and_populate_bucket('resume_bucket')

        self.s3_conn.create_bucket('resume_dest_bucket')
        dest_bucket = self.s3_conn.get_bucket('resume_bucket')
        key = dest_bucket.new_key('some_prefix/key1')
        key.set_contents_from_string('not overwritten')

        expected_final_keys = [
            'some_prefix/key1', 'some_prefix/key2', 'some_prefix/key3'
        ]

        # when:
        with closing(S3Bypass(options, meta())) as bypass:
            bypass.execute()

        # then:
        dest_bucket = self.s3_conn.get_bucket('resume_bucket')
        key1 = dest_bucket.get_key('some_prefix/key1')
        self.assertEqual(key1.get_contents_as_string(), 'not overwritten')
        bucket_keynames = [
            k.name for k in list(dest_bucket.list('some_prefix/'))
        ]
        self.assertEquals(expected_final_keys, bucket_keynames)
        self.assertEquals(bypass.total_items, 6,
                          'Wrong number of items written')
Example #19
0
 def test_should_meet_conditions(self):
     config = create_s3_bypass_simple_config()
     # shouldn't raise any exception
     S3Bypass.meets_conditions(config)
Example #20
0
 def test_should_meet_conditions(self):
     config = create_s3_bypass_simple_config()
     # shouldn't raise any exception
     S3Bypass.meets_conditions(config)