Exemplo n.º 1
0
    def test_check_writer_consistency(self):
        # given
        options = self.get_writer_config()
        options['options']['check_consistency'] = True

        # when:
        writer = FSWriter(options, meta())
        try:
            writer.write_batch(self.get_batch())
            writer.flush()

        finally:
            writer.close()

        # Consistency check passes
        writer.finish_writing()

        with open(os.path.join(self.tmp_dir, 'exporter_test0000.jl.gz'), 'w'):
            with self.assertRaisesRegexp(InconsistentWriteState,
                                         'Wrong size for file'):
                writer.finish_writing()

        os.remove(os.path.join(self.tmp_dir, 'exporter_test0000.jl.gz'))
        with self.assertRaisesRegexp(InconsistentWriteState,
                                     'file is not present at destination'):
            writer.finish_writing()
Exemplo n.º 2
0
    def test_get_file_number(self):
        writer_config = self.get_writer_config()
        writer = FSWriter(writer_config, meta())
        try:
            writer.write_batch(self.get_batch())
            writer.flush()

        finally:
            writer.close()
        expected_file = '{}/exporter_test0000.jl.gz'.format(self.tmp_dir)
        self.assertTrue(expected_file in writer.written_files)
Exemplo n.º 3
0
    def test_get_file_number(self):
        writer_config = self.get_writer_config()
        writer = FSWriter(writer_config, meta())
        try:
            writer.write_batch(self.get_batch())
            writer.flush()

        finally:
            writer.close()
        expected_file = '{}/exporter_test0000.jl.gz'.format(self.tmp_dir)
        self.assertTrue(expected_file in writer.written_files)
Exemplo n.º 4
0
    def test_no_compression(self):
        writer_config = self.get_writer_config()
        writer_config['options'].update({'compression': 'none'})
        writer = FSWriter(writer_config, meta())
        try:
            writer.write_batch(self.get_batch())
            writer.flush()

        finally:
            writer.close()
        expected_file = '{}/exporter_test0000.jl'.format(self.tmp_dir)
        self.assertTrue(expected_file in writer.written_files)

        written = []
        with open(expected_file, 'r') as fin:
            for line in fin:
                written.append(json.loads(line))
        self.assertEqual(written, self.get_batch())
Exemplo n.º 5
0
    def test_no_compression(self):
        writer_config = self.get_writer_config()
        writer_config['options'].update({'compression': 'none'})
        writer = FSWriter(writer_config, meta())
        try:
            writer.write_batch(self.get_batch())
            writer.flush()

        finally:
            writer.close()
        expected_file = '{}/exporter_test0000.jl'.format(self.tmp_dir)
        self.assertTrue(expected_file in writer.written_files)

        written = []
        with open(expected_file, 'r') as fin:
            for line in fin:
                written.append(json.loads(line))
        self.assertEqual(written, self.get_batch())
Exemplo n.º 6
0
    def test_get_file_number_with_date(self):
        file_path = '/tmp/%Y%m%d/'
        file_name = '{file_number}_exporter_test_%m%d%y'
        start_file_count = 1
        writer_config = self.get_writer_config()
        writer_config.update({'options': {
            'filebase': file_path + file_name,
            'start_file_count': start_file_count
        }})
        writer = FSWriter(writer_config, meta())
        try:
            writer.write_batch(self.get_batch())
            writer.flush()

        finally:
            writer.close()
        file_path = datetime.datetime.now().strftime(file_path).format(file_number=start_file_count)
        file_name = datetime.datetime.now().strftime(file_name).format(file_number=start_file_count)
        self.assertIn(file_path + file_name + '.jl.gz', writer.written_files)
Exemplo n.º 7
0
    def test_compression_zip_format(self):
        writer_config = self.get_writer_config()
        writer_config['options'].update({'compression': 'zip'})
        writer = FSWriter(writer_config, meta())
        try:
            writer.write_batch(self.get_batch())
            writer.flush()

        finally:
            writer.close()
        expected_file = '{}/exporter_test0000.jl.zip'.format(self.tmp_dir)
        self.assertTrue(expected_file in writer.written_files)

        import zipfile
        written = []
        with zipfile.ZipFile(expected_file) as z:
            with z.open('exporter_test0000.jl') as f:
                for line in f:
                    written.append(json.loads(line))
        self.assertEqual(written, self.get_batch())
Exemplo n.º 8
0
    def test_compression_zip_format(self):
        writer_config = self.get_writer_config()
        writer_config['options'].update({'compression': 'zip'})
        writer = FSWriter(writer_config, meta())
        try:
            writer.write_batch(self.get_batch())
            writer.flush()

        finally:
            writer.close()
        expected_file = '{}/exporter_test0000.jl.zip'.format(self.tmp_dir)
        self.assertTrue(expected_file in writer.written_files)

        import zipfile
        written = []
        with zipfile.ZipFile(expected_file) as z:
            with z.open('exporter_test0000.jl') as f:
                for line in f:
                    written.append(json.loads(line))
        self.assertEqual(written, self.get_batch())
Exemplo n.º 9
0
    def test_get_file_number_with_date(self):
        file_path = '/tmp/%Y%m%d/'
        file_name = '{file_number}_exporter_test_%m%d%y'
        start_file_count = 1
        writer_config = self.get_writer_config()
        writer_config.update({
            'options': {
                'filebase': file_path + file_name,
                'start_file_count': start_file_count
            }
        })
        writer = FSWriter(writer_config, meta())
        try:
            writer.write_batch(self.get_batch())
            writer.flush()

        finally:
            writer.close()
        file_path = datetime.datetime.now().strftime(file_path).format(
            file_number=start_file_count)
        file_name = datetime.datetime.now().strftime(file_name).format(
            file_number=start_file_count)
        self.assertIn(file_path + file_name + '.jl.gz', writer.written_files)
Exemplo n.º 10
0
    def test_check_writer_consistency(self):
        # given
        options = self.get_writer_config()
        options['options']['check_consistency'] = True

        # when:
        writer = FSWriter(options, meta())
        try:
            writer.write_batch(self.get_batch())
            writer.flush()

        finally:
            writer.close()

        # Consistency check passes
        writer.finish_writing()

        with open(os.path.join(self.tmp_dir, 'exporter_test0000.jl.gz'), 'w'):
            with self.assertRaisesRegexp(InconsistentWriteState, 'Wrong size for file'):
                writer.finish_writing()

        os.remove(os.path.join(self.tmp_dir, 'exporter_test0000.jl.gz'))
        with self.assertRaisesRegexp(InconsistentWriteState, 'file is not present at destination'):
            writer.finish_writing()