예제 #1
0
    def test_recovery_failed_nobackup(self):
        # Create the coverage
        cov, dset = self.get_cov()
        if cov._persistence_layer.master_manager.storage_type() != 'hdf':
            # TODO: Check for something Cassandra related
            self.assertTrue(True)
        else:
            cov_pth = cov.persistence_dir
            cov.close()

            # Analyze the valid coverage
            dr = CoverageDoctor(cov_pth, 'dprod', dset)

            dr_result = dr.analyze()

            # Corrupt the Master File
            fo = open(cov._persistence_layer.master_manager.file_path, "wb")
            fo.write('Junk')
            fo.close()
            # Corrupt the lon Parameter file
            fo = open(cov._persistence_layer.parameter_metadata['lon'].file_path, "wb")
            fo.write('Junk')
            fo.close()

            corrupt_res = dr.analyze(reanalyze=True)
            self.assertTrue(corrupt_res.is_corrupt)

            # Repair the metadata files
            with self.assertRaises(ValueError):
                temp_cov_dir = dr.repair(reanalyze=True, backup=False, copy_over=False, keep_temp=False)
                log.info('Temporary Coverage Directory: {0}', temp_cov_dir)
                self.assertFalse(os.path.exists(temp_cov_dir))

            nofix_res = dr.analyze(reanalyze=True)
            self.assertTrue(nofix_res.is_corrupt)
    def test_coverage_recovery(self):
        # Create the coverage
        dp_id, stream_id, route, stream_def_id, dataset_id = self.load_data_product()
        self.populate_dataset(dataset_id, 36)
        dset = self.dataset_management.read_dataset(dataset_id)
        dprod = self.dpsc_cli.read_data_product(dp_id)
        cov = DatasetManagementService._get_simplex_coverage(dataset_id)
        cov_pth = cov.persistence_dir
        cov.close()

        # Analyze the valid coverage
        dr = CoverageDoctor(cov_pth, dprod, dset)
        dr_result = dr.analyze()

        # Get original values (mock)
        orig_cov = AbstractCoverage.load(cov_pth)
        time_vals_orig = orig_cov.get_time_values()

        # TODO: Destroy the metadata files

        # TODO: RE-analyze coverage

        # TODO: Should be corrupt, take action to repair if so

        # Repair the metadata files
        dr.repair_metadata()

        # TODO: Re-analyze fixed coverage

        fixed_cov = AbstractCoverage.load(cov_pth)
        self.assertIsInstance(fixed_cov, AbstractCoverage)

        time_vals_fixed = fixed_cov.get_time_values()
        self.assertTrue(np.array_equiv(time_vals_orig, time_vals_fixed))
예제 #3
0
    def test_coverage_recovery(self):
        # Create the coverage
        cov, dset = self.get_cov()
        cov_pth = cov.persistence_dir
        cov.close()

        # Analyze the valid coverage
        dr = CoverageDoctor(cov_pth, 'dprod', dset)

        dr_result = dr.analyze()

        # TODO: Turn these into meaningful Asserts
        self.assertEqual(len(dr_result.get_brick_corruptions()), 0)
        self.assertEqual(len(dr_result.get_brick_size_ratios()), 6)
        self.assertEqual(len(dr_result.get_corruptions()), 0)
        self.assertEqual(len(dr_result.get_master_corruption()), 0)
        self.assertEqual(len(dr_result.get_param_corruptions()), 0)
        self.assertEqual(len(dr_result.get_param_size_ratios()), 3)
        self.assertEqual(len(dr_result.get_master_size_ratio()), 1)
        self.assertEqual(len(dr_result.get_size_ratios()), 10)
        self.assertEqual(dr_result.master_status[1], 'NORMAL')

        self.assertFalse(dr_result.is_corrupt)
        self.assertEqual(dr_result.param_file_count, 3)
        self.assertEqual(dr_result.brick_file_count, 6)
        self.assertEqual(dr_result.total_file_count, 10)

        # Get original values (mock)
        orig_cov = AbstractCoverage.load(cov_pth)
        time_vals_orig = orig_cov.get_time_values()
        orig_cov.close()

        # Corrupt the Master File
        fo = open(cov._persistence_layer.master_manager.file_path, "wb")
        fo.write('Junk')
        fo.close()
        # Corrupt the lon Parameter file
        fo = open(cov._persistence_layer.parameter_metadata['lon'].file_path,
                  "wb")
        fo.write('Junk')
        fo.close()

        corrupt_res = dr.analyze(reanalyze=True)
        self.assertTrue(corrupt_res.is_corrupt)

        # Repair the metadata files
        dr.repair(reanalyze=True)

        fixed_res = dr.analyze(reanalyze=True)
        self.assertFalse(fixed_res.is_corrupt)

        fixed_cov = AbstractCoverage.load(cov_pth)
        self.assertIsInstance(fixed_cov, AbstractCoverage)

        time_vals_fixed = fixed_cov.get_time_values()
        fixed_cov.close()
        self.assertTrue(np.array_equiv(time_vals_orig, time_vals_fixed))
예제 #4
0
    def test_coverage_recovery(self):
        # Create the coverage
        cov, dset = self.get_cov()
        if cov._persistence_layer.master_manager.storage_type() != 'hdf':
            # TODO: Check for something Cassandra related
            self.assertTrue(True)
        else:
            cov_pth = cov.persistence_dir
            cov.close()

            # Analyze the valid coverage
            dr = CoverageDoctor(cov_pth, 'dprod', dset)

            dr_result = dr.analyze()

            # TODO: Turn these into meaningful Asserts
            self.assertEqual(len(dr_result.get_brick_corruptions()), 0)
            self.assertEqual(len(dr_result.get_brick_size_ratios()), 6)
            self.assertEqual(len(dr_result.get_corruptions()), 0)
            self.assertEqual(len(dr_result.get_master_corruption()), 0)
            self.assertEqual(len(dr_result.get_param_corruptions()), 0)
            self.assertEqual(len(dr_result.get_param_size_ratios()), 3)
            self.assertEqual(len(dr_result.get_master_size_ratio()), 1)
            self.assertEqual(len(dr_result.get_size_ratios()), 10)
            self.assertEqual(dr_result.master_status[1], 'NORMAL')

            self.assertFalse(dr_result.is_corrupt)
            self.assertEqual(dr_result.param_file_count, 3)
            self.assertEqual(dr_result.brick_file_count, 6)
            self.assertEqual(dr_result.total_file_count, 10)

            # Get original values (mock)
            orig_cov = AbstractCoverage.load(cov_pth)
            time_vals_orig = orig_cov.get_time_values()
            orig_cov.close()

            # Corrupt the Master File
            fo = open(cov._persistence_layer.master_manager.file_path, "wb")
            fo.write('Junk')
            fo.close()
            # Corrupt the lon Parameter file
            fo = open(cov._persistence_layer.parameter_metadata['lon'].file_path, "wb")
            fo.write('Junk')
            fo.close()

            corrupt_res = dr.analyze(reanalyze=True)
            self.assertTrue(corrupt_res.is_corrupt)

            # Repair the metadata files
            dr.repair(reanalyze=True)

            fixed_res = dr.analyze(reanalyze=True)
            self.assertFalse(fixed_res.is_corrupt)

            fixed_cov = AbstractCoverage.load(cov_pth)
            self.assertIsInstance(fixed_cov, AbstractCoverage)

            time_vals_fixed = fixed_cov.get_time_values()
            fixed_cov.close()
            self.assertTrue(np.array_equiv(time_vals_orig, time_vals_fixed))
    def test_coverage_recovery(self):
        # Create the coverage
        dp_id, stream_id, route, stream_def_id, dataset_id = self.load_data_product()
        self.populate_dataset(dataset_id, 36)
        dset = self.dataset_management.read_dataset(dataset_id)
        dprod = self.dpsc_cli.read_data_product(dp_id)
        cov = DatasetManagementService._get_simplex_coverage(dataset_id)
        cov_pth = cov.persistence_dir
        cov.close()

        # Analyze the valid coverage
        dr = CoverageDoctor(cov_pth, dprod, dset)

        dr_result = dr.analyze()

        # TODO: Turn these into meaningful Asserts
        self.assertEqual(len(dr_result.get_brick_corruptions()), 0)
        self.assertEqual(len(dr_result.get_brick_size_ratios()), 8)
        self.assertEqual(len(dr_result.get_corruptions()), 0)
        self.assertEqual(len(dr_result.get_master_corruption()), 0)
        self.assertEqual(len(dr_result.get_param_corruptions()), 0)
        self.assertEqual(len(dr_result.get_param_size_ratios()), 64)
        self.assertEqual(len(dr_result.get_master_size_ratio()), 1)
        self.assertEqual(len(dr_result.get_size_ratios()), 73)
        self.assertEqual(dr_result.master_status[1], 'NORMAL')

        self.assertFalse(dr_result.is_corrupt)
        self.assertEqual(dr_result.param_file_count, 64)
        self.assertEqual(dr_result.brick_file_count, 8)
        self.assertEqual(dr_result.total_file_count, 73)

        # Get original values (mock)
        orig_cov = AbstractCoverage.load(cov_pth)
        time_vals_orig = orig_cov.get_time_values()
        orig_cov.close()

        # Corrupt the Master File
        fo = open(cov._persistence_layer.master_manager.file_path, "wb")
        fo.write('Junk')
        fo.close()
        # Corrupt the lon Parameter file
        fo = open(cov._persistence_layer.parameter_metadata['lon'].file_path, "wb")
        fo.write('Junk')
        fo.close()

        corrupt_res = dr.analyze(reanalyze=True)
        self.assertTrue(corrupt_res.is_corrupt)

        # Repair the metadata files
        dr.repair(reanalyze=True)

        fixed_res = dr.analyze(reanalyze=True)
        self.assertFalse(fixed_res.is_corrupt)

        fixed_cov = AbstractCoverage.load(cov_pth)
        self.assertIsInstance(fixed_cov, AbstractCoverage)

        time_vals_fixed = fixed_cov.get_time_values()
        fixed_cov.close()
        self.assertTrue(np.array_equiv(time_vals_orig, time_vals_fixed))
예제 #6
0
    def test_recovery_failed_nobackup(self):
        # Create the coverage
        cov, dset = self.get_cov()
        cov_pth = cov.persistence_dir
        cov.close()

        # Analyze the valid coverage
        dr = CoverageDoctor(cov_pth, 'dprod', dset)

        dr_result = dr.analyze()

        # Corrupt the Master File
        fo = open(cov._persistence_layer.master_manager.file_path, "wb")
        fo.write('Junk')
        fo.close()
        # Corrupt the lon Parameter file
        fo = open(cov._persistence_layer.parameter_metadata['lon'].file_path,
                  "wb")
        fo.write('Junk')
        fo.close()

        corrupt_res = dr.analyze(reanalyze=True)
        self.assertTrue(corrupt_res.is_corrupt)

        # Repair the metadata files
        with self.assertRaises(ValueError):
            temp_cov_dir = dr.repair(reanalyze=True,
                                     backup=False,
                                     copy_over=False,
                                     keep_temp=False)
            log.info('Temporary Coverage Directory: {0}', temp_cov_dir)
            self.assertFalse(os.path.exists(temp_cov_dir))

        nofix_res = dr.analyze(reanalyze=True)
        self.assertTrue(nofix_res.is_corrupt)
                 'unable to open file (File accessibilty: Unable to open file)',
                 'unable to open file (File accessability: Unable to open file)']
            if fs in ex.message:  # The view coverage couldn't load it's underlying reference coverage
                cpth = ex.message[len(fs):-1]

            for err in accessibility_errors:
                if err in ex.message:
                    cpth = self.get_coverage_path(dataset_id)
                    self.pause_ingestion(self.get_stream_id(dataset_id))
                    break
            else:
                log.critical("Unmatched error: %s", ex.message)
                raise

        # Return the CoverageDoctor instance
        return CoverageDoctor(cpth, dprod_obj, dset_obj)

    def run_coverage_doctor(self, dataset_id, data_product_id=None):
        log.error('Running coverage doctor')
        dr = self.get_coverage_doctor(dataset_id, data_product_id=data_product_id)

        if dr.analyze().is_corrupt:
            dr.repair()
        else:
            log.error("Repair Not Necessary")
            return "Repair Not Necessary"

        if not dr.analyze(reanalyze=True).is_corrupt:
            log.error("Repair Successful")
            return "Repair Successful"
        else:
예제 #8
0
    def test_coverage_recovery(self):
        # Create the coverage
        dp_id, stream_id, route, stream_def_id, dataset_id = self.load_data_product(
        )
        self.populate_dataset(dataset_id, 36)
        dset = self.dataset_management.read_dataset(dataset_id)
        dprod = self.dpsc_cli.read_data_product(dp_id)
        cov = DatasetManagementService._get_simplex_coverage(dataset_id)
        cov_pth = cov.persistence_dir
        cov.close()

        num_params = len(cov.list_parameters())
        num_bricks = 8
        total = num_params + num_bricks + 1

        # Analyze the valid coverage
        dr = CoverageDoctor(cov_pth, dprod, dset)

        dr_result = dr.analyze()

        # TODO: Turn these into meaningful Asserts
        self.assertEqual(len(dr_result.get_brick_corruptions()), 0)
        self.assertEqual(len(dr_result.get_brick_size_ratios()), num_bricks)
        self.assertEqual(len(dr_result.get_corruptions()), 0)
        self.assertEqual(len(dr_result.get_master_corruption()), 0)
        self.assertEqual(len(dr_result.get_param_corruptions()), 0)
        self.assertEqual(len(dr_result.get_param_size_ratios()), num_params)
        self.assertEqual(len(dr_result.get_master_size_ratio()), 1)
        self.assertEqual(len(dr_result.get_size_ratios()), total)
        self.assertEqual(dr_result.master_status[1], 'NORMAL')

        self.assertFalse(dr_result.is_corrupt)
        self.assertEqual(dr_result.param_file_count, num_params)
        self.assertEqual(dr_result.brick_file_count, num_bricks)
        self.assertEqual(dr_result.total_file_count, total)

        # Get original values (mock)
        orig_cov = AbstractCoverage.load(cov_pth)
        time_vals_orig = orig_cov.get_time_values()
        orig_cov.close()

        # Corrupt the Master File
        fo = open(cov._persistence_layer.master_manager.file_path, "wb")
        fo.write('Junk')
        fo.close()
        # Corrupt the lon Parameter file
        fo = open(cov._persistence_layer.parameter_metadata['lon'].file_path,
                  "wb")
        fo.write('Junk')
        fo.close()

        corrupt_res = dr.analyze(reanalyze=True)
        self.assertTrue(corrupt_res.is_corrupt)

        # Repair the metadata files
        dr.repair(reanalyze=True)

        fixed_res = dr.analyze(reanalyze=True)
        self.assertFalse(fixed_res.is_corrupt)

        fixed_cov = AbstractCoverage.load(cov_pth)
        self.assertIsInstance(fixed_cov, AbstractCoverage)

        time_vals_fixed = fixed_cov.get_time_values()
        fixed_cov.close()
        self.assertTrue(np.array_equiv(time_vals_orig, time_vals_fixed))