def test_end_to_end(self, download_mock): context = mock.Mock() context.bulk.endpoint = "https://test" context.bulk.create_job.return_value = "JOB" context.bulk.post_batch.side_effect = ["BATCH1", "BATCH2"] download_mock.return_value = io.StringIO("""id,success,created,error 003000000000001,true,true, 003000000000002,true,true, 003000000000003,false,false,error""") step = BulkApiDmlOperation( sobject="Contact", operation=DataOperationType.INSERT, api_options={}, context=context, fields=["LastName"], ) step._wait_for_job = mock.Mock() step._wait_for_job.return_value = DataOperationJobResult( DataOperationStatus.SUCCESS, [], 0, 0) step.start() step.load_records(iter([["Test"], ["Test2"], ["Test3"]])) step.end() assert step.job_result.status is DataOperationStatus.SUCCESS results = step.get_results() assert list(results) == [ DataOperationResult("003000000000001", True, None), DataOperationResult("003000000000002", True, None), DataOperationResult(None, False, "error"), ]
def test_get_results(self, download_mock): context = mock.Mock() context.bulk.endpoint = "https://test" download_mock.side_effect = [ io.StringIO("""id,success,created,error 003000000000001,true,true, 003000000000002,true,true,"""), io.StringIO("""id,success,created,error 003000000000003,false,false,error"""), ] step = BulkApiDmlOperation( sobject="Contact", operation=DataOperationType.INSERT, api_options={}, context=context, fields=["LastName"], ) step.job_id = "JOB" step.batch_ids = ["BATCH1", "BATCH2"] results = step.get_results() assert list(results) == [ DataOperationResult("003000000000001", True, None), DataOperationResult("003000000000002", True, None), DataOperationResult(None, False, "error"), ] download_mock.assert_has_calls([ mock.call("https://test/job/JOB/batch/BATCH1/result", context.bulk), mock.call("https://test/job/JOB/batch/BATCH2/result", context.bulk), ])
def _run_task(self): self._validate_and_inject_namespace() for obj in self.sobjects: query = f"SELECT Id FROM {obj}" if self.options["where"]: query += f" WHERE {self.options['where']}" self.logger.info(f"Querying for {obj} objects") qs = BulkApiQueryOperation(sobject=obj, api_options={}, context=self, query=query) qs.query() if qs.job_result.status is not DataOperationStatus.SUCCESS: raise BulkDataException( f"Unable to query records for {obj}: {','.join(qs.job_result.job_errors)}" ) if not qs.job_result.records_processed: self.logger.info("No records found, skipping delete operation") continue self.logger.info(f"Deleting {self._object_description(obj)} ") ds = BulkApiDmlOperation( sobject=obj, operation=(DataOperationType.HARD_DELETE if self.options["hardDelete"] else DataOperationType.DELETE), api_options={}, context=self, fields=["Id"], ) ds.start() ds.load_records(qs.get_results()) ds.end() if ds.job_result.status not in [ DataOperationStatus.SUCCESS, DataOperationStatus.ROW_FAILURE, ]: raise BulkDataException( f"Unable to delete records for {obj}: {','.join(qs.job_result.job_errors)}" ) error_checker = RowErrorChecker(self.logger, self.options["ignore_row_errors"], self.row_warning_limit) for result in ds.get_results(): error_checker.check_for_row_error(result, result.id)
def test_get_results__failure(self, download_mock): context = mock.Mock() context.bulk.endpoint = "https://test" download_mock.return_value.side_effect = Exception step = BulkApiDmlOperation( sobject="Contact", operation=DataOperationType.INSERT, api_options={}, context=context, fields=["LastName"], ) step.job_id = "JOB" step.batch_ids = ["BATCH1", "BATCH2"] with self.assertRaises(BulkDataException): list(step.get_results())