def test_end_to_end(self, download_mock):
context = mock.Mock()
context.bulk.endpoint = "https://test"
context.bulk.create_job.return_value = "JOB"
context.bulk.post_batch.side_effect = ["BATCH1", "BATCH2"]
download_mock.return_value = io.StringIO("""id,success,created,error
003000000000001,true,true,
003000000000002,true,true,
003000000000003,false,false,error""")
step = BulkApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={},
context=context,
fields=["LastName"],
)
step._wait_for_job = mock.Mock()
step._wait_for_job.return_value = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 0, 0)
step.start()
step.load_records(iter([["Test"], ["Test2"], ["Test3"]]))
step.end()
assert step.job_result.status is DataOperationStatus.SUCCESS
results = step.get_results()
assert list(results) == [
DataOperationResult("003000000000001", True, None),
DataOperationResult("003000000000002", True, None),
DataOperationResult(None, False, "error"),
]
def test_get_results(self, download_mock):
context = mock.Mock()
context.bulk.endpoint = "https://test"
download_mock.side_effect = [
io.StringIO("""id,success,created,error
003000000000001,true,true,
003000000000002,true,true,"""),
io.StringIO("""id,success,created,error
003000000000003,false,false,error"""),
]
step = BulkApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={},
context=context,
fields=["LastName"],
)
step.job_id = "JOB"
step.batch_ids = ["BATCH1", "BATCH2"]
results = step.get_results()
assert list(results) == [
DataOperationResult("003000000000001", True, None),
DataOperationResult("003000000000002", True, None),
DataOperationResult(None, False, "error"),
]
download_mock.assert_has_calls([
mock.call("https://test/job/JOB/batch/BATCH1/result",
context.bulk),
mock.call("https://test/job/JOB/batch/BATCH2/result",
context.bulk),
])
def _run_task(self):
self._validate_and_inject_namespace()
for obj in self.sobjects:
query = f"SELECT Id FROM {obj}"
if self.options["where"]:
query += f" WHERE {self.options['where']}"
self.logger.info(f"Querying for {obj} objects")
qs = BulkApiQueryOperation(sobject=obj,
api_options={},
context=self,
query=query)
qs.query()
if qs.job_result.status is not DataOperationStatus.SUCCESS:
raise BulkDataException(
f"Unable to query records for {obj}: {','.join(qs.job_result.job_errors)}"
)
if not qs.job_result.records_processed:
self.logger.info("No records found, skipping delete operation")
continue
self.logger.info(f"Deleting {self._object_description(obj)} ")
ds = BulkApiDmlOperation(
sobject=obj,
operation=(DataOperationType.HARD_DELETE
if self.options["hardDelete"] else
DataOperationType.DELETE),
api_options={},
context=self,
fields=["Id"],
)
ds.start()
ds.load_records(qs.get_results())
ds.end()
if ds.job_result.status not in [
DataOperationStatus.SUCCESS,
DataOperationStatus.ROW_FAILURE,
]:
raise BulkDataException(
f"Unable to delete records for {obj}: {','.join(qs.job_result.job_errors)}"
)
error_checker = RowErrorChecker(self.logger,
self.options["ignore_row_errors"],
self.row_warning_limit)
for result in ds.get_results():
error_checker.check_for_row_error(result, result.id)
def test_get_results__failure(self, download_mock):
context = mock.Mock()
context.bulk.endpoint = "https://test"
download_mock.return_value.side_effect = Exception
step = BulkApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={},
context=context,
fields=["LastName"],
)
step.job_id = "JOB"
step.batch_ids = ["BATCH1", "BATCH2"]
with self.assertRaises(BulkDataException):
list(step.get_results())
