def test_insert_dml_operation(self):
mock_describe_calls()
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite:///test.db",
"mapping": "mapping.yml",
}
},
)
task.project_config.project__package__api_version = "48.0"
task._init_task()
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[
{
"id": "003000000000001",
"success": True
},
{
"id": "003000000000002",
"success": True
},
],
status=200,
)
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[{
"id": "003000000000003",
"success": True
}],
status=200,
)
recs = [["Fred", "Narvaez"], [None, "De Vries"], ["Hiroko", "Aito"]]
dml_op = RestApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={"batch_size": 2},
context=task,
fields=["FirstName", "LastName"],
)
dml_op.start()
dml_op.load_records(iter(recs))
dml_op.end()
assert dml_op.job_result == DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 3, 0)
assert list(dml_op.get_results()) == [
DataOperationResult("003000000000001", True, ""),
DataOperationResult("003000000000002", True, ""),
DataOperationResult("003000000000003", True, ""),
]
def test_get_results(self, download_mock):
context = mock.Mock()
context.bulk.endpoint = "https://test"
download_mock.side_effect = [
io.StringIO("""id,success,created,error
003000000000001,true,true,
003000000000002,true,true,"""),
io.StringIO("""id,success,created,error
003000000000003,false,false,error"""),
]
step = BulkApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={},
context=context,
fields=["LastName"],
)
step.job_id = "JOB"
step.batch_ids = ["BATCH1", "BATCH2"]
results = step.get_results()
assert list(results) == [
DataOperationResult("003000000000001", True, None),
DataOperationResult("003000000000002", True, None),
DataOperationResult(None, False, "error"),
]
download_mock.assert_has_calls([
mock.call("https://test/job/JOB/batch/BATCH1/result",
context.bulk),
mock.call("https://test/job/JOB/batch/BATCH2/result",
context.bulk),
])
def test_end_to_end(self, download_mock):
context = mock.Mock()
context.bulk.endpoint = "https://test"
context.bulk.create_job.return_value = "JOB"
context.bulk.post_batch.side_effect = ["BATCH1", "BATCH2"]
download_mock.return_value = io.StringIO("""id,success,created,error
003000000000001,true,true,
003000000000002,true,true,
003000000000003,false,false,error""")
step = BulkApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={},
context=context,
fields=["LastName"],
)
step._wait_for_job = mock.Mock()
step._wait_for_job.return_value = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 0, 0)
step.start()
step.load_records(iter([["Test"], ["Test2"], ["Test3"]]))
step.end()
assert step.job_result.status is DataOperationStatus.SUCCESS
results = step.get_results()
assert list(results) == [
DataOperationResult("003000000000001", True, None),
DataOperationResult("003000000000002", True, None),
DataOperationResult(None, False, "error"),
]
def test_run(self, dml_mock, query_mock):
task = _make_task(DeleteData, {"options": {"objects": "Contact"}})
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", True, None),
])
dml_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 0, 0)
task()
query_mock.assert_called_once_with(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id FROM Contact",
)
query_mock.return_value.query.assert_called_once()
query_mock.return_value.get_results.assert_called_once()
dml_mock.assert_called_once_with(
sobject="Contact",
operation=DataOperationType.DELETE,
api_options={},
context=task,
fields=["Id"],
)
dml_mock.return_value.start.assert_called_once()
dml_mock.return_value.end.assert_called_once()
dml_mock.return_value.load_records.assert_called_once()
dml_mock.return_value.get_results.assert_called_once()
def test_generate_results_id_map__success(self):
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite://",
"mapping": "mapping.yml"
}
},
)
step = mock.Mock()
step.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", True, None),
DataOperationResult("001000000000002", True, None),
])
generator = task._generate_results_id_map(
step, ["001000000000009", "001000000000010", "001000000000011"])
assert list(generator) == [
("001000000000009", "001000000000000"),
("001000000000010", "001000000000001"),
("001000000000011", "001000000000002"),
]
def test_run__ignore_error_throttling(self, dml_mock, query_mock):
task = _make_task(
DeleteData,
{
"options": {
"objects": "Contact",
"ignore_row_errors": "true",
"hardDelete": "true",
}
},
)
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"] * 15)
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", False, None),
] * 15)
dml_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
with mock.patch.object(task.logger, "warning") as warning:
task()
assert len(warning.mock_calls) == task.row_warning_limit + 1 == 11
assert "warnings suppressed" in str(warning.mock_calls[-1])
def test_generate_results_id_map__exception_failure(self):
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite://",
"mapping": "mapping.yml"
}
},
)
step = mock.Mock()
step.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult(None, False, "error"),
DataOperationResult("001000000000002", True, None),
])
with self.assertRaises(BulkDataException) as ex:
list(
task._generate_results_id_map(
step,
["001000000000009", "001000000000010", "001000000000011"]))
self.assertIn("Error on record", str(ex.exception))
self.assertIn("001000000000010", str(ex.exception))
def test_run__autopk(self, step_mock):
responses.add(
method="GET",
url=
"https://example.com/services/data/v46.0/query/?q=SELECT+Id+FROM+RecordType+WHERE+SObjectType%3D%27Account%27AND+DeveloperName+%3D+%27HH_Account%27+LIMIT+1",
body=json.dumps({"records": [{
"Id": "1"
}]}),
status=200,
)
mapping_file = "mapping_v2.yml"
base_path = os.path.dirname(__file__)
db_path = os.path.join(base_path, "testdata.db")
mapping_path = os.path.join(base_path, mapping_file)
with temporary_dir() as d:
tmp_db_path = os.path.join(d, "testdata.db")
shutil.copyfile(db_path, tmp_db_path)
task = _make_task(
LoadData,
{
"options": {
"database_url": f"sqlite:///{tmp_db_path}",
"mapping": mapping_path,
}
},
)
task.bulk = mock.Mock()
task.sf = mock.Mock()
step = MockBulkApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={},
context=task,
fields=[],
)
step_mock.return_value = step
step.results = [
DataOperationResult("001000000000000", True, None),
DataOperationResult("003000000000000", True, None),
DataOperationResult("003000000000001", True, None),
]
task()
assert step.records == [
["TestHousehold", "1"],
["Test", "User", "*****@*****.**", "001000000000000"],
["Error", "User", "*****@*****.**", "001000000000000"],
]
hh_ids = task.session.query(
*task.metadata.tables["households_sf_ids"].columns).one()
assert hh_ids == ("1", "001000000000000")
task.session.close()
def test_run__job_error_delete(self, dml_mock, query_mock):
task = _make_task(DeleteData, {"options": {"objects": "Contact"}})
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", False, None),
])
with self.assertRaises(BulkDataException):
task()
def test_run__ignore_error(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(
DeleteData,
{
"options": {
"objects": "Contact",
"ignore_row_errors": "true",
"hardDelete": "true",
}
},
)
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", False, None),
])
dml_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
with mock.patch.object(task.logger, "warning") as warning:
task()
assert len(warning.mock_calls) == 1
query_mock.assert_called_once_with(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id FROM Contact",
api=DataApi.SMART,
fields=["Id"],
)
query_mock.return_value.query.assert_called_once()
query_mock.return_value.get_results.assert_called_once()
dml_mock.assert_called_once_with(
sobject="Contact",
operation=DataOperationType.HARD_DELETE,
api_options={},
context=task,
fields=["Id"],
api=DataApi.SMART,
volume=2,
)
dml_mock.return_value.start.assert_called_once()
dml_mock.return_value.end.assert_called_once()
dml_mock.return_value.load_records.assert_called_once()
dml_mock.return_value.get_results.assert_called_once()
def test_process_job_results__update_success(self):
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite://",
"mapping": "mapping.yml"
}
},
)
task.session = mock.Mock()
task._initialize_id_table = mock.Mock()
task._sql_bulk_insert_from_records = mock.Mock()
task.bulk = mock.Mock()
task.sf = mock.Mock()
local_ids = ["1"]
step = MockBulkApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={},
context=task,
fields=[],
)
step.results = [DataOperationResult("001111111111111", True, None)]
mapping = {"table": "Account", "action": "update"}
task._process_job_results(mapping, step, local_ids)
task.session.connection.assert_not_called()
task._initialize_id_table.assert_not_called()
task._sql_bulk_insert_from_records.assert_not_called()
task.session.commit.assert_not_called()
def test_process_job_results__insert_rows_fail(self):
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite://",
"mapping": "mapping.yml",
"ignore_row_errors": True,
}
},
)
task.session = mock.Mock()
task._initialize_id_table = mock.Mock()
task._sql_bulk_insert_from_records = mock.Mock()
task.bulk = mock.Mock()
task.sf = mock.Mock()
task.logger = mock.Mock()
local_ids = ["1", "2", "3", "4"]
step = MockBulkApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={},
context=task,
fields=[],
)
step.job_result = DataOperationJobResult(
DataOperationStatus.ROW_FAILURE, [], 4, 4)
step.end = mock.Mock()
step.results = [
DataOperationResult("001111111111111", False, None),
DataOperationResult("001111111111112", False, None),
DataOperationResult("001111111111113", False, None),
DataOperationResult("001111111111114", False, None),
]
mapping = {"table": "Account", "action": "insert"}
task._process_job_results(mapping, step, local_ids)
task.session.connection.assert_called_once()
task._initialize_id_table.assert_called_once_with(mapping, True)
task._sql_bulk_insert_from_records.assert_not_called()
task.session.commit.assert_called_once()
assert len(task.logger.mock_calls) == 4
def test_generate_results_id_map__respects_silent_error_flag(self):
task = _make_task(
LoadData,
{
"options": {
"ignore_row_errors": True,
"database_url": "sqlite://",
"mapping": "mapping.yml",
}
},
)
step = mock.Mock()
step.get_results.return_value = iter(
[DataOperationResult(None, False, None)] * 15)
with mock.patch.object(task.logger, "warning") as warning:
generator = task._generate_results_id_map(
step,
["001000000000009", "001000000000010", "001000000000011"] * 15)
_ = list(generator) # generate the errors
assert len(warning.mock_calls) == task.row_warning_limit + 1 == 11
assert "warnings suppressed" in str(warning.mock_calls[-1])
step = mock.Mock()
step.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult(None, False, None),
DataOperationResult("001000000000002", True, None),
])
generator = task._generate_results_id_map(
step, ["001000000000009", "001000000000010", "001000000000011"])
assert list(generator) == [
("001000000000009", "001000000000000"),
("001000000000011", "001000000000002"),
]
def test_process_job_results__exception_failure(self):
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite://",
"mapping": "mapping.yml"
}
},
)
task.session = mock.Mock()
task._initialize_id_table = mock.Mock()
task._sql_bulk_insert_from_records = mock.Mock()
task.bulk = mock.Mock()
task.sf = mock.Mock()
local_ids = ["1"]
step = MockBulkApiDmlOperation(
sobject="Contact",
operation=DataOperationType.UPDATE,
api_options={},
context=task,
fields=[],
)
step.results = [DataOperationResult(None, False, "message")]
step.end()
mapping = {"table": "Account", "action": "update"}
with self.assertRaises(BulkDataException) as ex:
task._process_job_results(mapping, step, local_ids)
self.assertIn("Error on record with id", str(ex.exception))
self.assertIn("message", str(ex.exception))
