def test_run__job_error_delete(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(DeleteData, {"options": {"objects": "Contact"}})
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", False, None),
])
with self.assertRaises(BulkDataException):
task()
def test_run__query_fails(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(
DeleteData,
{"options": {
"objects": "Contact",
"where": "Id != null"
}})
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.JOB_FAILURE, [], 0, 0)
with self.assertRaises(BulkDataException):
task()
def test_run__person_accounts_enabled(self, step_mock):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
mock_describe_calls()
task = _make_task(
ExtractData,
{
"options": {
"database_url": "sqlite://", # in memory
"mapping": mapping_path,
}
},
)
task.bulk = mock.Mock()
task.sf = mock.Mock()
task.org_config._is_person_accounts_enabled = True
mock_query_households = MockBulkQueryOperation(
sobject="Account",
api_options={},
context=task,
query="SELECT Id, IsPersonAccount FROM Account",
)
mock_query_contacts = MockBulkQueryOperation(
sobject="Contact",
api_options={},
context=task,
query=
"SELECT Id, FirstName, LastName, Email, IsPersonAccount, AccountId FROM Contact",
)
mock_query_households.results = [["1", "false"]]
mock_query_contacts.results = [[
"2", "First", "Last", "*****@*****.**", "true", "1"
]]
step_mock.side_effect = [mock_query_households, mock_query_contacts]
task()
household = task.session.query(task.models["households"]).one()
self.assertEqual("1", household.sf_id)
self.assertEqual("false", household.IsPersonAccount)
self.assertEqual("HH_Account", household.record_type)
contact = task.session.query(task.models["contacts"]).one()
self.assertEqual("2", contact.sf_id)
self.assertEqual("true", contact.IsPersonAccount)
self.assertEqual("1", contact.household_id)
def test_run__ignore_error(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(
DeleteData,
{
"options": {
"objects": "Contact",
"ignore_row_errors": "true",
"hardDelete": "true",
}
},
)
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", False, None),
])
dml_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
with mock.patch.object(task.logger, "warning") as warning:
task()
assert len(warning.mock_calls) == 1
query_mock.assert_called_once_with(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id FROM Contact",
api=DataApi.SMART,
fields=["Id"],
)
query_mock.return_value.query.assert_called_once()
query_mock.return_value.get_results.assert_called_once()
dml_mock.assert_called_once_with(
sobject="Contact",
operation=DataOperationType.HARD_DELETE,
api_options={},
context=task,
fields=["Id"],
api=DataApi.SMART,
volume=2,
)
dml_mock.return_value.start.assert_called_once()
dml_mock.return_value.end.assert_called_once()
dml_mock.return_value.load_records.assert_called_once()
dml_mock.return_value.get_results.assert_called_once()
def test_init_mapping_org_has_person_accounts_enabled(self):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
mock_describe_calls()
task = _make_task(
ExtractData,
{"options": {"database_url": "sqlite:///", "mapping": mapping_path}},
)
task.org_config._is_person_accounts_enabled = True
task._init_mapping()
assert "Insert Households" in task.mapping
# Person Accounts should not be added to mapping
self.assert_person_accounts_in_mapping(task.mapping, True)
def test_run__v2__person_accounts_disabled(self, step_mock):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v2)
mock_describe_calls()
task = _make_task(
ExtractData,
{
"options": {
"database_url": "sqlite://", # in memory
"mapping": mapping_path,
}
},
)
task.bulk = mock.Mock()
task.sf = mock.Mock()
task.org_config._is_person_accounts_enabled = False
mock_query_households = MockBulkQueryOperation(
sobject="Account",
api_options={},
context=task,
query="SELECT Id, Name FROM Account",
)
mock_query_contacts = MockBulkQueryOperation(
sobject="Contact",
api_options={},
context=task,
query=
"SELECT Id, FirstName, LastName, Email, AccountId FROM Contact",
)
mock_query_households.results = [["1", "TestHousehold"]]
mock_query_contacts.results = [[
"2", "First", "Last", "*****@*****.**", "1"
]]
step_mock.side_effect = [mock_query_households, mock_query_contacts]
task()
household = task.session.query(task.models["households"]).one()
assert household.name == "TestHousehold"
assert not hasattr(household, "IsPersonAccount")
assert household.record_type == "HH_Account"
contact = task.session.query(task.models["contacts"]).one()
assert contact.household_id == "1"
assert not hasattr(contact, "IsPersonAccount")
def test_insert_dml_operation__booleans(self):
mock_describe_calls()
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite:///test.db",
"mapping": "mapping.yml",
}
},
)
task.project_config.project__package__api_version = "48.0"
task._init_task()
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[{
"id": "003000000000001",
"success": True
}],
status=200,
)
recs = [["Narvaez", "True"]]
dml_op = RestApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={"batch_size": 2},
context=task,
fields=["LastName",
"IsEmailBounced"], # IsEmailBounced is a Boolean field.
)
dml_op.start()
dml_op.load_records(iter(recs))
dml_op.end()
json_body = json.loads(responses.calls[1].request.body)
assert json_body["records"] == [{
"LastName": "Narvaez",
"IsEmailBounced": True,
"attributes": {
"type": "Contact"
},
}]
def test_create_table__already_exists(self):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
db_path = os.path.join(base_path, "testdata.db")
mock_describe_calls()
task = _make_task(
ExtractData,
{
"options": {
"database_url": f"sqlite:///{db_path}",
"mapping": mapping_path,
}
},
)
task.org_config._is_person_accounts_enabled = False
with self.assertRaises(BulkDataException):
task()
def test_run__no_results(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(DeleteData, {"options": {"objects": "Contact"}})
query_mock.return_value.get_results.return_value = iter([])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 0, 0)
task()
query_mock.assert_called_once_with(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id FROM Contact",
)
query_mock.return_value.query.assert_called_once()
query_mock.return_value.get_results.assert_not_called()
dml_mock.assert_not_called()
def test_run__where(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(
DeleteData,
{"options": {
"objects": "Contact",
"where": "Id != null"
}})
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", True, None),
])
dml_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
task()
query_mock.assert_called_once_with(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id FROM Contact WHERE Id != null",
fields=["Id"],
api=DataApi.SMART,
)
query_mock.return_value.query.assert_called_once()
query_mock.return_value.get_results.assert_called_once()
dml_mock.assert_called_once_with(
sobject="Contact",
operation=DataOperationType.DELETE,
api_options={},
context=task,
fields=["Id"],
api=DataApi.SMART,
volume=2,
)
dml_mock.return_value.start.assert_called_once()
dml_mock.return_value.end.assert_called_once()
dml_mock.return_value.load_records.assert_called_once()
dml_mock.return_value.get_results.assert_called_once()
def test_run__sql(self, step_mock):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
mock_describe_calls()
with temporary_dir():
task = _make_task(
ExtractData,
{
"options": {
"sql_path": "testdata.sql",
"mapping": mapping_path
}
},
)
task.bulk = mock.Mock()
task.sf = mock.Mock()
task.org_config._is_person_accounts_enabled = False
mock_query_households = MockBulkQueryOperation(
sobject="Account",
api_options={},
context=task,
query="SELECT Id FROM Account",
)
mock_query_contacts = MockBulkQueryOperation(
sobject="Contact",
api_options={},
context=task,
query=
"SELECT Id, FirstName, LastName, Email, AccountId FROM Contact",
)
mock_query_households.results = [["1"]]
mock_query_contacts.results = [[
"2", "First", "Last", "*****@*****.**", "1"
]]
step_mock.side_effect = [
mock_query_households, mock_query_contacts
]
task()
assert os.path.exists("testdata.sql")
def test_validate_and_inject_mapping_injects_namespaces(self):
mock_describe_calls()
# Note: ns__Description__c is a mock field added to our stored, mock describes (in JSON)
ms = parse_from_yaml(
StringIO("""Insert Accounts:
sf_object: Account
table: Account
fields:
- Description__c"""))["Insert Accounts"]
org_config = DummyOrgConfig(
{
"instance_url": "https://example.com",
"access_token": "abc123"
}, "test")
assert ms.validate_and_inject_namespace(org_config,
"ns",
DataOperationType.INSERT,
inject_namespaces=True)
assert list(ms.fields.keys()) == ["ns__Description__c"]
def test_validate_and_inject_mapping_enforces_fls(self):
mock_describe_calls()
mapping = parse_from_yaml(
StringIO(
"Insert Accounts:\n sf_object: Account\n table: Account\n fields:\n - Nonsense__c"
))
org_config = DummyOrgConfig(
{
"instance_url": "https://example.com",
"access_token": "abc123"
}, "test")
with pytest.raises(BulkDataException):
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.INSERT,
inject_namespaces=False,
drop_missing=False,
)
def test_validate_and_inject_mapping_removes_steps_with_drop_missing(self):
mock_describe_calls()
mapping = parse_from_yaml(
StringIO(
"Insert Accounts:\n sf_object: NotAccount\n table: Account\n fields:\n - Nonsense__c"
))
org_config = DummyOrgConfig(
{
"instance_url": "https://example.com",
"access_token": "abc123"
}, "test")
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.INSERT,
inject_namespaces=False,
drop_missing=True,
)
assert "Insert Accounts" not in mapping
def test_insert_dml_operation__delete(self):
mock_describe_calls()
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite:///test.db",
"mapping": "mapping.yml",
}
},
)
task.project_config.project__package__api_version = "48.0"
task._init_task()
responses.add(
responses.DELETE,
url=
"https://example.com/services/data/v48.0/composite/sobjects?ids=003000000000001,003000000000002",
json=[
{
"id": "003000000000001",
"success": True
},
{
"id": "003000000000002",
"success": True
},
],
status=200,
)
responses.add(
responses.DELETE,
url=
"https://example.com/services/data/v48.0/composite/sobjects?ids=003000000000003",
json=[{
"id": "003000000000003",
"success": True
}],
status=200,
)
recs = [["003000000000001"], ["003000000000002"], ["003000000000003"]]
dml_op = RestApiDmlOperation(
sobject="Contact",
operation=DataOperationType.DELETE,
api_options={"batch_size": 2},
context=task,
fields=["Id"],
)
dml_op.start()
dml_op.load_records(iter(recs))
dml_op.end()
assert dml_op.job_result == DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 3, 0)
assert list(dml_op.get_results()) == [
DataOperationResult("003000000000001", True, ""),
DataOperationResult("003000000000002", True, ""),
DataOperationResult("003000000000003", True, ""),
]
def test_insert_dml_operation__row_failure(self):
mock_describe_calls()
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite:///test.db",
"mapping": "mapping.yml",
}
},
)
task.project_config.project__package__api_version = "48.0"
task._init_task()
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[
{
"id": "003000000000001",
"success": True
},
{
"id": "003000000000002",
"success": True
},
],
status=200,
)
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[{
"id":
"003000000000003",
"success":
False,
"errors": [{
"statusCode": "VALIDATION_ERR",
"message": "Bad data",
"fields": ["FirstName"],
}],
}],
status=200,
)
recs = [["Fred", "Narvaez"], [None, "De Vries"], ["Hiroko", "Aito"]]
dml_op = RestApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={"batch_size": 2},
context=task,
fields=["FirstName", "LastName"],
)
dml_op.start()
dml_op.load_records(iter(recs))
dml_op.end()
assert dml_op.job_result == DataOperationJobResult(
DataOperationStatus.ROW_FAILURE, [], 3, 1)
assert list(dml_op.get_results()) == [
DataOperationResult("003000000000001", True, ""),
DataOperationResult("003000000000002", True, ""),
DataOperationResult("003000000000003", False,
"VALIDATION_ERR: Bad data (FirstName)"),
]
