def test_run__ignore_error_throttling(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(
DeleteData,
{
"options": {
"objects": "Contact",
"ignore_row_errors": "true",
"hardDelete": "true",
}
},
)
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"] * 15)
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", False, None),
] * 15)
dml_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
with mock.patch.object(task.logger, "warning") as warning:
task()
assert len(warning.mock_calls) == task.row_warning_limit + 1 == 11
assert "warnings suppressed" in str(warning.mock_calls[-1])
def test_validate_and_inject_mapping_works_case_insensitively(self):
mock_describe_calls()
mapping = parse_from_yaml(
StringIO(
(
"Insert Accounts:\n sf_object: account\n table: account\n fields:\n - name\n"
"Insert Contacts:\n sf_object: contact\n table: contact\n fields:\n - fIRSTnAME\n lookups:\n accountid:\n table: account"
)
)
)
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
assert mapping["Insert Accounts"].sf_object != "Account"
assert mapping["Insert Accounts"].sf_object == "account"
assert "name" in mapping["Insert Accounts"].fields
assert "Name" not in mapping["Insert Accounts"].fields
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.INSERT,
inject_namespaces=False,
drop_missing=False,
)
assert mapping["Insert Accounts"].sf_object == "Account"
assert mapping["Insert Accounts"].sf_object != "account"
assert "Name" in mapping["Insert Accounts"].fields
assert "name" not in mapping["Insert Accounts"].fields
def test_map_autopks(self):
mock_describe_calls()
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v2)
mock_describe_calls()
task = _make_task(
ExtractData,
{
"options": {
"database_url": "sqlite://", # in memory
"mapping": mapping_path,
}
},
)
task._convert_lookups_to_id = mock.Mock()
task.metadata = mock.MagicMock()
task._init_mapping()
task._map_autopks()
task._convert_lookups_to_id.assert_called_once_with(
task.mapping["Insert Contacts"], ["AccountId"]
)
task.metadata.tables.__getitem__.return_value.drop.assert_has_calls(
[mock.call(), mock.call()]
)
task.metadata.tables.__getitem__.assert_has_calls(
[mock.call("contacts_sf_ids"), mock.call("households_sf_ids")],
any_order=True,
)
def test_validate_and_inject_mapping_removes_lookups_with_drop_missing(self):
mock_describe_calls()
mapping = parse_from_yaml(
StringIO(
(
"Insert Accounts:\n sf_object: NotAccount\n table: Account\n fields:\n - Nonsense__c\n"
"Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n AccountId:\n table: Account"
)
)
)
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.INSERT,
inject_namespaces=False,
drop_missing=True,
)
assert "Insert Accounts" not in mapping
assert "Insert Contacts" in mapping
assert "AccountId" not in mapping["Insert Contacts"].lookups
def test_import_results__relative_dates(self):
mock_describe_calls()
task = _make_task(
ExtractData,
{"options": {"database_url": "sqlite://", "mapping": "mapping.yml"}},
)
mapping = MappingStep(
sf_object="Opportunity",
fields={"Id": "sf_id", "CloseDate": "CloseDate"},
anchor_date="2020-07-01",
)
step = mock.Mock()
step.get_results.return_value = iter(
[["006000000000001", (date.today() + timedelta(days=9)).isoformat()]]
)
task.session = mock.Mock()
task._sql_bulk_insert_from_records = mock.Mock()
task._import_results(mapping, step)
task.session.connection.assert_called_once_with()
step.get_results.assert_called_once_with()
task._sql_bulk_insert_from_records.assert_called_once_with(
connection=task.session.connection.return_value,
table=mapping.table,
columns=["sf_id", "CloseDate"],
record_iterable=mock.ANY,
)
result = list(
task._sql_bulk_insert_from_records.call_args_list[0][1]["record_iterable"]
)
assert result == [["006000000000001", "2020-07-10"]]
def test_import_results__no_columns(self): # , query_op_mock):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
mock_describe_calls()
task = _make_task(
ExtractData,
{"options": {"database_url": "sqlite://", "mapping": mapping_path}},
)
mapping = MappingStep(
sf_object="Opportunity",
table="Opportunity",
fields={},
lookups={},
)
step = mock.Mock()
step.get_results.return_value = [[1], [2]]
task.session = mock.Mock()
task._init_mapping()
task.mapping["Opportunity"] = mapping
with task._init_db():
task._import_results(mapping, step)
output_Opportunties = list(
task.session.execute("select * from Opportunity")
)
assert output_Opportunties == [(1,), (2,)]
def test_validate_and_inject_mapping_throws_exception_required_lookup_dropped(self):
mock_describe_calls()
# This test uses a bit of gimmickry to validate exception behavior on dropping a required lookup.
# Since mapping_parser identifies target objects via the `table` clause rather than the actual schema,
# which makes us resilient to polymorphic lookups, we'll pretend the non-nillable `Id` field is a lookup.
mapping = parse_from_yaml(
StringIO(
(
"Insert Accounts:\n sf_object: NotAccount\n table: Account\n fields:\n - Nonsense__c\n"
"Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n Id:\n table: Account"
)
)
)
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
with pytest.raises(BulkDataException):
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.INSERT,
inject_namespaces=False,
drop_missing=True,
)
def test_validate_and_inject_mapping_queries_is_person_account_field(self):
mock_describe_calls()
mapping = parse_from_yaml(
StringIO(
(
"Insert Accounts:\n sf_object: Account\n table: Account\n fields:\n - Description__c\n"
"Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n AccountId:\n table: Account"
)
)
)
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.QUERY,
inject_namespaces=False,
drop_missing=True,
org_has_person_accounts_enabled=True,
)
assert "Insert Accounts" in mapping
assert "Insert Contacts" in mapping
assert "IsPersonAccount" in mapping["Insert Accounts"]["fields"]
assert "IsPersonAccount" in mapping["Insert Contacts"]["fields"]
def test_run__job_error_query(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(DeleteData, {"options": {"objects": "Contact"}})
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.JOB_FAILURE, [], 0, 0)
with self.assertRaises(BulkDataException):
task()
def test_validate_and_inject_namespace__missing_object(self):
mock_describe_calls()
task = _make_task(
DeleteData, {"options": {
"objects": "ApexTestQueueItem,Account"
}})
# ApexTestQueueItem is not deletable
with pytest.raises(BulkDataException):
task._validate_and_inject_namespace()
def test_validate_and_inject_namespace__standard(self):
mock_describe_calls()
task = _make_task(DeleteData,
{"options": {
"objects": "Contact,Account"
}})
task._validate_and_inject_namespace()
assert task.sobjects == ["Contact", "Account"]
def test_insert_dml_operation__delete(self):
mock_describe_calls()
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite:///test.db",
"mapping": "mapping.yml",
}
},
)
task.project_config.project__package__api_version = "48.0"
task._init_task()
responses.add(
responses.DELETE,
url="https://example.com/services/data/v48.0/composite/sobjects?ids=003000000000001,003000000000002",
json=[
{"id": "003000000000001", "success": True},
{"id": "003000000000002", "success": True},
],
status=200,
)
responses.add(
responses.DELETE,
url="https://example.com/services/data/v48.0/composite/sobjects?ids=003000000000003",
json=[{"id": "003000000000003", "success": True}],
status=200,
)
recs = [["003000000000001"], ["003000000000002"], ["003000000000003"]]
dml_op = RestApiDmlOperation(
sobject="Contact",
operation=DataOperationType.DELETE,
api_options={"batch_size": 2},
context=task,
fields=["Id"],
)
dml_op.start()
dml_op.load_records(iter(recs))
dml_op.end()
assert dml_op.job_result == DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 3, 0
)
assert list(dml_op.get_results()) == [
DataOperationResult("003000000000001", True, ""),
DataOperationResult("003000000000002", True, ""),
DataOperationResult("003000000000003", True, ""),
]
def test_run__person_accounts_enabled(self, query_op_mock):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
mock_describe_calls()
with temporary_dir() as d:
tmp_db_path = os.path.join(d, "testdata.db")
task = _make_task(
ExtractData,
{
"options": {
"database_url": f"sqlite:///{tmp_db_path}",
"mapping": mapping_path,
}
},
)
task.bulk = mock.Mock()
task.sf = mock.Mock()
task.org_config._is_person_accounts_enabled = True
mock_query_households = MockBulkQueryOperation(
sobject="Account",
api_options={},
context=task,
query="SELECT Id, IsPersonAccount FROM Account",
)
mock_query_contacts = MockBulkQueryOperation(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id, FirstName, LastName, Email, IsPersonAccount, AccountId FROM Contact",
)
mock_query_households.results = [["1", "false"]]
mock_query_contacts.results = [
["2", "First", "Last", "*****@*****.**", "true", "1"]
]
query_op_mock.side_effect = [mock_query_households, mock_query_contacts]
task()
with create_engine(task.options["database_url"]).connect() as conn:
household = next(conn.execute("select * from households"))
assert "1" == household.sf_id
assert "false" == household.IsPersonAccount
assert "HH_Account" == household.record_type
contact = next(conn.execute("select * from contacts"))
assert "2" == contact.sf_id
assert "true" == contact.IsPersonAccount
assert "1" == contact.household_id
def test_run__job_error_delete(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(DeleteData, {"options": {"objects": "Contact"}})
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", False, None),
])
with self.assertRaises(BulkDataException):
task()
def test_run__ignore_error(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(
DeleteData,
{
"options": {
"objects": "Contact",
"ignore_row_errors": "true",
"hardDelete": "true",
}
},
)
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", False, None),
])
dml_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
with mock.patch.object(task.logger, "warning") as warning:
task()
assert len(warning.mock_calls) == 1
query_mock.assert_called_once_with(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id FROM Contact",
api=DataApi.SMART,
fields=["Id"],
)
query_mock.return_value.query.assert_called_once()
query_mock.return_value.get_results.assert_called_once()
dml_mock.assert_called_once_with(
sobject="Contact",
operation=DataOperationType.HARD_DELETE,
api_options={},
context=task,
fields=["Id"],
api=DataApi.SMART,
volume=2,
)
dml_mock.return_value.start.assert_called_once()
dml_mock.return_value.end.assert_called_once()
dml_mock.return_value.load_records.assert_called_once()
dml_mock.return_value.get_results.assert_called_once()
def test_init_mapping_org_has_person_accounts_enabled(self):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
mock_describe_calls()
task = _make_task(
ExtractData,
{"options": {"database_url": "sqlite:///", "mapping": mapping_path}},
)
task.org_config._is_person_accounts_enabled = True
task._init_mapping()
assert "Insert Households" in task.mapping
# Person Accounts should not be added to mapping
self.assert_person_accounts_in_mapping(task.mapping, True)
def test_run__v2__person_accounts_disabled(self, query_op_mock):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v2)
mock_describe_calls()
with TemporaryDirectory() as t:
task = _make_task(
ExtractData,
{
"options": {
"database_url": f"sqlite:///{t}/temp_db", # in memory
"mapping": mapping_path,
}
},
)
task.bulk = mock.Mock()
task.sf = mock.Mock()
task.org_config._is_person_accounts_enabled = False
mock_query_households = MockBulkQueryOperation(
sobject="Account",
api_options={},
context=task,
query="SELECT Id, Name FROM Account",
)
mock_query_contacts = MockBulkQueryOperation(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id, FirstName, LastName, Email, AccountId FROM Contact",
)
mock_query_households.results = [["1", "TestHousehold"]]
mock_query_contacts.results = [
["2", "First", "Last", "*****@*****.**", "1"]
]
query_op_mock.side_effect = [mock_query_households, mock_query_contacts]
task()
with create_engine(task.options["database_url"]).connect() as conn:
household = next(conn.execute("select * from households"))
assert household.name == "TestHousehold"
assert not hasattr(household, "IsPersonAccount")
assert household.record_type == "HH_Account"
contact = next(conn.execute("select * from contacts"))
assert contact.household_id == "1"
assert not hasattr(contact, "IsPersonAccount")
def test_insert_dml_operation__boolean_conversion__fails(self):
mock_describe_calls()
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite:///test.db",
"mapping": "mapping.yml",
}
},
)
task.project_config.project__package__api_version = "48.0"
task._init_task()
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[
{"id": "003000000000001", "success": True},
{"id": "003000000000002", "success": True},
],
status=200,
)
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[{"id": "003000000000003", "success": True}],
status=200,
)
recs = [
["Narvaez", "xyzzy"],
]
dml_op = RestApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
context=task,
api_options={},
fields=["LastName", "IsEmailBounced"],
)
dml_op.start()
with pytest.raises(BulkDataException) as e:
dml_op.load_records(iter(recs))
assert "xyzzy" in str(e.value)
def test_import_results__autopk(self, create_task_fixture):
mock_describe_calls()
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_vanilla)
with temporary_dir() as d:
task = create_task_fixture(
ExtractData,
{"database_url": f"sqlite:///{d}/temp.db", "mapping": mapping_path},
)
extracted_records = {
# ['Name', 'Description', 'BillingStreet', 'BillingCity', 'BillingState', 'BillingPostalCode', 'BillingCountry', 'ParentId', 'ShippingStreet', 'ShippingCity', 'ShippingState', 'ShippingPostalCode', 'ShippingCountry', 'Phone', 'Fax', 'Website', 'NumberOfEmployees', 'AccountNumber', 'Site', 'Type', 'IsPersonAccount']
"Account": [
["001002", "Account1"] + [""] * 20 + [False],
["001003", "Account2"] + [""] * 20 + [False],
],
# Salutation|FirstName|LastName|Email|Phone|MobilePhone|OtherPhone|HomePhone|Title|Birthdate|MailingStreet|MailingCity|MailingState|MailingPostalCode|MailingCountry|AccountId
"Contact": [
["002001", "", "Bob 1", "Barker 2"] + [""] * 4,
["002002", "", "Sam 2", "Smith 2"] + [""] * 4,
], # id|Name|StageName|CloseDate|Amount|AccountId|ContactId|record_type
"Opportunity": [
[
"0003001",
"Dickenson Mobile Generators",
"Qualification",
"2020-07-18",
"15000.0",
"001003",
"002001",
]
],
}
with mock_extract_jobs(task, extracted_records), mock_salesforce_client(
task
):
task()
with create_engine(task.options["database_url"]).connect() as conn:
output_accounts = list(conn.execute("select * from Account"))
assert output_accounts[0][0:2] == (1, "Account1")
assert output_accounts[1][0:2] == (2, "Account2")
assert len(output_accounts) == 2
output_opportunities = list(conn.execute("select * from Opportunity"))
assert output_opportunities[0].AccountId == "2"
assert output_opportunities[0].ContactId == "1"
def test_create_table__already_exists(self):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
db_path = os.path.join(base_path, "testdata.db")
mock_describe_calls()
task = _make_task(
ExtractData,
{
"options": {
"database_url": f"sqlite:///{db_path}",
"mapping": mapping_path,
}
},
)
task.org_config._is_person_accounts_enabled = False
with pytest.raises(BulkDataException):
task()
def test_run__sql(self, query_op_mock):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
mock_describe_calls()
with temporary_dir():
task = _make_task(
ExtractData,
{"options": {"sql_path": "testdata.sql", "mapping": mapping_path}},
)
task.bulk = mock.Mock()
task.sf = mock.Mock()
task.org_config._is_person_accounts_enabled = False
mock_query_households = MockBulkQueryOperation(
sobject="Account",
api_options={},
context=task,
query="SELECT Id FROM Account",
)
mock_query_contacts = MockBulkQueryOperation(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id, FirstName, LastName, Email, AccountId FROM Contact",
)
mock_query_households.results = [["1"]]
mock_query_contacts.results = [
["2", "First☃", "Last", "*****@*****.**", "1"]
]
query_op_mock.side_effect = [mock_query_households, mock_query_contacts]
with mock.patch(
"cumulusci.tasks.bulkdata.extract.create_engine", wraps=create_engine
) as ce_mock:
task()
assert os.path.exists("testdata.sql")
assert ce_mock.mock_calls[0][1][0].endswith(
"temp_db.db"
), ce_mock.mock_calls[0][1][0]
assert ce_mock.mock_calls[0][1][0].startswith(
"sqlite:///"
), ce_mock.mock_calls[0][1][0]
def test_run__where(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(
DeleteData,
{"options": {
"objects": "Contact",
"where": "Id != null"
}})
query_mock.return_value.get_results.return_value = iter(
["001000000000000", "001000000000001"])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
dml_mock.return_value.get_results.return_value = iter([
DataOperationResult("001000000000000", True, None),
DataOperationResult("001000000000001", True, None),
])
dml_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 2, 0)
task()
query_mock.assert_called_once_with(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id FROM Contact WHERE Id != null",
fields=["Id"],
api=DataApi.SMART,
)
query_mock.return_value.query.assert_called_once()
query_mock.return_value.get_results.assert_called_once()
dml_mock.assert_called_once_with(
sobject="Contact",
operation=DataOperationType.DELETE,
api_options={},
context=task,
fields=["Id"],
api=DataApi.SMART,
volume=2,
)
dml_mock.return_value.start.assert_called_once()
dml_mock.return_value.end.assert_called_once()
dml_mock.return_value.load_records.assert_called_once()
dml_mock.return_value.get_results.assert_called_once()
def test_insert_dml_operation__booleans(self):
mock_describe_calls()
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite:///test.db",
"mapping": "mapping.yml",
}
},
)
task.project_config.project__package__api_version = "48.0"
task._init_task()
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[{"id": "003000000000001", "success": True}],
status=200,
)
recs = [["Narvaez", "True"]]
dml_op = RestApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={"batch_size": 2},
context=task,
fields=["LastName", "IsEmailBounced"], # IsEmailBounced is a Boolean field.
)
dml_op.start()
dml_op.load_records(iter(recs))
dml_op.end()
json_body = json.loads(responses.calls[1].request.body)
assert json_body["records"] == [
{
"LastName": "Narvaez",
"IsEmailBounced": True,
"attributes": {"type": "Contact"},
}
]
def test_extract_memory_usage(self, step_mock):
with TemporaryDirectory() as t:
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
mock_describe_calls()
task = _make_task(
ExtractData,
{
"options": {
"database_url": f"sqlite:///{t}/foo.db", # tempdir database
"mapping": mapping_path,
}
},
)
task.bulk = mock.Mock()
task.sf = mock.Mock()
task.org_config._is_person_accounts_enabled = False
mock_query_households = MockScalableBulkQueryOperation(
sobject="Account",
api_options={},
context=task,
query="SELECT Id FROM Account",
)
mock_query_contacts = MockScalableBulkQueryOperation(
sobject="Contact",
api_options={},
context=task,
query="SELECT Id, FirstName, LastName, Email, AccountId FROM Contact",
)
mock_query_households.results = ([str(num)] for num in range(1, 20000))
mock_query_households.result_len = 20000
mock_query_contacts.results = [
["2", "First", "Last", "*****@*****.**", "1"]
]
mock_query_contacts.result_len = 1
step_mock.side_effect = [mock_query_households, mock_query_contacts]
with assert_max_memory_usage(15 * 10 ** 6):
task()
def test_validate_and_inject_mapping_enforces_fls(self):
mock_describe_calls()
mapping = parse_from_yaml(
StringIO(
"Insert Accounts:\n sf_object: Account\n table: Account\n fields:\n - Nonsense__c"
)
)
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
with pytest.raises(BulkDataException):
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.INSERT,
inject_namespaces=False,
drop_missing=False,
)
def test_validate_and_inject_mapping_removes_namespaces(self):
mock_describe_calls()
# Note: History__c is a mock field added to our stored, mock describes (in JSON)
ms = parse_from_yaml(
StringIO(
"""Insert Accounts:
sf_object: Account
table: Account
fields:
- ns__History__c"""
)
)["Insert Accounts"]
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
assert ms.validate_and_inject_namespace(
org_config, "ns", DataOperationType.INSERT, inject_namespaces=True
)
assert list(ms.fields.keys()) == ["History__c"]
def test_run__no_results(self, dml_mock, query_mock):
mock_describe_calls()
task = _make_task(DeleteData, {"options": {"objects": "Contact"}})
query_mock.return_value.get_results.return_value = iter([])
query_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 0, 0)
task()
query_mock.assert_called_once_with(
sobject="Contact",
fields=["Id"],
api_options={},
context=task,
query="SELECT Id FROM Contact",
api=DataApi.SMART,
)
query_mock.return_value.query.assert_called_once()
query_mock.return_value.get_results.assert_not_called()
dml_mock.assert_not_called()
def test_insert_dml_operation__row_failure(self):
mock_describe_calls()
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite:///test.db",
"mapping": "mapping.yml",
}
},
)
task.project_config.project__package__api_version = "48.0"
task._init_task()
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[
{"id": "003000000000001", "success": True},
{"id": "003000000000002", "success": True},
],
status=200,
)
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[
{
"id": "003000000000003",
"success": False,
"errors": [
{
"statusCode": "VALIDATION_ERR",
"message": "Bad data",
"fields": ["FirstName"],
}
],
}
],
status=200,
)
recs = [["Fred", "Narvaez"], [None, "De Vries"], ["Hiroko", "Aito"]]
dml_op = RestApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={"batch_size": 2},
context=task,
fields=["FirstName", "LastName"],
)
dml_op.start()
dml_op.load_records(iter(recs))
dml_op.end()
assert dml_op.job_result == DataOperationJobResult(
DataOperationStatus.ROW_FAILURE, [], 3, 1
)
assert list(dml_op.get_results()) == [
DataOperationResult("003000000000001", True, ""),
DataOperationResult("003000000000002", True, ""),
DataOperationResult(
"003000000000003", False, "VALIDATION_ERR: Bad data (FirstName)"
),
]
def test_insert_dml_operation__boolean_conversion(self):
mock_describe_calls()
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite:///test.db",
"mapping": "mapping.yml",
}
},
)
task.project_config.project__package__api_version = "48.0"
task._init_task()
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[
{"id": "003000000000001", "success": True},
{"id": "003000000000002", "success": True},
],
status=200,
)
responses.add(
responses.POST,
url="https://example.com/services/data/v48.0/composite/sobjects",
json=[{"id": "003000000000003", "success": True}],
status=200,
)
recs = [
["Narvaez", "true"],
["Chalmers", "True"],
["De Vries", "False"],
["Aito", "false"],
["Boone", None],
["June", False],
["Zoom", True],
["Jewel", 0],
["Zule", 1],
["Jane", "0"],
["Zane", "1"],
]
dml_op = RestApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
context=task,
api_options={},
fields=["LastName", "IsEmailBounced"],
)
dml_op.start()
dml_op.load_records(iter(recs))
dml_op.end()
assert json.loads(responses.calls[1].request.body) == {
"allOrNone": False,
"records": [
{
"LastName": "Narvaez",
"IsEmailBounced": True,
"attributes": {"type": "Contact"},
},
{
"LastName": "Chalmers",
"IsEmailBounced": True,
"attributes": {"type": "Contact"},
},
{
"LastName": "De Vries",
"IsEmailBounced": False,
"attributes": {"type": "Contact"},
},
{
"LastName": "Aito",
"IsEmailBounced": False,
"attributes": {"type": "Contact"},
},
{
"LastName": "Boone",
"IsEmailBounced": False,
"attributes": {"type": "Contact"},
},
{
"LastName": "June",
"IsEmailBounced": False,
"attributes": {"type": "Contact"},
},
{
"LastName": "Zoom",
"IsEmailBounced": True,
"attributes": {"type": "Contact"},
},
{
"LastName": "Jewel",
"IsEmailBounced": False,
"attributes": {"type": "Contact"},
},
{
"LastName": "Zule",
"IsEmailBounced": True,
"attributes": {"type": "Contact"},
},
{
"LastName": "Jane",
"IsEmailBounced": False,
"attributes": {"type": "Contact"},
},
{
"LastName": "Zane",
"IsEmailBounced": True,
"attributes": {"type": "Contact"},
},
],
}
