def test_bad_mapping_batch_size(self):
base_path = Path(__file__).parent / "mapping_v2.yml"
with open(base_path, "r") as f:
data = f.read().replace("record_type: HH_Account",
"batch_size: 500")
with pytest.raises(ValidationError):
parse_from_yaml(StringIO(data))
def test_deprecation(self, caplog):
base_path = Path(__file__).parent / "mapping_v2.yml"
caplog.set_level(logging.WARNING)
parse_from_yaml(base_path)
assert "record_type" in caplog.text
with open(base_path) as f:
raw_mapping = safe_load(f)
raw_mapping["Insert Households"]["oid_as_pk"] = True
parse_from_yaml(StringIO(dump(raw_mapping)))
assert "oid_as_pk" in caplog.text
def _init_mapping(self):
"""Load a YAML mapping file."""
mapping_file_path = self.options["mapping"]
if not mapping_file_path:
raise TaskOptionsError("Mapping file path required")
self.mapping = parse_from_yaml(mapping_file_path)
def test_validate_and_inject_mapping_removes_lookups_with_drop_missing(self):
mock_describe_calls()
mapping = parse_from_yaml(
StringIO(
(
"Insert Accounts:\n sf_object: NotAccount\n table: Account\n fields:\n - Nonsense__c\n"
"Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n AccountId:\n table: Account"
)
)
)
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.INSERT,
inject_namespaces=False,
drop_missing=True,
)
assert "Insert Accounts" not in mapping
assert "Insert Contacts" in mapping
assert "AccountId" not in mapping["Insert Contacts"].lookups
def test_validate_and_inject_mapping_throws_exception_required_lookup_dropped(self):
mock_describe_calls()
# This test uses a bit of gimmickry to validate exception behavior on dropping a required lookup.
# Since mapping_parser identifies target objects via the `table` clause rather than the actual schema,
# which makes us resilient to polymorphic lookups, we'll pretend the non-nillable `Id` field is a lookup.
mapping = parse_from_yaml(
StringIO(
(
"Insert Accounts:\n sf_object: NotAccount\n table: Account\n fields:\n - Nonsense__c\n"
"Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n Id:\n table: Account"
)
)
)
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
with pytest.raises(BulkDataException):
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.INSERT,
inject_namespaces=False,
drop_missing=True,
)
def test_validate_and_inject_mapping_queries_is_person_account_field(self):
mock_describe_calls()
mapping = parse_from_yaml(
StringIO(
(
"Insert Accounts:\n sf_object: Account\n table: Account\n fields:\n - Description__c\n"
"Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n AccountId:\n table: Account"
)
)
)
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.QUERY,
inject_namespaces=False,
drop_missing=True,
org_has_person_accounts_enabled=True,
)
assert "Insert Accounts" in mapping
assert "Insert Contacts" in mapping
assert "IsPersonAccount" in mapping["Insert Accounts"]["fields"]
assert "IsPersonAccount" in mapping["Insert Contacts"]["fields"]
def test_validate_and_inject_mapping_works_case_insensitively(self):
mock_describe_calls()
mapping = parse_from_yaml(
StringIO(
(
"Insert Accounts:\n sf_object: account\n table: account\n fields:\n - name\n"
"Insert Contacts:\n sf_object: contact\n table: contact\n fields:\n - fIRSTnAME\n lookups:\n accountid:\n table: account"
)
)
)
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
assert mapping["Insert Accounts"].sf_object != "Account"
assert mapping["Insert Accounts"].sf_object == "account"
assert "name" in mapping["Insert Accounts"].fields
assert "Name" not in mapping["Insert Accounts"].fields
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.INSERT,
inject_namespaces=False,
drop_missing=False,
)
assert mapping["Insert Accounts"].sf_object == "Account"
assert mapping["Insert Accounts"].sf_object != "account"
assert "Name" in mapping["Insert Accounts"].fields
assert "name" not in mapping["Insert Accounts"].fields
def _generate_data(self, db_url, mapping_file_path, num_records, current_batch_num):
"""Generate all of the data"""
if mapping_file_path:
self.mapping = parse_from_yaml(mapping_file_path)
else:
self.mapping = {}
self.logger.info(f"Generating batch {current_batch_num} with {num_records}")
self.generate_data(db_url, num_records, current_batch_num)
def test_after(self):
base_path = Path(__file__).parent / "mapping_after.yml"
result = parse_from_yaml(base_path)
step = result["Insert Accounts"]
lookups = step["lookups"]
assert lookups
assert "after" in lookups["ParentId"]
after_list = {l["after"] for l in lookups.values() if "after" in l}
assert after_list
def test_fields_list_to_dict(self):
base_path = Path(__file__).parent / "mapping_v3.yml"
with open(base_path, "r") as f:
data = f.read()
ms = parse_from_yaml(StringIO(data))
assert ms["Insert Accounts"].fields == {"Name": "Name"}
assert ms["Insert Contacts"].fields == {
"FirstName": "FirstName",
"LastName": "LastName",
"Email": "Email",
}
def test_validate_and_inject_namespace__injection_fields(
self, mock_field, mock_sobject
):
ms = parse_from_yaml(
StringIO(
"""Insert Accounts:
sf_object: Account
table: Account
fields:
- Test__c"""
)
)["Insert Accounts"]
org_config = mock.Mock()
org_config.salesforce_client.describe.return_value = {
"sobjects": [{"name": "Account", "createable": True}]
}
org_config.salesforce_client.Account.describe.return_value = {
"fields": [{"name": "ns__Test__c", "createable": True}]
}
assert ms.validate_and_inject_namespace(
org_config, "ns", DataOperationType.INSERT, inject_namespaces=True
)
ms._validate_sobject.assert_called_once_with(
CaseInsensitiveDict({"Account": {"name": "Account", "createable": True}}),
mock.ANY, # This is a function def
DataOperationType.INSERT,
)
ms._validate_field_dict.assert_has_calls(
[
mock.call(
CaseInsensitiveDict(
{"ns__Test__c": {"name": "ns__Test__c", "createable": True}}
),
ms.fields,
mock.ANY, # local function def
mock.ANY, # local function def
False,
DataOperationType.INSERT,
),
mock.call(
{"ns__Test__c": {"name": "ns__Test__c", "createable": True}},
ms.lookups,
mock.ANY, # local function def
mock.ANY, # local function def
False,
DataOperationType.INSERT,
),
]
)
def test_create_table_modern_id_mapping(self):
mapping_file = os.path.join(os.path.dirname(__file__), "mapping_v2.yml")
content = parse_from_yaml(mapping_file)
account_mapping = content["Insert Contacts"]
with temporary_dir() as d:
tmp_db_path = os.path.join(d, "temp.db")
engine, metadata = create_db_file(tmp_db_path)
t = create_table(account_mapping, metadata)
assert t.name == "contacts"
assert isinstance(t.columns["id"].type, Integer)
assert isinstance(t.columns["first_name"].type, Unicode)
assert isinstance(t.columns["last_name"].type, Unicode)
assert isinstance(t.columns["email"].type, Unicode)
def _init_mapping(self):
"""Load a YAML mapping file."""
mapping_file_path = self.options["mapping"]
if not mapping_file_path:
raise TaskOptionsError("Mapping file path required")
self.mapping = parse_from_yaml(mapping_file_path)
validate_and_inject_mapping(
mapping=self.mapping,
org_config=self.org_config,
namespace=self.project_config.project__package__namespace,
data_operation=DataOperationType.INSERT,
inject_namespaces=self.options["inject_namespaces"],
drop_missing=self.options["drop_missing_schema"],
)
def test_bulk_attributes(self):
mapping = parse_from_yaml(
StringIO(
(
"""Insert Accounts:
sf_object: account
table: account
api: rest
bulk_mode: Serial
batch_size: 50
fields:
- name"""
)
)
)
assert mapping["Insert Accounts"].api == DataApi.REST
assert mapping["Insert Accounts"].bulk_mode == "Serial"
assert mapping["Insert Accounts"].batch_size == 50
def _init_mapping(self):
"""Load a YAML mapping file."""
mapping_file_path = self.options["mapping"]
if not mapping_file_path:
raise TaskOptionsError("Mapping file path required")
self.logger.info(f"Mapping file: {self.options['mapping']}")
self.mapping = parse_from_yaml(mapping_file_path)
validate_and_inject_mapping(
mapping=self.mapping,
org_config=self.org_config,
namespace=self.project_config.project__package__namespace,
data_operation=DataOperationType.QUERY,
inject_namespaces=self.options["inject_namespaces"],
drop_missing=self.options["drop_missing_schema"],
org_has_person_accounts_enabled=self.org_config.
is_person_accounts_enabled,
)
def test_validate_and_inject_mapping_enforces_fls(self):
mock_describe_calls()
mapping = parse_from_yaml(
StringIO(
"Insert Accounts:\n sf_object: Account\n table: Account\n fields:\n - Nonsense__c"
)
)
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
with pytest.raises(BulkDataException):
validate_and_inject_mapping(
mapping=mapping,
org_config=org_config,
namespace=None,
data_operation=DataOperationType.INSERT,
inject_namespaces=False,
drop_missing=False,
)
def test_validate_and_inject_mapping_removes_namespaces(self):
mock_describe_calls()
# Note: History__c is a mock field added to our stored, mock describes (in JSON)
ms = parse_from_yaml(
StringIO(
"""Insert Accounts:
sf_object: Account
table: Account
fields:
- ns__History__c"""
)
)["Insert Accounts"]
org_config = DummyOrgConfig(
{"instance_url": "https://example.com", "access_token": "abc123"}, "test"
)
assert ms.validate_and_inject_namespace(
org_config, "ns", DataOperationType.INSERT, inject_namespaces=True
)
assert list(ms.fields.keys()) == ["History__c"]
def test_bad_mapping_id_mode(self):
base_path = Path(__file__).parent / "mapping_v2.yml"
with open(base_path, "r") as f:
data = f.read().replace("Name: name", "Id: sf_id")
with pytest.raises(ValidationError):
parse_from_yaml(StringIO(data))
def _init_mapping(self):
"""Load a YAML mapping file."""
with open(self.options["mapping"], "r") as f:
# yaml.safe_load should also work here for now.
self.mapping = parse_from_yaml(f)
def test_fields_default_null(self):
base_path = Path(__file__).parent / "mapping_v3.yml"
with open(base_path, "r") as f:
data = f.read()
ms = parse_from_yaml(StringIO(data))
assert ms["Insert Other Junction Objects"].fields == {}
def test_load_from_bytes_stream(self):
base_path = Path(__file__).parent / "mapping_v2.yml"
with open(base_path, "rb") as f:
assert parse_from_yaml(f)
def test_simple_parse(self):
base_path = Path(__file__).parent / "mapping_v2.yml"
assert parse_from_yaml(base_path)
def test_default_table_to_sobject_name(self):
base_path = Path(__file__).parent / "mapping_v3.yml"
with open(base_path, "r") as f:
data = f.read()
ms = parse_from_yaml(StringIO(data))
assert ms["Insert Accounts"].table == "Account"
def test_deprecation(self, caplog):
base_path = Path(__file__).parent / "mapping_v2.yml"
caplog.set_level(logging.WARNING)
parse_from_yaml(base_path)
assert "record_type" in caplog.text
def test_validate_and_inject_namespace__fls_lookups_update_failure(
self, mock_field, mock_sobject
):
ms = parse_from_yaml(
StringIO(
"""Insert Accounts:
sf_object: Account
table: Account
fields:
- Name
lookups:
Lookup__c:
table: Stuff
after: Insert Stuff"""
)
)["Insert Accounts"]
org_config = mock.Mock()
org_config.salesforce_client.describe.return_value = {
"sobjects": [{"name": "Account", "createable": True}]
}
org_config.salesforce_client.Account.describe.return_value = {
"fields": [
{"name": "Name", "createable": True},
{"name": "Lookup__c", "updateable": False, "createable": True},
]
}
assert not ms.validate_and_inject_namespace(
org_config, "ns", DataOperationType.INSERT
)
ms._validate_sobject.assert_called_once_with(
{"Account": {"name": "Account", "createable": True}},
None,
DataOperationType.INSERT,
)
ms._validate_field_dict.assert_has_calls(
[
mock.call(
{
"Name": {"name": "Name", "createable": True},
"Lookup__c": {
"name": "Lookup__c",
"updateable": False,
"createable": True,
},
},
{"Name": "Name"},
None,
None,
False,
DataOperationType.INSERT,
),
mock.call(
{
"Name": {"name": "Name", "createable": True},
"Lookup__c": {
"name": "Lookup__c",
"updateable": False,
"createable": True,
},
},
ms.lookups,
None,
None,
False,
DataOperationType.INSERT,
),
]
)
def test_bad_mapping_syntax(self):
base_path = Path(__file__).parent / "mapping_v2.yml"
with open(base_path, "r") as f:
data = f.read().replace(":", ": abcd")
with pytest.raises(YAMLError):
parse_from_yaml(StringIO(data))
def _read_mappings(self, mapping_file_path):
if not mapping_file_path:
raise TaskOptionsError("Mapping file path required")
return parse_from_yaml(mapping_file_path)
def test_bad_mapping_grammar(self):
base_path = Path(__file__).parent / "mapping_v2.yml"
with open(base_path, "r") as f:
data = f.read().replace("record_type", "xyzzy")
with pytest.raises(ValidationError):
parse_from_yaml(StringIO(data))
def test_bad_mapping_oid_as_pk(self):
base_path = Path(__file__).parent / "mapping_v1.yml"
with open(base_path, "r") as f:
data = f.read().replace("api: bulk", "oid_as_pk: True`")
with pytest.raises(ValidationError):
parse_from_yaml(StringIO(data))