def test_bad_mapping_batch_size(self): base_path = Path(__file__).parent / "mapping_v2.yml" with open(base_path, "r") as f: data = f.read().replace("record_type: HH_Account", "batch_size: 500") with pytest.raises(ValidationError): parse_from_yaml(StringIO(data))
def test_deprecation(self, caplog): base_path = Path(__file__).parent / "mapping_v2.yml" caplog.set_level(logging.WARNING) parse_from_yaml(base_path) assert "record_type" in caplog.text with open(base_path) as f: raw_mapping = safe_load(f) raw_mapping["Insert Households"]["oid_as_pk"] = True parse_from_yaml(StringIO(dump(raw_mapping))) assert "oid_as_pk" in caplog.text
def _init_mapping(self): """Load a YAML mapping file.""" mapping_file_path = self.options["mapping"] if not mapping_file_path: raise TaskOptionsError("Mapping file path required") self.mapping = parse_from_yaml(mapping_file_path)
def test_validate_and_inject_mapping_removes_lookups_with_drop_missing(self): mock_describe_calls() mapping = parse_from_yaml( StringIO( ( "Insert Accounts:\n sf_object: NotAccount\n table: Account\n fields:\n - Nonsense__c\n" "Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n AccountId:\n table: Account" ) ) ) org_config = DummyOrgConfig( {"instance_url": "https://example.com", "access_token": "abc123"}, "test" ) validate_and_inject_mapping( mapping=mapping, org_config=org_config, namespace=None, data_operation=DataOperationType.INSERT, inject_namespaces=False, drop_missing=True, ) assert "Insert Accounts" not in mapping assert "Insert Contacts" in mapping assert "AccountId" not in mapping["Insert Contacts"].lookups
def test_validate_and_inject_mapping_throws_exception_required_lookup_dropped(self): mock_describe_calls() # This test uses a bit of gimmickry to validate exception behavior on dropping a required lookup. # Since mapping_parser identifies target objects via the `table` clause rather than the actual schema, # which makes us resilient to polymorphic lookups, we'll pretend the non-nillable `Id` field is a lookup. mapping = parse_from_yaml( StringIO( ( "Insert Accounts:\n sf_object: NotAccount\n table: Account\n fields:\n - Nonsense__c\n" "Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n Id:\n table: Account" ) ) ) org_config = DummyOrgConfig( {"instance_url": "https://example.com", "access_token": "abc123"}, "test" ) with pytest.raises(BulkDataException): validate_and_inject_mapping( mapping=mapping, org_config=org_config, namespace=None, data_operation=DataOperationType.INSERT, inject_namespaces=False, drop_missing=True, )
def test_validate_and_inject_mapping_queries_is_person_account_field(self): mock_describe_calls() mapping = parse_from_yaml( StringIO( ( "Insert Accounts:\n sf_object: Account\n table: Account\n fields:\n - Description__c\n" "Insert Contacts:\n sf_object: Contact\n table: Contact\n lookups:\n AccountId:\n table: Account" ) ) ) org_config = DummyOrgConfig( {"instance_url": "https://example.com", "access_token": "abc123"}, "test" ) validate_and_inject_mapping( mapping=mapping, org_config=org_config, namespace=None, data_operation=DataOperationType.QUERY, inject_namespaces=False, drop_missing=True, org_has_person_accounts_enabled=True, ) assert "Insert Accounts" in mapping assert "Insert Contacts" in mapping assert "IsPersonAccount" in mapping["Insert Accounts"]["fields"] assert "IsPersonAccount" in mapping["Insert Contacts"]["fields"]
def test_validate_and_inject_mapping_works_case_insensitively(self): mock_describe_calls() mapping = parse_from_yaml( StringIO( ( "Insert Accounts:\n sf_object: account\n table: account\n fields:\n - name\n" "Insert Contacts:\n sf_object: contact\n table: contact\n fields:\n - fIRSTnAME\n lookups:\n accountid:\n table: account" ) ) ) org_config = DummyOrgConfig( {"instance_url": "https://example.com", "access_token": "abc123"}, "test" ) assert mapping["Insert Accounts"].sf_object != "Account" assert mapping["Insert Accounts"].sf_object == "account" assert "name" in mapping["Insert Accounts"].fields assert "Name" not in mapping["Insert Accounts"].fields validate_and_inject_mapping( mapping=mapping, org_config=org_config, namespace=None, data_operation=DataOperationType.INSERT, inject_namespaces=False, drop_missing=False, ) assert mapping["Insert Accounts"].sf_object == "Account" assert mapping["Insert Accounts"].sf_object != "account" assert "Name" in mapping["Insert Accounts"].fields assert "name" not in mapping["Insert Accounts"].fields
def _generate_data(self, db_url, mapping_file_path, num_records, current_batch_num): """Generate all of the data""" if mapping_file_path: self.mapping = parse_from_yaml(mapping_file_path) else: self.mapping = {} self.logger.info(f"Generating batch {current_batch_num} with {num_records}") self.generate_data(db_url, num_records, current_batch_num)
def test_after(self): base_path = Path(__file__).parent / "mapping_after.yml" result = parse_from_yaml(base_path) step = result["Insert Accounts"] lookups = step["lookups"] assert lookups assert "after" in lookups["ParentId"] after_list = {l["after"] for l in lookups.values() if "after" in l} assert after_list
def test_fields_list_to_dict(self): base_path = Path(__file__).parent / "mapping_v3.yml" with open(base_path, "r") as f: data = f.read() ms = parse_from_yaml(StringIO(data)) assert ms["Insert Accounts"].fields == {"Name": "Name"} assert ms["Insert Contacts"].fields == { "FirstName": "FirstName", "LastName": "LastName", "Email": "Email", }
def test_validate_and_inject_namespace__injection_fields( self, mock_field, mock_sobject ): ms = parse_from_yaml( StringIO( """Insert Accounts: sf_object: Account table: Account fields: - Test__c""" ) )["Insert Accounts"] org_config = mock.Mock() org_config.salesforce_client.describe.return_value = { "sobjects": [{"name": "Account", "createable": True}] } org_config.salesforce_client.Account.describe.return_value = { "fields": [{"name": "ns__Test__c", "createable": True}] } assert ms.validate_and_inject_namespace( org_config, "ns", DataOperationType.INSERT, inject_namespaces=True ) ms._validate_sobject.assert_called_once_with( CaseInsensitiveDict({"Account": {"name": "Account", "createable": True}}), mock.ANY, # This is a function def DataOperationType.INSERT, ) ms._validate_field_dict.assert_has_calls( [ mock.call( CaseInsensitiveDict( {"ns__Test__c": {"name": "ns__Test__c", "createable": True}} ), ms.fields, mock.ANY, # local function def mock.ANY, # local function def False, DataOperationType.INSERT, ), mock.call( {"ns__Test__c": {"name": "ns__Test__c", "createable": True}}, ms.lookups, mock.ANY, # local function def mock.ANY, # local function def False, DataOperationType.INSERT, ), ] )
def test_create_table_modern_id_mapping(self): mapping_file = os.path.join(os.path.dirname(__file__), "mapping_v2.yml") content = parse_from_yaml(mapping_file) account_mapping = content["Insert Contacts"] with temporary_dir() as d: tmp_db_path = os.path.join(d, "temp.db") engine, metadata = create_db_file(tmp_db_path) t = create_table(account_mapping, metadata) assert t.name == "contacts" assert isinstance(t.columns["id"].type, Integer) assert isinstance(t.columns["first_name"].type, Unicode) assert isinstance(t.columns["last_name"].type, Unicode) assert isinstance(t.columns["email"].type, Unicode)
def _init_mapping(self): """Load a YAML mapping file.""" mapping_file_path = self.options["mapping"] if not mapping_file_path: raise TaskOptionsError("Mapping file path required") self.mapping = parse_from_yaml(mapping_file_path) validate_and_inject_mapping( mapping=self.mapping, org_config=self.org_config, namespace=self.project_config.project__package__namespace, data_operation=DataOperationType.INSERT, inject_namespaces=self.options["inject_namespaces"], drop_missing=self.options["drop_missing_schema"], )
def test_bulk_attributes(self): mapping = parse_from_yaml( StringIO( ( """Insert Accounts: sf_object: account table: account api: rest bulk_mode: Serial batch_size: 50 fields: - name""" ) ) ) assert mapping["Insert Accounts"].api == DataApi.REST assert mapping["Insert Accounts"].bulk_mode == "Serial" assert mapping["Insert Accounts"].batch_size == 50
def _init_mapping(self): """Load a YAML mapping file.""" mapping_file_path = self.options["mapping"] if not mapping_file_path: raise TaskOptionsError("Mapping file path required") self.logger.info(f"Mapping file: {self.options['mapping']}") self.mapping = parse_from_yaml(mapping_file_path) validate_and_inject_mapping( mapping=self.mapping, org_config=self.org_config, namespace=self.project_config.project__package__namespace, data_operation=DataOperationType.QUERY, inject_namespaces=self.options["inject_namespaces"], drop_missing=self.options["drop_missing_schema"], org_has_person_accounts_enabled=self.org_config. is_person_accounts_enabled, )
def test_validate_and_inject_mapping_enforces_fls(self): mock_describe_calls() mapping = parse_from_yaml( StringIO( "Insert Accounts:\n sf_object: Account\n table: Account\n fields:\n - Nonsense__c" ) ) org_config = DummyOrgConfig( {"instance_url": "https://example.com", "access_token": "abc123"}, "test" ) with pytest.raises(BulkDataException): validate_and_inject_mapping( mapping=mapping, org_config=org_config, namespace=None, data_operation=DataOperationType.INSERT, inject_namespaces=False, drop_missing=False, )
def test_validate_and_inject_mapping_removes_namespaces(self): mock_describe_calls() # Note: History__c is a mock field added to our stored, mock describes (in JSON) ms = parse_from_yaml( StringIO( """Insert Accounts: sf_object: Account table: Account fields: - ns__History__c""" ) )["Insert Accounts"] org_config = DummyOrgConfig( {"instance_url": "https://example.com", "access_token": "abc123"}, "test" ) assert ms.validate_and_inject_namespace( org_config, "ns", DataOperationType.INSERT, inject_namespaces=True ) assert list(ms.fields.keys()) == ["History__c"]
def test_bad_mapping_id_mode(self): base_path = Path(__file__).parent / "mapping_v2.yml" with open(base_path, "r") as f: data = f.read().replace("Name: name", "Id: sf_id") with pytest.raises(ValidationError): parse_from_yaml(StringIO(data))
def _init_mapping(self): """Load a YAML mapping file.""" with open(self.options["mapping"], "r") as f: # yaml.safe_load should also work here for now. self.mapping = parse_from_yaml(f)
def test_fields_default_null(self): base_path = Path(__file__).parent / "mapping_v3.yml" with open(base_path, "r") as f: data = f.read() ms = parse_from_yaml(StringIO(data)) assert ms["Insert Other Junction Objects"].fields == {}
def test_load_from_bytes_stream(self): base_path = Path(__file__).parent / "mapping_v2.yml" with open(base_path, "rb") as f: assert parse_from_yaml(f)
def test_simple_parse(self): base_path = Path(__file__).parent / "mapping_v2.yml" assert parse_from_yaml(base_path)
def test_default_table_to_sobject_name(self): base_path = Path(__file__).parent / "mapping_v3.yml" with open(base_path, "r") as f: data = f.read() ms = parse_from_yaml(StringIO(data)) assert ms["Insert Accounts"].table == "Account"
def test_deprecation(self, caplog): base_path = Path(__file__).parent / "mapping_v2.yml" caplog.set_level(logging.WARNING) parse_from_yaml(base_path) assert "record_type" in caplog.text
def test_validate_and_inject_namespace__fls_lookups_update_failure( self, mock_field, mock_sobject ): ms = parse_from_yaml( StringIO( """Insert Accounts: sf_object: Account table: Account fields: - Name lookups: Lookup__c: table: Stuff after: Insert Stuff""" ) )["Insert Accounts"] org_config = mock.Mock() org_config.salesforce_client.describe.return_value = { "sobjects": [{"name": "Account", "createable": True}] } org_config.salesforce_client.Account.describe.return_value = { "fields": [ {"name": "Name", "createable": True}, {"name": "Lookup__c", "updateable": False, "createable": True}, ] } assert not ms.validate_and_inject_namespace( org_config, "ns", DataOperationType.INSERT ) ms._validate_sobject.assert_called_once_with( {"Account": {"name": "Account", "createable": True}}, None, DataOperationType.INSERT, ) ms._validate_field_dict.assert_has_calls( [ mock.call( { "Name": {"name": "Name", "createable": True}, "Lookup__c": { "name": "Lookup__c", "updateable": False, "createable": True, }, }, {"Name": "Name"}, None, None, False, DataOperationType.INSERT, ), mock.call( { "Name": {"name": "Name", "createable": True}, "Lookup__c": { "name": "Lookup__c", "updateable": False, "createable": True, }, }, ms.lookups, None, None, False, DataOperationType.INSERT, ), ] )
def test_bad_mapping_syntax(self): base_path = Path(__file__).parent / "mapping_v2.yml" with open(base_path, "r") as f: data = f.read().replace(":", ": abcd") with pytest.raises(YAMLError): parse_from_yaml(StringIO(data))
def _read_mappings(self, mapping_file_path): if not mapping_file_path: raise TaskOptionsError("Mapping file path required") return parse_from_yaml(mapping_file_path)
def test_bad_mapping_grammar(self): base_path = Path(__file__).parent / "mapping_v2.yml" with open(base_path, "r") as f: data = f.read().replace("record_type", "xyzzy") with pytest.raises(ValidationError): parse_from_yaml(StringIO(data))
def test_bad_mapping_oid_as_pk(self): base_path = Path(__file__).parent / "mapping_v1.yml" with open(base_path, "r") as f: data = f.read().replace("api: bulk", "oid_as_pk: True`") with pytest.raises(ValidationError): parse_from_yaml(StringIO(data))