def test_run__job_error_delete(self, dml_mock, query_mock): mock_describe_calls() task = _make_task(DeleteData, {"options": {"objects": "Contact"}}) query_mock.return_value.get_results.return_value = iter( ["001000000000000", "001000000000001"]) query_mock.return_value.job_result = DataOperationJobResult( DataOperationStatus.SUCCESS, [], 2, 0) dml_mock.return_value.get_results.return_value = iter([ DataOperationResult("001000000000000", True, None), DataOperationResult("001000000000001", False, None), ]) with self.assertRaises(BulkDataException): task()
def test_run__query_fails(self, dml_mock, query_mock): mock_describe_calls() task = _make_task( DeleteData, {"options": { "objects": "Contact", "where": "Id != null" }}) query_mock.return_value.get_results.return_value = iter( ["001000000000000", "001000000000001"]) query_mock.return_value.job_result = DataOperationJobResult( DataOperationStatus.JOB_FAILURE, [], 0, 0) with self.assertRaises(BulkDataException): task()
def test_run__person_accounts_enabled(self, step_mock): base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, self.mapping_file_v1) mock_describe_calls() task = _make_task( ExtractData, { "options": { "database_url": "sqlite://", # in memory "mapping": mapping_path, } }, ) task.bulk = mock.Mock() task.sf = mock.Mock() task.org_config._is_person_accounts_enabled = True mock_query_households = MockBulkQueryOperation( sobject="Account", api_options={}, context=task, query="SELECT Id, IsPersonAccount FROM Account", ) mock_query_contacts = MockBulkQueryOperation( sobject="Contact", api_options={}, context=task, query= "SELECT Id, FirstName, LastName, Email, IsPersonAccount, AccountId FROM Contact", ) mock_query_households.results = [["1", "false"]] mock_query_contacts.results = [[ "2", "First", "Last", "*****@*****.**", "true", "1" ]] step_mock.side_effect = [mock_query_households, mock_query_contacts] task() household = task.session.query(task.models["households"]).one() self.assertEqual("1", household.sf_id) self.assertEqual("false", household.IsPersonAccount) self.assertEqual("HH_Account", household.record_type) contact = task.session.query(task.models["contacts"]).one() self.assertEqual("2", contact.sf_id) self.assertEqual("true", contact.IsPersonAccount) self.assertEqual("1", contact.household_id)
def test_run__ignore_error(self, dml_mock, query_mock): mock_describe_calls() task = _make_task( DeleteData, { "options": { "objects": "Contact", "ignore_row_errors": "true", "hardDelete": "true", } }, ) query_mock.return_value.get_results.return_value = iter( ["001000000000000", "001000000000001"]) query_mock.return_value.job_result = DataOperationJobResult( DataOperationStatus.SUCCESS, [], 2, 0) dml_mock.return_value.get_results.return_value = iter([ DataOperationResult("001000000000000", True, None), DataOperationResult("001000000000001", False, None), ]) dml_mock.return_value.job_result = DataOperationJobResult( DataOperationStatus.SUCCESS, [], 2, 0) with mock.patch.object(task.logger, "warning") as warning: task() assert len(warning.mock_calls) == 1 query_mock.assert_called_once_with( sobject="Contact", api_options={}, context=task, query="SELECT Id FROM Contact", api=DataApi.SMART, fields=["Id"], ) query_mock.return_value.query.assert_called_once() query_mock.return_value.get_results.assert_called_once() dml_mock.assert_called_once_with( sobject="Contact", operation=DataOperationType.HARD_DELETE, api_options={}, context=task, fields=["Id"], api=DataApi.SMART, volume=2, ) dml_mock.return_value.start.assert_called_once() dml_mock.return_value.end.assert_called_once() dml_mock.return_value.load_records.assert_called_once() dml_mock.return_value.get_results.assert_called_once()
def test_init_mapping_org_has_person_accounts_enabled(self): base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, self.mapping_file_v1) mock_describe_calls() task = _make_task( ExtractData, {"options": {"database_url": "sqlite:///", "mapping": mapping_path}}, ) task.org_config._is_person_accounts_enabled = True task._init_mapping() assert "Insert Households" in task.mapping # Person Accounts should not be added to mapping self.assert_person_accounts_in_mapping(task.mapping, True)
def test_run__v2__person_accounts_disabled(self, step_mock): base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, self.mapping_file_v2) mock_describe_calls() task = _make_task( ExtractData, { "options": { "database_url": "sqlite://", # in memory "mapping": mapping_path, } }, ) task.bulk = mock.Mock() task.sf = mock.Mock() task.org_config._is_person_accounts_enabled = False mock_query_households = MockBulkQueryOperation( sobject="Account", api_options={}, context=task, query="SELECT Id, Name FROM Account", ) mock_query_contacts = MockBulkQueryOperation( sobject="Contact", api_options={}, context=task, query= "SELECT Id, FirstName, LastName, Email, AccountId FROM Contact", ) mock_query_households.results = [["1", "TestHousehold"]] mock_query_contacts.results = [[ "2", "First", "Last", "*****@*****.**", "1" ]] step_mock.side_effect = [mock_query_households, mock_query_contacts] task() household = task.session.query(task.models["households"]).one() assert household.name == "TestHousehold" assert not hasattr(household, "IsPersonAccount") assert household.record_type == "HH_Account" contact = task.session.query(task.models["contacts"]).one() assert contact.household_id == "1" assert not hasattr(contact, "IsPersonAccount")
def test_insert_dml_operation__booleans(self): mock_describe_calls() task = _make_task( LoadData, { "options": { "database_url": "sqlite:///test.db", "mapping": "mapping.yml", } }, ) task.project_config.project__package__api_version = "48.0" task._init_task() responses.add( responses.POST, url="https://example.com/services/data/v48.0/composite/sobjects", json=[{ "id": "003000000000001", "success": True }], status=200, ) recs = [["Narvaez", "True"]] dml_op = RestApiDmlOperation( sobject="Contact", operation=DataOperationType.INSERT, api_options={"batch_size": 2}, context=task, fields=["LastName", "IsEmailBounced"], # IsEmailBounced is a Boolean field. ) dml_op.start() dml_op.load_records(iter(recs)) dml_op.end() json_body = json.loads(responses.calls[1].request.body) assert json_body["records"] == [{ "LastName": "Narvaez", "IsEmailBounced": True, "attributes": { "type": "Contact" }, }]
def test_create_table__already_exists(self): base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, self.mapping_file_v1) db_path = os.path.join(base_path, "testdata.db") mock_describe_calls() task = _make_task( ExtractData, { "options": { "database_url": f"sqlite:///{db_path}", "mapping": mapping_path, } }, ) task.org_config._is_person_accounts_enabled = False with self.assertRaises(BulkDataException): task()
def test_run__no_results(self, dml_mock, query_mock): mock_describe_calls() task = _make_task(DeleteData, {"options": {"objects": "Contact"}}) query_mock.return_value.get_results.return_value = iter([]) query_mock.return_value.job_result = DataOperationJobResult( DataOperationStatus.SUCCESS, [], 0, 0) task() query_mock.assert_called_once_with( sobject="Contact", api_options={}, context=task, query="SELECT Id FROM Contact", ) query_mock.return_value.query.assert_called_once() query_mock.return_value.get_results.assert_not_called() dml_mock.assert_not_called()
def test_run__where(self, dml_mock, query_mock): mock_describe_calls() task = _make_task( DeleteData, {"options": { "objects": "Contact", "where": "Id != null" }}) query_mock.return_value.get_results.return_value = iter( ["001000000000000", "001000000000001"]) query_mock.return_value.job_result = DataOperationJobResult( DataOperationStatus.SUCCESS, [], 2, 0) dml_mock.return_value.get_results.return_value = iter([ DataOperationResult("001000000000000", True, None), DataOperationResult("001000000000001", True, None), ]) dml_mock.return_value.job_result = DataOperationJobResult( DataOperationStatus.SUCCESS, [], 2, 0) task() query_mock.assert_called_once_with( sobject="Contact", api_options={}, context=task, query="SELECT Id FROM Contact WHERE Id != null", fields=["Id"], api=DataApi.SMART, ) query_mock.return_value.query.assert_called_once() query_mock.return_value.get_results.assert_called_once() dml_mock.assert_called_once_with( sobject="Contact", operation=DataOperationType.DELETE, api_options={}, context=task, fields=["Id"], api=DataApi.SMART, volume=2, ) dml_mock.return_value.start.assert_called_once() dml_mock.return_value.end.assert_called_once() dml_mock.return_value.load_records.assert_called_once() dml_mock.return_value.get_results.assert_called_once()
def test_run__sql(self, step_mock): base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, self.mapping_file_v1) mock_describe_calls() with temporary_dir(): task = _make_task( ExtractData, { "options": { "sql_path": "testdata.sql", "mapping": mapping_path } }, ) task.bulk = mock.Mock() task.sf = mock.Mock() task.org_config._is_person_accounts_enabled = False mock_query_households = MockBulkQueryOperation( sobject="Account", api_options={}, context=task, query="SELECT Id FROM Account", ) mock_query_contacts = MockBulkQueryOperation( sobject="Contact", api_options={}, context=task, query= "SELECT Id, FirstName, LastName, Email, AccountId FROM Contact", ) mock_query_households.results = [["1"]] mock_query_contacts.results = [[ "2", "First", "Last", "*****@*****.**", "1" ]] step_mock.side_effect = [ mock_query_households, mock_query_contacts ] task() assert os.path.exists("testdata.sql")
def test_validate_and_inject_mapping_injects_namespaces(self): mock_describe_calls() # Note: ns__Description__c is a mock field added to our stored, mock describes (in JSON) ms = parse_from_yaml( StringIO("""Insert Accounts: sf_object: Account table: Account fields: - Description__c"""))["Insert Accounts"] org_config = DummyOrgConfig( { "instance_url": "https://example.com", "access_token": "abc123" }, "test") assert ms.validate_and_inject_namespace(org_config, "ns", DataOperationType.INSERT, inject_namespaces=True) assert list(ms.fields.keys()) == ["ns__Description__c"]
def test_validate_and_inject_mapping_enforces_fls(self): mock_describe_calls() mapping = parse_from_yaml( StringIO( "Insert Accounts:\n sf_object: Account\n table: Account\n fields:\n - Nonsense__c" )) org_config = DummyOrgConfig( { "instance_url": "https://example.com", "access_token": "abc123" }, "test") with pytest.raises(BulkDataException): validate_and_inject_mapping( mapping=mapping, org_config=org_config, namespace=None, data_operation=DataOperationType.INSERT, inject_namespaces=False, drop_missing=False, )
def test_validate_and_inject_mapping_removes_steps_with_drop_missing(self): mock_describe_calls() mapping = parse_from_yaml( StringIO( "Insert Accounts:\n sf_object: NotAccount\n table: Account\n fields:\n - Nonsense__c" )) org_config = DummyOrgConfig( { "instance_url": "https://example.com", "access_token": "abc123" }, "test") validate_and_inject_mapping( mapping=mapping, org_config=org_config, namespace=None, data_operation=DataOperationType.INSERT, inject_namespaces=False, drop_missing=True, ) assert "Insert Accounts" not in mapping
def test_insert_dml_operation__delete(self): mock_describe_calls() task = _make_task( LoadData, { "options": { "database_url": "sqlite:///test.db", "mapping": "mapping.yml", } }, ) task.project_config.project__package__api_version = "48.0" task._init_task() responses.add( responses.DELETE, url= "https://example.com/services/data/v48.0/composite/sobjects?ids=003000000000001,003000000000002", json=[ { "id": "003000000000001", "success": True }, { "id": "003000000000002", "success": True }, ], status=200, ) responses.add( responses.DELETE, url= "https://example.com/services/data/v48.0/composite/sobjects?ids=003000000000003", json=[{ "id": "003000000000003", "success": True }], status=200, ) recs = [["003000000000001"], ["003000000000002"], ["003000000000003"]] dml_op = RestApiDmlOperation( sobject="Contact", operation=DataOperationType.DELETE, api_options={"batch_size": 2}, context=task, fields=["Id"], ) dml_op.start() dml_op.load_records(iter(recs)) dml_op.end() assert dml_op.job_result == DataOperationJobResult( DataOperationStatus.SUCCESS, [], 3, 0) assert list(dml_op.get_results()) == [ DataOperationResult("003000000000001", True, ""), DataOperationResult("003000000000002", True, ""), DataOperationResult("003000000000003", True, ""), ]
def test_insert_dml_operation__row_failure(self): mock_describe_calls() task = _make_task( LoadData, { "options": { "database_url": "sqlite:///test.db", "mapping": "mapping.yml", } }, ) task.project_config.project__package__api_version = "48.0" task._init_task() responses.add( responses.POST, url="https://example.com/services/data/v48.0/composite/sobjects", json=[ { "id": "003000000000001", "success": True }, { "id": "003000000000002", "success": True }, ], status=200, ) responses.add( responses.POST, url="https://example.com/services/data/v48.0/composite/sobjects", json=[{ "id": "003000000000003", "success": False, "errors": [{ "statusCode": "VALIDATION_ERR", "message": "Bad data", "fields": ["FirstName"], }], }], status=200, ) recs = [["Fred", "Narvaez"], [None, "De Vries"], ["Hiroko", "Aito"]] dml_op = RestApiDmlOperation( sobject="Contact", operation=DataOperationType.INSERT, api_options={"batch_size": 2}, context=task, fields=["FirstName", "LastName"], ) dml_op.start() dml_op.load_records(iter(recs)) dml_op.end() assert dml_op.job_result == DataOperationJobResult( DataOperationStatus.ROW_FAILURE, [], 3, 1) assert list(dml_op.get_results()) == [ DataOperationResult("003000000000001", True, ""), DataOperationResult("003000000000002", True, ""), DataOperationResult("003000000000003", False, "VALIDATION_ERR: Bad data (FirstName)"), ]