def test_loads_and_extracts_high_data_volume(self): # This is a single unit test rather than multiple to save on execution time. records = [] for i in range(100000): records.append({ "Id": "00Q000000{:06d}".format(i), "Company": "[not provided]", "LastName": "Lead {:06d}".format(i), }) op = amaxa.LoadOperation(Connection(self.connection, "52.0")) op.file_store = MockFileStore() op.file_store.records["Lead"] = records op.add_step(amaxa.LoadStep("Lead", set(["LastName", "Company"]))) op.initialize() op.execute() self.assertEqual( 100000, self.connection.query("SELECT count() FROM Lead").get("totalSize")) oc = amaxa.ExtractOperation(Connection(self.connection, "52.0")) oc.file_store = MockFileStore() extraction = amaxa.ExtractionStep("Lead", amaxa.ExtractionScope.ALL_RECORDS, ["Id", "LastName"]) oc.add_step(extraction) extraction.initialize() extraction.execute() self.assertEqual(100000, len(oc.get_extracted_ids("Lead")))
def test_bulk_api_query(self): # FIXME: test wait sf = Mock() sf.bulk_url = "https://salesforce.com" conn = Connection(sf) conn._bulk = Mock() retval = [{"Id": "001000000000001"}, {"Id": "001000000000002"}] conn._bulk.is_batch_done = Mock(side_effect=[False, True]) conn._bulk.create_query_job = Mock(return_value="075000000000000AAA") conn._bulk.get_all_results_for_query_batch = Mock( return_value=[IteratorBytesIO([json.dumps(retval).encode("utf-8")])] ) results = list(conn.bulk_api_query("Account", "SELECT Id FROM Account", [], 5)) conn._bulk.query.assert_called_once_with( "075000000000000AAA", "SELECT Id FROM Account" ) self.assertEqual( conn._bulk.is_batch_done.call_args_list, [call(conn._bulk.query.return_value), call(conn._bulk.query.return_value)], ) conn._bulk.get_all_results_for_query_batch.assert_called_once_with( conn._bulk.query.return_value ) self.assertEqual(retval, results)
def test_query_extracts_self_lookup_hierarchy(self): expected_names = { "Caprica Cosmetics", "Gemenon Gastronomy", "Aerilon Agrinomics", } oc = amaxa.ExtractOperation(Connection(self.connection)) oc.file_store = MockFileStore() rec = self.connection.query( "SELECT Id FROM Account WHERE Name = 'Caprica Cosmetics'" ) oc.add_dependency("Account", rec.get("records")[0]["Id"]) extraction = amaxa.ExtractionStep( "Account", amaxa.ExtractionScope.SELECTED_RECORDS, ["Id", "Name", "ParentId"], ) oc.add_step(extraction) extraction.initialize() extraction.execute() self.assertEqual(3, len(oc.get_extracted_ids("Account"))) for c in oc.file_store.get_csv( "Account", amaxa.FileType.OUTPUT ).writerow.call_args_list: self.assertIn(c[0][0]["Name"], expected_names) expected_names.remove(c[0][0]["Name"]) self.assertEqual(0, len(expected_names))
def test_caches_field_maps(self): sf = Mock() sf.bulk_url = "https://salesforce.com" conn = Connection(sf, "52.0") sf.Account.describe.return_value = { "fields": [{ "name": "Name" }, { "name": "Id" }] } retval = conn.get_sobject_field_map("Account") self.assertEqual({ "Name": { "name": "Name" }, "Id": { "name": "Id" } }, retval) sf.Account.describe.assert_called_once_with() sf.Account.describe.reset_mock() retval = conn.get_sobject_field_map("Account") self.assertEqual({ "Name": { "name": "Name" }, "Id": { "name": "Id" } }, retval) sf.Account.describe.assert_not_called()
def test_extracts_polymorphic_lookups(self): oc = amaxa.ExtractOperation(Connection(self.connection)) oc.file_store = MockFileStore() rec = self.connection.query( "SELECT Id FROM Account WHERE Name = 'Caprica Cosmetics'" ) oc.add_dependency("Account", rec.get("records")[0]["Id"]) oc.add_step( amaxa.ExtractionStep( "Account", amaxa.ExtractionScope.SELECTED_RECORDS, ["Id", "Name", "OwnerId"], ) ) oc.add_step( amaxa.ExtractionStep( "User", amaxa.ExtractionScope.DESCENDENTS, ["Id", "Username"] ) ) oc.initialize() oc.execute() self.assertEqual(1, len(oc.get_extracted_ids("Account"))) self.assertEqual(1, len(oc.get_extracted_ids("User")))
def test_descendents_extracts_object_network(self): expected_names = {"Elosha", "Gaius"} oc = amaxa.ExtractOperation(Connection(self.connection, "48.0")) oc.file_store = MockFileStore() rec = self.connection.query( "SELECT Id FROM Account WHERE Name = 'Caprica Cosmetics'") oc.add_dependency("Account", rec.get("records")[0]["Id"]) oc.add_step( amaxa.ExtractionStep( "Account", amaxa.ExtractionScope.SELECTED_RECORDS, ["Id", "Name", "ParentId"], )) oc.add_step( amaxa.ExtractionStep( "Contact", amaxa.ExtractionScope.DESCENDENTS, ["Id", "FirstName", "LastName", "AccountId"], )) oc.initialize() oc.execute() self.assertEqual(3, len(oc.get_extracted_ids("Account"))) self.assertEqual(2, len(oc.get_extracted_ids("Contact"))) for c in oc.file_store.get_csv( "Contact", amaxa.FileType.OUTPUT).writerow.call_args_list: self.assertIn(c[0][0]["FirstName"], expected_names) expected_names.remove(c[0][0]["FirstName"]) self.assertEqual(0, len(expected_names))
def test_query_records_by_reference_field(self): sf = Mock() sf.bulk_url = "https://salesforce.com" conn = Connection(sf) id_set = [] for i in range(400): new_id = str(amaxa.SalesforceId("00100000000" + str(i + 1).zfill(4))) id_set.append(new_id) api_return_value = { "records": [ {"Id": "001000000000001", "Name": "test", "Industry": "Finance"} ] } sf.query_all.return_value = api_return_value retval = list( conn.query_records_by_reference_field( "Account", ["Name", "Industry"], "ParentId", id_set ) ) self.assertGreater(sf.query_all.call_count, 1) self.assertEqual(api_return_value["records"] * sf.query_all.call_count, retval) # Validate that the WHERE clause length limits were respected # and that all of the Ids were queried total_ids = 0 for each_call in sf.query_all.call_args_list: argument = each_call[0][0] self.assertLessEqual(len(argument[argument.find("WHERE") :]), 4000) total_ids += argument.count("'001") self.assertEqual(len(id_set), total_ids)
def test_bulk_api_update(self): sf = Mock() sf.bulk_url = "https://salesforce.com" conn = Connection(sf, "48.0") conn._bulk = Mock() conn._bulk_api_insert_update = Mock(return_value=[]) self.assertEqual( [], list(conn.bulk_api_update("Account", [], 120, 5, 1, "Parallel")) ) conn._bulk.create_update_job.assert_called_once_with( "Account", contentType="JSON", concurrency="Parallel" ) conn._bulk_api_insert_update.assert_called_once() assert ( conn._bulk_api_insert_update.call_args[0][0] == conn._bulk.create_update_job.return_value ) assert conn._bulk_api_insert_update.call_args[0][1] == "Account" assert conn._bulk_api_insert_update.call_args[0][3] == 120 assert conn._bulk_api_insert_update.call_args[0][4] == 5 assert conn._bulk_api_insert_update.call_args[0][5] == 1
def test_bulk_query_converts_datetimes(self): sf = Mock() sf.bulk_url = "https://salesforce.com" conn = Connection(sf) conn._bulk = Mock() retval = [ {"Id": "001000000000001", "CreatedDate": 1546659665000}, {"Id": "001000000000002", "CreatedDate": None}, ] conn._bulk.is_batch_done = Mock(side_effect=[False, True]) conn._bulk.create_query_job = Mock(return_value="075000000000000AAA") conn._bulk.get_all_results_for_query_batch = Mock( return_value=[IteratorBytesIO([json.dumps(retval).encode("utf-8")])] ) results = list( conn.bulk_api_query( "Account", "SELECT Id, CreatedDate, FROM Account", ["CreatedDate"], 5 ) ) self.assertEqual( results[0], {"Id": "001000000000001", "CreatedDate": "2019-01-05T03:41:05.000+0000"}, ) self.assertEqual(results[1], {"Id": "001000000000002", "CreatedDate": None})
def test_maps_ids_to_sobject_types(self): sf = Mock() sf.bulk_url = "https://salesforce.com" conn = Connection(sf, "52.0") conn.get_global_describe = Mock() conn.get_global_describe.return_value = { "sobjects": [ { "name": "Account", "keyPrefix": "001" }, { "name": "Contact", "keyPrefix": "003" }, ] } self.assertEqual("Account", conn.get_sobject_name_for_id("001000000000000")) self.assertEqual("Contact", conn.get_sobject_name_for_id("003000000000000")) conn.get_global_describe.assert_called_once_with()
def test_get_global_describe_calls_salesforce(self): sf = Mock() sf.bulk_url = "https://salesforce.com" conn = Connection(sf) self.assertEqual(sf.describe.return_value, conn.get_global_describe()) sf.describe.assert_called_once_with()
def test_init_creates_bulk_instance(self, bulk_mock): sf = Mock() sf.bulk_url = "https://salesforce.com" Connection(sf, api_version="48.0") bulk_mock.assert_called_once_with( sessionId=sf.session_id, host="salesforce.com", API_version="48.0", )
def test_init_creates_bulk_instance(self, bulk_mock): sf = Mock() sf.bulk_url = "https://salesforce.com" Connection(sf) bulk_mock.assert_called_once_with( sessionId=sf.session_id, host="salesforce.com", API_version=amaxa.constants.API_VERSION, )
def test_bulk_api_insert_update(self): sf = Mock() sf.bulk_url = "https://salesforce.com" conn = Connection(sf) conn._bulk = Mock() job = Mock() retval = [ [{"Id": "001000000000001"}, {"Id": "001000000000002"}], [{"Id": "001000000000003"}], ] conn._bulk.is_batch_done = Mock(side_effect=[False, True]) conn._bulk.get_batch_results = Mock(side_effect=retval) input_data = [{"Name": "Test"}, {"Name": "Test2"}, {"Name": "Test3"}] results = list( conn._bulk_api_insert_update(job, "Account", input_data, 120, 5, 2) ) self.assertEqual(2, conn._bulk.post_batch.call_count) self.assertEqual( conn._bulk.wait_for_batch.call_args_list, [ call( job, conn._bulk.post_batch.return_value, timeout=120, sleep_interval=5, ), call( job, conn._bulk.post_batch.return_value, timeout=120, sleep_interval=5, ), ], ) conn._bulk.close_job.assert_called_once_with(job) self.assertEqual( conn._bulk.get_batch_results.call_args_list, [ call(conn._bulk.post_batch.return_value, job), call(conn._bulk.post_batch.return_value, job), ], ) self.assertEqual( results, [ {"Id": "001000000000001"}, {"Id": "001000000000002"}, {"Id": "001000000000003"}, ], )
def test_all_records_extracts_accounts(self): oc = amaxa.ExtractOperation(Connection(self.connection)) oc.file_store = MockFileStore() extraction = amaxa.ExtractionStep( "Account", amaxa.ExtractionScope.ALL_RECORDS, ["Id", "Name"] ) oc.add_step(extraction) extraction.initialize() extraction.execute() self.assertEqual(5, len(oc.get_extracted_ids("Account")))
def test_get_sobject_describe_calls_salesforce(self): sf = Mock() sf.bulk_url = "https://salesforce.com" conn = Connection(sf) sf.Account.describe.return_value = { "fields": [{"name": "Name"}, {"name": "Id"}] } self.assertEqual( sf.Account.describe.return_value, conn.get_sobject_describe("Account") ) sf.Account.describe.assert_called_once_with()
def test_retrieve_records_by_id(self): id_set = [] # Generate enough mock Ids to require two queries. for i in range(2005): new_id = str( amaxa.SalesforceId("00100000000" + str(i + 1).zfill(4))) id_set.append(new_id) self.assertEqual(2005, len(id_set)) complete_return_value = [{"Id": each_id} for each_id in id_set] api_return_value = [ complete_return_value[:2000], complete_return_value[2000:] ] api_return_value[0].append(None) sf = Mock() sf.bulk_url = "https://salesforce.com" sf.restful = Mock(side_effect=api_return_value) conn = Connection(sf) retval = conn.retrieve_records_by_id("Account", id_set, ["Name"]) self.assertEqual(complete_return_value, list(retval)) self.assertEqual(2, sf.restful.call_count) self.assertEqual( sf.restful.call_args_list, [ call( "composite/sobjects/Account", method="POST", data=json.dumps({ "ids": id_set[:2000], "fields": ["Name"] }), ), call( "composite/sobjects/Account", method="POST", data=json.dumps({ "ids": id_set[2000:], "fields": ["Name"] }), ), ], )
def test_extracts_dependencies(self): expected_account_names = { "Caprica Cosmetics", "Gemenon Gastronomy", "Aerilon Agrinomics", } expected_contact_names = {"Gaius"} oc = amaxa.ExtractOperation(Connection(self.connection)) oc.file_store = MockFileStore() rec = self.connection.query("SELECT Id FROM Contact WHERE LastName = 'Baltar'") oc.add_dependency("Contact", rec.get("records")[0]["Id"]) oc.add_step( amaxa.ExtractionStep( "Contact", amaxa.ExtractionScope.SELECTED_RECORDS, ["Id", "FirstName", "LastName", "AccountId"], ) ) oc.add_step( amaxa.ExtractionStep( "Account", amaxa.ExtractionScope.DESCENDENTS, ["Id", "Name", "ParentId"] ) ) oc.initialize() oc.execute() self.assertEqual(3, len(oc.get_extracted_ids("Account"))) self.assertEqual(1, len(oc.get_extracted_ids("Contact"))) for c in oc.file_store.get_csv( "Contact", amaxa.FileType.OUTPUT ).writerow.call_args_list: self.assertIn(c[0][0]["FirstName"], expected_contact_names) expected_contact_names.remove(c[0][0]["FirstName"]) self.assertEqual(0, len(expected_contact_names)) for c in oc.file_store.get_csv( "Account", amaxa.FileType.OUTPUT ).writerow.call_args_list: self.assertIn(c[0][0]["Name"], expected_account_names) expected_account_names.remove(c[0][0]["Name"]) self.assertEqual(0, len(expected_account_names))
def test_loads_single_object(self): records = [ { "Id": "01t000000000001", "Name": "Tauron Taffy", "IsActive": "True", "ProductCode": "TAFFY_TAUR", }, { "Id": "01t000000000002", "Name": "Gemenese Goulash", "IsActive": "True", "ProductCode": "GLSH", }, { "Id": "01t000000000003AAA", "Name": "Caprica Corn", "IsActive": "False", "ProductCode": "CPRCC", }, ] op = amaxa.LoadOperation(Connection(self.connection)) op.file_store = MockFileStore() op.file_store.records["Product2"] = records op.add_step( amaxa.LoadStep( "Product2", set(["Name", "IsActive", "ProductCode", "Description"]) ) ) op.initialize() op.execute() loaded_products = self.connection.query_all( "SELECT Id, Name, IsActive, ProductCode FROM Product2" ).get("records") self.assertEqual(3, len(loaded_products)) required_names = {x["Name"] for x in records} for r in loaded_products: self.register_case_record("Product2", r["Id"]) self.assertIn(r["Name"], required_names) required_names.remove(r["Name"]) self.assertEqual(0, len(required_names))
def test_loads_complex_hierarchy(self): accounts = [ { "Id": "001000000000001", "Name": "Tauron Tourist Commission", "ParentId": "", }, { "Id": "001000000000002", "Name": "Emporion Enterprises", "ParentId": "001000000000001", }, { "Id": "001000000000003AAA", "Name": "Caprica City Outreach", "ParentId": "001000000000001", }, ] contacts = [ {"Id": "003000000000000", "FirstName": "Joseph", "LastName": "Adama"}, {"Id": "003000000000001", "FirstName": "Sam", "LastName": "Adama"}, ] opportunities = [ { "Id": "006000000000001", "AccountId": "001000000000001", "Name": "End-of-Year Promotion", "CloseDate": "2019-06-01", "StageName": "Closed Won", }, { "Id": "006000000000002", "AccountId": "001000000000001", "Name": "New Initiative", "CloseDate": "2019-12-01", "StageName": "Closed Lost", }, ] opportunity_contact_roles = [ { "Id": "00K000000000001", "OpportunityId": "006000000000001", "ContactId": "003000000000000", "Role": "Decision Maker", }, { "Id": "00K000000000001", "OpportunityId": "006000000000001", "ContactId": "003000000000000", "Role": "Influencer", }, { "Id": "00K000000000002", "OpportunityId": "006000000000002", "ContactId": "003000000000001", "Role": "Decision Maker", }, ] op = amaxa.LoadOperation(Connection(self.connection)) op.file_store = MockFileStore() op.file_store.records["Account"] = accounts op.file_store.records["Contact"] = contacts op.file_store.records["Opportunity"] = opportunities op.file_store.records["OpportunityContactRole"] = opportunity_contact_roles op.add_step(amaxa.LoadStep("Account", set(["Name", "ParentId"]))) op.add_step( amaxa.LoadStep("Contact", set(["AccountId", "FirstName", "LastName"])) ) op.add_step( amaxa.LoadStep( "Opportunity", set(["AccountId", "Name", "CloseDate", "StageName"]) ) ) op.add_step( amaxa.LoadStep( "OpportunityContactRole", set(["ContactId", "OpportunityId", "Role"]) ) ) op.initialize() op.execute() loaded_accounts = self.connection.query_all( "SELECT Id, Name, (SELECT Name FROM ChildAccounts) FROM Account" ).get("records") self.assertEqual(len(accounts), len(loaded_accounts)) required_names = [x["Name"] for x in accounts] for r in loaded_accounts: self.register_case_record("Account", r["Id"]) self.assertIn(r["Name"], required_names) required_names.remove(r["Name"]) if r["Name"] == "Tauron Tourist Commission": self.assertIsNotNone(r["ChildAccounts"]) self.assertEqual(2, len(r["ChildAccounts"]["records"])) self.assertEqual(0, len(required_names)) loaded_opportunities = self.connection.query_all( "SELECT Id, Name, (SELECT Id FROM OpportunityContactRoles) FROM Opportunity" ).get("records") self.assertEqual(len(opportunities), len(loaded_opportunities)) required_names = [x["Name"] for x in opportunities] for r in loaded_opportunities: self.register_case_record("Opportunity", r["Id"]) self.assertIn(r["Name"], required_names) required_names.remove(r["Name"]) if r["Name"] == "End-of-Year Promotion": self.assertEqual(2, len(r["OpportunityContactRoles"]["records"])) else: self.assertEqual(1, len(r["OpportunityContactRoles"]["records"])) self.assertEqual(0, len(required_names)) loaded_contacts = self.connection.query_all( "SELECT Id, FirstName, LastName, (SELECT Id FROM OpportunityContactRoles) FROM Contact" ).get("records") self.assertEqual(len(contacts), len(loaded_contacts)) required_names = [x["LastName"] for x in contacts] for r in loaded_contacts: self.register_case_record("Contact", r["Id"]) self.assertIn(r["LastName"], required_names) required_names.remove(r["LastName"]) if r["FirstName"] == "Sam": self.assertEqual(1, len(r["OpportunityContactRoles"]["records"])) else: self.assertEqual(2, len(r["OpportunityContactRoles"]["records"])) self.assertEqual(0, len(required_names))