def test_bulk_query_converts_datetimes(self):
sf = Mock()
sf.bulk_url = "https://salesforce.com"
conn = Connection(sf)
conn._bulk = Mock()
retval = [
{"Id": "001000000000001", "CreatedDate": 1546659665000},
{"Id": "001000000000002", "CreatedDate": None},
]
conn._bulk.is_batch_done = Mock(side_effect=[False, True])
conn._bulk.create_query_job = Mock(return_value="075000000000000AAA")
conn._bulk.get_all_results_for_query_batch = Mock(
return_value=[IteratorBytesIO([json.dumps(retval).encode("utf-8")])]
)
results = list(
conn.bulk_api_query(
"Account", "SELECT Id, CreatedDate, FROM Account", ["CreatedDate"], 5
)
)
self.assertEqual(
results[0],
{"Id": "001000000000001", "CreatedDate": "2019-01-05T03:41:05.000+0000"},
)
self.assertEqual(results[1], {"Id": "001000000000002", "CreatedDate": None})
def test_bulk_api_query(self): # FIXME: test wait
sf = Mock()
sf.bulk_url = "https://salesforce.com"
conn = Connection(sf)
conn._bulk = Mock()
retval = [{"Id": "001000000000001"}, {"Id": "001000000000002"}]
conn._bulk.is_batch_done = Mock(side_effect=[False, True])
conn._bulk.create_query_job = Mock(return_value="075000000000000AAA")
conn._bulk.get_all_results_for_query_batch = Mock(
return_value=[IteratorBytesIO([json.dumps(retval).encode("utf-8")])]
)
results = list(conn.bulk_api_query("Account", "SELECT Id FROM Account", [], 5))
conn._bulk.query.assert_called_once_with(
"075000000000000AAA", "SELECT Id FROM Account"
)
self.assertEqual(
conn._bulk.is_batch_done.call_args_list,
[call(conn._bulk.query.return_value), call(conn._bulk.query.return_value)],
)
conn._bulk.get_all_results_for_query_batch.assert_called_once_with(
conn._bulk.query.return_value
)
self.assertEqual(retval, results)
def test_bulk_api_update(self):
sf = Mock()
sf.bulk_url = "https://salesforce.com"
conn = Connection(sf, "48.0")
conn._bulk = Mock()
conn._bulk_api_insert_update = Mock(return_value=[])
self.assertEqual(
[], list(conn.bulk_api_update("Account", [], 120, 5, 1, "Parallel"))
)
conn._bulk.create_update_job.assert_called_once_with(
"Account", contentType="JSON", concurrency="Parallel"
)
conn._bulk_api_insert_update.assert_called_once()
assert (
conn._bulk_api_insert_update.call_args[0][0]
== conn._bulk.create_update_job.return_value
)
assert conn._bulk_api_insert_update.call_args[0][1] == "Account"
assert conn._bulk_api_insert_update.call_args[0][3] == 120
assert conn._bulk_api_insert_update.call_args[0][4] == 5
assert conn._bulk_api_insert_update.call_args[0][5] == 1
def test_query_records_by_reference_field(self):
sf = Mock()
sf.bulk_url = "https://salesforce.com"
conn = Connection(sf)
id_set = []
for i in range(400):
new_id = str(amaxa.SalesforceId("00100000000" + str(i + 1).zfill(4)))
id_set.append(new_id)
api_return_value = {
"records": [
{"Id": "001000000000001", "Name": "test", "Industry": "Finance"}
]
}
sf.query_all.return_value = api_return_value
retval = list(
conn.query_records_by_reference_field(
"Account", ["Name", "Industry"], "ParentId", id_set
)
)
self.assertGreater(sf.query_all.call_count, 1)
self.assertEqual(api_return_value["records"] * sf.query_all.call_count, retval)
# Validate that the WHERE clause length limits were respected
# and that all of the Ids were queried
total_ids = 0
for each_call in sf.query_all.call_args_list:
argument = each_call[0][0]
self.assertLessEqual(len(argument[argument.find("WHERE") :]), 4000)
total_ids += argument.count("'001")
self.assertEqual(len(id_set), total_ids)
def test_loads_and_extracts_high_data_volume(self):
# This is a single unit test rather than multiple to save on execution time.
records = []
for i in range(100000):
records.append({
"Id": "00Q000000{:06d}".format(i),
"Company": "[not provided]",
"LastName": "Lead {:06d}".format(i),
})
op = amaxa.LoadOperation(Connection(self.connection, "52.0"))
op.file_store = MockFileStore()
op.file_store.records["Lead"] = records
op.add_step(amaxa.LoadStep("Lead", set(["LastName", "Company"])))
op.initialize()
op.execute()
self.assertEqual(
100000,
self.connection.query("SELECT count() FROM Lead").get("totalSize"))
oc = amaxa.ExtractOperation(Connection(self.connection, "52.0"))
oc.file_store = MockFileStore()
extraction = amaxa.ExtractionStep("Lead",
amaxa.ExtractionScope.ALL_RECORDS,
["Id", "LastName"])
oc.add_step(extraction)
extraction.initialize()
extraction.execute()
self.assertEqual(100000, len(oc.get_extracted_ids("Lead")))
def test_caches_field_maps(self):
sf = Mock()
sf.bulk_url = "https://salesforce.com"
conn = Connection(sf, "52.0")
sf.Account.describe.return_value = {
"fields": [{
"name": "Name"
}, {
"name": "Id"
}]
}
retval = conn.get_sobject_field_map("Account")
self.assertEqual({
"Name": {
"name": "Name"
},
"Id": {
"name": "Id"
}
}, retval)
sf.Account.describe.assert_called_once_with()
sf.Account.describe.reset_mock()
retval = conn.get_sobject_field_map("Account")
self.assertEqual({
"Name": {
"name": "Name"
},
"Id": {
"name": "Id"
}
}, retval)
sf.Account.describe.assert_not_called()
def test_maps_ids_to_sobject_types(self):
sf = Mock()
sf.bulk_url = "https://salesforce.com"
conn = Connection(sf, "52.0")
conn.get_global_describe = Mock()
conn.get_global_describe.return_value = {
"sobjects": [
{
"name": "Account",
"keyPrefix": "001"
},
{
"name": "Contact",
"keyPrefix": "003"
},
]
}
self.assertEqual("Account",
conn.get_sobject_name_for_id("001000000000000"))
self.assertEqual("Contact",
conn.get_sobject_name_for_id("003000000000000"))
conn.get_global_describe.assert_called_once_with()
def test_get_global_describe_calls_salesforce(self):
sf = Mock()
sf.bulk_url = "https://salesforce.com"
conn = Connection(sf)
self.assertEqual(sf.describe.return_value, conn.get_global_describe())
sf.describe.assert_called_once_with()
def test_bulk_api_insert_update(self):
sf = Mock()
sf.bulk_url = "https://salesforce.com"
conn = Connection(sf)
conn._bulk = Mock()
job = Mock()
retval = [
[{"Id": "001000000000001"}, {"Id": "001000000000002"}],
[{"Id": "001000000000003"}],
]
conn._bulk.is_batch_done = Mock(side_effect=[False, True])
conn._bulk.get_batch_results = Mock(side_effect=retval)
input_data = [{"Name": "Test"}, {"Name": "Test2"}, {"Name": "Test3"}]
results = list(
conn._bulk_api_insert_update(job, "Account", input_data, 120, 5, 2)
)
self.assertEqual(2, conn._bulk.post_batch.call_count)
self.assertEqual(
conn._bulk.wait_for_batch.call_args_list,
[
call(
job,
conn._bulk.post_batch.return_value,
timeout=120,
sleep_interval=5,
),
call(
job,
conn._bulk.post_batch.return_value,
timeout=120,
sleep_interval=5,
),
],
)
conn._bulk.close_job.assert_called_once_with(job)
self.assertEqual(
conn._bulk.get_batch_results.call_args_list,
[
call(conn._bulk.post_batch.return_value, job),
call(conn._bulk.post_batch.return_value, job),
],
)
self.assertEqual(
results,
[
{"Id": "001000000000001"},
{"Id": "001000000000002"},
{"Id": "001000000000003"},
],
)
def test_get_sobject_describe_calls_salesforce(self):
sf = Mock()
sf.bulk_url = "https://salesforce.com"
conn = Connection(sf)
sf.Account.describe.return_value = {
"fields": [{"name": "Name"}, {"name": "Id"}]
}
self.assertEqual(
sf.Account.describe.return_value, conn.get_sobject_describe("Account")
)
sf.Account.describe.assert_called_once_with()
def test_extracts_polymorphic_lookups(self):
oc = amaxa.ExtractOperation(Connection(self.connection))
oc.file_store = MockFileStore()
rec = self.connection.query(
"SELECT Id FROM Account WHERE Name = 'Caprica Cosmetics'"
)
oc.add_dependency("Account", rec.get("records")[0]["Id"])
oc.add_step(
amaxa.ExtractionStep(
"Account",
amaxa.ExtractionScope.SELECTED_RECORDS,
["Id", "Name", "OwnerId"],
)
)
oc.add_step(
amaxa.ExtractionStep(
"User", amaxa.ExtractionScope.DESCENDENTS, ["Id", "Username"]
)
)
oc.initialize()
oc.execute()
self.assertEqual(1, len(oc.get_extracted_ids("Account")))
self.assertEqual(1, len(oc.get_extracted_ids("User")))
def test_descendents_extracts_object_network(self):
expected_names = {"Elosha", "Gaius"}
oc = amaxa.ExtractOperation(Connection(self.connection, "48.0"))
oc.file_store = MockFileStore()
rec = self.connection.query(
"SELECT Id FROM Account WHERE Name = 'Caprica Cosmetics'")
oc.add_dependency("Account", rec.get("records")[0]["Id"])
oc.add_step(
amaxa.ExtractionStep(
"Account",
amaxa.ExtractionScope.SELECTED_RECORDS,
["Id", "Name", "ParentId"],
))
oc.add_step(
amaxa.ExtractionStep(
"Contact",
amaxa.ExtractionScope.DESCENDENTS,
["Id", "FirstName", "LastName", "AccountId"],
))
oc.initialize()
oc.execute()
self.assertEqual(3, len(oc.get_extracted_ids("Account")))
self.assertEqual(2, len(oc.get_extracted_ids("Contact")))
for c in oc.file_store.get_csv(
"Contact", amaxa.FileType.OUTPUT).writerow.call_args_list:
self.assertIn(c[0][0]["FirstName"], expected_names)
expected_names.remove(c[0][0]["FirstName"])
self.assertEqual(0, len(expected_names))
def test_query_extracts_self_lookup_hierarchy(self):
expected_names = {
"Caprica Cosmetics",
"Gemenon Gastronomy",
"Aerilon Agrinomics",
}
oc = amaxa.ExtractOperation(Connection(self.connection))
oc.file_store = MockFileStore()
rec = self.connection.query(
"SELECT Id FROM Account WHERE Name = 'Caprica Cosmetics'"
)
oc.add_dependency("Account", rec.get("records")[0]["Id"])
extraction = amaxa.ExtractionStep(
"Account",
amaxa.ExtractionScope.SELECTED_RECORDS,
["Id", "Name", "ParentId"],
)
oc.add_step(extraction)
extraction.initialize()
extraction.execute()
self.assertEqual(3, len(oc.get_extracted_ids("Account")))
for c in oc.file_store.get_csv(
"Account", amaxa.FileType.OUTPUT
).writerow.call_args_list:
self.assertIn(c[0][0]["Name"], expected_names)
expected_names.remove(c[0][0]["Name"])
self.assertEqual(0, len(expected_names))
def test_retrieve_records_by_id(self):
id_set = []
# Generate enough mock Ids to require two queries.
for i in range(2005):
new_id = str(
amaxa.SalesforceId("00100000000" + str(i + 1).zfill(4)))
id_set.append(new_id)
self.assertEqual(2005, len(id_set))
complete_return_value = [{"Id": each_id} for each_id in id_set]
api_return_value = [
complete_return_value[:2000], complete_return_value[2000:]
]
api_return_value[0].append(None)
sf = Mock()
sf.bulk_url = "https://salesforce.com"
sf.restful = Mock(side_effect=api_return_value)
conn = Connection(sf)
retval = conn.retrieve_records_by_id("Account", id_set, ["Name"])
self.assertEqual(complete_return_value, list(retval))
self.assertEqual(2, sf.restful.call_count)
self.assertEqual(
sf.restful.call_args_list,
[
call(
"composite/sobjects/Account",
method="POST",
data=json.dumps({
"ids": id_set[:2000],
"fields": ["Name"]
}),
),
call(
"composite/sobjects/Account",
method="POST",
data=json.dumps({
"ids": id_set[2000:],
"fields": ["Name"]
}),
),
],
)
def test_init_creates_bulk_instance(self, bulk_mock):
sf = Mock()
sf.bulk_url = "https://salesforce.com"
Connection(sf, api_version="48.0")
bulk_mock.assert_called_once_with(
sessionId=sf.session_id, host="salesforce.com", API_version="48.0",
)
def test_init_creates_bulk_instance(self, bulk_mock):
sf = Mock()
sf.bulk_url = "https://salesforce.com"
Connection(sf)
bulk_mock.assert_called_once_with(
sessionId=sf.session_id,
host="salesforce.com",
API_version=amaxa.constants.API_VERSION,
)
def test_all_records_extracts_accounts(self):
oc = amaxa.ExtractOperation(Connection(self.connection))
oc.file_store = MockFileStore()
extraction = amaxa.ExtractionStep(
"Account", amaxa.ExtractionScope.ALL_RECORDS, ["Id", "Name"]
)
oc.add_step(extraction)
extraction.initialize()
extraction.execute()
self.assertEqual(5, len(oc.get_extracted_ids("Account")))
def test_extracts_dependencies(self):
expected_account_names = {
"Caprica Cosmetics",
"Gemenon Gastronomy",
"Aerilon Agrinomics",
}
expected_contact_names = {"Gaius"}
oc = amaxa.ExtractOperation(Connection(self.connection))
oc.file_store = MockFileStore()
rec = self.connection.query("SELECT Id FROM Contact WHERE LastName = 'Baltar'")
oc.add_dependency("Contact", rec.get("records")[0]["Id"])
oc.add_step(
amaxa.ExtractionStep(
"Contact",
amaxa.ExtractionScope.SELECTED_RECORDS,
["Id", "FirstName", "LastName", "AccountId"],
)
)
oc.add_step(
amaxa.ExtractionStep(
"Account", amaxa.ExtractionScope.DESCENDENTS, ["Id", "Name", "ParentId"]
)
)
oc.initialize()
oc.execute()
self.assertEqual(3, len(oc.get_extracted_ids("Account")))
self.assertEqual(1, len(oc.get_extracted_ids("Contact")))
for c in oc.file_store.get_csv(
"Contact", amaxa.FileType.OUTPUT
).writerow.call_args_list:
self.assertIn(c[0][0]["FirstName"], expected_contact_names)
expected_contact_names.remove(c[0][0]["FirstName"])
self.assertEqual(0, len(expected_contact_names))
for c in oc.file_store.get_csv(
"Account", amaxa.FileType.OUTPUT
).writerow.call_args_list:
self.assertIn(c[0][0]["Name"], expected_account_names)
expected_account_names.remove(c[0][0]["Name"])
self.assertEqual(0, len(expected_account_names))
def test_loads_single_object(self):
records = [
{
"Id": "01t000000000001",
"Name": "Tauron Taffy",
"IsActive": "True",
"ProductCode": "TAFFY_TAUR",
},
{
"Id": "01t000000000002",
"Name": "Gemenese Goulash",
"IsActive": "True",
"ProductCode": "GLSH",
},
{
"Id": "01t000000000003AAA",
"Name": "Caprica Corn",
"IsActive": "False",
"ProductCode": "CPRCC",
},
]
op = amaxa.LoadOperation(Connection(self.connection))
op.file_store = MockFileStore()
op.file_store.records["Product2"] = records
op.add_step(
amaxa.LoadStep(
"Product2", set(["Name", "IsActive", "ProductCode", "Description"])
)
)
op.initialize()
op.execute()
loaded_products = self.connection.query_all(
"SELECT Id, Name, IsActive, ProductCode FROM Product2"
).get("records")
self.assertEqual(3, len(loaded_products))
required_names = {x["Name"] for x in records}
for r in loaded_products:
self.register_case_record("Product2", r["Id"])
self.assertIn(r["Name"], required_names)
required_names.remove(r["Name"])
self.assertEqual(0, len(required_names))
def test_loads_complex_hierarchy(self):
accounts = [
{
"Id": "001000000000001",
"Name": "Tauron Tourist Commission",
"ParentId": "",
},
{
"Id": "001000000000002",
"Name": "Emporion Enterprises",
"ParentId": "001000000000001",
},
{
"Id": "001000000000003AAA",
"Name": "Caprica City Outreach",
"ParentId": "001000000000001",
},
]
contacts = [
{"Id": "003000000000000", "FirstName": "Joseph", "LastName": "Adama"},
{"Id": "003000000000001", "FirstName": "Sam", "LastName": "Adama"},
]
opportunities = [
{
"Id": "006000000000001",
"AccountId": "001000000000001",
"Name": "End-of-Year Promotion",
"CloseDate": "2019-06-01",
"StageName": "Closed Won",
},
{
"Id": "006000000000002",
"AccountId": "001000000000001",
"Name": "New Initiative",
"CloseDate": "2019-12-01",
"StageName": "Closed Lost",
},
]
opportunity_contact_roles = [
{
"Id": "00K000000000001",
"OpportunityId": "006000000000001",
"ContactId": "003000000000000",
"Role": "Decision Maker",
},
{
"Id": "00K000000000001",
"OpportunityId": "006000000000001",
"ContactId": "003000000000000",
"Role": "Influencer",
},
{
"Id": "00K000000000002",
"OpportunityId": "006000000000002",
"ContactId": "003000000000001",
"Role": "Decision Maker",
},
]
op = amaxa.LoadOperation(Connection(self.connection))
op.file_store = MockFileStore()
op.file_store.records["Account"] = accounts
op.file_store.records["Contact"] = contacts
op.file_store.records["Opportunity"] = opportunities
op.file_store.records["OpportunityContactRole"] = opportunity_contact_roles
op.add_step(amaxa.LoadStep("Account", set(["Name", "ParentId"])))
op.add_step(
amaxa.LoadStep("Contact", set(["AccountId", "FirstName", "LastName"]))
)
op.add_step(
amaxa.LoadStep(
"Opportunity", set(["AccountId", "Name", "CloseDate", "StageName"])
)
)
op.add_step(
amaxa.LoadStep(
"OpportunityContactRole", set(["ContactId", "OpportunityId", "Role"])
)
)
op.initialize()
op.execute()
loaded_accounts = self.connection.query_all(
"SELECT Id, Name, (SELECT Name FROM ChildAccounts) FROM Account"
).get("records")
self.assertEqual(len(accounts), len(loaded_accounts))
required_names = [x["Name"] for x in accounts]
for r in loaded_accounts:
self.register_case_record("Account", r["Id"])
self.assertIn(r["Name"], required_names)
required_names.remove(r["Name"])
if r["Name"] == "Tauron Tourist Commission":
self.assertIsNotNone(r["ChildAccounts"])
self.assertEqual(2, len(r["ChildAccounts"]["records"]))
self.assertEqual(0, len(required_names))
loaded_opportunities = self.connection.query_all(
"SELECT Id, Name, (SELECT Id FROM OpportunityContactRoles) FROM Opportunity"
).get("records")
self.assertEqual(len(opportunities), len(loaded_opportunities))
required_names = [x["Name"] for x in opportunities]
for r in loaded_opportunities:
self.register_case_record("Opportunity", r["Id"])
self.assertIn(r["Name"], required_names)
required_names.remove(r["Name"])
if r["Name"] == "End-of-Year Promotion":
self.assertEqual(2, len(r["OpportunityContactRoles"]["records"]))
else:
self.assertEqual(1, len(r["OpportunityContactRoles"]["records"]))
self.assertEqual(0, len(required_names))
loaded_contacts = self.connection.query_all(
"SELECT Id, FirstName, LastName, (SELECT Id FROM OpportunityContactRoles) FROM Contact"
).get("records")
self.assertEqual(len(contacts), len(loaded_contacts))
required_names = [x["LastName"] for x in contacts]
for r in loaded_contacts:
self.register_case_record("Contact", r["Id"])
self.assertIn(r["LastName"], required_names)
required_names.remove(r["LastName"])
if r["FirstName"] == "Sam":
self.assertEqual(1, len(r["OpportunityContactRoles"]["records"]))
else:
self.assertEqual(2, len(r["OpportunityContactRoles"]["records"]))
self.assertEqual(0, len(required_names))
