def test_execute_dependent_updates_uses_bulk_api_options(self):
record_list = [
{
"Name": "Test",
"Id": "001000000000000",
"ParentId": "001000000000001"
},
{
"Name": "Test 2",
"Id": "001000000000001",
"ParentId": "001000000000000"
},
]
cleaned_record_list = [
{
"Id": "001000000000002AAA",
"ParentId": "001000000000003AAA"
},
{
"Id": "001000000000003AAA",
"ParentId": "001000000000002AAA"
},
]
connection = MockConnection(bulk_update_results=[
UploadResult("001000000000002", True, True, ""),
UploadResult("001000000000003", True, True, ""),
])
op = amaxa.LoadOperation(Mock(wraps=connection))
op.file_store = MockFileStore()
op.register_new_id(
"Account",
amaxa.SalesforceId("001000000000000"),
amaxa.SalesforceId("001000000000002"),
)
op.register_new_id(
"Account",
amaxa.SalesforceId("001000000000001"),
amaxa.SalesforceId("001000000000003"),
)
op.register_new_id = Mock()
op.register_error = Mock()
op.file_store.records["Account"] = record_list
load_step = amaxa.LoadStep(
"Account",
["Name", "ParentId"],
options={
"bulk-api-poll-interval": 10,
"bulk-api-timeout": 600,
"bulk-api-batch-size": 5000,
},
)
op.add_step(load_step)
load_step.initialize()
load_step.execute_dependent_updates()
op.connection.bulk_api_update.assert_called_once_with(
"Account", cleaned_record_list, 600, 10, 5000)
def test_execute_transforms_and_loads_records_with_lookups(self):
record_list = [
{
"Name": "Test",
"Id": "001000000000000",
"OwnerId": "500000000000000"
},
{
"Name": "Test 2",
"Id": "001000000000001",
"OwnerId": "500000000000001"
},
]
transformed_record_list = [
{
"Name": "Test",
"OwnerId": str(amaxa.SalesforceId("500000000000002"))
},
{
"Name": "Test 2",
"OwnerId": str(amaxa.SalesforceId("500000000000003"))
},
]
connection = MockConnection(bulk_insert_results=[
UploadResult("001000000000002", True, True, ""),
UploadResult("001000000000003", True, True, ""),
])
op = amaxa.LoadOperation(Mock(wraps=connection))
op.file_store = MockFileStore()
op.register_new_id(
"User",
amaxa.SalesforceId("500000000000000"),
amaxa.SalesforceId("500000000000002"),
)
op.register_new_id(
"User",
amaxa.SalesforceId("500000000000001"),
amaxa.SalesforceId("500000000000003"),
)
op.register_new_id = Mock()
op.file_store.records["Account"] = record_list
op.mappers["Account"] = Mock()
op.mappers["Account"].transform_record = Mock(side_effect=lambda x: x)
load_step = amaxa.LoadStep("Account", ["Name", "OwnerId"])
op.add_step(load_step)
load_step.primitivize = Mock(side_effect=lambda x: x)
load_step.initialize()
load_step.descendent_lookups = set(["OwnerId"])
load_step.execute()
op.mappers["Account"].transform_record.assert_has_calls(
[unittest.mock.call(x) for x in record_list])
load_step.primitivize.assert_has_calls(
[unittest.mock.call(x) for x in transformed_record_list])
op.connection.bulk_api_insert.assert_called_once_with(
"Account",
transformed_record_list,
load_step.get_option("bulk-api-timeout"),
load_step.get_option("bulk-api-poll-interval"),
load_step.get_option("bulk-api-batch-size"),
)
op.register_new_id.assert_has_calls([
unittest.mock.call(
"Account",
amaxa.SalesforceId("001000000000000"),
amaxa.SalesforceId("001000000000002"),
),
unittest.mock.call(
"Account",
amaxa.SalesforceId("001000000000001"),
amaxa.SalesforceId("001000000000003"),
),
])
def test_execute_dependent_updates_handles_lookups(self):
record_list = [
{
"Name": "Test",
"Id": "001000000000000",
"ParentId": "001000000000004"
},
{
"Name": "Test 2",
"Id": "001000000000001",
"ParentId": "001000000000005"
},
]
transformed_record_list = [
{
"Id": str(amaxa.SalesforceId("001000000000002")),
"ParentId": str(amaxa.SalesforceId("001000000000006")),
},
{
"Id": str(amaxa.SalesforceId("001000000000003")),
"ParentId": str(amaxa.SalesforceId("001000000000007")),
},
]
connection = MockConnection(bulk_update_results=[
UploadResult("001000000000002", True, True, ""),
UploadResult("001000000000003", True, True, ""),
])
op = amaxa.LoadOperation(Mock(wraps=connection))
op.file_store = MockFileStore()
op.register_new_id(
"Account",
amaxa.SalesforceId("001000000000004"),
amaxa.SalesforceId("001000000000006"),
)
op.register_new_id(
"Account",
amaxa.SalesforceId("001000000000005"),
amaxa.SalesforceId("001000000000007"),
)
op.register_new_id(
"Account",
amaxa.SalesforceId("001000000000000"),
amaxa.SalesforceId("001000000000002"),
)
op.register_new_id(
"Account",
amaxa.SalesforceId("001000000000001"),
amaxa.SalesforceId("001000000000003"),
)
op.register_new_id = Mock()
op.register_error = Mock()
op.file_store.records["Account"] = record_list
load_step = amaxa.LoadStep("Account", ["Name", "ParentId"])
op.add_step(load_step)
load_step.initialize()
load_step.execute_dependent_updates()
op.register_error.assert_not_called()
op.connection.bulk_api_update.assert_called_once_with(
"Account",
transformed_record_list,
load_step.get_option("bulk-api-timeout"),
load_step.get_option("bulk-api-poll-interval"),
load_step.get_option("bulk-api-batch-size"),
)
def test_execute_does_not_insert_records_prepopulated_in_id_map(self):
record_list = [
{
"Name": "Test",
"Id": "001000000000000"
},
{
"Name": "Test 2",
"Id": "001000000000001"
},
{
"Name": "Test 3",
"Id": "001000000000002"
},
]
clean_record_list = [{"Name": "Test 2"}, {"Name": "Test 3"}]
connection = MockConnection(bulk_insert_results=[
UploadResult("001000000000007", True, True, ""),
UploadResult("001000000000008", True, True, ""),
])
op = amaxa.LoadOperation(Mock(wraps=connection))
op.file_store = MockFileStore()
op.register_new_id(
"Account",
amaxa.SalesforceId("001000000000000"),
amaxa.SalesforceId("001000000000005"),
)
op.register_new_id = Mock()
op.file_store.records["Account"] = record_list
op.mappers["Account"] = Mock()
op.mappers["Account"].transform_record = Mock(side_effect=lambda x: x)
load_step = amaxa.LoadStep("Account", ["Name"])
op.add_step(load_step)
load_step.primitivize = Mock(side_effect=lambda x: x)
load_step.populate_lookups = Mock(side_effect=lambda x, y, z: x)
load_step.initialize()
load_step.execute()
op.connection.bulk_api_insert.assert_called_once_with(
"Account",
clean_record_list,
load_step.get_option("bulk-api-timeout"),
load_step.get_option("bulk-api-poll-interval"),
load_step.get_option("bulk-api-batch-size"),
load_step.get_option("bulk-api-mode"),
)
op.register_new_id.assert_has_calls([
unittest.mock.call(
"Account",
amaxa.SalesforceId("001000000000001"),
amaxa.SalesforceId("001000000000007"),
),
unittest.mock.call(
"Account",
amaxa.SalesforceId("001000000000002"),
amaxa.SalesforceId("001000000000008"),
),
])
def test_execute_dependent_updates_handles_errors(self, bulk_proxy):
record_list = [{
'Name': 'Test',
'Id': '001000000000000',
'Lookup__c': '001000000000001'
}, {
'Name': 'Test 2',
'Id': '001000000000001',
'Lookup__c': '001000000000000'
}]
dependent_record_list = [{
'Id': '001000000000000',
'Lookup__c': '001000000000001'
}, {
'Id': '001000000000001',
'Lookup__c': '001000000000000'
}]
connection = Mock()
op = amaxa.LoadOperation(connection)
op.file_store = MockFileStore()
op.get_field_map = Mock(
return_value={
'Name': {
'type': 'string '
},
'Id': {
'type': 'string'
},
'Lookup__c': {
'type': 'string'
}
})
op.register_new_id('Account', amaxa.SalesforceId('001000000000000'),
amaxa.SalesforceId('001000000000002'))
op.register_new_id('Account', amaxa.SalesforceId('001000000000001'),
amaxa.SalesforceId('001000000000003'))
op.register_new_id = Mock()
op.register_error = Mock()
op.file_store.records['Account'] = record_list
error = [{
'statusCode': 'DUPLICATES_DETECTED',
'message': 'There are duplicates',
'fields': [],
'extendedErrorDetails': None
}]
bulk_proxy.get_batch_results = Mock(return_value=[
UploadResult(None, False, False, error),
UploadResult(None, False, False, error)
])
l = amaxa.LoadStep('Account', ['Name', 'Lookup__c'])
l.context = op
l.initialize()
l.self_lookups = set(['Lookup__c'])
l.dependent_lookup_records = dependent_record_list
l.execute_dependent_updates()
self.assertEqual([
unittest.mock.call('Account', record_list[0]['Id'],
l.format_error(error)),
unittest.mock.call('Account', record_list[1]['Id'],
l.format_error(error))
], op.register_error.call_args_list)
def test_execute_dependent_updates_handles_lookups(self,
json_iterator_proxy,
bulk_proxy):
record_list = [{
'Name': 'Test',
'Id': '001000000000000',
'Lookup__c': '001000000000001'
}, {
'Name': 'Test 2',
'Id': '001000000000001',
'Lookup__c': '001000000000000'
}]
cleaned_record_list = [{
'Id': '001000000000000',
'Lookup__c': '001000000000001'
}, {
'Id': '001000000000001',
'Lookup__c': '001000000000000'
}]
transformed_record_list = [{
'Id':
str(amaxa.SalesforceId('001000000000002')),
'Lookup__c':
str(amaxa.SalesforceId('001000000000003'))
}, {
'Id':
str(amaxa.SalesforceId('001000000000003')),
'Lookup__c':
str(amaxa.SalesforceId('001000000000002'))
}]
connection = Mock()
op = amaxa.LoadOperation(connection)
op.file_store = MockFileStore()
op.get_field_map = Mock(
return_value={
'Name': {
'type': 'string '
},
'Id': {
'type': 'string'
},
'Lookup__c': {
'type': 'string'
}
})
op.register_new_id('Account', amaxa.SalesforceId('001000000000000'),
amaxa.SalesforceId('001000000000002'))
op.register_new_id('Account', amaxa.SalesforceId('001000000000001'),
amaxa.SalesforceId('001000000000003'))
op.register_new_id = Mock()
op.register_error = Mock()
op.file_store.records['Account'] = record_list
bulk_proxy.get_batch_results = Mock(return_value=[
UploadResult('001000000000002', True, True, ''),
UploadResult('001000000000003', True, True, '')
])
l = amaxa.LoadStep('Account', ['Name', 'Lookup__c'])
l.context = op
l.initialize()
l.self_lookups = set(['Lookup__c'])
l.dependent_lookup_records = cleaned_record_list
l.execute_dependent_updates()
op.register_error.assert_not_called()
bulk_proxy.create_update_job.assert_called_once_with(
'Account', contentType='JSON')
json_iterator_proxy.assert_called_once_with(transformed_record_list)
bulk_proxy.post_batch.assert_called_once_with(
bulk_proxy.create_update_job.return_value,
json_iterator_proxy.return_value)
def test_execute_transforms_and_loads_records_with_lookups(
self, json_iterator_proxy, bulk_proxy):
record_list = [{
'Name': 'Test',
'Id': '001000000000000',
'Lookup__c': '003000000000000'
}, {
'Name': 'Test 2',
'Id': '001000000000001',
'Lookup__c': '003000000000001'
}]
transformed_record_list = [{
'Name':
'Test',
'Lookup__c':
str(amaxa.SalesforceId('003000000000002'))
}, {
'Name':
'Test 2',
'Lookup__c':
str(amaxa.SalesforceId('003000000000003'))
}]
connection = Mock()
op = amaxa.LoadOperation(connection)
op.file_store = MockFileStore()
op.get_field_map = Mock(
return_value={
'Name': {
'type': 'string '
},
'Id': {
'type': 'string'
},
'Lookup__c': {
'type': 'string'
}
})
op.register_new_id('Account', amaxa.SalesforceId('003000000000000'),
amaxa.SalesforceId('003000000000002'))
op.register_new_id('Account', amaxa.SalesforceId('003000000000001'),
amaxa.SalesforceId('003000000000003'))
op.register_new_id = Mock()
op.file_store.records['Account'] = record_list
bulk_proxy.get_batch_results = Mock(return_value=[
UploadResult('001000000000002', True, True, ''),
UploadResult('001000000000003', True, True, '')
])
bulk_proxy.create_insert_job = Mock(return_value=Mock())
op.mappers['Account'] = Mock()
op.mappers['Account'].transform_record = Mock(side_effect=lambda x: x)
l = amaxa.LoadStep('Account', ['Name', 'Lookup__c'])
l.context = op
l.primitivize = Mock(side_effect=lambda x: x)
l.initialize()
l.descendent_lookups = set(['Lookup__c'])
l.execute()
op.mappers['Account'].transform_record.assert_has_calls(
[unittest.mock.call(x) for x in record_list])
l.primitivize.assert_has_calls(
[unittest.mock.call(x) for x in transformed_record_list])
json_iterator_proxy.assert_called_once_with(transformed_record_list)
bulk_proxy.post_batch.assert_called_once_with(
bulk_proxy.create_insert_job.return_value,
json_iterator_proxy.return_value)
op.register_new_id.assert_has_calls([
unittest.mock.call('Account',
amaxa.SalesforceId('001000000000000'),
amaxa.SalesforceId('001000000000002')),
unittest.mock.call('Account',
amaxa.SalesforceId('001000000000001'),
amaxa.SalesforceId('001000000000003'))
])