def test_validate_and_inject_namespace__fls_sobject_failure(
self, mock_field, mock_sobject
):
ms = MappingStep(
sf_object="Test__c", fields=["Name"], action=DataOperationType.INSERT
)
org_config = mock.Mock()
org_config.salesforce_client.describe.return_value = {
"sobjects": [{"name": "Test__c", "createable": False}]
}
org_config.salesforce_client.Test__c.describe.return_value = {
"fields": [{"name": "Name", "createable": True}]
}
assert not ms.validate_and_inject_namespace(
org_config, "ns", DataOperationType.INSERT
)
ms._validate_sobject.assert_called_once_with(
{"Test__c": {"name": "Test__c", "createable": False}},
None,
DataOperationType.INSERT,
)
ms._validate_field_dict.assert_not_called()
def test_get_relative_date_context(self):
mapping = MappingStep(
sf_object="Account",
fields=["Some_Date__c", "Some_Datetime__c"],
anchor_date="2020-07-01",
)
org_config = mock.Mock()
org_config.salesforce_client.Account.describe.return_value = {
"fields": [
{
"name": "Some_Date__c",
"type": "date"
},
{
"name": "Some_Datetime__c",
"type": "datetime"
},
{
"name": "Some_Bool__c",
"type": "boolean"
},
]
}
assert mapping.get_relative_date_context(org_config) == ([0], [1],
date.today())
def test_validate_field_dict__fls_checks(self):
ms = MappingStep(
sf_object="Account",
fields=["Id", "Name", "Website"],
action=DataOperationType.INSERT,
)
assert ms._validate_field_dict(
describe=CaseInsensitiveDict(
{"Name": {"createable": True}, "Website": {"createable": True}}
),
field_dict=ms.fields_,
inject=None,
strip=None,
drop_missing=False,
data_operation_type=DataOperationType.INSERT,
)
assert not ms._validate_field_dict(
describe=CaseInsensitiveDict(
{"Name": {"createable": True}, "Website": {"createable": False}}
),
field_dict=ms.fields_,
inject=None,
strip=None,
drop_missing=False,
data_operation_type=DataOperationType.INSERT,
)
def test_import_results__record_type_mapping(self):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, "recordtypes.yml")
task = _make_task(
ExtractData,
{"options": {"database_url": "sqlite://", "mapping": mapping_path}},
)
task._extract_record_types = mock.Mock()
task._sql_bulk_insert_from_records = mock.Mock()
task.session = mock.Mock()
task.org_config._is_person_accounts_enabled = False
step = mock.Mock()
step.get_results.return_value = [["000000000000001", "Test", "012000000000000"]]
mapping = MappingStep(
sf_object="Account",
fields={"Name": "Name", "RecordTypeId": "RecordTypeId"},
lookups={},
table="accounts",
)
task._import_results(
mapping,
step,
)
task._extract_record_types.assert_called_once_with(
"Account",
mapping.get_source_record_type_table(),
task.session.connection.return_value,
)
def _execute_step(
self,
mapping: MappingStep) -> Union[DataOperationJobResult, MagicMock]:
"""Load data for a single step."""
if mapping.get("fields", {}).get("RecordTypeId"):
conn = self.session.connection()
self._load_record_types([mapping["sf_object"]], conn)
self.session.commit()
mapping["oid_as_pk"] = bool(mapping.get("fields", {}).get("Id"))
bulk_mode = mapping.get("bulk_mode") or self.bulk_mode or "Parallel"
step = BulkApiDmlOperation(
sobject=mapping["sf_object"],
operation=(DataOperationType.INSERT if mapping.get("action")
== "insert" else DataOperationType.UPDATE),
api_options={"bulk_mode": bulk_mode},
context=self,
fields=self._get_columns(mapping),
)
local_ids = []
step.start()
step.load_records(self._stream_queried_data(mapping, local_ids))
step.end()
if step.job_result.status is not DataOperationStatus.JOB_FAILURE:
self._process_job_results(mapping, step, local_ids)
return step.job_result
def test_soql_for_mapping(self):
task = _make_task(
ExtractData,
{"options": {
"database_url": "sqlite:///",
"mapping": ""
}})
mapping = MappingStep(
sf_object="Contact",
oid_as_pk=True,
fields={
"Id": "sf_id",
"Test__c": "Test"
},
)
assert task._soql_for_mapping(
mapping) == "SELECT Id, Test__c FROM Contact"
mapping = MappingStep(
sf_object="Contact",
record_type="Devel",
oid_as_pk=True,
fields={
"Id": "sf_id",
"Test__c": "Test"
},
)
assert (
task._soql_for_mapping(mapping) ==
"SELECT Id, Test__c FROM Contact WHERE RecordType.DeveloperName = 'Devel'"
)
def test_validate_field_dict__injection(self):
ms = MappingStep(sf_object="Account",
fields=["Id", "Name", "Test__c"],
action="insert")
assert ms._validate_field_dict(
CaseInsensitiveDict({
"Name": {
"createable": True
},
"npsp__Test__c": {
"createable": True
}
}),
ms.fields_,
lambda field: f"npsp__{field}",
False,
DataOperationType.INSERT,
)
assert ms.fields_ == {
"Id": "Id",
"Name": "Name",
"npsp__Test__c": "Test__c"
}
def test_validate_field_dict__fls_checks(self):
ms = MappingStep(sf_object="Account",
fields=["Id", "Name", "Website"],
action="insert")
assert ms._validate_field_dict(
CaseInsensitiveDict({
"Name": {
"createable": True
},
"Website": {
"createable": True
}
}),
ms.fields_,
None,
False,
DataOperationType.INSERT,
)
assert not ms._validate_field_dict(
CaseInsensitiveDict({
"Name": {
"createable": True
},
"Website": {
"createable": False
}
}),
ms.fields_,
None,
False,
DataOperationType.INSERT,
)
def test_create_table__record_type_mapping(self):
task = _make_task(
ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}}
)
task.mapping = {
"Insert Accounts": MappingStep(
sf_object="Account",
table="accounts",
fields={"Name": "Name", "RecordTypeId": "RecordTypeId"},
lookups={},
),
"Insert Other Accounts": MappingStep(
sf_object="Account",
fields={"Name": "Name", "RecordTypeId": "RecordTypeId"},
lookups={},
table="accounts_2",
),
}
task.org_config._is_person_accounts_enabled = False
def create_table_mock(table_name):
task.models[table_name] = mock.Mock()
task._create_record_type_table = mock.Mock(side_effect=create_table_mock)
task._init_db()
task._create_record_type_table.assert_called_once_with("Account_rt_mapping")
def test_convert_lookups_to_id(self):
task = _make_task(
ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}}
)
task.session = mock.Mock()
task.models = {
"Account": mock.Mock(),
"Account_sf_ids": mock.Mock(),
"Opportunity": mock.Mock(),
"Opportunity_sf_ids": mock.Mock(),
}
task.mapping = {
"Account": MappingStep(sf_object="Account"),
"Opportunity": MappingStep(sf_object="Opportunity"),
}
task._convert_lookups_to_id(
MappingStep(
sf_object="Opportunity",
lookups={"AccountId": MappingLookup(table="Account", name="AccountId")},
),
["AccountId"],
)
task.session.query.return_value.filter.return_value.update.assert_called_once_with(
{task.models["Opportunity"].AccountId: task.models["Account_sf_ids"].id},
synchronize_session=False,
)
task.session.commit.assert_called_once_with()
def test_validate_sobject__injection(self):
ms = MappingStep(
sf_object="Test__c", fields=["Name"], action=DataOperationType.INSERT
)
assert ms._validate_sobject(
CaseInsensitiveDict({"npsp__Test__c": {"createable": True}}),
lambda obj: f"npsp__{obj}",
DataOperationType.INSERT,
)
assert ms.sf_object == "npsp__Test__c"
def test_validate_and_inject_namespace__fls(self, mock_field,
mock_sobject):
ms = MappingStep(sf_object="Test__c",
fields=["Field__c"],
action="insert")
org_config = mock.Mock()
org_config.salesforce_client.describe.return_value = {
"sobjects": [{
"name": "Test__c",
"createable": True
}]
}
org_config.salesforce_client.Test__c.describe.return_value = {
"fields": [{
"name": "Field__c",
"createable": True
}]
}
assert ms.validate_and_inject_namespace(org_config, "ns",
DataOperationType.INSERT)
ms._validate_sobject.assert_called_once_with(
CaseInsensitiveDict(
{"Test__c": {
"name": "Test__c",
"createable": True
}}),
None,
DataOperationType.INSERT,
)
ms._validate_field_dict.assert_has_calls([
mock.call(
{"Field__c": {
"name": "Field__c",
"createable": True
}},
{"Field__c": "Field__c"},
None,
False,
DataOperationType.INSERT,
),
mock.call(
{"Field__c": {
"name": "Field__c",
"createable": True
}},
{},
None,
False,
DataOperationType.INSERT,
),
])
def test_import_results__relative_dates(self):
mock_describe_calls()
task = _make_task(
ExtractData,
{"options": {"database_url": "sqlite://", "mapping": "mapping.yml"}},
)
mapping = MappingStep(
sf_object="Opportunity",
fields={"Id": "sf_id", "CloseDate": "CloseDate"},
anchor_date="2020-07-01",
)
step = mock.Mock()
step.get_results.return_value = iter(
[["006000000000001", (date.today() + timedelta(days=9)).isoformat()]]
)
task.session = mock.Mock()
task._sql_bulk_insert_from_records = mock.Mock()
task._import_results(mapping, step)
task.session.connection.assert_called_once_with()
step.get_results.assert_called_once_with()
task._sql_bulk_insert_from_records.assert_called_once_with(
connection=task.session.connection.return_value,
table=mapping.table,
columns=["sf_id", "CloseDate"],
record_iterable=mock.ANY,
)
result = list(
task._sql_bulk_insert_from_records.call_args_list[0][1]["record_iterable"]
)
assert result == [["006000000000001", "2020-07-10"]]
def _execute_step(
self,
mapping: MappingStep) -> Union[DataOperationJobResult, MagicMock]:
"""Load data for a single step."""
if "RecordTypeId" in mapping.fields:
conn = self.session.connection()
self._load_record_types([mapping.sf_object], conn)
self.session.commit()
local_ids = []
query = self._query_db(mapping)
bulk_mode = mapping.bulk_mode or self.bulk_mode or "Parallel"
step = get_dml_operation(
sobject=mapping.sf_object,
operation=mapping.action,
api_options={
"batch_size": mapping.batch_size,
"bulk_mode": bulk_mode
},
context=self,
fields=mapping.get_field_list(),
api=mapping.api,
volume=query.count(),
)
step.start()
step.load_records(self._stream_queried_data(mapping, local_ids, query))
step.end()
if step.job_result.status is not DataOperationStatus.JOB_FAILURE:
self._process_job_results(mapping, step, local_ids)
return step.job_result
def test_import_results__no_columns(self): # , query_op_mock):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, self.mapping_file_v1)
mock_describe_calls()
task = _make_task(
ExtractData,
{"options": {"database_url": "sqlite://", "mapping": mapping_path}},
)
mapping = MappingStep(
sf_object="Opportunity",
table="Opportunity",
fields={},
lookups={},
)
step = mock.Mock()
step.get_results.return_value = [[1], [2]]
task.session = mock.Mock()
task._init_mapping()
task.mapping["Opportunity"] = mapping
with task._init_db():
task._import_results(mapping, step)
output_Opportunties = list(
task.session.execute("select * from Opportunity")
)
assert output_Opportunties == [(1,), (2,)]
def test_relative_datetimes(self):
mapping = MappingStep(
sf_object="Account", fields=["Some_Datetime__c"], anchor_date="2020-07-01"
)
input_dt = datetime_from_salesforce("2020-07-08T09:37:57.373+0000")
target = datetime.combine(date.today() + timedelta(days=7), input_dt.time())
assert (
adjust_relative_dates(
mapping,
([], [0], date.today()),
[salesforce_from_datetime(input_dt)],
DataOperationType.INSERT,
)
== [salesforce_from_datetime(target)]
)
now = datetime.combine(mapping.anchor_date, datetime.now().time())
assert (
adjust_relative_dates(
mapping,
([], [0], date.today()),
[salesforce_from_datetime(now)],
DataOperationType.INSERT,
)
== [salesforce_from_datetime(datetime.combine(date.today(), now.time()))]
)
assert adjust_relative_dates(
mapping, ([], [0], date.today()), [""], DataOperationType.INSERT
) == [""]
def test_extract_respects_key_field(self, log_mock):
task = _make_task(
ExtractData,
{"options": {"database_url": "sqlite://", "mapping": "mapping.yml"}},
)
mapping = MappingStep(
sf_object="Opportunity",
table="Opportunity",
fields={"Id": "sf_id", "Name": "Name"},
lookups={
"AccountId": MappingLookup(
table="Account", key_field="account_id", name="AccountId"
)
},
)
step = mock.Mock()
task.session = mock.Mock()
task._sql_bulk_insert_from_records = mock.Mock()
task._import_results(mapping, step)
task.session.connection.assert_called_once_with()
step.get_results.assert_called_once_with()
task._sql_bulk_insert_from_records.assert_called_once_with(
connection=task.session.connection.return_value,
table="Opportunity",
columns=["sf_id", "Name", "account_id"],
record_iterable=log_mock.return_value,
)
def test_validate_sobject(self):
ms = MappingStep(
sf_object="Account", fields=["Name"], action=DataOperationType.INSERT
)
assert ms._validate_sobject(
CaseInsensitiveDict({"Account": {"createable": True}}),
None,
DataOperationType.INSERT,
)
assert ms._validate_sobject(
CaseInsensitiveDict({"Account": {"queryable": True}}),
None,
DataOperationType.QUERY,
)
ms = MappingStep(
sf_object="Account", fields=["Name"], action=DataOperationType.UPDATE
)
assert not ms._validate_sobject(
CaseInsensitiveDict({"Account": {"updateable": False}}),
None,
DataOperationType.INSERT,
)
def test_get_complete_field_map(self):
m = MappingStep(
sf_object="Account",
fields=["Name", "AccountSite"],
lookups={"ParentId": MappingLookup(table="Account")},
)
assert m.get_complete_field_map() == {
"Name": "Name",
"AccountSite": "AccountSite",
"ParentId": "ParentId",
}
assert m.get_complete_field_map(include_id=True) == {
"Id": "sf_id",
"Name": "Name",
"AccountSite": "AccountSite",
"ParentId": "ParentId",
}
def test_run_query__failure(self, query_op_mock):
task = _make_task(
ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}}
)
query_op_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.JOB_FAILURE, [], 1, 0
)
with self.assertRaises(BulkDataException):
task._run_query("SELECT Id FROM Contact", MappingStep(sf_object="Contact"))
def test_validate_field_dict__injection(self):
ms = MappingStep(
sf_object="Account",
fields=["Id", "Name", "Test__c"],
action=DataOperationType.INSERT,
)
assert ms._validate_field_dict(
describe=CaseInsensitiveDict(
{"Name": {"createable": True}, "npsp__Test__c": {"createable": True}}
),
field_dict=ms.fields_,
inject=lambda field: f"npsp__{field}",
strip=None,
drop_missing=False,
data_operation_type=DataOperationType.INSERT,
)
assert ms.fields_ == {"Id": "Id", "Name": "Name", "npsp__Test__c": "Test__c"}
def test_get_permission_type(self):
ms = MappingStep(sf_object="Account", fields=["Name"], action="insert")
assert ms._get_permission_type(
DataOperationType.INSERT) == "createable"
assert ms._get_permission_type(DataOperationType.QUERY) == "queryable"
ms = MappingStep(sf_object="Account", fields=["Name"], action="update")
assert ms._get_permission_type(
DataOperationType.INSERT) == "updateable"
def test_create_table__autopk(self, mapper_mock, table_mock, create_mock):
task = _make_task(
ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}}
)
mapping = MappingStep(
sf_object="Account",
fields={"Name": "Name"},
table="accounts",
)
task.models = {}
task.metadata = mock.Mock()
task.org_config._is_person_accounts_enabled = False
task._create_table(mapping)
create_mock.assert_called_once_with(mapping, task.metadata)
assert len(table_mock.mock_calls) == 1
assert "accounts" in task.models
assert mapping.get_sf_id_table() in task.models
def _expand_mapping(self):
"""Walk the mapping and generate any required 'after' steps
to handle dependent and self-lookups."""
# Expand the mapping to handle dependent lookups
self.after_steps = defaultdict(dict)
for step in self.mapping.values():
if any([lookup.after for lookup in step.lookups.values()]):
# We have deferred/dependent lookups.
# Synthesize mapping steps for them.
sobject = step.sf_object
after_list = {
lookup.after
for lookup in step.lookups.values() if lookup.after
}
for after in after_list:
lookups = {
lookup_field: lookup
for lookup_field, lookup in step.lookups.items()
if lookup.after == after
}
name = f"Update {sobject} Dependencies After {after}"
mapping = MappingStep(
sf_object=sobject,
api=step.api,
action="update",
table=step.table,
)
mapping.lookups["Id"] = MappingLookup(
name="Id",
table=step["table"],
key_field=self.models[step["table"]].__table__.
primary_key.columns.keys()[0],
)
for lookup in lookups:
mapping.lookups[lookup] = lookups[lookup].copy()
mapping.lookups[lookup].after = None
self.after_steps[after][name] = mapping
def test_execute_step__record_type_mapping(self, step_mock):
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite://",
"mapping": "mapping.yml"
}
},
)
task.session = mock.Mock()
task._load_record_types = mock.Mock()
task._process_job_results = mock.Mock()
task._execute_step(
MappingStep(
**{
"sf_object": "Account",
"action": "insert",
"fields": {
"Name": "Name"
},
}))
task._load_record_types.assert_not_called()
task._execute_step(
MappingStep(
**{
"sf_object": "Account",
"action": "insert",
"fields": {
"Name": "Name",
"RecordTypeId": "RecordTypeId"
},
}))
task._load_record_types.assert_called_once_with(
["Account"], task.session.connection.return_value)
def test_import_results__autopk(self, csv_mock):
task = _make_task(
ExtractData,
{"options": {"database_url": "sqlite://", "mapping": "mapping.yml"}},
)
mapping = MappingStep(
sf_object="Opportunity",
table="Opportunity",
fields={"Name": "Name"},
lookups={"AccountId": MappingLookup(table="Account", name="AccountId")},
)
step = mock.Mock()
step.get_results.return_value = iter(
[["111", "Test Opportunity", "1"], ["222", "Test Opportunity 2", "1"]]
)
task.session = mock.Mock()
task._sql_bulk_insert_from_records = mock.Mock()
task._import_results(mapping, step)
task.session.connection.assert_called_once_with()
step.get_results.assert_called_once_with()
task._sql_bulk_insert_from_records.assert_has_calls(
[
mock.call(
connection=task.session.connection.return_value,
table="Opportunity",
columns=["Name", "AccountId"],
record_iterable=csv_mock.return_value,
),
mock.call(
connection=task.session.connection.return_value,
table=mapping.get_sf_id_table(),
columns=["sf_id"],
record_iterable=csv_mock.return_value,
),
]
)
def test_run_query(self, query_op_mock):
task = _make_task(
ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}}
)
task._import_results = mock.Mock()
query_op_mock.return_value.job_result = DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 1, 0
)
task._run_query("SELECT Id FROM Contact", MappingStep(sf_object="Contact"))
query_op_mock.assert_called_once_with(
sobject="Contact",
fields=["Id"],
api=DataApi.SMART,
api_options={},
context=task,
query="SELECT Id FROM Contact",
)
query_op_mock.return_value.query.assert_called_once_with()
task._import_results.assert_called_once_with(
MappingStep(sf_object="Contact"), query_op_mock.return_value
)
def test_validate_field_dict__drop_missing(self):
ms = MappingStep(
sf_object="Account",
fields=["Id", "Name", "Website"],
action=DataOperationType.INSERT,
)
assert ms._validate_field_dict(
CaseInsensitiveDict({
"Name": {
"createable": True
},
"Website": {
"createable": False
}
}),
ms.fields_,
None,
True,
DataOperationType.INSERT,
)
assert ms.fields_ == {"Id": "Id", "Name": "Name"}
def test_convert_lookups_to_id__sqlite(self):
task = _make_task(
ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}}
)
task.session = mock.Mock()
task.models = {
"Account": mock.Mock(),
"Account_sf_ids": mock.Mock(),
"Opportunity": mock.Mock(),
"Opportunity_sf_ids": mock.Mock(),
}
task.mapping = {
"Account": MappingStep(sf_object="Account"),
"Opportunity": MappingStep(sf_object="Opportunity"),
}
task.session.query.return_value.filter.return_value.update.side_effect = (
NotImplementedError
)
item = mock.Mock()
task.session.query.return_value.join.return_value = [(item, "1")]
task._convert_lookups_to_id(
MappingStep(
sf_object="Opportunity",
lookups={"AccountId": MappingLookup(table="Account", name="AccountId")},
),
["AccountId"],
)
task.session.bulk_update_mappings.assert_called_once_with(
task.models["Opportunity"], [{"id": item.id, "AccountId": "1"}]
)
task.session.commit.assert_called_once_with()
def test_run_task__start_step(self):
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite://",
"mapping": "mapping.yml",
"start_step": "Insert Contacts",
}
},
)
task._init_db = mock.Mock()
task._init_mapping = mock.Mock()
task.mapping = {}
task.mapping["Insert Households"] = MappingStep(sf_object="one",
fields={})
task.mapping["Insert Contacts"] = MappingStep(sf_object="two",
fields={})
task.after_steps = {}
task._execute_step = mock.Mock(return_value=DataOperationJobResult(
DataOperationStatus.SUCCESS, [], 0, 0))
task()
task._execute_step.assert_called_once_with(
MappingStep(sf_object="two", fields={}))
