def test_validate_and_inject_namespace__fls_sobject_failure( self, mock_field, mock_sobject ): ms = MappingStep( sf_object="Test__c", fields=["Name"], action=DataOperationType.INSERT ) org_config = mock.Mock() org_config.salesforce_client.describe.return_value = { "sobjects": [{"name": "Test__c", "createable": False}] } org_config.salesforce_client.Test__c.describe.return_value = { "fields": [{"name": "Name", "createable": True}] } assert not ms.validate_and_inject_namespace( org_config, "ns", DataOperationType.INSERT ) ms._validate_sobject.assert_called_once_with( {"Test__c": {"name": "Test__c", "createable": False}}, None, DataOperationType.INSERT, ) ms._validate_field_dict.assert_not_called()
def test_get_relative_date_context(self): mapping = MappingStep( sf_object="Account", fields=["Some_Date__c", "Some_Datetime__c"], anchor_date="2020-07-01", ) org_config = mock.Mock() org_config.salesforce_client.Account.describe.return_value = { "fields": [ { "name": "Some_Date__c", "type": "date" }, { "name": "Some_Datetime__c", "type": "datetime" }, { "name": "Some_Bool__c", "type": "boolean" }, ] } assert mapping.get_relative_date_context(org_config) == ([0], [1], date.today())
def test_validate_field_dict__fls_checks(self): ms = MappingStep( sf_object="Account", fields=["Id", "Name", "Website"], action=DataOperationType.INSERT, ) assert ms._validate_field_dict( describe=CaseInsensitiveDict( {"Name": {"createable": True}, "Website": {"createable": True}} ), field_dict=ms.fields_, inject=None, strip=None, drop_missing=False, data_operation_type=DataOperationType.INSERT, ) assert not ms._validate_field_dict( describe=CaseInsensitiveDict( {"Name": {"createable": True}, "Website": {"createable": False}} ), field_dict=ms.fields_, inject=None, strip=None, drop_missing=False, data_operation_type=DataOperationType.INSERT, )
def test_import_results__record_type_mapping(self): base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, "recordtypes.yml") task = _make_task( ExtractData, {"options": {"database_url": "sqlite://", "mapping": mapping_path}}, ) task._extract_record_types = mock.Mock() task._sql_bulk_insert_from_records = mock.Mock() task.session = mock.Mock() task.org_config._is_person_accounts_enabled = False step = mock.Mock() step.get_results.return_value = [["000000000000001", "Test", "012000000000000"]] mapping = MappingStep( sf_object="Account", fields={"Name": "Name", "RecordTypeId": "RecordTypeId"}, lookups={}, table="accounts", ) task._import_results( mapping, step, ) task._extract_record_types.assert_called_once_with( "Account", mapping.get_source_record_type_table(), task.session.connection.return_value, )
def _execute_step( self, mapping: MappingStep) -> Union[DataOperationJobResult, MagicMock]: """Load data for a single step.""" if mapping.get("fields", {}).get("RecordTypeId"): conn = self.session.connection() self._load_record_types([mapping["sf_object"]], conn) self.session.commit() mapping["oid_as_pk"] = bool(mapping.get("fields", {}).get("Id")) bulk_mode = mapping.get("bulk_mode") or self.bulk_mode or "Parallel" step = BulkApiDmlOperation( sobject=mapping["sf_object"], operation=(DataOperationType.INSERT if mapping.get("action") == "insert" else DataOperationType.UPDATE), api_options={"bulk_mode": bulk_mode}, context=self, fields=self._get_columns(mapping), ) local_ids = [] step.start() step.load_records(self._stream_queried_data(mapping, local_ids)) step.end() if step.job_result.status is not DataOperationStatus.JOB_FAILURE: self._process_job_results(mapping, step, local_ids) return step.job_result
def test_soql_for_mapping(self): task = _make_task( ExtractData, {"options": { "database_url": "sqlite:///", "mapping": "" }}) mapping = MappingStep( sf_object="Contact", oid_as_pk=True, fields={ "Id": "sf_id", "Test__c": "Test" }, ) assert task._soql_for_mapping( mapping) == "SELECT Id, Test__c FROM Contact" mapping = MappingStep( sf_object="Contact", record_type="Devel", oid_as_pk=True, fields={ "Id": "sf_id", "Test__c": "Test" }, ) assert ( task._soql_for_mapping(mapping) == "SELECT Id, Test__c FROM Contact WHERE RecordType.DeveloperName = 'Devel'" )
def test_validate_field_dict__injection(self): ms = MappingStep(sf_object="Account", fields=["Id", "Name", "Test__c"], action="insert") assert ms._validate_field_dict( CaseInsensitiveDict({ "Name": { "createable": True }, "npsp__Test__c": { "createable": True } }), ms.fields_, lambda field: f"npsp__{field}", False, DataOperationType.INSERT, ) assert ms.fields_ == { "Id": "Id", "Name": "Name", "npsp__Test__c": "Test__c" }
def test_validate_field_dict__fls_checks(self): ms = MappingStep(sf_object="Account", fields=["Id", "Name", "Website"], action="insert") assert ms._validate_field_dict( CaseInsensitiveDict({ "Name": { "createable": True }, "Website": { "createable": True } }), ms.fields_, None, False, DataOperationType.INSERT, ) assert not ms._validate_field_dict( CaseInsensitiveDict({ "Name": { "createable": True }, "Website": { "createable": False } }), ms.fields_, None, False, DataOperationType.INSERT, )
def test_create_table__record_type_mapping(self): task = _make_task( ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}} ) task.mapping = { "Insert Accounts": MappingStep( sf_object="Account", table="accounts", fields={"Name": "Name", "RecordTypeId": "RecordTypeId"}, lookups={}, ), "Insert Other Accounts": MappingStep( sf_object="Account", fields={"Name": "Name", "RecordTypeId": "RecordTypeId"}, lookups={}, table="accounts_2", ), } task.org_config._is_person_accounts_enabled = False def create_table_mock(table_name): task.models[table_name] = mock.Mock() task._create_record_type_table = mock.Mock(side_effect=create_table_mock) task._init_db() task._create_record_type_table.assert_called_once_with("Account_rt_mapping")
def test_convert_lookups_to_id(self): task = _make_task( ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}} ) task.session = mock.Mock() task.models = { "Account": mock.Mock(), "Account_sf_ids": mock.Mock(), "Opportunity": mock.Mock(), "Opportunity_sf_ids": mock.Mock(), } task.mapping = { "Account": MappingStep(sf_object="Account"), "Opportunity": MappingStep(sf_object="Opportunity"), } task._convert_lookups_to_id( MappingStep( sf_object="Opportunity", lookups={"AccountId": MappingLookup(table="Account", name="AccountId")}, ), ["AccountId"], ) task.session.query.return_value.filter.return_value.update.assert_called_once_with( {task.models["Opportunity"].AccountId: task.models["Account_sf_ids"].id}, synchronize_session=False, ) task.session.commit.assert_called_once_with()
def test_validate_sobject__injection(self): ms = MappingStep( sf_object="Test__c", fields=["Name"], action=DataOperationType.INSERT ) assert ms._validate_sobject( CaseInsensitiveDict({"npsp__Test__c": {"createable": True}}), lambda obj: f"npsp__{obj}", DataOperationType.INSERT, ) assert ms.sf_object == "npsp__Test__c"
def test_validate_and_inject_namespace__fls(self, mock_field, mock_sobject): ms = MappingStep(sf_object="Test__c", fields=["Field__c"], action="insert") org_config = mock.Mock() org_config.salesforce_client.describe.return_value = { "sobjects": [{ "name": "Test__c", "createable": True }] } org_config.salesforce_client.Test__c.describe.return_value = { "fields": [{ "name": "Field__c", "createable": True }] } assert ms.validate_and_inject_namespace(org_config, "ns", DataOperationType.INSERT) ms._validate_sobject.assert_called_once_with( CaseInsensitiveDict( {"Test__c": { "name": "Test__c", "createable": True }}), None, DataOperationType.INSERT, ) ms._validate_field_dict.assert_has_calls([ mock.call( {"Field__c": { "name": "Field__c", "createable": True }}, {"Field__c": "Field__c"}, None, False, DataOperationType.INSERT, ), mock.call( {"Field__c": { "name": "Field__c", "createable": True }}, {}, None, False, DataOperationType.INSERT, ), ])
def test_import_results__relative_dates(self): mock_describe_calls() task = _make_task( ExtractData, {"options": {"database_url": "sqlite://", "mapping": "mapping.yml"}}, ) mapping = MappingStep( sf_object="Opportunity", fields={"Id": "sf_id", "CloseDate": "CloseDate"}, anchor_date="2020-07-01", ) step = mock.Mock() step.get_results.return_value = iter( [["006000000000001", (date.today() + timedelta(days=9)).isoformat()]] ) task.session = mock.Mock() task._sql_bulk_insert_from_records = mock.Mock() task._import_results(mapping, step) task.session.connection.assert_called_once_with() step.get_results.assert_called_once_with() task._sql_bulk_insert_from_records.assert_called_once_with( connection=task.session.connection.return_value, table=mapping.table, columns=["sf_id", "CloseDate"], record_iterable=mock.ANY, ) result = list( task._sql_bulk_insert_from_records.call_args_list[0][1]["record_iterable"] ) assert result == [["006000000000001", "2020-07-10"]]
def _execute_step( self, mapping: MappingStep) -> Union[DataOperationJobResult, MagicMock]: """Load data for a single step.""" if "RecordTypeId" in mapping.fields: conn = self.session.connection() self._load_record_types([mapping.sf_object], conn) self.session.commit() local_ids = [] query = self._query_db(mapping) bulk_mode = mapping.bulk_mode or self.bulk_mode or "Parallel" step = get_dml_operation( sobject=mapping.sf_object, operation=mapping.action, api_options={ "batch_size": mapping.batch_size, "bulk_mode": bulk_mode }, context=self, fields=mapping.get_field_list(), api=mapping.api, volume=query.count(), ) step.start() step.load_records(self._stream_queried_data(mapping, local_ids, query)) step.end() if step.job_result.status is not DataOperationStatus.JOB_FAILURE: self._process_job_results(mapping, step, local_ids) return step.job_result
def test_import_results__no_columns(self): # , query_op_mock): base_path = os.path.dirname(__file__) mapping_path = os.path.join(base_path, self.mapping_file_v1) mock_describe_calls() task = _make_task( ExtractData, {"options": {"database_url": "sqlite://", "mapping": mapping_path}}, ) mapping = MappingStep( sf_object="Opportunity", table="Opportunity", fields={}, lookups={}, ) step = mock.Mock() step.get_results.return_value = [[1], [2]] task.session = mock.Mock() task._init_mapping() task.mapping["Opportunity"] = mapping with task._init_db(): task._import_results(mapping, step) output_Opportunties = list( task.session.execute("select * from Opportunity") ) assert output_Opportunties == [(1,), (2,)]
def test_relative_datetimes(self): mapping = MappingStep( sf_object="Account", fields=["Some_Datetime__c"], anchor_date="2020-07-01" ) input_dt = datetime_from_salesforce("2020-07-08T09:37:57.373+0000") target = datetime.combine(date.today() + timedelta(days=7), input_dt.time()) assert ( adjust_relative_dates( mapping, ([], [0], date.today()), [salesforce_from_datetime(input_dt)], DataOperationType.INSERT, ) == [salesforce_from_datetime(target)] ) now = datetime.combine(mapping.anchor_date, datetime.now().time()) assert ( adjust_relative_dates( mapping, ([], [0], date.today()), [salesforce_from_datetime(now)], DataOperationType.INSERT, ) == [salesforce_from_datetime(datetime.combine(date.today(), now.time()))] ) assert adjust_relative_dates( mapping, ([], [0], date.today()), [""], DataOperationType.INSERT ) == [""]
def test_extract_respects_key_field(self, log_mock): task = _make_task( ExtractData, {"options": {"database_url": "sqlite://", "mapping": "mapping.yml"}}, ) mapping = MappingStep( sf_object="Opportunity", table="Opportunity", fields={"Id": "sf_id", "Name": "Name"}, lookups={ "AccountId": MappingLookup( table="Account", key_field="account_id", name="AccountId" ) }, ) step = mock.Mock() task.session = mock.Mock() task._sql_bulk_insert_from_records = mock.Mock() task._import_results(mapping, step) task.session.connection.assert_called_once_with() step.get_results.assert_called_once_with() task._sql_bulk_insert_from_records.assert_called_once_with( connection=task.session.connection.return_value, table="Opportunity", columns=["sf_id", "Name", "account_id"], record_iterable=log_mock.return_value, )
def test_validate_sobject(self): ms = MappingStep( sf_object="Account", fields=["Name"], action=DataOperationType.INSERT ) assert ms._validate_sobject( CaseInsensitiveDict({"Account": {"createable": True}}), None, DataOperationType.INSERT, ) assert ms._validate_sobject( CaseInsensitiveDict({"Account": {"queryable": True}}), None, DataOperationType.QUERY, ) ms = MappingStep( sf_object="Account", fields=["Name"], action=DataOperationType.UPDATE ) assert not ms._validate_sobject( CaseInsensitiveDict({"Account": {"updateable": False}}), None, DataOperationType.INSERT, )
def test_get_complete_field_map(self): m = MappingStep( sf_object="Account", fields=["Name", "AccountSite"], lookups={"ParentId": MappingLookup(table="Account")}, ) assert m.get_complete_field_map() == { "Name": "Name", "AccountSite": "AccountSite", "ParentId": "ParentId", } assert m.get_complete_field_map(include_id=True) == { "Id": "sf_id", "Name": "Name", "AccountSite": "AccountSite", "ParentId": "ParentId", }
def test_run_query__failure(self, query_op_mock): task = _make_task( ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}} ) query_op_mock.return_value.job_result = DataOperationJobResult( DataOperationStatus.JOB_FAILURE, [], 1, 0 ) with self.assertRaises(BulkDataException): task._run_query("SELECT Id FROM Contact", MappingStep(sf_object="Contact"))
def test_validate_field_dict__injection(self): ms = MappingStep( sf_object="Account", fields=["Id", "Name", "Test__c"], action=DataOperationType.INSERT, ) assert ms._validate_field_dict( describe=CaseInsensitiveDict( {"Name": {"createable": True}, "npsp__Test__c": {"createable": True}} ), field_dict=ms.fields_, inject=lambda field: f"npsp__{field}", strip=None, drop_missing=False, data_operation_type=DataOperationType.INSERT, ) assert ms.fields_ == {"Id": "Id", "Name": "Name", "npsp__Test__c": "Test__c"}
def test_get_permission_type(self): ms = MappingStep(sf_object="Account", fields=["Name"], action="insert") assert ms._get_permission_type( DataOperationType.INSERT) == "createable" assert ms._get_permission_type(DataOperationType.QUERY) == "queryable" ms = MappingStep(sf_object="Account", fields=["Name"], action="update") assert ms._get_permission_type( DataOperationType.INSERT) == "updateable"
def test_create_table__autopk(self, mapper_mock, table_mock, create_mock): task = _make_task( ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}} ) mapping = MappingStep( sf_object="Account", fields={"Name": "Name"}, table="accounts", ) task.models = {} task.metadata = mock.Mock() task.org_config._is_person_accounts_enabled = False task._create_table(mapping) create_mock.assert_called_once_with(mapping, task.metadata) assert len(table_mock.mock_calls) == 1 assert "accounts" in task.models assert mapping.get_sf_id_table() in task.models
def _expand_mapping(self): """Walk the mapping and generate any required 'after' steps to handle dependent and self-lookups.""" # Expand the mapping to handle dependent lookups self.after_steps = defaultdict(dict) for step in self.mapping.values(): if any([lookup.after for lookup in step.lookups.values()]): # We have deferred/dependent lookups. # Synthesize mapping steps for them. sobject = step.sf_object after_list = { lookup.after for lookup in step.lookups.values() if lookup.after } for after in after_list: lookups = { lookup_field: lookup for lookup_field, lookup in step.lookups.items() if lookup.after == after } name = f"Update {sobject} Dependencies After {after}" mapping = MappingStep( sf_object=sobject, api=step.api, action="update", table=step.table, ) mapping.lookups["Id"] = MappingLookup( name="Id", table=step["table"], key_field=self.models[step["table"]].__table__. primary_key.columns.keys()[0], ) for lookup in lookups: mapping.lookups[lookup] = lookups[lookup].copy() mapping.lookups[lookup].after = None self.after_steps[after][name] = mapping
def test_execute_step__record_type_mapping(self, step_mock): task = _make_task( LoadData, { "options": { "database_url": "sqlite://", "mapping": "mapping.yml" } }, ) task.session = mock.Mock() task._load_record_types = mock.Mock() task._process_job_results = mock.Mock() task._execute_step( MappingStep( **{ "sf_object": "Account", "action": "insert", "fields": { "Name": "Name" }, })) task._load_record_types.assert_not_called() task._execute_step( MappingStep( **{ "sf_object": "Account", "action": "insert", "fields": { "Name": "Name", "RecordTypeId": "RecordTypeId" }, })) task._load_record_types.assert_called_once_with( ["Account"], task.session.connection.return_value)
def test_import_results__autopk(self, csv_mock): task = _make_task( ExtractData, {"options": {"database_url": "sqlite://", "mapping": "mapping.yml"}}, ) mapping = MappingStep( sf_object="Opportunity", table="Opportunity", fields={"Name": "Name"}, lookups={"AccountId": MappingLookup(table="Account", name="AccountId")}, ) step = mock.Mock() step.get_results.return_value = iter( [["111", "Test Opportunity", "1"], ["222", "Test Opportunity 2", "1"]] ) task.session = mock.Mock() task._sql_bulk_insert_from_records = mock.Mock() task._import_results(mapping, step) task.session.connection.assert_called_once_with() step.get_results.assert_called_once_with() task._sql_bulk_insert_from_records.assert_has_calls( [ mock.call( connection=task.session.connection.return_value, table="Opportunity", columns=["Name", "AccountId"], record_iterable=csv_mock.return_value, ), mock.call( connection=task.session.connection.return_value, table=mapping.get_sf_id_table(), columns=["sf_id"], record_iterable=csv_mock.return_value, ), ] )
def test_run_query(self, query_op_mock): task = _make_task( ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}} ) task._import_results = mock.Mock() query_op_mock.return_value.job_result = DataOperationJobResult( DataOperationStatus.SUCCESS, [], 1, 0 ) task._run_query("SELECT Id FROM Contact", MappingStep(sf_object="Contact")) query_op_mock.assert_called_once_with( sobject="Contact", fields=["Id"], api=DataApi.SMART, api_options={}, context=task, query="SELECT Id FROM Contact", ) query_op_mock.return_value.query.assert_called_once_with() task._import_results.assert_called_once_with( MappingStep(sf_object="Contact"), query_op_mock.return_value )
def test_validate_field_dict__drop_missing(self): ms = MappingStep( sf_object="Account", fields=["Id", "Name", "Website"], action=DataOperationType.INSERT, ) assert ms._validate_field_dict( CaseInsensitiveDict({ "Name": { "createable": True }, "Website": { "createable": False } }), ms.fields_, None, True, DataOperationType.INSERT, ) assert ms.fields_ == {"Id": "Id", "Name": "Name"}
def test_convert_lookups_to_id__sqlite(self): task = _make_task( ExtractData, {"options": {"database_url": "sqlite:///", "mapping": ""}} ) task.session = mock.Mock() task.models = { "Account": mock.Mock(), "Account_sf_ids": mock.Mock(), "Opportunity": mock.Mock(), "Opportunity_sf_ids": mock.Mock(), } task.mapping = { "Account": MappingStep(sf_object="Account"), "Opportunity": MappingStep(sf_object="Opportunity"), } task.session.query.return_value.filter.return_value.update.side_effect = ( NotImplementedError ) item = mock.Mock() task.session.query.return_value.join.return_value = [(item, "1")] task._convert_lookups_to_id( MappingStep( sf_object="Opportunity", lookups={"AccountId": MappingLookup(table="Account", name="AccountId")}, ), ["AccountId"], ) task.session.bulk_update_mappings.assert_called_once_with( task.models["Opportunity"], [{"id": item.id, "AccountId": "1"}] ) task.session.commit.assert_called_once_with()
def test_run_task__start_step(self): task = _make_task( LoadData, { "options": { "database_url": "sqlite://", "mapping": "mapping.yml", "start_step": "Insert Contacts", } }, ) task._init_db = mock.Mock() task._init_mapping = mock.Mock() task.mapping = {} task.mapping["Insert Households"] = MappingStep(sf_object="one", fields={}) task.mapping["Insert Contacts"] = MappingStep(sf_object="two", fields={}) task.after_steps = {} task._execute_step = mock.Mock(return_value=DataOperationJobResult( DataOperationStatus.SUCCESS, [], 0, 0)) task() task._execute_step.assert_called_once_with( MappingStep(sf_object="two", fields={}))