def test_init_options__missing_input(self):
with self.assertRaises(TaskOptionsError):
_make_task(LoadData, {"options": {}})
def test_options_error(self):
with pytest.raises(TaskOptionsError):
_make_task(
GenerateMapping, {"options": {"path": "t", "break_cycles": "foo"}}
)
def test_run(self, step_mock):
responses.add(
method="GET",
url=
"https://example.com/services/data/v46.0/query/?q=SELECT+Id+FROM+RecordType+WHERE+SObjectType%3D%27Account%27AND+DeveloperName+%3D+%27HH_Account%27+LIMIT+1",
body=json.dumps({"records": [{
"Id": "1"
}]}),
status=200,
)
base_path = os.path.dirname(__file__)
db_path = os.path.join(base_path, "testdata.db")
mapping_path = os.path.join(base_path, self.mapping_file)
with temporary_dir() as d:
tmp_db_path = os.path.join(d, "testdata.db")
shutil.copyfile(db_path, tmp_db_path)
task = _make_task(
LoadData,
{
"options": {
"database_url": f"sqlite:///{tmp_db_path}",
"mapping": mapping_path,
}
},
)
task.bulk = mock.Mock()
task.sf = mock.Mock()
step = MockBulkApiDmlOperation(
sobject="Contact",
operation=DataOperationType.INSERT,
api_options={},
context=task,
fields=[],
)
step_mock.return_value = step
step.results = [
DataOperationResult("001000000000000", True, None),
DataOperationResult("003000000000000", True, None),
DataOperationResult("003000000000001", True, None),
]
task()
assert step.records == [
["TestHousehold", "1"],
["Test", "User", "*****@*****.**", "001000000000000"],
["Error", "User", "*****@*****.**", "001000000000000"],
]
hh_ids = task.session.query(
*task.metadata.tables["households_sf_ids"].columns).one()
assert hh_ids == ("1", "001000000000000")
task.session.close()
task.engine.dispose()
def test_is_any_custom_api_name(self):
t = _make_task(GenerateMapping, {"options": {"path": "t"}})
self.assertTrue(t._is_any_custom_api_name("Custom__c"))
self.assertFalse(t._is_any_custom_api_name("Standard"))
def test_is_core_field(self):
t = _make_task(GenerateMapping, {"options": {"path": "t"}})
self.assertTrue(t._is_core_field("Name"))
self.assertFalse(t._is_core_field("Custom__c"))
def test_accepts_include_list(self):
t = _make_task(
GenerateMapping, {"options": {"include": ["Foo", "Bar"], "path": "t"}}
)
self.assertEqual(["Foo", "Bar"], t.options["include"])
def test_build_mapping__strip_namespace(self, prompt):
t = _make_task(GenerateMapping, {"options": {"path": "t"}})
t.project_config.project__package__namespace = "ns"
prompt.return_value = "ns__Parent__c"
t.schema = {
"ns__Parent__c": {
"Name": self._mock_field("Name"),
"ns__Dependent__c": self._mock_field(
"ns__Dependent__c",
field_type="reference",
referenceTo=["ns__Child__c"],
),
},
"ns__Child__c": {
"Name": self._mock_field("Name"),
"ns__Parent__c": self._mock_field(
"ns__Parent__c",
field_type="reference",
referenceTo=["ns__Parent__c"],
),
"ns__Self__c": self._mock_field(
"ns__Self__c", field_type="reference", referenceTo=["ns__Child__c"]
),
},
}
t.refs = {
"ns__Child__c": {
"ns__Parent__c": {"ns__Parent__c": FieldData({"nillable": False})}
},
"ns__Parent__c": {
"ns__Child__c": {"ns__Dependent__c": FieldData({"nillable": True})}
},
}
t._build_mapping()
self.assertEqual(
["Insert Parent__c", "Insert Child__c"], list(t.mapping.keys())
)
self.assertEqual("Parent__c", t.mapping["Insert Parent__c"]["sf_object"])
self.assertEqual(["Name"], t.mapping["Insert Parent__c"]["fields"])
self.assertEqual(
["Dependent__c"], list(t.mapping["Insert Parent__c"]["lookups"].keys())
)
self.assertEqual(
"Child__c",
t.mapping["Insert Parent__c"]["lookups"]["Dependent__c"]["table"],
)
self.assertEqual("Child__c", t.mapping["Insert Child__c"]["sf_object"])
self.assertEqual(["Name"], t.mapping["Insert Child__c"]["fields"])
self.assertEqual(
["Parent__c", "Self__c"],
list(t.mapping["Insert Child__c"]["lookups"].keys()),
)
self.assertEqual(
"Parent__c", t.mapping["Insert Child__c"]["lookups"]["Parent__c"]["table"]
)
self.assertEqual(
"Child__c", t.mapping["Insert Child__c"]["lookups"]["Self__c"]["table"]
)
def test_postfixes_underscores_to_namespace(self):
t = _make_task(
GenerateMapping, {"options": {"namespace_prefix": "t", "path": "t"}}
)
self.assertEqual("t__", t.options["namespace_prefix"])
def test_splits_ignore_string(self):
t = _make_task(
GenerateMapping, {"options": {"ignore": "Account, Contact", "path": "t"}}
)
self.assertEqual(["Account", "Contact"], t.options["ignore"])
def test_expand_mapping_creates_after_steps(self):
base_path = os.path.dirname(__file__)
mapping_path = os.path.join(base_path, "mapping_after.yml")
task = _make_task(
LoadData,
{
"options": {
"database_url": "sqlite://",
"mapping": mapping_path
}
},
)
task._init_mapping()
model = mock.Mock()
model.__table__ = mock.Mock()
model.__table__.primary_key.columns.keys.return_value = ["sf_id"]
task.models = {"accounts": model, "contacts": model}
task._expand_mapping()
self.assertEqual({}, task.after_steps["Insert Opportunities"])
self.assertEqual(
[
"Update Account Dependencies After Insert Contacts",
"Update Contact Dependencies After Insert Contacts",
],
list(task.after_steps["Insert Contacts"].keys()),
)
lookups = {}
lookups["Id"] = {"table": "accounts", "key_field": "sf_id"}
lookups["Primary_Contact__c"] = MappingLookup(
table="contacts",
name="Primary_Contact__c",
)
self.assertEqual(
{
"sf_object": "Account",
"action": "update",
"table": "accounts",
"lookups": lookups,
"fields": {},
},
task.after_steps["Insert Contacts"]
["Update Account Dependencies After Insert Contacts"],
)
lookups = {}
lookups["Id"] = {"table": "contacts", "key_field": "sf_id"}
lookups["ReportsToId"] = MappingLookup(table="contacts",
name="ReportsToId")
self.assertEqual(
{
"sf_object": "Contact",
"action": "update",
"table": "contacts",
"fields": {},
"lookups": lookups,
},
task.after_steps["Insert Contacts"]
["Update Contact Dependencies After Insert Contacts"],
)
self.assertEqual(
["Update Account Dependencies After Insert Accounts"],
list(task.after_steps["Insert Accounts"].keys()),
)
lookups = {}
lookups["Id"] = {"table": "accounts", "key_field": "sf_id"}
lookups["ParentId"] = MappingLookup(table="accounts", name="ParentId")
self.assertEqual(
{
"sf_object": "Account",
"action": "update",
"table": "accounts",
"fields": {},
"lookups": lookups,
},
task.after_steps["Insert Accounts"]
["Update Account Dependencies After Insert Accounts"],
)
def test_is_field_mappable(self):
t = _make_task(
GenerateMapping, {"options": {"ignore": "Account.ParentId", "path": "t"}}
)
t.mapping_objects = ["Account", "Contact"]
self.assertTrue(
t._is_field_mappable(
"Account",
{"name": "Name", "type": "string", "label": "Name", "createable": True},
)
)
self.assertFalse(
t._is_field_mappable(
"Account",
{"name": "Name", "type": "base64", "label": "Name", "createable": True},
)
)
self.assertFalse(
t._is_field_mappable(
"Account",
{
"name": "Name",
"type": "string",
"label": "Name (Deprecated)",
"createable": True,
},
)
)
self.assertFalse(
t._is_field_mappable(
"Account",
{
"name": "ParentId",
"type": "reference",
"label": "Parent",
"createable": True,
"referenceTo": ["Account"],
},
)
)
self.assertFalse(
t._is_field_mappable(
"Account",
{
"name": "Name",
"type": "string",
"label": "Name",
"createable": False,
},
)
)
self.assertFalse(
t._is_field_mappable(
"Contact",
{
"name": "OwnerId",
"type": "reference",
"label": "Owner",
"createable": True,
"referenceTo": ["User", "Group"],
},
)
)
def test_init_options__bulk_mode_wrong(self):
with self.assertRaises(TaskOptionsError):
_make_task(LoadData, {"options": {"bulk_mode": "Test"}})
def test_init_options__missing_output(self):
with pytest.raises(TaskOptionsError):
_make_task(ExtractData, {"options": {}})
def test_no_options(self):
with self.assertRaises(Exception):
_make_task(GenerateDataFromYaml, {})
def test_build_mapping(self, prompt):
t = _make_task(GenerateMapping, {"options": {"path": "t"}})
prompt.return_value = "Account"
t.schema = {
"Account": {
"Id":
self._mock_field("Id"),
"Name":
self._mock_field("Name"),
"Dependent__c":
self._mock_field("Dependent__c",
field_type="reference",
referenceTo=["Child__c"]),
},
"Child__c": {
"Id":
self._mock_field("Id"),
"Name":
self._mock_field("Name"),
"Account__c":
self._mock_field("Account__c",
field_type="reference",
referenceTo=["Account"]),
"Self__c":
self._mock_field("Self__c",
field_type="reference",
referenceTo=["Child__c"]),
},
}
t.refs = {
"Child__c": {
"Account": set(["Account__c"])
},
"Account": {
"Child__c": set(["Dependent__c"])
},
}
t._build_mapping()
self.assertEqual(["Insert Account", "Insert Child__c"],
list(t.mapping.keys()))
self.assertEqual("Account", t.mapping["Insert Account"]["sf_object"])
self.assertEqual("Account", t.mapping["Insert Account"]["table"])
self.assertEqual(["Id", "Name"],
list(t.mapping["Insert Account"]["fields"].keys()))
self.assertEqual("sf_id", t.mapping["Insert Account"]["fields"]["Id"])
self.assertEqual("Name", t.mapping["Insert Account"]["fields"]["Name"])
self.assertEqual(["Dependent__c"],
list(t.mapping["Insert Account"]["lookups"].keys()))
self.assertEqual(
"Child__c",
t.mapping["Insert Account"]["lookups"]["Dependent__c"]["table"])
self.assertEqual("Child__c", t.mapping["Insert Child__c"]["sf_object"])
self.assertEqual("Child__c", t.mapping["Insert Child__c"]["table"])
self.assertEqual(["Id", "Name"],
list(t.mapping["Insert Child__c"]["fields"].keys()))
self.assertEqual(
["Account__c", "Self__c"],
list(t.mapping["Insert Child__c"]["lookups"].keys()),
)
self.assertEqual("sf_id", t.mapping["Insert Child__c"]["fields"]["Id"])
self.assertEqual("Name",
t.mapping["Insert Child__c"]["fields"]["Name"])
self.assertEqual(
"Account",
t.mapping["Insert Child__c"]["lookups"]["Account__c"]["table"])
self.assertEqual(
"Child__c",
t.mapping["Insert Child__c"]["lookups"]["Self__c"]["table"])
def test_defaults_options(self):
t = _make_task(GenerateMapping, {"options": {"path": "t"}})
self.assertEqual([], t.options["ignore"])
self.assertEqual("", t.options["namespace_prefix"])
