def test_main_stops_with_check_only(self, operation_mock, credential_mock):
context = Mock()
context.run.return_value = 0
credential_mock.return_value = Mock()
credential_mock.return_value.result = context
credential_mock.return_value.errors = []
operation_mock.return_value = Mock()
operation_mock.return_value.result = context
operation_mock.return_value.errors = []
m = Mock(side_effect=select_file)
with unittest.mock.patch("builtins.open", m):
with unittest.mock.patch(
"sys.argv",
[
"amaxa",
"-c",
"credentials-good.json",
"extraction-good.json",
"--check-only",
],
):
return_value = main()
credential_mock.assert_called_once_with(
json.loads(CREDENTIALS_GOOD_JSON), constants.OPTION_DEFAULTS["api-version"]
)
operation_mock.assert_called_once_with(
json.loads(EXTRACTION_GOOD_JSON), context
)
context.run.assert_not_called()
self.assertEqual(0, return_value)
def test_main_returns_error_with_errors_during_extraction(
self, operation_mock, credential_mock
):
context = Mock()
op = Mock()
op.run = Mock(return_value=-1)
op.stage = amaxa.LoadStage.INSERTS
op.global_id_map = {}
credential_mock.return_value = Mock()
credential_mock.return_value.result = context
credential_mock.return_value.errors = []
operation_mock.return_value = Mock()
operation_mock.return_value.result = op
operation_mock.return_value.errors = []
m = Mock(side_effect=select_file)
with unittest.mock.patch("builtins.open", m):
with unittest.mock.patch(
"sys.argv",
["amaxa", "-c", "credentials-good.yaml", "extraction-good.yaml"],
):
return_value = main()
self.assertEqual(-1, return_value)
def test_main_returns_error_with_bad_credentials(
self, operation_mock, credential_mock
):
context = Mock()
credential_mock.return_value = Mock()
credential_mock.return_value.result = None
credential_mock.return_value.errors = ["Test error occured."]
operation_mock.return_value = Mock()
operation_mock.return_value.result = context
operation_mock.return_value.errors = []
m = Mock(side_effect=select_file)
with unittest.mock.patch("builtins.open", m):
with unittest.mock.patch(
"sys.argv",
["amaxa", "-c", "credentials-bad.yaml", "extraction-good.yaml"],
):
return_value = main()
credential_mock.assert_called_once_with(
yaml.safe_load(CREDENTIALS_BAD), constants.OPTION_DEFAULTS["api-version"]
)
context.run.assert_not_called()
self.assertEqual(-1, return_value)
def test_main_returns_error_with_bad_state_file(
self, operation_mock, state_mock, credential_mock
):
credential_mock.return_value.errors = []
operation_mock.return_value.errors = []
state_mock.return_value.result = None
state_mock.return_value.errors = ["Test error occured."]
m = Mock(side_effect=select_file)
with unittest.mock.patch("builtins.open", m):
with unittest.mock.patch(
"sys.argv",
[
"amaxa",
"--load",
"-c",
"credentials-good.yaml",
"extraction-good.yaml",
"-s",
"state-good.yaml",
],
):
return_value = main()
credential_mock.assert_called_once_with(
yaml.safe_load(CREDENTIALS_GOOD_YAML),
constants.OPTION_DEFAULTS["api-version"],
)
state_mock.assert_called_once_with(
yaml.safe_load(STATE_GOOD_YAML), operation_mock.return_value.result
)
self.assertEqual(-1, return_value)
def test_main_calls_execute_with_yaml_input(self, operation_mock, credential_mock):
context = Mock()
context.run.return_value = 0
credential_mock.return_value = Mock()
credential_mock.return_value.result = context
credential_mock.return_value.errors = []
operation_mock.return_value = Mock()
operation_mock.return_value.result = context
operation_mock.return_value.errors = []
m = Mock(side_effect=select_file)
with unittest.mock.patch("builtins.open", m):
with unittest.mock.patch(
"sys.argv",
["amaxa", "-c", "credentials-good.yaml", "extraction-good.yaml"],
):
return_value = main()
credential_mock.assert_called_once_with(
yaml.safe_load(CREDENTIALS_GOOD_YAML),
constants.OPTION_DEFAULTS["api-version"],
)
operation_mock.assert_called_once_with(
yaml.safe_load(EXTRACTION_GOOD_YAML), context
)
context.run.assert_called_once_with()
self.assertEqual(0, return_value)
def test_main_returns_error_with_bad_extraction(self, operation_mock,
credential_mock):
context = Mock()
credential_mock.return_value = Mock()
credential_mock.return_value.result = context
credential_mock.return_value.errors = []
operation_mock.return_value = Mock()
operation_mock.return_value.result = None
operation_mock.return_value.errors = ["Test error occured."]
m = Mock(side_effect=select_file)
with unittest.mock.patch("builtins.open", m):
with unittest.mock.patch(
"sys.argv",
[
"amaxa", "-c", "credentials-good.yaml",
"extraction-bad.yaml"
],
):
return_value = main()
credential_mock.assert_called_once_with(
yaml.safe_load(CREDENTIALS_GOOD_YAML))
operation_mock.assert_called_once_with(yaml.safe_load(EXTRACTION_BAD),
context)
self.assertEqual(-1, return_value)
def test_main_calls_execute_with_json_input_load_mode(
self, operation_mock, credential_mock):
context = Mock()
context.run.return_value = 0
credential_mock.return_value = Mock()
credential_mock.return_value.result = context
credential_mock.return_value.errors = []
operation_mock.return_value = Mock()
operation_mock.return_value.result = context
operation_mock.return_value.errors = []
m = Mock(side_effect=select_file)
with unittest.mock.patch("builtins.open", m):
with unittest.mock.patch(
"sys.argv",
[
"amaxa",
"-c",
"credentials-good.json",
"--load",
"extraction-good.json",
],
):
return_value = main()
credential_mock.assert_called_once_with(
json.loads(CREDENTIALS_GOOD_JSON))
operation_mock.assert_called_once_with(
json.loads(EXTRACTION_GOOD_JSON), context, use_state=False)
context.run.assert_called_once_with()
self.assertEqual(0, return_value)
def test_round_trip_all_transforms(self):
with tempfile.TemporaryDirectory() as tempdir:
copy_tree("assets/test_data_transforms", tempdir)
with cd(tempdir):
with unittest.mock.patch(
"sys.argv",
[
"amaxa", "--load", "-c", "credentials-env.yml",
"test.yml"
],
):
self.assertEqual(0, main())
result = self.connection.query(
"SELECT Id, Name, Description FROM Account WHERE Name = 'Sandia Style Interiors'"
)
assert len(result["records"]) == 1
record = result["records"][0]
assert record["Name"] == "sandia style interiors"
assert record["Description"] == "LifestyleLifestyle"
self.connection.Account.update(
record["Id"], {"Name": record["Name"].upper()})
# Re-extract the data.
with unittest.mock.patch(
"sys.argv",
["amaxa", "-c", "credentials-env.yml", "test.yml"]):
self.assertEqual(0, main())
# Validate the results
with open("Account.csv", "r") as csv_file:
reader = csv.DictReader(csv_file)
for record in reader:
self.register_case_record("Account", record["Id"])
assert record["Name"].islower()
assert record["Description"] in [
"Lifestyle" * 4,
"VC" * 4,
"Government" * 4,
"Sustainability" * 4,
]
def test_main_saves_state_on_error(self, operation_mock, credential_mock):
context = Mock()
op = Mock()
op.run = Mock(return_value=-1)
op.stage = amaxa.LoadStage.INSERTS
op.global_id_map = {
amaxa.SalesforceId("001000000000001"):
amaxa.SalesforceId("001000000000002")
}
credential_mock.return_value = Mock()
credential_mock.return_value.result = context
credential_mock.return_value.errors = []
operation_mock.return_value = Mock()
operation_mock.return_value.result = op
operation_mock.return_value.errors = []
state_file.close = Mock()
m = Mock(side_effect=select_file)
with unittest.mock.patch("builtins.open", m):
with unittest.mock.patch(
"sys.argv",
[
"amaxa",
"-c",
"credentials-good.yaml",
"--load",
"extraction-good.yaml",
],
):
return_value = main()
self.assertEqual(-1, return_value)
contents = state_file.getvalue()
self.assertLess(0, len(contents))
state_file.close.assert_called_once_with()
yaml_state = yaml.safe_load(io.StringIO(contents))
self.assertIn("state", yaml_state)
self.assertIn("id-map", yaml_state["state"])
self.assertIn("stage", yaml_state["state"])
self.assertEqual(amaxa.LoadStage.INSERTS.value,
yaml_state["state"]["stage"])
self.assertEqual(
{str(k): str(v)
for k, v in op.global_id_map.items()},
yaml_state["state"]["id-map"],
)
def test_main_loads_state_with_use_state_option(
self, operation_mock, credential_mock
):
context = Mock()
op = Mock()
op.run = Mock(return_value=0)
credential_mock.return_value = Mock()
credential_mock.return_value.result = context
credential_mock.return_value.errors = []
operation_mock.return_value = Mock()
operation_mock.return_value.result = op
operation_mock.return_value.errors = []
m = Mock(side_effect=select_file)
with unittest.mock.patch("builtins.open", m):
with unittest.mock.patch(
"sys.argv",
[
"amaxa",
"-c",
"credentials-good.yaml",
"--load",
"extraction-good.yaml",
"--use-state",
"state-good.yaml",
],
):
return_value = main()
self.assertEqual(0, return_value)
self.assertEqual(amaxa.LoadStage.INSERTS, op.stage)
self.assertEqual(
{
amaxa.SalesforceId("001000000000001"): amaxa.SalesforceId(
"001000000000002"
),
amaxa.SalesforceId("001000000000003"): amaxa.SalesforceId(
"001000000000004"
),
},
op.global_id_map,
)
def test_main_errors_bad_api_version(self, operation_mock, credential_mock):
context = Mock()
context.run.return_value = 0
credential_mock.return_value = Mock()
credential_mock.return_value.result = context
credential_mock.return_value.errors = []
operation_mock.return_value = Mock()
operation_mock.return_value.result = context
operation_mock.return_value.errors = []
m = Mock(side_effect=select_file)
with unittest.mock.patch("builtins.open", m):
with unittest.mock.patch(
"sys.argv",
["amaxa", "-c", "credentials-good.yaml", "extraction-bad-api.yaml",],
):
return_value = main()
credential_mock.assert_not_called()
operation_mock.assert_not_called()
self.assertEqual(-1, return_value)
def test_round_trip_from_command_line(self):
with tempfile.TemporaryDirectory() as tempdir:
copy_tree("assets/test_data_csv", tempdir)
with cd(tempdir):
with unittest.mock.patch(
"sys.argv",
[
"amaxa", "--load", "-c", "credentials-env.yml",
"test.yml"
],
):
self.assertEqual(0, main())
# Read the data in so we can validate once it's re-extracted.
with open("test.yml", "r") as fh:
load_op = amaxa.loader.LoadOperationLoader(
yaml.safe_load(fh.read()),
amaxa.api.Connection(self.connection, "48.0"),
)
load_op.load()
sobject_list = [
s.sobjectname for s in load_op.result.steps
]
counts = defaultdict(lambda: 0)
names = defaultdict(lambda: set())
for sobject in sobject_list:
# For each object, look at the CSV to determine the record count
# and the name field(s) we should verify.
with open("{}.csv".format(sobject), "r") as csv_file:
reader = csv.DictReader(csv_file)
for record in reader:
counts[sobject] = counts[sobject] + 1
if "LastName" in record:
names[sobject].add(record["LastName"])
elif "Name" in record:
names[sobject].add(record["Name"])
# Re-extract the data.
with unittest.mock.patch(
"sys.argv",
["amaxa", "-c", "credentials-env.yml", "test.yml"]):
self.assertEqual(0, main())
# Validate the results
for sobject in sobject_list:
with open("{}.csv".format(sobject), "r") as csv_file:
reader = csv.DictReader(csv_file)
for record in reader:
self.register_case_record(sobject, record["Id"])
counts[sobject] = counts[sobject] - 1
if "LastName" in record:
names[sobject].remove(record["LastName"])
elif "Name" in record:
names[sobject].remove(record["Name"])
self.assertFalse(names[sobject])
self.assertEqual(0, counts[sobject])
