def test_validate_options(self):
with self.assertRaises(TaskOptionsError):
_make_task(bulkdata.DeleteData, {"options": {"objects": ""}})
with self.assertRaises(TaskOptionsError):
_make_task(
bulkdata.DeleteData, {"options": {"objects": "a,b", "where": "x='y'"}}
)
def test_run_with_where(self):
query_job = "1"
query_batch = "2"
delete_job = "3"
api = self._configure_mocks(query_job, query_batch, delete_job)
task = _make_task(
bulkdata.DeleteData,
{"options": {"objects": "Contact", "where": "city='Goshen'"}},
)
def _init_class():
task.bulk = api
task._init_class = _init_class
task()
api.create_query_job.assert_called_once_with("Contact", contentType="CSV")
api.query.assert_called_once_with(
query_job, "SELECT Id FROM Contact WHERE city='Goshen'"
)
api.is_batch_done.assert_has_calls(
[mock.call(query_batch, query_job), mock.call(query_batch, query_job)]
)
api.create_job.assert_called_once_with("Contact", "delete")
api.close_job.assert_has_calls([mock.call(query_job), mock.call(delete_job)])
def test_with_continuation_file(self):
continuation_data = """
!snowfakery_globals
id_manager: !snowfakery_ids
last_used_ids:
Account: 5
last_seen_obj_of_type:
Account: &id001 !snowfakery_objectrow
_tablename: Account
_values:
Name: Johnston incorporated
id: 5
named_objects:
blah: blah
"""
with NamedTemporaryFile() as temp_db:
database_url = f"sqlite:///{temp_db.name}"
with NamedTemporaryFile("w+") as continuation_file:
continuation_file.write(continuation_data)
continuation_file.flush()
task = _make_task(
GenerateDataFromYaml,
{
"options": {
"generator_yaml": sample_yaml,
"database_url": database_url,
"mapping": vanilla_mapping_file,
"continuation_file": continuation_file.name,
}
},
)
task()
rows = self.assertRowsCreated(database_url)
assert dict(rows[0])["id"] == 6
def test_with_nonexistent_continuation_file(self):
with self.assertRaises(TaskOptionsError) as e:
with NamedTemporaryFile() as temp_db:
database_url = f"sqlite:///{temp_db.name}"
task = _make_task(
GenerateDataFromYaml,
{
"options": {
"generator_yaml":
sample_yaml,
"database_url":
database_url,
"mapping":
vanilla_mapping_file,
"continuation_file":
"/tmp/foobar/baz/jazz/continuation.yml",
}
},
)
task()
rows = self.assertRowsCreated(database_url)
assert dict(rows[0])["id"] == 6
assert "jazz" in str(e.exception)
assert "does not exist" in str(e.exception)
def test_batching(self, _dataload):
with NamedTemporaryFile() as t:
database_url = f"sqlite:///{t.name}"
task = _make_task(
GenerateAndLoadDataFromYaml,
{
"options": {
"generator_yaml": simple_yaml,
"num_records": 14,
"batch_size": 6,
"database_url": database_url,
"num_records_tablename": "Account",
"data_generation_task":
"cumulusci.tasks.bulkdata.generate_from_yaml.GenerateDataFromYaml",
"reset_oids": False,
}
},
)
task()
assert len(_dataload.mock_calls) == 3
engine = create_engine(database_url)
connection = engine.connect()
records = list(connection.execute("select * from Account"))
assert len(records) == 14 % 6 # leftovers
def test_create_job__no_records(self):
task = _make_task(bulkdata.DeleteData, {"options": {"objects": "Contact"}})
task._query_salesforce_for_records_to_delete = mock.Mock(return_value=[])
task.logger = mock.Mock()
task._create_job("Contact")
task.logger.info.assert_called_with(
" No Contact objects found, skipping delete"
)
def test_inaccessible_generator_yaml(self):
with self.assertRaises(TaskOptionsError):
task = _make_task(
GenerateDataFromYaml,
{
"options": {
"generator_yaml": sample_yaml / "junk",
"num_records": 10,
"num_records_tablename": "Account",
}
},
)
task()
def test_mismatched_options(self):
with self.assertRaises(TaskOptionsError) as e:
task = _make_task(
GenerateDataFromYaml,
{
"options": {
"generator_yaml": sample_yaml,
"num_records": 10
}
},
)
task()
assert "without num_records_tablename" in str(e.exception)
def test_default_database(self):
mapping_file = os.path.join(os.path.dirname(__file__), "mapping_v2.yml")
with mock.patch(
"cumulusci.tasks.bulkdata.base_generate_data_task.BaseGenerateDataTask._generate_data"
) as gen_data:
task = _make_task(
DummyBaseBatchDataTask,
{"options": {"num_records": NUM_RECORDS, "mapping": mapping_file}},
)
task()
gen_data.assert_called_once_with(
"sqlite:///generated_data.db", mock.ANY, 20, 0
)
def test_upload_batches__error(self):
task = _make_task(bulkdata.DeleteData, {"options": {"objects": "Contact"}})
api = mock.Mock()
api.endpoint = "http://api"
api.headers.return_value = {}
api.raise_error.side_effect = Exception
def _init_class():
task.bulk = api
task._init_class = _init_class
responses.add(responses.POST, "http://api/job/1/batch", body=b"", status=500)
with self.assertRaises(Exception):
list(task._upload_batches("1", [{"Id": "1"}]))
def test_parse_job_state(self):
task = _make_task(bulkdata.DeleteData,
{"options": {
"objects": "Contact"
}})
api = mock.Mock()
api.jobNS = "http://ns"
task.bulk = api
self.assertEqual(
("InProgress", None),
task._parse_job_state(
'<root xmlns="http://ns">'
" <batch><state>InProgress</state></batch>"
" <batch><state>Failed</state><stateMessage>test</stateMessage></batch>"
" <batch><state>Completed</state></batch>"
"</root>"),
)
self.assertEqual(
("Failed", ["test"]),
task._parse_job_state(
'<root xmlns="http://ns">'
" <batch><state>Failed</state><stateMessage>test</stateMessage></batch>"
" <batch><state>Completed</state></batch>"
"</root>"),
)
self.assertEqual(
("Completed", None),
task._parse_job_state('<root xmlns="http://ns">'
" <batch><state>Completed</state></batch>"
" <batch><state>Completed</state></batch>"
"</root>"),
)
self.assertEqual(
("Aborted", None),
task._parse_job_state(
'<root xmlns="http://ns">'
" <batch><state>Not Processed</state></batch>"
" <batch><state>Completed</state></batch>"
"</root>"),
)
self.assertEqual(
("CompletedWithFailures", ["Failures detected: 200"]),
task._parse_job_state(
'<root xmlns="http://ns">'
" <batch><state>Completed</state></batch>"
" <numberRecordsFailed>200</numberRecordsFailed>"
"</root>"),
)
def test_vars(self):
with NamedTemporaryFile() as t:
database_url = f"sqlite:///{t.name}"
with self.assertWarns(UserWarning):
task = _make_task(
GenerateDataFromYaml,
{
"options": {
"generator_yaml": sample_yaml,
"vars": "xyzzy:foo",
"database_url": database_url,
}
},
)
task()
self.assertRowsCreated(database_url)
def test_use_mapping_file(self):
assert vanilla_mapping_file.exists()
with NamedTemporaryFile() as temp_db:
database_url = f"sqlite:///{temp_db.name}"
task = _make_task(
GenerateDataFromYaml,
{
"options": {
"generator_yaml": sample_yaml,
"database_url": database_url,
"mapping": vanilla_mapping_file,
}
},
)
task()
self.assertRowsCreated(database_url)
def test_simple(self):
with NamedTemporaryFile() as t:
database_url = f"sqlite:///{t.name}"
task = _make_task(
GenerateDataFromYaml,
{
"options": {
"generator_yaml": sample_yaml,
"num_records": 1,
"database_url": database_url,
"num_records_tablename": "Account",
}
},
)
task()
self.assertRowsCreated(database_url)
def test_generate_mapping_file(self):
with NamedTemporaryFile() as temp_mapping:
with NamedTemporaryFile() as temp_db:
database_url = f"sqlite:///{temp_db.name}"
task = _make_task(
GenerateDataFromYaml,
{
"options": {
"generator_yaml": sample_yaml,
"database_url": database_url,
"generate_mapping_file": temp_mapping.name,
}
},
)
task()
mapping = yaml.safe_load(open(temp_mapping.name))
assert mapping["Insert Account"]["fields"]
def test_factory(self):
mapping_file = os.path.join(os.path.dirname(__file__),
"mapping_v2.yml")
with temporary_dir() as d:
tmp_db_path = os.path.join(d, "temp.db")
dburl = "sqlite:///" + tmp_db_path
task = _make_task(
GenerateDummyData,
{
"options": {
"num_records": 10,
"mapping": mapping_file,
"database_url": dburl,
}
},
)
task()
def test_BaseBatchDataTask(self):
mapping_file = os.path.join(os.path.dirname(__file__), "mapping_v2.yml")
with temporary_dir() as d:
tmp_db_path = os.path.join(d, "temp.db")
dburl = "sqlite:///" + tmp_db_path
task = _make_task(
DummyBaseBatchDataTask,
{
"options": {
"num_records": NUM_RECORDS,
"mapping": mapping_file,
"database_url": dburl,
}
},
)
task()
assert DummyBaseBatchDataTask.was_called
def test_generate_continuation_file(self):
with NamedTemporaryFile() as temp_continuation_file:
with NamedTemporaryFile() as temp_db:
database_url = f"sqlite:///{temp_db.name}"
task = _make_task(
GenerateDataFromYaml,
{
"options": {
"generator_yaml":
sample_yaml,
"database_url":
database_url,
"generate_continuation_file":
temp_continuation_file.name,
}
},
)
task()
mapping = yaml.safe_load(open(temp_continuation_file.name))
assert mapping # internals of this file are not important to MetaCI
def test_no_options(self):
with self.assertRaises(Exception):
_make_task(GenerateDataFromYaml, {})
