def test_validate_options(self): with self.assertRaises(TaskOptionsError): _make_task(bulkdata.DeleteData, {"options": {"objects": ""}}) with self.assertRaises(TaskOptionsError): _make_task( bulkdata.DeleteData, {"options": {"objects": "a,b", "where": "x='y'"}} )
def test_run_with_where(self): query_job = "1" query_batch = "2" delete_job = "3" api = self._configure_mocks(query_job, query_batch, delete_job) task = _make_task( bulkdata.DeleteData, {"options": {"objects": "Contact", "where": "city='Goshen'"}}, ) def _init_class(): task.bulk = api task._init_class = _init_class task() api.create_query_job.assert_called_once_with("Contact", contentType="CSV") api.query.assert_called_once_with( query_job, "SELECT Id FROM Contact WHERE city='Goshen'" ) api.is_batch_done.assert_has_calls( [mock.call(query_batch, query_job), mock.call(query_batch, query_job)] ) api.create_job.assert_called_once_with("Contact", "delete") api.close_job.assert_has_calls([mock.call(query_job), mock.call(delete_job)])
def test_with_continuation_file(self): continuation_data = """ !snowfakery_globals id_manager: !snowfakery_ids last_used_ids: Account: 5 last_seen_obj_of_type: Account: &id001 !snowfakery_objectrow _tablename: Account _values: Name: Johnston incorporated id: 5 named_objects: blah: blah """ with NamedTemporaryFile() as temp_db: database_url = f"sqlite:///{temp_db.name}" with NamedTemporaryFile("w+") as continuation_file: continuation_file.write(continuation_data) continuation_file.flush() task = _make_task( GenerateDataFromYaml, { "options": { "generator_yaml": sample_yaml, "database_url": database_url, "mapping": vanilla_mapping_file, "continuation_file": continuation_file.name, } }, ) task() rows = self.assertRowsCreated(database_url) assert dict(rows[0])["id"] == 6
def test_with_nonexistent_continuation_file(self): with self.assertRaises(TaskOptionsError) as e: with NamedTemporaryFile() as temp_db: database_url = f"sqlite:///{temp_db.name}" task = _make_task( GenerateDataFromYaml, { "options": { "generator_yaml": sample_yaml, "database_url": database_url, "mapping": vanilla_mapping_file, "continuation_file": "/tmp/foobar/baz/jazz/continuation.yml", } }, ) task() rows = self.assertRowsCreated(database_url) assert dict(rows[0])["id"] == 6 assert "jazz" in str(e.exception) assert "does not exist" in str(e.exception)
def test_batching(self, _dataload): with NamedTemporaryFile() as t: database_url = f"sqlite:///{t.name}" task = _make_task( GenerateAndLoadDataFromYaml, { "options": { "generator_yaml": simple_yaml, "num_records": 14, "batch_size": 6, "database_url": database_url, "num_records_tablename": "Account", "data_generation_task": "cumulusci.tasks.bulkdata.generate_from_yaml.GenerateDataFromYaml", "reset_oids": False, } }, ) task() assert len(_dataload.mock_calls) == 3 engine = create_engine(database_url) connection = engine.connect() records = list(connection.execute("select * from Account")) assert len(records) == 14 % 6 # leftovers
def test_create_job__no_records(self): task = _make_task(bulkdata.DeleteData, {"options": {"objects": "Contact"}}) task._query_salesforce_for_records_to_delete = mock.Mock(return_value=[]) task.logger = mock.Mock() task._create_job("Contact") task.logger.info.assert_called_with( " No Contact objects found, skipping delete" )
def test_inaccessible_generator_yaml(self): with self.assertRaises(TaskOptionsError): task = _make_task( GenerateDataFromYaml, { "options": { "generator_yaml": sample_yaml / "junk", "num_records": 10, "num_records_tablename": "Account", } }, ) task()
def test_mismatched_options(self): with self.assertRaises(TaskOptionsError) as e: task = _make_task( GenerateDataFromYaml, { "options": { "generator_yaml": sample_yaml, "num_records": 10 } }, ) task() assert "without num_records_tablename" in str(e.exception)
def test_default_database(self): mapping_file = os.path.join(os.path.dirname(__file__), "mapping_v2.yml") with mock.patch( "cumulusci.tasks.bulkdata.base_generate_data_task.BaseGenerateDataTask._generate_data" ) as gen_data: task = _make_task( DummyBaseBatchDataTask, {"options": {"num_records": NUM_RECORDS, "mapping": mapping_file}}, ) task() gen_data.assert_called_once_with( "sqlite:///generated_data.db", mock.ANY, 20, 0 )
def test_upload_batches__error(self): task = _make_task(bulkdata.DeleteData, {"options": {"objects": "Contact"}}) api = mock.Mock() api.endpoint = "http://api" api.headers.return_value = {} api.raise_error.side_effect = Exception def _init_class(): task.bulk = api task._init_class = _init_class responses.add(responses.POST, "http://api/job/1/batch", body=b"", status=500) with self.assertRaises(Exception): list(task._upload_batches("1", [{"Id": "1"}]))
def test_parse_job_state(self): task = _make_task(bulkdata.DeleteData, {"options": { "objects": "Contact" }}) api = mock.Mock() api.jobNS = "http://ns" task.bulk = api self.assertEqual( ("InProgress", None), task._parse_job_state( '<root xmlns="http://ns">' " <batch><state>InProgress</state></batch>" " <batch><state>Failed</state><stateMessage>test</stateMessage></batch>" " <batch><state>Completed</state></batch>" "</root>"), ) self.assertEqual( ("Failed", ["test"]), task._parse_job_state( '<root xmlns="http://ns">' " <batch><state>Failed</state><stateMessage>test</stateMessage></batch>" " <batch><state>Completed</state></batch>" "</root>"), ) self.assertEqual( ("Completed", None), task._parse_job_state('<root xmlns="http://ns">' " <batch><state>Completed</state></batch>" " <batch><state>Completed</state></batch>" "</root>"), ) self.assertEqual( ("Aborted", None), task._parse_job_state( '<root xmlns="http://ns">' " <batch><state>Not Processed</state></batch>" " <batch><state>Completed</state></batch>" "</root>"), ) self.assertEqual( ("CompletedWithFailures", ["Failures detected: 200"]), task._parse_job_state( '<root xmlns="http://ns">' " <batch><state>Completed</state></batch>" " <numberRecordsFailed>200</numberRecordsFailed>" "</root>"), )
def test_vars(self): with NamedTemporaryFile() as t: database_url = f"sqlite:///{t.name}" with self.assertWarns(UserWarning): task = _make_task( GenerateDataFromYaml, { "options": { "generator_yaml": sample_yaml, "vars": "xyzzy:foo", "database_url": database_url, } }, ) task() self.assertRowsCreated(database_url)
def test_use_mapping_file(self): assert vanilla_mapping_file.exists() with NamedTemporaryFile() as temp_db: database_url = f"sqlite:///{temp_db.name}" task = _make_task( GenerateDataFromYaml, { "options": { "generator_yaml": sample_yaml, "database_url": database_url, "mapping": vanilla_mapping_file, } }, ) task() self.assertRowsCreated(database_url)
def test_simple(self): with NamedTemporaryFile() as t: database_url = f"sqlite:///{t.name}" task = _make_task( GenerateDataFromYaml, { "options": { "generator_yaml": sample_yaml, "num_records": 1, "database_url": database_url, "num_records_tablename": "Account", } }, ) task() self.assertRowsCreated(database_url)
def test_generate_mapping_file(self): with NamedTemporaryFile() as temp_mapping: with NamedTemporaryFile() as temp_db: database_url = f"sqlite:///{temp_db.name}" task = _make_task( GenerateDataFromYaml, { "options": { "generator_yaml": sample_yaml, "database_url": database_url, "generate_mapping_file": temp_mapping.name, } }, ) task() mapping = yaml.safe_load(open(temp_mapping.name)) assert mapping["Insert Account"]["fields"]
def test_factory(self): mapping_file = os.path.join(os.path.dirname(__file__), "mapping_v2.yml") with temporary_dir() as d: tmp_db_path = os.path.join(d, "temp.db") dburl = "sqlite:///" + tmp_db_path task = _make_task( GenerateDummyData, { "options": { "num_records": 10, "mapping": mapping_file, "database_url": dburl, } }, ) task()
def test_BaseBatchDataTask(self): mapping_file = os.path.join(os.path.dirname(__file__), "mapping_v2.yml") with temporary_dir() as d: tmp_db_path = os.path.join(d, "temp.db") dburl = "sqlite:///" + tmp_db_path task = _make_task( DummyBaseBatchDataTask, { "options": { "num_records": NUM_RECORDS, "mapping": mapping_file, "database_url": dburl, } }, ) task() assert DummyBaseBatchDataTask.was_called
def test_generate_continuation_file(self): with NamedTemporaryFile() as temp_continuation_file: with NamedTemporaryFile() as temp_db: database_url = f"sqlite:///{temp_db.name}" task = _make_task( GenerateDataFromYaml, { "options": { "generator_yaml": sample_yaml, "database_url": database_url, "generate_continuation_file": temp_continuation_file.name, } }, ) task() mapping = yaml.safe_load(open(temp_continuation_file.name)) assert mapping # internals of this file are not important to MetaCI
def test_no_options(self): with self.assertRaises(Exception): _make_task(GenerateDataFromYaml, {})