def test_kairntech(self, mock_post): task_runner = FileTaskRunner("task_runner", self.temp_path) file_reader = FileReader("xl reader", self.test_file_path) fp = FileFormater("fp", "VERBATIM", "DATE_INTER", "ID", []) task_runner.run_task(file_reader) task_runner.run_task(fp, [file_reader]) mock_post.return_value.json.return_value = {"access_token": "token"} client = KairntechClient('test', 'test', "http://mockurl.com") mock_post.return_value.json.return_value = { "categories": [{"label": "cat1"}]} topic_annotator = Annotator(client, "project", "model") sentiment_annotator = Annotator(client, "project", "model") k_task = KairntechTask("k_task", topic_annotator, sentiment_annotator) task_runner.run_task(k_task, [fp]) self.assertTrue(task_runner.output_filename(k_task).exists()) with open(task_runner.output_filename(k_task)) as f: data = json.load(f) self.assertEqual(data[0]["tonality"], "cat1") self.assertEqual(data[0]["themes"], [ {"key": "cat1", "tonality": "cat1"}])
def test_kairntech_on_csv_with_sentencizer(self, mock_post): task_runner = FileTaskRunner("task_runner", self.temp_path) file_reader = FileReader("xl reader", self.test_file_path_csv) fp = FileFormater("fp", "Verbatim", "Date_Reponse", "id", [ ], encoding="ISO 8859-1", generate_id=True, csv_separator=";") task_runner.run_task(file_reader) task_runner.run_task(fp, [file_reader]) mock_post.return_value.json.return_value = {"access_token": "token"} client = KairntechClient('test', 'test', "http://mockurl.com") mock_post.return_value.json.return_value = { "categories": [{"label": "cat1"}]} topic_annotator = Annotator(client, "project", "model") sentiment_annotator = Annotator(client, "project", "model") sentencizer = CustomSentencizer() k_task = KairntechTask("k_task", topic_annotator, sentiment_annotator, sentencizer) task_runner.run_task(k_task, [fp]) self.assertTrue(task_runner.output_filename(k_task).exists()) with open(task_runner.output_filename(k_task)) as f: data = json.load(f) self.assertEqual(data[0]["tonality"], "cat1") self.assertEqual(data[0]["themes"], [ {"key": "cat1", "tonality": "cat1"}])
def test_run(self): fp = FileFormater("fp", "VERBATIM", "DATE_INTER", "ID", []) task_runner = FileTaskRunner("task_runner", self.temp_path) file_reader = FileReader("xl reader", self.test_file_path) task_runner.run_task(file_reader) task_runner.run_task(fp, [file_reader]) self.assertTrue(task_runner.is_task_completed(fp))
def test_run_pipeline(self): fp = FileFormater("fp", "VERBATIM", "DATE_INTER", "ID", []) task_runner = FileTaskRunner("task_runner", self.temp_path) file_reader = FileReader("xl reader", self.test_file_path) pipeline = Pipeline(task_runner, [file_reader, fp]) pipeline.run() self.assertTrue(task_runner.output_filename(fp).exists()) self.assertTrue(task_runner.output_filename(file_reader).exists())
def test_file_task_reader(self): """Test running a file reader task""" task = FileReader("file_reader_task", str(self.test_file_path)) task_runner = FileTaskRunner("file_task_runner", folder=Path(self.temp_path), date="today", survey="me") task_runner.run_task(task) self.assertTrue(task_runner.is_task_completed(task))
def test_file_task_reader_as_dep(self): """Test using a file reader task as a dep""" dep_task = FileReader("file_reader_task_dep", str(self.test_file_path)) task = DummyTask("dummy task with dep") task_runner = FileTaskRunner("file_task_runner", folder=Path(self.temp_path), test="test") task_runner.run_task(task, [dep_task]) self.assertFalse(task_runner.is_task_completed(task)) task_runner.run_task(dep_task) task_runner.run_task(task, [dep_task]) self.assertTrue(task_runner.is_task_completed(task))
def test_recoding__with_drop(self): """Test recoding a column and creating new column""" meta = MetaColumn("Net Promoter Score") fp = FileFormater("fp", "VERBATIM", "DATE_INTER", "ID", [meta]) task_runner = FileTaskRunner("task_runner", self.temp_path) file_reader = FileReader("xl reader", self.test_file_path) recoder = Recoder("nps_recod", "Net Promoter Score", recod_nps, False, "NPS", True) task_runner.run_task(file_reader) task_runner.run_task(fp, [file_reader]) task_runner.run_task(recoder, [fp]) with open(task_runner.output_filename(recoder), "r") as f: output = json.load(f) self.assertIn(output[0]["data"]["NPS"], ["Promoteur", "Neutre", "Détracteur"]) self.assertNotIn("Net Promoter Score", output[0]["data"])
def test_meta_column(self): """Test renaming a column""" meta = MetaColumn("Country", "Pays") fp = FileFormater("fp", "VERBATIM", "DATE_INTER", "ID", [meta]) task_runner = FileTaskRunner("task_runner", self.temp_path) file_reader = FileReader("xl reader", self.test_file_path) task_runner.run_task(file_reader) task_runner.run_task(fp, [file_reader]) with open(task_runner.output_filename(fp), "r") as f: output = json.load(f) filters = [f for f in output[0]["data"]] self.assertTrue("Pays" in filters) self.assertTrue("text" in output[0]) self.assertTrue("dateInterview" in output[0]) self.assertTrue("id" in output[0]) self.assertFalse("Country" in filters) self.assertFalse("Net Promoter Score" in filters)