def test_kairntech(self, mock_post):
task_runner = FileTaskRunner("task_runner", self.temp_path)
file_reader = FileReader("xl reader", self.test_file_path)
fp = FileFormater("fp", "VERBATIM", "DATE_INTER", "ID", [])
task_runner.run_task(file_reader)
task_runner.run_task(fp, [file_reader])
mock_post.return_value.json.return_value = {"access_token": "token"}
client = KairntechClient('test', 'test', "http://mockurl.com")
mock_post.return_value.json.return_value = {
"categories": [{"label": "cat1"}]}
topic_annotator = Annotator(client, "project", "model")
sentiment_annotator = Annotator(client, "project", "model")
k_task = KairntechTask("k_task", topic_annotator, sentiment_annotator)
task_runner.run_task(k_task, [fp])
self.assertTrue(task_runner.output_filename(k_task).exists())
with open(task_runner.output_filename(k_task)) as f:
data = json.load(f)
self.assertEqual(data[0]["tonality"], "cat1")
self.assertEqual(data[0]["themes"], [
{"key": "cat1", "tonality": "cat1"}])
def test_kairntech_on_csv_with_sentencizer(self, mock_post):
task_runner = FileTaskRunner("task_runner", self.temp_path)
file_reader = FileReader("xl reader", self.test_file_path_csv)
fp = FileFormater("fp", "Verbatim", "Date_Reponse", "id", [
], encoding="ISO 8859-1", generate_id=True, csv_separator=";")
task_runner.run_task(file_reader)
task_runner.run_task(fp, [file_reader])
mock_post.return_value.json.return_value = {"access_token": "token"}
client = KairntechClient('test', 'test', "http://mockurl.com")
mock_post.return_value.json.return_value = {
"categories": [{"label": "cat1"}]}
topic_annotator = Annotator(client, "project", "model")
sentiment_annotator = Annotator(client, "project", "model")
sentencizer = CustomSentencizer()
k_task = KairntechTask("k_task", topic_annotator,
sentiment_annotator, sentencizer)
task_runner.run_task(k_task, [fp])
self.assertTrue(task_runner.output_filename(k_task).exists())
with open(task_runner.output_filename(k_task)) as f:
data = json.load(f)
self.assertEqual(data[0]["tonality"], "cat1")
self.assertEqual(data[0]["themes"], [
{"key": "cat1", "tonality": "cat1"}])
def test_run(self):
fp = FileFormater("fp", "VERBATIM", "DATE_INTER", "ID", [])
task_runner = FileTaskRunner("task_runner", self.temp_path)
file_reader = FileReader("xl reader", self.test_file_path)
task_runner.run_task(file_reader)
task_runner.run_task(fp, [file_reader])
self.assertTrue(task_runner.is_task_completed(fp))
def test_run_pipeline(self):
fp = FileFormater("fp", "VERBATIM", "DATE_INTER", "ID", [])
task_runner = FileTaskRunner("task_runner", self.temp_path)
file_reader = FileReader("xl reader", self.test_file_path)
pipeline = Pipeline(task_runner, [file_reader, fp])
pipeline.run()
self.assertTrue(task_runner.output_filename(fp).exists())
self.assertTrue(task_runner.output_filename(file_reader).exists())
def test_file_task_reader(self):
"""Test running a file reader task"""
task = FileReader("file_reader_task", str(self.test_file_path))
task_runner = FileTaskRunner("file_task_runner",
folder=Path(self.temp_path),
date="today",
survey="me")
task_runner.run_task(task)
self.assertTrue(task_runner.is_task_completed(task))
def test_file_task_reader_as_dep(self):
"""Test using a file reader task as a dep"""
dep_task = FileReader("file_reader_task_dep", str(self.test_file_path))
task = DummyTask("dummy task with dep")
task_runner = FileTaskRunner("file_task_runner",
folder=Path(self.temp_path),
test="test")
task_runner.run_task(task, [dep_task])
self.assertFalse(task_runner.is_task_completed(task))
task_runner.run_task(dep_task)
task_runner.run_task(task, [dep_task])
self.assertTrue(task_runner.is_task_completed(task))
def test_recoding__with_drop(self):
"""Test recoding a column and creating new column"""
meta = MetaColumn("Net Promoter Score")
fp = FileFormater("fp", "VERBATIM", "DATE_INTER", "ID", [meta])
task_runner = FileTaskRunner("task_runner", self.temp_path)
file_reader = FileReader("xl reader", self.test_file_path)
recoder = Recoder("nps_recod", "Net Promoter Score", recod_nps, False,
"NPS", True)
task_runner.run_task(file_reader)
task_runner.run_task(fp, [file_reader])
task_runner.run_task(recoder, [fp])
with open(task_runner.output_filename(recoder), "r") as f:
output = json.load(f)
self.assertIn(output[0]["data"]["NPS"],
["Promoteur", "Neutre", "Détracteur"])
self.assertNotIn("Net Promoter Score", output[0]["data"])
def test_meta_column(self):
"""Test renaming a column"""
meta = MetaColumn("Country", "Pays")
fp = FileFormater("fp", "VERBATIM", "DATE_INTER", "ID", [meta])
task_runner = FileTaskRunner("task_runner", self.temp_path)
file_reader = FileReader("xl reader", self.test_file_path)
task_runner.run_task(file_reader)
task_runner.run_task(fp, [file_reader])
with open(task_runner.output_filename(fp), "r") as f:
output = json.load(f)
filters = [f for f in output[0]["data"]]
self.assertTrue("Pays" in filters)
self.assertTrue("text" in output[0])
self.assertTrue("dateInterview" in output[0])
self.assertTrue("id" in output[0])
self.assertFalse("Country" in filters)
self.assertFalse("Net Promoter Score" in filters)