def test_create_task(self, mock_mkdir, mock_dir):
session = spark.SparkSession(self.master_url, self.web_url,
self.working_dir)
with self.assertRaises(AttributeError):
session.create_task(None)
with self.assertRaises(AttributeError):
session.create_task("abc")
template = mock.create_autospec(submission.Submission,
is_template=True)
with self.assertRaises(ValueError):
session.create_task(template)
del_sub = mock.create_autospec(submission.Submission,
is_template=False,
is_deleted=True)
with self.assertRaises(ValueError):
session.create_task(del_sub)
sub = mock.create_autospec(submission.Submission,
is_deleted=False,
is_template=False,
priority=const.PRIORITY_0,
uid="123",
payload={},
system_code=spark.SPARK_SYSTEM_CODE)
sub.name = "abc"
mock_dir.return_value = "/mock-work-dir/123"
task = session.create_task(sub)
self.assertEqual(task.master_url, self.master_url)
self.assertEqual(task.web_url, self.web_url)
self.assertEqual(task.name, "abc")
self.assertEqual(task.working_directory, "/mock-work-dir/123")
mock_mkdir.assert_called_once_with("./work/123", 0775)
mock_dir.assert_called_once_with("./work/123")
def test_create_task_fail_sys_code(self, mock_mkdir, mock_dir):
session = spark.SparkSession(self.master_url, self.web_url,
self.working_dir)
sub = mock.create_autospec(submission.Submission,
is_template=True,
system_code="ABC")
with self.assertRaises(ValueError):
session.create_task(sub)
self.assertEqual(mock_mkdir.call_count, 0)
self.assertEqual(mock_dir.call_count, 0)
def test_status(self, mock_applications):
session = spark.SparkSession(self.master_url, self.web_url,
self.working_dir)
mock_applications.return_value = None
self.assertEqual(session.status(), const.SYSTEM_UNAVAILABLE)
mock_applications.return_value = []
self.assertEqual(session.status(), const.SYSTEM_AVAILABLE)
mock_applications.return_value = [{
"completed": True
}, {
"completed": True
}]
self.assertEqual(session.status(), const.SYSTEM_AVAILABLE)
mock_applications.return_value = [{
"completed": True
}, {
"completed": False
}]
self.assertEqual(session.status(), const.SYSTEM_BUSY)
def test_init(self):
with self.assertRaises(StandardError):
spark.SparkSession("abc", self.web_url, self.working_dir)
with self.assertRaises(StandardError):
spark.SparkSession(self.master_url, "abc", self.working_dir)
with self.assertRaises(ValueError):
spark.SparkSession(self.master_url,
self.web_url,
self.working_dir,
num_executors="abc")
with self.assertRaises(ValueError):
spark.SparkSession(self.master_url,
self.web_url,
self.working_dir,
num_executors=1,
timeout="abc")
with self.assertRaises(AttributeError):
spark.SparkSession(self.master_url,
self.web_url,
self.working_dir,
num_executors=0)
with self.assertRaises(AttributeError):
spark.SparkSession(self.master_url,
self.web_url,
self.working_dir,
num_executors=1,
timeout=0.0)
# valid instance
session = spark.SparkSession(self.master_url, self.web_url,
self.working_dir)
self.assertEqual(session.master_url, self.master_url)
self.assertEqual(session.web_url, self.web_url)
self.assertEqual(session.working_dir, self.working_dir)
self.assertEqual(session.num_executors, 1)
self.assertEqual(session.timeout, 1.0)
self.assertNotEqual(session.scheduler, None)
def test_scheduler(self):
session = spark.SparkSession(self.master_url, self.web_url,
self.working_dir)
self.assertTrue(isinstance(session.scheduler, scheduler.Scheduler))
def test_system_uri(self):
session = spark.SparkSession(self.master_url, self.web_url,
self.working_dir)
self.assertEqual(session.system_uri().url, self.web_url)
self.assertEqual(session.system_uri().alias, "Spark Web UI")
def test_system_code(self):
session = spark.SparkSession(self.master_url, self.web_url,
self.working_dir)
self.assertEqual(session.system_code(), spark.SPARK_SYSTEM_CODE)