def _validate_mongodb_url(self, url):
try:
return util.URI(url).url
except:
raise StandardError(
"Invalid Mongo DB url '%s', expected mongodb://[user:password@]host:port"
% url)
def validate_web_url(value):
"""
Validate web url for Spark (Spark Web UI), this is also used to access Spark REST API.
Has format: http(s)://host:port.
:param value: unresolved web url value
:return: valid Spark web url as string
"""
try:
return util.URI(value).url
except StandardError as err:
raise StandardError(
"Failed to assign Spark web url for '%s', reason: %s" %
(value, err))
def validate_master_url(value):
"""
Validate Spark master address, which has a format: spark://host:port.
:param value: unresolved master address value
:return: valid Spark master address as string
"""
try:
uri = util.URI(value)
except StandardError as err:
raise StandardError(
"Failed to assign Spark master url for '%s', reason: %s" %
(value, err))
else:
if uri.scheme != "spark":
raise ValueError(
"Expected scheme to be 'spark' for master url '%s'" % value)
return uri.url
test_scheduler = mock.create_autospec(spark.SparkStandaloneScheduler,
spec_set=True,
instance=True)
test_scheduler.get_num_executors.return_value = 5
test_scheduler.executor_class.return_value = spark.SparkStandaloneExecutor
test_scheduler.get_metrics.return_value = {"a": 1, "b": 2}
test_scheduler.get_is_alive_statuses.return_value = {
"ex1": util.utcnow(),
"ex2": util.utcnow()
}
test_session = mock.create_autospec(spark.SparkSession,
spec_set=True,
instance=True)
test_session.system_code.return_value = "TEST"
test_session.system_uri.return_value = util.URI("http://*****:*****@mock.patch("src.queue.util")
@mock.patch("src.queue.pymongo")
def setUpModule(mock_pymongo, mock_util):
def test_invalid_port(self):
with self.assertRaises(StandardError):
util.URI("http://localhost")
with self.assertRaises(ValueError):
util.URI("http://localhost:ABC")
def test_invalid_host(self):
with self.assertRaises(StandardError):
util.URI("http://:8080")
def test_invalid_scheme(self):
with self.assertRaises(StandardError):
util.URI("localhost:8080")
def test_init(self):
uri = util.URI("http://localhost:8080")
self.assertNotEqual(uri, None)
def setUp(self):
self.uri1 = util.URI("http://*****:*****@127.0.0.1:3001", "Mongo")
def system_uri(self):
return util.URI(self.web_url, "Spark Web UI")