from pyspark.sql import SparkSession
from config_factory import ConfigFactory
from src.step_factory import StepFactory
# Example from https://spark.apache.org/docs/latest/ml-pipeline.html#example-pipeline
if __name__ == "__main__":
spark = SparkSession \
.builder \
.appName("MLPipeline") \
.getOrCreate()
step_config = ConfigFactory.create_config("config/pipeline_config.json")
prediction = StepFactory.create_step(spark, step_config).execute()
selected = prediction.select("id", "text", "probability", "prediction")
for row in selected.collect():
rid, text, prob, prediction = row
print("(%d, %s) --> prob=%s, prediction=%f" %
(rid, text, str(prob), prediction))
spark.stop()
import sys
from pyspark.sql import SparkSession
from config_factory import ConfigFactory
from step_factory import StepFactory
if __name__ == "__main__":
if len(sys.argv) == 1:
print("No config file given")
exit(0)
config_path = sys.argv[1]
spark = SparkSession \
.builder \
.appName("MLPipeline") \
.getOrCreate()
print("Loading config from: " + config_path)
step_config = ConfigFactory.create_config(config_path)
print("Executing step...")
StepFactory.create_step(spark, step_config).execute()
class TestNamecheapConfig(unittest.TestCase):
def setUp(self):
self.domain = 'example.com'
self.password = '******'
self.hosts = '@, www'
self.striped_hosts = map(lambda s: s.strip(), self.hosts.split(','))
self.ip = '127.0.0.1'
self.factory = ConfigFactory()
def test_not_found(self):
with self.assertRaises(lib.config.ConfigNotFound):
lib.config.read_config('does_not_exist.cfg')
def test_no_sections(self):
config_path = self.factory.create_config()
with self.assertRaises(lib.config.ConfigMissingDomain):
lib.config.read_config(config_path)
def test_no_hosts(self):
config_path = self.factory.create_config({
'domain': self.domain,
'password': self.password,
'ip': self.ip
})
with self.assertRaises(lib.config.ConfigMissingParameter):
lib.config.read_config(config_path)
def test_empty_hosts(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': '',
'password': self.password,
'ip': self.ip
})
with self.assertRaises(lib.config.ConfigEmptyParameter):
lib.config.read_config(config_path)
def test_no_password(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': '@',
'ip': self.ip
})
with self.assertRaises(lib.config.ConfigMissingParameter):
lib.config.read_config(config_path)
def test_empty_password(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': self.hosts,
'password': '',
'ip': self.ip
})
with self.assertRaises(lib.config.ConfigEmptyParameter):
lib.config.read_config(config_path)
def test_no_ip(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': '@',
'password': self.password
})
with self.assertRaises(lib.config.ConfigMissingParameter):
lib.config.read_config(config_path)
def test_empty_ip(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': self.hosts,
'password': self.password,
'ip': ''
})
with self.assertRaises(lib.config.ConfigEmptyParameter):
lib.config.read_config(config_path)
def test_config_ok(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': self.hosts,
'password': self.password,
'ip': self.ip
})
domains = lib.config.read_config(config_path)
for domain in domains:
self.assertEqual(domain.name, self.domain)
self.assertListEqual(domain.hosts, self.striped_hosts)
self.assertEqual(domain.password, self.password)
self.assertEqual(domain.ip, self.ip)
def tearDown(self):
self.factory.destroy_configs()