def setUp(self):
self.domain = 'example.com'
self.password = '******'
self.hosts = '@, www'
self.striped_hosts = map(lambda s: s.strip(), self.hosts.split(','))
self.ip = '127.0.0.1'
self.factory = ConfigFactory()
def prepare_to_run_command(self, cmd):
# Check for config
if not os.path.exists(constants.REPOS_CONFIG_FILE):
print(f"{constants.REPOS_CONFIG_FILE} not found.")
if not os.path.exists(constants.WORK_DIR):
os.makedirs(constants.WORK_DIR)
with open(constants.REPOS_CONFIG_FILE, "wb") as conf_file:
remote_file = requests.get(constants.DEFAULT_CONFIG_URL)
conf_file.write(remote_file.content)
print("Generated default config")
_ = ConfigFactory.get_instance()
def take_action(self, parsed_args):
repo_name = parsed_args.repo if parsed_args.repo else constants.DEFAULT_REPO_NAME
artifact_name = parsed_args.artifact
repo = ConfigFactory.get_instance().get_repo(repo_name)
repo_url, catalog_name = repo["url"], repo["catalog"]
artifact_url = f"{repo_url}/catalogs/{catalog_name}/{artifact_name}.tar.gz"
artifact = requests.get(artifact_url)
with open(f"{os.getcwd()}/{artifact_name}", "wb") as artifact_archive:
artifact_archive.write(artifact.content)
artifact_file = tarfile.open(f"{os.getcwd()}/{artifact_name}", "r:gz")
try:
artifact_file.extractall()
finally:
artifact_file.close()
import sys
from pyspark.sql import SparkSession
from config_factory import ConfigFactory
from step_factory import StepFactory
if __name__ == "__main__":
if len(sys.argv) == 1:
print("No config file given")
exit(0)
config_path = sys.argv[1]
spark = SparkSession \
.builder \
.appName("MLPipeline") \
.getOrCreate()
print("Loading config from: " + config_path)
step_config = ConfigFactory.create_config(config_path)
print("Executing step...")
StepFactory.create_step(spark, step_config).execute()
from pyspark.sql import SparkSession
from config_factory import ConfigFactory
from src.step_factory import StepFactory
# Example from https://spark.apache.org/docs/latest/ml-pipeline.html#example-pipeline
if __name__ == "__main__":
spark = SparkSession \
.builder \
.appName("MLPipeline") \
.getOrCreate()
step_config = ConfigFactory.create_config("config/pipeline_config.json")
prediction = StepFactory.create_step(spark, step_config).execute()
selected = prediction.select("id", "text", "probability", "prediction")
for row in selected.collect():
rid, text, prob, prediction = row
print("(%d, %s) --> prob=%s, prediction=%f" %
(rid, text, str(prob), prediction))
spark.stop()
class TestNamecheapConfig(unittest.TestCase):
def setUp(self):
self.domain = 'example.com'
self.password = '******'
self.hosts = '@, www'
self.striped_hosts = map(lambda s: s.strip(), self.hosts.split(','))
self.ip = '127.0.0.1'
self.factory = ConfigFactory()
def test_not_found(self):
with self.assertRaises(lib.config.ConfigNotFound):
lib.config.read_config('does_not_exist.cfg')
def test_no_sections(self):
config_path = self.factory.create_config()
with self.assertRaises(lib.config.ConfigMissingDomain):
lib.config.read_config(config_path)
def test_no_hosts(self):
config_path = self.factory.create_config({
'domain': self.domain,
'password': self.password,
'ip': self.ip
})
with self.assertRaises(lib.config.ConfigMissingParameter):
lib.config.read_config(config_path)
def test_empty_hosts(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': '',
'password': self.password,
'ip': self.ip
})
with self.assertRaises(lib.config.ConfigEmptyParameter):
lib.config.read_config(config_path)
def test_no_password(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': '@',
'ip': self.ip
})
with self.assertRaises(lib.config.ConfigMissingParameter):
lib.config.read_config(config_path)
def test_empty_password(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': self.hosts,
'password': '',
'ip': self.ip
})
with self.assertRaises(lib.config.ConfigEmptyParameter):
lib.config.read_config(config_path)
def test_no_ip(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': '@',
'password': self.password
})
with self.assertRaises(lib.config.ConfigMissingParameter):
lib.config.read_config(config_path)
def test_empty_ip(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': self.hosts,
'password': self.password,
'ip': ''
})
with self.assertRaises(lib.config.ConfigEmptyParameter):
lib.config.read_config(config_path)
def test_config_ok(self):
config_path = self.factory.create_config({
'domain': self.domain,
'hosts': self.hosts,
'password': self.password,
'ip': self.ip
})
domains = lib.config.read_config(config_path)
for domain in domains:
self.assertEqual(domain.name, self.domain)
self.assertListEqual(domain.hosts, self.striped_hosts)
self.assertEqual(domain.password, self.password)
self.assertEqual(domain.ip, self.ip)
def tearDown(self):
self.factory.destroy_configs()