コード例 #1
0
 def setUp(self):
     self.domain = 'example.com'
     self.password = '******'
     self.hosts = '@, www'
     self.striped_hosts = map(lambda s: s.strip(), self.hosts.split(','))
     self.ip = '127.0.0.1'
     self.factory = ConfigFactory()
コード例 #2
0
    def prepare_to_run_command(self, cmd):
        # Check for config
        if not os.path.exists(constants.REPOS_CONFIG_FILE):
            print(f"{constants.REPOS_CONFIG_FILE} not found.")

            if not os.path.exists(constants.WORK_DIR):
                os.makedirs(constants.WORK_DIR)

            with open(constants.REPOS_CONFIG_FILE, "wb") as conf_file:
                remote_file = requests.get(constants.DEFAULT_CONFIG_URL)
                conf_file.write(remote_file.content)

                print("Generated default config")

        _ = ConfigFactory.get_instance()
コード例 #3
0
ファイル: artifacts.py プロジェクト: realbucksavage/pyarch
    def take_action(self, parsed_args):
        repo_name = parsed_args.repo if parsed_args.repo else constants.DEFAULT_REPO_NAME
        artifact_name = parsed_args.artifact
        repo = ConfigFactory.get_instance().get_repo(repo_name)

        repo_url, catalog_name = repo["url"], repo["catalog"]
        artifact_url = f"{repo_url}/catalogs/{catalog_name}/{artifact_name}.tar.gz"

        artifact = requests.get(artifact_url)
        with open(f"{os.getcwd()}/{artifact_name}", "wb") as artifact_archive:
            artifact_archive.write(artifact.content)

        artifact_file = tarfile.open(f"{os.getcwd()}/{artifact_name}", "r:gz")
        try:
            artifact_file.extractall()
        finally:
            artifact_file.close()
コード例 #4
0
import sys

from pyspark.sql import SparkSession

from config_factory import ConfigFactory
from step_factory import StepFactory

if __name__ == "__main__":

    if len(sys.argv) == 1:
        print("No config file given")
        exit(0)

    config_path = sys.argv[1]

    spark = SparkSession \
        .builder \
        .appName("MLPipeline") \
        .getOrCreate()

    print("Loading config from: " + config_path)
    step_config = ConfigFactory.create_config(config_path)
    print("Executing step...")
    StepFactory.create_step(spark, step_config).execute()
コード例 #5
0
from pyspark.sql import SparkSession

from config_factory import ConfigFactory
from src.step_factory import StepFactory

# Example from https://spark.apache.org/docs/latest/ml-pipeline.html#example-pipeline
if __name__ == "__main__":
    spark = SparkSession \
        .builder \
        .appName("MLPipeline") \
        .getOrCreate()

    step_config = ConfigFactory.create_config("config/pipeline_config.json")
    prediction = StepFactory.create_step(spark, step_config).execute()

    selected = prediction.select("id", "text", "probability", "prediction")
    for row in selected.collect():
        rid, text, prob, prediction = row
        print("(%d, %s) --> prob=%s, prediction=%f" %
              (rid, text, str(prob), prediction))

    spark.stop()
コード例 #6
0
class TestNamecheapConfig(unittest.TestCase):

    def setUp(self):
        self.domain = 'example.com'
        self.password = '******'
        self.hosts = '@, www'
        self.striped_hosts = map(lambda s: s.strip(), self.hosts.split(','))
        self.ip = '127.0.0.1'
        self.factory = ConfigFactory()

    def test_not_found(self):
        with self.assertRaises(lib.config.ConfigNotFound):
            lib.config.read_config('does_not_exist.cfg')

    def test_no_sections(self):
        config_path = self.factory.create_config()
        with self.assertRaises(lib.config.ConfigMissingDomain):
            lib.config.read_config(config_path)

    def test_no_hosts(self):
        config_path = self.factory.create_config({
            'domain': self.domain,
            'password': self.password,
            'ip': self.ip
        })
        with self.assertRaises(lib.config.ConfigMissingParameter):
            lib.config.read_config(config_path)

    def test_empty_hosts(self):
        config_path = self.factory.create_config({
            'domain': self.domain,
            'hosts': '',
            'password': self.password,
            'ip': self.ip
        })
        with self.assertRaises(lib.config.ConfigEmptyParameter):
            lib.config.read_config(config_path)

    def test_no_password(self):
        config_path = self.factory.create_config({
            'domain': self.domain,
            'hosts': '@',
            'ip': self.ip
        })
        with self.assertRaises(lib.config.ConfigMissingParameter):
            lib.config.read_config(config_path)

    def test_empty_password(self):
        config_path = self.factory.create_config({
            'domain': self.domain,
            'hosts': self.hosts,
            'password': '',
            'ip': self.ip
        })
        with self.assertRaises(lib.config.ConfigEmptyParameter):
            lib.config.read_config(config_path)

    def test_no_ip(self):
        config_path = self.factory.create_config({
            'domain': self.domain,
            'hosts': '@',
            'password': self.password
        })
        with self.assertRaises(lib.config.ConfigMissingParameter):
            lib.config.read_config(config_path)

    def test_empty_ip(self):
        config_path = self.factory.create_config({
            'domain': self.domain,
            'hosts': self.hosts,
            'password': self.password,
            'ip': ''
        })
        with self.assertRaises(lib.config.ConfigEmptyParameter):
            lib.config.read_config(config_path)

    def test_config_ok(self):
        config_path = self.factory.create_config({
            'domain': self.domain,
            'hosts': self.hosts,
            'password': self.password,
            'ip': self.ip
        })
        domains = lib.config.read_config(config_path)
        for domain in domains:
            self.assertEqual(domain.name, self.domain)
            self.assertListEqual(domain.hosts, self.striped_hosts)
            self.assertEqual(domain.password, self.password)
            self.assertEqual(domain.ip, self.ip)

    def tearDown(self):
        self.factory.destroy_configs()