Exemplo n.º 1
0
 def test_with_param_query(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig '
                                        '-param_file params.properties '
                                        '-f "wordcount.pig"')) \
         .load_parameters_from_file("params.properties") \
         .run()
Exemplo n.º 2
0
 def test_configure_logging(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig '
                                        '-logfile pig.log -brief -debug '
                                        '-f "wordcount.pig"')) \
         .log_config(logfile="pig.log", debug=True, brief=True) \
         .run()
Exemplo n.º 3
0
 def test_log4j_configs_injections(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig '
                                        '-log4jconf ~/log4j.properties '
                                        '-f "wordcount.pig"')) \
         .log4j_config("~/log4j.properties") \
         .run()
Exemplo n.º 4
0
 def test_configure_logging(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig '
                                        '-logfile pig.log -brief -debug '
                                        '-f "wordcount.pig"')) \
         .log_config(logfile="pig.log", debug=True, brief=True) \
         .run()
Exemplo n.º 5
0
 def test_with_property_file(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig '
                                        '-propertyFile pig.properties '
                                        '-x mapreduce '
                                        '-f "wordcount.pig"')) \
         .with_property_file("pig.properties").using_mode().run()
Exemplo n.º 6
0
 def test_with_param_query(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig '
                                        '-param_file params.properties '
                                        '-f "wordcount.pig"')) \
         .load_parameters_from_file("params.properties") \
         .run()
Exemplo n.º 7
0
 def test_with_property_file(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig '
                                        '-propertyFile pig.properties '
                                        '-x mapreduce '
                                        '-f "wordcount.pig"')) \
         .with_property_file("pig.properties").using_mode().run()
Exemplo n.º 8
0
 def test_use_database(self):
     _command = 'hive -e "test" --database hello'
     hive = (
         Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command))
         .use_database("blabla")
         .use_database("hello")
     )
     hive.run()
Exemplo n.º 9
0
 def test_log4j_configs_injections(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig '
                                        '-log4jconf ~/log4j.properties '
                                        '-f "wordcount.pig"')) \
         .log4j_config("~/log4j.properties") \
         .run()
Exemplo n.º 10
0
 def test_agent_with_options(self):
     command = "flume-ng agent --name a1 --conf-file path/to/file --conf path/to/dir " \
               "--plugins-path path/to/plugins1,path/to/plugins2,path/to/plugins3 -X4=test"
     flume = Flume.agent(agent="a1",
                         conf_file="path/to/file",
                         executor=mock_executor(expected_command=command))
     flume.with_jvm_X_option(value="test", name=4).load_configs_from_dir("path/to/dir")
     flume.load_plugins_from_dirs(pathes=["path/to/plugins1,path/to/plugins2", "path/to/plugins3"])
     flume.run()
Exemplo n.º 11
0
 def test_load_preconfigured_job(self):
     _command = 'pig -brief -optimizer_off SplitFilter -optimizer_off ColumnMapKeyPrune -e "ls /"'
     metastore = IniFileMetaStore(file=os.path.join(os.path.dirname(__file__), 'resources/pig/pig.ini'))
     pig = Pig.load_preconfigured_job(job_name='pig test',
                                      config=Configuration.load(
                                          metastore=metastore,
                                          readonly=False, accepts_nulls=True),
                                      command_executor=mock_executor(expected_command=_command))
     pig.without_split_filter().run()
Exemplo n.º 12
0
 def test_with_param_file(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig '
                                        '-param param001=value001 '
                                        '-param param002=value002 '
                                        '-x mapreduce '
                                        '-f "wordcount.pig"')) \
         .with_parameter("param001", "value001").using_mode() \
         .with_parameter("param002", "value002").run()
Exemplo n.º 13
0
 def test_with_param_file(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig '
                                        '-param param001=value001 '
                                        '-param param002=value002 '
                                        '-x mapreduce '
                                        '-f "wordcount.pig"')) \
         .with_parameter("param001", "value001").using_mode() \
         .with_parameter("param002", "value002").run()
Exemplo n.º 14
0
 def test_load_preconfigured_job(self):
     _command = 'pig -brief -optimizer_off SplitFilter -optimizer_off ColumnMapKeyPrune -e "ls /"'
     metastore = IniFileMetaStore(file=os.path.join(
         os.path.dirname(__file__), 'resources/pig/pig.ini'))
     pig = Pig.load_preconfigured_job(
         job_name='pig test',
         config=Configuration.load(metastore=metastore,
                                   readonly=False,
                                   accepts_nulls=True),
         command_executor=mock_executor(expected_command=_command))
     pig.without_split_filter().run()
Exemplo n.º 15
0
 def test_agent_with_options(self):
     command = "flume-ng agent --name a1 --conf-file path/to/file --conf path/to/dir " \
               "--plugins-path path/to/plugins1,path/to/plugins2,path/to/plugins3 -X4=test"
     flume = Flume.agent(agent="a1",
                         conf_file="path/to/file",
                         executor=mock_executor(expected_command=command))
     flume.with_jvm_X_option(value="test",
                             name=4).load_configs_from_dir("path/to/dir")
     flume.load_plugins_from_dirs(
         pathes=["path/to/plugins1,path/to/plugins2", "path/to/plugins3"])
     flume.run()
Exemplo n.º 16
0
 def test_load_config(self):
     _command = "hive -e \"test\" --define A=B --define C=D --hiveconf hello=world " \
                "--hivevar A=B --hivevar C=D --database hive"
     metastore = IniFileMetaStore(file=os.path.join(
         os.path.dirname(__file__), 'resources/hive/hive.ini'))
     hive = Hive.load_preconfigured_job(name='hive test',
                                        config=Configuration.load(
                                            metastore=metastore,
                                            readonly=False, accepts_nulls=True),
                                        executor=mock_executor(expected_command=_command)) \
         .with_hive_conf("hello", "world")
     hive.run()
Exemplo n.º 17
0
 def test_load_config(self):
     _command = (
         'hive -e "test" --define A=B --define C=D --hiveconf hello=world '
         "--hivevar A=B --hivevar C=D --database hive"
     )
     metastore = IniFileMetaStore(file=os.path.join(os.path.dirname(__file__), "resources/hive/hive.ini"))
     hive = Hive.load_preconfigured_job(
         name="hive test",
         config=Configuration.load(metastore=metastore, readonly=False, accepts_nulls=True),
         executor=mock_executor(expected_command=_command),
     ).with_hive_conf("hello", "world")
     hive.run()
Exemplo n.º 18
0
    def test_with_auxpath(self):
        _command = "hive " \
                   "--auxpath dear user,hello "\
                   "-e \"test\" " \
                   "--define key=value " \
                   "--hivevar hello=user " \
                   "--database hello" \


        hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
            .with_auxillary_jars("dear user,hello") \
            .add_hivevar("hello", "user") \
            .use_database("hello") \
            .define_variable("key", "value")
        hive.run()
Exemplo n.º 19
0
    def test_with_auxpath(self):
        _command = (
            "hive "
            "--auxpath dear user,hello "
            '-e "test" '
            "--define key=value "
            "--hivevar hello=user "
            "--database hello"
        )

        hive = (
            Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command))
            .with_auxillary_jars("dear user,hello")
            .add_hivevar("hello", "user")
            .use_database("hello")
            .define_variable("key", "value")
        )
        hive.run()
Exemplo n.º 20
0
 def test_spark_submit_from_ini(self):
     _command = "spark-submit " \
                "--master local[10] " \
                "--class test.SparkApp " \
                "--name test_app " \
                "--jars lib001.jar,lib002.jar,lib003.jar " \
                "--files dim001.cache.txt,dim002.cache.txt " \
                "--properties-file spark.app.configs " \
                "--conf \"spark.app.name=test_app spark.executor.memory=512m " \
                "spark.serializer=org.apache.spark.serializer.KryoSerializer\" " \
                "application.jar " \
                "10 test"
     metastore=IniFileMetaStore(file=os.path.join(os.path.dirname(__file__), "resources", "spark", "spark.app.ini"))
     spark = SparkApplication.load_preconfigured_job(
         config=Configuration.load(metastore,
                                    readonly=False),
         name="test_spark_app",
         executor=mock_executor(expected_command=_command)).application_jar("application.jar")
     spark.run(10, "test")
Exemplo n.º 21
0
    def test_spark_submit_command_generation(self):
        _command = "spark-submit " \
                   "--master local[10] " \
                   "--class test.SparkApp " \
                   "--name test_app " \
                   "--jars lib001.jar,lib002.jar,lib003.jar " \
                   "--files dim001.cache.txt,dim002.cache.txt " \
                   "--properties-file spark.app.configs " \
                   "--conf \"spark.app.name=test_app spark.executor.memory=512m\" " \
                   "application.jar " \
                   "10"

        spark = SparkApplication(executor=mock_executor(expected_command=_command)) \
            .master(SparkMaster.local(10)) \
            .application(application_jar='application.jar', app_name="test_app", main_class="test.SparkApp") \
            .classpath("lib001.jar", "lib002.jar").classpath("lib003.jar") \
            .add_files("dim001.cache.txt") \
            .add_files("dim002.cache.txt") \
            .config_file(path="spark.app.configs") \
            .with_config_option("spark.app.name", "test_app") \
            .with_config_option("spark.executor.memory", "512m")
        spark.run(10)
Exemplo n.º 22
0
 def test_run_script_from_string(self):
     Pig.load_commands_from_string(
         commands="ls /",
         command_executor=mock_executor('pig -e "ls /"')).run()
Exemplo n.º 23
0
 def test_define_variable(self):
     _command = "hive -e \"test\" --define hello=world"
     hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
         .define_variable("hello", "world")
     hive.run()
Exemplo n.º 24
0
 def test_add_hivevar(self):
     _command = 'hive -e "test" --hivevar hello=world'
     hive = Hive.load_queries_from_string(
         query="test", executor=mock_executor(expected_command=_command)
     ).add_hivevar("hello", "world")
     hive.run()
Exemplo n.º 25
0
 def test_define_variable(self):
     _command = 'hive -e "test" --define hello=world'
     hive = Hive.load_queries_from_string(
         query="test", executor=mock_executor(expected_command=_command)
     ).define_variable("hello", "world")
     hive.run()
Exemplo n.º 26
0
    def test_optimization_disabling(self):
        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off SplitFilter -f "wordcount.pig"')) \
            .without_split_filter().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off PushUpFilter -f "wordcount.pig"')) \
            .without_pushup_filter().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off MergeFilter -f "wordcount.pig"')) \
            .without_merge_filter().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off PushDownForeachFlatten -f "wordcount.pig"')) \
            .without_push_down_foreach_flatten().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off LimitOptimizer -f "wordcount.pig"')) \
            .without_limit_optimizer().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off ColumnMapKeyPrune -f "wordcount.pig"')) \
            .without_column_map_key_prune().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off AddForEach -f "wordcount.pig"')) \
            .without_add_foreach().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off MergeForEach -f "wordcount.pig"')) \
            .without_merge_foreach().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off GroupByConstParallelSetter -f "wordcount.pig"')) \
            .without_groupby_const_parallel_setter().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off All -f "wordcount.pig"')) \
            .disable_all_optimizations().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig '
                                           '-optimizer_off LimitOptimizer '
                                           '-optimizer_off AddForEach '
                                           '-f "wordcount.pig"')) \
            .without_add_foreach().without_limit_optimizer().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig '
                                           '-x tez '
                                           '-optimizer_off LimitOptimizer '
                                           '-optimizer_off AddForEach '
                                           '-no_multiquery '
                                           '-f "wordcount.pig"')) \
            .without_add_foreach().using_mode(type="tez")\
            .without_limit_optimizer() \
            .without_multiquery().run()
Exemplo n.º 27
0
    def test_optimization_disabling(self):
        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off SplitFilter -f "wordcount.pig"')) \
            .without_split_filter().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off PushUpFilter -f "wordcount.pig"')) \
            .without_pushup_filter().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off MergeFilter -f "wordcount.pig"')) \
            .without_merge_filter().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off PushDownForeachFlatten -f "wordcount.pig"')) \
            .without_push_down_foreach_flatten().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off LimitOptimizer -f "wordcount.pig"')) \
            .without_limit_optimizer().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off ColumnMapKeyPrune -f "wordcount.pig"')) \
            .without_column_map_key_prune().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off AddForEach -f "wordcount.pig"')) \
            .without_add_foreach().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off MergeForEach -f "wordcount.pig"')) \
            .without_merge_foreach().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off GroupByConstParallelSetter -f "wordcount.pig"')) \
            .without_groupby_const_parallel_setter().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig -optimizer_off All -f "wordcount.pig"')) \
            .disable_all_optimizations().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig '
                                           '-optimizer_off LimitOptimizer '
                                           '-optimizer_off AddForEach '
                                           '-f "wordcount.pig"')) \
            .without_add_foreach().without_limit_optimizer().run()

        Pig.load_commands_from_file(
            path='wordcount.pig',
            command_executor=mock_executor('pig '
                                           '-x tez '
                                           '-optimizer_off LimitOptimizer '
                                           '-optimizer_off AddForEach '
                                           '-no_multiquery '
                                           '-f "wordcount.pig"')) \
            .without_add_foreach().using_mode(type="tez")\
            .without_limit_optimizer() \
            .without_multiquery().run()
Exemplo n.º 28
0
 def test_stop_broker(self):
     _command = "kafka-server-stop.sh path/to/config"
     Kafka.stop_broker(path_to_config="path/to/config", executor=mock_executor(expected_command=_command))
Exemplo n.º 29
0
 def test_create_topic(self):
     _command = "kafka-run-class.sh kafka.admin.TopicCommand --create --zookeeper master1.mycluster:2181" \
                " --topic Topic --partitions 3 --replication-factor 1"
     Kafka.create_topic(name='Topic', replication_factor=1,
                        partitions=3, zookeeper_host="master1.mycluster:2181",
                        executor=mock_executor(expected_command=_command))
Exemplo n.º 30
0
 def test_run_producer(self):
     _command = "kafka-run-class.sh Producer conf1 conf2"
     Kafka.run_producer(name='Producer', args=["conf1", "conf2"], executor=mock_executor(expected_command=_command))
Exemplo n.º 31
0
 def test_run_consumer(self):
     _command = "kafka-run-class.sh Consumer --zookeeper lcoalhost"
     Kafka.run_consumer(name='Consumer', args={"--zookeeper": "lcoalhost"}, executor=mock_executor(expected_command=_command))
Exemplo n.º 32
0
 def test_agent_failed(self):
     command = "flume-ng agent --name a1"
     flume = Flume.agent(agent="a1",
                         executor=mock_executor(expected_command=command))
     flume.run()
Exemplo n.º 33
0
 def test_use_database(self):
     _command = "hive -e \"test\" --database hello"
     hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
         .use_database("blabla") \
         .use_database("hello")
     hive.run()
Exemplo n.º 34
0
 def test_run_script_from_file_verbose(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig -verbose -f "wordcount.pig"')) \
         .debug()
Exemplo n.º 35
0
 def test_get_list_topics(self):
     _command = "kafka-run-class.sh kafka.admin.TopicCommand --zookeeper master1.mycluster:2181 --list"
     Kafka.get_list_topics(zookeeper_host="master1.mycluster:2181", executor=mock_executor(expected_command=_command))
Exemplo n.º 36
0
 def test_get_metadta(self):
     _command = "kafka-run-class.sh kafka.admin.TopicCommand --zookeeper localhost:2181 --topic Topic --describe"
     topic = Topic(name="Topic", zookeeper_host="localhost:2181", executor=mock_executor(expected_command=_command))
     topic.get_metadata()
Exemplo n.º 37
0
 def test_delete_config(self):
     _command = "kafka-run-class.sh kafka.admin.TopicCommand --zookeeper localhost:2181 --topic Topic --alter deleteConfig kye1=value1"
     topic = Topic(name="Topic", zookeeper_host="localhost:2181", executor=mock_executor(expected_command=_command))
     topic.delete_config(key="kye1", value="value1")
Exemplo n.º 38
0
 def test_run_script_from_file_verbose(self):
     Pig.load_commands_from_file(
         path='wordcount.pig',
         command_executor=mock_executor('pig -verbose -f "wordcount.pig"')) \
         .debug()
Exemplo n.º 39
0
 def test_agent(self):
     command = "flume-ng agent --name a1 --conf-file path/to/file"
     flume = Flume.agent(agent="a1",
                         conf_file="path/to/file",
                         executor=mock_executor(expected_command=command))
     flume.run()
Exemplo n.º 40
0
 def test_run_script_from_string(self):
     Pig.load_commands_from_string(
         commands="ls /",
         command_executor=mock_executor('pig -e "ls /"')).run()
Exemplo n.º 41
0
 def test_agent_failed(self):
     command = "flume-ng agent --name a1"
     flume = Flume.agent(agent="a1",
                         executor=mock_executor(expected_command=command))
     flume.run()
Exemplo n.º 42
0
 def test_agent(self):
     command = "flume-ng agent --name a1 --conf-file path/to/file"
     flume = Flume.agent(agent="a1",
                         conf_file="path/to/file",
                         executor=mock_executor(expected_command=command))
     flume.run()
Exemplo n.º 43
0
 def test_add_hivevar(self):
     _command = "hive -e \"test\" --hivevar hello=world"
     hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
         .add_hivevar("hello", "world")
     hive.run()