def test_with_param_query(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-param_file params.properties '
'-f "wordcount.pig"')) \
.load_parameters_from_file("params.properties") \
.run()
def test_configure_logging(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-logfile pig.log -brief -debug '
'-f "wordcount.pig"')) \
.log_config(logfile="pig.log", debug=True, brief=True) \
.run()
def test_log4j_configs_injections(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-log4jconf ~/log4j.properties '
'-f "wordcount.pig"')) \
.log4j_config("~/log4j.properties") \
.run()
def test_configure_logging(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-logfile pig.log -brief -debug '
'-f "wordcount.pig"')) \
.log_config(logfile="pig.log", debug=True, brief=True) \
.run()
def test_with_property_file(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-propertyFile pig.properties '
'-x mapreduce '
'-f "wordcount.pig"')) \
.with_property_file("pig.properties").using_mode().run()
def test_with_param_query(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-param_file params.properties '
'-f "wordcount.pig"')) \
.load_parameters_from_file("params.properties") \
.run()
def test_with_property_file(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-propertyFile pig.properties '
'-x mapreduce '
'-f "wordcount.pig"')) \
.with_property_file("pig.properties").using_mode().run()
def test_use_database(self):
_command = 'hive -e "test" --database hello'
hive = (
Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command))
.use_database("blabla")
.use_database("hello")
)
hive.run()
def test_log4j_configs_injections(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-log4jconf ~/log4j.properties '
'-f "wordcount.pig"')) \
.log4j_config("~/log4j.properties") \
.run()
def test_agent_with_options(self):
command = "flume-ng agent --name a1 --conf-file path/to/file --conf path/to/dir " \
"--plugins-path path/to/plugins1,path/to/plugins2,path/to/plugins3 -X4=test"
flume = Flume.agent(agent="a1",
conf_file="path/to/file",
executor=mock_executor(expected_command=command))
flume.with_jvm_X_option(value="test", name=4).load_configs_from_dir("path/to/dir")
flume.load_plugins_from_dirs(pathes=["path/to/plugins1,path/to/plugins2", "path/to/plugins3"])
flume.run()
def test_load_preconfigured_job(self):
_command = 'pig -brief -optimizer_off SplitFilter -optimizer_off ColumnMapKeyPrune -e "ls /"'
metastore = IniFileMetaStore(file=os.path.join(os.path.dirname(__file__), 'resources/pig/pig.ini'))
pig = Pig.load_preconfigured_job(job_name='pig test',
config=Configuration.load(
metastore=metastore,
readonly=False, accepts_nulls=True),
command_executor=mock_executor(expected_command=_command))
pig.without_split_filter().run()
def test_with_param_file(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-param param001=value001 '
'-param param002=value002 '
'-x mapreduce '
'-f "wordcount.pig"')) \
.with_parameter("param001", "value001").using_mode() \
.with_parameter("param002", "value002").run()
def test_with_param_file(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-param param001=value001 '
'-param param002=value002 '
'-x mapreduce '
'-f "wordcount.pig"')) \
.with_parameter("param001", "value001").using_mode() \
.with_parameter("param002", "value002").run()
def test_load_preconfigured_job(self):
_command = 'pig -brief -optimizer_off SplitFilter -optimizer_off ColumnMapKeyPrune -e "ls /"'
metastore = IniFileMetaStore(file=os.path.join(
os.path.dirname(__file__), 'resources/pig/pig.ini'))
pig = Pig.load_preconfigured_job(
job_name='pig test',
config=Configuration.load(metastore=metastore,
readonly=False,
accepts_nulls=True),
command_executor=mock_executor(expected_command=_command))
pig.without_split_filter().run()
def test_agent_with_options(self):
command = "flume-ng agent --name a1 --conf-file path/to/file --conf path/to/dir " \
"--plugins-path path/to/plugins1,path/to/plugins2,path/to/plugins3 -X4=test"
flume = Flume.agent(agent="a1",
conf_file="path/to/file",
executor=mock_executor(expected_command=command))
flume.with_jvm_X_option(value="test",
name=4).load_configs_from_dir("path/to/dir")
flume.load_plugins_from_dirs(
pathes=["path/to/plugins1,path/to/plugins2", "path/to/plugins3"])
flume.run()
def test_load_config(self):
_command = "hive -e \"test\" --define A=B --define C=D --hiveconf hello=world " \
"--hivevar A=B --hivevar C=D --database hive"
metastore = IniFileMetaStore(file=os.path.join(
os.path.dirname(__file__), 'resources/hive/hive.ini'))
hive = Hive.load_preconfigured_job(name='hive test',
config=Configuration.load(
metastore=metastore,
readonly=False, accepts_nulls=True),
executor=mock_executor(expected_command=_command)) \
.with_hive_conf("hello", "world")
hive.run()
def test_load_config(self):
_command = (
'hive -e "test" --define A=B --define C=D --hiveconf hello=world '
"--hivevar A=B --hivevar C=D --database hive"
)
metastore = IniFileMetaStore(file=os.path.join(os.path.dirname(__file__), "resources/hive/hive.ini"))
hive = Hive.load_preconfigured_job(
name="hive test",
config=Configuration.load(metastore=metastore, readonly=False, accepts_nulls=True),
executor=mock_executor(expected_command=_command),
).with_hive_conf("hello", "world")
hive.run()
def test_with_auxpath(self):
_command = "hive " \
"--auxpath dear user,hello "\
"-e \"test\" " \
"--define key=value " \
"--hivevar hello=user " \
"--database hello" \
hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
.with_auxillary_jars("dear user,hello") \
.add_hivevar("hello", "user") \
.use_database("hello") \
.define_variable("key", "value")
hive.run()
def test_with_auxpath(self):
_command = (
"hive "
"--auxpath dear user,hello "
'-e "test" '
"--define key=value "
"--hivevar hello=user "
"--database hello"
)
hive = (
Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command))
.with_auxillary_jars("dear user,hello")
.add_hivevar("hello", "user")
.use_database("hello")
.define_variable("key", "value")
)
hive.run()
def test_spark_submit_from_ini(self):
_command = "spark-submit " \
"--master local[10] " \
"--class test.SparkApp " \
"--name test_app " \
"--jars lib001.jar,lib002.jar,lib003.jar " \
"--files dim001.cache.txt,dim002.cache.txt " \
"--properties-file spark.app.configs " \
"--conf \"spark.app.name=test_app spark.executor.memory=512m " \
"spark.serializer=org.apache.spark.serializer.KryoSerializer\" " \
"application.jar " \
"10 test"
metastore=IniFileMetaStore(file=os.path.join(os.path.dirname(__file__), "resources", "spark", "spark.app.ini"))
spark = SparkApplication.load_preconfigured_job(
config=Configuration.load(metastore,
readonly=False),
name="test_spark_app",
executor=mock_executor(expected_command=_command)).application_jar("application.jar")
spark.run(10, "test")
def test_spark_submit_command_generation(self):
_command = "spark-submit " \
"--master local[10] " \
"--class test.SparkApp " \
"--name test_app " \
"--jars lib001.jar,lib002.jar,lib003.jar " \
"--files dim001.cache.txt,dim002.cache.txt " \
"--properties-file spark.app.configs " \
"--conf \"spark.app.name=test_app spark.executor.memory=512m\" " \
"application.jar " \
"10"
spark = SparkApplication(executor=mock_executor(expected_command=_command)) \
.master(SparkMaster.local(10)) \
.application(application_jar='application.jar', app_name="test_app", main_class="test.SparkApp") \
.classpath("lib001.jar", "lib002.jar").classpath("lib003.jar") \
.add_files("dim001.cache.txt") \
.add_files("dim002.cache.txt") \
.config_file(path="spark.app.configs") \
.with_config_option("spark.app.name", "test_app") \
.with_config_option("spark.executor.memory", "512m")
spark.run(10)
def test_run_script_from_string(self):
Pig.load_commands_from_string(
commands="ls /",
command_executor=mock_executor('pig -e "ls /"')).run()
def test_define_variable(self):
_command = "hive -e \"test\" --define hello=world"
hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
.define_variable("hello", "world")
hive.run()
def test_add_hivevar(self):
_command = 'hive -e "test" --hivevar hello=world'
hive = Hive.load_queries_from_string(
query="test", executor=mock_executor(expected_command=_command)
).add_hivevar("hello", "world")
hive.run()
def test_define_variable(self):
_command = 'hive -e "test" --define hello=world'
hive = Hive.load_queries_from_string(
query="test", executor=mock_executor(expected_command=_command)
).define_variable("hello", "world")
hive.run()
def test_optimization_disabling(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off SplitFilter -f "wordcount.pig"')) \
.without_split_filter().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off PushUpFilter -f "wordcount.pig"')) \
.without_pushup_filter().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off MergeFilter -f "wordcount.pig"')) \
.without_merge_filter().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off PushDownForeachFlatten -f "wordcount.pig"')) \
.without_push_down_foreach_flatten().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off LimitOptimizer -f "wordcount.pig"')) \
.without_limit_optimizer().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off ColumnMapKeyPrune -f "wordcount.pig"')) \
.without_column_map_key_prune().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off AddForEach -f "wordcount.pig"')) \
.without_add_foreach().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off MergeForEach -f "wordcount.pig"')) \
.without_merge_foreach().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off GroupByConstParallelSetter -f "wordcount.pig"')) \
.without_groupby_const_parallel_setter().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off All -f "wordcount.pig"')) \
.disable_all_optimizations().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-optimizer_off LimitOptimizer '
'-optimizer_off AddForEach '
'-f "wordcount.pig"')) \
.without_add_foreach().without_limit_optimizer().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-x tez '
'-optimizer_off LimitOptimizer '
'-optimizer_off AddForEach '
'-no_multiquery '
'-f "wordcount.pig"')) \
.without_add_foreach().using_mode(type="tez")\
.without_limit_optimizer() \
.without_multiquery().run()
def test_optimization_disabling(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off SplitFilter -f "wordcount.pig"')) \
.without_split_filter().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off PushUpFilter -f "wordcount.pig"')) \
.without_pushup_filter().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off MergeFilter -f "wordcount.pig"')) \
.without_merge_filter().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off PushDownForeachFlatten -f "wordcount.pig"')) \
.without_push_down_foreach_flatten().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off LimitOptimizer -f "wordcount.pig"')) \
.without_limit_optimizer().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off ColumnMapKeyPrune -f "wordcount.pig"')) \
.without_column_map_key_prune().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off AddForEach -f "wordcount.pig"')) \
.without_add_foreach().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off MergeForEach -f "wordcount.pig"')) \
.without_merge_foreach().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off GroupByConstParallelSetter -f "wordcount.pig"')) \
.without_groupby_const_parallel_setter().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -optimizer_off All -f "wordcount.pig"')) \
.disable_all_optimizations().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-optimizer_off LimitOptimizer '
'-optimizer_off AddForEach '
'-f "wordcount.pig"')) \
.without_add_foreach().without_limit_optimizer().run()
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig '
'-x tez '
'-optimizer_off LimitOptimizer '
'-optimizer_off AddForEach '
'-no_multiquery '
'-f "wordcount.pig"')) \
.without_add_foreach().using_mode(type="tez")\
.without_limit_optimizer() \
.without_multiquery().run()
def test_stop_broker(self):
_command = "kafka-server-stop.sh path/to/config"
Kafka.stop_broker(path_to_config="path/to/config", executor=mock_executor(expected_command=_command))
def test_create_topic(self):
_command = "kafka-run-class.sh kafka.admin.TopicCommand --create --zookeeper master1.mycluster:2181" \
" --topic Topic --partitions 3 --replication-factor 1"
Kafka.create_topic(name='Topic', replication_factor=1,
partitions=3, zookeeper_host="master1.mycluster:2181",
executor=mock_executor(expected_command=_command))
def test_run_producer(self):
_command = "kafka-run-class.sh Producer conf1 conf2"
Kafka.run_producer(name='Producer', args=["conf1", "conf2"], executor=mock_executor(expected_command=_command))
def test_run_consumer(self):
_command = "kafka-run-class.sh Consumer --zookeeper lcoalhost"
Kafka.run_consumer(name='Consumer', args={"--zookeeper": "lcoalhost"}, executor=mock_executor(expected_command=_command))
def test_agent_failed(self):
command = "flume-ng agent --name a1"
flume = Flume.agent(agent="a1",
executor=mock_executor(expected_command=command))
flume.run()
def test_use_database(self):
_command = "hive -e \"test\" --database hello"
hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
.use_database("blabla") \
.use_database("hello")
hive.run()
def test_run_script_from_file_verbose(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -verbose -f "wordcount.pig"')) \
.debug()
def test_get_list_topics(self):
_command = "kafka-run-class.sh kafka.admin.TopicCommand --zookeeper master1.mycluster:2181 --list"
Kafka.get_list_topics(zookeeper_host="master1.mycluster:2181", executor=mock_executor(expected_command=_command))
def test_get_metadta(self):
_command = "kafka-run-class.sh kafka.admin.TopicCommand --zookeeper localhost:2181 --topic Topic --describe"
topic = Topic(name="Topic", zookeeper_host="localhost:2181", executor=mock_executor(expected_command=_command))
topic.get_metadata()
def test_delete_config(self):
_command = "kafka-run-class.sh kafka.admin.TopicCommand --zookeeper localhost:2181 --topic Topic --alter deleteConfig kye1=value1"
topic = Topic(name="Topic", zookeeper_host="localhost:2181", executor=mock_executor(expected_command=_command))
topic.delete_config(key="kye1", value="value1")
def test_run_script_from_file_verbose(self):
Pig.load_commands_from_file(
path='wordcount.pig',
command_executor=mock_executor('pig -verbose -f "wordcount.pig"')) \
.debug()
def test_agent(self):
command = "flume-ng agent --name a1 --conf-file path/to/file"
flume = Flume.agent(agent="a1",
conf_file="path/to/file",
executor=mock_executor(expected_command=command))
flume.run()
def test_run_script_from_string(self):
Pig.load_commands_from_string(
commands="ls /",
command_executor=mock_executor('pig -e "ls /"')).run()
def test_agent_failed(self):
command = "flume-ng agent --name a1"
flume = Flume.agent(agent="a1",
executor=mock_executor(expected_command=command))
flume.run()
def test_agent(self):
command = "flume-ng agent --name a1 --conf-file path/to/file"
flume = Flume.agent(agent="a1",
conf_file="path/to/file",
executor=mock_executor(expected_command=command))
flume.run()
def test_add_hivevar(self):
_command = "hive -e \"test\" --hivevar hello=world"
hive = Hive.load_queries_from_string(query="test", executor=mock_executor(expected_command=_command)) \
.add_hivevar("hello", "world")
hive.run()