def _create_job_binary(id, type):
binary = mock.Mock()
binary.id = id
binary.url = "savanna-db://42"
if edp.compare_job_type(type, 'Pig'):
binary.name = "script.pig"
elif edp.compare_job_type(type, 'MapReduce', 'Java'):
binary.name = "main.jar"
else:
binary.name = "script.q"
return binary
def test_compare_job_type(self):
self.assertTrue(
edp.compare_job_type("Java", "Java", "MapReduce", strict=True))
self.assertFalse(
edp.compare_job_type(MAPRED_STREAMING,
"Java",
"MapReduce",
strict=True))
self.assertTrue(
edp.compare_job_type(MAPRED_STREAMING, "Java", "MapReduce"))
self.assertFalse(
edp.compare_job_type("MapReduce", "Java", MAPRED_STREAMING))
def _create_job_exec(job_id, type, configs=None):
j_exec = mock.Mock()
j_exec.job_id = job_id
j_exec.job_configs = configs
if edp.compare_job_type(type, "Java"):
j_exec.job_configs['configs']['edp.java.main_class'] = _java_main_class
j_exec.job_configs['configs']['edp.java.java_opts'] = _java_opts
return j_exec
def _create_job(id, job_binary, type):
job = mock.Mock()
job.id = id
job.type = type
job.name = 'special_name'
if edp.compare_job_type(type, 'Pig', 'Hive'):
job.mains = [job_binary]
job.libs = None
else:
job.libs = [job_binary]
job.mains = None
return job
def get_possible_job_config(job_type):
if not edp.compare_job_type(job_type, *get_possible_job_types()):
return None
if edp.compare_job_type(job_type, 'Java'):
return {'job_config': {'configs': [], 'args': []}}
if edp.compare_job_type(job_type, 'MapReduce', 'Pig'):
#TODO(nmakhotkin) Savanna should return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/v1_2_1/resources/mapred-default.xml')
if edp.compare_job_type(job_type, 'MapReduce'):
cfg += xmlutils.load_hadoop_xml_defaults(
'service/edp/resources/mapred-job-config.xml')
elif edp.compare_job_type(job_type, 'Hive'):
#TODO(nmakhotkin) Savanna should return config based on specific plugin
cfg = xmlutils.load_hadoop_xml_defaults(
'plugins/vanilla/v1_2_1/resources/hive-default.xml')
# TODO(tmckay): args should be a list when bug #269968
# is fixed on the UI side
config = {'configs': cfg, "args": {}}
if not edp.compare_job_type('MapReduce', 'Java'):
config.update({'params': {}})
return {'job_config': config}
def edp_testing(self,
job_type,
job_data_list,
lib_data_list=None,
configs=None,
pass_input_output_args=False):
try:
swift = self.connect_to_swift()
container_name = 'Edp-test-%s' % str(uuid.uuid4())[:8]
swift.put_container(container_name)
swift.put_object(
container_name, 'input', ''.join(
random.choice(':' + ' ' + '\n' + string.ascii_lowercase)
for x in range(10000)))
except Exception as e:
with excutils.save_and_reraise_exception():
self.delete_swift_container(swift, container_name)
print(str(e))
input_id = None
output_id = None
job_id = None
job_execution = None
try:
job_binary_list = []
lib_binary_list = []
job_binary_internal_list = []
swift_input_url = 'swift://%s.savanna/input' % container_name
swift_output_url = 'swift://%s.savanna/output' % container_name
# Java jobs don't use data sources. Input/output paths must
# be passed as args with corresponding username/password configs
if not edp.compare_job_type(job_type, "Java"):
input_id = self._create_data_source(
'input-%s' % str(uuid.uuid4())[:8], 'swift',
swift_input_url)
output_id = self._create_data_source(
'output-%s' % str(uuid.uuid4())[:8], 'swift',
swift_output_url)
if job_data_list:
self._create_job_binaries(job_data_list,
job_binary_internal_list,
job_binary_list)
if lib_data_list:
self._create_job_binaries(lib_data_list,
job_binary_internal_list,
lib_binary_list)
job_id = self._create_job(
'Edp-test-job-%s' % str(uuid.uuid4())[:8], job_type,
job_binary_list, lib_binary_list)
if not configs:
configs = {}
# Append the input/output paths with the swift configs
# if the caller has requested it...
if edp.compare_job_type(job_type,
"Java") and pass_input_output_args:
self._add_swift_configs(configs)
if "args" in configs:
configs["args"].extend([swift_input_url, swift_output_url])
else:
configs["args"] = [swift_input_url, swift_output_url]
job_execution = self.savanna.job_executions.create(job_id,
self.cluster_id,
input_id,
output_id,
configs=configs)
if job_execution:
self._await_job_execution(job_execution)
except Exception as e:
with excutils.save_and_reraise_exception():
print(str(e))
finally:
self.delete_swift_container(swift, container_name)
self._delete_job(job_execution, job_id,
job_binary_list + lib_binary_list,
job_binary_internal_list, input_id, output_id)
def validate(input_data, output_data, job):
if not edp.compare_job_type(job.type, 'Pig', 'MapReduce', 'Hive', 'Java'):
raise RuntimeError
def run_job(job_execution):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, job_execution.cluster_id)
if cluster.status != 'Active':
return job_execution
job = conductor.job_get(ctx, job_execution.job_id)
if not edp.compare_job_type(job.type, 'Java'):
input_source = conductor.data_source_get(ctx, job_execution.input_id)
output_source = conductor.data_source_get(ctx, job_execution.output_id)
else:
input_source = None
output_source = None
#TODO(nprivalova): should be removed after all features implemented
validate(input_source, output_source, job)
for data_source in [input_source, output_source]:
if data_source and data_source.type == 'hdfs':
h.configure_cluster_for_hdfs(cluster, data_source)
hdfs_user = _get_hdfs_user(cluster)
oozie_server = _get_oozie_server(cluster)
wf_dir = create_workflow_dir(oozie_server, job, hdfs_user)
upload_job_files(oozie_server, wf_dir, job, hdfs_user)
creator = workflow_factory.get_creator(job)
# Do other job type specific setup here, for example
# uploading hive configuration
creator.configure_workflow_if_needed(cluster, wf_dir)
wf_xml = creator.get_workflow_xml(job_execution, input_source,
output_source)
path_to_workflow = upload_workflow_file(oozie_server, wf_dir, wf_xml,
hdfs_user)
rm_path = _get_resource_manager_path(cluster)
nn_path = cluster['info']['HDFS']['NameNode']
client = o.OozieClient(cluster['info']['JobFlow']['Oozie'] + "/oozie/",
_get_oozie_server(cluster))
job_parameters = {
"jobTracker": rm_path,
"nameNode": nn_path,
"user.name": hdfs_user,
"oozie.wf.application.path": "%s%s" % (nn_path, path_to_workflow),
"oozie.use.system.libpath": "true"
}
oozie_job_id = client.add_job(x.create_hadoop_xml(job_parameters),
job_execution)
job_execution = conductor.job_execution_update(
ctx, job_execution, {
'oozie_job_id': oozie_job_id,
'start_time': datetime.datetime.now()
})
client.run_job(job_execution, oozie_job_id)
return job_execution