def default_spark_home_path():
response = _service.request(service_pb2.SingleStringRequest(),
"default_spark_home_path")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.SingleStringResponse())
return res.response.encode("utf8"), res.status
def default_tmp_hdfs_path(unique_id):
request = service_pb2.SingleStringRequest()
request.request = unique_id
response = _service.request(request, "tmp_hdfs_path")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.SingleStringResponse())
return res.response.encode("utf8"), res.status
def default_hadoop_client_path():
response = _service.request(service_pb2.SingleStringRequest(),
"default_hadoop_client_path")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.SingleStringResponse())
#logger.info(res)
return res.response.encode("utf8"), res.status
def default_hadoop_job_ugi(hadoop_config_path=None):
"""
request default hadoop.job.ugi from hadoop config
"""
if hadoop_config_path is None:
hadoop_config_path = default_hadoop_config_path()
request = service_pb2.SingleStringRequest()
request.request = hadoop_config_path
response = _service.request(request, "default_hadoop_job_ugi")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.SingleStringResponse())
return res.response.encode("utf8"), res.status
def default_fs_defaultfs(hadoop_config_path=None):
"""
request default fs.defaultFS from hadoop config
"""
if hadoop_config_path is None:
hadoop_config_path = default_hadoop_config_path()
request = service_pb2.SingleStringRequest()
request.request = hadoop_config_path
response = _service.request(request, "default_fs_defaultfs")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.SingleStringResponse())
#logger.info(res)
return res.response.encode("utf8"), res.status
def get_toft_style_path(input_path, hadoop_config_path=None, \
fs_defaultfs=None, hadoop_job_ugi=None):
"""
request toft style path from flume
"""
if hadoop_config_path is None:
hadoop_config_path = default_hadoop_config_path()
request = service_pb2.GetToftStylePathRequest()
request.input_path = input_path
request.hadoop_conf_path = hadoop_config_path
if fs_defaultfs is not None:
request.fs_defaultfs = fs_defaultfs
if hadoop_job_ugi is not None:
request.hadoop_job_ugi = hadoop_job_ugi
response = _service.request(request, "get_toft_style_path")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.SingleStringResponse())
#logger.info(res)
return res.response.encode("utf8"), res.status