def register_pipeline(pb_pipeline, pipeline_id):
request = service_pb2.RegisterPipelineRequest()
request.pipeline.CopyFrom(pb_pipeline)
request.pipeline_id = pipeline_id
response = _service.request(request, "register_pipeline")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.VoidResponse())
#logger.info(res)
return None, res.status
def unregister_pipeline(pipeline_id):
"""
unregister pipeline to backend
"""
request = service_pb2.UnRegisterPipelineRequest()
request.pipeline_id = pipeline_id
response = _service.request(request, "unregister_pipeline")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.VoidResponse())
#logger.info(res)
return None, res.status
def write_record(file_path, records, objector):
"""
write records to file path with objector
"""
request = service_pb2.WriteLocalSeqFileRequest()
request.file_path = file_path
for record in records:
request.key.append("")
request.value.append(objector.serialize(record))
response = _service.request(request, "write_local_seqfile")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.VoidResponse())
return None, res.status
def hadoop_commit(args, hadoop_client_path=None, hadoop_config_path=None):
if hadoop_client_path is None:
hadoop_client_path = default_hadoop_client_path()
if hadoop_config_path is None:
hadoop_config_path = default_hadoop_config_path()
request = service_pb2.HadoopCommitRequest()
request.hadoop_client_path = hadoop_client_path
request.hadoop_config_path = hadoop_config_path
request.args.extend(args)
response = _service.request(request, "hadoop_commit")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.VoidResponse())
return res.status.success
def launch(pipeline_id,
logical_plan_message,
resource_message,
commit_args=None,
context=None):
"""
Send the rpc command to the other end to launch the logical plan
Args:
Raises:
error.BigflowRPCException: if any error happened
"""
request = service_pb2.LaunchRequest()
request.pipeline_id = pipeline_id
request.logical_plan.CopyFrom(logical_plan_message)
request.resource.CopyFrom(resource_message)
if commit_args is not None:
request.hadoop_commit_args.extend(commit_args)
if context is not None:
assert isinstance(context, str)
request.pipeline_context = context
response = _service.request(request, "launch")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.VoidResponse())
#logger.info(res)
if not res.status.success:
backend_log_path = os.getenv("BIGFLOW_LOG_FILE_BACKEND", "")
error_message = "Job ran failed"
if len(_message) > 0:
error_message += ", possible_reason: \n" + "".join(_message)
if backend_log_path:
error_message += "Please check backend log['%s.log'] for details" % backend_log_path
raise error.BigflowRuntimeException(error_message)
def kill(pipeline_id, logical_plan_message, commit_args=None):
"""
Send the rpc command to the other end to kill the logical plan
Args:
Raises:
error.BigflowRPCException: if any error happened
"""
request = service_pb2.KillRequest()
request.pipeline_id = pipeline_id
request.logical_plan.CopyFrom(logical_plan_message)
if commit_args is not None:
request.hadoop_commit_args.extend(commit_args)
response = _service.request(request, "kill")
import google.protobuf.json_format as json_format
res = json_format.Parse(response, service_pb2.VoidResponse())
return None, res.status