def generate(self, sub_graph: AISubGraph, project_desc: ProjectDesc) -> AbstractJob:
"""
Generate local flink job.
:param sub_graph: Sub graph generates from ai nodes.
:param project_desc: Description of the project.
:return: Base job Object.
"""
if sub_graph.config.exec_mode == ExecutionMode.BATCH:
flink_context = JobContext(ExecutionMode.BATCH)
else:
flink_context = JobContext(ExecutionMode.STREAM)
flink_context.project_config = project_desc.project_config
local_flink_job_config: LocalFlinkJobConfig = sub_graph.config
job = LocalFlinkJob(ai_graph=sub_graph, job_context=flink_context, job_config=local_flink_job_config)
if job.job_config.language_type is None:
job.job_config.language_type = self.get_language_type(sub_graph)
# set jar and class
job.job_config.main_class = version.main_class
job.job_config.jar_path = version.jar_path
# set python main
job.job_config.py_entry_file = version.py_main_file
return job
def generate(self, sub_graph: AISubGraph, project_desc: ProjectDesc) -> AbstractJob:
assert 1 == len(sub_graph.nodes)
node: ExecutableNode = list(sub_graph.nodes.values())[0]
if sub_graph.config.exec_mode == ExecutionMode.BATCH:
context = JobContext(ExecutionMode.BATCH)
else:
context = JobContext(ExecutionMode.STREAM)
executor: CmdExecutor = node.executor
return KubernetesCMDJob(job_context=context, exec_cmd=executor.cmd_line, job_config=sub_graph.config)
def generate(self, sub_graph: AISubGraph,
project_desc: ProjectDesc) -> VVPJob:
if sub_graph.config.exec_mode == ExecutionMode.BATCH:
flink_context = JobContext(ExecutionMode.BATCH)
else:
flink_context = JobContext(ExecutionMode.STREAM)
flink_context.project_config = project_desc.project_config
job_config: VVPJobConfig = sub_graph.config
job = VVPJob(job_context=flink_context, job_config=job_config)
return job
def generate(self, sub_graph: AISubGraph, project_desc: ProjectDesc) -> KubernetesPythonJob:
if sub_graph.config.exec_mode == ExecutionMode.BATCH:
run_func = serialize(batch_run_func)
py_context = JobContext(ExecutionMode.BATCH)
else:
run_func = serialize(stream_run_func)
py_context = JobContext(ExecutionMode.STREAM)
py_context.project_config = project_desc.project_config
run_graph: RunGraph = self.build_run_graph(sub_graph, py_context)
job_config: KubernetesPythonJobConfig = sub_graph.config
return KubernetesPythonJob(run_graph=run_graph, run_func=run_func,
job_context=py_context, job_config=job_config)
def compile_node(self, node: AINode, context: JobContext) -> Executor:
exec_mode: ExecutionMode = context.get_execution_mode()
def component_execute(component: BaseComponent):
if exec_mode == ExecutionMode.BATCH:
executor = component.batch_executor()
else:
executor = component.stream_executor()
return executor
executor: Executor = None
if isinstance(node, Example):
n: Example = node
if node.executor is not None:
example_type = 'udf'
else:
example_type = n.example_meta.data_type
if example_type != 'udf' and example_type != "pandas" and example_type != "numpy":
raise Exception(
"Base example node only supports pandas and numpy now")
example_component_type: type(
BaseExampleComponent) = self.example_registry.get_object(
example_type)
example_component = example_component_type(node, context)
executor = component_execute(example_component)
elif isinstance(node, ExecutableNode):
n: ExecutableNode = node
c = ExecuteComponent(n, context)
executor = component_execute(c)
else:
pass
return executor
def __init__(self,
run_func: bytes,
run_graph: Optional[RunGraph],
job_context: JobContext = JobContext(),
job_config: LocalPythonJobConfig = LocalPythonJobConfig()):
super().__init__(job_context=job_context, job_config=job_config)
self.run_func = run_func
self.run_graph = run_graph
self.exec_func_file: Text = None
self.exec_args_file: Text = None
def __init__(self,
exec_cmd,
job_context: JobContext = JobContext(),
job_config: AbstractJobConfig = KubernetesCMDJobConfig()):
super().__init__(job_context, job_config)
self.exec_cmd = exec_cmd
def __init__(self,
run_func: bytes,
run_graph: Optional[RunGraph],
job_context: JobContext = JobContext(),
job_config: KubernetesPythonJobConfig = KubernetesPythonJobConfig()):
super().__init__(run_func, run_graph, job_context, job_config)
def __init__(self,
job_context: JobContext = JobContext(),
job_config: AbstractJobConfig = LocalDummyJobConfig()):
super().__init__(job_context, job_config)
