def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] if (error == ErrorTypes.NO_ERROR): my_args = { "node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } gen_code = CodeGenerationUtils.handle_instantination_or_call( node["parameters"], 'df_' + node["id"] + '=' + df_name + '.' + node["ddfo_name"] + '(', my_args) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist={"df_count": {0}, "model_count": {0}} error, extra=IncomingEdgeValidityChecker.check_validity(node["id"], requireds_info, edges, checklist) final_code=[] shared_function_set = set() additional_local_code=[] errors=[] if(error == ErrorTypes.NO_ERROR): error, is_schema_appropriate=DataSourceValidityChecker.check_validity(node) if(error == ErrorTypes.NO_ERROR): my_args = {"node_id": node["id"], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors} # Must be a valid schema at this point. additional_code, param_string = CodeGenerationUtils.handle_parameter(node["parameter"]["schema"], my_args) gen_code=[] gen_code.extend(additional_code) gen_code.append("df_" + node["id"] + ' = spark.readStream.format("kafka").option("kafka.bootstrap.servers", ') gen_code.append(CodeGenerationUtils.handle_primitive(node["parameters"]["host"]["value"] + ":" + node["parameters"]["port"]["value"]) + ")") gen_code.append('.option("subscribe", ' + CodeGenerationUtils.handle_primitive(node["parameters"]["topic"]["value"] + ")")) gen_code.append('.load().select(from_json(col("value").cast("string"), '+ param_string +")") # For streams, we will use timestamp as a key while writing to kafka topic in case. gen_code.extend(['.alias("value"), "timestamp").select("value.*", "timestamp")', os.linesep]) final_code = CodeGenerationUtils.merge_with_additional_code(gen_code, additional_local_code) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist={"df_count": {1}, "model_count": {0}} error, extra=IncomingEdgeValidityChecker.check_validity(node["id"], requireds_info, edges, checklist) final_code=[] shared_function_set = set() additional_local_code=[] errors=[] if(error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str(extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args={"node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors} updated_function_name = CodeGenerationUtils.handle_parameter(node["parameters"]["udf_function"], my_args) gen_code=[] gen_code.extend(["udf_"+node["id"]+" = udf("+updated_function_name+", "+node["parameters"]["udf_return_type"]["value"]+"())", os.linesep]) gen_code.extend(["tuple_list = " + CodeGenerationUtils.handle_parameter(node["parameters"]["udf_input_tuples"], my_args), os.linesep]) gen_code.extend(["output_list = " + CodeGenerationUtils.handle_parameter(node["parameters"]["udf_outputs"], my_args), os.linesep]) gen_code.extend(["df_" + node["id"] + "=" + df_name, os.linesep]) gen_code.extend(["for index in range(len(tuple_list)):", os.linesep]) gen_code.extend(["\tdf_"+node["id"]+" = df_"+node["id"]+".withColumn(output_list[index], udf_"+node["id"]+"(*tuple_list[index]))", os.linesep, os.linesep]) final_code = CodeGenerationUtils.merge_with_additional_code(gen_code, additional_local_code) return final_code, shared_function_set, error
def __create_bash_operator(task_id, args): op_task_id = args["app_id"] + "_" + task_id return [ 'Task_' + op_task_id + ' = BashOperator(task_id=' + CodeGenerationUtils.handle_primitive(op_task_id) + ", bash_command='" + args["bash_command"] + ' ' + CodeGenerationUtils.handle_primitive(op_task_id) + " ', dag=dag)" ]
def __create_spark_operator(task_id, args): op_task_id = args["app_id"] + "_" + task_id op_name = 'Task_' + op_task_id operator_args_str = str(args["spark_operator_conf"]) script_path = os.path.join(args["code_base_path"], op_task_id + '.py') return [ "operator_args = " + operator_args_str, os.linesep, op_name + ' = SparkSubmitOperator(task_id=' + CodeGenerationUtils.handle_primitive(op_task_id) + ', application=' + CodeGenerationUtils.handle_primitive(script_path) + ', dag=dag, **operator_args)' ]
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = { "node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } gen_code = [] shared_function_set.add(SharedFunctionTypes.SELECT_EXPR_HELPERS) gen_code.append("df_" + node["id"] + "=" + df_name + ".selectExpr(") for expr in node["parameters"]["expressions"]["value"]: gen_code.extend([ 'single_select_expr_generator(' + CodeGenerationUtils.handle_parameter(expr["input_cols"], my_args) + ', ' + CodeGenerationUtils.handle_parameter( expr["output_cols"], my_args) + ', ' + CodeGenerationUtils.handle_parameter( expr["operation"], my_args) + ')', ', ' ]) gen_code.pop() gen_code.extend([")", os.linesep]) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) code = [] shared_function_set = set() if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] code.append( df_name + '.selectExpr("CAST(' + node["parameters"]["unique_column_name"]["value"] + ' AS STRING) AS key", "to_json(struct(*)) AS value").writeStream.format("kafka").option("kafka.bootstrap.servers", ' ) code.append( CodeGenerationUtils.handle_primitive( node["parameters"]["host"]["value"] + ":" + node["parameters"]["port"]["value"]) + ")") code.append(".trigger(" + __generate_trigger_code(node) + ")") code.append('.option("topic", ' + CodeGenerationUtils.handle_primitive( node["parameters"]["topic"]["value"]) + ")") code.append('.option("checkpointLocation", ' + CodeGenerationUtils.handle_primitive( node["parameters"]["checkpointLocation"]["value"]) + ").start()") code.extend([ os.linesep, "query_" + node["id"], ".awaitTermination()", os.linesep ]) args["additional_info"]["written_topics"].append({ "topic_name": node["parameters"]["topic"]["value"], "host": node["parameters"]["host"]["value"], "port": node["parameters"]["port"]["value"] }) return code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) shared_function_set = set() errors = [] code = [] if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] shared_function_set.add(SharedFunctionTypes.VECTOR_DISASSEMBLER) code = [ "df_" + node["id"] + " = " + "vector_disassembler(" + df_name + ", " + CodeGenerationUtils.handle_primitive( node["parameters"]["vector_column"]["value"]) + ")", os.linesep ] return code, shared_function_set, error
def __generate_trigger_code(node): trigger_type = node["parameters"]["trigger_type"]["value"] if (trigger_type == "once"): return "once=True" else: return trigger_type + "=" + "'" + CodeGenerationUtils.handle_primitive( node["parameters"]["trigger_value"]["value"])
def __generate_code_for_pipeline_instantination(node, args): code = [] non_indicator_params = {} for param in node["parameters"]: if (param in node["multi_instance_indicator"]): code.extend([ "mmi_value_" + param + "_" + node["id"] + " = " + CodeGenerationUtils.handle_parameter( node["parameters"][param], args), os.linesep ]) else: non_indicator_params[param] = node["parameters"][param] code.extend(["stages_" + node["id"], " = ", "[]", os.linesep]) code.extend([ "for i in ", "range(len(mmi_value_" + node["multi_instance_indicator"][0] + "_" + node["id"], ")):", os.linesep ]) code.extend([ "\t", __generate_stage_template(node, non_indicator_params, args), os.linesep ]) if (not ("in_pipeline" in args and args["in_pipeline"])): code.extend([ 'pipeline_' + node["id"] + "=Pipeline(stages=", "stages_" + node["id"] + ")", os.linesep ]) return code
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist={"df_count": {1}, "model_count": {0}} error, extra=IncomingEdgeValidityChecker.check_validity(node["id"], requireds_info, edges, checklist) final_code=[] shared_function_set = set() additional_local_code = [] errors = [] if(error == ErrorTypes.NO_ERROR): if("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str(extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = {"node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors} # Depending on the column that multi_instance_indicator indicates, we will decide to apply whether to multi-instance generation or usual generation if(MultiInstanceHandlerUtils.should_generate_multiple_instances(node)): gen_code = MultiInstanceHandlerUtils.multi_instance_generation(node, df_name, my_args) else: gen_code = __single_generation(node, df_name, my_args) final_code = CodeGenerationUtils.merge_with_additional_code(gen_code, additional_local_code) return final_code, shared_function_set, error
def __generate_stage_template(node, non_indicator_params, args): code = [] class_name = "" # Do not allow other families than estimator and transformer. Handle the "else" case later... if (node["family"] == NodeFamilyTypes.Estimator.value): class_name = node["estimator_name"] elif (node["family"] == NodeFamilyTypes.Transformer.value): class_name = node["transformer_name"] mmi_part = [] for i in range(len(node["multi_instance_indicator"])): mmi_part.extend([ node["multi_instance_indicator"][i] + "=" + "mmi_value_" + node["multi_instance_indicator"][i] + "_" + node["id"] + "[i]", ", " ]) arg_part = CodeGenerationUtils.handle_arguments(non_indicator_params, args) if (not bool(arg_part)): mmi_part.pop() code.extend(["stages_" + node["id"], ".append(", class_name + '(']) code.extend(mmi_part) code.extend(arg_part) code.extend(["))", os.linesep]) return ''.join(code)
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {0}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): error, is_schema_appropriate = DataSourceValidityChecker.check_validity( node) if (error == ErrorTypes.NO_ERROR): my_args = { "node_id": node["id"], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } if (is_schema_appropriate): gen_code = CodeGenerationUtils.handle_instantination_or_call( node["parameters"], "df_" + node["id"] + "=" + "spark.read." + node["file_type"] + "(", my_args) else: # For safety, but consider it again if ("schema" in node["parameters"]): del node["parameters"]["schema"] if (node["can_infer_schema"]): node["parameters"]["inferSchema"] = { "value": True, "type": "boolean" } gen_code = CodeGenerationUtils.handle_instantination_or_call( node["parameters"], "df_" + node["id"] + "=" + "spark.read.format(" + CodeGenerationUtils.handle_primitive(node["file_type"]) + ").load(", my_args) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def __handle_expression_string_only(node, args): code = ["df_" + args["node_id"] + " = " + args["input_dfs"][0], os.linesep] code.extend([ "df_" + args["node_id"] + " = " + "df_" + args["node_id"] + "." + node["sql_name"] + "(" + CodeGenerationUtils.handle_primitive( node["parameters"]["expression"]["value"]) + ")", os.linesep ]) return code
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): error, extra2 = CVValiditiyChecker.check_validity( node["nodes"], node["edges"]) if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = { "node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } gen_code = [] gen_code.extend( __generate_code_for_estimator_instantination( node["nodes"][extra2["estimator_node_id"]], my_args)) gen_code.extend( __generate_code_for_evaluator_instantination( node["nodes"][extra2["evaluator_node_id"]], my_args)) gen_code.extend( __generate_code_for_param_grid( node, 'estimator_' + extra2["estimator_node_id"], my_args)) gen_code.extend( __generate_code_for_cv_instantination(node, extra2["estimator_node_id"], extra2["evaluator_node_id"])) gen_code.extend([ 'model_' + node["id"] + "=" + 'cv_' + node["id"] + ".fit(" + df_name + ")", os.linesep ]) # Following might not be logical unless you aim to predict on training data for some specific needs. gen_code.extend([ 'df_' + node["id"] + "=" + 'model_' + node["id"] + '.transform(' + df_name + ')', os.linesep ]) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def __generate_code_for_transformer_instantination(node, df_name, args): if (MultiInstanceHandlerUtils.should_generate_multiple_instances(node)): args["in_pipeline"] = True return MultiInstanceHandlerUtils.multi_instance_generation( node, df_name, args) else: return CodeGenerationUtils.handle_instantination_or_call( node["parameters"], 'pipeline_stage_' + node["id"] + ' = ' + node["transformer_name"] + '(', args)
def __arg_dict_to_string(args): # Assuming that corresponding argument is a string which is appropriate for pre-defined datetime format. code = ["{"] for arg in args: if (arg in __set_of_datetime_arguments): code.extend([ CodeGenerationUtils.handle_primitive(arg), ": ", 'datetime.strptime("' + args[arg] + '", "' + __datetime_format + '")', ", " ]) else: code.extend([ CodeGenerationUtils.handle_primitive(arg), ": ", CodeGenerationUtils.handle_primitive(args[arg]), ", " ]) if (len(args) > 0): code.pop() code.append("}") return ''.join(code)
def __handle_in_op_out_trio(node, args): in_op_out_trio_list = node["parameters"]["in_op_out_trio_list"]["value"] code = ["df_" + args["node_id"] + " = " + args["input_dfs"][0], os.linesep] code.extend( ["df_" + args["node_id"] + " = " + "df_" + args["node_id"] + " .agg("]) for elem in in_op_out_trio_list: code.extend([ "F." + elem["operation"]["value"] + "(" + CodeGenerationUtils.handle_primitive(elem["input_column"]["value"]) + ").alias(" + CodeGenerationUtils.handle_primitive( elem["output_column"]["value"]) + ")", ", " ]) # Assuming that there is at least 1 agg request. However, check this and produce error before got here... code.pop() code.extend([")", os.linesep]) return code
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist={"df_count": {1}, "model_count": {0}} error, extra=IncomingEdgeValidityChecker.check_validity(node["id"], requireds_info, edges, checklist) final_code=[] shared_function_set = set() additional_local_code = [] errors = [] if(error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str(extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = {"node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors} input_cols = CodeGenerationUtils.handle_parameter(node["parameters"]["input_cols"], my_args) output_cols = CodeGenerationUtils.handle_parameter(node["parameters"]["output_cols"], my_args) window_size = node["parameters"]["window_size"]["value"] partitioning_column = node["parameters"]["partitioning_column"]["value"] ordering_column = node["parameters"]["ordering_column"]["value"] ordering_direction = node["parameters"]["ordering_direction"]["value"] gen_code=[] gen_code.extend(["input_cols = " + output_cols, os.linesep]) gen_code.extend(["output_cols = " + input_cols, os.linesep]) gen_code.extend(["df_" + node["id"] + "=" + df_name, os.linesep]) gen_code.extend(["for inC, outC in zip(input_cols, output_cols):", os.linesep]) gen_code.extend(["\tdf_" + node["id"] + " = df_" + node["id"] + ".withColumn('temp', col(inC))", os.linesep]) gen_code.extend(["\twSpec = Window.partitionBy('" + partitioning_column + "').orderBy(col('" + ordering_column + "')." + ordering_direction + "())", os.linesep]) gen_code.extend(["\tfor j in range(" + str(window_size) + "):", os.linesep]) gen_code.extend(["\t\tlag_values = lag('temp', default=0).over(wSpec)", os.linesep]) gen_code.extend(["\t\tdf_" + node["id"] + " = df_" + node["id"] + ".withColumn('temp', F.when((col('temp')==1) | (lag_values==None) | (lag_values<1) | (lag_values>=" + str(window_size + 1) + "), col('temp')).otherwise(lag_values+1))", os.linesep]) gen_code.extend(["\tdf_" + node["id"] + " = df_" + node["id"] + ".withColumn(outC, F.when(col('temp') > 0, 1.0).otherwise(0.0))", os.linesep]) final_code = CodeGenerationUtils.merge_with_additional_code(gen_code, additional_local_code) return final_code, shared_function_set, error
def __handle_col_list_plus_kwargs(node, args): kwargs_str = [] if ("kwargs" in node["parameters"]): kwargs = node["parameters"]["kwargs"]["value"] # If there are parameters for kwargs and all them is optional; and if user do not provide any of them, then we assume that there will be an empty dictionary here... for elem in kwargs: kwargs_str.extend([ ", ", elem, "=", CodeGenerationUtils.handle_parameter(kwargs[elem], args) ]) code = ["df_" + args["node_id"] + " = " + args["input_dfs"][0], os.linesep] code.extend([ "df_" + args["node_id"] + " = " + "df_" + args["node_id"] + "." + node["sql_name"] + "(" + CodeGenerationUtils.handle_parameter( node["parameters"]["input_columns"], args) ]) code.extend(kwargs_str) code.extend([")", os.linesep]) return code
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] final_code = [] shared_function_set = set() additional_local_code = [] errors = [] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) if (error == ErrorTypes.NO_ERROR): error, pipeline_order = PipelineValidityChecker.check_validity( node["nodes"], node["edges"]) if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = { "node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } gen_code, error = __generate_stages(node["nodes"], pipeline_order, df_name, my_args) if (error == ErrorTypes.NO_ERROR): gen_code.append(os.linesep) gen_code.extend( __generate_code_for_pipeline_instantination( node, pipeline_order, my_args)) gen_code.extend([ 'model_' + node["id"] + "=" + 'pipeline_' + node["id"] + ".fit(" + df_name + ")", os.linesep ]) # Following might not be logical for pipelines with an estimator gen_code.extend([ 'df_' + node["id"] + "=" + 'model_' + node["id"] + '.transform(' + df_name + ')', os.linesep ]) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = { "node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } gen_code = CodeGenerationUtils.handle_instantination_or_call( node["parameters"], df_name + ".write.format(" + CodeGenerationUtils.handle_primitive(node["file_type"]) + ").save(", my_args) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) args["additional_info"]["written_tables"].append( {"table_path": node["parameters"]["path"]["value"]}) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {0}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): error, is_schema_appropriate = DataSourceValidityChecker.check_validity( node) if (error == ErrorTypes.NO_ERROR): my_args = { "node_id": node["id"], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } # Must be a valid schema at this point. additional_code, param_string = CodeGenerationUtils.handle_parameter( node["parameter"]["schema"], my_args) gen_code = [] gen_code.extend(additional_code) gen_code.extend([ "df_" + node["id"] + ' = spark.readStream.schema(' + param_string + ")." + node["file_type"] + "(" + CodeGenerationUtils.handle_primitive( node["parameters"]["path"]["value"]) + ")", os.linesep ]) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] if (error == ErrorTypes.NO_ERROR): final_code = [ "correlation_" + node["id"] + " = " + "Correlation.corr(" + df_name + ", " + CodeGenerationUtils.handle_primitive( node["parameters"]["column"]["value"]) + ", " + CodeGenerationUtils.handle_primitive( node["parameters"]["method"]["value"]) + ")", os.linesep ] final_code.extend([ "result_array_" + node["id"] + " = ", "correlation_" + node["id"] + ".head()[0].toArray()", os.linesep ]) # In the future, dynamically name columns according to an appropriate convention... final_code.extend([ "df_" + node["id"] + " = sc.parallelize(" + "result_array_" + node["id"] + ")", ".map(lambda x: [float(i) for i in x]).toDF()", os.linesep ]) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist={"df_count": {1}, "model_count": {0}} error, extra=IncomingEdgeValidityChecker.check_validity(node["id"], requireds_info, edges, checklist) code=[] shared_function_set = set() if(error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str(extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] code.append("query_" + node["id"] + "=" + df_name + ".writeStream.format(" + CodeGenerationUtils.handle_primitive(node["file_type"]) + ")") code.append(".trigger("+ __generate_trigger_code(node) +")") code.append('.option("path", ' + CodeGenerationUtils.handle_primitive(node["parameters"]["path"]["value"]) + ")") code.append('.option("checkpointLocation", ' + CodeGenerationUtils.handle_primitive(node["parameters"]["checkpointLocation"]["value"]) + ").start()") code.extend([os.linesep, "query_" + node["id"], ".awaitTermination()", os.linesep]) args["additional_info"]["written_tables"].append({"table_path": node["parameters"]["path"]["value"]}) return code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {2}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) code = [] shared_function_set = set() if (error == ErrorTypes.NO_ERROR): df_names = __get_dfs_to_join(extra) code.extend([ "df_" + node["id"] + "=" + df_names[0] + ".join(" + df_names[1] + ", " + CodeGenerationUtils.handle_primitive( node["parameters"]["join_column"]["value"]) + ")", os.linesep ]) return code, shared_function_set, error
def __generate_code_for_param_grid(node, cur_estimator_name, args): # In the future handle this in special requirement handler for parameters code = ["param_grid_" + node["id"] + "=", "None", os.linesep] # Assuming that fix parameters are given in the estimator itself. # Maybe reconsider this part. grid_params = node["parameters"]["parameter_grid"] if (bool(grid_params)): code.pop() code.pop() code.extend(["ParamGridBuilder()"]) for param in grid_params: code.extend([ ".addGrid(" + cur_estimator_name + "." + param + ", " + CodeGenerationUtils.handle_parameter( grid_params[param], args) + ")" ]) code.extend([".build()", os.linesep]) return code
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {0}, "model_count": {1}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) code = [] shared_function_set = set() if (error == ErrorTypes.NO_ERROR): model_id = extra["models"][0]["source_id"] code = [ "model_" + model_id + ".save(" + CodeGenerationUtils.handle_primitive( node["parameters"]["model_path"]["value"]) + ")", os.linesep ] return code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist={"df_count": {1}, "model_count": {0}} error, extra=IncomingEdgeValidityChecker.check_validity(node["id"], requireds_info, edges, checklist) final_code=[] shared_function_set = set() if(error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str(extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] if(error==ErrorTypes.NO_ERROR): # For now, directly returns the correlation result (as a dataframe) which includes (in a row~): # pValues: DenseVector # degreesOfFreedom: list # statistics: DenseVector final_code=["df_"+ node["id"] + " = " + node["parameters"]["test_type"]["value"] + ".test(" + df_name + ", " + CodeGenerationUtils.handle_primitive(node["parameters"]["features_column"]["value"]) + ", " + CodeGenerationUtils.handle_primitive(node["parameters"]["label_column"]["value"]) + ")", os.linesep] return final_code, shared_function_set, error
def __single_generation(node, df_name, args): code=CodeGenerationUtils.handle_instantination_or_call(node["parameters"], 'estimator_' + node["id"] + ' = ' + node["estimator_name"] + '(', args) code.extend(['model_' + node["id"] + "=" + 'estimator_' + node["id"] + ".fit(" + df_name + ")", os.linesep]) code.extend(['df_' + node["id"] + "=" + 'model_' + node["id"] + '.transform(' + df_name + ')', os.linesep]) return code