def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = { "node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } updated_function_name = CodeGenerationUtils.handle_parameter( node["parameters"]["udf_function"], my_args) gen_code = [] gen_code.extend([ "udf_" + node["id"] + " = udf(" + updated_function_name + ", " + node["parameters"]["udf_return_type"]["value"] + "())", os.linesep ]) gen_code.extend([ "tuple_list = " + CodeGenerationUtils.handle_parameter( node["parameters"]["udf_input_tuples"], my_args), os.linesep ]) gen_code.extend([ "output_list = " + CodeGenerationUtils.handle_parameter( node["parameters"]["udf_outputs"], my_args), os.linesep ]) gen_code.extend(["df_" + node["id"] + "=" + df_name, os.linesep]) gen_code.extend(["for index in range(len(tuple_list)):", os.linesep]) gen_code.extend([ "\tdf_" + node["id"] + " = df_" + node["id"] + ".withColumn(output_list[index], udf_" + node["id"] + "(*tuple_list[index]))", os.linesep, os.linesep ]) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = { "node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } gen_code = [] shared_function_set.add(SharedFunctionTypes.SELECT_EXPR_HELPERS) gen_code.extend(["expressions_" + node["id"] + "=[]", os.linesep]) for expr in node["parameters"]["expressions"]["value"]: gen_code.extend([ "expressions_" + node["id"] + ".extend(", 'single_select_expr_generator(' + CodeGenerationUtils.handle_parameter(expr["input_cols"], my_args) + ', ' + CodeGenerationUtils.handle_parameter( expr["output_cols"], my_args) + ', ' + CodeGenerationUtils.handle_parameter( expr["operation"], my_args) + '))', os.linesep ]) gen_code.extend([ "df_" + node["id"] + "=" + df_name + ".selectExpr(" + "*expressions_" + node["id"] + ")", os.linesep ]) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def __generate_code_for_pipeline_instantination(node, args): code = [] non_indicator_params = {} for param in node["parameters"]: if (param in node["multi_instance_indicator"]): code.extend([ "mmi_value_" + param + "_" + node["id"] + " = " + CodeGenerationUtils.handle_parameter( node["parameters"][param], args), os.linesep ]) else: non_indicator_params[param] = node["parameters"][param] code.extend(["stages_" + node["id"], " = ", "[]", os.linesep]) code.extend([ "for i in ", "range(len(mmi_value_" + node["multi_instance_indicator"][0] + "_" + node["id"], ")):", os.linesep ]) code.extend([ "\t", __generate_stage_template(node, non_indicator_params, args), os.linesep ]) if (not ("in_pipeline" in args and args["in_pipeline"])): code.extend([ 'pipeline_' + node["id"] + "=Pipeline(stages=", "stages_" + node["id"] + ")", os.linesep ]) return code
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {0}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): error, is_schema_appropriate = DataSourceValidityChecker.check_validity( node) if (error == ErrorTypes.NO_ERROR): my_args = { "node_id": node["id"], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } # Must be a valid schema at this point. param_string = CodeGenerationUtils.handle_parameter( node["parameters"]["schema"], my_args) gen_code = [] gen_code.append( "df_" + node["id"] + ' = spark.readStream.format("kafka").option("kafka.bootstrap.servers", ' ) gen_code.append( CodeGenerationUtils.handle_primitive( node["parameters"]["host"]["value"] + ":" + node["parameters"]["port"]["value"]) + ")") gen_code.append('.option("subscribe", ' + CodeGenerationUtils.handle_primitive( node["parameters"]["topic"]["value"]) + ")") gen_code.append( '.option("startingOffsets", ' + CodeGenerationUtils.handle_primitive( node["parameters"]["startingOffsets"]["value"]) + ")") gen_code.append( '.load().select(from_json(col("value").cast("string"), ' + param_string + ")") # For streams, we will use timestamp as a key while writing to kafka topic in case. gen_code.extend([ '.alias("value"), "timestamp").select("value.*", "timestamp")', os.linesep ]) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist={"df_count": {1}, "model_count": {0}} error, extra= IncomingEdgeValidityChecker.check_validity(node["id"], requireds_info, edges, checklist) final_code=[] shared_function_set = set() additional_local_code = [] errors = [] if(error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str(extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = {"node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors} input_cols = CodeGenerationUtils.handle_parameter(node["parameters"]["input_cols"], my_args) output_cols = CodeGenerationUtils.handle_parameter(node["parameters"]["output_cols"], my_args) window_size = node["parameters"]["window_size"]["value"] partitioning_column = node["parameters"]["partitioning_column"]["value"] ordering_column = node["parameters"]["ordering_column"]["value"] ordering_direction = node["parameters"]["ordering_direction"]["value"] gen_code=[] gen_code.extend(["input_cols = " + output_cols, os.linesep]) gen_code.extend(["output_cols = " + input_cols, os.linesep]) gen_code.extend(["df_" + node["id"] + "=" + df_name, os.linesep]) gen_code.extend(["for inC, outC in zip(input_cols, output_cols):", os.linesep]) gen_code.extend(["\tdf_" + node["id"] + " = df_" + node["id"] + ".withColumn('temp', col(inC))", os.linesep]) gen_code.extend(["\twSpec = Window.partitionBy('" + partitioning_column + "').orderBy(col('" + ordering_column + "')." + ordering_direction + "())", os.linesep]) gen_code.extend(["\tfor j in range(" + str(window_size) + "):", os.linesep]) gen_code.extend(["\t\tlag_values = lag('temp', default=0).over(wSpec)", os.linesep]) gen_code.extend(["\t\tdf_" + node["id"] + " = df_" + node["id"] + ".withColumn('temp', F.when((col('temp')==1) | (lag_values==None) | (lag_values<1) | (lag_values>=" + str(window_size + 1) + "), col('temp')).otherwise(lag_values+1))", os.linesep]) gen_code.extend(["\tdf_" + node["id"] + " = df_" + node["id"] + ".withColumn(outC, F.when(col('temp') > 0, 1.0).otherwise(0.0))", os.linesep]) final_code = CodeGenerationUtils.merge_with_additional_code(gen_code, additional_local_code) return final_code, shared_function_set, error
def __handle_col_list_plus_kwargs(node, args): kwargs_str = [] if ("kwargs" in node["parameters"]): kwargs = node["parameters"]["kwargs"]["value"] # If there are parameters for kwargs and all them is optional; and if user do not provide any of them, then we assume that there will be an empty dictionary here... for elem in kwargs: kwargs_str.extend([ ", ", elem, "=", CodeGenerationUtils.handle_parameter(kwargs[elem], args) ]) code = ["df_" + args["node_id"] + " = " + args["input_dfs"][0], os.linesep] code.extend([ "df_" + args["node_id"] + " = " + "df_" + args["node_id"] + "." + node["sql_name"] + "(" + CodeGenerationUtils.handle_parameter( node["parameters"]["input_columns"], args) ]) code.extend(kwargs_str) code.extend([")", os.linesep]) return code
def test_handle_parameter_for_primitive( self, create_parameter_for_primitive, create_accompanying_arg_for_parameter): # parameter_val=create_parameter_for_primitive # arg_val=create_accompanying_arg_for_parameter # assert CodeGenerationUtils.handle_parameter(parameter_val, arg_val) == "\"" + str(parameter_val["value"]) + "\"" expected_val = str(create_parameter_for_primitive["value"]) if (isinstance(create_parameter_for_primitive["value"], str)): expected_val = "\"" + create_parameter_for_primitive["value"] + "\"" assert CodeGenerationUtils.handle_parameter( create_parameter_for_primitive, create_accompanying_arg_for_parameter) == expected_val
def __generate_code_for_param_grid(node, cur_estimator_name, args): # In the future handle this in special requirement handler for parameters code=["param_grid_" + node["id"] + "=", "None", os.linesep] # Assuming that fix parameters are given in the estimator itself. # Maybe reconsider this part. grid_params = node["parameters"]["parameter_grid"] if(bool(grid_params)): code.pop() code.pop() code.extend(["ParamGridBuilder()"]) for param in grid_params: code.extend([".addGrid(" + cur_estimator_name + "." + param + ", " + CodeGenerationUtils.handle_parameter(grid_params[param], args) + ")"]) code.extend([".build()", os.linesep]) return code
def test_handle_parameter_for_primitive_array( self, create_parameter_for_primitive_array, create_accompanying_arg_for_parameter): primitive_array = create_parameter_for_primitive_array["value"] code = ["["] for elem in primitive_array: if (isinstance(elem, str)): code.append("\"" + elem + "\"") else: code.append(str(elem)) code.append(", ") code.pop() code.append("]") expected_val = ''.join(code) actual_val = CodeGenerationUtils.handle_parameter( create_parameter_for_primitive_array, create_accompanying_arg_for_parameter) assert all([a == b for a, b in zip(actual_val, expected_val)])
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {0}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): error, is_schema_appropriate = DataSourceValidityChecker.check_validity( node) if (error == ErrorTypes.NO_ERROR): my_args = { "node_id": node["id"], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } # Must be a valid schema at this point. param_string = CodeGenerationUtils.handle_parameter( node["parameter"]["schema"], my_args) gen_code = [] gen_code.extend([ "df_" + node["id"] + ' = spark.readStream.schema(' + param_string + ")." + node["file_type"] + "(" + CodeGenerationUtils.handle_primitive( node["parameters"]["path"]["value"]) + ")", os.linesep ]) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = { "node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } gen_code = [] gen_code.extend(["df_" + node["id"] + "=" + df_name, os.linesep]) between_operation = node["parameters"]["rolling_stats_info"]["value"][ "between_operation"]["value"] first_argument_input_cols = CodeGenerationUtils.handle_parameter( node["parameters"]["rolling_stats_info"]["value"]["first_argument"] ["value"]["input_cols"], my_args) first_argument_operation = node["parameters"]["rolling_stats_info"][ "value"]["first_argument"]["value"]["operation"]["value"] gen_code.extend( ["first_cols = " + first_argument_input_cols, os.linesep]) output_cols = CodeGenerationUtils.handle_parameter( node["parameters"]["rolling_stats_info"]["value"]["output_cols"], my_args) gen_code.extend(["output_cols = " + output_cols, os.linesep]) partitioning_column = node["parameters"]["rolling_stats_info"][ "value"]["partitioning_column"]["value"] ordering_column = node["parameters"]["rolling_stats_info"]["value"][ "ordering_column"]["value"] ordering_direction = node["parameters"]["rolling_stats_info"]["value"][ "ordering_direction"]["value"] lags = node["parameters"]["rolling_stats_info"]["value"]["lags"] lags_str = CodeGenerationUtils.handle_parameter(lags, my_args) window_str = "over (partition by " + partitioning_column + " order by " + ordering_column + " " + ordering_direction + " rows " + "'+ str(lag) +'" + " preceding) " # if window_size == -1: # window_str = "over (partition by " + partition_column + " order by " + ordering_column + " " + ordering_direction + " rows unbounded preceding) " # else: # window_str = "over (partition by " + partition_column + " order by " + ordering_column + " " + ordering_direction + " rows " + str(window_size) + " preceding) " if between_operation != 'Identity': second_argument_input_cols = CodeGenerationUtils.handle_parameter( node["parameters"]["rolling_stats_info"]["value"] ["second_argument"]["value"]["input_cols"], my_args) second_argument_operation = node["parameters"][ "rolling_stats_info"]["value"]["second_argument"]["value"][ "operation"]["value"] gen_code.extend( ["second_cols = " + second_argument_input_cols, os.linesep]) loop_str = "for col_1,col_2,out_col in zip(first_cols, second_cols, output_cols):" if first_argument_operation == 'Identity': if second_argument_operation == 'Identity': select_str = "df_" + node["id"] + " = df_" + node[ "id"] + ".selectExpr('*', col_1 + ' " + between_operation + " '+ col_2 + ' as out_col' + str(lag))" else: select_str = "df_" + node["id"] + " = df_" + node[ "id"] + ".selectExpr('*', col_1 + ' " + between_operation + " ' + '" + second_argument_operation + "(' + col_2 + ') " + window_str + "as out_col' + str(lag))" else: if second_argument_operation == 'Identity': select_str = "df_" + node["id"] + " = df_" + node[ "id"] + ".selectExpr('*', '" + first_argument_operation + "(' + col_1 + ') " + window_str + between_operation + " ' + col_2 + ' as out_col' + str(lag))" else: select_str = "df_" + node["id"] + " = df_" + node[ "id"] + ".selectExpr('*', '" + first_argument_operation + "(' + col_1 + ') " + window_str + between_operation + " " + second_argument_operation + "(' + col_2 + ') " + window_str + "as out_col' + str(lag))" else: loop_str = "for col_1,out_col in zip(first_cols, output_cols):" if first_argument_operation == 'Identity': select_str = "df_" + node["id"] + " = df_" + node[ "id"] + ".selectExpr('*', col_1 + ' as out_col' + str(lag))" else: select_str = "df_" + node["id"] + " = df_" + node[ "id"] + ".selectExpr('*', '" + first_argument_operation + "(' + col_1 + ') " + window_str + "as out_col' + str(lag))" gen_code.extend(["lags = " + lags_str, os.linesep]) gen_code.extend(["for lag in lags:", os.linesep]) gen_code.extend(["\t", loop_str, os.linesep]) gen_code.extend(["\t\t" + select_str, os.linesep]) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error
def generate_code(args): node = args["node"] requireds_info = args["requireds_info"] edges = args["edges"] checklist = {"df_count": {1}, "model_count": {0}} error, extra = IncomingEdgeValidityChecker.check_validity( node["id"], requireds_info, edges, checklist) final_code = [] shared_function_set = set() additional_local_code = [] errors = [] if (error == ErrorTypes.NO_ERROR): if ("portion" in extra["dfs"][0]): df_name = "df_" + extra["dfs"][0]["source_id"] + "[" + str( extra["dfs"][0]["portion"]) + "]" else: df_name = "df_" + extra["dfs"][0]["source_id"] my_args = { "node_id": node["id"], "input_dfs": [df_name], "shared_function_set": shared_function_set, "additional_local_code": additional_local_code, "errors": errors } df_name = "df_" + my_args["node_id"] gen_code = [df_name + " = " + my_args["input_dfs"][0], os.linesep] input_columns = ["["] conditions = ["["] values = ["["] otherwises = ["["] output_columns = ["["] for exp in node["parameters"]["expressions"]["value"]: input_columns.extend([ CodeGenerationUtils.handle_parameter(exp["input_columns"], my_args), ", " ]) conditions.extend([ CodeGenerationUtils.handle_parameter(exp["condition"], my_args), ", " ]) values.extend([ CodeGenerationUtils.handle_parameter(exp["value"], my_args), ", " ]) otherwises.extend([ CodeGenerationUtils.handle_parameter(exp["otherwise"], my_args), ", " ]) output_columns.extend([ CodeGenerationUtils.handle_parameter(exp["output_columns"], my_args), ", " ]) # Check there are at least 1 elememnt in expressions input_columns.pop() conditions.pop() values.pop() otherwises.pop() output_columns.pop() input_columns.extend(["]"]) conditions.extend(["]"]) values.extend(["]"]) otherwises.extend(["]"]) output_columns.extend(["]"]) gen_code.extend( ["input_columns = " + ''.join(input_columns), os.linesep]) gen_code.extend(["conditions = " + ''.join(conditions), os.linesep]) gen_code.extend(["values = " + ''.join(values), os.linesep]) gen_code.extend(["otherwises = " + ''.join(otherwises), os.linesep]) gen_code.extend( ["output_columns = " + ''.join(output_columns), os.linesep]) gen_code.extend([ "for in_cols, cond, val, otw, out_cols in zip(input_columns, conditions, values, otherwises, output_columns):", os.linesep ]) gen_code.extend( ["\tfor in_col, out_col in zip(in_cols, out_cols):", os.linesep]) gen_code.extend([ "\t\tcur_cond = eval(cond.replace('$','" + df_name + "[\"'+in_col+'\"]'" + "))", os.linesep ]) gen_code.extend([ "\t\t" + df_name + " = " + df_name + ".withColumn(out_col, F.when(cur_cond, val).otherwise(otw))", os.linesep ]) final_code = CodeGenerationUtils.merge_with_additional_code( gen_code, additional_local_code) return final_code, shared_function_set, error