def value_error(var=None, data_values=None): """ Raise a ValueError exception :param var: :type var: :param data_values: values accepted by the variable :type data_values: str/list :return: """ from optimus.helpers.debug import get_var_name if not isinstance(data_values, list): data_values = [data_values] # data_values = val_to_list(data_values) if len(data_values) == 1: divisor = "" elif len(data_values) == 2: divisor = " or " elif len(data_values) > 2: divisor = ", " raise ValueError("'{var_name}' must be {type}, received '{var_type}'" .format(var_name=get_var_name(var), type=divisor.join(map( lambda x: "'" + x + "'", data_values)), var_type=one_list_to_val(var)))
def value_error(var=None, data_values=None, extra_text=""): """ Raise a ValueError exception :param var: :param data_values: values accepted by the variable. str/list :param extra_text: Additional final info about the error :return: """ from optimus.helpers.debug import get_var_name if not isinstance(data_values, list): data_values = [data_values] if len(data_values) == 1: divisor = "" elif len(data_values) == 2: divisor = " or " elif len(data_values) > 2: divisor = ", " raise ValueError("'{var_name}' must be {type}, received '{var_type}'. {extra_text}" .format(var_name=get_var_name(var), type=divisor.join(map( lambda x: "'" + x + "'", data_values)), var_type=one_list_to_val(var), extra_text=extra_text))
def type(cls, var): """ Raise and exception of the type specified :param var: :return: """ from optimus.helpers.debug import get_var_name raise cls("'{var_name}' error".format(var_name=get_var_name(var), var_type=var))
def length_error(var1: list, var2: (list, int)) -> Exception: """ Raise a ValueError exception :param var1: variable to check for length :param var2: variable or integer to compare with var1 :return: """ from optimus.helpers.debug import get_var_name if isinstance(var2, int): length_var2 = str(var2) else: length_var2 = str(len(var2)) raise ValueError("'{var2_name}' must be length '{var1_length}', received '{var2_length}'" .format(var2_name=get_var_name(var2), var1_length=str(len(var1)), var2_length=length_var2))
def type_error(var, data_types): """ Raise a TypeError exception :param var: :param data_types: data types expected as strings :return: """ from optimus.helpers.debug import get_var_name if len(data_types) == 1: divisor = "" elif len(data_types) == 2: divisor = " or " elif len(data_types) > 2: divisor = ", " _type = divisor.join(map(lambda x: "'" + x + "'", data_types)) raise TypeError( "'{var_name}' must be of type {type}, received '{var_type}'". format(var_name=get_var_name(var), type=_type, var_type=type(var)))
def create(self, obj, method, suffix=None, output="df", additional_method=None, *args, **kwargs): """ This is a helper function that output python tests for Spark DataFrames. :param obj: Object to be tested :param method: Method to be tested :param suffix: The test name will be create using the method param. suffix will add a string in case you want to customize the test name. :param output: can be a 'df' or a 'json' :param additional_method: :param args: Arguments to be used in the method :param kwargs: Keyword arguments to be used in the functions :return: """ buffer = [] def add_buffer(value): buffer.append("\t" + value) # Create name name = [] if method is not None: name.append(method.replace(".", "_")) if additional_method is not None: name.append(additional_method) if suffix is not None: name.append(suffix) test_name = "_".join(name) func_test_name = "test_" + test_name + "()" print("Creating {test} test function...".format(test=func_test_name)) logger.print(func_test_name) if not output == "dict": add_buffer("@staticmethod\n") func_test_name = "test_" + test_name + "()" else: func_test_name = "test_" + test_name + "(self)" filename = test_name + ".test" add_buffer("def " + func_test_name + ":\n") source = "source_df" if obj is None: # Use the main df df_func = self.df elif isinstance(obj, pyspark.sql.dataframe.DataFrame): source_df = "\tsource_df=op.create.df(" + obj.export() + ")\n" df_func = obj add_buffer(source_df) else: source = get_var_name(obj) df_func = obj # Process simple arguments _args = [] for v in args: if is_str(v): _args.append("'" + v + "'") elif is_numeric(v): _args.append(str(v)) elif is_list(v): if is_list_of_strings(v): lst = ["'" + x + "'" for x in v] elif is_list_of_numeric(v) or is_list_of_tuples(v): lst = [str(x) for x in v] elif is_list_of_tuples(v): lst = [str(x) for x in v] _args.append('[' + ','.join(lst) + ']') elif is_dict(v): _args.append(json.dumps(v)) elif is_function(v): _args.append(v.__qualname__) else: # _args.append(get_var_name(v)) _args.append(str(v)) # else: # import marshal # code_string = marshal.dumps(v.__code__) # add_buffer("\tfunction = '" + code_string + "'\n") # import marshal, types # # code = marshal.loads(code_string) # func = types.FunctionType(code, globals(), "some_func_name") print(_args) _args = ','.join(_args) _kwargs = [] # print(_args) # Process keywords arguments for k, v in kwargs.items(): if is_str(v): v = "'" + v + "'" _kwargs.append(k + "=" + str(v)) # Separator if we have positional and keyword arguments separator = "" if (not is_list_empty(args)) & (not is_list_empty(kwargs)): separator = "," if method is None: add_buffer("\tactual_df = source_df\n") else: am = "" if additional_method: am = "." + additional_method + "()" add_buffer("\tactual_df =" + source + "." + method + "(" + _args + separator + ','.join(_kwargs) + ")" + am + "\n") # Apply function to the dataframe if method is None: df_result = self.op.create.df(*args, **kwargs) else: # Here we construct the method to be applied to the source object for f in method.split("."): df_func = getattr(df_func, f) df_result = df_func(*args, **kwargs) # Additional Methods if additional_method is not None: df_result = getattr(df_result, additional_method)() if output == "df": df_result.table() expected = "\texpected_df = op.create.df(" + df_result.export( ) + ")\n" elif output == "json": print(df_result) if is_str(df_result): df_result = "'" + df_result + "'" else: df_result = str(df_result) add_buffer("\tactual_df =json_enconding(actual_df)\n") expected = "\texpected_value =json_enconding(" + df_result + ")\n" elif output == "dict": print(df_result) expected = "\texpected_value =" + str(df_result) + "\n" else: expected = "\t\n" add_buffer(expected) # Output if output == "df": add_buffer( "\tassert (expected_df.collect() == actual_df.collect())\n") elif output == "json": add_buffer("\tassert(expected_value == actual_df)\n") elif output == "dict": add_buffer( "\tself.assertDictEqual(deep_sort(expected_value), deep_sort(actual_df))\n" ) filename = self.path + "//" + filename if not os.path.exists(os.path.dirname(filename)): try: os.makedirs(os.path.dirname(filename)) except OSError as exc: # Guard against race condition if exc.errno != errno.EEXIST: raise # Write file test_file = open(filename, 'w', encoding='utf-8') for b in buffer: test_file.write(b)