def test_build_lambda_processor_config(self):
parser = TransformationsParser(["a: config('input.options.port')"])
parser.run()
operations = TransformationOperations(self.config)
transformations_validator = TransformationsValidator(
operations, self.data_structure)
_ = transformations_validator.validate(parser.expanded_transformation)
creator = TransformationCreator(self.data_structure,
parser.expanded_transformation,
TransformationOperations(self.config))
transformation = creator.build_lambda()
self.assertIsInstance(transformation, types.LambdaType,
"Transformation type should be lambda")
spark = SparkSession.builder.getOrCreate()
file = spark.read.csv(DATA_PATH, self.data_structure_pyspark)
result = file.rdd.map(transformation)
result = result.collect()
self.assertListEqual(result,
[(29092, ), (29092, ), (29092, ), (29092, ),
(29092, )], "List of tuples should be equal")
spark.stop()
def test_run(self):
config = TransformationsParserConfig(CONFIG_PATH)
parser = TransformationsParser(
config.content["processing"]["transformation"])
parser.run()
#self.assertEqual(len(parser.expanded_transformation), 5, 'Transformations should contain 5 elements')
self.assertEqual(
parser.expanded_transformation[1], 'dst_ip',
"2 element in expanded transformation should be 'dst_ip'")
for index in [0, 2, 3]:
self.assertIsInstance(
parser.expanded_transformation[index], FieldTransformation,
"{} element expanded transformation should has FieldTransformation type"
.format(index))
self.assertEqual(
parser.expanded_transformation[index].name,
stub['run_test'][index]['field_name'],
"expanded_transformation[{}].field_name should be {}".format(
index, stub["run_test"][index]["field_name"]))
self.assertIsInstance(
parser.expanded_transformation[index].body,
stub["run_test"][index]["type"],
'expanded_transformation[{}].operation should be instance of {}'
.format(index, stub["run_test"][index]["type"]))
def test_build_lambda_processor_add(self):
self.maxDiff = None
parser = TransformationsParser([
"dst_ip: add(-13.5, 2)", "src_ip:add(-13.5,2)",
"foobar: 'add(-13.5,2)'", "foobar2: 'add\\'(-13.5,2)'"
])
parser.run()
operations = TransformationOperations(self.config)
transformations_validator = TransformationsValidator(
operations, self.data_structure)
_ = transformations_validator.validate(parser.expanded_transformation)
creator = TransformationCreator(self.data_structure,
parser.expanded_transformation,
TransformationOperations(self.config))
transformation = creator.build_lambda()
self.assertIsInstance(transformation, types.LambdaType,
"Transformation type should be lambda")
spark = SparkSession.builder.getOrCreate()
file = spark.read.csv(DATA_PATH, self.data_structure_pyspark)
result = file.rdd.map(transformation)
result = result.collect()
self.assertListEqual(result,
[(-11.5, -11.5, 'add(-13.5,2)', "add'(-13.5,2)"),
(-11.5, -11.5, 'add(-13.5,2)', "add'(-13.5,2)"),
(-11.5, -11.5, 'add(-13.5,2)', "add'(-13.5,2)"),
(-11.5, -11.5, 'add(-13.5,2)', "add'(-13.5,2)"),
(-11.5, -11.5, 'add(-13.5,2)', "add'(-13.5,2)")],
"List of tuples should be equal")
spark.stop()