def test_output(self):
conf = Config()
conf.load(
Path(
'/home/patrizio/PycharmProjects/pyspark-config/tests/resource/configuration_tests/output.yaml'
))
expected = Config(input=None,
output=[
Csv_output(type='Csv',
name='outputi',
path='ksandkjanskda',
transformations=None,
delimiter=','),
Json(type='Json',
name='outputi',
path='ksandkjanskda',
transformations=None),
Parquet_Output(type='Parquet',
name='outputi',
path='ksandkjanskda',
transformations=None,
partitionCols=['hiJN', 'ADSAD'])
],
transformation=None)
self.assertEqual(conf, expected)
def test_input(self):
conf = Config()
conf.load(
Path(
'/home/patrizio/PycharmProjects/pyspark-config/tests/resource/configuration_tests/input.yaml'
))
expected = Config(input=Input(sources=[
Parquet(type='Parquet',
label='events',
path='heloooo.parquet',
parquet_path='heloooo.pacucucu'),
Csv(type='Csv',
label='events',
path='heloooo.csv',
csv_path='heloooo.csucsu',
delimiter='demiliiiiii')
],
transformations=[
Split(type='Split',
column='A',
newCol='new',
delimiter=',')
]),
output=None,
transformation=None)
self.assertEqual(conf, expected)
def test_transformations(self):
conf = Config()
conf.load(
Path(
'/home/patrizio/PycharmProjects/pyspark-config/tests/resource/configuration_tests/transformations.yaml'
))
expected = Config(input=None,
output=None,
transformations=[
Select(type='Select', cols='sql_condition'),
Filter(type='Filter',
sql_condition=['A', 'B', 'C'])
])
self.assertEqual(conf, expected)
def test_fileHandling(self):
"""
Test if exception is raised, when file does not exist
Test if no exception is raised when file exists
Test if exception is raised when file type is not yaml
"""
FOLDER = '../../resource/yamlConfig/'
conf_avble = '%stest_fileHandling_conf.yaml' % (FOLDER)
conf_not_avble = '%stest_fileHandling_conf_wrong.yaml' % (FOLDER)
conf_wrong_type = '%stest_fileHandling_conf.txt' % (FOLDER)
configuration = Config()
self.assertEqual(None, configuration.load(conf_avble))
self.assertRaises(NotFoundError, configuration.load, conf_not_avble)
self.assertRaises(InvalidTypeError, configuration.load,
conf_wrong_type)
from pathlib import Path
import unittest
from pyspark_config.config import Config
from pyspark_config.input.input import Input
from pyspark_config.transformations.transformations import *
from pyspark_config.output import *
from pyspark_config.input import *
conf = Config()
conf.load('../../resource/configuration_tests/testing.yaml')
conf.apply()