def transform(runner: Optional[str], stream_dtc_file: Optional[str],
window_dtc_file: Optional[str], data_processor: Optional[str],
raw_json_files: List[str]) -> int:
if stream_dtc_file is None and window_dtc_file is None:
stream_dtc_file, window_dtc_file = get_stream_window_dtc_files(
get_valid_yml_files(get_yml_files()))
if stream_dtc_file is None:
eprint('Streaming DTC file not provided and could not be found in '
'the current directory.')
return 1
if not runner:
runner = 'local'
if not data_processor:
data_processor = 'simple'
if runner not in RUNNER_CLASS:
eprint('Unknown runner: \'{}\'. Possible values: {}'.format(
runner, list(RUNNER_CLASS.keys())))
return 1
if data_processor not in DATA_PROCESSOR_CLASS:
eprint('Unknown data-processor: \'{}\'. Possible values: {}'.format(
runner, list(DATA_PROCESSOR_CLASS.keys())))
return 1
data_processor_obj = DATA_PROCESSOR_CLASS[data_processor]()
if runner == 'local':
return transform_local(stream_dtc_file, window_dtc_file,
raw_json_files, data_processor_obj)
else:
return transform_spark(stream_dtc_file, window_dtc_file,
raw_json_files, data_processor_obj)
def test_get_stream_window_dtc_files_valid():
assert get_stream_window_dtc_files(
['stream1.yml', 'stream2.yml',
'window1.yml']) == ('stream1.yml', 'window1.yml')
def test_get_stream_window_dtc_files_missing_window():
assert get_stream_window_dtc_files(['invalid.yml', 'window2.yml'
]) == (None, 'window2.yml')
def test_get_stream_window_dtc_files_missing_stream():
assert get_stream_window_dtc_files(['invalid.yml', 'stream1.yml'
]) == ('stream1.yml', None)
def test_get_stream_window_dtc_files_bad_files():
assert get_stream_window_dtc_files([]) == (None, None)
assert get_stream_window_dtc_files(['invalid.yml']) == (None, None)