def rendered_result(self) -> Tuple[str, str]:
return self.template.render(
name=self.name, source1=self.source1,
source2=self.source2), self.template_ext.render(
name=self.name,
type_left=format_types(
self.named_modules.get(self.source1).get_out_type()),
type_right=format_types(
self.named_modules.get(self.source2).get_out_type()),
type_out=format_types(self.get_out_type()),
source_extract=self.source_extract,
target_extract=self.target_extract,
collect_tuple=self._get_collection_tuple(),
source1=self.source1,
source2=self.source2)
def __init__(self, module, env: Environment, named_modules):
super().__init__(module, env, named_modules)
# delimiter and quote have default values if not in module
self.field_delimiter = module.get('fieldDelimiter')
self.quote_character = module.get('quoteCharacter')
self.named_fields = module.get('namedFields')
# By default, all columns and lines are imported
self.column_list = None
self.ignore_first_line = None
# if fields are named in the modules, ignore first line and
# compute the column numbers to be passed to the reader
# (!! This implies opening the file first)
# TODO: Implement for remote file
if self.named_fields is not None:
self.ignore_first_line = True
self.column_list = self._get_column_list()
self.data_type = module.get('dataType')
if self.data_type is None:
raise ValueError(
"No dataType provided for module {}".format(module))
self.formatted_type = format_types(self.data_type)
self.template_path = os.path.join(self.template_path,
'scala_csv_loader.template')
self.template = self.env.get_template(self.template_path)
def add_to_graph(self, graph: Digraph):
graph.node(str(hash(self)),
label=self.to_graph_repr(),
fillcolor='lightblue',
style='filled',
shape='component')
graph.edge(
str(hash(self.named_modules.get(self.source1))),
str(hash(self)),
label=format_types(
self.named_modules.get(self.source1).get_out_type())
)
graph.edge(
str(hash(self.named_modules.get(self.source2))),
str(hash(self)),
label=format_types(
self.named_modules.get(self.source2).get_out_type())
)
def add_to_graph(self, graph: Digraph):
graph.node(str(hash(self)),
label=self.to_graph_repr(),
fillcolor='orange',
style='filled',
shape='note')
graph.edge(str(hash(self.named_modules.get(self.source))),
str(hash(self)),
label=format_types(
self.named_modules.get(self.source).get_out_type()))
def rendered_result(self) -> Tuple[str, str]:
return self.template.render(
name=self.name,
field_types=[
'createTypeInformation[{}]'.format(i) for i in self.data_type
],
field_names=[quote(i) for i in self.field_names],
data_type=format_types(self.data_type),
driver=self.driver,
db_url=self.db_url,
query=self.query,
filter_null=self.filter_null,
filter_fields=[
't._{} != null'.format(i + 1)
for i in range(len(self.data_type))
]), self.template_ext.render()
def test_render(self, mock_get_template: unittest.mock.patch):
""" Should call render method with right parameters
"""
mock_get_template.return_value = self.template_mock
csv_importer = CsvImporter(self.module, self.env, [])
result = csv_importer.rendered_result()
self.assertEqual(result, ("", ""))
self.template_mock.render.assert_called_once_with(
file_path=self.module.get('path'),
name=self.module.get('name'),
ignore_first_line=None,
included_fields=None,
field_delimiter=self.module.get('fieldDelimiter'),
quote_character=self.module.get('quoteCharacter'),
type=format_types(self.module.get('dataType'))
)
def test_render(self, mock_get_template: unittest.mock.patch):
""" Should call render method with right parameters
"""
mock_get_template.return_value = self.template_mock
json_importer = JsonImporter(self.module, self.env, [])
result = json_importer.rendered_result()
self.assertEqual(result, ("", ""))
self.template_mock.render.assert_called_once_with(
file_path=self.module.get('path'),
name=self.module.get('name'),
main_field=self.module.get('mainField'),
required_fields=[
quote(i) for i in
self.module.get('requiredFields')
],
type=format_types(
len(self.module.get('requiredFields'))*["String"])
)
def rendered_result(self) -> Tuple[str, str]:
return self.template.render(file_path=self.file_path,
name=self.name,
main_field=self.main_field,
type=format_types(self.data_type),
required_fields=self.required_fields), ''
def test_multiple_types(self):
for i in range(2, 5):
self.assertEqual("({})".format(', '.join(i * ["TEST"])),
format_types(i * ["TEST"]))
def test_one_type(self):
self.assertEqual("Tuple1[String]", format_types(["String"]))
def rendered_result(self) -> Tuple[str, str]:
return self.template.render(name=self.name,
db_name=self.db_name,
collection=self.collection,
type=format_types(self.data_type),
required_fields=self.required_fields), ''
