def test_column_type_checking(self):
model = MappingPreviewModel()
model.reset_model([["1", "0h", "2018-01-01 00:00"], ["2", "1h", "2018-01-01 00:00"]])
model.set_type(0, value_to_convert_spec('float'))
self.assertEqual(model._column_type_errors, {})
self.assertEqual(model._row_type_errors, {})
model.set_type(1, value_to_convert_spec('duration'))
self.assertEqual(model._column_type_errors, {})
self.assertEqual(model._row_type_errors, {})
model.set_type(2, value_to_convert_spec('datetime'))
self.assertEqual(model._column_type_errors, {})
self.assertEqual(model._row_type_errors, {})
def test_row_type_checking_produces_error(self):
model = MappingPreviewModel()
model.reset_model([["1", "2.4"], ["Not a valid number", "3"]])
model.set_type(1,
value_to_convert_spec('float'),
orientation=Qt.Vertical)
error_index = (1, 0)
self.assertEqual(len(model._row_type_errors), 1)
self.assertEqual(model._column_type_errors, {})
self.assertTrue(error_index in model._row_type_errors)
# Error should only be shown if we have a pivot mapping on that row.
self.assertEqual(model.data(model.index(*error_index)),
"Not a valid number")
# if we add mapping error should be shown.
mapping = MappingSpecModel(
dict_to_map({
"map_type": "ObjectClass",
"name": {
"map_type": "row",
"value_reference": 1
}
}))
model.set_mapping(mapping)
self.assertEqual(model.data(model.index(*error_index)), "Error")
def test_column_type_checking_produces_error(self):
model = MappingPreviewModel()
model.reset_model([["Not a valid number", "2.4"], ["1", "3"]])
model.set_type(0, value_to_convert_spec('float'))
error_index = (0, 0)
self.assertEqual(len(model._column_type_errors), 1)
self.assertEqual(model._row_type_errors, {})
self.assertTrue(error_index in model._column_type_errors)
self.assertEqual(model.data(model.index(*error_index)), "Error")
# if we add a pivoted mapping for the row with the error, the error should not be shown
mapping = MappingSpecModel(
dict_to_map({
"map_type": "ObjectClass",
"name": {
"map_type": "row",
"value_reference": 0
}
}))
model.set_mapping(mapping)
self.assertEqual(model.data(model.index(*error_index)),
"Not a valid number")
# or if we add a mapping where there reading starts from a row bellow the error, the error should not be shown.
mapping = MappingSpecModel(
dict_to_map({
"map_type": "ObjectClass",
"read_start_row": 1
}))
model.set_mapping(mapping)
self.assertEqual(model.data(model.index(*error_index)),
"Not a valid number")
def test_interger_sequence_datetime(self):
self.assertIsInstance(
value_to_convert_spec({
"start_datetime": "2019-01-01T00:00",
"start_int": 0,
"duration": "1h"
}),
IntegerSequenceDateTimeConvertSpec,
)
def test_Duration(self):
self.assertIsInstance(value_to_convert_spec("duration"),
DurationConvertSpec)
def test_DateTime(self):
self.assertIsInstance(value_to_convert_spec("datetime"),
DateTimeConvertSpec)
def test_float(self):
self.assertIsInstance(value_to_convert_spec("float"), FloatConvertSpec)
def test_string(self):
self.assertIsInstance(value_to_convert_spec("string"),
StringConvertSpec)
def run(checked_files, all_import_settings, all_source_settings, urls_downstream, logs_dir, cancel_on_error):
print("starting importer program")
all_data = []
all_errors = []
for source in checked_files:
settings = all_import_settings.get(source, None)
if settings == "deselected":
continue
if settings is None or not settings:
print(f"There are no mappings defined for {source}, moving on...")
continue
source_type = settings["source_type"]
source_settings = all_source_settings.get(source_type)
connector = {
"CSVConnector": CSVConnector,
"ExcelConnector": ExcelConnector,
"GdxConnector": GdxConnector,
"JSONConnector": JSONConnector,
}[source_type](source_settings)
try:
connector.connect_to_source(source)
except IOError as error:
print(f"Failed to connect to source: {error}", file=sys.stderr)
sys.exit(1)
table_mappings = {
name: mapping
for name, mapping in settings.get("table_mappings", {}).items()
if name in settings["selected_tables"]
}
table_options = {
name: options
for name, options in settings.get("table_options", {}).items()
if name in settings["selected_tables"]
}
table_types = {
tn: {int(col): value_to_convert_spec(spec) for col, spec in cols.items()}
for tn, cols in settings.get("table_types", {}).items()
}
table_row_types = {
tn: {int(col): value_to_convert_spec(spec) for col, spec in cols.items()}
for tn, cols in settings.get("table_row_types", {}).items()
}
try:
data, errors = connector.get_mapped_data(
table_mappings, table_options, table_types, table_row_types, max_rows=-1
)
except spinedb_api.InvalidMapping as error:
print(f"Failed to imoport '{source}': {error}", file=sys.stderr)
if cancel_on_error:
sys.exit(1)
else:
continue
print(f"Read {sum(len(d) for d in data.values())} data from {source} with {len(errors)} errors")
all_data.append(data)
all_errors.extend(errors)
if all_errors:
# Log errors in a time stamped file into the logs directory
timestamp = _create_log_file_timestamp()
logfilepath = os.path.abspath(os.path.join(logs_dir, timestamp + "_error.log"))
with open(logfilepath, 'w') as f:
for err in all_errors:
f.write(f"{err}\n")
# Make error log file anchor with path as tooltip
logfile_anchor = (
"<a style='color:#BB99FF;' title='" + logfilepath + "' href='file:///" + logfilepath + "'>error log</a>"
)
print("Import errors. Logfile: {0}".format(logfile_anchor), file=sys.stderr)
if cancel_on_error:
sys.exit(-1)
if all_data:
for url in urls_downstream:
_import(all_data, url, logs_dir, cancel_on_error)
def run(checked_files, all_settings, urls_downstream, logs_dir,
cancel_on_error):
all_data = []
all_errors = []
for source in checked_files:
settings = all_settings.get(source, None)
if settings is None or not settings:
print("There are no mappings defined for {0}, moving on...".format(
source))
continue
source_type = settings["source_type"]
connector = {
"CSVConnector": CSVConnector,
"ExcelConnector": ExcelConnector,
"GdxConnector": GdxConnector
}[source_type]()
connector.connect_to_source(source)
table_mappings = {
name: mapping
for name, mapping in settings.get("table_mappings", {}).items()
if name in settings["selected_tables"]
}
table_options = {
name: options
for name, options in settings.get("table_options", {}).items()
if name in settings["selected_tables"]
}
table_types = {
tn: {
int(col): value_to_convert_spec(spec)
for col, spec in cols.items()
}
for tn, cols in settings.get("table_types", {}).items()
}
table_row_types = {
tn: {
int(col): value_to_convert_spec(spec)
for col, spec in cols.items()
}
for tn, cols in settings.get("table_row_types", {}).items()
}
data, errors = connector.get_mapped_data(table_mappings,
table_options,
table_types,
table_row_types,
max_rows=-1)
print("Read {0} data from {1} with {2} errors".format(
sum(len(d) for d in data.values()), source, len(errors)))
all_data.append(data)
all_errors.extend(errors)
if all_errors:
# Log errors in a time stamped file into the logs directory
timestamp = _create_log_file_timestamp()
logfilepath = os.path.abspath(
os.path.join(logs_dir, timestamp + "_error.log"))
with open(logfilepath, 'w') as f:
for err in all_errors:
f.write("{0}\n".format(err))
# Make error log file anchor with path as tooltip
logfile_anchor = ("<a style='color:#BB99FF;' title='" + logfilepath +
"' href='file:///" + logfilepath + "'>error log</a>")
print("Import errors. Logfile: {0}".format(logfile_anchor),
file=sys.stderr)
if cancel_on_error:
sys.exit(-1)
if all_data:
for url in urls_downstream:
_import(all_data, url, logs_dir, cancel_on_error)
