def test_decimal(self):
GobType = get_gob_type("GOB.Decimal")
self.assertEqual(GobType.name, "Decimal")
self.assertEqual('null', GobType.from_value(None).json)
self.assertEqual('null', GobType.from_value("nan").json)
self.assertEqual('123.0', GobType.from_value(123).json)
self.assertEqual('123.123', GobType.from_value(123.123).json)
# DB ouptut is float
self.assertIsInstance(GobType.from_value('123').to_db, float)
# Python value is string
self.assertIsInstance(GobType.from_value('123').to_value, str)
with self.assertRaises(GOBException):
GobType.from_value("123,123")
self.assertEqual(
'123.123',
GobType.from_value("123,123", decimal_separator=',').json)
self.assertEqual('123.1',
GobType.from_value("123.123", precision=1).json)
self.assertEqual('123.1000',
str(GobType.from_value("123.1", precision=4)))
self.assertEqual('123.1',
GobType.from_value("123.1", precision=4).json)
with self.assertRaises(GOBException):
GobType.from_value('N')
with self.assertRaises(GOBException):
GobType.from_value(True)
with self.assertRaises(GOBException):
GobType.from_value('Overtime')
def test_date(self):
GobType = get_gob_type("GOB.Date")
self.assertEqual(GobType.name, "Date")
self.assertEqual('null', GobType.from_value(None).json)
self.assertEqual("2016-05-04", str(GobType.from_value('2016-05-04')))
self.assertEqual("2016-05-04",
str(GobType.from_value('20160504', format="%Y%m%d")))
self.assertEqual('"2016-05-04"', GobType.from_value('2016-05-04').json)
self.assertEqual('"2016-05-04"',
GobType.from_value('20160504', format="%Y%m%d").json)
# Test edge case https://bugs.python.org/issue13305 strftime is not consistent for years < 1000
self.assertEqual('"0001-05-04"', GobType.from_value('0001-05-04').json)
with self.assertRaises(GOBException):
GobType.from_value('N')
with self.assertRaises(GOBException):
GobType.from_value('Overtime')
with self.assertRaises(GOBException):
GobType.from_value(1)
# DB ouptut is datetime
self.assertIsInstance(GobType.from_value('2016-05-04').to_db, datetime)
# Python value is datetime
self.assertIsInstance(GobType.from_value('2016-05-04').to_value, date)
def create_temporary_table(self):
""" Create a new temporary table based on the current table for a collection
Message data is inserted to be compared with the current state
:param data: the imported data
:return:
"""
self.collection = self.gob_model.get_collection(
self.metadata.catalogue, self.metadata.entity)
table_name = self.gob_model.get_table_name(self.metadata.catalogue,
self.metadata.entity)
tmp_table_name = self._get_tmp_table_name(table_name)
self.fields = ['_tid', '_source', '_hash']
# Drop any existing temporary table
self.drop_temporary_table(tmp_table_name)
columns = [
get_column(c, self.collection['all_fields'][c])
for c in self.fields
]
columns.append(
get_gob_type("GOB.JSON").get_column_definition("_original_value"))
self.tmp_table = Table(tmp_table_name,
self.base.metadata,
*columns,
extend_existing=True)
self.tmp_table.create(self.engine)
self.temporary_rows = []
return tmp_table_name
def test_reference(self):
GobType = get_gob_type("GOB.Reference")
self.assertEqual(GobType.name, "Reference")
# Test that a value of id is ignored when comparing references. Only test bronwaarde
self.assertEqual(
GobType.from_value('{"bronwaarde": "123456"}'),
GobType.from_value('{"bronwaarde": "123456", "id": "123456"}'))
def test_datetime(self):
GobType = get_gob_type("GOB.DateTime")
self.assertEqual(GobType.name, "DateTime")
self.assertEqual('null', GobType.from_value(None).json)
self.assertEqual("2016-05-04T12:00:00.123000",
str(GobType.from_value('2016-05-04T12:00:00.123000')))
self.assertEqual(
"2016-05-04T12:00:00.123000",
str(
GobType.from_value('20160504 12:00:00.123000',
format="%Y%m%d %H:%M:%S.%f")))
self.assertEqual("2016-05-04T12:00:00.000000",
str(GobType.from_value('2016-05-04T12:00:00')))
self.assertEqual(
"2016-05-04T12:00:00.000000",
str(
GobType.from_value('20160504 12:00:00',
format="%Y%m%d %H:%M:%S")))
self.assertEqual('"2016-05-04T12:00:00.123000"',
GobType.from_value('2016-05-04T12:00:00.123000').json)
self.assertEqual(
'"2016-05-04T12:00:00.123000"',
GobType.from_value('20160504 12:00:00.123000',
format="%Y%m%d %H:%M:%S.%f").json)
# Test edge case https://bugs.python.org/issue13305 strftime is not consistent for years < 1000
self.assertEqual('"0005-05-04T12:00:00.123000"',
GobType.from_value('0005-05-04T12:00:00.123000').json)
self.assertEqual(
'"0005-05-04T12:00:00.123000"',
GobType.from_value('00050504 12:00:00.123000',
format="%Y%m%d %H:%M:%S.%f").json)
with self.assertRaises(GOBException):
GobType.from_value('N')
with self.assertRaises(GOBException):
GobType.from_value('Overtime')
with self.assertRaises(GOBException):
GobType.from_value(1)
# DB ouptut is datetime
self.assertIsInstance(
GobType.from_value('2016-05-04T12:00:00.123000').to_db, datetime)
# unless an empty string is entered
self.assertIsNone(GobType.from_value(None).to_db)
# Python value is datetime
self.assertIsInstance(
GobType.from_value('2016-05-04T12:00:00.123000').to_value,
datetime)
# unless an empty string is entered
self.assertIsNone(GobType.from_value(None).to_value)
def test_gob_geometry(self):
GobType = get_gob_type("GOB.Geo.Geometry")
self.assertEqual(GobType.name, "Geometry")
self.assertEqual('POINT (1.000 2.000)',
str(GobType.from_value('POINT (1 2)')))
empty_geometry = GobType('')
self.assertEqual('null', empty_geometry.json)
def test_gob_polygon(self):
GobType = get_gob_type("GOB.Geo.Polygon")
self.assertEqual(GobType.name, "Polygon")
self.assertEqual(
'POLYGON(112.0 22.0, 113.0 22.0, 113.0 21.0)',
str(
GobType.from_value(
'POLYGON(112.0 22.0, 113.0 22.0, 113.0 21.0)')))
empty_polygon = GobType('')
self.assertEqual('null', empty_polygon.json)
def get_column(column_name, column_specification):
"""Utility method to convert GOB type to a SQLAlchemy Column
Get the SQLAlchemy columndefinition for the gob type as exposed by the gob_type
:param column: (name, type)
:return: sqlalchemy.Column
"""
gob_type = get_gob_type(column_specification["type"])
column = gob_type.get_column_definition(column_name)
column.doc = column_specification["description"]
return column
def test_incomplete_date(self):
GobType = get_gob_type("GOB.IncompleteDate")
self.assertEqual(GobType.name, "IncompleteDate")
def test_result(t, year, month, day, formatted):
self.assertEqual(year, t.year)
self.assertEqual(month, t.month)
self.assertEqual(day, t.day)
self.assertEqual(
json.dumps(
{
'year': year,
'month': month,
'day': day,
'formatted': formatted
},
sort_keys=True), t._string)
t = GobType.from_value(None)
self.assertIsNone(t.to_value)
t = GobType.from_value("2020-03-21")
test_result(t, 2020, 3, 21, '2020-03-21')
t = GobType.from_value({'year': 2020, 'month': 3, 'day': 21})
test_result(t, 2020, 3, 21, '2020-03-21')
t = GobType.from_value("2020-03-00")
test_result(t, 2020, 3, None, '2020-03-00')
t = GobType.from_value({'year': 2020, 'month': None, 'day': 21})
test_result(t, 2020, None, 21, '2020-00-21')
t = GobType.from_value(
json.dumps({
'year': 2020,
'month': None,
'day': 21
}))
test_result(t, 2020, None, 21, '2020-00-21')
with self.assertRaises(GOBTypeException):
GobType.from_value(
json.dumps({
'year': 2020,
'month': None,
'day': 21
})[:-2])
with self.assertRaises(GOBTypeException):
GobType.from_value({'year': 2020})
with self.assertRaises(GOBTypeException):
GobType.from_value('2020-invalid')
def get_field_value(entity, field, spec):
"""
Get the value of the given field in the given entity.
Use GOB type to correctly interpret the entity value
:param entity:
:param field:
:param spec:
:return:
"""
gob_type = get_gob_type(spec['type'])
entity_value = getattr(entity, field, None)
return None if entity_value is None else gob_type.from_value(
entity_value).to_value
def test_json(self):
GobType = get_gob_type("GOB.JSON")
self.assertEqual(GobType.name, "JSON")
self.assertEqual('null', GobType.from_value(None).json)
key = fixtures.random_string()
value = fixtures.random_string()
generated_json = GobType.from_value({key: value}).json
self.assertEqual(f'{{"{key}": "{value}"}}',
GobType.from_value({
key: value
}).json)
self.assertEqual(f'["{key}", "{value}"]',
GobType.from_value([key, value]).json)
value = random.randint(9, 33)
self.assertEqual(f'{{"{key}": {value}}}',
GobType.from_value({key: value}))
value = False
self.assertEqual(f'{{"{key}": false}}',
GobType.from_value({key: value}))
value = None
self.assertEqual(f'{{"{key}": null}}', GobType.from_value({key:
value}))
self.assertEqual(generated_json, GobType.from_value(generated_json))
self.assertEqual(generated_json, GobType(generated_json))
out_of_order = '{"b": "c", "a": "d"}'
in_order = '{"a": "d", "b": "c"}'
self.assertEqual(in_order, GobType(out_of_order))
self.assertEqual(in_order, GobType.from_value(GobType(out_of_order)))
# Test unknown format
with self.assertRaises(GOBTypeException):
GobType.from_value('{"test" = "test"}')
# DB ouptut is json
self.assertIsInstance(
GobType.from_value('{"key": "value"}').to_db, dict)
# Python value is dict
self.assertIsInstance(
GobType.from_value('{"key": "value"}').to_value, dict)
def test_gob_type_and_string(self):
GobType = get_gob_type("GOB.String")
self.assertEqual(GobType.name, "String")
self.assertEqual('None', str(GobType(None)))
fixture = fixtures.random_string()
# Gobtype can be instantiated with a string
gob_type = GobType(fixture)
# Gobtype can be compared with string
self.assertTrue(gob_type == fixture)
# Gobtype has a json representation
self.assertEqual(gob_type.json, f'"{fixture}"')
# Gobtype can be constructed from a value
gob_type1 = GobType.from_value(fixture)
# Gobtype can be compared with other gob_type
self.assertTrue(gob_type == gob_type1)
# DB ouptut is string
self.assertIsInstance(gob_type.to_db, str)
# Gobtype cannot be instantiated with something different than a string
with self.assertRaises(GOBException):
GobType(fixtures.random_bool())
with self.assertRaises(GOBException):
GobType(random.randint(0, 10))
# Gobtype can be constructed from a value
gob_type1 = GobType.from_value(int)
gob_type2 = GobType.from_value(bool)
# JSON values are String representations
self.assertEqual('"1"', GobType.from_value(1).json)
self.assertEqual('"True"', GobType.from_value(True).json)
# Gobtype can be constructed from None
self.assertNotEqual(GobType.from_value(None),
GobType.from_value("None"))
self.assertEqual(GobType.from_value(None), GobType.from_value(None))
self.assertEqual('null', GobType.from_value(None).json)
self.assertEqual('"None"', GobType.from_value("None").json)
self.assertEqual('null', GobType.from_value(float('nan')).json)
def get_attribute_dict(self):
"""Gets an dict with attributes to insert entities in bulk
:return:
"""
entity = {
self.timestamp_field: self._metadata.timestamp,
'_application': self._metadata.application,
'_source': self._metadata.source
}
for key, value in self._data.items():
if key not in self.skip:
gob_type = get_gob_type(self._model['all_fields'][key]['type'])
entity[key] = gob_type.from_value(value).to_db
return entity
def write_temporary_entity(self, entity):
"""
Writes an entity to the temporary table
:param entity:
:return:
"""
row = {}
for field in self.fields:
gob_type = get_gob_type(
self.collection['all_fields'][field]['type'])
if field == '_source':
row[field] = gob_type.from_value(self.metadata.source).to_db
else:
row[field] = gob_type.from_value(entity[field]).to_db
row["_original_value"] = entity
self.temporary_rows.append(row)
self._write_temporary_entities(write_per=10000)
def test_reference(self):
GobType = get_gob_type("GOB.Reference")
self.assertEqual(GobType.name, "Reference")
# Test that a value of id is ignored when comparing references. Only test bronwaarde
v1 = GobType.from_value('{"bronwaarde": "123456"}')
v2 = GobType.from_value(
'{"bronwaarde": "123456", "id": "123456", "volgnummer": "1"}')
self.assertEqual(v1, v2)
# Test that None value is accepted
self.assertTrue(GobType.from_value(None) == GobType.from_value(None))
self.assertTrue(GobType.from_value({}) == GobType.from_value({}))
self.assertTrue(GobType.from_value(None) != GobType.from_value({}))
self.assertTrue(GobType.from_value({}) != GobType.from_value(None))
self.assertTrue(GobType.from_value(v1) != GobType.from_value(None))
self.assertTrue(GobType.from_value(None) != GobType.from_value(v1))
def test_char(self):
GobType = get_gob_type("GOB.Character")
self.assertEqual(GobType.name, "Character")
self.assertEqual('null', GobType.from_value(None).json)
with self.assertRaises(GOBException):
self.assertEqual('"1"', GobType.from_value(123).json)
with self.assertRaises(GOBException):
self.assertEqual('"1"', GobType.from_value("123").json)
self.assertEqual('"1"', GobType.from_value(1).json)
self.assertEqual('"N"', GobType.from_value('N').json)
with self.assertRaises(GOBException):
self.assertEqual('"O"', GobType.from_value('Overtime').json)
# DB ouptut is string
self.assertIsInstance(GobType.from_value('O').to_db, str)
# Python value is string
self.assertIsInstance(GobType.from_value('O').to_value, str)
def test_gob_point(self):
GobType = get_gob_type("GOB.Geo.Point")
self.assertEqual(GobType.name, "Point")
self.assertEqual('POINT(112.0 22.0)',
str(GobType.from_values(x=112, y=22, precision=1)))
self.assertEqual('POINT(112.000 22.000)',
str(GobType.from_values(x=112, y=22)))
self.assertEqual(
'POINT(52.3063972 4.9627873)',
str(GobType.from_values(x=52.3063972, y=4.9627873, precision=7)))
self.assertEqual(
'POINT(52.3063972 4.9627873)',
str(
GobType.from_values(x="52,3063972",
y="4,9627873",
precision=7,
decimal_separator=",")))
self.assertEqual('{"type": "Point", "coordinates": [112.0, 22.0]}',
GobType.from_values(x=112, y=22).json)
self.assertEqual(
'{"type": "Point", "coordinates": [52.3063972, 4.9627873]}',
GobType.from_values(x=52.3063972, y=4.9627873, precision=7).json)
self.assertEqual('POINT (112.000 22.000)',
str(GobType.from_value('POINT(112.0 22.0)')))
self.assertEqual(
'POINT (52.3063972 4.9627873)',
str(GobType.from_value('POINT(52.3063972 4.9627873)',
precision=7)))
empty_point = GobType('')
self.assertEqual('null', empty_point.json)
mock_db_field = MagicMock(spec=geoalchemy2.elements.WKBElement)
mock_db_field.srid = 28992
mock_db_field.data = '01010000204071000000000000f0abfd400000000050a11d41'
self.assertEqual('POINT (121535.000 485460.000)',
str(GobType.from_value(mock_db_field)))
self.assertEqual('POINT (1.000 2.000)',
str(GobType.from_value('POINT (1 2)')))
def test_int(self):
GobType = get_gob_type("GOB.Integer")
self.assertEqual(GobType.name, "Integer")
self.assertEqual('null', GobType.from_value(None).json)
self.assertEqual('null', GobType.from_value("nan").json)
self.assertEqual('123', GobType.from_value(123).json)
self.assertEqual('123', GobType.from_value("123").json)
# DB ouptut is int
self.assertIsInstance(GobType.from_value('123').to_db, int)
# Python value is int
self.assertIsInstance(GobType.from_value('123').to_value, int)
with self.assertRaises(GOBException):
GobType.from_value('N')
with self.assertRaises(GOBException):
GobType.from_value(True)
with self.assertRaises(GOBException):
GobType.from_value(1.3)
with self.assertRaises(GOBException):
GobType.from_value('Overtime')
def test_boolean(self):
GobType = get_gob_type("GOB.Boolean")
self.assertEqual(GobType.name, "Boolean")
self.assertEqual('null', GobType.from_value(None).json)
self.assertEqual('true', GobType.from_value(True).json)
self.assertEqual("False", str(GobType.from_value(False)))
self.assertEqual('false', GobType.from_value(False).json)
self.assertIsNone(GobType.from_value(None).to_db)
self.assertTrue(GobType.from_value(True).to_db)
self.assertFalse(GobType.from_value(False).to_db)
self.assertIsNone(GobType.from_value(None).to_value)
self.assertTrue(GobType.from_value(True).to_value)
self.assertFalse(GobType.from_value(False).to_value)
with self.assertRaises(GOBException):
GobType.from_value('N')
with self.assertRaises(GOBException):
GobType.from_value('Overtime')
with self.assertRaises(GOBException):
GobType.from_value(1)
self.assertEqual('true', GobType.from_value(1, format='10').json)
self.assertEqual('false', GobType.from_value(0, format='10').json)
self.assertIsNone(GobType.from_value('J', format='10').to_db)
self.assertEqual('true', GobType.from_value('J', format='JN').json)
self.assertEqual('false', GobType.from_value('N', format='JN').json)
self.assertIsNone(GobType.from_value('Y', format='JN').to_db)
self.assertEqual('true', GobType.from_value('Y', format='YN').json)
self.assertEqual('false', GobType.from_value('N', format='YN').json)
self.assertIsNone(GobType.from_value('J', format='YN').to_db)
# Test unknown format
with self.assertRaises(GOBTypeException):
GobType.from_value('Yes', format='YesNo')
def _build_timeslot_rows(
collections,
entities_with_timeslots,
primary_collection_name, # noqa: C901
relations,
collections_with_state,
offset,
limit):
"""Builds the output of the timeslot rows within the given offset and limit.
A timeslot row is a state for an entity within a certain timeslot.
Returns a list timeslot rows and the total count of timeslot rows
:param collections: The collections for which states are returned
:param entities_with_timeslots: A dict of timeslots grouped by entity id
:param primary_collection_name: The name of the base collection we are exporting
:param relations: A dictionary for relations between the given collections
:param collections_with_state: A dictionary with all states grouped by collection name
:return timeslot_rows, total_count: A list of timeslot rows and the total count of timeslot rows
"""
row_count = 0
timeslot_rows = []
# for each timeslot get the valid state and related states
for entity_id, timeslots in entities_with_timeslots.items():
for count, timeslot in enumerate(timeslots):
timeslot_start = timeslot
timeslot_end = timeslots[
count + 1] if count + 1 < len(timeslots) else END_OF_TIME
valid_states = _get_valid_states_in_timeslot(
timeslot_start=timeslot_start,
timeslot_end=timeslot_end,
collection_name=primary_collection_name,
entity_id=entity_id,
relations=relations,
collections_with_state=collections_with_state)
# Only add a row if a valid state has been found for the primary collection
if (valid_states[primary_collection_name]):
row_count += 1
# Don't store the row if it doesn't match the requested offset and limit
if row_count <= offset or (row_count - offset) > limit:
continue
# First fill the primary state
catalog_name, collection_name = primary_collection_name.split(
':')
model = GOBModel().get_collection(catalog_name,
collection_name)
entity_convert = _get_convert_for_state(model)
row = entity_convert(valid_states.pop(primary_collection_name))
gob_date = get_gob_type("GOB.Date")
row['begin_tijdvak'] = gob_date.from_value(timeslot_start)
row['eind_tijdvak'] = gob_date.from_value(
timeslot_end if timeslot_end != END_OF_TIME else None)
# Add the related states, so skip the first collection
itercollections = iter(collections)
next(itercollections)
for collection in itercollections:
collection_name = ':'.join(collection)
model = GOBModel().get_collection(*collection)
entity_convert = _get_convert_for_state(
model, RELATED_COLLECTION_FIELDS)
try:
related_row = entity_convert(
valid_states[collection_name])
related_row = {
f'{collection_name}_{k}': v
for k, v in related_row.items()
}
row.update(related_row)
except KeyError:
# If a relation can't be found, add null values
related_row = {
f'{collection_name}_{k}': None
for k in RELATED_COLLECTION_FIELDS
}
row.update(related_row)
timeslot_rows.append(row)
return timeslot_rows, row_count
