def test_decimal(self): GobType = get_gob_type("GOB.Decimal") self.assertEqual(GobType.name, "Decimal") self.assertEqual('null', GobType.from_value(None).json) self.assertEqual('null', GobType.from_value("nan").json) self.assertEqual('123.0', GobType.from_value(123).json) self.assertEqual('123.123', GobType.from_value(123.123).json) # DB ouptut is float self.assertIsInstance(GobType.from_value('123').to_db, float) # Python value is string self.assertIsInstance(GobType.from_value('123').to_value, str) with self.assertRaises(GOBException): GobType.from_value("123,123") self.assertEqual( '123.123', GobType.from_value("123,123", decimal_separator=',').json) self.assertEqual('123.1', GobType.from_value("123.123", precision=1).json) self.assertEqual('123.1000', str(GobType.from_value("123.1", precision=4))) self.assertEqual('123.1', GobType.from_value("123.1", precision=4).json) with self.assertRaises(GOBException): GobType.from_value('N') with self.assertRaises(GOBException): GobType.from_value(True) with self.assertRaises(GOBException): GobType.from_value('Overtime')
def test_date(self): GobType = get_gob_type("GOB.Date") self.assertEqual(GobType.name, "Date") self.assertEqual('null', GobType.from_value(None).json) self.assertEqual("2016-05-04", str(GobType.from_value('2016-05-04'))) self.assertEqual("2016-05-04", str(GobType.from_value('20160504', format="%Y%m%d"))) self.assertEqual('"2016-05-04"', GobType.from_value('2016-05-04').json) self.assertEqual('"2016-05-04"', GobType.from_value('20160504', format="%Y%m%d").json) # Test edge case https://bugs.python.org/issue13305 strftime is not consistent for years < 1000 self.assertEqual('"0001-05-04"', GobType.from_value('0001-05-04').json) with self.assertRaises(GOBException): GobType.from_value('N') with self.assertRaises(GOBException): GobType.from_value('Overtime') with self.assertRaises(GOBException): GobType.from_value(1) # DB ouptut is datetime self.assertIsInstance(GobType.from_value('2016-05-04').to_db, datetime) # Python value is datetime self.assertIsInstance(GobType.from_value('2016-05-04').to_value, date)
def create_temporary_table(self): """ Create a new temporary table based on the current table for a collection Message data is inserted to be compared with the current state :param data: the imported data :return: """ self.collection = self.gob_model.get_collection( self.metadata.catalogue, self.metadata.entity) table_name = self.gob_model.get_table_name(self.metadata.catalogue, self.metadata.entity) tmp_table_name = self._get_tmp_table_name(table_name) self.fields = ['_tid', '_source', '_hash'] # Drop any existing temporary table self.drop_temporary_table(tmp_table_name) columns = [ get_column(c, self.collection['all_fields'][c]) for c in self.fields ] columns.append( get_gob_type("GOB.JSON").get_column_definition("_original_value")) self.tmp_table = Table(tmp_table_name, self.base.metadata, *columns, extend_existing=True) self.tmp_table.create(self.engine) self.temporary_rows = [] return tmp_table_name
def test_reference(self): GobType = get_gob_type("GOB.Reference") self.assertEqual(GobType.name, "Reference") # Test that a value of id is ignored when comparing references. Only test bronwaarde self.assertEqual( GobType.from_value('{"bronwaarde": "123456"}'), GobType.from_value('{"bronwaarde": "123456", "id": "123456"}'))
def test_datetime(self): GobType = get_gob_type("GOB.DateTime") self.assertEqual(GobType.name, "DateTime") self.assertEqual('null', GobType.from_value(None).json) self.assertEqual("2016-05-04T12:00:00.123000", str(GobType.from_value('2016-05-04T12:00:00.123000'))) self.assertEqual( "2016-05-04T12:00:00.123000", str( GobType.from_value('20160504 12:00:00.123000', format="%Y%m%d %H:%M:%S.%f"))) self.assertEqual("2016-05-04T12:00:00.000000", str(GobType.from_value('2016-05-04T12:00:00'))) self.assertEqual( "2016-05-04T12:00:00.000000", str( GobType.from_value('20160504 12:00:00', format="%Y%m%d %H:%M:%S"))) self.assertEqual('"2016-05-04T12:00:00.123000"', GobType.from_value('2016-05-04T12:00:00.123000').json) self.assertEqual( '"2016-05-04T12:00:00.123000"', GobType.from_value('20160504 12:00:00.123000', format="%Y%m%d %H:%M:%S.%f").json) # Test edge case https://bugs.python.org/issue13305 strftime is not consistent for years < 1000 self.assertEqual('"0005-05-04T12:00:00.123000"', GobType.from_value('0005-05-04T12:00:00.123000').json) self.assertEqual( '"0005-05-04T12:00:00.123000"', GobType.from_value('00050504 12:00:00.123000', format="%Y%m%d %H:%M:%S.%f").json) with self.assertRaises(GOBException): GobType.from_value('N') with self.assertRaises(GOBException): GobType.from_value('Overtime') with self.assertRaises(GOBException): GobType.from_value(1) # DB ouptut is datetime self.assertIsInstance( GobType.from_value('2016-05-04T12:00:00.123000').to_db, datetime) # unless an empty string is entered self.assertIsNone(GobType.from_value(None).to_db) # Python value is datetime self.assertIsInstance( GobType.from_value('2016-05-04T12:00:00.123000').to_value, datetime) # unless an empty string is entered self.assertIsNone(GobType.from_value(None).to_value)
def test_gob_geometry(self): GobType = get_gob_type("GOB.Geo.Geometry") self.assertEqual(GobType.name, "Geometry") self.assertEqual('POINT (1.000 2.000)', str(GobType.from_value('POINT (1 2)'))) empty_geometry = GobType('') self.assertEqual('null', empty_geometry.json)
def test_gob_polygon(self): GobType = get_gob_type("GOB.Geo.Polygon") self.assertEqual(GobType.name, "Polygon") self.assertEqual( 'POLYGON(112.0 22.0, 113.0 22.0, 113.0 21.0)', str( GobType.from_value( 'POLYGON(112.0 22.0, 113.0 22.0, 113.0 21.0)'))) empty_polygon = GobType('') self.assertEqual('null', empty_polygon.json)
def get_column(column_name, column_specification): """Utility method to convert GOB type to a SQLAlchemy Column Get the SQLAlchemy columndefinition for the gob type as exposed by the gob_type :param column: (name, type) :return: sqlalchemy.Column """ gob_type = get_gob_type(column_specification["type"]) column = gob_type.get_column_definition(column_name) column.doc = column_specification["description"] return column
def test_incomplete_date(self): GobType = get_gob_type("GOB.IncompleteDate") self.assertEqual(GobType.name, "IncompleteDate") def test_result(t, year, month, day, formatted): self.assertEqual(year, t.year) self.assertEqual(month, t.month) self.assertEqual(day, t.day) self.assertEqual( json.dumps( { 'year': year, 'month': month, 'day': day, 'formatted': formatted }, sort_keys=True), t._string) t = GobType.from_value(None) self.assertIsNone(t.to_value) t = GobType.from_value("2020-03-21") test_result(t, 2020, 3, 21, '2020-03-21') t = GobType.from_value({'year': 2020, 'month': 3, 'day': 21}) test_result(t, 2020, 3, 21, '2020-03-21') t = GobType.from_value("2020-03-00") test_result(t, 2020, 3, None, '2020-03-00') t = GobType.from_value({'year': 2020, 'month': None, 'day': 21}) test_result(t, 2020, None, 21, '2020-00-21') t = GobType.from_value( json.dumps({ 'year': 2020, 'month': None, 'day': 21 })) test_result(t, 2020, None, 21, '2020-00-21') with self.assertRaises(GOBTypeException): GobType.from_value( json.dumps({ 'year': 2020, 'month': None, 'day': 21 })[:-2]) with self.assertRaises(GOBTypeException): GobType.from_value({'year': 2020}) with self.assertRaises(GOBTypeException): GobType.from_value('2020-invalid')
def get_field_value(entity, field, spec): """ Get the value of the given field in the given entity. Use GOB type to correctly interpret the entity value :param entity: :param field: :param spec: :return: """ gob_type = get_gob_type(spec['type']) entity_value = getattr(entity, field, None) return None if entity_value is None else gob_type.from_value( entity_value).to_value
def test_json(self): GobType = get_gob_type("GOB.JSON") self.assertEqual(GobType.name, "JSON") self.assertEqual('null', GobType.from_value(None).json) key = fixtures.random_string() value = fixtures.random_string() generated_json = GobType.from_value({key: value}).json self.assertEqual(f'{{"{key}": "{value}"}}', GobType.from_value({ key: value }).json) self.assertEqual(f'["{key}", "{value}"]', GobType.from_value([key, value]).json) value = random.randint(9, 33) self.assertEqual(f'{{"{key}": {value}}}', GobType.from_value({key: value})) value = False self.assertEqual(f'{{"{key}": false}}', GobType.from_value({key: value})) value = None self.assertEqual(f'{{"{key}": null}}', GobType.from_value({key: value})) self.assertEqual(generated_json, GobType.from_value(generated_json)) self.assertEqual(generated_json, GobType(generated_json)) out_of_order = '{"b": "c", "a": "d"}' in_order = '{"a": "d", "b": "c"}' self.assertEqual(in_order, GobType(out_of_order)) self.assertEqual(in_order, GobType.from_value(GobType(out_of_order))) # Test unknown format with self.assertRaises(GOBTypeException): GobType.from_value('{"test" = "test"}') # DB ouptut is json self.assertIsInstance( GobType.from_value('{"key": "value"}').to_db, dict) # Python value is dict self.assertIsInstance( GobType.from_value('{"key": "value"}').to_value, dict)
def test_gob_type_and_string(self): GobType = get_gob_type("GOB.String") self.assertEqual(GobType.name, "String") self.assertEqual('None', str(GobType(None))) fixture = fixtures.random_string() # Gobtype can be instantiated with a string gob_type = GobType(fixture) # Gobtype can be compared with string self.assertTrue(gob_type == fixture) # Gobtype has a json representation self.assertEqual(gob_type.json, f'"{fixture}"') # Gobtype can be constructed from a value gob_type1 = GobType.from_value(fixture) # Gobtype can be compared with other gob_type self.assertTrue(gob_type == gob_type1) # DB ouptut is string self.assertIsInstance(gob_type.to_db, str) # Gobtype cannot be instantiated with something different than a string with self.assertRaises(GOBException): GobType(fixtures.random_bool()) with self.assertRaises(GOBException): GobType(random.randint(0, 10)) # Gobtype can be constructed from a value gob_type1 = GobType.from_value(int) gob_type2 = GobType.from_value(bool) # JSON values are String representations self.assertEqual('"1"', GobType.from_value(1).json) self.assertEqual('"True"', GobType.from_value(True).json) # Gobtype can be constructed from None self.assertNotEqual(GobType.from_value(None), GobType.from_value("None")) self.assertEqual(GobType.from_value(None), GobType.from_value(None)) self.assertEqual('null', GobType.from_value(None).json) self.assertEqual('"None"', GobType.from_value("None").json) self.assertEqual('null', GobType.from_value(float('nan')).json)
def get_attribute_dict(self): """Gets an dict with attributes to insert entities in bulk :return: """ entity = { self.timestamp_field: self._metadata.timestamp, '_application': self._metadata.application, '_source': self._metadata.source } for key, value in self._data.items(): if key not in self.skip: gob_type = get_gob_type(self._model['all_fields'][key]['type']) entity[key] = gob_type.from_value(value).to_db return entity
def write_temporary_entity(self, entity): """ Writes an entity to the temporary table :param entity: :return: """ row = {} for field in self.fields: gob_type = get_gob_type( self.collection['all_fields'][field]['type']) if field == '_source': row[field] = gob_type.from_value(self.metadata.source).to_db else: row[field] = gob_type.from_value(entity[field]).to_db row["_original_value"] = entity self.temporary_rows.append(row) self._write_temporary_entities(write_per=10000)
def test_reference(self): GobType = get_gob_type("GOB.Reference") self.assertEqual(GobType.name, "Reference") # Test that a value of id is ignored when comparing references. Only test bronwaarde v1 = GobType.from_value('{"bronwaarde": "123456"}') v2 = GobType.from_value( '{"bronwaarde": "123456", "id": "123456", "volgnummer": "1"}') self.assertEqual(v1, v2) # Test that None value is accepted self.assertTrue(GobType.from_value(None) == GobType.from_value(None)) self.assertTrue(GobType.from_value({}) == GobType.from_value({})) self.assertTrue(GobType.from_value(None) != GobType.from_value({})) self.assertTrue(GobType.from_value({}) != GobType.from_value(None)) self.assertTrue(GobType.from_value(v1) != GobType.from_value(None)) self.assertTrue(GobType.from_value(None) != GobType.from_value(v1))
def test_char(self): GobType = get_gob_type("GOB.Character") self.assertEqual(GobType.name, "Character") self.assertEqual('null', GobType.from_value(None).json) with self.assertRaises(GOBException): self.assertEqual('"1"', GobType.from_value(123).json) with self.assertRaises(GOBException): self.assertEqual('"1"', GobType.from_value("123").json) self.assertEqual('"1"', GobType.from_value(1).json) self.assertEqual('"N"', GobType.from_value('N').json) with self.assertRaises(GOBException): self.assertEqual('"O"', GobType.from_value('Overtime').json) # DB ouptut is string self.assertIsInstance(GobType.from_value('O').to_db, str) # Python value is string self.assertIsInstance(GobType.from_value('O').to_value, str)
def test_gob_point(self): GobType = get_gob_type("GOB.Geo.Point") self.assertEqual(GobType.name, "Point") self.assertEqual('POINT(112.0 22.0)', str(GobType.from_values(x=112, y=22, precision=1))) self.assertEqual('POINT(112.000 22.000)', str(GobType.from_values(x=112, y=22))) self.assertEqual( 'POINT(52.3063972 4.9627873)', str(GobType.from_values(x=52.3063972, y=4.9627873, precision=7))) self.assertEqual( 'POINT(52.3063972 4.9627873)', str( GobType.from_values(x="52,3063972", y="4,9627873", precision=7, decimal_separator=","))) self.assertEqual('{"type": "Point", "coordinates": [112.0, 22.0]}', GobType.from_values(x=112, y=22).json) self.assertEqual( '{"type": "Point", "coordinates": [52.3063972, 4.9627873]}', GobType.from_values(x=52.3063972, y=4.9627873, precision=7).json) self.assertEqual('POINT (112.000 22.000)', str(GobType.from_value('POINT(112.0 22.0)'))) self.assertEqual( 'POINT (52.3063972 4.9627873)', str(GobType.from_value('POINT(52.3063972 4.9627873)', precision=7))) empty_point = GobType('') self.assertEqual('null', empty_point.json) mock_db_field = MagicMock(spec=geoalchemy2.elements.WKBElement) mock_db_field.srid = 28992 mock_db_field.data = '01010000204071000000000000f0abfd400000000050a11d41' self.assertEqual('POINT (121535.000 485460.000)', str(GobType.from_value(mock_db_field))) self.assertEqual('POINT (1.000 2.000)', str(GobType.from_value('POINT (1 2)')))
def test_int(self): GobType = get_gob_type("GOB.Integer") self.assertEqual(GobType.name, "Integer") self.assertEqual('null', GobType.from_value(None).json) self.assertEqual('null', GobType.from_value("nan").json) self.assertEqual('123', GobType.from_value(123).json) self.assertEqual('123', GobType.from_value("123").json) # DB ouptut is int self.assertIsInstance(GobType.from_value('123').to_db, int) # Python value is int self.assertIsInstance(GobType.from_value('123').to_value, int) with self.assertRaises(GOBException): GobType.from_value('N') with self.assertRaises(GOBException): GobType.from_value(True) with self.assertRaises(GOBException): GobType.from_value(1.3) with self.assertRaises(GOBException): GobType.from_value('Overtime')
def test_boolean(self): GobType = get_gob_type("GOB.Boolean") self.assertEqual(GobType.name, "Boolean") self.assertEqual('null', GobType.from_value(None).json) self.assertEqual('true', GobType.from_value(True).json) self.assertEqual("False", str(GobType.from_value(False))) self.assertEqual('false', GobType.from_value(False).json) self.assertIsNone(GobType.from_value(None).to_db) self.assertTrue(GobType.from_value(True).to_db) self.assertFalse(GobType.from_value(False).to_db) self.assertIsNone(GobType.from_value(None).to_value) self.assertTrue(GobType.from_value(True).to_value) self.assertFalse(GobType.from_value(False).to_value) with self.assertRaises(GOBException): GobType.from_value('N') with self.assertRaises(GOBException): GobType.from_value('Overtime') with self.assertRaises(GOBException): GobType.from_value(1) self.assertEqual('true', GobType.from_value(1, format='10').json) self.assertEqual('false', GobType.from_value(0, format='10').json) self.assertIsNone(GobType.from_value('J', format='10').to_db) self.assertEqual('true', GobType.from_value('J', format='JN').json) self.assertEqual('false', GobType.from_value('N', format='JN').json) self.assertIsNone(GobType.from_value('Y', format='JN').to_db) self.assertEqual('true', GobType.from_value('Y', format='YN').json) self.assertEqual('false', GobType.from_value('N', format='YN').json) self.assertIsNone(GobType.from_value('J', format='YN').to_db) # Test unknown format with self.assertRaises(GOBTypeException): GobType.from_value('Yes', format='YesNo')
def _build_timeslot_rows( collections, entities_with_timeslots, primary_collection_name, # noqa: C901 relations, collections_with_state, offset, limit): """Builds the output of the timeslot rows within the given offset and limit. A timeslot row is a state for an entity within a certain timeslot. Returns a list timeslot rows and the total count of timeslot rows :param collections: The collections for which states are returned :param entities_with_timeslots: A dict of timeslots grouped by entity id :param primary_collection_name: The name of the base collection we are exporting :param relations: A dictionary for relations between the given collections :param collections_with_state: A dictionary with all states grouped by collection name :return timeslot_rows, total_count: A list of timeslot rows and the total count of timeslot rows """ row_count = 0 timeslot_rows = [] # for each timeslot get the valid state and related states for entity_id, timeslots in entities_with_timeslots.items(): for count, timeslot in enumerate(timeslots): timeslot_start = timeslot timeslot_end = timeslots[ count + 1] if count + 1 < len(timeslots) else END_OF_TIME valid_states = _get_valid_states_in_timeslot( timeslot_start=timeslot_start, timeslot_end=timeslot_end, collection_name=primary_collection_name, entity_id=entity_id, relations=relations, collections_with_state=collections_with_state) # Only add a row if a valid state has been found for the primary collection if (valid_states[primary_collection_name]): row_count += 1 # Don't store the row if it doesn't match the requested offset and limit if row_count <= offset or (row_count - offset) > limit: continue # First fill the primary state catalog_name, collection_name = primary_collection_name.split( ':') model = GOBModel().get_collection(catalog_name, collection_name) entity_convert = _get_convert_for_state(model) row = entity_convert(valid_states.pop(primary_collection_name)) gob_date = get_gob_type("GOB.Date") row['begin_tijdvak'] = gob_date.from_value(timeslot_start) row['eind_tijdvak'] = gob_date.from_value( timeslot_end if timeslot_end != END_OF_TIME else None) # Add the related states, so skip the first collection itercollections = iter(collections) next(itercollections) for collection in itercollections: collection_name = ':'.join(collection) model = GOBModel().get_collection(*collection) entity_convert = _get_convert_for_state( model, RELATED_COLLECTION_FIELDS) try: related_row = entity_convert( valid_states[collection_name]) related_row = { f'{collection_name}_{k}': v for k, v in related_row.items() } row.update(related_row) except KeyError: # If a relation can't be found, add null values related_row = { f'{collection_name}_{k}': None for k in RELATED_COLLECTION_FIELDS } row.update(related_row) timeslot_rows.append(row) return timeslot_rows, row_count