def setUp(self): signals.post_save.disconnect(layer_post_save, sender=Layer) signals.post_save.disconnect(service_post_save, sender=Service) for s in range(0, SERVICE_NUMBER): service = Service( url='http://%s.fakeurl.com' % s, title='Title %s' % s, type='OGC:WMS', ) service.save() for l in range(0, LAYER_PER_SERVICE_NUMBER): layer = Layer(name='Layer %s, from service %s' % (l, s), bbox_x0=-179, bbox_x1=179, bbox_y0=-89, bbox_y1=89, service=service) layer.save() service.layer_set.add(layer) for c in range(0, TIMES_TO_CHECK): for s in range(0, SERVICE_NUMBER): service = Service.objects.all()[s] service.check_available() for layer in service.layer_set.all(): layer.check_available()
def create_wm_service(): catalog, created = Catalog.objects.get_or_create(name="hypermap", slug="hypermap", url="search_api") service = Service(type='Hypermap:WorldMap', catalog=catalog) service.save()
def test_create_wms_service(self): # create the service service = Service( type='OGC:WMS', url='http://wms.example.com/ows?', ) service.save() # check layer number self.assertEqual(service.layer_set.all().count(), 9) # check layer 0 (public) layer_0 = service.layer_set.all()[0] self.assertEqual(layer_0.name, 'geonode:_30river_project1_1') self.assertEqual(layer_0.title, 'Rivers') self.assertTrue(layer_0.is_public) self.assertEqual(layer_0.keywords.all().count(), 3) self.assertEqual(layer_0.srs.all().count(), 2) self.assertEqual(layer_0.check_set.all().count(), 1) # TODO test layer_0.layerdate_set # test that if creating the service and is already exiting it is not being duplicated # create the service def create_duplicated_service(): duplicated_service = Service( type='OGC:WMS', url='http://wms.example.com/ows?', ) duplicated_service.save() self.assertRaises(Exception, create_duplicated_service)
def test_create_wms_service(self): catalog, created = Catalog.objects.get_or_create(name="hypermap", slug="hypermap", url="search_api") # create the service service = Service(type='OGC:WMS', url='http://wms.example.com/ows130?', catalog=catalog) service.save() # check layer number self.assertEqual(service.layer_set.all().count(), 3) # check layer nexrad-n0r-wmst (public) layer_0 = service.layer_set.get(name='nexrad-n0r-wmst') self.assertEqual(layer_0.title, 'NEXRAD BASE REFLECT') self.assertTrue(layer_0.is_public) self.assertEqual(layer_0.keywords.all().count(), 0) self.assertEqual(layer_0.service.srs.all().count(), 4) self.assertEqual(layer_0.check_set.all().count(), 1) # TODO test layer_0.layerdate_set # test that if creating the service and is already exiting it is not being duplicated # create the service def create_duplicated_service(): duplicated_service = Service( type='OGC:WMS', url='http://wms.example.com/ows130?', ) duplicated_service.save() self.assertRaises(Exception, create_duplicated_service)
def create_warper_service(): catalog, created = Catalog.objects.get_or_create(name="hypermap", slug="hypermap", url="search_api") service = Service(type='Hypermap:WARPER', url='http://warper.example.com/warper/maps', catalog=catalog) service.save()
def create_wms_service(): catalog, created = Catalog.objects.get_or_create(name="hypermap", slug="hypermap", url="search_api") service = Service(type='OGC:WMS', url='http://wms.example.com/ows111?', catalog=catalog) service.save()
def _insert_or_update(self, resourcetype, source, mode='insert'): ''' Insert or update a record in the repository ''' try: s = Service(type=HYPERMAP_SERVICE_TYPES[resourcetype], url=source) s.save() except Exception as err: raise RuntimeError('HHypermap error: %s' % err) # return a list of ids that were inserted or updated ids = [] for res in Service.objects.filter(url=source).all(): ids.append({'identifier': res.id_string, 'title': res.title}) return ids
def test_create_warper_service(self): # apparently signals are disconnected when running this test # TODO figure out which test is disconnecting them without reconnecting from django.db.models import signals from hypermap.aggregator.models import Layer from hypermap.aggregator.models import layer_post_save, service_post_save signals.post_save.connect(layer_post_save, sender=Layer) signals.post_save.connect(service_post_save, sender=Service) catalog, created = Catalog.objects.get_or_create(name="hypermap", slug="hypermap", url="search_api") # create the service service = Service(type='Hypermap:WARPER', url='http://warper.example.com/warper/maps', catalog=catalog) service.save() # check layer number self.assertEqual(service.layer_set.all().count(), 15) # check layer name 29568 (public) layer_0 = service.layer_set.get(name='29568') self.assertEqual(layer_0.name, '29568') self.assertEqual(layer_0.title, 'Plate 24: Map bounded by Myrtle Avenue') self.assertTrue(layer_0.is_public) self.assertEqual(layer_0.keywords.all().count(), 0) self.assertEqual(layer_0.service.srs.all().count(), 3) self.assertEqual(layer_0.check_set.all().count(), 1) self.assertEqual(layer_0.layerdate_set.all()[0].date, '1855-01-01') # a layer with no bbox must be stored with None coordinates layer_no_bbox = service.layer_set.get(name='16239') self.assertEqual(layer_no_bbox.bbox_x0, None) self.assertEqual(layer_no_bbox.bbox_y0, None) self.assertEqual(layer_no_bbox.bbox_x1, None) self.assertEqual(layer_no_bbox.bbox_y1, None) # test that if creating the service and is already exiting it is not being duplicated # create the service def create_duplicated_service(): duplicated_service = Service( type='Hypermap:WARPER', url='http://warper.example.com/warper/maps', ) duplicated_service.save() self.assertRaises(Exception, create_duplicated_service)
def test_create_wms_service(self): catalog, created = Catalog.objects.get_or_create( name="hypermap", slug="hypermap", url="search_api" ) # create the service service = Service( type='Hypermap:WARPER', url='http://warper.example.com/warper/maps', catalog=catalog ) service.save() # check layer number self.assertEqual(service.layer_set.all().count(), 15) # check layer name 29568 (public) layer_0 = service.layer_set.get(name='29568') self.assertEqual(layer_0.name, '29568') self.assertEqual(layer_0.title, 'Plate 24: Map bounded by Myrtle Avenue') self.assertTrue(layer_0.is_public) self.assertEqual(layer_0.keywords.all().count(), 0) self.assertEqual(layer_0.service.srs.all().count(), 3) self.assertEqual(layer_0.check_set.all().count(), 1) self.assertEqual(layer_0.layerdate_set.all()[0].date, '1855-01-01') # a layer with no bbox must be stored with None coordinates layer_no_bbox = service.layer_set.get(name='16239') self.assertEqual(layer_no_bbox.bbox_x0, None) self.assertEqual(layer_no_bbox.bbox_y0, None) self.assertEqual(layer_no_bbox.bbox_x1, None) self.assertEqual(layer_no_bbox.bbox_y1, None) # test that if creating the service and is already exiting it is not being duplicated # create the service def create_duplicated_service(): duplicated_service = Service( type='Hypermap:WARPER', url='http://warper.example.com/warper/maps', ) duplicated_service.save() self.assertRaises(Exception, create_duplicated_service)
def test_create_wms_service(self): catalog, created = Catalog.objects.get_or_create(name="hypermap", slug="hypermap", url="search_api") # create the service service = Service(type='OGC:WMS', url='http://wms.example.com/ows-invalid?', catalog=catalog) service.save() service.refresh_from_db() # check service is invalid self.assertEqual(service.is_valid, False) # check layers number self.assertEqual(service.layer_set.all().count(), 3) # check all layers are invalid for layer in service.layer_set.all(): self.assertEqual(layer.is_valid, False)
def setUp(self): signals.post_save.disconnect(layer_post_save, sender=Layer) signals.post_save.disconnect(service_post_save, sender=Service) catalog, created = Catalog.objects.get_or_create( name="hypermap", slug="hypermap", url="search_api" ) for s in range(0, SERVICE_NUMBER): service = Service( url='http://%s.fakeurl.com' % s, title='Title %s' % s, type='OGC:WMS', catalog=catalog ) service.save() for l in range(0, LAYER_PER_SERVICE_NUMBER): layer = Layer( name='Layer %s, from service %s' % (l, s), bbox_x0=-179, bbox_x1=179, bbox_y0=-89, bbox_y1=89, service=service, catalog=service.catalog ) layer.save() service.layer_set.add(layer) for c in range(0, TIMES_TO_CHECK): for s in range(0, SERVICE_NUMBER): service = Service.objects.all()[s] service.check_available() for layer in service.layer_set.all(): layer.check_available()
def create_duplicated_service(): duplicated_service = Service( type='OGC:WMS', url='http://wms.example.com/ows130?', ) duplicated_service.save()
def setUp(self): signals.post_save.disconnect(layer_post_save, sender=Layer) signals.post_save.disconnect(service_post_save, sender=Service) catalog_test_slug = "hypermap" if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print('> updating schema'.format(SEARCH_URL)) self.solr.update_schema(catalog=catalog_test_slug) print('> clearing SEARCH_URL={0}'.format(SEARCH_URL)) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) catalog, created = Catalog.objects.get_or_create( name=catalog_test_slug ) service = Service( url='http://fakeurl.com', title='Title', type='OGC:WMS', catalog=catalog ) service.save() layer = Layer( name='Layer 1', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 2', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 3', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 4', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) # solr have commitWithin 1500. # before to proceed with the tests wait for 2 secs. # otherwise it will return zero docs in the next test. service.index_layers(with_cache=False) time.sleep(2) self.api_url = "{0}{1}".format( settings.SITE_URL, reverse("search_api", args=[catalog_test_slug]) ) self.default_params = { "search_engine": SEARCH_TYPE, "search_engine_endpoint": self.search_engine_endpoint, "q_time": "[* TO *]", "q_geo": "[-90,-180 TO 90,180]", "d_docs_limit": 0, "d_docs_page": 1, "d_docs_sort": "score" }
def create_duplicated_service(): duplicated_service = Service( type='Hypermap:WorldMap', ) duplicated_service.save()
def create_duplicated_service(): duplicated_service = Service(type='Hypermap:WorldMap', url='http://worldmap.harvard.edu/') duplicated_service.save()
def _insert_or_update(self, resourcetype, source, mode='insert', hhclass='Service'): """ Insert or update a record in the repository """ keywords = [] if self.filter is not None: catalog = Catalog.objects.get(id=int(self.filter.split()[-1])) try: if hhclass == 'Layer': # TODO: better way of figuring out duplicates match = Layer.objects.filter(name=source.name, title=source.title, abstract=source.abstract, is_monitored=False) matches = match.all() if matches: if mode == 'insert': raise RuntimeError('HHypermap error: Layer %d \'%s\' already exists' % ( matches[0].id, source.title)) elif mode == 'update': match.update( name=source.name, title=source.title, abstract=source.abstract, is_monitored=False, xml=source.xml, wkt_geometry=source.wkt_geometry, anytext=util.get_anytext([source.title, source.abstract, source.keywords_csv]) ) service = get_service(source.xml) res, keywords = create_layer_from_metadata_xml(resourcetype, source.xml, monitor=False, service=service, catalog=catalog) res.save() LOGGER.debug('Indexing layer with id %s on search engine' % res.uuid) index_layer(res.id, use_cache=True) else: if resourcetype == 'http://www.opengis.net/cat/csw/2.0.2': res = Endpoint(url=source, catalog=catalog) else: res = Service(type=HYPERMAP_SERVICE_TYPES[resourcetype], url=source, catalog=catalog) res.save() if keywords: for kw in keywords: res.keywords.add(kw) except Exception as err: raise RuntimeError('HHypermap error: %s' % err) # return a list of ids that were inserted or updated ids = [] if hhclass == 'Layer': ids.append({'identifier': res.uuid, 'title': res.title}) else: if resourcetype == 'http://www.opengis.net/cat/csw/2.0.2': for res in Endpoint.objects.filter(url=source).all(): ids.append({'identifier': res.uuid, 'title': res.url}) else: for res in Service.objects.filter(url=source).all(): ids.append({'identifier': res.uuid, 'title': res.title}) return ids
def create_wm_service(): service = Service(type='Hypermap:WorldMap', ) service.save()
def create_warper_service(): service = Service( type='Hypermap:WARPER', url='http://warper.example.com/warper/maps', ) service.save()
def create_wms_service(): service = Service( type='OGC:WMS', url='http://wms.example.com/ows?', ) service.save()
def create_duplicated_service(): duplicated_service = Service( type='Hypermap:WARPER', url='http://warper.example.com/warper/maps', ) duplicated_service.save()
def test_create_worldmap_service(self): catalog, created = Catalog.objects.get_or_create(name="hypermap", slug="hypermap", url="search_api") # create the service service = Service(type='Hypermap:WorldMap', url='http://worldmap.harvard.edu/', catalog=catalog) service.save() service.refresh_from_db() # check title is correct self.assertTrue(service.title, 'Harvard WorldMap') # check layer number self.assertEqual(service.layer_set.all().count(), 10) # check deleted layers number self.assertEqual(service.layer_set.filter(was_deleted=True).count(), 2) # check layer geonode:River_basin_num2 (public) layer_0 = service.layer_set.get(name='geonode:River_basin_num2') self.assertEqual(layer_0.title, 'China River Basins') self.assertTrue(layer_0.is_public) self.assertEqual(layer_0.layerwm.temporal_extent_start, '1971-02-06') self.assertEqual(layer_0.layerwm.temporal_extent_end, '1974-09-30') self.assertEqual(layer_0.layerwm.category, 'Rivers, Streams, Lakes') self.assertEqual(layer_0.layerwm.username, 'matt') self.assertEqual(layer_0.keywords.all().count(), 4) self.assertEqual(layer_0.service.srs.all().count(), 3) self.assertEqual(layer_0.check_set.all().count(), 1) # test dates #1 layer_with_many_dates = service.layer_set.get( name='geonode:layer_with_many_dates') # this layer has the following dates # in title: 1999, 1882 # in abstract: 1632, 1661, 1992, 1338 # from metadata: temporal_extent_start: 2011-01-24 # from metadata: temporal_extent_end: 2015-09-30 # check detected dates for date in ('1999-01-01', '1882-01-01', '1632-01-01', '1661-01-01', '1992-01-01'): date_filtered = layer_with_many_dates.layerdate_set.filter( date=date) self.assertEqual( date_filtered.filter(type=DATE_DETECTED).count(), 1) # check metadata dates for date in ('2011-01-24', '2015-09-30'): date_filtered = layer_with_many_dates.layerdate_set.filter( date=date) self.assertEqual( date_filtered.filter(type=DATE_FROM_METADATA).count(), 1) # test dates #2 layer_with_few_dates = service.layer_set.get( name='geonode:layer_with_few_dates') # this layer has the following dates # in title: 1990 # in abstract: 1990, 1996, 2000 # from metadata: temporal_extent_start: 1990-01-01 # from metadata: temporal_extent_end: Date in invalid format # check detected dates for date in ('1990-01-01', '1996-01-01', '2000-01-01'): self.assertEqual( layer_with_few_dates.layerdate_set.filter(date=date).filter( type=DATE_DETECTED).count(), 1) # check metadata dates for date in ('1990-01-01', ): date_filtered = layer_with_few_dates.layerdate_set.filter( date=date) self.assertEqual( date_filtered.filter(type=DATE_FROM_METADATA).count(), 1) # test dates #3 layer_with_complex_dates = service.layer_set.get( name='geonode:layer_with_complex_dates') # this layer has the following dates # in abstract: 1900 BCE, 2000 BCE, Xia dynasty # from metadata: temporal_extent_start: -1900-01-01 # from metadata: temporal_extent_end: -2000-01-01 for date in ('-1900-01-01', '-2000-01-01', '-1600-01-01', '-2100-01-01'): date_filtered = layer_with_complex_dates.layerdate_set.filter( date=date) self.assertEqual( date_filtered.filter(type=DATE_DETECTED).count(), 1) # check metadata dates for date in ( '-1900-01-01', '-2000-01-01', ): date_filtered = layer_with_complex_dates.layerdate_set.filter( date=date) self.assertEqual( date_filtered.filter(type=DATE_FROM_METADATA).count(), 1) # test dates #4 layer_with_dates_in_abstract = service.layer_set.get( name='geonode:layer_with_dates_in_abstract') # this layer has the following dates # in abstract: 1901, 1902 for date in ('1901-01-01', '1902-01-01'): self.assertEqual( layer_with_dates_in_abstract.layerdate_set.filter( date=date).filter(type=DATE_DETECTED).count(), 1) # test dates #5 layer_with_html_tag = service.layer_set.get( name='geonode:layer_with_html_tag') self.assertEqual( layer_with_html_tag.layerdate_set.filter(date=date).filter( type=DATE_DETECTED).count(), 0) # check layer geonode:layer_with_many_dates (private) layer_1 = service.layer_set.get(name='geonode:layer_with_many_dates') self.assertFalse(layer_1.is_public) # test that if creating the service and is already exiting it is not being duplicated # create the service def create_duplicated_service(): duplicated_service = Service(type='Hypermap:WorldMap', url='http://worldmap.harvard.edu/') duplicated_service.save() self.assertRaises(Exception, create_duplicated_service)