def index_layer(self, layer): # TODO: Make this function more DRY # by abstracting the common bits. if settings.SEARCH_TYPE == 'solr': from hypermap.aggregator.solr import SolrHypermap print 'Syncing layer %s to solr' % layer.name try: solrobject = SolrHypermap() success, message = solrobject.layer_to_solr(layer) if not success: from hypermap.aggregator.models import TaskError task_error = TaskError(task_name=self.name, args=layer.id, message=message) task_error.save() except: print 'There was an exception here!' self.retry(layer) elif settings.SEARCH_TYPE == 'elasticsearch': from hypermap.aggregator.elasticsearch_client import ESHypermap print 'Syncing layer %s to es' % layer.name esobject = ESHypermap() success, message = esobject.layer_to_es(layer) if not success: from hypermap.aggregator.models import TaskError task_error = TaskError(task_name=self.name, args=layer.id, message=message) task_error.save()
def index_layer(self, layer_id, use_cache=False): """ Index a layer in the search backend. If cache is set, append it to the list, if it isn't send the transaction right away. cache needs memcached to be available. """ from hypermap.aggregator.models import Layer layer = Layer.objects.get(id=layer_id) if not layer.is_valid: LOGGER.debug('Not indexing or removing layer with id %s in search engine as it is not valid' % layer.id) unindex_layer(layer.id, use_cache) return if layer.was_deleted: LOGGER.debug('Not indexing or removing layer with id %s in search engine as was_deleted is true' % layer.id) unindex_layer(layer.id, use_cache) return # 1. if we use cache if use_cache: LOGGER.debug('Caching layer with id %s for syncing with search engine' % layer.id) layers = cache.get('layers') if layers is None: layers = set([layer.id]) else: layers.add(layer.id) cache.set('layers', layers) return # 2. if we don't use cache # TODO: Make this function more DRY # by abstracting the common bits. if SEARCH_TYPE == 'solr': from hypermap.aggregator.solr import SolrHypermap LOGGER.debug('Syncing layer %s to solr' % layer.name) solrobject = SolrHypermap() success, message = solrobject.layer_to_solr(layer) # update the error message if using celery if not settings.REGISTRY_SKIP_CELERY: if not success: self.update_state( state=states.FAILURE, meta=message ) raise Ignore() elif SEARCH_TYPE == 'elasticsearch': from hypermap.aggregator.elasticsearch_client import ESHypermap LOGGER.debug('Syncing layer %s to es' % layer.name) esobject = ESHypermap() success, message = esobject.layer_to_es(layer) # update the error message if using celery if not settings.REGISTRY_SKIP_CELERY: if not success: self.update_state( state=states.FAILURE, meta=message ) raise Ignore()
def unindex_layer(self, layer_id, use_cache=False): """ Remove the index for a layer in the search backend. If cache is set, append it to the list of removed layers, if it isn't send the transaction right away. """ from hypermap.aggregator.models import Layer layer = Layer.objects.get(id=layer_id) if use_cache: LOGGER.debug('Caching layer with id %s for being removed from search engine' % layer.id) deleted_layers = cache.get('deleted_layers') if deleted_layers is None: deleted_layers = set([layer.id]) else: deleted_layers.add(layer.id) cache.set('deleted_layers', deleted_layers) return if SEARCH_TYPE == 'solr': from hypermap.aggregator.solr import SolrHypermap LOGGER.debug('Removing layer %s from solr' % layer.id) try: solrobject = SolrHypermap() solrobject.remove_layer(layer.uuid) except Exception: LOGGER.error('Layer NOT correctly removed from Solr') elif SEARCH_TYPE == 'elasticsearch': # TODO implement me pass
def index_layer(self, layer): # TODO: Make this function more DRY # by abstracting the common bits. if settings.SEARCH_TYPE == 'solr': from hypermap.aggregator.solr import SolrHypermap print 'Syncing layer %s to solr' % layer.name try: solrobject = SolrHypermap() success, message = solrobject.layer_to_solr(layer) if not success: from hypermap.aggregator.models import TaskError task_error = TaskError( task_name=self.name, args=layer.id, message=message ) task_error.save() except: print 'There was an exception here!' self.retry(layer) elif settings.SEARCH_TYPE == 'elasticsearch': from hypermap.aggregator.elasticsearch_client import ESHypermap print 'Syncing layer %s to es' % layer.name esobject = ESHypermap() success, message = esobject.layer_to_es(layer) if not success: from hypermap.aggregator.models import TaskError task_error = TaskError( task_name=self.name, args=layer.id, message=message ) task_error.save()
def setUp(self): if not SELENIUM_HUB_URL: # run test on firefox of this machine. self.driver = webdriver.Firefox() else: # run test on stand alone node machine in docker: selenium-firefox self.driver = webdriver.Remote( command_executor=SELENIUM_HUB_URL, desired_capabilities=DesiredCapabilities.FIREFOX) self.driver.implicitly_wait(30) self.base_url = BROWSER_HYPERMAP_URL self.verificationErrors = [] self.accept_next_alert = True print('> clearing SEARCH_URL={0}'.format(SEARCH_URL)) if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print('> updating schema'.format(SEARCH_URL)) self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
def handle(self, *args, **options): """ reset core: rm -Rf SOLR_HOME/server/solr/hypermap_test solr6 create_core -c hypermap_test """ client = SolrHypermap() client.update_schema()
def clear_index(): if SEARCH_TYPE == 'solr': LOGGER.debug('Clearing the solr indexes') from hypermap.aggregator.solr import SolrHypermap solrobject = SolrHypermap() solrobject.clear_solr() elif SEARCH_TYPE == 'elasticsearch': LOGGER.debug('Clearing the ES indexes') from hypermap.aggregator.elasticsearch_client import ESHypermap esobject = ESHypermap() esobject.clear_es()
def setUp(self): self.client = Client() user = User.objects.create(username='******') user.set_password('admin') user.save() self.client.login(username="******", password="******") Catalog.objects.get_or_create(name=catalog_test_slug) Layer.objects.all().delete() Service.objects.all().delete() if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() es.clear_es() else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
def index_layer(self, layer): # TODO: Make this function more DRY # by abstracting the common bits. if SEARCH_TYPE == 'solr': from hypermap.aggregator.solr import SolrHypermap LOGGER.debug('Syncing layer %s to solr' % layer.name) try: solrobject = SolrHypermap() success, message = solrobject.layer_to_solr(layer) if not success: from hypermap.aggregator.models import TaskError task_error = TaskError( task_name=self.name, args=layer.id, message=message ) task_error.save() except Exception, e: LOGGER.error('Layers NOT indexed correctly') LOGGER.error(e, exc_info=True) self.retry(layer)
def index_layer(self, layer, use_cache=False): """Index a layer in the search backend. If cache is set, append it to the list, if it isn't send the transaction right away. cache needs memcached to be available. """ if use_cache: LOGGER.debug('Caching layer with id %s for syncing with search engine' % layer.id) layers = cache.get('layers') if layers is None: layers = set([layer.id]) else: layers.add(layer.id) cache.set('layers', layers) return # TODO: Make this function more DRY # by abstracting the common bits. if SEARCH_TYPE == 'solr': from hypermap.aggregator.solr import SolrHypermap LOGGER.debug('Syncing layer %s to solr' % layer.name) try: solrobject = SolrHypermap() success, message = solrobject.layer_to_solr(layer) if not success: from hypermap.aggregator.models import TaskError task_error = TaskError( task_name=self.name, args=layer.id, message=message ) task_error.save() except Exception, e: LOGGER.error('Layers NOT indexed correctly') LOGGER.error(e, exc_info=True) self.retry(layer)
def setUp(self): self.client = Client() user = User.objects.create(username='******') user.set_password('admin') user.save() self.client.login(username="******", password="******") Catalog.objects.get_or_create( name=catalog_test_slug ) Layer.objects.all().delete() Service.objects.all().delete() if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() es.clear_es() else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
def setUp(self): if not SELENIUM_HUB_URL: # run test on firefox of this machine. self.driver = webdriver.Firefox() else: # run test on stand alone node machine in docker: selenium-firefox self.driver = webdriver.Remote( command_executor=SELENIUM_HUB_URL, desired_capabilities=DesiredCapabilities.FIREFOX ) self.driver.implicitly_wait(30) self.base_url = BROWSER_HYPERMAP_URL self.verificationErrors = [] self.accept_next_alert = True print '> clearing SEARCH_URL={0}'.format(SEARCH_URL) if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print '> updating schema'.format(SEARCH_URL) self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
class SearchApiTestCase(TestCase): """ run me python manage.py test hypermap.search_api --settings=hypermap.settings.test --failfast """ def tearDown(self): signals.post_save.connect(layer_post_save, sender=Layer) signals.post_save.connect(service_post_save, sender=Service) def setUp(self): signals.post_save.disconnect(layer_post_save, sender=Layer) signals.post_save.disconnect(service_post_save, sender=Service) catalog_test_slug = "hypermap" if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print '> updating schema'.format(SEARCH_URL) self.solr.update_schema(catalog=catalog_test_slug) print '> clearing SEARCH_URL={0}'.format(SEARCH_URL) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) catalog, created = Catalog.objects.get_or_create( name=catalog_test_slug ) service = Service( url='http://fakeurl.com', title='Title', type='OGC:WMS', catalog=catalog ) service.save() layer = Layer( name='Layer 1', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 2', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 3', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 4', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) # solr have commitWithin 1500. # before to proceed with the tests wait for 2 secs. # otherwise it will return zero docs in the next test. service.index_layers(with_cache=False) time.sleep(2) self.api_url = "{0}{1}".format( settings.SITE_URL, reverse("search_api", args=[catalog_test_slug]) ) self.default_params = { "search_engine": SEARCH_TYPE, "search_engine_endpoint": self.search_engine_endpoint, "q_time": "[* TO *]", "q_geo": "[-90,-180 TO 90,180]", "d_docs_limit": 0, "d_docs_page": 1, "d_docs_sort": "score" } def test_catalogs(self): print '> testing catalogs' url = settings.SITE_URL + reverse("catalog-list") res = self.client.get(url) self.assertEqual(res.status_code, 200) catalogs = json.loads(res.content) self.assertEqual(len(catalogs), Catalog.objects.all().count()) def test_all_match_docs(self): print '> testing match all docs' params = self.default_params print "searching on [{}]".format(self.api_url) results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) def test_q_text(self): print '> testing q text' layer = Layer.objects.all()[0] params = self.default_params params["q_text"] = "title:\"{0}\"".format(layer.title) params["d_docs_limit"] = 100 results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) for doc in results.get("d.docs", []): self.assertEqual(doc["title"], layer.title) def test_q_geo(self): print '> testing q geo' params = self.default_params # top right square params["q_geo"] = "[0,0 TO 30,30]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) # bottom left square params["q_geo"] = "[-30,-30 TO 0,0]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) # big square params["q_geo"] = "[-30,-30 TO 30,30]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 4) # center where no layers params["q_geo"] = "[-5,-5 TO 5,5]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 0) # bad format params["q_geo"] = "[-5,-5 5,5]" results = self.client.get(self.api_url, params) # validate the format print '> testing q geo (format validations)' self.assertEqual(results.status_code, 400) def test_q_time(self): print '> testing q time (format validations)' params = self.default_params # test validations params["q_time"] = "[2000-01-01 - 2001-01-01T00:00:00]" results = self.client.get(self.api_url, params) # requires [X TO Y] self.assertEqual(400, results.status_code) print '> testing q time' # test asterisks # all times params["q_time"] = "[* TO *]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) # all records self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) # test range # entire year 2000 params["q_time"] = "[2000-01-01 TO 2001-01-01T00:00:00]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) # 1 in year 2000 self.assertEqual(results["a.matchDocs"], 1) # test complete min and max when q time is asterisks params["q_time"] = "[* TO *]" params["a_time_limit"] = 1 if SEARCH_TYPE == SEARCH_TYPE_ES: # TODO: a_time_limit is WIP in ES, today requires a a_time_gap to be completed. params["a_time_gap"] = "P1Y" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) if SEARCH_TYPE == SEARCH_TYPE_SOLR: # TODO: fix on Solr or ES? see next TODO. # * TO * to first date and last date. self.assertEqual(results["a.time"]["start"].upper(), "2000-03-01T00:00:00Z") self.assertEqual(results["a.time"]["end"].upper(), "2003-03-01T00:00:00Z") else: # TODO: ES and SOLR returns facets by default spliting the data yearly. first record is on 2000-03, # ES facets are returned from 2000-01... to 2003-01. # SOLR facets are returned from 2000-03... to 2003-03. # SOLR data seems more accurate since first and last Layers are in month 03. # Example: http://panchicore.d.pr/12ESP # * TO * to first date and last date. self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z") self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z") # test facets params["q_time"] = "[2000 TO 2022]" params["a_time_limit"] = 1 params["a_time_gap"] = "P1Y" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) # 2000 to complete datetime format self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z") # 2022 to complete datetime format if SEARCH_TYPE == SEARCH_TYPE_SOLR: # TODO: solr creates entire time span and brings facets with empty entries. (2000 to 2022) # fix by removing facets with zero counts?. self.assertEqual(results["a.time"]["end"].upper(), "2022-01-01T00:00:00Z") else: self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z") # the facet counters are all facets excluding < 2000 self.assertEqual(len(results["a.time"]["counts"]), Layer.objects.all().count()) def test_utilities(self): print '> testing utilities functions' # test_parse_datetime_range start, end = utils.parse_datetime_range("[2013-03-01 TO 2014-05-02T23:00:00]") self.assertTrue(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime").year, 2013) self.assertEqual(start.get("parsed_datetime").month, 3) self.assertEqual(start.get("parsed_datetime").day, 1) self.assertTrue(end.get("is_common_era")) self.assertEqual(end.get("parsed_datetime").year, 2014) self.assertEqual(end.get("parsed_datetime").month, 5) self.assertEqual(end.get("parsed_datetime").day, 2) self.assertEqual(end.get("parsed_datetime").hour, 23) self.assertEqual(end.get("parsed_datetime").minute, 0) self.assertEqual(end.get("parsed_datetime").second, 0) start, end = utils.parse_datetime_range("[-500000000 TO 2014-05-02T23:00:00]") self.assertFalse(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime"), "-500000000-01-01T00:00:00Z") start, end = utils.parse_datetime_range("[* TO *]") self.assertTrue(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime"), None) self.assertEqual(end.get("parsed_datetime"), None) # test_parse_ISO8601 quantity, units = utils.parse_ISO8601("P3D") self.assertEqual(quantity, 3) self.assertEqual(units[0], "DAYS") # test_gap_to_sorl value = utils.gap_to_sorl("P3D") self.assertEqual(value, "+3DAYS") # test_parse_geo_box value = utils.parse_geo_box("[-90,-180 TO 90,180]") self.assertEqual(value.bounds[0], -90) self.assertEqual(value.bounds[1], -180) self.assertEqual(value.bounds[2], 90) self.assertEqual(value.bounds[3], 180) # test_request_time_facet d = utils.request_time_facet("x", "[2000 TO 2014-01-02T11:12:13]", None, 1000) self.assertEqual(type(d), dict) self.assertEqual(d['f.x.facet.range.start'], '2000-01-01T00:00:00Z') self.assertEqual(d['f.x.facet.range.end'], '2014-01-02T11:12:13Z') self.assertEqual(d['f.x.facet.range.gap'], '+6DAYS') self.assertEqual(d['facet.range'], 'x') d = utils.request_time_facet("y", "[-5000000 TO 2016]", "P1D", 1) self.assertEqual(d['f.y.facet.range.start'], '-5000000-01-01T00:00:00Z') self.assertEqual(d['f.y.facet.range.end'], '2016-01-01T00:00:00Z') self.assertEqual(d['f.y.facet.range.gap'], '+1DAYS') self.assertEqual(d['facet.range'], 'y')
def index_cached_layers(self): """ Index and unindex all layers in the Django cache (Index all layers who have been checked). """ from hypermap.aggregator.models import Layer if SEARCH_TYPE == 'solr': from hypermap.aggregator.solr import SolrHypermap solrobject = SolrHypermap() else: from hypermap.aggregator.elasticsearch_client import ESHypermap from elasticsearch import helpers es_client = ESHypermap() layers_cache = cache.get('layers') deleted_layers_cache = cache.get('deleted_layers') # 1. added layers cache if layers_cache: layers_list = list(layers_cache) LOGGER.debug('There are %s layers in cache: %s' % (len(layers_list), layers_list)) batch_size = settings.REGISTRY_SEARCH_BATCH_SIZE batch_lists = [layers_list[i:i+batch_size] for i in range(0, len(layers_list), batch_size)] for batch_list_ids in batch_lists: layers = Layer.objects.filter(id__in=batch_list_ids) if batch_size > len(layers): batch_size = len(layers) LOGGER.debug('Syncing %s/%s layers to %s: %s' % (batch_size, len(layers_cache), layers, SEARCH_TYPE)) try: # SOLR if SEARCH_TYPE == 'solr': success, layers_errors_ids = solrobject.layers_to_solr(layers) if success: # remove layers from cache here layers_cache = layers_cache.difference(set(batch_list_ids)) LOGGER.debug('Removing layers with id %s from cache' % batch_list_ids) cache.set('layers', layers_cache) # ES elif SEARCH_TYPE == 'elasticsearch': with_bulk, success = True, False layers_to_index = [es_client.layer_to_es(layer, with_bulk) for layer in layers] message = helpers.bulk(es_client.es, layers_to_index) # Check that all layers where indexed...if not, don't clear cache. # TODO: Check why es does not index all layers at first. len_indexed_layers = message[0] if len_indexed_layers == len(layers): LOGGER.debug('%d layers indexed successfully' % (len_indexed_layers)) success = True if success: # remove layers from cache here layers_cache = layers_cache.difference(set(batch_list_ids)) cache.set('layers', layers_cache) else: raise Exception("Incorrect SEARCH_TYPE=%s" % SEARCH_TYPE) except Exception as e: LOGGER.error('Layers were NOT indexed correctly') LOGGER.error(e, exc_info=True) else: LOGGER.debug('No cached layers to add in search engine.') # 2. deleted layers cache if deleted_layers_cache: layers_list = list(deleted_layers_cache) LOGGER.debug('There are %s layers in cache for deleting: %s' % (len(layers_list), layers_list)) # TODO implement me: batch layer index deletion for layer_id in layers_list: # SOLR if SEARCH_TYPE == 'solr': if Layer.objects.filter(pk=layer_id).exists(): layer = Layer.objects.get(id=layer_id) unindex_layer(layer.id, use_cache=False) deleted_layers_cache = deleted_layers_cache.difference(set([layer_id])) cache.set('deleted_layers', deleted_layers_cache) else: # TODO implement me raise NotImplementedError else: LOGGER.debug('No cached layers to remove in search engine.')
def clear_solr(): print 'Clearing the solr core and indexes' from hypermap.aggregator.solr import SolrHypermap solrobject = SolrHypermap() solrobject.clear_solr()
def setUp(self): signals.post_save.disconnect(layer_post_save, sender=Layer) signals.post_save.disconnect(service_post_save, sender=Service) catalog_test_slug = "hypermap" if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print '> updating schema'.format(SEARCH_URL) self.solr.update_schema(catalog=catalog_test_slug) print '> clearing SEARCH_URL={0}'.format(SEARCH_URL) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) catalog, created = Catalog.objects.get_or_create( name=catalog_test_slug ) service = Service( url='http://fakeurl.com', title='Title', type='OGC:WMS', catalog=catalog ) service.save() layer = Layer( name='Layer 1', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 2', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 3', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 4', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) # solr have commitWithin 1500. # before to proceed with the tests wait for 2 secs. # otherwise it will return zero docs in the next test. service.index_layers(with_cache=False) time.sleep(2) self.api_url = "{0}{1}".format( settings.SITE_URL, reverse("search_api", args=[catalog_test_slug]) ) self.default_params = { "search_engine": SEARCH_TYPE, "search_engine_endpoint": self.search_engine_endpoint, "q_time": "[* TO *]", "q_geo": "[-90,-180 TO 90,180]", "d_docs_limit": 0, "d_docs_page": 1, "d_docs_sort": "score" }
class TestCSWTransactions(unittest.TestCase): def setUp(self): self.client = Client() user = User.objects.create(username='******') user.set_password('admin') user.save() self.client.login(username="******", password="******") Catalog.objects.get_or_create(name=catalog_test_slug) Layer.objects.all().delete() Service.objects.all().delete() if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() es.clear_es() else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) def test_post(self): """ test CSV transactions. :return: """ print("") print(">>> with env:") print("REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY) print("REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS) print("REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD) print("REGISTRY_SEARCH_URL: %s" % settings.REGISTRY_SEARCH_URL) print("REGISTRY_HARVEST_SERVICES: %s" % settings.REGISTRY_HARVEST_SERVICES) print("") # Post the 10 Layers contained in this file: data/cswt_insert.xml path = os.path.join(settings.PROJECT_DIR, "..", "data", "cswt_insert.xml") payload = open(path, 'rb').read() content_type = "application/xml" url = "/registry/{0}/csw".format(catalog_test_slug) res = self.client.post(url, data=payload, content_type=content_type) self.assertEqual(res.status_code, 200) self.assertEqual(Layer.objects.all().count(), 10) # List the Layers posted above url = "/registry/{0}/csw?service=CSW&version=2.0.2&request=" \ "GetRecords&typenames=csw:Record&elementsetname=full&" \ "resulttype=results".format(catalog_test_slug) res = self.client.get(url) self.assertEqual(res.status_code, 200) self.assertEqual(res.content.count("Airports (OSM)"), 1) self.assertEqual(res.content.count("Manaus Roads (OSM May 2016)"), 2) # Search one Layer posted above url = "/registry/{0}/csw?mode=opensearch&service=CSW&version" \ "=2.0.2&request=GetRecords&elementsetname=full&typenames=" \ "csw:Record&resulttype=results" \ "&q=Airport".format(catalog_test_slug) res = self.client.get(url) self.assertEqual(res.status_code, 200) self.assertEqual(res.content.count("Airports (OSM)"), 1) # Flush layers in the cache. index_cached_layers() # Give celery some time. time.sleep(3) # are Layers in index? url = "{0}hypermap/_search".format(SEARCH_URL) res = requests.get(url) results_ok_in_search_backend = res.json() self.assertTrue("hits" in results_ok_in_search_backend) self.assertTrue("total" in results_ok_in_search_backend["hits"]) self.assertEqual(results_ok_in_search_backend["hits"]["total"], 10) def tearDown(self): pass
class TestBrowser(unittest.TestCase): def setUp(self): if not SELENIUM_HUB_URL: # run test on firefox of this machine. self.driver = webdriver.Firefox() else: # run test on stand alone node machine in docker: selenium-firefox self.driver = webdriver.Remote( command_executor=SELENIUM_HUB_URL, desired_capabilities=DesiredCapabilities.FIREFOX ) self.driver.implicitly_wait(30) self.base_url = BROWSER_HYPERMAP_URL self.verificationErrors = [] self.accept_next_alert = True print '> clearing SEARCH_URL={0}'.format(SEARCH_URL) if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print '> updating schema'.format(SEARCH_URL) self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) def test_browser(self): ENDPOINT_FILE = os.path.join("/usr/src/app", "hypermap", "tests", "mesonet.agron.iastate.edu.txt") print "" print ">>> with env:" print "REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY print "REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS print "REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD print "" print "SELENIUM_HUB_URL: %s" % SELENIUM_HUB_URL print "BROWSER_HYPERMAP_URL: %s" % BROWSER_HYPERMAP_URL print "BROWSER_SEARCH_URL: %s" % BROWSER_SEARCH_URL print "BROWSER_MAPLOOM_URL: %s" % BROWSER_MAPLOOM_URL print "WAIT_FOR_CELERY_JOB_PERIOD: %s" % WAIT_FOR_CELERY_JOB_PERIOD print "ENDPOINT FILE: %s" % ENDPOINT_FILE print "" print "Starting..." driver = self.driver time.sleep(3) driver.get(self.base_url + "/admin/login/?next=/admin/") print driver.current_url driver.find_element_by_id("id_password").clear() driver.find_element_by_id("id_password").send_keys("admin") driver.find_element_by_id("id_username").clear() driver.find_element_by_id("id_username").send_keys("admin") driver.find_element_by_css_selector("input[type=\"submit\"]").click() print driver.current_url driver.find_element_by_link_text("Periodic tasks").click() print driver.current_url print "> assert 3 periodic tasks. means beat is alive." self.assertEqual("3 periodic tasks", driver.find_element_by_css_selector( "p.paginator").text) driver.find_element_by_link_text("Home").click() print driver.current_url driver.find_element_by_link_text("Endpoint lists").click() print driver.current_url driver.find_element_by_link_text("Add endpoint list").click() print driver.current_url print "> uploading Endpoint List..." driver.find_element_by_id("id_upload").clear() driver.find_element_by_id("id_upload").send_keys(ENDPOINT_FILE) driver.find_element_by_name("_save").click() print driver.current_url print "> waiting {0} seconds for celery do the job....".format( WAIT_FOR_CELERY_JOB_PERIOD ) time.sleep(WAIT_FOR_CELERY_JOB_PERIOD) driver.find_element_by_link_text("Aggregator").click() time.sleep(1) print driver.current_url driver.find_element_by_link_text("Endpoints").click() print driver.current_url print "> assert Endpoint created." time.sleep(1) self.assertEqual( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi", driver.find_element_by_link_text( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").text) driver.find_element_by_link_text( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").click() # self.assertEqual("1 service/s created", driver.find_element_by_id("id_message").text) driver.find_element_by_link_text("Endpoints").click() print driver.current_url time.sleep(1) driver.find_element_by_link_text("Aggregator").click() print driver.current_url time.sleep(1) driver.find_element_by_link_text("Services").click() print driver.current_url print "> assert 1 Service created." time.sleep(1) self.assertEqual("1 service", driver.find_element_by_css_selector( "p.paginator").text) self.assertEqual( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi", driver.find_element_by_css_selector("td.field-url").text) driver.find_element_by_link_text("1").click() print driver.current_url print "> assert Service details." time.sleep(1) try: self.assertEqual("IEM NWS Warnings WMS Service", driver.find_element_by_id( "id_title").get_attribute("value")) except AssertionError as e: self.verificationErrors.append(str(e)) driver.find_element_by_link_text("Services").click() print driver.current_url driver.find_element_by_link_text("Aggregator").click() print driver.current_url driver.find_element_by_link_text("Layers").click() print driver.current_url print "> assert 3 layers created." time.sleep(1) self.assertEqual("3 layers", driver.find_element_by_css_selector( "p.paginator").text) driver.get(self.base_url + "/registry/") print driver.current_url print "> go to /registry/." for i in range(1, 11): print "> try assert checks count > 0. (%i of 10)" % i try: self.assertNotEqual("0", driver.find_element_by_xpath( "//td[4]").text) print "> found" break except AssertionError as e: print "> wait and reload page" time.sleep(10) driver.get(self.base_url + "/registry/") try: self.assertNotEqual("0", driver.find_element_by_xpath("//td[4]").text) except AssertionError as e: self.verificationErrors.append(str(e)) driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL)) print driver.current_url time.sleep(2) for i in range(1, 11): print "> assert layers indexed are 3. (%i of 10)" % i try: self.assertRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.refresh() self.assertRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") driver.get(self.base_url + "/registry/") print driver.current_url driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url print "> remove checks." driver.find_element_by_name("remove").click() print driver.current_url driver.find_element_by_link_text("Home").click() print driver.current_url print "> assert checks = 0." self.assertEqual("0", driver.find_element_by_xpath("//td[4]").text) driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url print "> trigger check." driver.find_element_by_name("check").click() print driver.current_url driver.find_element_by_link_text("Home").click() print driver.current_url for i in range(1, 11): try: print "> assert checks = 1. (%i of 10)" % i self.assertTrue( int(driver.find_element_by_xpath("//td[4]").text) > 0) print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.refresh() driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url driver.find_element_by_link_text("wwa").click() print driver.current_url print "> remove checks from Layer." driver.find_element_by_name("remove").click() print driver.current_url print "> assert text [No checks performed so far]." self.assertEqual("No checks performed so far", driver.find_element_by_xpath("//tr[11]/td[2]").text) print "> check Layer." driver.find_element_by_name("check").click() print driver.current_url for i in range(1, 11): try: print "> assert text [Total Checks: N>0]. (%i of 10)" % i src = driver.page_source text_found_TOTAL_CHECKS_LTE_1 = re.search( r'Total Checks: (1|2|3|4|5|6|7)', src) self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None) print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.get(driver.current_url) src = driver.page_source text_found_TOTAL_CHECKS_LTE_1 = re.search( r'Total Checks: (1|2|3|4|5|6|7)', src) self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None) driver.find_element_by_link_text("Home").click() print driver.current_url driver.find_element_by_link_text("Monitor").click() print driver.current_url print "> clean Search index and wait" driver.find_element_by_name("clear_index").click() print driver.current_url time.sleep(5) driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL)) print driver.current_url print "> assert count != 3 layers" try: self.assertNotRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") except AssertionError as e: self.verificationErrors.append(str(e)) driver.get(self.base_url + "/registry/") print driver.current_url print "> finish hypermap page" print "" # TODO: activate this to test maploom, now dat app looks very buggy. """ print ">> start maploom" driver.get(BROWSER_MAPLOOM_URL) print driver.current_url print ">> open registry modal" driver.find_element_by_xpath( "//div[@id='pulldown-content']/div[2]/div/div").click() print ">> assert Hypermap catalog" time.sleep(10) self.assertEqual("Hypermap", driver.find_element_by_xpath( "//div[@id='explore']/div/nav/div/form/div/div[2]/select").text) print ">> assert [Showing 3 of 3 - Page 1 / 1]" self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) driver.find_element_by_id("text_search_input_exp").clear() print ">> search IEM" driver.find_element_by_id("text_search_input_exp").send_keys("IEM") driver.find_element_by_id("text_search_btn").click() time.sleep(10) print ">> assert [Showing 1 of 1 - Page 1 / 1]" self.assertEqual("Showing 1 of 1 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) print ">> click reset" driver.find_element_by_name("button").click() time.sleep(10) print ">> assert [Showing 3 of 3 - Page 1 / 1]" self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) print ">> click on 3 layers to select" driver.find_element_by_css_selector("td.ellipsis.ng-binding").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click() print ">> click on 3 layers to unselect" driver.find_element_by_css_selector("td.ellipsis.ng-binding").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click() """ def is_element_present(self, how, what): try: self.driver.find_element(by=how, value=what) except NoSuchElementException as e: print e return False return True def is_alert_present(self): try: self.driver.switch_to_alert() except NoAlertPresentException as e: print e return False return True def close_alert_and_get_its_text(self): try: alert = self.driver.switch_to_alert() alert_text = alert.text if self.accept_next_alert: alert.accept() else: alert.dismiss() return alert_text finally: self.accept_next_alert = True def tearDown(self): self.driver.quit() self.assertEqual([], self.verificationErrors)
def setUp(self): signals.post_save.disconnect(layer_post_save, sender=Layer) signals.post_save.disconnect(service_post_save, sender=Service) catalog_test_slug = "hypermap" if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print('> updating schema'.format(SEARCH_URL)) self.solr.update_schema(catalog=catalog_test_slug) print('> clearing SEARCH_URL={0}'.format(SEARCH_URL)) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) catalog, created = Catalog.objects.get_or_create( name=catalog_test_slug ) service = Service( url='http://fakeurl.com', title='Title', type='OGC:WMS', catalog=catalog ) service.save() layer = Layer( name='Layer 1', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 2', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 3', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 4', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) # solr have commitWithin 1500. # before to proceed with the tests wait for 2 secs. # otherwise it will return zero docs in the next test. service.index_layers(with_cache=False) time.sleep(2) self.api_url = "{0}{1}".format( settings.SITE_URL, reverse("search_api", args=[catalog_test_slug]) ) self.default_params = { "search_engine": SEARCH_TYPE, "search_engine_endpoint": self.search_engine_endpoint, "q_time": "[* TO *]", "q_geo": "[-90,-180 TO 90,180]", "d_docs_limit": 0, "d_docs_page": 1, "d_docs_sort": "score" }
class SearchApiTestCase(TestCase): """ run me python manage.py test hypermap.search_api --settings=hypermap.settings.test --failfast """ def tearDown(self): signals.post_save.connect(layer_post_save, sender=Layer) signals.post_save.connect(service_post_save, sender=Service) def setUp(self): signals.post_save.disconnect(layer_post_save, sender=Layer) signals.post_save.disconnect(service_post_save, sender=Service) catalog_test_slug = "hypermap" if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print('> updating schema'.format(SEARCH_URL)) self.solr.update_schema(catalog=catalog_test_slug) print('> clearing SEARCH_URL={0}'.format(SEARCH_URL)) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) catalog, created = Catalog.objects.get_or_create( name=catalog_test_slug ) service = Service( url='http://fakeurl.com', title='Title', type='OGC:WMS', catalog=catalog ) service.save() layer = Layer( name='Layer 1', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 2', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 3', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 4', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) # solr have commitWithin 1500. # before to proceed with the tests wait for 2 secs. # otherwise it will return zero docs in the next test. service.index_layers(with_cache=False) time.sleep(2) self.api_url = "{0}{1}".format( settings.SITE_URL, reverse("search_api", args=[catalog_test_slug]) ) self.default_params = { "search_engine": SEARCH_TYPE, "search_engine_endpoint": self.search_engine_endpoint, "q_time": "[* TO *]", "q_geo": "[-90,-180 TO 90,180]", "d_docs_limit": 0, "d_docs_page": 1, "d_docs_sort": "score" } def test_catalogs(self): print('> testing catalogs') url = settings.SITE_URL + reverse("catalog-list") res = self.client.get(url) self.assertEqual(res.status_code, 200) catalogs = json.loads(res.content) self.assertEqual(len(catalogs), Catalog.objects.all().count()) def test_all_match_docs(self): print('> testing match all docs') params = self.default_params print("searching on [{}]".format(self.api_url)) results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) def test_q_text(self): print('> testing q text') layer = Layer.objects.all()[0] params = self.default_params params["q_text"] = "title:\"{0}\"".format(layer.title) params["d_docs_limit"] = 100 results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) for doc in results.get("d.docs", []): self.assertEqual(doc["title"], layer.title) def test_q_geo(self): print('> testing q geo') params = self.default_params # top right square params["q_geo"] = "[0,0 TO 30,30]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) # bottom left square params["q_geo"] = "[-30,-30 TO 0,0]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) # big square params["q_geo"] = "[-30,-30 TO 30,30]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 4) # center where no layers params["q_geo"] = "[-5,-5 TO 5,5]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 0) # bad format params["q_geo"] = "[-5,-5 5,5]" results = self.client.get(self.api_url, params) # validate the format print('> testing q geo (format validations)') self.assertEqual(results.status_code, 400) def test_q_time(self): print('> testing q time (format validations)') params = self.default_params # test validations params["q_time"] = "[2000-01-01 - 2001-01-01T00:00:00]" results = self.client.get(self.api_url, params) # requires [X TO Y] self.assertEqual(400, results.status_code) print('> testing q time') # test asterisks # all times params["q_time"] = "[* TO *]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) # all records self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) # test range # entire year 2000 params["q_time"] = "[2000-01-01 TO 2001-01-01T00:00:00]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) # 1 in year 2000 self.assertEqual(results["a.matchDocs"], 1) # test complete min and max when q time is asterisks params["q_time"] = "[* TO *]" params["a_time_limit"] = 1 if SEARCH_TYPE == SEARCH_TYPE_ES: # TODO: a_time_limit is WIP in ES, today requires a a_time_gap to be completed. params["a_time_gap"] = "P1Y" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) if SEARCH_TYPE == SEARCH_TYPE_SOLR: # TODO: fix on Solr or ES? see next TODO. # * TO * to first date and last date. self.assertEqual(results["a.time"]["start"].upper(), "2000-03-01T00:00:00Z") self.assertEqual(results["a.time"]["end"].upper(), "2003-03-01T00:00:00Z") else: # TODO: ES and SOLR returns facets by default spliting the data yearly. first record is on 2000-03, # ES facets are returned from 2000-01... to 2003-01. # SOLR facets are returned from 2000-03... to 2003-03. # SOLR data seems more accurate since first and last Layers are in month 03. # Example: http://panchicore.d.pr/12ESP # * TO * to first date and last date. self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z") self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z") # test facets params["q_time"] = "[2000 TO 2022]" params["a_time_limit"] = 1 params["a_time_gap"] = "P1Y" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) # 2000 to complete datetime format self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z") # 2022 to complete datetime format if SEARCH_TYPE == SEARCH_TYPE_SOLR: # TODO: solr creates entire time span and brings facets with empty entries. (2000 to 2022) # fix by removing facets with zero counts?. self.assertEqual(results["a.time"]["end"].upper(), "2022-01-01T00:00:00Z") else: self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z") # the facet counters are all facets excluding < 2000 self.assertEqual(len(results["a.time"]["counts"]), Layer.objects.all().count()) def test_utilities(self): print('> testing utilities functions') # test_parse_datetime_range start, end = utils.parse_datetime_range("[2013-03-01 TO 2014-05-02T23:00:00]") self.assertTrue(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime").year, 2013) self.assertEqual(start.get("parsed_datetime").month, 3) self.assertEqual(start.get("parsed_datetime").day, 1) self.assertTrue(end.get("is_common_era")) self.assertEqual(end.get("parsed_datetime").year, 2014) self.assertEqual(end.get("parsed_datetime").month, 5) self.assertEqual(end.get("parsed_datetime").day, 2) self.assertEqual(end.get("parsed_datetime").hour, 23) self.assertEqual(end.get("parsed_datetime").minute, 0) self.assertEqual(end.get("parsed_datetime").second, 0) start, end = utils.parse_datetime_range("[-500000000 TO 2014-05-02T23:00:00]") self.assertFalse(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime"), "-500000000-01-01T00:00:00Z") start, end = utils.parse_datetime_range("[* TO *]") self.assertTrue(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime"), None) self.assertEqual(end.get("parsed_datetime"), None) # test_parse_ISO8601 quantity, units = utils.parse_ISO8601("P3D") self.assertEqual(quantity, 3) self.assertEqual(units[0], "DAYS") # test_gap_to_sorl value = utils.gap_to_sorl("P3D") self.assertEqual(value, "+3DAYS") # test_parse_geo_box value = utils.parse_geo_box("[-90,-180 TO 90,180]") self.assertEqual(value.bounds[0], -90) self.assertEqual(value.bounds[1], -180) self.assertEqual(value.bounds[2], 90) self.assertEqual(value.bounds[3], 180) # test_request_time_facet d = utils.request_time_facet("x", "[2000 TO 2014-01-02T11:12:13]", None, 1000) self.assertEqual(type(d), dict) self.assertEqual(d['f.x.facet.range.start'], '2000-01-01T00:00:00Z') self.assertEqual(d['f.x.facet.range.end'], '2014-01-02T11:12:13Z') self.assertEqual(d['f.x.facet.range.gap'], '+6DAYS') self.assertEqual(d['facet.range'], 'x') d = utils.request_time_facet("y", "[-5000000 TO 2016]", "P1D", 1) self.assertEqual(d['f.y.facet.range.start'], '-5000000-01-01T00:00:00Z') self.assertEqual(d['f.y.facet.range.end'], '2016-01-01T00:00:00Z') self.assertEqual(d['f.y.facet.range.gap'], '+1DAYS') self.assertEqual(d['facet.range'], 'y')
class TestCSWTransactions(unittest.TestCase): def setUp(self): self.client = Client() user = User.objects.create(username='******') user.set_password('admin') user.save() self.client.login(username="******", password="******") Catalog.objects.get_or_create( name=catalog_test_slug ) Layer.objects.all().delete() Service.objects.all().delete() if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() es.clear_es() else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) def test_post(self): """ test CSV transactions. :return: """ print "" print ">>> with env:" print "REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY print "REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS print "REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD print "REGISTRY_SEARCH_URL: %s" % settings.REGISTRY_SEARCH_URL print "REGISTRY_HARVEST_SERVICES: %s" % settings.REGISTRY_HARVEST_SERVICES print "" # Post the 10 Layers contained in this file: data/cswt_insert.xml path = os.path.join(settings.PROJECT_DIR, "..", "data", "cswt_insert.xml") payload = open(path, 'rb').read() content_type = "application/xml" url = "/registry/{0}/csw".format(catalog_test_slug) res = self.client.post(url, data=payload, content_type=content_type) self.assertEqual(res.status_code, 200) self.assertEqual(Layer.objects.all().count(), 10) # List the Layers posted above url = "/registry/{0}/csw?service=CSW&version=2.0.2&request=" \ "GetRecords&typenames=csw:Record&elementsetname=full&" \ "resulttype=results".format(catalog_test_slug) res = self.client.get(url) self.assertEqual(res.status_code, 200) self.assertEqual(res.content.count("Airports (OSM)"), 1) self.assertEqual(res.content.count("Manaus Roads (OSM May 2016)"), 2) # Search one Layer posted above url = "/registry/{0}/csw?mode=opensearch&service=CSW&version" \ "=2.0.2&request=GetRecords&elementsetname=full&typenames=" \ "csw:Record&resulttype=results" \ "&q=Airport".format(catalog_test_slug) res = self.client.get(url) self.assertEqual(res.status_code, 200) self.assertEqual(res.content.count("Airports (OSM)"), 1) # Flush layers in the cache. index_cached_layers() # Give celery some time. time.sleep(3) # are Layers in index? url = "{0}hypermap/_search".format( SEARCH_URL ) res = requests.get(url) results_ok_in_search_backend = res.json() self.assertTrue("hits" in results_ok_in_search_backend) self.assertTrue("total" in results_ok_in_search_backend["hits"]) self.assertEqual(results_ok_in_search_backend["hits"]["total"], 10) def tearDown(self): pass
def index_cached_layers(self): """ Index all layers in the Django cache (Index all layers who have been checked). """ from hypermap.aggregator.models import Layer from hypermap.aggregator.models import TaskError if SEARCH_TYPE == 'solr': from hypermap.aggregator.solr import SolrHypermap solrobject = SolrHypermap() else: from hypermap.aggregator.elasticsearch_client import ESHypermap from elasticsearch import helpers es_client = ESHypermap() layers_cache = cache.get('layers') if layers_cache: layers_list = list(layers_cache) LOGGER.debug('There are %s layers in cache: %s' % (len(layers_list), layers_list)) batch_size = settings.REGISTRY_SEARCH_BATCH_SIZE batch_lists = [layers_list[i:i+batch_size] for i in range(0, len(layers_list), batch_size)] for batch_list_ids in batch_lists: layers = Layer.objects.filter(id__in=batch_list_ids) if batch_size > len(layers): batch_size = len(layers) LOGGER.debug('Syncing %s/%s layers to %s: %s' % (batch_size, len(layers_cache), layers, SEARCH_TYPE)) try: if SEARCH_TYPE == 'solr': success, message = solrobject.layers_to_solr(layers) elif SEARCH_TYPE == 'elasticsearch': with_bulk, success = True, False layers_to_index = [es_client.layer_to_es(layer, with_bulk) for layer in layers] message = helpers.bulk(es_client.es, layers_to_index) # Check that all layers where indexed...if not, don't clear cache. # TODO: Check why es does not index all layers at first. len_indexed_layers = message[0] if len_indexed_layers == len(layers): LOGGER.debug('%d layers indexed successfully' % (len_indexed_layers)) success = True else: raise Exception("Incorrect SEARCH_TYPE=%s" % SEARCH_TYPE) if success: # remove layers from cache here layers_cache = layers_cache.difference(set(batch_list_ids)) cache.set('layers', layers_cache) else: task_error = TaskError( task_name=self.name, args=batch_list_ids, message=message ) task_error.save() except Exception as e: LOGGER.error('Layers were NOT indexed correctly') LOGGER.error(e, exc_info=True) else: LOGGER.debug('No cached layers.')
class TestBrowser(unittest.TestCase): def setUp(self): if not SELENIUM_HUB_URL: # run test on firefox of this machine. self.driver = webdriver.Firefox() else: # run test on stand alone node machine in docker: selenium-firefox self.driver = webdriver.Remote( command_executor=SELENIUM_HUB_URL, desired_capabilities=DesiredCapabilities.FIREFOX ) self.driver.implicitly_wait(30) self.base_url = BROWSER_HYPERMAP_URL self.verificationErrors = [] self.accept_next_alert = True print '> clearing SEARCH_URL={0}'.format(SEARCH_URL) if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print '> updating schema'.format(SEARCH_URL) self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) def test_browser(self): ENDPOINT_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "mesonet.agron.iastate.edu.txt") print "" print ">>> with env:" print "REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY print "REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS print "REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD print "" print "SELENIUM_HUB_URL: %s" % SELENIUM_HUB_URL print "BROWSER_HYPERMAP_URL: %s" % BROWSER_HYPERMAP_URL print "BROWSER_SEARCH_URL: %s" % BROWSER_SEARCH_URL print "BROWSER_MAPLOOM_URL: %s" % BROWSER_MAPLOOM_URL print "WAIT_FOR_CELERY_JOB_PERIOD: %s" % WAIT_FOR_CELERY_JOB_PERIOD print "ENDPOINT FILE: %s" % ENDPOINT_FILE print "" print "Starting..." driver = self.driver time.sleep(3) driver.get(self.base_url + "/admin/login/?next=/admin/") print driver.current_url driver.find_element_by_id("id_password").clear() driver.find_element_by_id("id_password").send_keys("admin") driver.find_element_by_id("id_username").clear() driver.find_element_by_id("id_username").send_keys("admin") driver.find_element_by_css_selector("input[type=\"submit\"]").click() print driver.current_url driver.find_element_by_link_text("Periodic tasks").click() print driver.current_url print "> assert 3 periodic tasks. means beat is alive." self.assertEqual("3 periodic tasks", driver.find_element_by_css_selector( "p.paginator").text) driver.find_element_by_link_text("Home").click() print driver.current_url driver.find_element_by_link_text("Endpoint lists").click() print driver.current_url driver.find_element_by_link_text("Add endpoint list").click() print driver.current_url print "> uploading Endpoint List..." driver.find_element_by_id("id_upload").clear() driver.find_element_by_id("id_upload").send_keys(ENDPOINT_FILE) driver.find_element_by_name("_save").click() print driver.current_url print "> waiting {0} seconds for celery do the job....".format( WAIT_FOR_CELERY_JOB_PERIOD ) time.sleep(WAIT_FOR_CELERY_JOB_PERIOD) driver.find_element_by_link_text("Aggregator").click() time.sleep(1) print driver.current_url driver.find_element_by_link_text("Endpoints").click() print driver.current_url print "> assert Endpoint created." time.sleep(1) self.assertEqual( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi", driver.find_element_by_link_text( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").text) driver.find_element_by_link_text( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").click() # self.assertEqual("1 service/s created", driver.find_element_by_id("id_message").text) driver.find_element_by_link_text("Endpoints").click() print driver.current_url time.sleep(1) driver.find_element_by_link_text("Aggregator").click() print driver.current_url time.sleep(1) driver.find_element_by_link_text("Services").click() print driver.current_url print "> assert 1 Service created." time.sleep(1) self.assertEqual("1 service", driver.find_element_by_css_selector( "p.paginator").text) self.assertEqual( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi", driver.find_element_by_css_selector("td.field-url").text) driver.find_element_by_xpath( '//*[@id="result_list"]/tbody/tr/th/a').click() print driver.current_url print "> assert Service details." time.sleep(1) self.assertEqual("IEM NWS Warnings WMS Service", driver.find_element_by_id( "id_title").get_attribute("value")) driver.find_element_by_link_text("Services").click() print driver.current_url driver.find_element_by_link_text("Aggregator").click() print driver.current_url driver.find_element_by_link_text("Layers").click() print driver.current_url print "> assert 3 layers created." time.sleep(1) self.assertEqual("3 layers", driver.find_element_by_css_selector( "p.paginator").text) driver.get(self.base_url + "/registry/") print driver.current_url print "> go to /registry/." for i in range(1, 11): print "> try assert checks count > 0. (%i of 10)" % i try: self.assertNotEqual("0", driver.find_element_by_xpath( "//td[4]").text) print "> found" break except AssertionError as e: print "> wait and reload page" time.sleep(10) driver.get(self.base_url + "/registry/") try: self.assertNotEqual("0", driver.find_element_by_xpath("//td[4]").text) except AssertionError as e: self.verificationErrors.append(str(e)) driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL)) print driver.current_url time.sleep(2) for i in range(1, 11): print "> assert layers indexed are 3. (%i of 10)" % i try: self.assertRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.refresh() self.assertRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") driver.get(self.base_url + "/registry/") print driver.current_url driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url print "> remove checks." driver.find_element_by_name("remove").click() print driver.current_url driver.find_element_by_link_text("Home").click() print driver.current_url print "> assert checks = 0." self.assertEqual("0", driver.find_element_by_xpath("//td[4]").text) driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url print "> trigger check." driver.find_element_by_name("check").click() print driver.current_url driver.find_element_by_link_text("Home").click() print driver.current_url for i in range(1, 11): try: print "> assert checks = 1. (%i of 10)" % i self.assertTrue( int(driver.find_element_by_xpath("//td[4]").text) > 0) print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.refresh() driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url driver.find_element_by_link_text("wwa").click() print driver.current_url print "> remove checks from Layer." driver.find_element_by_name("remove").click() print driver.current_url print "> assert text [No checks performed so far]." self.assertEqual("No checks performed so far", driver.find_element_by_xpath("//tr[11]/td[2]").text) print "> check Layer." driver.find_element_by_name("check").click() print driver.current_url for i in range(1, 11): try: print "> assert text [Total Checks: N>0]. (%i of 10)" % i src = driver.page_source text_found_TOTAL_CHECKS_LTE_1 = re.search( r'Total Checks: (1|2|3|4|5|6|7)', src) self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None) print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.get(driver.current_url) src = driver.page_source text_found_TOTAL_CHECKS_LTE_1 = re.search( r'Total Checks: (1|2|3|4|5|6|7)', src) self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None) driver.find_element_by_link_text("Home").click() print driver.current_url driver.find_element_by_link_text("Monitor").click() print driver.current_url print "> clean Search index and wait" driver.find_element_by_name("clear_index").click() print driver.current_url time.sleep(5) driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL)) print driver.current_url print "> assert count != 3 layers" try: self.assertNotRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") except AssertionError as e: self.verificationErrors.append(str(e)) driver.get(self.base_url + "/registry/") print driver.current_url print "> finish hypermap page" print "" # TODO: activate this to test maploom, now dat app looks very buggy. """ print ">> start maploom" driver.get(BROWSER_MAPLOOM_URL) print driver.current_url print ">> open registry modal" driver.find_element_by_xpath( "//div[@id='pulldown-content']/div[2]/div/div").click() print ">> assert Hypermap catalog" time.sleep(10) self.assertEqual("Hypermap", driver.find_element_by_xpath( "//div[@id='explore']/div/nav/div/form/div/div[2]/select").text) print ">> assert [Showing 3 of 3 - Page 1 / 1]" self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) driver.find_element_by_id("text_search_input_exp").clear() print ">> search IEM" driver.find_element_by_id("text_search_input_exp").send_keys("IEM") driver.find_element_by_id("text_search_btn").click() time.sleep(10) print ">> assert [Showing 1 of 1 - Page 1 / 1]" self.assertEqual("Showing 1 of 1 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) print ">> click reset" driver.find_element_by_name("button").click() time.sleep(10) print ">> assert [Showing 3 of 3 - Page 1 / 1]" self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) print ">> click on 3 layers to select" driver.find_element_by_css_selector("td.ellipsis.ng-binding").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click() print ">> click on 3 layers to unselect" driver.find_element_by_css_selector("td.ellipsis.ng-binding").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click() """ def is_element_present(self, how, what): try: self.driver.find_element(by=how, value=what) except NoSuchElementException as e: print e return False return True def is_alert_present(self): try: self.driver.switch_to_alert() except NoAlertPresentException as e: print e return False return True def close_alert_and_get_its_text(self): try: alert = self.driver.switch_to_alert() alert_text = alert.text if self.accept_next_alert: alert.accept() else: alert.dismiss() return alert_text finally: self.accept_next_alert = True def tearDown(self): self.driver.quit() self.assertEqual([], self.verificationErrors)