def clear_index(): if SEARCH_TYPE == 'solr': LOGGER.debug('Clearing the solr indexes') from hypermap.aggregator.solr import SolrHypermap solrobject = SolrHypermap() solrobject.clear_solr() elif SEARCH_TYPE == 'elasticsearch': LOGGER.debug('Clearing the ES indexes') from hypermap.aggregator.elasticsearch_client import ESHypermap esobject = ESHypermap() esobject.clear_es()
def clear_index(): if SEARCH_TYPE == 'solr': LOGGER.debug('Clearing the solr indexes') from hypermap.aggregator.solr import SolrHypermap solrobject = SolrHypermap() solrobject.clear_solr() elif SEARCH_TYPE == 'elasticsearch': LOGGER.debug('Clearing the ES indexes') from hypermap.aggregator.elasticsearch_client import ESHypermap esobject = ESHypermap() esobject.clear_es()
class SearchApiTestCase(TestCase): """ run me python manage.py test hypermap.search_api --settings=hypermap.settings.test --failfast """ def tearDown(self): signals.post_save.connect(layer_post_save, sender=Layer) signals.post_save.connect(service_post_save, sender=Service) def setUp(self): signals.post_save.disconnect(layer_post_save, sender=Layer) signals.post_save.disconnect(service_post_save, sender=Service) catalog_test_slug = "hypermap" if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print('> updating schema'.format(SEARCH_URL)) self.solr.update_schema(catalog=catalog_test_slug) print('> clearing SEARCH_URL={0}'.format(SEARCH_URL)) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) catalog, created = Catalog.objects.get_or_create( name=catalog_test_slug ) service = Service( url='http://fakeurl.com', title='Title', type='OGC:WMS', catalog=catalog ) service.save() layer = Layer( name='Layer 1', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 2', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 3', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 4', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) # solr have commitWithin 1500. # before to proceed with the tests wait for 2 secs. # otherwise it will return zero docs in the next test. service.index_layers(with_cache=False) time.sleep(2) self.api_url = "{0}{1}".format( settings.SITE_URL, reverse("search_api", args=[catalog_test_slug]) ) self.default_params = { "search_engine": SEARCH_TYPE, "search_engine_endpoint": self.search_engine_endpoint, "q_time": "[* TO *]", "q_geo": "[-90,-180 TO 90,180]", "d_docs_limit": 0, "d_docs_page": 1, "d_docs_sort": "score" } def test_catalogs(self): print('> testing catalogs') url = settings.SITE_URL + reverse("catalog-list") res = self.client.get(url) self.assertEqual(res.status_code, 200) catalogs = json.loads(res.content) self.assertEqual(len(catalogs), Catalog.objects.all().count()) def test_all_match_docs(self): print('> testing match all docs') params = self.default_params print("searching on [{}]".format(self.api_url)) results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) def test_q_text(self): print('> testing q text') layer = Layer.objects.all()[0] params = self.default_params params["q_text"] = "title:\"{0}\"".format(layer.title) params["d_docs_limit"] = 100 results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) for doc in results.get("d.docs", []): self.assertEqual(doc["title"], layer.title) def test_q_geo(self): print('> testing q geo') params = self.default_params # top right square params["q_geo"] = "[0,0 TO 30,30]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) # bottom left square params["q_geo"] = "[-30,-30 TO 0,0]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) # big square params["q_geo"] = "[-30,-30 TO 30,30]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 4) # center where no layers params["q_geo"] = "[-5,-5 TO 5,5]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 0) # bad format params["q_geo"] = "[-5,-5 5,5]" results = self.client.get(self.api_url, params) # validate the format print('> testing q geo (format validations)') self.assertEqual(results.status_code, 400) def test_q_time(self): print('> testing q time (format validations)') params = self.default_params # test validations params["q_time"] = "[2000-01-01 - 2001-01-01T00:00:00]" results = self.client.get(self.api_url, params) # requires [X TO Y] self.assertEqual(400, results.status_code) print('> testing q time') # test asterisks # all times params["q_time"] = "[* TO *]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) # all records self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) # test range # entire year 2000 params["q_time"] = "[2000-01-01 TO 2001-01-01T00:00:00]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) # 1 in year 2000 self.assertEqual(results["a.matchDocs"], 1) # test complete min and max when q time is asterisks params["q_time"] = "[* TO *]" params["a_time_limit"] = 1 if SEARCH_TYPE == SEARCH_TYPE_ES: # TODO: a_time_limit is WIP in ES, today requires a a_time_gap to be completed. params["a_time_gap"] = "P1Y" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) if SEARCH_TYPE == SEARCH_TYPE_SOLR: # TODO: fix on Solr or ES? see next TODO. # * TO * to first date and last date. self.assertEqual(results["a.time"]["start"].upper(), "2000-03-01T00:00:00Z") self.assertEqual(results["a.time"]["end"].upper(), "2003-03-01T00:00:00Z") else: # TODO: ES and SOLR returns facets by default spliting the data yearly. first record is on 2000-03, # ES facets are returned from 2000-01... to 2003-01. # SOLR facets are returned from 2000-03... to 2003-03. # SOLR data seems more accurate since first and last Layers are in month 03. # Example: http://panchicore.d.pr/12ESP # * TO * to first date and last date. self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z") self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z") # test facets params["q_time"] = "[2000 TO 2022]" params["a_time_limit"] = 1 params["a_time_gap"] = "P1Y" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) # 2000 to complete datetime format self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z") # 2022 to complete datetime format if SEARCH_TYPE == SEARCH_TYPE_SOLR: # TODO: solr creates entire time span and brings facets with empty entries. (2000 to 2022) # fix by removing facets with zero counts?. self.assertEqual(results["a.time"]["end"].upper(), "2022-01-01T00:00:00Z") else: self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z") # the facet counters are all facets excluding < 2000 self.assertEqual(len(results["a.time"]["counts"]), Layer.objects.all().count()) def test_utilities(self): print('> testing utilities functions') # test_parse_datetime_range start, end = utils.parse_datetime_range("[2013-03-01 TO 2014-05-02T23:00:00]") self.assertTrue(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime").year, 2013) self.assertEqual(start.get("parsed_datetime").month, 3) self.assertEqual(start.get("parsed_datetime").day, 1) self.assertTrue(end.get("is_common_era")) self.assertEqual(end.get("parsed_datetime").year, 2014) self.assertEqual(end.get("parsed_datetime").month, 5) self.assertEqual(end.get("parsed_datetime").day, 2) self.assertEqual(end.get("parsed_datetime").hour, 23) self.assertEqual(end.get("parsed_datetime").minute, 0) self.assertEqual(end.get("parsed_datetime").second, 0) start, end = utils.parse_datetime_range("[-500000000 TO 2014-05-02T23:00:00]") self.assertFalse(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime"), "-500000000-01-01T00:00:00Z") start, end = utils.parse_datetime_range("[* TO *]") self.assertTrue(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime"), None) self.assertEqual(end.get("parsed_datetime"), None) # test_parse_ISO8601 quantity, units = utils.parse_ISO8601("P3D") self.assertEqual(quantity, 3) self.assertEqual(units[0], "DAYS") # test_gap_to_sorl value = utils.gap_to_sorl("P3D") self.assertEqual(value, "+3DAYS") # test_parse_geo_box value = utils.parse_geo_box("[-90,-180 TO 90,180]") self.assertEqual(value.bounds[0], -90) self.assertEqual(value.bounds[1], -180) self.assertEqual(value.bounds[2], 90) self.assertEqual(value.bounds[3], 180) # test_request_time_facet d = utils.request_time_facet("x", "[2000 TO 2014-01-02T11:12:13]", None, 1000) self.assertEqual(type(d), dict) self.assertEqual(d['f.x.facet.range.start'], '2000-01-01T00:00:00Z') self.assertEqual(d['f.x.facet.range.end'], '2014-01-02T11:12:13Z') self.assertEqual(d['f.x.facet.range.gap'], '+6DAYS') self.assertEqual(d['facet.range'], 'x') d = utils.request_time_facet("y", "[-5000000 TO 2016]", "P1D", 1) self.assertEqual(d['f.y.facet.range.start'], '-5000000-01-01T00:00:00Z') self.assertEqual(d['f.y.facet.range.end'], '2016-01-01T00:00:00Z') self.assertEqual(d['f.y.facet.range.gap'], '+1DAYS') self.assertEqual(d['facet.range'], 'y')
class TestBrowser(unittest.TestCase): def setUp(self): if not SELENIUM_HUB_URL: # run test on firefox of this machine. self.driver = webdriver.Firefox() else: # run test on stand alone node machine in docker: selenium-firefox self.driver = webdriver.Remote( command_executor=SELENIUM_HUB_URL, desired_capabilities=DesiredCapabilities.FIREFOX ) self.driver.implicitly_wait(30) self.base_url = BROWSER_HYPERMAP_URL self.verificationErrors = [] self.accept_next_alert = True print '> clearing SEARCH_URL={0}'.format(SEARCH_URL) if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print '> updating schema'.format(SEARCH_URL) self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) def test_browser(self): ENDPOINT_FILE = os.path.join("/usr/src/app", "hypermap", "tests", "mesonet.agron.iastate.edu.txt") print "" print ">>> with env:" print "REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY print "REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS print "REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD print "" print "SELENIUM_HUB_URL: %s" % SELENIUM_HUB_URL print "BROWSER_HYPERMAP_URL: %s" % BROWSER_HYPERMAP_URL print "BROWSER_SEARCH_URL: %s" % BROWSER_SEARCH_URL print "BROWSER_MAPLOOM_URL: %s" % BROWSER_MAPLOOM_URL print "WAIT_FOR_CELERY_JOB_PERIOD: %s" % WAIT_FOR_CELERY_JOB_PERIOD print "ENDPOINT FILE: %s" % ENDPOINT_FILE print "" print "Starting..." driver = self.driver time.sleep(3) driver.get(self.base_url + "/admin/login/?next=/admin/") print driver.current_url driver.find_element_by_id("id_password").clear() driver.find_element_by_id("id_password").send_keys("admin") driver.find_element_by_id("id_username").clear() driver.find_element_by_id("id_username").send_keys("admin") driver.find_element_by_css_selector("input[type=\"submit\"]").click() print driver.current_url driver.find_element_by_link_text("Periodic tasks").click() print driver.current_url print "> assert 3 periodic tasks. means beat is alive." self.assertEqual("3 periodic tasks", driver.find_element_by_css_selector( "p.paginator").text) driver.find_element_by_link_text("Home").click() print driver.current_url driver.find_element_by_link_text("Endpoint lists").click() print driver.current_url driver.find_element_by_link_text("Add endpoint list").click() print driver.current_url print "> uploading Endpoint List..." driver.find_element_by_id("id_upload").clear() driver.find_element_by_id("id_upload").send_keys(ENDPOINT_FILE) driver.find_element_by_name("_save").click() print driver.current_url print "> waiting {0} seconds for celery do the job....".format( WAIT_FOR_CELERY_JOB_PERIOD ) time.sleep(WAIT_FOR_CELERY_JOB_PERIOD) driver.find_element_by_link_text("Aggregator").click() time.sleep(1) print driver.current_url driver.find_element_by_link_text("Endpoints").click() print driver.current_url print "> assert Endpoint created." time.sleep(1) self.assertEqual( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi", driver.find_element_by_link_text( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").text) driver.find_element_by_link_text( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").click() # self.assertEqual("1 service/s created", driver.find_element_by_id("id_message").text) driver.find_element_by_link_text("Endpoints").click() print driver.current_url time.sleep(1) driver.find_element_by_link_text("Aggregator").click() print driver.current_url time.sleep(1) driver.find_element_by_link_text("Services").click() print driver.current_url print "> assert 1 Service created." time.sleep(1) self.assertEqual("1 service", driver.find_element_by_css_selector( "p.paginator").text) self.assertEqual( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi", driver.find_element_by_css_selector("td.field-url").text) driver.find_element_by_link_text("1").click() print driver.current_url print "> assert Service details." time.sleep(1) try: self.assertEqual("IEM NWS Warnings WMS Service", driver.find_element_by_id( "id_title").get_attribute("value")) except AssertionError as e: self.verificationErrors.append(str(e)) driver.find_element_by_link_text("Services").click() print driver.current_url driver.find_element_by_link_text("Aggregator").click() print driver.current_url driver.find_element_by_link_text("Layers").click() print driver.current_url print "> assert 3 layers created." time.sleep(1) self.assertEqual("3 layers", driver.find_element_by_css_selector( "p.paginator").text) driver.get(self.base_url + "/registry/") print driver.current_url print "> go to /registry/." for i in range(1, 11): print "> try assert checks count > 0. (%i of 10)" % i try: self.assertNotEqual("0", driver.find_element_by_xpath( "//td[4]").text) print "> found" break except AssertionError as e: print "> wait and reload page" time.sleep(10) driver.get(self.base_url + "/registry/") try: self.assertNotEqual("0", driver.find_element_by_xpath("//td[4]").text) except AssertionError as e: self.verificationErrors.append(str(e)) driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL)) print driver.current_url time.sleep(2) for i in range(1, 11): print "> assert layers indexed are 3. (%i of 10)" % i try: self.assertRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.refresh() self.assertRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") driver.get(self.base_url + "/registry/") print driver.current_url driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url print "> remove checks." driver.find_element_by_name("remove").click() print driver.current_url driver.find_element_by_link_text("Home").click() print driver.current_url print "> assert checks = 0." self.assertEqual("0", driver.find_element_by_xpath("//td[4]").text) driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url print "> trigger check." driver.find_element_by_name("check").click() print driver.current_url driver.find_element_by_link_text("Home").click() print driver.current_url for i in range(1, 11): try: print "> assert checks = 1. (%i of 10)" % i self.assertTrue( int(driver.find_element_by_xpath("//td[4]").text) > 0) print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.refresh() driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url driver.find_element_by_link_text("wwa").click() print driver.current_url print "> remove checks from Layer." driver.find_element_by_name("remove").click() print driver.current_url print "> assert text [No checks performed so far]." self.assertEqual("No checks performed so far", driver.find_element_by_xpath("//tr[11]/td[2]").text) print "> check Layer." driver.find_element_by_name("check").click() print driver.current_url for i in range(1, 11): try: print "> assert text [Total Checks: N>0]. (%i of 10)" % i src = driver.page_source text_found_TOTAL_CHECKS_LTE_1 = re.search( r'Total Checks: (1|2|3|4|5|6|7)', src) self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None) print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.get(driver.current_url) src = driver.page_source text_found_TOTAL_CHECKS_LTE_1 = re.search( r'Total Checks: (1|2|3|4|5|6|7)', src) self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None) driver.find_element_by_link_text("Home").click() print driver.current_url driver.find_element_by_link_text("Monitor").click() print driver.current_url print "> clean Search index and wait" driver.find_element_by_name("clear_index").click() print driver.current_url time.sleep(5) driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL)) print driver.current_url print "> assert count != 3 layers" try: self.assertNotRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") except AssertionError as e: self.verificationErrors.append(str(e)) driver.get(self.base_url + "/registry/") print driver.current_url print "> finish hypermap page" print "" # TODO: activate this to test maploom, now dat app looks very buggy. """ print ">> start maploom" driver.get(BROWSER_MAPLOOM_URL) print driver.current_url print ">> open registry modal" driver.find_element_by_xpath( "//div[@id='pulldown-content']/div[2]/div/div").click() print ">> assert Hypermap catalog" time.sleep(10) self.assertEqual("Hypermap", driver.find_element_by_xpath( "//div[@id='explore']/div/nav/div/form/div/div[2]/select").text) print ">> assert [Showing 3 of 3 - Page 1 / 1]" self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) driver.find_element_by_id("text_search_input_exp").clear() print ">> search IEM" driver.find_element_by_id("text_search_input_exp").send_keys("IEM") driver.find_element_by_id("text_search_btn").click() time.sleep(10) print ">> assert [Showing 1 of 1 - Page 1 / 1]" self.assertEqual("Showing 1 of 1 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) print ">> click reset" driver.find_element_by_name("button").click() time.sleep(10) print ">> assert [Showing 3 of 3 - Page 1 / 1]" self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) print ">> click on 3 layers to select" driver.find_element_by_css_selector("td.ellipsis.ng-binding").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click() print ">> click on 3 layers to unselect" driver.find_element_by_css_selector("td.ellipsis.ng-binding").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click() """ def is_element_present(self, how, what): try: self.driver.find_element(by=how, value=what) except NoSuchElementException as e: print e return False return True def is_alert_present(self): try: self.driver.switch_to_alert() except NoAlertPresentException as e: print e return False return True def close_alert_and_get_its_text(self): try: alert = self.driver.switch_to_alert() alert_text = alert.text if self.accept_next_alert: alert.accept() else: alert.dismiss() return alert_text finally: self.accept_next_alert = True def tearDown(self): self.driver.quit() self.assertEqual([], self.verificationErrors)
class TestBrowser(unittest.TestCase): def setUp(self): if not SELENIUM_HUB_URL: # run test on firefox of this machine. self.driver = webdriver.Firefox() else: # run test on stand alone node machine in docker: selenium-firefox self.driver = webdriver.Remote( command_executor=SELENIUM_HUB_URL, desired_capabilities=DesiredCapabilities.FIREFOX ) self.driver.implicitly_wait(30) self.base_url = BROWSER_HYPERMAP_URL self.verificationErrors = [] self.accept_next_alert = True print '> clearing SEARCH_URL={0}'.format(SEARCH_URL) if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print '> updating schema'.format(SEARCH_URL) self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) def test_browser(self): ENDPOINT_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "mesonet.agron.iastate.edu.txt") print "" print ">>> with env:" print "REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY print "REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS print "REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD print "" print "SELENIUM_HUB_URL: %s" % SELENIUM_HUB_URL print "BROWSER_HYPERMAP_URL: %s" % BROWSER_HYPERMAP_URL print "BROWSER_SEARCH_URL: %s" % BROWSER_SEARCH_URL print "BROWSER_MAPLOOM_URL: %s" % BROWSER_MAPLOOM_URL print "WAIT_FOR_CELERY_JOB_PERIOD: %s" % WAIT_FOR_CELERY_JOB_PERIOD print "ENDPOINT FILE: %s" % ENDPOINT_FILE print "" print "Starting..." driver = self.driver time.sleep(3) driver.get(self.base_url + "/admin/login/?next=/admin/") print driver.current_url driver.find_element_by_id("id_password").clear() driver.find_element_by_id("id_password").send_keys("admin") driver.find_element_by_id("id_username").clear() driver.find_element_by_id("id_username").send_keys("admin") driver.find_element_by_css_selector("input[type=\"submit\"]").click() print driver.current_url driver.find_element_by_link_text("Periodic tasks").click() print driver.current_url print "> assert 3 periodic tasks. means beat is alive." self.assertEqual("3 periodic tasks", driver.find_element_by_css_selector( "p.paginator").text) driver.find_element_by_link_text("Home").click() print driver.current_url driver.find_element_by_link_text("Endpoint lists").click() print driver.current_url driver.find_element_by_link_text("Add endpoint list").click() print driver.current_url print "> uploading Endpoint List..." driver.find_element_by_id("id_upload").clear() driver.find_element_by_id("id_upload").send_keys(ENDPOINT_FILE) driver.find_element_by_name("_save").click() print driver.current_url print "> waiting {0} seconds for celery do the job....".format( WAIT_FOR_CELERY_JOB_PERIOD ) time.sleep(WAIT_FOR_CELERY_JOB_PERIOD) driver.find_element_by_link_text("Aggregator").click() time.sleep(1) print driver.current_url driver.find_element_by_link_text("Endpoints").click() print driver.current_url print "> assert Endpoint created." time.sleep(1) self.assertEqual( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi", driver.find_element_by_link_text( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").text) driver.find_element_by_link_text( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").click() # self.assertEqual("1 service/s created", driver.find_element_by_id("id_message").text) driver.find_element_by_link_text("Endpoints").click() print driver.current_url time.sleep(1) driver.find_element_by_link_text("Aggregator").click() print driver.current_url time.sleep(1) driver.find_element_by_link_text("Services").click() print driver.current_url print "> assert 1 Service created." time.sleep(1) self.assertEqual("1 service", driver.find_element_by_css_selector( "p.paginator").text) self.assertEqual( "http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi", driver.find_element_by_css_selector("td.field-url").text) driver.find_element_by_xpath( '//*[@id="result_list"]/tbody/tr/th/a').click() print driver.current_url print "> assert Service details." time.sleep(1) self.assertEqual("IEM NWS Warnings WMS Service", driver.find_element_by_id( "id_title").get_attribute("value")) driver.find_element_by_link_text("Services").click() print driver.current_url driver.find_element_by_link_text("Aggregator").click() print driver.current_url driver.find_element_by_link_text("Layers").click() print driver.current_url print "> assert 3 layers created." time.sleep(1) self.assertEqual("3 layers", driver.find_element_by_css_selector( "p.paginator").text) driver.get(self.base_url + "/registry/") print driver.current_url print "> go to /registry/." for i in range(1, 11): print "> try assert checks count > 0. (%i of 10)" % i try: self.assertNotEqual("0", driver.find_element_by_xpath( "//td[4]").text) print "> found" break except AssertionError as e: print "> wait and reload page" time.sleep(10) driver.get(self.base_url + "/registry/") try: self.assertNotEqual("0", driver.find_element_by_xpath("//td[4]").text) except AssertionError as e: self.verificationErrors.append(str(e)) driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL)) print driver.current_url time.sleep(2) for i in range(1, 11): print "> assert layers indexed are 3. (%i of 10)" % i try: self.assertRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.refresh() self.assertRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") driver.get(self.base_url + "/registry/") print driver.current_url driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url print "> remove checks." driver.find_element_by_name("remove").click() print driver.current_url driver.find_element_by_link_text("Home").click() print driver.current_url print "> assert checks = 0." self.assertEqual("0", driver.find_element_by_xpath("//td[4]").text) driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url print "> trigger check." driver.find_element_by_name("check").click() print driver.current_url driver.find_element_by_link_text("Home").click() print driver.current_url for i in range(1, 11): try: print "> assert checks = 1. (%i of 10)" % i self.assertTrue( int(driver.find_element_by_xpath("//td[4]").text) > 0) print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.refresh() driver.find_element_by_link_text( "IEM NWS Warnings WMS Service").click() print driver.current_url driver.find_element_by_link_text("wwa").click() print driver.current_url print "> remove checks from Layer." driver.find_element_by_name("remove").click() print driver.current_url print "> assert text [No checks performed so far]." self.assertEqual("No checks performed so far", driver.find_element_by_xpath("//tr[11]/td[2]").text) print "> check Layer." driver.find_element_by_name("check").click() print driver.current_url for i in range(1, 11): try: print "> assert text [Total Checks: N>0]. (%i of 10)" % i src = driver.page_source text_found_TOTAL_CHECKS_LTE_1 = re.search( r'Total Checks: (1|2|3|4|5|6|7)', src) self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None) print "> found" break except AssertionError: print "> wait and reload page" time.sleep(10) driver.get(driver.current_url) src = driver.page_source text_found_TOTAL_CHECKS_LTE_1 = re.search( r'Total Checks: (1|2|3|4|5|6|7)', src) self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None) driver.find_element_by_link_text("Home").click() print driver.current_url driver.find_element_by_link_text("Monitor").click() print driver.current_url print "> clean Search index and wait" driver.find_element_by_name("clear_index").click() print driver.current_url time.sleep(5) driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL)) print driver.current_url print "> assert count != 3 layers" try: self.assertNotRegexpMatches( driver.find_element_by_css_selector("pre").text, "^\\{\"count\":3[\\s\\S]*$") except AssertionError as e: self.verificationErrors.append(str(e)) driver.get(self.base_url + "/registry/") print driver.current_url print "> finish hypermap page" print "" # TODO: activate this to test maploom, now dat app looks very buggy. """ print ">> start maploom" driver.get(BROWSER_MAPLOOM_URL) print driver.current_url print ">> open registry modal" driver.find_element_by_xpath( "//div[@id='pulldown-content']/div[2]/div/div").click() print ">> assert Hypermap catalog" time.sleep(10) self.assertEqual("Hypermap", driver.find_element_by_xpath( "//div[@id='explore']/div/nav/div/form/div/div[2]/select").text) print ">> assert [Showing 3 of 3 - Page 1 / 1]" self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) driver.find_element_by_id("text_search_input_exp").clear() print ">> search IEM" driver.find_element_by_id("text_search_input_exp").send_keys("IEM") driver.find_element_by_id("text_search_btn").click() time.sleep(10) print ">> assert [Showing 1 of 1 - Page 1 / 1]" self.assertEqual("Showing 1 of 1 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) print ">> click reset" driver.find_element_by_name("button").click() time.sleep(10) print ">> assert [Showing 3 of 3 - Page 1 / 1]" self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(), driver.find_element_by_css_selector( "span.text-muted.ng-binding").text.lower()) print ">> click on 3 layers to select" driver.find_element_by_css_selector("td.ellipsis.ng-binding").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click() print ">> click on 3 layers to unselect" driver.find_element_by_css_selector("td.ellipsis.ng-binding").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click() driver.find_element_by_xpath( "//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click() """ def is_element_present(self, how, what): try: self.driver.find_element(by=how, value=what) except NoSuchElementException as e: print e return False return True def is_alert_present(self): try: self.driver.switch_to_alert() except NoAlertPresentException as e: print e return False return True def close_alert_and_get_its_text(self): try: alert = self.driver.switch_to_alert() alert_text = alert.text if self.accept_next_alert: alert.accept() else: alert.dismiss() return alert_text finally: self.accept_next_alert = True def tearDown(self): self.driver.quit() self.assertEqual([], self.verificationErrors)
def clear_solr(): print 'Clearing the solr core and indexes' from hypermap.aggregator.solr import SolrHypermap solrobject = SolrHypermap() solrobject.clear_solr()
class TestCSWTransactions(unittest.TestCase): def setUp(self): self.client = Client() user = User.objects.create(username='******') user.set_password('admin') user.save() self.client.login(username="******", password="******") Catalog.objects.get_or_create(name=catalog_test_slug) Layer.objects.all().delete() Service.objects.all().delete() if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() es.clear_es() else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) def test_post(self): """ test CSV transactions. :return: """ print("") print(">>> with env:") print("REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY) print("REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS) print("REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD) print("REGISTRY_SEARCH_URL: %s" % settings.REGISTRY_SEARCH_URL) print("REGISTRY_HARVEST_SERVICES: %s" % settings.REGISTRY_HARVEST_SERVICES) print("") # Post the 10 Layers contained in this file: data/cswt_insert.xml path = os.path.join(settings.PROJECT_DIR, "..", "data", "cswt_insert.xml") payload = open(path, 'rb').read() content_type = "application/xml" url = "/registry/{0}/csw".format(catalog_test_slug) res = self.client.post(url, data=payload, content_type=content_type) self.assertEqual(res.status_code, 200) self.assertEqual(Layer.objects.all().count(), 10) # List the Layers posted above url = "/registry/{0}/csw?service=CSW&version=2.0.2&request=" \ "GetRecords&typenames=csw:Record&elementsetname=full&" \ "resulttype=results".format(catalog_test_slug) res = self.client.get(url) self.assertEqual(res.status_code, 200) self.assertEqual(res.content.count("Airports (OSM)"), 1) self.assertEqual(res.content.count("Manaus Roads (OSM May 2016)"), 2) # Search one Layer posted above url = "/registry/{0}/csw?mode=opensearch&service=CSW&version" \ "=2.0.2&request=GetRecords&elementsetname=full&typenames=" \ "csw:Record&resulttype=results" \ "&q=Airport".format(catalog_test_slug) res = self.client.get(url) self.assertEqual(res.status_code, 200) self.assertEqual(res.content.count("Airports (OSM)"), 1) # Flush layers in the cache. index_cached_layers() # Give celery some time. time.sleep(3) # are Layers in index? url = "{0}hypermap/_search".format(SEARCH_URL) res = requests.get(url) results_ok_in_search_backend = res.json() self.assertTrue("hits" in results_ok_in_search_backend) self.assertTrue("total" in results_ok_in_search_backend["hits"]) self.assertEqual(results_ok_in_search_backend["hits"]["total"], 10) def tearDown(self): pass
def clear_solr(): print 'Clearing the solr core and indexes' from hypermap.aggregator.solr import SolrHypermap solrobject = SolrHypermap() solrobject.clear_solr()
class SearchApiTestCase(TestCase): """ run me python manage.py test hypermap.search_api --settings=hypermap.settings.test --failfast """ def tearDown(self): signals.post_save.connect(layer_post_save, sender=Layer) signals.post_save.connect(service_post_save, sender=Service) def setUp(self): signals.post_save.disconnect(layer_post_save, sender=Layer) signals.post_save.disconnect(service_post_save, sender=Service) catalog_test_slug = "hypermap" if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() # delete solr documents # add the schema print '> updating schema'.format(SEARCH_URL) self.solr.update_schema(catalog=catalog_test_slug) print '> clearing SEARCH_URL={0}'.format(SEARCH_URL) self.solr.clear_solr(catalog=catalog_test_slug) self.search_engine_endpoint = '{0}/solr/{1}/select'.format( SEARCH_URL, catalog_test_slug ) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() # delete ES documents es.clear_es() self.search_engine_endpoint = '{0}/{1}/_search'.format( SEARCH_URL, catalog_test_slug ) else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) catalog, created = Catalog.objects.get_or_create( name=catalog_test_slug ) service = Service( url='http://fakeurl.com', title='Title', type='OGC:WMS', catalog=catalog ) service.save() layer = Layer( name='Layer 1', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 2', bbox_x0=-40.0, bbox_x1=-20.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 3', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=20.0, bbox_y1=40.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) layer = Layer( name='Layer 4', bbox_x0=20.0, bbox_x1=40.0, bbox_y0=-40.0, bbox_y1=-20.0, service=service, catalog=catalog ) layer.title = layer.name layer.save() layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0) layer.save() service.layer_set.add(layer) # solr have commitWithin 1500. # before to proceed with the tests wait for 2 secs. # otherwise it will return zero docs in the next test. service.index_layers(with_cache=False) time.sleep(2) self.api_url = "{0}{1}".format( settings.SITE_URL, reverse("search_api", args=[catalog_test_slug]) ) self.default_params = { "search_engine": SEARCH_TYPE, "search_engine_endpoint": self.search_engine_endpoint, "q_time": "[* TO *]", "q_geo": "[-90,-180 TO 90,180]", "d_docs_limit": 0, "d_docs_page": 1, "d_docs_sort": "score" } def test_catalogs(self): print '> testing catalogs' url = settings.SITE_URL + reverse("catalog-list") res = self.client.get(url) self.assertEqual(res.status_code, 200) catalogs = json.loads(res.content) self.assertEqual(len(catalogs), Catalog.objects.all().count()) def test_all_match_docs(self): print '> testing match all docs' params = self.default_params print "searching on [{}]".format(self.api_url) results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) def test_q_text(self): print '> testing q text' layer = Layer.objects.all()[0] params = self.default_params params["q_text"] = "title:\"{0}\"".format(layer.title) params["d_docs_limit"] = 100 results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) for doc in results.get("d.docs", []): self.assertEqual(doc["title"], layer.title) def test_q_geo(self): print '> testing q geo' params = self.default_params # top right square params["q_geo"] = "[0,0 TO 30,30]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) # bottom left square params["q_geo"] = "[-30,-30 TO 0,0]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 1) # big square params["q_geo"] = "[-30,-30 TO 30,30]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 4) # center where no layers params["q_geo"] = "[-5,-5 TO 5,5]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], 0) # bad format params["q_geo"] = "[-5,-5 5,5]" results = self.client.get(self.api_url, params) # validate the format print '> testing q geo (format validations)' self.assertEqual(results.status_code, 400) def test_q_time(self): print '> testing q time (format validations)' params = self.default_params # test validations params["q_time"] = "[2000-01-01 - 2001-01-01T00:00:00]" results = self.client.get(self.api_url, params) # requires [X TO Y] self.assertEqual(400, results.status_code) print '> testing q time' # test asterisks # all times params["q_time"] = "[* TO *]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) # all records self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) # test range # entire year 2000 params["q_time"] = "[2000-01-01 TO 2001-01-01T00:00:00]" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) # 1 in year 2000 self.assertEqual(results["a.matchDocs"], 1) # test complete min and max when q time is asterisks params["q_time"] = "[* TO *]" params["a_time_limit"] = 1 if SEARCH_TYPE == SEARCH_TYPE_ES: # TODO: a_time_limit is WIP in ES, today requires a a_time_gap to be completed. params["a_time_gap"] = "P1Y" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) if SEARCH_TYPE == SEARCH_TYPE_SOLR: # TODO: fix on Solr or ES? see next TODO. # * TO * to first date and last date. self.assertEqual(results["a.time"]["start"].upper(), "2000-03-01T00:00:00Z") self.assertEqual(results["a.time"]["end"].upper(), "2003-03-01T00:00:00Z") else: # TODO: ES and SOLR returns facets by default spliting the data yearly. first record is on 2000-03, # ES facets are returned from 2000-01... to 2003-01. # SOLR facets are returned from 2000-03... to 2003-03. # SOLR data seems more accurate since first and last Layers are in month 03. # Example: http://panchicore.d.pr/12ESP # * TO * to first date and last date. self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z") self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z") # test facets params["q_time"] = "[2000 TO 2022]" params["a_time_limit"] = 1 params["a_time_gap"] = "P1Y" results = self.client.get(self.api_url, params) self.assertEqual(results.status_code, 200) results = json.loads(results.content) self.assertEqual(results["a.matchDocs"], Layer.objects.all().count()) # 2000 to complete datetime format self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z") # 2022 to complete datetime format if SEARCH_TYPE == SEARCH_TYPE_SOLR: # TODO: solr creates entire time span and brings facets with empty entries. (2000 to 2022) # fix by removing facets with zero counts?. self.assertEqual(results["a.time"]["end"].upper(), "2022-01-01T00:00:00Z") else: self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z") # the facet counters are all facets excluding < 2000 self.assertEqual(len(results["a.time"]["counts"]), Layer.objects.all().count()) def test_utilities(self): print '> testing utilities functions' # test_parse_datetime_range start, end = utils.parse_datetime_range("[2013-03-01 TO 2014-05-02T23:00:00]") self.assertTrue(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime").year, 2013) self.assertEqual(start.get("parsed_datetime").month, 3) self.assertEqual(start.get("parsed_datetime").day, 1) self.assertTrue(end.get("is_common_era")) self.assertEqual(end.get("parsed_datetime").year, 2014) self.assertEqual(end.get("parsed_datetime").month, 5) self.assertEqual(end.get("parsed_datetime").day, 2) self.assertEqual(end.get("parsed_datetime").hour, 23) self.assertEqual(end.get("parsed_datetime").minute, 0) self.assertEqual(end.get("parsed_datetime").second, 0) start, end = utils.parse_datetime_range("[-500000000 TO 2014-05-02T23:00:00]") self.assertFalse(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime"), "-500000000-01-01T00:00:00Z") start, end = utils.parse_datetime_range("[* TO *]") self.assertTrue(start.get("is_common_era")) self.assertEqual(start.get("parsed_datetime"), None) self.assertEqual(end.get("parsed_datetime"), None) # test_parse_ISO8601 quantity, units = utils.parse_ISO8601("P3D") self.assertEqual(quantity, 3) self.assertEqual(units[0], "DAYS") # test_gap_to_sorl value = utils.gap_to_sorl("P3D") self.assertEqual(value, "+3DAYS") # test_parse_geo_box value = utils.parse_geo_box("[-90,-180 TO 90,180]") self.assertEqual(value.bounds[0], -90) self.assertEqual(value.bounds[1], -180) self.assertEqual(value.bounds[2], 90) self.assertEqual(value.bounds[3], 180) # test_request_time_facet d = utils.request_time_facet("x", "[2000 TO 2014-01-02T11:12:13]", None, 1000) self.assertEqual(type(d), dict) self.assertEqual(d['f.x.facet.range.start'], '2000-01-01T00:00:00Z') self.assertEqual(d['f.x.facet.range.end'], '2014-01-02T11:12:13Z') self.assertEqual(d['f.x.facet.range.gap'], '+6DAYS') self.assertEqual(d['facet.range'], 'x') d = utils.request_time_facet("y", "[-5000000 TO 2016]", "P1D", 1) self.assertEqual(d['f.y.facet.range.start'], '-5000000-01-01T00:00:00Z') self.assertEqual(d['f.y.facet.range.end'], '2016-01-01T00:00:00Z') self.assertEqual(d['f.y.facet.range.gap'], '+1DAYS') self.assertEqual(d['facet.range'], 'y')
class TestCSWTransactions(unittest.TestCase): def setUp(self): self.client = Client() user = User.objects.create(username='******') user.set_password('admin') user.save() self.client.login(username="******", password="******") Catalog.objects.get_or_create( name=catalog_test_slug ) Layer.objects.all().delete() Service.objects.all().delete() if SEARCH_TYPE == SEARCH_TYPE_SOLR: self.solr = SolrHypermap() self.solr.update_schema(catalog=catalog_test_slug) self.solr.clear_solr(catalog=catalog_test_slug) elif SEARCH_TYPE == SEARCH_TYPE_ES: es = ESHypermap() es.clear_es() else: raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE) def test_post(self): """ test CSV transactions. :return: """ print "" print ">>> with env:" print "REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY print "REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS print "REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD print "REGISTRY_SEARCH_URL: %s" % settings.REGISTRY_SEARCH_URL print "REGISTRY_HARVEST_SERVICES: %s" % settings.REGISTRY_HARVEST_SERVICES print "" # Post the 10 Layers contained in this file: data/cswt_insert.xml path = os.path.join(settings.PROJECT_DIR, "..", "data", "cswt_insert.xml") payload = open(path, 'rb').read() content_type = "application/xml" url = "/registry/{0}/csw".format(catalog_test_slug) res = self.client.post(url, data=payload, content_type=content_type) self.assertEqual(res.status_code, 200) self.assertEqual(Layer.objects.all().count(), 10) # List the Layers posted above url = "/registry/{0}/csw?service=CSW&version=2.0.2&request=" \ "GetRecords&typenames=csw:Record&elementsetname=full&" \ "resulttype=results".format(catalog_test_slug) res = self.client.get(url) self.assertEqual(res.status_code, 200) self.assertEqual(res.content.count("Airports (OSM)"), 1) self.assertEqual(res.content.count("Manaus Roads (OSM May 2016)"), 2) # Search one Layer posted above url = "/registry/{0}/csw?mode=opensearch&service=CSW&version" \ "=2.0.2&request=GetRecords&elementsetname=full&typenames=" \ "csw:Record&resulttype=results" \ "&q=Airport".format(catalog_test_slug) res = self.client.get(url) self.assertEqual(res.status_code, 200) self.assertEqual(res.content.count("Airports (OSM)"), 1) # Flush layers in the cache. index_cached_layers() # Give celery some time. time.sleep(3) # are Layers in index? url = "{0}hypermap/_search".format( SEARCH_URL ) res = requests.get(url) results_ok_in_search_backend = res.json() self.assertTrue("hits" in results_ok_in_search_backend) self.assertTrue("total" in results_ok_in_search_backend["hits"]) self.assertEqual(results_ok_in_search_backend["hits"]["total"], 10) def tearDown(self): pass