def index_layer(self, layer):
# TODO: Make this function more DRY
# by abstracting the common bits.
if settings.SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
print 'Syncing layer %s to solr' % layer.name
try:
solrobject = SolrHypermap()
success, message = solrobject.layer_to_solr(layer)
if not success:
from hypermap.aggregator.models import TaskError
task_error = TaskError(task_name=self.name,
args=layer.id,
message=message)
task_error.save()
except:
print 'There was an exception here!'
self.retry(layer)
elif settings.SEARCH_TYPE == 'elasticsearch':
from hypermap.aggregator.elasticsearch_client import ESHypermap
print 'Syncing layer %s to es' % layer.name
esobject = ESHypermap()
success, message = esobject.layer_to_es(layer)
if not success:
from hypermap.aggregator.models import TaskError
task_error = TaskError(task_name=self.name,
args=layer.id,
message=message)
task_error.save()
def index_layer(self, layer_id, use_cache=False):
"""
Index a layer in the search backend.
If cache is set, append it to the list, if it isn't send the transaction right away.
cache needs memcached to be available.
"""
from hypermap.aggregator.models import Layer
layer = Layer.objects.get(id=layer_id)
if not layer.is_valid:
LOGGER.debug('Not indexing or removing layer with id %s in search engine as it is not valid' % layer.id)
unindex_layer(layer.id, use_cache)
return
if layer.was_deleted:
LOGGER.debug('Not indexing or removing layer with id %s in search engine as was_deleted is true' % layer.id)
unindex_layer(layer.id, use_cache)
return
# 1. if we use cache
if use_cache:
LOGGER.debug('Caching layer with id %s for syncing with search engine' % layer.id)
layers = cache.get('layers')
if layers is None:
layers = set([layer.id])
else:
layers.add(layer.id)
cache.set('layers', layers)
return
# 2. if we don't use cache
# TODO: Make this function more DRY
# by abstracting the common bits.
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
LOGGER.debug('Syncing layer %s to solr' % layer.name)
solrobject = SolrHypermap()
success, message = solrobject.layer_to_solr(layer)
# update the error message if using celery
if not settings.REGISTRY_SKIP_CELERY:
if not success:
self.update_state(
state=states.FAILURE,
meta=message
)
raise Ignore()
elif SEARCH_TYPE == 'elasticsearch':
from hypermap.aggregator.elasticsearch_client import ESHypermap
LOGGER.debug('Syncing layer %s to es' % layer.name)
esobject = ESHypermap()
success, message = esobject.layer_to_es(layer)
# update the error message if using celery
if not settings.REGISTRY_SKIP_CELERY:
if not success:
self.update_state(
state=states.FAILURE,
meta=message
)
raise Ignore()
def unindex_layer(self, layer_id, use_cache=False):
"""
Remove the index for a layer in the search backend.
If cache is set, append it to the list of removed layers, if it isn't send the transaction right away.
"""
from hypermap.aggregator.models import Layer
layer = Layer.objects.get(id=layer_id)
if use_cache:
LOGGER.debug('Caching layer with id %s for being removed from search engine' % layer.id)
deleted_layers = cache.get('deleted_layers')
if deleted_layers is None:
deleted_layers = set([layer.id])
else:
deleted_layers.add(layer.id)
cache.set('deleted_layers', deleted_layers)
return
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
LOGGER.debug('Removing layer %s from solr' % layer.id)
try:
solrobject = SolrHypermap()
solrobject.remove_layer(layer.uuid)
except Exception:
LOGGER.error('Layer NOT correctly removed from Solr')
elif SEARCH_TYPE == 'elasticsearch':
# TODO implement me
pass
def index_layer(self, layer):
# TODO: Make this function more DRY
# by abstracting the common bits.
if settings.SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
print 'Syncing layer %s to solr' % layer.name
try:
solrobject = SolrHypermap()
success, message = solrobject.layer_to_solr(layer)
if not success:
from hypermap.aggregator.models import TaskError
task_error = TaskError(
task_name=self.name,
args=layer.id,
message=message
)
task_error.save()
except:
print 'There was an exception here!'
self.retry(layer)
elif settings.SEARCH_TYPE == 'elasticsearch':
from hypermap.aggregator.elasticsearch_client import ESHypermap
print 'Syncing layer %s to es' % layer.name
esobject = ESHypermap()
success, message = esobject.layer_to_es(layer)
if not success:
from hypermap.aggregator.models import TaskError
task_error = TaskError(
task_name=self.name,
args=layer.id,
message=message
)
task_error.save()
def index_layer(self, layer_id, use_cache=False):
"""
Index a layer in the search backend.
If cache is set, append it to the list, if it isn't send the transaction right away.
cache needs memcached to be available.
"""
from hypermap.aggregator.models import Layer
layer = Layer.objects.get(id=layer_id)
if not layer.is_valid:
LOGGER.debug('Not indexing or removing layer with id %s in search engine as it is not valid' % layer.id)
unindex_layer(layer.id, use_cache)
return
if layer.was_deleted:
LOGGER.debug('Not indexing or removing layer with id %s in search engine as was_deleted is true' % layer.id)
unindex_layer(layer.id, use_cache)
return
# 1. if we use cache
if use_cache:
LOGGER.debug('Caching layer with id %s for syncing with search engine' % layer.id)
layers = cache.get('layers')
if layers is None:
layers = set([layer.id])
else:
layers.add(layer.id)
cache.set('layers', layers)
return
# 2. if we don't use cache
# TODO: Make this function more DRY
# by abstracting the common bits.
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
LOGGER.debug('Syncing layer %s to solr' % layer.name)
solrobject = SolrHypermap()
success, message = solrobject.layer_to_solr(layer)
# update the error message if using celery
if not settings.REGISTRY_SKIP_CELERY:
if not success:
self.update_state(
state=states.FAILURE,
meta=message
)
raise Ignore()
elif SEARCH_TYPE == 'elasticsearch':
from hypermap.aggregator.elasticsearch_client import ESHypermap
LOGGER.debug('Syncing layer %s to es' % layer.name)
esobject = ESHypermap()
success, message = esobject.layer_to_es(layer)
# update the error message if using celery
if not settings.REGISTRY_SKIP_CELERY:
if not success:
self.update_state(
state=states.FAILURE,
meta=message
)
raise Ignore()
def unindex_layer(self, layer_id, use_cache=False):
"""
Remove the index for a layer in the search backend.
If cache is set, append it to the list of removed layers, if it isn't send the transaction right away.
"""
from hypermap.aggregator.models import Layer
layer = Layer.objects.get(id=layer_id)
if use_cache:
LOGGER.debug('Caching layer with id %s for being removed from search engine' % layer.id)
deleted_layers = cache.get('deleted_layers')
if deleted_layers is None:
deleted_layers = set([layer.id])
else:
deleted_layers.add(layer.id)
cache.set('deleted_layers', deleted_layers)
return
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
LOGGER.debug('Removing layer %s from solr' % layer.id)
try:
solrobject = SolrHypermap()
solrobject.remove_layer(layer.uuid)
except Exception:
LOGGER.error('Layer NOT correctly removed from Solr')
elif SEARCH_TYPE == 'elasticsearch':
# TODO implement me
pass
def setUp(self):
if not SELENIUM_HUB_URL:
# run test on firefox of this machine.
self.driver = webdriver.Firefox()
else:
# run test on stand alone node machine in docker: selenium-firefox
self.driver = webdriver.Remote(
command_executor=SELENIUM_HUB_URL,
desired_capabilities=DesiredCapabilities.FIREFOX)
self.driver.implicitly_wait(30)
self.base_url = BROWSER_HYPERMAP_URL
self.verificationErrors = []
self.accept_next_alert = True
print('> clearing SEARCH_URL={0}'.format(SEARCH_URL))
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
# delete solr documents
# add the schema
print('> updating schema'.format(SEARCH_URL))
self.solr.update_schema(catalog=catalog_test_slug)
self.solr.clear_solr(catalog=catalog_test_slug)
self.search_engine_endpoint = '{0}/solr/{1}/select'.format(
SEARCH_URL, catalog_test_slug)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
# delete ES documents
es.clear_es()
self.search_engine_endpoint = '{0}/{1}/_search'.format(
SEARCH_URL, catalog_test_slug)
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
def handle(self, *args, **options):
"""
reset core:
rm -Rf SOLR_HOME/server/solr/hypermap_test
solr6 create_core -c hypermap_test
"""
client = SolrHypermap()
client.update_schema()
def handle(self, *args, **options):
"""
reset core:
rm -Rf SOLR_HOME/server/solr/hypermap_test
solr6 create_core -c hypermap_test
"""
client = SolrHypermap()
client.update_schema()
def clear_index():
if SEARCH_TYPE == 'solr':
LOGGER.debug('Clearing the solr indexes')
from hypermap.aggregator.solr import SolrHypermap
solrobject = SolrHypermap()
solrobject.clear_solr()
elif SEARCH_TYPE == 'elasticsearch':
LOGGER.debug('Clearing the ES indexes')
from hypermap.aggregator.elasticsearch_client import ESHypermap
esobject = ESHypermap()
esobject.clear_es()
def clear_index():
if SEARCH_TYPE == 'solr':
LOGGER.debug('Clearing the solr indexes')
from hypermap.aggregator.solr import SolrHypermap
solrobject = SolrHypermap()
solrobject.clear_solr()
elif SEARCH_TYPE == 'elasticsearch':
LOGGER.debug('Clearing the ES indexes')
from hypermap.aggregator.elasticsearch_client import ESHypermap
esobject = ESHypermap()
esobject.clear_es()
def setUp(self):
self.client = Client()
user = User.objects.create(username='******')
user.set_password('admin')
user.save()
self.client.login(username="******", password="******")
Catalog.objects.get_or_create(name=catalog_test_slug)
Layer.objects.all().delete()
Service.objects.all().delete()
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
self.solr.update_schema(catalog=catalog_test_slug)
self.solr.clear_solr(catalog=catalog_test_slug)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
es.clear_es()
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
def index_layer(self, layer):
# TODO: Make this function more DRY
# by abstracting the common bits.
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
LOGGER.debug('Syncing layer %s to solr' % layer.name)
try:
solrobject = SolrHypermap()
success, message = solrobject.layer_to_solr(layer)
if not success:
from hypermap.aggregator.models import TaskError
task_error = TaskError(
task_name=self.name,
args=layer.id,
message=message
)
task_error.save()
except Exception, e:
LOGGER.error('Layers NOT indexed correctly')
LOGGER.error(e, exc_info=True)
self.retry(layer)
def index_layer(self, layer, use_cache=False):
"""Index a layer in the search backend.
If cache is set, append it to the list, if it isn't send the transaction right away.
cache needs memcached to be available.
"""
if use_cache:
LOGGER.debug('Caching layer with id %s for syncing with search engine' % layer.id)
layers = cache.get('layers')
if layers is None:
layers = set([layer.id])
else:
layers.add(layer.id)
cache.set('layers', layers)
return
# TODO: Make this function more DRY
# by abstracting the common bits.
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
LOGGER.debug('Syncing layer %s to solr' % layer.name)
try:
solrobject = SolrHypermap()
success, message = solrobject.layer_to_solr(layer)
if not success:
from hypermap.aggregator.models import TaskError
task_error = TaskError(
task_name=self.name,
args=layer.id,
message=message
)
task_error.save()
except Exception, e:
LOGGER.error('Layers NOT indexed correctly')
LOGGER.error(e, exc_info=True)
self.retry(layer)
def setUp(self):
self.client = Client()
user = User.objects.create(username='******')
user.set_password('admin')
user.save()
self.client.login(username="******", password="******")
Catalog.objects.get_or_create(
name=catalog_test_slug
)
Layer.objects.all().delete()
Service.objects.all().delete()
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
self.solr.update_schema(catalog=catalog_test_slug)
self.solr.clear_solr(catalog=catalog_test_slug)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
es.clear_es()
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
def setUp(self):
if not SELENIUM_HUB_URL:
# run test on firefox of this machine.
self.driver = webdriver.Firefox()
else:
# run test on stand alone node machine in docker: selenium-firefox
self.driver = webdriver.Remote(
command_executor=SELENIUM_HUB_URL,
desired_capabilities=DesiredCapabilities.FIREFOX
)
self.driver.implicitly_wait(30)
self.base_url = BROWSER_HYPERMAP_URL
self.verificationErrors = []
self.accept_next_alert = True
print '> clearing SEARCH_URL={0}'.format(SEARCH_URL)
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
# delete solr documents
# add the schema
print '> updating schema'.format(SEARCH_URL)
self.solr.update_schema(catalog=catalog_test_slug)
self.solr.clear_solr(catalog=catalog_test_slug)
self.search_engine_endpoint = '{0}/solr/{1}/select'.format(
SEARCH_URL, catalog_test_slug
)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
# delete ES documents
es.clear_es()
self.search_engine_endpoint = '{0}/{1}/_search'.format(
SEARCH_URL, catalog_test_slug
)
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
class SearchApiTestCase(TestCase):
"""
run me
python manage.py test hypermap.search_api --settings=hypermap.settings.test --failfast
"""
def tearDown(self):
signals.post_save.connect(layer_post_save, sender=Layer)
signals.post_save.connect(service_post_save, sender=Service)
def setUp(self):
signals.post_save.disconnect(layer_post_save, sender=Layer)
signals.post_save.disconnect(service_post_save, sender=Service)
catalog_test_slug = "hypermap"
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
# delete solr documents
# add the schema
print '> updating schema'.format(SEARCH_URL)
self.solr.update_schema(catalog=catalog_test_slug)
print '> clearing SEARCH_URL={0}'.format(SEARCH_URL)
self.solr.clear_solr(catalog=catalog_test_slug)
self.search_engine_endpoint = '{0}/solr/{1}/select'.format(
SEARCH_URL, catalog_test_slug
)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
# delete ES documents
es.clear_es()
self.search_engine_endpoint = '{0}/{1}/_search'.format(
SEARCH_URL, catalog_test_slug
)
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
catalog, created = Catalog.objects.get_or_create(
name=catalog_test_slug
)
service = Service(
url='http://fakeurl.com',
title='Title',
type='OGC:WMS',
catalog=catalog
)
service.save()
layer = Layer(
name='Layer 1',
bbox_x0=-40.0,
bbox_x1=-20.0,
bbox_y0=-40.0,
bbox_y1=-20.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 2',
bbox_x0=-40.0,
bbox_x1=-20.0,
bbox_y0=20.0,
bbox_y1=40.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 3',
bbox_x0=20.0,
bbox_x1=40.0,
bbox_y0=20.0,
bbox_y1=40.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 4',
bbox_x0=20.0,
bbox_x1=40.0,
bbox_y0=-40.0,
bbox_y1=-20.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
# solr have commitWithin 1500.
# before to proceed with the tests wait for 2 secs.
# otherwise it will return zero docs in the next test.
service.index_layers(with_cache=False)
time.sleep(2)
self.api_url = "{0}{1}".format(
settings.SITE_URL, reverse("search_api", args=[catalog_test_slug])
)
self.default_params = {
"search_engine": SEARCH_TYPE,
"search_engine_endpoint": self.search_engine_endpoint,
"q_time": "[* TO *]",
"q_geo": "[-90,-180 TO 90,180]",
"d_docs_limit": 0,
"d_docs_page": 1,
"d_docs_sort": "score"
}
def test_catalogs(self):
print '> testing catalogs'
url = settings.SITE_URL + reverse("catalog-list")
res = self.client.get(url)
self.assertEqual(res.status_code, 200)
catalogs = json.loads(res.content)
self.assertEqual(len(catalogs), Catalog.objects.all().count())
def test_all_match_docs(self):
print '> testing match all docs'
params = self.default_params
print "searching on [{}]".format(self.api_url)
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], Layer.objects.all().count())
def test_q_text(self):
print '> testing q text'
layer = Layer.objects.all()[0]
params = self.default_params
params["q_text"] = "title:\"{0}\"".format(layer.title)
params["d_docs_limit"] = 100
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], 1)
for doc in results.get("d.docs", []):
self.assertEqual(doc["title"], layer.title)
def test_q_geo(self):
print '> testing q geo'
params = self.default_params
# top right square
params["q_geo"] = "[0,0 TO 30,30]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], 1)
# bottom left square
params["q_geo"] = "[-30,-30 TO 0,0]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], 1)
# big square
params["q_geo"] = "[-30,-30 TO 30,30]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], 4)
# center where no layers
params["q_geo"] = "[-5,-5 TO 5,5]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], 0)
# bad format
params["q_geo"] = "[-5,-5 5,5]"
results = self.client.get(self.api_url, params)
# validate the format
print '> testing q geo (format validations)'
self.assertEqual(results.status_code, 400)
def test_q_time(self):
print '> testing q time (format validations)'
params = self.default_params
# test validations
params["q_time"] = "[2000-01-01 - 2001-01-01T00:00:00]"
results = self.client.get(self.api_url, params)
# requires [X TO Y]
self.assertEqual(400, results.status_code)
print '> testing q time'
# test asterisks
# all times
params["q_time"] = "[* TO *]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
# all records
self.assertEqual(results["a.matchDocs"], Layer.objects.all().count())
# test range
# entire year 2000
params["q_time"] = "[2000-01-01 TO 2001-01-01T00:00:00]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
# 1 in year 2000
self.assertEqual(results["a.matchDocs"], 1)
# test complete min and max when q time is asterisks
params["q_time"] = "[* TO *]"
params["a_time_limit"] = 1
if SEARCH_TYPE == SEARCH_TYPE_ES:
# TODO: a_time_limit is WIP in ES, today requires a a_time_gap to be completed.
params["a_time_gap"] = "P1Y"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], Layer.objects.all().count())
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
# TODO: fix on Solr or ES? see next TODO.
# * TO * to first date and last date.
self.assertEqual(results["a.time"]["start"].upper(), "2000-03-01T00:00:00Z")
self.assertEqual(results["a.time"]["end"].upper(), "2003-03-01T00:00:00Z")
else:
# TODO: ES and SOLR returns facets by default spliting the data yearly. first record is on 2000-03,
# ES facets are returned from 2000-01... to 2003-01.
# SOLR facets are returned from 2000-03... to 2003-03.
# SOLR data seems more accurate since first and last Layers are in month 03.
# Example: http://panchicore.d.pr/12ESP
# * TO * to first date and last date.
self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z")
self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z")
# test facets
params["q_time"] = "[2000 TO 2022]"
params["a_time_limit"] = 1
params["a_time_gap"] = "P1Y"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], Layer.objects.all().count())
# 2000 to complete datetime format
self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z")
# 2022 to complete datetime format
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
# TODO: solr creates entire time span and brings facets with empty entries. (2000 to 2022)
# fix by removing facets with zero counts?.
self.assertEqual(results["a.time"]["end"].upper(), "2022-01-01T00:00:00Z")
else:
self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z")
# the facet counters are all facets excluding < 2000
self.assertEqual(len(results["a.time"]["counts"]), Layer.objects.all().count())
def test_utilities(self):
print '> testing utilities functions'
# test_parse_datetime_range
start, end = utils.parse_datetime_range("[2013-03-01 TO 2014-05-02T23:00:00]")
self.assertTrue(start.get("is_common_era"))
self.assertEqual(start.get("parsed_datetime").year, 2013)
self.assertEqual(start.get("parsed_datetime").month, 3)
self.assertEqual(start.get("parsed_datetime").day, 1)
self.assertTrue(end.get("is_common_era"))
self.assertEqual(end.get("parsed_datetime").year, 2014)
self.assertEqual(end.get("parsed_datetime").month, 5)
self.assertEqual(end.get("parsed_datetime").day, 2)
self.assertEqual(end.get("parsed_datetime").hour, 23)
self.assertEqual(end.get("parsed_datetime").minute, 0)
self.assertEqual(end.get("parsed_datetime").second, 0)
start, end = utils.parse_datetime_range("[-500000000 TO 2014-05-02T23:00:00]")
self.assertFalse(start.get("is_common_era"))
self.assertEqual(start.get("parsed_datetime"), "-500000000-01-01T00:00:00Z")
start, end = utils.parse_datetime_range("[* TO *]")
self.assertTrue(start.get("is_common_era"))
self.assertEqual(start.get("parsed_datetime"), None)
self.assertEqual(end.get("parsed_datetime"), None)
# test_parse_ISO8601
quantity, units = utils.parse_ISO8601("P3D")
self.assertEqual(quantity, 3)
self.assertEqual(units[0], "DAYS")
# test_gap_to_sorl
value = utils.gap_to_sorl("P3D")
self.assertEqual(value, "+3DAYS")
# test_parse_geo_box
value = utils.parse_geo_box("[-90,-180 TO 90,180]")
self.assertEqual(value.bounds[0], -90)
self.assertEqual(value.bounds[1], -180)
self.assertEqual(value.bounds[2], 90)
self.assertEqual(value.bounds[3], 180)
# test_request_time_facet
d = utils.request_time_facet("x", "[2000 TO 2014-01-02T11:12:13]", None, 1000)
self.assertEqual(type(d), dict)
self.assertEqual(d['f.x.facet.range.start'], '2000-01-01T00:00:00Z')
self.assertEqual(d['f.x.facet.range.end'], '2014-01-02T11:12:13Z')
self.assertEqual(d['f.x.facet.range.gap'], '+6DAYS')
self.assertEqual(d['facet.range'], 'x')
d = utils.request_time_facet("y", "[-5000000 TO 2016]", "P1D", 1)
self.assertEqual(d['f.y.facet.range.start'], '-5000000-01-01T00:00:00Z')
self.assertEqual(d['f.y.facet.range.end'], '2016-01-01T00:00:00Z')
self.assertEqual(d['f.y.facet.range.gap'], '+1DAYS')
self.assertEqual(d['facet.range'], 'y')
def index_cached_layers(self):
"""
Index and unindex all layers in the Django cache (Index all layers who have been checked).
"""
from hypermap.aggregator.models import Layer
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
solrobject = SolrHypermap()
else:
from hypermap.aggregator.elasticsearch_client import ESHypermap
from elasticsearch import helpers
es_client = ESHypermap()
layers_cache = cache.get('layers')
deleted_layers_cache = cache.get('deleted_layers')
# 1. added layers cache
if layers_cache:
layers_list = list(layers_cache)
LOGGER.debug('There are %s layers in cache: %s' % (len(layers_list), layers_list))
batch_size = settings.REGISTRY_SEARCH_BATCH_SIZE
batch_lists = [layers_list[i:i+batch_size] for i in range(0, len(layers_list), batch_size)]
for batch_list_ids in batch_lists:
layers = Layer.objects.filter(id__in=batch_list_ids)
if batch_size > len(layers):
batch_size = len(layers)
LOGGER.debug('Syncing %s/%s layers to %s: %s' % (batch_size, len(layers_cache), layers, SEARCH_TYPE))
try:
# SOLR
if SEARCH_TYPE == 'solr':
success, layers_errors_ids = solrobject.layers_to_solr(layers)
if success:
# remove layers from cache here
layers_cache = layers_cache.difference(set(batch_list_ids))
LOGGER.debug('Removing layers with id %s from cache' % batch_list_ids)
cache.set('layers', layers_cache)
# ES
elif SEARCH_TYPE == 'elasticsearch':
with_bulk, success = True, False
layers_to_index = [es_client.layer_to_es(layer, with_bulk) for layer in layers]
message = helpers.bulk(es_client.es, layers_to_index)
# Check that all layers where indexed...if not, don't clear cache.
# TODO: Check why es does not index all layers at first.
len_indexed_layers = message[0]
if len_indexed_layers == len(layers):
LOGGER.debug('%d layers indexed successfully' % (len_indexed_layers))
success = True
if success:
# remove layers from cache here
layers_cache = layers_cache.difference(set(batch_list_ids))
cache.set('layers', layers_cache)
else:
raise Exception("Incorrect SEARCH_TYPE=%s" % SEARCH_TYPE)
except Exception as e:
LOGGER.error('Layers were NOT indexed correctly')
LOGGER.error(e, exc_info=True)
else:
LOGGER.debug('No cached layers to add in search engine.')
# 2. deleted layers cache
if deleted_layers_cache:
layers_list = list(deleted_layers_cache)
LOGGER.debug('There are %s layers in cache for deleting: %s' % (len(layers_list), layers_list))
# TODO implement me: batch layer index deletion
for layer_id in layers_list:
# SOLR
if SEARCH_TYPE == 'solr':
if Layer.objects.filter(pk=layer_id).exists():
layer = Layer.objects.get(id=layer_id)
unindex_layer(layer.id, use_cache=False)
deleted_layers_cache = deleted_layers_cache.difference(set([layer_id]))
cache.set('deleted_layers', deleted_layers_cache)
else:
# TODO implement me
raise NotImplementedError
else:
LOGGER.debug('No cached layers to remove in search engine.')
def clear_solr():
print 'Clearing the solr core and indexes'
from hypermap.aggregator.solr import SolrHypermap
solrobject = SolrHypermap()
solrobject.clear_solr()
def setUp(self):
signals.post_save.disconnect(layer_post_save, sender=Layer)
signals.post_save.disconnect(service_post_save, sender=Service)
catalog_test_slug = "hypermap"
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
# delete solr documents
# add the schema
print '> updating schema'.format(SEARCH_URL)
self.solr.update_schema(catalog=catalog_test_slug)
print '> clearing SEARCH_URL={0}'.format(SEARCH_URL)
self.solr.clear_solr(catalog=catalog_test_slug)
self.search_engine_endpoint = '{0}/solr/{1}/select'.format(
SEARCH_URL, catalog_test_slug
)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
# delete ES documents
es.clear_es()
self.search_engine_endpoint = '{0}/{1}/_search'.format(
SEARCH_URL, catalog_test_slug
)
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
catalog, created = Catalog.objects.get_or_create(
name=catalog_test_slug
)
service = Service(
url='http://fakeurl.com',
title='Title',
type='OGC:WMS',
catalog=catalog
)
service.save()
layer = Layer(
name='Layer 1',
bbox_x0=-40.0,
bbox_x1=-20.0,
bbox_y0=-40.0,
bbox_y1=-20.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 2',
bbox_x0=-40.0,
bbox_x1=-20.0,
bbox_y0=20.0,
bbox_y1=40.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 3',
bbox_x0=20.0,
bbox_x1=40.0,
bbox_y0=20.0,
bbox_y1=40.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 4',
bbox_x0=20.0,
bbox_x1=40.0,
bbox_y0=-40.0,
bbox_y1=-20.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
# solr have commitWithin 1500.
# before to proceed with the tests wait for 2 secs.
# otherwise it will return zero docs in the next test.
service.index_layers(with_cache=False)
time.sleep(2)
self.api_url = "{0}{1}".format(
settings.SITE_URL, reverse("search_api", args=[catalog_test_slug])
)
self.default_params = {
"search_engine": SEARCH_TYPE,
"search_engine_endpoint": self.search_engine_endpoint,
"q_time": "[* TO *]",
"q_geo": "[-90,-180 TO 90,180]",
"d_docs_limit": 0,
"d_docs_page": 1,
"d_docs_sort": "score"
}
class TestCSWTransactions(unittest.TestCase):
def setUp(self):
self.client = Client()
user = User.objects.create(username='******')
user.set_password('admin')
user.save()
self.client.login(username="******", password="******")
Catalog.objects.get_or_create(name=catalog_test_slug)
Layer.objects.all().delete()
Service.objects.all().delete()
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
self.solr.update_schema(catalog=catalog_test_slug)
self.solr.clear_solr(catalog=catalog_test_slug)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
es.clear_es()
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
def test_post(self):
"""
test CSV transactions.
:return:
"""
print("")
print(">>> with env:")
print("REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY)
print("REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS)
print("REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD)
print("REGISTRY_SEARCH_URL: %s" % settings.REGISTRY_SEARCH_URL)
print("REGISTRY_HARVEST_SERVICES: %s" %
settings.REGISTRY_HARVEST_SERVICES)
print("")
# Post the 10 Layers contained in this file: data/cswt_insert.xml
path = os.path.join(settings.PROJECT_DIR, "..", "data",
"cswt_insert.xml")
payload = open(path, 'rb').read()
content_type = "application/xml"
url = "/registry/{0}/csw".format(catalog_test_slug)
res = self.client.post(url, data=payload, content_type=content_type)
self.assertEqual(res.status_code, 200)
self.assertEqual(Layer.objects.all().count(), 10)
# List the Layers posted above
url = "/registry/{0}/csw?service=CSW&version=2.0.2&request=" \
"GetRecords&typenames=csw:Record&elementsetname=full&" \
"resulttype=results".format(catalog_test_slug)
res = self.client.get(url)
self.assertEqual(res.status_code, 200)
self.assertEqual(res.content.count("Airports (OSM)"), 1)
self.assertEqual(res.content.count("Manaus Roads (OSM May 2016)"), 2)
# Search one Layer posted above
url = "/registry/{0}/csw?mode=opensearch&service=CSW&version" \
"=2.0.2&request=GetRecords&elementsetname=full&typenames=" \
"csw:Record&resulttype=results" \
"&q=Airport".format(catalog_test_slug)
res = self.client.get(url)
self.assertEqual(res.status_code, 200)
self.assertEqual(res.content.count("Airports (OSM)"), 1)
# Flush layers in the cache.
index_cached_layers()
# Give celery some time.
time.sleep(3)
# are Layers in index?
url = "{0}hypermap/_search".format(SEARCH_URL)
res = requests.get(url)
results_ok_in_search_backend = res.json()
self.assertTrue("hits" in results_ok_in_search_backend)
self.assertTrue("total" in results_ok_in_search_backend["hits"])
self.assertEqual(results_ok_in_search_backend["hits"]["total"], 10)
def tearDown(self):
pass
def index_cached_layers(self):
"""
Index and unindex all layers in the Django cache (Index all layers who have been checked).
"""
from hypermap.aggregator.models import Layer
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
solrobject = SolrHypermap()
else:
from hypermap.aggregator.elasticsearch_client import ESHypermap
from elasticsearch import helpers
es_client = ESHypermap()
layers_cache = cache.get('layers')
deleted_layers_cache = cache.get('deleted_layers')
# 1. added layers cache
if layers_cache:
layers_list = list(layers_cache)
LOGGER.debug('There are %s layers in cache: %s' % (len(layers_list), layers_list))
batch_size = settings.REGISTRY_SEARCH_BATCH_SIZE
batch_lists = [layers_list[i:i+batch_size] for i in range(0, len(layers_list), batch_size)]
for batch_list_ids in batch_lists:
layers = Layer.objects.filter(id__in=batch_list_ids)
if batch_size > len(layers):
batch_size = len(layers)
LOGGER.debug('Syncing %s/%s layers to %s: %s' % (batch_size, len(layers_cache), layers, SEARCH_TYPE))
try:
# SOLR
if SEARCH_TYPE == 'solr':
success, layers_errors_ids = solrobject.layers_to_solr(layers)
if success:
# remove layers from cache here
layers_cache = layers_cache.difference(set(batch_list_ids))
LOGGER.debug('Removing layers with id %s from cache' % batch_list_ids)
cache.set('layers', layers_cache)
# ES
elif SEARCH_TYPE == 'elasticsearch':
with_bulk, success = True, False
layers_to_index = [es_client.layer_to_es(layer, with_bulk) for layer in layers]
message = helpers.bulk(es_client.es, layers_to_index)
# Check that all layers where indexed...if not, don't clear cache.
# TODO: Check why es does not index all layers at first.
len_indexed_layers = message[0]
if len_indexed_layers == len(layers):
LOGGER.debug('%d layers indexed successfully' % (len_indexed_layers))
success = True
if success:
# remove layers from cache here
layers_cache = layers_cache.difference(set(batch_list_ids))
cache.set('layers', layers_cache)
else:
raise Exception("Incorrect SEARCH_TYPE=%s" % SEARCH_TYPE)
except Exception as e:
LOGGER.error('Layers were NOT indexed correctly')
LOGGER.error(e, exc_info=True)
else:
LOGGER.debug('No cached layers to add in search engine.')
# 2. deleted layers cache
if deleted_layers_cache:
layers_list = list(deleted_layers_cache)
LOGGER.debug('There are %s layers in cache for deleting: %s' % (len(layers_list), layers_list))
# TODO implement me: batch layer index deletion
for layer_id in layers_list:
# SOLR
if SEARCH_TYPE == 'solr':
if Layer.objects.filter(pk=layer_id).exists():
layer = Layer.objects.get(id=layer_id)
unindex_layer(layer.id, use_cache=False)
deleted_layers_cache = deleted_layers_cache.difference(set([layer_id]))
cache.set('deleted_layers', deleted_layers_cache)
else:
# TODO implement me
raise NotImplementedError
else:
LOGGER.debug('No cached layers to remove in search engine.')
class TestBrowser(unittest.TestCase):
def setUp(self):
if not SELENIUM_HUB_URL:
# run test on firefox of this machine.
self.driver = webdriver.Firefox()
else:
# run test on stand alone node machine in docker: selenium-firefox
self.driver = webdriver.Remote(
command_executor=SELENIUM_HUB_URL,
desired_capabilities=DesiredCapabilities.FIREFOX
)
self.driver.implicitly_wait(30)
self.base_url = BROWSER_HYPERMAP_URL
self.verificationErrors = []
self.accept_next_alert = True
print '> clearing SEARCH_URL={0}'.format(SEARCH_URL)
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
# delete solr documents
# add the schema
print '> updating schema'.format(SEARCH_URL)
self.solr.update_schema(catalog=catalog_test_slug)
self.solr.clear_solr(catalog=catalog_test_slug)
self.search_engine_endpoint = '{0}/solr/{1}/select'.format(
SEARCH_URL, catalog_test_slug
)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
# delete ES documents
es.clear_es()
self.search_engine_endpoint = '{0}/{1}/_search'.format(
SEARCH_URL, catalog_test_slug
)
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
def test_browser(self):
ENDPOINT_FILE = os.path.join("/usr/src/app",
"hypermap",
"tests",
"mesonet.agron.iastate.edu.txt")
print ""
print ">>> with env:"
print "REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY
print "REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS
print "REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD
print ""
print "SELENIUM_HUB_URL: %s" % SELENIUM_HUB_URL
print "BROWSER_HYPERMAP_URL: %s" % BROWSER_HYPERMAP_URL
print "BROWSER_SEARCH_URL: %s" % BROWSER_SEARCH_URL
print "BROWSER_MAPLOOM_URL: %s" % BROWSER_MAPLOOM_URL
print "WAIT_FOR_CELERY_JOB_PERIOD: %s" % WAIT_FOR_CELERY_JOB_PERIOD
print "ENDPOINT FILE: %s" % ENDPOINT_FILE
print ""
print "Starting..."
driver = self.driver
time.sleep(3)
driver.get(self.base_url + "/admin/login/?next=/admin/")
print driver.current_url
driver.find_element_by_id("id_password").clear()
driver.find_element_by_id("id_password").send_keys("admin")
driver.find_element_by_id("id_username").clear()
driver.find_element_by_id("id_username").send_keys("admin")
driver.find_element_by_css_selector("input[type=\"submit\"]").click()
print driver.current_url
driver.find_element_by_link_text("Periodic tasks").click()
print driver.current_url
print "> assert 3 periodic tasks. means beat is alive."
self.assertEqual("3 periodic tasks",
driver.find_element_by_css_selector(
"p.paginator").text)
driver.find_element_by_link_text("Home").click()
print driver.current_url
driver.find_element_by_link_text("Endpoint lists").click()
print driver.current_url
driver.find_element_by_link_text("Add endpoint list").click()
print driver.current_url
print "> uploading Endpoint List..."
driver.find_element_by_id("id_upload").clear()
driver.find_element_by_id("id_upload").send_keys(ENDPOINT_FILE)
driver.find_element_by_name("_save").click()
print driver.current_url
print "> waiting {0} seconds for celery do the job....".format(
WAIT_FOR_CELERY_JOB_PERIOD
)
time.sleep(WAIT_FOR_CELERY_JOB_PERIOD)
driver.find_element_by_link_text("Aggregator").click()
time.sleep(1)
print driver.current_url
driver.find_element_by_link_text("Endpoints").click()
print driver.current_url
print "> assert Endpoint created."
time.sleep(1)
self.assertEqual(
"http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi",
driver.find_element_by_link_text(
"http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").text)
driver.find_element_by_link_text(
"http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").click()
# self.assertEqual("1 service/s created", driver.find_element_by_id("id_message").text)
driver.find_element_by_link_text("Endpoints").click()
print driver.current_url
time.sleep(1)
driver.find_element_by_link_text("Aggregator").click()
print driver.current_url
time.sleep(1)
driver.find_element_by_link_text("Services").click()
print driver.current_url
print "> assert 1 Service created."
time.sleep(1)
self.assertEqual("1 service", driver.find_element_by_css_selector(
"p.paginator").text)
self.assertEqual(
"http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi",
driver.find_element_by_css_selector("td.field-url").text)
driver.find_element_by_link_text("1").click()
print driver.current_url
print "> assert Service details."
time.sleep(1)
try:
self.assertEqual("IEM NWS Warnings WMS Service",
driver.find_element_by_id(
"id_title").get_attribute("value"))
except AssertionError as e:
self.verificationErrors.append(str(e))
driver.find_element_by_link_text("Services").click()
print driver.current_url
driver.find_element_by_link_text("Aggregator").click()
print driver.current_url
driver.find_element_by_link_text("Layers").click()
print driver.current_url
print "> assert 3 layers created."
time.sleep(1)
self.assertEqual("3 layers", driver.find_element_by_css_selector(
"p.paginator").text)
driver.get(self.base_url + "/registry/")
print driver.current_url
print "> go to /registry/."
for i in range(1, 11):
print "> try assert checks count > 0. (%i of 10)" % i
try:
self.assertNotEqual("0", driver.find_element_by_xpath(
"//td[4]").text)
print "> found"
break
except AssertionError as e:
print "> wait and reload page"
time.sleep(10)
driver.get(self.base_url + "/registry/")
try:
self.assertNotEqual("0",
driver.find_element_by_xpath("//td[4]").text)
except AssertionError as e:
self.verificationErrors.append(str(e))
driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL))
print driver.current_url
time.sleep(2)
for i in range(1, 11):
print "> assert layers indexed are 3. (%i of 10)" % i
try:
self.assertRegexpMatches(
driver.find_element_by_css_selector("pre").text,
"^\\{\"count\":3[\\s\\S]*$")
print "> found"
break
except AssertionError:
print "> wait and reload page"
time.sleep(10)
driver.refresh()
self.assertRegexpMatches(
driver.find_element_by_css_selector("pre").text,
"^\\{\"count\":3[\\s\\S]*$")
driver.get(self.base_url + "/registry/")
print driver.current_url
driver.find_element_by_link_text(
"IEM NWS Warnings WMS Service").click()
print driver.current_url
print "> remove checks."
driver.find_element_by_name("remove").click()
print driver.current_url
driver.find_element_by_link_text("Home").click()
print driver.current_url
print "> assert checks = 0."
self.assertEqual("0", driver.find_element_by_xpath("//td[4]").text)
driver.find_element_by_link_text(
"IEM NWS Warnings WMS Service").click()
print driver.current_url
print "> trigger check."
driver.find_element_by_name("check").click()
print driver.current_url
driver.find_element_by_link_text("Home").click()
print driver.current_url
for i in range(1, 11):
try:
print "> assert checks = 1. (%i of 10)" % i
self.assertTrue(
int(driver.find_element_by_xpath("//td[4]").text) > 0)
print "> found"
break
except AssertionError:
print "> wait and reload page"
time.sleep(10)
driver.refresh()
driver.find_element_by_link_text(
"IEM NWS Warnings WMS Service").click()
print driver.current_url
driver.find_element_by_link_text("wwa").click()
print driver.current_url
print "> remove checks from Layer."
driver.find_element_by_name("remove").click()
print driver.current_url
print "> assert text [No checks performed so far]."
self.assertEqual("No checks performed so far",
driver.find_element_by_xpath("//tr[11]/td[2]").text)
print "> check Layer."
driver.find_element_by_name("check").click()
print driver.current_url
for i in range(1, 11):
try:
print "> assert text [Total Checks: N>0]. (%i of 10)" % i
src = driver.page_source
text_found_TOTAL_CHECKS_LTE_1 = re.search(
r'Total Checks: (1|2|3|4|5|6|7)', src)
self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None)
print "> found"
break
except AssertionError:
print "> wait and reload page"
time.sleep(10)
driver.get(driver.current_url)
src = driver.page_source
text_found_TOTAL_CHECKS_LTE_1 = re.search(
r'Total Checks: (1|2|3|4|5|6|7)', src)
self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None)
driver.find_element_by_link_text("Home").click()
print driver.current_url
driver.find_element_by_link_text("Monitor").click()
print driver.current_url
print "> clean Search index and wait"
driver.find_element_by_name("clear_index").click()
print driver.current_url
time.sleep(5)
driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL))
print driver.current_url
print "> assert count != 3 layers"
try:
self.assertNotRegexpMatches(
driver.find_element_by_css_selector("pre").text,
"^\\{\"count\":3[\\s\\S]*$")
except AssertionError as e:
self.verificationErrors.append(str(e))
driver.get(self.base_url + "/registry/")
print driver.current_url
print "> finish hypermap page"
print ""
# TODO: activate this to test maploom, now dat app looks very buggy.
"""
print ">> start maploom"
driver.get(BROWSER_MAPLOOM_URL)
print driver.current_url
print ">> open registry modal"
driver.find_element_by_xpath(
"//div[@id='pulldown-content']/div[2]/div/div").click()
print ">> assert Hypermap catalog"
time.sleep(10)
self.assertEqual("Hypermap",
driver.find_element_by_xpath(
"//div[@id='explore']/div/nav/div/form/div/div[2]/select").text)
print ">> assert [Showing 3 of 3 - Page 1 / 1]"
self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(),
driver.find_element_by_css_selector(
"span.text-muted.ng-binding").text.lower())
driver.find_element_by_id("text_search_input_exp").clear()
print ">> search IEM"
driver.find_element_by_id("text_search_input_exp").send_keys("IEM")
driver.find_element_by_id("text_search_btn").click()
time.sleep(10)
print ">> assert [Showing 1 of 1 - Page 1 / 1]"
self.assertEqual("Showing 1 of 1 - Page 1 / 1".lower(),
driver.find_element_by_css_selector(
"span.text-muted.ng-binding").text.lower())
print ">> click reset"
driver.find_element_by_name("button").click()
time.sleep(10)
print ">> assert [Showing 3 of 3 - Page 1 / 1]"
self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(),
driver.find_element_by_css_selector(
"span.text-muted.ng-binding").text.lower())
print ">> click on 3 layers to select"
driver.find_element_by_css_selector("td.ellipsis.ng-binding").click()
driver.find_element_by_xpath(
"//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click()
driver.find_element_by_xpath(
"//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click()
print ">> click on 3 layers to unselect"
driver.find_element_by_css_selector("td.ellipsis.ng-binding").click()
driver.find_element_by_xpath(
"//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click()
driver.find_element_by_xpath(
"//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click()
"""
def is_element_present(self, how, what):
try:
self.driver.find_element(by=how, value=what)
except NoSuchElementException as e:
print e
return False
return True
def is_alert_present(self):
try:
self.driver.switch_to_alert()
except NoAlertPresentException as e:
print e
return False
return True
def close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally:
self.accept_next_alert = True
def tearDown(self):
self.driver.quit()
self.assertEqual([], self.verificationErrors)
def clear_solr():
print 'Clearing the solr core and indexes'
from hypermap.aggregator.solr import SolrHypermap
solrobject = SolrHypermap()
solrobject.clear_solr()
def setUp(self):
signals.post_save.disconnect(layer_post_save, sender=Layer)
signals.post_save.disconnect(service_post_save, sender=Service)
catalog_test_slug = "hypermap"
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
# delete solr documents
# add the schema
print('> updating schema'.format(SEARCH_URL))
self.solr.update_schema(catalog=catalog_test_slug)
print('> clearing SEARCH_URL={0}'.format(SEARCH_URL))
self.solr.clear_solr(catalog=catalog_test_slug)
self.search_engine_endpoint = '{0}/solr/{1}/select'.format(
SEARCH_URL, catalog_test_slug
)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
# delete ES documents
es.clear_es()
self.search_engine_endpoint = '{0}/{1}/_search'.format(
SEARCH_URL, catalog_test_slug
)
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
catalog, created = Catalog.objects.get_or_create(
name=catalog_test_slug
)
service = Service(
url='http://fakeurl.com',
title='Title',
type='OGC:WMS',
catalog=catalog
)
service.save()
layer = Layer(
name='Layer 1',
bbox_x0=-40.0,
bbox_x1=-20.0,
bbox_y0=-40.0,
bbox_y1=-20.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 2',
bbox_x0=-40.0,
bbox_x1=-20.0,
bbox_y0=20.0,
bbox_y1=40.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 3',
bbox_x0=20.0,
bbox_x1=40.0,
bbox_y0=20.0,
bbox_y1=40.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 4',
bbox_x0=20.0,
bbox_x1=40.0,
bbox_y0=-40.0,
bbox_y1=-20.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
# solr have commitWithin 1500.
# before to proceed with the tests wait for 2 secs.
# otherwise it will return zero docs in the next test.
service.index_layers(with_cache=False)
time.sleep(2)
self.api_url = "{0}{1}".format(
settings.SITE_URL, reverse("search_api", args=[catalog_test_slug])
)
self.default_params = {
"search_engine": SEARCH_TYPE,
"search_engine_endpoint": self.search_engine_endpoint,
"q_time": "[* TO *]",
"q_geo": "[-90,-180 TO 90,180]",
"d_docs_limit": 0,
"d_docs_page": 1,
"d_docs_sort": "score"
}
class SearchApiTestCase(TestCase):
"""
run me
python manage.py test hypermap.search_api --settings=hypermap.settings.test --failfast
"""
def tearDown(self):
signals.post_save.connect(layer_post_save, sender=Layer)
signals.post_save.connect(service_post_save, sender=Service)
def setUp(self):
signals.post_save.disconnect(layer_post_save, sender=Layer)
signals.post_save.disconnect(service_post_save, sender=Service)
catalog_test_slug = "hypermap"
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
# delete solr documents
# add the schema
print('> updating schema'.format(SEARCH_URL))
self.solr.update_schema(catalog=catalog_test_slug)
print('> clearing SEARCH_URL={0}'.format(SEARCH_URL))
self.solr.clear_solr(catalog=catalog_test_slug)
self.search_engine_endpoint = '{0}/solr/{1}/select'.format(
SEARCH_URL, catalog_test_slug
)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
# delete ES documents
es.clear_es()
self.search_engine_endpoint = '{0}/{1}/_search'.format(
SEARCH_URL, catalog_test_slug
)
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
catalog, created = Catalog.objects.get_or_create(
name=catalog_test_slug
)
service = Service(
url='http://fakeurl.com',
title='Title',
type='OGC:WMS',
catalog=catalog
)
service.save()
layer = Layer(
name='Layer 1',
bbox_x0=-40.0,
bbox_x1=-20.0,
bbox_y0=-40.0,
bbox_y1=-20.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2000, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 2',
bbox_x0=-40.0,
bbox_x1=-20.0,
bbox_y0=20.0,
bbox_y1=40.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2001, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 3',
bbox_x0=20.0,
bbox_x1=40.0,
bbox_y0=20.0,
bbox_y1=40.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2002, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
layer = Layer(
name='Layer 4',
bbox_x0=20.0,
bbox_x1=40.0,
bbox_y0=-40.0,
bbox_y1=-20.0,
service=service,
catalog=catalog
)
layer.title = layer.name
layer.save()
layer.created = datetime.datetime(2003, 3, 1, 0, 0, 0)
layer.save()
service.layer_set.add(layer)
# solr have commitWithin 1500.
# before to proceed with the tests wait for 2 secs.
# otherwise it will return zero docs in the next test.
service.index_layers(with_cache=False)
time.sleep(2)
self.api_url = "{0}{1}".format(
settings.SITE_URL, reverse("search_api", args=[catalog_test_slug])
)
self.default_params = {
"search_engine": SEARCH_TYPE,
"search_engine_endpoint": self.search_engine_endpoint,
"q_time": "[* TO *]",
"q_geo": "[-90,-180 TO 90,180]",
"d_docs_limit": 0,
"d_docs_page": 1,
"d_docs_sort": "score"
}
def test_catalogs(self):
print('> testing catalogs')
url = settings.SITE_URL + reverse("catalog-list")
res = self.client.get(url)
self.assertEqual(res.status_code, 200)
catalogs = json.loads(res.content)
self.assertEqual(len(catalogs), Catalog.objects.all().count())
def test_all_match_docs(self):
print('> testing match all docs')
params = self.default_params
print("searching on [{}]".format(self.api_url))
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], Layer.objects.all().count())
def test_q_text(self):
print('> testing q text')
layer = Layer.objects.all()[0]
params = self.default_params
params["q_text"] = "title:\"{0}\"".format(layer.title)
params["d_docs_limit"] = 100
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], 1)
for doc in results.get("d.docs", []):
self.assertEqual(doc["title"], layer.title)
def test_q_geo(self):
print('> testing q geo')
params = self.default_params
# top right square
params["q_geo"] = "[0,0 TO 30,30]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], 1)
# bottom left square
params["q_geo"] = "[-30,-30 TO 0,0]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], 1)
# big square
params["q_geo"] = "[-30,-30 TO 30,30]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], 4)
# center where no layers
params["q_geo"] = "[-5,-5 TO 5,5]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], 0)
# bad format
params["q_geo"] = "[-5,-5 5,5]"
results = self.client.get(self.api_url, params)
# validate the format
print('> testing q geo (format validations)')
self.assertEqual(results.status_code, 400)
def test_q_time(self):
print('> testing q time (format validations)')
params = self.default_params
# test validations
params["q_time"] = "[2000-01-01 - 2001-01-01T00:00:00]"
results = self.client.get(self.api_url, params)
# requires [X TO Y]
self.assertEqual(400, results.status_code)
print('> testing q time')
# test asterisks
# all times
params["q_time"] = "[* TO *]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
# all records
self.assertEqual(results["a.matchDocs"], Layer.objects.all().count())
# test range
# entire year 2000
params["q_time"] = "[2000-01-01 TO 2001-01-01T00:00:00]"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
# 1 in year 2000
self.assertEqual(results["a.matchDocs"], 1)
# test complete min and max when q time is asterisks
params["q_time"] = "[* TO *]"
params["a_time_limit"] = 1
if SEARCH_TYPE == SEARCH_TYPE_ES:
# TODO: a_time_limit is WIP in ES, today requires a a_time_gap to be completed.
params["a_time_gap"] = "P1Y"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], Layer.objects.all().count())
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
# TODO: fix on Solr or ES? see next TODO.
# * TO * to first date and last date.
self.assertEqual(results["a.time"]["start"].upper(), "2000-03-01T00:00:00Z")
self.assertEqual(results["a.time"]["end"].upper(), "2003-03-01T00:00:00Z")
else:
# TODO: ES and SOLR returns facets by default spliting the data yearly. first record is on 2000-03,
# ES facets are returned from 2000-01... to 2003-01.
# SOLR facets are returned from 2000-03... to 2003-03.
# SOLR data seems more accurate since first and last Layers are in month 03.
# Example: http://panchicore.d.pr/12ESP
# * TO * to first date and last date.
self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z")
self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z")
# test facets
params["q_time"] = "[2000 TO 2022]"
params["a_time_limit"] = 1
params["a_time_gap"] = "P1Y"
results = self.client.get(self.api_url, params)
self.assertEqual(results.status_code, 200)
results = json.loads(results.content)
self.assertEqual(results["a.matchDocs"], Layer.objects.all().count())
# 2000 to complete datetime format
self.assertEqual(results["a.time"]["start"].upper(), "2000-01-01T00:00:00Z")
# 2022 to complete datetime format
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
# TODO: solr creates entire time span and brings facets with empty entries. (2000 to 2022)
# fix by removing facets with zero counts?.
self.assertEqual(results["a.time"]["end"].upper(), "2022-01-01T00:00:00Z")
else:
self.assertEqual(results["a.time"]["end"].upper(), "2003-01-01T00:00:00Z")
# the facet counters are all facets excluding < 2000
self.assertEqual(len(results["a.time"]["counts"]), Layer.objects.all().count())
def test_utilities(self):
print('> testing utilities functions')
# test_parse_datetime_range
start, end = utils.parse_datetime_range("[2013-03-01 TO 2014-05-02T23:00:00]")
self.assertTrue(start.get("is_common_era"))
self.assertEqual(start.get("parsed_datetime").year, 2013)
self.assertEqual(start.get("parsed_datetime").month, 3)
self.assertEqual(start.get("parsed_datetime").day, 1)
self.assertTrue(end.get("is_common_era"))
self.assertEqual(end.get("parsed_datetime").year, 2014)
self.assertEqual(end.get("parsed_datetime").month, 5)
self.assertEqual(end.get("parsed_datetime").day, 2)
self.assertEqual(end.get("parsed_datetime").hour, 23)
self.assertEqual(end.get("parsed_datetime").minute, 0)
self.assertEqual(end.get("parsed_datetime").second, 0)
start, end = utils.parse_datetime_range("[-500000000 TO 2014-05-02T23:00:00]")
self.assertFalse(start.get("is_common_era"))
self.assertEqual(start.get("parsed_datetime"), "-500000000-01-01T00:00:00Z")
start, end = utils.parse_datetime_range("[* TO *]")
self.assertTrue(start.get("is_common_era"))
self.assertEqual(start.get("parsed_datetime"), None)
self.assertEqual(end.get("parsed_datetime"), None)
# test_parse_ISO8601
quantity, units = utils.parse_ISO8601("P3D")
self.assertEqual(quantity, 3)
self.assertEqual(units[0], "DAYS")
# test_gap_to_sorl
value = utils.gap_to_sorl("P3D")
self.assertEqual(value, "+3DAYS")
# test_parse_geo_box
value = utils.parse_geo_box("[-90,-180 TO 90,180]")
self.assertEqual(value.bounds[0], -90)
self.assertEqual(value.bounds[1], -180)
self.assertEqual(value.bounds[2], 90)
self.assertEqual(value.bounds[3], 180)
# test_request_time_facet
d = utils.request_time_facet("x", "[2000 TO 2014-01-02T11:12:13]", None, 1000)
self.assertEqual(type(d), dict)
self.assertEqual(d['f.x.facet.range.start'], '2000-01-01T00:00:00Z')
self.assertEqual(d['f.x.facet.range.end'], '2014-01-02T11:12:13Z')
self.assertEqual(d['f.x.facet.range.gap'], '+6DAYS')
self.assertEqual(d['facet.range'], 'x')
d = utils.request_time_facet("y", "[-5000000 TO 2016]", "P1D", 1)
self.assertEqual(d['f.y.facet.range.start'], '-5000000-01-01T00:00:00Z')
self.assertEqual(d['f.y.facet.range.end'], '2016-01-01T00:00:00Z')
self.assertEqual(d['f.y.facet.range.gap'], '+1DAYS')
self.assertEqual(d['facet.range'], 'y')
class TestCSWTransactions(unittest.TestCase):
def setUp(self):
self.client = Client()
user = User.objects.create(username='******')
user.set_password('admin')
user.save()
self.client.login(username="******", password="******")
Catalog.objects.get_or_create(
name=catalog_test_slug
)
Layer.objects.all().delete()
Service.objects.all().delete()
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
self.solr.update_schema(catalog=catalog_test_slug)
self.solr.clear_solr(catalog=catalog_test_slug)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
es.clear_es()
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
def test_post(self):
"""
test CSV transactions.
:return:
"""
print ""
print ">>> with env:"
print "REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY
print "REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS
print "REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD
print "REGISTRY_SEARCH_URL: %s" % settings.REGISTRY_SEARCH_URL
print "REGISTRY_HARVEST_SERVICES: %s" % settings.REGISTRY_HARVEST_SERVICES
print ""
# Post the 10 Layers contained in this file: data/cswt_insert.xml
path = os.path.join(settings.PROJECT_DIR, "..",
"data", "cswt_insert.xml")
payload = open(path, 'rb').read()
content_type = "application/xml"
url = "/registry/{0}/csw".format(catalog_test_slug)
res = self.client.post(url, data=payload, content_type=content_type)
self.assertEqual(res.status_code, 200)
self.assertEqual(Layer.objects.all().count(), 10)
# List the Layers posted above
url = "/registry/{0}/csw?service=CSW&version=2.0.2&request=" \
"GetRecords&typenames=csw:Record&elementsetname=full&" \
"resulttype=results".format(catalog_test_slug)
res = self.client.get(url)
self.assertEqual(res.status_code, 200)
self.assertEqual(res.content.count("Airports (OSM)"), 1)
self.assertEqual(res.content.count("Manaus Roads (OSM May 2016)"), 2)
# Search one Layer posted above
url = "/registry/{0}/csw?mode=opensearch&service=CSW&version" \
"=2.0.2&request=GetRecords&elementsetname=full&typenames=" \
"csw:Record&resulttype=results" \
"&q=Airport".format(catalog_test_slug)
res = self.client.get(url)
self.assertEqual(res.status_code, 200)
self.assertEqual(res.content.count("Airports (OSM)"), 1)
# Flush layers in the cache.
index_cached_layers()
# Give celery some time.
time.sleep(3)
# are Layers in index?
url = "{0}hypermap/_search".format(
SEARCH_URL
)
res = requests.get(url)
results_ok_in_search_backend = res.json()
self.assertTrue("hits" in results_ok_in_search_backend)
self.assertTrue("total" in results_ok_in_search_backend["hits"])
self.assertEqual(results_ok_in_search_backend["hits"]["total"], 10)
def tearDown(self):
pass
def index_cached_layers(self):
"""
Index all layers in the Django cache (Index all layers who have been checked).
"""
from hypermap.aggregator.models import Layer
from hypermap.aggregator.models import TaskError
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
solrobject = SolrHypermap()
else:
from hypermap.aggregator.elasticsearch_client import ESHypermap
from elasticsearch import helpers
es_client = ESHypermap()
layers_cache = cache.get('layers')
if layers_cache:
layers_list = list(layers_cache)
LOGGER.debug('There are %s layers in cache: %s' % (len(layers_list), layers_list))
batch_size = settings.REGISTRY_SEARCH_BATCH_SIZE
batch_lists = [layers_list[i:i+batch_size] for i in range(0, len(layers_list), batch_size)]
for batch_list_ids in batch_lists:
layers = Layer.objects.filter(id__in=batch_list_ids)
if batch_size > len(layers):
batch_size = len(layers)
LOGGER.debug('Syncing %s/%s layers to %s: %s' % (batch_size, len(layers_cache), layers, SEARCH_TYPE))
try:
if SEARCH_TYPE == 'solr':
success, message = solrobject.layers_to_solr(layers)
elif SEARCH_TYPE == 'elasticsearch':
with_bulk, success = True, False
layers_to_index = [es_client.layer_to_es(layer, with_bulk) for layer in layers]
message = helpers.bulk(es_client.es, layers_to_index)
# Check that all layers where indexed...if not, don't clear cache.
# TODO: Check why es does not index all layers at first.
len_indexed_layers = message[0]
if len_indexed_layers == len(layers):
LOGGER.debug('%d layers indexed successfully' % (len_indexed_layers))
success = True
else:
raise Exception("Incorrect SEARCH_TYPE=%s" % SEARCH_TYPE)
if success:
# remove layers from cache here
layers_cache = layers_cache.difference(set(batch_list_ids))
cache.set('layers', layers_cache)
else:
task_error = TaskError(
task_name=self.name,
args=batch_list_ids,
message=message
)
task_error.save()
except Exception as e:
LOGGER.error('Layers were NOT indexed correctly')
LOGGER.error(e, exc_info=True)
else:
LOGGER.debug('No cached layers.')
class TestBrowser(unittest.TestCase):
def setUp(self):
if not SELENIUM_HUB_URL:
# run test on firefox of this machine.
self.driver = webdriver.Firefox()
else:
# run test on stand alone node machine in docker: selenium-firefox
self.driver = webdriver.Remote(
command_executor=SELENIUM_HUB_URL,
desired_capabilities=DesiredCapabilities.FIREFOX
)
self.driver.implicitly_wait(30)
self.base_url = BROWSER_HYPERMAP_URL
self.verificationErrors = []
self.accept_next_alert = True
print '> clearing SEARCH_URL={0}'.format(SEARCH_URL)
if SEARCH_TYPE == SEARCH_TYPE_SOLR:
self.solr = SolrHypermap()
# delete solr documents
# add the schema
print '> updating schema'.format(SEARCH_URL)
self.solr.update_schema(catalog=catalog_test_slug)
self.solr.clear_solr(catalog=catalog_test_slug)
self.search_engine_endpoint = '{0}/solr/{1}/select'.format(
SEARCH_URL, catalog_test_slug
)
elif SEARCH_TYPE == SEARCH_TYPE_ES:
es = ESHypermap()
# delete ES documents
es.clear_es()
self.search_engine_endpoint = '{0}/{1}/_search'.format(
SEARCH_URL, catalog_test_slug
)
else:
raise Exception("SEARCH_TYPE not valid=%s" % SEARCH_TYPE)
def test_browser(self):
ENDPOINT_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"mesonet.agron.iastate.edu.txt")
print ""
print ">>> with env:"
print "REGISTRY_SKIP_CELERY: %s" % settings.REGISTRY_SKIP_CELERY
print "REGISTRY_LIMIT_LAYERS: %s" % settings.REGISTRY_LIMIT_LAYERS
print "REGISTRY_CHECK_PERIOD: %s" % settings.REGISTRY_CHECK_PERIOD
print ""
print "SELENIUM_HUB_URL: %s" % SELENIUM_HUB_URL
print "BROWSER_HYPERMAP_URL: %s" % BROWSER_HYPERMAP_URL
print "BROWSER_SEARCH_URL: %s" % BROWSER_SEARCH_URL
print "BROWSER_MAPLOOM_URL: %s" % BROWSER_MAPLOOM_URL
print "WAIT_FOR_CELERY_JOB_PERIOD: %s" % WAIT_FOR_CELERY_JOB_PERIOD
print "ENDPOINT FILE: %s" % ENDPOINT_FILE
print ""
print "Starting..."
driver = self.driver
time.sleep(3)
driver.get(self.base_url + "/admin/login/?next=/admin/")
print driver.current_url
driver.find_element_by_id("id_password").clear()
driver.find_element_by_id("id_password").send_keys("admin")
driver.find_element_by_id("id_username").clear()
driver.find_element_by_id("id_username").send_keys("admin")
driver.find_element_by_css_selector("input[type=\"submit\"]").click()
print driver.current_url
driver.find_element_by_link_text("Periodic tasks").click()
print driver.current_url
print "> assert 3 periodic tasks. means beat is alive."
self.assertEqual("3 periodic tasks",
driver.find_element_by_css_selector(
"p.paginator").text)
driver.find_element_by_link_text("Home").click()
print driver.current_url
driver.find_element_by_link_text("Endpoint lists").click()
print driver.current_url
driver.find_element_by_link_text("Add endpoint list").click()
print driver.current_url
print "> uploading Endpoint List..."
driver.find_element_by_id("id_upload").clear()
driver.find_element_by_id("id_upload").send_keys(ENDPOINT_FILE)
driver.find_element_by_name("_save").click()
print driver.current_url
print "> waiting {0} seconds for celery do the job....".format(
WAIT_FOR_CELERY_JOB_PERIOD
)
time.sleep(WAIT_FOR_CELERY_JOB_PERIOD)
driver.find_element_by_link_text("Aggregator").click()
time.sleep(1)
print driver.current_url
driver.find_element_by_link_text("Endpoints").click()
print driver.current_url
print "> assert Endpoint created."
time.sleep(1)
self.assertEqual(
"http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi",
driver.find_element_by_link_text(
"http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").text)
driver.find_element_by_link_text(
"http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi").click()
# self.assertEqual("1 service/s created", driver.find_element_by_id("id_message").text)
driver.find_element_by_link_text("Endpoints").click()
print driver.current_url
time.sleep(1)
driver.find_element_by_link_text("Aggregator").click()
print driver.current_url
time.sleep(1)
driver.find_element_by_link_text("Services").click()
print driver.current_url
print "> assert 1 Service created."
time.sleep(1)
self.assertEqual("1 service", driver.find_element_by_css_selector(
"p.paginator").text)
self.assertEqual(
"http://mesonet.agron.iastate.edu/cgi-bin/wms/us/wwa.cgi",
driver.find_element_by_css_selector("td.field-url").text)
driver.find_element_by_xpath(
'//*[@id="result_list"]/tbody/tr/th/a').click()
print driver.current_url
print "> assert Service details."
time.sleep(1)
self.assertEqual("IEM NWS Warnings WMS Service",
driver.find_element_by_id(
"id_title").get_attribute("value"))
driver.find_element_by_link_text("Services").click()
print driver.current_url
driver.find_element_by_link_text("Aggregator").click()
print driver.current_url
driver.find_element_by_link_text("Layers").click()
print driver.current_url
print "> assert 3 layers created."
time.sleep(1)
self.assertEqual("3 layers", driver.find_element_by_css_selector(
"p.paginator").text)
driver.get(self.base_url + "/registry/")
print driver.current_url
print "> go to /registry/."
for i in range(1, 11):
print "> try assert checks count > 0. (%i of 10)" % i
try:
self.assertNotEqual("0", driver.find_element_by_xpath(
"//td[4]").text)
print "> found"
break
except AssertionError as e:
print "> wait and reload page"
time.sleep(10)
driver.get(self.base_url + "/registry/")
try:
self.assertNotEqual("0",
driver.find_element_by_xpath("//td[4]").text)
except AssertionError as e:
self.verificationErrors.append(str(e))
driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL))
print driver.current_url
time.sleep(2)
for i in range(1, 11):
print "> assert layers indexed are 3. (%i of 10)" % i
try:
self.assertRegexpMatches(
driver.find_element_by_css_selector("pre").text,
"^\\{\"count\":3[\\s\\S]*$")
print "> found"
break
except AssertionError:
print "> wait and reload page"
time.sleep(10)
driver.refresh()
self.assertRegexpMatches(
driver.find_element_by_css_selector("pre").text,
"^\\{\"count\":3[\\s\\S]*$")
driver.get(self.base_url + "/registry/")
print driver.current_url
driver.find_element_by_link_text(
"IEM NWS Warnings WMS Service").click()
print driver.current_url
print "> remove checks."
driver.find_element_by_name("remove").click()
print driver.current_url
driver.find_element_by_link_text("Home").click()
print driver.current_url
print "> assert checks = 0."
self.assertEqual("0", driver.find_element_by_xpath("//td[4]").text)
driver.find_element_by_link_text(
"IEM NWS Warnings WMS Service").click()
print driver.current_url
print "> trigger check."
driver.find_element_by_name("check").click()
print driver.current_url
driver.find_element_by_link_text("Home").click()
print driver.current_url
for i in range(1, 11):
try:
print "> assert checks = 1. (%i of 10)" % i
self.assertTrue(
int(driver.find_element_by_xpath("//td[4]").text) > 0)
print "> found"
break
except AssertionError:
print "> wait and reload page"
time.sleep(10)
driver.refresh()
driver.find_element_by_link_text(
"IEM NWS Warnings WMS Service").click()
print driver.current_url
driver.find_element_by_link_text("wwa").click()
print driver.current_url
print "> remove checks from Layer."
driver.find_element_by_name("remove").click()
print driver.current_url
print "> assert text [No checks performed so far]."
self.assertEqual("No checks performed so far",
driver.find_element_by_xpath("//tr[11]/td[2]").text)
print "> check Layer."
driver.find_element_by_name("check").click()
print driver.current_url
for i in range(1, 11):
try:
print "> assert text [Total Checks: N>0]. (%i of 10)" % i
src = driver.page_source
text_found_TOTAL_CHECKS_LTE_1 = re.search(
r'Total Checks: (1|2|3|4|5|6|7)', src)
self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None)
print "> found"
break
except AssertionError:
print "> wait and reload page"
time.sleep(10)
driver.get(driver.current_url)
src = driver.page_source
text_found_TOTAL_CHECKS_LTE_1 = re.search(
r'Total Checks: (1|2|3|4|5|6|7)', src)
self.assertNotEqual(text_found_TOTAL_CHECKS_LTE_1, None)
driver.find_element_by_link_text("Home").click()
print driver.current_url
driver.find_element_by_link_text("Monitor").click()
print driver.current_url
print "> clean Search index and wait"
driver.find_element_by_name("clear_index").click()
print driver.current_url
time.sleep(5)
driver.get("{0}/hypermap/_count".format(BROWSER_SEARCH_URL))
print driver.current_url
print "> assert count != 3 layers"
try:
self.assertNotRegexpMatches(
driver.find_element_by_css_selector("pre").text,
"^\\{\"count\":3[\\s\\S]*$")
except AssertionError as e:
self.verificationErrors.append(str(e))
driver.get(self.base_url + "/registry/")
print driver.current_url
print "> finish hypermap page"
print ""
# TODO: activate this to test maploom, now dat app looks very buggy.
"""
print ">> start maploom"
driver.get(BROWSER_MAPLOOM_URL)
print driver.current_url
print ">> open registry modal"
driver.find_element_by_xpath(
"//div[@id='pulldown-content']/div[2]/div/div").click()
print ">> assert Hypermap catalog"
time.sleep(10)
self.assertEqual("Hypermap",
driver.find_element_by_xpath(
"//div[@id='explore']/div/nav/div/form/div/div[2]/select").text)
print ">> assert [Showing 3 of 3 - Page 1 / 1]"
self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(),
driver.find_element_by_css_selector(
"span.text-muted.ng-binding").text.lower())
driver.find_element_by_id("text_search_input_exp").clear()
print ">> search IEM"
driver.find_element_by_id("text_search_input_exp").send_keys("IEM")
driver.find_element_by_id("text_search_btn").click()
time.sleep(10)
print ">> assert [Showing 1 of 1 - Page 1 / 1]"
self.assertEqual("Showing 1 of 1 - Page 1 / 1".lower(),
driver.find_element_by_css_selector(
"span.text-muted.ng-binding").text.lower())
print ">> click reset"
driver.find_element_by_name("button").click()
time.sleep(10)
print ">> assert [Showing 3 of 3 - Page 1 / 1]"
self.assertEqual("Showing 3 of 3 - Page 1 / 1".lower(),
driver.find_element_by_css_selector(
"span.text-muted.ng-binding").text.lower())
print ">> click on 3 layers to select"
driver.find_element_by_css_selector("td.ellipsis.ng-binding").click()
driver.find_element_by_xpath(
"//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click()
driver.find_element_by_xpath(
"//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click()
print ">> click on 3 layers to unselect"
driver.find_element_by_css_selector("td.ellipsis.ng-binding").click()
driver.find_element_by_xpath(
"//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[3]/td").click()
driver.find_element_by_xpath(
"//div[@id='registry-layers']/div/div/div/div[2]/div[2]/div/table/tbody/tr[4]/td").click()
"""
def is_element_present(self, how, what):
try:
self.driver.find_element(by=how, value=what)
except NoSuchElementException as e:
print e
return False
return True
def is_alert_present(self):
try:
self.driver.switch_to_alert()
except NoAlertPresentException as e:
print e
return False
return True
def close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally:
self.accept_next_alert = True
def tearDown(self):
self.driver.quit()
self.assertEqual([], self.verificationErrors)