def sync_es(session, batch_size=1000):
last_update, date_now = es_sync.get_status(session)
log.info('Last update time: {}'.format(last_update))
if not last_update:
raise Exception('No last update time, run `fill_index` to do an '
'initial import into ElasticSearch')
# get all documents that have changed since the last update
changed_documents = \
get_changed_documents(session, last_update) + \
get_changed_users(session, last_update) + \
get_changed_documents_for_associations(session, last_update) + \
get_deleted_locale_documents(session, last_update) + \
get_tagged_documents(session, last_update)
log.info('Number of changed documents: {}'.format(len(changed_documents)))
if changed_documents:
sync_documents(session, changed_documents, batch_size)
# get list of documents deleted since the last update
deleted_documents = get_deleted_documents(session, last_update)
log.info('Number of deleted documents: {}'.format(len(deleted_documents)))
if deleted_documents:
sync_deleted_documents(session, deleted_documents, batch_size)
es_sync.mark_as_updated(session, date_now)
log.info('Sync has finished')
def fill_index(session, batch_size=1000):
client = elasticsearch_config['client']
index_name = elasticsearch_config['index']
status = {'start_time': datetime.now(), 'last_progress_update': None}
_, date_now = es_sync.get_status(session)
total = session.query(Document). \
filter(Document.redirects_to.is_(None)).count()
def progress(count, total_count):
if status['last_progress_update'] is None or \
status['last_progress_update'] + timedelta(seconds=1) < \
datetime.now():
print('{0} of {1}'.format(count, total_count))
status['last_progress_update'] = datetime.now()
batch = ElasticBatch(client, batch_size)
count = 0
with batch:
for doc_type in document_types:
print('Importing document type {}'.format(doc_type))
to_search_document = search_documents[doc_type].to_search_document
for doc in sync.get_documents(session, doc_type, batch_size):
batch.add(to_search_document(doc, index_name))
count += 1
progress(count, total)
es_sync.mark_as_updated(session, date_now)
duration = datetime.now() - status['start_time']
print('Done (duration: {0})'.format(duration))
def test_get_changed_documents(self):
last_update, _ = es_sync.get_status(self.session)
changed_documents = get_changed_documents(self.session, last_update)
self.assertEqual(len(changed_documents), 4)
self.assertEqual('r', changed_documents[0][1])
self.assertEqual('w', changed_documents[1][1])
self.assertEqual(
len(get_changed_documents(
self.session, last_update + datetime.timedelta(0, 1))),
0)
def test_get_changed_documents(self):
last_update, _ = es_sync.get_status(self.session)
changed_documents = get_changed_documents(self.session, last_update)
self.assertEqual(len(changed_documents), 4)
self.assertEqual('r', changed_documents[0][1])
self.assertEqual('w', changed_documents[1][1])
self.assertEqual(
len(
get_changed_documents(self.session,
last_update + datetime.timedelta(0, 1))),
0)
def _add_database_status(self, status):
last_es_syncer_run = None
success = False
try:
last_es_syncer_run, _ = es_sync.get_status(DBSession)
success = True
except:
log.exception('Getting last es syncer run failed')
self.request.response.status_code = 500
status['pg'] = 'ok' if success else 'error'
status['last_es_syncer_run'] = last_es_syncer_run.isoformat() \
if last_es_syncer_run else ''
def _add_database_status(self, status):
last_es_syncer_run = None
success = False
try:
last_es_syncer_run, _ = es_sync.get_status(DBSession)
success = True
except Exception:
log.exception('Getting last es syncer run failed')
self.request.response.status_code = 500
status['pg'] = 'ok' if success else 'error'
status['last_es_syncer_run'] = last_es_syncer_run.isoformat() \
if last_es_syncer_run else ''
def test_get_deleted_locale_documents(self):
# simulate removing a locale
self.session.add(
ESDeletedLocale(document_id=self.waypoint1.document_id,
type='w',
lang='fr'))
self.session.flush()
last_update, _ = es_sync.get_status(self.session)
changed_documents = get_deleted_locale_documents(
self.session, last_update)
self.assertEqual(len(changed_documents), 1)
self.assertEqual('w', changed_documents[0][1])
def test_get_changed_documents_for_associations(self):
last_update, _ = es_sync.get_status(self.session)
changed_documents = get_changed_documents_for_associations(
self.session, last_update)
self.assertEqual(len(changed_documents), 4)
self.assertEqual(changed_documents[0][0], self.route1.document_id)
self.assertEqual(changed_documents[1][0], self.outing1.document_id)
self.assertEqual(changed_documents[2][0], self.route1.document_id)
self.assertEqual(changed_documents[3][0], self.outing1.document_id)
self.assertEqual(
len(get_changed_documents_for_associations(
self.session, last_update + datetime.timedelta(0, 1))),
0)
def test_get_changed_documents_for_associations(self):
last_update, _ = es_sync.get_status(self.session)
changed_documents = get_changed_documents_for_associations(
self.session, last_update)
self.assertEqual(len(changed_documents), 4)
self.assertEqual(changed_documents[0][0], self.route1.document_id)
self.assertEqual(changed_documents[1][0], self.outing1.document_id)
self.assertEqual(changed_documents[2][0], self.route1.document_id)
self.assertEqual(changed_documents[3][0], self.outing1.document_id)
self.assertEqual(
len(
get_changed_documents_for_associations(
self.session, last_update + datetime.timedelta(0, 1))), 0)
def sync_es(session):
last_update, date_now = es_sync.get_status(session)
if not last_update:
raise Exception('No last update time, run `fill_index` to do an '
'initial import into ElasticSearch')
# get all documents that have changed since the last update
# TODO also check changes to associations
changed_documents = get_changed_documents(session, last_update) + \
get_changed_users(session, last_update)
if changed_documents:
sync_documents(session, changed_documents)
es_sync.mark_as_updated(session, date_now)
def test_get_changed_users(self):
user_id = self.global_userids['contributor']
user = self.session.query(User).get(user_id)
user.name = 'changed'
self.session.flush()
last_update, _ = es_sync.get_status(self.session)
changed_users = get_changed_users(self.session, last_update)
self.assertEqual(len(changed_users), 1)
self.assertEqual(user_id, changed_users[0][0])
self.assertEqual('u', changed_users[0][1])
self.assertEqual(
len(get_changed_documents(
self.session, last_update + datetime.timedelta(0, 1))),
0)
def test_get_changed_users(self):
user_id = self.global_userids['contributor']
user = self.session.query(User).get(user_id)
user.name = 'changed'
self.session.flush()
last_update, _ = es_sync.get_status(self.session)
changed_users = get_changed_users(self.session, last_update)
self.assertEqual(len(changed_users), 1)
self.assertEqual(user_id, changed_users[0][0])
self.assertEqual('u', changed_users[0][1])
self.assertEqual(
len(
get_changed_documents(self.session,
last_update + datetime.timedelta(0, 1))),
0)
def sync_es(session, batch_size=1000):
last_update, date_now = es_sync.get_status(session)
log.info('Last update time: {}'.format(last_update))
if not last_update:
raise Exception('No last update time, run `fill_index` to do an '
'initial import into ElasticSearch')
# get all documents that have changed since the last update
changed_documents = \
get_changed_documents(session, last_update) + \
get_changed_users(session, last_update) + \
get_changed_documents_for_associations(session, last_update)
log.info('Number of changed documents: {}'.format(len(changed_documents)))
if changed_documents:
sync_documents(session, changed_documents, batch_size)
es_sync.mark_as_updated(session, date_now)
log.info('Sync has finished')
def fill_index(session):
client = elasticsearch_config['client']
index_name = elasticsearch_config['index']
status = {
'start_time': datetime.now(),
'last_progress_update': None
}
_, date_now = es_sync.get_status(session)
total = session.query(Document). \
filter(Document.redirects_to.is_(None)).count()
def progress(count, total_count):
if status['last_progress_update'] is None or \
status['last_progress_update'] + timedelta(seconds=1) < \
datetime.now():
print('{0} of {1}'.format(count, total_count))
status['last_progress_update'] = datetime.now()
batch = ElasticBatch(client, batch_size)
count = 0
with batch:
for doc_type in document_types:
print('Importing document type {}'.format(doc_type))
to_search_document = search_documents[doc_type].to_search_document
for doc in sync.get_documents(session, doc_type):
batch.add(to_search_document(doc, index_name))
count += 1
progress(count, total)
es_sync.mark_as_updated(session, date_now)
duration = datetime.now() - status['start_time']
print('Done (duration: {0})'.format(duration))
def _add_test_data(self):
_, date_now = es_sync.get_status(self.session)
es_sync.mark_as_updated(self.session, date_now)
self.waypoint1 = Waypoint(
document_id=71171,
waypoint_type='summit', elevation=2000,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='fr', title='Mont Granier',
description='...',
summary='Le Mont [b]Granier[/b]'),
WaypointLocale(
lang='en', title='Mont Granier',
description='...',
summary='The Mont Granier')
])
self.waypoint2 = Waypoint(
document_id=71172,
waypoint_type='summit', elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='en', title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
])
self.route1 = Route(
document_id=71173,
activities=['skitouring'], elevation_max=1500, elevation_min=700,
height_diff_up=800, height_diff_down=800, durations='1',
locales=[
RouteLocale(
lang='en', title='Face N',
description='...', gear='paraglider',
title_prefix='Mont Blanc'
)
]
)
self.waypoint3 = Waypoint(
document_id=71174,
redirects_to=71171,
waypoint_type='summit', elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='en', title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
])
self.session.add_all([
self.waypoint1, self.waypoint2, self.waypoint3, self.route1])
self.session.flush()
user_id = self.global_userids['contributor']
DocumentRest.create_new_version(self.waypoint1, user_id)
DocumentRest.create_new_version(self.waypoint2, user_id)
DocumentRest.create_new_version(self.waypoint3, user_id)
DocumentRest.create_new_version(self.route1, user_id)
def test_fill_index(self):
"""Tests that documents are inserted into the ElasticSearch index.
"""
self.session.add(Waypoint(
document_id=71171,
waypoint_type='summit', elevation=2000, quality='medium',
access_time='15min',
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='fr', title='Mont Granier',
description='...',
summary='Le Mont [b]Granier[/b]'),
WaypointLocale(
lang='en', title='Mont Granier',
description='...',
summary='The Mont Granier')
]))
self.session.add(Waypoint(
document_id=71172,
waypoint_type='summit', elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='en', title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
]))
self.session.add(Route(
document_id=71173,
activities=['skitouring'], elevation_max=1500, elevation_min=700,
height_diff_up=800, height_diff_down=800, durations=['1'],
locales=[
RouteLocale(
lang='en', title='Face N',
description='...', gear='paraglider',
title_prefix='Mont Blanc'
)
]
))
self.session.add(Waypoint(
document_id=71174,
redirects_to=71171,
waypoint_type='summit', elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='en', title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
]))
self.session.add(Outing(
document_id=71175,
activities=['skitouring'], date_start=datetime.date(2016, 1, 1),
date_end=datetime.date(2016, 1, 1), frequentation='overcrowded',
locales=[
OutingLocale(
lang='en', title='Mont Blanc : Face N !',
description='...', weather='sunny')
]
))
self.session.add(Article(
document_id=71176,
categories=['site_info'], activities=['hiking'],
article_type='collab',
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
DocumentLocale(
lang='en', title='Lac d\'Annecy',
description='...',
summary=''),
DocumentLocale(
lang='fr', title='Lac d\'Annecy',
description='...',
summary='')
]))
self.session.flush()
self.session.add(Book(
document_id=71177,
activities=['hiking'],
book_types=['biography'],
author='Denis Dainat',
publication_date='',
locales=[
DocumentLocale(
lang='fr', title='Escalades au Thaurac',
description='...',
summary=''),
DocumentLocale(
lang='en', title='Escalades au Thaurac',
description='...',
summary=''
)
]
))
self.session.flush()
self.session.add(Xreport(
document_id=71178,
event_type=['roped_fall'],
activities=['skitouring'],
nb_participants=10,
elevation=1500,
date=datetime.date(2016, 1, 1),
locales=[
XreportLocale(
lang='en', title='Death in the mountains',
description='...',
place='some place description'
)
]
))
self.session.flush()
# fill the ElasticSearch index
fill_index(self.session)
waypoint1 = SearchWaypoint.get(id=71171)
self.assertIsNotNone(waypoint1)
self.assertEqual(waypoint1.title_en, 'Mont Granier')
self.assertEqual(waypoint1.title_fr, 'Mont Granier')
# self.assertEqual(waypoint1.summary_fr, 'Le Mont Granier ')
self.assertEqual(waypoint1.doc_type, 'w')
self.assertEqual(waypoint1.quality, 2)
self.assertEqual(waypoint1.access_time, 3)
self.assertAlmostEqual(waypoint1.geom[0], 5.71288994)
self.assertAlmostEqual(waypoint1.geom[1], 45.64476395)
waypoint2 = SearchWaypoint.get(id=71172)
self.assertIsNotNone(waypoint2)
self.assertEqual(waypoint2.title_en, 'Mont Blanc')
self.assertEqual(waypoint2.title_fr, '')
self.assertEqual(waypoint2.doc_type, 'w')
route = SearchRoute.get(id=71173)
self.assertIsNotNone(route)
self.assertEqual(route.title_en, 'Mont Blanc : Face N')
self.assertEqual(route.title_fr, '')
self.assertEqual(route.doc_type, 'r')
self.assertEqual(route.durations, [0])
outing = SearchOuting.get(id=71175)
self.assertIsNotNone(outing)
self.assertEqual(outing.title_en, 'Mont Blanc : Face N !')
self.assertEqual(outing.title_fr, '')
self.assertEqual(outing.doc_type, 'o')
self.assertEqual(outing.frequentation, 3)
article = SearchArticle.get(id=71176)
self.assertIsNotNone(article)
self.assertEqual(article.title_en, 'Lac d\'Annecy')
self.assertEqual(article.title_fr, 'Lac d\'Annecy')
self.assertEqual(article.doc_type, 'c')
book = SearchBook.get(id=71177)
self.assertIsNotNone(book)
self.assertEqual(book.title_en, 'Escalades au Thaurac')
self.assertEqual(book.title_fr, 'Escalades au Thaurac')
self.assertEqual(book.doc_type, 'b')
self.assertEqual(book.book_types, ['biography'])
xreport = SearchXreport.get(id=71178)
self.assertIsNotNone(xreport)
self.assertEqual(xreport.title_en, 'Death in the mountains')
self.assertEqual(xreport.doc_type, 'x')
# merged document is ignored
self.assertIsNone(SearchWaypoint.get(id=71174, ignore=404))
# check that the sync. status was updated
last_update, _ = es_sync.get_status(self.session)
self.assertIsNotNone(last_update)
def test_fill_index(self):
"""Tests that documents are inserted into the ElasticSearch index.
"""
self.session.add(
Waypoint(document_id=71171,
waypoint_type='summit',
elevation=2000,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='fr',
title='Mont Granier',
description='...',
summary='Le Mont [b]Granier[/b]'),
WaypointLocale(lang='en',
title='Mont Granier',
description='...',
summary='The Mont Granier')
]))
self.session.add(
Waypoint(document_id=71172,
waypoint_type='summit',
elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='en',
title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
]))
self.session.add(
Route(document_id=71173,
activities=['skitouring'],
elevation_max=1500,
elevation_min=700,
height_diff_up=800,
height_diff_down=800,
durations='1',
locales=[
RouteLocale(lang='en',
title='Face N',
description='...',
gear='paraglider',
title_prefix='Mont Blanc')
]))
self.session.add(
Waypoint(document_id=71174,
redirects_to=71171,
waypoint_type='summit',
elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='en',
title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
]))
self.session.flush()
# fill the ElasticSearch index
fill_index(self.session)
waypoint1 = SearchWaypoint.get(id=71171)
self.assertIsNotNone(waypoint1)
self.assertEqual(waypoint1.title_en, 'Mont Granier')
self.assertEqual(waypoint1.title_fr, 'Mont Granier')
self.assertEqual(waypoint1.summary_fr, 'Le Mont Granier ')
self.assertEqual(waypoint1.doc_type, 'w')
self.assertAlmostEqual(waypoint1.geom[0], 5.71288994)
self.assertAlmostEqual(waypoint1.geom[1], 45.64476395)
waypoint2 = SearchWaypoint.get(id=71172)
self.assertIsNotNone(waypoint2)
self.assertEqual(waypoint2.title_en, 'Mont Blanc')
self.assertEqual(waypoint2.title_fr, '')
self.assertEqual(waypoint2.doc_type, 'w')
route = SearchRoute.get(id=71173)
self.assertIsNotNone(route)
self.assertEqual(route.title_en, 'Mont Blanc : Face N')
self.assertEqual(route.title_fr, '')
self.assertEqual(route.doc_type, 'r')
# merged document is ignored
self.assertIsNone(SearchWaypoint.get(id=71174, ignore=404))
# check that the sync. status was updated
last_update, _ = es_sync.get_status(self.session)
self.assertIsNotNone(last_update)
def _add_test_data(self):
_, date_now = es_sync.get_status(self.session)
es_sync.mark_as_updated(self.session, date_now)
self.waypoint1 = Waypoint(
document_id=71171,
waypoint_type='summit',
elevation=2000,
geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='fr',
title='Mont Granier',
description='...',
summary='Le Mont [b]Granier[/b]'),
WaypointLocale(lang='en',
title='Mont Granier',
description='...',
summary='The Mont Granier')
])
self.waypoint2 = Waypoint(
document_id=71172,
waypoint_type='summit',
elevation=4985,
geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='en',
title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
])
self.route1 = Route(document_id=71173,
activities=['skitouring'],
elevation_max=1500,
elevation_min=700,
height_diff_up=800,
height_diff_down=800,
durations='1',
locales=[
RouteLocale(lang='en',
title='Face N',
description='...',
gear='paraglider',
title_prefix='Mont Blanc')
])
self.waypoint3 = Waypoint(
document_id=71174,
redirects_to=71171,
waypoint_type='summit',
elevation=4985,
geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='en',
title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
])
self.outing1 = Outing(activities=['skitouring'],
date_start=datetime.date(2016, 1, 1),
date_end=datetime.date(2016, 1, 1),
locales=[
OutingLocale(lang='en',
title='...',
description='...',
weather='sunny')
])
self.session.add_all([
self.waypoint1, self.waypoint2, self.waypoint3, self.route1,
self.outing1
])
self.session.flush()
user_id = self.global_userids['contributor']
DocumentRest.create_new_version(self.waypoint1, user_id)
DocumentRest.create_new_version(self.waypoint2, user_id)
DocumentRest.create_new_version(self.waypoint3, user_id)
DocumentRest.create_new_version(self.route1, user_id)
DocumentRest.create_new_version(self.outing1, user_id)
association_wr = Association.create(self.waypoint1, self.route1)
association_ww = Association.create(self.waypoint2, self.waypoint1)
association_ro = Association.create(self.route1, self.outing1)
user = self.session.query(UserProfile).get(
self.global_userids['contributor'])
association_uo = Association.create(user, self.outing1)
self.session.add_all([
association_wr, association_ww, association_ro, association_uo,
association_wr.get_log(user_id),
association_ww.get_log(user_id),
association_ro.get_log(user_id),
association_uo.get_log(user_id)
])
self.session.flush()
def test_fill_index(self):
"""Tests that documents are inserted into the ElasticSearch index.
"""
self.session.add(
Waypoint(document_id=71171,
waypoint_type='summit',
elevation=2000,
quality='medium',
access_time='15min',
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='fr',
title='Mont Granier',
description='...',
summary='Le Mont [b]Granier[/b]'),
WaypointLocale(lang='en',
title='Mont Granier',
description='...',
summary='The Mont Granier')
]))
self.session.add(
Waypoint(document_id=71172,
waypoint_type='summit',
elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='en',
title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
]))
self.session.add(
Route(document_id=71173,
activities=['skitouring'],
elevation_max=1500,
elevation_min=700,
height_diff_up=800,
height_diff_down=800,
durations=['1'],
locales=[
RouteLocale(lang='en',
title='Face N',
description='...',
gear='paraglider',
title_prefix='Mont Blanc')
]))
self.session.add(
Waypoint(document_id=71174,
redirects_to=71171,
waypoint_type='summit',
elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='en',
title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
]))
self.session.add(
Outing(document_id=71175,
activities=['skitouring'],
date_start=datetime.date(2016, 1, 1),
date_end=datetime.date(2016, 1, 1),
frequentation='overcrowded',
locales=[
OutingLocale(lang='en',
title='Mont Blanc : Face N !',
description='...',
weather='sunny')
]))
self.session.add(
Article(document_id=71176,
categories=['site_info'],
activities=['hiking'],
article_type='collab',
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
DocumentLocale(lang='en',
title='Lac d\'Annecy',
description='...',
summary=''),
DocumentLocale(lang='fr',
title='Lac d\'Annecy',
description='...',
summary='')
]))
self.session.flush()
self.session.add(
Book(document_id=71177,
activities=['hiking'],
book_types=['biography'],
author='Denis Dainat',
publication_date='',
locales=[
DocumentLocale(lang='fr',
title='Escalades au Thaurac',
description='...',
summary=''),
DocumentLocale(lang='en',
title='Escalades au Thaurac',
description='...',
summary='')
]))
self.session.flush()
self.session.add(
Xreport(document_id=71178,
event_type='person_fall',
event_activity='skitouring',
nb_participants=10,
elevation=1500,
date=datetime.date(2016, 1, 1),
locales=[
XreportLocale(lang='en',
title='Death in the mountains',
description='...',
place='some place description')
]))
self.session.flush()
# fill the ElasticSearch index
fill_index(self.session)
waypoint1 = SearchWaypoint.get(id=71171)
self.assertIsNotNone(waypoint1)
self.assertEqual(waypoint1.title_en, 'Mont Granier')
self.assertEqual(waypoint1.title_fr, 'Mont Granier')
# self.assertEqual(waypoint1.summary_fr, 'Le Mont Granier ')
self.assertEqual(waypoint1.doc_type, 'w')
self.assertEqual(waypoint1.quality, 2)
self.assertEqual(waypoint1.access_time, 3)
self.assertAlmostEqual(waypoint1.geom[0], 5.71288994)
self.assertAlmostEqual(waypoint1.geom[1], 45.64476395)
waypoint2 = SearchWaypoint.get(id=71172)
self.assertIsNotNone(waypoint2)
self.assertEqual(waypoint2.title_en, 'Mont Blanc')
self.assertIsNone(waypoint2.title_fr)
self.assertEqual(waypoint2.doc_type, 'w')
route = SearchRoute.get(id=71173)
self.assertIsNotNone(route)
self.assertEqual(route.title_en, 'Mont Blanc : Face N')
self.assertIsNone(route.title_fr)
self.assertEqual(route.doc_type, 'r')
self.assertEqual(route.durations, [0])
outing = SearchOuting.get(id=71175)
self.assertIsNotNone(outing)
self.assertEqual(outing.title_en, 'Mont Blanc : Face N !')
self.assertIsNone(outing.title_fr)
self.assertEqual(outing.doc_type, 'o')
self.assertEqual(outing.frequentation, 3)
article = SearchArticle.get(id=71176)
self.assertIsNotNone(article)
self.assertEqual(article.title_en, 'Lac d\'Annecy')
self.assertEqual(article.title_fr, 'Lac d\'Annecy')
self.assertEqual(article.doc_type, 'c')
book = SearchBook.get(id=71177)
self.assertIsNotNone(book)
self.assertEqual(book.title_en, 'Escalades au Thaurac')
self.assertEqual(book.title_fr, 'Escalades au Thaurac')
self.assertEqual(book.doc_type, 'b')
self.assertEqual(book.book_types, ['biography'])
xreport = SearchXreport.get(id=71178)
self.assertIsNotNone(xreport)
self.assertEqual(xreport.title_en, 'Death in the mountains')
self.assertEqual(xreport.doc_type, 'x')
# merged document is ignored
self.assertIsNone(SearchWaypoint.get(id=71174, ignore=404))
# check that the sync. status was updated
last_update, _ = es_sync.get_status(self.session)
self.assertIsNotNone(last_update)
def test_mark_as_updated(self):
_, date_now = get_status(self.session)
mark_as_updated(self.session, date_now)
last_update, _ = get_status(self.session)
self.assertEqual(last_update, date_now)
def test_get_status(self):
last_update, date_now = get_status(self.session)
self.assertIsNotNone(last_update)
self.assertIsNotNone(date_now)
def test_get_status(self):
last_update, date_now = get_status(self.session)
self.assertIsNotNone(last_update)
self.assertIsNotNone(date_now)
def test_fill_index(self):
"""Tests that documents are inserted into the ElasticSearch index.
"""
self.session.add(Waypoint(
document_id=71171,
waypoint_type='summit', elevation=2000,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='fr', title='Mont Granier',
description='...',
summary='Le Mont [b]Granier[/b]'),
WaypointLocale(
lang='en', title='Mont Granier',
description='...',
summary='The Mont Granier')
]))
self.session.add(Waypoint(
document_id=71172,
waypoint_type='summit', elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='en', title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
]))
self.session.add(Route(
document_id=71173,
activities=['skitouring'], elevation_max=1500, elevation_min=700,
height_diff_up=800, height_diff_down=800, durations='1',
locales=[
RouteLocale(
lang='en', title='Face N',
description='...', gear='paraglider',
title_prefix='Mont Blanc'
)
]
))
self.session.add(Waypoint(
document_id=71174,
redirects_to=71171,
waypoint_type='summit', elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='en', title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
]))
self.session.flush()
# fill the ElasticSearch index
fill_index(self.session)
waypoint1 = SearchWaypoint.get(id=71171)
self.assertIsNotNone(waypoint1)
self.assertEqual(waypoint1.title_en, 'Mont Granier')
self.assertEqual(waypoint1.title_fr, 'Mont Granier')
self.assertEqual(waypoint1.summary_fr, 'Le Mont Granier ')
self.assertEqual(waypoint1.doc_type, 'w')
self.assertAlmostEqual(waypoint1.geom[0], 5.71288994)
self.assertAlmostEqual(waypoint1.geom[1], 45.64476395)
waypoint2 = SearchWaypoint.get(id=71172)
self.assertIsNotNone(waypoint2)
self.assertEqual(waypoint2.title_en, 'Mont Blanc')
self.assertEqual(waypoint2.title_fr, '')
self.assertEqual(waypoint2.doc_type, 'w')
route = SearchRoute.get(id=71173)
self.assertIsNotNone(route)
self.assertEqual(route.title_en, 'Mont Blanc : Face N')
self.assertEqual(route.title_fr, '')
self.assertEqual(route.doc_type, 'r')
# merged document is ignored
self.assertIsNone(SearchWaypoint.get(id=71174, ignore=404))
# check that the sync. status was updated
last_update, _ = es_sync.get_status(self.session)
self.assertIsNotNone(last_update)
def _add_test_data(self):
_, date_now = es_sync.get_status(self.session)
es_sync.mark_as_updated(self.session, date_now)
self.waypoint1 = Waypoint(
document_id=71171,
waypoint_type='summit',
elevation=2000,
geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='fr',
title='Mont Granier',
description='...',
summary='Le Mont [b]Granier[/b]'),
WaypointLocale(lang='en',
title='Mont Granier',
description='...',
summary='The Mont Granier')
])
self.waypoint2 = Waypoint(
document_id=71172,
waypoint_type='summit',
elevation=4985,
geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='en',
title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
])
self.route1 = Route(document_id=71173,
activities=['skitouring'],
elevation_max=1500,
elevation_min=700,
height_diff_up=800,
height_diff_down=800,
durations='1',
locales=[
RouteLocale(lang='en',
title='Face N',
description='...',
gear='paraglider',
title_prefix='Mont Blanc')
])
self.waypoint3 = Waypoint(
document_id=71174,
redirects_to=71171,
waypoint_type='summit',
elevation=4985,
geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(lang='en',
title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
])
self.session.add_all(
[self.waypoint1, self.waypoint2, self.waypoint3, self.route1])
self.session.flush()
user_id = self.global_userids['contributor']
DocumentRest.create_new_version(self.waypoint1, user_id)
DocumentRest.create_new_version(self.waypoint2, user_id)
DocumentRest.create_new_version(self.waypoint3, user_id)
DocumentRest.create_new_version(self.route1, user_id)
def _add_test_data(self):
_, date_now = es_sync.get_status(self.session)
es_sync.mark_as_updated(self.session, date_now)
self.waypoint1 = Waypoint(
document_id=71171,
waypoint_type='summit', elevation=2000,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='fr', title='Mont Granier',
description='...',
summary='Le Mont [b]Granier[/b]'),
WaypointLocale(
lang='en', title='Mont Granier',
description='...',
summary='The Mont Granier')
])
self.waypoint2 = Waypoint(
document_id=71172,
waypoint_type='summit', elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='en', title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
])
self.route1 = Route(
document_id=71173,
activities=['skitouring'], elevation_max=1500, elevation_min=700,
height_diff_up=800, height_diff_down=800, durations='1',
locales=[
RouteLocale(
lang='en', title='Face N',
description='...', gear='paraglider',
title_prefix='Mont Blanc'
)
]
)
self.waypoint3 = Waypoint(
document_id=71174,
redirects_to=71171,
waypoint_type='summit', elevation=4985,
geometry=DocumentGeometry(
geom='SRID=3857;POINT(635956 5723604)'),
locales=[
WaypointLocale(
lang='en', title='Mont Blanc',
description='...',
summary='The heighest point in Europe')
])
self.outing1 = Outing(
activities=['skitouring'], date_start=datetime.date(2016, 1, 1),
date_end=datetime.date(2016, 1, 1),
locales=[
OutingLocale(
lang='en', title='...', description='...',
weather='sunny')
]
)
self.session.add_all([
self.waypoint1, self.waypoint2, self.waypoint3, self.route1,
self.outing1
])
self.session.flush()
user_id = self.global_userids['contributor']
DocumentRest.create_new_version(self.waypoint1, user_id)
DocumentRest.create_new_version(self.waypoint2, user_id)
DocumentRest.create_new_version(self.waypoint3, user_id)
DocumentRest.create_new_version(self.route1, user_id)
DocumentRest.create_new_version(self.outing1, user_id)
association_wr = Association.create(self.waypoint1, self.route1)
association_ww = Association.create(self.waypoint2, self.waypoint1)
association_ro = Association.create(self.route1, self.outing1)
user = self.session.query(UserProfile).get(
self.global_userids['contributor'])
association_uo = Association.create(user, self.outing1)
self.session.add_all([
association_wr,
association_ww,
association_ro,
association_uo,
association_wr.get_log(user_id),
association_ww.get_log(user_id),
association_ro.get_log(user_id),
association_uo.get_log(user_id)
])
self.session.flush()
def test_mark_as_updated(self):
_, date_now = get_status(self.session)
mark_as_updated(self.session, date_now)
last_update, _ = get_status(self.session)
self.assertEqual(last_update, date_now)