Example #1
0
def sync_es(session, batch_size=1000):
    last_update, date_now = es_sync.get_status(session)
    log.info('Last update time: {}'.format(last_update))

    if not last_update:
        raise Exception('No last update time, run `fill_index` to do an '
                        'initial import into ElasticSearch')

    # get all documents that have changed since the last update
    changed_documents = \
        get_changed_documents(session, last_update) + \
        get_changed_users(session, last_update) + \
        get_changed_documents_for_associations(session, last_update) + \
        get_deleted_locale_documents(session, last_update) + \
        get_tagged_documents(session, last_update)
    log.info('Number of changed documents: {}'.format(len(changed_documents)))
    if changed_documents:
        sync_documents(session, changed_documents, batch_size)

    # get list of documents deleted since the last update
    deleted_documents = get_deleted_documents(session, last_update)
    log.info('Number of deleted documents: {}'.format(len(deleted_documents)))
    if deleted_documents:
        sync_deleted_documents(session, deleted_documents, batch_size)

    es_sync.mark_as_updated(session, date_now)
    log.info('Sync has finished')
Example #2
0
def fill_index(session, batch_size=1000):
    client = elasticsearch_config['client']
    index_name = elasticsearch_config['index']

    status = {'start_time': datetime.now(), 'last_progress_update': None}

    _, date_now = es_sync.get_status(session)

    total = session.query(Document). \
        filter(Document.redirects_to.is_(None)).count()

    def progress(count, total_count):
        if status['last_progress_update'] is None or \
                status['last_progress_update'] + timedelta(seconds=1) < \
                datetime.now():
            print('{0} of {1}'.format(count, total_count))
            status['last_progress_update'] = datetime.now()

    batch = ElasticBatch(client, batch_size)
    count = 0
    with batch:
        for doc_type in document_types:
            print('Importing document type {}'.format(doc_type))
            to_search_document = search_documents[doc_type].to_search_document

            for doc in sync.get_documents(session, doc_type, batch_size):
                batch.add(to_search_document(doc, index_name))

                count += 1
                progress(count, total)

    es_sync.mark_as_updated(session, date_now)

    duration = datetime.now() - status['start_time']
    print('Done (duration: {0})'.format(duration))
Example #3
0
    def test_get_changed_documents(self):
        last_update, _ = es_sync.get_status(self.session)

        changed_documents = get_changed_documents(self.session, last_update)
        self.assertEqual(len(changed_documents), 4)
        self.assertEqual('r', changed_documents[0][1])
        self.assertEqual('w', changed_documents[1][1])

        self.assertEqual(
            len(get_changed_documents(
                self.session, last_update + datetime.timedelta(0, 1))),
            0)
Example #4
0
    def test_get_changed_documents(self):
        last_update, _ = es_sync.get_status(self.session)

        changed_documents = get_changed_documents(self.session, last_update)
        self.assertEqual(len(changed_documents), 4)
        self.assertEqual('r', changed_documents[0][1])
        self.assertEqual('w', changed_documents[1][1])

        self.assertEqual(
            len(
                get_changed_documents(self.session,
                                      last_update + datetime.timedelta(0, 1))),
            0)
Example #5
0
    def _add_database_status(self, status):
        last_es_syncer_run = None
        success = False

        try:
            last_es_syncer_run, _ = es_sync.get_status(DBSession)
            success = True
        except:
            log.exception('Getting last es syncer run failed')
            self.request.response.status_code = 500

        status['pg'] = 'ok' if success else 'error'
        status['last_es_syncer_run'] = last_es_syncer_run.isoformat() \
            if last_es_syncer_run else ''
Example #6
0
    def _add_database_status(self, status):
        last_es_syncer_run = None
        success = False

        try:
            last_es_syncer_run, _ = es_sync.get_status(DBSession)
            success = True
        except Exception:
            log.exception('Getting last es syncer run failed')
            self.request.response.status_code = 500

        status['pg'] = 'ok' if success else 'error'
        status['last_es_syncer_run'] = last_es_syncer_run.isoformat() \
            if last_es_syncer_run else ''
Example #7
0
    def test_get_deleted_locale_documents(self):
        # simulate removing a locale
        self.session.add(
            ESDeletedLocale(document_id=self.waypoint1.document_id,
                            type='w',
                            lang='fr'))
        self.session.flush()

        last_update, _ = es_sync.get_status(self.session)

        changed_documents = get_deleted_locale_documents(
            self.session, last_update)

        self.assertEqual(len(changed_documents), 1)
        self.assertEqual('w', changed_documents[0][1])
Example #8
0
    def test_get_changed_documents_for_associations(self):
        last_update, _ = es_sync.get_status(self.session)

        changed_documents = get_changed_documents_for_associations(
            self.session, last_update)
        self.assertEqual(len(changed_documents), 4)
        self.assertEqual(changed_documents[0][0], self.route1.document_id)
        self.assertEqual(changed_documents[1][0], self.outing1.document_id)
        self.assertEqual(changed_documents[2][0], self.route1.document_id)
        self.assertEqual(changed_documents[3][0], self.outing1.document_id)

        self.assertEqual(
            len(get_changed_documents_for_associations(
                self.session, last_update + datetime.timedelta(0, 1))),
            0)
Example #9
0
    def test_get_changed_documents_for_associations(self):
        last_update, _ = es_sync.get_status(self.session)

        changed_documents = get_changed_documents_for_associations(
            self.session, last_update)
        self.assertEqual(len(changed_documents), 4)
        self.assertEqual(changed_documents[0][0], self.route1.document_id)
        self.assertEqual(changed_documents[1][0], self.outing1.document_id)
        self.assertEqual(changed_documents[2][0], self.route1.document_id)
        self.assertEqual(changed_documents[3][0], self.outing1.document_id)

        self.assertEqual(
            len(
                get_changed_documents_for_associations(
                    self.session, last_update + datetime.timedelta(0, 1))), 0)
Example #10
0
def sync_es(session):
    last_update, date_now = es_sync.get_status(session)

    if not last_update:
        raise Exception('No last update time, run `fill_index` to do an '
                        'initial import into ElasticSearch')

    # get all documents that have changed since the last update
    # TODO also check changes to associations
    changed_documents = get_changed_documents(session, last_update) + \
        get_changed_users(session, last_update)

    if changed_documents:
        sync_documents(session, changed_documents)

    es_sync.mark_as_updated(session, date_now)
Example #11
0
    def test_get_changed_users(self):
        user_id = self.global_userids['contributor']
        user = self.session.query(User).get(user_id)
        user.name = 'changed'
        self.session.flush()

        last_update, _ = es_sync.get_status(self.session)

        changed_users = get_changed_users(self.session, last_update)
        self.assertEqual(len(changed_users), 1)
        self.assertEqual(user_id, changed_users[0][0])
        self.assertEqual('u', changed_users[0][1])

        self.assertEqual(
            len(get_changed_documents(
                self.session, last_update + datetime.timedelta(0, 1))),
            0)
Example #12
0
    def test_get_changed_users(self):
        user_id = self.global_userids['contributor']
        user = self.session.query(User).get(user_id)
        user.name = 'changed'
        self.session.flush()

        last_update, _ = es_sync.get_status(self.session)

        changed_users = get_changed_users(self.session, last_update)
        self.assertEqual(len(changed_users), 1)
        self.assertEqual(user_id, changed_users[0][0])
        self.assertEqual('u', changed_users[0][1])

        self.assertEqual(
            len(
                get_changed_documents(self.session,
                                      last_update + datetime.timedelta(0, 1))),
            0)
Example #13
0
def sync_es(session, batch_size=1000):
    last_update, date_now = es_sync.get_status(session)
    log.info('Last update time: {}'.format(last_update))

    if not last_update:
        raise Exception('No last update time, run `fill_index` to do an '
                        'initial import into ElasticSearch')

    # get all documents that have changed since the last update
    changed_documents = \
        get_changed_documents(session, last_update) + \
        get_changed_users(session, last_update) + \
        get_changed_documents_for_associations(session, last_update)

    log.info('Number of changed documents: {}'.format(len(changed_documents)))
    if changed_documents:
        sync_documents(session, changed_documents, batch_size)

    es_sync.mark_as_updated(session, date_now)
    log.info('Sync has finished')
Example #14
0
def fill_index(session):
    client = elasticsearch_config['client']
    index_name = elasticsearch_config['index']

    status = {
        'start_time': datetime.now(),
        'last_progress_update': None
    }

    _, date_now = es_sync.get_status(session)

    total = session.query(Document). \
        filter(Document.redirects_to.is_(None)).count()

    def progress(count, total_count):
        if status['last_progress_update'] is None or \
                status['last_progress_update'] + timedelta(seconds=1) < \
                datetime.now():
            print('{0} of {1}'.format(count, total_count))
            status['last_progress_update'] = datetime.now()

    batch = ElasticBatch(client, batch_size)
    count = 0
    with batch:
        for doc_type in document_types:
            print('Importing document type {}'.format(doc_type))
            to_search_document = search_documents[doc_type].to_search_document

            for doc in sync.get_documents(session, doc_type):
                batch.add(to_search_document(doc, index_name))

                count += 1
                progress(count, total)

    es_sync.mark_as_updated(session, date_now)

    duration = datetime.now() - status['start_time']
    print('Done (duration: {0})'.format(duration))
Example #15
0
    def _add_test_data(self):
        _, date_now = es_sync.get_status(self.session)
        es_sync.mark_as_updated(self.session, date_now)

        self.waypoint1 = Waypoint(
            document_id=71171,
            waypoint_type='summit', elevation=2000,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='fr', title='Mont Granier',
                    description='...',
                    summary='Le Mont [b]Granier[/b]'),
                WaypointLocale(
                    lang='en', title='Mont Granier',
                    description='...',
                    summary='The Mont Granier')
            ])
        self.waypoint2 = Waypoint(
            document_id=71172,
            waypoint_type='summit', elevation=4985,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='en', title='Mont Blanc',
                    description='...',
                    summary='The heighest point in Europe')
            ])
        self.route1 = Route(
            document_id=71173,
            activities=['skitouring'], elevation_max=1500, elevation_min=700,
            height_diff_up=800, height_diff_down=800, durations='1',
            locales=[
                RouteLocale(
                    lang='en', title='Face N',
                    description='...', gear='paraglider',
                    title_prefix='Mont Blanc'
                )
            ]
        )
        self.waypoint3 = Waypoint(
            document_id=71174,
            redirects_to=71171,
            waypoint_type='summit', elevation=4985,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='en', title='Mont Blanc',
                    description='...',
                    summary='The heighest point in Europe')
            ])
        self.session.add_all([
            self.waypoint1, self.waypoint2, self.waypoint3, self.route1])
        self.session.flush()

        user_id = self.global_userids['contributor']
        DocumentRest.create_new_version(self.waypoint1, user_id)
        DocumentRest.create_new_version(self.waypoint2, user_id)
        DocumentRest.create_new_version(self.waypoint3, user_id)
        DocumentRest.create_new_version(self.route1, user_id)
Example #16
0
    def test_fill_index(self):
        """Tests that documents are inserted into the ElasticSearch index.
        """
        self.session.add(Waypoint(
            document_id=71171,
            waypoint_type='summit', elevation=2000, quality='medium',
            access_time='15min',
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='fr', title='Mont Granier',
                    description='...',
                    summary='Le Mont [b]Granier[/b]'),
                WaypointLocale(
                    lang='en', title='Mont Granier',
                    description='...',
                    summary='The Mont Granier')
            ]))
        self.session.add(Waypoint(
            document_id=71172,
            waypoint_type='summit', elevation=4985,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='en', title='Mont Blanc',
                    description='...',
                    summary='The heighest point in Europe')
            ]))
        self.session.add(Route(
            document_id=71173,
            activities=['skitouring'], elevation_max=1500, elevation_min=700,
            height_diff_up=800, height_diff_down=800, durations=['1'],
            locales=[
                RouteLocale(
                    lang='en', title='Face N',
                    description='...', gear='paraglider',
                    title_prefix='Mont Blanc'
                )
            ]
        ))
        self.session.add(Waypoint(
            document_id=71174,
            redirects_to=71171,
            waypoint_type='summit', elevation=4985,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='en', title='Mont Blanc',
                    description='...',
                    summary='The heighest point in Europe')
            ]))
        self.session.add(Outing(
            document_id=71175,
            activities=['skitouring'], date_start=datetime.date(2016, 1, 1),
            date_end=datetime.date(2016, 1, 1), frequentation='overcrowded',
            locales=[
                OutingLocale(
                    lang='en', title='Mont Blanc : Face N !',
                    description='...', weather='sunny')
            ]
        ))
        self.session.add(Article(
            document_id=71176,
            categories=['site_info'], activities=['hiking'],
            article_type='collab',
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                DocumentLocale(
                    lang='en', title='Lac d\'Annecy',
                    description='...',
                    summary=''),
                DocumentLocale(
                    lang='fr', title='Lac d\'Annecy',
                    description='...',
                    summary='')
            ]))
        self.session.flush()

        self.session.add(Book(
            document_id=71177,
            activities=['hiking'],
            book_types=['biography'],
            author='Denis Dainat',
            publication_date='',
            locales=[
                DocumentLocale(
                    lang='fr', title='Escalades au Thaurac',
                    description='...',
                    summary=''),
                DocumentLocale(
                    lang='en', title='Escalades au Thaurac',
                    description='...',
                    summary=''
                )
            ]
        ))
        self.session.flush()

        self.session.add(Xreport(
            document_id=71178,
            event_type=['roped_fall'],
            activities=['skitouring'],
            nb_participants=10,
            elevation=1500,
            date=datetime.date(2016, 1, 1),
            locales=[
                XreportLocale(
                    lang='en', title='Death in the mountains',
                    description='...',
                    place='some place description'
                )
            ]
        ))
        self.session.flush()

        # fill the ElasticSearch index
        fill_index(self.session)

        waypoint1 = SearchWaypoint.get(id=71171)
        self.assertIsNotNone(waypoint1)
        self.assertEqual(waypoint1.title_en, 'Mont Granier')
        self.assertEqual(waypoint1.title_fr, 'Mont Granier')
        # self.assertEqual(waypoint1.summary_fr, 'Le Mont  Granier ')
        self.assertEqual(waypoint1.doc_type, 'w')
        self.assertEqual(waypoint1.quality, 2)
        self.assertEqual(waypoint1.access_time, 3)
        self.assertAlmostEqual(waypoint1.geom[0], 5.71288994)
        self.assertAlmostEqual(waypoint1.geom[1], 45.64476395)

        waypoint2 = SearchWaypoint.get(id=71172)
        self.assertIsNotNone(waypoint2)
        self.assertEqual(waypoint2.title_en, 'Mont Blanc')
        self.assertEqual(waypoint2.title_fr, '')
        self.assertEqual(waypoint2.doc_type, 'w')

        route = SearchRoute.get(id=71173)
        self.assertIsNotNone(route)
        self.assertEqual(route.title_en, 'Mont Blanc : Face N')
        self.assertEqual(route.title_fr, '')
        self.assertEqual(route.doc_type, 'r')
        self.assertEqual(route.durations, [0])

        outing = SearchOuting.get(id=71175)
        self.assertIsNotNone(outing)
        self.assertEqual(outing.title_en, 'Mont Blanc : Face N !')
        self.assertEqual(outing.title_fr, '')
        self.assertEqual(outing.doc_type, 'o')
        self.assertEqual(outing.frequentation, 3)

        article = SearchArticle.get(id=71176)
        self.assertIsNotNone(article)
        self.assertEqual(article.title_en, 'Lac d\'Annecy')
        self.assertEqual(article.title_fr, 'Lac d\'Annecy')
        self.assertEqual(article.doc_type, 'c')

        book = SearchBook.get(id=71177)
        self.assertIsNotNone(book)
        self.assertEqual(book.title_en, 'Escalades au Thaurac')
        self.assertEqual(book.title_fr, 'Escalades au Thaurac')
        self.assertEqual(book.doc_type, 'b')
        self.assertEqual(book.book_types, ['biography'])

        xreport = SearchXreport.get(id=71178)
        self.assertIsNotNone(xreport)
        self.assertEqual(xreport.title_en, 'Death in the mountains')
        self.assertEqual(xreport.doc_type, 'x')

        # merged document is ignored
        self.assertIsNone(SearchWaypoint.get(id=71174, ignore=404))

        # check that the sync. status was updated
        last_update, _ = es_sync.get_status(self.session)
        self.assertIsNotNone(last_update)
Example #17
0
    def test_fill_index(self):
        """Tests that documents are inserted into the ElasticSearch index.
        """
        self.session.add(
            Waypoint(document_id=71171,
                     waypoint_type='summit',
                     elevation=2000,
                     geometry=DocumentGeometry(
                         geom='SRID=3857;POINT(635956 5723604)'),
                     locales=[
                         WaypointLocale(lang='fr',
                                        title='Mont Granier',
                                        description='...',
                                        summary='Le Mont [b]Granier[/b]'),
                         WaypointLocale(lang='en',
                                        title='Mont Granier',
                                        description='...',
                                        summary='The Mont Granier')
                     ]))
        self.session.add(
            Waypoint(document_id=71172,
                     waypoint_type='summit',
                     elevation=4985,
                     geometry=DocumentGeometry(
                         geom='SRID=3857;POINT(635956 5723604)'),
                     locales=[
                         WaypointLocale(lang='en',
                                        title='Mont Blanc',
                                        description='...',
                                        summary='The heighest point in Europe')
                     ]))
        self.session.add(
            Route(document_id=71173,
                  activities=['skitouring'],
                  elevation_max=1500,
                  elevation_min=700,
                  height_diff_up=800,
                  height_diff_down=800,
                  durations='1',
                  locales=[
                      RouteLocale(lang='en',
                                  title='Face N',
                                  description='...',
                                  gear='paraglider',
                                  title_prefix='Mont Blanc')
                  ]))
        self.session.add(
            Waypoint(document_id=71174,
                     redirects_to=71171,
                     waypoint_type='summit',
                     elevation=4985,
                     geometry=DocumentGeometry(
                         geom='SRID=3857;POINT(635956 5723604)'),
                     locales=[
                         WaypointLocale(lang='en',
                                        title='Mont Blanc',
                                        description='...',
                                        summary='The heighest point in Europe')
                     ]))
        self.session.flush()

        # fill the ElasticSearch index
        fill_index(self.session)

        waypoint1 = SearchWaypoint.get(id=71171)
        self.assertIsNotNone(waypoint1)
        self.assertEqual(waypoint1.title_en, 'Mont Granier')
        self.assertEqual(waypoint1.title_fr, 'Mont Granier')
        self.assertEqual(waypoint1.summary_fr, 'Le Mont  Granier ')
        self.assertEqual(waypoint1.doc_type, 'w')
        self.assertAlmostEqual(waypoint1.geom[0], 5.71288994)
        self.assertAlmostEqual(waypoint1.geom[1], 45.64476395)

        waypoint2 = SearchWaypoint.get(id=71172)
        self.assertIsNotNone(waypoint2)
        self.assertEqual(waypoint2.title_en, 'Mont Blanc')
        self.assertEqual(waypoint2.title_fr, '')
        self.assertEqual(waypoint2.doc_type, 'w')

        route = SearchRoute.get(id=71173)
        self.assertIsNotNone(route)
        self.assertEqual(route.title_en, 'Mont Blanc : Face N')
        self.assertEqual(route.title_fr, '')
        self.assertEqual(route.doc_type, 'r')

        # merged document is ignored
        self.assertIsNone(SearchWaypoint.get(id=71174, ignore=404))

        # check that the sync. status was updated
        last_update, _ = es_sync.get_status(self.session)
        self.assertIsNotNone(last_update)
Example #18
0
    def _add_test_data(self):
        _, date_now = es_sync.get_status(self.session)
        es_sync.mark_as_updated(self.session, date_now)

        self.waypoint1 = Waypoint(
            document_id=71171,
            waypoint_type='summit',
            elevation=2000,
            geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(lang='fr',
                               title='Mont Granier',
                               description='...',
                               summary='Le Mont [b]Granier[/b]'),
                WaypointLocale(lang='en',
                               title='Mont Granier',
                               description='...',
                               summary='The Mont Granier')
            ])
        self.waypoint2 = Waypoint(
            document_id=71172,
            waypoint_type='summit',
            elevation=4985,
            geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(lang='en',
                               title='Mont Blanc',
                               description='...',
                               summary='The heighest point in Europe')
            ])
        self.route1 = Route(document_id=71173,
                            activities=['skitouring'],
                            elevation_max=1500,
                            elevation_min=700,
                            height_diff_up=800,
                            height_diff_down=800,
                            durations='1',
                            locales=[
                                RouteLocale(lang='en',
                                            title='Face N',
                                            description='...',
                                            gear='paraglider',
                                            title_prefix='Mont Blanc')
                            ])
        self.waypoint3 = Waypoint(
            document_id=71174,
            redirects_to=71171,
            waypoint_type='summit',
            elevation=4985,
            geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(lang='en',
                               title='Mont Blanc',
                               description='...',
                               summary='The heighest point in Europe')
            ])
        self.outing1 = Outing(activities=['skitouring'],
                              date_start=datetime.date(2016, 1, 1),
                              date_end=datetime.date(2016, 1, 1),
                              locales=[
                                  OutingLocale(lang='en',
                                               title='...',
                                               description='...',
                                               weather='sunny')
                              ])
        self.session.add_all([
            self.waypoint1, self.waypoint2, self.waypoint3, self.route1,
            self.outing1
        ])
        self.session.flush()

        user_id = self.global_userids['contributor']
        DocumentRest.create_new_version(self.waypoint1, user_id)
        DocumentRest.create_new_version(self.waypoint2, user_id)
        DocumentRest.create_new_version(self.waypoint3, user_id)
        DocumentRest.create_new_version(self.route1, user_id)
        DocumentRest.create_new_version(self.outing1, user_id)

        association_wr = Association.create(self.waypoint1, self.route1)
        association_ww = Association.create(self.waypoint2, self.waypoint1)
        association_ro = Association.create(self.route1, self.outing1)
        user = self.session.query(UserProfile).get(
            self.global_userids['contributor'])
        association_uo = Association.create(user, self.outing1)
        self.session.add_all([
            association_wr, association_ww, association_ro, association_uo,
            association_wr.get_log(user_id),
            association_ww.get_log(user_id),
            association_ro.get_log(user_id),
            association_uo.get_log(user_id)
        ])
        self.session.flush()
Example #19
0
    def test_fill_index(self):
        """Tests that documents are inserted into the ElasticSearch index.
        """
        self.session.add(
            Waypoint(document_id=71171,
                     waypoint_type='summit',
                     elevation=2000,
                     quality='medium',
                     access_time='15min',
                     geometry=DocumentGeometry(
                         geom='SRID=3857;POINT(635956 5723604)'),
                     locales=[
                         WaypointLocale(lang='fr',
                                        title='Mont Granier',
                                        description='...',
                                        summary='Le Mont [b]Granier[/b]'),
                         WaypointLocale(lang='en',
                                        title='Mont Granier',
                                        description='...',
                                        summary='The Mont Granier')
                     ]))
        self.session.add(
            Waypoint(document_id=71172,
                     waypoint_type='summit',
                     elevation=4985,
                     geometry=DocumentGeometry(
                         geom='SRID=3857;POINT(635956 5723604)'),
                     locales=[
                         WaypointLocale(lang='en',
                                        title='Mont Blanc',
                                        description='...',
                                        summary='The heighest point in Europe')
                     ]))
        self.session.add(
            Route(document_id=71173,
                  activities=['skitouring'],
                  elevation_max=1500,
                  elevation_min=700,
                  height_diff_up=800,
                  height_diff_down=800,
                  durations=['1'],
                  locales=[
                      RouteLocale(lang='en',
                                  title='Face N',
                                  description='...',
                                  gear='paraglider',
                                  title_prefix='Mont Blanc')
                  ]))
        self.session.add(
            Waypoint(document_id=71174,
                     redirects_to=71171,
                     waypoint_type='summit',
                     elevation=4985,
                     geometry=DocumentGeometry(
                         geom='SRID=3857;POINT(635956 5723604)'),
                     locales=[
                         WaypointLocale(lang='en',
                                        title='Mont Blanc',
                                        description='...',
                                        summary='The heighest point in Europe')
                     ]))
        self.session.add(
            Outing(document_id=71175,
                   activities=['skitouring'],
                   date_start=datetime.date(2016, 1, 1),
                   date_end=datetime.date(2016, 1, 1),
                   frequentation='overcrowded',
                   locales=[
                       OutingLocale(lang='en',
                                    title='Mont Blanc : Face N !',
                                    description='...',
                                    weather='sunny')
                   ]))
        self.session.add(
            Article(document_id=71176,
                    categories=['site_info'],
                    activities=['hiking'],
                    article_type='collab',
                    geometry=DocumentGeometry(
                        geom='SRID=3857;POINT(635956 5723604)'),
                    locales=[
                        DocumentLocale(lang='en',
                                       title='Lac d\'Annecy',
                                       description='...',
                                       summary=''),
                        DocumentLocale(lang='fr',
                                       title='Lac d\'Annecy',
                                       description='...',
                                       summary='')
                    ]))
        self.session.flush()

        self.session.add(
            Book(document_id=71177,
                 activities=['hiking'],
                 book_types=['biography'],
                 author='Denis Dainat',
                 publication_date='',
                 locales=[
                     DocumentLocale(lang='fr',
                                    title='Escalades au Thaurac',
                                    description='...',
                                    summary=''),
                     DocumentLocale(lang='en',
                                    title='Escalades au Thaurac',
                                    description='...',
                                    summary='')
                 ]))
        self.session.flush()

        self.session.add(
            Xreport(document_id=71178,
                    event_type='person_fall',
                    event_activity='skitouring',
                    nb_participants=10,
                    elevation=1500,
                    date=datetime.date(2016, 1, 1),
                    locales=[
                        XreportLocale(lang='en',
                                      title='Death in the mountains',
                                      description='...',
                                      place='some place description')
                    ]))
        self.session.flush()

        # fill the ElasticSearch index
        fill_index(self.session)

        waypoint1 = SearchWaypoint.get(id=71171)
        self.assertIsNotNone(waypoint1)
        self.assertEqual(waypoint1.title_en, 'Mont Granier')
        self.assertEqual(waypoint1.title_fr, 'Mont Granier')
        # self.assertEqual(waypoint1.summary_fr, 'Le Mont  Granier ')
        self.assertEqual(waypoint1.doc_type, 'w')
        self.assertEqual(waypoint1.quality, 2)
        self.assertEqual(waypoint1.access_time, 3)
        self.assertAlmostEqual(waypoint1.geom[0], 5.71288994)
        self.assertAlmostEqual(waypoint1.geom[1], 45.64476395)

        waypoint2 = SearchWaypoint.get(id=71172)
        self.assertIsNotNone(waypoint2)
        self.assertEqual(waypoint2.title_en, 'Mont Blanc')
        self.assertIsNone(waypoint2.title_fr)
        self.assertEqual(waypoint2.doc_type, 'w')

        route = SearchRoute.get(id=71173)
        self.assertIsNotNone(route)
        self.assertEqual(route.title_en, 'Mont Blanc : Face N')
        self.assertIsNone(route.title_fr)
        self.assertEqual(route.doc_type, 'r')
        self.assertEqual(route.durations, [0])

        outing = SearchOuting.get(id=71175)
        self.assertIsNotNone(outing)
        self.assertEqual(outing.title_en, 'Mont Blanc : Face N !')
        self.assertIsNone(outing.title_fr)
        self.assertEqual(outing.doc_type, 'o')
        self.assertEqual(outing.frequentation, 3)

        article = SearchArticle.get(id=71176)
        self.assertIsNotNone(article)
        self.assertEqual(article.title_en, 'Lac d\'Annecy')
        self.assertEqual(article.title_fr, 'Lac d\'Annecy')
        self.assertEqual(article.doc_type, 'c')

        book = SearchBook.get(id=71177)
        self.assertIsNotNone(book)
        self.assertEqual(book.title_en, 'Escalades au Thaurac')
        self.assertEqual(book.title_fr, 'Escalades au Thaurac')
        self.assertEqual(book.doc_type, 'b')
        self.assertEqual(book.book_types, ['biography'])

        xreport = SearchXreport.get(id=71178)
        self.assertIsNotNone(xreport)
        self.assertEqual(xreport.title_en, 'Death in the mountains')
        self.assertEqual(xreport.doc_type, 'x')

        # merged document is ignored
        self.assertIsNone(SearchWaypoint.get(id=71174, ignore=404))

        # check that the sync. status was updated
        last_update, _ = es_sync.get_status(self.session)
        self.assertIsNotNone(last_update)
Example #20
0
    def test_mark_as_updated(self):
        _, date_now = get_status(self.session)
        mark_as_updated(self.session, date_now)

        last_update, _ = get_status(self.session)
        self.assertEqual(last_update, date_now)
Example #21
0
 def test_get_status(self):
     last_update, date_now = get_status(self.session)
     self.assertIsNotNone(last_update)
     self.assertIsNotNone(date_now)
Example #22
0
 def test_get_status(self):
     last_update, date_now = get_status(self.session)
     self.assertIsNotNone(last_update)
     self.assertIsNotNone(date_now)
Example #23
0
    def test_fill_index(self):
        """Tests that documents are inserted into the ElasticSearch index.
        """
        self.session.add(Waypoint(
            document_id=71171,
            waypoint_type='summit', elevation=2000,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='fr', title='Mont Granier',
                    description='...',
                    summary='Le Mont [b]Granier[/b]'),
                WaypointLocale(
                    lang='en', title='Mont Granier',
                    description='...',
                    summary='The Mont Granier')
            ]))
        self.session.add(Waypoint(
            document_id=71172,
            waypoint_type='summit', elevation=4985,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='en', title='Mont Blanc',
                    description='...',
                    summary='The heighest point in Europe')
            ]))
        self.session.add(Route(
            document_id=71173,
            activities=['skitouring'], elevation_max=1500, elevation_min=700,
            height_diff_up=800, height_diff_down=800, durations='1',
            locales=[
                RouteLocale(
                    lang='en', title='Face N',
                    description='...', gear='paraglider',
                    title_prefix='Mont Blanc'
                )
            ]
        ))
        self.session.add(Waypoint(
            document_id=71174,
            redirects_to=71171,
            waypoint_type='summit', elevation=4985,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='en', title='Mont Blanc',
                    description='...',
                    summary='The heighest point in Europe')
            ]))
        self.session.flush()

        # fill the ElasticSearch index
        fill_index(self.session)

        waypoint1 = SearchWaypoint.get(id=71171)
        self.assertIsNotNone(waypoint1)
        self.assertEqual(waypoint1.title_en, 'Mont Granier')
        self.assertEqual(waypoint1.title_fr, 'Mont Granier')
        self.assertEqual(waypoint1.summary_fr, 'Le Mont  Granier ')
        self.assertEqual(waypoint1.doc_type, 'w')
        self.assertAlmostEqual(waypoint1.geom[0], 5.71288994)
        self.assertAlmostEqual(waypoint1.geom[1], 45.64476395)

        waypoint2 = SearchWaypoint.get(id=71172)
        self.assertIsNotNone(waypoint2)
        self.assertEqual(waypoint2.title_en, 'Mont Blanc')
        self.assertEqual(waypoint2.title_fr, '')
        self.assertEqual(waypoint2.doc_type, 'w')

        route = SearchRoute.get(id=71173)
        self.assertIsNotNone(route)
        self.assertEqual(route.title_en, 'Mont Blanc : Face N')
        self.assertEqual(route.title_fr, '')
        self.assertEqual(route.doc_type, 'r')

        # merged document is ignored
        self.assertIsNone(SearchWaypoint.get(id=71174, ignore=404))

        # check that the sync. status was updated
        last_update, _ = es_sync.get_status(self.session)
        self.assertIsNotNone(last_update)
Example #24
0
    def _add_test_data(self):
        _, date_now = es_sync.get_status(self.session)
        es_sync.mark_as_updated(self.session, date_now)

        self.waypoint1 = Waypoint(
            document_id=71171,
            waypoint_type='summit',
            elevation=2000,
            geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(lang='fr',
                               title='Mont Granier',
                               description='...',
                               summary='Le Mont [b]Granier[/b]'),
                WaypointLocale(lang='en',
                               title='Mont Granier',
                               description='...',
                               summary='The Mont Granier')
            ])
        self.waypoint2 = Waypoint(
            document_id=71172,
            waypoint_type='summit',
            elevation=4985,
            geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(lang='en',
                               title='Mont Blanc',
                               description='...',
                               summary='The heighest point in Europe')
            ])
        self.route1 = Route(document_id=71173,
                            activities=['skitouring'],
                            elevation_max=1500,
                            elevation_min=700,
                            height_diff_up=800,
                            height_diff_down=800,
                            durations='1',
                            locales=[
                                RouteLocale(lang='en',
                                            title='Face N',
                                            description='...',
                                            gear='paraglider',
                                            title_prefix='Mont Blanc')
                            ])
        self.waypoint3 = Waypoint(
            document_id=71174,
            redirects_to=71171,
            waypoint_type='summit',
            elevation=4985,
            geometry=DocumentGeometry(geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(lang='en',
                               title='Mont Blanc',
                               description='...',
                               summary='The heighest point in Europe')
            ])
        self.session.add_all(
            [self.waypoint1, self.waypoint2, self.waypoint3, self.route1])
        self.session.flush()

        user_id = self.global_userids['contributor']
        DocumentRest.create_new_version(self.waypoint1, user_id)
        DocumentRest.create_new_version(self.waypoint2, user_id)
        DocumentRest.create_new_version(self.waypoint3, user_id)
        DocumentRest.create_new_version(self.route1, user_id)
Example #25
0
    def _add_test_data(self):
        _, date_now = es_sync.get_status(self.session)
        es_sync.mark_as_updated(self.session, date_now)

        self.waypoint1 = Waypoint(
            document_id=71171,
            waypoint_type='summit', elevation=2000,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='fr', title='Mont Granier',
                    description='...',
                    summary='Le Mont [b]Granier[/b]'),
                WaypointLocale(
                    lang='en', title='Mont Granier',
                    description='...',
                    summary='The Mont Granier')
            ])
        self.waypoint2 = Waypoint(
            document_id=71172,
            waypoint_type='summit', elevation=4985,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='en', title='Mont Blanc',
                    description='...',
                    summary='The heighest point in Europe')
            ])
        self.route1 = Route(
            document_id=71173,
            activities=['skitouring'], elevation_max=1500, elevation_min=700,
            height_diff_up=800, height_diff_down=800, durations='1',
            locales=[
                RouteLocale(
                    lang='en', title='Face N',
                    description='...', gear='paraglider',
                    title_prefix='Mont Blanc'
                )
            ]
        )
        self.waypoint3 = Waypoint(
            document_id=71174,
            redirects_to=71171,
            waypoint_type='summit', elevation=4985,
            geometry=DocumentGeometry(
                geom='SRID=3857;POINT(635956 5723604)'),
            locales=[
                WaypointLocale(
                    lang='en', title='Mont Blanc',
                    description='...',
                    summary='The heighest point in Europe')
            ])
        self.outing1 = Outing(
            activities=['skitouring'], date_start=datetime.date(2016, 1, 1),
            date_end=datetime.date(2016, 1, 1),
            locales=[
                OutingLocale(
                    lang='en', title='...', description='...',
                    weather='sunny')
            ]
        )
        self.session.add_all([
            self.waypoint1, self.waypoint2, self.waypoint3, self.route1,
            self.outing1
        ])
        self.session.flush()

        user_id = self.global_userids['contributor']
        DocumentRest.create_new_version(self.waypoint1, user_id)
        DocumentRest.create_new_version(self.waypoint2, user_id)
        DocumentRest.create_new_version(self.waypoint3, user_id)
        DocumentRest.create_new_version(self.route1, user_id)
        DocumentRest.create_new_version(self.outing1, user_id)

        association_wr = Association.create(self.waypoint1, self.route1)
        association_ww = Association.create(self.waypoint2, self.waypoint1)
        association_ro = Association.create(self.route1, self.outing1)
        user = self.session.query(UserProfile).get(
            self.global_userids['contributor'])
        association_uo = Association.create(user, self.outing1)
        self.session.add_all([
            association_wr,
            association_ww,
            association_ro,
            association_uo,
            association_wr.get_log(user_id),
            association_ww.get_log(user_id),
            association_ro.get_log(user_id),
            association_uo.get_log(user_id)
        ])
        self.session.flush()
Example #26
0
    def test_mark_as_updated(self):
        _, date_now = get_status(self.session)
        mark_as_updated(self.session, date_now)

        last_update, _ = get_status(self.session)
        self.assertEqual(last_update, date_now)