Beispiel #1
0
    def _create_dummy_routes(self, climbing_sites):
        route_batch = DocumentBatch(
            self.session_target, self.batch_size,
            Route, ArchiveRoute,
            DocumentGeometry, ArchiveDocumentGeometry)
        route_locale_batch = DocumentBatch(
            self.session_target, self.batch_size,
            RouteLocale, ArchiveRouteLocale,
            None, None)
        history_batch = SimpleBatch(
            self.session_target, self.batch_size, HistoryMetaData)
        version_batch = SimpleBatch(
            self.session_target, self.batch_size, DocumentVersion)

        with route_batch:
            links_for_climbing_site, route_locales, route_locale_archives, \
                history_entries, version_entries = \
                self._create_routes(climbing_sites, route_batch)

        with route_locale_batch:
            for route_locale in route_locales:
                route_locale_batch.add_document(route_locale)
            for route_locale_archive in route_locale_archives:
                route_locale_batch.add_archive_documents(
                    [route_locale_archive])

        with history_batch:
            for entry in history_entries:
                history_batch.add(entry)

        with version_batch:
            for entry in version_entries:
                version_batch.add(entry)

        return links_for_climbing_site
Beispiel #2
0
    def _create_dummy_routes(self, climbing_sites):
        route_batch = DocumentBatch(self.session_target, self.batch_size,
                                    Route, ArchiveRoute, DocumentGeometry,
                                    ArchiveDocumentGeometry)
        route_locale_batch = DocumentBatch(self.session_target,
                                           self.batch_size, RouteLocale,
                                           ArchiveRouteLocale, None, None)
        history_batch = SimpleBatch(self.session_target, self.batch_size,
                                    HistoryMetaData)
        version_batch = SimpleBatch(self.session_target, self.batch_size,
                                    DocumentVersion)

        with route_batch:
            links_for_climbing_site, route_locales, route_locale_archives, \
                history_entries, version_entries = \
                self._create_routes(climbing_sites, route_batch)

        with route_locale_batch:
            for route_locale in route_locales:
                route_locale_batch.add_document(route_locale)
            for route_locale_archive in route_locale_archives:
                route_locale_batch.add_archive_documents(
                    [route_locale_archive])

        with history_batch:
            for entry in history_entries:
                history_batch.add(entry)

        with version_batch:
            for entry in version_entries:
                version_batch.add(entry)

        return links_for_climbing_site
Beispiel #3
0
    def _migrate(self, locales=False):
        self.start('{0}{1}'.format('locales of ' if locales else '',
                                   self.get_name()))

        query_count = self.get_count_query_locales() if locales else \
            self.get_count_query()
        total_count = self.connection_source.execute(
            text(query_count)).fetchone()[0]

        print('Total: {0} rows'.format(total_count))

        query = text(self.get_query_locales() if locales else self.get_query())

        model_document = self.get_model_document(locales)
        # make sure that the version is not managed by SQLAlchemy, the value
        # that is provided as version should be used
        document_mapper = class_mapper(model_document)
        document_mapper.version_id_prop = None
        document_mapper.version_id_col = None
        document_mapper.version_id_generator = None

        if not locales:
            document_mapper = class_mapper(DocumentGeometry)
            document_mapper.version_id_prop = None
            document_mapper.version_id_col = None
            document_mapper.version_id_generator = None

        batch = DocumentBatch(self.session_target, self.batch_size,
                              model_document,
                              self.get_model_archive_document(locales),
                              self.get_model_geometry(),
                              self.get_model_archive_geometry())
        with transaction.manager, batch:
            count = 0
            current_document_id = None
            current_locale = None
            version = 1
            archives = []
            geometry_archives = []

            for document_in in self.connection_source.execute(query):
                count += 1
                if current_document_id is None:
                    current_document_id = document_in.id
                    version = 1
                    if locales:
                        current_locale = document_in.culture
                else:
                    if current_document_id != document_in.id:
                        print('WARNING: no latest version for {0}'.format(
                            current_document_id))
                        archives = []
                        geometry_archives = []
                        version = 1
                        current_document_id = document_in.id
                        if locales:
                            current_locale = document_in.culture

                if locales:
                    document_archive = self.get_document_locale_archive(
                        document_in, version)
                else:
                    document_archive = self.get_document_archive(
                        document_in, version)
                    # do not migrate any empty geom for non geom types
                    geometry_archive = self.get_document_geometry_archive(
                        document_in, version)
                    if geometry_archive is not None:
                        geometry_archives.append(geometry_archive)
                archives.append(document_archive)

                if document_in.is_latest_version:
                    if locales:
                        if current_locale != document_in.culture:
                            raise Exception(
                                'locale of the latest version does not match '
                                'locale of the first version {0}'.format(
                                    current_document_id))
                        document = self.get_document_locale(
                            document_in, version)
                    else:
                        document = self.get_document(document_in, version)
                        geometry = self.get_document_geometry(
                            document_in, version)
                        # do not migrate any empty geom for non geom types
                        if geometry is not None:
                            batch.add_geometry(geometry)
                            batch.add_geometry_archives(geometry_archives)
                    batch.add_archive_documents(archives)
                    batch.add_document(document)

                    archives = []
                    geometry_archives = []
                    version = 1
                    current_document_id = None
                    current_locale = None
                else:
                    version += 1
                self.progress(count, total_count)

            # the transaction will not be commited automatically when doing
            # a bulk insertion. `mark_changed` forces a commit.
            zope.sqlalchemy.mark_changed(self.session_target)
        self.stop()
Beispiel #4
0
    def _migrate(self, locales=False):
        self.start('{0}{1}'.format('locales of ' if locales else '',
                                   self.get_name()))

        query_count = self.get_count_query_locales() if locales else \
            self.get_count_query()
        total_count = self.connection_source.execute(
            text(query_count)).fetchone()[0]

        print('Total: {0} rows'.format(total_count))

        query = text(self.get_query_locales() if locales else self.get_query())
        batch = DocumentBatch(self.session_target, self.batch_size,
                              self.get_model_document(locales),
                              self.get_model_archive_document(locales),
                              self.get_model_geometry(),
                              self.get_model_archive_geometry())
        with transaction.manager, batch:
            count = 0
            current_document_id = None
            version = 1
            archives = []
            geometry_archives = []

            for document_in in self.connection_source.execute(query):
                count += 1
                if current_document_id is None:
                    current_document_id = document_in.id
                else:
                    if current_document_id != document_in.id:
                        print('WARNING: no latest version for {0}'.format(
                            current_document_id))
                        archives = []
                        geometry_archives = []
                        version = 1
                        current_document_id = document_in.id

                if locales:
                    document_archive = self.get_document_locale_archive(
                        document_in, version)
                else:
                    document_archive = self.get_document_archive(
                        document_in, version)
                    geometry_archive = self.get_document_geometry_archive(
                        document_in, version)
                    geometry_archives.append(geometry_archive)
                archives.append(document_archive)

                if document_in.is_latest_version:
                    version = 1
                    current_document_id = None

                    if locales:
                        document = self.get_document_locale(
                            document_in, version)
                    else:
                        document = self.get_document(document_in, version)
                        geometry = self.get_document_geometry(
                            document_in, version)
                        batch.add_geometry(geometry)
                    batch.add_geometry_archives(geometry_archives)
                    batch.add_archive_documents(archives)
                    batch.add_document(document)
                    archives = []
                    geometry_archives = []
                else:
                    version += 1
                self.progress(count, total_count)

            # the transaction will not be commited automatically when doing
            # a bulk insertion. `mark_changed` forces a commit.
            zope.sqlalchemy.mark_changed(self.session_target)
        self.stop()
Beispiel #5
0
    def _migrate(self, locales=False):
        self.start('{0}{1}'.format(
            'locales of ' if locales else '',
            self.get_name()))

        query_count = self.get_count_query_locales() if locales else \
            self.get_count_query()
        total_count = self.connection_source.execute(
            text(query_count)).fetchone()[0]

        print('Total: {0} rows'.format(total_count))

        query = text(self.get_query_locales() if locales else self.get_query())
        batch = DocumentBatch(
            self.session_target, self.batch_size,
            self.get_model_document(locales),
            self.get_model_archive_document(locales),
            self.get_model_geometry(),
            self.get_model_archive_geometry())
        with transaction.manager, batch:
            count = 0
            current_document_id = None
            version = 1
            archives = []
            geometry_archives = []

            for document_in in self.connection_source.execute(query):
                count += 1
                if current_document_id is None:
                    current_document_id = document_in.id
                else:
                    if current_document_id != document_in.id:
                        print('WARNING: no latest version for {0}'.format(
                                current_document_id))
                        archives = []
                        geometry_archives = []
                        version = 1
                        current_document_id = document_in.id

                if locales:
                    document_archive = self.get_document_locale_archive(
                        document_in, version)
                else:
                    document_archive = self.get_document_archive(
                        document_in, version)
                    geometry_archive = self.get_document_geometry_archive(
                            document_in, version)
                    geometry_archives.append(geometry_archive)
                archives.append(document_archive)

                if document_in.is_latest_version:
                    version = 1
                    current_document_id = None

                    if locales:
                        document = self.get_document_locale(
                            document_in, version)
                    else:
                        document = self.get_document(
                            document_in, version)
                        geometry = self.get_document_geometry(
                                document_in, version)
                        batch.add_geometry(geometry)
                    batch.add_geometry_archives(geometry_archives)
                    batch.add_archive_documents(archives)
                    batch.add_document(document)
                    archives = []
                    geometry_archives = []
                else:
                    version += 1
                self.progress(count, total_count)

            # the transaction will not be commited automatically when doing
            # a bulk insertion. `mark_changed` forces a commit.
            zope.sqlalchemy.mark_changed(self.session_target)
        self.stop()
Beispiel #6
0
    def _migrate(self, locales=False):
        self.start('{0}{1}'.format(
            'locales of ' if locales else '',
            self.get_name()))

        query_count = self.get_count_query_locales() if locales else \
            self.get_count_query()
        total_count = self.connection_source.execute(
            text(query_count)).fetchone()[0]

        print('Total: {0} rows'.format(total_count))

        query = text(self.get_query_locales() if locales else self.get_query())

        model_document = self.get_model_document(locales)
        # make sure that the version is not managed by SQLAlchemy, the value
        # that is provided as version should be used
        document_mapper = class_mapper(model_document)
        document_mapper.version_id_prop = None
        document_mapper.version_id_col = None
        document_mapper.version_id_generator = None

        if not locales:
            document_mapper = class_mapper(DocumentGeometry)
            document_mapper.version_id_prop = None
            document_mapper.version_id_col = None
            document_mapper.version_id_generator = None

        batch = DocumentBatch(
            self.session_target, self.batch_size,
            model_document,
            self.get_model_archive_document(locales),
            self.get_model_geometry(),
            self.get_model_archive_geometry())
        with transaction.manager, batch:
            count = 0
            current_document_id = None
            current_locale = None
            version = 1
            archives = []
            geometry_archives = []

            for document_in in self.connection_source.execute(query):
                count += 1
                if current_document_id is None:
                    current_document_id = document_in.id
                    version = 1
                    if locales:
                        current_locale = document_in.culture
                else:
                    if current_document_id != document_in.id:
                        print('WARNING: no latest version for {0}'.format(
                                current_document_id))
                        archives = []
                        geometry_archives = []
                        version = 1
                        current_document_id = document_in.id
                        if locales:
                            current_locale = document_in.culture

                if locales:
                    document_archive = self.get_document_locale_archive(
                        document_in, version)
                else:
                    document_archive = self.get_document_archive(
                        document_in, version)
                    # do not migrate any empty geom for non geom types
                    geometry_archive = self.get_document_geometry_archive(
                            document_in, version)
                    if geometry_archive is not None:
                        geometry_archives.append(geometry_archive)
                archives.append(document_archive)

                if document_in.is_latest_version:
                    if locales:
                        if current_locale != document_in.culture:
                            raise Exception(
                                'locale of the latest version does not match '
                                'locale of the first version {0}'.format(
                                    current_document_id))
                        document = self.get_document_locale(
                            document_in, version)
                    else:
                        document = self.get_document(
                            document_in, version)
                        geometry = self.get_document_geometry(
                                document_in, version)
                        # do not migrate any empty geom for non geom types
                        if geometry is not None:
                            batch.add_geometry(geometry)
                            batch.add_geometry_archives(geometry_archives)
                    batch.add_archive_documents(archives)
                    batch.add_document(document)

                    archives = []
                    geometry_archives = []
                    version = 1
                    current_document_id = None
                    current_locale = None
                else:
                    version += 1
                self.progress(count, total_count)

            # the transaction will not be commited automatically when doing
            # a bulk insertion. `mark_changed` forces a commit.
            zope.sqlalchemy.mark_changed(self.session_target)
        self.stop()