Exemplo n.º 1
0
    def setup(self):
        """
        Create users and organizations
        """
        reset_db()
        search.clear()

        self.sysadmin = factories.Sysadmin()
        self.test_member_1 = factories.User()
        self.test_member_2 = factories.User()
        self.test_editor = factories.User()

        users = [{
            'name': self.test_member_1['name'],
            'capacity': 'member'
        }, {
            'name': self.test_member_2['name'],
            'capacity': 'member'
        }, {
            'name': self.test_editor['name'],
            'capacity': 'editor'
        }]

        self.test_org_1 = factories.Organization(users=users,
                                                 name="testjournal")
        self.test_org_2 = factories.Organization()
        self.test_package = self._dara_testpackage()
Exemplo n.º 2
0
def clear(dataset_name):
    from ckan.lib.search import clear, clear_all

    if dataset_name:
        clear(dataset_name)
    else:
        clear_all()
Exemplo n.º 3
0
 def setup_class(cls):
     search.clear()
     CreateTestData.create()
     cls.testsysadmin = model.User.by_name(u'testsysadmin')
     cls.comment = u'Comment umlaut: \xfc.'
     cls.user_name = u'annafan'  # created in CreateTestData
     cls.init_extra_environ(cls.user_name)
Exemplo n.º 4
0
    def setup_class(self):
        try:
            search.clear()
            setup_test_search_index()
            super(OnsLoaderBase, self).setup_class()

            # make annafan a sysadmin to allow package creation
            rev = model.repo.new_revision()
            user = model.User.by_name(u'annafan')
            model.add_user_to_role(user, model.Role.ADMIN, model.System())
            model.repo.commit_and_remove()

            publist = [g.name for g in model.Session.query(model.Group).all()]

            # create test publishers
            rev = model.repo.new_revision()
            for name, title in publishers.items():
                if not name in publist:
                    model.Session.add(model.Group(name=unicode(name), title=title, type='publisher'))
            model.repo.commit_and_remove()
        except Exception, e:
            # ensure that mock_drupal is destroyed
            print e
            MockDrupalCase.teardown_class()
            #model.repo.rebuild_db()
            raise
Exemplo n.º 5
0
    def command(self):
        self._load_config()
        from ckan.lib.search import rebuild, check, show, clear

        if not self.args:
            # default to printing help
            print self.usage
            return

        cmd = self.args[0]        
        if cmd == 'rebuild':
            if len(self.args) > 1:
                rebuild(self.args[1])
            else:
                rebuild()
        elif cmd == 'check':
            check()
        elif cmd == 'show':
            if not len(self.args) == 2:
                import pdb; pdb.set_trace()
                self.args
            show(self.args[1])
        elif cmd == 'clear':
            clear()
        else:
            print 'Command %s not recognized' % cmd
Exemplo n.º 6
0
    def setup_class(self):
        try:
            search.clear()
            setup_test_search_index()
            super(OnsLoaderBase, self).setup_class()

            # make annafan a sysadmin to allow package creation
            rev = model.repo.new_revision()
            user = model.User.by_name(u'annafan')
            model.add_user_to_role(user, model.Role.ADMIN, model.System())
            model.repo.commit_and_remove()

            publist = [g.name for g in model.Session.query(model.Group).all()]

            # create test publishers
            rev = model.repo.new_revision()
            for name, title in publishers.items():
                if not name in publist:
                    model.Session.add(model.Group(name=unicode(name), title=title, type='publisher'))
            model.repo.commit_and_remove()
        except Exception, e:
            # ensure that mock_drupal is destroyed
            print e
            MockDrupalCase.teardown_class()
            #model.repo.rebuild_db()
            raise
Exemplo n.º 7
0
def setup_test_search_index():
    from ckan import plugins

    if not is_search_supported():
        raise SkipTest("Search not supported")
    search.clear()
    plugins.load("synchronous_search")
Exemplo n.º 8
0
    def command(self):
        self._load_config()
        from ckan.lib.search import rebuild, check, show, clear

        if not self.args:
            # default to printing help
            print self.usage
            return

        cmd = self.args[0]
        if cmd == 'rebuild':
            if len(self.args) > 1:
                rebuild(self.args[1])
            else:
                rebuild()
        elif cmd == 'check':
            check()
        elif cmd == 'show':
            if not len(self.args) == 2:
                import pdb
                pdb.set_trace()
                self.args
            show(self.args[1])
        elif cmd == 'clear':
            clear()
        else:
            print 'Command %s not recognized' % cmd
Exemplo n.º 9
0
 def setup_class(cls):
     search.clear()
     CreateTestData.create()
     cls.testsysadmin = model.User.by_name(u"testsysadmin")
     cls.comment = u"Comment umlaut: \xfc."
     cls.user_name = u"annafan"  # created in CreateTestData
     cls.init_extra_environ(cls.user_name)
Exemplo n.º 10
0
def clear(dataset_name: str):
    from ckan.lib.search import clear, clear_all

    if dataset_name:
        clear(dataset_name)
    else:
        clear_all()
Exemplo n.º 11
0
    def setup(self):
        import ckan.model as model

        # Reset the db before each test method.
        model.repo.rebuild_db()

        # Clear the search index
        search.clear()
Exemplo n.º 12
0
 def setup_class(cls):
     search.clear()
     cls.tsi = TestSearchIndexer()
     DguCreateTestData.create_dgu_test_data()
     DguCreateTestData.create_arbitrary({'name': 'latest',
                                         'notes': '<b>Latest</b> dataset.',
                                         'groups': ['national-health-service']})
     cls.tsi.index()
Exemplo n.º 13
0
    def setup(self):
        import ckan.model as model

        # Reset the db before each test method.
        model.repo.rebuild_db()

        # Clear the search index
        search.clear()
Exemplo n.º 14
0
    def setup_class(cls):
#         p.load('metadata_fields')
#         TestMetadataFields.get_app()
        search.clear()
        model.Session.remove()
        ctd.CreateTestData.create()
        # Need to invalidate the caches to make sure the new data is being served
        caching.invalidate_group_caches()
Exemplo n.º 15
0
    def setup_class(self):
        search.clear()
        model.Session.remove()
        CreateTestData.create()

        # reduce extraneous logging
        from ckan.lib import activity_streams_session_extension
        activity_streams_session_extension.logger.level = 100
Exemplo n.º 16
0
 def setup_class(cls):
     search.clear()
     cls.tsi = TestSearchIndexer()
     DguCreateTestData.create_dgu_test_data()
     DguCreateTestData.create_arbitrary(
         {"name": "latest", "notes": "<b>Latest</b> dataset.", "groups": ["national-health-service"]}
     )
     cls.tsi.index()
Exemplo n.º 17
0
    def setup_class(cls):
        search.clear()
        CreateTestData.create()
        cls.sysadmin_user = model.User.get("testsysadmin")

        cls.sysadmin_action = TestAppCKAN(cls.app, str(cls.sysadmin_user.apikey)).action

        cls.example_pkg = [json.loads(j) for j in EXAMPLE_JSON_LINES.strip().split("\n")]
Exemplo n.º 18
0
    def setup_class(self):
        search.clear()
        model.Session.remove()
        CreateTestData.create()

        # reduce extraneous logging
        from ckan.lib import activity_streams_session_extension
        activity_streams_session_extension.logger.level = 100
Exemplo n.º 19
0
 def setup_class(cls):
     search.clear()
     cls.tsi = TestSearchIndexer()
     DguCreateTestData.create_dgu_test_data()
     DguCreateTestData.create_arbitrary({'name': 'latest',
                                         'notes': '<b>Latest</b> dataset.',
                                         'groups': ['national-health-service']})
     cls.tsi.index()
Exemplo n.º 20
0
    def setup_class(cls):
        cls.original_config = config.copy()

        cls._load_plugins()
        cls.app = _get_test_app()

        search.clear()
        helpers.reset_db()
        cls._create_test_data()
Exemplo n.º 21
0
    def setup_class(cls):
        search.clear()
        CreateTestData.create()
        cls.sysadmin_user = model.User.get('testsysadmin')

        cls.sysadmin_action = TestAppCKAN(cls.app,
                                          str(cls.sysadmin_user.apikey)).action

        cls.example_pkg = [
            json.loads(j) for j in EXAMPLE_JSON_LINES.strip().split('\n')
        ]
Exemplo n.º 22
0
    def setup_class(cls):
        search.clear()
        CreateTestData.create()
        cls.sysadmin_user = model.User.get('testsysadmin')
        cls.normal_user = model.User.get('annafan')
        cls.publisher_user = model.User.get('russianfan')

        cls.sysadmin_action = TestAppCKAN(cls.app,
                                          str(cls.sysadmin_user.apikey)).action
        cls.normal_action = TestAppCKAN(cls.app,
                                        str(cls.normal_user.apikey)).action
        cls.publisher_action = TestAppCKAN(
            cls.app, str(cls.publisher_user.apikey)).action
        cls.action = TestAppCKAN(cls.app).action

        cls.sysadmin_action.organization_member_create(username='******',
                                                       id='nrcan-rncan',
                                                       role='editor')

        cls.sysadmin_action.organization_member_create(username='******',
                                                       id='tb-ct',
                                                       role='editor')

        cls.incomplete_pkg = {
            'title':
            u'A Novel By Tolstoy',
            'license_id':
            'ca-ogl-lgo',
            'resources': [{
                'name': u'Full text.',
                'name_fra': u'Full text.',
                'format': u'TXT',
                'url': u'http://www.annakarenina.com/download/',
                'size': 42,
                'resource_type': 'file',
                'language': 'zxx; CAN',
            }],
        }

        cls.override_possible_pkg = dict(cls.incomplete_pkg,
                                         owner_org='nrcan-rncan')

        cls.complete_pkg = dict(
            cls.override_possible_pkg,
            catalog_type=u'Data | Données',
            title_fra=u'Un novel par Tolstoy',
            maintenance_and_update_frequency=u'As Needed | Au besoin',
            notes=u'...',
            notes_fra=u'...',
            subject=[u'Persons  Personnes'],
            date_published=u'2013-01-01',
            keywords=u'book',
            keywords_fra=u'livre')
Exemplo n.º 23
0
    def test_purged_dataset_is_not_in_search_results(self):
        search.clear()
        dataset = factories.Dataset()

        def get_search_results():
            results = helpers.call_action('package_search',
                                          q=dataset['title'])['results']
            return [d['name'] for d in results]
        assert_equals(get_search_results(), [dataset['name']])

        helpers.call_action('dataset_purge', id=dataset['name'])

        assert_equals(get_search_results(), [])
Exemplo n.º 24
0
    def test_purged_dataset_is_not_in_search_results(self):
        search.clear()
        dataset = factories.Dataset()

        def get_search_results():
            results = helpers.call_action("package_search", q=dataset["title"])["results"]
            return [d["name"] for d in results]

        assert_equals(get_search_results(), [dataset["name"]])

        helpers.call_action("dataset_purge", id=dataset["name"])

        assert_equals(get_search_results(), [])
Exemplo n.º 25
0
    def test_purged_org_is_not_in_search_results_for_its_ex_dataset(self):
        search.clear()
        org = factories.Organization()
        dataset = factories.Dataset(owner_org=org['id'])

        def get_search_result_owner_org():
            results = helpers.call_action('package_search',
                                          q=dataset['title'])['results']
            return results[0]['owner_org']
        assert_equals(get_search_result_owner_org(), org['id'])

        helpers.call_action('organization_purge', id=org['name'])

        assert_equals(get_search_result_owner_org(), None)
Exemplo n.º 26
0
    def test_purged_group_is_not_in_search_results_for_its_ex_dataset(self):
        search.clear()
        group = factories.Group()
        dataset = factories.Dataset(groups=[{'name': group['name']}])

        def get_search_result_groups():
            results = helpers.call_action('package_search',
                                          q=dataset['title'])['results']
            return [g['name'] for g in results[0]['groups']]
        assert_equals(get_search_result_groups(), [group['name']])

        helpers.call_action('group_purge', id=group['name'])

        assert_equals(get_search_result_groups(), [])
Exemplo n.º 27
0
    def setup_class(cls):
        '''
        Set up test class
        '''
        super(TestSearchDataset, cls).setup_class()
        search.clear()

        data_dict = copy.deepcopy(cls.TEST_DATADICT)    # Create public dataset
        data_dict['private'] = False

        # Create a dataset for this test class
        output = call_action_api(cls.app, 'package_create', apikey=cls.user_sysadmin.apikey,
                                 status=200, **data_dict)

        cls.package_id = output.get('id')
Exemplo n.º 28
0
    def setup_class(cls):
        search.clear()
        CreateTestData.create()
        cls.sysadmin_user = model.User.get('testsysadmin')
        cls.normal_user = model.User.get('annafan')
        cls.publisher_user = model.User.get('russianfan')

        cls.sysadmin_action = TestAppCKAN(cls.app,
            str(cls.sysadmin_user.apikey)).action
        cls.normal_action = TestAppCKAN(cls.app,
            str(cls.normal_user.apikey)).action
        cls.publisher_action = TestAppCKAN(cls.app,
            str(cls.publisher_user.apikey)).action
        cls.action = TestAppCKAN(cls.app).action

        cls.sysadmin_action.organization_member_create(
            username='******', id='nrcan-rncan', role='editor')

        cls.sysadmin_action.organization_member_create(
            username='******', id='tb-ct', role='editor')

        cls.incomplete_pkg = {
            'title': u'A Novel By Tolstoy',
            'license_id': 'ca-ogl-lgo',
            'resources': [{
                'name': u'Full text.',
                'name_fra': u'Full text.',
                'format': u'TXT',
                'url': u'http://www.annakarenina.com/download/',
                'size': 42,
                'resource_type': 'file',
                'language': 'zxx; CAN',
            }],
        }

        cls.override_possible_pkg = dict(cls.incomplete_pkg,
            owner_org='nrcan-rncan')

        cls.complete_pkg = dict(cls.override_possible_pkg,
            catalog_type=u'Data | Données',
            title_fra=u'Un novel par Tolstoy',
            maintenance_and_update_frequency=u'As Needed | Au besoin',
            notes=u'...',
            notes_fra=u'...',
            subject=[u'Persons  Personnes'],
            date_published=u'2013-01-01',
            keywords=u'book',
            keywords_fra=u'livre')
Exemplo n.º 29
0
    def setup_class(self):
        search.clear()
        setup_test_search_index()
        super(OnsLoaderBase, self).setup_class()

        # make annafan a sysadmin to allow package creation
        rev = model.repo.new_revision()
        user = model.User.by_name(u'annafan')
        model.add_user_to_role(user, model.Role.ADMIN, model.System())
        model.repo.commit_and_remove()

        # create test publishers
        rev = model.repo.new_revision()
        for name, title in publishers.items():
            model.Session.add(model.Group(name=unicode(name), title=title, type='publisher'))
        model.repo.commit_and_remove()
Exemplo n.º 30
0
    def command(self):
        # Avoids vdm logging warning
        logging.basicConfig(level=logging.ERROR)

        self._load_config()
        from ckan import model
        import ckan.lib.search as search

        cmd = self.args[0]
        if cmd == 'init':
            model.repo.init_db()
            if self.verbose:
                print 'Initialising DB: SUCCESS'
        elif cmd == 'clean' or cmd == 'drop':
            model.repo.clean_db()
            search.clear()
            if self.verbose:
                print 'Cleaning DB: SUCCESS'
        elif cmd == 'upgrade':
            if len(self.args) > 1:
                model.repo.upgrade_db(self.args[1])
            else:
                model.repo.upgrade_db()
        elif cmd == 'version':
            self.version()
        elif cmd == 'dump':
            self.dump()
        elif cmd == 'load':
            self.load()
        elif cmd == 'load-only':
            self.load(only_load=True)
        elif cmd == 'simple-dump-csv':
            self.simple_dump_csv()
        elif cmd == 'simple-dump-json':
            self.simple_dump_json()
        elif cmd == 'dump-rdf':
            self.dump_rdf()
        elif cmd == 'create-from-model':
            model.repo.create_db()
            if self.verbose:
                print 'Creating DB: SUCCESS'
        elif cmd == 'send-rdf':
            self.send_rdf()
        else:
            print 'Command %s not recognized' % cmd
            sys.exit(1)
Exemplo n.º 31
0
    def setup_class(self):
        search.clear()
        setup_test_search_index()
        super(OnsLoaderBase, self).setup_class()

        # make annafan a sysadmin to allow package creation
        rev = model.repo.new_revision()
        user = model.User.by_name(u'annafan')
        model.add_user_to_role(user, model.Role.ADMIN, model.System())
        model.repo.commit_and_remove()

        # create test publishers
        rev = model.repo.new_revision()
        for name, title in publishers.items():
            model.Session.add(
                model.Group(name=unicode(name), title=title, type='publisher'))
        model.repo.commit_and_remove()
Exemplo n.º 32
0
    def command(self):
        # Avoids vdm logging warning
        logging.basicConfig(level=logging.ERROR)
        
        self._load_config()
        from ckan import model
        import ckan.lib.search as search

        cmd = self.args[0]
        if cmd == 'init':
            model.repo.init_db()
            if self.verbose:
                print 'Initialising DB: SUCCESS'
        elif cmd == 'clean' or cmd == 'drop':
            model.repo.clean_db()
            search.clear()
            if self.verbose:
                print 'Cleaning DB: SUCCESS'
        elif cmd == 'upgrade':
            if len(self.args) > 1:
                model.repo.upgrade_db(self.args[1])
            else:
                model.repo.upgrade_db()
        elif cmd == 'version':
            self.version()
        elif cmd == 'dump':
            self.dump()
        elif cmd == 'load':
            self.load()
        elif cmd == 'load-only':
            self.load(only_load=True)
        elif cmd == 'simple-dump-csv':
            self.simple_dump_csv()
        elif cmd == 'simple-dump-json':
            self.simple_dump_json()
        elif cmd == 'dump-rdf':
            self.dump_rdf()
        elif cmd == 'create-from-model':
            model.repo.create_db()
            if self.verbose:
                print 'Creating DB: SUCCESS'
        elif cmd == 'send-rdf':
            self.send_rdf()
        else:
            print 'Command %s not recognized' % cmd
            sys.exit(1)
Exemplo n.º 33
0
    def setup(self):
        """
        Create users and organizations
        """
        reset_db()
        search.clear()
        
        self.sysadmin = factories.Sysadmin()
        self.test_member_1 = factories.User()
        self.test_member_2 = factories.User()
        self.test_editor = factories.User()

        users = [{'name': self.test_member_1['name'], 'capacity': 'member'},
                 {'name': self.test_member_2['name'], 'capacity': 'member'},
                 {'name': self.test_editor['name'], 'capacity': 'editor'}]

        self.test_org_1 = factories.Organization(users=users,
            name="testjournal")
        self.test_org_2 = factories.Organization()
        self.test_package = self._dara_testpackage()
Exemplo n.º 34
0
    def setup_class(cls):
        # clean the db so we can run these tests on their own
        model.repo.rebuild_db()
        search.clear()
        CreateTestData.create()

        cls.package_expected = {
            u'author': None,
            u'author_email': None,
            u'creator_user_id': None,
            'extras': [
               {u'key': u'genre',
                u'state': u'active',
                u'value': 'romantic novel'},
               {u'key': u'original media', u'state': u'active', u'value': u'book'}],
            'groups': [{
                        u'name': u'david',
                        u'capacity': u'public',
                        u'image_url': u'',
                        u'image_display_url': u'',
                        u'description': u'These are books that David likes.',
                        u'display_name': u"Dave's books",
                        u'type': u'group',
                        u'state': u'active',
                        u'is_organization': False,
                        u'title': u"Dave's books",
                        u"approval_status": u"approved"},
                       {
                        u'name': u'roger',
                        u'capacity': u'public',
                        u'description': u'Roger likes these books.',
                        u'image_url': u'',
                        'image_display_url': u'',
                        'display_name': u"Roger's books",
                        u'type': u'group',
                        u'state': u'active',
                        u'is_organization': False,
                        u'title': u"Roger's books",
                        u"approval_status": u"approved"}],
            'isopen': True,
            u'license_id': u'other-open',
            'license_title': u'Other (Open)',
            'organization': None,
            u'owner_org': None,
            u'maintainer': None,
            u'maintainer_email': None,
            u'name': u'annakarenina',
            u'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n\nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
            'num_resources': 2,
            'num_tags': 3,
            u'private': False,
            'relationships_as_object': [],
            'relationships_as_subject': [],
            'resources': [{u'alt_url': u'alt123',
                            u'cache_last_updated': None,
                            u'cache_url': None,
                            u'description': u'Full text. Needs escaping: " Umlaut: \xfc',
                            u'format': u'plain text',
                            u'hash': u'abc123',
                            u'last_modified': None,
                            u'mimetype': None,
                            u'mimetype_inner': None,
                            u'name': None,
                            u'position': 0,
                            u'resource_type': None,
                            u'size': None,
                            u'size_extra': u'123',
                            u'url_type': None,
                            u'state': u'active',
                            u'url': u'http://www.annakarenina.com/download/x=1&y=2',
                            u'webstore_last_updated': None,
                            u'webstore_url': None},
                           {u'alt_url': u'alt345',
                            u'cache_last_updated': None,
                            u'cache_url': None,
                            u'description': u'Index of the novel',
                            u'format': u'JSON',
                            u'hash': u'def456',
                            u'last_modified': None,
                            u'mimetype': None,
                            u'mimetype_inner': None,
                            u'name': None,
                            u'position': 1,
                            u'resource_type': None,
                            u'url_type': None,
                            u'size': None,
                            u'size_extra': u'345',
                            u'state': u'active',
                            u'url': u'http://www.annakarenina.com/index.json',
                            u'webstore_last_updated': None,
                            u'webstore_url': None}],
            u'state': u'active',
            'tags': [{u'name': u'Flexible \u30a1',
                        'display_name': u'Flexible \u30a1',
                        u'state': u'active'},
                     {'display_name': u'russian',
                         u'name': u'russian',
                         u'state': u'active'},
                     {'display_name': u'tolstoy',
                         u'name': u'tolstoy',
                         u'state': u'active'}],
            u'title': u'A Novel By Tolstoy',
            u'type': u'dataset',
            u'url': u'http://www.annakarenina.com',
            u'version': u'0.7a',
            }
Exemplo n.º 35
0
 def teardown_class(cls):
     search.clear()
     model.repo.rebuild_db()
Exemplo n.º 36
0
 def setup(self):
     helpers.reset_db()
     search.clear()
Exemplo n.º 37
0
 def teardown_class(self):
     model.repo.rebuild_db()
     model.Session.remove()
     search.clear()
Exemplo n.º 38
0
def setup_test_search_index():
    #from ckan import plugins
    if not is_search_supported():
        raise SkipTest("Search not supported")
    search.clear()
Exemplo n.º 39
0
 def setup(self):
     helpers.reset_db()
     search.clear()
Exemplo n.º 40
0
 def setup_class(cls):
     search.clear()
     CreateTestData.create()
     cls.sysadmin_user = model.User.get('testsysadmin')
     cls.normal_user = model.User.get('annafan')
     CreateTestData.make_some_vocab_tags()
Exemplo n.º 41
0
 def teardown(self):
     search.clear()
Exemplo n.º 42
0
 def setup_class(self):
     search.clear()
     model.Session.remove()
     CreateTestData.create()
Exemplo n.º 43
0
    def teardown_class(cls):

        helpers.reset_db()
        search.clear()
Exemplo n.º 44
0
    def db_solr_sync(self):

        print str(
            datetime.datetime.now()) + ' Entering Database Solr Sync function.'

        url = config.get(
            'solr_url'
        ) + "/select?q=*%3A*&sort=id+asc&fl=id%2Cmetadata_modified&wt=json&indent=true"
        response = get_response(url)

        if (response != 'error'):

            print str(datetime.datetime.now()
                      ) + ' Deleting records from solr_pkg_ids.'
            sql = '''delete from solr_pkg_ids'''
            model.Session.execute(sql)
            model.Session.commit()

            f = response.read()
            data = json.loads(f)
            rows = data.get('response').get('numFound')

            start = 0
            chunk_size = 1000

            print str(datetime.datetime.now()
                      ) + ' Starting insertion of records in solr_pkg_ids .'

            for x in range(0, int(math.ceil(rows / chunk_size)) + 1):

                if (x == 0):
                    start = 0

                print str(datetime.datetime.now()
                          ) + ' Fetching ' + url + "&rows=" + str(
                              chunk_size) + "&start=" + str(start)

                response = get_response(url + "&rows=" + str(chunk_size) +
                                        "&start=" + str(start))
                f = response.read()
                data = json.loads(f)
                results = data.get('response').get('docs')

                print str(datetime.datetime.now()) + ' Inserting ' + str(
                    start) + ' - ' + str(start + int(
                        data.get('responseHeader').get('params').get('rows')) -
                                         1) + ' of ' + str(rows)

                for x in range(0, len(results)):
                    sql = '''select count(id) as count from package where id = :pkg_id;'''
                    q = model.Session.execute(sql,
                                              {'pkg_id': results[x]['id']})
                    for row in q:
                        if (row['count'] == 0):
                            sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
                            model.Session.execute(sql, {
                                'pkg_id': results[x]['id'],
                                'action': 'notfound'
                            })
                            model.Session.commit()
                        else:
                            pkg_dict = logic.get_action('package_show')(
                                {
                                    'model': model,
                                    'ignore_auth': True,
                                    'validate': False
                                }, {
                                    'id': results[x]['id']
                                })
                            if (str(results[x]['metadata_modified'])[:19] !=
                                    pkg_dict['metadata_modified'][:19]):
                                print str(
                                    datetime.datetime.now()
                                ) + ' Action Type : outsync for Package Id: ' + results[
                                    x]['id']
                                print ' ' * 26 + ' Modified Date from Solr: ' + str(
                                    results[x]['metadata_modified'])
                                print ' ' * 26 + ' Modified Date from Db: ' + pkg_dict[
                                    'metadata_modified']
                                sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
                                model.Session.execute(
                                    sql, {
                                        'pkg_id': results[x]['id'],
                                        'action': 'outsync'
                                    })
                                model.Session.commit()
                            else:
                                sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
                                model.Session.execute(
                                    sql, {
                                        'pkg_id': results[x]['id'],
                                        'action': 'insync'
                                    })
                                model.Session.commit()

                start = int(
                    data.get('responseHeader').get('params').get(
                        'start')) + chunk_size

            print str(
                datetime.datetime.now()) + ' Starting Database to Solr Sync'

            #sql = '''Select id from package where id not in (select pkg_id from solr_pkg_ids); '''
            sql = '''Select p.id as pkg_id from package p
                   left join solr_pkg_ids sp on sp.pkg_id = p.id
                   where sp.pkg_id is null; '''

            q = model.Session.execute(sql)
            pkg_ids = set()
            for row in q:
                pkg_ids.add(row['pkg_id'])
            for pkg_id in pkg_ids:
                try:
                    print str(
                        datetime.datetime.now()) + ' Building Id: ' + pkg_id
                    search.rebuild(pkg_id)
                except ckan.logic.NotFound:
                    print "Error: Not Found."
                except KeyboardInterrupt:
                    print "Stopped."
                    return
                except:
                    raise

            sql = '''Select pkg_id from solr_pkg_ids where action = 'outsync'; '''
            q = model.Session.execute(sql)
            pkg_ids = set()
            for row in q:
                pkg_ids.add(row['pkg_id'])
            for pkg_id in pkg_ids:
                try:
                    print str(
                        datetime.datetime.now()) + ' Rebuilding Id: ' + pkg_id
                    search.rebuild(pkg_id)
                except ckan.logic.NotFound:
                    print "Error: Not Found."
                except KeyboardInterrupt:
                    print "Stopped."
                    return
                except:
                    raise

            print str(
                datetime.datetime.now()) + ' Starting Solr to Database Sync'

            sql = '''Select pkg_id from solr_pkg_ids where action = 'notfound'; '''
            q = model.Session.execute(sql)
            pkg_ids = set()
            for row in q:
                pkg_ids.add(row['pkg_id'])
            for pkg_id in pkg_ids:
                try:
                    search.clear(pkg_id)
                except ckan.logic.NotFound:
                    print "Error: Not Found."
                except KeyboardInterrupt:
                    print "Stopped."
                    return
                except:
                    raise

            print str(datetime.datetime.now()) + " All Sync Done."
Exemplo n.º 45
0
 def setup_class(cls):
     search.clear()
     CreateTestData.create()
     cls.sysadmin_user = model.User.get('testsysadmin')
     cls.normal_user = model.User.get('annafan')
     CreateTestData.make_some_vocab_tags()
Exemplo n.º 46
0
 def setup_class(self):
     search.clear()
     super(OnsLoaderBase, self).setup_class()
Exemplo n.º 47
0
    def format_mapping(self):
        try:
            tk.check_access('sysadmin', {'user': g.user, model: model})
        except tk.NotAuthorized:
            return tk.abort(403)
        if request.method == 'POST':
            old = request.POST.get('from')
            new = request.POST.get('to')
            if old and new:
                ids = set()
                res_query = model.Session.query(model.Resource).filter_by(
                    format=old, state='active'
                )
                for res in res_query:
                    ids.add(res.package_id)

                res_query.update({'format': new})
                model.Session.commit()
                for id in ids:
                    clear(id)
                    rebuild(id, defer_commit=True)
                commit()
                tk.h.flash_success(
                    'Updated. Records changed: {}'.format(len(ids))
                )
            return tk.redirect_to('format_mapping')

        defined = set(
            map(lambda (_1, fmt, _3): fmt,
                h.resource_formats().values())
        )
        db_formats = model.Session.query(
            model.Resource.format, func.count(model.Resource.id),
            func.count(model.PackageExtra.value)
        ).outerjoin(
            model.PackageExtra,
            (model.Resource.package_id == model.PackageExtra.package_id)
            & ((model.PackageExtra.key == 'harvest_portal')
               | (model.PackageExtra.key.is_(None)))
        ).group_by(model.Resource.format).filter(
            model.Resource.format != '', model.Resource.state == 'active'
        )
        db_formats = db_formats.all()

        format_types = {
            f: {
                True: 'Partially external',
                e == 0: 'Local',
                t - e == 0: 'External'
            }[True]
            for (f, t, e) in db_formats
        }
        used = set(format_types)
        undefined = used - defined

        extra_vars = {
            'undefined': undefined,
            'defined': defined,
            'format_types': format_types
        }
        return tk.render('admin/format_mapping.html', extra_vars)
Exemplo n.º 48
0
 def teardown(self):
     helpers.reset_db()
     search.clear()
Exemplo n.º 49
0
 def setup(self):
     '''Reset the database and clear the search indexes.'''
     reset_db()
     search.clear()
Exemplo n.º 50
0
    def setup(self):
        helpers.reset_db()

        # Clear the search index
        search.clear()
Exemplo n.º 51
0
def setup_test_search_index():
    from ckan import plugins
    if not is_search_supported():
        raise SkipTest("Search not supported")
    search.clear()
    plugins.load('synchronous_search')
Exemplo n.º 52
0
 def teardown(self):
     search.clear()
Exemplo n.º 53
0
    def setup_class(cls):
        # clean the db so we can run these tests on their own
        model.repo.rebuild_db()
        search.clear()
        CreateTestData.create()

        cls.package_expected = {
            'author':
            None,
            'author_email':
            None,
            'extras': [{
                'key': u'genre',
                'state': u'active',
                'value': 'romantic novel'
            }, {
                'key': u'original media',
                'state': u'active',
                'value': u'book'
            }],
            'groups': [{
                'description': u'These are books that David likes.',
                'name': u'david',
                'capacity': 'public',
                'image_url': u'',
                'image_display_url': u'',
                'display_name': u"Dave's books",
                'type': u'group',
                'state': u'active',
                'is_organization': False,
                'title': u"Dave's books",
                "approval_status": u"approved"
            }, {
                'description': u'Roger likes these books.',
                'name': u'roger',
                'capacity': 'public',
                'image_url': u'',
                'image_display_url': u'',
                'display_name': u"Roger's books",
                'type': u'group',
                'state': u'active',
                'is_organization': False,
                'title': u"Roger's books",
                "approval_status": u"approved"
            }],
            'isopen':
            True,
            'license_id':
            u'other-open',
            'license_title':
            u'Other (Open)',
            'creator_user_id':
            None,
            'owner_org':
            None,
            'private':
            False,
            'organization':
            None,
            'maintainer':
            None,
            'maintainer_email':
            None,
            'type':
            u'dataset',
            'name':
            u'annakarenina',
            'notes':
            u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n\nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
            'relationships_as_object': [],
            'relationships_as_subject': [],
            'resources': [{
                u'alt_url': u'alt123',
                u'cache_last_updated': None,
                u'cache_url': None,
                u'description': u'Full text. Needs escaping: " Umlaut: \xfc',
                u'format': u'plain text',
                u'hash': u'abc123',
                u'last_modified': None,
                u'mimetype': None,
                u'mimetype_inner': None,
                u'name': None,
                u'position': 0,
                u'resource_type': None,
                u'size': None,
                u'size_extra': u'123',
                'url_type': None,
                u'state': u'active',
                u'url': u'http://www.annakarenina.com/download/x=1&y=2',
                u'webstore_last_updated': None,
                u'webstore_url': None
            }, {
                u'alt_url': u'alt345',
                u'cache_last_updated': None,
                u'cache_url': None,
                u'description': u'Index of the novel',
                u'format': u'JSON',
                u'hash': u'def456',
                u'last_modified': None,
                u'mimetype': None,
                u'mimetype_inner': None,
                u'name': None,
                u'position': 1,
                u'resource_type': None,
                'url_type': None,
                u'size': None,
                u'size_extra': u'345',
                u'state': u'active',
                u'url': u'http://www.annakarenina.com/index.json',
                u'webstore_last_updated': None,
                u'webstore_url': None
            }],
            'state':
            u'active',
            'tags': [{
                'name': u'Flexible \u30a1',
                'display_name': u'Flexible \u30a1',
                'state': u'active'
            }, {
                'name': u'russian',
                'display_name': u'russian',
                'state': u'active'
            }, {
                'name': u'tolstoy',
                'display_name': u'tolstoy',
                'state': u'active'
            }],
            'title':
            u'A Novel By Tolstoy',
            'url':
            u'http://www.annakarenina.com',
            'version':
            u'0.7a',
            'num_tags':
            3,
            'num_resources':
            2,
        }
Exemplo n.º 54
0
 def teardown_class(self):
     model.repo.rebuild_db()
     model.Session.remove()
     search.clear()
Exemplo n.º 55
0
 def teardown_class(cls):
     model.repo.rebuild_db()
     search.clear()
Exemplo n.º 56
0
    def setup(self):
        helpers.reset_db()

        # Clear the search index
        search.clear()
Exemplo n.º 57
0
    def db_solr_sync(self):

        print str(datetime.datetime.now()) + ' Entering Database Solr Sync function.'

        url = config.get('solr_url') + "/select?q=*%3A*&sort=id+asc&fl=id%2Cmetadata_modified&wt=json&indent=true"
        response = get_response(url)
    
        if (response != 'error'):

          print str(datetime.datetime.now()) + ' Deleting records from solr_pkg_ids.'		
          sql = '''delete from solr_pkg_ids'''
          model.Session.execute(sql)
          model.Session.commit()
		
          f = response.read()
          data = json.loads(f)
          rows = data.get('response').get('numFound')

          start = 0
          chunk_size = 1000         

          print str(datetime.datetime.now()) + ' Starting insertion of records in solr_pkg_ids .'
 
          for x in range(0, int(math.ceil(rows/chunk_size))+1):
		  
            if(x == 0):
               start = 0
			
            print str(datetime.datetime.now()) + ' Fetching ' + url + "&rows=" + str(chunk_size) + "&start=" + str(start)			  
			  
            response = get_response(url + "&rows=" + str(chunk_size) + "&start=" + str(start))
            f = response.read()
            data = json.loads(f)
            results = data.get('response').get('docs')

            print str(datetime.datetime.now()) + ' Inserting ' + str(start) + ' - ' + str(start + int(data.get('responseHeader').get('params').get('rows')) - 1) + ' of ' + str(rows)			
			
            for x in range(0, len(results)):
                sql = '''select count(id) as count from package where id = :pkg_id;'''
                q = model.Session.execute(sql, {'pkg_id' : results[x]['id']})            
                for row in q:
                   if(row['count'] == 0):
                     sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
                     model.Session.execute(sql, {'pkg_id' : results[x]['id'], 'action' : 'notfound' })
                     model.Session.commit()			
                   else:
                     pkg_dict = logic.get_action('package_show')(
                                    {'model': model, 'ignore_auth': True, 'validate': False},
                                    {'id': results[x]['id']})
                     if(str(results[x]['metadata_modified'])[:19] != pkg_dict['metadata_modified'][:19]):
                       print str(datetime.datetime.now()) + ' Action Type : outsync for Package Id: ' + results[x]['id']
                       print ' ' * 26 +                     ' Modified Date from Solr: ' + str(results[x]['metadata_modified'])
                       print ' ' * 26 +                     ' Modified Date from Db: ' + pkg_dict['metadata_modified']
                       sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
                       model.Session.execute(sql, {'pkg_id' : results[x]['id'], 'action' : 'outsync' })
                       model.Session.commit()
                     else:
                       sql = '''insert into solr_pkg_ids (pkg_id, action) values (:pkg_id, :action);'''
                       model.Session.execute(sql, {'pkg_id' : results[x]['id'], 'action' : 'insync' })
                       model.Session.commit()
                     
            start = int(data.get('responseHeader').get('params').get('start')) + chunk_size			       
          
          print str(datetime.datetime.now()) + ' Starting Database to Solr Sync'           
          
          #sql = '''Select id from package where id not in (select pkg_id from solr_pkg_ids); '''
          sql = '''Select p.id as pkg_id from package p
                   left join solr_pkg_ids sp on sp.pkg_id = p.id
                   where sp.pkg_id is null; '''
          
          q = model.Session.execute(sql)
          pkg_ids = set()
          for row in q:
            pkg_ids.add(row['pkg_id'])
          for pkg_id in pkg_ids:
            try:
              print str(datetime.datetime.now()) + ' Building Id: ' + pkg_id
              search.rebuild(pkg_id)
            except ckan.logic.NotFound:
              print "Error: Not Found."
            except KeyboardInterrupt:
              print "Stopped."
              return
            except:
              raise
          
          sql = '''Select pkg_id from solr_pkg_ids where action = 'outsync'; '''
          q = model.Session.execute(sql)          
          pkg_ids = set()
          for row in q:
            pkg_ids.add(row['pkg_id'])
          for pkg_id in pkg_ids:
            try:
              print str(datetime.datetime.now()) + ' Rebuilding Id: ' + pkg_id
              search.rebuild(pkg_id)
            except ckan.logic.NotFound:
              print "Error: Not Found."
            except KeyboardInterrupt:
              print "Stopped."
              return
            except:
              raise
          
          print str(datetime.datetime.now()) + ' Starting Solr to Database Sync'
          
          sql = '''Select pkg_id from solr_pkg_ids where action = 'notfound'; '''
          q = model.Session.execute(sql)
          pkg_ids = set()
          for row in q:
            pkg_ids.add(row['pkg_id'])
          for pkg_id in pkg_ids:
            try:
              search.clear(pkg_id)
            except ckan.logic.NotFound:
              print "Error: Not Found."
            except KeyboardInterrupt:
              print "Stopped."
              return
            except:
              raise
          
          print str(datetime.datetime.now()) + " All Sync Done."
Exemplo n.º 58
0
Arquivo: cli.py Projeto: arkka/ckan
    def clear(self):
        from ckan.lib.search import clear

        package_id =self.args[1] if len(self.args) > 1 else None
        clear(package_id)