예제 #1
0
    def test_attach_does_not_duplicate(self):
        attached_datasets = []
        for i in range(2):
            dataset = DatasetFactory.build()
            dataset.extras["harvest:domain"] = "test.org"
            dataset.extras["harvest:remote_id"] = str(i)
            dataset.save()
            attached_datasets.append(dataset)

        datasets = DatasetFactory.create_batch(3)

        with NamedTemporaryFile() as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=["local", "remote"], delimiter=b";", quotechar=b'"')

            writer.writeheader()
            for index, dataset in enumerate(datasets):
                writer.writerow({"local": str(dataset.id), "remote": str(index)})
            csvfile.flush()

            result = actions.attach("test.org", csvfile.name)

        dbcount = Dataset.objects(**{"extras__harvest:remote_id__exists": True}).count()
        self.assertEqual(result.success, len(datasets))
        self.assertEqual(dbcount, result.success)
        for index, dataset in enumerate(datasets):
            dataset.reload()
            self.assertEqual(dataset.extras["harvest:domain"], "test.org")
            self.assertEqual(dataset.extras["harvest:remote_id"], str(index))
예제 #2
0
    def test_datasets_csv(self):
        with self.autoindex():
            datasets = [DatasetFactory(resources=[ResourceFactory()])
                        for _ in range(5)]
            hidden_dataset = DatasetFactory()

        response = self.get(url_for('site.datasets_csv'))

        self.assert200(response)
        self.assertEqual(response.mimetype, 'text/csv')
        self.assertEqual(response.charset, 'utf-8')

        csvfile = StringIO.StringIO(response.data)
        reader = csv.get_reader(csvfile)
        header = reader.next()

        self.assertEqual(header[0], 'id')
        self.assertIn('title', header)
        self.assertIn('description', header)
        self.assertIn('created_at', header)
        self.assertIn('last_modified', header)
        self.assertIn('tags', header)
        self.assertIn('metric.reuses', header)

        rows = list(reader)
        ids = [row[0] for row in rows]

        self.assertEqual(len(rows), len(datasets))
        for dataset in datasets:
            self.assertIn(str(dataset.id), ids)
        self.assertNotIn(str(hidden_dataset.id), ids)
예제 #3
0
    def test_render_home(self):
        '''It should render the home page'''
        for i in range(3):
            org = OrganizationFactory()
            DatasetFactory(organization=org)
            ReuseFactory(organization=org)

        current_site.settings.home_datasets = [
            DatasetFactory() for _ in range(3)]
        current_site.settings.home_reuses = [
            ReuseFactory() for _ in range(3)]

        response = self.get(url_for('site.home'))
        self.assert200(response)
예제 #4
0
    def test_attach(self):
        datasets = DatasetFactory.create_batch(3)

        with NamedTemporaryFile() as csvfile:
            writer = csv.DictWriter(csvfile,
                                    fieldnames=['local', 'remote'],
                                    delimiter=b';',
                                    quotechar=b'"')

            writer.writeheader()
            for index, dataset in enumerate(datasets):
                writer.writerow({
                    'local': str(dataset.id),
                    'remote': str(index)
                })
            csvfile.flush()

            result = actions.attach('test.org', csvfile.name)

        self.assertEqual(result.success, len(datasets))
        self.assertEqual(result.errors, 0)
        for index, dataset in enumerate(datasets):
            dataset.reload()
            self.assertEqual(dataset.extras['harvest:domain'], 'test.org')
            self.assertEqual(dataset.extras['harvest:remote_id'], str(index))
예제 #5
0
    def test_attach(self):
        datasets = DatasetFactory.create_batch(3)

        with NamedTemporaryFile() as csvfile:
            writer = csv.DictWriter(csvfile,
                                    fieldnames=['local', 'remote'],
                                    delimiter=b';',
                                    quotechar=b'"')

            writer.writeheader()
            for index, dataset in enumerate(datasets):
                writer.writerow({
                    'local': str(dataset.id),
                    'remote': str(index)
                })
            csvfile.flush()

            result = actions.attach('test.org', csvfile.name)

        self.assertEqual(result.success, len(datasets))
        self.assertEqual(result.errors, 0)
        for index, dataset in enumerate(datasets):
            dataset.reload()
            self.assertEqual(dataset.extras['harvest:domain'], 'test.org')
            self.assertEqual(dataset.extras['harvest:remote_id'], str(index))
예제 #6
0
    def test_attach_skip_not_found(self):
        datasets = DatasetFactory.create_batch(3)

        with NamedTemporaryFile() as csvfile:
            writer = csv.DictWriter(csvfile,
                                    fieldnames=['local', 'remote'],
                                    delimiter=b';',
                                    quotechar=b'"')

            writer.writeheader()
            writer.writerow({
                'local': 'not-found',
                'remote': '42'
            })
            for index, dataset in enumerate(datasets):
                writer.writerow({
                    'local': str(dataset.id),
                    'remote': str(index)
                })
            csvfile.flush()

            result = actions.attach('test.org', csvfile.name)

        self.assertEqual(result.success, len(datasets))
        self.assertEqual(result.errors, 1)
예제 #7
0
    def test_resources_csv_with_filters(self):
        '''Should handle filtering but ignore paging or facets'''
        with self.autoindex():
            filtered_datasets = [
                DatasetFactory(
                    resources=[ResourceFactory(),
                               ResourceFactory()],
                    tags=['selected']) for _ in range(6)
            ]
            [DatasetFactory(resources=[ResourceFactory()]) for _ in range(3)]
            DatasetFactory()

        response = self.get(
            url_for('site.resources_csv',
                    tag='selected',
                    page_size=3,
                    facets=True))

        self.assert200(response)
        self.assertEqual(response.mimetype, 'text/csv')
        self.assertEqual(response.charset, 'utf-8')

        csvfile = StringIO.StringIO(response.data)
        reader = csv.get_reader(csvfile)
        header = reader.next()

        self.assertEqual(header[0], 'dataset.id')
        self.assertIn('dataset.title', header)
        self.assertIn('dataset.url', header)
        self.assertIn('title', header)
        self.assertIn('description', header)
        self.assertIn('type', header)
        self.assertIn('url', header)
        self.assertIn('created_at', header)
        self.assertIn('modified', header)
        self.assertIn('downloads', header)

        resource_id_index = header.index('id')

        rows = list(reader)
        ids = [(row[0], row[resource_id_index]) for row in rows]

        self.assertEqual(len(rows),
                         sum(len(d.resources) for d in filtered_datasets))
        for dataset in filtered_datasets:
            for resource in dataset.resources:
                self.assertIn((str(dataset.id), str(resource.id)), ids)
예제 #8
0
 def test_render_metrics(self):
     '''It should render the search page'''
     for i in range(3):
         org = OrganizationFactory()
         DatasetFactory(organization=org)
         ReuseFactory(organization=org)
     response = self.get(url_for('site.dashboard'))
     self.assert200(response)
예제 #9
0
    def test_render_reuse_page(self):
        '''It should render the reuse page'''
        org = OrganizationFactory()
        dataset = DatasetFactory(organization=org)
        reuse = ReuseFactory(organization=org, datasets=[dataset])

        response = self.get(url_for('reuses.show', reuse=reuse))
        self.assert200(response)
예제 #10
0
    def test_render_dataset_page(self):
        '''It should render the dataset page'''
        org = OrganizationFactory()
        dataset = DatasetFactory(organization=org)
        ReuseFactory(organization=org, datasets=[dataset])

        response = self.get(url_for('datasets.show', dataset=dataset))
        self.assert200(response)
예제 #11
0
    def test_datasets_csv_with_filters(self):
        '''Should handle filtering but ignore paging or facets'''
        with self.autoindex():
            filtered_datasets = [
                DatasetFactory(resources=[ResourceFactory()],
                               tags=['selected']) for _ in range(6)
            ]
            datasets = [
                DatasetFactory(resources=[ResourceFactory()]) for _ in range(3)
            ]
            hidden_dataset = DatasetFactory()

        response = self.get(
            url_for('site.datasets_csv',
                    tag='selected',
                    page_size=3,
                    facets=True))

        self.assert200(response)
        self.assertEqual(response.mimetype, 'text/csv')
        self.assertEqual(response.charset, 'utf-8')

        csvfile = StringIO.StringIO(response.data)
        reader = csv.get_reader(csvfile)
        header = reader.next()

        self.assertEqual(header[0], 'id')
        self.assertIn('title', header)
        self.assertIn('description', header)
        self.assertIn('created_at', header)
        self.assertIn('last_modified', header)
        self.assertIn('tags', header)
        self.assertIn('metric.reuses', header)

        rows = list(reader)
        ids = [row[0] for row in rows]

        # Should ignore paging
        self.assertEqual(len(rows), len(filtered_datasets))
        # SHoulf pass filter
        for dataset in filtered_datasets:
            self.assertIn(str(dataset.id), ids)
        for dataset in datasets:
            self.assertNotIn(str(dataset.id), ids)
        self.assertNotIn(str(hidden_dataset.id), ids)
예제 #12
0
    def test_render_organization_page(self):
        '''It should render the organization page'''
        org = OrganizationFactory()
        datasets = [DatasetFactory(organization=org) for _ in range(3)]
        for dataset in datasets:
            ReuseFactory(organization=org, datasets=[dataset])

        response = self.get(url_for('organizations.show', org=org))
        self.assert200(response)
예제 #13
0
    def test_render_home(self):
        '''It should render the home page'''
        with self.autoindex():
            for i in range(3):
                org = OrganizationFactory()
                DatasetFactory(organization=org)
                ReuseFactory(organization=org)

        response = self.get(url_for('site.home'))
        self.assert200(response)
예제 #14
0
    def test_render_search(self):
        '''It should render the search page'''
        with self.autoindex():
            for i in range(3):
                org = OrganizationFactory()
                DatasetFactory(organization=org)
                ReuseFactory(organization=org)

        response = self.get(url_for('front.search'))
        self.assert200(response)
예제 #15
0
    def test_resources_csv(self):
        with self.autoindex():
            datasets = [
                DatasetFactory(
                    resources=[ResourceFactory(),
                               ResourceFactory()]) for _ in range(3)
            ]
            hidden_dataset = DatasetFactory()

        response = self.get(url_for('site.resources_csv'))

        self.assert200(response)
        self.assertEqual(response.mimetype, 'text/csv')
        self.assertEqual(response.charset, 'utf-8')

        csvfile = StringIO.StringIO(response.data)
        reader = csv.get_reader(csvfile)
        header = reader.next()

        self.assertEqual(header[0], 'dataset.id')
        self.assertIn('dataset.title', header)
        self.assertIn('dataset.url', header)
        self.assertIn('title', header)
        self.assertIn('description', header)
        self.assertIn('type', header)
        self.assertIn('url', header)
        self.assertIn('created_at', header)
        self.assertIn('modified', header)
        self.assertIn('downloads', header)

        resource_id_index = header.index('id')

        rows = list(reader)
        ids = [(row[0], row[resource_id_index]) for row in rows]

        self.assertEqual(len(rows), sum(len(d.resources) for d in datasets))
        for dataset in datasets:
            for resource in dataset.resources:
                self.assertIn((str(dataset.id), str(resource.id)), ids)
예제 #16
0
    def test_attach_does_not_duplicate(self):
        attached_datasets = []
        for i in range(2):
            dataset = DatasetFactory.build()
            dataset.extras['harvest:domain'] = 'test.org'
            dataset.extras['harvest:remote_id'] = str(i)
            dataset.last_modified = datetime.now()
            dataset.save()
            attached_datasets.append(dataset)

        datasets = DatasetFactory.create_batch(3)

        with NamedTemporaryFile() as csvfile:
            writer = csv.DictWriter(csvfile,
                                    fieldnames=['local', 'remote'],
                                    delimiter=b';',
                                    quotechar=b'"')

            writer.writeheader()
            for index, dataset in enumerate(datasets):
                writer.writerow({
                    'local': str(dataset.id),
                    'remote': str(index)
                })
            csvfile.flush()

            result = actions.attach('test.org', csvfile.name)

        dbcount = Dataset.objects(**{
            'extras__harvest:remote_id__exists': True
        }).count()
        self.assertEqual(result.success, len(datasets))
        self.assertEqual(dbcount, result.success)
        for index, dataset in enumerate(datasets):
            dataset.reload()
            self.assertEqual(dataset.extras['harvest:domain'], 'test.org')
            self.assertEqual(dataset.extras['harvest:remote_id'], str(index))
예제 #17
0
    def test_attach_does_not_duplicate(self):
        attached_datasets = []
        for i in range(2):
            dataset = DatasetFactory.build()
            dataset.extras['harvest:domain'] = 'test.org'
            dataset.extras['harvest:remote_id'] = str(i)
            dataset.last_modified = datetime.now()
            dataset.save()
            attached_datasets.append(dataset)

        datasets = DatasetFactory.create_batch(3)

        with NamedTemporaryFile() as csvfile:
            writer = csv.DictWriter(csvfile,
                                    fieldnames=['local', 'remote'],
                                    delimiter=b';',
                                    quotechar=b'"')

            writer.writeheader()
            for index, dataset in enumerate(datasets):
                writer.writerow({
                    'local': str(dataset.id),
                    'remote': str(index)
                })
            csvfile.flush()

            result = actions.attach('test.org', csvfile.name)

        dbcount = Dataset.objects(**{
            'extras__harvest:remote_id__exists': True
        }).count()
        self.assertEqual(result.success, len(datasets))
        self.assertEqual(dbcount, result.success)
        for index, dataset in enumerate(datasets):
            dataset.reload()
            self.assertEqual(dataset.extras['harvest:domain'], 'test.org')
            self.assertEqual(dataset.extras['harvest:remote_id'], str(index))
예제 #18
0
    def test_attach_skip_not_found(self):
        datasets = DatasetFactory.create_batch(3)

        with NamedTemporaryFile() as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=["local", "remote"], delimiter=b";", quotechar=b'"')

            writer.writeheader()
            writer.writerow({"local": "not-found", "remote": "42"})
            for index, dataset in enumerate(datasets):
                writer.writerow({"local": str(dataset.id), "remote": str(index)})
            csvfile.flush()

            result = actions.attach("test.org", csvfile.name)

        self.assertEqual(result.success, len(datasets))
        self.assertEqual(result.errors, 1)
예제 #19
0
    def test_attach_skip_not_found(self):
        datasets = DatasetFactory.create_batch(3)

        with NamedTemporaryFile() as csvfile:
            writer = csv.DictWriter(csvfile,
                                    fieldnames=['local', 'remote'],
                                    delimiter=b';',
                                    quotechar=b'"')

            writer.writeheader()
            writer.writerow({'local': 'not-found', 'remote': '42'})
            for index, dataset in enumerate(datasets):
                writer.writerow({
                    'local': str(dataset.id),
                    'remote': str(index)
                })
            csvfile.flush()

            result = actions.attach('test.org', csvfile.name)

        self.assertEqual(result.success, len(datasets))
        self.assertEqual(result.errors, 1)
예제 #20
0
 def test_redirect_datasets(self):
     dataset = DatasetFactory()
     response = self.client.get('/en/dataset/%s/' % dataset.slug)
     self.assertRedirects(
         response, url_for('datasets.show', dataset=dataset))
예제 #21
0
 def process(self, item):
     mock_process.send(self, item=item)
     return DatasetFactory.build(title='dataset-{0}'.format(item.remote_id))
예제 #22
0
 def process(self, item):
     mock_process.send(self, item=item)
     return DatasetFactory.build(title='dataset-{0}'.format(item.remote_id))