コード例 #1
0
ファイル: plugin.py プロジェクト: keitaroinc/ckanext-sweden
def latest_post():
    '''Return the most recent blog post.

    Returns None if there are no blog posts.

    :rtype: ckanext.sweden.blog.model.post.Post or None

    '''
    try:
        from ckanext.sweden.blog.model.post import Post
        post = Session.query(Post).\
            filter(Post.visible == True).\
            order_by('created desc').\
            first()
    except NoResultFound:
        return None

    if post is None:
        return None

    post.content_markdown = markdown(
        unicode(truncate(post.content, length=320, indicator='...',
                         whole_word=True)))
    post.post_author = (model.User.get(post.user_id)
                        or Session.query(model.User).filter_by(
                        id=post.user_id).first())

    return post
コード例 #2
0
def get_or_create_doi(package_id):
    '''Create or retrieve the unique identifier for this package_id.

    :param package_id:

    '''
    datacite_api = DOIDataCiteAPI()

    doi = get_doi(package_id)
    if doi is None:
        while True:
            identifier = os.path.join(
                get_prefix(), u'{0:07}'.format(random.randint(1, 100000)))

            # Check this identifier doesn't exist in the table
            if Session.query(DOI).filter(DOI.identifier == identifier).count():
                continue

            # And check against the datacite service
            try:
                datacite_doi = datacite_api.get(identifier)
            except HTTPError:
                pass
            else:
                if datacite_doi.text:
                    continue

            doi = DOI(package_id=package_id, identifier=identifier)
            Session.add(doi)
            Session.commit()
            break

    return doi
コード例 #3
0
 def setUp(self):
     def get_path(fname):
         return os.path.join(os.path.dirname(__file__),
                     '..', '..', '..', 'examples', fname)
     licenses = get_path('licenses.rdf')
     load_from_graph(path=licenses)
     Session.flush()
コード例 #4
0
    def test_create_extent(self):

        package = factories.Dataset()

        geojson = json.loads(self.geojson_examples["point"])

        shape = asShape(geojson)
        package_extent = PackageExtent(
            package_id=package["id"],
            the_geom=WKTElement(shape.wkt, self.db_srid),
        )
        package_extent.save()

        assert (package_extent.package_id == package["id"])
        if legacy_geoalchemy:
            assert (Session.scalar(
                package_extent.the_geom.x) == geojson["coordinates"][0])
            assert (Session.scalar(
                package_extent.the_geom.y) == geojson["coordinates"][1])
            assert (Session.scalar(
                package_extent.the_geom.srid) == self.db_srid)
        else:
            from sqlalchemy import func

            assert (Session.query(func.ST_X(package_extent.the_geom)).first()
                    [0] == geojson["coordinates"][0])
            assert (Session.query(func.ST_Y(package_extent.the_geom)).first()
                    [0] == geojson["coordinates"][1])
            assert (package_extent.the_geom.srid == self.db_srid)
コード例 #5
0
ファイル: cli.py プロジェクト: kindly/ckantest
 def moderate_change_apply(self, change):
     print "Change summary:"
     self.print_changes([change])
     print ""
     answer = raw_input("Do you want to apply this change? [Y/n] ").strip() or "y"
     answer = answer[0].lower()
     print ""
     if answer == 'y':
         return True
     else:
         print 
         answer = raw_input("Do you want to mask changes to this ref? [Y/n] ").strip() or "y"
         answer = answer[0].lower()
         print ""
         if answer == 'y':
             from ckan.model.changeset import ChangemaskRegister, Session
             register = ChangemaskRegister()
             mask = register.create_entity(change.ref)
             Session.add(mask)
             Session.commit()
             print "Mask has been set for ref: %s" % change.ref
             print ""
         else:
             print "Warning: Not setting a mask after not applying changes may lead to conflicts."
             import time
             time.sleep(5)
             print ""
コード例 #6
0
    def test_zfaulty_xml_unknown_errors(self):
        harv, job = self._create_harvester()
        res = "http://www.fsd.uta.fi/fi/aineistot/luettelo/FSD0115/FSD0115.xml"
        urllib2.urlopen = mock.Mock(return_value=StringIO(res))
        gathered = harv.gather_stage(job)

        urllib2.urlopen = mock.Mock(return_value=open("FSD2355.xml"))
        harvest_obj = HarvestObject.get(gathered[0])
        self.assert_(harv.fetch_stage(harvest_obj))
        self.assert_(harv.import_stage(harvest_obj))
        print Package.text_search(\
                            Session.query(Package),
                            'Kansalaiskeskustelu ydinvoimasta 2006').all()
        self.assert_(len(Package.text_search(\
                            Session.query(Package),
                            'Kansalaiskeskustelu ydinvoimasta 2006').all()) >= 1)

        res = "http://www.fsd.uta.fi/fi/aineistot/luettelo/FSD0115/FSD0115.xml"
        urllib2.urlopen = mock.Mock(return_value=StringIO(res))
        gathered = harv.gather_stage(job)
        urllib2.urlopen = mock.Mock(return_value=open("FSD2362.xml"))
        harvest_obj = HarvestObject.get(gathered[0])
        self.assert_(harv.fetch_stage(harvest_obj))
        self.assert_(harv.import_stage(harvest_obj))
        self.assert_(len(Package.text_search(\
                                Session.query(Package),
                                'Energia-asennetutkimus 2004').all()) >= 1)
コード例 #7
0
def load_themes():
    vocab_file_path = _get_path('data-theme-skos.rdf', 'vocabularies')

    class Opts(object):
        def __init__(self, filename, name, format):
            self.filename = filename
            self.url = None  #filename
            self.name = name
            self.format = format

    themes_loader.options = Opts(vocab_file_path, 'eu_themes', None)
    themes_loader.load()

    tag_localized = interfaces.get_localized_tag_name('ECON')
    Session.flush()
    assert tag_localized
    q = Session.query(Vocabulary).filter_by(name='eu_themes')
    vocab = q.first()
    assert vocab

    map_f = _get_path(MAPPING_FILE)
    voc_f = _get_path(EUROVOC_FILE)
    clear_subthemes()
    load_subthemes(map_f, voc_f)
    assert Subtheme.q().first()
コード例 #8
0
 def moderate_change_apply(self, change):
     print "Change summary:"
     self.print_changes([change])
     print ""
     answer = raw_input("Do you want to apply this change? [Y/n] ").strip() or "y"
     answer = answer[0].lower()
     print ""
     if answer == 'y':
         return True
     else:
         print 
         answer = raw_input("Do you want to mask changes to this ref? [Y/n] ").strip() or "y"
         answer = answer[0].lower()
         print ""
         if answer == 'y':
             from ckan.model.changeset import ChangemaskRegister, Session
             register = ChangemaskRegister()
             mask = register.create_entity(change.ref)
             Session.add(mask)
             Session.commit()
             print "Mask has been set for ref: %s" % change.ref
             print ""
         else:
             print "Warning: Not setting a mask after not applying changes may lead to conflicts."
             import time
             time.sleep(5)
             print ""
コード例 #9
0
    def test_update_extent(self):

        package = Package.get('annakarenina')

        geojson = json.loads(self.geojson_examples['point'])

        shape = asShape(geojson)
        package_extent = PackageExtent(package_id=package.id,
                                       the_geom=WKTSpatialElement(
                                           shape.wkt, self.db_srid))
        package_extent.save()
        assert Session.scalar(
            package_extent.the_geom.geometry_type) == 'ST_Point'

        # Update the geometry (Point -> Polygon)
        geojson = json.loads(self.geojson_examples['polygon'])

        shape = asShape(geojson)
        package_extent.the_geom = WKTSpatialElement(shape.wkt, self.db_srid)
        package_extent.save()

        assert package_extent.package_id == package.id
        assert Session.scalar(
            package_extent.the_geom.geometry_type) == 'ST_Polygon'
        assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
コード例 #10
0
    def setup(self):

        model.Session.remove()
        CreateTestData.create(auth_profile='publisher')
        self.sysadmin_user = model.User.get('testsysadmin')
        self.normal_user = model.User.get(
            'annafan')  # Does not belong to a publisher
        self.publisher1_user = model.User.by_name('russianfan')
        self.publisher2_user = model.User.by_name('tester')

        # Create two Publishers
        rev = model.repo.new_revision()
        self.publisher1 = model.Group(name=u'test-publisher1',
                                      title=u'Test Publihser 1',
                                      type=u'publisher')
        Session.add(self.publisher1)
        self.publisher2 = model.Group(name=u'test-publisher2',
                                      title=u'Test Publihser 2',
                                      type=u'publisher')
        Session.add(self.publisher2)

        member1 = model.Member(table_name='user',
                               table_id=self.publisher1_user.id,
                               group=self.publisher1,
                               capacity='admin')
        Session.add(member1)
        member2 = model.Member(table_name='user',
                               table_id=self.publisher2_user.id,
                               group=self.publisher2,
                               capacity='admin')
        Session.add(member2)

        Session.commit()
コード例 #11
0
    def test_tokenizer(self):

        load_from_graph(path=self.licenses)
        Session.flush()
        tokens = License.get_as_tokens()
        self.assertTrue(len(tokens.keys())>0)

        from_token, default = License.find_by_token('cc-by-sa')
        self.assertFalse(default)
        self.assertTrue(from_token)
        self.assertTrue('ccbysa' in from_token.uri.lower())

        from_token, default = License.find_by_token('cc-zero') #http://opendefinition.org/licenses/cc-zero/')
        self.assertFalse(default)
        self.assertTrue(from_token)

        self.assertTrue('PublicDomain' in from_token.license_type)
        
        from_token, default = License.find_by_token('Creative Commons Attribuzione') #http://opendefinition.org/licenses/cc-zero/')
        self.assertFalse(default)
        self.assertTrue(from_token)

        self.assertTrue('Attribution' in from_token.license_type)

        odbl = """["Open Data Commons Open Database License / OSM (ODbL/OSM): You are free to copy, distribute, transmit and adapt our data, as long as you credit OpenStreetMap and its contributors\nIf you alter or build upon our data, you may distribute the result only under the same licence. (http://www.openstreetmap.org/copyright)"]"""

        from_token, default = License.find_by_token(odbl, 'other')
        self.assertFalse(default)
        self.assertTrue(from_token)
        self.assertTrue('odbl' in from_token.default_name.lower())
コード例 #12
0
    def _run_job_for_single_document(
        self, job, force_import=False, expect_gather_errors=False, expect_obj_errors=False
    ):

        harvester = GeminiDocHarvester()

        harvester.force_import = force_import

        object_ids = harvester.gather_stage(job)
        assert object_ids, len(object_ids) == 1
        if expect_gather_errors:
            assert len(job.gather_errors) > 0
        else:
            assert len(job.gather_errors) == 0

        assert harvester.fetch_stage(object_ids) == True

        obj = HarvestObject.get(object_ids[0])
        assert obj, obj.content

        harvester.import_stage(obj)
        Session.refresh(obj)
        if expect_obj_errors:
            assert len(obj.errors) > 0
        else:
            assert len(obj.errors) == 0

        job.status = u"Finished"
        job.save()

        return obj
コード例 #13
0
def persist_tag_multilang(tag: model.Tag, lang, label, vocab):
    log.debug('DCAT-AP_IT: persisting tag multilang for tag %r ...', tag.name)

    tag_loc = TagLocalization.by_tag_id(tag.id, lang)

    if tag_loc:
        # Update the existing record
        if label:
            if label != tag_loc.text:
                try:
                    tag_loc.text = label
                    tag_loc.save()
                    return DBAction.UPDATED, tag_loc.id
                except Exception as err:
                    # on rollback, the same closure of state
                    # as that of commit proceeds.
                    Session.rollback()

                    log.error(
                        'Exception occurred while persisting DB objects: %s',
                        err)
                    raise
            else:
                return DBAction.NONE, tag_loc.id
        else:
            log.warning(
                f'Skipping empty label V:{vocab.name} T:{tag.name} L:{lang}')
            return DBAction.ERROR, tag_loc.id
    else:
        # Create a new localized record
        tag_loc = TagLocalization.persist(tag, label, lang)
        return DBAction.CREATED, tag_loc.id
コード例 #14
0
    def make_package_name(self, title, exclude_existing_package):
        '''
        Creates a URL friendly name from a title

        If the name already exists, it will add some random characters at the end
        '''

        name = munge_title_to_name(title).replace('_', '-')
        while '--' in name:
            name = name.replace('--', '-')
        name = name[0:90] # max length is 100

        # Is this slug already in use (and if we're updating a package, is it in
        # use by a different package?).
        pkg_obj = Session.query(Package).filter(Package.name == name).filter(Package.id != exclude_existing_package).first()
        if not pkg_obj:
            # The name is available, so use it. Note that if we're updating an
            # existing package we will be updating this package's URL, so incoming
            # links may break.
            return name

        if exclude_existing_package:
            # The name is not available, and we're updating a package. Chances
            # are the package's name already had some random string attached
            # to it last time. Prevent spurrious updates to the package's URL
            # (choosing new random text) by just reusing the existing package's
            # name.
            pkg_obj = Session.query(Package).filter(Package.id == exclude_existing_package).first()
            if pkg_obj: # the package may not exist yet because we may be passed the desired package GUID before a new package is instantiated
                return pkg_obj.name

        # Append some random text to the URL. Hope that with five character
        # there will be no collsion.
        return name + "-" + str(uuid.uuid4())[:5]
コード例 #15
0
    def _test_auth_not_allowed(self, user_name=None, source=None, status=401):

        if not source:
            # Create harvest source
            source = HarvestSource(url=u'http://test-source.com', type='ckan')
            Session.add(source)
            Session.commit()

        if user_name:
            extra_environ = {'REMOTE_USER': user_name.encode('utf8')}
        else:
            extra_environ = {}

        # List
        res = self.app.get('/harvest',
                           status=status,
                           extra_environ=extra_environ)
        # Create
        res = self.app.get('/harvest/new',
                           status=status,
                           extra_environ=extra_environ)
        # Read
        res = self.app.get('/harvest/%s' % source.id,
                           status=status,
                           extra_environ=extra_environ)
        # Edit
        res = self.app.get('/harvest/edit/%s' % source.id,
                           status=status,
                           extra_environ=extra_environ)
        # Refresh
        res = self.app.get('/harvest/refresh/%s' % source.id,
                           status=status,
                           extra_environ=extra_environ)
コード例 #16
0
ファイル: lib.py プロジェクト: okfn/ckanext-doi
def create_doi_from_identifier(package_id, identifier):
    """Can be called when an identifier has already been created elsewhere.
    Does not ensure the identifier is unique"""
    doi = DOI(package_id=package_id, identifier=identifier)
    Session.add(doi)
    Session.commit()
    return doi
コード例 #17
0
    def test_auth_publisher_profile_different_publisher(self):

        # Create a source for publisher 1
        source = HarvestSource(url=u'http://test-source.com',
                               type='ckan',
                               publisher_id=self.publisher1.id)
        Session.add(source)
        Session.commit()

        extra_environ = {
            'REMOTE_USER': self.publisher2_user.name.encode('utf8')
        }

        # List (Publihsers can see the sources list)
        res = self.app.get('/harvest', extra_environ=extra_environ)
        assert 'Harvesting Sources' in res
        # Create
        res = self.app.get('/harvest/new', extra_environ=extra_environ)
        assert 'New harvest source' in res
        assert 'publisher_id' in res

        # Check that this publihser is not allowed to manage sources from other publishers
        status = 401
        # Read
        res = self.app.get('/harvest/%s' % source.id,
                           status=status,
                           extra_environ=extra_environ)
        # Edit
        res = self.app.get('/harvest/edit/%s' % source.id,
                           status=status,
                           extra_environ=extra_environ)
        # Refresh
        res = self.app.get('/harvest/refresh/%s' % source.id,
                           status=status,
                           extra_environ=extra_environ)
コード例 #18
0
def setup(srid=None):

    if package_extent_table is None:
        define_spatial_tables(srid)
        log.debug('Spatial tables defined in memory')

    if model.package_table.exists():
        if not Table('geometry_columns', meta.metadata).exists() or \
           not Table('spatial_ref_sys', meta.metadata).exists():
            raise Exception(
                'The spatial extension is enabled, but PostGIS '
                'has not been set up in the database. '
                'Please refer to the "Setting up PostGIS" section in the README.'
            )

        if not package_extent_table.exists():
            try:
                package_extent_table.create()
            except Exception as e:
                # Make sure the table does not remain incorrectly created
                # (eg without geom column or constraints)
                if package_extent_table.exists():
                    Session.execute('DROP TABLE package_extent')
                    Session.commit()

                raise e

            log.debug('Spatial tables created')
        else:
            log.debug('Spatial tables already exist')
            # Future migrations go here

    else:
        log.debug('Spatial tables creation deferred')
コード例 #19
0
 def _refresh_harvest_objects(self, harvest_object, package_id):
     """
     Perform harvester housekeeping:
         - Flag the other objects of the source as not current
         - Set a refernce to the package in the harvest object
         - Flag it as current
         - And save the changes
     """
     # Flag the other objects of this source as not current
     from ckanext.harvest.model import harvest_object_table
     u = update(harvest_object_table) \
         .where(harvest_object_table.c.package_id == bindparam('pkg_id')) \
         .values(current=False)
     Session.execute(u, params={'pkg_id': package_id})
     Session.commit()
     # Refresh current object from session, otherwise the
     # import paster command fails
     # (Copied from the Gemini harvester--not sure if necessary)
     Session.remove()
     Session.add(harvest_object)
     Session.refresh(harvest_object)
     # Set reference to package in the HarvestObject and flag it as
     # the current one
     if not harvest_object.package_id:
         harvest_object.package_id = package_id
     harvest_object.current = True
     harvest_object.save()
コード例 #20
0
def setup(srid=None):

    if package_extent_table is None:
        define_spatial_tables(srid)
        # log.debug('Spatial tables defined in memory')

    if model.package_table.exists():
        if not Table('geometry_columns',meta.metadata).exists() or \
            not Table('spatial_ref_sys',meta.metadata).exists():
            raise Exception('The spatial extension is enabled, but PostGIS ' + \
                    'has not been set up in the database. ' + \
                    'Please refer to the "Setting up PostGIS" section in the README.')


        if not package_extent_table.exists():
            try:
                package_extent_table.create()
            except Exception,e:
                # Make sure the table does not remain incorrectly created
                # (eg without geom column or constraints)
                if package_extent_table.exists():
                    Session.execute('DROP TABLE package_extent')
                    Session.commit()

                raise e

            log.debug('Spatial tables created')
        else:
            # log.debug('Spatial tables already exist')
			doNothing = "true"
コード例 #21
0
 def setup_class(cls):
     """
     Remove any initial sessions.
     """
     Session.remove()
     # TODO: Should also remove test data
     CreateTestData.create()
コード例 #22
0
    def delete_tests(self):

        print 'Deleting all test DOIs'
        Session.query(DOI).filter(
            DOI.identifier.like('%' + TEST_PREFIX +
                                '%')).delete(synchronize_session=False)
        Session.commit()
コード例 #23
0
def create_unique_identifier(package_id):
    """
    Create a unique identifier, using the prefix and a random number: 10.5072/0044634
    Checks the random number doesn't exist in the table or the datacite repository
    All unique identifiers are created with
    @return:
    """
    datacite_api = DOIDataCiteAPI()

    while True:

        identifier = os.path.join(get_prefix(), '{0:07}'.format(random.randint(1, 100000)))

        # Check this identifier doesn't exist in the table
        if not Session.query(DOI).filter(DOI.identifier == identifier).count():

            # And check against the datacite service
            try:
                datacite_doi = datacite_api.get(identifier)
            except HTTPError:
                pass
            else:
                if datacite_doi.text:
                    continue

        doi = DOI(package_id=package_id, identifier=identifier)
        Session.add(doi)
        Session.commit()

        return doi
コード例 #24
0
def create_application(data_dict, image):
    if 'image' in data_dict:
        del data_dict['image']
    data, errors = validate(data_dict, application_schema())
    if errors:
        raise ValidationError(errors, error_summary(errors))

    application = Application(
            name=Application.generate_name(data.get('title')),
            title=data.get('title'),
            url=data.get('url'),
            description=data.get('description'),
            featured=data.get('featured'),
            submitter=data.get('submitter'),
            developer=data.get('developer'),
            developer_url=data.get('developer_url'),
            license=data.get('license'),
            code_url=data.get('code_url'),
            api_url=data.get('api_url'),
        )

    tags = data.get('tags', '').split(' ')
    application.update_tags(tags)

    if image and image.filename and image.file:
        image = ApplicationImage(name=image.filename, 
            data=image.file.read(),
            application=application)
        Session.add(image)
    application.save()
    return application
コード例 #25
0
 def create(cls, term, label, lang):
     inst = cls(term=term,
                label=label,
                lang=lang)
     Session.add(inst)
     Session.flush()
     return inst
コード例 #26
0
    def test_spatial_extra_base(self, app):

        user = factories.User()
        env = {"REMOTE_USER": user["name"].encode("ascii")}
        dataset = factories.Dataset(user=user)

        offset = url_for("dataset.edit", id=dataset["id"])
        res = app.get(offset, extra_environ=env)

        data = {
            "name": dataset['name'],
            "extras__0__key": u"spatial",
            "extras__0__value": self.geojson_examples["point"]
        }

        res = app.post(offset, environ_overrides=env, data=data)

        assert "Error" not in res, res

        package_extent = (Session.query(PackageExtent).filter(
            PackageExtent.package_id == dataset["id"]).first())

        geojson = json.loads(self.geojson_examples["point"])

        assert package_extent.package_id == dataset["id"]
        from sqlalchemy import func

        assert (Session.query(func.ST_X(
            package_extent.the_geom)).first()[0] == geojson["coordinates"][0])
        assert (Session.query(func.ST_Y(
            package_extent.the_geom)).first()[0] == geojson["coordinates"][1])
        assert package_extent.the_geom.srid == self.db_srid
コード例 #27
0
def _create_unique_identifier(package_doi_identifier=None):
    datacite_api = DOIDataCiteAPI()
    while True:
        if package_doi_identifier:
            identifier = os.path.join(package_doi_identifier,
                                      '{0:03}'.format(random.randint(1, 999)))
            query = Session.query(CeonResourceDOI)
            query = query.filter(CeonResourceDOI.identifier == identifier)
            exists = query.count()
        else:
            identifier = os.path.join(
                get_doi_prefix(), '{0:07}'.format(random.randint(1, 9999999)))
            query = Session.query(CeonPackageDOI)
            query = query.filter(CeonPackageDOI.identifier == identifier)
            exists = query.count()
        # Check this identifier doesn't exist in the table
        if not exists:
            # And check against the datacite service
            try:
                datacite_doi = datacite_api.get(identifier)
            except HTTPError:
                pass
            # TODO remove the nest 2 lines (ConnectionError) ignoring
            except ConnectionError:
                pass
            else:
                if datacite_doi.text:
                    continue
        return identifier
コード例 #28
0
    def authenticate(self, environ, identity):
        """ Mimic most of UsernamePasswordAuthenticator.authenticate
        but add account lockout after 10 failed attempts.
        """
        if 'login' not in identity or 'password' not in identity:
            return None
        user = User.by_name(identity.get('login'))
        if user is None:
            LOG.debug('Login failed - username %r not found',
                      identity.get('login'))
            return None

        qgov_user = Session.query(QGOVUser).filter_by(
            name=identity.get('login')).first()
        if qgov_user.login_attempts >= 10:
            LOG.debug('Login as %r failed - account is locked',
                      identity.get('login'))
        elif user.validate_password(identity.get('password')):
            # reset attempt count to 0
            qgov_user.login_attempts = 0
            Session.commit()
            return user.name
        else:
            LOG.debug('Login as %r failed - password not valid',
                      identity.get('login'))

        qgov_user.login_attempts += 1
        Session.commit()
        return None
コード例 #29
0
    def test_create_extent(self):

        package = factories.Dataset()

        geojson = json.loads(self.geojson_examples['point'])

        shape = asShape(geojson)
        package_extent = PackageExtent(package_id=package['id'],
                                       the_geom=WKTElement(shape.wkt,
                                                           self.db_srid))
        package_extent.save()

        assert_equals(package_extent.package_id, package['id'])
        if legacy_geoalchemy:
            assert_equals(Session.scalar(package_extent.the_geom.x),
                          geojson['coordinates'][0])
            assert_equals(Session.scalar(package_extent.the_geom.y),
                          geojson['coordinates'][1])
            assert_equals(Session.scalar(package_extent.the_geom.srid),
                          self.db_srid)
        else:
            from sqlalchemy import func
            assert_equals(
                Session.query(func.ST_X(package_extent.the_geom)).first()[0],
                geojson['coordinates'][0])
            assert_equals(
                Session.query(func.ST_Y(package_extent.the_geom)).first()[0],
                geojson['coordinates'][1])
            assert_equals(package_extent.the_geom.srid, self.db_srid)
コード例 #30
0
    def test_new(self):
        name = 'test-spatial-dataset-1'

        offset = url_for(controller='package', action='new')
        res = self.app.get(offset, extra_environ=self.extra_environ)
        assert 'Add - Datasets' in res
        fv = res.forms['dataset-edit']
        prefix = ''
        fv[prefix + 'name'] = name
        fv[prefix + 'extras__0__key'] = u'spatial'
        fv[prefix + 'extras__0__value'] = self.geojson_examples['point']

        res = fv.submit('save', extra_environ=self.extra_environ)
        assert not 'Error' in res, res

        package = Package.get(name)

        # Check that a PackageExtent object has been created
        package_extent = Session.query(PackageExtent).filter(
            PackageExtent.package_id == package.id).first()

        geojson = json.loads(self.geojson_examples['point'])

        assert package_extent
        assert package_extent.package_id == package.id
        assert Session.scalar(
            package_extent.the_geom.x) == geojson['coordinates'][0]
        assert Session.scalar(
            package_extent.the_geom.y) == geojson['coordinates'][1]
        assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
コード例 #31
0
    def _run_job_for_single_document(self,job,force_import=False,expect_gather_errors=False,expect_obj_errors=False):

        harvester = GeminiDocHarvester()

        harvester.force_import = force_import


        object_ids = harvester.gather_stage(job)
        assert object_ids, len(object_ids) == 1
        if expect_gather_errors:
            assert len(job.gather_errors) > 0
        else:
            assert len(job.gather_errors) == 0

        assert harvester.fetch_stage(object_ids) == True

        obj = HarvestObject.get(object_ids[0])
        assert obj, obj.content

        harvester.import_stage(obj)
        Session.refresh(obj)
        if expect_obj_errors:
            assert len(obj.errors) > 0
        else:
            assert len(obj.errors) == 0

        job.status = u'Finished'
        job.save()

        return obj
コード例 #32
0
ファイル: controller.py プロジェクト: Psykar/ckanext-ands
    def handle_submit(self, id):
        data = clean_dict(dict_fns.unflatten(tuplize_dict(parse_params(
            request.params))))

        data['dataset_url'] = toolkit.url_for(
            controller='package',
            action='read',
            id=id,
            qualified=True
        )

        package = get_action('package_show')(None, {'id': id})
        self.fail_if_private(package, data['dataset_url'])

        # Comma separated config var
        to_addrs = config['ckanext.ands.support_emails'].split(',')

        subject = 'DataPortal Support: Request to publish dataset'

        body = base.render(
            'package/doi_email.text',
            extra_vars=data)

        for email in to_addrs:
            mail_recipient('Dataportal support', email, subject, body)

        data['package_id'] = package['id']
        data['user_id'] = c.userobj.id

        doi_request = DoiRequest(**data)
        Session.add(doi_request)
        Session.commit()

        h.flash_success("DOI Request sent")
        return toolkit.redirect_to(data['dataset_url'])
コード例 #33
0
ファイル: controller.py プロジェクト: Psykar/ckanext-ands
    def handle_submit(self, id):
        data = clean_dict(
            dict_fns.unflatten(tuplize_dict(parse_params(request.params))))

        data['dataset_url'] = toolkit.url_for(controller='package',
                                              action='read',
                                              id=id,
                                              qualified=True)

        package = get_action('package_show')(None, {'id': id})
        self.fail_if_private(package, data['dataset_url'])

        # Comma separated config var
        to_addrs = config['ckanext.ands.support_emails'].split(',')

        subject = 'DataPortal Support: Request to publish dataset'

        body = base.render('package/doi_email.text', extra_vars=data)

        for email in to_addrs:
            mail_recipient('Dataportal support', email, subject, body)

        data['package_id'] = package['id']
        data['user_id'] = c.userobj.id

        doi_request = DoiRequest(**data)
        Session.add(doi_request)
        Session.commit()

        h.flash_success("DOI Request sent")
        return toolkit.redirect_to(data['dataset_url'])
コード例 #34
0
    def authenticate(self, environ, identity):
        if not 'login' in identity or not 'password' in identity:
            return None
        user = User.by_name(identity.get('login'))
        if user is None:
            log.debug('Login failed - username %r not found',
                      identity.get('login'))
            return None

        seedUser = Session.query(SEEDUser).filter_by(
            name=identity.get('login')).first()
        if seedUser.login_attempts >= 10:
            log.debug('Login as %r failed - account is locked',
                      identity.get('login'))
        elif user.validate_password(identity.get('password')):
            # reset attempt count to 0
            seedUser.login_attempts = 0
            Session.commit()
            return user.name
        else:
            log.debug('Login as %r failed - password not valid',
                      identity.get('login'))

        seedUser.login_attempts += 1
        Session.commit()
        return None
コード例 #35
0
ファイル: __init__.py プロジェクト: tilemapjp/ckanext-spatial
def bbox_query_ordered(bbox, srid=None):
    '''
    Performs a spatial query of a bounding box. Returns packages in order
    of how similar the data\'s bounding box is to the search box (best first).

    bbox - bounding box dict

    Returns a query object of PackageExtents, which each reference a package
    by ID.
    '''

    input_geometry = _bbox_2_wkt(bbox, srid)

    params = {'query_bbox': str(input_geometry),
              'query_srid': input_geometry.srid}

    # First get the area of the query box
    sql = "SELECT ST_Area(GeomFromText(:query_bbox, :query_srid));"
    params['search_area'] = Session.execute(sql, params).fetchone()[0]

    # Uses spatial ranking method from "USGS - 2006-1279" (Lanfear)
    sql = """SELECT ST_AsBinary(package_extent.the_geom) AS package_extent_the_geom,
                    POWER(ST_Area(ST_Intersection(package_extent.the_geom, GeomFromText(:query_bbox, :query_srid))),2)/ST_Area(package_extent.the_geom)/:search_area as spatial_ranking,
                    package_extent.package_id AS package_id
             FROM package_extent, package
             WHERE package_extent.package_id = package.id
                AND ST_Intersects(package_extent.the_geom, GeomFromText(:query_bbox, :query_srid))
                AND package.state = 'active'
             ORDER BY spatial_ranking desc"""
    extents = Session.execute(sql, params).fetchall()
    log.debug('Spatial results: %r',
              [('%.2f' % extent.spatial_ranking, extent.package_id) for extent in extents[:20]])
    return extents
コード例 #36
0
ファイル: lib.py プロジェクト: ERC-data/ckanext-doi
def create_unique_identifier(package_id):
    """
    Create a unique identifier, using the prefix and a random number: 10.5072/0044634
    Checks the random number doesn't exist in the table or the datacite repository
    All unique identifiers are created with
    @return:
    """
    datacite_api = DOIDataCiteAPI()

    while True:

        identifier = os.path.join(get_prefix(), '{0:07}'.format(random.randint(1, 100000)))

        # Check this identifier doesn't exist in the table
        if not Session.query(DOI).filter(DOI.identifier == identifier).count():

            # And check against the datacite service
            try:
                datacite_doi = datacite_api.get(identifier)
            except HTTPError:
                pass
            else:
                if datacite_doi.text:
                    continue

        doi = DOI(package_id=package_id, identifier=identifier)
        Session.add(doi)
        Session.commit()

        return doi
コード例 #37
0
    def test_create_extent(self):

        package = factories.Dataset()

        geojson = json.loads(self.geojson_examples['point'])

        shape = asShape(geojson)
        package_extent = PackageExtent(package_id=package['id'],
                                       the_geom=WKTElement(
                                           shape.wkt, self.db_srid))
        package_extent.save()

        assert_equals(package_extent.package_id, package['id'])
        if legacy_geoalchemy:
            assert_equals(Session.scalar(package_extent.the_geom.x),
                          geojson['coordinates'][0])
            assert_equals(Session.scalar(package_extent.the_geom.y),
                          geojson['coordinates'][1])
            assert_equals(Session.scalar(package_extent.the_geom.srid),
                          self.db_srid)
        else:
            from sqlalchemy import func
            assert_equals(
                Session.query(func.ST_X(package_extent.the_geom)).first()[0],
                geojson['coordinates'][0])
            assert_equals(
                Session.query(func.ST_Y(package_extent.the_geom)).first()[0],
                geojson['coordinates'][1])
            assert_equals(package_extent.the_geom.srid, self.db_srid)
コード例 #38
0
def update_extents():
    from ckan.model import PackageExtra, Package, Session
    conn = Session.connection()
    packages = [extra.package \
                for extra in \
                Session.query(PackageExtra).filter(PackageExtra.key == 'spatial').all()]

    errors = []
    count = 0
    for package in packages:
        try:
            value = package.extras['spatial']
            log.debug('Received: %r' % value)
            geometry = json.loads(value)

            count += 1
        except ValueError as e:
            errors.append(u'Package %s - Error decoding JSON object: %s' %
                          (package.id, six.text_type(e)))
        except TypeError as e:
            errors.append(u'Package %s - Error decoding JSON object: %s' %
                          (package.id, six.text_type(e)))

        save_package_extent(package.id, geometry)

    Session.commit()

    if errors:
        msg = 'Errors were found:\n%s' % '\n'.join(errors)
        print(msg)

    msg = "Done. Extents generated for %i out of %i packages" % (count,
                                                                 len(packages))

    print(msg)
コード例 #39
0
ファイル: base.py プロジェクト: raphaelstolt/ckanext-harvest
 def _save_object_error(self, message, obj, stage=u"Fetch", line=None):
     err = HarvestObjectError(message=message, object=obj, stage=stage, line=line)
     try:
         err.save()
     except InvalidRequestError, e:
         Session.rollback()
         err.save()
コード例 #40
0
    def test_new(self):
        name = "test-spatial-dataset-1"

        offset = url_for(controller="package", action="new")
        res = self.app.get(offset, extra_environ=self.extra_environ)
        assert "Add - Datasets" in res
        fv = res.forms["dataset-edit"]
        prefix = ""
        fv[prefix + "name"] = name
        fv[prefix + "extras__0__key"] = u"spatial"
        fv[prefix + "extras__0__value"] = self.geojson_examples["point"]

        res = fv.submit("save", extra_environ=self.extra_environ)
        assert not "Error" in res, res

        package = Package.get(name)

        # Check that a PackageExtent object has been created
        package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id == package.id).first()

        geojson = json.loads(self.geojson_examples["point"])

        assert package_extent
        assert package_extent.package_id == package.id
        assert Session.scalar(package_extent.the_geom.x) == geojson["coordinates"][0]
        assert Session.scalar(package_extent.the_geom.y) == geojson["coordinates"][1]
        assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
コード例 #41
0
ファイル: __init__.py プロジェクト: GSA/ckanext-spatial
def bbox_query_ordered(bbox, srid=None):
    '''
    Performs a spatial query of a bounding box. Returns packages in order
    of how similar the data\'s bounding box is to the search box (best first).

    bbox - bounding box dict

    Returns a query object of PackageExtents, which each reference a package
    by ID.
    '''

    input_geometry = _bbox_2_wkt(bbox, srid)

    params = {'query_bbox': str(input_geometry),
              'query_srid': input_geometry.srid}

    # First get the area of the query box
    sql = "SELECT ST_Area(ST_GeomFromText(:query_bbox, :query_srid));"
    params['search_area'] = Session.execute(sql, params).fetchone()[0]

    # Uses spatial ranking method from "USGS - 2006-1279" (Lanfear)
    sql = """SELECT ST_AsBinary(package_extent.the_geom) AS package_extent_the_geom,
                    POWER(ST_Area(ST_Intersection(package_extent.the_geom, ST_GeomFromText(:query_bbox, :query_srid))),2)/ST_Area(package_extent.the_geom)/:search_area as spatial_ranking,
                    package_extent.package_id AS package_id
             FROM package_extent, package
             WHERE package_extent.package_id = package.id
                AND ST_Intersects(package_extent.the_geom, ST_GeomFromText(:query_bbox, :query_srid))
                AND package.state = 'active'
             ORDER BY spatial_ranking desc"""
    extents = Session.execute(sql, params).fetchall()
    log.debug('Spatial results: %r',
              [('%.2f' % extent.spatial_ranking, extent.package_id) for extent in extents[:20]])
    return extents
コード例 #42
0
ファイル: doi.py プロジェクト: kchod/ckanext-ceon
def _create_unique_identifier(package_doi_identifier=None):
    datacite_api = DOIDataCiteAPI()
    while True:
        if package_doi_identifier:
            identifier = os.path.join(package_doi_identifier,
                    '{0:03}'.format(random.randint(1, 999)))
            query = Session.query(CeonResourceDOI)
            query = query.filter(CeonResourceDOI.identifier == identifier)
            exists = query.count()
        else:
            identifier = os.path.join(get_doi_prefix(),
                    '{0:07}'.format(random.randint(1, 9999999)))
            query = Session.query(CeonPackageDOI)
            query = query.filter(CeonPackageDOI.identifier == identifier)
            exists = query.count()
        # Check this identifier doesn't exist in the table
        if not exists:
            # And check against the datacite service
            try:
                datacite_doi = datacite_api.get(identifier)
            except HTTPError:
                pass
            # TODO remove the nest 2 lines (ConnectionError) ignoring
            except ConnectionError:
                pass
            else:
                if datacite_doi.text:
                    continue
        return identifier
コード例 #43
0
    def test_auth_publisher_profile_different_publisher(self):

        # Create a source for publisher 1
        source = HarvestSource(url=u'http://test-source.com',type='ckan',
                               publisher_id=self.publisher1.id)
        Session.add(source)
        Session.commit()

        extra_environ = {'REMOTE_USER': self.publisher2_user.name.encode('utf8')}

        # List (Publihsers can see the sources list)
        res = self.app.get('/harvest', extra_environ=extra_environ)
        assert 'Harvesting Sources' in res
        # Create
        res = self.app.get('/harvest/new', extra_environ=extra_environ)
        assert 'New harvest source' in res
        assert 'publisher_id' in res

        # Check that this publihser is not allowed to manage sources from other publishers
        status = 401
        # Read
        res = self.app.get('/harvest/%s' % source.id, status=status, extra_environ=extra_environ)
        # Edit
        res = self.app.get('/harvest/edit/%s' % source.id, status=status, extra_environ=extra_environ)
        # Refresh
        res = self.app.get('/harvest/refresh/%s' % source.id, status=status, extra_environ=extra_environ)
コード例 #44
0
    def setup(self):

        model.Session.remove()
        CreateTestData.create(auth_profile='publisher')
        self.sysadmin_user = model.User.get('testsysadmin')
        self.normal_user = model.User.get('annafan') # Does not belong to a publisher
        self.publisher1_user = model.User.by_name('russianfan')
        self.publisher2_user = model.User.by_name('tester')

        # Create two Publishers
        rev = model.repo.new_revision()
        self.publisher1 = model.Group(name=u'test-publisher1',title=u'Test Publihser 1',type=u'publisher')
        Session.add(self.publisher1)
        self.publisher2 = model.Group(name=u'test-publisher2',title=u'Test Publihser 2',type=u'publisher')
        Session.add(self.publisher2)

        member1 = model.Member(table_name = 'user',
                         table_id = self.publisher1_user.id,
                         group=self.publisher1,
                         capacity='admin')
        Session.add(member1)
        member2 = model.Member(table_name = 'user',
                         table_id = self.publisher2_user.id,
                         group=self.publisher2,
                         capacity='admin')
        Session.add(member2)

        Session.commit()
コード例 #45
0
    def make_package_name(self, title, exclude_existing_package):
        '''
        Creates a URL friendly name from a title

        If the name already exists, it will add some random characters at the end
        '''

        name = munge_title_to_name(title).replace('_', '-')
        while '--' in name:
            name = name.replace('--', '-')
        name = name[0:90] # max length is 100

        # Is this slug already in use (and if we're updating a package, is it in
        # use by a different package?).
        pkg_obj = Session.query(Package).filter(Package.name == name).filter(Package.id != exclude_existing_package).first()
        if not pkg_obj:
            # The name is available, so use it. Note that if we're updating an
            # existing package we will be updating this package's URL, so incoming
            # links may break.
            return name

        if exclude_existing_package:
            # The name is not available, and we're updating a package. Chances
            # are the package's name already had some random string attached
            # to it last time. Prevent spurrious updates to the package's URL
            # (choosing new random text) by just reusing the existing package's
            # name.
            pkg_obj = Session.query(Package).filter(Package.id == exclude_existing_package).first()
            if pkg_obj: # the package may not exist yet because we may be passed the desired package GUID before a new package is instantiated
                return pkg_obj.name

        # Append some random text to the URL. Hope that with five character
        # there will be no collsion.
        return name + "-" + str(uuid.uuid4())[:5]
コード例 #46
0
    def test_ckan_duplicated_name(self):
        dataset0 = {
            'owner_org': self.org['id'],
            'holder_name': 'test holder',
            'holder_identifier': 'abcdef',
            'notes': 'some notes',
            'modified': '2000-01-01',
            'theme': 'AGRI',
            'frequency': 'UNKNOWN',
            'publisher_name': 'publisher',
            'identifier': 'aasdfa',
            'publisher_identifier': 'publisher',
            'resources': [],
            'extras': [],
        }

        dataset1 = {
            'owner_org': self.org['id'],
            'title': 'duplicated title',
            'name': 'duplicated-title',
            'id': 'dummyid'
        }
        dataset1.update(dataset0)
        data = json.dumps(dataset1)

        harvest_dict = self._create_harvest_obj('http://mock/source/', name='dupname1', owner_org=self.org['id'])
        harvest_obj = HarvestObject.get(harvest_dict['id'])
        harvest_obj.content = data
        h = DCATRDFHarvester()
        import_successful = h.import_stage(harvest_obj)
        self.assertTrue(import_successful, harvest_obj.errors)
        Session.flush()
        dataset1['_id'] = harvest_obj.package_id

        dataset2 = {'title': 'duplicated title',
                    'name': 'duplicated-title',
                    'id': 'dummyid2'}

        dataset2.update(dataset0)
        dataset2['identifier'] = 'otherid'
        data = json.dumps(dataset2)

        harvest_dict = self._create_harvest_obj('http://mock/source/', name='dupname2', owner_org=self.org['id'])
        harvest_obj = HarvestObject.get(harvest_dict['id'])
        harvest_obj.content = data
        h = DCATRDFHarvester()
        import_successful = h.import_stage(harvest_obj)
        self.assertTrue(import_successful, harvest_obj.errors)
        Session.flush()
        dataset2['_id'] = harvest_obj.package_id

        # duplicated names are mangled, one should have numeric suffix
        pkg_dict = helpers.call_action('package_show', context={}, name_or_id=dataset1['_id'])
        self.assertEqual(pkg_dict['title'], dataset1['title'])
        self.assertEqual(pkg_dict['name'], 'duplicated-title')

        pkg_dict = helpers.call_action('package_show', context={}, name_or_id=dataset2['_id'])
        self.assertEqual(pkg_dict['title'], dataset2['title'])
        self.assertEqual(pkg_dict['name'], 'duplicated-title1')
コード例 #47
0
ファイル: test_irods.py プロジェクト: kata-csc/ckanext-irods
 def setup_class(self):
     """
     Remove any initial sessions.
     """
     Session.remove()
     # TODO: Should also remove test data
     CreateTestData.create()
     self.sysadmin = model.User.get('testsysadmin')
コード例 #48
0
 def get(cls, reference):
     """Returns a IssueCategory object referenced by its id or name."""
     if type(reference) is int:
         # if reference is an integer, get by ID
         return Session.query(cls).filter(cls.id == reference).first()
     else:
         # if not, get by name
         return Session.query(cls).filter(cls.name == reference).first()
コード例 #49
0
ファイル: base.py プロジェクト: okfn/ckanext-ngds
    def _execute_sql(cls, script):
        engine = create_engine(cls.sqlalchemy_url)
        Session.bind = engine
        
        connection = Session.connection()

        connection.execute(script)
        Session.commit() 
コード例 #50
0
 def _create_pkg(self):
     model.repo.new_revision()
     pkg = Package.get("annakarenina")
     pkg.name = "fookarenina"
     pkg.add_resource("www.google.com", description="foo", name="foo")
     Session.add(pkg)
     Session.commit()
     return pkg
コード例 #51
0
ファイル: test_sitemap.py プロジェクト: skvisha/hdx-ckan
 def _create_pkg(self):
     model.repo.new_revision()
     pkg = Package.get('annakarenina')
     pkg.name = "fookarenina"
     pkg.add_resource('www.google.com', description='foo', name="foo")
     Session.add(pkg)
     Session.commit()
     return pkg
コード例 #52
0
    def import_stage(self, harvest_object):
        log.debug('In SwisstopoHarvester import_stage')

        if not harvest_object:
            log.error('No harvest object received')
            return False

        try:
            package_dict = json.loads(harvest_object.content)

            package_dict['id'] = harvest_object.guid
            package_dict['name'] = munge_title_to_name(
                package_dict['layer_name']
            )
            user = model.User.get(self.HARVEST_USER)
            context = {
                'model': model,
                'session': Session,
                'user': self.HARVEST_USER
                }

            # Find or create group the dataset should get assigned to
            package_dict['groups'] = self._find_or_create_groups(context)

            # Find or create the organization
            # the dataset should get assigned to
            package_dict['owner_org'] = self._find_or_create_organization(
                context,
                package_dict
            )

            # Save license url in extras
            extras = []
            if 'license_url' in package_dict:
                extras.append(('license_url', package_dict['license_url']))
            package_dict['extras'] = extras

            package = model.Package.get(package_dict['id'])
            model.PackageRole(
                package=package,
                user=user,
                role=model.Role.ADMIN
            )

            log.debug(
                'Save or update package %s (%s)'
                % (package_dict['name'], package_dict['id'])
            )
            self._create_or_update_package(package_dict, harvest_object)

            log.debug('Save or update term translations')
            self._submit_term_translations(context, package_dict)
            Session.commit()

        except Exception, e:
            log.exception(e)
            raise
コード例 #53
0
def update_stat_metameta_table(org_id, metameta_id, state):
    """
     update stat on metameta table with new status
    """

    update_sql = "UPDATE metameta SET state='%s', modified_date='%s' WHERE id ='%s' AND org_id='%s';"%(state, datetime.datetime.utcnow().isoformat(), metameta_id, org_id)

    result = Session.execute(update_sql)
    Session.commit()
コード例 #54
0
 def setup_class(cls):
     """
     Remove any initial sessions.
     """
     Session.remove()
     CreateTestData.create()
     url = url_for(controller="ckanext.sitemap.controller:SitemapController", action="view")
     cls.cont = cls.app.get(url)
     cls.content_file = StringIO(cls.cont.body)
コード例 #55
0
ファイル: base.py プロジェクト: coreyerickson/ckanext-harvest
 def _save_gather_error(self, message, job):
     err = HarvestGatherError(message=message, job=job)
     try:
         err.save()
     except InvalidRequestError:
         Session.rollback()
         err.save()
     finally:
         log.error(message)
コード例 #56
0
def _execute_script(script_path):

    conn = Session.connection()
    script = open(script_path, 'r').read()
    for cmd in script.split(';'):
        cmd = re.sub(r'--(.*)|[\n\t]', '', cmd)
        if len(cmd):
            conn.execute(cmd)

    Session.commit()
コード例 #57
0
def insert_metameta_options(org_id, key, option_value):
    """
     creates entries in metameta options table for the organisation.
     value fields are null
    """
    sql = "INSERT INTO metameta_options(id, org_id, key, option_value, modified_date) VALUES ('%s', '%s', '%s', '%s', '%s');"%(str(uuid.uuid4()), org_id, key, option_value, datetime.datetime.utcnow().isoformat())

    result = Session.execute(sql)
    Session.commit()
    return
コード例 #58
0
    def populate_harvest_job(self, harvest_job, set_ids, config, client):
        # Check if this source has been harvested before
        previous_job = Session.query(HarvestJob) \
            .filter(HarvestJob.source == harvest_job.source) \
            .filter(HarvestJob.gather_finished != None) \
            .filter(HarvestJob.id != harvest_job.id) \
            .order_by(HarvestJob.gather_finished.desc()) \
            .limit(1).first()

        last_time = None
        if previous_job and previous_job.finished and model.Package.get(harvest_job.source.id).metadata_modified < previous_job.gather_started:
            last_time = previous_job.gather_started.isoformat()

        # Collect package ids
        package_ids = list(self.get_package_ids(set_ids, config, last_time, client))
        log.debug('Identifiers: %s', package_ids)

        if not self._recreate(harvest_job) and package_ids:
            converted_identifiers = {}
            for identifier in package_ids:
                converted_identifiers[datapid_to_name(identifier)] = identifier
                if identifier.endswith(u'm'):
                    converted_identifiers[datapid_to_name(u"%ss" % identifier[0:-1])] = identifier

            for package in model.Session.query(model.Package).filter(model.Package.name.in_(converted_identifiers.keys())).all():
                converted_name = package.name
                if converted_identifiers[converted_name] not in package_ids:
                    converted_name = "%sm" % converted_name[0:-1]
                package_ids.remove(converted_identifiers[converted_name])

        if previous_job:
            for previous_error in [error.guid for error in Session.query(HarvestObject).
                                   filter(HarvestObject.harvest_job_id == previous_job.id).
                                   filter(HarvestObject.state == 'ERROR').all()]:
                if previous_error not in package_ids:
                    package_ids.append(previous_error)

        try:
            object_ids = []
            if len(package_ids):
                for package_id in islice(package_ids, config['limit']) if 'limit' in config else package_ids:
                    # Create a new HarvestObject for this identifier
                    obj = HarvestObject(guid=package_id, job=harvest_job)
                    obj.save()
                    object_ids.append(obj.id)
                log.debug('Object ids: {i}'.format(i=object_ids))
                return object_ids
            else:
                self._save_gather_error('No packages received for URL: {u}'.format(
                    u=harvest_job.source.url), harvest_job)
                return None
        except Exception as e:
            self._save_gather_error('Gather: {e}'.format(e=e), harvest_job)
            raise
コード例 #59
0
def setup_postgis_tables():

    conn = Session.connection()
    script_path = os.path.join(os.path.dirname(os.path.abspath( __file__ )), 'scripts', 'postgis.sql')
    script = open(script_path,'r').read()
    for cmd in script.split(';'):
        cmd = re.sub(r'--(.*)|[\n\t]','',cmd)
        if len(cmd):
            conn.execute(cmd)

    Session.commit()
コード例 #60
0
def init_db():
    #Setting up custom fiels table for locales    
    try:
        custom_field_table.create()
    except Exception,e:
        # Make sure the table does not remain incorrectly created
        if custom_field_table.exists():
            Session.execute('DROP TABLE custom_field_table')
            Session.commit()

        raise e