def create_unique_identifier(package_id): """ Create a unique identifier, using the prefix and a random number: 10.5072/0044634 Checks the random number doesn't exist in the table or the datacite repository All unique identifiers are created with @return: """ datacite_api = DOIDataCiteAPI() while True: identifier = os.path.join(get_prefix(), '{0:07}'.format(random.randint(1, 100000))) # Check this identifier doesn't exist in the table if not Session.query(DOI).filter(DOI.identifier == identifier).count(): # And check against the datacite service try: datacite_doi = datacite_api.get(identifier) except HTTPError: pass else: if datacite_doi.text: continue doi = DOI(package_id=package_id, identifier=identifier) Session.add(doi) Session.commit() return doi
def record_existing_unique_identifier(package_id, identifier): """ Based on a provided identifier, checks datacite for an existing DOI Saves to local CKAN database :param package_id: string :param identifier: string :return DOI object if saved, false if it didn't exist in datacite """ datacite_api = DOIDataCiteAPI() # Check this identifier doesn't exist in the table existing_doi = Session.query(DOI).filter(DOI.identifier == identifier).first() if not existing_doi: # And check against the datacite service try: datacite_doi = datacite_api.get(identifier) if datacite_doi.text: # Determine whether or not we need to delete a doi that points to the current dataset doi_for_this_pkg = Session.query(DOI).filter(DOI.package_id == package_id).first() if doi_for_this_pkg: datacite_api doi = DOI(package_id=package_id, identifier=identifier) Session.add(doi) Session.commit() return doi except HTTPError: pass
def setup(self): model.Session.remove() CreateTestData.create(auth_profile='publisher') self.sysadmin_user = model.User.get('testsysadmin') self.normal_user = model.User.get( 'annafan') # Does not belong to a publisher self.publisher1_user = model.User.by_name('russianfan') self.publisher2_user = model.User.by_name('tester') # Create two Publishers rev = model.repo.new_revision() self.publisher1 = model.Group(name=u'test-publisher1', title=u'Test Publihser 1', type=u'publisher') Session.add(self.publisher1) self.publisher2 = model.Group(name=u'test-publisher2', title=u'Test Publihser 2', type=u'publisher') Session.add(self.publisher2) member1 = model.Member(table_name='user', table_id=self.publisher1_user.id, group=self.publisher1, capacity='admin') Session.add(member1) member2 = model.Member(table_name='user', table_id=self.publisher2_user.id, group=self.publisher2, capacity='admin') Session.add(member2) Session.commit()
def setup(srid=None): if package_extent_table is None: define_spatial_tables(srid) # log.debug('Spatial tables defined in memory') if model.package_table.exists(): if not Table('geometry_columns',meta.metadata).exists() or \ not Table('spatial_ref_sys',meta.metadata).exists(): raise Exception('The spatial extension is enabled, but PostGIS ' + \ 'has not been set up in the database. ' + \ 'Please refer to the "Setting up PostGIS" section in the README.') if not package_extent_table.exists(): try: package_extent_table.create() except Exception,e: # Make sure the table does not remain incorrectly created # (eg without geom column or constraints) if package_extent_table.exists(): Session.execute('DROP TABLE package_extent') Session.commit() raise e log.debug('Spatial tables created') else: # log.debug('Spatial tables already exist') doNothing = "true"
def _refresh_harvest_objects(self, harvest_object, package_id): """ Perform harvester housekeeping: - Flag the other objects of the source as not current - Set a refernce to the package in the harvest object - Flag it as current - And save the changes """ # Flag the other objects of this source as not current from ckanext.harvest.model import harvest_object_table u = update(harvest_object_table) \ .where(harvest_object_table.c.package_id == bindparam('pkg_id')) \ .values(current=False) Session.execute(u, params={'pkg_id': package_id}) Session.commit() # Refresh current object from session, otherwise the # import paster command fails # (Copied from the Gemini harvester--not sure if necessary) Session.remove() Session.add(harvest_object) Session.refresh(harvest_object) # Set reference to package in the HarvestObject and flag it as # the current one if not harvest_object.package_id: harvest_object.package_id = package_id harvest_object.current = True harvest_object.save()
def test_auth_publisher_profile_different_publisher(self): # Create a source for publisher 1 source = HarvestSource(url=u'http://test-source.com',type='ckan', publisher_id=self.publisher1.id) Session.add(source) Session.commit() extra_environ = {'REMOTE_USER': self.publisher2_user.name.encode('utf8')} # List (Publihsers can see the sources list) res = self.app.get('/harvest', extra_environ=extra_environ) assert 'Harvesting Sources' in res # Create res = self.app.get('/harvest/new', extra_environ=extra_environ) assert 'New harvest source' in res assert 'publisher_id' in res # Check that this publihser is not allowed to manage sources from other publishers status = 401 # Read res = self.app.get('/harvest/%s' % source.id, status=status, extra_environ=extra_environ) # Edit res = self.app.get('/harvest/edit/%s' % source.id, status=status, extra_environ=extra_environ) # Refresh res = self.app.get('/harvest/refresh/%s' % source.id, status=status, extra_environ=extra_environ)
def moderate_change_apply(self, change): print "Change summary:" self.print_changes([change]) print "" answer = raw_input("Do you want to apply this change? [Y/n] ").strip() or "y" answer = answer[0].lower() print "" if answer == 'y': return True else: print answer = raw_input("Do you want to mask changes to this ref? [Y/n] ").strip() or "y" answer = answer[0].lower() print "" if answer == 'y': from ckan.model.changeset import ChangemaskRegister, Session register = ChangemaskRegister() mask = register.create_entity(change.ref) Session.add(mask) Session.commit() print "Mask has been set for ref: %s" % change.ref print "" else: print "Warning: Not setting a mask after not applying changes may lead to conflicts." import time time.sleep(5) print ""
def setup_class(cls): try: from ckanext.harvest.model import HarvestObject, HarvestJob, HarvestSource, HarvestObjectExtra except ImportError: raise SkipTest('The harvester extension is needed for these tests') cls.content1 = '<xml>Content 1</xml>' ho1 = HarvestObject( guid='test-ho-1', job=HarvestJob(source=HarvestSource(url='http://', type='xx')), content=cls.content1) cls.content2 = '<xml>Content 2</xml>' cls.original_content2 = '<xml>Original Content 2</xml>' ho2 = HarvestObject( guid='test-ho-2', job=HarvestJob(source=HarvestSource(url='http://', type='xx')), content=cls.content2) hoe = HarvestObjectExtra(key='original_document', value=cls.original_content2, object=ho2) Session.add(ho1) Session.add(ho2) Session.add(hoe) Session.commit() cls.object_id_1 = ho1.id cls.object_id_2 = ho2.id
def authenticate(self, environ, identity): if not 'login' in identity or not 'password' in identity: return None user = User.by_name(identity.get('login')) if user is None: log.debug('Login failed - username %r not found', identity.get('login')) return None seedUser = Session.query(SEEDUser).filter_by( name=identity.get('login')).first() if seedUser.login_attempts >= 10: log.debug('Login as %r failed - account is locked', identity.get('login')) elif user.validate_password(identity.get('password')): # reset attempt count to 0 seedUser.login_attempts = 0 Session.commit() return user.name else: log.debug('Login as %r failed - password not valid', identity.get('login')) seedUser.login_attempts += 1 Session.commit() return None
def setup(srid=None): if package_extent_table is None: define_spatial_tables(srid) log.debug('Spatial tables defined in memory') if model.package_table.exists(): if not Table('geometry_columns', meta.metadata).exists() or \ not Table('spatial_ref_sys', meta.metadata).exists(): raise Exception( 'The spatial extension is enabled, but PostGIS ' 'has not been set up in the database. ' 'Please refer to the "Setting up PostGIS" section in the README.' ) if not package_extent_table.exists(): try: package_extent_table.create() except Exception as e: # Make sure the table does not remain incorrectly created # (eg without geom column or constraints) if package_extent_table.exists(): Session.execute('DROP TABLE package_extent') Session.commit() raise e log.debug('Spatial tables created') else: log.debug('Spatial tables already exist') # Future migrations go here else: log.debug('Spatial tables creation deferred')
def handle_submit(self, id): data = clean_dict(dict_fns.unflatten(tuplize_dict(parse_params( request.params)))) data['dataset_url'] = toolkit.url_for( controller='package', action='read', id=id, qualified=True ) package = get_action('package_show')(None, {'id': id}) self.fail_if_private(package, data['dataset_url']) # Comma separated config var to_addrs = config['ckanext.ands.support_emails'].split(',') subject = 'DataPortal Support: Request to publish dataset' body = base.render( 'package/doi_email.text', extra_vars=data) for email in to_addrs: mail_recipient('Dataportal support', email, subject, body) data['package_id'] = package['id'] data['user_id'] = c.userobj.id doi_request = DoiRequest(**data) Session.add(doi_request) Session.commit() h.flash_success("DOI Request sent") return toolkit.redirect_to(data['dataset_url'])
def update_extents(): from ckan.model import PackageExtra, Package, Session conn = Session.connection() packages = [extra.package \ for extra in \ Session.query(PackageExtra).filter(PackageExtra.key == 'spatial').all()] errors = [] count = 0 for package in packages: try: value = package.extras['spatial'] log.debug('Received: %r' % value) geometry = json.loads(value) count += 1 except ValueError as e: errors.append(u'Package %s - Error decoding JSON object: %s' % (package.id, six.text_type(e))) except TypeError as e: errors.append(u'Package %s - Error decoding JSON object: %s' % (package.id, six.text_type(e))) save_package_extent(package.id, geometry) Session.commit() if errors: msg = 'Errors were found:\n%s' % '\n'.join(errors) print(msg) msg = "Done. Extents generated for %i out of %i packages" % (count, len(packages)) print(msg)
def create_doi_from_identifier(package_id, identifier): """Can be called when an identifier has already been created elsewhere. Does not ensure the identifier is unique""" doi = DOI(package_id=package_id, identifier=identifier) Session.add(doi) Session.commit() return doi
def get_or_create_doi(package_id): '''Create or retrieve the unique identifier for this package_id. :param package_id: ''' datacite_api = DOIDataCiteAPI() doi = get_doi(package_id) if doi is None: while True: identifier = os.path.join( get_prefix(), u'{0:07}'.format(random.randint(1, 100000))) # Check this identifier doesn't exist in the table if Session.query(DOI).filter(DOI.identifier == identifier).count(): continue # And check against the datacite service try: datacite_doi = datacite_api.get(identifier) except HTTPError: pass else: if datacite_doi.text: continue doi = DOI(package_id=package_id, identifier=identifier) Session.add(doi) Session.commit() break return doi
def delete_tests(self): print 'Deleting all test DOIs' Session.query(DOI).filter( DOI.identifier.like('%' + TEST_PREFIX + '%')).delete(synchronize_session=False) Session.commit()
def setup(self): model.Session.remove() CreateTestData.create(auth_profile='publisher') self.sysadmin_user = model.User.get('testsysadmin') self.normal_user = model.User.get('annafan') # Does not belong to a publisher self.publisher1_user = model.User.by_name('russianfan') self.publisher2_user = model.User.by_name('tester') # Create two Publishers rev = model.repo.new_revision() self.publisher1 = model.Group(name=u'test-publisher1',title=u'Test Publihser 1',type=u'publisher') Session.add(self.publisher1) self.publisher2 = model.Group(name=u'test-publisher2',title=u'Test Publihser 2',type=u'publisher') Session.add(self.publisher2) member1 = model.Member(table_name = 'user', table_id = self.publisher1_user.id, group=self.publisher1, capacity='admin') Session.add(member1) member2 = model.Member(table_name = 'user', table_id = self.publisher2_user.id, group=self.publisher2, capacity='admin') Session.add(member2) Session.commit()
def test_auth_publisher_profile_different_publisher(self): # Create a source for publisher 1 source = HarvestSource(url=u'http://test-source.com', type='ckan', publisher_id=self.publisher1.id) Session.add(source) Session.commit() extra_environ = { 'REMOTE_USER': self.publisher2_user.name.encode('utf8') } # List (Publihsers can see the sources list) res = self.app.get('/harvest', extra_environ=extra_environ) assert 'Harvesting Sources' in res # Create res = self.app.get('/harvest/new', extra_environ=extra_environ) assert 'New harvest source' in res assert 'publisher_id' in res # Check that this publihser is not allowed to manage sources from other publishers status = 401 # Read res = self.app.get('/harvest/%s' % source.id, status=status, extra_environ=extra_environ) # Edit res = self.app.get('/harvest/edit/%s' % source.id, status=status, extra_environ=extra_environ) # Refresh res = self.app.get('/harvest/refresh/%s' % source.id, status=status, extra_environ=extra_environ)
def authenticate(self, environ, identity): """ Mimic most of UsernamePasswordAuthenticator.authenticate but add account lockout after 10 failed attempts. """ if 'login' not in identity or 'password' not in identity: return None user = User.by_name(identity.get('login')) if user is None: LOG.debug('Login failed - username %r not found', identity.get('login')) return None qgov_user = Session.query(QGOVUser).filter_by( name=identity.get('login')).first() if qgov_user.login_attempts >= 10: LOG.debug('Login as %r failed - account is locked', identity.get('login')) elif user.validate_password(identity.get('password')): # reset attempt count to 0 qgov_user.login_attempts = 0 Session.commit() return user.name else: LOG.debug('Login as %r failed - password not valid', identity.get('login')) qgov_user.login_attempts += 1 Session.commit() return None
def handle_submit(self, id): data = clean_dict( dict_fns.unflatten(tuplize_dict(parse_params(request.params)))) data['dataset_url'] = toolkit.url_for(controller='package', action='read', id=id, qualified=True) package = get_action('package_show')(None, {'id': id}) self.fail_if_private(package, data['dataset_url']) # Comma separated config var to_addrs = config['ckanext.ands.support_emails'].split(',') subject = 'DataPortal Support: Request to publish dataset' body = base.render('package/doi_email.text', extra_vars=data) for email in to_addrs: mail_recipient('Dataportal support', email, subject, body) data['package_id'] = package['id'] data['user_id'] = c.userobj.id doi_request = DoiRequest(**data) Session.add(doi_request) Session.commit() h.flash_success("DOI Request sent") return toolkit.redirect_to(data['dataset_url'])
def _test_auth_not_allowed(self, user_name=None, source=None, status=401): if not source: # Create harvest source source = HarvestSource(url=u'http://test-source.com', type='ckan') Session.add(source) Session.commit() if user_name: extra_environ = {'REMOTE_USER': user_name.encode('utf8')} else: extra_environ = {} # List res = self.app.get('/harvest', status=status, extra_environ=extra_environ) # Create res = self.app.get('/harvest/new', status=status, extra_environ=extra_environ) # Read res = self.app.get('/harvest/%s' % source.id, status=status, extra_environ=extra_environ) # Edit res = self.app.get('/harvest/edit/%s' % source.id, status=status, extra_environ=extra_environ) # Refresh res = self.app.get('/harvest/refresh/%s' % source.id, status=status, extra_environ=extra_environ)
def _create_pkg(self): model.repo.new_revision() pkg = Package.get("annakarenina") pkg.name = "fookarenina" pkg.add_resource("www.google.com", description="foo", name="foo") Session.add(pkg) Session.commit() return pkg
def _create_pkg(self): model.repo.new_revision() pkg = Package.get('annakarenina') pkg.name = "fookarenina" pkg.add_resource('www.google.com', description='foo', name="foo") Session.add(pkg) Session.commit() return pkg
def save(self, commit=True): if not hasattr(self, 'slug') or not self.slug: self.slug = slugify(self.name) session = Session() log.debug(self) session.add(self) if commit: session.commit()
def delete_tests(self): '''Delete all test DOIs.''' print u'Deleting all test DOIs' Session.query(DOI).filter( DOI.identifier.like(u'%' + get_prefix() + u'%')).delete(synchronize_session=False) Session.commit()
def _execute_sql(cls, script): engine = create_engine(cls.sqlalchemy_url) Session.bind = engine connection = Session.connection() connection.execute(script) Session.commit()
def import_stage(self, harvest_object): log.debug('In SwisstopoHarvester import_stage') if not harvest_object: log.error('No harvest object received') return False try: package_dict = json.loads(harvest_object.content) package_dict['id'] = harvest_object.guid package_dict['name'] = munge_title_to_name( package_dict['layer_name'] ) user = model.User.get(self.HARVEST_USER) context = { 'model': model, 'session': Session, 'user': self.HARVEST_USER } # Find or create group the dataset should get assigned to package_dict['groups'] = self._find_or_create_groups(context) # Find or create the organization # the dataset should get assigned to package_dict['owner_org'] = self._find_or_create_organization( context, package_dict ) # Save license url in extras extras = [] if 'license_url' in package_dict: extras.append(('license_url', package_dict['license_url'])) package_dict['extras'] = extras package = model.Package.get(package_dict['id']) model.PackageRole( package=package, user=user, role=model.Role.ADMIN ) log.debug( 'Save or update package %s (%s)' % (package_dict['name'], package_dict['id']) ) self._create_or_update_package(package_dict, harvest_object) log.debug('Save or update term translations') self._submit_term_translations(context, package_dict) Session.commit() except Exception, e: log.exception(e) raise
def update_stat_metameta_table(org_id, metameta_id, state): """ update stat on metameta table with new status """ update_sql = "UPDATE metameta SET state='%s', modified_date='%s' WHERE id ='%s' AND org_id='%s';"%(state, datetime.datetime.utcnow().isoformat(), metameta_id, org_id) result = Session.execute(update_sql) Session.commit()
def create(cls, **kwargs): ''' Create a new record of type :class:`~m`. ''' item_dict = cls._columns(**kwargs) new_item = cls.m(**item_dict) Session.add(new_item) Session.commit() return new_item
def _execute_script(script_path): conn = Session.connection() script = open(script_path, 'r').read() for cmd in script.split(';'): cmd = re.sub(r'--(.*)|[\n\t]', '', cmd) if len(cmd): conn.execute(cmd) Session.commit()
def update_stat_metameta_table(org_id, metameta_id, state): """ update stat on metameta table with new status """ update_sql = "UPDATE metameta SET state='%s', modified_date='%s' WHERE id ='%s' AND org_id='%s';" % ( state, datetime.datetime.utcnow().isoformat(), metameta_id, org_id) result = Session.execute(update_sql) Session.commit()
def _execute_script(script_path): conn = Session.connection() script = open(script_path, "r").read() for cmd in script.split(";"): cmd = re.sub(r"--(.*)|[\n\t]", "", cmd) if len(cmd): conn.execute(cmd) Session.commit()
def insert_metameta_options(org_id, key, option_value): """ creates entries in metameta options table for the organisation. value fields are null """ sql = "INSERT INTO metameta_options(id, org_id, key, option_value, modified_date) VALUES ('%s', '%s', '%s', '%s', '%s');"%(str(uuid.uuid4()), org_id, key, option_value, datetime.datetime.utcnow().isoformat()) result = Session.execute(sql) Session.commit() return
def import_stage(self, harvest_object): ''' The import stage will receive a HarvestObject object and will be responsible for: - performing any necessary action with the fetched object (e.g create a CKAN package). Note: if this stage creates or updates a package, a reference to the package must be added to the HarvestObject. Additionally, the HarvestObject must be flagged as current. - creating the HarvestObject - Package relation (if necessary) - creating and storing any suitable HarvestObjectErrors that may occur. - returning True if everything went as expected, False otherwise. :param harvest_object: HarvestObject object :returns: True if everything went right, False if errors were found ''' logger.debug("in import stage: %s" % harvest_object.guid) if not harvest_object: logger.error('No harvest object received') self._save_object_error('No harvest object received') return False try: self._set_config(harvest_object.job.source.config) context = {'model': model, 'session': Session, 'user': self.user} package_dict = json.loads(harvest_object.content) package_dict['id'] = munge_title_to_name(harvest_object.guid) package_dict['name'] = package_dict['id'] # add owner_org source_dataset = get_action('package_show')( { 'ignore_auth': True }, { 'id': harvest_object.source.id }) owner_org = source_dataset.get('owner_org') package_dict['owner_org'] = owner_org # logger.debug('Create/update package using dict: %s' % package_dict) self._create_or_update_package(package_dict, harvest_object, 'package_show') Session.commit() logger.debug("Finished record") except: logger.exception('Something went wrong!') self._save_object_error('Exception in import stage', harvest_object) return False return True
def delete(cls, item_id): ''' :param item_id: ''' to_delete = Session.query(cls.m).get(item_id) if to_delete is not None: Session.delete(to_delete) Session.commit()
def init_db(): #Setting up custom fiels table for locales try: custom_field_table.create() except Exception,e: # Make sure the table does not remain incorrectly created if custom_field_table.exists(): Session.execute('DROP TABLE custom_field_table') Session.commit() raise e
def setup_postgis_tables(): conn = Session.connection() script_path = os.path.join(os.path.dirname(os.path.abspath( __file__ )), 'scripts', 'postgis.sql') script = open(script_path,'r').read() for cmd in script.split(';'): cmd = re.sub(r'--(.*)|[\n\t]','',cmd) if len(cmd): conn.execute(cmd) Session.commit()
def init_db(): #Setting up custom fiels table for locales try: custom_field_table.create() except Exception, e: # Make sure the table does not remain incorrectly created if custom_field_table.exists(): Session.execute('DROP TABLE custom_field_table') Session.commit() raise e
def insert_metameta_table(org_id, data): """ creates entries in metameta table for the organisation. value fields are null """ errors = {} #key if not (data.has_key('key') and len(data['key'].strip())): #TODO validation errors.update({'key': _('%s field is required.' % ('key'))}) return errors key = data['key'].strip() #label: if not set, insert key if data.has_key('label') and len(data['label'].strip()): label = data['label'].strip() else: label = key.title() #convert non-alphanumeric to underscore key = create_search_key(key) if org_has_defaults(org_id, key): #TODO validation errors.update({ 'key': _('%s has been already used by the organization.' % (key)) }) return errors #readonly attribute readonly = str(data.get('readonly', False)).lower() == 'True'.lower() #validator validator = data.get('validator', validator_not_empty) #default_value default_value = data.get('default_value', '') #field_type field_type = data.get('field_type', field_type_text) min_value = max_value = 0.0 if field_type == field_type_text and validator == validator_range: min_value = data.get('min_value', 0.0) max_value = data.get('max_value', 0.0) sql = "INSERT INTO metameta(id, org_id, key, default_value, label, validator, state, custom, presettable, readonly, field_type, min_value, max_value, sort_order, modified_date) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', %r, %r, %r, '%s', %f, %f, (CASE WHEN (SELECT MAX(sort_order)+1 FROM metameta WHERE org_id='%s')>101 THEN (SELECT MAX(sort_order)+1 FROM metameta WHERE org_id='%s') ELSE 101 END), '%s');" % ( str(uuid.uuid4()), org_id, key, default_value, label, validator, state_active, True, True, readonly, field_type, float(min_value), float(max_value), org_id, org_id, datetime.datetime.utcnow().isoformat()) result = Session.execute(sql) Session.commit() return errors
def _unlock_account(account_id): """ Unlock an account (erase the failed login attempts). """ qgov_user = Session.query(QGOVUser).filter( QGOVUser.id == account_id).first() if qgov_user: LOG.debug("Clearing failed login attempts for %s", account_id) qgov_user.login_attempts = 0 Session.commit() else: LOG.debug("Account %s not found", account_id)
def logged_in(self): controller = UserController() if not c.user: # a number of failed login attempts greater than 10 # indicates that the locked user is associated with the current request qgovUser = Session.query(QGOVUser).filter(QGOVUser.login_attempts > 10).first() if qgovUser: qgovUser.login_attempts = 10 Session.commit() return controller.login("account-locked") return controller.logged_in()
def delete_application(application_name): application = Application.by_name(application_name) if not application_name: abort(404) app_tags = Session.query(ApplicationTag) \ .filter(ApplicationTag.application_id==application.id) \ .all() for app_tag in app_tags: app_tag.delete() application.delete() Session.commit()
def setup(self): # Add sysadmin user harvest_user = model.User(name=u'harvest', password=u'test', sysadmin=True) Session.add(harvest_user) Session.commit() package_schema = default_update_package_schema() self.context ={'model':model, 'session':Session, 'user':u'harvest', 'schema':package_schema, 'api_version': '2'}
def update_package(cls, package_id, **kwargs): ''' Update the package_id and/or published fields of a record associated with a given package. :param package_id: the id of the package :param kwargs: the values to be updated :return: the updated record object ''' update_dict = {k: v for k, v in kwargs.items() if k in cls.cols} Session.query(DOI).filter( DOI.package_id == package_id).update(update_dict) Session.commit() return cls.read_package(package_id)
def delete_idea(idea_name): idea = Idea.by_name(idea_name) if not idea_name: abort(404) idea_tags = Session.query(IdeaTag) \ .filter(IdeaTag.idea_id==idea.id) \ .all() for idea_tag in idea_tags: idea_tag.delete() idea.delete() Session.commit()
def remove_metameta_options(org_id, key, option_value=None): """ creates entries in metameta options table for the organisation. value fields are null """ sql = "DELETE FROM metameta_options WHERE org_id = '%s' AND key = '%s' "%(org_id, key) if option_value: sql = sql + " AND option_value='%s' "%(option_value) sql = sql + ";" result = Session.execute(sql) Session.commit() return
def setup(srid=None): if not srid: srid = DEFAULT_SRID srid = str(srid) connection = Session.connection() connection.execute('CREATE TABLE package_extent(package_id text PRIMARY KEY)') connection.execute('SELECT AddGeometryColumn(\'package_extent\',\'the_geom\', %s, \'GEOMETRY\', 2)',srid) Session.commit()
def create_package_doi(pkg_dict): """ Create a unique identifier, using the prefix and a random number: 10.5072/0044634 Checks the random number doesn't exist in the table or the datacite repository All unique identifiers are created with @return: """ _validate_package(pkg_dict) identifier = _create_unique_identifier() package_doi = CeonPackageDOI(package_id=pkg_dict['id'], identifier=identifier) Session.add(package_doi) Session.commit() log.debug(u"Created DOI {} for package {}".format(package_doi.identifier, pkg_dict['id'])) return package_doi
def update_resource_stats(resource_id, total_visits, total_downloads): q_result = Session.query(PiwikResource).filter(PiwikResource.resource_id == resource_id).first() if q_result: # update if resource is already in table q_result.total_visits = total_visits q_result.total_downloads = total_downloads else: # add new entry for resource rt = PiwikResource() rt.resource_id = resource_id rt.total_visits = total_visits rt.total_downloads = total_downloads Session.add(rt) Session.commit()
def update_package_stats(package_name, total_visits, recent_visits): q_result = Session.query(PiwikPackage).filter(PiwikPackage.package_name == package_name).first() if q_result: # update if package is already in table q_result.total_visits = total_visits q_result.recent_visits = recent_visits Session.commit() else: # add new entry for package pt = PiwikPackage() pt.package_name = package_name pt.total_visits = total_visits pt.recent_visits = recent_visits Session.add(pt) Session.commit()
def setup(): log.debug('Multilingual tables defined in memory') #Setting up package multilang table if not package_multilang_table.exists(): try: package_multilang_table.create() except Exception,e: # Make sure the table does not remain incorrectly created if package_multilang_table.exists(): Session.execute('DROP TABLE package_multilang') Session.commit() raise e log.info('Package Multilingual table created')
def insert_metameta_table(org_id, data): """ creates entries in metameta table for the organisation. value fields are null """ errors = {} #key if not (data.has_key('key') and len(data['key'].strip())): #TODO validation errors.update({'key': _('%s field is required.'%('key'))}) return errors key = data['key'].strip() #label: if not set, insert key if data.has_key('label') and len(data['label'].strip()): label = data['label'].strip() else: label = key.title() #convert non-alphanumeric to underscore key = create_search_key(key) if org_has_defaults(org_id, key): #TODO validation errors.update({'key': _('%s has been already used by the organization.'%(key))}) return errors #readonly attribute readonly = str(data.get('readonly', False)).lower()=='True'.lower() #validator validator = data.get('validator', validator_not_empty) #default_value default_value = data.get('default_value', '') #field_type field_type = data.get('field_type', field_type_text) min_value = max_value = 0.0 if field_type == field_type_text and validator == validator_range: min_value = data.get('min_value', 0.0) max_value = data.get('max_value', 0.0) sql = "INSERT INTO metameta(id, org_id, key, default_value, label, validator, state, custom, presettable, readonly, field_type, min_value, max_value, sort_order, modified_date) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', %r, %r, %r, '%s', %f, %f, (CASE WHEN (SELECT MAX(sort_order)+1 FROM metameta WHERE org_id='%s')>101 THEN (SELECT MAX(sort_order)+1 FROM metameta WHERE org_id='%s') ELSE 101 END), '%s');"%(str(uuid.uuid4()), org_id, key, default_value, label, validator, state_active, True, True, readonly, field_type, float(min_value), float(max_value), org_id, org_id, datetime.datetime.utcnow().isoformat()) result = Session.execute(sql) Session.commit() return errors
def update_metameta_table(org_id, metameta_id, data): """ update metameta table with new values """ errors = {} #key if not (data.has_key('key') and len(data['key'].strip())): #TODO validation errors.update({'key': _('%s field is required.'%('key'))}) return errors key = data['key'].strip() #label: if not set, insert key if data.has_key('label') and len(data['label'].strip()): label = data['label'].strip() else: label = key.title() #convert non-alphanumeric to underscore key = create_search_key(key) if org_has_defaults(org_id, key, metameta_id): #TODO validation errors.update({'key': _('%s has been already used by the organization.'%(key))}) return errors #readonly attribute readonly = str(data.get('readonly', False)).lower()=='True'.lower() #validator validator = data.get('validator', validator_not_empty) #default_value default_value = data.get('default_value', '') #field_type field_type = data.get('field_type', field_type_text) min_value = max_value = 0.0 if field_type == field_type_text and validator == validator_range: min_value = data.get('min_value', 0.0) max_value = data.get('max_value', 0.0) update_sql = "UPDATE metameta SET key='%s', default_value='%s', label='%s', validator='%s', readonly=%r, field_type='%s', min_value=%f, max_value=%f, modified_date='%s' WHERE id ='%s' AND org_id='%s';"%(key, default_value, label, validator, readonly, field_type, float(min_value), float(max_value), datetime.datetime.utcnow().isoformat(), metameta_id, org_id) result = Session.execute(update_sql) Session.commit() return errors
def create_resource_doi(pkg_dict, res_dict): """ Create a unique identifier, using the prefix and a random number: 10.5072/0044634 Checks the random number doesn't exist in the table or the datacite repository All unique identifiers are created with @return: """ _validate_resource(res_dict) resource_id = res_dict['id'] package_doi = CeonPackageDOI.get(pkg_dict['id']) if not package_doi: create_package_doi(pkg_dict) identifier = _create_unique_identifier(package_doi.identifier) resource_doi = CeonResourceDOI(resource_id=resource_id, identifier=identifier) Session.add(resource_doi) Session.commit() log.debug(u"Created DOI {} for resource {}".format(resource_doi.identifier, res_dict['id'])) return resource_doi
def authenticate(self, environ, identity): if 'repoze.who.plugins.openid.userid' in identity: openid = identity.get('repoze.who.plugins.openid.userid') user = User.by_openid(openid) if user is None: # TODO: Implement a mask to ask for an alternative user # name instead of just using the OpenID identifier. name = identity.get('repoze.who.plugins.openid.nickname') if not User.check_name_valid(name): name = openid if not User.check_name_available(name): name = openid user = User(openid=openid, name=name, fullname=identity.get('repoze.who.plugins.openid.fullname'), email=identity.get('repoze.who.plugins.openid.email')) Session.add(user) Session.commit() Session.remove() return user.name return None
def _test_auth_not_allowed(self, user_name=None, source=None, status=401): if not source: # Create harvest source source = HarvestSource(url=u'http://test-source.com', type='ckan') Session.add(source) Session.commit() if user_name: extra_environ = {'REMOTE_USER': user_name.encode('utf8')} else: extra_environ = {} # List res = self.app.get( '/harvest', status=status, extra_environ=extra_environ) # Create res = self.app.get( '/harvest/new', status=status, extra_environ=extra_environ) # Read res = self.app.get( '/harvest/%s' % source.id, status=status, extra_environ=extra_environ) # Edit res = self.app.get( '/harvest/edit/%s' % source.id, status=status, extra_environ=extra_environ) # Refresh res = self.app.get( '/harvest/refresh/%s' % source.id, status=status, extra_environ=extra_environ)