def create_unique_identifier(package_id):
"""
Create a unique identifier, using the prefix and a random number: 10.5072/0044634
Checks the random number doesn't exist in the table or the datacite repository
All unique identifiers are created with
@return:
"""
datacite_api = DOIDataCiteAPI()
while True:
identifier = os.path.join(get_prefix(), '{0:07}'.format(random.randint(1, 100000)))
# Check this identifier doesn't exist in the table
if not Session.query(DOI).filter(DOI.identifier == identifier).count():
# And check against the datacite service
try:
datacite_doi = datacite_api.get(identifier)
except HTTPError:
pass
else:
if datacite_doi.text:
continue
doi = DOI(package_id=package_id, identifier=identifier)
Session.add(doi)
Session.commit()
return doi
def record_existing_unique_identifier(package_id, identifier):
"""
Based on a provided identifier, checks datacite for an existing DOI
Saves to local CKAN database
:param package_id: string
:param identifier: string
:return DOI object if saved, false if it didn't exist in datacite
"""
datacite_api = DOIDataCiteAPI()
# Check this identifier doesn't exist in the table
existing_doi = Session.query(DOI).filter(DOI.identifier == identifier).first()
if not existing_doi:
# And check against the datacite service
try:
datacite_doi = datacite_api.get(identifier)
if datacite_doi.text:
# Determine whether or not we need to delete a doi that points to the current dataset
doi_for_this_pkg = Session.query(DOI).filter(DOI.package_id == package_id).first()
if doi_for_this_pkg:
datacite_api
doi = DOI(package_id=package_id, identifier=identifier)
Session.add(doi)
Session.commit()
return doi
except HTTPError:
pass
def setup(self):
model.Session.remove()
CreateTestData.create(auth_profile='publisher')
self.sysadmin_user = model.User.get('testsysadmin')
self.normal_user = model.User.get(
'annafan') # Does not belong to a publisher
self.publisher1_user = model.User.by_name('russianfan')
self.publisher2_user = model.User.by_name('tester')
# Create two Publishers
rev = model.repo.new_revision()
self.publisher1 = model.Group(name=u'test-publisher1',
title=u'Test Publihser 1',
type=u'publisher')
Session.add(self.publisher1)
self.publisher2 = model.Group(name=u'test-publisher2',
title=u'Test Publihser 2',
type=u'publisher')
Session.add(self.publisher2)
member1 = model.Member(table_name='user',
table_id=self.publisher1_user.id,
group=self.publisher1,
capacity='admin')
Session.add(member1)
member2 = model.Member(table_name='user',
table_id=self.publisher2_user.id,
group=self.publisher2,
capacity='admin')
Session.add(member2)
Session.commit()
def setup(srid=None):
if package_extent_table is None:
define_spatial_tables(srid)
# log.debug('Spatial tables defined in memory')
if model.package_table.exists():
if not Table('geometry_columns',meta.metadata).exists() or \
not Table('spatial_ref_sys',meta.metadata).exists():
raise Exception('The spatial extension is enabled, but PostGIS ' + \
'has not been set up in the database. ' + \
'Please refer to the "Setting up PostGIS" section in the README.')
if not package_extent_table.exists():
try:
package_extent_table.create()
except Exception,e:
# Make sure the table does not remain incorrectly created
# (eg without geom column or constraints)
if package_extent_table.exists():
Session.execute('DROP TABLE package_extent')
Session.commit()
raise e
log.debug('Spatial tables created')
else:
# log.debug('Spatial tables already exist')
doNothing = "true"
def _refresh_harvest_objects(self, harvest_object, package_id):
"""
Perform harvester housekeeping:
- Flag the other objects of the source as not current
- Set a refernce to the package in the harvest object
- Flag it as current
- And save the changes
"""
# Flag the other objects of this source as not current
from ckanext.harvest.model import harvest_object_table
u = update(harvest_object_table) \
.where(harvest_object_table.c.package_id == bindparam('pkg_id')) \
.values(current=False)
Session.execute(u, params={'pkg_id': package_id})
Session.commit()
# Refresh current object from session, otherwise the
# import paster command fails
# (Copied from the Gemini harvester--not sure if necessary)
Session.remove()
Session.add(harvest_object)
Session.refresh(harvest_object)
# Set reference to package in the HarvestObject and flag it as
# the current one
if not harvest_object.package_id:
harvest_object.package_id = package_id
harvest_object.current = True
harvest_object.save()
def test_auth_publisher_profile_different_publisher(self):
# Create a source for publisher 1
source = HarvestSource(url=u'http://test-source.com',type='ckan',
publisher_id=self.publisher1.id)
Session.add(source)
Session.commit()
extra_environ = {'REMOTE_USER': self.publisher2_user.name.encode('utf8')}
# List (Publihsers can see the sources list)
res = self.app.get('/harvest', extra_environ=extra_environ)
assert 'Harvesting Sources' in res
# Create
res = self.app.get('/harvest/new', extra_environ=extra_environ)
assert 'New harvest source' in res
assert 'publisher_id' in res
# Check that this publihser is not allowed to manage sources from other publishers
status = 401
# Read
res = self.app.get('/harvest/%s' % source.id, status=status, extra_environ=extra_environ)
# Edit
res = self.app.get('/harvest/edit/%s' % source.id, status=status, extra_environ=extra_environ)
# Refresh
res = self.app.get('/harvest/refresh/%s' % source.id, status=status, extra_environ=extra_environ)
def moderate_change_apply(self, change):
print "Change summary:"
self.print_changes([change])
print ""
answer = raw_input("Do you want to apply this change? [Y/n] ").strip() or "y"
answer = answer[0].lower()
print ""
if answer == 'y':
return True
else:
print
answer = raw_input("Do you want to mask changes to this ref? [Y/n] ").strip() or "y"
answer = answer[0].lower()
print ""
if answer == 'y':
from ckan.model.changeset import ChangemaskRegister, Session
register = ChangemaskRegister()
mask = register.create_entity(change.ref)
Session.add(mask)
Session.commit()
print "Mask has been set for ref: %s" % change.ref
print ""
else:
print "Warning: Not setting a mask after not applying changes may lead to conflicts."
import time
time.sleep(5)
print ""
def setup_class(cls):
try:
from ckanext.harvest.model import HarvestObject, HarvestJob, HarvestSource, HarvestObjectExtra
except ImportError:
raise SkipTest('The harvester extension is needed for these tests')
cls.content1 = '<xml>Content 1</xml>'
ho1 = HarvestObject(
guid='test-ho-1',
job=HarvestJob(source=HarvestSource(url='http://', type='xx')),
content=cls.content1)
cls.content2 = '<xml>Content 2</xml>'
cls.original_content2 = '<xml>Original Content 2</xml>'
ho2 = HarvestObject(
guid='test-ho-2',
job=HarvestJob(source=HarvestSource(url='http://', type='xx')),
content=cls.content2)
hoe = HarvestObjectExtra(key='original_document',
value=cls.original_content2,
object=ho2)
Session.add(ho1)
Session.add(ho2)
Session.add(hoe)
Session.commit()
cls.object_id_1 = ho1.id
cls.object_id_2 = ho2.id
def authenticate(self, environ, identity):
if not 'login' in identity or not 'password' in identity:
return None
user = User.by_name(identity.get('login'))
if user is None:
log.debug('Login failed - username %r not found',
identity.get('login'))
return None
seedUser = Session.query(SEEDUser).filter_by(
name=identity.get('login')).first()
if seedUser.login_attempts >= 10:
log.debug('Login as %r failed - account is locked',
identity.get('login'))
elif user.validate_password(identity.get('password')):
# reset attempt count to 0
seedUser.login_attempts = 0
Session.commit()
return user.name
else:
log.debug('Login as %r failed - password not valid',
identity.get('login'))
seedUser.login_attempts += 1
Session.commit()
return None
def setup(srid=None):
if package_extent_table is None:
define_spatial_tables(srid)
log.debug('Spatial tables defined in memory')
if model.package_table.exists():
if not Table('geometry_columns', meta.metadata).exists() or \
not Table('spatial_ref_sys', meta.metadata).exists():
raise Exception(
'The spatial extension is enabled, but PostGIS '
'has not been set up in the database. '
'Please refer to the "Setting up PostGIS" section in the README.'
)
if not package_extent_table.exists():
try:
package_extent_table.create()
except Exception as e:
# Make sure the table does not remain incorrectly created
# (eg without geom column or constraints)
if package_extent_table.exists():
Session.execute('DROP TABLE package_extent')
Session.commit()
raise e
log.debug('Spatial tables created')
else:
log.debug('Spatial tables already exist')
# Future migrations go here
else:
log.debug('Spatial tables creation deferred')
def moderate_change_apply(self, change):
print "Change summary:"
self.print_changes([change])
print ""
answer = raw_input("Do you want to apply this change? [Y/n] ").strip() or "y"
answer = answer[0].lower()
print ""
if answer == 'y':
return True
else:
print
answer = raw_input("Do you want to mask changes to this ref? [Y/n] ").strip() or "y"
answer = answer[0].lower()
print ""
if answer == 'y':
from ckan.model.changeset import ChangemaskRegister, Session
register = ChangemaskRegister()
mask = register.create_entity(change.ref)
Session.add(mask)
Session.commit()
print "Mask has been set for ref: %s" % change.ref
print ""
else:
print "Warning: Not setting a mask after not applying changes may lead to conflicts."
import time
time.sleep(5)
print ""
def handle_submit(self, id):
data = clean_dict(dict_fns.unflatten(tuplize_dict(parse_params(
request.params))))
data['dataset_url'] = toolkit.url_for(
controller='package',
action='read',
id=id,
qualified=True
)
package = get_action('package_show')(None, {'id': id})
self.fail_if_private(package, data['dataset_url'])
# Comma separated config var
to_addrs = config['ckanext.ands.support_emails'].split(',')
subject = 'DataPortal Support: Request to publish dataset'
body = base.render(
'package/doi_email.text',
extra_vars=data)
for email in to_addrs:
mail_recipient('Dataportal support', email, subject, body)
data['package_id'] = package['id']
data['user_id'] = c.userobj.id
doi_request = DoiRequest(**data)
Session.add(doi_request)
Session.commit()
h.flash_success("DOI Request sent")
return toolkit.redirect_to(data['dataset_url'])
def update_extents():
from ckan.model import PackageExtra, Package, Session
conn = Session.connection()
packages = [extra.package \
for extra in \
Session.query(PackageExtra).filter(PackageExtra.key == 'spatial').all()]
errors = []
count = 0
for package in packages:
try:
value = package.extras['spatial']
log.debug('Received: %r' % value)
geometry = json.loads(value)
count += 1
except ValueError as e:
errors.append(u'Package %s - Error decoding JSON object: %s' %
(package.id, six.text_type(e)))
except TypeError as e:
errors.append(u'Package %s - Error decoding JSON object: %s' %
(package.id, six.text_type(e)))
save_package_extent(package.id, geometry)
Session.commit()
if errors:
msg = 'Errors were found:\n%s' % '\n'.join(errors)
print(msg)
msg = "Done. Extents generated for %i out of %i packages" % (count,
len(packages))
print(msg)
def create_doi_from_identifier(package_id, identifier):
"""Can be called when an identifier has already been created elsewhere.
Does not ensure the identifier is unique"""
doi = DOI(package_id=package_id, identifier=identifier)
Session.add(doi)
Session.commit()
return doi
def get_or_create_doi(package_id):
'''Create or retrieve the unique identifier for this package_id.
:param package_id:
'''
datacite_api = DOIDataCiteAPI()
doi = get_doi(package_id)
if doi is None:
while True:
identifier = os.path.join(
get_prefix(), u'{0:07}'.format(random.randint(1, 100000)))
# Check this identifier doesn't exist in the table
if Session.query(DOI).filter(DOI.identifier == identifier).count():
continue
# And check against the datacite service
try:
datacite_doi = datacite_api.get(identifier)
except HTTPError:
pass
else:
if datacite_doi.text:
continue
doi = DOI(package_id=package_id, identifier=identifier)
Session.add(doi)
Session.commit()
break
return doi
def delete_tests(self):
print 'Deleting all test DOIs'
Session.query(DOI).filter(
DOI.identifier.like('%' + TEST_PREFIX +
'%')).delete(synchronize_session=False)
Session.commit()
def setup(self):
model.Session.remove()
CreateTestData.create(auth_profile='publisher')
self.sysadmin_user = model.User.get('testsysadmin')
self.normal_user = model.User.get('annafan') # Does not belong to a publisher
self.publisher1_user = model.User.by_name('russianfan')
self.publisher2_user = model.User.by_name('tester')
# Create two Publishers
rev = model.repo.new_revision()
self.publisher1 = model.Group(name=u'test-publisher1',title=u'Test Publihser 1',type=u'publisher')
Session.add(self.publisher1)
self.publisher2 = model.Group(name=u'test-publisher2',title=u'Test Publihser 2',type=u'publisher')
Session.add(self.publisher2)
member1 = model.Member(table_name = 'user',
table_id = self.publisher1_user.id,
group=self.publisher1,
capacity='admin')
Session.add(member1)
member2 = model.Member(table_name = 'user',
table_id = self.publisher2_user.id,
group=self.publisher2,
capacity='admin')
Session.add(member2)
Session.commit()
def test_auth_publisher_profile_different_publisher(self):
# Create a source for publisher 1
source = HarvestSource(url=u'http://test-source.com',
type='ckan',
publisher_id=self.publisher1.id)
Session.add(source)
Session.commit()
extra_environ = {
'REMOTE_USER': self.publisher2_user.name.encode('utf8')
}
# List (Publihsers can see the sources list)
res = self.app.get('/harvest', extra_environ=extra_environ)
assert 'Harvesting Sources' in res
# Create
res = self.app.get('/harvest/new', extra_environ=extra_environ)
assert 'New harvest source' in res
assert 'publisher_id' in res
# Check that this publihser is not allowed to manage sources from other publishers
status = 401
# Read
res = self.app.get('/harvest/%s' % source.id,
status=status,
extra_environ=extra_environ)
# Edit
res = self.app.get('/harvest/edit/%s' % source.id,
status=status,
extra_environ=extra_environ)
# Refresh
res = self.app.get('/harvest/refresh/%s' % source.id,
status=status,
extra_environ=extra_environ)
def create_unique_identifier(package_id):
"""
Create a unique identifier, using the prefix and a random number: 10.5072/0044634
Checks the random number doesn't exist in the table or the datacite repository
All unique identifiers are created with
@return:
"""
datacite_api = DOIDataCiteAPI()
while True:
identifier = os.path.join(get_prefix(), '{0:07}'.format(random.randint(1, 100000)))
# Check this identifier doesn't exist in the table
if not Session.query(DOI).filter(DOI.identifier == identifier).count():
# And check against the datacite service
try:
datacite_doi = datacite_api.get(identifier)
except HTTPError:
pass
else:
if datacite_doi.text:
continue
doi = DOI(package_id=package_id, identifier=identifier)
Session.add(doi)
Session.commit()
return doi
def authenticate(self, environ, identity):
""" Mimic most of UsernamePasswordAuthenticator.authenticate
but add account lockout after 10 failed attempts.
"""
if 'login' not in identity or 'password' not in identity:
return None
user = User.by_name(identity.get('login'))
if user is None:
LOG.debug('Login failed - username %r not found',
identity.get('login'))
return None
qgov_user = Session.query(QGOVUser).filter_by(
name=identity.get('login')).first()
if qgov_user.login_attempts >= 10:
LOG.debug('Login as %r failed - account is locked',
identity.get('login'))
elif user.validate_password(identity.get('password')):
# reset attempt count to 0
qgov_user.login_attempts = 0
Session.commit()
return user.name
else:
LOG.debug('Login as %r failed - password not valid',
identity.get('login'))
qgov_user.login_attempts += 1
Session.commit()
return None
def handle_submit(self, id):
data = clean_dict(
dict_fns.unflatten(tuplize_dict(parse_params(request.params))))
data['dataset_url'] = toolkit.url_for(controller='package',
action='read',
id=id,
qualified=True)
package = get_action('package_show')(None, {'id': id})
self.fail_if_private(package, data['dataset_url'])
# Comma separated config var
to_addrs = config['ckanext.ands.support_emails'].split(',')
subject = 'DataPortal Support: Request to publish dataset'
body = base.render('package/doi_email.text', extra_vars=data)
for email in to_addrs:
mail_recipient('Dataportal support', email, subject, body)
data['package_id'] = package['id']
data['user_id'] = c.userobj.id
doi_request = DoiRequest(**data)
Session.add(doi_request)
Session.commit()
h.flash_success("DOI Request sent")
return toolkit.redirect_to(data['dataset_url'])
def _test_auth_not_allowed(self, user_name=None, source=None, status=401):
if not source:
# Create harvest source
source = HarvestSource(url=u'http://test-source.com', type='ckan')
Session.add(source)
Session.commit()
if user_name:
extra_environ = {'REMOTE_USER': user_name.encode('utf8')}
else:
extra_environ = {}
# List
res = self.app.get('/harvest',
status=status,
extra_environ=extra_environ)
# Create
res = self.app.get('/harvest/new',
status=status,
extra_environ=extra_environ)
# Read
res = self.app.get('/harvest/%s' % source.id,
status=status,
extra_environ=extra_environ)
# Edit
res = self.app.get('/harvest/edit/%s' % source.id,
status=status,
extra_environ=extra_environ)
# Refresh
res = self.app.get('/harvest/refresh/%s' % source.id,
status=status,
extra_environ=extra_environ)
def _create_pkg(self):
model.repo.new_revision()
pkg = Package.get("annakarenina")
pkg.name = "fookarenina"
pkg.add_resource("www.google.com", description="foo", name="foo")
Session.add(pkg)
Session.commit()
return pkg
def _create_pkg(self):
model.repo.new_revision()
pkg = Package.get('annakarenina')
pkg.name = "fookarenina"
pkg.add_resource('www.google.com', description='foo', name="foo")
Session.add(pkg)
Session.commit()
return pkg
def save(self, commit=True):
if not hasattr(self, 'slug') or not self.slug:
self.slug = slugify(self.name)
session = Session()
log.debug(self)
session.add(self)
if commit:
session.commit()
def delete_tests(self):
'''Delete all test DOIs.'''
print u'Deleting all test DOIs'
Session.query(DOI).filter(
DOI.identifier.like(u'%' + get_prefix() +
u'%')).delete(synchronize_session=False)
Session.commit()
def _execute_sql(cls, script):
engine = create_engine(cls.sqlalchemy_url)
Session.bind = engine
connection = Session.connection()
connection.execute(script)
Session.commit()
def import_stage(self, harvest_object):
log.debug('In SwisstopoHarvester import_stage')
if not harvest_object:
log.error('No harvest object received')
return False
try:
package_dict = json.loads(harvest_object.content)
package_dict['id'] = harvest_object.guid
package_dict['name'] = munge_title_to_name(
package_dict['layer_name']
)
user = model.User.get(self.HARVEST_USER)
context = {
'model': model,
'session': Session,
'user': self.HARVEST_USER
}
# Find or create group the dataset should get assigned to
package_dict['groups'] = self._find_or_create_groups(context)
# Find or create the organization
# the dataset should get assigned to
package_dict['owner_org'] = self._find_or_create_organization(
context,
package_dict
)
# Save license url in extras
extras = []
if 'license_url' in package_dict:
extras.append(('license_url', package_dict['license_url']))
package_dict['extras'] = extras
package = model.Package.get(package_dict['id'])
model.PackageRole(
package=package,
user=user,
role=model.Role.ADMIN
)
log.debug(
'Save or update package %s (%s)'
% (package_dict['name'], package_dict['id'])
)
self._create_or_update_package(package_dict, harvest_object)
log.debug('Save or update term translations')
self._submit_term_translations(context, package_dict)
Session.commit()
except Exception, e:
log.exception(e)
raise
def update_stat_metameta_table(org_id, metameta_id, state):
"""
update stat on metameta table with new status
"""
update_sql = "UPDATE metameta SET state='%s', modified_date='%s' WHERE id ='%s' AND org_id='%s';"%(state, datetime.datetime.utcnow().isoformat(), metameta_id, org_id)
result = Session.execute(update_sql)
Session.commit()
def create(cls, **kwargs):
'''
Create a new record of type :class:`~m`.
'''
item_dict = cls._columns(**kwargs)
new_item = cls.m(**item_dict)
Session.add(new_item)
Session.commit()
return new_item
def _execute_script(script_path):
conn = Session.connection()
script = open(script_path, 'r').read()
for cmd in script.split(';'):
cmd = re.sub(r'--(.*)|[\n\t]', '', cmd)
if len(cmd):
conn.execute(cmd)
Session.commit()
def update_stat_metameta_table(org_id, metameta_id, state):
"""
update stat on metameta table with new status
"""
update_sql = "UPDATE metameta SET state='%s', modified_date='%s' WHERE id ='%s' AND org_id='%s';" % (
state, datetime.datetime.utcnow().isoformat(), metameta_id, org_id)
result = Session.execute(update_sql)
Session.commit()
def _execute_script(script_path):
conn = Session.connection()
script = open(script_path, "r").read()
for cmd in script.split(";"):
cmd = re.sub(r"--(.*)|[\n\t]", "", cmd)
if len(cmd):
conn.execute(cmd)
Session.commit()
def insert_metameta_options(org_id, key, option_value):
"""
creates entries in metameta options table for the organisation.
value fields are null
"""
sql = "INSERT INTO metameta_options(id, org_id, key, option_value, modified_date) VALUES ('%s', '%s', '%s', '%s', '%s');"%(str(uuid.uuid4()), org_id, key, option_value, datetime.datetime.utcnow().isoformat())
result = Session.execute(sql)
Session.commit()
return
def import_stage(self, harvest_object):
'''
The import stage will receive a HarvestObject object and will be
responsible for:
- performing any necessary action with the fetched object (e.g
create a CKAN package).
Note: if this stage creates or updates a package, a reference
to the package must be added to the HarvestObject.
Additionally, the HarvestObject must be flagged as current.
- creating the HarvestObject - Package relation (if necessary)
- creating and storing any suitable HarvestObjectErrors that may
occur.
- returning True if everything went as expected, False otherwise.
:param harvest_object: HarvestObject object
:returns: True if everything went right, False if errors were found
'''
logger.debug("in import stage: %s" % harvest_object.guid)
if not harvest_object:
logger.error('No harvest object received')
self._save_object_error('No harvest object received')
return False
try:
self._set_config(harvest_object.job.source.config)
context = {'model': model, 'session': Session, 'user': self.user}
package_dict = json.loads(harvest_object.content)
package_dict['id'] = munge_title_to_name(harvest_object.guid)
package_dict['name'] = package_dict['id']
# add owner_org
source_dataset = get_action('package_show')(
{
'ignore_auth': True
}, {
'id': harvest_object.source.id
})
owner_org = source_dataset.get('owner_org')
package_dict['owner_org'] = owner_org
# logger.debug('Create/update package using dict: %s' % package_dict)
self._create_or_update_package(package_dict, harvest_object,
'package_show')
Session.commit()
logger.debug("Finished record")
except:
logger.exception('Something went wrong!')
self._save_object_error('Exception in import stage',
harvest_object)
return False
return True
def delete(cls, item_id):
'''
:param item_id:
'''
to_delete = Session.query(cls.m).get(item_id)
if to_delete is not None:
Session.delete(to_delete)
Session.commit()
def init_db():
#Setting up custom fiels table for locales
try:
custom_field_table.create()
except Exception,e:
# Make sure the table does not remain incorrectly created
if custom_field_table.exists():
Session.execute('DROP TABLE custom_field_table')
Session.commit()
raise e
def setup_postgis_tables():
conn = Session.connection()
script_path = os.path.join(os.path.dirname(os.path.abspath( __file__ )), 'scripts', 'postgis.sql')
script = open(script_path,'r').read()
for cmd in script.split(';'):
cmd = re.sub(r'--(.*)|[\n\t]','',cmd)
if len(cmd):
conn.execute(cmd)
Session.commit()
def init_db():
#Setting up custom fiels table for locales
try:
custom_field_table.create()
except Exception, e:
# Make sure the table does not remain incorrectly created
if custom_field_table.exists():
Session.execute('DROP TABLE custom_field_table')
Session.commit()
raise e
def insert_metameta_table(org_id, data):
"""
creates entries in metameta table for the organisation.
value fields are null
"""
errors = {}
#key
if not (data.has_key('key') and len(data['key'].strip())):
#TODO validation
errors.update({'key': _('%s field is required.' % ('key'))})
return errors
key = data['key'].strip()
#label: if not set, insert key
if data.has_key('label') and len(data['label'].strip()):
label = data['label'].strip()
else:
label = key.title()
#convert non-alphanumeric to underscore
key = create_search_key(key)
if org_has_defaults(org_id, key):
#TODO validation
errors.update({
'key':
_('%s has been already used by the organization.' % (key))
})
return errors
#readonly attribute
readonly = str(data.get('readonly', False)).lower() == 'True'.lower()
#validator
validator = data.get('validator', validator_not_empty)
#default_value
default_value = data.get('default_value', '')
#field_type
field_type = data.get('field_type', field_type_text)
min_value = max_value = 0.0
if field_type == field_type_text and validator == validator_range:
min_value = data.get('min_value', 0.0)
max_value = data.get('max_value', 0.0)
sql = "INSERT INTO metameta(id, org_id, key, default_value, label, validator, state, custom, presettable, readonly, field_type, min_value, max_value, sort_order, modified_date) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', %r, %r, %r, '%s', %f, %f, (CASE WHEN (SELECT MAX(sort_order)+1 FROM metameta WHERE org_id='%s')>101 THEN (SELECT MAX(sort_order)+1 FROM metameta WHERE org_id='%s') ELSE 101 END), '%s');" % (
str(uuid.uuid4()), org_id, key, default_value, label,
validator, state_active, True, True, readonly, field_type,
float(min_value), float(max_value), org_id, org_id,
datetime.datetime.utcnow().isoformat())
result = Session.execute(sql)
Session.commit()
return errors
def _unlock_account(account_id):
""" Unlock an account (erase the failed login attempts).
"""
qgov_user = Session.query(QGOVUser).filter(
QGOVUser.id == account_id).first()
if qgov_user:
LOG.debug("Clearing failed login attempts for %s", account_id)
qgov_user.login_attempts = 0
Session.commit()
else:
LOG.debug("Account %s not found", account_id)
def logged_in(self):
controller = UserController()
if not c.user:
# a number of failed login attempts greater than 10
# indicates that the locked user is associated with the current request
qgovUser = Session.query(QGOVUser).filter(QGOVUser.login_attempts > 10).first()
if qgovUser:
qgovUser.login_attempts = 10
Session.commit()
return controller.login("account-locked")
return controller.logged_in()
def delete_application(application_name):
application = Application.by_name(application_name)
if not application_name:
abort(404)
app_tags = Session.query(ApplicationTag) \
.filter(ApplicationTag.application_id==application.id) \
.all()
for app_tag in app_tags:
app_tag.delete()
application.delete()
Session.commit()
def setup(self):
# Add sysadmin user
harvest_user = model.User(name=u'harvest', password=u'test', sysadmin=True)
Session.add(harvest_user)
Session.commit()
package_schema = default_update_package_schema()
self.context ={'model':model,
'session':Session,
'user':u'harvest',
'schema':package_schema,
'api_version': '2'}
def update_package(cls, package_id, **kwargs):
'''
Update the package_id and/or published fields of a record associated with a given package.
:param package_id: the id of the package
:param kwargs: the values to be updated
:return: the updated record object
'''
update_dict = {k: v for k, v in kwargs.items() if k in cls.cols}
Session.query(DOI).filter(
DOI.package_id == package_id).update(update_dict)
Session.commit()
return cls.read_package(package_id)
def delete_idea(idea_name):
idea = Idea.by_name(idea_name)
if not idea_name:
abort(404)
idea_tags = Session.query(IdeaTag) \
.filter(IdeaTag.idea_id==idea.id) \
.all()
for idea_tag in idea_tags:
idea_tag.delete()
idea.delete()
Session.commit()
def remove_metameta_options(org_id, key, option_value=None):
"""
creates entries in metameta options table for the organisation.
value fields are null
"""
sql = "DELETE FROM metameta_options WHERE org_id = '%s' AND key = '%s' "%(org_id, key)
if option_value:
sql = sql + " AND option_value='%s' "%(option_value)
sql = sql + ";"
result = Session.execute(sql)
Session.commit()
return
def setup(self):
# Add sysadmin user
harvest_user = model.User(name=u'harvest', password=u'test', sysadmin=True)
Session.add(harvest_user)
Session.commit()
package_schema = default_update_package_schema()
self.context ={'model':model,
'session':Session,
'user':u'harvest',
'schema':package_schema,
'api_version': '2'}
def setup(srid=None):
if not srid:
srid = DEFAULT_SRID
srid = str(srid)
connection = Session.connection()
connection.execute('CREATE TABLE package_extent(package_id text PRIMARY KEY)')
connection.execute('SELECT AddGeometryColumn(\'package_extent\',\'the_geom\', %s, \'GEOMETRY\', 2)',srid)
Session.commit()
def create_package_doi(pkg_dict):
"""
Create a unique identifier, using the prefix and a random number: 10.5072/0044634
Checks the random number doesn't exist in the table or the datacite repository
All unique identifiers are created with
@return:
"""
_validate_package(pkg_dict)
identifier = _create_unique_identifier()
package_doi = CeonPackageDOI(package_id=pkg_dict['id'], identifier=identifier)
Session.add(package_doi)
Session.commit()
log.debug(u"Created DOI {} for package {}".format(package_doi.identifier, pkg_dict['id']))
return package_doi
def update_resource_stats(resource_id, total_visits, total_downloads):
q_result = Session.query(PiwikResource).filter(PiwikResource.resource_id == resource_id).first()
if q_result:
# update if resource is already in table
q_result.total_visits = total_visits
q_result.total_downloads = total_downloads
else:
# add new entry for resource
rt = PiwikResource()
rt.resource_id = resource_id
rt.total_visits = total_visits
rt.total_downloads = total_downloads
Session.add(rt)
Session.commit()
def update_package_stats(package_name, total_visits, recent_visits):
q_result = Session.query(PiwikPackage).filter(PiwikPackage.package_name == package_name).first()
if q_result:
# update if package is already in table
q_result.total_visits = total_visits
q_result.recent_visits = recent_visits
Session.commit()
else:
# add new entry for package
pt = PiwikPackage()
pt.package_name = package_name
pt.total_visits = total_visits
pt.recent_visits = recent_visits
Session.add(pt)
Session.commit()
def setup():
log.debug('Multilingual tables defined in memory')
#Setting up package multilang table
if not package_multilang_table.exists():
try:
package_multilang_table.create()
except Exception,e:
# Make sure the table does not remain incorrectly created
if package_multilang_table.exists():
Session.execute('DROP TABLE package_multilang')
Session.commit()
raise e
log.info('Package Multilingual table created')
def insert_metameta_table(org_id, data):
"""
creates entries in metameta table for the organisation.
value fields are null
"""
errors = {}
#key
if not (data.has_key('key') and len(data['key'].strip())):
#TODO validation
errors.update({'key': _('%s field is required.'%('key'))})
return errors
key = data['key'].strip()
#label: if not set, insert key
if data.has_key('label') and len(data['label'].strip()):
label = data['label'].strip()
else:
label = key.title()
#convert non-alphanumeric to underscore
key = create_search_key(key)
if org_has_defaults(org_id, key):
#TODO validation
errors.update({'key': _('%s has been already used by the organization.'%(key))})
return errors
#readonly attribute
readonly = str(data.get('readonly', False)).lower()=='True'.lower()
#validator
validator = data.get('validator', validator_not_empty)
#default_value
default_value = data.get('default_value', '')
#field_type
field_type = data.get('field_type', field_type_text)
min_value = max_value = 0.0
if field_type == field_type_text and validator == validator_range:
min_value = data.get('min_value', 0.0)
max_value = data.get('max_value', 0.0)
sql = "INSERT INTO metameta(id, org_id, key, default_value, label, validator, state, custom, presettable, readonly, field_type, min_value, max_value, sort_order, modified_date) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', %r, %r, %r, '%s', %f, %f, (CASE WHEN (SELECT MAX(sort_order)+1 FROM metameta WHERE org_id='%s')>101 THEN (SELECT MAX(sort_order)+1 FROM metameta WHERE org_id='%s') ELSE 101 END), '%s');"%(str(uuid.uuid4()), org_id, key, default_value, label, validator, state_active, True, True, readonly, field_type, float(min_value), float(max_value), org_id, org_id, datetime.datetime.utcnow().isoformat())
result = Session.execute(sql)
Session.commit()
return errors
def update_metameta_table(org_id, metameta_id, data):
"""
update metameta table with new values
"""
errors = {}
#key
if not (data.has_key('key') and len(data['key'].strip())):
#TODO validation
errors.update({'key': _('%s field is required.'%('key'))})
return errors
key = data['key'].strip()
#label: if not set, insert key
if data.has_key('label') and len(data['label'].strip()):
label = data['label'].strip()
else:
label = key.title()
#convert non-alphanumeric to underscore
key = create_search_key(key)
if org_has_defaults(org_id, key, metameta_id):
#TODO validation
errors.update({'key': _('%s has been already used by the organization.'%(key))})
return errors
#readonly attribute
readonly = str(data.get('readonly', False)).lower()=='True'.lower()
#validator
validator = data.get('validator', validator_not_empty)
#default_value
default_value = data.get('default_value', '')
#field_type
field_type = data.get('field_type', field_type_text)
min_value = max_value = 0.0
if field_type == field_type_text and validator == validator_range:
min_value = data.get('min_value', 0.0)
max_value = data.get('max_value', 0.0)
update_sql = "UPDATE metameta SET key='%s', default_value='%s', label='%s', validator='%s', readonly=%r, field_type='%s', min_value=%f, max_value=%f, modified_date='%s' WHERE id ='%s' AND org_id='%s';"%(key, default_value, label, validator, readonly, field_type, float(min_value), float(max_value), datetime.datetime.utcnow().isoformat(), metameta_id, org_id)
result = Session.execute(update_sql)
Session.commit()
return errors
def create_resource_doi(pkg_dict, res_dict):
"""
Create a unique identifier, using the prefix and a random number: 10.5072/0044634
Checks the random number doesn't exist in the table or the datacite repository
All unique identifiers are created with
@return:
"""
_validate_resource(res_dict)
resource_id = res_dict['id']
package_doi = CeonPackageDOI.get(pkg_dict['id'])
if not package_doi:
create_package_doi(pkg_dict)
identifier = _create_unique_identifier(package_doi.identifier)
resource_doi = CeonResourceDOI(resource_id=resource_id, identifier=identifier)
Session.add(resource_doi)
Session.commit()
log.debug(u"Created DOI {} for resource {}".format(resource_doi.identifier, res_dict['id']))
return resource_doi
def authenticate(self, environ, identity):
if 'repoze.who.plugins.openid.userid' in identity:
openid = identity.get('repoze.who.plugins.openid.userid')
user = User.by_openid(openid)
if user is None:
# TODO: Implement a mask to ask for an alternative user
# name instead of just using the OpenID identifier.
name = identity.get('repoze.who.plugins.openid.nickname')
if not User.check_name_valid(name):
name = openid
if not User.check_name_available(name):
name = openid
user = User(openid=openid, name=name,
fullname=identity.get('repoze.who.plugins.openid.fullname'),
email=identity.get('repoze.who.plugins.openid.email'))
Session.add(user)
Session.commit()
Session.remove()
return user.name
return None
def _test_auth_not_allowed(self, user_name=None, source=None, status=401):
if not source:
# Create harvest source
source = HarvestSource(url=u'http://test-source.com', type='ckan')
Session.add(source)
Session.commit()
if user_name:
extra_environ = {'REMOTE_USER': user_name.encode('utf8')}
else:
extra_environ = {}
# List
res = self.app.get(
'/harvest',
status=status,
extra_environ=extra_environ)
# Create
res = self.app.get(
'/harvest/new',
status=status,
extra_environ=extra_environ)
# Read
res = self.app.get(
'/harvest/%s' %
source.id,
status=status,
extra_environ=extra_environ)
# Edit
res = self.app.get(
'/harvest/edit/%s' %
source.id,
status=status,
extra_environ=extra_environ)
# Refresh
res = self.app.get(
'/harvest/refresh/%s' %
source.id,
status=status,
extra_environ=extra_environ)
