def send_email(req):
requestee = User.get(req.user_id)
pkg = Package.get(req.pkg_id)
selrole = False
for role in pkg.roles:
if role.role == "admin":
selrole = role
if not selrole:
return
admin = User.get(selrole.user_id)
msg = _("""%s (%s) is requesting editor access to a dataset you have created
%s.
Please click this link if you want to give this user write access:
%s%s""")
controller = 'ckanext.kata.controllers:AccessRequestController'
body = msg % (requestee.name, requestee.email, pkg.title if pkg.title else pkg.name,
config.get('ckan.site_url', ''),
h.url_for(controller=controller,
action="unlock_access",
id=req.id))
email_dict = {}
email_dict["subject"] = _("Access request for dataset %s" % pkg.title if pkg.title else pkg.name)
email_dict["body"] = body
send_notification(admin.as_dict(), email_dict)
def send_notifications_on_new_post(post, lang):
from ckan.model import User
template_dir = os.path.join(os.path.dirname(__file__), 'templates')
locale_dir = os.path.join(os.path.dirname(__file__), 'i18n')
env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir),
extensions=['jinja2.ext.i18n'])
translations = Translations.load(locale_dir, [lang],
domain='ckanext-forum')
env.install_gettext_translations(translations)
env.globals['get_locale'] = lambda: lang
post_author = User.get(post.author_id)
thread = Thread.get_by_id(post.thread_id)
author_ids = set([p.author_id
for p in thread.forum_posts] + [thread.author_id])
author_ids -= set([
u.user_id for u in Unsubscription.filter_by_thread_id(post.thread_id)
])
for author_id in author_ids:
if author_id == post_author.id:
continue
user = User.get(author_id)
unsubscribe_url = tk.url_for('forum_unsubscribe',
base64_name=base64.b64encode(user.name),
thread_id=thread.id)
context = {
'user_name':
user.name,
'site_title':
tk.config.get('ckan.site_title'),
'site_url':
tk.config.get('ckan.site_url'),
'post_content':
post.content,
'title':
env.globals['gettext']('New post'),
'unsubscribe_url':
urljoin(tk.config['ckan.site_url'], unsubscribe_url),
'username':
post_author.name,
'thread_url':
urljoin(tk.config['ckan.site_url'], thread.get_absolute_url()),
}
template = env.get_template('forum_new_post_mail.html')
body = template.render(context)
log.debug('Email body %s', body)
tk.get_action('send_mail')(
{}, {
'to': user.email,
'subject': env.globals['gettext']('New post'),
'message_html': body
})
def harvest_job_show(context, data_dict):
model = context['model']
user = context.get('user')
job = get_job_object(context, data_dict)
if not user:
return {
'success': False,
'msg':
_('Non-logged in users are not authorized to see harvest jobs')
}
if Authorizer().is_sysadmin(user):
return {'success': True}
user_obj = User.get(user)
if not user_obj or not job.source.publisher_id in [
g.id for g in user_obj.get_groups(u'publisher')
]:
return {
'success':
False,
'msg':
_('User %s not authorized to read harvest job %s') %
(str(user), job.id)
}
else:
return {'success': True}
def harvesters_info_show(context, data_dict):
model = context['model']
user = context.get('user', '')
# Non-logged users can not create sources
if not user:
return {
'success': False,
'msg': _('Non-logged in users can not see the harvesters info')
}
# Sysadmins and the rest of logged users can see the harvesters info,
# as long as they belong to a publisher
user_obj = User.get(user)
if not user_obj or not Authorizer().is_sysadmin(user) and len(
user_obj.get_groups(u'publisher')) == 0:
return {
'success':
False,
'msg':
_('User %s must belong to a publisher to see the harvesters info')
% str(user)
}
else:
return {'success': True}
def harvest_jobs_run(context,data_dict):
model = context['model']
user = context.get('user')
# Check user is logged in
if not user:
return {'success': False, 'msg': _('Only logged users are authorized to run harvest jobs')}
user_obj = User.get(user)
# Checks for non sysadmin users
if not Authorizer().is_sysadmin(user):
if not user_obj or len(user_obj.get_groups(u'publisher')) == 0:
return {'success': False, 'msg': _('User %s must belong to a publisher to run harvest jobs') % str(user)}
source_id = data_dict.get('source_id',False)
if not source_id:
return {'success': False, 'msg': _('Only sysadmins can run all harvest jobs') % str(user)}
source = HarvestSource.get(source_id)
if not source:
raise NotFound
if not source.publisher_id in [g.id for g in user_obj.get_groups(u'publisher')]:
return {'success': False, 'msg': _('User %s not authorized to run jobs from source %s') % (str(user),source.id)}
return {'success': True}
def resource_tracker_create(context, data_dict):
'''Append a new resource tracker to the list of resource log
:param resource_id: the id of the resource
:param event: the action which the user take
:param obj_type: object type which the user action is applied to.
:param user_id: the username of the user
'''
check_access('resource_tracker_create', context, data_dict)
data, errors = validate(data_dict, resource_tracker_create_schema(),
context)
if errors:
raise ValidationError(errors)
logger = User.get(context.get('user'))
if logger:
tracker = ResourceLog(
resource_id=data.get('resource_id'),
event=data.get('event'),
obj_type=data.get('obj_type'),
user_id=logger.name,
)
else:
tracker = ResourceLog(
resource_id=data.get('resource_id'),
event=data.get('event'),
obj_type=data.get('obj_type'),
user_id=None,
)
tracker.save()
return tracker.as_dict()
def harvest_source_list(context, data_dict):
model = context['model']
user = context.get('user')
# Here we will just check that the user is logged in.
# The logic action will return an empty list if the user does not
# have permissons on any source.
if not user:
return {
'success': False,
'msg': _('Only logged users are authorized to see their sources')
}
else:
user_obj = User.get(user)
assert user_obj
# Only users belonging to a publisher can list sources,
# unless they are sysadmins
if Authorizer().is_sysadmin(user_obj):
return {'success': True}
if len(user_obj.get_groups(u'publisher')) > 0:
return {'success': True}
else:
return {
'success':
False,
'msg':
_('User %s must belong to a publisher to list harvest sources')
% str(user)
}
def harvest_job_create(context, data_dict):
model = context['model']
user = context.get('user')
source_id = data_dict['source_id']
if not user:
return {
'success':
False,
'msg':
_('Non-logged in users are not authorized to create harvest jobs')
}
if ckan.new_authz.is_sysadmin(user):
return {'success': True}
user_obj = User.get(user)
source = HarvestSource.get(source_id)
if not source:
raise NotFound
if not user_obj or not source.publisher_id in [
g.id for g in user_obj.get_groups(u'organization')
]:
return {
'success':
False,
'msg':
_('User %s not authorized to create a job for source %s') %
(str(user), source.id)
}
else:
return {'success': True}
def harvest_source_update(context, data_dict):
model = context['model']
user = context.get('user', '')
source = get_source_object(context, data_dict)
# Non-logged users can not update this source
if not user:
return {
'success':
False,
'msg':
_('Non-logged in users are not authorized to update harvest sources'
)
}
# Sysadmins can update the source
if Authorizer().is_sysadmin(user):
return {'success': True}
# Check if the source publisher id exists on the user's groups
user_obj = User.get(user)
if not user_obj or not source.publisher_id in [
g.id for g in user_obj.get_groups(u'publisher')
]:
return {
'success':
False,
'msg':
_('User %s not authorized to update harvest source %s') %
(str(user), source.id)
}
else:
return {'success': True}
def harvest_source_create(context, data_dict):
model = context['model']
user = context.get('user', '')
# Non-logged users can not create sources
if not user:
return {
'success':
False,
'msg':
_('Non-logged in users are not authorized to create harvest sources'
)
}
# Sysadmins and the rest of logged users can create sources,
# as long as they belong to a publisher
user_obj = User.get(user)
if not user_obj or not ckan.new_authz.is_sysadmin(user) and len(
user_obj.get_groups(u'organization')) == 0:
return {
'success':
False,
'msg':
_('User %s must belong to a publisher to create harvest sources') %
str(user)
}
else:
return {'success': True}
def command(self):
self._load_config()
self._user = User.get(self.site_user['name'])
with app_context() as context:
context.g.user = self.site_user['name']
context.g.userobj = self._user
self.migrate_all_resources()
def send(self, pkg_id):
package = Package.get(pkg_id)
url = h.url_for(controller='package',
action="read",
id=package.id)
if c.user:
userid = None
for role in package.roles:
if role.role == "admin":
userid = role.user_id
break
if userid:
owner = User.get(userid)
msg = request.params.get('msg', '')
if msg:
send_contact_email(owner, c.userobj, package,\
msg)
else:
h.flash_error(_("No message"))
return redirect(url)
else:
h.flash_error(_("No owner found"))
return redirect(url)
h.flash_notice(_("Message sent"))
else:
h.flash_error(_("Please login"))
return redirect(url)
def is_owner(context, data_dict):
'''
This is used in "request edit rights" feature.
Checks if the user is admin or editor of the
package in question
:param context: context
:param data_dict: package data
:type data_dict: dictionary
:rtype: dictionary
'''
pkg = context.get('package', None)
roles = pkg.roles if pkg else Package.get(data_dict['id']).roles
user = context.get('user', False)
if user:
for role in roles:
ruser = User.get(role.user.id)
if user == ruser.name and role.role in ('admin', 'editor'):
return {'success': True}
# Check if the user has editor rights to this dataset through an organization
package = get_package_object(context, data_dict)
if new_authz.has_user_permission_for_group_or_org(package.owner_org, user, 'delete_dataset'):
return {'success': True}
return {'success': False}
def _get_sources_for_user(context,
data_dict,
organization_id=None,
limit=None):
session = context['session']
user = context.get('user', '')
only_active = data_dict.get('only_active', False)
only_to_run = data_dict.get('only_to_run', False)
query = session.query(HarvestSource) \
.order_by(HarvestSource.created.desc())
if organization_id:
query = query.join(Package, HarvestSource.id == Package.id).filter(
Package.owner_org == organization_id)
if only_active:
query = query.filter(
HarvestSource.active == True # noqa: E712
) \
if only_to_run:
query = query.filter(HarvestSource.frequency != 'MANUAL')
query = query.filter(
or_(
HarvestSource.next_run <= datetime.datetime.utcnow(),
HarvestSource.next_run == None # noqa: E711
))
user_obj = User.get(user)
# Sysadmins will get all sources
if user_obj and not user_obj.sysadmin:
# This only applies to a non sysadmin user when using the
# publisher auth profile. When using the default profile,
# normal users will never arrive at this point, but even if they
# do, they will get an empty list.
publisher_filters = []
publishers_for_the_user = user_obj.get_groups(u'publisher')
for publisher_id in [g.id for g in publishers_for_the_user]:
publisher_filters.append(
HarvestSource.publisher_id == publisher_id)
if len(publisher_filters):
query = query.filter(or_(*publisher_filters))
else:
# This user does not belong to a publisher yet, no sources for him/her
return []
log.debug('User %s with publishers %r has Harvest Sources: %r', user,
publishers_for_the_user, [(hs.id, hs.url) for hs in query])
sources = query.limit(limit).all() if limit else query.all()
return sources
def user_organizations(user):
u = User.get(user['name'])
groups = u.get_groups(group_type="organization")
groups_data = []
for group in groups:
context = {'model': model}
data_dict = {'id': group.id, "include_extra": True, 'all_fields': True}
group_dict = get_action('organization_show')(context, data_dict)
groups_data.append(group_dict)
return groups_data
def unsubscribe(self, base64_name, thread_id):
log.debug('Unsubscribing %s %s', base64.b64decode(base64_name),
thread_id)
thread = Thread.get_by_id(thread_id)
user = User.get(base64.b64decode(base64_name))
if not thread or not user:
abort(404)
Unsubscription.add(user.id, thread.id)
flash_success(tk._('You successfully unsibsribed'))
tk.redirect_to(thread.get_absolute_url())
def identify(self):
'''Identify which user (if any) is logged-in
If a logged-in user is found, set toolkit.c.user to be their user name.
'''
# Try to get the item that login() placed in the session.
user_id = pylons.session.get('ckanext-discourse-sso-client-user-id')
if user_id:
# We've found a logged-in user. Set c.user to let CKAN know.
user = User.get(user_id)
toolkit.c.user = user.name
def authenticate(self, environ, identity):
if 'shibboleth_auth' in identity:
userid = identity['shibboleth_auth']
user = User.get(userid)
if user is None or not user.is_active():
log.info("ShibbolethAuthenticator: user not found: %s", userid)
return None
else:
log.info("ShibbolethAuthenticator: user found %s", userid)
return user.name
return None
def is_owner(context, data_dict):
pkg = context.get('package', None)
roles = pkg.roles if pkg else Package.get(data_dict['id']).roles
user = context.get('user', False)
if user:
for role in roles:
ruser = User.get(role.user_id)
if user == ruser.name and role.role in ('admin', 'editor'):
return {'success': True}
else:
return {'success': False}
return {'success': False}
def authenticate(self, environ, identity):
if 'provbz_auth' in identity:
userid = identity['provbz_auth']
user = User.get(userid)
if user is None or not user.is_active():
log.info("ProvBzAuthenticator: user not found: %s", userid)
return None
else:
log.info("ProvBzAuthenticator: user found %s", userid)
return user.name
return None
def harvest_objects_import(context, data_dict):
model = context['model']
user = context.get('user')
# Check user is logged in
if not user:
return {
'success':
False,
'msg':
_('Only logged users are authorized to reimport harvest objects')
}
user_obj = User.get(user)
# Checks for non sysadmin users
if not Authorizer().is_sysadmin(user):
if not user_obj or len(user_obj.get_groups(u'publisher')) == 0:
return {
'success':
False,
'msg':
_('User %s must belong to a publisher to reimport harvest objects'
) % str(user)
}
source_id = data_dict.get('source_id', False)
if not source_id:
return {
'success':
False,
'msg':
_('Only sysadmins can reimport all harvest objects') %
str(user)
}
source = HarvestSource.get(source_id)
if not source:
raise NotFound
if not source.publisher_id in [
g.id for g in user_obj.get_groups(u'publisher')
]:
return {
'success':
False,
'msg':
_('User %s not authorized to reimport objects from source %s')
% (str(user), source.id)
}
return {'success': True}
def get_package_count(c, organization):
user = User.get(c.user)
member = AuthMember.by_group_and_user_id(group_id=organization['id'],
user_id=user.id)
if (member is None or member.role is None) and not user.sysadmin:
return meta.Session.query(Package).filter_by(
owner_org=organization['id'], state='active',
private=False).count()
else:
return meta.Session.query(Package).filter_by(
owner_org=organization['id'], state='active').count()
def send_edit_access_request_email(req):
"""
Send edit access request email.
:param user_id: user who requests access
:param pkg_id: dataset's id
"""
requester = User.get(req.user_id)
pkg = Package.get(req.pkg_id)
selrole = False
for role in pkg.roles:
if role.role == "admin":
selrole = role
if not selrole:
return
admin = User.get(selrole.user_id)
admin_dict = admin.as_dict()
admin_dict['name'] = admin.fullname if admin.fullname else admin.name
msg = u'{a} ({b}) is requesting editing rights to the metadata in dataset\n\n{c}\n\n\
for which you are currently an administrator. Please click this \
link if you want to allow this user to edit the metadata of the dataset:\n\
{d}\n\n{a} ({b}) pyytää muokkausoikeuksia tietoaineiston\n\n{c}\n\n\
metatietoihin, joiden ylläpitäjä olet. Klikkaa linkkiä, jos haluat tämän käyttäjän \
saavan muokkausoikeudet aineiston metatietoihin:\n\
{d}\n'
controller = 'ckanext.kata.controllers:EditAccessRequestController'
requester_name = requester.fullname if requester.fullname else requester.name
accessurl = config.get('ckan.site_url', '') + h.url_for(controller=controller, action="unlock_access", id=req.id)
body = msg.format(a=requester_name, b=requester.email, c=pkg.title if pkg.title else pkg.name, d=accessurl)
email_dict = {}
email_dict["subject"] = u"Access request for dataset / pyyntö koskien tietoaineistoa %s" % pkg.title if pkg.title else pkg.name
email_dict["body"] = body
send_notification(admin_dict, email_dict)
def after_search(self, search_params, search_results):
context = {
'model': ckan.model,
'session': ckan.model.Session,
'user': pylons.c.user
}
#set permission level: read (default is edit)
data_dict = {'user': pylons.c.user, 'permission': 'read'}
#get list of organisations that the user is a member of
orgs = ckan.logic.get_action('organization_list_for_user')(context,
data_dict)
#user doesn't belong to an organisation
if not orgs:
print('User is not a member of any organisations!')
c.maintainers = []
return search_params
#get a distinct list of members who belong to the organisations
members = []
for org in orgs:
params = {'id': org['id'], 'object_type': 'user'}
member_list = ckan.logic.get_action('member_list')(context, params)
for m in member_list:
members.append(m)
memberset = set(members)
#need the user name to match with the maintainer field
current_user_name = None
member_names = []
for member in memberset:
user = User.get(member[0]) #user id
member_names.append(user.name)
#get all maintainers
maintainers = [
p[0] for p in meta.Session.query(distinct(Package.maintainer))
if p[0]
]
maintset = set(maintainers)
#filter maintainers by user-related organisation members
results = maintset.intersection(member_names)
c.maintainers = results
return search_params
def _get_sources_for_user(context, data_dict):
model = context['model']
session = context['session']
user = context.get('user', '')
only_active = data_dict.get('only_active', False)
only_to_run = data_dict.get('only_to_run', False)
query = session.query(HarvestSource) \
.order_by(HarvestSource.created.desc())
if only_active:
query = query.filter(HarvestSource.active) \
if only_to_run:
query = query.filter(HarvestSource.frequency != 'MANUAL')
query = query.filter(or_(HarvestSource.next_run <=
datetime.datetime.utcnow(),
HarvestSource.next_run is None))
user_obj = User.get(user)
# Sysadmins will get all sources
if user_obj and not user_obj.sysadmin:
# This only applies to a non sysadmin user when using the
# publisher auth profile. When using the default profile,
# normal users will never arrive at this point, but even if they
# do, they will get an empty list.
publisher_filters = []
publishers_for_the_user = user_obj.get_groups(u'publisher')
for publisher_id in [g.id for g in publishers_for_the_user]:
publisher_filters.append(
HarvestSource.publisher_id == publisher_id)
if len(publisher_filters):
query = query.filter(or_(*publisher_filters))
else:
# This user does not belong to a publisher yet, no sources for
# him/her
return []
log.debug('User %s with publishers %r has Harvest Sources: %r',
user, publishers_for_the_user,
[(hs.id, hs.url) for hs in query])
sources = query.all()
return sources
def harvest_source_create(context, data_dict):
model = context["model"]
user = context.get("user", "")
# Non-logged users can not create sources
if not user:
return {"success": False, "msg": _("Non-logged in users are not authorized to create harvest sources")}
# Sysadmins and the rest of logged users can create sources,
# as long as they belong to a publisher
user_obj = User.get(user)
if not user_obj or not authz.is_sysadmin(user) and len(user_obj.get_groups(u"publisher")) == 0:
return {"success": False, "msg": _("User %s must belong to a publisher to create harvest sources") % str(user)}
else:
return {"success": True}
def harvesters_info_show(context,data_dict):
model = context['model']
user = context.get('user','')
# Non-logged users can not create sources
if not user:
return {'success': False, 'msg': _('Non-logged in users can not see the harvesters info')}
# Sysadmins and the rest of logged users can see the harvesters info,
# as long as they belong to a publisher
user_obj = User.get(user)
if not user_obj or not authz.is_sysadmin(user) and len(user_obj.get_groups(u'publisher')) == 0:
return {'success': False, 'msg': _('User %s must belong to a publisher to see the harvesters info') % str(user)}
else:
return {'success': True}
def harvest_source_create(context,data_dict):
model = context['model']
user = context.get('user','')
# Non-logged users can not create sources
if not user:
return {'success': False, 'msg': _('Non-logged in users are not authorized to create harvest sources')}
# Sysadmins and the rest of logged users can create sources,
# as long as they belong to a publisher
user_obj = User.get(user)
if not user_obj or not Authorizer().is_sysadmin(user) and len(user_obj.get_groups(u'publisher')) == 0:
return {'success': False, 'msg': _('User %s must belong to a publisher to create harvest sources') % str(user)}
else:
return {'success': True}
def test_tokens_related_to_user(self):
user_1 = factories.User()
num_tokens_1 = 5
user_2 = factories.User()
num_tokens_2 = 3
for i in range(num_tokens_1):
Session.add(ApiToken(user_1[u"id"]))
for i in range(num_tokens_2):
Session.add(ApiToken(user_2[u"id"]))
Session.commit()
user_1 = User.get(user_1[u"id"])
assert len(user_1.api_tokens) == num_tokens_1
for token in user_1.api_tokens:
assert token.owner == user_1
user_2 = User.get(user_2[u"id"])
assert len(user_2.api_tokens) == num_tokens_2
for token in user_2.api_tokens:
assert token.owner == user_2
def unlock_access(self, id):
q = model.Session.query(KataAccessRequest)
q = q.filter_by(id=id)
req = q.first()
if req:
user = User.get(req.user_id)
pkg = Package.get(req.pkg_id)
add_user_to_role(user, 'editor', pkg)
url = h.url_for(controller='package', action='read', id=req.pkg_id)
h.flash_success(_("%s now has editor rights to package %s" % (user.name, pkg.name)))
req.delete()
meta.Session.commit()
redirect(url)
else:
h.flash_error(_("No such request found!"))
redirect('/')
def user_autocomplete(context, data_dict):
'''
Override to explicitly allow logged in users to have
user autocompletion even if user_list is disallowed.
:param context:
:param data_dict:
:return:
'''
user_name = context.get('user')
user_obj = User.get(user_name) if user_name else None
if user_obj:
return {'success': True}
else:
return {'success': False}
def user_autocomplete(context, data_dict):
'''
Override to explicitly allow logged in users to have
user autocompletion even if user_list is disallowed.
:param context:
:param data_dict:
:return:
'''
user_name = context.get('user')
user_obj = User.get(user_name) if user_name else None
if user_obj:
return {'success': True}
else:
return {'success': False}
def after_search(self, search_params, search_results):
context = {'model': ckan.model,
'session': ckan.model.Session,
'user': pylons.c.user}
#set permission level: read (default is edit)
data_dict = {'user': pylons.c.user, 'permission': 'read'}
#get list of organisations that the user is a member of
orgs = ckan.logic.get_action('organization_list_for_user')(context, data_dict)
#user doesn't belong to an organisation
if not orgs:
print ('User is not a member of any organisations!')
c.maintainers = []
return search_params
#get a distinct list of members who belong to the organisations
members = []
for org in orgs:
params = {'id': org['id'], 'object_type': 'user'}
member_list = ckan.logic.get_action('member_list')(context, params)
for m in member_list:
members.append(m)
memberset = set(members)
#need the user name to match with the maintainer field
current_user_name = None
member_names = []
for member in memberset:
user = User.get(member[0]) #user id
member_names.append(user.name)
#get all maintainers
maintainers = [p[0] for p in meta.Session.query(distinct(Package.maintainer)) if p[0]]
maintset = set(maintainers)
#filter maintainers by user-related organisation members
results = maintset.intersection(member_names)
c.maintainers = results
return search_params
def harvest_job_show(context,data_dict):
model = context['model']
user = context.get('user')
job = get_job_object(context,data_dict)
if not user:
return {'success': False, 'msg': _('Non-logged in users are not authorized to see harvest jobs')}
if Authorizer().is_sysadmin(user):
return {'success': True}
user_obj = User.get(user)
if not user_obj or not job.source.publisher_id in [g.id for g in user_obj.get_groups(u'publisher')]:
return {'success': False, 'msg': _('User %s not authorized to read harvest job %s') % (str(user),job.id)}
else:
return {'success': True}
def _get_sources_for_user(context,data_dict):
model = context['model']
session = context['session']
user = context.get('user','')
only_mine = data_dict.get('only_mine', False)
only_active = data_dict.get('only_active',False)
only_organization = data_dict.get('organization') or data_dict.get('group')
query = session.query(HarvestSource) \
.order_by(HarvestSource.created.desc())
if only_active:
query = query.filter(HarvestSource.active==True) \
if only_mine:
# filter to only harvest sources from this user's organizations
user_obj = User.get(user)
publisher_filters = []
publishers_for_the_user = user_obj.get_groups(u'organization')
for publisher_id in [g.id for g in publishers_for_the_user]:
publisher_filters.append(HarvestSource.publisher_id==publisher_id)
if len(publisher_filters):
query = query.filter(or_(*publisher_filters))
else:
# This user does not belong to a publisher yet, no sources for him/her
return []
log.debug('User %s with publishers %r has Harvest Sources: %r',
user, publishers_for_the_user, [(hs.id, hs.url) for hs in query])
if only_organization:
org = model.Group.get(only_organization)
if not org:
raise p.toolkit.ObjectNotFound('Could not find: %s' % only_organization)
query = query.filter(HarvestSource.publisher_id==org.id)
sources = query.all()
return sources
def test_harvester_import(self):
harv, job = self._create_harvester()
res = "http://www.fsd.uta.fi/fi/aineistot/luettelo/FSD0115/FSD0115.xml"
urllib2.urlopen = mock.Mock(return_value=StringIO(res))
gathered = harv.gather_stage(job)
urllib2.urlopen = mock.Mock(return_value=StringIO(testdata.nr1))
harvest_obj = HarvestObject.get(gathered[0])
self.assert_(harv.fetch_stage(harvest_obj))
self.assert_(isinstance(json.loads(harvest_obj.content), dict))
self.assert_(harv.import_stage(harvest_obj))
self.assert_(len(Session.query(Package).all()) == 1)
# Lets see if the package is ok, according to test data
pkg = Session.query(Package).filter(Package.title == "Puolueiden ajankohtaistutkimus 1981").one()
self.assert_(pkg.title == "Puolueiden ajankohtaistutkimus 1981")
log.debug(pkg.extras)
self.assert_(len(pkg.get_groups()) == 2)
self.assert_(len(pkg.resources) == 4)
self.assert_(len(pkg.get_tags()) == 9)
self.assert_(pkg.url == "http://www.fsd.uta.fi/fi/aineistot/luettelo/FSD0115/FSD0115.xml")
self.assert_(isinstance(pkg.extras, _AssociationDict))
self.assert_(len(pkg.extras.items()) > 1)
urllib2.urlopen = mock.Mock(return_value=StringIO(testdata.nr2))
harvest_obj = HarvestObject.get(gathered[0])
harvest_obj.content = json.dumps({'url': 'http://foo'})
self.assert_(harv.fetch_stage(harvest_obj))
self.assert_(isinstance(json.loads(harvest_obj.content), dict))
self.assert_(harv.import_stage(harvest_obj))
self.assert_(len(Session.query(Package).all()) == 2)
# Test user access
user = User.get('testlogin2')
grp = pkg.get_groups()[0]
context = {'user': user.name, 'model': model}
data_dict = {'id': pkg.id}
auth_dict = package_show(context, data_dict)
self.assert_(auth_dict['success'])
data_dict = {'id': grp.id}
context = {'user': '', 'model': model}
auth_dict = group_show(context, data_dict)
self.assert_(auth_dict['success'])
def harvest_source_show(context,data_dict):
model = context['model']
user = context.get('user','')
source = get_source_object(context,data_dict)
# Non-logged users can not read the source
if not user:
return {'success': False, 'msg': _('Non-logged in users are not authorized to see harvest sources')}
# Sysadmins can read the source
if Authorizer().is_sysadmin(user):
return {'success': True}
# Check if the source publisher id exists on the user's groups
user_obj = User.get(user)
if not user_obj or not source.publisher_id in [g.id for g in user_obj.get_groups(u'publisher')]:
return {'success': False, 'msg': _('User %s not authorized to read harvest source %s') % (str(user),source.id)}
else:
return {'success': True}
def harvest_object_show(context,data_dict):
model = context['model']
user = context.get('user')
obj = get_obj_object(context,data_dict)
if context.get('ignore_auth', False):
return {'success': True}
if not user:
return {'success': False, 'msg': _('Non-logged in users are not authorized to see harvest objects')}
if authz.is_sysadmin(user):
return {'success': True}
user_obj = User.get(user)
if not user_obj or not obj.source.publisher_id in [g.id for g in user_obj.get_groups(u'publisher')]:
return {'success': False, 'msg': _('User %s not authorized to read harvest object %s') % (str(user),obj.id)}
else:
return {'success': True}
def harvest_source_delete(context,data_dict):
model = context['model']
user = context.get('user','')
source = get_source_object(context,data_dict)
# Non-logged users cannot delete this source
if not user:
return {'success': False, 'msg': _('Non-logged in users are not authorized to delete harvest sources')}
# Sysadmins can delete the source
if ckan.new_authz.is_sysadmin(user):
return {'success': True}
# Check if the source publisher id exists on the user's groups
user_obj = User.get(user)
if not user_obj or not source.publisher_id in [g.id for g in user_obj.get_groups(u'organization')]:
return {'success': False, 'msg': _('User %s not authorized to delete harvest source %s') % (str(user),source.id)}
else:
return {'success': True}
def check_private(key, data, errors, context):
'''
Changes to owner_org_validator requires checking of private value.
:param key: key
:param data: data
:param errors: errors
:param context: context
:return: nothing. Raise invalid if not organisation editor and private == False
'''
value = data.get(key)
is_editor = False
if not value or value == u'False':
user = context.get('user', False)
if user:
if utils.get_member_role(data.get((u'owner_org',)), User.get(user).id) in ('admin', 'editor'):
is_editor = True
if not is_editor:
raise Invalid(_('Only organization\'s editors and admins can create a public dataset'))
def harvest_job_create(context,data_dict):
model = context['model']
user = context.get('user')
source_id = data_dict['source_id']
if not user:
return {'success': False, 'msg': _('Non-logged in users are not authorized to create harvest jobs')}
if Authorizer().is_sysadmin(user):
return {'success': True}
user_obj = User.get(user)
source = HarvestSource.get(source_id)
if not source:
raise NotFound
if not user_obj or not source.publisher_id in [g.id for g in user_obj.get_groups(u'publisher')]:
return {'success': False, 'msg': _('User %s not authorized to create a job for source %s') % (str(user),source.id)}
else:
return {'success': True}
def _get_sources_for_user(context, data_dict):
model = context['model']
session = context['session']
user = context.get('user', '')
only_active = data_dict.get('only_active', False)
query = session.query(HarvestSource) \
.order_by(HarvestSource.created.desc())
if only_active:
query = query.filter(HarvestSource.active==True) \
# Sysadmins will get all sources
if not Authorizer().is_sysadmin(user):
# This only applies to a non sysadmin user when using the
# publisher auth profile. When using the default profile,
# normal users will never arrive at this point, but even if they
# do, they will get an empty list.
user_obj = User.get(user)
publisher_filters = []
publishers_for_the_user = user_obj.get_groups(u'publisher')
for publisher_id in [g.id for g in publishers_for_the_user]:
publisher_filters.append(
HarvestSource.publisher_id == publisher_id)
if len(publisher_filters):
query = query.filter(or_(*publisher_filters))
else:
# This user does not belong to a publisher yet, no sources for him/her
return []
log.debug('User %s with publishers %r has Harvest Sources: %r', user,
publishers_for_the_user, [(hs.id, hs.url) for hs in query])
sources = query.all()
return sources
def harvest_source_list(context,data_dict):
model = context['model']
user = context.get('user')
# Here we will just check that the user is logged in.
# The logic action will return an empty list if the user does not
# have permissons on any source.
if not user:
return {'success': False, 'msg': _('Only logged users are authorized to see their sources')}
else:
user_obj = User.get(user)
assert user_obj
# Only users belonging to a publisher can list sources,
# unless they are sysadmins
if Authorizer().is_sysadmin(user_obj):
return {'success': True}
if len(user_obj.get_groups(u'publisher')) > 0:
return {'success': True}
else:
return {'success': False, 'msg': _('User %s must belong to a publisher to list harvest sources') % str(user)}
def setUp(self):
licenses = get_voc_file(LICENSES_FILE)
load_licenses(load_graph(licenses))
Session.flush()
user = User.get('dummy')
if not user:
user = call_action('user_create',
name='dummy',
password='******',
email='*****@*****.**')
user_name = user['name']
else:
user_name = user.name
org = Group.by_name('dummy')
if org:
self.org = org.__dict__
else:
self.org = call_action('organization_create',
context={'user': user_name},
name='dummy',
identifier='aaaaaa')
def get_first_admin(id):
'''
Get the url of the first one with an admin role
:param id: the package id
:returns: profile url
:rtype: string
'''
pkg = Package.get(id)
if pkg:
data = pkg.as_dict()
user = None
if pkg.roles:
owner = [role for role in pkg.roles if role.role == 'admin']
if len(owner):
user = User.get(owner[0].user_id)
profileurl = ""
if user:
profileurl = config.get('ckan.site_url', '') + \
h.url_for(controller="user", action="read",
id=user.name)
return profileurl
return False
def send_notifications_on_change_app_status(app, status, lang):
""" Send mail when app changes status """
from ckan.model import User
template_dir = os.path.join(os.path.dirname(__file__), 'templates')
locale_dir = os.path.join(os.path.dirname(__file__), 'i18n')
env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir),
extensions=['jinja2.ext.i18n'])
translations = Translations.load(locale_dir, [lang], domain='ckanext-apps')
env.install_gettext_translations(translations)
env.globals['get_locale'] = lambda: lang
app_author = User.get(app.author_id)
data = {
'author_name': app_author.name,
'site_title': tk.config.get('ckan.site_title'),
'site_url': tk.config.get('ckan.site_url')
}
template_name = 'email/app_{0}_app_mail.html'.format(status)
template = env.get_template(template_name)
if status == 'close':
data['closed_message'] = app.closed_message
body = template.render(data)
if status == 'pending':
subject = tk._('You added an application')
elif status == 'active':
subject = tk._('Your application has been approved')
elif status == 'close':
subject = tk._('Your application has been rejected')
else:
subject = status
tk.get_action('send_mail')({}, {
'to': app_author.email,
'subject': env.globals['gettext'](subject),
'message_html': body
})
def harvest_job_create(context, data_dict):
model = context["model"]
user = context.get("user")
source_id = data_dict["source_id"]
if not user:
return {"success": False, "msg": _("Non-logged in users are not authorized to create harvest jobs")}
if authz.is_sysadmin(user):
return {"success": True}
user_obj = User.get(user)
source = HarvestSource.get(source_id)
if not source:
raise NotFound
if not user_obj or not source.publisher_id in [g.id for g in user_obj.get_groups(u"publisher")]:
return {
"success": False,
"msg": _("User %s not authorized to create a job for source %s") % (str(user), source.id),
}
else:
return {"success": True}
def user_organizations(user):
u = User.get(user['name'])
return u.get_groups(group_type = "organization")
def test_mapping(self):
# multilang requires lang to be set
from pylons.i18n.translation import set_lang, get_lang
import pylons
class dummyreq(object):
class p(object):
translator = object()
environ = {'pylons.pylons': p()}
pylons.request = dummyreq()
pylons.translator.pylons_lang = ['en_GB']
set_lang('en_GB')
assert get_lang() == ['en_GB']
assert 'dcatapit_theme_group_mapper' in config['ckan.plugins'], "No dcatapit_theme_group_mapper plugin in config"
contents = self._get_file_contents('dataset.rdf')
p = RDFParser(profiles=['it_dcat_ap'])
p.parse(contents)
datasets = [d for d in p.datasets()]
eq_(len(datasets), 1)
package_dict = datasets[0]
user = User.get('dummy')
if not user:
user = call_action('user_create',
name='dummy',
password='******',
email='*****@*****.**')
user_name = user['name']
else:
user_name = user.name
org = Group.by_name('dummy')
if org is None:
org = call_action('organization_create',
context={'user': user_name},
name='dummy',
identifier='aaaaaa')
existing_g = Group.by_name('existing-group')
if existing_g is None:
existing_g = call_action('group_create',
context={'user': user_name},
name='existing-group')
context = {'user': '******',
'ignore_auth': True,
'defer_commit': False}
package_schema = schema.default_create_package_schema()
context['schema'] = package_schema
_p = {'frequency': 'manual',
'publisher_name': 'dummy',
'extras': [{'key':'theme', 'value':['non-mappable', 'thememap1']}],
'groups': [],
'title': 'dummy',
'holder_name': 'dummy',
'holder_identifier': 'dummy',
'name': 'dummy',
'notes': 'dummy',
'owner_org': 'dummy',
'modified': datetime.now(),
'publisher_identifier': 'dummy',
'metadata_created' : datetime.now(),
'metadata_modified': datetime.now(),
'guid': unicode(uuid.uuid4),
'identifier': 'dummy'}
package_dict.update(_p)
config[DCATAPIT_THEME_TO_MAPPING_SOURCE] = ''
package_data = call_action('package_create', context=context, **package_dict)
p = Package.get(package_data['id'])
# no groups should be assigned at this point (no map applied)
assert {'theme': ['non-mappable', 'thememap1']} == p.extras, '{} vs {}'.format(_p['extras'], p.extras)
assert [] == p.get_groups(group_type='group'), 'should be {}, got {}'.format([], p.get_groups(group_type='group'))
package_data = call_action('package_show', context=context, id=package_data['id'])
# use test mapping, which replaces thememap1 to thememap2 and thememap3
test_map_file = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'examples', 'test_map.ini')
config[DCATAPIT_THEME_TO_MAPPING_SOURCE] = test_map_file
package_dict['theme'] = ['non-mappable', 'thememap1']
expected_groups_existing = ['existing-group']
expected_groups_new = expected_groups_existing + ['somegroup1', 'somegroup2']
expected_groups_multi = expected_groups_new + ['othergroup']
package_dict.pop('extras', None)
p = Package.get(package_data['id'])
context['package'] = p
package_data = call_action('package_update',
context=context,
**package_dict)
#meta.Session.flush()
#meta.Session.revision = repo.new_revision()
# check - only existing group should be assigned
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
assert expected_groups_existing == groups, (expected_groups_existing, 'vs', groups,)
config[DCATAPIT_THEME_TO_MAPPING_ADD_NEW_GROUPS] = 'true'
package_dict['theme'] = ['non-mappable', 'thememap1']
package_data = call_action('package_update', context=context, **package_dict)
meta.Session.flush()
meta.Session.revision = repo.new_revision()
# recheck - this time, new groups should appear
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
assert len(expected_groups_new) == len(groups), (expected_groups_new, 'vs', groups,)
assert set(expected_groups_new) == set(groups), (expected_groups_new, 'vs', groups,)
package_dict['theme'] = ['non-mappable', 'thememap1', 'thememap-multi']
package_data = call_action('package_update', context=context, **package_dict)
meta.Session.flush()
meta.Session.revision = repo.new_revision()
# recheck - there should be no duplicates
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
assert len(expected_groups_multi) == len(groups), (expected_groups_multi, 'vs', groups,)
assert set(expected_groups_multi) == set(groups), (expected_groups_multi, 'vs', groups,)
package_data = call_action('package_update', context=context, **package_dict)
meta.Session.flush()
meta.Session.revision = repo.new_revision()
# recheck - there still should be no duplicates
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
assert len(expected_groups_multi) == len(groups), (expected_groups_multi, 'vs', groups,)
assert set(expected_groups_multi) == set(groups), (expected_groups_multi, 'vs', groups,)
meta.Session.rollback()
def _user(self, username):
tk.c.username = username
user = User.get(username)
tk.c.user_info = h.get_user(user.id)
tk.c.sub_title = _('User Activities: %s'%(user.fullname or user.name) )
tk.c.user_activity_list = h.list_activities_for_user(user_id=user.id)
def test_theme_to_group_mapping(self):
# multilang requires lang to be set
# class dummyreq(object):
# class p(object):
# translator = object()
# environ = {'pylons.pylons': p()}
# CKANRequest(dummyreq)
# pylons.request = dummyreq()
# pylons.translator.pylons_lang = ['en_GB']
#set_lang('en_GB')
#assert get_lang() == ['en_GB']
assert 'dcatapit_theme_group_mapper' in config[
'ckan.plugins'], 'No dcatapit_theme_group_mapper plugin in config'
with open(get_example_file('dataset.rdf'), 'r') as f:
contents = f.read()
p = RDFParser(profiles=['it_dcat_ap'])
p.parse(contents)
datasets = [d for d in p.datasets()]
self.assertEqual(len(datasets), 1)
package_dict = datasets[0]
user = User.get('dummy')
if not user:
user = call_action('user_create',
name='dummy',
password='******',
email='*****@*****.**')
user_name = user['name']
else:
user_name = user.name
org = Group.by_name('dummy')
if org is None:
org = call_action('organization_create',
context={'user': user_name},
name='dummy',
identifier='aaaaaa')
existing_g = Group.by_name('existing-group')
if existing_g is None:
existing_g = call_action('group_create',
context={'user': user_name},
name='existing-group')
context = {'user': '******', 'ignore_auth': True, 'defer_commit': False}
package_schema = schema.default_create_package_schema()
context['schema'] = package_schema
_p = {
'frequency': 'manual',
'publisher_name': 'dummy',
'extras': [{
'key': 'theme',
'value': ['non-mappable', 'thememap1']
}],
'groups': [], # [{'name':existing_g.name}],
'title': 'dummy',
'holder_name': 'dummy',
'holder_identifier': 'dummy',
'name': 'dummy-' + uuid4().hex,
'identifier': 'dummy' + uuid4().hex,
'notes': 'dummy',
'owner_org': 'dummy',
'modified': datetime.now(),
'publisher_identifier': 'dummy',
'metadata_created': datetime.now(),
'metadata_modified': datetime.now(),
'guid': str(uuid.uuid4),
}
package_dict.update(_p)
config[DCATAPIT_THEME_TO_MAPPING_SOURCE] = ''
config[DCATAPIT_THEME_TO_MAPPING_ADD_NEW_GROUPS] = 'false'
package_data = call_action('package_create',
context=context,
**package_dict)
p = Package.get(package_data['id'])
# no groups should be assigned at this point (no map applied)
assert {
'theme': ['non-mappable', 'thememap1']
} == p.extras, '{} vs {}'.format(_p['extras'], p.extras)
assert [] == p.get_groups(
group_type='group'), 'should be {}, got {}'.format(
[], p.get_groups(group_type='group'))
package_data = call_action('package_show',
context=context,
id=package_data['id'])
# use test mapping, which replaces thememap1 to thememap2 and thememap3
test_map_file = os.path.join(os.path.dirname(__file__), '..', '..',
'..', 'examples', 'test_map.ini')
config[DCATAPIT_THEME_TO_MAPPING_SOURCE] = test_map_file
config[DCATAPIT_THEME_TO_MAPPING_ADD_NEW_GROUPS] = 'false'
# package_dict['theme'] = ['non-mappable', 'thememap1']
package_dict.pop('extras', None)
p = Package.get(package_data['id'])
context['package'] = p
package_data = call_action('package_update',
context=context,
**package_dict)
# check - only existing group should be assigned
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
# the map file maps ECON to existing group, and 2 other unexisting groups that will not be created
expected_groups = ['existing-group']
self.assertSetEqual(set(expected_groups), set(groups),
'Error in assigned groups')
config[DCATAPIT_THEME_TO_MAPPING_SOURCE] = test_map_file
config[DCATAPIT_THEME_TO_MAPPING_ADD_NEW_GROUPS] = 'true'
# package_dict['theme'] = ['non-mappable', 'thememap1']
package_data = call_action('package_update',
context=context,
**package_dict)
meta.Session.flush()
# recheck - this time, new groups should appear
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
# the map file maps ECON to existing group and 2 other groups that have been automatically created
expected_groups = expected_groups + ['somegroup1', 'somegroup2']
self.assertSetEqual(set(expected_groups), set(groups), 'Groups differ')
# package_dict['theme'] = ['non-mappable', 'thememap1', 'thememap-multi']
aggr = json.loads(package_dict[FIELD_THEMES_AGGREGATE])
aggr.append({'theme': 'thememap-multi', 'subthemes': []})
package_dict[FIELD_THEMES_AGGREGATE] = json.dumps(aggr)
package_data = call_action('package_update',
context=context,
**package_dict)
meta.Session.flush()
# recheck - there should be no duplicates
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
# added theme 'thememap-multi', that maps to 'othergroup' and other already exisintg groups
expected_groups = expected_groups + ['othergroup']
self.assertEqual(len(expected_groups), len(groups),
'New groups differ - there may be duplicated groups')
self.assertSetEqual(set(expected_groups), set(groups),
'New groups differ')
package_data = call_action('package_update',
context=context,
**package_dict)
meta.Session.flush()
# recheck - there still should be no duplicates
p = Package.get(package_data['id'])
groups = [g.name for g in p.get_groups(group_type='group')]
self.assertEqual(len(expected_groups), len(groups),
'New groups differ - there may be duplicated groups')
self.assertSetEqual(set(expected_groups), set(groups),
'New groups differ')
meta.Session.rollback()
def test_clean_tags(self):
# Create source
source_fixture = {
'title': 'Test Source',
'name': 'test-source',
'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml',
'source_type': u'gemini-single',
'owner_org': 'test-org',
'metadata_created': datetime.now().strftime('%YYYY-%MM-%DD %HH:%MM:%s'),
'metadata_modified': datetime.now().strftime('%YYYY-%MM-%DD %HH:%MM:%s'),
}
user = User.get('dummy')
if not user:
user = call_action('user_create',
name='dummy',
password='******',
email='*****@*****.**')
user_name = user['name']
else:
user_name = user.name
org = Group.by_name('test-org')
if org is None:
org = call_action('organization_create',
context={'user': user_name},
name='test-org')
existing_g = Group.by_name('existing-group')
if existing_g is None:
existing_g = call_action('group_create',
context={'user': user_name},
name='existing-group')
context = {'user': '******'}
package_schema = default_update_package_schema()
context['schema'] = package_schema
package_dict = {'frequency': 'manual',
'publisher_name': 'dummy',
'extras': [{'key':'theme', 'value':['non-mappable', 'thememap1']}],
'groups': [],
'title': 'fakename',
'holder_name': 'dummy',
'holder_identifier': 'dummy',
'name': 'fakename',
'notes': 'dummy',
'owner_org': 'test-org',
'modified': datetime.now(),
'publisher_identifier': 'dummy',
'metadata_created' : datetime.now(),
'metadata_modified' : datetime.now(),
'guid': unicode(uuid4()),
'identifier': 'dummy'}
package_data = call_action('package_create', context=context, **package_dict)
package = Package.get('fakename')
source, job = self._create_source_and_job(source_fixture)
job.package = package
job.guid = uuid4()
harvester = SpatialHarvester()
with open(os.path.join('..', 'data', 'dataset.json')) as f:
dataset = json.load(f)
# long tags are invalid in all cases
TAG_LONG_INVALID = 'abcdefghij' * 20
# if clean_tags is not set to true, tags will be truncated to 50 chars
TAG_LONG_VALID = TAG_LONG_INVALID[:50]
# default truncate to 100
TAG_LONG_VALID_LONG = TAG_LONG_INVALID[:100]
assert len(TAG_LONG_VALID) == 50
assert TAG_LONG_VALID[-1] == 'j'
TAG_CHARS_INVALID = '[email protected]!'
TAG_CHARS_VALID = 'pretty-invlidtag'
dataset['tags'].append(TAG_LONG_INVALID)
dataset['tags'].append(TAG_CHARS_INVALID)
harvester.source_config = {'clean_tags': False}
out = harvester.get_package_dict(dataset, job)
tags = out['tags']
# no clean tags, so invalid chars are in
# but tags are truncated to 50 chars
assert {'name': TAG_CHARS_VALID} not in tags
assert {'name': TAG_CHARS_INVALID} in tags
assert {'name': TAG_LONG_VALID_LONG} in tags
assert {'name': TAG_LONG_INVALID} not in tags
harvester.source_config = {'clean_tags': True}
out = harvester.get_package_dict(dataset, job)
tags = out['tags']
assert {'name': TAG_CHARS_VALID} in tags
assert {'name': TAG_LONG_VALID_LONG} in tags
def tordf(self, id, format):
graph = Graph()
pkg = Package.get(id)
if pkg:
data = pkg.as_dict()
metadoc = URIRef('')
user = None
if pkg.roles:
owner = [role for role in pkg.roles if role.role == 'admin']
if len(owner):
user = User.get(owner[0].user_id)
profileurl = ""
if user:
profileurl = URIRef(config.get('ckan.site_url', '') +\
h.url_for(controller="user", action="read", id=user.name))
graph.add((metadoc, DC.identifier, Literal(data["id"])\
if 'identifier' not in data["extras"]\
else URIRef(data["extras"]["identifier"])))
graph.add((metadoc, DC.modified, Literal(data["metadata_modified"],
datatype=XSD.dateTime)))
graph.add((metadoc, FOAF.primaryTopic, Identifier(data['name'])))
uri = URIRef(data['name'])
if data["license"]:
graph.add((uri, DC.rights, Literal(data["license"])))
if "versionPID" in data["extras"]:
graph.add((uri, DC.identifier, Literal(data["extras"]["versionPID"])))
graph.add((uri, DC.identifier, Literal(data["name"])))
graph.add((uri, DC.modified, Literal(data.get("version", ''),
datatype=XSD.dateTime)))
org = URIRef(FOAF.Person)
if profileurl:
graph.add((uri, DC.publisher, profileurl))
graph.add((profileurl, RDF.type, org))
graph.add((profileurl, FOAF.name, Literal(data["extras"]["publisher"])))
graph.add((profileurl, FOAF.phone, Identifier(data["extras"]["phone"])))
graph.add((profileurl, FOAF.homepage, Identifier(data["extras"]["contactURL"])))
graph.add((uri, DC.rightsHolder, URIRef(data["extras"]["owner"])
if data["extras"]["owner"].startswith(('http','urn'))\
else Literal(data["extras"]["owner"])))
log.debug(data["extras"])
if all((k in data["extras"] and data["extras"][k] != "") for k in ("project_name",\
"project_homepage",\
"project_funding",\
"funder")):
project = URIRef(FOAF.Project)
projecturl = URIRef(data["extras"]["project_homepage"])
graph.add((uri, DC.contributor, projecturl))
graph.add((projecturl, RDF.type, project))
graph.add((projecturl, FOAF.name, Literal(data["extras"]["project_name"])))
graph.add((projecturl, FOAF.homepage, Identifier(data["extras"]["project_homepage"])))
graph.add((projecturl, RDFS.comment,
Literal(data["extras"]["project_funder"])))
for key in data["extras"]:
log.debug(key)
if key.startswith('author'):
graph.add((uri, DC.creator, URIRef(data["extras"][key])\
if data["extras"][key].startswith(('http','urn'))\
else Literal(data["extras"][key])))
if key.startswith("title"):
lastlangnum = key.split('_')[-1]
log.debug(lastlangnum)
lastlang = data["extras"]["lang_title_%s" % lastlangnum]
graph.add((uri, DC.title, Literal(data["extras"][key],
lang=lastlang)))
for tag in data['tags']:
graph.add((uri, DC.subject, Literal(tag)))
for lang in data["extras"].get("language", "").split(','):
graph.add((uri, DC.language, Literal(lang.strip())))
if "access" in data["extras"]:
graph.add((uri, DC.rights, self._make_rights_element(data["extras"])))
# Extended metadatamodel
if all(k in data["extras"] for k in ("temporal_coverage_begin",
"temporal_coverage_end")):
graph.add((uri, DC.temporal, Literal(self._make_temporal(data["extras"]))))
if "geographical_coverage" in data["extras"]:
graph.add((uri, DC.spatial, Literal(data["extras"]["geographical_coverage"])))
for rel in Related.get_for_dataset(pkg):
graph.add((uri, DC.isReferencedBy, Literal(rel.related.title)))
if "notes_rendered" in data:
graph.add((uri, DC.description, Literal(data["notes_rendered"])))
response.headers['Content-type'] = 'text/xml'
if format == 'rdf':
format = 'pretty-xml'
return graph.serialize(format=format)
else:
return ""
def test_clean_tags(self):
# Create source
source_fixture = {
'title': 'Test Source',
'name': 'test-source',
'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml',
'source_type': u'gemini-single',
'owner_org': 'test-org',
'metadata_created': datetime.now().strftime('%YYYY-%MM-%DD %HH:%MM:%s'),
'metadata_modified': datetime.now().strftime('%YYYY-%MM-%DD %HH:%MM:%s'),
}
user = User.get('dummy')
if not user:
user = call_action('user_create',
name='dummy',
password='******',
email='*****@*****.**')
user_name = user['name']
else:
user_name = user.name
org = Group.by_name('test-org')
if org is None:
org = call_action('organization_create',
context={'user': user_name},
name='test-org')
existing_g = Group.by_name('existing-group')
if existing_g is None:
existing_g = call_action('group_create',
context={'user': user_name},
name='existing-group')
context = {'user': '******'}
package_schema = default_update_package_schema()
context['schema'] = package_schema
package_dict = {'frequency': 'manual',
'publisher_name': 'dummy',
'extras': [{'key':'theme', 'value':['non-mappable', 'thememap1']}],
'groups': [],
'title': 'fakename',
'holder_name': 'dummy',
'holder_identifier': 'dummy',
'name': 'fakename',
'notes': 'dummy',
'owner_org': 'test-org',
'modified': datetime.now(),
'publisher_identifier': 'dummy',
'metadata_created' : datetime.now(),
'metadata_modified' : datetime.now(),
'guid': unicode(uuid4()),
'identifier': 'dummy'}
package_data = call_action('package_create', context=context, **package_dict)
package = Package.get('fakename')
source, job = self._create_source_and_job(source_fixture)
job.package = package
job.guid = uuid4()
harvester = SpatialHarvester()
with open(os.path.join('..', 'data', 'dataset.json')) as f:
dataset = json.load(f)
# long tags are invalid in all cases
TAG_LONG_INVALID = 'abcdefghij' * 20
# if clean_tags is not set to true, tags will be truncated to 50 chars
TAG_LONG_VALID = TAG_LONG_INVALID[:50]
# default truncate to 100
TAG_LONG_VALID_LONG = TAG_LONG_INVALID[:100]
assert len(TAG_LONG_VALID) == 50
assert TAG_LONG_VALID[-1] == 'j'
TAG_CHARS_INVALID = '[email protected]!'
TAG_CHARS_VALID = 'pretty-invlidtag'
dataset['tags'].append(TAG_LONG_INVALID)
dataset['tags'].append(TAG_CHARS_INVALID)
harvester.source_config = {'clean_tags': False}
out = harvester.get_package_dict(dataset, job)
tags = out['tags']
# no clean tags, so invalid chars are in
# but tags are truncated to 50 chars
assert {'name': TAG_CHARS_VALID} not in tags
assert {'name': TAG_CHARS_INVALID} in tags
assert {'name': TAG_LONG_VALID_LONG} in tags
assert {'name': TAG_LONG_INVALID} not in tags
harvester.source_config = {'clean_tags': True}
out = harvester.get_package_dict(dataset, job)
tags = out['tags']
assert {'name': TAG_CHARS_VALID} in tags
assert {'name': TAG_LONG_VALID_LONG} in tags
def user_organizations(user):
u = User.get(user['name'])
return u.get_groups(group_type = "organization")
def _get_user_obj(username):
user = User.get(username)
if not user:
raise ObjectNotFound()
return user