def validate_resource_edit(self,
id,
resource_id,
data=None,
errors=None,
error_summary=None):
'''
Intercept save_edit
Replace author, maintainer, maintainer_email
'''
if 'validation_schema' in request.POST and 'format' in request.POST:
resource_format = request.POST.getone('format')
validation_schema = request.POST.getone('validation_schema')
if resource_format == 'CSV' and validation_schema and validation_schema != '':
schema_url = plugin.generate_download_url(id, validation_schema)
data_url = plugin.generate_download_url(id, resource_id)
validation_url = "http://goodtables.okfnlabs.org/api/run?format=csv&schema={0}&data={1}&row_limit=100000&report_limit=1000&report_type=grouped".format(
schema_url, data_url)
req = requests.get(validation_url, verify=False)
if req.status_code == requests.codes.ok:
response_text = json.loads(req.text)
if response_text['success']:
h.flash_success(
"CSV was validated successfully against the selected schema"
)
else:
h.flash_error(
"CSV was NOT validated against the selected schema")
return RESOURCE_EDIT(self, id, resource_id, data, errors, error_summary)
def action_add_form(users_or_authz_groups):
# The user is attempting to set new roles for a named user
new_user = request.params.get('new_user_name')
# this is the list of roles whose boxes were ticked
checked_roles = [ a for (a,b) in request.params.items() if (b == u'on')]
# this is the list of all the roles that were in the submitted form
submitted_roles = [ a for (a,b) in request.params.items() if (b == u'submitted')]
# from this we can make a dictionary of the desired states
# i.e. true for the ticked boxes, false for the unticked
desired_roles = {}
for r in submitted_roles:
desired_roles[r]=False
for r in checked_roles:
desired_roles[r]=True
# again, in order to avoid either creating a role twice or deleting one which is
# non-existent, we need to get the users' current roles (if any)
current_uors = get_userobjectroles()
if users_or_authz_groups=='users':
current_roles = [uor.role for uor in current_uors if ( uor.user and uor.user.name == new_user )]
user_object = model.User.by_name(new_user)
if user_object==None:
# The submitted user does not exist. Bail with flash message
h.flash_error('unknown user:'******'authz_groups':
current_roles = [uor.role for uor in current_uors if ( uor.authorized_group and uor.authorized_group.name == new_user )]
user_object = model.AuthorizationGroup.by_name(new_user)
if user_object==None:
# The submitted user does not exist. Bail with flash message
h.flash_error('unknown authorization group:' + str (new_user))
else:
# Whenever our desired state is different from our current state, change it.
for (r,val) in desired_roles.items():
if val:
if (r not in current_roles):
model.add_authorization_group_to_role(user_object, r, pkg)
else:
if (r in current_roles):
model.remove_authorization_group_from_role(user_object, r, pkg)
h.flash_success("Authorization Group Added")
else:
assert False, "shouldn't be here"
# and finally commit all these changes to the database
model.repo.commit_and_remove()
def action_add_form(users_or_authz_groups):
# The user is attempting to set new roles for a named user
new_user = request.params.get('new_user_name')
# this is the list of roles whose boxes were ticked
checked_roles = [ a for (a,b) in request.params.items() if (b == u'on')]
# this is the list of all the roles that were in the submitted form
submitted_roles = [ a for (a,b) in request.params.items() if (b == u'submitted')]
# from this we can make a dictionary of the desired states
# i.e. true for the ticked boxes, false for the unticked
desired_roles = {}
for r in submitted_roles:
desired_roles[r]=False
for r in checked_roles:
desired_roles[r]=True
# again, in order to avoid either creating a role twice or deleting one which is
# non-existent, we need to get the users' current roles (if any)
current_uors = get_userobjectroles()
if users_or_authz_groups=='users':
current_roles = [uor.role for uor in current_uors if ( uor.user and uor.user.name == new_user )]
user_object = model.User.by_name(new_user)
if user_object==None:
# The submitted user does not exist. Bail with flash message
h.flash_error('unknown user:'******'authz_groups':
current_roles = [uor.role for uor in current_uors if ( uor.authorized_group and uor.authorized_group.name == new_user )]
user_object = model.AuthorizationGroup.by_name(new_user)
if user_object==None:
# The submitted user does not exist. Bail with flash message
h.flash_error('unknown authorization group:' + str (new_user))
else:
# Whenever our desired state is different from our current state, change it.
for (r,val) in desired_roles.items():
if val:
if (r not in current_roles):
model.add_authorization_group_to_role(user_object, r, group)
else:
if (r in current_roles):
model.remove_authorization_group_from_role(user_object, r, group)
h.flash_success("Authorization Group Added")
else:
assert False, "shouldn't be here"
# and finally commit all these changes to the database
model.repo.commit_and_remove()
def _add_users( self, group, parameters ):
if not group:
h.flash_error(_("There was a problem with your submission, "
"please correct it and try again"))
errors = {"reason": ["No reason was supplied"]}
return self.apply(group.id, errors=errors,
error_summary=action.error_summary(errors))
data_dict = logic.clean_dict(dict_func.unflatten(
logic.tuplize_dict(logic.parse_params(request.params))))
data_dict['id'] = group.id
# Temporary fix for strange caching during dev
l = data_dict['users']
for d in l:
d['capacity'] = d.get('capacity','editor')
context = {
"group" : group,
"schema": schema.default_group_schema(),
"model": model,
"session": model.Session
}
# Temporary cleanup of a capacity being sent without a name
users = [d for d in data_dict['users'] if len(d) == 2]
data_dict['users'] = users
model.repo.new_revision()
model_save.group_member_save(context, data_dict, 'users')
model.Session.commit()
h.redirect_to( controller='group', action='edit', id=group.name)
def render_request_form(self, pkg_id):
"""
Render the access request contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
if not c.package:
abort(404, _(u"Dataset not found"))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts]
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/dataset_request_form.html')
def render_request_form(self, pkg_id):
"""
Render the access request contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if not c.package:
abort(404, _(u"Dataset not found"))
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts]
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/dataset_request_form.html')
def logged_in(self):
# we need to set the language via a redirect
lang = session.pop('lang', None)
session.save()
# we need to set the language explicitly here or the flash
# messages will not be translated.
i18n.set_lang(lang)
if c.user:
is_new = False
is_sysadmin = new_authz.is_sysadmin(c.user)
# Retrieve information about the current user
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': schema.user_new_form_schema()}
data_dict = {'id': c.user}
user_dict = get_action('user_show')(context, data_dict)
# This check is not needed (or correct) for sys admins
if not is_sysadmin:
# Get all organizations and all groups the user belongs to
orgs_q = model.Session.query(model.Group) \
.filter(model.Group.is_organization == True) \
.filter(model.Group.state == 'active')
q = model.Session.query(model.Member) \
.filter(model.Member.table_name == 'user') \
.filter(model.Member.table_id == user_dict['id'])
group_ids = []
for row in q.all():
group_ids.append(row.group_id)
if not group_ids:
is_new = True
else:
orgs_q = orgs_q.filter(model.Group.id.in_(group_ids))
orgs_list = model_dictize.group_list_dictize(orgs_q.all(), context)
if len(orgs_list) == 0:
is_new = True
h.flash_success(_("<p><strong>Note</strong></p>"
"<p>%s is now logged in</p>") %
user_dict['display_name'], allow_html=True)
if is_new:
return h.redirect_to(controller='ckanext.canada.controller:CanadaController',
action='view_new_user', locale=lang)
else:
return h.redirect_to(controller='package',
action='search', locale=lang)
else:
h.flash_error(_('Login failed. Bad username or password.'))
return h.redirect_to(controller='user',
action='login', locale=lang)
def abort_harvest_job(self, harvest_job_id):
try:
toolkit.get_action('harvest_job_abort')(None, {
"id": harvest_job_id
})
h.flash_success(_('Harvest job ID %s - aborted' % harvest_job_id))
except Exception, e:
h.flash_error(
_('ERROR aborting harvest job ID %s' % harvest_job_id))
def create_request(self, pkg_id):
pkg = Package.get(pkg_id)
user = c.userobj if c.userobj else None
if user:
req = KataAccessRequest(user.id, pkg.id)
req.save()
url = h.url_for(controller='package', action='read', id=pkg_id)
h.flash_success(_("You now requested editor rights to package %s" % pkg.name))
redirect(url)
else:
url = h.url_for(controller='package', action='read', id=pkg_id)
h.flash_error(_("Please log in!"))
redirect(url)
def authorize(method, bucket, key, user, ofs):
if not method in ['POST', 'GET', 'PUT', 'DELETE']:
abort(400)
if method != 'GET':
# do not allow overwriting
if ofs.exists(bucket, key):
abort(409)
# now check user stuff
username = user.name if user else ''
is_authorized = authz.Authorizer.is_authorized(username, UPLOAD_ACTION, model.System())
if not is_authorized:
h.flash_error('Not authorized to upload files.')
abort(401)
def apply(self, id=None, data=None, errors=None, error_summary=None):
"""
Form for a user to request to be an editor for a organisation.
It sends an email to a suitable admin.
"""
if not c.user:
abort(401, _('You must be logged in to apply for membership'))
if 'parent' in request.params and not id:
id = request.params['parent']
if id:
c.group = model.Group.get(id)
if not c.group:
log.warning('Could not find organisation for name %s', id)
abort(404, _('Publisher not found'))
if 'save' in request.params and not errors:
from ckanext.dgu_orgs.model.organisation_request import OrganisationRequest
reason = request.params.get('reason', None)
if model.Session.query(OrganisationRequest).filter_by(
user_name=c.user, group_name=id).all():
h.flash_error(
'A request for this organisation is already in the system. If you have waited more than a couple of days then <a href="http://data.gov.uk/contact">contact the data.gov.uk team</a>',
allow_html=True)
h.redirect_to('organisation_apply', id=id)
return
else:
req = OrganisationRequest(user_name=c.user,
group_name=id,
reason=reason)
model.Session.add(req)
model.Session.commit()
return self._send_application(c.group, reason)
else:
c.possible_parents = model.Session.query(model.Group)\
.filter(model.Group.state=='active').order_by(model.Group.title).all()
data = data or {}
errors = errors or {}
error_summary = error_summary or {}
data.update(request.params)
vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
c.form = render('organisation/apply_form.html', extra_vars=vars)
return render('organisation/apply.html')
def _validate_uploadfile(self, data_file, resource_path, resource_list):
"""
This function is responsible for validating a bulk upload.
"""
err_msg = ""
try:
validator = ngdsvalidator.NGDSValidator(filepath=data_file, resource_path=resource_path,
resource_list=resource_list)
validator.validate()
status = "VALID"
h.flash_notice(toolkit._('Files Uploaded Successfully.'), allow_html=True)
except Exception, e:
err_msg = e.__str__()
h.flash_error(toolkit._('Uploaded files are invalid.: %s ') % err_msg, allow_html=True)
status = "INVALID"
def _upload_xml(self, errors=None, error_summary=None):
'''
Allow filling dataset form by parsing a user uploaded metadata file.
'''
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
try:
t.check_access('package_create', context)
except t.NotAuthorized:
t.abort(401, _('Unauthorized to upload metadata'))
xmlfile = u''
field_storage = request.params.get('xmlfile')
if isinstance(field_storage, FieldStorage):
bffr = field_storage.file
xmlfile = bffr.read()
url = request.params.get('url', u'')
xmltype = request.params.get('xml-format', u'')
log.info('Importing from {src}'.format(
src='file: ' + field_storage.filename if field_storage else 'url: ' + url))
for harvester in plugins.PluginImplementations(h_interfaces.IHarvester):
info = harvester.info()
if not info or 'name' not in info:
log.error('Harvester %r does not provide the harvester name in the info response' % str(harvester))
continue
if xmltype == info['name']:
log.debug('_upload_xml: Found harvester for import: {nam}'.format(nam=info['name']))
try:
if xmlfile:
pkg_dict = harvester.parse_xml(xmlfile, context)
elif url:
pkg_dict = harvester.fetch_xml(url, context)
else:
h.flash_error(_('Give upload URL or file.'))
return h.redirect_to(controller='package', action='new')
return super(KataPackageController, self).new(pkg_dict, errors, error_summary)
except (urllib2.URLError, urllib2.HTTPError):
log.debug('Could not fetch from url {ur}'.format(ur=url))
h.flash_error(_('Could not fetch from url {ur}'.format(ur=url)))
except ValueError, e:
log.debug(e)
h.flash_error(_('Invalid upload URL'))
except etree.XMLSyntaxError:
h.flash_error(_('Invalid XML content'))
except Exception, e:
log.debug(e)
log.debug(type(e))
h.flash_error(_("Failed to load file"))
def _upload_xml(self, errors=None, error_summary=None):
'''
Allow filling dataset form by parsing a user uploaded metadata file.
'''
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
try:
t.check_access('package_create', context)
except t.NotAuthorized:
t.abort(401, _('Unauthorized to upload metadata'))
xmlfile = u''
field_storage = request.params.get('xmlfile')
if isinstance(field_storage, FieldStorage):
bffr = field_storage.file
xmlfile = bffr.read()
url = request.params.get('url', u'')
xmltype = request.params.get('xml-format', u'')
log.info('Importing from {src}'.format(
src='file: ' + field_storage.filename if field_storage else 'url: ' + url))
for harvester in plugins.PluginImplementations(h_interfaces.IHarvester):
info = harvester.info()
if not info or 'name' not in info:
log.error('Harvester %r does not provide the harvester name in the info response' % str(harvester))
continue
if xmltype == info['name']:
log.debug('_upload_xml: Found harvester for import: {nam}'.format(nam=info['name']))
try:
if xmlfile:
pkg_dict = harvester.parse_xml(xmlfile, context)
elif url:
pkg_dict = harvester.fetch_xml(url, context)
else:
h.flash_error(_('Give upload URL or file.'))
return h.redirect_to(controller='package', action='new')
return super(KataPackageController, self).new(pkg_dict, errors, error_summary)
except (urllib2.URLError, urllib2.HTTPError):
log.debug('Could not fetch from url {ur}'.format(ur=url))
h.flash_error(_('Could not fetch from url {ur}'.format(ur=url)))
except ValueError, e:
log.debug(e)
h.flash_error(_('Invalid upload URL'))
except etree.XMLSyntaxError:
h.flash_error(_('Invalid XML content'))
except Exception, e:
log.debug(e)
log.debug(type(e))
h.flash_error(_("Failed to load file"))
def unlock_access(self, id):
q = model.Session.query(KataAccessRequest)
q = q.filter_by(id=id)
req = q.first()
if req:
user = User.get(req.user_id)
pkg = Package.get(req.pkg_id)
add_user_to_role(user, 'editor', pkg)
url = h.url_for(controller='package', action='read', id=req.pkg_id)
h.flash_success(_("%s now has editor rights to package %s" % (user.name, pkg.name)))
req.delete()
meta.Session.commit()
redirect(url)
else:
h.flash_error(_("No such request found!"))
redirect('/')
def follow(self, id):
'''Start following this group.'''
context = {'model': model,
'session': model.Session,
'user': c.user or c.author}
data_dict = {'id': id}
try:
get_action('follow_group')(context, data_dict)
h.flash_success(_("You are now following {0}").format(id))
except ValidationError as e:
error_message = (e.extra_msg or e.message or e.error_summary
or e.error_dict)
h.flash_error(error_message)
except NotAuthorized as e:
h.flash_error(e.extra_msg)
h.redirect_to(controller='group', action='read', id=id)
def authorize(method, bucket, key, user, ofs):
"""
Check authz for the user with a given bucket/key combo within a
particular ofs implementation.
"""
if not method in ["POST", "GET", "PUT", "DELETE"]:
abort(400)
if method != "GET":
# do not allow overwriting
if ofs.exists(bucket, key):
abort(409)
# now check user stuff
username = user.name if user else ""
is_authorized = authz.Authorizer.is_authorized(username, "file-upload", model.System())
if not is_authorized:
h.flash_error("Not authorized to upload files.")
abort(401)
def authorize(method, bucket, key, user, ofs):
"""
Check authz for the user with a given bucket/key combo within a
particular ofs implementation.
"""
if not method in ['POST', 'GET', 'PUT', 'DELETE']:
abort(400)
if method != 'GET':
# do not allow overwriting
if ofs.exists(bucket, key):
abort(409)
# now check user stuff
username = user.name if user else ''
is_authorized = authz.Authorizer.is_authorized(username, 'file-upload', model.System())
if not is_authorized:
h.flash_error('Not authorized to upload files.')
abort(401)
def authorize(method, bucket, key, user, ofs):
"""
Check authz for the user with a given bucket/key combo within a
particular ofs implementation.
"""
if not method in ['POST', 'GET', 'PUT', 'DELETE']:
abort(400)
if method != 'GET':
# do not allow overwriting
if ofs.exists(bucket, key):
abort(409)
# now check user stuff
username = user.name if user else ''
is_authorized = authz.Authorizer.is_authorized(username, 'file-upload', model.System())
if not is_authorized:
h.flash_error('Not authorized to upload files.')
abort(401)
def authorize(method, bucket, key, user, ofs):
"""
Check authz for the user with a given bucket/key combo within a
particular ofs implementation.
"""
if not method in ['POST', 'GET', 'PUT', 'DELETE']:
abort(400)
if method != 'GET':
# do not allow overwriting
if ofs.exists(bucket, key):
abort(409)
# now check user stuff
context = {'user': c.user, 'model': model}
is_authorized = new_authz.is_authorized_boolean(
'file_upload', context, {})
if not is_authorized:
h.flash_error('Not authorized to upload files.')
abort(401)
def authorize(method, bucket, key, user, ofs):
"""
Check authz for the user with a given bucket/key combo within a
particular ofs implementation.
"""
if not method in ['POST', 'GET', 'PUT', 'DELETE']:
abort(400)
if method != 'GET':
# do not allow overwriting
if ofs.exists(bucket, key):
abort(409)
# now check user stuff
context = {'user': c.user,
'model': model}
is_authorized = new_authz.is_authorized_boolean('file_upload', context, {})
if not is_authorized:
h.flash_error('Not authorized to upload files.')
abort(401)
def logged_in(self):
# we need to set the language via a redirect
# Lang is not being retrieved properly by the Babel i18n lib in
# this redirect, so using this clunky workaround for now.
lang = session.pop('lang', None)
if lang is None:
came_from = request.params.get('came_from', '')
if came_from.startswith('/fr'):
lang = 'fr'
else:
lang = 'en'
session.save()
# we need to set the language explicitly here or the flash
# messages will not be translated.
i18n.set_lang(lang)
if c.user:
context = None
data_dict = {'id': c.user}
user_dict = get_action('user_show')(context, data_dict)
h.flash_success(
_('<strong>Note</strong><br>{0} is now logged in').format(
user_dict['display_name']
),
allow_html=True
)
notice_no_access()
return h.redirect_to(
controller='ckanext.canada.controller:CanadaController',
action='home',
locale=lang)
else:
h.flash_error(_('Login failed. Bad username or password.'))
return h.redirect_to(
controller='user',
action='login', locale=lang
)
def _add_users(self, group, parameters):
from ckan.logic.schema import default_group_schema
from ckan.logic.action import error_summary
from ckan.lib.dictization.model_save import group_member_save
if not group:
h.flash_error(_("There was a problem with your submission, \
please correct it and try again"))
errors = {"reason": ["No reason was supplied"]}
return self.apply(group.id, errors=errors,
error_summary=error_summary(errors))
data_dict = clean_dict(unflatten(
tuplize_dict(parse_params(request.params))))
data_dict['id'] = group.id
# Temporary fix for strange caching during dev
l = data_dict['users']
for d in l:
# Form javascript creates d['capacity'] == 'undefined' for
# newly added users.
# If javascript in users form is not working (such as in tests)
# it will not create a capacity value.
if 'capacity' not in d or d['capacity'] == 'undefined':
# default to 'editor'
d['capacity'] = 'editor'
context = {
"group" : group,
"schema": default_group_schema(),
"model": model,
"session": model.Session
}
# Temporary cleanup of a capacity being sent without a name
users = [d for d in data_dict['users'] if len(d) == 2]
data_dict['users'] = users
model.repo.new_revision()
group_member_save(context, data_dict, 'users')
model.Session.commit()
h.redirect_to('/publisher/%s' % str(group.name))
def unfollow(self, id):
'''Stop following this group.'''
context = {'model': model,
'session': model.Session,
'user': c.user or c.author}
data_dict = {'id': id}
try:
get_action('unfollow_group')(context, data_dict)
group_dict = get_action('group_show')(context, data_dict)
h.flash_success(_("You are no longer following {0}").format(
group_dict['title']))
except ValidationError as e:
error_message = (e.extra_msg or e.message or e.error_summary
or e.error_dict)
h.flash_error(error_message)
except (NotFound, NotAuthorized) as e:
error_message = e.extra_msg or e.message
h.flash_error(error_message)
h.redirect_to(controller='group', action='read', id=id)
def _validate_uploadfile(self, data_file, resource_path, resource_list):
"""
This function is responsible for validating a bulk upload.
"""
err_msg = ""
try:
validator = ngdsvalidator.NGDSValidator(
filepath=data_file,
resource_path=resource_path,
resource_list=resource_list)
validator.validate()
status = "VALID"
h.flash_notice(toolkit._('Files Uploaded Successfully.'),
allow_html=True)
except Exception, e:
err_msg = e.__str__()
h.flash_error(toolkit._('Uploaded files are invalid.: %s ') %
err_msg,
allow_html=True)
status = "INVALID"
def render(self, pkg_id):
c.package = Package.get(pkg_id)
url = h.url_for(controller='package',
action="read",
id=c.package.id)
if c.user:
if pkg_id not in c.userobj.extras.get('contacted', []):
model.repo.new_revision()
if "contacted" in c.userobj.extras:
c.userobj.extras['contacted'].append(pkg_id)
else:
c.userobj.extras['contacted'] = []
c.userobj.extras['contacted'].append(pkg_id)
c.userobj.save()
return render('contact/contact_form.html')
else:
h.flash_error(_("Already contacted"))
return redirect(url)
else:
h.flash_error(_("Please login"))
return redirect(url)
def logged_in(self):
"""Minor rewrite to redirect the user to the own profile page instead of
the dashboard.
"""
# we need to set the language via a redirect
lang = session.pop('lang', None)
session.save()
came_from = request.params.get('came_from', '')
if came_from and not came_from.isspace():
came_from = came_from\
.replace('\n', ' ')\
.replace('\r', '')
# we need to set the language explicitly here or the flash
# messages will not be translated.
ckan.lib.i18n.set_lang(lang)
if h.url_is_local(came_from):
return h.redirect_to(unquote(str(came_from)))
if c.user:
context = {'model': model,
'user': c.user}
data_dict = {'id': c.user}
user_dict = get_action('user_show')(context, data_dict)
#h.flash_success(_("%s is now logged in") %
# user_dict['display_name'])
return h.redirect_to(controller='user', action='read', id=c.userobj.name)
else:
err = _('Login failed. Bad username or password.')
if asbool(config.get('ckan.legacy_templates', 'false')):
h.flash_error(err)
h.redirect_to(controller='user',
action='login', came_from=came_from)
else:
return self.login(error=err)
def logged_in(self):
"""Minor rewrite to redirect the user to the own profile page instead of
the dashboard.
"""
# we need to set the language via a redirect
lang = session.pop('lang', None)
session.save()
came_from = request.params.get('came_from', '')
if came_from and not came_from.isspace():
came_from = came_from\
.replace('\n', ' ')\
.replace('\r', '')
# we need to set the language explicitly here or the flash
# messages will not be translated.
ckan.lib.i18n.set_lang(lang)
if h.url_is_local(came_from):
return h.redirect_to(unquote(str(came_from)))
if c.user:
context = {'model': model,
'user': c.user}
data_dict = {'id': c.user}
user_dict = get_action('user_show')(context, data_dict)
#h.flash_success(_("%s is now logged in") %
# user_dict['display_name'])
return h.redirect_to(controller='user', action='read', id=c.userobj.name)
else:
err = _('Login failed. Bad username or password.')
if asbool(config.get('ckan.legacy_templates', 'false')):
h.flash_error(err)
h.redirect_to(controller='user',
action='login', came_from=came_from)
else:
return self.login(error=err)
def send(self, pkg_id):
package = Package.get(pkg_id)
url = h.url_for(controller='package',
action="read",
id=package.id)
if c.user:
userid = None
for role in package.roles:
if role.role == "admin":
userid = role.user_id
break
if userid:
owner = User.get(userid)
msg = request.params.get('msg', '')
if msg:
send_contact_email(owner, c.userobj, package,\
msg)
else:
h.flash_error(_("No message"))
return redirect(url)
else:
h.flash_error(_("No owner found"))
return redirect(url)
h.flash_notice(_("Message sent"))
else:
h.flash_error(_("Please login"))
return redirect(url)
def _send_application( self, group, reason ):
from ckan.logic.action import error_summary
from ckan.lib.mailer import mail_recipient
from genshi.template.text import NewTextTemplate
from pylons import config
if not reason:
h.flash_error(_("There was a problem with your submission, \
please correct it and try again"))
errors = {"reason": ["No reason was supplied"]}
return self.apply(group.id, errors=errors,
error_summary=error_summary(errors))
# look for publisher admins up the tree
recipients = []
recipient_publisher = None
for publisher in go_up_tree(group):
admins = publisher.members_of_type(model.User, 'admin').all()
if admins:
recipients = [(u.fullname,u.email) for u in admins]
recipient_publisher = publisher.title
break
if not recipients:
if not config.get('dgu.admin.email'):
log.error('User "%s" prevented from applying for publisher access for "%s" '
'because: dgu.admin.email is not setup in CKAN config.',
c.user, group.name)
h.flash_error(_("There is a problem with the system configuration"))
errors = {"reason": ["%s does not have an administrator user to contact" % group.name]}
return self.apply(group.id, data=data, errors=errors,
error_summary=error_summary(errors))
recipients = [(config.get('dgu.admin.name', "DGU Admin"),
config['dgu.admin.email'])]
recipient_publisher = 'data.gov.uk admin'
log.debug('User "%s" requested publisher access for "%s" which was sent to admin %s (%r) with reason: %r',
c.user, group.name, recipient_publisher, recipients, reason)
extra_vars = {
'group' : group,
'requester': c.userobj,
'reason' : reason
}
email_msg = render("email/join_publisher_request.txt", extra_vars=extra_vars,
loader_class=NewTextTemplate)
try:
for (name,recipient) in recipients:
mail_recipient(name,
recipient,
"Publisher request",
email_msg)
except Exception, e:
h.flash_error(_("There is a problem with the system configuration"))
errors = {"reason": ["No mail server was found"]}
log.error('User "%s" prevented from applying for publisher access for "%s" because of mail configuration error: %s',
c.user, group.name, e)
return self.apply(group.id, errors=errors,
error_summary=error_summary(errors))
def unfollow(self, id):
'''Stop following this group.'''
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
data_dict = {'id': id}
try:
get_action('unfollow_group')(context, data_dict)
group_dict = get_action('group_show')(context, data_dict)
h.flash_success(
_("You are no longer following {0}").format(
group_dict['title']))
except ValidationError as e:
error_message = (e.extra_msg or e.message or e.error_summary
or e.error_dict)
h.flash_error(error_message)
except (NotFound, NotAuthorized) as e:
error_message = e.extra_msg or e.message
h.flash_error(error_message)
h.redirect_to(controller='group', action='read', id=id)
def logged_in(self):
# we need to set the language via a redirect
lang = session.pop("lang", None)
session.save()
# we need to set the language explicitly here or the flash
# messages will not be translated.
i18n.set_lang(lang)
if c.user:
context = None
data_dict = {"id": c.user}
user_dict = get_action("user_show")(context, data_dict)
h.flash_success(
_("<p><strong>Note</strong></p>" "<p>%s is now logged in</p>") % user_dict["display_name"],
allow_html=True,
)
return h.redirect_to(controller="package", action="search", locale=lang)
else:
h.flash_error(_("Login failed. Bad username or password."))
return h.redirect_to(controller="user", action="login", locale=lang)
def render_contact_form(self, pkg_id):
"""
Render the contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if not c.package:
abort(404, _(u"Dataset not found"))
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = []
for contact in contacts:
if 'name' in contact:
text_val = contact['name']
else:
at_idx = contact['email'].find('@')
text_val = contact['email'][0:at_idx]
text_val = text_val.replace(".", " ").title()
print(text_val)
c.recipient_options.append({'text': text_val, 'value': contact['id']})
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/contact_form.html')
def _add_users(self, group, parameters):
if not group:
h.flash_error(
_("There was a problem with your submission, "
"please correct it and try again"))
errors = {"reason": ["No reason was supplied"]}
return self.apply(group.id,
errors=errors,
error_summary=action.error_summary(errors))
data_dict = logic.clean_dict(
dict_func.unflatten(
logic.tuplize_dict(logic.parse_params(request.params))))
data_dict['id'] = group.id
# Temporary fix for strange caching during dev
l = data_dict['users']
for d in l:
d['capacity'] = d.get('capacity', 'editor')
context = {
"group": group,
"schema": schema.default_group_schema(),
"model": model,
"session": model.Session
}
# Temporary cleanup of a capacity being sent without a name
users = [d for d in data_dict['users'] if len(d) == 2]
data_dict['users'] = users
model.repo.new_revision()
model_save.group_member_save(context, data_dict, 'users')
model.Session.commit()
h.redirect_to(controller='group', action='edit', id=group.name)
def logged_in(self):
# we need to set the language via a redirect
lang = session.pop('lang', None)
session.save()
# we need to set the language explicitly here or the flash
# messages will not be translated.
i18n.set_lang(lang)
if c.user:
context = None
data_dict = {'id': c.user}
user_dict = get_action('user_show')(context, data_dict)
h.flash_success(_("<p><strong>Note</strong></p>"
"<p>%s is now logged in</p>") %
user_dict['display_name'], allow_html=True)
return h.redirect_to(controller='package',
action='search', locale=lang)
else:
h.flash_error(_('Login failed. Bad username or password.'))
return h.redirect_to(controller='user',
action='login', locale=lang)
def _send_application(self, group, reason):
from genshi.template.text import NewTextTemplate
if not reason:
h.flash_error(
_("There was a problem with your submission, \
please correct it and try again"))
errors = {"reason": ["No reason was supplied"]}
return self.apply(group.id,
errors=errors,
error_summary=action.error_summary(errors))
admins = group.members_of_type(model.User, 'admin').all()
recipients = [(u.fullname,u.email) for u in admins] if admins else \
[(config.get('ckan.admin.name', "CKAN Administrator"),
config.get('ckan.admin.email', None), )]
if not recipients:
h.flash_error(
_("There is a problem with the system configuration"))
errors = {"reason": ["No group administrator exists"]}
return self.apply(group.id,
data=data,
errors=errors,
error_summary=action.error_summary(errors))
extra_vars = {'group': group, 'requester': c.userobj, 'reason': reason}
email_msg = render("organizations/email/join_publisher_request.txt",
extra_vars=extra_vars,
loader_class=NewTextTemplate)
try:
for (name, recipient) in recipients:
mailer.mail_recipient(name, recipient, "Publisher request",
email_msg)
except:
h.flash_error(
_("There is a problem with the system configuration"))
errors = {"reason": ["No mail server was found"]}
return self.apply(group.id,
errors=errors,
error_summary=action.error_summary(errors))
h.flash_success(_("Your application has been submitted"))
h.redirect_to('publisher_read', id=group.name)
def _send_application( self, group, reason ):
from genshi.template.text import NewTextTemplate
if not reason:
h.flash_error(_("There was a problem with your submission, \
please correct it and try again"))
errors = {"reason": ["No reason was supplied"]}
return self.apply(group.id, errors=errors,
error_summary=action.error_summary(errors))
admins = group.members_of_type( model.User, 'admin' ).all()
recipients = [(u.fullname,u.email) for u in admins] if admins else \
[(config.get('ckan.admin.name', "CKAN Administrator"),
config.get('ckan.admin.email', None), )]
if not recipients:
h.flash_error(_("There is a problem with the system configuration"))
errors = {"reason": ["No group administrator exists"]}
return self.apply(group.id, data=data, errors=errors,
error_summary=action.error_summary(errors))
extra_vars = {
'group' : group,
'requester': c.userobj,
'reason' : reason
}
email_msg = render("organizations/email/join_publisher_request.txt",
extra_vars=extra_vars,
loader_class=NewTextTemplate)
try:
for (name,recipient) in recipients:
mailer.mail_recipient(name,
recipient,
"Publisher request",
email_msg)
except:
h.flash_error(_("There is a problem with the system configuration"))
errors = {"reason": ["No mail server was found"]}
return self.apply(group.id, errors=errors,
error_summary=action.error_summary(errors))
h.flash_success(_("Your application has been submitted"))
h.redirect_to( 'publisher_read', id=group.name)
def index(self):
group_type = self._guess_group_type()
page = h.get_page_number(toolkit.request.params) or 1
items_per_page = 50
context = {'model': model, 'session': model.Session,
'user': toolkit.c.user, 'for_view': True,
'with_private': False}
q = toolkit.c.q = toolkit.request.params.get('q', '')
sort_by = toolkit.c.sort_by_selected = toolkit.request.params.get('sort')
startswith = toolkit.request.params.get('startswith')
try:
self._check_access('site_read', context)
self._check_access('group_list', context)
except toolkit.NotAuthorized:
toolkit.abort(403, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if toolkit.c.userobj:
context['user_id'] = toolkit.c.userobj.id
context['user_is_admin'] = toolkit.c.userobj.sysadmin
try:
data_dict_global_results = {
'all_fields': True,
'q': q,
'sort': sort_by,
'type': group_type or 'group',
'include_extras': True,
}
global_results = self._action('group_list')(
context, data_dict_global_results)
except toolkit.ValidationError as e:
if e.error_dict and e.error_dict.get('message'):
msg = e.error_dict['message']
else:
msg = str(e)
h.flash_error(msg)
toolkit.c.page = h.Page([], 0)
return toolkit.render(
self._index_template(group_type),
extra_vars={'group_type': group_type}
)
start = items_per_page * (page - 1)
end = start + items_per_page
page_results = global_results
if startswith is not None:
page_results = [x for x in global_results if x['name'][0].lower() == startswith]
toolkit.c.page = h.Page(
collection=page_results,
page=page,
url=h.pager_url,
items_per_page=items_per_page,
)
toolkit.c.page.items = page_results[start:end]
return toolkit.render(
self._index_template(group_type),
extra_vars={
'total_orgs': len(global_results),
'group_type': group_type,
'org_initials': set([x['name'][0].lower() for x in global_results]),
}
)
def new_resource(self, id, data=None, errors=None, error_summary=None):
''' FIXME: This is a temporary action to allow styling of the
forms. '''
euosme_path = config.get('ckan.CustomInspireDataset.euosme.path', "")
ckan_url = config.get('ckan.site_url')
if request.method == 'POST' and not data:
save_action = request.params.get('save')
data = data or \
clean_dict(dict_fns.unflatten(tuplize_dict(parse_params(
request.POST))))
# we don't want to include save as it is part of the form
del data['save']
resource_id = data['id']
del data['id']
context = {'model': model, 'session': model.Session,
'user': c.user, 'auth_user_obj': c.userobj}
# see if we have any data that we are trying to save
data_provided = False
for key, value in data.iteritems():
if ((value or isinstance(value, cgi.FieldStorage))
and key != 'resource_type'):
data_provided = True
break
if not data_provided and save_action != "go-dataset-complete":
if save_action == 'go-dataset':
# go to final stage of adddataset
h.redirect_to(controller='package', action='edit', id=id)
# see if we have added any resources
try:
data_dict = get_action('package_show')(context, {'id': id})
except NotAuthorized:
abort(403, _('Unauthorized to update dataset'))
except NotFound:
abort(404, _('The dataset {id} could not be found.'
).format(id=id))
if not len(data_dict['resources']):
# no data so keep on page
msg = _('You must add at least one data resource')
# On new templates do not use flash message
if asbool(config.get('ckan.legacy_templates')):
h.flash_error(msg)
h.redirect_to(controller='package',
action='new_resource', id=id)
else:
errors = {}
error_summary = {_('Error'): msg}
return self.new_resource(id, data, errors,
error_summary)
# XXX race condition if another user edits/deletes
data_dict = get_action('package_show')(context, {'id': id})
get_action('package_update')(
dict(context, allow_state_change=True),
dict(data_dict, state='active'))
h.redirect_to(controller='package', action='read', id=id)
data['package_id'] = id
try:
if resource_id:
data['id'] = resource_id
get_action('resource_update')(context, data)
else:
if(request.params.get('format')=='INSPIRE'):
home_dir = os.path.join(euosme_path)
random=request.POST['inspire_random_name'];
inspire_file_name = ""
if(data["name"]):
inspire_file_name = str(data["name"])
else:
inspire_file_name = random
filelocation=home_dir+'/'+random+'.xml'
api = c.userobj.apikey
descr = request.params.get('description')
if(descr):
descr = str(descr)
else:
descr = ""
r = requests.post(ckan_url+'/api/action/resource_create',
data={'package_id': id,
'name': inspire_file_name,
'format': 'INSPIRE',
'description': descr,
'url': 'upload', # Needed to pass validation
},
headers={'Authorization': api},#api_key},
files=[('upload', file(filelocation))])
else:
get_action('resource_create')(context, data)
#get_action('resource_create')(context, data)
except ValidationError, e:
errors = e.error_dict
error_summary = e.error_summary
return self.new_resource(id, data, errors, error_summary)
except NotAuthorized:
abort(403, _('Unauthorized to create a resource'))
def _prepare_and_send(self, pkg_id, recipient_id, subject, prefix_template, suffix):
"""
Sends a message by email from the logged in user to the appropriate
contact address of the given dataset.
The prefix template should have formatting placeholders for the following arguments:
{sender_name}, {sender_email}, {package_title}, {package_id}
:param pkg_id: package id
:type pkg_id: string
:param recipient_id: id of the recipient, as returned by utils.get_package_contacts
:type recipient_id: string
:param subject: the subject of the message
:type subject: string
:param prefix_template: the template for the prefix to be automatically included before the user message
:type prefix_template: unicode
:param suffix: an additional note to be automatically included after the user message
:type suffix: unicode
"""
url = h.url_for(controller='package', action='read', id=pkg_id)
if asbool(config.get('kata.contact_captcha')):
try:
captcha.check_recaptcha(request)
except captcha.CaptchaError:
h.flash_error(_(u'Bad Captcha. Please try again.'))
redirect(url)
if not request.params.get('accept_logging'):
h.flash_error(_(u"Message not sent as logging wasn't permitted"))
return redirect(url)
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. Please try again later or contact customer "
u"service."))
return redirect(url)
package = Package.get(pkg_id)
package_title = package.title if package.title else package.name
sender_addr = request.params.get('from_address')
sender_name = request.params.get('from_name')
recipient = self._get_contact_email(pkg_id, recipient_id)
if not recipient:
abort(404, _('Recipient not found'))
user_msg = request.params.get('msg', '')
if request.params.get('hp'):
h.flash_error(_(u"Message not sent. Couldn't confirm human interaction (spam bot control)"))
return redirect(url)
if sender_addr and sender_name and \
isinstance(sender_name, basestring) and len(sender_name) >= 3:
if user_msg:
prefix = prefix_template.format(
sender_name=sender_name,
sender_email=sender_addr,
package_title=package_title,
package_id=pkg_id
# above line was: metadata_pid=utils.get_primary_data_pid_from_package(package)
)
log.info(u"Message {m} sent from {a} ({b}) to {r} about {c}, IP: {d}"
.format(m=user_msg, a=sender_name, b=sender_addr, r=recipient, c=pkg_id,
d=request.environ.get('REMOTE_ADDR', 'No remote address')))
full_msg = u"{a}{b}{c}".format(a=prefix, b=user_msg, c=suffix)
self._send_message(subject, full_msg, recipient.get('email'), recipient.get('name'))
h.flash_success(_(u"Message sent"))
else:
h.flash_error(_(u"No message"))
else:
h.flash_error(_(u"Message not sent. Please, provide reply address and name. Name must contain \
at least three letters."))
return redirect(url)
def _send_application(self, group, reason):
from ckan.logic.action import error_summary
from ckan.lib.mailer import mail_recipient
from genshi.template.text import NewTextTemplate
from pylons import config
if not reason:
h.flash_error(
_("There was a problem with your submission, \
please correct it and try again"))
errors = {"reason": ["No reason was supplied"]}
return self.apply(group.id,
errors=errors,
error_summary=error_summary(errors))
recipients, recipient_organisation = find_group_admins(group)
if not recipients:
if not config.get('dgu.admin.email'):
log.error(
'User "%s" prevented from applying for organisation access for "%s" '
'because: dgu.admin.email is not setup in CKAN config.',
c.user, group.name)
h.flash_error(
_("There is a problem with the system configuration"))
errors = {"reason": ["data.gov.uk error"]}
return self.apply(group.id,
errors=errors,
error_summary=error_summary(errors))
recipients = [(config.get('dgu.admin.name',
"DGU Admin"), config['dgu.admin.email'])]
recipient_organisation = 'data.gov.uk admin team'
url = urljoin(
g.site_url,
h.url_for(
controller=
'ckanext.dgu_orgs.controllers.organisation:OrganisationController',
action='users',
id=group.name))
log.debug(
'User "%s" requested organisation access for "%s" which was sent to admin %s (%r) with reason: %r',
c.user, group.name, recipient_organisation, recipients, reason)
extra_vars = {
'group': group,
'requester': c.userobj,
'reason': reason,
'accept_url': url
}
email_msg = render("email/join_publisher_request.txt",
extra_vars=extra_vars,
loader_class=NewTextTemplate)
try:
for (name, recipient) in recipients:
mail_recipient(
name,
recipient,
subject=
'DGUKOrganisationRequest: Please add me as a data.gov.uk organisation',
body=email_msg)
except Exception, e:
h.flash_error(
'There is a problem with the system configuration. Please instead <a href="http://data.gov.uk/contact">contact the data.gov.uk team</a>',
allow_html=True)
errors = {"reason": ["data.gov.uk error"]}
log.error(
'User "%s" prevented from applying for organisation access for "%s" because of mail configuration error: %s',
c.user, group.name, e)
return self.apply(group.id,
errors=errors,
error_summary=error_summary(errors))
try:
c.group_dict = get_action('organization_show')(context, {"id": id})
c.group = context['group']
except ObjectNotFound:
abort(404, 'Organization not found')
except NotAuthorized:
abort(401, 'Unauthorized to read group %s' % id)
try:
context['group'] = c.group
check_access('organization_update', context)
except NotAuthorized, e:
abort(401, 'User %r not authorized to upload inventory' % (c.user))
if not 'upload' in request.POST or not hasattr(request.POST['upload'], "filename"):
h.flash_error("No file was selected, please choose a file before uploading", allow_html=True)
return h.redirect_to( controller="ckanext.dgu.controllers.inventory:InventoryController",
action="edit", id=c.group.name)
incoming = request.POST['upload'].filename
file_root = config.get('inventory.temporary.storage', '/tmp')
filename = os.path.join(file_root, make_uuid()) + "-{0}".format(incoming)
with inventory_lib.UploadFileHelper(incoming, request.POST['upload'].file) as f:
open(filename, 'wb').write(f.read())
job_id, timestamp = inventory_lib.enqueue_document(c.userobj, filename, c.group)
jobdict = json.loads(c.group.extras.get('inventory.jobs', '{}'))
jobdict[job_id] = timestamp
def _add_users(self, group, parameters):
from ckan.logic.schema import default_group_schema
from ckan.logic.action import error_summary
from ckan.lib.dictization.model_save import group_member_save
if not group:
h.flash_error(
_("There was a problem with your submission, \
please correct it and try again"))
errors = {"reason": ["No reason was supplied"]}
return self.users(group.name,
errors=errors,
error_summary=error_summary(errors))
data_dict = clean_dict(
unflatten(tuplize_dict(parse_params(request.params))))
data_dict['id'] = group.id
# Check that the user being added, if they are a Drupal user, has
# verified their email address
new_users = [user['name'] for user in data_dict['users'] \
if not 'capacity' in user or user['capacity'] == 'undefined']
# Removed Drupal auth stuff.
# for user_name in new_users:
# drupal_id = DrupalUserMapping.ckan_user_name_to_drupal_id(user_name)
# if drupal_id:
# if not is_drupal_auth_activated():
# # joint auth with Drupal is not activated, so cannot
# # check with Drupal
# log.warning('Drupal user made editor/admin but without checking email is verified.')
# break
# if 'drupal_client' not in dir(self):
# self.drupal_client = DrupalClient()
# user_properties = self.drupal_client.get_user_properties(drupal_id)
# roles = user_properties['roles'].values()
# if 'unverified user' in roles:
# user = model.User.by_name(user_name)
# h.flash_error("There was a problem with your submission - see the error message below.")
# errors = {"reason": ['User "%s" has not verified their email address yet. '
# 'Please ask them to do this and then try again. ' % \
# user.fullname]}
# log.warning('Trying to add user (%r %s) who is not verified to group %s',
# user.fullname, user_name, group.name)
# # NB Other values in the form are lost, but that is probably ok
# return self.users(group.name, errors=errors,
# error_summary=error_summary(errors))
# Temporary fix for strange caching during dev
l = data_dict['users']
for d in l:
# Form javascript creates d['capacity'] == 'undefined' for
# newly added users.
# If javascript in users form is not working (such as in tests)
# it will not create a capacity value.
if 'capacity' not in d or d['capacity'] == 'undefined':
# default to 'editor'
d['capacity'] = 'editor'
context = {
"group": group,
"schema": default_group_schema(),
"model": model,
"session": model.Session
}
# Temporary cleanup of a capacity being sent without a name
users = [d for d in data_dict['users'] if len(d) == 2]
data_dict['users'] = users
model.repo.new_revision()
group_member_save(context, data_dict, 'users')
model.Session.commit()
h.redirect_to('/organisation/%s' % str(group.name))
def read_data(self, id, resource_id):
res = Resource.get(resource_id)
pkg = Package.get(id)
c.pkg_dict = pkg.as_dict()
c.package = pkg
c.resource = get_action('resource_show')({'model': model},
{'id': resource_id})
label = res.url.split(config.get('ckan.site_url') + '/storage/f/')[-1]
label = urllib2.unquote(label)
ofs = get_ofs()
try:
furl = ofs.get_url(BUCKET, label).split('file://')[-1]
except FileNotFoundException:
h.flash_error(_('Cannot do data mining on remote resource!'))
url = h.url_for(controller='package', action='resource_read',
id=id, resource_id=resource_id)
return redirect(url)
wordstats = {}
ret = {}
if res.format in ('TXT', 'txt'):
wdsf, wdspath = tempfile.mkstemp()
os.write(wdsf, "%s\nmetadata description title information" % furl)
with os.fdopen(wdsf, 'r') as wordfile:
preproc = orngText.Preprocess()
table = orngText.loadFromListWithCategories(wdspath)
data = orngText.bagOfWords(table, preprocessor=preproc)
words = orngText.extractWordNGram(data, threshold=10.0, measure='MI')
for i in range(len(words)):
d = words[i]
wordstats = d.get_metas(str)
for k, v in wordstats.items():
if v.value > 10.0:
ret[unicode(k, 'utf8')] = v.value
from operator import itemgetter
c.data_tags = sorted(ret.iteritems(), key=itemgetter(1), reverse=True)[:30]
os.remove(wdspath)
for i in range(len(data)):
d = words[i]
wordstats = d.get_metas(str)
words = []
for k, v in wordstats.items():
words.append(k)
model.repo.new_revision()
if not 'autoextracted_description' in pkg.extras:
pkg.extras['autoextracted_description'] = ' '.join(words)
pkg.save()
return render('datamining/read.html')
elif res.format in ('odt', 'doc', 'xls', 'ods', 'odp', 'ppt', 'doc', 'html'):
textfd, textpath = convert_to_text(res, furl)
if not textpath:
h.flash_error(_('This file could not be mined for any data!'))
os.close(textfd)
return render('datamining/read.html')
else:
wdsf, wdspath = tempfile.mkstemp()
os.write(wdsf, "%s\nmetadata description title information" % textpath)
preproc = orngText.Preprocess()
table = orngText.loadFromListWithCategories(wdspath)
data = orngText.bagOfWords(table, preprocessor=preproc)
words = orngText.extractWordNGram(data, threshold=10.0, measure='MI')
for i in range(len(words)):
d = words[i]
wordstats = d.get_metas(str)
for k, v in wordstats.items():
if v.value > 10.0:
ret[unicode(k, 'utf8')] = v.value
from operator import itemgetter
c.data_tags = sorted(ret.iteritems(), key=itemgetter(1), reverse=True)[:30]
os.close(textfd)
os.close(wdsf)
os.remove(wdspath)
os.remove(textpath)
for i in range(len(data)):
d = words[i]
wordstats = d.get_metas(str)
words = []
for k, v in wordstats.items():
log.debug(k)
words.append(substitute_ascii_equivalents(k))
model.repo.new_revision()
if not 'autoextracted_description' in pkg.extras:
pkg.extras['autoextracted_description'] = ' '.join(words)
pkg.save()
return render('datamining/read.html')
else:
h.flash_error(_('This metadata document is not in proper format for data mining!'))
url = h.url_for(controller='package', action='resource_read',
id=id, resource_id=resource_id)
return redirect(url)
def _prepare_and_send(self, pkg_id, recipient_id, subject, prefix_template, suffix):
"""
Sends a message by email from the logged in user to the appropriate
contact address of the given dataset.
The prefix template should have formatting placeholders for the following arguments:
{sender_name}, {sender_email}, {package_title}, {data_pid}
:param pkg_id: package id
:type pkg_id: string
:param recipient_id: id of the recipient, as returned by utils.get_package_contacts
:type recipient_id: string
:param subject: the subject of the message
:type subject: string
:param prefix_template: the template for the prefix to be automatically included before the user message
:type prefix_template: unicode
:param suffix: an additional note to be automatically included after the user message
:type suffix: unicode
"""
url = h.url_for(controller='package', action='read', id=pkg_id)
if asbool(config.get('kata.contact_captcha')):
try:
captcha.check_recaptcha(request)
except captcha.CaptchaError:
h.flash_error(_(u'Bad Captcha. Please try again.'))
redirect(url)
if not request.params.get('accept_logging'):
h.flash_error(_(u"Message not sent as logging wasn't permitted"))
return redirect(url)
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. Please try again later or contact customer "
u"service."))
return redirect(url)
package = Package.get(pkg_id)
package_title = package.title if package.title else package.name
sender_addr = request.params.get('from_address')
sender_name = request.params.get('from_name')
recipient = self._get_contact_email(pkg_id, recipient_id)
if not recipient:
abort(404, _('Recipient not found'))
user_msg = request.params.get('msg', '')
ct = int(time.time())
try:
check = self.crypto.decrypt(base64.b64decode(request.params.get('check_this_out')))
check = re.sub(' ', '', check)
except TypeError:
h.flash_error(_(u"Message not sent. Couldn't confirm human interaction (spam bot control)"))
return redirect(url)
hp = request.params.get('hp')
if hp or not check or (ct - int(check) < 20) or (ct - int(check) > 1200):
h.flash_error(_(u"Message not sent. Couldn't confirm human interaction (spam bot control)"))
return redirect(url)
if sender_addr and sender_name and \
isinstance(sender_name, basestring) and len(sender_name) >= 3:
if user_msg:
prefix = prefix_template.format(
sender_name=sender_name,
sender_email=sender_addr,
package_title=package_title,
data_pid=utils.get_primary_data_pid_from_package(package)
)
log.info(u"Message {m} sent from {a} ({b}) to {r} about {c}, IP: {d}"
.format(m=user_msg, a=sender_name, b=sender_addr, r=recipient, c=pkg_id,
d=request.environ.get('REMOTE_ADDR', 'No remote address')))
full_msg = u"{a}{b}{c}".format(a=prefix, b=user_msg, c=suffix)
self._send_message(subject, full_msg, recipient.get('email'), recipient.get('name'))
h.flash_success(_(u"Message sent"))
else:
h.flash_error(_(u"No message"))
else:
h.flash_error(_(u"Message not sent. Please, provide reply address and name. Name must contain \
at least three letters."))
return redirect(url)