def apply(self, id=None, data=None, errors=None, error_summary=None):
"""
Form for a user to request to be an editor for a publisher.
It sends an email to a suitable admin.
"""
if 'parent' in request.params and not id:
id = request.params['parent']
if id:
c.group = model.Group.get(id)
if not c.group:
abort(404, _('Publisher not found'))
if 'save' in request.params and not errors:
return self._send_application(c.group, request.params.get('reason', None))
self._add_publisher_list()
data = data or {}
errors = errors or {}
error_summary = error_summary or {}
data.update(request.params)
vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
c.form = render('publisher/apply_form.html', extra_vars=vars)
return render('publisher/apply.html')
def named_theme(self, name):
"""
Shows the theme home page for a specified theme containing information relevant
to *just* that theme, popular and new datasets as well as recent apps+ideas from
the front end.
"""
raise NotImplementedError("Cannot view theme pages yet.")
c.theme = name
c.theme_name = THEMES.get(name)
if not c.theme_name:
abort(404)
c.dataset_count = (
model.Session.query(model.Package)
.join(model.PackageExtra)
.filter(model.PackageExtra.key == "theme-primary")
.filter(model.PackageExtra.value == c.theme_name)
.filter(model.Package.state == "active")
.count()
)
c.latest = self._search(theme=c.theme_name)
c.popular = self._search(theme=c.theme_name, sort_string="popularity asc")
return render("themed/theme.html")
def openspending_report(self):
if not dgu_helpers.is_sysadmin():
abort(401, 'User must be a sysadmin to view this page.')
self._set_openspending_reports_dir()
c.content = open (c.openspending_report_dir + "/index.html").read()
return render('data/openspending_report.html')
def contracts_archive(self, relative_url='/'):
import requests
from pylons import response
if request.method == 'POST':
abort(405) # Method Not Allowed
headers = {'X-Script-Name': '/data/contracts-finder-archive'}
contracts_url = pylons.config.get('dgu.contracts_url')
url = urlparse.urljoin(contracts_url, relative_url)
r = requests.get(url,
headers=headers,
params=dict(request.params),
stream=True)
if r.status_code != 200:
abort(r.status_code)
if relative_url.startswith(('/static/', '/download/')):
# CSS will only get loaded if it has the right content type
response.content_type = r.headers.get('Content-Type', 'text/html')
if r.headers.get('Content-Disposition'):
response.headers['Content-Disposition'] = r.headers.get('Content-Disposition')
return r.raw.read() # Some of the static files are binary
else:
extra_vars = {'content': r.text}
return render('contracts_archive/front_page.html',
extra_vars=extra_vars)
def linked_data_admin(self):
"""
"""
import git
if not dgu_helpers.is_sysadmin() and not c.user in ['user_d24373', 'user_d102361']:
abort(403)
prefix = 'dgu.jekyll.ukgovld.'
c.repo_url = pylons.config.get(prefix + "repo.url", None)
c.repo_branch = pylons.config.get(prefix + "repo.branch", None)
source_repo_path = pylons.config.get(prefix + "local.source", None)
build_path = pylons.config.get(prefix + "local.build", None)
deploy_path = pylons.config.get(prefix + "local.deploy", None)
if not all([c.repo_url, c.repo_branch, source_repo_path, build_path,
deploy_path]) or \
not os.path.exists(source_repo_path):
c.error = "System not configured, please setup ckan.ini"
if not os.path.exists(source_repo_path):
c.error = "Repo path %s does not exist" % source_repo_path
return render('data/ukgovld.html')
# Ensure repo exists locally
try:
repo = git.Repo(source_repo_path)
except git.InvalidGitRepositoryError, e:
repo = git.Repo.init(source_repo_path)
repo.create_remote('origin', c.repo_url)
def index(self):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
data_dict = {'all_fields': True}
try:
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# TODO: Fix this up, we only really need to do this when we are
# showing the hierarchy (and then we should load on demand really).
c.all_groups = model.Session.query(model.Group).\
filter(model.Group.type == 'publisher').\
filter(model.Group.state == 'active').\
order_by('title')
c.page = AlphaPage(
controller_name="ckanext.dgu.controllers.publisher:PublisherController",
collection=c.all_groups,
page=request.params.get('page', 'A'),
alpha_attribute='title',
other_text=_('Other'),
)
return render('publisher/index.html')
def commitments(self, id):
"""
Shows all of the commitments for the specified publisher
"""
from ckanext.dgu.model.commitment import Commitment
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'extras_as_string': True,
'save': 'save' in request.params
}
try:
c.publisher_dict = get_action('organization_show')(context, {
'id': id
})
except NotAuthorized:
abort(401, _('Unauthorized to read commitments'))
except ObjectNotFound:
abort(404, _('Publisher not found'))
c.publisher = context.get('group')
c.commitments = Commitment.get_for_publisher(c.publisher.name)\
.order_by('commitment.dataset_name').all()
return render('commitment/read.html')
def challenge(self, environ, status, app_headers, forget_headers):
# redirect to login_form
res = Response()
res.status = 401
res.unicode_body = render('not_authorized.html')
#res.location = '/data/not_authorized' #self.login_form_url+"?%s=%s" %(self.came_from_field, request.url)
return res
def contracts_archive(self, relative_url='/'):
import requests
from pylons import response
if request.method == 'POST':
abort(405) # Method Not Allowed
headers = {'X-Script-Name': '/data/contracts-finder-archive'}
contracts_url = pylons.config.get('dgu.contracts_url')
url = urlparse.urljoin(contracts_url, relative_url)
r = requests.get(url,
headers=headers,
params=dict(request.params),
stream=True)
if r.status_code != 200:
abort(r.status_code)
if relative_url.startswith(('/static/', '/download/')):
# CSS will only get loaded if it has the right content type
response.content_type = r.headers.get('Content-Type', 'text/html')
if r.headers.get('Content-Disposition'):
response.headers['Content-Disposition'] = r.headers.get(
'Content-Disposition')
return r.raw.read() # Some of the static files are binary
else:
extra_vars = {'content': r.text}
return render('contracts_archive/front_page.html',
extra_vars=extra_vars)
def linked_data_admin(self):
"""
"""
import git
if not dgu_helpers.is_sysadmin() and not c.user in [
'user_d24373', 'user_d102361'
]:
abort(403)
prefix = 'dgu.jekyll.ukgovld.'
c.repo_url = pylons.config.get(prefix + "repo.url", None)
c.repo_branch = pylons.config.get(prefix + "repo.branch", None)
source_repo_path = pylons.config.get(prefix + "local.source", None)
build_path = pylons.config.get(prefix + "local.build", None)
deploy_path = pylons.config.get(prefix + "local.deploy", None)
if not all([c.repo_url, c.repo_branch, source_repo_path, build_path,
deploy_path]) or \
not os.path.exists(source_repo_path):
c.error = "System not configured, please setup ckan.ini"
if not os.path.exists(source_repo_path):
c.error = "Repo path %s does not exist" % source_repo_path
return render('data/ukgovld.html')
# Ensure repo exists locally
try:
repo = git.Repo(source_repo_path)
except git.InvalidGitRepositoryError, e:
repo = git.Repo.init(source_repo_path)
repo.create_remote('origin', c.repo_url)
def openspending_report(self):
if not dgu_helpers.is_sysadmin():
abort(401, 'User must be a sysadmin to view this page.')
self._set_openspending_reports_dir()
c.content = open(c.openspending_report_dir + "/index.html").read()
return render('data/openspending_report.html')
def home(self):
extra_vars = {}
# Get the dataset count using search
# (shouldn't cache, as it makes it more likely to be out of sync with
# the data page)
from ckan.lib.search import SearchError, SearchQueryError
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'auth_user_obj': c.userobj
}
data_dict = {
'q': '*:*',
'fq': '+dataset_type:dataset',
#'facet.field': facets.keys(),
'rows': 0,
}
query = get_action('package_search')(context, data_dict)
extra_vars['num_datasets'] = query['count']
extra_vars['themes'] = get_themes()
return render('data/home.html', extra_vars=extra_vars)
def challenge(self, environ, status, app_headers, forget_headers):
# redirect to login_form
res = Response()
res.status = 401
res.unicode_body = render('not_authorized.html')
#res.location = '/data/not_authorized' #self.login_form_url+"?%s=%s" %(self.came_from_field, request.url)
return res
def _send_application( self, group, reason ):
from ckan.logic.action import error_summary
from ckan.lib.mailer import mail_recipient
from genshi.template.text import NewTextTemplate
from pylons import config
if not reason:
h.flash_error(_("There was a problem with your submission, \
please correct it and try again"))
errors = {"reason": ["No reason was supplied"]}
return self.apply(group.id, errors=errors,
error_summary=error_summary(errors))
# look for publisher admins up the tree
recipients = []
recipient_publisher = None
for publisher in go_up_tree(group):
admins = publisher.members_of_type(model.User, 'admin').all()
if admins:
recipients = [(u.fullname,u.email) for u in admins]
recipient_publisher = publisher.title
break
if not recipients:
if not config.get('dgu.admin.email'):
log.error('User "%s" prevented from applying for publisher access for "%s" '
'because: dgu.admin.email is not setup in CKAN config.',
c.user, group.name)
h.flash_error(_("There is a problem with the system configuration"))
errors = {"reason": ["%s does not have an administrator user to contact" % group.name]}
return self.apply(group.id, data=data, errors=errors,
error_summary=error_summary(errors))
recipients = [(config.get('dgu.admin.name', "DGU Admin"),
config['dgu.admin.email'])]
recipient_publisher = 'data.gov.uk admin'
log.debug('User "%s" requested publisher access for "%s" which was sent to admin %s (%r) with reason: %r',
c.user, group.name, recipient_publisher, recipients, reason)
extra_vars = {
'group' : group,
'requester': c.userobj,
'reason' : reason
}
email_msg = render("email/join_publisher_request.txt", extra_vars=extra_vars,
loader_class=NewTextTemplate)
try:
for (name,recipient) in recipients:
mail_recipient(name,
recipient,
"Publisher request",
email_msg)
except Exception, e:
h.flash_error(_("There is a problem with the system configuration"))
errors = {"reason": ["No mail server was found"]}
log.error('User "%s" prevented from applying for publisher access for "%s" because of mail configuration error: %s',
c.user, group.name, e)
return self.apply(group.id, errors=errors,
error_summary=error_summary(errors))
def index(self):
from ckanext.dgu.model.commitment import Commitment, ODS_ORGS
c.publishers = model.Session.query(model.Group)\
.filter(model.Group.state=='active')\
.filter(model.Group.name.in_(ODS_ORGS.values()))\
.order_by(model.Group.title).all()
c.commitments = model.Session.query(Commitment).filter(Commitment.state=='active').all()
return render('commitment/index.html')
def index(self):
c.q = request.params.get('q', '')
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True
}
# This is the one difference from ckan core.
# No need for all vocab / translation stuff, so save the massive
# effort of dictizing every tag. This improves page load from
# 60s to 10s.
data_dict = {'all_fields': False}
if c.q:
try:
page = int(request.params.get('page', 1))
except ValueError:
abort(404, _('Not found'))
data_dict['q'] = c.q
data_dict['limit'] = LIMIT
data_dict['offset'] = (page - 1) * LIMIT
data_dict['return_objects'] = True
result_dict = get_action('tag_search')(context, data_dict)
def pager_url(q=None, page=None):
return h.url_for(
controller='ckanext.dgu.controllers.tag:TagController',
action='index',
q=request.params['q'],
page=page)
c.page = h.Page(
collection=result_dict['results'],
page=page,
item_count=result_dict['count'],
items_per_page=LIMIT,
url=pager_url,
)
c.page.items = [
tag_dict['name'] for tag_dict in result_dict['results']
]
else:
results = get_action('tag_list')(context, data_dict)
c.page = AlphaPageLarge(
collection=results,
page=request.params.get('page', 'A'),
alpha_attribute='name',
other_text=_('Other'),
controller_name='ckanext.dgu.controllers.tag:TagController',
)
return render('tag/index.html')
def index(self):
from ckanext.dgu.model.commitment import Commitment, ODS_ORGS
c.publishers = model.Session.query(model.Group)\
.filter(model.Group.state=='active')\
.filter(model.Group.name.in_(ODS_ORGS.values()))\
.order_by(model.Group.title).all()
c.commitments = model.Session.query(Commitment).filter(
Commitment.state == 'active').all()
return render('commitment/index.html')
def linked_data_admin(self):
"""
Instructions for installing Ruby via RVM:
\curl -sSL https://get.rvm.io | bash -s stable
# Comment out lines in .bashrc and .bash_profile
source "$HOME/.rvm/scripts/rvm"
rvm autolibs distable
rvm install 1.9.3
rvm use 1.9.3
rvm gemset create ukgovld
rvm alias create ukgovldwg 1.9.3@ukgovld
cd /vagrant/src/ckanext-dgu/
~/.rvm/wrappers/ukgovldwg/bundle
rvm wrapper ukgovldwg jekyll
"""
import git
if not dgu_helpers.is_sysadmin() and not c.user in [
'user_d24373', 'user_d102361'
]:
abort(401)
prefix = 'dgu.jekyll.ukgovld.'
c.repo_url = pylons.config.get(prefix + "repo.url", None)
c.repo_branch = pylons.config.get(prefix + "repo.branch", None)
source_repo_path = pylons.config.get(prefix + "local.source", None)
build_path = pylons.config.get(prefix + "local.build", None)
deploy_path = pylons.config.get(prefix + "local.deploy", None)
if source_repo_path and (not os.path.exists(source_repo_path)
and source_repo_path.startswith('/tmp/')):
# Directories in /tmp won't survive a reboot
os.makedirs(source_repo_path)
if build_path and (not os.path.exists(build_path)
and build_path.startswith('/tmp/')):
# Directories in /tmp won't survive a reboot
os.makedirs(build_path)
if not all([c.repo_url, c.repo_branch, source_repo_path, build_path,
deploy_path]) or \
not os.path.exists(source_repo_path):
c.error = "System not configured, please setup ckan.ini"
if source_repo_path and not os.path.exists(source_repo_path):
c.error = "Repo path %s does not exist" % source_repo_path
return render('data/ukgovld.html')
# Ensure repo exists locally
try:
repo = git.Repo(source_repo_path)
except git.InvalidGitRepositoryError, e:
repo = git.Repo.init(source_repo_path)
repo.create_remote('origin', c.repo_url)
def view(self, id):
""" View all feedback for the specified package """
from ckanext.dgu.model.feedback import Feedback
self._get_package(id)
c.items = model.Session.query(Feedback).\
filter(Feedback.package_id == c.pkg.id).\
filter(Feedback.visible == True).\
filter(Feedback.active==True).order_by('created desc')
return render('feedback/view.html')
def view(self, id):
""" View all feedback for the specified package """
from ckanext.dgu.model.feedback import Feedback
self._get_package(id)
c.items = model.Session.query(Feedback).\
filter(Feedback.package_id == c.pkg.id).\
filter(Feedback.visible == True).\
filter(Feedback.active==True).order_by('created desc')
return render('feedback/view.html')
def all_packages(self):
ctx = {'model': model, 'session': model.Session}
package_list = get_action("package_list")(ctx, {})
def linkify_packages(pkgs):
x = 0
for pkg in pkgs:
yield '<a href="/dataset/{p}">{p}</a><br/>'.format(p=pkg)
c.datasets = linkify_packages(package_list)
return render("package/all_datasets.html")
def all_packages(self):
ctx = {'model': model, 'session': model.Session}
package_list = get_action("package_list")(ctx, {})
def linkify_packages(pkgs):
x = 0
for pkg in pkgs:
yield '<a href="/dataset/{p}">{p}</a><br/>'.format(p=pkg)
c.datasets = linkify_packages(package_list)
return render("package/all_datasets.html")
def linked_data_admin(self):
"""
Instructions for installing Ruby via RVM:
\curl -sSL https://get.rvm.io | bash -s stable
# Comment out lines in .bashrc and .bash_profile
source "$HOME/.rvm/scripts/rvm"
rvm autolibs distable
rvm install 1.9.3
rvm use 1.9.3
rvm gemset create ukgovld
rvm alias create ukgovldwg 1.9.3@ukgovld
cd /vagrant/src/ckanext-dgu/
~/.rvm/wrappers/ukgovldwg/bundle
rvm wrapper ukgovldwg jekyll
"""
import git
if not dgu_helpers.is_sysadmin() and not c.user in ['user_d24373', 'user_d102361']:
abort(401)
prefix = 'dgu.jekyll.ukgovld.'
c.repo_url = pylons.config.get(prefix + "repo.url", None)
c.repo_branch = pylons.config.get(prefix + "repo.branch", None)
source_repo_path = pylons.config.get(prefix + "local.source", None)
build_path = pylons.config.get(prefix + "local.build", None)
deploy_path = pylons.config.get(prefix + "local.deploy", None)
if source_repo_path and (not os.path.exists(source_repo_path) and
source_repo_path.startswith('/tmp/')):
# Directories in /tmp won't survive a reboot
os.makedirs(source_repo_path)
if build_path and (not os.path.exists(build_path) and build_path.startswith('/tmp/')):
# Directories in /tmp won't survive a reboot
os.makedirs(build_path)
if not all([c.repo_url, c.repo_branch, source_repo_path, build_path,
deploy_path]) or \
not os.path.exists(source_repo_path):
c.error = "System not configured, please setup ckan.ini"
if source_repo_path and not os.path.exists(source_repo_path):
c.error = "Repo path %s does not exist" % source_repo_path
return render('data/ukgovld.html')
# Ensure repo exists locally
try:
repo = git.Repo(source_repo_path)
except git.InvalidGitRepositoryError, e:
repo = git.Repo.init(source_repo_path)
repo.create_remote('origin', c.repo_url)
class CommentController(BaseController):
def moderation(self):
context = {'model': model, 'user': c.user}
check_access('moderation_queue_show', context)
try:
res = get_action('moderation_queue_show')(context, {})
except Exception, e:
abort(403)
c.comments = res.get('comments')
return render('comments/moderation.html')
def index(self):
c.q = request.params.get("q", "")
context = {"model": model, "session": model.Session, "user": c.user or c.author, "for_view": True}
# This is the one difference from ckan core.
# No need for all vocab / translation stuff, so save the massive
# effort of dictizing every tag. This improves page load from
# 60s to 10s.
data_dict = {"all_fields": False}
if c.q:
try:
page = int(request.params.get("page", 1))
except ValueError:
abort(404, _("Not found"))
data_dict["q"] = c.q
data_dict["limit"] = LIMIT
data_dict["offset"] = (page - 1) * LIMIT
data_dict["return_objects"] = True
result_dict = get_action("tag_search")(context, data_dict)
def pager_url(q=None, page=None):
return h.url_for(
controller="ckanext.dgu.controllers.tag:TagController",
action="index",
q=request.params["q"],
page=page,
)
c.page = h.Page(
collection=result_dict["results"],
page=page,
item_count=result_dict["count"],
items_per_page=LIMIT,
url=pager_url,
)
c.page.items = [tag_dict["name"] for tag_dict in result_dict["results"]]
else:
results = get_action("tag_list")(context, data_dict)
c.page = AlphaPageLarge(
collection=results,
page=request.params.get("page", "A"),
alpha_attribute="name",
other_text=_("Other"),
controller_name="ckanext.dgu.controllers.tag:TagController",
)
return render("tag/index.html")
def openspending_publisher_report(self, id):
id = id.replace('.html', '')
if id.startswith('publisher-'):
publisher_name = id.replace('publisher-', '')
# Check the publisher actually exists, for security
publisher = model.Group.by_name(publisher_name)
if publisher:
c.report_name = id
else:
abort(404, 'Publisher not found')
self._set_openspending_reports_dir()
return render('data/openspending_publisher_report.html')
else:
abort(404)
def openspending_publisher_report(self, id):
id = id.replace('.html', '')
if id.startswith('publisher-'):
publisher_name = id.replace('publisher-', '')
# Check the publisher actually exists, for security
publisher = model.Group.by_name(publisher_name)
if publisher:
c.report_name = id
else:
abort(404, 'Publisher not found')
self._set_openspending_reports_dir()
return render('data/openspending_publisher_report.html')
else:
abort(404)
def report_groups_without_admins(self):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
try:
check_access('group_create', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
g_query = """SELECT g.* FROM public.group g WHERE id NOT IN
(SELECT group_id FROM public.member WHERE capacity='admin')
ORDER BY g.name;"""
c.non_admin = model.Session.query(model.Group).from_statement(g_query).all()
c.non_admin_count = len(c.non_admin)
return render('publisher/report_groups_without_admins.html')
def report_users_not_assigned_to_groups(self):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
try:
check_access('group_create', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
query = """SELECT * FROM public.user WHERE id NOT IN
(SELECT table_id FROM public.member WHERE table_name='user')
ORDER BY created desc;"""
c.unassigned_users = model.Session.query(model.User).from_statement(query).all()
c.unassigned_users_count = len(c.unassigned_users)
return render('publisher/report_users_not_assigned_to_groups.html')
def random_datasets(self):
import random
from routes import url_for
seed = request.params.get('seed')
sample_size = request.params.get('sample_size')
needs_redirect = False
if seed is None:
seed = random.randint(1000, 9999)
needs_redirect = True
if sample_size is None:
sample_size = 10
needs_redirect = True
if needs_redirect:
url_params = dict(
seed=seed,
**request.environ['pylons.routes_dict'])
if sample_size:
url_params['sample_size'] = sample_size
redirect(url_for(**url_params))
try:
seed = int(seed)
except:
abort(401, 'Bad seed')
random.seed(seed)
if sample_size:
try:
sample_size = int(sample_size)
except:
abort(401, 'Bad sample_size')
dataset_q = \
model.Session.query(model.Package.name).\
filter_by(state='active').\
order_by(model.Package.name)
num_datasets = dataset_q.count()
dataset_indexes = random.sample(range(num_datasets), sample_size)
dataset_names = dataset_q.all()
datasets = [dataset_names[i][0] for i in dataset_indexes]
else:
datasets = None
return render('data/random_datasets.html',
extra_vars=dict(sample_size=sample_size,
datasets=datasets,
next_seed=random.randint(1000, 9999)))
def report_users(self):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
try:
check_access('group_create', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
q = model.Session.query(model.User).order_by(model.User.created.desc())
c.count = q.count()
c.page = h.Page(
collection=q,
page=int(request.params.get('page', 1)),
url=h.pager_url,
items_per_page=report_limit,
)
return render('publisher/report_users.html')
def publisher_files(self, rel_path):
if not has_user_got_publisher_permissions():
abort(
401,
_('You need to be logged in as an editor or admin for a'
' publisher to see these files'))
path = self._get_path_relative_to_publisher_files_dir(rel_path)
if not os.path.exists(path):
abort(404, 'File not found')
# resolve symbolic link, so it downloads with the actual filename, not
# the link name
if os.path.islink(path):
path = os.path.realpath(path)
if os.path.isdir(path):
root, dirs, files = os.walk(path).next()
rel_path_with_trailing_slash = rel_path.strip('/') + '/'
dirs = [
dict(
name=dir_,
rel_path=rel_path_with_trailing_slash + dir_,
) for dir_ in dirs
]
if rel_path not in ('/', '', None):
dirs.insert(
0,
dict(name='..',
rel_path=os.path.dirname(rel_path.rstrip('/'))))
file_details = [
dict(
name=file,
rel_path=rel_path_with_trailing_slash + file,
size=os.path.getsize(root + '/' + file),
modified=time.ctime(os.path.getmtime(root + '/' + file)),
) for file in files
]
extra_vars = dict(dirs=dirs, file_details=file_details)
return render('data/publisher_files.html', extra_vars=extra_vars)
elif os.path.isfile(path):
return self._serve_file(path)
elif os.path.islink(path):
abort(401, 'Cannot use links')
else:
abort(401, 'Unknown path %s' % path)
def openspending_publisher_report(self, id):
id = id.replace('.html', '')
if id.startswith('publisher-'):
publisher_name = id.replace('publisher-', '')
# Check the publisher actually exists, for security
publisher = model.Group.by_name(publisher_name)
if publisher:
c.report_name = id
else:
abort(404, 'Publisher not found')
self._set_openspending_reports_dir()
path = c.openspending_report_dir + "/" + c.report_name + ".html"
c.content = open(path).read()
# Jinja needs it as unicode
c.content = c.content.decode('utf8')
return render('data/openspending_publisher_report.html')
else:
abort(404)
def openspending_publisher_report(self, id):
id = id.replace('.html', '')
if id.startswith('publisher-'):
publisher_name = id.replace('publisher-', '')
# Check the publisher actually exists, for security
publisher = model.Group.by_name(publisher_name)
if publisher:
c.report_name = id
else:
abort(404, 'Publisher not found')
self._set_openspending_reports_dir()
path = c.openspending_report_dir + "/" + c.report_name + ".html"
c.content = open(path).read()
# Jinja needs it as unicode
c.content = c.content.decode('utf8')
return render('data/openspending_publisher_report.html')
else:
abort(404)
def random_datasets(self):
import random
from routes import url_for
seed = request.params.get('seed')
sample_size = request.params.get('sample_size')
needs_redirect = False
if seed is None:
seed = random.randint(1000, 9999)
needs_redirect = True
if sample_size is None:
sample_size = 10
needs_redirect = True
if needs_redirect:
url_params = dict(seed=seed,
**request.environ['pylons.routes_dict'])
if sample_size:
url_params['sample_size'] = sample_size
redirect(url_for(**url_params))
try:
seed = int(seed)
except:
abort(401, 'Bad seed')
random.seed(seed)
if sample_size:
try:
sample_size = int(sample_size)
except:
abort(401, 'Bad sample_size')
dataset_q = \
model.Session.query(model.Package.name).\
filter_by(state='active').\
order_by(model.Package.name)
num_datasets = dataset_q.count()
dataset_indexes = random.sample(range(num_datasets), sample_size)
dataset_names = dataset_q.all()
datasets = [dataset_names[i][0] for i in dataset_indexes]
else:
datasets = None
return render('data/random_datasets.html',
extra_vars=dict(sample_size=sample_size,
datasets=datasets,
next_seed=random.randint(1000, 9999)))
def commitments(self, id):
"""
Shows all of the commitments for the specified publisher
"""
from ckanext.dgu.model.commitment import Commitment
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'save': 'save' in request.params}
try:
c.publisher_dict = get_action('organization_show')(context, {'id': id})
except NotAuthorized:
abort(401, _('Unauthorized to read commitments'))
except ObjectNotFound:
abort(404, _('Publisher not found'))
c.publisher = context.get('group')
c.commitments = Commitment.get_for_publisher(c.publisher.name).order_by('commitment.dataset_name').all()
return render('commitment/read.html')
def edit(self, id):
"""
Allows editing of commitments for a specific publisher
"""
from ckanext.dgu.model.commitment import Commitment
from ckanext.dgu.lib import helpers as dgu_helpers
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'save': 'save' in request.params}
if not dgu_helpers.is_sysadmin():
abort(401, "You are not allowed to edit the commitments for this publisher")
try:
c.publisher_dict = get_action('organization_show')(context, {'id': id})
except NotAuthorized:
abort(401, _('Unauthorized to read commitments'))
except ObjectNotFound:
abort(404, _('Publisher not found'))
c.publisher = context.get('group')
c.publishers = model.Session.query(model.Group).filter(model.Group.state=='active')\
.order_by(model.Group.title).all()
c.errors = {}
if request.method == "POST":
# need to flatten the request into some commitments, if there is an ID then
# we should update them, if there isn't then add them.
# TODO: Work out how to remove them. Perhaps get the current IDs and do a diff
# If successful redirect to read()
h.redirect_to(h.url_for(controller='ckanext.dgu.controllers.commitment:CommitmentController',
action='commitments', id=c.publisher.name))
c.commitments = Commitment.get_for_publisher(c.publisher.name).order_by('commitment.commitment_text')
return render('commitment/edit.html')
def named_theme(self, name):
"""
Shows the theme home page for a specified theme containing information relevant
to *just* that theme, popular and new datasets as well as recent apps+ideas from
the front end.
"""
log.warning('Cannot view theme pages yet.')
abort(404)
c.theme = c.theme_name = name
if not c.theme_name:
abort(404)
c.dataset_count = model.Session.query(model.Package)\
.join(model.PackageExtra)\
.filter(model.PackageExtra.key=='theme-primary')\
.filter(model.PackageExtra.value==c.theme_name)\
.filter(model.Package.state=='active').count()
c.latest = self._search(theme=c.theme_name)
c.popular = self._search(theme=c.theme_name, sort_string='popularity asc')
return render('themed/theme.html')
def named_theme(self, name):
"""
Shows the theme home page for a specified theme containing information relevant
to *just* that theme, popular and new datasets as well as recent apps+ideas from
the front end.
"""
raise NotImplementedError('Cannot view theme pages yet.')
c.theme = c.theme_name = name
if not c.theme_name:
abort(404)
c.dataset_count = model.Session.query(model.Package)\
.join(model.PackageExtra)\
.filter(model.PackageExtra.key=='theme-primary')\
.filter(model.PackageExtra.value==c.theme_name)\
.filter(model.Package.state=='active').count()
c.latest = self._search(theme=c.theme_name)
c.popular = self._search(theme=c.theme_name,
sort_string='popularity asc')
return render('themed/theme.html')
def home(self):
extra_vars = {}
# Get the dataset count using search
# (shouldn't cache, as it makes it more likely to be out of sync with
# the data page)
from ckan.lib.search import SearchError, SearchQueryError
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'auth_user_obj': c.userobj}
data_dict = {
'q': '*:*',
'fq': '+dataset_type:dataset',
#'facet.field': facets.keys(),
'rows': 0,
}
query = get_action('package_search')(context, data_dict)
extra_vars['num_datasets'] = query['count']
extra_vars['themes'] = get_themes()
return render('data/home.html', extra_vars=extra_vars)
def publisher_files(self, rel_path):
if not has_user_got_publisher_permissions():
abort(401, _('You need to be logged in as an editor or admin for a'
' publisher to see these files'))
path = self._get_path_relative_to_publisher_files_dir(rel_path)
if not os.path.exists(path):
abort(404, 'File not found')
# resolve symbolic link, so it downloads with the actual filename, not
# the link name
if os.path.islink(path):
path = os.path.realpath(path)
if os.path.isdir(path):
root, dirs, files = os.walk(path).next()
rel_path_with_trailing_slash = rel_path.strip('/') + '/'
dirs = [
dict(name=dir_,
rel_path=rel_path_with_trailing_slash + dir_,
)
for dir_ in dirs]
if rel_path not in ('/', '', None):
dirs.insert(0, dict(
name='..', rel_path=os.path.dirname(rel_path.rstrip('/'))))
file_details = [
dict(name=file,
rel_path=rel_path_with_trailing_slash + file,
size=os.path.getsize(root + '/' + file),
modified=time.ctime(os.path.getmtime(root + '/' + file)),
)
for file in files]
extra_vars = dict(dirs=dirs, file_details=file_details)
return render('data/publisher_files.html', extra_vars=extra_vars)
elif os.path.isfile(path):
return self._serve_file(path)
elif os.path.islink(path):
abort(401, 'Cannot use links')
else:
abort(401, 'Unknown path %s' % path)
def openspending_report(self):
self._set_openspending_reports_dir()
return render('data/openspending_report.html')
def resource_cache(self, root, resource_id, filename):
"""
Called when a request is made for an item in the resource cache and
is responsible for rendering the data. When the data to be rendered
is HTML it will add a header to show that the content is cached, and
set a <base> header if not present to make sure all relative links are
resolved correctly.
"""
from pylons import response
from paste.fileapp import FileApp
from ckanext.dgu.lib.helpers import tidy_url
from ckanext.qa.model import QA
archive_root = pylons.config.get('ckanext-archiver.archive_dir')
if not archive_root:
# Bad configuration likely to cause this.
abort(404, "Could not find archive folder")
resource = model.Resource.get(resource_id)
fmt = ""
if resource:
qa = QA.get_for_resource(resource.id)
if qa:
fmt = qa.format
is_html = fmt == "HTML"
filepath = os.path.join(archive_root, root, resource_id,
filename).encode('utf-8')
filepath = urllib.quote(filepath)
if not os.path.exists(filepath):
abort(404, "Resource is not cached")
file_size = os.path.getsize(filepath)
if not is_html:
# Content-Type is determined by FileApp based on the extension.
# Using the format provided by QA isn't an option currently as
# for zip files it gives the format of the content of the zip.
headers = [('Content-Length', str(file_size))]
fapp = FileApp(filepath, headers=headers)
return fapp(request.environ, self.start_response)
origin = tidy_url(resource.url)
parts = urlparse.urlparse(origin)
url = "{0}://{1}".format(parts.scheme, parts.netloc)
base_string = "<head><base href='{0}'>".format(url)
response.headers['Content-Type'] = 'text/html; charset=utf-8'
try:
f = open(filepath, "r")
except IOError:
log.error('Error reading resource cache file: %s', filepath)
abort(
403,
"The system was unable to read this resource from the cache. Admins have been notified"
)
content = f.read()
f.close()
if not re.search("<base ", content, re.IGNORECASE):
compiled_head = re.compile(re.escape("<head>"), re.IGNORECASE)
content = compiled_head.sub(base_string, content, re.IGNORECASE)
if not '__archiver__cache__header__' in content:
# We should insert our HTML block at the bottom of the page with
# the appropriate CSS to render it at the top. Easier to insert
# before </body>.
c.url = resource.url
replacement = render("data/cache_header.html")
try:
compiled_body = re.compile(re.escape("</body>"), re.IGNORECASE)
content = compiled_body.sub("{0}</body>".format(replacement),
content, re.IGNORECASE)
except Exception, e:
log.warn(
"Failed to do the replacement in resource<{0}> and file: {1}"
.format(resource.id, filepath))
return
def system_dashboard(self):
is_sysadmin = ckan.authz.Authorizer().is_sysadmin(c.userobj) if c.userobj else False
if not is_sysadmin:
abort(401, 'User must be a sysadmin to view this page.')
return render('data/system_dashboard.html')
def openspending_browse(self):
return render('data/openspending_browse.html')
def resource_cache(self, root, resource_id, filename):
"""
Called when a request is made for an item in the resource cache and
is responsible for rendering the data. When the data to be rendered
is HTML it will add a header to show that the content is cached, and
set a <base> header if not present to make sure all relative links are
resolved correctly.
"""
from pylons import response
from paste.fileapp import FileApp
from ckanext.dgu.lib.helpers import tidy_url
archive_root = pylons.config.get('ckanext-archiver.archive_dir')
if not archive_root:
# Bad configuration likely to cause this.
abort(404, "Could not find archive folder")
resource = model.Resource.get(resource_id)
is_html = False
content_type = "application/octet-stream"
fmt = ""
if resource:
task_status = model.Session.query(model.TaskStatus).\
filter(model.TaskStatus.task_type=='qa').\
filter(model.TaskStatus.key=='status').\
filter(model.TaskStatus.entity_id==resource.id).first()
if task_status:
status = json.loads(task_status.error)
fmt = status['format']
# Make an attempt at getting the correct content type but fail with
# application/octet-stream in cases where we don't know.
formats = {
"CSV": "application/csv",
"XLS": "application/vnd.ms-excel",
"HTML": 'text/html; charset=utf-8' }
content_type = formats.get(fmt, "application/octet-stream")
is_html = fmt == "HTML"
filepath = os.path.join(archive_root, root, resource_id, filename).encode('utf-8')
if not os.path.exists(filepath):
abort(404, "Resource is not cached")
file_size = os.path.getsize(filepath)
if not is_html:
headers = [('Content-Type', content_type),
('Content-Length', str(file_size))]
fapp = FileApp(filepath, headers=headers)
return fapp(request.environ, self.start_response)
origin = tidy_url(resource.url)
parts = urlparse.urlparse(origin)
url = "{0}://{1}".format(parts.scheme, parts.netloc)
base_string = "<head><base href='{0}'>".format(url)
response.headers['Content-Type'] = content_type
try:
f = open(filepath, "r")
except IOError:
log.error('Error reading resource cache file: %s', filepath)
abort(403, "The system was unable to read this resource from the cache. Admins have been notified")
content = f.read()
f.close()
if not re.search("<base ", content, re.IGNORECASE):
compiled_head = re.compile(re.escape("<head>"), re.IGNORECASE)
content = compiled_head.sub( base_string, content, re.IGNORECASE)
if not '__archiver__cache__header__' in content:
# We should insert our HTML block at the bottom of the page with
# the appropriate CSS to render it at the top. Easier to insert
# before </body>.
c.url = resource.url
replacement = render("data/cache_header.html")
try:
compiled_body = re.compile(re.escape("</body>"), re.IGNORECASE)
content = compiled_body.sub( "{0}</body>".format(replacement), content, re.IGNORECASE)
except Exception, e:
log.error("Failed to do the replacement in resource<{0}> and file: {1}".format(resource.id, filepath))
return
def api(self):
return render('data/api.html')
def api(self):
return render('data/api.html')
def carparks(self):
return render('data/carparks.html')
def system_dashboard(self):
if not dgu_helpers.is_sysadmin():
abort(401, 'User must be a sysadmin to view this page.')
return render('data/system_dashboard.html')
def viz_upload(self):
"""
Provides direct upload to DGU for users in specific publishers.
This is specifically for the social investment publishers so that
they can host their files somewhere.
"""
import ckan.model as model
ALLOWED_PUBLISHERS = set([
'seedbed'
'health-social-ventures',
'big-issue-cooperate',
'social-incubator-east',
'young-academy',
'dotforge-social-ventures',
'wayra-unltd',
'social-incubator-north',
'bethnal-green-ventures',
'hub-launchpad',
'the-social-investment-business-group',
])
# Check auth .. user must be in one of selected groups.
context = {"model": model, "session": model.Session, "user": c.user}
if dgu_helpers.is_sysadmin():
user_orgs = ALLOWED_PUBLISHERS
else:
res = get_action("organization_list_for_user")(
context, {
"permission": "create_dataset"
})
user_orgs = set([o['name'] for o in res]) & ALLOWED_PUBLISHERS
if not user_orgs:
abort(401)
publisher_q = " OR ".join(
["publisher:{}".format(o) for o in user_orgs])
res = get_action("package_search")(context, {
"fq": "({})".format(publisher_q),
'rows': 100
})
c.package_names = [(
p['name'],
u"{} ({})".format(p['title'], p['organization']['title']),
) for p in res['results']]
if not c.package_names:
flash_error(
"There are no datasets available in your organisation. You will be unable to upload."
)
return render('viz/upload.html')
c.package_names = sorted(c.package_names)
if request.method == 'POST':
success, errors = self._validate_viz_upload(request.POST)
c.title = request.POST.get('title', '')
c.format = request.POST.get('format', '')
c.dataset = request.POST.get('dataset', '')
if not success:
error_list = "<li>" + "</li><li>".join(errors) + "</li>"
flash_error(
'There was a problem with your submission<br/><ul>{}</ul>'.
format(error_list),
allow_html=True)
return render('viz/upload.html')
extension, url = self._store_file(request.POST['upload'],
c.dataset)
if not c.format:
c.format = extension[1:].upper()
# Create resource
resource = {
u'description': c.title,
u'url': url,
u'format': c.format
}
# Show and update package
pkg = get_action('package_show')(context, {'id': c.dataset})
pkg['resources'].append(resource)
res = get_action('package_update')(context, pkg)
log.info('Added viz to dataset %s: %s', pkg['name'], resource)
flash_success(
'The file has been uploaded and added to the dataset. <a href="/dataset/{}" style="text-decoration:underline;">View dataset</a>'
.format(c.dataset),
allow_html=True)
return redirect('/data/viz/upload')
return render('viz/upload.html')
def openspending_report(self):
self._set_openspending_reports_dir()
c.content = open(c.openspending_report_dir + "/index.html").read()
return render('data/openspending_report.html')
def viz_social_investment_and_foundations(self):
return render('viz/social_investment_and_foundations.html')
def viz_investment_readiness_programme(self):
return render('viz/investment_readiness_programme.html')
