def clear_key(self, filepath):
'''Deletes the contents of the key at `filepath` on `self.bucket`.'''
try:
self.get_s3_resource().Object(self.bucket_name, filepath).delete()
log.info("Removed %s from S3", filepath)
if self.signed_url_cache_enabled:
connect_to_redis().delete(_get_cache_key(filepath))
except Exception as e:
raise e
def plugin_edit_clear_settings_cache(entity):
if config.get(
u'datacity.settings_group_id') and entity.type == "settings" and (
entity.name == config[u'datacity.settings_group_id']
or entity.id == config[u'datacity.settings_group_id']):
conn = connect_to_redis()
key = "%s:%s" % (SETTINGS_REDIS_KEY_PREFIX, config[u'ckan.site_id'])
conn.delete(key)
conn.delete(HOMEPAGE_POPULAR_DATASETS_REDIS_KEY)
conn.delete(HOMEPAGE_LAST_UPDATED_DATASETS_REDIS_KEY)
elif entity.type == "group" or entity.type == "dataset":
conn = connect_to_redis()
for key in conn.keys("ckanext:datacity:homepage:*"):
conn.delete(key)
def _check_new_user_quota():
redis_conn = connect_to_redis()
new_users_list = 'new_latest_users'
if 'new_latest_users' not in redis_conn.keys():
redis_conn.lpush(new_users_list, datetime.now().isoformat())
else:
# TODO: read this rom config
max_new_users = 10
period = 60 * 10
begin_date = datetime.now() - timedelta(seconds=period)
count = 0
elements_to_remove = []
for i in range(0, redis_conn.llen(new_users_list)):
value = redis_conn.lindex(new_users_list, i)
new_user_creation_date = dateutil.parser.parse(value)
if new_user_creation_date >= begin_date:
count += 1
else:
elements_to_remove += [value]
for value in elements_to_remove:
redis_conn.lrem(new_users_list, value)
if count >= max_new_users:
log.error("new user temporary quota exceeded ({0})".format(count))
raise logic.ValidationError({
'user':
"******"
.format(period / 60)
})
else:
# add new user creation
redis_conn.lpush(new_users_list, datetime.now().isoformat())
def get_setting(setting, default=None):
value = None
if config.get(u'datacity.settings_group_id'):
conn = connect_to_redis()
key = "%s:%s" % (SETTINGS_REDIS_KEY_PREFIX, config[u'ckan.site_id'])
raw_value = conn.get(key)
if raw_value is None:
try:
value = get_action("group_show")(
data_dict={
"id": config[u'datacity.settings_group_id'],
"include_extras": True,
"include_dataset_count": False,
"include_users": False,
"include_groups": False,
"include_tags": False,
"include_followers": False
})
except NotFound:
value = {}
conn.set(key, json.dumps(value))
value = value.get(setting)
else:
value = json.loads(raw_value).get(setting)
if value:
value = unicode(value).strip()
if not value and default != None:
return default
else:
return value
def authenticate(self, environ, identity):
""" Mimic most of UsernamePasswordAuthenticator.authenticate
but add account lockout after 10 failed attempts.
"""
if 'login' not in identity or 'password' not in identity:
return None
login_name = identity.get('login')
user = User.by_name(login_name)
if user is None:
LOG.debug('Login failed - username %r not found', login_name)
return None
cache_key = '{}.ckanext.qgov.login_attempts.{}'.format(
g.site_id, login_name)
redis_conn = connect_to_redis()
try:
login_attempts = int(redis_conn.get(cache_key) or 0)
except ValueError:
# shouldn't happen but let's play it safe
login_attempts = 0
if login_attempts >= 10:
LOG.debug('Login as %r failed - account is locked', login_name)
elif user.validate_password(identity.get('password')):
if login_attempts > 0:
LOG.debug("Clearing failed login attempts for %s", login_name)
# reset attempt count to 0
redis_conn.delete(cache_key)
return user.name
else:
LOG.debug('Login as %r failed - password not valid', login_name)
redis_conn.set(cache_key, login_attempts + 1, ex=LOGIN_THROTTLE_EXPIRY)
return None
def _check_reset_attempts(email):
redis_conn = connect_to_redis()
if email not in redis_conn.keys():
log.debug("Redis: first login attempt for {0}".format(email))
redis_conn.hmset(email, {
'attempts': 1,
'latest': datetime.now().isoformat()
})
else:
base = 3
attempts = int(redis_conn.hmget(email, 'attempts')[0])
latest = dateutil.parser.parse(redis_conn.hmget(email, 'latest')[0])
waiting_seconds = base**attempts
limit_date = latest + timedelta(seconds=waiting_seconds)
log.debug(
'Redis: wait {0} seconds after {1} attempts => after date {2}'.
format(waiting_seconds, attempts, limit_date.isoformat()))
if limit_date > datetime.now():
raise logic.ValidationError({
'user':
"******"
.format(int((limit_date - datetime.now()).total_seconds()),
limit_date.isoformat())
})
else:
# increase counter
redis_conn.hmset(email, {
'attempts': attempts + 1,
'latest': datetime.now().isoformat()
})
def _save_token_data(token_data):
"""
Save the token data to redis.
"""
expires_in = int(token_data['expiry_timestamp'] - time.time())
redis = connect_to_redis()
key = 'oidc_token_data:' + token_data['user_id']
redis.setex(key, json.dumps(token_data), expires_in)
def __init__(self, worker_id, heartbeat_interval=60000, redis=None):
self.worker_id = worker_id
self.logger = getLogger('ckanext.Worker:' + worker_id)
self._redis = redis or connect_to_redis()
self.heartbeat_interval = heartbeat_interval
self.running = False
self._hb_thread = None
self._hb_thread_stop = Event()
def _save_token(user_id, token):
"""
Save a user's auth token to Redis, with the expiry time specified within the token.
"""
expires_in = token.get('expires_in', 300)
redis = connect_to_redis()
key = 'oidc_token:' + user_id
redis.setex(key, json.dumps(token), expires_in)
def _save_state(state):
"""
Save a state string, used for verifying an OAuth2 login callback, to Redis,
with an expiry time of 5 minutes.
"""
redis = connect_to_redis()
key = 'oidc_state:' + state
redis.setex(key, '', 300)
def _wrapper():
conn = connect_to_redis()
raw_value = conn.get(key)
if raw_value is None:
value = get_value()
conn.set(key, json.dumps(value), ex=key_ex)
return value
else:
return json.loads(raw_value)
def _verify_state(state):
"""
Check that the state string provided with a callback matches one that was sent
to the auth server.
"""
redis = connect_to_redis()
key = 'oidc_state:' + state
if key not in redis:
raise OpenIDConnectError(_("Invalid authorization state"))
def all_jobs(self):
u'''
Get a list of all RQ jobs.
'''
jobs = []
redis_conn = connect_to_redis()
for queue in rq.Queue.all(connection=redis_conn):
jobs.extend(queue.jobs)
return jobs
def upload_to_key(self, filepath, upload_file, make_public=False):
'''Uploads the `upload_file` to `filepath` on `self.bucket`.'''
upload_file.seek(0)
log.debug(
"ckanext.s3filestore.uploader: going to upload %s to bucket %s with mimetype %s",
filepath, self.bucket_name, getattr(self, 'mimetype', None))
try:
self.get_s3_resource().Object(self.bucket_name, filepath).put(
Body=upload_file.read(),
ACL=self.acl,
ContentType=getattr(self, 'mimetype', None))
log.info("Successfully uploaded %s to S3!", filepath)
if self.signed_url_cache_enabled:
connect_to_redis().delete(_get_cache_key(filepath))
except Exception as e:
log.error('Something went very very wrong for %s', str(e))
raise e
def _connect():
u'''
Connect to Redis and tell RQ about it.
Workaround for https://github.com/nvie/rq/issues/479.
'''
conn = connect_to_redis()
push_connection(conn)
return conn
def all_jobs(self):
u"""
Get a list of all RQ jobs.
"""
jobs = []
redis_conn = connect_to_redis()
for queue in rq.Queue.all(connection=redis_conn):
jobs.extend(queue.jobs)
return jobs
def setup(self):
u'''
Delete all RQ queues and jobs.
'''
# See https://github.com/nvie/rq/issues/731
redis_conn = connect_to_redis()
for queue in rq.Queue.all(connection=redis_conn):
queue.empty()
redis_conn.srem(rq.Queue.redis_queues_keys, queue._key)
redis_conn.delete(queue._key)
def setup(self):
u"""
Delete all RQ queues and jobs.
"""
# See https://github.com/nvie/rq/issues/731
redis_conn = connect_to_redis()
for queue in rq.Queue.all(connection=redis_conn):
queue.empty()
redis_conn.srem(rq.Queue.redis_queues_keys, queue._key)
redis_conn.delete(queue._key)
def _load_token(user_id):
"""
Retrieve a user's auth token from Redis.
"""
if not user_id:
return {}
redis = connect_to_redis()
key = 'oidc_token:' + user_id
token = redis.get(key) or '{}'
token = json.loads(token)
return token
def logged_in(self):
""" Provide a custom error code when login fails due to account lockout.
"""
if not c.user:
# a number of failed login attempts greater than 10 indicates
# that the locked user is associated with the current request
redis_conn = connect_to_redis()
for key in redis_conn.keys('{}.ckanext.qgov.login_attempts.*'.format(g.site_id)):
login_attempts = redis_conn.get(key)
if login_attempts > 10:
redis_conn.set(key, 10, ex=LOGIN_THROTTLE_EXPIRY)
return self.login('account-locked')
return LOGGED_IN(self)
def unlock_account(account_id):
""" Unlock an account (erase the failed login attempts).
"""
qgov_user = Session.query(User).filter(User.id == account_id).first()
if qgov_user:
login_name = qgov_user.name
cache_key = '{}.ckanext.qgov.login_attempts.{}'.format(
g.site_id, login_name)
redis_conn = connect_to_redis()
if redis_conn.get(cache_key):
LOG.debug("Clearing failed login attempts for %s", login_name)
redis_conn.delete(cache_key)
else:
LOG.debug("Account %s not found", account_id)
def cacheable_wrapper(*args, **kwargs):
expiry = cacheable_kwargs.get("expiry", 100)
site_id = config.get("ckan.site_id", "default")
lang = h.lang()
cache_key = cacheable_kwargs.get(
"key",
"CACHE:" + site_id + "::" + lang + "::" + cacheable_func.__name__)
conn = connect_to_redis()
if conn.exists(cache_key):
return json.loads(conn.get(cache_key))
value = cacheable_func(*args, **kwargs)
conn.setex(name=cache_key, value=json.dumps(value), time=expiry)
return value
def wrapper(*args, **kwargs):
conn = redis.connect_to_redis()
key = pickle.dumps((args, kwargs))
value = conn.get(key)
if value:
return cast(T, json.loads(value))
value = func(*args, **kwargs)
cache_duration = tk.asint(
tk.config.get(CONFIG_CACHE_DURATION, DEFAULT_CACHE_DURATION))
if isinstance(value, DontCache):
value = cast(T, value.unwrap())
conn.set(key, json.dumps(value), ex=cache_duration)
return value
def get_signed_url_to_key(self, key, extra_params={}):
'''Generates a pre-signed URL giving access to an S3 object.
If a download_proxy is configured, then the URL will be
generated using the true S3 host, and then the hostname will be
rewritten afterward. Note that the Host header is part of a
version 4 signature, so the resulting request, as it stands,
will fail signature verification; the download_proxy server must
be configured to set the Host header back to the true value when
forwarding the request (CloudFront does this automatically).
'''
client = self.get_s3_client()
# check whether the object exists in S3
metadata = client.head_object(Bucket=self.bucket_name, Key=key)
if self.signed_url_cache_enabled:
redis_conn = connect_to_redis()
cache_key = _get_cache_key(key)
cache_url = redis_conn.get(cache_key)
if cache_url:
log.debug('Returning cached URL for path %s', key)
return cache_url
else:
log.debug('No cache found for %s; generating a new URL', key)
params = {'Bucket': self.bucket_name, 'Key': key}
if metadata['ContentType'] != 'application/pdf':
filename = key.split('/')[-1]
params[
'ResponseContentDisposition'] = 'attachment; filename=' + filename
params.update(extra_params)
url = client.generate_presigned_url(ClientMethod='get_object',
Params=params,
ExpiresIn=self.signed_url_expiry)
if self.download_proxy:
url = URL_HOST.sub(self.download_proxy + '/', url, 1)
if self.signed_url_cache_enabled:
redis_conn.set(cache_key, url, ex=self.signed_url_cache_window)
return url
def check_privs(
context,
require_admin=False,
require_curator=False,
require_harvester=False,
require_contributor=False,
require_organization=None,
):
"""
Check whether the user has the specified privileges.
This is done by looking up the info directly in Redis, which gets written by
ckanext/accesscontrol/logic/openidconnect.py (see functions _extract_token_data
and _save_token_data for details).
Note: this is not the usual CKAN way of doing things; it effectively makes this extension
dependent on ckanext-accesscontrol. At this point, however, it's the simplest means of
implementing role based access control.
Roles are cumulative, i.e. a given role can do everything that any lower role can do.
admin > curator > harvester > contributor > member
:param require_admin: the user must have the administrator role in the admin organization
:param require_curator: the user must have the curator role either in the admin organization or the specified require_organization
:param require_harvester: the user must have the harvester role in the specified require_organization
:param require_contributor: the user must have the contributor role in the specified require_organization
:param require_organization: the organization (id or name) associated with the resource being requested or updated
:return: bool
"""
admin_org = config.get('ckan.metadata.admin_org')
admin_role = config.get('ckan.metadata.admin_role')
curator_role = config.get('ckan.metadata.curator_role')
harvester_role = config.get('ckan.metadata.harvester_role')
contributor_role = config.get('ckan.metadata.contributor_role')
model = context['model']
user = context['user']
user_id = model.User.by_name(user.decode('utf8')).id
redis = connect_to_redis()
key = 'oidc_token_data:' + user_id
token_json = redis.get(key)
if not token_json:
return False
token_data = json.loads(token_json)
if token_data['superuser']:
return True
is_admin = False
is_curator = False
is_harvester = False
is_contributor = False
is_member = False
if require_organization:
require_organization = model.Group.get(require_organization).name
for privilege in token_data['privileges']:
if privilege['institution'] == admin_org and privilege[
'role'] == admin_role:
is_admin = True
if privilege['institution'] in (
admin_org,
require_organization) and privilege['role'] == curator_role:
is_curator = True
if privilege['institution'] == require_organization and privilege[
'role'] == harvester_role:
is_harvester = True
if privilege['institution'] == require_organization and privilege[
'role'] == contributor_role:
is_contributor = True
if privilege['institution'] == require_organization:
is_member = True
if require_admin:
return is_admin
if require_curator:
return is_admin or is_curator
if require_harvester:
return is_admin or is_curator or is_harvester
if require_contributor:
return is_admin or is_curator or is_harvester or is_contributor
if require_organization:
return is_admin or is_curator or is_harvester or is_contributor or is_member
return True
'validation', 'transformations', 'dcat', 'solr',
'properties_to_remove'
]
contents = json.load(config_file)
redis_conn.set(cache_key, current_date)
for redis_config_key in config_keys:
redis_conn.set(config_key + redis_config_key,
json.dumps(contents.get(redis_config_key)))
[
redis_conn.set(
redis_key + 'vocabulary.{0}'.format(key),
json.dumps(_load_list(key, voc['local'], 'vocabulary'))) for
key, voc in contents.get('validation')['vocabularies'].iteritems()
]
[
redis_conn.set(
redis_key + 'taxonomy.{0}'.format(key),
json.dumps(_load_list(key, tax['local'], 'taxonomy'))) for key,
tax in contents.get('validation')['taxonomies'].iteritems()
]
redis_key = 'ckanext.dataoverheid:'
config_key = redis_key + 'config.'
redis_conn = connect_to_redis()
_load_config_file()
def _login(context, data_dict):
if toolkit.c.user:
# Don't offer the reset form if already logged in
log.warning("User already logged in")
raise toolkit.NotAuthorized('user already logged in, logout first')
# Check if parameters are present
try:
user_id = data_dict.get('id')
if not user_id:
email = data_dict['email'].lower()
# Check email is valid
if not util.check_email(email):
raise toolkit.ValidationError({'email': 'invalid email'})
# get the user id
user_id = util.get_user_id(email)
if not user_id:
raise toolkit.ValidationError({
'email':
'email does not correspond to a registered user'
})
except KeyError:
raise toolkit.ValidationError({'email': 'missing email'})
try:
orig_key = data_dict['key']
except KeyError:
raise toolkit.ValidationError({'key': 'missing token'})
if len(orig_key) <= 32 and not orig_key.startswith("b'"):
key = "b'{0}'".format(orig_key)
else:
key = orig_key
log.debug('login: {0} ({1}) => {2}'.format(user_id, orig_key, key))
# get whether to return context (UI) or just a message (API)
return_context = data_dict.get('return_context', False)
try:
data_dict = {'id': user_id}
user_dict = logic.get_action('user_show')(context, data_dict)
user_obj = context['user_obj']
email = user_dict.get('email', user_obj.email)
except logic.NotFound:
raise logic.NotFound('"%s" matched several users' % user_id)
except toolkit.NotAuthorized:
raise toolkit.NotAuthorized('Exception (Not Authorized) email = ' +
str(email) + 'id = ' + str(user_id))
if not user_obj or not mailer.verify_reset_link(user_obj, key):
raise toolkit.ValidationError({'key': 'token provided is not valid'})
flask.session['ckanext-passwordless-user'] = user_dict['name']
# remove token
mailer.create_reset_key(user_obj)
# log the user in programmatically
try:
_set_repoze_user_only(user_dict['name'])
except TypeError as e:
log.warning("Exception at login: {0}".format(e))
# delete attempts from Redis
log.debug("Redis: reset attempts for {0}".format(email))
redis_conn = connect_to_redis()
redis_conn.delete(email)
# make sure the master API key exists
apikey = util.renew_master_token(user_dict['name'])
# return message or context
if return_context:
return context
else:
user_obj = context.get('user_obj', None)
result_json = {
'user': {
'email': user_obj.email,
'id': user_obj.id,
'name': user_obj.name,
'apikey': apikey,
'fullname': user_obj.fullname
},
'message': "login success"
}
return result_json
