def clear_key(self, filepath): '''Deletes the contents of the key at `filepath` on `self.bucket`.''' try: self.get_s3_resource().Object(self.bucket_name, filepath).delete() log.info("Removed %s from S3", filepath) if self.signed_url_cache_enabled: connect_to_redis().delete(_get_cache_key(filepath)) except Exception as e: raise e
def plugin_edit_clear_settings_cache(entity): if config.get( u'datacity.settings_group_id') and entity.type == "settings" and ( entity.name == config[u'datacity.settings_group_id'] or entity.id == config[u'datacity.settings_group_id']): conn = connect_to_redis() key = "%s:%s" % (SETTINGS_REDIS_KEY_PREFIX, config[u'ckan.site_id']) conn.delete(key) conn.delete(HOMEPAGE_POPULAR_DATASETS_REDIS_KEY) conn.delete(HOMEPAGE_LAST_UPDATED_DATASETS_REDIS_KEY) elif entity.type == "group" or entity.type == "dataset": conn = connect_to_redis() for key in conn.keys("ckanext:datacity:homepage:*"): conn.delete(key)
def _check_new_user_quota(): redis_conn = connect_to_redis() new_users_list = 'new_latest_users' if 'new_latest_users' not in redis_conn.keys(): redis_conn.lpush(new_users_list, datetime.now().isoformat()) else: # TODO: read this rom config max_new_users = 10 period = 60 * 10 begin_date = datetime.now() - timedelta(seconds=period) count = 0 elements_to_remove = [] for i in range(0, redis_conn.llen(new_users_list)): value = redis_conn.lindex(new_users_list, i) new_user_creation_date = dateutil.parser.parse(value) if new_user_creation_date >= begin_date: count += 1 else: elements_to_remove += [value] for value in elements_to_remove: redis_conn.lrem(new_users_list, value) if count >= max_new_users: log.error("new user temporary quota exceeded ({0})".format(count)) raise logic.ValidationError({ 'user': "******" .format(period / 60) }) else: # add new user creation redis_conn.lpush(new_users_list, datetime.now().isoformat())
def get_setting(setting, default=None): value = None if config.get(u'datacity.settings_group_id'): conn = connect_to_redis() key = "%s:%s" % (SETTINGS_REDIS_KEY_PREFIX, config[u'ckan.site_id']) raw_value = conn.get(key) if raw_value is None: try: value = get_action("group_show")( data_dict={ "id": config[u'datacity.settings_group_id'], "include_extras": True, "include_dataset_count": False, "include_users": False, "include_groups": False, "include_tags": False, "include_followers": False }) except NotFound: value = {} conn.set(key, json.dumps(value)) value = value.get(setting) else: value = json.loads(raw_value).get(setting) if value: value = unicode(value).strip() if not value and default != None: return default else: return value
def authenticate(self, environ, identity): """ Mimic most of UsernamePasswordAuthenticator.authenticate but add account lockout after 10 failed attempts. """ if 'login' not in identity or 'password' not in identity: return None login_name = identity.get('login') user = User.by_name(login_name) if user is None: LOG.debug('Login failed - username %r not found', login_name) return None cache_key = '{}.ckanext.qgov.login_attempts.{}'.format( g.site_id, login_name) redis_conn = connect_to_redis() try: login_attempts = int(redis_conn.get(cache_key) or 0) except ValueError: # shouldn't happen but let's play it safe login_attempts = 0 if login_attempts >= 10: LOG.debug('Login as %r failed - account is locked', login_name) elif user.validate_password(identity.get('password')): if login_attempts > 0: LOG.debug("Clearing failed login attempts for %s", login_name) # reset attempt count to 0 redis_conn.delete(cache_key) return user.name else: LOG.debug('Login as %r failed - password not valid', login_name) redis_conn.set(cache_key, login_attempts + 1, ex=LOGIN_THROTTLE_EXPIRY) return None
def _check_reset_attempts(email): redis_conn = connect_to_redis() if email not in redis_conn.keys(): log.debug("Redis: first login attempt for {0}".format(email)) redis_conn.hmset(email, { 'attempts': 1, 'latest': datetime.now().isoformat() }) else: base = 3 attempts = int(redis_conn.hmget(email, 'attempts')[0]) latest = dateutil.parser.parse(redis_conn.hmget(email, 'latest')[0]) waiting_seconds = base**attempts limit_date = latest + timedelta(seconds=waiting_seconds) log.debug( 'Redis: wait {0} seconds after {1} attempts => after date {2}'. format(waiting_seconds, attempts, limit_date.isoformat())) if limit_date > datetime.now(): raise logic.ValidationError({ 'user': "******" .format(int((limit_date - datetime.now()).total_seconds()), limit_date.isoformat()) }) else: # increase counter redis_conn.hmset(email, { 'attempts': attempts + 1, 'latest': datetime.now().isoformat() })
def _save_token_data(token_data): """ Save the token data to redis. """ expires_in = int(token_data['expiry_timestamp'] - time.time()) redis = connect_to_redis() key = 'oidc_token_data:' + token_data['user_id'] redis.setex(key, json.dumps(token_data), expires_in)
def __init__(self, worker_id, heartbeat_interval=60000, redis=None): self.worker_id = worker_id self.logger = getLogger('ckanext.Worker:' + worker_id) self._redis = redis or connect_to_redis() self.heartbeat_interval = heartbeat_interval self.running = False self._hb_thread = None self._hb_thread_stop = Event()
def _save_token(user_id, token): """ Save a user's auth token to Redis, with the expiry time specified within the token. """ expires_in = token.get('expires_in', 300) redis = connect_to_redis() key = 'oidc_token:' + user_id redis.setex(key, json.dumps(token), expires_in)
def _save_state(state): """ Save a state string, used for verifying an OAuth2 login callback, to Redis, with an expiry time of 5 minutes. """ redis = connect_to_redis() key = 'oidc_state:' + state redis.setex(key, '', 300)
def _wrapper(): conn = connect_to_redis() raw_value = conn.get(key) if raw_value is None: value = get_value() conn.set(key, json.dumps(value), ex=key_ex) return value else: return json.loads(raw_value)
def _verify_state(state): """ Check that the state string provided with a callback matches one that was sent to the auth server. """ redis = connect_to_redis() key = 'oidc_state:' + state if key not in redis: raise OpenIDConnectError(_("Invalid authorization state"))
def all_jobs(self): u''' Get a list of all RQ jobs. ''' jobs = [] redis_conn = connect_to_redis() for queue in rq.Queue.all(connection=redis_conn): jobs.extend(queue.jobs) return jobs
def upload_to_key(self, filepath, upload_file, make_public=False): '''Uploads the `upload_file` to `filepath` on `self.bucket`.''' upload_file.seek(0) log.debug( "ckanext.s3filestore.uploader: going to upload %s to bucket %s with mimetype %s", filepath, self.bucket_name, getattr(self, 'mimetype', None)) try: self.get_s3_resource().Object(self.bucket_name, filepath).put( Body=upload_file.read(), ACL=self.acl, ContentType=getattr(self, 'mimetype', None)) log.info("Successfully uploaded %s to S3!", filepath) if self.signed_url_cache_enabled: connect_to_redis().delete(_get_cache_key(filepath)) except Exception as e: log.error('Something went very very wrong for %s', str(e)) raise e
def _connect(): u''' Connect to Redis and tell RQ about it. Workaround for https://github.com/nvie/rq/issues/479. ''' conn = connect_to_redis() push_connection(conn) return conn
def all_jobs(self): u""" Get a list of all RQ jobs. """ jobs = [] redis_conn = connect_to_redis() for queue in rq.Queue.all(connection=redis_conn): jobs.extend(queue.jobs) return jobs
def setup(self): u''' Delete all RQ queues and jobs. ''' # See https://github.com/nvie/rq/issues/731 redis_conn = connect_to_redis() for queue in rq.Queue.all(connection=redis_conn): queue.empty() redis_conn.srem(rq.Queue.redis_queues_keys, queue._key) redis_conn.delete(queue._key)
def setup(self): u""" Delete all RQ queues and jobs. """ # See https://github.com/nvie/rq/issues/731 redis_conn = connect_to_redis() for queue in rq.Queue.all(connection=redis_conn): queue.empty() redis_conn.srem(rq.Queue.redis_queues_keys, queue._key) redis_conn.delete(queue._key)
def _load_token(user_id): """ Retrieve a user's auth token from Redis. """ if not user_id: return {} redis = connect_to_redis() key = 'oidc_token:' + user_id token = redis.get(key) or '{}' token = json.loads(token) return token
def logged_in(self): """ Provide a custom error code when login fails due to account lockout. """ if not c.user: # a number of failed login attempts greater than 10 indicates # that the locked user is associated with the current request redis_conn = connect_to_redis() for key in redis_conn.keys('{}.ckanext.qgov.login_attempts.*'.format(g.site_id)): login_attempts = redis_conn.get(key) if login_attempts > 10: redis_conn.set(key, 10, ex=LOGIN_THROTTLE_EXPIRY) return self.login('account-locked') return LOGGED_IN(self)
def unlock_account(account_id): """ Unlock an account (erase the failed login attempts). """ qgov_user = Session.query(User).filter(User.id == account_id).first() if qgov_user: login_name = qgov_user.name cache_key = '{}.ckanext.qgov.login_attempts.{}'.format( g.site_id, login_name) redis_conn = connect_to_redis() if redis_conn.get(cache_key): LOG.debug("Clearing failed login attempts for %s", login_name) redis_conn.delete(cache_key) else: LOG.debug("Account %s not found", account_id)
def cacheable_wrapper(*args, **kwargs): expiry = cacheable_kwargs.get("expiry", 100) site_id = config.get("ckan.site_id", "default") lang = h.lang() cache_key = cacheable_kwargs.get( "key", "CACHE:" + site_id + "::" + lang + "::" + cacheable_func.__name__) conn = connect_to_redis() if conn.exists(cache_key): return json.loads(conn.get(cache_key)) value = cacheable_func(*args, **kwargs) conn.setex(name=cache_key, value=json.dumps(value), time=expiry) return value
def wrapper(*args, **kwargs): conn = redis.connect_to_redis() key = pickle.dumps((args, kwargs)) value = conn.get(key) if value: return cast(T, json.loads(value)) value = func(*args, **kwargs) cache_duration = tk.asint( tk.config.get(CONFIG_CACHE_DURATION, DEFAULT_CACHE_DURATION)) if isinstance(value, DontCache): value = cast(T, value.unwrap()) conn.set(key, json.dumps(value), ex=cache_duration) return value
def get_signed_url_to_key(self, key, extra_params={}): '''Generates a pre-signed URL giving access to an S3 object. If a download_proxy is configured, then the URL will be generated using the true S3 host, and then the hostname will be rewritten afterward. Note that the Host header is part of a version 4 signature, so the resulting request, as it stands, will fail signature verification; the download_proxy server must be configured to set the Host header back to the true value when forwarding the request (CloudFront does this automatically). ''' client = self.get_s3_client() # check whether the object exists in S3 metadata = client.head_object(Bucket=self.bucket_name, Key=key) if self.signed_url_cache_enabled: redis_conn = connect_to_redis() cache_key = _get_cache_key(key) cache_url = redis_conn.get(cache_key) if cache_url: log.debug('Returning cached URL for path %s', key) return cache_url else: log.debug('No cache found for %s; generating a new URL', key) params = {'Bucket': self.bucket_name, 'Key': key} if metadata['ContentType'] != 'application/pdf': filename = key.split('/')[-1] params[ 'ResponseContentDisposition'] = 'attachment; filename=' + filename params.update(extra_params) url = client.generate_presigned_url(ClientMethod='get_object', Params=params, ExpiresIn=self.signed_url_expiry) if self.download_proxy: url = URL_HOST.sub(self.download_proxy + '/', url, 1) if self.signed_url_cache_enabled: redis_conn.set(cache_key, url, ex=self.signed_url_cache_window) return url
def check_privs( context, require_admin=False, require_curator=False, require_harvester=False, require_contributor=False, require_organization=None, ): """ Check whether the user has the specified privileges. This is done by looking up the info directly in Redis, which gets written by ckanext/accesscontrol/logic/openidconnect.py (see functions _extract_token_data and _save_token_data for details). Note: this is not the usual CKAN way of doing things; it effectively makes this extension dependent on ckanext-accesscontrol. At this point, however, it's the simplest means of implementing role based access control. Roles are cumulative, i.e. a given role can do everything that any lower role can do. admin > curator > harvester > contributor > member :param require_admin: the user must have the administrator role in the admin organization :param require_curator: the user must have the curator role either in the admin organization or the specified require_organization :param require_harvester: the user must have the harvester role in the specified require_organization :param require_contributor: the user must have the contributor role in the specified require_organization :param require_organization: the organization (id or name) associated with the resource being requested or updated :return: bool """ admin_org = config.get('ckan.metadata.admin_org') admin_role = config.get('ckan.metadata.admin_role') curator_role = config.get('ckan.metadata.curator_role') harvester_role = config.get('ckan.metadata.harvester_role') contributor_role = config.get('ckan.metadata.contributor_role') model = context['model'] user = context['user'] user_id = model.User.by_name(user.decode('utf8')).id redis = connect_to_redis() key = 'oidc_token_data:' + user_id token_json = redis.get(key) if not token_json: return False token_data = json.loads(token_json) if token_data['superuser']: return True is_admin = False is_curator = False is_harvester = False is_contributor = False is_member = False if require_organization: require_organization = model.Group.get(require_organization).name for privilege in token_data['privileges']: if privilege['institution'] == admin_org and privilege[ 'role'] == admin_role: is_admin = True if privilege['institution'] in ( admin_org, require_organization) and privilege['role'] == curator_role: is_curator = True if privilege['institution'] == require_organization and privilege[ 'role'] == harvester_role: is_harvester = True if privilege['institution'] == require_organization and privilege[ 'role'] == contributor_role: is_contributor = True if privilege['institution'] == require_organization: is_member = True if require_admin: return is_admin if require_curator: return is_admin or is_curator if require_harvester: return is_admin or is_curator or is_harvester if require_contributor: return is_admin or is_curator or is_harvester or is_contributor if require_organization: return is_admin or is_curator or is_harvester or is_contributor or is_member return True
'validation', 'transformations', 'dcat', 'solr', 'properties_to_remove' ] contents = json.load(config_file) redis_conn.set(cache_key, current_date) for redis_config_key in config_keys: redis_conn.set(config_key + redis_config_key, json.dumps(contents.get(redis_config_key))) [ redis_conn.set( redis_key + 'vocabulary.{0}'.format(key), json.dumps(_load_list(key, voc['local'], 'vocabulary'))) for key, voc in contents.get('validation')['vocabularies'].iteritems() ] [ redis_conn.set( redis_key + 'taxonomy.{0}'.format(key), json.dumps(_load_list(key, tax['local'], 'taxonomy'))) for key, tax in contents.get('validation')['taxonomies'].iteritems() ] redis_key = 'ckanext.dataoverheid:' config_key = redis_key + 'config.' redis_conn = connect_to_redis() _load_config_file()
def _login(context, data_dict): if toolkit.c.user: # Don't offer the reset form if already logged in log.warning("User already logged in") raise toolkit.NotAuthorized('user already logged in, logout first') # Check if parameters are present try: user_id = data_dict.get('id') if not user_id: email = data_dict['email'].lower() # Check email is valid if not util.check_email(email): raise toolkit.ValidationError({'email': 'invalid email'}) # get the user id user_id = util.get_user_id(email) if not user_id: raise toolkit.ValidationError({ 'email': 'email does not correspond to a registered user' }) except KeyError: raise toolkit.ValidationError({'email': 'missing email'}) try: orig_key = data_dict['key'] except KeyError: raise toolkit.ValidationError({'key': 'missing token'}) if len(orig_key) <= 32 and not orig_key.startswith("b'"): key = "b'{0}'".format(orig_key) else: key = orig_key log.debug('login: {0} ({1}) => {2}'.format(user_id, orig_key, key)) # get whether to return context (UI) or just a message (API) return_context = data_dict.get('return_context', False) try: data_dict = {'id': user_id} user_dict = logic.get_action('user_show')(context, data_dict) user_obj = context['user_obj'] email = user_dict.get('email', user_obj.email) except logic.NotFound: raise logic.NotFound('"%s" matched several users' % user_id) except toolkit.NotAuthorized: raise toolkit.NotAuthorized('Exception (Not Authorized) email = ' + str(email) + 'id = ' + str(user_id)) if not user_obj or not mailer.verify_reset_link(user_obj, key): raise toolkit.ValidationError({'key': 'token provided is not valid'}) flask.session['ckanext-passwordless-user'] = user_dict['name'] # remove token mailer.create_reset_key(user_obj) # log the user in programmatically try: _set_repoze_user_only(user_dict['name']) except TypeError as e: log.warning("Exception at login: {0}".format(e)) # delete attempts from Redis log.debug("Redis: reset attempts for {0}".format(email)) redis_conn = connect_to_redis() redis_conn.delete(email) # make sure the master API key exists apikey = util.renew_master_token(user_dict['name']) # return message or context if return_context: return context else: user_obj = context.get('user_obj', None) result_json = { 'user': { 'email': user_obj.email, 'id': user_obj.id, 'name': user_obj.name, 'apikey': apikey, 'fullname': user_obj.fullname }, 'message': "login success" } return result_json