def __init__(self, *args, **kwargs): super(CollsController, self).__init__(*args, **kwargs) config = kwargs['config'] self.solr_mgr = kwargs.get('solr_mgr') self.allow_external = get_bool(os.environ.get('ALLOW_EXTERNAL', False)) self.is_search_auto = get_bool(os.environ.get('SEARCH_AUTO', False))
def get_collections(): user = self.get_user(api=True, redir_check=False) kwargs = {'include_recordings': get_bool(request.query.get('include_recordings')), 'include_lists': get_bool(request.query.get('include_lists')), 'include_pages': get_bool(request.query.get('include_pages')), } collections = user.get_collections() return {'collections': [coll.serialize(**kwargs) for coll in collections]}
def get_collections(): user = self.get_user(api=True, redir_check=False) kwargs = {'include_recordings': get_bool(request.query.get('include_recordings')), 'include_lists': get_bool(request.query.get('include_lists')), 'include_pages': get_bool(request.query.get('include_pages')), } collections = user.get_collections() return {'collections': [coll.serialize(**kwargs) for coll in collections]}
def __init__(self, *args, **kwargs): super(CollsController, self).__init__(*args, **kwargs) config = kwargs['config'] self.allow_external = get_bool(os.environ.get('ALLOW_EXTERNAL', False)) self.cork = kwargs['cork']
def load_user(self, username=None): include_colls = get_bool(request.query.get('include_colls', False)) if username: user = self.get_user(user=username) else: user = self.access.session_user return {'user': user.serialize(include_colls=include_colls)}
def load_user(self, username=None): include_colls = get_bool(request.query.get('include_colls', False)) if username: user = self.get_user(user=username) else: user = self.access.session_user return {'user': user.serialize(include_colls=include_colls)}
def get_bool_prop(self, prop, default_val=False): """Get boolean value of property. :param str prop: name of property :param bool default_val: default value :returns: boolean value of property :rtype: bool """ return get_bool(self.get_prop(prop, default_val=False))
def __init__(self, app, jinja_env, manager, config): self.app = app self.jinja_env = jinja_env self.manager = manager self.config = config self.app_host = os.environ['APP_HOST'] self.content_host = os.environ['CONTENT_HOST'] self.cache_template = config.get('cache_template') self.anon_disabled = get_bool(os.environ.get('ANON_DISABLED')) self.init_routes()
def __init__(self, redis): self.redis = redis self.dat_enabled = get_bool(os.environ.get('ALLOW_DAT', False)) self.dat_host = os.environ.get('DAT_SHARE_HOST', 'dat') self.dat_port = int(os.environ.get('DAT_SHARE_PORT', 3000)) self.dat_url = 'http://{dat_host}:{dat_port}'.format(dat_host=self.dat_host, dat_port=self.dat_port) self.running = True if self.dat_enabled: spawn_once(self.dat_sync_check_loop, worker=1)
def __init__(self, redis): self.redis = redis self.dat_enabled = get_bool(os.environ.get('ALLOW_DAT', False)) self.dat_host = os.environ.get('DAT_SHARE_HOST', 'dat') self.dat_port = int(os.environ.get('DAT_SHARE_PORT', 3000)) self.dat_url = 'http://{dat_host}:{dat_port}'.format(dat_host=self.dat_host, dat_port=self.dat_port) self.running = True if self.dat_enabled: spawn_once(self.dat_sync_check_loop, worker=1)
def __init__(self, config, browser_redis, user_manager): self.browser_redis = browser_redis self.browser_req_url = config['browser_req_url'] self.browser_list_url = config['browser_list_url'] self.browsers = {} if not get_bool(os.environ.get('NO_REMOTE_BROWSERS')): self.load_all_browsers() gevent.spawn(self.browser_load_loop) self.user_manager = user_manager self.inactive_time = os.environ.get('INACTIVE_TIME', 60)
def login_user(self, input_data): """Authenticate users""" username = input_data.get('username', '') password = input_data.get('password', '') try: move_info = self.get_move_temp_info(input_data) except ValidationException as ve: return {'error': str(ve)} # first, authenticate the user # if failing, see if case-insensitive username and try that if not self.cork.is_authenticate(username, password): username = self.find_case_insensitive_username(username) if not username or not self.cork.is_authenticate( username, password): return {'error': 'invalid_login'} # if not enough space, don't continue with login if move_info: if not self.has_space_for_new_collection( username, move_info['from_user'], 'temp'): #return {'error': 'Sorry, not enough space to import this Temporary Collection into your account.'} return {'error': 'out_of_space'} user = self.all_users[username] new_collection = None try: if move_info: new_collection = self.move_temp_coll(user, move_info) except DupeNameException as de: return {'error': 'duplicate_name'} #return {'error': 'Collection "{0}" already exists'.format(move_info['to_title'])} remember_me = get_bool(input_data.get('remember_me')) # login session and access system self.access.log_in(username, remember_me) user.update_last_login() return { 'success': '1', 'new_coll_name': new_collection.name if new_collection else None, 'user': user }
def __init__(self, *args, **kwargs): self.app = kwargs['app'] self.jinja_env = kwargs['jinja_env'] self.user_manager = kwargs['user_manager'] self.config = kwargs['config'] self.redis = kwargs['redis'] self.cork = kwargs['cork'] self.api = api_decorator self.app_host = os.environ.get('APP_HOST', '') self.content_host = os.environ.get('CONTENT_HOST', '') self.cache_template = self.config.get('cache_template') self.anon_disabled = get_bool(os.environ.get('ANON_DISABLED')) self.init_routes()
def login_user(self, input_data): """Authenticate users""" username = input_data.get('username', '') password = input_data.get('password', '') try: move_info = self.get_move_temp_info(input_data) except ValidationException as ve: return {'error': str(ve)} # first, authenticate the user # if failing, see if case-insensitive username and try that if not self.cork.is_authenticate(username, password): username = self.find_case_insensitive_username(username) if not username or not self.cork.is_authenticate(username, password): return {'error': 'invalid_login'} # if not enough space, don't continue with login if move_info: if not self.has_space_for_new_collection(username, move_info['from_user'], 'temp'): #return {'error': 'Sorry, not enough space to import this Temporary Collection into your account.'} return {'error': 'out_of_space'} user = self.all_users[username] new_collection = None try: if move_info: new_collection = self.move_temp_coll(user, move_info) except DupeNameException as de: return {'error': 'duplicate_name'} #return {'error': 'Collection "{0}" already exists'.format(move_info['to_title'])} remember_me = get_bool(input_data.get('remember_me')) # login session and access system self.access.log_in(username, remember_me) user.update_last_login() return {'success': '1', 'new_coll_name': new_collection.name if new_collection else None, 'user': user}
def generate_derivs(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) if not self.is_search_auto: self._raise_error(400, 'not_supported') title = 'Derivates Regenerated on ' + datetime.datetime.now().isoformat() derivs_recording = collection.create_recording(title=title, rec_type='derivs') res = collection.requeue_pages_for_derivs(derivs_recording.my_id, get_bool(request.query.get('include_existing'))) if res > 0: collection.set_bool_prop('autoindexed', True) return {'queued': res}
def __init__(self, *args, **kwargs): self.app = kwargs['app'] self.jinja_env = kwargs['jinja_env'] self.user_manager = kwargs['user_manager'] self.config = kwargs['config'] self.redis = kwargs['redis'] self.cork = kwargs['cork'] self.api = api_decorator self.app_host = os.environ.get('APP_HOST', '') self.content_host = os.environ.get('CONTENT_HOST', '') self.cache_template = self.config.get('cache_template') self.anon_disabled = get_bool(os.environ.get('ANON_DISABLED')) self.allow_beta_features_role = os.environ.get('ALLOW_BETA_FEATURES_ROLE', 'beta-archivist') self.init_routes()
def login(): """Authenticate users""" if not self.access.is_anon(): return self._raise_error(403, 'already_logged_in') include_colls = get_bool(request.query.get('include_colls', False)) result = self.user_manager.login_user(request.json or {}) if 'success' in result: data = {'user': self.access.session_user.serialize(include_colls)} if result.get('new_coll_name'): data['new_coll_name'] = result['new_coll_name'] return data #self._raise_error(401, result.get('error', '')) response.status = 401 return result
def login(): """Authenticate users""" if not self.access.is_anon(): return self._raise_error(403, 'already_logged_in') include_colls = get_bool(request.query.get('include_colls', False)) result = self.user_manager.login_user(request.json or {}) if 'success' in result: data = {'user': self.access.session_user.serialize(include_colls)} if result.get('new_coll_name'): data['new_coll_name'] = result['new_coll_name'] return data #self._raise_error(401, result.get('error', '')) response.status = 401 return result
def __init__(self, *args, **kwargs): super(UserController, self).__init__(*args, **kwargs) config = kwargs['config'] self.default_user_desc = config['user_desc'] self.allow_external = get_bool(os.environ.get('ALLOW_EXTERNAL', False))
def get_bool_prop(self, prop, default_val=False): return get_bool(self.get_prop(prop, default_val=False))
def __init__(self, *args, **kwargs): super(CollsController, self).__init__(*args, **kwargs) config = kwargs['config'] self.allow_external = get_bool(os.environ.get('ALLOW_EXTERNAL', False))
def wasapi_list(self): username = request.query.getunicode('user') # some clients use collection rather than coll_name so we must check for both coll_name = request.query.getunicode('collection') commit = get_bool(request.query.getunicode('commit')) user = self._get_wasapi_user() self.access.assert_is_curr_user(user) colls = None if coll_name: collection = user.get_collection_by_name(coll_name) if collection: colls = [collection] else: self._raise_error(404, 'no_such_collection') else: colls = user.get_collections() files = [] download_path = self.get_origin( ) + '/api/v1/download/{user}/{coll}/{filename}' for collection in colls: if commit: commit_id = collection.commit_all() while commit_id: gevent.sleep(10) commit_id = collection.commit_all(commit_id) storage = collection.get_storage() for recording in collection.get_recordings(): if not recording.is_fully_committed(): continue for name, path in recording.iter_all_files( include_index=False): full_warc_path = collection.get_warc_path(name) local_download = download_path.format(user=user.name, coll=collection.name, filename=name) remote_download_url = storage.get_remote_presigned_url( full_warc_path) # if remote download url exists (eg. for s3), include that first # always include local download url as well if remote_download_url: locations = [remote_download_url, local_download] else: locations = [local_download] kind, check_sum, size = storage.get_checksum_and_size( full_warc_path) files.append({ 'content-type': 'application/warc', 'filetype': 'application/warc', 'filename': name, 'size': size, 'recording': recording.my_id, 'recording_date': recording.get_prop('created_at'), 'collection': collection.name, 'checksums': { kind: check_sum }, 'locations': locations, }) return {'files': files, 'include-extra': True}
import json import hashlib import os from webrecorder.utils import get_bool search_auto = get_bool(os.environ.get('SEARCH_AUTO')) # ============================================================================ class PagesMixin(object): """Recording pages. :cvar str PAGES_KEY: pages Redis key template :cvar str PAGE_BOOKMARKS_CACHE_KEY: temporary list of pages->bookmarks Redis key template :ivar _pages_cache: n.s. """ PAGES_KEY = 'c:{coll}:p' PAGE_BOOKMARKS_CACHE_KEY = 'c:{coll}:p_to_b' PAGES_Q = 'c:{coll}:pq' NEW_PAGES_Q = 'new_pages:q' def __init__(self, **kwargs): """Initialize recording pages.""" super(PagesMixin, self).__init__(**kwargs) self._pages_cache = None @property def pages_key(self): """Read-only property pages_key.
def __init__(self, *args, **kwargs): super(UserController, self).__init__(*args, **kwargs) config = kwargs['config'] self.default_user_desc = config['user_desc'] self.allow_external = get_bool(os.environ.get('ALLOW_EXTERNAL', False))