def __init__(self, *args, **kwargs):
super(CollsController, self).__init__(*args, **kwargs)
config = kwargs['config']
self.solr_mgr = kwargs.get('solr_mgr')
self.allow_external = get_bool(os.environ.get('ALLOW_EXTERNAL', False))
self.is_search_auto = get_bool(os.environ.get('SEARCH_AUTO', False))
def get_collections():
user = self.get_user(api=True, redir_check=False)
kwargs = {'include_recordings': get_bool(request.query.get('include_recordings')),
'include_lists': get_bool(request.query.get('include_lists')),
'include_pages': get_bool(request.query.get('include_pages')),
}
collections = user.get_collections()
return {'collections': [coll.serialize(**kwargs) for coll in collections]}
def get_collections():
user = self.get_user(api=True, redir_check=False)
kwargs = {'include_recordings': get_bool(request.query.get('include_recordings')),
'include_lists': get_bool(request.query.get('include_lists')),
'include_pages': get_bool(request.query.get('include_pages')),
}
collections = user.get_collections()
return {'collections': [coll.serialize(**kwargs) for coll in collections]}
def __init__(self, *args, **kwargs):
super(CollsController, self).__init__(*args, **kwargs)
config = kwargs['config']
self.allow_external = get_bool(os.environ.get('ALLOW_EXTERNAL', False))
self.cork = kwargs['cork']
def load_user(self, username=None):
include_colls = get_bool(request.query.get('include_colls', False))
if username:
user = self.get_user(user=username)
else:
user = self.access.session_user
return {'user': user.serialize(include_colls=include_colls)}
def load_user(self, username=None):
include_colls = get_bool(request.query.get('include_colls', False))
if username:
user = self.get_user(user=username)
else:
user = self.access.session_user
return {'user': user.serialize(include_colls=include_colls)}
def get_bool_prop(self, prop, default_val=False):
"""Get boolean value of property.
:param str prop: name of property
:param bool default_val: default value
:returns: boolean value of property
:rtype: bool
"""
return get_bool(self.get_prop(prop, default_val=False))
def __init__(self, app, jinja_env, manager, config):
self.app = app
self.jinja_env = jinja_env
self.manager = manager
self.config = config
self.app_host = os.environ['APP_HOST']
self.content_host = os.environ['CONTENT_HOST']
self.cache_template = config.get('cache_template')
self.anon_disabled = get_bool(os.environ.get('ANON_DISABLED'))
self.init_routes()
def __init__(self, redis):
self.redis = redis
self.dat_enabled = get_bool(os.environ.get('ALLOW_DAT', False))
self.dat_host = os.environ.get('DAT_SHARE_HOST', 'dat')
self.dat_port = int(os.environ.get('DAT_SHARE_PORT', 3000))
self.dat_url = 'http://{dat_host}:{dat_port}'.format(dat_host=self.dat_host,
dat_port=self.dat_port)
self.running = True
if self.dat_enabled:
spawn_once(self.dat_sync_check_loop, worker=1)
def __init__(self, redis):
self.redis = redis
self.dat_enabled = get_bool(os.environ.get('ALLOW_DAT', False))
self.dat_host = os.environ.get('DAT_SHARE_HOST', 'dat')
self.dat_port = int(os.environ.get('DAT_SHARE_PORT', 3000))
self.dat_url = 'http://{dat_host}:{dat_port}'.format(dat_host=self.dat_host,
dat_port=self.dat_port)
self.running = True
if self.dat_enabled:
spawn_once(self.dat_sync_check_loop, worker=1)
def __init__(self, config, browser_redis, user_manager):
self.browser_redis = browser_redis
self.browser_req_url = config['browser_req_url']
self.browser_list_url = config['browser_list_url']
self.browsers = {}
if not get_bool(os.environ.get('NO_REMOTE_BROWSERS')):
self.load_all_browsers()
gevent.spawn(self.browser_load_loop)
self.user_manager = user_manager
self.inactive_time = os.environ.get('INACTIVE_TIME', 60)
def login_user(self, input_data):
"""Authenticate users"""
username = input_data.get('username', '')
password = input_data.get('password', '')
try:
move_info = self.get_move_temp_info(input_data)
except ValidationException as ve:
return {'error': str(ve)}
# first, authenticate the user
# if failing, see if case-insensitive username and try that
if not self.cork.is_authenticate(username, password):
username = self.find_case_insensitive_username(username)
if not username or not self.cork.is_authenticate(
username, password):
return {'error': 'invalid_login'}
# if not enough space, don't continue with login
if move_info:
if not self.has_space_for_new_collection(
username, move_info['from_user'], 'temp'):
#return {'error': 'Sorry, not enough space to import this Temporary Collection into your account.'}
return {'error': 'out_of_space'}
user = self.all_users[username]
new_collection = None
try:
if move_info:
new_collection = self.move_temp_coll(user, move_info)
except DupeNameException as de:
return {'error': 'duplicate_name'}
#return {'error': 'Collection "{0}" already exists'.format(move_info['to_title'])}
remember_me = get_bool(input_data.get('remember_me'))
# login session and access system
self.access.log_in(username, remember_me)
user.update_last_login()
return {
'success': '1',
'new_coll_name': new_collection.name if new_collection else None,
'user': user
}
def __init__(self, *args, **kwargs):
self.app = kwargs['app']
self.jinja_env = kwargs['jinja_env']
self.user_manager = kwargs['user_manager']
self.config = kwargs['config']
self.redis = kwargs['redis']
self.cork = kwargs['cork']
self.api = api_decorator
self.app_host = os.environ.get('APP_HOST', '')
self.content_host = os.environ.get('CONTENT_HOST', '')
self.cache_template = self.config.get('cache_template')
self.anon_disabled = get_bool(os.environ.get('ANON_DISABLED'))
self.init_routes()
def login_user(self, input_data):
"""Authenticate users"""
username = input_data.get('username', '')
password = input_data.get('password', '')
try:
move_info = self.get_move_temp_info(input_data)
except ValidationException as ve:
return {'error': str(ve)}
# first, authenticate the user
# if failing, see if case-insensitive username and try that
if not self.cork.is_authenticate(username, password):
username = self.find_case_insensitive_username(username)
if not username or not self.cork.is_authenticate(username, password):
return {'error': 'invalid_login'}
# if not enough space, don't continue with login
if move_info:
if not self.has_space_for_new_collection(username,
move_info['from_user'],
'temp'):
#return {'error': 'Sorry, not enough space to import this Temporary Collection into your account.'}
return {'error': 'out_of_space'}
user = self.all_users[username]
new_collection = None
try:
if move_info:
new_collection = self.move_temp_coll(user, move_info)
except DupeNameException as de:
return {'error': 'duplicate_name'}
#return {'error': 'Collection "{0}" already exists'.format(move_info['to_title'])}
remember_me = get_bool(input_data.get('remember_me'))
# login session and access system
self.access.log_in(username, remember_me)
user.update_last_login()
return {'success': '1',
'new_coll_name': new_collection.name if new_collection else None,
'user': user}
def generate_derivs(coll_name):
user, collection = self.load_user_coll(coll_name=coll_name)
self.access.assert_can_admin_coll(collection)
if not self.is_search_auto:
self._raise_error(400, 'not_supported')
title = 'Derivates Regenerated on ' + datetime.datetime.now().isoformat()
derivs_recording = collection.create_recording(title=title,
rec_type='derivs')
res = collection.requeue_pages_for_derivs(derivs_recording.my_id, get_bool(request.query.get('include_existing')))
if res > 0:
collection.set_bool_prop('autoindexed', True)
return {'queued': res}
def __init__(self, *args, **kwargs):
self.app = kwargs['app']
self.jinja_env = kwargs['jinja_env']
self.user_manager = kwargs['user_manager']
self.config = kwargs['config']
self.redis = kwargs['redis']
self.cork = kwargs['cork']
self.api = api_decorator
self.app_host = os.environ.get('APP_HOST', '')
self.content_host = os.environ.get('CONTENT_HOST', '')
self.cache_template = self.config.get('cache_template')
self.anon_disabled = get_bool(os.environ.get('ANON_DISABLED'))
self.allow_beta_features_role = os.environ.get('ALLOW_BETA_FEATURES_ROLE', 'beta-archivist')
self.init_routes()
def login():
"""Authenticate users"""
if not self.access.is_anon():
return self._raise_error(403, 'already_logged_in')
include_colls = get_bool(request.query.get('include_colls', False))
result = self.user_manager.login_user(request.json or {})
if 'success' in result:
data = {'user': self.access.session_user.serialize(include_colls)}
if result.get('new_coll_name'):
data['new_coll_name'] = result['new_coll_name']
return data
#self._raise_error(401, result.get('error', ''))
response.status = 401
return result
def login():
"""Authenticate users"""
if not self.access.is_anon():
return self._raise_error(403, 'already_logged_in')
include_colls = get_bool(request.query.get('include_colls', False))
result = self.user_manager.login_user(request.json or {})
if 'success' in result:
data = {'user': self.access.session_user.serialize(include_colls)}
if result.get('new_coll_name'):
data['new_coll_name'] = result['new_coll_name']
return data
#self._raise_error(401, result.get('error', ''))
response.status = 401
return result
def __init__(self, *args, **kwargs):
super(UserController, self).__init__(*args, **kwargs)
config = kwargs['config']
self.default_user_desc = config['user_desc']
self.allow_external = get_bool(os.environ.get('ALLOW_EXTERNAL', False))
def get_bool_prop(self, prop, default_val=False):
return get_bool(self.get_prop(prop, default_val=False))
def __init__(self, *args, **kwargs):
super(CollsController, self).__init__(*args, **kwargs)
config = kwargs['config']
self.allow_external = get_bool(os.environ.get('ALLOW_EXTERNAL', False))
def wasapi_list(self):
username = request.query.getunicode('user')
# some clients use collection rather than coll_name so we must check for both
coll_name = request.query.getunicode('collection')
commit = get_bool(request.query.getunicode('commit'))
user = self._get_wasapi_user()
self.access.assert_is_curr_user(user)
colls = None
if coll_name:
collection = user.get_collection_by_name(coll_name)
if collection:
colls = [collection]
else:
self._raise_error(404, 'no_such_collection')
else:
colls = user.get_collections()
files = []
download_path = self.get_origin(
) + '/api/v1/download/{user}/{coll}/{filename}'
for collection in colls:
if commit:
commit_id = collection.commit_all()
while commit_id:
gevent.sleep(10)
commit_id = collection.commit_all(commit_id)
storage = collection.get_storage()
for recording in collection.get_recordings():
if not recording.is_fully_committed():
continue
for name, path in recording.iter_all_files(
include_index=False):
full_warc_path = collection.get_warc_path(name)
local_download = download_path.format(user=user.name,
coll=collection.name,
filename=name)
remote_download_url = storage.get_remote_presigned_url(
full_warc_path)
# if remote download url exists (eg. for s3), include that first
# always include local download url as well
if remote_download_url:
locations = [remote_download_url, local_download]
else:
locations = [local_download]
kind, check_sum, size = storage.get_checksum_and_size(
full_warc_path)
files.append({
'content-type':
'application/warc',
'filetype':
'application/warc',
'filename':
name,
'size':
size,
'recording':
recording.my_id,
'recording_date':
recording.get_prop('created_at'),
'collection':
collection.name,
'checksums': {
kind: check_sum
},
'locations':
locations,
})
return {'files': files, 'include-extra': True}
import json
import hashlib
import os
from webrecorder.utils import get_bool
search_auto = get_bool(os.environ.get('SEARCH_AUTO'))
# ============================================================================
class PagesMixin(object):
"""Recording pages.
:cvar str PAGES_KEY: pages Redis key template
:cvar str PAGE_BOOKMARKS_CACHE_KEY: temporary list of pages->bookmarks Redis key template
:ivar _pages_cache: n.s.
"""
PAGES_KEY = 'c:{coll}:p'
PAGE_BOOKMARKS_CACHE_KEY = 'c:{coll}:p_to_b'
PAGES_Q = 'c:{coll}:pq'
NEW_PAGES_Q = 'new_pages:q'
def __init__(self, **kwargs):
"""Initialize recording pages."""
super(PagesMixin, self).__init__(**kwargs)
self._pages_cache = None
@property
def pages_key(self):
"""Read-only property pages_key.
def __init__(self, *args, **kwargs):
super(UserController, self).__init__(*args, **kwargs)
config = kwargs['config']
self.default_user_desc = config['user_desc']
self.allow_external = get_bool(os.environ.get('ALLOW_EXTERNAL', False))
