def make_collection(self, user, filename, info, rec_info=None):
"""Create collection.
:param User user: user
:param str filename: WARC archive filename
:param dict info: collection information
:param rec_info: recording information
:type: dict or None
:returns: collection
:rtype: Collection
"""
self.prepare_coll_desc(filename, info, rec_info)
public = info.get('public', False)
public_index = info.get('public_index', False)
info['id'] = sanitize_title(info['title'])
collection = user.create_collection(info['id'],
title=info['title'],
desc=info['desc'],
public=public,
public_index=public_index,
allow_dupe=True)
info['id'] = collection.name
info['type'] = 'collection'
collection.data['_updated_at'] = info.get('updated_at')
collection.data['_created_at'] = info.get('created_at')
return collection
def make_collection(self, user, filename, info, rec_info=None):
"""Create collection.
:param User user: user
:param str filename: WARC archive filename
:param dict info: collection information
:param rec_info: recording information
:type: dict or None
:returns: collection
:rtype: Collection
"""
self.prepare_coll_desc(filename, info, rec_info)
public = info.get('public', False)
public_index = info.get('public_index', False)
info['id'] = sanitize_title(info['title'])
collection = user.create_collection(info['id'],
title=info['title'],
desc=info['desc'],
public=public,
public_index=public_index,
allow_dupe=True)
info['id'] = collection.name
info['type'] = 'collection'
collection.data['_updated_at'] = info.get('updated_at')
collection.data['_created_at'] = info.get('created_at')
return collection
def update_list_slug(self, new_title, bookmark_list):
old_title = bookmark_list.get_prop('title')
if old_title == new_title:
return False
new_slug = self.list_names.rename(bookmark_list, sanitize_title(new_title))
return new_slug is not None
def get_list_slug(self, title):
if not title:
return
slug = sanitize_title(title)
if not slug:
return
return self.list_names.reserve_obj_name(slug, allow_dupe=True)
def _runner_init(self):
os.environ['WR_USER_CONFIG'] = 'pkg://webrecorder/config/standalone_recorder.yaml'
os.environ['SECRET_KEY'] = base64.b32encode(os.urandom(75)).decode('utf-8')
os.environ['RECORD_ROOT'] = os.path.join(self.root_dir, 'warcs', '')
os.environ['STORAGE_ROOT'] = os.path.join(self.root_dir, 'storage', '')
os.environ['REDIS_BROWSER_URL'] = 'redis://*****:*****@localhost'.format(self.default_user),
username=self.default_user,
passwd='LocalUser1',
role='admin',
name=self.default_user)
print('DEFAULT_USER='******'max_size', max_size)
except Exception as e:
print(e)
os.environ['AUTO_LOGIN_USER'] = self.default_user
def _create_coll(self, user, coll_title, public=False):
# Collection
params = {'title': coll_title, 'public': public}
coll_name = sanitize_title(coll_title)
res = self.testapp.post_json(
'/api/v1/collections?user={0}'.format(user), params=params)
collection = res.json['collection']
assert collection['public'] == public
assert collection['public_index'] == False
return coll_name
def update_list_slug(self, new_title, bookmark_list):
"""Rename list field name.
:param str new_title: new field name
:param BookmarkList bookmark_list: list of bookmarks
:returns: whether successful or not
:rtype: bool or None
"""
old_title = bookmark_list.get_prop('title')
if old_title == new_title:
return False
new_slug = self.list_names.rename(bookmark_list, sanitize_title(new_title))
return new_slug is not None
def update_list_slug(self, new_title, bookmark_list):
"""Rename list field name.
:param str new_title: new field name
:param BookmarkList bookmark_list: list of bookmarks
:returns: whether successful or not
:rtype: bool or None
"""
old_title = bookmark_list.get_prop('title')
if old_title == new_title:
return False
new_slug = self.list_names.rename(bookmark_list, sanitize_title(new_title))
return new_slug is not None
def get_list_slug(self, title):
"""Return reserved field name.
:param str title: title
:returns: reserved field name
:rtype: str
"""
if not title:
return
slug = sanitize_title(title)
if not slug:
return
return self.list_names.reserve_obj_name(slug, allow_dupe=True)
def get_list_slug(self, title):
"""Return reserved field name.
:param str title: title
:returns: reserved field name
:rtype: str
"""
if not title:
return
slug = sanitize_title(title)
if not slug:
return
return self.list_names.reserve_obj_name(slug, allow_dupe=True)
def get_move_temp_info(self, input_data):
move_temp = input_data.get('moveTemp')
if not move_temp:
return None
to_coll_title = input_data.get('toColl', '')
to_coll = sanitize_title(to_coll_title)
if not to_coll:
raise ValidationException('invalid_coll_name')
if not self.access.session_user.is_anon():
raise ValidationException('invalid_user_import')
return {'from_user': self.access.session_user.name,
'to_coll': to_coll,
'to_title': to_coll_title,
}
def get_move_temp_info(self, input_data):
move_temp = input_data.get('moveTemp')
if not move_temp:
return None
to_coll_title = input_data.get('toColl', '')
to_coll = sanitize_title(to_coll_title)
if not to_coll:
raise ValidationException('invalid_coll_name')
if not self.access.session_user.is_anon():
raise ValidationException('invalid_user_import')
return {'from_user': self.access.session_user.name,
'to_coll': to_coll,
'to_title': to_coll_title,
}
def make_collection(self, user, filename, info, rec_info=None):
self.prepare_coll_desc(filename, info, rec_info)
public = info.get('public', False)
public_index = info.get('public_index', False)
info['id'] = sanitize_title(info['title'])
collection = user.create_collection(info['id'],
title=info['title'],
desc=info['desc'],
public=public,
public_index=public_index,
allow_dupe=True)
info['id'] = collection.name
info['type'] = 'collection'
collection.data['_updated_at'] = info.get('updated_at')
collection.data['_created_at'] = info.get('created_at')
return collection
def migrate_collection(self, user, old_user, old_coll):
# get old coll info data
old_info_key = 'c:{user}:{coll}:info'.format(user=old_user,
coll=old_coll)
old_coll_data = self.old_redis.hgetall(old_info_key)
if not old_coll_data or not old_coll_data.get('id'):
print(' SKIPPING INVALID: ' + old_coll)
return 0
collection = user.create_collection(
old_coll,
allow_dupe=False,
title=old_coll_data.get('title', ''),
desc=old_coll_data.get('desc', ''),
public=old_coll_data.get('r:@public') == '1')
print(' New Collection Created: ' + collection.my_id)
if old_coll_data.get('created_at'):
collection.set_prop('created_at',
old_coll_data.get('created_at'),
update_ts=False)
else:
print(' OBJ ERR: created_at missing')
#collection.set_prop('size', old_coll_data.get('size', '0'))
collection.bookmarks_list = None
recs = self.old_redis.smembers('c:{user}:{coll}:recs'.format(
user=old_user, coll=old_coll))
patch_recordings = {}
total_size = 0
for rec in recs:
old_rec_base_key = 'r:{user}:{coll}:{rec}:'.format(user=old_user,
coll=old_coll,
rec=rec)
print(' Processing Recording: ' + rec)
recording, size = self.migrate_recording(collection, rec,
old_rec_base_key)
total_size += size
# track patch recordings
if recording and recording.get_prop('rec_type') == 'patch':
print(' Patch: yes')
id_ = self.old_redis.hget(old_rec_base_key + 'info', 'id')
patch_recordings[id_] = recording
collection.set_prop('size', total_size, update_ts=False)
# add bookmarks to Bookmarks list
if hasattr(collection, 'pages_for_bookmarks'):
# sort pages oldest to newest
pages = sorted(collection.pages_for_bookmarks,
key=lambda x: x.get('timestamp') or x.get('ts', ''))
for page in pages:
bookmark = collection.bookmarks_list.create_bookmark(page)
# map source to patch recordings, if any
if not patch_recordings:
return total_size
all_recordings = collection.get_recordings()
for recording in all_recordings:
title = recording.get_prop('title')
if not title:
continue
patch_title = 'patch-of-' + sanitize_title(title)
patch_recording = patch_recordings.get(patch_title)
if patch_recording:
print('Patch Mapped ({0}) {1} -> {2}'.format(
title, recording.my_id, patch_recording.my_id))
recording.set_patch_recording(patch_recording, update_ts=False)
return total_size
def sanitize_title(self, title):
return sanitize_title(title)
def migrate_collection(self, user, old_user, old_coll):
# get old coll info data
old_info_key = 'c:{user}:{coll}:info'.format(user=old_user, coll=old_coll)
old_coll_data = self.old_redis.hgetall(old_info_key)
if not old_coll_data or not old_coll_data.get('id'):
print(' SKIPPING INVALID: ' + old_coll)
return 0
collection = user.create_collection(old_coll,
allow_dupe=False,
title=old_coll_data.get('title', ''),
desc=old_coll_data.get('desc', ''),
public=old_coll_data.get('r:@public') == '1')
print(' New Collection Created: ' + collection.my_id)
if old_coll_data.get('created_at'):
collection.set_prop('created_at', old_coll_data.get('created_at'), update_ts=False)
else:
print(' OBJ ERR: created_at missing')
#collection.set_prop('size', old_coll_data.get('size', '0'))
collection.bookmarks_list = None
recs = self.old_redis.smembers('c:{user}:{coll}:recs'.format(user=old_user, coll=old_coll))
patch_recordings = {}
total_size = 0
for rec in recs:
old_rec_base_key = 'r:{user}:{coll}:{rec}:'.format(user=old_user, coll=old_coll, rec=rec)
print(' Processing Recording: ' + rec)
recording, size = self.migrate_recording(collection, rec, old_rec_base_key)
total_size += size
# track patch recordings
if recording and recording.get_prop('rec_type') == 'patch':
print(' Patch: yes')
id_ = self.old_redis.hget(old_rec_base_key + 'info', 'id')
patch_recordings[id_] = recording
collection.set_prop('size', total_size, update_ts=False)
# add bookmarks to Bookmarks list
if hasattr(collection, 'pages_for_bookmarks'):
# sort pages oldest to newest
pages = sorted(collection.pages_for_bookmarks, key=lambda x: x.get('timestamp') or x.get('ts', ''))
for page in pages:
bookmark = collection.bookmarks_list.create_bookmark(page)
# map source to patch recordings, if any
if not patch_recordings:
return total_size
all_recordings = collection.get_recordings()
for recording in all_recordings:
title = recording.get_prop('title')
if not title:
continue
patch_title = 'patch-of-' + sanitize_title(title)
patch_recording = patch_recordings.get(patch_title)
if patch_recording:
print('Patch Mapped ({0}) {1} -> {2}'.format(title, recording.my_id, patch_recording.my_id))
recording.set_patch_recording(patch_recording, update_ts=False)
return total_size
