def _index_merge_warcs(self, new_warcs, index_file, rel_root=None):
cdx_file = os.path.join(self.indexes_dir, index_file)
temp_file = cdx_file + '.tmp.' + timestamp20_now()
self._cdx_index(temp_file, new_warcs, rel_root)
# no existing file, so just make it the new file
if not os.path.isfile(cdx_file):
shutil.move(temp_file, cdx_file)
return
merged_file = temp_file + '.merged'
last_line = None
with open(cdx_file, 'rb') as orig_index:
with open(temp_file, 'rb') as new_index:
with open(merged_file, 'w+b') as merged:
for line in heapq.merge(orig_index, new_index):
if last_line != line:
merged.write(line)
last_line = line
shutil.move(merged_file, cdx_file)
#os.rename(merged_file, cdx_file)
os.remove(temp_file)
def _index_merge_warcs(self, new_warcs, index_file, rel_root=None):
cdx_file = os.path.join(self.indexes_dir, index_file)
temp_file = cdx_file + '.tmp.' + timestamp20_now()
self._cdx_index(temp_file, new_warcs, rel_root)
# no existing file, so just make it the new file
if not os.path.isfile(cdx_file):
shutil.move(temp_file, cdx_file)
return
merged_file = temp_file + '.merged'
last_line = None
with open(cdx_file, 'rb') as orig_index:
with open(temp_file, 'rb') as new_index:
with open(merged_file, 'w+b') as merged:
for line in heapq.merge(orig_index, new_index):
if last_line != line:
merged.write(line)
last_line = line
shutil.move(merged_file, cdx_file)
#os.rename(merged_file, cdx_file)
os.remove(temp_file)
def get_new_filename(self, dir_, params):
timestamp = timestamp20_now()
randstr = base64.b32encode(os.urandom(5)).decode('utf-8')
filename = dir_ + res_template(self.filename_template, params,
hostname=self.hostname,
timestamp=timestamp,
random=randstr)
return filename
def get_new_filename(self, dir_, params):
timestamp = timestamp20_now()
randstr = base64.b32encode(os.urandom(5)).decode('utf-8')
filename = dir_ + res_template(self.filename_template, params,
hostname=self.hostname,
timestamp=timestamp,
random=randstr)
return filename
def copy_data_from_recording(self, source, delete_source=False):
"""Copy given recording building block entries.
:param RedisUniqueComponent source: building block
:param bool delete_source: whether to delete source building block
:returns: whether successful or not
:rtype: bool
"""
if self == source:
return False
if not self.is_open():
return False
errored = False
self._copy_prop(source, 'title')
self._copy_prop(source, 'desc')
self._copy_prop(source, 'rec_type')
self._copy_prop(source, 'recorded_at')
#self._copy_prop(source, 'patch_rec')
collection = self.get_owner()
user = collection.get_owner()
target_dirname = user.get_user_temp_warc_path()
target_warc_key = self.COLL_WARC_KEY.format(coll=collection.my_id)
# Copy WARCs
loader = BlockLoader()
for n, url in source.iter_all_files(include_index=True):
local_filename = n + '.' + timestamp20_now()
target_file = os.path.join(target_dirname, local_filename)
src = loader.load(url)
try:
with open(target_file, 'wb') as dest:
logger.debug('Copying {0} -> {1}'.format(url, target_file))
shutil.copyfileobj(src, dest)
size = dest.tell()
if n != self.INDEX_FILE_KEY:
self.incr_size(size)
self.redis.hset(target_warc_key, n, add_local_store_prefix(target_file))
else:
self.set_prop(n, target_file)
except:
traceback.print_exc()
errored = True
# COPY cdxj, if exists
source_key = self.CDXJ_KEY.format(rec=source.my_id)
target_key = self.CDXJ_KEY.format(rec=self.my_id)
self.redis.zunionstore(target_key, [source_key])
# recreate pages, if any, in new recording
source_coll = source.get_owner()
source_pages = source_coll.list_rec_pages(source)
collection.import_pages(source_pages, self)
# COPY remote archives, if any
self.redis.sunionstore(self.RA_KEY.format(rec=self.my_id),
self.RA_KEY.format(rec=source.my_id))
# COPY recording warc keys
self.redis.sunionstore(self.REC_WARC_KEY.format(rec=self.my_id),
self.REC_WARC_KEY.format(rec=source.my_id))
# sync collection cdxj, if exists
collection.sync_coll_index(exists=True, do_async=True)
if not errored and delete_source:
collection = source.get_owner()
collection.remove_recording(source, delete=True)
return not errored
def copy_data_from_recording(self, source, delete_source=False):
"""Copy given recording building block entries.
:param RedisUniqueComponent source: building block
:param bool delete_source: whether to delete source building block
:returns: whether successful or not
:rtype: bool
"""
if self == source:
return False
if not self.is_open():
return False
errored = False
self._copy_prop(source, 'title')
self._copy_prop(source, 'desc')
self._copy_prop(source, 'rec_type')
self._copy_prop(source, 'recorded_at')
#self._copy_prop(source, 'patch_rec')
collection = self.get_owner()
user = collection.get_owner()
target_dirname = user.get_user_temp_warc_path()
target_warc_key = self.COLL_WARC_KEY.format(coll=collection.my_id)
# Copy WARCs
loader = BlockLoader()
for n, url in source.iter_all_files(include_index=True):
local_filename = n + '.' + timestamp20_now()
target_file = os.path.join(target_dirname, local_filename)
src = loader.load(url)
try:
with open(target_file, 'wb') as dest:
print('Copying {0} -> {1}'.format(url, target_file))
shutil.copyfileobj(src, dest)
size = dest.tell()
if n != self.INDEX_FILE_KEY:
self.incr_size(size)
self.redis.hset(target_warc_key, n,
add_local_store_prefix(target_file))
else:
self.set_prop(n, target_file)
except:
import traceback
traceback.print_exc()
errored = True
# COPY cdxj, if exists
source_key = self.CDXJ_KEY.format(rec=source.my_id)
target_key = self.CDXJ_KEY.format(rec=self.my_id)
self.redis.zunionstore(target_key, [source_key])
# recreate pages, if any, in new recording
source_coll = source.get_owner()
source_pages = source_coll.list_rec_pages(source)
collection.import_pages(source_pages, self)
# COPY remote archives, if any
self.redis.sunionstore(self.RA_KEY.format(rec=self.my_id),
self.RA_KEY.format(rec=source.my_id))
# COPY recording warc keys
self.redis.sunionstore(self.REC_WARC_KEY.format(rec=self.my_id),
self.REC_WARC_KEY.format(rec=source.my_id))
# sync collection cdxj, if exists
collection.sync_coll_index(exists=True, do_async=True)
if not errored and delete_source:
collection = source.get_owner()
collection.remove_recording(source, delete=True)
return not errored
