def compare_fast(cls, repo_iter):
"""Compare rorps (metadata only) quickly, return report iter"""
src_iter = cls.get_source_select()
for src_rorp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter):
report = get_basic_report(src_rorp, mir_rorp)
if report: yield report
else: log_success(src_rorp, mir_rorp)
def get_diffiter(self, new_iter, old_iter):
"""Iterate meta diffs of new_iter -> old_iter"""
for new_rorp, old_rorp in rorpiter.Collate2Iters(new_iter, old_iter):
if not old_rorp: yield rpath.RORPath(new_rorp.index)
elif not new_rorp or new_rorp.data != old_rorp.data:
# exact compare here, can't use == on rorps
yield old_rorp
def ListChangedSince(mirror_rp, inc_rp, restore_to_time):
"""List the changed files under mirror_rp since rest time
Notice the output is an iterator of RORPs. We do this because we
want to give the remote connection the data in buffered
increments, and this is done automatically for rorp iterators.
Encode the lines in the first element of the rorp's index.
"""
assert mirror_rp.conn is Globals.local_connection, "Run locally only"
MirrorStruct.set_mirror_and_rest_times(restore_to_time)
MirrorStruct.initialize_rf_cache(mirror_rp, inc_rp)
old_iter = MirrorStruct.get_mirror_rorp_iter(MirrorStruct._rest_time, 1)
cur_iter = MirrorStruct.get_mirror_rorp_iter(MirrorStruct._mirror_time, 1)
collated = rorpiter.Collate2Iters(old_iter, cur_iter)
for old_rorp, cur_rorp in collated:
if not old_rorp: change = "new"
elif not cur_rorp: change = "deleted"
elif old_rorp == cur_rorp: continue
else: change = "changed"
path_desc = (old_rorp and old_rorp.get_indexpath() or
cur_rorp.get_indexpath())
yield rpath.RORPath(("%-7s %s" % (change, path_desc),))
MirrorStruct.close_rf_cache()
def set_rorp_cache(cls, baserp, source_iter, for_increment): """Initialize cls.CCPP, the destination rorp cache for_increment should be true if we are mirror+incrementing, false if we are just mirroring. """ dest_iter = cls.get_dest_select(baserp, for_increment) collated = rorpiter.Collate2Iters(source_iter, dest_iter) cls.CCPP = CacheCollatedPostProcess( collated, Globals.pipeline_max_length*4, baserp)
def get_diffs(cls, target_iter): """Given rorp iter of target files, return diffs Here the target_iter doesn't contain any actual data, just attribute listings. Thus any diffs we generate will be snapshots. """ mir_iter = cls.subtract_indicies(cls.mirror_base.index, cls.get_mirror_rorp_iter()) collated = rorpiter.Collate2Iters(mir_iter, target_iter) return cls.get_diffs_from_collated(collated)
def compare_hash(cls, repo_iter):
"""Like above, but also compare sha1 sums of any regular files"""
def hashes_changed(src_rp, mir_rorp):
"""Return 0 if their data hashes same, 1 otherwise"""
if not mir_rorp.has_sha1():
log.Log(
"Warning: Metadata file has no digest for %s, "
"unable to compare." % (mir_rorp.get_indexpath(), ), 2)
return 0
elif (src_rp.getsize() == mir_rorp.getsize()
and hash.compute_sha1(src_rp) == mir_rorp.get_sha1()):
return 0
return 1
src_iter = cls.get_source_select()
for src_rp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter):
report = get_basic_report(src_rp, mir_rorp, hashes_changed)
if report: yield report
else: log_success(src_rp, mir_rorp)
def yield_sub_rfs(self): """Return RestoreFiles under current RestoreFile (which is dir)""" if not self.mirror_rp.isdir() and not self.inc_rp.isdir(): return if self.mirror_rp.isdir(): mirror_iter = self.yield_mirrorrps(self.mirror_rp) else: mirror_iter = iter([]) if self.inc_rp.isdir(): inc_pair_iter = self.yield_inc_complexes(self.inc_rp) else: inc_pair_iter = iter([]) collated = rorpiter.Collate2Iters(mirror_iter, inc_pair_iter) for mirror_rp, inc_pair in collated: if not inc_pair: inc_rp = self.inc_rp.new_index(mirror_rp.index) inc_list = [] else: inc_rp, inc_list = inc_pair if not mirror_rp: mirror_rp = self.mirror_rp.new_index_empty(inc_rp.index) yield self.__class__(mirror_rp, inc_rp, inc_list)
def iterate_meta_rfs(mirror_rp, inc_rp):
"""Yield RegressFile objects with extra metadata information added
Each RegressFile will have an extra object variable .metadata_rorp
which will contain the metadata attributes of the mirror file at
regress_time.
"""
raw_rfs = iterate_raw_rfs(mirror_rp, inc_rp)
collated = rorpiter.Collate2Iters(raw_rfs, yield_metadata())
for raw_rf, metadata_rorp in collated:
raw_rf = longname.update_regressfile(raw_rf, metadata_rorp, mirror_rp)
if not raw_rf:
log.Log(
"Warning, metadata file has entry for %s,\n"
"but there are no associated files." %
(metadata_rorp.get_indexpath(), ), 2)
continue
raw_rf.set_metadata_rorp(metadata_rorp)
yield raw_rf
def attach_files(cls, src_iter, mirror_rp, inc_rp, compare_time):
"""Attach data to all the files that need checking
Return an iterator of repo rorps that includes all the files
that may have changed, and has the fileobj set on all rorps
that need it.
"""
repo_iter = cls.init_and_get_iter(mirror_rp, inc_rp, compare_time)
base_index = cls.mirror_base.index
for src_rorp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter):
index = src_rorp and src_rorp.index or mir_rorp.index
if src_rorp and mir_rorp:
if not src_rorp.isreg() and src_rorp == mir_rorp:
log_success(src_rorp, mir_rorp)
continue # They must be equal, nothing else to check
if (src_rorp.isreg() and mir_rorp.isreg()
and src_rorp.getsize() == mir_rorp.getsize()):
fp = cls.rf_cache.get_fp(base_index + index, mir_rorp)
mir_rorp.setfile(fp)
mir_rorp.set_attached_filetype('snapshot')
if mir_rorp: yield mir_rorp
else: yield rpath.RORPath(index) # indicate deleted mir_rorp