def _find_exact_renames(self):
add_map = defaultdict(list)
for add in self._adds:
add_map[add.new.sha].append(add.new)
delete_map = defaultdict(list)
for delete in self._deletes:
# Keep track of whether the delete was actually marked as a delete.
# If not, it must have been added due to find_copies_harder, and
# needs to be marked as a copy.
is_delete = delete.type == CHANGE_DELETE
delete_map[delete.old.sha].append((delete.old, is_delete))
add_paths = set()
delete_paths = set()
for sha, sha_deletes in delete_map.iteritems():
sha_adds = add_map[sha]
for (old, is_delete), new in itertools.izip(sha_deletes, sha_adds):
if stat.S_IFMT(old.mode) != stat.S_IFMT(new.mode):
continue
delete_paths.add(old.path)
add_paths.add(new.path)
new_type = is_delete and CHANGE_RENAME or CHANGE_COPY
self._changes.append(TreeChange(new_type, old, new))
num_extra_adds = len(sha_adds) - len(sha_deletes)
# TODO(dborowitz): Less arbitrary way of dealing with extra copies.
old = sha_deletes[0][0]
if num_extra_adds:
for new in sha_adds[-num_extra_adds:]:
add_paths.add(new.path)
self._changes.append(TreeChange(CHANGE_COPY, old, new))
self._prune(add_paths, delete_paths)
def _find_exact_renames(self):
add_map = defaultdict(list)
for add in self._adds:
add_map[add.new.sha].append(add.new)
delete_map = defaultdict(list)
for delete in self._deletes:
# Keep track of whether the delete was actually marked as a delete.
# If not, it must have been added due to find_copies_harder, and
# needs to be marked as a copy.
is_delete = delete.type == CHANGE_DELETE
delete_map[delete.old.sha].append((delete.old, is_delete))
add_paths = set()
delete_paths = set()
for sha, sha_deletes in delete_map.iteritems():
sha_adds = add_map[sha]
for (old, is_delete), new in itertools.izip(sha_deletes, sha_adds):
if stat.S_IFMT(old.mode) != stat.S_IFMT(new.mode):
continue
delete_paths.add(old.path)
add_paths.add(new.path)
new_type = is_delete and CHANGE_RENAME or CHANGE_COPY
self._changes.append(TreeChange(new_type, old, new))
num_extra_adds = len(sha_adds) - len(sha_deletes)
# TODO(dborowitz): Less arbitrary way of dealing with extra copies.
old = sha_deletes[0][0]
if num_extra_adds:
for new in sha_adds[-num_extra_adds:]:
add_paths.add(new.path)
self._changes.append(TreeChange(CHANGE_COPY, old, new))
self._prune(add_paths, delete_paths)
def _count_blocks(obj):
"""Count the blocks in an object.
Splits the data into blocks either on lines or <=64-byte chunks of lines.
:param obj: The object to count blocks for.
:return: A dict of block hashcode -> total bytes occurring.
"""
block_counts = defaultdict(int)
block = StringIO()
n = 0
# Cache attrs as locals to avoid expensive lookups in the inner loop.
block_write = block.write
block_seek = block.seek
block_truncate = block.truncate
block_getvalue = block.getvalue
for c in itertools.chain(*obj.as_raw_chunks()):
block_write(c)
n += 1
if c == '\n' or n == _BLOCK_SIZE:
value = block_getvalue()
block_counts[hash(value)] += len(value)
block_seek(0)
block_truncate()
n = 0
if n > 0:
last_block = block_getvalue()
block_counts[hash(last_block)] += len(last_block)
return block_counts
def _count_blocks(obj):
"""Count the blocks in an object.
Splits the data into blocks either on lines or <=64-byte chunks of lines.
:param obj: The object to count blocks for.
:return: A dict of block hashcode -> total bytes occurring.
"""
block_counts = defaultdict(int)
block = StringIO()
n = 0
# Cache attrs as locals to avoid expensive lookups in the inner loop.
block_write = block.write
block_seek = block.seek
block_truncate = block.truncate
block_getvalue = block.getvalue
for c in itertools.chain(*obj.as_raw_chunks()):
block_write(c)
n += 1
if c == '\n' or n == _BLOCK_SIZE:
value = block_getvalue()
block_counts[hash(value)] += len(value)
block_seek(0)
block_truncate()
n = 0
if n > 0:
last_block = block_getvalue()
block_counts[hash(last_block)] += len(last_block)
return block_counts
