def fix_trees(a_files, b_files):
"""
Given two sequences of File objects 'a' and 'b', use the tuple of two
integers returned by align_trees() to remove the number of path segments
required to create equal paths for two files that are the same in 'a' and
'b'.
"""
a_offset, b_offset = align_trees(a_files, b_files)
for a_file in a_files:
a_file.original_path = a_file.path
a_file.path = '/'.join(paths.split(a_file.path)[a_offset:])
for b_file in b_files:
b_file.original_path = b_file.path
b_file.path = '/'.join(paths.split(b_file.path)[b_offset:])
def _match(path, patterns):
"""
Return a message if `path` is matched by a pattern from the `patterns` map
or False.
"""
if not path or not patterns:
return False
path = fileutils.as_posixpath(path).lower()
pathstripped = path.lstrip('/')
if not pathstripped:
return False
segments = paths.split(pathstripped)
if DEBUG:
logger.debug('_match: path: %(path)r patterns:%(patterns)r.' %
locals())
mtch = False
for pat, msg in patterns.items():
if not pat and not pat.strip():
continue
msg = msg or ''
pat = pat.lstrip('/').lower()
is_plain = '/' not in pat
if is_plain:
if any(fnmatch.fnmatchcase(s, pat) for s in segments):
mtch = msg
break
elif (fnmatch.fnmatchcase(path, pat)
or fnmatch.fnmatchcase(pathstripped, pat)):
mtch = msg
break
if DEBUG:
logger.debug('_match: match is %(mtch)r' % locals())
return mtch
def _match(path, patterns):
"""
Return a message if `path` is matched by a pattern from the `patterns` map
or False.
"""
if not path or not patterns:
return False
path = fileutils.as_posixpath(path).lower()
pathstripped = path.lstrip(POSIX_PATH_SEP)
if not pathstripped:
return False
segments = paths.split(pathstripped)
if DEBUG:
logger.debug('_match: path: %(path)r patterns:%(patterns)r.' % locals())
mtch = False
for pat, msg in patterns.items():
if not pat and not pat.strip():
continue
msg = msg or EMPTY_STRING
pat = pat.lstrip(POSIX_PATH_SEP).lower()
is_plain = POSIX_PATH_SEP not in pat
if is_plain:
if any(fnmatch.fnmatchcase(s, pat) for s in segments):
mtch = msg
break
elif (fnmatch.fnmatchcase(path, pat)
or fnmatch.fnmatchcase(pathstripped, pat)):
mtch = msg
break
if DEBUG:
logger.debug('_match: match is %(mtch)r' % locals())
return mtch
def compute_path_depth(root_path, dir_path):
"""
Compute the depth of ``dir_path`` below ``root_path`` as the number of paths
segments that extend below the root.
"""
if not dir_path:
return 0
dir_path = dir_path.strip('/')
if not root_path:
return len(split(dir_path))
root_path = root_path.strip('/')
suffix = dir_path[len(root_path):]
return len(split(suffix))
def get_matches(path, patterns, all_matches=False):
"""
Return a list of values (which are values from the matched `patterns`
mappint of {pattern: value or message} if `path` is matched by any of the
pattern from the `patterns` map or an empty list.
If `all_matches` is False, stop and return on the first matched pattern.
"""
if not path or not patterns:
return False
path = fileutils.as_posixpath(path).lower()
pathstripped = path.lstrip(POSIX_PATH_SEP)
if not pathstripped:
return False
segments = paths.split(pathstripped)
if TRACE:
logger.debug('_match: path: %(path)r patterns:%(patterns)r.' %
locals())
matches = []
if not isinstance(patterns, dict):
assert isinstance(
patterns, (list, tuple)), 'Invalid patterns: {}'.format(patterns)
patterns = {p: p for p in patterns}
for pat, value in patterns.items():
if not pat or not pat.strip():
continue
value = value or EMPTY_STRING
pat = pat.lstrip(POSIX_PATH_SEP).lower()
is_plain = POSIX_PATH_SEP not in pat
if is_plain:
if any(fnmatch.fnmatchcase(s, pat) for s in segments):
matches.append(value)
if not all_matches:
break
elif (fnmatch.fnmatchcase(path, pat)
or fnmatch.fnmatchcase(pathstripped, pat)):
matches.append(value)
if not all_matches:
break
if TRACE:
logger.debug('_match: matches: %(matches)r' % locals())
if not all_matches:
if matches:
return matches[0]
else:
return False
return matches
def align_trees(a_files, b_files):
"""
Given two sequences of File objects 'a' and 'b', return a tuple of
two integers that represent the number path segments to remove
respectively from a File path in 'a' or a File path in 'b' to obtain the
equal paths for two files that are the same in 'a' and 'b'.
"""
# we need to find one uniquly named file that exists in 'a' and 'b'.
a_names = defaultdict(list)
for a_file in a_files:
a_names[a_file.name].append(a_file)
a_uniques = {k: v[0] for k, v in a_names.items() if len(v) == 1}
b_names = defaultdict(list)
for b_file in b_files:
b_names[b_file.name].append(b_file)
b_uniques = {k: v[0] for k, v in b_names.items() if len(v) == 1}
candidate_found = False
for a_name, a_unique in a_uniques.items():
if a_name not in b_uniques:
continue
b_unique = b_uniques.get(a_name)
if a_unique and a_unique.sha1 == b_unique.sha1:
candidate_found = True
break
if not candidate_found:
raise AlignmentException
if a_unique.path == b_unique.path:
return 0, 0
common_suffix, common_segments = paths.common_path_suffix(
a_unique.path, b_unique.path)
a_segments = len(paths.split(a_unique.path))
b_segments = len(paths.split(b_unique.path))
return a_segments - common_segments, b_segments - common_segments
def parse_7z_listing(location, utf=False):
"""
Return a list Entry objects from parsing a long format 7zip listing from a
file at `location`.
If `utf` is True or if on Python 3, the console output will treated as
utf-8-encoded text. Otherwise it is treated as bytes.
The 7zip -slt format looks like this:
1. a header with:
- copyright and version details
- '--' line
- archive header info, varying based on the archive types and subtype
- lines of key=value pairs
- ERRORS: followed by one or more message lines
- WARNINGS: followed by one or more message lines
- blank line
2. blocks of path aka. entry data, one for each path with:
- '----------' line once as the indicator of path blocks starting
- for each archive member:
- lines of either
- key = value pairs, with a possible twist that the Path may
contain a line return since a filename may. The first key is the Path.
- Errors: followed by one or more message lines
- Warnings: followed by one or more message lines
- Open Warning: : followed by one or more message lines
- blank line
3. a footer
- blank line
- footer sometimes with lines with summary stats
such as Warnings: 1 Errors: 1
- a line with two or more dashes or an empty line
We ignore the header and footer in a listing.
"""
if utf or py3:
# read to unicode
with io.open(location, 'r', encoding='utf-8') as listing:
text = listing.read()
text = text.replace(u'\r\n', u'\n')
end_of_header = u'----------\n'
path_key = u'Path'
kv_sep = u'='
path_blocks_sep = u'\n\n'
line_sep = u'\n'
else:
# read to bytes
with io.open(location, 'rb') as listing:
text = listing.read()
text = text.replace(b'\r\n', b'\n')
end_of_header = b'----------\n'
path_key = b'Path'
kv_sep = b'='
path_blocks_sep = b'\n\n'
line_sep = b'\n'
if TRACE:
logger.debug('parse_7z_listing: initial text: type: ' +
repr(type(text)))
print('--------------------------------------')
print(text)
print('--------------------------------------')
# for now we ignore the header
_header, _, paths = text.rpartition(end_of_header)
if not paths:
# we have only a header, likely an error condition or an empty archive
return []
# each block representing one path or file:
# - starts with a "Path = <some/path>" key/value
# - continues with key = value pairs each on a single line
# (unless there is a \n in file name which is an error condition)
# - ends with an empty line
# then we have a global footer
path_blocks = [
pb for pb in paths.split(path_blocks_sep) if pb and path_key in pb
]
entries = []
for path_block in path_blocks:
# we ignore empty lines as well as lines that do not contain a key
lines = [
line.strip() for line in path_block.splitlines(False)
if line.strip()
]
if not lines:
continue
# we have a weird case of path with line returns in the file name
# we concatenate these in the first Path line
while len(lines) > 1 and lines[0].startswith(
path_key) and kv_sep not in lines[1]:
first_line = lines[0]
second_line = lines.pop(1)
first_line = line_sep.join([first_line, second_line])
lines[0] = first_line
dangling_lines = [line for line in lines if kv_sep not in line]
entry_errors = []
if dangling_lines:
emsg = 'Invalid 7z listing path block missing "=" as key/value separator: {}'.format(
repr(path_block))
entry_errors.append(emsg)
entry_attributes = {}
key_lines = [line for line in lines if kv_sep in line]
for line in key_lines:
k, _, v = line.partition(kv_sep)
k = k.strip()
v = v.strip()
entry_attributes[k] = v
entries.append(
Entry.from_dict(infos=entry_attributes, errors=entry_errors))
if TRACE_ENTRIES:
logger.debug('parse_7z_listing: entries# {}\n'.format(len(entries)))
for entry in entries:
logger.debug(' ' + repr(entry.to_dict()))
return entries