def is_in_git_raw(file, data, _dummy, _options):
debug_level = 0
report_fail = lambda dir, command: \
f'When executing in directory: {dir}\n>{command} failed'
git_dir = get_git_dir(file)
if git_dir is None:
return False
# Make a pushd to the git dir
curr_dir = os.getcwd()
os.chdir(git_dir)
commando = f'git ls-files -s "{file}"'
reply, _exit_code = simur.run_process(commando, False) # git may complain
if len(reply) == 0: # if it is not a repo
os.chdir(curr_dir)
return False
if reply.startswith('fatal'): # fatal: not a git repository ...
if debug_level > 4:
print(report_fail(curr_dir, commando)) # so it is not a fail
os.chdir(curr_dir)
return False
reply = reply.rstrip()
if debug_level > 4:
print(f'GIT: |{reply}|')
repo = re.match(r'^(\d+)\s*([a-fA-F0-9]+)\s*(\d+)\s*(.+)$', reply)
if repo:
data['reporoot'] = str(git_dir)
data['relpath'] = repo.group(4)
data['revision'] = repo.group(2)
data['sha1'] = repo.group(2)
data['local'] = str(git_dir)
else:
print(report_fail(git_dir, commando))
print(f'When executing in directory: {git_dir}')
print(f'>{commando} failed')
os.chdir(curr_dir)
return False
#Look for remote:s
commando = 'git remote -v'
reply, _exit_code = simur.run_process(commando, True)
lines = reply.splitlines()
for line in lines:
remote = re.match(r'^origin\s*(.+)\s+\(fetch\)$', line)
if remote:
data['reporoot'] = remote.group(1)
data['remote'] = remote.group(1)
data['vcs'] = 'git'
os.chdir(curr_dir)
return True
def is_in_svn_raw(file, data, _dummy, _options):
debug_level = 0
svn_dir = get_svn_dir(file)
if svn_dir is None:
return False
# Make a pushd to the svn dir
curr_dir = os.getcwd()
os.chdir(svn_dir)
commando = 'svn info --show-item url .'
reply, _exit_code = simur.run_process(commando, True)
reporoot = reply.strip()
disk_rel = os.path.relpath(file)
url_rel = disk_rel.replace('\\', '/')
commando = f'svn info "{file}"'
reply, _exit_code = simur.run_process(commando, True)
if len(reply) < 2:
print(f'svn info returned: {reply}')
os.chdir(curr_dir)
return False
lines = reply.splitlines()
hits = 0
for line in lines:
if debug_level > 4:
print(line)
# Just check if we get an URL: but we ignore what it says
url = line.startswith('URL: ')
if url:
data['reporoot'] = reporoot
data['relpath'] = url_rel
hits += 1
continue
rev = re.match(r'^Revision: (.*)$', line)
if rev:
data['revision'] = rev.group(1)
hits += 1
continue
sha1 = re.match(r'^Checksum: ([a-fA-F0-9]+)$', line)
if sha1:
data['sha1'] = sha1.group(1)
hits += 1
continue
os.chdir(curr_dir)
if hits != 3:
return False
data['vcs'] = 'svn'
return True
def make_log(srcsrv, explicit_cvdump, elapsed):
bins_used = [
sys.executable,
os.path.join(srcsrv, 'srctool.exe'),
os.path.join(srcsrv, 'pdbstr.exe'),
]
maybe_bins = [explicit_cvdump, 'git.exe', 'svn.exe', 'hg.exe']
found_bins, unfound_bins = get_available_bins(maybe_bins)
bins_used += found_bins
print(f'Executed by : {os.getenv("USERNAME")}')
print(f' on machine : {os.getenv("COMPUTERNAME")}')
print(f' SIMUR_REPO_CACHE: {os.getenv("SIMUR_REPO_CACHE")}')
print(f' elapsed time : {make_time_string(elapsed)}')
codepage, _exit_code = simur.run_process('cmd /c CHCP', False)
the_cp = re.match(r'^.*:\s+(\d*)$', codepage)
if the_cp:
codepage = the_cp.group(1)
print(f' CodePage : {codepage}')
print('Script:')
print(f' {os.path.realpath(sys.argv[0])}')
print('Using binaries:')
for the_exe in bins_used:
the_exe = os.path.join(srcsrv, the_exe)
print(f' {the_exe}:')
props = simur.getFileProperties(the_exe)
if props:
print(f' {props["StringFileInfo"]["FileVersion"]}')
print(f' {props["FileVersion"]}')
if unfound_bins:
print('Binaries not found/used:')
for the_exe in unfound_bins:
print(f' {the_exe}:')
def main():
'''Just as processPDBs only that as the first argument give the PDB of
interest
Example:
prepPDB.py armLibSupport.pdb . //ExternalAccess/WinKit10Debuggers/srcsrv
'''
if len(sys.argv) < 2:
print("Too few arguments")
usage()
sys.exit(3)
the_pdb = sys.argv[1]
root = sys.argv[2]
cvdump = 'cvdump.exe'
srcsrv = 'C:\\Program Files (x86)\\Windows Kits\\10\\Debuggers\\x64\\srcsrv'
if len(sys.argv) > 3:
srcsrv = sys.argv[3]
if len(sys.argv) > 4:
cvdump = sys.argv[4]
cvdump = libSrcTool.check_cvdump(cvdump, srcsrv)
# Invoke indexPDBs.py as its own process to keep the distance
this_dir = os.path.dirname(os.path.abspath(__file__))
script = os.path.join(this_dir, 'indexPDBs.py')
commando = [
'python', script, '-u', the_pdb, '-t', root, '-s', srcsrv, '-c', cvdump
]
outputs, exit_code = simur.run_process(commando, True)
print(outputs)
return exit_code
def get_lib_source_files(pdb_file, cvdump, srcsrv, options):
# First check if srctool returns anything - then it is NOT a lib-PDB
srctool_files = prepPDB.get_non_indexed_files(pdb_file, srcsrv, options)
if srctool_files:
print(f'{pdb_file} is not a lib-PDB file - skipped')
return []
commando = [cvdump, pdb_file]
raw_data, _exit_code = simur.run_process(commando, True)
files = process_raw_cvdump_data(raw_data)
return files
def dump_stream_to_pdb(pdb_file, srcsrv, stream, options):
tempfile = make_stream_file(pdb_file, stream)
'''
To restore the pdb:s their .orig's
---
for /R %%I in (*.*.orig) do call :doit %%I %%~nI %%~pI
goto :EOF
:doit
pushd %3
del %2
ren %1 %2
popd
---
'''
if options.backup:
make_backup_file(pdb_file, '.orig')
pdbstr = os.path.join(srcsrv, 'pdbstr.exe')
commando = [pdbstr, '-w', '-s:srcsrv', f'-p:{pdb_file}', f'-i:{tempfile}']
simur.run_process(commando, True)
os.remove(tempfile) # Or keep it for debugging
def handle_local_git(reporoot, revision):
# Make a pushd to the git dir
curr_dir = os.getcwd()
os.chdir(reporoot)
command = f'git show {revision}'
reply, _exit_code = simur.run_process(command,
True,
extra_dir=reporoot,
as_text=False)
os.chdir(curr_dir)
return reply
def compare_srcsrv_content(test_pdbs, facit_pdbs, options):
pdb_to_find = options.beyond_compare
intersection = test_pdbs.keys() & facit_pdbs.keys()
print('Compare .pdb - srcsrv content:')
if not pdb_to_find in intersection:
print(f'Cannot find {pdb_to_find} in current selections')
return 3
pdbstr = os.path.join(options.srcsrv_dir, 'pdbstr.exe')
test_pdb = test_pdbs[pdb_to_find]
facit_pdb = facit_pdbs[pdb_to_find]
test_commando = [pdbstr, '-r', '-s:srcsrv', f'-p:{test_pdb}']
facit_commando = [pdbstr, '-r', '-s:srcsrv', f'-p:{facit_pdb}']
test_output, _exit_code = simur.run_process(test_commando, False)
facit_output, _exit_code = simur.run_process(facit_commando, False)
test_output = test_output.splitlines()
facit_output = facit_output.splitlines()
print(f'{pdb_to_find}:')
diff_on_saved_files(test_output, facit_output, options)
return 0
def diff_on_saved_files(test_output, facit_output, options):
test_file = 'test.srcsrv'
facit_file = 'facit.srcsrv'
temp_dir = os.environ['TEMP']
test_file = os.path.join(temp_dir, test_file)
facit_file = os.path.join(temp_dir, facit_file)
write_set_to_file(test_file, test_output, options)
write_set_to_file(facit_file, facit_output, options)
visualdiff_commando = [
'C:\\Program Files\\Beyond Compare 4\\BComp.exe', test_file, facit_file
]
test_output, _exit_code = simur.run_process(visualdiff_commando, False)
os.unlink(test_file)
os.unlink(facit_file)
def handle_svn(reporoot, relpath, revision):
url = reporoot + '/' + relpath
command = f'svn cat {url}@{revision}'
reply, _exit_code = simur.run_process(command,
True,
extra_dir=None,
as_text=False)
# Try to work around any http - https redirecting
if reply.startswith(b'Redirecting to URL'):
temp = bytearray(reply)
slicer = temp.index(b'\r\n') + 2
reply = temp[slicer:]
return reply
def handle_remote_git(reporoot, revision):
git_dir = simur.find_and_update_git_cache(reporoot)
if not git_dir:
reply = f'Could not find a .git dir from {reporoot}\n'
reply += f'when looking in {git_dir}'
else:
curr_dir = os.getcwd()
os.chdir(git_dir)
command = f'git show {revision}'
reply, _exit_code = simur.run_process(command,
True,
extra_dir=git_dir,
as_text=False)
os.chdir(curr_dir)
return reply
def is_indexed(root, srcsrv, options):
pdbstr = os.path.join(srcsrv, 'pdbstr.exe')
# read the pdb and dump its 'srcsrv' stream
# - if there is a stream then it is indexed
# Use a list to avoid whitespace problems in paths
commando = [pdbstr, '-r', f'-p:{root}', '-s:srcsrv']
reply, exit_code = simur.run_process(commando, False)
if options.debug_level > 3:
print(f'{commando} returned {exit_code = }')
print(f'{reply = }')
# I will look at an empty reply as not indexed
if len(reply) == 0:
return False
if not options.quiet:
print(f'Sorry, {root} is already indexed or has no debug information')
return True
def accumulate_srctool_reply(the_pdb, commando, accumulated_set, options):
# checker = 'AmpSyncClientPool.h'
raw_data, exit_code = simur.run_process(commando, False)
my_lines = raw_data.splitlines()
if len(my_lines) in range(1, 3):
if my_lines[0].endswith('is not source indexed.'):
print(my_lines[0])
elif my_lines[0].startswith('No source information in pdb'):
# Could be a static library - skip
pass
else:
print(f'{the_pdb} has funny indexing')
my_lines = my_lines[:-1]
# for line in my_lines:
# if os.path.basename(line) == checker:
# print(f'Unprocessed {line} in {commando}')
accumulated_set |= set(my_lines)
return accumulated_set
def get_non_indexed_files(root, srcsrv, options):
srctool = os.path.join(srcsrv, 'srctool.exe')
commando = [srctool, '-r', root]
# srctool returns the number of files - not an exit code
filestring, exit_code = simur.run_process(commando, False)
if options.debug_level > 3:
print(f'{commando} returned {exit_code = }')
print(filestring)
all_files = filestring.splitlines()[:-1] # Last line is no source file
files = []
for file in all_files:
if file[0] == '*':
# Do not know what this is, but lines starting with a star ('*')
# seems to refer to non-existing .inj files in the object directory
continue
absolute_path = os.path.abspath(file)
files.append(absolute_path)
return files
def filter_pdbs(pdbs, cvdump, srcsrv, options):
lib_pdbs = []
exe_pdbs = []
for pdb_file in pdbs:
# First exclude the default vcNNN.pdb files, they are from the compiler
internal_pdb = re.match(r'.*\\vc\d+\.pdb$', pdb_file)
if internal_pdb:
print(f'Skipping {pdb_file}')
continue
# First check if srctool returns anything - then it is NOT a lib-PDB
exe_files = prepPDB.get_non_indexed_files(pdb_file, srcsrv, options)
if exe_files:
exe_pdbs.append(pdb_file)
elif cvdump:
commando = [cvdump, pdb_file]
raw_data, _exit_code = simur.run_process(commando, True)
files = libSrcTool.process_raw_cvdump_data(raw_data)
# The .pdb contained source files, append it
if files:
lib_pdbs.append(pdb_file)
return lib_pdbs, exe_pdbs
def main():
if len(sys.argv) < 2:
print("Too few arguments")
usage()
sys.exit(3)
root = sys.argv[1]
cvdump = 'cvdump.exe'
srcsrv = 'C:\\Program Files (x86)\\Windows Kits\\10\\Debuggers\\x64\\srcsrv'
if len(sys.argv) > 2:
srcsrv = sys.argv[2]
if len(sys.argv) > 3:
cvdump = sys.argv[3]
# Invoke as indexPDBs.py but keep the distance
this_dir = os.path.dirname(os.path.abspath(__file__))
script = os.path.join(this_dir, 'indexPDBs.py')
commando = ['python', script, '-t', root, '-s', srcsrv, '-c', cvdump]
outputs, exit_code = simur.run_process(commando, True)
print(outputs)
return exit_code
def get_last_line_of_srctool_reply(commando, options):
raw_data, _exit_code = simur.run_process(commando, False)
my_line = raw_data.splitlines()[-1]
return my_line
def is_in_git(file, data, git_cache, options):
debug_level = 0
report_fail = lambda dir, command: \
f'When executing in directory: {dir}\n>{command} failed'
for cached_dir in git_cache.keys(): # dict on git roots
if file.startswith(cached_dir):
git_content = git_cache[cached_dir] # dict on abs path file
if file in git_content.keys():
copy_cache_response(data, git_content[file])
return True
if debug_level > 4:
print(f'in cached directory {cached_dir}:')
print(f' {file} was not found')
# Do not return False here - it could be a WC further down the path
git_dir = get_git_dir(file)
if git_dir is None:
return False
# Make a pushd to the git dir
curr_dir = os.getcwd()
os.chdir(git_dir)
if debug_level > 4:
print(f'git-caching: {git_dir}')
#Look for remote:s
commando = 'git remote -v'
git_remote = None
reply, _exit_code = simur.run_process(commando, True)
lines = reply.splitlines()
for line in lines:
remote = re.match(r'^origin\s*(.+)\s+\(fetch\)$', line)
if remote:
git_remote = remote.group(1)
# No else - you cannot know if there is a remote
if git_remote is None:
print(f'Warning: {git_dir} has no remote')
# Get the contents of the repository
commando = 'git ls-files -s'
reply, _exit_code = simur.run_process(commando, True)
if len(reply) == 0:
os.chdir(curr_dir)
return False
if reply.startswith('fatal'): # fatal: not a git repository ...
os.chdir(curr_dir) # so it is not a fail
return False
git_dir = str(git_dir)
git_cache[git_dir] = {}
dir_cache = git_cache[git_dir]
# Iterate on lines
for line in reply.splitlines():
# 100644 2520fa373ff004b2fd4f9fa3e285b0d7d36c9319 0 script/prepPDB.py
repo = re.match(r'^\d+\s*([a-fA-F0-9]+)\s*\d+\s*(.+)$', line)
if repo:
revision = repo.group(1)
rel_key = repo.group(2)
# Make the key, i.e. that is the file path
key = os.path.join(git_dir, rel_key)
try:
key = os.path.abspath(key)
except Exception:
if debug_level > 4:
print(f'cannot handle {line}')
continue
key = str(key) # json cannot have WindowsPath as key
if options.lower_case_pdb:
key = key.lower()
dir_cache[key] = {}
cache_entry = dir_cache[key]
cache_entry['reporoot'] = git_remote
cache_entry['relpath'] = rel_key
cache_entry['revision'] = revision
cache_entry['sha1'] = revision
cache_entry['local'] = git_dir
cache_entry['remote'] = git_remote
cache_entry['vcs'] = 'git'
else:
print(report_fail(git_dir, commando))
print(f'When executing in directory: {git_dir}')
print(f'>{commando} failed')
os.chdir(curr_dir)
return False
os.chdir(curr_dir)
# We may have looked in this directory but 'file' maybe isn't under VC
if file in dir_cache.keys():
copy_cache_response(data, dir_cache[file])
return True
return False
def is_in_svn(file, data, svn_cache, options):
debug_level = 0
for cached_dir in svn_cache.keys(): # dict on svn roots
# Since svn may have externals this may fail if we have a narrower root
if file.startswith(cached_dir):
svn_content = svn_cache[cached_dir] # dict on abs path file
if file in svn_content.keys():
copy_cache_response(data, svn_content[file])
if debug_level > 4:
print(f'Found in cache: {file}')
return True
svn_dir = get_svn_dir(file)
if svn_dir is None:
return False
if str(svn_dir) in svn_cache.keys():
if debug_level > 4:
print(f'Already cached {svn_dir} - {file}')
return False
# Make a pushd to the svn dir
curr_dir = os.getcwd()
os.chdir(svn_dir)
if debug_level > 4:
print(f'svn-caching: {svn_dir} - {file}')
commando = 'svn info -R'
reply, _exit_code = simur.run_process(commando, True)
if len(reply) < 2:
if debug_level > 4:
print(f'svn info returned: {reply}')
os.chdir(curr_dir)
return False
svn_dir = str(svn_dir)
svn_cache[svn_dir] = {}
dir_cache = svn_cache[svn_dir]
lines = reply.splitlines()
path_str = 'Path: '
url_str = 'URL: '
rev_str = 'Revision: '
sha_str = 'Checksum: '
nod_str = 'Node Kind: '
# In case of doubt: use brute force
no_of_lines = len(lines)
curr_line = 0
hits = 0
# Eat the first entry - it is the root dir
while curr_line < no_of_lines:
line = lines[curr_line]
if line.startswith(url_str):
url = line[len(url_str):] # Get the repository root
curr_line += 1
if len(line) == 0:
break
path = rev = sha = node_kind = None
while curr_line < no_of_lines:
line = lines[curr_line]
if debug_level > 4:
print(f'IN: {hits} {line}')
curr_line += 1
if hits >= 3 or curr_line >= no_of_lines or len(line) == 0:
# Make the key
if debug_level > 4:
print(f'hits; {hits}')
print(f'cnt ; {curr_line}:{no_of_lines}')
print(f'len ; {len(line)}')
if not url:
print("No url")
if not path:
print("No path")
if not rev:
print("No rev")
if not sha:
print("No sha")
if not path:
path = 'None'
node_kind = 'went_wrong'
if not (url and rev and (sha or node_kind)): # DeLorean
node_kind = 'went_wrong'
try:
key = os.path.join(svn_dir, path)
key = os.path.abspath(key)
except Exception:
if debug_level > 4:
print('cannot handle the path')
# Incapacitate in case we encounter them in the future
path = 'throw_on_path'
key = 'throw_on_key'
hits = 0
node_kind = 'exception'
if node_kind == 'file':
key = str(key) # json cannot have WindowsPath as key
if options.lower_case_pdb:
key = key.lower()
cache_entry = {}
disk_rel = os.path.relpath(path)
url_rel = disk_rel.replace('\\', '/') # since disk_rel is str
cache_entry['reporoot'] = url
cache_entry['relpath'] = url_rel
cache_entry['revision'] = rev
cache_entry['sha1'] = sha
cache_entry['vcs'] = 'svn'
dir_cache[key] = cache_entry
if debug_level > 4:
print(f'Inserts: {key}')
else:
if debug_level > 4:
print(f'Skips: {node_kind} - {path}')
path = rev = sha = node_kind = None
hits = 0
if line.startswith(path_str):
path = line[len(path_str):]
hits += 1
if line.startswith(rev_str):
rev = line[len(rev_str):]
hits += 1
if line.startswith(sha_str):
sha = line[len(sha_str):]
hits += 1
if line.startswith(nod_str):
node_kind = line[len(nod_str):]
os.chdir(curr_dir)
# We may have looked in this directory but 'file' maybe isn't under VC
if file in dir_cache.keys():
copy_cache_response(data, dir_cache[file])
return True
return False
