def is_in_git_raw(file, data, _dummy, _options): debug_level = 0 report_fail = lambda dir, command: \ f'When executing in directory: {dir}\n>{command} failed' git_dir = get_git_dir(file) if git_dir is None: return False # Make a pushd to the git dir curr_dir = os.getcwd() os.chdir(git_dir) commando = f'git ls-files -s "{file}"' reply, _exit_code = simur.run_process(commando, False) # git may complain if len(reply) == 0: # if it is not a repo os.chdir(curr_dir) return False if reply.startswith('fatal'): # fatal: not a git repository ... if debug_level > 4: print(report_fail(curr_dir, commando)) # so it is not a fail os.chdir(curr_dir) return False reply = reply.rstrip() if debug_level > 4: print(f'GIT: |{reply}|') repo = re.match(r'^(\d+)\s*([a-fA-F0-9]+)\s*(\d+)\s*(.+)$', reply) if repo: data['reporoot'] = str(git_dir) data['relpath'] = repo.group(4) data['revision'] = repo.group(2) data['sha1'] = repo.group(2) data['local'] = str(git_dir) else: print(report_fail(git_dir, commando)) print(f'When executing in directory: {git_dir}') print(f'>{commando} failed') os.chdir(curr_dir) return False #Look for remote:s commando = 'git remote -v' reply, _exit_code = simur.run_process(commando, True) lines = reply.splitlines() for line in lines: remote = re.match(r'^origin\s*(.+)\s+\(fetch\)$', line) if remote: data['reporoot'] = remote.group(1) data['remote'] = remote.group(1) data['vcs'] = 'git' os.chdir(curr_dir) return True
def is_in_svn_raw(file, data, _dummy, _options): debug_level = 0 svn_dir = get_svn_dir(file) if svn_dir is None: return False # Make a pushd to the svn dir curr_dir = os.getcwd() os.chdir(svn_dir) commando = 'svn info --show-item url .' reply, _exit_code = simur.run_process(commando, True) reporoot = reply.strip() disk_rel = os.path.relpath(file) url_rel = disk_rel.replace('\\', '/') commando = f'svn info "{file}"' reply, _exit_code = simur.run_process(commando, True) if len(reply) < 2: print(f'svn info returned: {reply}') os.chdir(curr_dir) return False lines = reply.splitlines() hits = 0 for line in lines: if debug_level > 4: print(line) # Just check if we get an URL: but we ignore what it says url = line.startswith('URL: ') if url: data['reporoot'] = reporoot data['relpath'] = url_rel hits += 1 continue rev = re.match(r'^Revision: (.*)$', line) if rev: data['revision'] = rev.group(1) hits += 1 continue sha1 = re.match(r'^Checksum: ([a-fA-F0-9]+)$', line) if sha1: data['sha1'] = sha1.group(1) hits += 1 continue os.chdir(curr_dir) if hits != 3: return False data['vcs'] = 'svn' return True
def make_log(srcsrv, explicit_cvdump, elapsed): bins_used = [ sys.executable, os.path.join(srcsrv, 'srctool.exe'), os.path.join(srcsrv, 'pdbstr.exe'), ] maybe_bins = [explicit_cvdump, 'git.exe', 'svn.exe', 'hg.exe'] found_bins, unfound_bins = get_available_bins(maybe_bins) bins_used += found_bins print(f'Executed by : {os.getenv("USERNAME")}') print(f' on machine : {os.getenv("COMPUTERNAME")}') print(f' SIMUR_REPO_CACHE: {os.getenv("SIMUR_REPO_CACHE")}') print(f' elapsed time : {make_time_string(elapsed)}') codepage, _exit_code = simur.run_process('cmd /c CHCP', False) the_cp = re.match(r'^.*:\s+(\d*)$', codepage) if the_cp: codepage = the_cp.group(1) print(f' CodePage : {codepage}') print('Script:') print(f' {os.path.realpath(sys.argv[0])}') print('Using binaries:') for the_exe in bins_used: the_exe = os.path.join(srcsrv, the_exe) print(f' {the_exe}:') props = simur.getFileProperties(the_exe) if props: print(f' {props["StringFileInfo"]["FileVersion"]}') print(f' {props["FileVersion"]}') if unfound_bins: print('Binaries not found/used:') for the_exe in unfound_bins: print(f' {the_exe}:')
def main(): '''Just as processPDBs only that as the first argument give the PDB of interest Example: prepPDB.py armLibSupport.pdb . //ExternalAccess/WinKit10Debuggers/srcsrv ''' if len(sys.argv) < 2: print("Too few arguments") usage() sys.exit(3) the_pdb = sys.argv[1] root = sys.argv[2] cvdump = 'cvdump.exe' srcsrv = 'C:\\Program Files (x86)\\Windows Kits\\10\\Debuggers\\x64\\srcsrv' if len(sys.argv) > 3: srcsrv = sys.argv[3] if len(sys.argv) > 4: cvdump = sys.argv[4] cvdump = libSrcTool.check_cvdump(cvdump, srcsrv) # Invoke indexPDBs.py as its own process to keep the distance this_dir = os.path.dirname(os.path.abspath(__file__)) script = os.path.join(this_dir, 'indexPDBs.py') commando = [ 'python', script, '-u', the_pdb, '-t', root, '-s', srcsrv, '-c', cvdump ] outputs, exit_code = simur.run_process(commando, True) print(outputs) return exit_code
def get_lib_source_files(pdb_file, cvdump, srcsrv, options): # First check if srctool returns anything - then it is NOT a lib-PDB srctool_files = prepPDB.get_non_indexed_files(pdb_file, srcsrv, options) if srctool_files: print(f'{pdb_file} is not a lib-PDB file - skipped') return [] commando = [cvdump, pdb_file] raw_data, _exit_code = simur.run_process(commando, True) files = process_raw_cvdump_data(raw_data) return files
def dump_stream_to_pdb(pdb_file, srcsrv, stream, options): tempfile = make_stream_file(pdb_file, stream) ''' To restore the pdb:s their .orig's --- for /R %%I in (*.*.orig) do call :doit %%I %%~nI %%~pI goto :EOF :doit pushd %3 del %2 ren %1 %2 popd --- ''' if options.backup: make_backup_file(pdb_file, '.orig') pdbstr = os.path.join(srcsrv, 'pdbstr.exe') commando = [pdbstr, '-w', '-s:srcsrv', f'-p:{pdb_file}', f'-i:{tempfile}'] simur.run_process(commando, True) os.remove(tempfile) # Or keep it for debugging
def handle_local_git(reporoot, revision): # Make a pushd to the git dir curr_dir = os.getcwd() os.chdir(reporoot) command = f'git show {revision}' reply, _exit_code = simur.run_process(command, True, extra_dir=reporoot, as_text=False) os.chdir(curr_dir) return reply
def compare_srcsrv_content(test_pdbs, facit_pdbs, options): pdb_to_find = options.beyond_compare intersection = test_pdbs.keys() & facit_pdbs.keys() print('Compare .pdb - srcsrv content:') if not pdb_to_find in intersection: print(f'Cannot find {pdb_to_find} in current selections') return 3 pdbstr = os.path.join(options.srcsrv_dir, 'pdbstr.exe') test_pdb = test_pdbs[pdb_to_find] facit_pdb = facit_pdbs[pdb_to_find] test_commando = [pdbstr, '-r', '-s:srcsrv', f'-p:{test_pdb}'] facit_commando = [pdbstr, '-r', '-s:srcsrv', f'-p:{facit_pdb}'] test_output, _exit_code = simur.run_process(test_commando, False) facit_output, _exit_code = simur.run_process(facit_commando, False) test_output = test_output.splitlines() facit_output = facit_output.splitlines() print(f'{pdb_to_find}:') diff_on_saved_files(test_output, facit_output, options) return 0
def diff_on_saved_files(test_output, facit_output, options): test_file = 'test.srcsrv' facit_file = 'facit.srcsrv' temp_dir = os.environ['TEMP'] test_file = os.path.join(temp_dir, test_file) facit_file = os.path.join(temp_dir, facit_file) write_set_to_file(test_file, test_output, options) write_set_to_file(facit_file, facit_output, options) visualdiff_commando = [ 'C:\\Program Files\\Beyond Compare 4\\BComp.exe', test_file, facit_file ] test_output, _exit_code = simur.run_process(visualdiff_commando, False) os.unlink(test_file) os.unlink(facit_file)
def handle_svn(reporoot, relpath, revision): url = reporoot + '/' + relpath command = f'svn cat {url}@{revision}' reply, _exit_code = simur.run_process(command, True, extra_dir=None, as_text=False) # Try to work around any http - https redirecting if reply.startswith(b'Redirecting to URL'): temp = bytearray(reply) slicer = temp.index(b'\r\n') + 2 reply = temp[slicer:] return reply
def handle_remote_git(reporoot, revision): git_dir = simur.find_and_update_git_cache(reporoot) if not git_dir: reply = f'Could not find a .git dir from {reporoot}\n' reply += f'when looking in {git_dir}' else: curr_dir = os.getcwd() os.chdir(git_dir) command = f'git show {revision}' reply, _exit_code = simur.run_process(command, True, extra_dir=git_dir, as_text=False) os.chdir(curr_dir) return reply
def is_indexed(root, srcsrv, options): pdbstr = os.path.join(srcsrv, 'pdbstr.exe') # read the pdb and dump its 'srcsrv' stream # - if there is a stream then it is indexed # Use a list to avoid whitespace problems in paths commando = [pdbstr, '-r', f'-p:{root}', '-s:srcsrv'] reply, exit_code = simur.run_process(commando, False) if options.debug_level > 3: print(f'{commando} returned {exit_code = }') print(f'{reply = }') # I will look at an empty reply as not indexed if len(reply) == 0: return False if not options.quiet: print(f'Sorry, {root} is already indexed or has no debug information') return True
def accumulate_srctool_reply(the_pdb, commando, accumulated_set, options): # checker = 'AmpSyncClientPool.h' raw_data, exit_code = simur.run_process(commando, False) my_lines = raw_data.splitlines() if len(my_lines) in range(1, 3): if my_lines[0].endswith('is not source indexed.'): print(my_lines[0]) elif my_lines[0].startswith('No source information in pdb'): # Could be a static library - skip pass else: print(f'{the_pdb} has funny indexing') my_lines = my_lines[:-1] # for line in my_lines: # if os.path.basename(line) == checker: # print(f'Unprocessed {line} in {commando}') accumulated_set |= set(my_lines) return accumulated_set
def get_non_indexed_files(root, srcsrv, options): srctool = os.path.join(srcsrv, 'srctool.exe') commando = [srctool, '-r', root] # srctool returns the number of files - not an exit code filestring, exit_code = simur.run_process(commando, False) if options.debug_level > 3: print(f'{commando} returned {exit_code = }') print(filestring) all_files = filestring.splitlines()[:-1] # Last line is no source file files = [] for file in all_files: if file[0] == '*': # Do not know what this is, but lines starting with a star ('*') # seems to refer to non-existing .inj files in the object directory continue absolute_path = os.path.abspath(file) files.append(absolute_path) return files
def filter_pdbs(pdbs, cvdump, srcsrv, options): lib_pdbs = [] exe_pdbs = [] for pdb_file in pdbs: # First exclude the default vcNNN.pdb files, they are from the compiler internal_pdb = re.match(r'.*\\vc\d+\.pdb$', pdb_file) if internal_pdb: print(f'Skipping {pdb_file}') continue # First check if srctool returns anything - then it is NOT a lib-PDB exe_files = prepPDB.get_non_indexed_files(pdb_file, srcsrv, options) if exe_files: exe_pdbs.append(pdb_file) elif cvdump: commando = [cvdump, pdb_file] raw_data, _exit_code = simur.run_process(commando, True) files = libSrcTool.process_raw_cvdump_data(raw_data) # The .pdb contained source files, append it if files: lib_pdbs.append(pdb_file) return lib_pdbs, exe_pdbs
def main(): if len(sys.argv) < 2: print("Too few arguments") usage() sys.exit(3) root = sys.argv[1] cvdump = 'cvdump.exe' srcsrv = 'C:\\Program Files (x86)\\Windows Kits\\10\\Debuggers\\x64\\srcsrv' if len(sys.argv) > 2: srcsrv = sys.argv[2] if len(sys.argv) > 3: cvdump = sys.argv[3] # Invoke as indexPDBs.py but keep the distance this_dir = os.path.dirname(os.path.abspath(__file__)) script = os.path.join(this_dir, 'indexPDBs.py') commando = ['python', script, '-t', root, '-s', srcsrv, '-c', cvdump] outputs, exit_code = simur.run_process(commando, True) print(outputs) return exit_code
def get_last_line_of_srctool_reply(commando, options): raw_data, _exit_code = simur.run_process(commando, False) my_line = raw_data.splitlines()[-1] return my_line
def is_in_git(file, data, git_cache, options): debug_level = 0 report_fail = lambda dir, command: \ f'When executing in directory: {dir}\n>{command} failed' for cached_dir in git_cache.keys(): # dict on git roots if file.startswith(cached_dir): git_content = git_cache[cached_dir] # dict on abs path file if file in git_content.keys(): copy_cache_response(data, git_content[file]) return True if debug_level > 4: print(f'in cached directory {cached_dir}:') print(f' {file} was not found') # Do not return False here - it could be a WC further down the path git_dir = get_git_dir(file) if git_dir is None: return False # Make a pushd to the git dir curr_dir = os.getcwd() os.chdir(git_dir) if debug_level > 4: print(f'git-caching: {git_dir}') #Look for remote:s commando = 'git remote -v' git_remote = None reply, _exit_code = simur.run_process(commando, True) lines = reply.splitlines() for line in lines: remote = re.match(r'^origin\s*(.+)\s+\(fetch\)$', line) if remote: git_remote = remote.group(1) # No else - you cannot know if there is a remote if git_remote is None: print(f'Warning: {git_dir} has no remote') # Get the contents of the repository commando = 'git ls-files -s' reply, _exit_code = simur.run_process(commando, True) if len(reply) == 0: os.chdir(curr_dir) return False if reply.startswith('fatal'): # fatal: not a git repository ... os.chdir(curr_dir) # so it is not a fail return False git_dir = str(git_dir) git_cache[git_dir] = {} dir_cache = git_cache[git_dir] # Iterate on lines for line in reply.splitlines(): # 100644 2520fa373ff004b2fd4f9fa3e285b0d7d36c9319 0 script/prepPDB.py repo = re.match(r'^\d+\s*([a-fA-F0-9]+)\s*\d+\s*(.+)$', line) if repo: revision = repo.group(1) rel_key = repo.group(2) # Make the key, i.e. that is the file path key = os.path.join(git_dir, rel_key) try: key = os.path.abspath(key) except Exception: if debug_level > 4: print(f'cannot handle {line}') continue key = str(key) # json cannot have WindowsPath as key if options.lower_case_pdb: key = key.lower() dir_cache[key] = {} cache_entry = dir_cache[key] cache_entry['reporoot'] = git_remote cache_entry['relpath'] = rel_key cache_entry['revision'] = revision cache_entry['sha1'] = revision cache_entry['local'] = git_dir cache_entry['remote'] = git_remote cache_entry['vcs'] = 'git' else: print(report_fail(git_dir, commando)) print(f'When executing in directory: {git_dir}') print(f'>{commando} failed') os.chdir(curr_dir) return False os.chdir(curr_dir) # We may have looked in this directory but 'file' maybe isn't under VC if file in dir_cache.keys(): copy_cache_response(data, dir_cache[file]) return True return False
def is_in_svn(file, data, svn_cache, options): debug_level = 0 for cached_dir in svn_cache.keys(): # dict on svn roots # Since svn may have externals this may fail if we have a narrower root if file.startswith(cached_dir): svn_content = svn_cache[cached_dir] # dict on abs path file if file in svn_content.keys(): copy_cache_response(data, svn_content[file]) if debug_level > 4: print(f'Found in cache: {file}') return True svn_dir = get_svn_dir(file) if svn_dir is None: return False if str(svn_dir) in svn_cache.keys(): if debug_level > 4: print(f'Already cached {svn_dir} - {file}') return False # Make a pushd to the svn dir curr_dir = os.getcwd() os.chdir(svn_dir) if debug_level > 4: print(f'svn-caching: {svn_dir} - {file}') commando = 'svn info -R' reply, _exit_code = simur.run_process(commando, True) if len(reply) < 2: if debug_level > 4: print(f'svn info returned: {reply}') os.chdir(curr_dir) return False svn_dir = str(svn_dir) svn_cache[svn_dir] = {} dir_cache = svn_cache[svn_dir] lines = reply.splitlines() path_str = 'Path: ' url_str = 'URL: ' rev_str = 'Revision: ' sha_str = 'Checksum: ' nod_str = 'Node Kind: ' # In case of doubt: use brute force no_of_lines = len(lines) curr_line = 0 hits = 0 # Eat the first entry - it is the root dir while curr_line < no_of_lines: line = lines[curr_line] if line.startswith(url_str): url = line[len(url_str):] # Get the repository root curr_line += 1 if len(line) == 0: break path = rev = sha = node_kind = None while curr_line < no_of_lines: line = lines[curr_line] if debug_level > 4: print(f'IN: {hits} {line}') curr_line += 1 if hits >= 3 or curr_line >= no_of_lines or len(line) == 0: # Make the key if debug_level > 4: print(f'hits; {hits}') print(f'cnt ; {curr_line}:{no_of_lines}') print(f'len ; {len(line)}') if not url: print("No url") if not path: print("No path") if not rev: print("No rev") if not sha: print("No sha") if not path: path = 'None' node_kind = 'went_wrong' if not (url and rev and (sha or node_kind)): # DeLorean node_kind = 'went_wrong' try: key = os.path.join(svn_dir, path) key = os.path.abspath(key) except Exception: if debug_level > 4: print('cannot handle the path') # Incapacitate in case we encounter them in the future path = 'throw_on_path' key = 'throw_on_key' hits = 0 node_kind = 'exception' if node_kind == 'file': key = str(key) # json cannot have WindowsPath as key if options.lower_case_pdb: key = key.lower() cache_entry = {} disk_rel = os.path.relpath(path) url_rel = disk_rel.replace('\\', '/') # since disk_rel is str cache_entry['reporoot'] = url cache_entry['relpath'] = url_rel cache_entry['revision'] = rev cache_entry['sha1'] = sha cache_entry['vcs'] = 'svn' dir_cache[key] = cache_entry if debug_level > 4: print(f'Inserts: {key}') else: if debug_level > 4: print(f'Skips: {node_kind} - {path}') path = rev = sha = node_kind = None hits = 0 if line.startswith(path_str): path = line[len(path_str):] hits += 1 if line.startswith(rev_str): rev = line[len(rev_str):] hits += 1 if line.startswith(sha_str): sha = line[len(sha_str):] hits += 1 if line.startswith(nod_str): node_kind = line[len(nod_str):] os.chdir(curr_dir) # We may have looked in this directory but 'file' maybe isn't under VC if file in dir_cache.keys(): copy_cache_response(data, dir_cache[file]) return True return False