def f_open(row, p4open={}): if not p4open: dep = _depot_path().replace('/...', '') p4open.update({ Pyforce.unescape(p['depotFile'])[len(dep) + 1:]: p['action'] for p in Pyforce('opened', dep + '/...') }) p4open['populated'] = True return row[F_PATH] in p4open
def _depot_path(): """ Returns the depot path of CWD. Result is cached in env var $DEPOT_PATH. """ if 'DEPOT_PATH' not in os.environ: os.environ['DEPOT_PATH'] = os.path.dirname( Pyforce.unescape(list(Pyforce('where', 'dummy'))[0]['depotFile'])) return os.environ['DEPOT_PATH']
def o4_clean(changelist, quick=False, resume=False, discard=False): def move_except(from_dir, to_dir, but_not): with chdir(from_dir): for f in os.listdir('.'): if f != but_not: shutil.move(f, f'{to_dir}/{f}') target = os.getcwd() source = f'{target}/.o4/cleaning' if resume: if not os.path.exists(source): sys.exit( f'*** ERROR: Cannot resume cleaning; {source} does not exist.') else: os.makedirs(f'{source}/.o4', exist_ok=True) move_except(f'{target}/.o4', f'{source}/.o4', but_not='cleaning') move_except(target, source, but_not='.o4') dep = _depot_path().replace('/...', '') p4open = [ Pyforce.unescape(p['depotFile'])[len(dep) + 1:] for p in Pyforce('opened', dep + '/...') if 'delete' not in p['action'] ] print(f"*** INFO: Not cleaning {len(p4open)} files opened for edit.") for of in p4open: if os.path.dirname(of): os.makedirs(os.path.dirname(of), exist_ok=True) shutil.move(os.path.join(source, of), of) os.chdir(target) o4bin = find_o4bin() cmd = [ o4bin, 'sync', f'.@{changelist}', '-f', '+o', '-s', source, '--move' ] if quick: cmd.append('-q') check_call(cmd) if not discard: savedir = source.replace( 'cleaning', time.strftime('cleaned')) # @%Y-%m-%d,%H:%M')) shutil.move(source, savedir) err_print( f'*** INFO: Directory is clean @{changelist}; detritus is in {savedir}' ) else: assert source.endswith('cleaning') shutil.rmtree(source)
def o4_seed_from(seed_dir, seed_fstat, op): """ For each target fstat on stdin, copy the matching file from the seed directory if 1) the seed fstat agrees, or 2) if no fstat, the checksum agrees. Output the fstat entries that were not copied. """ def no_uchg(*fnames): check_call(['chflags', 'nouchg'] + [fname for fname in fnames if os.path.exists(fname)]) def update_target(src, dest, fsop): try: try: os.makedirs(os.path.dirname(dest), exist_ok=True) fsop(src, dest) except IOError as e: if e.errno == EPERM and sys.platform == 'darwin': no_uchg(src, dest) fsop(src, dest) else: raise except IOError as e: print(f'# ERROR MOVING {src}: {e!r}') fsop = shutil.move if op == 'move' else shutil.copy2 seed_checksum = None if seed_fstat: seed_checksum = { f[F_PATH]: f[F_CHECKSUM] for f in fstat_from_csv(seed_fstat, fstat_split) if f } target_dir = os.getcwd() with chdir(seed_dir): for line in sys.stdin: if line.startswith('#o4pass'): print(line, end='') continue f = fstat_split(line) if not f: continue if f[F_CHECKSUM]: dest = os.path.join(target_dir, f[F_PATH]) if os.path.lexists(dest): try: os.unlink(dest) except IOError as e: if e.errno == EPERM and sys.platform == 'darwin': no_uchg(dest) os.unlink(dest) else: raise if seed_fstat: checksum = seed_checksum.get(f[F_PATH]) else: checksum = Pyforce.checksum(f[F_PATH], f[F_FILE_SIZE]) if f[F_FILE_SIZE].endswith( 'symlink') or checksum == f[F_CHECKSUM]: update_target(f[F_PATH], dest, fsop) print(line, end='') # line already ends with '\n'
def o4_head_update(args): res = list(args) for s in Pyforce('changes', '-s', 'submitted', '-m1', *res): for i, arg in enumerate(res): if type(arg) is int: continue # p4 rewrites path if there are no files until further down if s['path'].startswith(arg[:-3]) or arg.startswith( s['path'][:-3]): res[i] = int(s['change']) o4dir = os.environ['CLIENT_ROOT'] + arg[1:].replace( '/...', '/.o4') os.makedirs(o4dir, exist_ok=True) with open(o4dir + '/head', 'wt') as fout: print(f"{s['change']}", file=fout) break else: print("*** WARNING: Could not map result", s, file=sys.stderr) for r in res: if type(r) is not int: try: o4dir = os.environ['CLIENT_ROOT'] + r[1:].replace( '/...', '/.o4') os.unlink(o4dir + '/head') except FileNotFoundError: pass sys.exit(f"*** ERROR: Could not get HEAD for {r}") return res
def f_checksum(row): if os.path.lexists(row[F_PATH]): if not row[F_CHECKSUM]: # File is deleted if os.path.isdir(row[F_PATH]): return True elif row[F_FILE_SIZE].endswith('symlink') or Pyforce.checksum( row[F_PATH], row[F_FILE_SIZE]) == row[F_CHECKSUM]: return True else: if not row[F_CHECKSUM]: return True
def fstat_from_perforce(depot_path, upper, lower=None): """ Returns an iterator of Fstat objects where changelist is in (lower, upper]. If lower is not given, it is assumed to be 0. """ from o4_pyforce import Pyforce def fstatify(r, head=len(depot_path.replace('...', ''))): try: if r[b'code'] == b'mute': return ('0', '', '0', '0', '') t = r[b'headType'].decode('utf8') c = r.get(b'digest', b'').decode('utf8') sz = r.get(b'fileSize', b'0').decode('utf8') if t.startswith('utf') or t == 'symlink': sz = sz + '/' + t return [ r[b'headChange'].decode('utf8'), Pyforce.unescape(r[b'depotFile'].decode('utf8'))[head:], r[b'headRev'].decode('utf8'), sz, c ] except StopIteration: raise except Exception as e: print("*** ERROR: Got {!r} while fstatify({!r})".format(e, r), file=sys.stderr) raise lower = lower or 0 revs = f'@{upper}' if lower > 1: # A range going back to the beginning will get a Perforce error. assert lower <= upper revs = f'@{lower},@{upper}' pyf = Pyforce( 'fstat', '-Rc', '-Ol', '-Os', '-T', 'headType, digest, fileSize, depotFile, headChange, headRev', Pyforce.escape(depot_path) + revs) pyf.transform = fstatify return pyf
def o4_head(paths): def o4_head_update(args): res = list(args) for s in Pyforce('changes', '-s', 'submitted', '-m1', *res): for i, arg in enumerate(res): if type(arg) is int: continue # p4 rewrites path if there are no files until further down if s['path'].startswith(arg[:-3]) or arg.startswith( s['path'][:-3]): res[i] = int(s['change']) o4dir = os.environ['CLIENT_ROOT'] + arg[1:].replace( '/...', '/.o4') os.makedirs(o4dir, exist_ok=True) with open(o4dir + '/head', 'wt') as fout: print(f"{s['change']}", file=fout) break else: print("*** WARNING: Could not map result", s, file=sys.stderr) for r in res: if type(r) is not int: try: o4dir = os.environ['CLIENT_ROOT'] + r[1:].replace( '/...', '/.o4') os.unlink(o4dir + '/head') except FileNotFoundError: pass sys.exit(f"*** ERROR: Could not get HEAD for {r}") return res args = [] for depot_path in paths: if not depot_path.endswith('/...'): depot_path += '/...' args.append(Pyforce.escape(depot_path)) for retry in range(3): try: end = '' if len(args) > 1 else args[0] print( f"# {CLR}*** INFO: ({retry+1}/3) Retrieving HEAD changelist for", end, file=sys.stderr) if not end: for path in args: print(f" {path}") return o4_head_update(args) except (P4TimeoutError, IndexError): continue sys.exit( f"{CLR}*** ERROR: There was an error retrieving head change for {args}" )
def fstatify(r, head=len(depot_path.replace('...', ''))): try: if r[b'code'] == b'mute': return ('0', '', '0', '0', '') t = r[b'headType'].decode('utf8') c = r.get(b'digest', b'').decode('utf8') sz = r.get(b'fileSize', b'0').decode('utf8') if t.startswith('utf') or t == 'symlink': sz = sz + '/' + t return [ r[b'headChange'].decode('utf8'), Pyforce.unescape(r[b'depotFile'].decode('utf8'))[head:], r[b'headRev'].decode('utf8'), sz, c ] except StopIteration: raise except Exception as e: print("*** ERROR: Got {!r} while fstatify({!r})".format(e, r), file=sys.stderr) raise
def o4_sync(changelist, seed=None, seed_move=False, quick=False, force=False, skip_opened=False, verbose=True, gatling=True, manifold=True): """ Syncs CWD to changelist, as efficiently as possible. seed: Input dir for seeding. seed_move: Move seed files instead of copy. force: Go through every single file not just what's new. quick: Skip post p4 sync verification. gatling: Set to false to disable the use of gatling manifold: Set to false to disable the use of manifold Pseudo code to use also as inspiration for fault finding: CL: target changelist CUR_CL: Currently synced changelist RETRIES: How many times to attempt force sync for files that fail verification Sync the files open for edit (if the file is missing, it must be reverted): o4 fstat .<CL> [--changed <cur_CL>] | o4 keep -—open | gatling o4 pyforce sync | gatling o4 pyforce resolve -am | o4 drop --existence | o4 pyforce revert | o4 drop --existence | o4 fail Sync the files not open for edit, supply --case on macOS: o4 fstat .<CL> [--changed <cur_CL>] | o4 drop -—open [--case] | [| gatling -n 4 o4 seed-from --copy <seed>] | gatling o4 pyforce sync | | tee tmp_sync [| gatling -n 4 o4 drop --checksum | gatling o4 pyforce sync -f] * <RETRIES> o4 diff .<CL> [<cur_CL>] | o4 filter —unopen [--case] | [| gatling -n 4 o4 seed-from --copy <seed>] | gatling o4 pyforce sync | | tee tmp_sync [| gatling -n 4 o4 verify | gatling o4 pyforce sync -f] * <RETRIES> Ensure the have-list is in sync with the files: if seed or force: o4 diff .<CL> | o4 drop --havelist | gatling o4 pyforce sync -k else: cat tmp_sync | o4 drop --havelist | gatling o4 pyforce sync -k Note: manifold starts processes up front, so it's better suited for work that do not tax other equipment, such as locally calculating checksums. gatling starts and fills one process at a time and is best used with p4-related programs, to avoid lots of connections to the server. """ from tempfile import NamedTemporaryFile def clientspec_is_vanilla(): 'Return True if every View line is the same on the left and the right.' # We also want to accept a clientspec which has the same prefix # on every line on the right. E.g. # //depot/dir1 /client/pre/fix/dir1 # is acceptable if every mapping has /client/pre/fix # (This is to accomodate Autobuild clientspecs, which locate the workspace at autobuild/client) # (Turns out ABR has non-vanilla clientspecs even aside from the # prefix. Just give it an escape.) import o4_config if o4_config.allow_nonflat_clientspec(): return True client = pyforce_client() cname = client['Client'] view = [ v[1:].split(' //' + cname) for k, v in client.items() if k.startswith('View') and not v.startswith('-//') ] # Get the prefix (possibly zero-length) from the first entry. # If the first doesn't even match, it'll be empty, but then will # fail the match anyway. left, right = view[0] prefix = right[:-len(left)] for left, right in view: if prefix + left != right: return False return True def run_cmd(cmd, inputstream=None): timecmd = 'time ' if verbose else '' cmd = [c.strip() for c in cmd.split('|')] print("*** INFO: [{}]".format(os.getcwd()), ' |\n '.join(cmd).replace(o4bin, 'o4')) cmd = '|'.join(cmd) try: check_call([ '/bin/bash', '-c', f'set -o pipefail;{timecmd}{cmd}' + '|| (echo PIPESTATUS ${PIPESTATUS[@]} >.o4-pipefails; false)' ]) print() except CalledProcessError: cwd = os.getcwd() with open('.o4-pipefails') as f: fails = f.readline().rstrip().split()[1:] os.remove('.o4-pipefails') cmd = cmd.split('|') msg = [f"{CLR}*** ERROR: Pipeline failed in {cwd}:"] failures = [] for status, cmd in zip(fails, cmd): cmd = cmd.replace(o4bin, 'o4') if status == '1': status = ' FAILED ' failures.append(cmd) else: status = ' OK ' msg.append(f'{status} {cmd}') # Print the process list only if something besides "fail" failed. if len(failures) > 1 or not failures[0].endswith('o4 fail'): err_print('\n'.join(msg)) sys.exit(1) def gat(cmd): if not gatling: return '' return cmd def man(cmd): if not manifold: return '' return cmd if not clientspec_is_vanilla(): # If there was no cached client, or if we refresh it and # it's still bad, then abort. if not clear_cache('client') or not clientspec_is_vanilla(): clear_cache('client') sys.exit( '*** ERROR: o4 does not support a clientspec that maps a depot ' 'path to a non-matching local path. ' 'Are you aware that you have such a mapping? Do you need it? ' 'If not, please remove it and sync again. If so, ' 'please post to the BLT chatter group that you have such a ' 'clientspec; meanwhile you must use p4/p4v to sync.') o4bin = find_o4bin() previous_cl = 0 if os.path.exists('.o4/changelist'): with open('.o4/changelist') as fin: try: previous_cl = int(fin.read().strip()) except ValueError: print( "{CLR}*** WARNING: {os.getcwd()}/.o4/changelist could not be read", file=sys.stderr) o4_log('sync', changelist=changelist, previous_cl=previous_cl, seed=seed, seed_move=seed_move, quick=quick, force=force, skip_opened=skip_opened, verbose=verbose, gatling=gatling, manifold=manifold) verbose = ' -v' if verbose else '' force = ' -f' if force else '' fstat = f"{o4bin} fstat{force} ...@{changelist}" gatling_low = gat(f"gatling{verbose} -n 4") if previous_cl and not force: fstat += f" --changed {previous_cl}" gatling_low = '' manifold_big = man(f"manifold{verbose} -m {10*1024*1024}") gatling_verbose = gat(f"gatling{verbose}") manifold_verbose = man(f"manifold{verbose}") progress = f"| {o4bin} progress" if sys.stdin.isatty( ) and progress_enabled() else '' pyforce = 'pyforce' #pyforce = 'pyforce' + (' --debug --' if os.environ.get('DEBUG', '') else '') casefilter = ' --case' if sys.platform == 'darwin' else '' keep_case = f'| {o4bin} keep --case' if casefilter else '' if previous_cl == changelist and not force: print( f'*** INFO: {os.getcwd()} is already synced to {changelist}, use -f to force a' f' full verification.') return if os.path.exists(INCOMPLETE_INDICATOR): # Remove the indicator. If it is recreated, we will not create the # changelist file because the system is not exactly at that changelist. os.remove(INCOMPLETE_INDICATOR) if os.path.exists(SYNCED_CL_FILE): os.remove(SYNCED_CL_FILE) has_open = list(Pyforce('opened', '...')) openf = NamedTemporaryFile(dir='.o4', mode='w+t') if has_open: dep = _depot_path().replace('/...', '') print(f'*** INFO: Opened for edit in {dep}:') for i, p in enumerate(has_open): open_file_name = Pyforce.unescape(p['depotFile'])[len(dep) + 1:] print(open_file_name, file=openf) if i < 10: print(f'*** INFO: --keeping {open_file_name}') if len(has_open) > 10: print(f' (and {len(has_open) - 10} more)') openf.flush() # Resolve before syncing in case there are unresolved files for other reasons cmd = ( f"{fstat} --keep {openf.name}" f"| {gatling_verbose} -- {o4bin} {pyforce} --no-rev -- resolve -am" f"| {o4bin} {pyforce} sync" f"| {gatling_verbose} -- {o4bin} {pyforce} --no-rev -- resolve -am" f"{progress}" f"| {o4bin} drop --existence" f"| {gatling_verbose} -- {o4bin} {pyforce} --no-rev -- revert" f"| {o4bin} drop --existence" f"| {o4bin} fail") # Unopened only from here on fstat += f' --drop {openf.name}' if not skip_opened: run_cmd(cmd) else: print( f"*** INFO: Not syncing {len(has_open)} files opened for edit." ) else: print(f"{CLR}*** INFO: There are no opened files.") quiet = '-q' if seed else '' retry = (f"| {manifold_big} {o4bin} drop --checksum" f"| {gatling_verbose} {o4bin} {quiet} {pyforce} sync -f" f"| {manifold_big} {o4bin} drop --checksum" f"| {gatling_verbose} {o4bin} {quiet} {pyforce} sync -f" f"| {manifold_big} {o4bin} drop --checksum" f"| {o4bin} fail") syncit = f"| {gatling_verbose} {o4bin} {quiet} {pyforce} sync{force}" if seed: syncit = f"| {manifold_verbose} {o4bin} seed-from {seed}" _, seed_fstat = get_fstat_cache(10_000_000_000, seed + '/.o4') if seed_fstat: syncit += f" --fstat {os.path.abspath(seed_fstat)}" if seed_move: syncit += " --move" cmd = (f"{fstat} | {o4bin} drop --not-deletes --existence" f"{keep_case}" f"{progress}" f"{syncit}" f"{retry}") run_cmd(cmd) if seed: if not previous_cl: print( f"*** INFO: Flushing to changelist {changelist}, please do not interrupt" ) t0 = time.time() consume(Pyforce('-q', 'sync', '-k', f'...@{changelist}')) print("*** INFO: Flushing took {:.2f} minutes".format( (time.time() - t0) / 60)) cmd = (f"{fstat} " f"| {o4bin} drop --deletes" f"{keep_case}" f"{progress}" f"| {manifold_big} {o4bin} drop --checksum" f"{syncit}" f"{retry}") run_cmd(cmd) actual_cl, _ = get_fstat_cache(changelist) if os.path.exists(INCOMPLETE_INDICATOR): os.remove(INCOMPLETE_INDICATOR) else: with open(SYNCED_CL_FILE, 'wt') as fout: print(actual_cl, file=fout) if seed or not quick: print( "*** INFO: Sync is now locally complete, verifying server havelist." ) cmd = (f"{fstat}" f"| {o4bin} drop --havelist" f"{keep_case}" f"{progress}" f"| {gatling_low} {o4bin} {pyforce} sync -k" f"| {o4bin} drop --havelist" f"| {o4bin} fail") run_cmd(cmd) if actual_cl != changelist: print( f'*** INFO: Changelist {changelist} does not affect this directory.' ) print( f' Synced to {actual_cl} (the closest previous change that does).' ) if previous_cl == actual_cl and not force: print( f'*** INFO: {os.getcwd()} is already synced to {actual_cl}, use -f to force a' f' full verification.')
def o4_pyforce(debug, no_revision, args: list, quiet=False): """ Encapsulates Pyforce, does book keeping to ensure that all files that should be operated on are in fact dealt with by p4. Handles retry and strips out asks for files that are caseful mismatches on the current file system (macOS). """ from tempfile import NamedTemporaryFile from collections import defaultdict class LogAndAbort(Exception): 'Dumps debug information on errors.' o4_log('pyforce', no_revision=no_revision, quiet=quiet, *args) tmpf = NamedTemporaryFile(dir='.o4') fstats = [] for line in sys.stdin.read().splitlines(): if line.startswith('#o4pass'): print(line) continue f = fstat_split(line) if f and caseful_accurate(f[F_PATH]): fstats.append(f) elif f: print( f"*** WARNING: Pyforce is skipping {f[F_PATH]} because it is casefully", "mismatching a local file.", file=sys.stderr) retries = 3 head = _depot_path().replace('/...', '') while fstats: if no_revision: p4paths = [Pyforce.escape(f[F_PATH]) for f in fstats] else: p4paths = [ f"{Pyforce.escape(f[F_PATH])}#{f[F_REVISION]}" for f in fstats ] tmpf.seek(0) tmpf.truncate() not_yet = [] pargs = [] xargs = [] # This is a really bad idea, files are output to stdout before the actual # sync happens, causing checksum tests to start too early: # if len(p4paths) > 30 and 'sync' in args: # xargs.append('--parallel=threads=5') if sum(len(s) for s in p4paths) > 30000: pargs.append('-x') pargs.append(tmpf.name) for f in p4paths: tmpf.write(f.encode('utf8')) tmpf.write(b'\n') tmpf.flush() else: xargs.extend(p4paths) try: # TODO: Verbose #print('# PYFORCE({}, {}{})'.format(','.join(repr(a) for a in args), ','.join( # repr(a) for a in paths[:3]), ', ...' if len(paths) > 3 else '')) errs = [] repeats = defaultdict(list) infos = [] for res in Pyforce(*pargs, *args, *xargs): if debug: err_print("*** DEBUG: Received", repr(res)) # FIXME: Delete this if-statement: if res.get('code', '') == 'info': infos.append(res) if res.get('data', '').startswith('Diff chunks: '): continue if res.get('code', '') == 'error': errs.append(res) continue if 'resolveFlag' in res: # TODO: resolveFlag can be ...? # m: merge # c: copy from (not conflict!) # We skip this entry as it is the second returned from p4 # for one input file continue res_str = res.get('depotFile') or res.get('fromFile') if not res_str and res.get('data'): res_str = head + '/' + res['data'] if not res_str: errs.append(res) continue res_str = Pyforce.unescape(res_str) for i, f in enumerate(fstats): if f"{head}/{f[F_PATH]}" in res_str: repeats[f"{head}/{f[F_PATH]}"].append(res) not_yet.append(fstats.pop(i)) break else: for f in repeats.keys(): if f in res_str: if debug: err_print( f"*** DEBUG: REPEAT: {res_str}\n {res}\n {repeats[f]}" ) break else: if debug: err_print("*** DEBUG: ERRS APPEND", res) errs.append(res) if errs: raise LogAndAbort('Unexpected reply from p4') if len(p4paths) == len(fstats): raise LogAndAbort('Nothing recognized from p4') except P4Error as e: non_recoverable = False for a in e.args: if 'clobber writable file' in a['data']: fname = a['data'].split('clobber writable file')[1].strip() print("*** WARNING: Saving writable file as .bak:", fname, file=sys.stderr) if os.path.exists(fname + '.bak'): now = time.time() print( f"*** WARNING: Moved previous .bak to {fname}.{now}", file=sys.stderr) os.rename(fname + '.bak', f'{fname}.bak.{now}') shutil.copy(fname, fname + '.bak') os.chmod(fname, 0o400) else: non_recoverable = True if non_recoverable: raise except P4TimeoutError as e: e = str(e).replace('\n', ' ') print(f"# P4 TIMEOUT, RETRIES {retries}: {e}", file=sys.stderr) retries -= 1 if not retries: sys.exit( f"{CLR}*** ERROR: Perforce timed out too many times:\n{e}") except LogAndAbort as e: import json fname = f'debug-pyforce.{os.getpid()}.{int(time.time())}' d = { 'args': args, 'fstats': fstats, 'errs': errs, 'repeats': repeats, 'infos': infos, } json.dump(d, open(f'.o4/{fname}', 'wt')) sys.exit(f'{CLR}*** ERROR: {e}; detail in {fname}') finally: if not quiet: for fstat in not_yet: # Printing the fstats after the p4 process has ended, because p4 marshals # its objects before operation, as in "And for my next act... !" # This premature printing leads to false checksum errors during sync. print(fstat_join(fstat))
def p4_local(depot_path): py = list(Pyforce('where', depot_path))[0] if 'path' not in py: raise KeyError(f'Error resolving depot path {depot_path}') dir = py['path'].replace('/...', '') return dir