def o4_pyforce(debug, no_revision, args: list, quiet=False):
"""
Encapsulates Pyforce, does book keeping to ensure that all files
that should be operated on are in fact dealt with by p4. Handles
retry and strips out asks for files that are caseful mismatches on
the current file system (macOS).
"""
from tempfile import NamedTemporaryFile
from collections import defaultdict
class LogAndAbort(Exception):
'Dumps debug information on errors.'
o4_log('pyforce', no_revision=no_revision, quiet=quiet, *args)
tmpf = NamedTemporaryFile(dir='.o4')
fstats = []
for line in sys.stdin.read().splitlines():
if line.startswith('#o4pass'):
print(line)
continue
f = fstat_split(line)
if f and caseful_accurate(f[F_PATH]):
fstats.append(f)
elif f:
print(
f"*** WARNING: Pyforce is skipping {f[F_PATH]} because it is casefully",
"mismatching a local file.",
file=sys.stderr)
retries = 3
head = _depot_path().replace('/...', '')
while fstats:
if no_revision:
p4paths = [Pyforce.escape(f[F_PATH]) for f in fstats]
else:
p4paths = [
f"{Pyforce.escape(f[F_PATH])}#{f[F_REVISION]}" for f in fstats
]
tmpf.seek(0)
tmpf.truncate()
not_yet = []
pargs = []
xargs = []
# This is a really bad idea, files are output to stdout before the actual
# sync happens, causing checksum tests to start too early:
# if len(p4paths) > 30 and 'sync' in args:
# xargs.append('--parallel=threads=5')
if sum(len(s) for s in p4paths) > 30000:
pargs.append('-x')
pargs.append(tmpf.name)
for f in p4paths:
tmpf.write(f.encode('utf8'))
tmpf.write(b'\n')
tmpf.flush()
else:
xargs.extend(p4paths)
try:
# TODO: Verbose
#print('# PYFORCE({}, {}{})'.format(','.join(repr(a) for a in args), ','.join(
# repr(a) for a in paths[:3]), ', ...' if len(paths) > 3 else ''))
errs = []
repeats = defaultdict(list)
infos = []
for res in Pyforce(*pargs, *args, *xargs):
if debug:
err_print("*** DEBUG: Received", repr(res))
# FIXME: Delete this if-statement:
if res.get('code', '') == 'info':
infos.append(res)
if res.get('data', '').startswith('Diff chunks: '):
continue
if res.get('code', '') == 'error':
errs.append(res)
continue
if 'resolveFlag' in res:
# TODO: resolveFlag can be ...?
# m: merge
# c: copy from (not conflict!)
# We skip this entry as it is the second returned from p4
# for one input file
continue
res_str = res.get('depotFile') or res.get('fromFile')
if not res_str and res.get('data'):
res_str = head + '/' + res['data']
if not res_str:
errs.append(res)
continue
res_str = Pyforce.unescape(res_str)
for i, f in enumerate(fstats):
if f"{head}/{f[F_PATH]}" in res_str:
repeats[f"{head}/{f[F_PATH]}"].append(res)
not_yet.append(fstats.pop(i))
break
else:
for f in repeats.keys():
if f in res_str:
if debug:
err_print(
f"*** DEBUG: REPEAT: {res_str}\n {res}\n {repeats[f]}"
)
break
else:
if debug:
err_print("*** DEBUG: ERRS APPEND", res)
errs.append(res)
if errs:
raise LogAndAbort('Unexpected reply from p4')
if len(p4paths) == len(fstats):
raise LogAndAbort('Nothing recognized from p4')
except P4Error as e:
non_recoverable = False
for a in e.args:
if 'clobber writable file' in a['data']:
fname = a['data'].split('clobber writable file')[1].strip()
print("*** WARNING: Saving writable file as .bak:",
fname,
file=sys.stderr)
if os.path.exists(fname + '.bak'):
now = time.time()
print(
f"*** WARNING: Moved previous .bak to {fname}.{now}",
file=sys.stderr)
os.rename(fname + '.bak', f'{fname}.bak.{now}')
shutil.copy(fname, fname + '.bak')
os.chmod(fname, 0o400)
else:
non_recoverable = True
if non_recoverable:
raise
except P4TimeoutError as e:
e = str(e).replace('\n', ' ')
print(f"# P4 TIMEOUT, RETRIES {retries}: {e}", file=sys.stderr)
retries -= 1
if not retries:
sys.exit(
f"{CLR}*** ERROR: Perforce timed out too many times:\n{e}")
except LogAndAbort as e:
import json
fname = f'debug-pyforce.{os.getpid()}.{int(time.time())}'
d = {
'args': args,
'fstats': fstats,
'errs': errs,
'repeats': repeats,
'infos': infos,
}
json.dump(d, open(f'.o4/{fname}', 'wt'))
sys.exit(f'{CLR}*** ERROR: {e}; detail in {fname}')
finally:
if not quiet:
for fstat in not_yet:
# Printing the fstats after the p4 process has ended, because p4 marshals
# its objects before operation, as in "And for my next act... !"
# This premature printing leads to false checksum errors during sync.
print(fstat_join(fstat))
def o4_sync(changelist,
seed=None,
seed_move=False,
quick=False,
force=False,
skip_opened=False,
verbose=True,
gatling=True,
manifold=True):
""" Syncs CWD to changelist, as efficiently as possible.
seed: Input dir for seeding.
seed_move: Move seed files instead of copy.
force: Go through every single file not just what's new.
quick: Skip post p4 sync verification.
gatling: Set to false to disable the use of gatling
manifold: Set to false to disable the use of manifold
Pseudo code to use also as inspiration for fault finding:
CL: target changelist
CUR_CL: Currently synced changelist
RETRIES: How many times to attempt force sync for files that fail verification
Sync the files open for edit (if the file is missing, it must be reverted):
o4 fstat .<CL> [--changed <cur_CL>] | o4 keep -—open | gatling o4 pyforce sync |
gatling o4 pyforce resolve -am | o4 drop --existence | o4 pyforce revert |
o4 drop --existence | o4 fail
Sync the files not open for edit, supply --case on macOS:
o4 fstat .<CL> [--changed <cur_CL>] | o4 drop -—open [--case] |
[| gatling -n 4 o4 seed-from --copy <seed>] | gatling o4 pyforce sync |
| tee tmp_sync
[| gatling -n 4 o4 drop --checksum | gatling o4 pyforce sync -f] * <RETRIES>
o4 diff .<CL> [<cur_CL>] | o4 filter —unopen [--case] |
[| gatling -n 4 o4 seed-from --copy <seed>] | gatling o4 pyforce sync |
| tee tmp_sync
[| gatling -n 4 o4 verify | gatling o4 pyforce sync -f] * <RETRIES>
Ensure the have-list is in sync with the files:
if seed or force:
o4 diff .<CL> | o4 drop --havelist | gatling o4 pyforce sync -k
else:
cat tmp_sync | o4 drop --havelist | gatling o4 pyforce sync -k
Note: manifold starts processes up front, so it's better suited for work
that do not tax other equipment, such as locally calculating checksums.
gatling starts and fills one process at a time and is best used
with p4-related programs, to avoid lots of connections to the server.
"""
from tempfile import NamedTemporaryFile
def clientspec_is_vanilla():
'Return True if every View line is the same on the left and the right.'
# We also want to accept a clientspec which has the same prefix
# on every line on the right. E.g.
# //depot/dir1 /client/pre/fix/dir1
# is acceptable if every mapping has /client/pre/fix
# (This is to accomodate Autobuild clientspecs, which locate the workspace at autobuild/client)
# (Turns out ABR has non-vanilla clientspecs even aside from the
# prefix. Just give it an escape.)
import o4_config
if o4_config.allow_nonflat_clientspec():
return True
client = pyforce_client()
cname = client['Client']
view = [
v[1:].split(' //' + cname) for k, v in client.items()
if k.startswith('View') and not v.startswith('-//')
]
# Get the prefix (possibly zero-length) from the first entry.
# If the first doesn't even match, it'll be empty, but then will
# fail the match anyway.
left, right = view[0]
prefix = right[:-len(left)]
for left, right in view:
if prefix + left != right:
return False
return True
def run_cmd(cmd, inputstream=None):
timecmd = 'time ' if verbose else ''
cmd = [c.strip() for c in cmd.split('|')]
print("*** INFO: [{}]".format(os.getcwd()),
' |\n '.join(cmd).replace(o4bin, 'o4'))
cmd = '|'.join(cmd)
try:
check_call([
'/bin/bash', '-c', f'set -o pipefail;{timecmd}{cmd}' +
'|| (echo PIPESTATUS ${PIPESTATUS[@]} >.o4-pipefails; false)'
])
print()
except CalledProcessError:
cwd = os.getcwd()
with open('.o4-pipefails') as f:
fails = f.readline().rstrip().split()[1:]
os.remove('.o4-pipefails')
cmd = cmd.split('|')
msg = [f"{CLR}*** ERROR: Pipeline failed in {cwd}:"]
failures = []
for status, cmd in zip(fails, cmd):
cmd = cmd.replace(o4bin, 'o4')
if status == '1':
status = ' FAILED '
failures.append(cmd)
else:
status = ' OK '
msg.append(f'{status} {cmd}')
# Print the process list only if something besides "fail" failed.
if len(failures) > 1 or not failures[0].endswith('o4 fail'):
err_print('\n'.join(msg))
sys.exit(1)
def gat(cmd):
if not gatling:
return ''
return cmd
def man(cmd):
if not manifold:
return ''
return cmd
if not clientspec_is_vanilla():
# If there was no cached client, or if we refresh it and
# it's still bad, then abort.
if not clear_cache('client') or not clientspec_is_vanilla():
clear_cache('client')
sys.exit(
'*** ERROR: o4 does not support a clientspec that maps a depot '
'path to a non-matching local path. '
'Are you aware that you have such a mapping? Do you need it? '
'If not, please remove it and sync again. If so, '
'please post to the BLT chatter group that you have such a '
'clientspec; meanwhile you must use p4/p4v to sync.')
o4bin = find_o4bin()
previous_cl = 0
if os.path.exists('.o4/changelist'):
with open('.o4/changelist') as fin:
try:
previous_cl = int(fin.read().strip())
except ValueError:
print(
"{CLR}*** WARNING: {os.getcwd()}/.o4/changelist could not be read",
file=sys.stderr)
o4_log('sync',
changelist=changelist,
previous_cl=previous_cl,
seed=seed,
seed_move=seed_move,
quick=quick,
force=force,
skip_opened=skip_opened,
verbose=verbose,
gatling=gatling,
manifold=manifold)
verbose = ' -v' if verbose else ''
force = ' -f' if force else ''
fstat = f"{o4bin} fstat{force} ...@{changelist}"
gatling_low = gat(f"gatling{verbose} -n 4")
if previous_cl and not force:
fstat += f" --changed {previous_cl}"
gatling_low = ''
manifold_big = man(f"manifold{verbose} -m {10*1024*1024}")
gatling_verbose = gat(f"gatling{verbose}")
manifold_verbose = man(f"manifold{verbose}")
progress = f"| {o4bin} progress" if sys.stdin.isatty(
) and progress_enabled() else ''
pyforce = 'pyforce' #pyforce = 'pyforce' + (' --debug --' if os.environ.get('DEBUG', '') else '')
casefilter = ' --case' if sys.platform == 'darwin' else ''
keep_case = f'| {o4bin} keep --case' if casefilter else ''
if previous_cl == changelist and not force:
print(
f'*** INFO: {os.getcwd()} is already synced to {changelist}, use -f to force a'
f' full verification.')
return
if os.path.exists(INCOMPLETE_INDICATOR):
# Remove the indicator. If it is recreated, we will not create the
# changelist file because the system is not exactly at that changelist.
os.remove(INCOMPLETE_INDICATOR)
if os.path.exists(SYNCED_CL_FILE):
os.remove(SYNCED_CL_FILE)
has_open = list(Pyforce('opened', '...'))
openf = NamedTemporaryFile(dir='.o4', mode='w+t')
if has_open:
dep = _depot_path().replace('/...', '')
print(f'*** INFO: Opened for edit in {dep}:')
for i, p in enumerate(has_open):
open_file_name = Pyforce.unescape(p['depotFile'])[len(dep) + 1:]
print(open_file_name, file=openf)
if i < 10:
print(f'*** INFO: --keeping {open_file_name}')
if len(has_open) > 10:
print(f' (and {len(has_open) - 10} more)')
openf.flush()
# Resolve before syncing in case there are unresolved files for other reasons
cmd = (
f"{fstat} --keep {openf.name}"
f"| {gatling_verbose} -- {o4bin} {pyforce} --no-rev -- resolve -am"
f"| {o4bin} {pyforce} sync"
f"| {gatling_verbose} -- {o4bin} {pyforce} --no-rev -- resolve -am"
f"{progress}"
f"| {o4bin} drop --existence"
f"| {gatling_verbose} -- {o4bin} {pyforce} --no-rev -- revert"
f"| {o4bin} drop --existence"
f"| {o4bin} fail")
# Unopened only from here on
fstat += f' --drop {openf.name}'
if not skip_opened:
run_cmd(cmd)
else:
print(
f"*** INFO: Not syncing {len(has_open)} files opened for edit."
)
else:
print(f"{CLR}*** INFO: There are no opened files.")
quiet = '-q' if seed else ''
retry = (f"| {manifold_big} {o4bin} drop --checksum"
f"| {gatling_verbose} {o4bin} {quiet} {pyforce} sync -f"
f"| {manifold_big} {o4bin} drop --checksum"
f"| {gatling_verbose} {o4bin} {quiet} {pyforce} sync -f"
f"| {manifold_big} {o4bin} drop --checksum"
f"| {o4bin} fail")
syncit = f"| {gatling_verbose} {o4bin} {quiet} {pyforce} sync{force}"
if seed:
syncit = f"| {manifold_verbose} {o4bin} seed-from {seed}"
_, seed_fstat = get_fstat_cache(10_000_000_000, seed + '/.o4')
if seed_fstat:
syncit += f" --fstat {os.path.abspath(seed_fstat)}"
if seed_move:
syncit += " --move"
cmd = (f"{fstat} | {o4bin} drop --not-deletes --existence"
f"{keep_case}"
f"{progress}"
f"{syncit}"
f"{retry}")
run_cmd(cmd)
if seed:
if not previous_cl:
print(
f"*** INFO: Flushing to changelist {changelist}, please do not interrupt"
)
t0 = time.time()
consume(Pyforce('-q', 'sync', '-k', f'...@{changelist}'))
print("*** INFO: Flushing took {:.2f} minutes".format(
(time.time() - t0) / 60))
cmd = (f"{fstat} "
f"| {o4bin} drop --deletes"
f"{keep_case}"
f"{progress}"
f"| {manifold_big} {o4bin} drop --checksum"
f"{syncit}"
f"{retry}")
run_cmd(cmd)
actual_cl, _ = get_fstat_cache(changelist)
if os.path.exists(INCOMPLETE_INDICATOR):
os.remove(INCOMPLETE_INDICATOR)
else:
with open(SYNCED_CL_FILE, 'wt') as fout:
print(actual_cl, file=fout)
if seed or not quick:
print(
"*** INFO: Sync is now locally complete, verifying server havelist."
)
cmd = (f"{fstat}"
f"| {o4bin} drop --havelist"
f"{keep_case}"
f"{progress}"
f"| {gatling_low} {o4bin} {pyforce} sync -k"
f"| {o4bin} drop --havelist"
f"| {o4bin} fail")
run_cmd(cmd)
if actual_cl != changelist:
print(
f'*** INFO: Changelist {changelist} does not affect this directory.'
)
print(
f' Synced to {actual_cl} (the closest previous change that does).'
)
if previous_cl == actual_cl and not force:
print(
f'*** INFO: {os.getcwd()} is already synced to {actual_cl}, use -f to force a'
f' full verification.')
def o4_fstat(changelist,
previous_cl,
drop=None,
keep=None,
quiet=False,
force=False):
"""
changelist: Target changelist
previous_cl: Previous_cl if known (otherwise 0)
drop: Input file name with a line-by-line file list of filenames
to exclude from output
keep: Input file name with a line-by-line file list of filenames
to limit output to
force: output all fstat lines even though previous_cl is set. This only affects
fstat when previous_cl is more recent than changelist (reverse sync).
Missing fstat files for changelist and previous_cl are generated automatically.
Streams to stdout the fstat CSV from .o4/<changelist>.fstat.gz
IF not previous_cl:
Stream every entry from changelist (essentially gzcat), while
applying drop_keep if given.
IF previous_cl == changelist:
Stream nothing.
IF previous_cl < changelist:
Only items in changelist that are newer than previous_cl are
streamed. Apply drop_keep.
IF previous_cl > changelist
This reverse sync scenario is a little complicated:
* Use the forward sync iterator to determine all files that
should be synced.
* Find branched or added files and generate false entries to
have them deleted:
'<changelist>,0,0,reverse_sync/delete,text,,<path>'
* Add all other file names to KEEP and stream matches at
<changelist>.
DROP: Exclude fstat from the stream, if the fstat path is in the
drop-file.
KEEP: Limit fstat from the stream to paths listed in the
keep-file.
"""
if os.environ.get('DEBUG', ''):
print(f"""# o4 fstat {os.getcwd()}
# changelist: {changelist}
# previous_cl: {previous_cl}
# drop: {drop}
# keep: {keep}
# quiet: {quiet}""",
file=sys.stderr)
o4_log('fstat',
_depot_path(),
changelist=changelist,
previous_cl=previous_cl,
drop=drop,
keep=keep,
quiet=quiet,
force=force)
if previous_cl:
previous_cl = int(previous_cl)
if previous_cl == changelist:
return changelist
else:
previous_cl = 0
if quiet:
if drop or keep:
sys.exit("*** ERROR: Quiet fstat does not support drop or keep.")
actual_cl = max(
int(f.split(',', 1)[0])
for f in fstat_iter(_depot_path(), changelist, previous_cl))
print(f'*** INFO: Created {os.getcwd()}/.o4/{actual_cl}.fstat.gz')
return actual_cl
if drop:
with open(drop, 'rt', encoding='utf8') as fin:
drop = set(f[:-1] for f in fin)
if keep:
with open(keep, 'rt', encoding='utf8') as fin:
keep = set(f[:-1] for f in fin)
if previous_cl and previous_cl > changelist:
# Syncing backwards requires us to delete files that were added
# between the lower and higher changelist. All other files must
# be synced to their state at the lower changelist.
past_filenames = set(p for p, _ in map(
fstat_path,
progress_iter(fstat_iter(_depot_path(), changelist),
os.getcwd() + '/.o4/.fstat', 'fstat-reverse')) if p)
if not keep:
keep = set()
if not drop:
drop = set()
for f in map(
fstat_split,
progress_iter(
fstat_iter(_depot_path(), previous_cl, changelist),
os.getcwd() + '/.o4/.fstat', 'fstat-reverse')):
if not f:
continue
if f[F_PATH] not in past_filenames:
print(f'{changelist},{f[F_PATH]},0,0,')
if force:
drop.add(f[F_PATH])
elif not force:
keep.add(f[F_PATH])
previous_cl = 0
if drop and keep:
# Prioritize dropping over keeping, if a file is in both.
# Any file that is currently opened by Perforce must be dropped.
# This function assumes that a supplied drop list is a list of
# open files (which then gets augmented (above) with files that
# did not exist at the lower changelist).
keep = keep.difference(drop)
drop_n = 0 if not drop else len(drop)
keep_n = 0 if not keep else len(keep)
if not drop:
drop = None
if not keep:
keep = None
fstats = progress_iter(fstat_iter(_depot_path(), changelist, previous_cl),
os.getcwd() + '/.o4/.fstat', 'fstat')
# Can't break out of fstat_iter without risking that the local
# cache is not created, causing fstat_from_perforce to be called
# twice, so we use an iterator that we can drain.
for line in fstats:
if keep is not None or drop is not None:
path, line = fstat_path(line)
if not path:
continue
if drop is not None:
drop.discard(path)
if len(drop) != drop_n:
drop_n -= 1
if not drop_n:
drop = None
continue
if keep is not None:
keep.discard(path)
if len(keep) == keep_n:
continue
keep_n -= 1
if not keep_n:
print(line)
# Make sure the iterator is consumed, so that
# local cache is created.
sum(0 for line in fstats)
break
print(line)
actual_cl, fname = get_fstat_cache(changelist)
return actual_cl