def o4_seed_from(seed_dir, seed_fstat, op):
"""
For each target fstat on stdin, copy the matching file from the
seed directory if 1) the seed fstat agrees, or 2) if no fstat, the
checksum agrees. Output the fstat entries that were not copied.
"""
def no_uchg(*fnames):
check_call(['chflags', 'nouchg'] +
[fname for fname in fnames if os.path.exists(fname)])
def update_target(src, dest, fsop):
try:
try:
os.makedirs(os.path.dirname(dest), exist_ok=True)
fsop(src, dest)
except IOError as e:
if e.errno == EPERM and sys.platform == 'darwin':
no_uchg(src, dest)
fsop(src, dest)
else:
raise
except IOError as e:
print(f'# ERROR MOVING {src}: {e!r}')
fsop = shutil.move if op == 'move' else shutil.copy2
seed_checksum = None
if seed_fstat:
seed_checksum = {
f[F_PATH]: f[F_CHECKSUM]
for f in fstat_from_csv(seed_fstat, fstat_split) if f
}
target_dir = os.getcwd()
with chdir(seed_dir):
for line in sys.stdin:
if line.startswith('#o4pass'):
print(line, end='')
continue
f = fstat_split(line)
if not f:
continue
if f[F_CHECKSUM]:
dest = os.path.join(target_dir, f[F_PATH])
if os.path.lexists(dest):
try:
os.unlink(dest)
except IOError as e:
if e.errno == EPERM and sys.platform == 'darwin':
no_uchg(dest)
os.unlink(dest)
else:
raise
if seed_fstat:
checksum = seed_checksum.get(f[F_PATH])
else:
checksum = Pyforce.checksum(f[F_PATH], f[F_FILE_SIZE])
if f[F_FILE_SIZE].endswith(
'symlink') or checksum == f[F_CHECKSUM]:
update_target(f[F_PATH], dest, fsop)
print(line, end='') # line already ends with '\n'
def o4_fail():
files = []
passthroughs = []
n = 0
for line in sys.stdin:
if line.startswith('#o4pass-'):
msgtype, _, msg = line.replace('#o4pass-', '').partition('#')
if msgtype == 'warn':
passthroughs.append(msg)
continue
f = fstat_split(line)
if not f:
continue
n += 1
if n < 100:
files.append(f" {f[F_PATH]}#{f[F_REVISION]}")
if not files and not passthroughs:
sys.exit(0)
err_print()
hdr = ' o4 ERROR ' if files else ' o4 WARNING '
l = (78 - len(hdr)) // 2
hdr = '*' * l + hdr + '*' * l
ftr = '*' * len(hdr)
err_print(f'{CLR}{hdr}')
if files:
err_print('These files did not sync and are ERRORs')
err_print('\n'.join(sorted(files)))
if len(files) != n:
err_print(f' ...and {n-len(files)} others!')
err_print(f'\n{ftr}')
if passthroughs:
err_print('These files did not sync and are WARNINGs')
err_print('\n'.join(sorted(passthroughs)))
err_print(ftr)
if not files:
with open(INCOMPLETE_INDICATOR, 'w') as f:
pass
err_print(
f'{CLR} o4 IS EXITING WITH SUCCESS EVEN THOUGH THOSE FILES ARE NOT UP-TO-DATE'
)
err_print(ftr)
sys.exit(0)
err_print('BECAUSE o4 DID NOT COMPLETE, THERE MAY BE OTHER FILES')
err_print('BESIDES THOSE LISTED THAT ARE NOT CORRECTLY SYNCED.')
s = '' if n == 1 else 's'
sys.exit(f'{CLR}*** ERROR: Pipeline ended with {n} file{s} rejected.')
def o4_drop_have(verbose=False):
import time
from bisect import bisect_left
pre = len(_depot_path().replace('/...', '')) + 1
have = None
# We have to wait with pulling the server havelist until we have the input in its entirety
lines = sys.stdin.read().splitlines()
for line in lines:
if line.startswith('#o4pass'):
print(line)
continue
if not have:
if have is not None:
print(line)
continue
t0 = time.time()
# We are getting have list in text mode because marshalled python objects are too slow
have = check_output(['p4', 'have', '...'],
encoding=sys.stdout.encoding,
stderr=DEVNULL)
if verbose:
t0, t1 = time.time(), t0
err_print("# HAVELIST", t0 - t1)
have = [h[pre:] for h in have.splitlines()]
if verbose:
t0, t1 = time.time(), t0
err_print("# SPLIT", t0 - t1)
have.sort()
if verbose:
t0, t1 = time.time(), t0
err_print("# SORT", t0 - t1)
f = fstat_split(line)
if not f:
continue
needle = f"{Pyforce.escape(f[F_PATH])}#{f[F_REVISION]} -"
i = bisect_left(have, needle)
miss = (i == len(have)) or not have[i].startswith(needle)
if miss and f[F_CHECKSUM]:
print(line)
if verbose:
t0, t1 = time.time(), t0
err_print("# BISECT", t0 - t1)
def o4_pyforce(debug, no_revision, args: list, quiet=False):
"""
Encapsulates Pyforce, does book keeping to ensure that all files
that should be operated on are in fact dealt with by p4. Handles
retry and strips out asks for files that are caseful mismatches on
the current file system (macOS).
"""
from tempfile import NamedTemporaryFile
from collections import defaultdict
class LogAndAbort(Exception):
'Dumps debug information on errors.'
o4_log('pyforce', no_revision=no_revision, quiet=quiet, *args)
tmpf = NamedTemporaryFile(dir='.o4')
fstats = []
for line in sys.stdin.read().splitlines():
if line.startswith('#o4pass'):
print(line)
continue
f = fstat_split(line)
if f and caseful_accurate(f[F_PATH]):
fstats.append(f)
elif f:
print(
f"*** WARNING: Pyforce is skipping {f[F_PATH]} because it is casefully",
"mismatching a local file.",
file=sys.stderr)
retries = 3
head = _depot_path().replace('/...', '')
while fstats:
if no_revision:
p4paths = [Pyforce.escape(f[F_PATH]) for f in fstats]
else:
p4paths = [
f"{Pyforce.escape(f[F_PATH])}#{f[F_REVISION]}" for f in fstats
]
tmpf.seek(0)
tmpf.truncate()
not_yet = []
pargs = []
xargs = []
# This is a really bad idea, files are output to stdout before the actual
# sync happens, causing checksum tests to start too early:
# if len(p4paths) > 30 and 'sync' in args:
# xargs.append('--parallel=threads=5')
if sum(len(s) for s in p4paths) > 30000:
pargs.append('-x')
pargs.append(tmpf.name)
for f in p4paths:
tmpf.write(f.encode('utf8'))
tmpf.write(b'\n')
tmpf.flush()
else:
xargs.extend(p4paths)
try:
# TODO: Verbose
#print('# PYFORCE({}, {}{})'.format(','.join(repr(a) for a in args), ','.join(
# repr(a) for a in paths[:3]), ', ...' if len(paths) > 3 else ''))
errs = []
repeats = defaultdict(list)
infos = []
for res in Pyforce(*pargs, *args, *xargs):
if debug:
err_print("*** DEBUG: Received", repr(res))
# FIXME: Delete this if-statement:
if res.get('code', '') == 'info':
infos.append(res)
if res.get('data', '').startswith('Diff chunks: '):
continue
if res.get('code', '') == 'error':
errs.append(res)
continue
if 'resolveFlag' in res:
# TODO: resolveFlag can be ...?
# m: merge
# c: copy from (not conflict!)
# We skip this entry as it is the second returned from p4
# for one input file
continue
res_str = res.get('depotFile') or res.get('fromFile')
if not res_str and res.get('data'):
res_str = head + '/' + res['data']
if not res_str:
errs.append(res)
continue
res_str = Pyforce.unescape(res_str)
for i, f in enumerate(fstats):
if f"{head}/{f[F_PATH]}" in res_str:
repeats[f"{head}/{f[F_PATH]}"].append(res)
not_yet.append(fstats.pop(i))
break
else:
for f in repeats.keys():
if f in res_str:
if debug:
err_print(
f"*** DEBUG: REPEAT: {res_str}\n {res}\n {repeats[f]}"
)
break
else:
if debug:
err_print("*** DEBUG: ERRS APPEND", res)
errs.append(res)
if errs:
raise LogAndAbort('Unexpected reply from p4')
if len(p4paths) == len(fstats):
raise LogAndAbort('Nothing recognized from p4')
except P4Error as e:
non_recoverable = False
for a in e.args:
if 'clobber writable file' in a['data']:
fname = a['data'].split('clobber writable file')[1].strip()
print("*** WARNING: Saving writable file as .bak:",
fname,
file=sys.stderr)
if os.path.exists(fname + '.bak'):
now = time.time()
print(
f"*** WARNING: Moved previous .bak to {fname}.{now}",
file=sys.stderr)
os.rename(fname + '.bak', f'{fname}.bak.{now}')
shutil.copy(fname, fname + '.bak')
os.chmod(fname, 0o400)
else:
non_recoverable = True
if non_recoverable:
raise
except P4TimeoutError as e:
e = str(e).replace('\n', ' ')
print(f"# P4 TIMEOUT, RETRIES {retries}: {e}", file=sys.stderr)
retries -= 1
if not retries:
sys.exit(
f"{CLR}*** ERROR: Perforce timed out too many times:\n{e}")
except LogAndAbort as e:
import json
fname = f'debug-pyforce.{os.getpid()}.{int(time.time())}'
d = {
'args': args,
'fstats': fstats,
'errs': errs,
'repeats': repeats,
'infos': infos,
}
json.dump(d, open(f'.o4/{fname}', 'wt'))
sys.exit(f'{CLR}*** ERROR: {e}; detail in {fname}')
finally:
if not quiet:
for fstat in not_yet:
# Printing the fstats after the p4 process has ended, because p4 marshals
# its objects before operation, as in "And for my next act... !"
# This premature printing leads to false checksum errors during sync.
print(fstat_join(fstat))
def o4_filter(filtertype, filters, verbose):
from functools import partial
# Each function implements a filter. It is called with an Fstat tuple.
# If it "likes" the row (e.g., "checksum" likes the row if the checksum
# matches the local file's checksum), it returns the row; otherwise it
# returns None. It also has the option of returning an altered copy.
def f_deletes(row):
return not row[F_CHECKSUM]
def f_case(row):
return caseful_accurate(row[F_PATH])
def f_open(row, p4open={}):
if not p4open:
dep = _depot_path().replace('/...', '')
p4open.update({
Pyforce.unescape(p['depotFile'])[len(dep) + 1:]: p['action']
for p in Pyforce('opened', dep + '/...')
})
p4open['populated'] = True
return row[F_PATH] in p4open
def f_existence(row):
"""
Returns True if the file presence matches the fstat. That is True
if fstat says 'delete' and file is missing or True if fstat is
not 'delete' and file exists.
"""
return (os.path.lexists(row[F_PATH])
and not os.path.isdir(row[F_PATH])) == bool(row[F_CHECKSUM])
def f_checksum(row):
if os.path.lexists(row[F_PATH]):
if not row[F_CHECKSUM]: # File is deleted
if os.path.isdir(row[F_PATH]):
return True
elif row[F_FILE_SIZE].endswith('symlink') or Pyforce.checksum(
row[F_PATH], row[F_FILE_SIZE]) == row[F_CHECKSUM]:
return True
else:
if not row[F_CHECKSUM]:
return True
def inverter(fname, invert):
if invert:
return f"not({fname})"
return fname
funcs = [
inverter(f'f_{fname}(x)', invert) for fname, doit, invert in filters
if doit
]
if not funcs:
sys.exit(f'*** ERROR: No arguments supplied to filter')
elif len(funcs) == 1 and filtertype != 'drop' and not funcs[0].startswith(
'not('):
if verbose:
print(f"# Filter {filtertype}:", funcs, file=sys.stderr)
combo_func = locals()[funcs[0].split('(')[0]]
elif filtertype == 'drop':
combo_func = 'lambda x: not(' + ' or '.join(f for f in funcs) + ')'
if verbose:
print(f"# Filter {filtertype}:", combo_func, file=sys.stderr)
combo_func = eval(combo_func, locals())
elif filtertype == 'keep-any':
combo_func = 'lambda x: ' + ' or '.join(f for f in funcs)
if verbose:
print(f"# Filter {filtertype}:", combo_func, file=sys.stderr)
combo_func = eval(combo_func, locals())
elif filtertype == 'keep':
combo_func = 'lambda x: ' + ' and '.join(f for f in funcs)
if verbose:
print(f"# Filter {filtertype}:", combo_func, file=sys.stderr)
combo_func = eval(combo_func, locals())
else:
sys.exit(f"*** ERROR: Invalid filtertype: {filtertype}")
try:
for line in sys.stdin:
if line.startswith('#o4pass'):
print(line, end='')
continue
row = fstat_split(line)
if row:
if combo_func(row):
print(fstat_join(row))
elif verbose:
print('#', row)
except KeyboardInterrupt:
raise