def o4_head(paths):
def o4_head_update(args):
res = list(args)
for s in Pyforce('changes', '-s', 'submitted', '-m1', *res):
for i, arg in enumerate(res):
if type(arg) is int:
continue
# p4 rewrites path if there are no files until further down
if s['path'].startswith(arg[:-3]) or arg.startswith(
s['path'][:-3]):
res[i] = int(s['change'])
o4dir = os.environ['CLIENT_ROOT'] + arg[1:].replace(
'/...', '/.o4')
os.makedirs(o4dir, exist_ok=True)
with open(o4dir + '/head', 'wt') as fout:
print(f"{s['change']}", file=fout)
break
else:
print("*** WARNING: Could not map result", s, file=sys.stderr)
for r in res:
if type(r) is not int:
try:
o4dir = os.environ['CLIENT_ROOT'] + r[1:].replace(
'/...', '/.o4')
os.unlink(o4dir + '/head')
except FileNotFoundError:
pass
sys.exit(f"*** ERROR: Could not get HEAD for {r}")
return res
args = []
for depot_path in paths:
if not depot_path.endswith('/...'):
depot_path += '/...'
args.append(Pyforce.escape(depot_path))
for retry in range(3):
try:
end = '' if len(args) > 1 else args[0]
print(
f"# {CLR}*** INFO: ({retry+1}/3) Retrieving HEAD changelist for",
end,
file=sys.stderr)
if not end:
for path in args:
print(f" {path}")
return o4_head_update(args)
except (P4TimeoutError, IndexError):
continue
sys.exit(
f"{CLR}*** ERROR: There was an error retrieving head change for {args}"
)
def fstat_from_perforce(depot_path, upper, lower=None):
"""
Returns an iterator of Fstat objects where changelist is in
(lower, upper]. If lower is not given, it is assumed to be 0.
"""
from o4_pyforce import Pyforce
def fstatify(r, head=len(depot_path.replace('...', ''))):
try:
if r[b'code'] == b'mute':
return ('0', '', '0', '0', '')
t = r[b'headType'].decode('utf8')
c = r.get(b'digest', b'').decode('utf8')
sz = r.get(b'fileSize', b'0').decode('utf8')
if t.startswith('utf') or t == 'symlink':
sz = sz + '/' + t
return [
r[b'headChange'].decode('utf8'),
Pyforce.unescape(r[b'depotFile'].decode('utf8'))[head:],
r[b'headRev'].decode('utf8'), sz, c
]
except StopIteration:
raise
except Exception as e:
print("*** ERROR: Got {!r} while fstatify({!r})".format(e, r),
file=sys.stderr)
raise
lower = lower or 0
revs = f'@{upper}'
if lower > 1:
# A range going back to the beginning will get a Perforce error.
assert lower <= upper
revs = f'@{lower},@{upper}'
pyf = Pyforce(
'fstat', '-Rc', '-Ol', '-Os', '-T',
'headType, digest, fileSize, depotFile, headChange, headRev',
Pyforce.escape(depot_path) + revs)
pyf.transform = fstatify
return pyf
def o4_pyforce(debug, no_revision, args: list, quiet=False):
"""
Encapsulates Pyforce, does book keeping to ensure that all files
that should be operated on are in fact dealt with by p4. Handles
retry and strips out asks for files that are caseful mismatches on
the current file system (macOS).
"""
from tempfile import NamedTemporaryFile
from collections import defaultdict
class LogAndAbort(Exception):
'Dumps debug information on errors.'
o4_log('pyforce', no_revision=no_revision, quiet=quiet, *args)
tmpf = NamedTemporaryFile(dir='.o4')
fstats = []
for line in sys.stdin.read().splitlines():
if line.startswith('#o4pass'):
print(line)
continue
f = fstat_split(line)
if f and caseful_accurate(f[F_PATH]):
fstats.append(f)
elif f:
print(
f"*** WARNING: Pyforce is skipping {f[F_PATH]} because it is casefully",
"mismatching a local file.",
file=sys.stderr)
retries = 3
head = _depot_path().replace('/...', '')
while fstats:
if no_revision:
p4paths = [Pyforce.escape(f[F_PATH]) for f in fstats]
else:
p4paths = [
f"{Pyforce.escape(f[F_PATH])}#{f[F_REVISION]}" for f in fstats
]
tmpf.seek(0)
tmpf.truncate()
not_yet = []
pargs = []
xargs = []
# This is a really bad idea, files are output to stdout before the actual
# sync happens, causing checksum tests to start too early:
# if len(p4paths) > 30 and 'sync' in args:
# xargs.append('--parallel=threads=5')
if sum(len(s) for s in p4paths) > 30000:
pargs.append('-x')
pargs.append(tmpf.name)
for f in p4paths:
tmpf.write(f.encode('utf8'))
tmpf.write(b'\n')
tmpf.flush()
else:
xargs.extend(p4paths)
try:
# TODO: Verbose
#print('# PYFORCE({}, {}{})'.format(','.join(repr(a) for a in args), ','.join(
# repr(a) for a in paths[:3]), ', ...' if len(paths) > 3 else ''))
errs = []
repeats = defaultdict(list)
infos = []
for res in Pyforce(*pargs, *args, *xargs):
if debug:
err_print("*** DEBUG: Received", repr(res))
# FIXME: Delete this if-statement:
if res.get('code', '') == 'info':
infos.append(res)
if res.get('data', '').startswith('Diff chunks: '):
continue
if res.get('code', '') == 'error':
errs.append(res)
continue
if 'resolveFlag' in res:
# TODO: resolveFlag can be ...?
# m: merge
# c: copy from (not conflict!)
# We skip this entry as it is the second returned from p4
# for one input file
continue
res_str = res.get('depotFile') or res.get('fromFile')
if not res_str and res.get('data'):
res_str = head + '/' + res['data']
if not res_str:
errs.append(res)
continue
res_str = Pyforce.unescape(res_str)
for i, f in enumerate(fstats):
if f"{head}/{f[F_PATH]}" in res_str:
repeats[f"{head}/{f[F_PATH]}"].append(res)
not_yet.append(fstats.pop(i))
break
else:
for f in repeats.keys():
if f in res_str:
if debug:
err_print(
f"*** DEBUG: REPEAT: {res_str}\n {res}\n {repeats[f]}"
)
break
else:
if debug:
err_print("*** DEBUG: ERRS APPEND", res)
errs.append(res)
if errs:
raise LogAndAbort('Unexpected reply from p4')
if len(p4paths) == len(fstats):
raise LogAndAbort('Nothing recognized from p4')
except P4Error as e:
non_recoverable = False
for a in e.args:
if 'clobber writable file' in a['data']:
fname = a['data'].split('clobber writable file')[1].strip()
print("*** WARNING: Saving writable file as .bak:",
fname,
file=sys.stderr)
if os.path.exists(fname + '.bak'):
now = time.time()
print(
f"*** WARNING: Moved previous .bak to {fname}.{now}",
file=sys.stderr)
os.rename(fname + '.bak', f'{fname}.bak.{now}')
shutil.copy(fname, fname + '.bak')
os.chmod(fname, 0o400)
else:
non_recoverable = True
if non_recoverable:
raise
except P4TimeoutError as e:
e = str(e).replace('\n', ' ')
print(f"# P4 TIMEOUT, RETRIES {retries}: {e}", file=sys.stderr)
retries -= 1
if not retries:
sys.exit(
f"{CLR}*** ERROR: Perforce timed out too many times:\n{e}")
except LogAndAbort as e:
import json
fname = f'debug-pyforce.{os.getpid()}.{int(time.time())}'
d = {
'args': args,
'fstats': fstats,
'errs': errs,
'repeats': repeats,
'infos': infos,
}
json.dump(d, open(f'.o4/{fname}', 'wt'))
sys.exit(f'{CLR}*** ERROR: {e}; detail in {fname}')
finally:
if not quiet:
for fstat in not_yet:
# Printing the fstats after the p4 process has ended, because p4 marshals
# its objects before operation, as in "And for my next act... !"
# This premature printing leads to false checksum errors during sync.
print(fstat_join(fstat))
