def commitTut(msg):
os.chdir(f"{TUT_BASE}")
console(f"In {os.getcwd()}")
run(["git", "add", "--all", "."])
run(["git", "commit", "-m", msg])
run(["git", "push", "origin", "master"])
os.chdir(f"{TF_BASE}")
def commitTut(msg):
os.chdir(f'{TUT_BASE}')
console(f'In {os.getcwd()}')
run(['git', 'add', '--all', '.'])
run(['git', 'commit', '-m', msg])
run(['git', 'push', 'origin', 'master'])
os.chdir(f'{TF_BASE}')
def commitApps(msg):
for app in apps:
os.chdir(f'{AN_BASE}/app-{app}')
console(f'In {os.getcwd()}')
run(['git', 'add', '--all', '.'])
run(['git', 'commit', '-m', msg])
run(['git', 'push', 'origin', 'master'])
os.chdir(f'{TF_BASE}')
def commitApps(msg):
for app in apps:
os.chdir(f"{AN_BASE}/app-{app}")
console(f"In {os.getcwd()}")
run(["git", "add", "--all", "."])
run(["git", "commit", "-m", msg])
run(["git", "push", "origin", "master"])
os.chdir(f"{TF_BASE}")
def getImagery(app, silent, checkout=""):
aContext = app.context
org = aContext.org
repo = aContext.repo
graphicsRelative = aContext.graphicsRelative
(imageRelease, imageCommit, imageLocal, imageBase,
imageDir) = checkoutRepo(
app._browse,
org=org,
repo=repo,
folder=graphicsRelative,
version="",
checkout=checkout,
withPaths=True,
keep=True,
silent=silent,
)
if not imageBase:
app.api = None
return
app.imageDir = f"{imageBase}/{org}/{repo}/{graphicsRelative}"
app._imagery = {}
for (dirFmt, ext, kind, objectType) in (
(IDEO_TO, LINEART_EXT, "lineart", "ideograph"),
(TABLET_TO, LINEART_EXT, "lineart", "tablet"),
(PHOTO_TO, PHOTO_EXT, "photo", "tablet"),
):
srcDir = dirFmt.format(app.imageDir)
filePaths = glob(f"{srcDir}/*.{ext}")
images = {}
idPat = re.compile("P[0-9]+")
for filePath in filePaths:
(fileDir, fileName) = os.path.split(filePath)
(base, thisExt) = os.path.splitext(fileName)
if kind == "lineart" and objectType == "tablet":
ids = idPat.findall(base)
if not ids:
console(f'skipped non-{objectType} "{fileName}"')
continue
identifier = ids[0]
key = base.replace("_l", "").replace(identifier, "")
else:
identifier = base
if identifier.startswith("["):
identifier = "|" + identifier[1:]
if identifier.endswith("]"):
identifier = identifier[0:-1] + "|"
key = ""
images.setdefault(identifier, {})[key] = filePath
app._imagery.setdefault(objectType, {})[kind] = images
if not silent:
dh(f"Found {len(images)} {objectType} {kind}s<br>")
def killProcesses():
myself = os.getpid()
for proc in psutil.process_iter(attrs=['pid', 'name']):
pid = proc.info['pid']
if pid == myself:
continue
if filterProcess(proc):
try:
proc.terminate()
console(f'mkdocs [{pid}] terminated')
except psutil.NoSuchProcess:
console(f'mkdocs [{pid}] already terminated')
def testSet(self, apps=None):
if apps is None:
apps = APPS
elif type(apps) is str:
apps = apps.split()
self.doApps = set()
doApps = self.doApps
for app in apps:
if app not in self.apps:
console(f"No such app: {app}", error=True)
continue
doApps.add(app)
def diffFeature(f):
with open(f"{path1}/{f}.tf") as h:
eLines = (h.readlines() if f == "otext" else
(d for d in h.readlines() if not d.startswith("@")))
with open(f"{path2}/{f}.tf") as h:
nLines = (h.readlines() if f == "otext" else
(d for d in h.readlines() if not d.startswith("@")))
i = 0
equal = True
cutOff = 40
limit = 4
nUnequal = 0
for (e, n) in zip_longest(eLines, nLines, fillvalue="<empty>"):
i += 1
if e != n:
if nUnequal == 0:
console("differences{}".format(
"" if f == "otext" else " after the metadata"))
shortE = e[0:cutOff] + (" ..." if len(e) > cutOff else "")
shortN = n[0:cutOff] + (" ..." if len(n) > cutOff else "")
console("\tline {:>6} OLD -->{}<--".format(
i, shortE.rstrip("\n")))
console("\tline {:>6} NEW -->{}<--".format(
i, shortN.rstrip("\n")))
equal = False
nUnequal += 1
if nUnequal >= limit:
break
console("no changes" if equal else "")
def tftest(suite, remaining):
suiteDir = f'{TEST_BASE}/generic'
suiteFile = f'{suite}.py'
good = True
try:
os.chdir(suiteDir)
except Exception:
good = False
console(f'Cannot find TF test directory "{suiteDir}"')
if not good:
return
if not os.path.exists(suiteFile):
console(f'Cannot find TF test suite "{suite}"')
return
rargs = ' '.join(remaining)
cmdLine = f'python3 {suiteFile} -v {rargs}'
try:
run(cmdLine, shell=True)
except KeyboardInterrupt:
pass
def shipData(app, remaining):
result = findApp(app, 'clone', silent=False)
appDir = f'{result[3]}/{result[4]}'
if not appDir:
console('Data not shipped')
config = findAppConfig(app, appDir)
if not config:
console('Data not shipped')
return
seen = set()
for r in config.ZIP:
(org, repo, relative) = r if type(r) is tuple else (config.ORG, r,
config.RELATIVE)
keep = (org, repo) in seen
zipData(org,
repo,
relative=relative,
tf=relative.endswith('tf'),
keep=keep)
seen.add((org, repo))
def readArgs():
args = sys.argv[1:]
if not len(args) or args[0] in {"-h", "--help", "help"}:
console(HELP)
return (False, None, [])
arg = args[0]
if arg not in {
"a",
"t",
"adocs",
"docs",
"clean",
"l",
"lp",
"i",
"g",
"data",
"apps",
"tut",
"r",
"r1",
"r2",
"r3",
}:
console(HELP)
return (False, None, [])
if arg in {"g", "apps", "tut", "r", "r1", "r2", "r3"}:
if len(args) < 2:
console("Provide a commit message")
return (False, None, [])
return (arg, args[1], args[2:])
if arg in {"a", "t", "data"}:
if len(args) < 2:
if arg in {"a", "data"}:
console(f"Provide a data source [{appStr}]")
elif arg in {"t"}:
console("Provide a test suite [relations, qperf]")
return (False, None, [])
return (arg, args[1], args[2:])
return (arg, None, [])
def readArgs():
args = sys.argv[1:]
if not len(args) or args[0] in {'-h', '--help', 'help'}:
console(HELP)
return (False, None, [])
arg = args[0]
if arg not in {
'a',
't',
'docs',
'clean',
'l',
'lp',
'i',
'g',
'data',
'apps',
'tut',
'r',
'r1',
'r2',
'r3',
}:
console(HELP)
return (False, None, [])
if arg in {'g', 'apps', 'tut', 'r', 'r1', 'r2', 'r3'}:
if len(args) < 2:
console('Provide a commit message')
return (False, None, [])
return (arg, args[1], args[2:])
if arg in {'a', 't', 'data'}:
if len(args) < 2:
if arg in {'a', 'data'}:
console(f'Provide a data source [{appStr}]')
elif arg in {'t'}:
console('Provide a test suite [relations, qperf]')
return (False, None, [])
return (arg, args[1], args[2:])
return (arg, None, [])
def adjustVersion(task):
for (key, c) in VERSION_CONFIG.items():
console(f'Adjusting version in {c["file"]}')
with open(c['file']) as fh:
text = fh.read()
text = c['re'].sub(
replaceVersion(task, c['mask']),
text,
)
with open(c['file'], 'w') as fh:
fh.write(text)
if currentVersion == newVersion:
console(f'Rebuilding version {newVersion}')
else:
console(f'Replacing version {currentVersion} by {newVersion}')
def checkDiffs(path1, path2):
def diffFeature(f):
with open(f"{path1}/{f}.tf") as h:
eLines = (h.readlines() if f == "otext" else
(d for d in h.readlines() if not d.startswith("@")))
with open(f"{path2}/{f}.tf") as h:
nLines = (h.readlines() if f == "otext" else
(d for d in h.readlines() if not d.startswith("@")))
i = 0
equal = True
cutOff = 40
limit = 4
nUnequal = 0
for (e, n) in zip_longest(eLines, nLines, fillvalue="<empty>"):
i += 1
if e != n:
if nUnequal == 0:
console("differences{}".format(
"" if f == "otext" else " after the metadata"))
shortE = e[0:cutOff] + (" ..." if len(e) > cutOff else "")
shortN = n[0:cutOff] + (" ..." if len(n) > cutOff else "")
console("\tline {:>6} OLD -->{}<--".format(
i, shortE.rstrip("\n")))
console("\tline {:>6} NEW -->{}<--".format(
i, shortN.rstrip("\n")))
equal = False
nUnequal += 1
if nUnequal >= limit:
break
console("no changes" if equal else "")
console(f"Check differences between TF files in {path1} and {path2}")
files1 = glob(f"{path1}/*.tf")
files2 = glob(f"{path2}/*.tf")
features1 = {os.path.basename(os.path.splitext(f)[0]) for f in files1}
features2 = {os.path.basename(os.path.splitext(f)[0]) for f in files2}
addedOnes = features2 - features1
deletedOnes = features1 - features2
commonOnes = features2 & features1
if addedOnes:
console(f"\t{len(addedOnes)} features to add")
for f in sorted(addedOnes):
console(f"\t\t{f}")
else:
console(f"\tno features to add")
if deletedOnes:
console(f"\t{len(deletedOnes)} features to delete")
for f in sorted(deletedOnes):
console(f"\t\t{f}")
else:
console(f"\tno features to delete")
console(f"\t{len(commonOnes)} features in common")
for f in sorted(commonOnes):
console(f"{f}")
diffFeature(f)
console("Done")
def makeRelease(org, repo, tag, name, msg, silent=True):
ghPerson = os.environ.get("GHPERS", None)
if ghPerson:
ghConn = Github(ghPerson)
rate = ghConn.get_rate_limit().core
if not silent:
console(f"rate limit is {rate.limit} requests per hour,"
f" with {rate.remaining} left for this hour")
if rate.limit < 100:
console(
"To increase the rate,"
"see https://annotation.github.io/text-fabric/tf/advanced/repo.html"
)
try:
if not silent:
console(
f"\tconnecting to online GitHub repo {org}/{repo} ... ",
newline=False,
)
repoOnline = ghConn.get_repo(f"{org}/{repo}")
if not silent:
console("connected")
except GithubException as why:
console("failed", error=True)
console(f"GitHub says: {why}", error=True)
except IOError:
console("no internet", error=True)
if not repoOnline:
return 1
try:
repoOnline.create_git_release(tag, name, msg)
result = 0
except GithubException:
console("failed", error=True)
console("GitHub says: {why}", error=True)
result = 1
except IOError:
console("no internet", error=True)
result = 1
if result == 0:
if not silent:
console(f"{org}/{repo} @ {tag} ({name}: {msg})")
else:
console("done")
return result
