def _ontology_data_files():
resources = 'resources'
relpaths = [
'ttl/phenotype-core.ttl',
'ttl/phenotype-indicators.ttl',
'ttl/phenotypes.ttl',
'ttl/generated/part-of-self.ttl',
]
if RELEASE:
from augpathlib import RepoPath as Path
### KILL IT WITH FIRE
try:
from neurondm.core import auth ### this is NOT ok
except Exception:
# can't catch an error that you can never import because
# it will be raised before you can import it ... SIGH
import orthauth as oa
from pyontutils.config import auth as pauth
auth = oa.configure(Path('neurondm/auth-config.py').resolve(),
include=pauth)
###
olr = Path(auth.get_path('ontology-local-repo'))
### KILL IT WITH FIRE
if not olr.exists():
original = auth.get('ontology-local-repo')
raise FileNotFoundError(
f'ontology local repo does not exist: {olr}'
f'path expanded from {original}')
elif olr.repo.active_branch.name != auth.get('neurons-branch'):
# FIXME yes indeed having to call Config in a way that is
# invoked at import time is REALLY REALLY BAD :/
raise ValueError('git is on the wrong branch! '
f'{olr.repo.active_branch}')
###
resources = Path(resources)
resources.mkdir(
) # if we add resources to git, this will error before we delete by accident
paths = [olr / rp for rp in relpaths]
for p in paths:
p.copy_to(resources / p.name)
else:
from pathlib import Path
resources = Path(resources)
paths = [Path(rp) for rp in relpaths]
return resources.absolute(), [(resources / p.name).as_posix()
for p in paths]
def __call__(self, match):
name, suffix, rest = match.groups()
if rest is None:
rest = ''
if not any(name.startswith(p) for p in ('$', '#')):
if self.working_dir is None:
log.warning(
'no working directory found, you will have broken links'
)
rel = Path(name + suffix)
else:
try:
rel = ((self.current_file.parent /
(name + suffix)).resolve().relative_to(
self.working_dir))
except ValueError as e:
log.error('path went outside current repo')
return name + suffix + rest
if suffix in ('.md', '.org', '.ipynb'):
rel = rel.with_suffix('.html')
rel_path = rel.as_posix() + rest
#print('aaaaaaaaaaa', suffix, rel, rel_path)
return self.base + rel_path
elif name.startswith('$'):
raise self.MakeMeAnInlineSrcBlock(match.group())
else:
#print('bbbbbbbbbbb', match.group())
return match.group() # rel_path = name + ext + rest
def __init__(self, current_file):
self.current_file = Path(current_file).resolve()
self.working_dir = self.current_file.working_dir
if self.working_dir is None:
# cannot proceed without git
raise FileNotFoundError(
f'No working directory found for {self.current_file}')
#self.repo_name = 'NOREPO'
#self.netloc = 'NOWHERE'
#self.netloc_raw = 'NOWHERE'
#self.group = 'NOGROUP'
self.repo = self.working_dir.repo
self.repo_name = self.working_dir.name
self.remote_url = next(self.repo.remote().urls)
if self.remote_url.startswith('git@'):
self.remote_url = 'ssh://' + self.remote_url
duh = urlparse(self.remote_url)
self.netloc = duh.netloc
if 'github' in self.netloc:
self.netloc_raw = 'raw.githubusercontent.com'
else:
raise NotImplementedError("don't know raw for: {self.remote_url}")
if self.netloc.startswith('*****@*****.**'):
_, self.group = self.netloc.split(':')
self.netloc = 'github.com'
elif self.netloc == 'github.com':
_, self.group, *rest = duh.path.split('/')
else:
raise NotImplementedError(self.remote_url)
def setUp(self):
super().setUp(init_cache=False)
self.SshRemote = SshRemote._new(Path, SshCache)
project_path = Path(self.test_path)
self.project_path = project_path
remote_root = PurePath(Path(__file__).parent) # the 'remote' target
remote_id = remote_root.as_posix()
hostname = gethostname()
# this_folder.meta is sort of one extra level of host keys
try:
anchor = project_path.cache_init(hostname + ':' + remote_id, anchor=True)
except TypeError: # pxssh fail
anchor = project_path.cache_init(hostname + '-local:' + remote_id, anchor=True)
# FIXME remote_root doesn't actually work for ssh remotes, it is always '/'
#anchor = project_path.cache_init('/') # this_folder.meta is sort of one extra level of host keys
self.this_file = Path(__file__)
self.this_file_darkly = self.SshRemote(__file__)
tfd_cache = self.this_file_darkly.cache_init()
def test_meta(self):
#hrm = this_file_darkly.meta.__dict__, this_file_darkly.local.meta.__dict__
#assert hrm[0] == hrm[1]
rm = self.this_file_darkly.meta
lm = Path(__file__).meta
rmnid = {k:v for k, v in rm.items() if k != 'id'}
lmnid = {k:v for k, v in lm.items() if k != 'id'}
bads = []
for k, rv in rmnid.items():
lv = lmnid[k]
if rv != lv:
bads.append((lv, rv))
assert not bads, bads
def makeKwargs(repo_path, filepath):
path = repo_path / filepath
kwargs = {}
kwargs['title'] = filepath
kwargs['authors'] = sorted(name.strip() for name in set(
repo_path.repo.git.log(
['--follow', '--pretty=format:%an%x09', filepath]).split('\n')))
kwargs['date'] = isoformat(path.latest_commit().authored_datetime)
repo_url = Path(
next(
next(r for r in repo_path.repo.remotes
if r.name == 'origin').urls))
kwargs['org'] = repo_url.parent.name
kwargs['repo'] = repo_url.stem
kwargs[
'branch'] = 'master' # TODO figure out how to get the default branch on the remote
kwargs['crumbs'] = _crumbs
return kwargs
def default(self):
out_path = self.options.out_path
BUILD = self.options.BUILD
glb = Path(auth.get_path('git-local-base'))
theme_repo = glb / 'org-html-themes'
theme = theme_repo / 'setup/theme-readtheorg-local.setup'
prepare_paths(BUILD, out_path, theme_repo, theme)
doc_config = self._doc_config
names = tuple(doc_config['repos']) + tuple(
self.options.repo) # TODO fetch if missing ?
repo_paths = [(glb / name).resolve() for name in names]
repos = [p.repo for p in repo_paths]
skip_folders = doc_config.get('skip-folders', tuple())
rskip = doc_config.get('skip', {})
# TODO move this into run_all
docstring_kwargs = makeDocstrings(BUILD, repo_paths, skip_folders,
rskip)
wd_docs_kwargs = [docstring_kwargs]
if self.options.docstring_only:
[
kwargs.update({'theme': theme})
for _, _, kwargs in wd_docs_kwargs
]
outname, rendered = render_docs(wd_docs_kwargs,
out_path,
titles=None,
n_jobs=1,
debug=self.options.debug)[0]
if not outname.parent.exists():
outname.parent.mkdir(parents=True)
with open(outname.as_posix(), 'wt') as f:
f.write(rendered)
return
et = tuple()
wd_docs_kwargs += [
(rp, rp / f, makeKwargs(rp, f)) for rp in repo_paths
for f in rp.repo.git.ls_files().split('\n')
if Path(f).suffix in suffixFuncs and only(rp, f) and noneMembers(
f, *skip_folders) and f not in rskip.get(rp.name, et)
]
[kwargs.update({'theme': theme}) for _, _, kwargs in wd_docs_kwargs]
if self.options.spell:
spell((f.as_posix() for _, f, _ in wd_docs_kwargs))
return
titles = doc_config['titles']
outname_rendered = render_docs(wd_docs_kwargs,
out_path,
titles,
self.options.jobs,
debug=self.options.debug)
index = [
f'<b class="{heading}">{heading}</b>'
for heading in doc_config['index']
]
_NOTITLE = object()
for outname, rendered in outname_rendered:
apath = outname.relative_to(self.options.out_path)
title = titles.get(apath.as_posix(), _NOTITLE)
# TODO parse out/add titles
if title is not None:
value = (hfn.atag(apath) if title is _NOTITLE else hfn.atag(
apath, title))
index.append(value)
if not outname.parent.exists():
outname.parent.mkdir(parents=True)
with open(outname.as_posix(), 'wt') as f:
f.write(rendered)
lt = list(titles)
def title_key(a):
title = a.split('"')[1]
if title not in lt:
msg = (f'{title} missing from {self.options.config}')
raise ValueError(msg)
return lt.index(title)
index_body = '<br>\n'.join(['<h1>Documentation Index</h1>'] +
sorted(index, key=title_key))
with open((out_path / 'index.html').as_posix(), 'wt') as f:
f.write(hfn.htmldoc(index_body, title=doc_config['title']))
def out_path(self):
return Path(self._args['--out-path']).expanduser()
def config(self):
return Path(self._args['--config'])
#!/usr/bin/env python3.7
import tempfile
from pyontutils.config import auth
from augpathlib import RepoPath as Path
temp_path = Path(tempfile.tempdir)
_ddconf = auth.get_path('resources') / 'doc-config.yaml'
_ddpath = temp_path / 'build-ont-docs' / 'docs'
__doc__ = f"""Compile all documentation from git repos.
Usage:
ont-docs [options] [--repo=<REPO>...]
Options:
-h --help show this
-c --config=<PATH> path to doc-index.yaml [default: {_ddconf}]
-o --out-path=<PATH> path inside which docs are built [default: {_ddpath}]
-b --html-root=<REL> relative path to the html root [default: ..]
-s --spell run hunspell on all docs
-d --docstring-only build docstrings only
-j --jobs=NJOBS number of jobs [default: 9]
-r --repo=<REPO> additional repos to crawl for docs
--debug redirect stderr to debug pipeline errors
"""
import os
import re
import ast
import shutil
import logging
import subprocess
from pathlib import PurePath