def makeDocstrings(BUILD, repo_paths, skip_folders, skip):
docstr_file = 'docstrings.org'
docstr_path = BUILD / docstr_file
title = 'Command line programs and libraries'
authors = ['various']
date = TODAY() # FIXME midnight issues
docstr_kwargs = (docstr_path, docstr_path, {
'authors': authors,
'date': date,
'title': title,
'org': '',
'repo': '',
'branch': 'master',
'crumbs': _crumbs,
})
docstrings = get__doc__s(repo_paths, skip_folders, skip)
done = []
dslist = []
for type, module, docstring in docstrings:
if type not in done:
done.append(type)
dslist.append(f'* {type}')
if docstring is not None:
dslist.append(f'** {module}\n#+BEGIN_SRC\n{docstring}\n#+END_SRC')
docstrings_org = '\n'.join(dslist)
with open(docstr_path.as_posix(), 'wt') as f:
f.write(docstrings_org)
return docstr_kwargs
def docstrings(theme=theme):
docstr_file = 'docstrings.org'
docstr_path = working_dir / docstr_file
title = 'Command line programs and libraries'
authors = 'various'
date = TODAY()
docstr_kwargs = (working_dir, docstr_path, {
'authors': authors,
'date': date,
'title': title
})
docstrings = get__doc__s()
header = makeOrgHeader(title, authors, date, theme)
done = []
dslist = []
for type, module, docstring in docstrings:
if type not in done:
done.append(type)
dslist.append(f'* {type}')
if docstring is not None:
dslist.append(f'** {module}\n#+BEGIN_SRC\n{docstring}\n#+END_SRC')
docstrings_org = header + '\n'.join(dslist)
with open(docstr_path.as_posix(), 'wt') as f:
f.write(docstrings_org)
return docstr_kwargs
class CoCoMac(genericPScheme):
ont = OntMeta(
'http://ontology.neuinfo.org/NIF/ttl/generated/parcellation/',
'cocomacslim', 'CoCoMac terminology', 'CoCoMac',
('This file is automatically generated from the CoCoMac '
'database on the terms from BrainMaps_BrainSiteAcronyms.' +
'**FIXME**'), TODAY())
concept = PScheme(ilx['cocomac/uris/readable/BrainSiteAcronym'],
'CoCoMac terminology parcellation concept',
'NCBITaxon:9544', 'ilxtr:various')
atlas = PSArtifact(
ilx['cocomac/uris/readable/BrainSiteAcronymTable'],
'CoCoMac terminology',
None, #'no version info',
None, #'no date',
'http://cocomac.g-node.org',
'scholarly things',
tuple(),
tuple())
PREFIXES = makePrefixes('NIFRID')
PREFIXES[
'cocomac'] = 'http://cocomac.g-node.org/services/custom_sql_query.php?sql=SELECT%20*%20from%20BrainMaps_BrainSiteAcronyms%20where%20ID=' # looking for better options
@classmethod
def datagetter(cls):
url = 'http://cocomac.g-node.org/services/custom_sql_query.php?sql=SELECT * from BrainMaps_BrainSiteAcronyms;&format=json'
table = requests.get(url).json()
fields = table['fields']
data = [fields] + list(table['data'].values())
return data
@classmethod
def dataproc(cls, graph, data):
class cocomac(rowParse):
def ID(self, value):
self.identifier = 'cocomac:' + value # safe because reset every row (ish)
graph.add_class(self.identifier, cls.concept.curie)
def Key(self, value):
pass
def Summary(self, value):
pass
def Acronym(self, value):
graph.add_trip(self.identifier, ACRONYM, value)
def FullName(self, value):
graph.add_trip(self.identifier, rdfs.label,
'(%s) ' % cls.ont.shortname + value)
graph.add_trip(self.identifier, PARCLAB, value)
def LegacyID(self, value):
graph.add_trip(self.identifier, ACRONYM, value)
def BrainInfoID(self, value):
pass
cocomac(data)
def zip_name(wild=False):
return (repo_name +
'-' + branch +
'-services' +
'-' + ('*' if wild else TODAY()) +
'-' + scigraph_commit_short +
'.zip')
def folder_name(scigraph_commit, wild=False):
return (repo_name +
'-' + branch +
'-graph' +
'-' + ('*' if wild else TODAY()) +
'-' + scigraph_commit[:COMMIT_HASH_HEAD_LEN] +
'-' + ontology_commit)
def parcellation_schemes(ontids_atlases):
ont = OntMeta(
GENERATED, 'parcellation',
'NIF collected parcellation schemes ontology', 'NIF Parcellations',
'Brain parcellation schemes as represented by root concepts.', TODAY())
ontid = ont.path + ont.filename + '.ttl'
PREFIXES = makePrefixes('ilxtr', 'owl', 'skos', 'NIFRID')
graph = makeGraph(ont.filename, PREFIXES, writeloc='/tmp/')
graph.add_ont(ontid, *ont[2:])
for import_id, atlas in sorted(ontids_atlases):
graph.add_trip(ontid, owl.imports, import_id)
add_triples(graph, atlas, make_atlas)
graph.add_class(ATLAS_SUPER, label=atname)
graph.add_class(PARC_SUPER, label=psname)
graph.write()
def make_graphload_config(graphload_config_template, graphload_ontologies,
graph_path, remote_base, local_base, zip_location,
config_path=None):
config_n = 'graphload-' + TODAY() + '.yaml'
config_raw = config_n + '.raw'
if graphload_ontologies is not None:
with open(graphload_config_template, 'rt') as f1, open(graphload_ontologies, 'rt') as f2, open(zip_location / config_raw, 'wt') as out: # LOL PYTHON
out.write(f1.read())
out.write(f2.read())
else: # nothing will load, but that's ok
with open(graphload_config_template, 'rt') as f1, open(zip_location / config_raw, 'wt') as out: # LOL PYTHON
out.write(f1.read())
# config graphload.yaml from template
with open(zip_location / config_raw, 'rt') as f:
config = yaml.safe_load(f)
if 'ontologies' not in config:
# FIXME log a warning?
config['ontologies'] = []
config['graphConfiguration']['location'] = graph_path.as_posix()
if isinstance(local_base, Path):
lbasposix = local_base.as_posix()
else:
lbasposix = local_base
config['ontologies'] = [{k:v.replace(remote_base, lbasposix)
if k == 'url'
else v
for k, v in ont.items()}
for ont in config['ontologies']]
if config_path is None:
config_path = zip_location / config_n
with open(config_path, 'wt') as f:
yaml.dump(config, f, default_flow_style=False)
ontologies = [ont['url'] for ont in config['ontologies']]
return config, config_path, ontologies
def make_graphload_config(graphload_config,
graph_path,
remote_base,
local_base,
zip_location,
config_path=None):
# config graphload.yaml from template
graphload_config_template = graphload_config + '.template'
with open(graphload_config_template, 'rt') as f:
config = yaml.safe_load(f)
config['graphConfiguration']['location'] = graph_path
config['ontologies'] = [{
k: v.replace(remote_base, local_base) if k == 'url' else v
for k, v in ont.items()
} for ont in config['ontologies']]
if config_path is None:
config_path = jpth(zip_location, 'graphload-' + TODAY() + '.yaml')
with open(config_path, 'wt') as f:
yaml.dump(config, f, default_flow_style=False)
ontologies = [ont['url'] for ont in config['ontologies']]
return config, config_path, ontologies
def do_deprecation(replaced_by, g, additional_edges, conflated):
bmeta = OntMeta(
'http://ontology.neuinfo.org/NIF/ttl/bridge/', 'uberon-bridge',
'NIFSTD Uberon Bridge', 'UBERON Bridge',
('This is the bridge file that holds local NIFSTD additions to uberon. '
'This is also staging for any changes that we want to push upstream.'
), TODAY())
ontid = bmeta.path + bmeta.filename + '.ttl'
bridge = makeGraph('uberon-bridge', PREFIXES)
bridge.add_ont(ontid, *bmeta[2:])
graph = makeGraph('NIF-GrossAnatomy', NIFPREFIXES, graph=g)
#graph.g.namespace_manager._NamespaceManager__cache = {}
#g.namespace_manager.bind('UBERON','http://purl.obolibrary.org/obo/UBERON_') # this has to go in again because we reset g FIXME
udone = set('NOREP')
uedges = defaultdict(lambda: defaultdict(set))
def inner(nifga, uberon):
# check neuronames id TODO
udepr = sgv.findById(
uberon)['deprecated'] if uberon != 'NOREP' else False
if udepr:
# add xref to the now deprecated uberon term
graph.add_trip(nifga, 'oboInOwl:hasDbXref', uberon)
#print('Replacement is deprecated, not replacing:', uberon)
graph.add_trip(
nifga, RDFS.comment,
'xref %s is deprecated, so not using replacedBy:' % uberon)
else:
# add replaced by -> uberon
graph.add_trip(nifga, 'replacedBy:', uberon)
# add deprecated true (ok to do twice...)
graph.add_trip(nifga, OWL.deprecated, True)
# review nifga relations, specifically has_proper_part, proper_part_of
# put those relations on the uberon term in the
# if there is no uberon term raise an error so we can look into it
#if uberon not in uedges:
#uedges[uberon] = defaultdict(set)
resp = sgg.getNeighbors(nifga)
edges = resp['edges']
if nifga in additional_edges:
edges.append(additional_edges[nifga])
include = False # set this to True when running anns
for edge in edges: # FIXME TODO hierarchy extraction and porting
#print(edge)
if udepr: # skip everything if uberon is deprecated
include = False
hier = False
break
sub = edge['sub']
obj = edge['obj']
pred = edge['pred']
hier = False
if pred == 'subClassOf':
pred = RDFS.subClassOf
continue
elif pred == 'equivalentClass':
pred = OWL.equivalentClass
continue
elif pred == 'isDefinedBy':
pred = RDFS.isDefinedBy
continue
elif pred == 'http://www.obofoundry.org/ro/ro.owl#has_proper_part':
hier = True
include = True
elif pred == 'http://www.obofoundry.org/ro/ro.owl#proper_part_of':
hier = True
include = True
elif pred == 'ilx:partOf':
hier = True
include = True
if sub == nifga:
try:
obj = replaced_by[obj]
if obj == 'NOREP':
hier = False
except KeyError:
print('not in replaced_by', obj)
if type(obj) == tuple: continue # TODO
if hier:
if uberon not in uedges[obj][pred]:
uedges[obj][pred].add(uberon)
bridge.add_hierarchy(obj, pred, uberon)
else:
#bridge.add_trip(uberon, pred, obj)
pass
elif obj == nifga:
try:
sub = replaced_by[sub]
if sub == 'NOREP':
hier = False
except KeyError:
print('not in replaced_by', sub)
if type(sub) == tuple: continue # TODO
if hier:
if sub not in uedges[uberon][pred]:
uedges[uberon][pred].add(sub)
bridge.add_hierarchy(uberon, pred, sub)
else:
#bridge.add_trip(sub, pred, uberon)
pass
if False and uberon not in udone and include: # skip porting annotations and labels for now
#udone.add(uberon)
try:
label = sgv.findById(uberon)['labels'][0]
except IndexError:
WAT = sgv.findById(uberon)
embed()
bridge.add_class(uberon, label=label)
# annotations to port
for p in anns_to_port:
os_ = list(graph.g.objects(graph.expand(nifga), p))
for o in os_:
if label.lower() != o.lower(
): # we can simply capitalize labels
print(label.lower())
print(o.lower())
print()
bridge.add_trip(uberon, p, o)
if p == SKOS.prefLabel and not os_:
if uberon not in conflated or (uberon in conflated
and nifga in preflabs):
l = list(
graph.g.objects(graph.expand(nifga),
RDFS.label))[0]
bridge.add_trip(
uberon, SKOS.prefLabel,
l) # port label to prefLabel if no prefLabel
for nifga, uberon in replaced_by.items():
if type(uberon) == tuple:
print(uberon)
for ub in uberon:
print(ub)
inner(nifga, ub)
elif uberon == 'NOREP':
graph.add_trip(nifga, OWL.deprecated,
True) # TODO check for missing edges?
elif uberon is None:
continue # BUT TODAY IS NOT THAT DAY!
else:
inner(nifga, uberon)
return graph, bridge, uedges
def run(args):
# modes
graph = args['graph']
scigraph = args['scigraph']
config = args['config']
imports = args['imports']
chain = args['chain']
extra = args['extra']
# required
repo_name = args['<repo>']
remote_base = args['<remote_base>']
ontologies = args['<ontologies>']
# options
git_remote = args['--git-remote']
git_local = Path(args['--git-local']).resolve()
zip_location = Path(args['--zip-location']).resolve()
graphload_config = Path(args['--graphload-config']).resolve()
graphload_config_template = graphload_config # NOTE XXX
if args['--graphload-ontologies'] is not None:
graphload_ontologies = Path(args['--graphload-ontologies']).resolve()
else:
graphload_ontologies = None
org = args['--org']
branch = args['--branch']
commit = args['--commit']
scp = args['--scp-loc']
sorg = args['--scigraph-org']
sbranch = args['--scigraph-branch']
scommit = args['--scigraph-commit']
sscp = args['--scigraph-scp-loc']
scigraph_quiet = args['--scigraph-quiet']
patch_config = args['--patch-config']
curies_location = args['--curies']
patch = args['--patch']
check_built = args['--check-built']
debug = args['--debug']
log = args['--logfile'] # TODO
fix_imports_only = args['--fix-imports-only']
load_base = 'scigraph-load -c {config_path}' # now _this_ is easier
if args['--view-defaults']:
for k, v in defaults.items():
print(f'{k:<22} {v}')
return
# post parse mods
if remote_base == 'NIF':
remote_base = 'http://ontology.neuinfo.org/NIF'
itrips = None
if repo_name is not None:
local_base = jpth(git_local, repo_name)
if graph:
if args['--path-build-scigraph']: # path-build-scigraph
path_build_scigraph = Path(args['--path-build-scigraph'])
(scigraph_commit, services_zip,
scigraph_reset_state) = scigraph_build(path_build_scigraph,
git_remote,
sorg,
path_build_scigraph,
sbranch,
scommit,
check_built=check_built,
cleanup_later=True,
quiet=scigraph_quiet)
else:
scigraph_commit = 'dev-9999'
services_zip = 'None'
scigraph_reset_state = lambda: None
with execute_regardless(scigraph_reset_state):
rl = ReproLoader(
zip_location,
git_remote,
org,
git_local,
repo_name,
branch,
commit,
remote_base,
load_base,
graphload_config_template,
graphload_ontologies,
patch_config,
patch,
scigraph_commit,
fix_imports_only=fix_imports_only,
check_built=check_built,
)
if not fix_imports_only:
FILE_NAME_ZIP = Path(rl.zip_path).name
LATEST = Path(zip_location) / 'LATEST'
if LATEST.exists() and LATEST.is_symlink():
LATEST.unlink()
LATEST.symlink_to(FILE_NAME_ZIP)
itrips, config = rl.itrips, rl.config
if not ontologies:
ontologies = rl.ontologies
print(services_zip)
print(rl.zip_path)
if '--local' in args:
return
elif scigraph:
(scigraph_commit, services_zip,
_) = scigraph_build(zip_location,
git_remote,
sorg,
git_local,
sbranch,
scommit,
check_built=check_built,
quiet=scigraph_quiet)
print(services_zip)
if '--local' in args:
return
elif config:
#graph_path = Path(args['<graph_path>']).resolve()
config_path = Path(args['--graph-config-out']).resolve()
#local_base = Path(git_local, repo_name).resolve()
date_today = TODAY()
ReproLoader.make_graphload_config(graphload_config_template,
graphload_ontologies, zip_location,
date_today, config_path)
elif imports:
# TODO mismatch between import name and file name needs a better fix
itrips = local_imports(remote_base, local_base, ontologies)
elif chain:
itrips = local_imports(remote_base,
local_base,
ontologies,
readonly=True)
elif extra:
from nifstd_tools.utils import memoryCheck
curies = getCuries(curies_location)
curie_prefixes = set(curies.values())
memoryCheck(2665488384)
graph = loadall(git_local, repo_name)
new_graph = normalize_prefixes(graph, curies)
for_burak(new_graph)
debug = True
elif patch:
local_base = jpth(git_local, repo_name)
local_versions = tuple(do_patch(patch_config, local_base))
else:
raise BaseException('How did we possibly get here docopt?')
if itrips:
import_graph = OntGraph()
[import_graph.add(t) for t in itrips]
for tree, extra in import_tree(import_graph, ontologies):
name = Path(next(iter(tree.keys()))).name
with open(jpth(zip_location, f'{name}-import-closure.html'),
'wt') as f:
f.write(extra.html.replace('NIFTTL:',
'')) # much more readable
if debug:
breakpoint()
def __init__(self,
zip_location,
git_remote,
org,
git_local,
repo_name,
branch,
commit,
remote_base,
load_base,
graphload_config_template,
graphload_ontologies,
patch_config,
patch,
scigraph_commit,
post_clone=lambda: None,
fix_imports_only=False,
check_built=False):
date_today = TODAY()
load_from_repo = True
local_base = jpth(git_local, repo_name)
if load_from_repo:
repo, nob = self._set_up_repo_state(local_base, git_remote, org,
git_local, repo_name, branch,
commit, post_clone)
ontology_commit = repo.head.object.hexsha[:COMMIT_HASH_HEAD_LEN]
else:
ontology_commit = 'NONE'
config_path, config = self.make_graphload_config(
graphload_config_template, graphload_ontologies, zip_location,
date_today)
config_hash = identity_json(config, sort_lists=True).hex()
(graph_path, zip_path, zip_command,
wild_zip_path) = self._set_up_paths(zip_location, repo_name, branch,
scigraph_commit, ontology_commit,
config_hash, date_today)
# NOTE config is modified in place
ontologies = self.configure_config(config, graph_path, remote_base,
local_base, config_path)
load_command = load_base.format(
config_path=config_path) # 'exit 1' to test
log.info(load_command)
if load_from_repo:
# replace raw github imports with ontology.neuinfor iris to simplify import chain
# FIXME this is hardcoded and will not generalize ...
fix_imports = ("find " + local_base + (
" -name '*.ttl' -exec sed -i"
" 's,<http.\+/ttl/,<http://ontology.neuinfo.org/NIF/ttl/,' {} \;"
))
os.system(fix_imports)
if load_from_repo and not fix_imports_only:
def reset_state(original_branch=nob):
repo.git.checkout('--', local_base)
original_branch.checkout()
else:
reset_state = lambda: None
with execute_regardless(
reset_state
): # FIXME start this immediately after we obtain nob?
# main
if load_from_repo:
if patch:
# FIXME TODO XXX does scigraph load from the catalog!??!??
# because it seems like doid loads correctly without using local_versions
# which would be cool, if confusing
local_versions = tuple(do_patch(patch_config, local_base))
else:
local_versions = tuple()
itrips = local_imports(
remote_base,
local_base,
ontologies,
local_versions=local_versions,
dobig=True) # SciGraph doesn't support catalog.xml
catalog = make_catalog(itrips)
with open(Path(local_base, 'catalog.xml'), 'wt') as f:
f.write(catalog)
else:
itrips = []
pass
maybe_zip_path = glob(wild_zip_path)
if fix_imports_only:
pass
elif not maybe_zip_path:
if check_built:
print('The graph has not been loaded.')
raise NotBuiltError('The graph has not been loaded.')
#breakpoint()
failure = os.system(load_command)
if failure:
if os.path.exists(graph_path):
shutil.rmtree(graph_path)
else:
os.rename(
config_path, # save the config for eaiser debugging
graph_path / config_path.name)
cpr = config_path.with_suffix(config_path.suffix + '.raw')
os.rename(cpr, graph_path / cpr.name)
failure = os.system(zip_command) # graphload zip
else:
zip_path = maybe_zip_path[0] # this way we get the actual date
print('Graph already loaded at', zip_path)
# this needs to be run when the branch is checked out
# FIXME might be worth adding this to the load config?
self.ontologies = [
get_iri(load_header(rec['url']))
for rec in config['ontologies']
]
self.zip_path = zip_path
self.itrips = itrips
self.config = config
def zip_name(wild=False):
return (repo_name + '-' + branch + '-services' + '-' +
('*' if wild else TODAY()) + '-' +
scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '.zip')