def build_services_config(self):
if self.services_config is None:
raise self.MissingTemplateError(
f'You have not provided a services config!')
services_config_template = self.locate_config_template(
self.services_config)
curies_location = self.curies
curies = getCuries(curies_location)
with open(services_config_template, 'rt') as f:
services_config = yaml.safe_load(f)
services_config['graphConfiguration']['curies'] = curies
if self.graph_folder != combined_defaults['--graph-folder']:
services_config['graphConfiguration'][
'location'] = self.graph_folder
else:
self.graph_folder = services_config['graphConfiguration'][
'location']
port = services_config['server']['connector']['port']
url = services_config['serviceMetadata']['preview']['url']
services_config['serviceMetadata']['preview']['url'] = url.format(
HOSTNAME=self.services_host, PORT=port)
url = services_config['serviceMetadata']['view']['url']
services_config['serviceMetadata']['view']['url'] = url.format(
HOSTNAME=self.services_host, PORT=port)
#print(self.graph_folder)
services_config_path = jpth(self.zip_location,
self.services_config) # save loc
p = Path(services_config_path)
if not p.parent.exists(): p.parent.mkdir(parents=True)
with open(services_config_path, 'wt') as f:
yaml.dump(services_config, f, default_flow_style=False)
def run(args):
# modes
graph = args['graph']
scigraph = args['scigraph']
config = args['config']
imports = args['imports']
chain = args['chain']
extra = args['extra']
# required
repo_name = args['<repo>']
remote_base = args['<remote_base>']
ontologies = args['<ontologies>']
# options
git_remote = args['--git-remote']
git_local = Path(args['--git-local']).resolve()
zip_location = Path(args['--zip-location']).resolve()
graphload_config = Path(args['--graphload-config']).resolve()
graphload_config_template = graphload_config # NOTE XXX
if args['--graphload-ontologies'] is not None:
graphload_ontologies = Path(args['--graphload-ontologies']).resolve()
else:
graphload_ontologies = None
org = args['--org']
branch = args['--branch']
commit = args['--commit']
scp = args['--scp-loc']
sorg = args['--scigraph-org']
sbranch = args['--scigraph-branch']
scommit = args['--scigraph-commit']
sscp = args['--scigraph-scp-loc']
scigraph_quiet = args['--scigraph-quiet']
patch_config = args['--patch-config']
curies_location = args['--curies']
patch = args['--patch']
check_built = args['--check-built']
debug = args['--debug']
log = args['--logfile'] # TODO
fix_imports_only = args['--fix-imports-only']
load_base = 'scigraph-load -c {config_path}' # now _this_ is easier
if args['--view-defaults']:
for k, v in defaults.items():
print(f'{k:<22} {v}')
return
# post parse mods
if remote_base == 'NIF':
remote_base = 'http://ontology.neuinfo.org/NIF'
itrips = None
if repo_name is not None:
local_base = jpth(git_local, repo_name)
if graph:
if args['--path-build-scigraph']: # path-build-scigraph
path_build_scigraph = Path(args['--path-build-scigraph'])
(scigraph_commit, services_zip,
scigraph_reset_state) = scigraph_build(path_build_scigraph,
git_remote,
sorg,
path_build_scigraph,
sbranch,
scommit,
check_built=check_built,
cleanup_later=True,
quiet=scigraph_quiet)
else:
scigraph_commit = 'dev-9999'
services_zip = 'None'
scigraph_reset_state = lambda: None
with execute_regardless(scigraph_reset_state):
rl = ReproLoader(
zip_location,
git_remote,
org,
git_local,
repo_name,
branch,
commit,
remote_base,
load_base,
graphload_config_template,
graphload_ontologies,
patch_config,
patch,
scigraph_commit,
fix_imports_only=fix_imports_only,
check_built=check_built,
)
if not fix_imports_only:
FILE_NAME_ZIP = Path(rl.zip_path).name
LATEST = Path(zip_location) / 'LATEST'
if LATEST.exists() and LATEST.is_symlink():
LATEST.unlink()
LATEST.symlink_to(FILE_NAME_ZIP)
itrips, config = rl.itrips, rl.config
if not ontologies:
ontologies = rl.ontologies
print(services_zip)
print(rl.zip_path)
if '--local' in args:
return
elif scigraph:
(scigraph_commit, services_zip,
_) = scigraph_build(zip_location,
git_remote,
sorg,
git_local,
sbranch,
scommit,
check_built=check_built,
quiet=scigraph_quiet)
print(services_zip)
if '--local' in args:
return
elif config:
#graph_path = Path(args['<graph_path>']).resolve()
config_path = Path(args['--graph-config-out']).resolve()
#local_base = Path(git_local, repo_name).resolve()
date_today = TODAY()
ReproLoader.make_graphload_config(graphload_config_template,
graphload_ontologies, zip_location,
date_today, config_path)
elif imports:
# TODO mismatch between import name and file name needs a better fix
itrips = local_imports(remote_base, local_base, ontologies)
elif chain:
itrips = local_imports(remote_base,
local_base,
ontologies,
readonly=True)
elif extra:
from nifstd_tools.utils import memoryCheck
curies = getCuries(curies_location)
curie_prefixes = set(curies.values())
memoryCheck(2665488384)
graph = loadall(git_local, repo_name)
new_graph = normalize_prefixes(graph, curies)
for_burak(new_graph)
debug = True
elif patch:
local_base = jpth(git_local, repo_name)
local_versions = tuple(do_patch(patch_config, local_base))
else:
raise BaseException('How did we possibly get here docopt?')
if itrips:
import_graph = OntGraph()
[import_graph.add(t) for t in itrips]
for tree, extra in import_tree(import_graph, ontologies):
name = Path(next(iter(tree.keys()))).name
with open(jpth(zip_location, f'{name}-import-closure.html'),
'wt') as f:
f.write(extra.html.replace('NIFTTL:',
'')) # much more readable
if debug:
breakpoint()
def run(args):
# modes
graph = args['graph']
scigraph = args['scigraph']
config = args['config']
imports = args['imports']
chain = args['chain']
extra = args['extra']
# required
repo_name = args['<repo>']
remote_base = args['<remote_base>']
ontologies = args['<ontologies>']
# options
git_remote = args['--git-remote']
git_local = args['--git-local']
zip_location = args['--zip-location']
graphload_config = args['--graphload-config']
org = args['--org']
branch = args['--branch']
commit = args['--commit']
scp = args['--scp-loc']
sorg = args['--scigraph-org']
sbranch = args['--scigraph-branch']
scommit = args['--scigraph-commit']
sscp = args['--scigraph-scp-loc']
patch_config = args['--patch-config']
curies_location = args['--curies']
patch = args['--patch']
check_built = args['--check-built']
debug = args['--debug']
log = args['--logfile'] # TODO
if args['--view-defaults']:
for k, v in defaults.items():
print(f'{k:<22} {v}')
return
# post parse mods
if remote_base == 'NIF':
remote_base = 'http://ontology.neuinfo.org/NIF'
curies = getCuries(curies_location)
curie_prefixes = set(curies.values())
itrips = None
if repo_name is not None:
local_base = jpth(git_local, repo_name)
if graph:
if args['--build-scigraph']:
(scigraph_commit, load_base, services_zip,
scigraph_reset_state) = scigraph_build(zip_location,
git_remote,
sorg,
git_local,
sbranch,
scommit,
check_built=check_built,
cleanup_later=True)
else:
scigraph_commit = 'dev-9999'
services_zip = 'None'
load_base = 'scigraph-load -c {config_path}' # now _this_ is easier
scigraph_reset_state = lambda: None
with execute_regardless(scigraph_reset_state):
rl = ReproLoader(zip_location,
git_remote,
org,
git_local,
repo_name,
branch,
commit,
remote_base,
load_base,
graphload_config,
patch_config,
patch,
scigraph_commit,
check_built=check_built)
itrips, config = rl.itrips, rl.config
if not check_built:
deploy_scp(services_zip, sscp)
deploy_scp(rl.zip_path, scp)
if not ontologies:
ontologies = rl.ontologies
print(services_zip)
print(rl.zip_path)
if '--local' in args:
return
elif scigraph:
(scigraph_commit, load_base, services_zip,
_) = scigraph_build(zip_location,
git_remote,
sorg,
git_local,
sbranch,
scommit,
check_built=check_built)
if not check_built:
deploy_scp(services_zip, sscp)
print(services_zip)
if '--local' in args:
return
elif config:
graph_path = args['<graph_path>']
config_path = args['--graph-config-out']
local_base = jpth(git_local, repo_name)
ReproLoader.make_graphload_config(graphload_config, graph_path,
remote_base, local_base,
zip_location, config_path)
elif imports:
# TODO mismatch between import name and file name needs a better fix
itrips = local_imports(remote_base, local_base, ontologies)
elif chain:
itrips = local_imports(remote_base,
local_base,
ontologies,
readonly=True)
elif extra:
graph = loadall(git_local, repo_name)
mg, ng_ = normalize_prefixes(graph, curies)
ng_.add_known_namespaces(
'NIFRID') # not officially in the curies yet...
for_burak(ng_)
debug = True
elif patch:
local_base = jpth(git_local, repo_name)
local_versions = tuple(do_patch(patch_config, local_base))
else:
raise BaseException('How did we possibly get here docopt?')
if itrips:
import_graph = rdflib.Graph()
[import_graph.add(t) for t in itrips]
for tree, extra in import_tree(import_graph, ontologies):
name = Path(next(iter(tree.keys()))).name
with open(jpth(zip_location, f'{name}-import-closure.html'),
'wt') as f:
f.write(extra.html.replace('NIFTTL:',
'')) # much more readable
if debug:
embed()
def main():
from docopt import docopt, parse_defaults
args = docopt(__doc__, version='ontutils 0.0.1')
defaults = {
o.name: o.value if o.argcount else None
for o in parse_defaults(__doc__)
}
verbose = args['--verbose']
debug = args['--debug']
repo_name = args['<repo>']
git_local = os.path.expanduser(args['--git-local'])
epoch = args['--epoch']
curies_location = args['--curies']
curies = getCuries(curies_location)
curie_prefixes = set(curies.values())
filenames = args['<file>']
filenames.sort(key=lambda f: os.path.getsize(f),
reverse=True) # make sure the big boys go first
refactor_skip = ('nif.ttl', 'resources.ttl', 'generated/chebislim.ttl',
'unused/ro_bfo_bridge.ttl', 'generated/ncbigeneslim.ttl',
'generated/NIF-NIFSTD-mapping.ttl')
rfilenames = [f for f in filenames if f not in refactor_skip]
if args['set']:
from pyontutils.config import auth
uc = auth.user_config
def set_uc(var, value):
with open(uc._path, 'rt') as f:
text = f.read()
if '#' in text:
msg = f'Comments detected! Not writing config! {uc._path}'
raise ValueError(msg)
blob = uc.load()
# XXX NEVER DUMP A CONFIG THIS YOU _WILL_ KLOBBER IT
# BY ACCIDENT AT SOME POINT AND WILL ERASE ANY/ALL COMMENTS
# THERE IS NO SAFETY WITH THIS IMPLEMENTATION
# USERS SHOULD EDIT THEIR CONFIGS DIRECTLY
# except that it makes giving instructions for
# setting values a bit more complicated
blob['auth-variables'][var] = value
uc.dump(blob)
if args['ontology-local-repo']:
var = 'ontology-local-repo'
olr = Path(args['<path>']).expanduser().resolve()
olr_string = olr.as_posix()
set_uc(var, olr_string)
value2 = auth.get_path(var)
if not value2.exists():
msg = f'{var} path does not exist! {value2}'
print(tc.red('WARNING'), msg)
msg = f'{var} path {value2} written to {uc._path}'
print(msg)
assert olr == value2
elif args['scigraph-api-key']:
# FIXME this is a hack on top of orthauth, which will not
#
# check the secrets path first to make sure it is ok
# be implementing programmtic modification of user config
# files any time soon, though it might make sense to have a
# "machine config path" in addition to auth and user config
path = ['scigraph', 'api', 'key']
spath = auth._pathit(uc.get_blob('auth-stores', 'secrets')['path'])
if not spath.parent.exists():
spath.parent.mkdir(parents=True)
spath.parent.chmod(0o0700)
if spath.suffix != '.yaml':
msg = f"Can't write secrets file of type {spath.suffix}"
args = None
raise NotImplementedError(msg)
v = None
try:
s = uc.secrets
v = s(*path)
except:
pass
if v is not None:
v = None
raise ValueError(f'Path already in secrets! {path} in {spath}')
# safely append to the secrets file
key = args['<key>']
path_key = f'\nscigraph:\n api:\n key: {key}'
if not spath.exists():
spath.touch()
spath.chmod(0o0600)
with open(spath, 'a+') as f:
f.write(path_key)
# set the config var
var = 'scigraph-api-key'
value = {'path': ' '.join(path)}
set_uc(var, value) # set the path
# XXX NOTE yes, it is correct to do this only after secrets succeeds
# otherwise it is possible to get into a state where secrets does
# not exist but there is a path pointing to it, so load this
# ontutils file will fail during import time
# test that we got the value we expected
value2 = auth.get(var)
msg = (f'Key written to secrets. {spath} and path to '
f'key was written to config {uc._path}')
print(msg)
assert key == value2, 'Key retrieved does not match key set!'
elif args['devconfig']:
if args['--write']:
file = devconfig.write(args['--output-file'])
print(f'config written to {file}')
elif args['<field>']:
for f in args['<field>']:
print(getattr(devconfig, f, ''))
else:
print(devconfig)
elif args['catalog-extras']:
catalog_extras(args['--fetch'])
elif args['version-iri']:
version_iris(*filenames, epoch=epoch)
elif args['scigraph-stress']:
scigraph_stress(int(args['--rate']), int(args['--timeout']), verbose,
debug)
elif args['deadlinks']:
deadlinks(filenames, int(args['--rate']), int(args['--timeout']),
verbose, debug)
elif args['spell']:
spell(filenames, debug)
elif args['iri-commit']:
make_git_commit_command(git_local, repo_name)
elif args['uri-switch']:
uri_switch(rfilenames, uri_switch_values)
elif args['backend-refactor']:
backend_refactor(rfilenames, backend_refactor_values)
elif args['todo']:
graph = loadall(git_local, repo_name, local=True)
graph_todo(graph, curie_prefixes, uri_switch_values)
breakpoint()
elif args['expand']:
curies['NLXWIKI'] = 'http://legacy.neurolex.org/wiki/'
for curie in args['<curie>']:
prefix, suffix = curie.split(':')
print(curies[prefix] + suffix)
def main():
from docopt import docopt, parse_defaults
args = docopt(__doc__, version='ontutils 0.0.1')
defaults = {
o.name: o.value if o.argcount else None
for o in parse_defaults(__doc__)
}
verbose = args['--verbose']
debug = args['--debug']
repo_name = args['<repo>']
git_local = os.path.expanduser(args['--git-local'])
epoch = args['--epoch']
curies_location = args['--curies']
curies = getCuries(curies_location)
curie_prefixes = set(curies.values())
filenames = args['<file>']
filenames.sort(key=lambda f: os.path.getsize(f),
reverse=True) # make sure the big boys go first
refactor_skip = ('nif.ttl', 'resources.ttl', 'generated/chebislim.ttl',
'unused/ro_bfo_bridge.ttl', 'generated/ncbigeneslim.ttl',
'generated/NIF-NIFSTD-mapping.ttl')
rfilenames = [f for f in filenames if f not in refactor_skip]
if args['devconfig']:
if args['--write']:
file = devconfig.write(args['--output-file'])
print(f'config written to {file}')
elif args['<field>']:
for f in args['<field>']:
print(getattr(devconfig, f, ''))
else:
print(devconfig)
elif args['catalog-extras']:
catalog_extras(args['--fetch'])
elif args['version-iri']:
version_iris(*filenames, epoch=epoch)
elif args['scigraph-stress']:
scigraph_stress(int(args['--rate']), int(args['--timeout']), verbose,
debug)
elif args['deadlinks']:
deadlinks(filenames, int(args['--rate']), int(args['--timeout']),
verbose, debug)
elif args['spell']:
spell(filenames, debug)
elif args['iri-commit']:
make_git_commit_command(git_local, repo_name)
elif args['uri-switch']:
uri_switch(rfilenames, uri_switch_values)
elif args['backend-refactor']:
backend_refactor(rfilenames, backend_refactor_values)
elif args['todo']:
graph = loadall(git_local, repo_name, local=True)
graph_todo(graph, curie_prefixes, uri_switch_values)
embed()
elif args['expand']:
curies['NLXWIKI'] = 'http://legacy.neurolex.org/wiki/'
for curie in args['<curie>']:
prefix, suffix = curie.split(':')
print(curies[prefix] + suffix)