def run_all(doc,
wd,
out_path,
titles=None,
debug=False,
logfix=False,
**kwargs):
# workaround for joblib 692 that only creates the logger once
# workaround for incorrect behavior in old versions of makeSimpleLogger
log = logging.getLogger('ont-docs')
if not log.handlers:
log = makeSimpleLogger('ont-docs')
return (outFile(doc, wd, out_path),
renderDoc(doc, titles=titles, debug=debug, **kwargs))
from urllib.parse import urlparse
from importlib import import_module
import yaml
import htmlfn as hfn
import nbformat
from git import Repo
from joblib import Parallel, delayed
from nbconvert import HTMLExporter
from augpathlib import exceptions as aexc
from pyontutils import clifun as clif
from pyontutils.utils import TODAY, noneMembers, makeSimpleLogger, isoformat
from pyontutils.utils import TermColors as tc, get_working_dir
from pyontutils.utils import asStr, findAssignToName
from pyontutils.ontutils import tokstrip, _bads
log = makeSimpleLogger('ont-docs')
working_dir = Path(__file__).resolve().working_dir
try:
import hunspell
except ImportError:
hunspell = None
suffixFuncs = {}
def patch_theme_setup(theme):
with open(theme, 'rt+') as f:
dat = f.read()
f.seek(0)
f.write(dat.replace('="styles/', '="/docs/styles/'))
import io
import json
import hashlib
import inspect
from pathlib import Path
from functools import wraps
from pyontutils.utils import makeSimpleLogger, python_identifier # FIXME update imports
from pyontutils.namespaces import sparc
from sparcur.config import config
log = makeSimpleLogger('sparcur')
logd = log.getChild('data')
class _log:
""" logging prevents nice ipython recurions error printing
so rename this class to log when you need fake logging """
@staticmethod
def debug(nothing): pass
@staticmethod
def info(nothing): pass
@staticmethod
def warning(nothing): print(nothing)
@staticmethod
def error(nothing): pass
@staticmethod
def critical(nothing): pass
default_cypher = hashlib.blake2b
from pyontutils.utils import Async, deferred, UTCNOWISO
from pyontutils.config import auth
from pyontutils.hierarchies import (Query,
creatTree,
dematerialize,
flatten as flatten_tree)
from pyontutils.namespaces import rdfs, OntCuries
from pyontutils.scigraph_codegen import moduleDirect
from nifstd_tools.sheets_sparc import (hyperlink_tree,
tag_row,
open_custom_sparc_view_yml,
YML_DELIMITER)
from nifstd_tools.simplify import simplify, cleanBad
from nifstd_tools import __version__
log = makeSimpleLogger('ontree')
# FIXME these will go to network which is :/
sgg = scigraph.Graph(cache=False, verbose=True)
sgv = scigraph.Vocabulary(cache=False, verbose=True)
sgc = scigraph.Cypher(cache=False, verbose=True)
sgd = scigraph.Dynamic(cache=False, verbose=True)
# This was for ttl creation extension for sparc view
# ixr.setup(instrumented=OntTerm)
a = 'rdfs:subClassOf'
_hpp = 'RO_OLD:has_proper_part' # and apparently this fails too
hpp = 'http://www.obofoundry.org/ro/ro.owl#has_proper_part'
hpp = 'NIFRID:has_proper_part'
po = 'BFO:0000050' # how?! WHY does this fail!? the curie is there!
def datame(d,
ca,
timestamp,
helpers=None,
log_level=logging.INFO,
dp=_p,
evil=[False],
dumb=False):
""" sigh, pickles """
log_names = ('sparcur', 'idlib', 'protcur', 'orthauth', 'ontquery',
'augpathlib', 'pyontutils')
for log_name in log_names:
log = logging.getLogger(log_name)
if not log.handlers:
log = makeSimpleLogger(log_name)
log.setLevel(log_level)
log.info(f'{log_name} had no handler')
else:
if log.level != log_level:
log.setLevel(log_level)
rc = d.path._cache_class._remote_class
if not hasattr(rc, '_cache_anchor'):
rc._setup()
rc.anchorTo(ca)
if not hasattr(BlackfynnCache, '_anchor'):
# the fact that we only needed this much later in time
# tells me that we had actually done an excellent job
# of firewalling the validation pipeline from anything
# related to the cache beyond the xatter data
# can't use ca.__class__ because it is the posix variant of # _cache_class
BlackfynnCache._anchor = ca
prp = d.path.project_relative_path
if helpers is not None:
d.add_helpers(helpers)
out_path = (dp / d.id).with_suffix('.json')
if out_path.exists() and dumb:
if not evil[0]: # FIXME this is SO DUMB to do in here, but ...
from pysercomb.pyr import units as pyru
[register_type(c, c.tag) for c in (pyru._Quant, pyru.Range)]
pyru.Term._OntTerm = OntTerm # the tangled web grows ever deeper :x
evil[0] = True
log.warning(f'loading from path {out_path}')
# FIXME this is _idiotically_ slow with joblib
# multiple orders of magnitude faster just using listcomp
with open(out_path, 'rt') as f:
return fromJson(json.load(f))
blob_dataset = d.data_for_export(timestamp)
with open(out_path.with_suffix('.raw.json'),
'wt') as f: # FIXME XXXXXXXXXXXXXXXXXXXXXXXXXXXX
json.dump(blob_dataset, f, sort_keys=True, indent=2, cls=JEncode)
try:
pipe = pipes.IrToExportJsonPipeline(
blob_dataset) # FIXME network sandbox violation
blob_export = pipe.data
with open(out_path, 'wt') as f: # FIXME XXXXXXXXXXXXXXXXXXXXXXXXXXXX
json.dump(blob_export, f, sort_keys=True, indent=2, cls=JEncode)
except Exception as e:
log.exception(e)
log.critical(f'error during fancy json export, see previous log entry')
return blob_dataset
from copy import deepcopy
from pprint import pformat
import rdflib
import networkx as nx
from rdflib.extras import external_graph_libs as egl
from pyontutils.core import OntGraph, Edge
from pyontutils.utils import listIn, makeSimpleLogger
log = makeSimpleLogger('simplify') # FIXME nifstd-tools might need its own hierarchy?
def cleanBad(json):
deep = deepcopy(json)
bads = 'fma:continuous_with',
edges = deep['edges']
putative_bads = [e for e in edges if e['pred'] in bads]
skip = []
for e in tuple(putative_bads):
if e not in skip:
rev = {'sub': e['obj'], 'pred': e['pred'], 'obj':e['sub'], 'meta': e['meta']}
if rev in putative_bads:
skip.append(rev)
edges.remove(e)
remed = []
for e in skip:
for ae in tuple(edges):
if ae not in remed and ae != e:
if (
e['sub'] == ae['sub'] or
e['obj'] == ae['obj'] or
from pyontutils.utils import makeSimpleLogger
log = makeSimpleLogger('protbot')
logd = log.getChild('data')
import inspect
from types import MethodType
from datetime import datetime
from getpass import getuser
from collections import OrderedDict
import rdflib
from docopt import docopt
from pyontutils.core import OntId
from pyontutils.utils import makeSimpleLogger
from pyontutils.qnamefix import cull_prefixes
from pyontutils.namespaces import makeNamespaces, NIFRID, definition
from pyontutils.namespaces import TEMP, PREFIXES as uPREFIXES
from pyontutils.closed_namespaces import rdf, rdfs, owl, oboInOwl
from IPython import embed
log = makeSimpleLogger('obo-io')
fobo, obo, NIFSTD, NOPE = makeNamespaces('fobo', 'obo', 'NIFSTD', '')
N = -1 # use to define 'many ' for tag counts
TW = 4 # tab width
class od(OrderedDict):
pass
od.__repr__ = dict.__repr__
# this is our current (horrible) conversion from obo to ttl
obo_tag_to_ttl = {
import psutil
from pyontutils.utils import makeSimpleLogger
log = makeSimpleLogger('nifstd-tools')
logd = log.getChild('data')
def memoryCheck(vms_max_kb):
""" Lookup vms_max using getCurrentVMSKb """
safety_factor = 1.2
vms_max = vms_max_kb
vms_gigs = vms_max / 1024 ** 2
buffer = safety_factor * vms_max
buffer_gigs = buffer / 1024 ** 2
vm = psutil.virtual_memory()
free_gigs = vm.available / 1024 ** 2
if vm.available < buffer:
raise MemoryError('Running this requires quite a bit of memory ~ '
f'{vms_gigs:.2f}, you have {free_gigs:.2f} of the '
f'{buffer_gigs:.2f} needed')
def currentVMSKb():
p = psutil.Process(os.getpid())
return p.memory_info().vms
from pyontutils.core import OntId
from pyontutils.config import auth
from pathlib import Path
import requests