def run_all(doc, wd, out_path, titles=None, debug=False, logfix=False, **kwargs): # workaround for joblib 692 that only creates the logger once # workaround for incorrect behavior in old versions of makeSimpleLogger log = logging.getLogger('ont-docs') if not log.handlers: log = makeSimpleLogger('ont-docs') return (outFile(doc, wd, out_path), renderDoc(doc, titles=titles, debug=debug, **kwargs))
from urllib.parse import urlparse from importlib import import_module import yaml import htmlfn as hfn import nbformat from git import Repo from joblib import Parallel, delayed from nbconvert import HTMLExporter from augpathlib import exceptions as aexc from pyontutils import clifun as clif from pyontutils.utils import TODAY, noneMembers, makeSimpleLogger, isoformat from pyontutils.utils import TermColors as tc, get_working_dir from pyontutils.utils import asStr, findAssignToName from pyontutils.ontutils import tokstrip, _bads log = makeSimpleLogger('ont-docs') working_dir = Path(__file__).resolve().working_dir try: import hunspell except ImportError: hunspell = None suffixFuncs = {} def patch_theme_setup(theme): with open(theme, 'rt+') as f: dat = f.read() f.seek(0) f.write(dat.replace('="styles/', '="/docs/styles/'))
import io import json import hashlib import inspect from pathlib import Path from functools import wraps from pyontutils.utils import makeSimpleLogger, python_identifier # FIXME update imports from pyontutils.namespaces import sparc from sparcur.config import config log = makeSimpleLogger('sparcur') logd = log.getChild('data') class _log: """ logging prevents nice ipython recurions error printing so rename this class to log when you need fake logging """ @staticmethod def debug(nothing): pass @staticmethod def info(nothing): pass @staticmethod def warning(nothing): print(nothing) @staticmethod def error(nothing): pass @staticmethod def critical(nothing): pass default_cypher = hashlib.blake2b
from pyontutils.utils import Async, deferred, UTCNOWISO from pyontutils.config import auth from pyontutils.hierarchies import (Query, creatTree, dematerialize, flatten as flatten_tree) from pyontutils.namespaces import rdfs, OntCuries from pyontutils.scigraph_codegen import moduleDirect from nifstd_tools.sheets_sparc import (hyperlink_tree, tag_row, open_custom_sparc_view_yml, YML_DELIMITER) from nifstd_tools.simplify import simplify, cleanBad from nifstd_tools import __version__ log = makeSimpleLogger('ontree') # FIXME these will go to network which is :/ sgg = scigraph.Graph(cache=False, verbose=True) sgv = scigraph.Vocabulary(cache=False, verbose=True) sgc = scigraph.Cypher(cache=False, verbose=True) sgd = scigraph.Dynamic(cache=False, verbose=True) # This was for ttl creation extension for sparc view # ixr.setup(instrumented=OntTerm) a = 'rdfs:subClassOf' _hpp = 'RO_OLD:has_proper_part' # and apparently this fails too hpp = 'http://www.obofoundry.org/ro/ro.owl#has_proper_part' hpp = 'NIFRID:has_proper_part' po = 'BFO:0000050' # how?! WHY does this fail!? the curie is there!
def datame(d, ca, timestamp, helpers=None, log_level=logging.INFO, dp=_p, evil=[False], dumb=False): """ sigh, pickles """ log_names = ('sparcur', 'idlib', 'protcur', 'orthauth', 'ontquery', 'augpathlib', 'pyontutils') for log_name in log_names: log = logging.getLogger(log_name) if not log.handlers: log = makeSimpleLogger(log_name) log.setLevel(log_level) log.info(f'{log_name} had no handler') else: if log.level != log_level: log.setLevel(log_level) rc = d.path._cache_class._remote_class if not hasattr(rc, '_cache_anchor'): rc._setup() rc.anchorTo(ca) if not hasattr(BlackfynnCache, '_anchor'): # the fact that we only needed this much later in time # tells me that we had actually done an excellent job # of firewalling the validation pipeline from anything # related to the cache beyond the xatter data # can't use ca.__class__ because it is the posix variant of # _cache_class BlackfynnCache._anchor = ca prp = d.path.project_relative_path if helpers is not None: d.add_helpers(helpers) out_path = (dp / d.id).with_suffix('.json') if out_path.exists() and dumb: if not evil[0]: # FIXME this is SO DUMB to do in here, but ... from pysercomb.pyr import units as pyru [register_type(c, c.tag) for c in (pyru._Quant, pyru.Range)] pyru.Term._OntTerm = OntTerm # the tangled web grows ever deeper :x evil[0] = True log.warning(f'loading from path {out_path}') # FIXME this is _idiotically_ slow with joblib # multiple orders of magnitude faster just using listcomp with open(out_path, 'rt') as f: return fromJson(json.load(f)) blob_dataset = d.data_for_export(timestamp) with open(out_path.with_suffix('.raw.json'), 'wt') as f: # FIXME XXXXXXXXXXXXXXXXXXXXXXXXXXXX json.dump(blob_dataset, f, sort_keys=True, indent=2, cls=JEncode) try: pipe = pipes.IrToExportJsonPipeline( blob_dataset) # FIXME network sandbox violation blob_export = pipe.data with open(out_path, 'wt') as f: # FIXME XXXXXXXXXXXXXXXXXXXXXXXXXXXX json.dump(blob_export, f, sort_keys=True, indent=2, cls=JEncode) except Exception as e: log.exception(e) log.critical(f'error during fancy json export, see previous log entry') return blob_dataset
from copy import deepcopy from pprint import pformat import rdflib import networkx as nx from rdflib.extras import external_graph_libs as egl from pyontutils.core import OntGraph, Edge from pyontutils.utils import listIn, makeSimpleLogger log = makeSimpleLogger('simplify') # FIXME nifstd-tools might need its own hierarchy? def cleanBad(json): deep = deepcopy(json) bads = 'fma:continuous_with', edges = deep['edges'] putative_bads = [e for e in edges if e['pred'] in bads] skip = [] for e in tuple(putative_bads): if e not in skip: rev = {'sub': e['obj'], 'pred': e['pred'], 'obj':e['sub'], 'meta': e['meta']} if rev in putative_bads: skip.append(rev) edges.remove(e) remed = [] for e in skip: for ae in tuple(edges): if ae not in remed and ae != e: if ( e['sub'] == ae['sub'] or e['obj'] == ae['obj'] or
from pyontutils.utils import makeSimpleLogger log = makeSimpleLogger('protbot') logd = log.getChild('data')
import inspect from types import MethodType from datetime import datetime from getpass import getuser from collections import OrderedDict import rdflib from docopt import docopt from pyontutils.core import OntId from pyontutils.utils import makeSimpleLogger from pyontutils.qnamefix import cull_prefixes from pyontutils.namespaces import makeNamespaces, NIFRID, definition from pyontutils.namespaces import TEMP, PREFIXES as uPREFIXES from pyontutils.closed_namespaces import rdf, rdfs, owl, oboInOwl from IPython import embed log = makeSimpleLogger('obo-io') fobo, obo, NIFSTD, NOPE = makeNamespaces('fobo', 'obo', 'NIFSTD', '') N = -1 # use to define 'many ' for tag counts TW = 4 # tab width class od(OrderedDict): pass od.__repr__ = dict.__repr__ # this is our current (horrible) conversion from obo to ttl obo_tag_to_ttl = {
import psutil from pyontutils.utils import makeSimpleLogger log = makeSimpleLogger('nifstd-tools') logd = log.getChild('data') def memoryCheck(vms_max_kb): """ Lookup vms_max using getCurrentVMSKb """ safety_factor = 1.2 vms_max = vms_max_kb vms_gigs = vms_max / 1024 ** 2 buffer = safety_factor * vms_max buffer_gigs = buffer / 1024 ** 2 vm = psutil.virtual_memory() free_gigs = vm.available / 1024 ** 2 if vm.available < buffer: raise MemoryError('Running this requires quite a bit of memory ~ ' f'{vms_gigs:.2f}, you have {free_gigs:.2f} of the ' f'{buffer_gigs:.2f} needed') def currentVMSKb(): p = psutil.Process(os.getpid()) return p.memory_info().vms from pyontutils.core import OntId from pyontutils.config import auth from pathlib import Path import requests