def read_schemas(loadExtensions=False):
"""Read/parse/ingest schemas from data/*.rdfa. Also data/*examples.txt"""
load_start = datetime.datetime.now()
global schemasInitialized
schemasInitialized = True
if (not schemasInitialized or DYNALOAD):
log.debug("[%s] (re)loading core and annotations." %
getInstanceId(short=True))
files = glob.glob("data/*.rdfa")
jfiles = glob.glob("data/*.jsonld")
for jf in jfiles:
rdfequiv = jf[:-7] + ".rdfa"
if not rdfequiv in files: #Only add .jsonld files if no equivalent .rdfa
files.append(jf)
file_paths = []
for f in files:
file_paths.append(full_path(f))
apirdflib.load_graph('core', file_paths)
log.info("[%s] Loaded core graphs in %s" %
(getInstanceId(short=True),
(datetime.datetime.now() - load_start)))
load_start = datetime.datetime.now()
files = glob.glob("data/2015-04-vocab_counts.txt")
for file in files:
usage_data = read_file(file)
parser = parsers.UsageFileParser(None)
parser.parse(usage_data)
log.debug("[%s]Loaded usage data in %s" %
(getInstanceId(short=True),
(datetime.datetime.now() - load_start)))
schemasInitialized = True
def read_extensions(extensions):
global extensionsLoaded
extfiles = []
expfiles = []
load_start = datetime.datetime.now()
if not extensionsLoaded: #2nd load will throw up errors and duplicate terms
log.info("[%s] extensions %s " %
(getInstanceId(short=True), extensions))
for i in extensions:
all_layers[i] = "1"
extfiles = glob.glob("data/ext/%s/*.rdfa" % i)
jextfiles = glob.glob("data/ext/%s/*.jsonld" % i)
for jf in jextfiles:
rdfequiv = jf[:-7] + ".rdfa"
if not rdfequiv in extfiles: #Only add .jsonld files if no equivalent .rdfa
extfiles.append(jf)
file_paths = []
for f in extfiles:
file_paths.append(full_path(f))
apirdflib.load_graph(i, file_paths)
log.info("[%s]Loaded extension graphs in %s" %
(getInstanceId(short=True),
(datetime.datetime.now() - load_start)))
extensionsLoaded = True
def read_schemas(loadExtensions=False):
"""Read/parse/ingest schemas from data/*.rdfa. Also data/*examples.txt"""
import os.path
import glob
import re
global schemasInitialized
schemasInitialized = True
if (not schemasInitialized or DYNALOAD):
log.info("(re)loading core and annotations.")
files = glob.glob("data/*.rdfa")
jfiles = glob.glob("data/*.jsonld")
for jf in jfiles:
rdfequiv = jf[:-7]+".rdfa"
if not rdfequiv in files: #Only add .jsonld files if no equivalent .rdfa
files.append(jf)
file_paths = []
for f in files:
file_paths.append(full_path(f))
apirdflib.load_graph('core',file_paths)
files = glob.glob("data/*examples.txt")
read_examples(files,'core')
files = glob.glob("data/2015-04-vocab_counts.txt")
for file in files:
usage_data = read_file(file)
parser = parsers.UsageFileParser(None)
parser.parse(usage_data)
schemasInitialized = True
def read_schemas(loadExtensions=False):
"""Read/parse/ingest schemas from data/*.rdfa. Also data/*examples.txt"""
load_start = datetime.datetime.now()
global schemasInitialized
schemasInitialized = True
if (not schemasInitialized or DYNALOAD):
log.debug("[%s] (re)loading core and annotations." % getInstanceId(short=True))
files = glob.glob("data/*.rdfa")
jfiles = glob.glob("data/*.jsonld")
for jf in jfiles:
rdfequiv = jf[:-7]+".rdfa"
if not rdfequiv in files: #Only add .jsonld files if no equivalent .rdfa
files.append(jf)
file_paths = []
for f in files:
file_paths.append(full_path(f))
apirdflib.load_graph('core',file_paths)
log.info("[%s] Loaded core graphs in %s" % (getInstanceId(short=True),(datetime.datetime.now() - load_start)))
load_start = datetime.datetime.now()
files = glob.glob("data/2015-04-vocab_counts.txt")
for file in files:
usage_data = read_file(file)
parser = parsers.UsageFileParser(None)
parser.parse(usage_data)
log.debug("[%s]Loaded usage data in %s" % (getInstanceId(short=True),(datetime.datetime.now() - load_start)))
schemasInitialized = True
def read_extensions(extensions):
import os.path
import glob
import re
global extensionsLoaded
extfiles = []
expfiles = []
if not extensionsLoaded: #2nd load will throw up errors and duplicate terms
log.info("(re)scanning for extensions %s " % extensions)
for i in extensions:
all_layers[i] = "1"
extfiles = glob.glob("data/ext/%s/*.rdfa" % i)
jextfiles = glob.glob("data/ext/%s/*.jsonld" % i)
for jf in jextfiles:
rdfequiv = jf[:-7] + ".rdfa"
if not rdfequiv in extfiles: #Only add .jsonld files if no equivalent .rdfa
extfiles.append(jf)
# log.info("FILES: %s" % extfiles)
file_paths = []
for f in extfiles:
file_paths.append(full_path(f))
apirdflib.load_graph(i, file_paths)
expfiles = glob.glob("data/ext/%s/*examples.txt" % i)
read_examples(expfiles, i)
log.info("Extensions found: %s ." % " , ".join(extfiles))
# fnstrip_re = re.compile("\/.*")
# for ext in extfiles:
# ext_file_path = full_path(ext)
# extid = ext.replace('data/ext/', '')
# extid = re.sub(fnstrip_re,'',extid)
# log.info("Preparing to parse extension data: %s as '%s'" % (ext_file_path, "%s" % extid))
extensionsLoaded = True
def read_extensions(extensions):
global extensionsLoaded
extfiles = []
expfiles = []
load_start = datetime.datetime.now()
if not extensionsLoaded: #2nd load will throw up errors and duplicate terms
log.info("[%s] extensions %s " % (getInstanceId(short=True),extensions))
for i in extensions:
all_layers[i] = "1"
extfiles = glob.glob("data/ext/%s/*.rdfa" % i)
jextfiles = glob.glob("data/ext/%s/*.jsonld" % i)
for jf in jextfiles:
rdfequiv = jf[:-7]+".rdfa"
if not rdfequiv in extfiles: #Only add .jsonld files if no equivalent .rdfa
extfiles.append(jf)
file_paths = []
for f in extfiles:
file_paths.append(full_path(f))
apirdflib.load_graph(i,file_paths)
log.info("[%s]Loaded extension graphs in %s" % (getInstanceId(short=True),(datetime.datetime.now() - load_start)))
extensionsLoaded = True
def read_extensions(extensions):
import os.path
import glob
import re
global extensionsLoaded
extfiles = []
expfiles = []
if not extensionsLoaded: #2nd load will throw up errors and duplicate terms
log.info("(re)scanning for extensions %s " % extensions)
for i in extensions:
all_layers[i] = "1"
extfiles = glob.glob("data/ext/%s/*.rdfa" % i)
jextfiles = glob.glob("data/ext/%s/*.jsonld" % i)
for jf in jextfiles:
rdfequiv = jf[:-7]+".rdfa"
if not rdfequiv in extfiles: #Only add .jsonld files if no equivalent .rdfa
extfiles.append(jf)
# log.info("FILES: %s" % extfiles)
file_paths = []
for f in extfiles:
file_paths.append(full_path(f))
apirdflib.load_graph(i,file_paths)
expfiles = glob.glob("data/ext/%s/*examples.txt" % i)
read_examples(expfiles,i)
log.info("Extensions found: %s ." % " , ".join(extfiles) )
# fnstrip_re = re.compile("\/.*")
# for ext in extfiles:
# ext_file_path = full_path(ext)
# extid = ext.replace('data/ext/', '')
# extid = re.sub(fnstrip_re,'',extid)
# log.info("Preparing to parse extension data: %s as '%s'" % (ext_file_path, "%s" % extid))
extensionsLoaded = True