def _load_owl(self, owl_file):
if owl_file in self.owl_readers:
self.owl = self.owl_readers[owl_file]
else:
# Retreive owl file for NIDM-Results
# owl_file = os.path.join(RELPATH, 'terms', 'nidm-results.owl')
# check the file exists
assert os.path.exists(owl_file)
# Read owl (turtle) file
owl_path = os.path.dirname(owl_file)
if not "extension" in os.path.dirname(owl_file):
import_files = glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, "imports", '*.ttl'))
else:
import_files = glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, os.pardir, os.pardir, "imports", '*.ttl'))
# Main ontology file
import_files += glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, os.pardir, "terms", '*.owl'))
self.owl = OwlReader(owl_file, import_files)
self.owl_readers[owl_file] = self.owl
def __init__(self,
owl_file,
import_files,
spec_name,
subcomponents=None,
used_by=None,
generated_by=None,
derived_from=None,
attributed_to=None,
prefix=None,
commentable=False,
intro=None):
self.owl = OwlReader(owl_file, import_files)
self.owl.graph.bind('dct', 'http://purl.org/dc/terms/')
self.owl.graph.bind('dicom', 'http://purl.org/nidash/dicom#')
self.owl.graph.bind('nidm', 'http://purl.org/nidash/nidm#')
self.owl.graph.bind('bids', 'http://purl.org/nidash/bids#')
self.owl.graph.bind(
'onli',
'http://neurolog.unice.fr/ontoneurolog/v3.0/instrument.owl#')
self.owl.graph.bind('pato', 'http://purl.obolibrary.org/obo/pato#')
self.owl.graph.bind('prov', 'http://www.w3.org/ns/prov')
self.owl.graph.bind(
'qibo', 'http://www.owl-ontologies.com/Ontology1298855822.owl')
self.owl.graph.bind('sio', 'http://semanticscience.org/resource/')
self.name = spec_name
self.component = self.name.lower().replace("-", "_")
self.section_open = 0
self.already_defined_classes = list()
self.commentable = commentable
self.attributes_done = set()
self.text = ""
self.create_specification(subcomponents, used_by, generated_by,
derived_from, attributed_to, prefix, intro)
def _load_owl(self, owl_file):
if owl_file in self.owl_readers:
self.owl = self.owl_readers[owl_file]
else:
# Retreive owl file for NIDM-Results
# owl_file = os.path.join(RELPATH, 'terms', 'nidm-results.owl')
# check the file exists
assert os.path.exists(owl_file)
# Read owl (turtle) file
owl_path = os.path.dirname(owl_file)
if not "extension" in os.path.dirname(owl_file):
import_files = glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, "imports", '*.ttl'))
else:
import_files = glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, os.pardir, os.pardir, "imports", '*.ttl'))
# Main ontology file
import_files += glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, os.pardir, "terms", '*.owl'))
self.owl = OwlReader(owl_file, import_files)
self.owl_readers[owl_file] = self.owl
def __init__(self, owl_file, import_files, spec_name, subcomponents=None,
used_by=None, generated_by=None, derived_from=None,
attributed_to=None, prefix=None, commentable=False,
intro=None):
self.owl = OwlReader(owl_file, import_files)
self.owl.graph.bind('nidm', 'http://purl.org/nidash/nidm#')
self.name = spec_name
self.component = self.name.lower().replace("-", "_")
self.section_open = 0
self.already_defined_classes = list()
self.commentable = commentable
self.attributes_done = set()
self.text = ""
self.create_specification(subcomponents, used_by, generated_by,
derived_from, attributed_to, prefix, intro)
def setUp(self):
self.my_execption = ""
owl_file = os.path.join(os.path.dirname(
os.path.dirname(
os.path.realpath(__file__))),
'nidmresults', 'owl',
'nidm-results_130.owl')
self.owl = OwlReader(owl_file)
pwd = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
# Store test data in a 'data' folder until 'test'
data_dir = os.path.join(pwd, 'data')
if not os.path.exists(data_dir):
os.makedirs(data_dir)
# Collection containing examples of NIDM-Results packs (1.3.0)
req = Request(
"http://neurovault.org/api/collections/2210/nidm_results")
rep = urlopen(req)
response = rep.read()
data = json.loads(response.decode('utf-8'))
# Download the NIDM-Results packs from NeuroVault if not available
# locally
self.packs = list()
for nidm_res in data["results"]:
url = nidm_res["zip_file"]
study = nidm_res["name"]
nidmpack = os.path.join(data_dir, study + ".zip")
if not os.path.isfile(nidmpack):
f = urlopen(url)
print("downloading " + url + " at " + nidmpack)
with open(nidmpack, "wb") as local_file:
local_file.write(f.read())
self.packs.append(nidmpack)
self.packs = glob.glob(os.path.join(data_dir, '*.nidm.zip'))
self.out_dir = os.path.join(data_dir, 'recomputed')
if os.path.isdir(self.out_dir):
shutil.rmtree(self.out_dir)
os.mkdir(self.out_dir)
def __init__(self,
nidm_classes,
example_file,
one_file_per_class=False,
owl_file=None,
remove_att=None):
self.nidm_classes = nidm_classes
self.one_file_per_class = one_file_per_class
self.remove_att = remove_att
self.owl = None
if owl_file is None:
import_files = glob.glob(os.path.join(NIDMPATH, "imports",
'*.ttl'))
owl_file = os.path.join(NIDM_TERMS_DIR, 'nidm-results.owl')
self.owl = OwlReader(owl_file, import_files)
if not one_file_per_class:
self.file = example_file
else:
self.dir = example_file
def __init__(self, name, ttl_file, gt_ttl_files, exact_comparison,
version):
self.name = name
self.ttl_file = ttl_file
self.gt_ttl_files = gt_ttl_files
self.exact_comparison = exact_comparison
self.graph = Graph()
print(ttl_file)
self.graph.parse(ttl_file, format='turtle')
# Get NIDM-Results version for each example
self.version = version
if self.version != "dev":
self.gt_ttl_files = [
x.replace(os.path.join("nidm", "nidm"),
os.path.join("nidm_releases", self.version, "nidm"))
for x in self.gt_ttl_files
]
# Owl file corresponding to version
owl_file = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'owl',
"nidm-results_" + version.replace(".", "") + ".owl")
self.owl_file = owl_file
owl_imports = None
if self.version == "dev":
owl_imports = glob.glob(
os.path.join(os.path.dirname(owl_file), os.pardir, os.pardir,
"imports", '*.ttl'))
self.owl = OwlReader(self.owl_file, owl_imports)
def main(sid, aid, owl_file, template_files, script_files,
constants_file):
owl_txt = get_file_text(owl_file)
templates_txt = dict()
for template_file in template_files:
templates_txt[template_file] = get_file_text(template_file)
scripts_txt = dict()
for script_file in script_files:
scripts_txt[script_file] = get_file_text(script_file)
cst_txt = get_file_text(constants_file)
sid_name = sid.split(":")[1]
sid_namespace = sid.split(":")[0]
if sid_namespace == "nidm":
uri = NIDM[sid_name]
pref = "NIDM"
elif sid_namespace == "fsl":
uri = FSL[sid_name]
pref = "FSL"
elif sid_namespace == "spm":
uri = SPM[sid_name]
pref = "SPM"
# If alphanumeric identifier was not defined, find the next available
if aid is None:
before_alnum = sid_namespace + ":" + pref + "_"
# Find all alphanumeric identifiers in the owl file
alphanum_ids = set(re.findall("("+before_alnum+'\d+)\s+', owl_txt))
# Get identifier number for next alphanumeric identifier
last_id = sorted(list(alphanum_ids))[-1]
new_id_num = int(last_id.replace(before_alnum, ""))+1
aid = before_alnum+"{0:0>7}".format(new_id_num)
owl = OwlReader
owl = OwlReader(owl_file)
label = owl.get_label(uri).split(":")[1].replace("'", "")
# Replace all occurences of semantic id
owl_txt = owl_txt.replace(sid+" ", aid+" ")
# Replace ids in templates
for tpl, tpl_txt in templates_txt.items():
templates_txt[tpl] = tpl_txt.replace(sid+" ", aid+" ")
for scr, scr_txt in scripts_txt.items():
scripts_txt[scr] = scr_txt.replace('"'+sid+'"', '"'+aid+'"')
new_constant = pref + "_" + \
label.upper().replace(" ", "_").replace("-", "_") + \
" = " + pref + "['"+aid.replace(sid_namespace + ":", "")+"']"
cst_txt = cst_txt.replace("# NIDM constants",
"# NIDM constants\n"+new_constant)
replace_file_txt(owl_file, owl_txt)
replace_file_txt(constants_file, cst_txt)
for tpl, tpl_txt in templates_txt.items():
replace_file_txt(tpl, tpl_txt)
for scr, scr_txt in scripts_txt.items():
replace_file_txt(scr, scr_txt)
def __init__(self, owl_file):
self.owl = OwlReader(owl_file)
class UpdateExpTermReadme():
def __init__(self, owl_file):
self.owl = OwlReader(owl_file)
# Write out Readme
def write_readme(self, readme_file, readme_txt):
readme_file_open = open(readme_file, 'w')
readme_file_open.write(readme_txt)
readme_file_open.close()
def create_term_row(self, term_name, definition, same_as, editor, note,
color, range_value=None, domain=None, indiv_type=None):
img_color = ""
if color:
img_color = '<img src="../../../doc/content/specs/img/'+color+'.png?raw=true"/> '
if same_as:
same_as = "(same as: <a href="+same_as+">"+same_as+"</a>)"
range_domain_type = ""
if range_value is not None:
range_domain_type = """
<td>"""+domain+"""</td>
<td>"""+range_value+"""</td>"""
if indiv_type is not None:
range_domain_type += """
<td>"""+indiv_type+"""</td>"""
# Github mardow-like links
nidm_repo = "https://github.com/incf-nidash/nidm/"
#stato_repo = "https://github.com/ISA-tools/stato/"
nidm_pr_issue = re.compile(nidm_repo+r'[a-zA-Z]*/(\d+)')
note = nidm_pr_issue.sub(r'<a href="'+nidm_repo+r'pull/\1">'+r'#\1</a>', note)
#stato_pr_issue = re.compile(stato_repo+r'[a-zA-Z]*/(\d+)')
#note = stato_pr_issue.sub(r'<a href="'+stato_repo+r'pull/\1">'+r'ISA-tools/stato#\1</a>', note)
if note:
note = note+"<br/>"
# Add a search link (to check current state of the repo)
if "Under discussion" in note:
search_text = "more"
else:
search_text = "find issues/PR"
note = note+"<a href=\""+nidm_repo+"/issues?&q="+term_name.split(":")[1]+"\"> ["+search_text+"] </a>"
term_row = """
<tr>
<td>"""+img_color+"""</td>
<td>"""+note+"""</td>
<td><b>"""+term_name+""": </b>"""+definition+same_as+editor+"""</td>"""+range_domain_type+"""
</tr>"""
return term_row
def create_curation_legend(self, order):
curation_legend = "<b>Curation status</b>: \n"
curation_colors_sorted = [(key, CURATION_COLORS.get(key)) for key in order]
covered_colors = list()
for curation_color in curation_colors_sorted:
# curation_status = str(self.owl.qname(curation_color[0]))
# curation_status_labels = self.owl.objects(curation_color[0], RDFS['label'])
# curation_status = ", ".join(list(curation_status_labels))
color = curation_color[1]
if not color in covered_colors:
curation_legend = curation_legend+\
'<img src="../../../doc/content/specs/img/'+color+\
'.png?raw=true"/> '+CURATION_LEGEND[color]+";\n"
covered_colors.append(color)
return curation_legend
# Get README text according to owl file information
def update_readme(self, readme_file):
class_terms = dict()
prpty_terms = dict()
indiv_terms = dict()
definitions = dict()
editors = dict()
notes = dict()
ranges = dict()
domains = dict()
sameas = dict()
types = dict()
for owl_term in self.owl.classes.union(self.owl.properties).union(self.owl.individuals):
curation_status = self.owl.get_curation_status(owl_term)
definition = self.owl.get_definition(owl_term)
if definition == "":
definition = "<undefined>"
editor = self.owl.get_editor(owl_term)
note = self.owl.get_editor_note(owl_term)
range_value = self.owl.get_range(owl_term)
domain = self.owl.get_domain(owl_term)
same = self.owl.get_same_as(owl_term)
indiv_type = self.owl.get_individual_type(owl_term)
if curation_status:
curation_key = curation_status
term_key = self.owl.get_label(owl_term)
if term_key.startswith("nidm"):
if owl_term in self.owl.classes:
class_terms.setdefault(curation_key, list()).append(term_key)
else:
if owl_term in self.owl.properties:
prpty_terms.setdefault(curation_key, list()).append(term_key)
else:
if owl_term in self.owl.individuals:
indiv_terms.setdefault(curation_key, list()).append(term_key)
definitions[term_key] = definition
editors[term_key] = editor
notes[term_key] = note
ranges[term_key] = range_value
domains[term_key] = domain
sameas[term_key] = same
types[term_key] = indiv_type
# Include missing keys and do not display ready for release terms
order=CURATION_ORDER+(list(set(class_terms.keys()).union(set(prpty_terms.keys())) - set(CURATION_ORDER+list([OBO_READY]))))
print 'order = ', order
class_terms_sorted = [(key, class_terms.get(key)) for key in order]
prpty_terms_sorted = [(key, prpty_terms.get(key)) for key in order]
indiv_terms_sorted = [(key, indiv_terms.get(key)) for key in order]
class_table_txt = "<h2>Classes</h2>\n<table>\n<tr><th>Curation Status</th><th>Issue/PR</th><th>Term</th></tr>"
for tuple_status_term in class_terms_sorted:
curation_status = tuple_status_term[0]
class_names = tuple_status_term[1]
if class_names:
for class_name in sorted(class_names):
class_table_txt += self.create_term_row(class_name, \
definitions[class_name], \
sameas[class_name], \
editors[class_name], \
notes[class_name], \
CURATION_COLORS.setdefault(curation_status, ""))
class_table_txt = class_table_txt+"\n</table>"
prpty_table_txt = "<h2>Properties</h2>\n<table>\n<tr><th>Curation Status</th><th>Issue/PR</th><th>Term</th><th>Domain</th><th>Range</th></tr>"
for tuple_status_term in prpty_terms_sorted:
curation_status = tuple_status_term[0]
term_names = tuple_status_term[1]
if term_names:
for term_name in sorted(term_names):
prpty_table_txt += self.create_term_row(term_name, \
definitions[term_name], \
sameas[term_name], \
editors[term_name], \
notes[term_name], \
CURATION_COLORS.setdefault(curation_status, ""), \
ranges[term_name], \
domains[term_name])
prpty_table_txt = prpty_table_txt+"\n</table>"
indiv_table_txt = "<h2>Individuals</h2>\n<table>\n<tr><th>Curation Status</th><th>Issue/PR</th><th>Term</th><th>Type</th></tr>"
for tuple_status_term in indiv_terms_sorted:
curation_status = tuple_status_term[0]
term_names = tuple_status_term[1]
if term_names:
for term_name in sorted(term_names):
indiv_table_txt += self.create_term_row(term_name, \
definitions[term_name], \
sameas[term_name], \
editors[term_name], \
notes[term_name], \
CURATION_COLORS.setdefault(curation_status, ""), \
None, None,
types[term_name])
indiv_table_txt = indiv_table_txt+"\n</table>"
curation_legend = self.create_curation_legend(order)
title = "<h1>NIDM-Experiment Terms curation status</h1>"
intro = """You will find below a listing of the NIDM-Experiment terms that \
need to be curated. If you would like to help with the curation of a term, \
please follow those steps: first, check if the term is already under discussion in an \
issue and if it is please contribute to the discussion there. If not, create a new issue, \
including the current definition (available in\
the table below) and your proposed update.
If possible, priority should be given to uncurated terms (in red).
Thank you in advance for taking part in NIDM-Experiment term curation!\n\n"""
self.write_readme(readme_file, title+intro+\
curation_legend+class_table_txt+prpty_table_txt+indiv_table_txt)
class TestExamples(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestExamples, self).__init__(*args, **kwargs)
namespaces_def = os.path.join(RELPATH, "terms", "templates", 'Namespaces.txt')
fid = open(namespaces_def, "r")
namespaces = fid.read()
fid.close()
self.term_examples = glob.glob(os.path.join(RELPATH, "terms", "examples", '*.txt'))
self.example_files = example_filenames.union(self.term_examples)
self.examples = dict()
self.owl_files = dict()
for example_file in self.example_files:
ttl_file = os.path.join(os.path.dirname(os.path.dirname(
os.path.abspath(__file__))), example_file)
# ttl_file_url = get_turtle(provn_file)
# ttl_file = provn_file.replace(".provn", ".ttl")
# Read turtle
self.examples[example_file] = Graph()
if example_file in self.term_examples:
fid = open(ttl_file, "r")
ttl_txt = fid.read()
fid.close()
self.examples[example_file].parse(data=namespaces+ttl_txt, format='turtle')
else:
self.examples[example_file].parse(ttl_file, format='turtle')
term_dir = os.path.join(os.path.dirname(ttl_file), os.pardir, 'terms')
if not os.path.isdir(term_dir):
term_dir = os.path.join(os.path.dirname(ttl_file), os.pardir, os.pardir, 'terms')
# Retreive owl file for minimal examples
if not os.path.isdir(term_dir):
term_dir = os.path.join(
os.path.dirname(ttl_file), os.pardir, os.pardir,
os.pardir, 'terms')
owl_files = glob.glob(os.path.join(term_dir, '*.owl'))
self.owl_files[example_file] = owl_files[0]
self.owl_readers = dict()
def _load_owl(self, owl_file):
if owl_file in self.owl_readers:
self.owl = self.owl_readers[owl_file]
else:
# Retreive owl file for NIDM-Results
# owl_file = os.path.join(RELPATH, 'terms', 'nidm-results.owl')
# check the file exists
assert os.path.exists(owl_file)
# Read owl (turtle) file
owl_path = os.path.dirname(owl_file)
if not "extension" in os.path.dirname(owl_file):
import_files = glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, "imports", '*.ttl'))
else:
import_files = glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, os.pardir, os.pardir, "imports", '*.ttl'))
# Main ontology file
import_files += glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, os.pardir, "terms", '*.owl'))
self.owl = OwlReader(owl_file, import_files)
self.owl_readers[owl_file] = self.owl
def test_check_classes(self):
logger.info("TestExamples: test_check_classes")
my_exception = dict()
for example_file in self.example_files:
example_name = example_file
example_graph = self.examples[example_file]
owl = self.owl_files[example_file]
self._load_owl(owl)
# Check that all entity, activity, agent are defined in the data model
exception_msg = self.owl.check_class_names(example_graph, example_name)
my_exception = merge_exception_dict(my_exception, exception_msg)
# Aggredate errors over examples for conciseness
if my_exception:
error_msg = ""
for unrecognised_class_name, examples in my_exception.items():
error_msg += unrecognised_class_name+" (from "+', '.join(examples)+")"
raise Exception(error_msg)
def test_check_attributes(self):
logger.info("TestExamples: test_check_attributes")
my_exception = dict()
my_range_exception = dict()
my_restriction_exception = dict()
for example_file in self.example_files:
example_name = example_file
example_graph = self.examples[example_file]
owl = self.owl_files[example_file]
self._load_owl(owl)
exception_msg = self.owl.check_attributes(example_graph, example_name)
my_exception = merge_exception_dict(my_exception, exception_msg[0])
if not example_file in self.term_examples:
my_range_exception = merge_exception_dict(my_range_exception, exception_msg[1])
else:
# Ignore range exceptions for test examples (as for object
# properties the linked object will be missing)
my_range_exception = dict()
my_restriction_exception = merge_exception_dict(my_restriction_exception, exception_msg[2])
# Aggregate errors over examples for conciseness
error_msg = ""
for found_exception in list([my_exception, my_range_exception, my_restriction_exception]):
if found_exception:
for unrecognised_attribute, example_names in found_exception.items():
error_msg += unrecognised_attribute+" (from "+', '.join(example_names)+")"
# if my_range_exception:
# for unrecognised_range, example_names in my_range_exception.items():
# error_msg += unrecognised_range+" (from "+', '.join(example_names)+")"
if error_msg:
raise Exception(error_msg)
class TestExamples(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestExamples, self).__init__(*args, **kwargs)
namespaces_def = os.path.join(RELPATH, "terms", "templates",
'Namespaces.txt')
fid = open(namespaces_def, "r")
namespaces = fid.read()
fid.close()
self.term_examples = glob.glob(
os.path.join(RELPATH, "terms", "examples", '*.txt'))
self.example_files = example_filenames.union(self.term_examples)
self.examples = dict()
self.owl_files = dict()
for example_file in self.example_files:
ttl_file = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
example_file)
# ttl_file_url = get_turtle(provn_file)
# ttl_file = provn_file.replace(".provn", ".ttl")
# Read turtle
self.examples[example_file] = Graph()
if example_file in self.term_examples:
fid = open(ttl_file, "r")
ttl_txt = fid.read()
fid.close()
self.examples[example_file].parse(data=namespaces + ttl_txt,
format='turtle')
else:
self.examples[example_file].parse(ttl_file, format='turtle')
term_dir = os.path.join(os.path.dirname(ttl_file), os.pardir,
'terms')
if not os.path.isdir(term_dir):
term_dir = os.path.join(os.path.dirname(ttl_file), os.pardir,
os.pardir, 'terms')
# Retreive owl file for minimal examples
if not os.path.isdir(term_dir):
term_dir = os.path.join(os.path.dirname(ttl_file), os.pardir,
os.pardir, os.pardir, 'terms')
owl_files = glob.glob(os.path.join(term_dir, '*.owl'))
self.owl_files[example_file] = owl_files[0]
self.owl_readers = dict()
def _load_owl(self, owl_file):
if owl_file in self.owl_readers:
self.owl = self.owl_readers[owl_file]
else:
# Retreive owl file for NIDM-Results
# owl_file = os.path.join(RELPATH, 'terms', 'nidm-results.owl')
# check the file exists
assert os.path.exists(owl_file)
# Read owl (turtle) file
owl_path = os.path.dirname(owl_file)
if not "extension" in os.path.dirname(owl_file):
import_files = glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, "imports", '*.ttl'))
else:
import_files = glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, os.pardir, os.pardir, "imports", '*.ttl'))
# Main ontology file
import_files += glob.glob(os.path.join(owl_path, \
os.pardir, os.pardir, os.pardir, "terms", '*.owl'))
self.owl = OwlReader(owl_file, import_files)
self.owl_readers[owl_file] = self.owl
def test_check_classes(self):
logger.info("TestExamples: test_check_classes")
my_exception = dict()
for example_file in self.example_files:
example_name = example_file
example_graph = self.examples[example_file]
owl = self.owl_files[example_file]
self._load_owl(owl)
# Check that all entity, activity, agent are defined in the data model
exception_msg = self.owl.check_class_names(example_graph,
example_name)
my_exception = merge_exception_dict(my_exception, exception_msg)
# Aggredate errors over examples for conciseness
if my_exception:
error_msg = ""
for unrecognised_class_name, examples in my_exception.items():
error_msg += unrecognised_class_name + " (from " + ', '.join(
examples) + ")"
raise Exception(error_msg)
def test_check_attributes(self):
logger.info("TestExamples: test_check_attributes")
my_exception = dict()
my_range_exception = dict()
my_restriction_exception = dict()
for example_file in self.example_files:
example_name = example_file
example_graph = self.examples[example_file]
owl = self.owl_files[example_file]
self._load_owl(owl)
exception_msg = self.owl.check_attributes(example_graph,
example_name)
my_exception = merge_exception_dict(my_exception, exception_msg[0])
if not example_file in self.term_examples:
my_range_exception = merge_exception_dict(
my_range_exception, exception_msg[1])
else:
# Ignore range exceptions for test examples (as for object
# properties the linked object will be missing)
my_range_exception = dict()
my_restriction_exception = merge_exception_dict(
my_restriction_exception, exception_msg[2])
# Aggregate errors over examples for conciseness
error_msg = ""
for found_exception in list(
[my_exception, my_range_exception, my_restriction_exception]):
if found_exception:
for unrecognised_attribute, example_names in found_exception.items(
):
error_msg += unrecognised_attribute + " (from " + ', '.join(
example_names) + ")"
# if my_range_exception:
# for unrecognised_range, example_names in my_range_exception.items():
# error_msg += unrecognised_range+" (from "+', '.join(example_names)+")"
if error_msg:
raise Exception(error_msg)
class OwlSpecification(object):
def __init__(self, owl_file, import_files, spec_name, subcomponents=None,
used_by=None, generated_by=None, derived_from=None,
attributed_to=None, prefix=None, commentable=False,
intro=None):
self.owl = OwlReader(owl_file, import_files)
self.owl.graph.bind('nidm', 'http://purl.org/nidash/nidm#')
self.name = spec_name
self.component = self.name.lower().replace("-", "_")
self.section_open = 0
self.already_defined_classes = list()
self.commentable = commentable
self.attributes_done = set()
self.text = ""
self.create_specification(subcomponents, used_by, generated_by,
derived_from, attributed_to, prefix, intro)
def create_specification(self, subcomponents, used_by, generated_by,
derived_from, attributed_to, prefix, intro=None):
self.create_title(self.name+": Types and relations", "definitions")
if intro is not None:
self.text += intro
table_num = 3
for subcomponent_name, classes in subcomponents.items():
classes_by_types = self.owl.get_class_names_by_prov_type(
classes,
prefix=prefix, but=self.already_defined_classes)
self.already_defined_classes += classes
self.create_subcomponent_table(classes_by_types, table_num,
subcomponent_name)
table_num = table_num + 1
all_classes = \
classes_by_types[PROV['Activity']] + \
classes_by_types[PROV['Entity']] + \
classes_by_types[PROV['Agent']] + \
classes_by_types[None]
for class_name in all_classes:
self.create_class_section(
class_name,
self.owl.get_definition(class_name),
self.owl.attributes.setdefault(class_name, None),
used_by, generated_by, derived_from, attributed_to,
children=not (
self.owl.get_prov_class(class_name) == PROV['Entity']))
if subcomponent_name:
self.text += """
</section>"""
self.close_sections()
def create_subcomponent_table(self, classes, table_num,
subcomponent_name=None):
if subcomponent_name:
self.text += """
<section><h1>"""+subcomponent_name+"""</h1>"""
# Check if there is a header file to include here
fname = os.path.join(
INCLUDE_FOLDER,
self.component+"_" +
subcomponent_name.split(" ")[0].lower()+".html")
if os.path.isfile(fname):
fid = open(fname, "r")
self.text += fid.read()
fid.close()
else:
subcomponent_name = ""
# Did not find how to handle table numbering and ids with Respec as we
# did for figures?
table_id = "prov-mapping-"""+subcomponent_name.lower()
self.text += """
<div style="text-align: left;">
<table class="thinborder" \
style="margin-left: auto; margin-right: auto;">
<caption id=\""""+table_id+"""\">\
<a class="internalDFN" href=\"#"""+table_id+"""\">\
Table """+str(table_num)+"""</a>:"""+self.name+"""\
"""+subcomponent_name + """ Concepts</caption> \
<tbody>
<tr>
<th align="center"><b>"""+self.name+""" Concept</b>\
</th>
<th align="center"><b>PROV type</b></th>
<th align="center"><b>Identifier</b></th>
</tr>
"""
self.text += """
<!-- HERE ------------- Beginning of PROV Entities ------------- -->
"""
for prov_class in list([
PROV['Activity'],
PROV['Entity'],
PROV['Agent']]):
sorted_classes = classes[prov_class]
for class_uri in sorted_classes:
self.text += """
<tr>
<td>"""+self.term_link(class_uri)+"""
</td>
"""
# First iteration
if class_uri is sorted_classes[0]:
self.text += """
<td rowspan=\""""+str(len(sorted_classes)) + \
"""\" style="text-align: center;"> """ + \
self.owl.get_label(prov_class) + \
"""</td>
"""
self.text += """
<td>"""+self.owl.graph.qname(class_uri) + \
"""</td>
</tr>
"""
self.text += """
</tbody>
</table>
</div>"""
def create_title(self, title, id=None):
if id is None:
self.text += """
<section>
"""
else:
self.text += """
<section id=\""""+id+"""\">
"""
self.text += """
<h1>"""+title+"""</h1>
"""
self.section_open += 1
def _format_markdown(self, text):
# Replace links specified in markdown by html
text = markdown2.markdown(text).replace("<p>", "").replace("</p>", "")
# Remove trailing new line
text = text[0:-1]
return text
def format_definition(self, definition):
# Capitalize first letter, format markdown and end with dot
if definition:
definition = definition[0].upper() + definition[1:]
definition = self._format_markdown(definition)
definition += "."
return definition
def linked_listing(self, uri_list, prefix="", suffix="", sort=True):
linked_listing = prefix
if sort:
uri_list = self.owl.sorted_by_labels(uri_list)
for i, uri in enumerate(uri_list):
if i == 0:
sep = ""
elif i == len(uri_list):
sep = " and "
else:
sep = ", "
linked_listing += sep+self.term_link(uri)
return linked_listing+suffix
def term_link(self, term_uri, tag="a", text=None):
href = ""
if self.owl.is_external_namespace(term_uri):
href = " href =\""+str(term_uri)+"\""
if text is None:
text = self.owl.get_label(term_uri)
term_link = "<" + tag + " title=\"" + self.owl.get_name(term_uri) + \
"\"" + href + ">" + text+"</"+tag+">"
# # This could be handled by Respec, here we overwrite the id and href
# # fields in order to be able to have an id that is not generated from
# # the title field. e.g. title = nidm_0000001 (nidm:Map) and
# # id = nidm_0000001
# name_lw = self.owl.get_name(term_uri).lower()
# if tag is "dfn":
# link_info = " id=\"dfn-" + name_lw + "\""
# elif tag is "a":
# link_info = " href=\"#dfn-" + name_lw + "\""
# term_link = "<" + tag + link_info + \
# " class=\"internalDFN\"" + \
# " title=\"" + self.owl.get_name(term_uri) + \
# " (" + self.owl.get_label(term_uri) + ")" + \
# "\"" + href + ">" + text + "</" + tag + ">"
if tag is "dfn":
issue_url = "https://github.com/incf-nidash/nidm/issues"
# Add link to current definition
term_link = self.term_link(term_uri, text=term_link)
if self.commentable:
term_link = term_link + \
" <a href=\""+issue_url+"?&q=is%3Aopen+'" + text + \
"'\"\"><sup>☆</sup></a>" + \
"<a href=\""+issue_url+"/new\";\"><sup>+</sup></a>"
return term_link
def create_class_section(self, class_uri, definition, attributes,
used_by=None, generated_by=None,
derived_from=None, attributed_to=None,
children=False,
is_range=False):
class_label = self.owl.get_label(class_uri)
class_name = self.owl.get_name(class_uri)
definition = self.format_definition(definition)
self.text += """
<!-- """+class_label+""" ("""+class_name+""")"""+""" -->
<section id="section-"""+class_label+"""">
<h1 label=\""""+class_name+"""\">"""+class_label+"""</h1>
<div class="glossary-ref">
"""+self.term_link(class_uri, "dfn") + ": " + definition
self.text += "<p> "+self.term_link(class_uri)+" is"
nidm_class = self.owl.get_nidm_parent(class_uri)
if nidm_class:
self.text += " a "+self.term_link(nidm_class)
else:
prov_class = self.owl.get_prov_class(class_uri)
if prov_class:
self.text += " a "+self.owl.get_label(prov_class)
found_used_by = False
if used_by:
if class_uri in used_by:
self.text += self.linked_listing(used_by[class_uri],
" used by ")
found_used_by = True
used_entities = list()
for used_entity, used_activities in used_by.items():
for used_act in used_activities:
if used_act == class_uri:
used_entities.append(used_entity)
if used_entities:
self.text += self.linked_listing(used_entities,
" that uses ",
" entities")
found_attr_to = False
if attributed_to:
if class_uri in attributed_to:
if found_used_by:
self.text += " and "
self.text += self.linked_listing(attributed_to[class_uri],
" attributed to ")
found_attr_to = True
found_generated_by = False
if generated_by:
if class_uri in generated_by:
if found_used_by or found_generated_by:
self.text += " and "
self.text += self.linked_listing(
list([generated_by[class_uri]]), " generated by ")
found_generated_by = True
if class_uri in generated_by.values():
generated_entities = list()
for generated_entity, generated_act in generated_by.items():
if generated_act == class_uri:
generated_entities.append(generated_entity)
if generated_entities:
self.text += self.linked_listing(
generated_entities,
". This activity generates ", " entities")
if derived_from:
if class_uri in derived_from:
if found_used_by or found_generated_by or found_attr_to:
self.text += " and "
self.text += self.linked_listing(
list([derived_from[class_uri]]), " derived from ")
class_children = self.owl.get_direct_children(class_uri)
if class_children:
if found_used_by or found_generated_by or found_attr_to:
self.text += ". It "
else:
self.text += " and "
self.text += " has the following child"
if len(class_children) > 1:
self.text += "ren"
self.text += ": " + \
self.linked_listing(class_children)
self.text += "."
self.text += "</p>"
range_classes = list()
self.text += """
</div>"""
if attributes and (attributes != set([CRYPTO['sha512']])):
self.text += """
<p></p>
<div class="attributes" id="attributes-"""+class_label + \
""""> A """ + \
self.term_link(class_uri)+""" has attributes:
<ul>
<li><span class="attribute" id=\"""" + \
class_label+""".label">rdfs:label</span>: \
(<em class="rfc2119" title="OPTIONAL">OPTIONAL</em>) """\
"""Human readable description of the """ + \
self.term_link(class_uri)+""".</li>"""
for att in sorted(attributes):
# Do not display prov relations as attributes
# (except prov:atLocation...)
if not self.owl.is_prov(att) or (att == PROV['atLocation']):
if att not in self.attributes_done:
# First definition of this attribute
att_tag = "dfn"
else:
att_tag = "a"
self.attributes_done.add(att)
# if att_label.startswith("nidm:"):
att_def = self.owl.get_definition(att)
self.text += """
<li>"""+self.term_link(att, att_tag) + \
'</span>: (<em class="rfc2119" title="OPTIONAL">' + \
'OPTIONAL</em>) ' + self.format_definition(att_def)
if att in self.owl.parent_ranges:
child_ranges = list()
for parent_range in self.owl.parent_ranges[att]:
child_ranges += self.owl.get_direct_children(
parent_range)
if self.owl.get_label(parent_range).\
startswith('nidm'):
range_classes.append(parent_range)
child_ranges = sorted(child_ranges)
# if nidm_namespace:
child_range_txt = ""
if child_ranges:
# Get all child ranges
child_range_txt = self.linked_listing(
child_ranges, " such as ")
self.text += self.linked_listing(
self.owl.parent_ranges[att],
" (range ", child_range_txt+")")
self.text += "."
self.text += "</li>"
self.text += """
</ul>
</div>"""
BASE_REPOSITORY = "https://raw.githubusercontent.com/" + \
"incf-nidash/nidm/master/"
for title, example in self.owl.get_example(class_uri, BASE_REPOSITORY):
self.text += """
</ul>
</div>
<pre class='example highlight' title=\""""+title+"""\">""" + \
cgi.escape(example) + """</pre>"""
# For object property list also children (in sub-sections)
if children:
direct_children = self.owl.sorted_by_labels(
self.owl.get_direct_children(class_uri))
for child in direct_children:
if not child in self.already_defined_classes:
self.create_class_section(
child,
self.owl.get_definition(child),
self.owl.attributes.setdefault(child, None),
children=True)
self.already_defined_classes.append(child)
# Display individuals
individuals = self.owl.sorted_by_labels(
self.owl.get_individuals(class_uri))
if individuals:
self.text += \
" Examples of "+self.term_link(class_uri)+" includes " + \
"<ul>"
for indiv in individuals:
self.text += "<li>" + self.term_link(indiv, "dfn") + ": " + \
self.format_definition(
self.owl.get_definition(indiv)) + \
"</li>"
self.text += "</ul>"
if is_range:
self.text += """
</section>"""
for range_name in self.owl.sorted_by_labels(range_classes):
if not range_name in self.already_defined_classes:
self.already_defined_classes.append(range_name)
self.create_class_section(
range_name,
self.owl.get_definition(range_name),
self.owl.attributes.setdefault(range_name, None),
children=True, is_range=True)
if not is_range:
self.text += """
</section>"""
def close_sections(self):
for x in range(0, self.section_open):
self.text += "\t"*x+"</section>\n"
# Write out specification
def write_specification(self, spec_file=None, component=None,
version=None):
if component and version:
spec_file = os.path.join(DOC_FOLDER, component+"_"+version+".html")
spec_open = codecs.open(spec_file, 'w', "utf-8")
spec_open.write(self.text)
spec_open.close()
def _header_footer(self, prev_file=None, follow_file=None, component=None,
version=None):
release_notes = None
if component:
prev_file = os.path.join(
INCLUDE_FOLDER, component+"_"+version+"_head.html")
if not os.path.isfile(prev_file):
prev_file = os.path.join(
INCLUDE_FOLDER, component+"_head.html")
follow_file = os.path.join(
INCLUDE_FOLDER, component+"_"+version+"_foot.html")
if not os.path.isfile(follow_file):
follow_file = os.path.join(
INCLUDE_FOLDER, component+"_foot.html")
if version:
release_notes = os.path.join(
os.path.dirname(self.owl.file),
component+"_"+version+"_notes.html")
if not os.path.isfile(release_notes):
release_notes = None
if prev_file is not None:
prev_file_open = open(prev_file, 'r')
self.text = prev_file_open.read().decode('utf-8')+self.text
prev_file_open.close()
if release_notes is not None:
release_note_open = open(release_notes, 'r')
self.text = self.text+release_note_open.read()
release_note_open.close()
if follow_file is not None:
follow_file_open = open(follow_file, 'r')
self.text = self.text+follow_file_open.read()
follow_file_open.close()
def main(sid, aid, owl_file, template_files, script_files, constants_file):
owl_txt = get_file_text(owl_file)
templates_txt = dict()
for template_file in template_files:
templates_txt[template_file] = get_file_text(template_file)
scripts_txt = dict()
for script_file in script_files:
scripts_txt[script_file] = get_file_text(script_file)
cst_txt = get_file_text(constants_file)
sid_name = sid.split(":")[1]
sid_namespace = sid.split(":")[0]
if sid_namespace == "nidm":
uri = NIDM[sid_name]
pref = "NIDM"
elif sid_namespace == "fsl":
uri = FSL[sid_name]
pref = "FSL"
elif sid_namespace == "spm":
uri = SPM[sid_name]
pref = "SPM"
# If alphanumeric identifier was not defined, find the next available
if aid is None:
before_alnum = sid_namespace + ":" + pref + "_"
# Find all alphanumeric identifiers in the owl file
alphanum_ids = set(re.findall("(" + before_alnum + '\d+)\s+', owl_txt))
# Get identifier number for next alphanumeric identifier
last_id = sorted(list(alphanum_ids))[-1]
new_id_num = int(last_id.replace(before_alnum, "")) + 1
aid = before_alnum + "{0:0>7}".format(new_id_num)
owl = OwlReader
owl = OwlReader(owl_file)
label = owl.get_label(uri).split(":")[1].replace("'", "")
# Replace all occurences of semantic id
owl_txt = owl_txt.replace(sid + " ", aid + " ")
# Replace ids in templates
for tpl, tpl_txt in templates_txt.items():
templates_txt[tpl] = tpl_txt.replace(sid + " ", aid + " ")
for scr, scr_txt in scripts_txt.items():
scripts_txt[scr] = scr_txt.replace('"' + sid + '"', '"' + aid + '"')
new_constant = pref + "_" + \
label.upper().replace(" ", "_").replace("-", "_") + \
" = " + pref + "['"+aid.replace(sid_namespace + ":", "")+"']"
cst_txt = cst_txt.replace("# NIDM constants",
"# NIDM constants\n" + new_constant)
replace_file_txt(owl_file, owl_txt)
replace_file_txt(constants_file, cst_txt)
for tpl, tpl_txt in templates_txt.items():
replace_file_txt(tpl, tpl_txt)
for scr, scr_txt in scripts_txt.items():
replace_file_txt(scr, scr_txt)
class UpdateTermReadme():
def __init__(self, owl_file):
self.owl = OwlReader(owl_file)
# Write out Readme
def write_readme(self, readme_file, readme_txt):
readme_file_open = open(readme_file, 'w')
readme_file_open.write(readme_txt)
readme_file_open.close()
def create_term_row(self, term_name, definition, same_as, editor, note,
color, range_value=None, domain=None, indiv_type=None):
img_color = ""
if color:
img_color = '<img src="../../../doc/content/specs/img/'+color+'.png?raw=true"/> '
if same_as:
same_as = "(same as: <a href="+same_as+">"+same_as+"</a>)"
range_domain_type = ""
if range_value is not None:
range_domain_type = """
<td>"""+domain+"""</td>
<td>"""+range_value+"""</td>"""
if indiv_type is not None:
range_domain_type += """
<td>"""+indiv_type+"""</td>"""
# Github mardow-like links
nidm_repo = "https://github.com/incf-nidash/nidm/"
stato_repo = "https://github.com/ISA-tools/stato/"
nidm_pr_issue = re.compile(nidm_repo+r'[a-zA-Z]*/(\d+)')
note = nidm_pr_issue.sub(r'<a href="'+nidm_repo+r'pull/\1">'+r'#\1</a>', note)
stato_pr_issue = re.compile(stato_repo+r'[a-zA-Z]*/(\d+)')
note = stato_pr_issue.sub(r'<a href="'+stato_repo+r'pull/\1">'+r'ISA-tools/stato#\1</a>', note)
if note:
note = note+"<br/>"
# Add a search link (to check current state of the repo)
if "Under discussion" in note:
search_text = "more"
else:
search_text = "find issues/PR"
note = note+"<a href=\""+nidm_repo+"/issues?&q="+term_name.split(":")[1]+"\"> ["+search_text+"] </a>"
term_row = """
<tr>
<td>"""+img_color+"""</td>
<td>"""+note+"""</td>
<td><b>"""+term_name+""": </b>"""+definition+same_as+editor+"""</td>"""+range_domain_type+"""
</tr>"""
return term_row
def create_curation_legend(self, order):
curation_legend = "<b>Curation status</b>: \n"
curation_colors_sorted = [(key, CURATION_COLORS.get(key)) for key in order]
covered_colors = list()
for curation_color in curation_colors_sorted:
# curation_status = str(self.owl.qname(curation_color[0]))
# curation_status_labels = self.owl.objects(curation_color[0], RDFS['label'])
# curation_status = ", ".join(list(curation_status_labels))
color = curation_color[1]
if not color in covered_colors:
curation_legend = curation_legend+'<img src="../../../doc/content/specs/img/'+color+'.png?raw=true"/> '+\
CURATION_LEGEND[color]+";\n"
covered_colors.append(color)
return curation_legend
# Get README text according to owl file information
def update_readme(self, readme_file):
class_terms = dict()
prpty_terms = dict()
indiv_terms = dict()
definitions = dict()
editors = dict()
notes = dict()
ranges = dict()
domains = dict()
sameas = dict()
types = dict()
for owl_term in self.owl.classes.union(self.owl.properties).union(self.owl.individuals):
curation_status = self.owl.get_curation_status(owl_term)
definition = self.owl.get_definition(owl_term)
if definition == "":
definition = "<undefined>"
editor = self.owl.get_editor(owl_term)
note = self.owl.get_editor_note(owl_term)
range_value = self.owl.get_range(owl_term)
domain = self.owl.get_domain(owl_term)
same = self.owl.get_same_as(owl_term)
indiv_type = self.owl.get_individual_type(owl_term)
if curation_status:
curation_key = curation_status
term_key = self.owl.get_label(owl_term)
if term_key.startswith("nidm") or term_key.startswith("spm") or\
term_key.startswith("fsl") or term_key.startswith("afni"):
if owl_term in self.owl.classes:
class_terms.setdefault(curation_key, list()).append(term_key)
else:
if owl_term in self.owl.properties:
prpty_terms.setdefault(curation_key, list()).append(term_key)
else:
if owl_term in self.owl.individuals:
indiv_terms.setdefault(curation_key, list()).append(term_key)
definitions[term_key] = definition
editors[term_key] = editor
notes[term_key] = note
ranges[term_key] = range_value
domains[term_key] = domain
sameas[term_key] = same
types[term_key] = indiv_type
# Include missing keys and do not display ready for release terms
order=CURATION_ORDER+(list(set(class_terms.keys()).union(set(prpty_terms.keys())) - set(CURATION_ORDER+list([OBO_READY]))))
class_terms_sorted = [(key, class_terms.get(key)) for key in order]
prpty_terms_sorted = [(key, prpty_terms.get(key)) for key in order]
indiv_terms_sorted = [(key, indiv_terms.get(key)) for key in order]
class_table_txt = "<h2>Classes</h2>\n<table>\n<tr><th>Curation Status</th><th>Issue/PR</th><th>Term</th></tr>"
for tuple_status_term in class_terms_sorted:
curation_status = tuple_status_term[0]
class_names = tuple_status_term[1]
if class_names:
for class_name in sorted(class_names):
class_table_txt += self.create_term_row(class_name, \
definitions[class_name], \
sameas[class_name], \
editors[class_name], \
notes[class_name], \
CURATION_COLORS.setdefault(curation_status, ""))
class_table_txt = class_table_txt+"\n</table>"
prpty_table_txt = "<h2>Properties</h2>\n<table>\n<tr><th>Curation Status</th><th>Issue/PR</th><th>Term</th><th>Domain</th><th>Range</th></tr>"
for tuple_status_term in prpty_terms_sorted:
curation_status = tuple_status_term[0]
term_names = tuple_status_term[1]
if term_names:
for term_name in sorted(term_names):
prpty_table_txt += self.create_term_row(term_name, \
definitions[term_name], \
sameas[term_name], \
editors[term_name], \
notes[term_name], \
CURATION_COLORS.setdefault(curation_status, ""), \
ranges[term_name], \
domains[term_name])
prpty_table_txt = prpty_table_txt+"\n</table>"
indiv_table_txt = "<h2>Individuals</h2>\n<table>\n<tr><th>Curation Status</th><th>Issue/PR</th><th>Term</th><th>Type</th></tr>"
for tuple_status_term in indiv_terms_sorted:
curation_status = tuple_status_term[0]
term_names = tuple_status_term[1]
if term_names:
for term_name in sorted(term_names):
indiv_table_txt += self.create_term_row(term_name, \
definitions[term_name], \
sameas[term_name], \
editors[term_name], \
notes[term_name], \
CURATION_COLORS.setdefault(curation_status, ""), \
None, None,
types[term_name])
indiv_table_txt = indiv_table_txt+"\n</table>"
curation_legend = self.create_curation_legend(order)
title = "<h1>NIDM-Results Terms curation status</h1>"
intro = """You will find below a listing of the NIDM-Results terms that \
need to be curated. If you would like **to help with the curation of a term, \
please follow those steps**:
1. Check if the terms is already under discussion in an issue.
2. If not, create a new issue including the current definition (available in\
the table below) and your proposed update.
If possible, priority should be given to uncurated terms (in red).
Thank you in advance for taking part in NIDM-Results term curation!\n\n"""
self.write_readme(readme_file, title+intro+\
curation_legend+class_table_txt+prpty_table_txt+indiv_table_txt)
def __init__(self, owl_file):
self.owl = OwlReader(owl_file)
class OwlSpecification(object):
def __init__(self, owl_file, import_files, spec_name, subcomponents=None,
used_by=None, generated_by=None, derived_from=None,
attributed_to=None, prefix=None, commentable=False,
intro=None):
self.owl = OwlReader(owl_file, import_files)
self.owl.graph.bind('nidm', 'http://purl.org/nidash/nidm#')
self.owl.graph.bind('sio', 'http://semanticscience.org/resource/')
self.owl.graph.bind('onli', 'http://neurolog.unice.fr/ontoneurolog/v3.0/instrument.owl#')
self.name = spec_name
self.component = self.name.lower().replace("-", "_")
self.section_open = 0
self.already_defined_classes = list()
self.commentable = commentable
self.attributes_done = set()
self.text = ""
self.create_specification(subcomponents, used_by, generated_by,
derived_from, attributed_to, prefix, intro)
def create_specification(self, subcomponents, used_by, generated_by,
derived_from, attributed_to, prefix, intro=None):
self.create_title(self.name+": Types and relations", "definitions")
if intro is not None:
self.text += intro
table_num = 3
for subcomponent_name, classes in subcomponents.items():
classes_by_types = self.owl.get_class_names_by_prov_type(
classes,
prefix=prefix, but=self.already_defined_classes)
self.already_defined_classes += classes
self.create_subcomponent_table(classes_by_types, table_num,
subcomponent_name)
table_num = table_num + 1
all_classes = \
classes_by_types[PROV['Activity']] + \
classes_by_types[PROV['Entity']] + \
classes_by_types[PROV['Agent']] + \
classes_by_types[None]
for class_name in all_classes:
self.create_class_section(
class_name,
self.owl.get_definition(class_name),
self.owl.attributes.setdefault(class_name, None),
used_by, generated_by, derived_from, attributed_to,
children=not (
self.owl.get_prov_class(class_name) == PROV['Entity']))
if subcomponent_name:
self.text += """
</section>"""
self.close_sections()
def create_subcomponent_table(self, classes, table_num,
subcomponent_name=None):
if subcomponent_name:
self.text += """
<section><h1>"""+subcomponent_name+"""</h1>"""
# Check if there is a header file to include here
fname = os.path.join(
INCLUDE_FOLDER,
self.component+"_" +
subcomponent_name.split(" ")[0].lower()+".html")
if os.path.isfile(fname):
fid = open(fname, "r")
self.text += fid.read()
fid.close()
else:
subcomponent_name = ""
# Did not find how to handle table numbering and ids with Respec as we
# did for figures?
table_id = "prov-mapping-"""+subcomponent_name.lower()
self.text += """
<div style="text-align: left;">
<table class="thinborder" \
style="margin-left: auto; margin-right: auto;">
<caption id=\""""+table_id+"""\">\
<a class="internalDFN" href=\"#"""+table_id+"""\">\
Table """+str(table_num)+"""</a>:"""+self.name+"""\
"""+subcomponent_name + """ Concepts</caption> \
<tbody>
<tr>
<th align="center"><b>"""+self.name+""" Concept</b>\
</th>
<th align="center"><b>PROV type</b></th>
<th align="center"><b>Identifier</b></th>
</tr>
"""
self.text += """
<!-- HERE ------------- Beginning of PROV Entities ------------- -->
"""
for prov_class in list([
PROV['Activity'],
PROV['Entity'],
PROV['Agent']]):
sorted_classes = classes[prov_class]
for class_uri in sorted_classes:
self.text += """
<tr>
<td>"""+self.term_link(class_uri)+"""
</td>
"""
# First iteration
if class_uri is sorted_classes[0]:
self.text += """
<td rowspan=\""""+str(len(sorted_classes)) + \
"""\" style="text-align: center;"> """ + \
self.owl.get_label(prov_class) + \
"""</td>
"""
self.text += """
<td>"""+self.owl.graph.qname(class_uri) + \
"""</td>
</tr>
"""
self.text += """
</tbody>
</table>
</div>"""
def create_title(self, title, id=None):
if id is None:
self.text += """
<section>
"""
else:
self.text += """
<section id=\""""+id+"""\">
"""
self.text += """
<h1>"""+title+"""</h1>
"""
self.section_open += 1
def _format_markdown(self, text):
# Replace links specified in markdown by html
text = markdown2.markdown(text).replace("<p>", "").replace("</p>", "")
# Remove trailing new line
text = text[0:-1]
return text
def format_definition(self, definition):
# Capitalize first letter, format markdown and end with dot
if definition:
definition = definition[0].upper() + definition[1:]
definition = self._format_markdown(definition)
definition += "."
return definition
def linked_listing(self, uri_list, prefix="", suffix="", sort=True):
linked_listing = prefix
if sort:
uri_list = self.owl.sorted_by_labels(uri_list)
for i, uri in enumerate(uri_list):
if i == 0:
sep = ""
elif i == len(uri_list):
sep = " and "
else:
sep = ", "
linked_listing += sep+self.term_link(uri)
return linked_listing+suffix
def term_link(self, term_uri, tag="a", text=None):
href = ""
if self.owl.is_external_namespace(term_uri):
href = " href =\""+str(term_uri)+"\""
if text is None:
text = self.owl.get_label(term_uri)
term_link = "<" + tag + " title=\"" + self.owl.get_name(term_uri) + \
"\"" + href + ">" + text+"</"+tag+">"
# # This could be handled by Respec, here we overwrite the id and href
# # fields in order to be able to have an id that is not generated from
# # the title field. e.g. title = nidm_0000001 (nidm:Map) and
# # id = nidm_0000001
# name_lw = self.owl.get_name(term_uri).lower()
# if tag is "dfn":
# link_info = " id=\"dfn-" + name_lw + "\""
# elif tag is "a":
# link_info = " href=\"#dfn-" + name_lw + "\""
# term_link = "<" + tag + link_info + \
# " class=\"internalDFN\"" + \
# " title=\"" + self.owl.get_name(term_uri) + \
# " (" + self.owl.get_label(term_uri) + ")" + \
# "\"" + href + ">" + text + "</" + tag + ">"
if tag is "dfn":
issue_url = "https://github.com/incf-nidash/nidm/issues"
# Add link to current definition
term_link = self.term_link(term_uri, text=term_link)
if self.commentable:
term_link = term_link + \
" <a href=\""+issue_url+"?&q=is%3Aopen+'" + text + \
"'\"\"><sup>☆</sup></a>" + \
"<a href=\""+issue_url+"/new\";\"><sup>+</sup></a>"
return term_link
def create_class_section(self, class_uri, definition, attributes,
used_by=None, generated_by=None,
derived_from=None, attributed_to=None,
children=False,
is_range=False):
class_label = self.owl.get_label(class_uri)
class_name = self.owl.get_name(class_uri)
definition = self.format_definition(definition)
self.text += """
<!-- """+class_label+""" ("""+class_name+""")"""+""" -->
<section id="section-"""+class_label+"""">
<h1 label=\""""+class_name+"""\">"""+class_label+"""</h1>
<div class="glossary-ref">
"""+self.term_link(class_uri, "dfn") + ": " + definition
self.text += "<p> "+self.term_link(class_uri)+" is"
nidm_class = self.owl.get_nidm_parent(class_uri)
if nidm_class:
self.text += " a "+self.term_link(nidm_class)
else:
prov_class = self.owl.get_prov_class(class_uri)
if prov_class:
self.text += " a "+self.owl.get_label(prov_class)
found_used_by = False
if used_by:
if class_uri in used_by:
self.text += self.linked_listing(used_by[class_uri],
" used by ")
found_used_by = True
used_entities = list()
for used_entity, used_activities in used_by.items():
for used_act in used_activities:
if used_act == class_uri:
used_entities.append(used_entity)
if used_entities:
self.text += self.linked_listing(used_entities,
" that uses ",
" entities")
found_attr_to = False
if attributed_to:
if class_uri in attributed_to:
if found_used_by:
self.text += " and "
self.text += self.linked_listing(attributed_to[class_uri],
" attributed to ")
found_attr_to = True
found_generated_by = False
if generated_by:
if class_uri in generated_by:
if found_used_by or found_generated_by:
self.text += " and "
self.text += self.linked_listing(
list([generated_by[class_uri]]), " generated by ")
found_generated_by = True
if class_uri in generated_by.values():
generated_entities = list()
for generated_entity, generated_act in generated_by.items():
if generated_act == class_uri:
generated_entities.append(generated_entity)
if generated_entities:
self.text += self.linked_listing(
generated_entities,
". This activity generates ", " entities")
if derived_from:
if class_uri in derived_from:
if found_used_by or found_generated_by or found_attr_to:
self.text += " and "
self.text += self.linked_listing(
list([derived_from[class_uri]]), " derived from ")
class_children = self.owl.get_direct_children(class_uri)
if class_children:
if found_used_by or found_generated_by or found_attr_to:
self.text += ". It "
else:
self.text += " and "
self.text += " has the following child"
if len(class_children) > 1:
self.text += "ren"
self.text += ": " + \
self.linked_listing(class_children)
self.text += "."
self.text += "</p>"
range_classes = list()
self.text += """
</div>"""
if attributes and (attributes != set([CRYPTO['sha512']])):
self.text += """
<p></p>
<div class="attributes" id="attributes-"""+class_label + \
""""> A """ + \
self.term_link(class_uri)+""" has attributes:
<ul>
<li><span class="attribute" id=\"""" + \
class_label+""".label">rdfs:label</span>: \
(<em class="rfc2119" title="OPTIONAL">OPTIONAL</em>) """\
"""Human readable description of the """ + \
self.term_link(class_uri)+""".</li>"""
for att in sorted(attributes):
# Do not display prov relations as attributes
# (except prov:atLocation...)
if not self.owl.is_prov(att) or (att == PROV['atLocation']):
if att not in self.attributes_done:
# First definition of this attribute
att_tag = "dfn"
else:
att_tag = "a"
self.attributes_done.add(att)
# if att_label.startswith("nidm:"):
att_def = self.owl.get_definition(att)
self.text += """
<li>"""+self.term_link(att, att_tag) + \
'</span>: (<em class="rfc2119" title="OPTIONAL">' + \
'OPTIONAL</em>) ' + self.format_definition(att_def)
if att in self.owl.parent_ranges:
child_ranges = list()
for parent_range in self.owl.parent_ranges[att]:
child_ranges += self.owl.get_direct_children(
parent_range)
if self.owl.get_label(parent_range).\
startswith('nidm'):
range_classes.append(parent_range)
child_ranges = sorted(child_ranges)
# if nidm_namespace:
child_range_txt = ""
if child_ranges:
# Get all child ranges
child_range_txt = self.linked_listing(
child_ranges, " such as ")
self.text += self.linked_listing(
self.owl.parent_ranges[att],
" (range ", child_range_txt+")")
self.text += "."
self.text += "</li>"
self.text += """
</ul>
</div>"""
BASE_REPOSITORY = "https://raw.githubusercontent.com/" + \
"incf-nidash/nidm/master/"
for title, example in self.owl.get_example(class_uri, BASE_REPOSITORY):
self.text += """
</ul>
</div>
<pre class='example highlight' title=\""""+title+"""\">""" + \
cgi.escape(example) + """</pre>"""
# For object property list also children (in sub-sections)
if children:
direct_children = self.owl.sorted_by_labels(
self.owl.get_direct_children(class_uri))
for child in direct_children:
if not child in self.already_defined_classes:
self.create_class_section(
child,
self.owl.get_definition(child),
self.owl.attributes.setdefault(child, None),
children=True)
self.already_defined_classes.append(child)
# Display individuals
individuals = self.owl.sorted_by_labels(
self.owl.get_individuals(class_uri))
if individuals:
self.text += \
" Examples of "+self.term_link(class_uri)+" includes " + \
"<ul>"
for indiv in individuals:
self.text += "<li>" + self.term_link(indiv, "dfn") + ": " + \
self.format_definition(
self.owl.get_definition(indiv)) + \
"</li>"
self.text += "</ul>"
if is_range:
self.text += """
</section>"""
for range_name in self.owl.sorted_by_labels(range_classes):
if not range_name in self.already_defined_classes:
self.already_defined_classes.append(range_name)
self.create_class_section(
range_name,
self.owl.get_definition(range_name),
self.owl.attributes.setdefault(range_name, None),
children=True, is_range=True)
if not is_range:
self.text += """
</section>"""
def close_sections(self):
for x in range(0, self.section_open):
self.text += "\t"*x+"</section>\n"
# Write out specification
def write_specification(self, spec_file=None, component=None,
version=None):
if component and version:
spec_file = os.path.join(DOC_FOLDER, component+"_"+version+".html")
spec_open = codecs.open(spec_file, 'w', "utf-8")
spec_open.write(self.text)
spec_open.close()
def _header_footer(self, prev_file=None, follow_file=None, component=None,
version=None):
release_notes = None
if component:
prev_file = os.path.join(
INCLUDE_FOLDER, component+"_"+version+"_head.html")
if not os.path.isfile(prev_file):
prev_file = os.path.join(
INCLUDE_FOLDER, component+"_head.html")
follow_file = os.path.join(
INCLUDE_FOLDER, component+"_"+version+"_foot.html")
if not os.path.isfile(follow_file):
follow_file = os.path.join(
INCLUDE_FOLDER, component+"_foot.html")
if version:
release_notes = os.path.join(
os.path.dirname(self.owl.file),
component+"_"+version+"_notes.html")
if not os.path.isfile(release_notes):
release_notes = None
if prev_file is not None:
prev_file_open = open(prev_file, 'r')
self.text = prev_file_open.read().decode('utf-8')+self.text
prev_file_open.close()
if release_notes is not None:
release_note_open = open(release_notes, 'r')
self.text = self.text+release_note_open.read()
release_note_open.close()
if follow_file is not None:
follow_file_open = open(follow_file, 'r')
self.text = self.text+follow_file_open.read()
follow_file_open.close()
