def _con(login: bool = True):
server = "http://0.0.0.0:3333"
email = "*****@*****.**"
password = "******"
# projectcode = "00FE"
# ontoname = "KPT"
con = Knora(server)
if (login):
con.login(email, password)
return con
def do_import(input_dir_path):
temp_dir_path = tempfile.mkdtemp()
print(f"Using temporary directory {temp_dir_path}")
input_filenames = [
file_path for file_path in listdir(input_dir_path)
if isfile(join(input_dir_path, file_path))
and file_path[len(file_path) - 4:] == ".txt"
]
con = Knora("http://0.0.0.0:3333")
con.login("*****@*****.**", "test")
schema = con.create_schema("00FD", "books")
for input_filename in input_filenames:
print(f"Processing {input_filename}...")
input_filename_without_ext, _ = splitext(input_filename)
input_file_path = join(input_dir_path, input_filename)
output_file_base_path = join(temp_dir_path,
f"{input_filename_without_ext}")
author, title, fragment_paths = add_markup(input_file_path,
output_file_base_path)
fragment_number = 0
fragment_iris = []
for fragment_path in fragment_paths:
with open(fragment_path, "r", encoding="utf-8") as xml_file:
xml_content = xml_file.read()
resource_info = con.create_resource(
schema, "BookFragment",
f"{input_filename_without_ext}_{fragment_number}", {
"seqnum": fragment_number,
"hasText": {
"value":
KnoraStandoffXml(xml_content),
"mapping":
"http://rdfh.ch/projects/00FD/mappings/LinguisticMapping"
}
})
fragment_iri = resource_info['iri']
print(f"Created BookFragment resource {fragment_iri}")
fragment_iris.append(fragment_iri)
fragment_number += 1
resource_info = con.create_resource(schema, "Book",
f"{input_filename_without_ext}", {
"hasAuthor": author,
"hasTitle": title,
"hasFragment": fragment_iris
})
print(f"Created Book resource {resource_info['iri']}")
def do_import(input_dir_path, upload):
temp_dir_path = tempfile.mkdtemp()
print(f"Using temporary directory {temp_dir_path}")
input_filenames = [
file_path for file_path in listdir(input_dir_path)
if isfile(join(input_dir_path, file_path))
and file_path[len(file_path) - 4:] == ".txt"
]
con = None
schema = None
if upload:
con = Knora("http://0.0.0.0:3333")
con.login("*****@*****.**", "test")
schema = con.create_schema("00FD", "books")
for input_filename in input_filenames:
print(f"Processing {input_filename}...")
input_filename_without_ext, _ = splitext(input_filename)
input_file_path = join(input_dir_path, input_filename)
output_file_path = join(temp_dir_path,
f"{input_filename_without_ext}.xml")
author, title = add_markup(input_file_path, output_file_path)
print(f"Wrote {output_file_path}")
if upload:
with open(output_file_path, "r", encoding="utf-8") as xml_file:
xml_content = xml_file.read()
resource_info = con.create_resource(
schema, "Book", f"{input_filename_without_ext}", {
"hasAuthor": author,
"hasTitle": title,
"hasText": {
"value":
KnoraStandoffXml(xml_content),
"mapping":
"http://rdfh.ch/projects/00FD/mappings/LinguisticMapping"
}
})
print(f"Created resource {resource_info['iri']}")
def program(args):
# parse the arguments of the command line
parser = argparse.ArgumentParser()
parser.add_argument("-s",
"--server",
type=str,
default="http://0.0.0.0:3333",
help="URL of the Knora server")
parser.add_argument("-u",
"--user",
default="*****@*****.**",
help="Username for Knora")
parser.add_argument("-p",
"--password",
default="test",
help="The password for login")
args = parser.parse_args(args)
# create the knora connection object
con = Knora(args.server)
con.login(args.user, args.password)
con.reset_triplestore_content()
"--password",
default="test",
help="The password for login")
parser.add_argument("-P",
"--projectcode",
default="0826",
help="Project short code")
parser.add_argument("-O",
"--ontoname",
default="teimww",
help="Shortname of ontology")
args = parser.parse_args()
con = Knora(args.server)
con.login(args.user, args.password)
schema = con.create_schema(args.projectcode, args.ontoname)
#res_info1 = con.create_resource(schema, "book", "test-book", {
# "title": "Romeo und Julia"
#})
#pprint(res_info1)
#
# res_info2 = con.create_resource(schema, "person", "test-person", {
# "internalID": "&000001",
# "firstName": "William",
# "lastName": "Shakespeare",
# "description": "English Dramatist",
# "birthDate": "GREGORIAN:1564",
# "deathDate": "GREGORIAN:1616",
import csv
from pprint import pprint
import os
import requests
from knora import Knora, Sipi
server = "http://0.0.0.0:3333"
#user = "******"
user = "******"
password = "******"
projectcode = "0805"
ontoname = "tdk_onto"
con = Knora(server)
con.login(user, password)
sipi = Sipi("http://0.0.0.0:1024", con.get_token())
graph = con.get_ontology_graph(projectcode, ontoname)
schema = con.create_schema(projectcode, ontoname)
json = {
"lageNr": "1234",
"lageGrab": 10,
"lageUmgebung": "Umgebung",
"lageAreal": "Areal",
"lageRaum": "Raum",
"lageSchnitt": "Schnitt"
}
result = con.create_resource(schema, "Lage", "test_resource", json)
pprint(result)
def program(args) -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
"inproject", help="Shortname of project the data should be added to")
parser.add_argument("-s",
"--server",
type=str,
default="http://0.0.0.0:3333",
help="URL of the Knora server")
parser.add_argument("-S",
"--sipi",
type=str,
default="http://0.0.0.0:1024",
help="URL of SIPI server")
parser.add_argument("-u",
"--user",
type=str,
default="*****@*****.**",
help="Username for Knora")
parser.add_argument("-p",
"--password",
type=str,
default="test",
help="The password for login")
parser.add_argument("-F", "--folder", default="-", help="Input folder.")
parser.add_argument("-i", "--infile", default="-", help="Input file.")
parser.add_argument("-a", "--assets", default="-", help="Assets folder.")
parser.add_argument("-I", "--images", default="-", help="images folder.")
parser.add_argument(
"-V",
"--validate",
action='store_true',
help="Do only validation of JSON, no upload of the ontology")
args = parser.parse_args(args)
current_dir = os.path.dirname(os.path.realpath(__file__))
if args.folder == '-':
folder = args.inproject + ".dir"
else:
folder = args.folder
if args.assets == '-':
assets_path = os.path.join(folder, 'assets')
else:
assets_path = args.assets
if args.images == '-':
images_path = os.path.join(folder, 'images')
else:
images_path = args.images
if args.infile == '-':
infile_path = os.path.join(folder, args.inproject) + '.xml'
else:
infile_path = args.infile
xmlschema_doc = etree.parse(
os.path.join(current_dir, 'knora-data-schema.xsd'))
xmlschema = etree.XMLSchema(xmlschema_doc)
doc = etree.parse(infile_path)
xmlschema.assertValid(doc)
del xmlschema
del doc
del xmlschema_doc
print(
"The imput data file is syntactically correct and passed validation!")
if args.validate:
exit(0)
#
# read the XML file containing the data, including project shortcode
#
context: etree.iterparse = etree.iterparse(infile_path,
events=("start", "end"))
resources: List[KnoraResource] = []
permissions: Dict[str, Permission] = {}
while True:
event, node = next(context)
if event == 'start':
if node.tag == 'knora':
ontology = node.attrib['ontology']
shortcode = node.attrib['shortcode']
elif event == 'start' and node.tag == 'resource':
resources.append(KnoraResource(context, node))
elif event == 'start' and node.tag == 'permissions':
permission = Permission(context, node)
permissions[permission.id] = permission
elif event == 'end':
if node.tag == 'knora':
break
context = None # delete XML tree tto save memory
#
# sort the resources so that resources which do not likt to others come first
#
resources = do_sortorder(resources)
#
# connect to Knora
#
con = Knora(args.server)
con.login(args.user, args.password)
sipi = Sipi(args.sipi, con.get_token())
graph = con.get_ontology_graph(shortcode, ontology)
schema = con.create_schema(shortcode, ontology)
permissions_lookup: StrDict = {}
for p in permissions.items():
permissions_lookup[p[0]] = create_permission(con, p[1])
resiri_lookup: StrDict = {}
# cnt: int = 0
for resource in resources:
if resource.image is not None:
print('Uploading ' + resource.image + '...')
imgres = sipi.upload_image(resource.image)
print('Upload done: ' +
imgres['uploadedFiles'][0]['internalFilename'])
fileref = imgres['uploadedFiles'][0]['internalFilename']
print('Uploading data...')
resinfo = con.create_resource(
schema=schema,
res_class=resource.restype,
label=resource.label,
values=resource.get_propvals(resiri_lookup,
permissions_lookup),
permissions=permissions_lookup.get(resource.permissions),
stillimage=fileref)
else:
resinfo = con.create_resource(
schema=schema,
res_class=resource.restype,
label=resource.label,
values=resource.get_propvals(resiri_lookup,
permissions_lookup),
permissions=permissions_lookup.get(resource.permissions))
resiri_lookup[resource.unique_id] = resinfo['iri']
def program(args):
# parse the arguments of the command line
parser = argparse.ArgumentParser()
parser.add_argument("ontofile", help="path to ontology file")
parser.add_argument("-s",
"--server",
type=str,
default="http://0.0.0.0:3333",
help="URL of the Knora server")
parser.add_argument("-u",
"--user",
default="*****@*****.**",
help="Username for Knora")
parser.add_argument("-p",
"--password",
default="test",
help="The password for login")
parser.add_argument(
"-V",
"--validate",
action='store_true',
help="Do only validation of JSON, no upload of the ontology")
parser.add_argument("-l",
"--lists",
action='store_true',
help="Only create the lists")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="Verbose feedback")
args = parser.parse_args(args)
current_dir = os.path.dirname(os.path.realpath(__file__))
# let's read the schema for the ontology definition
if args.lists:
with open(os.path.join(current_dir, 'knora-schema-lists.json')) as s:
schema = json.load(s)
else:
with open(os.path.join(current_dir, 'knora-schema.json')) as s:
schema = json.load(s)
# read the ontology definition
with open(args.ontofile) as f:
ontology = json.load(f)
# validate the ontology definition in order to be sure that it is correct
validate(ontology, schema)
print("Ontology is syntactically correct and passed validation!")
if args.validate:
exit(0)
# create the knora connection object
con = Knora(args.server, ontology.get("prefixes"))
con.login(args.user, args.password)
# bulk_templ = con.create_schema(ontology["project"]["shortcode"], ontology["project"]["ontology"]["name"])
if not args.lists:
# create or update the project
try:
project = con.get_project(ontology["project"]["shortcode"])
except KnoraError as err:
proj_iri = con.create_project(
shortcode=ontology["project"]["shortcode"],
shortname=ontology["project"]["shortname"],
longname=ontology["project"]["longname"],
descriptions=ontology["project"].get("descriptions"),
keywords=ontology["project"].get("keywords"))
else:
if args.verbose is not None:
print("Updating existing project!")
pprint(ontology["project"].get("keywords"))
proj_iri = con.update_project(
shortcode=ontology["project"]["shortcode"],
shortname=ontology["project"]["shortname"],
longname=ontology["project"]["longname"],
descriptions=ontology["project"].get("descriptions"),
keywords=ontology["project"].get("keywords"))
project = con.get_project(ontology["project"]["shortcode"])
if args.verbose is not None:
print("Project-IRI: " + proj_iri)
else:
project = con.get_project(ontology["project"]["shortcode"])
proj_iri = project["id"]
#--------------------------------------------------------------------------
# now let's create the lists
#
if args.verbose is not None:
print("Creating lists...")
lists = ontology["project"].get('lists')
listrootnodes = {}
if lists is not None:
for rootnode in lists:
if args.verbose is not None:
print(" Creating list:" + rootnode['name'])
rootnode_iri = con.create_list_node(
project_iri=proj_iri,
name=rootnode['name'],
labels=rootnode['labels'],
comments=rootnode.get('comments'))
listnodes = list_creator(con, proj_iri, rootnode_iri, rootnode_iri,
rootnode['nodes'])
listrootnodes[rootnode['name']] = {
"id": rootnode_iri,
"nodes": listnodes
}
with open('lists.json', 'w', encoding="utf-8") as fp:
json.dump(listrootnodes, fp, indent=3, sort_keys=True)
if args.lists:
print(
"The definitions of the node-id's can be found in \"lists.json\"!")
exit(0)
#--------------------------------------------------------------------------
# now we add the groups if existing
#
if args.verbose is not None:
print("Adding groups...")
group_iris = []
groups = ontology["project"].get('groups')
if groups is not None:
for group in groups:
try:
group_iri = con.create_group(
project_iri=proj_iri,
name=group["name"],
description=group["description"],
selfjoin=group["selfjoin"]
if group.get("selfjoin") is not None else False,
status=group["status"]
if group.get("status") is not None else True)
except KnoraError as err:
print("Creating group failed: " + err.message)
if args.verbose is not None:
print(" Group added: " + group['name'] + ' (' + group_iri +
')')
#--------------------------------------------------------------------------
# now we add the users if existing
#
if args.verbose is not None:
print("Adding users...")
users = ontology["project"].get('users')
if users is not None:
for user in users:
try:
user_iri = con.create_user(
username=user["username"],
email=user["email"],
given_name=user["givenName"],
family_name=user["familyName"],
password=user["password"],
lang=user["lang"]
if user.get("lang") is not None else "en")
except KnoraError as err:
print("Creating user failed: " + err.message)
userinfo = con.get_user_by_email(email=user["email"])
user_iri = userinfo['id']
try:
con.add_user_to_project(user_iri, proj_iri)
except KnoraError as err:
print('Adding user to project failed: ' + err.message)
if args.verbose is not None:
print(" User added: " + user['username'] + ' (' + user_iri +
')')
if args.verbose is not None:
print(" Adding " + user['username'] + " to groups...")
groupnames = user["groups"]
for groupname in groupnames:
tmp = groupname.split(':')
try:
if len(tmp) > 1:
if tmp[0]: # we have 'proj_shortname:groupname'
group_iri = con.get_group_by_pshortname_and_gname(
tmp[0], tmp[1])
else: # we have ':groupname' and add to currnt project
group_iri = con.get_group_by_piri_and_gname(
proj_iri, tmp[1])
con.add_user_to_group(user_iri, group_iri)
print(" " + user['username'] +
" added to group " + groupname)
else:
if tmp[0] == "ProjectAdmin":
con.add_user_to_project_admin(
user_iri, proj_iri)
print(" " + user['username'] +
" added to group " + groupname)
elif tmp[0] == "SystemAdmin":
con.add_user_to_sysadmin(user_iri)
print(" " + user['username'] +
" added to group " + groupname)
else:
print(" Unknown System group: " + tmp[0])
except KnoraError as err:
print(' Added user to group failed: ' + err.message)
#--------------------------------------------------------------------------
# now we start creating the ontology
#
# first we assemble the ontology IRI
onto_iri = args.server + "/ontology/" + ontology["project"]["shortcode"]\
+ "/" + ontology["project"]["ontology"]["name"] + "/v2"
if args.verbose is not None:
print("Creating the ontology...")
# test, if the ontolgy already exists. if so, let's delete it!
ontos = con.get_project_ontologies(ontology["project"]["shortcode"])
if ontos is not None:
for onto in ontos:
if onto['iri'] == onto_iri:
try:
con.delete_ontology(onto_iri, onto['moddate'])
except KnoraError as err:
print("Deleting ontolopgy failed: " + err.message)
onto_data = con.create_ontology(
onto_name=ontology["project"]["ontology"]["name"],
project_iri=proj_iri,
label=ontology["project"]["ontology"]["label"])
onto_iri = onto_data['onto_iri']
last_onto_date = onto_data['last_onto_date']
# let's create the resources
resource_ids = {}
if args.verbose is not None:
print("Creating the resclasses...")
for resource in ontology["project"]["ontology"]["resources"]:
result = con.create_res_class(
onto_iri=onto_iri,
onto_name=ontology["project"]["ontology"]["name"],
last_onto_date=last_onto_date,
class_name=resource["name"],
super_class=resource["super"]
if ':' in resource["super"] else "knora-api:" + resource["super"],
labels=resource["labels"])
last_onto_date = result["last_onto_date"]
resource_ids[resource["name"]] = result["class_iri"]
if args.verbose is not None:
print("Created resclass: " + resource["name"])
#
# find properties that have been used for multiple resources
#
property_names = []
duplicate_properties = []
for resource in ontology["project"]["ontology"]["resources"]:
for prop in resource["properties"]:
if prop['name'] not in property_names:
property_names.append(prop['name'])
else:
duplicate_properties.append(prop['name'])
if args.verbose is not None:
print("Creating the properties...")
# let's create the properties
property_ids = {}
property_names = []
for resource in ontology["project"]["ontology"]["resources"]:
for prop in resource["properties"]:
if property_ids.get(prop['name']) is None:
guiattrs = prop.get("gui_attributes")
if guiattrs is not None:
new_guiattrs = []
for guiattr in guiattrs:
parts = guiattr.split("=")
if parts[0] == "hlist":
new_guiattrs.append("hlist=<" +
listrootnodes[parts[1]]["id"] +
">")
else:
new_guiattrs.append(guiattr)
guiattrs = new_guiattrs
if prop.get("super") is not None:
super_props = list(
map(lambda a: a
if ':' in a else "knora-api:" + a, prop["super"]))
else:
super_props = ["knora-api:hasValue"]
if prop.get("object") is not None:
tmp = prop["object"].split(':')
if len(tmp) > 1:
if tmp[0]:
object = prop["object"]
else:
object = ontology["project"]["ontology"][
"name"] + ':' + tmp[1]
else:
object = "knora-api:" + prop["object"]
else:
object = None
if prop.get("subject") is not None:
psubject = prop["subject"]
else:
psubject = ontology["project"]["ontology"][
"name"] + ':' + resource["name"]
result = con.create_property(
onto_iri=onto_iri,
onto_name=ontology["project"]["ontology"]["name"],
last_onto_date=last_onto_date,
prop_name=prop["name"],
super_props=super_props,
labels=prop["labels"],
gui_element="salsah-gui:" + prop["gui_element"],
gui_attributes=guiattrs,
subject=psubject
if prop['name'] not in duplicate_properties else None,
object=object,
comments=prop.get("comments"))
last_onto_date = result["last_onto_date"]
property_ids[prop["name"]] = result['prop_iri']
if args.verbose is not None:
print("Property created: {} ({})".format(
prop["name"], result['prop_iri']))
else:
print('Property \"{}\" reused!'.format(prop["name"]))
if args.verbose is not None:
print("Adding cardinalities...")
# add the cardinalities
for resource in ontology["project"]["ontology"]["resources"]:
for prop in resource["properties"]:
result = con.create_cardinality(
onto_iri=onto_iri,
onto_name=ontology["project"]["ontology"]["name"],
last_onto_date=last_onto_date,
class_iri=ontology["project"]["ontology"]["name"] + ':' +
resource["name"],
prop_iri=ontology["project"]["ontology"]["name"] + ':' +
prop["name"],
occurrence=prop["cardinality"],
gui_order=int(prop["gui_order"]),
)
last_onto_date = result["last_onto_date"]
if args.verbose is not None:
print("Cardinality for {} added: {}".format(
prop["name"], prop["cardinality"]))
con = None # force logout by deleting the connection object.
def start():
try:
with open('00_data_as_json/author.json') as authors_file:
authors = json.load(authors_file)
with open('00_data_as_json/book.json') as books_file:
books = json.load(books_file)
with open('00_data_as_json/passage.json') as passages_file:
passages = json.load(passages_file)
with open('00_data_as_json/contributor.json') as contributors_file:
contributors = json.load(contributors_file)
with open('00_data_as_json/lexia.json') as lexias_files:
lexias = json.load(lexias_files)
with open('00_data_as_json/company.json') as company_files:
companies = json.load(company_files)
with open('00_data_as_json/venue.json') as venue_files:
venues = json.load(venue_files)
except Exception as err:
print(err, "Import Fail")
raise SystemExit(0)
parser = argparse.ArgumentParser()
parser.add_argument("-s",
"--server",
type=str,
default="http://0.0.0.0:3333",
help="URL of the Knora server")
parser.add_argument("-u",
"--user",
default="*****@*****.**",
help="Username for Knora")
parser.add_argument("-p",
"--password",
default="test",
help="The password for login")
parser.add_argument("-P",
"--projectcode",
default="0826",
help="Project short code")
parser.add_argument("-O",
"--ontoname",
default="teimww",
help="Shortname of ontology")
args = parser.parse_args()
con = Knora(args.server)
con.login(args.user, args.password)
schema = con.create_schema(args.projectcode, args.ontoname)
ww_bulk_xml = "./test-bulk-output.xml"
ww_bulk_object = BulkImport(schema)
for author in authors:
ww_bulk_object.add_resource(
"person", author, "{} {}".format(authors[author]["hasFirstName"],
authors[author]["hasLastName"]),
authors[author])
for contributor in contributors:
ww_bulk_object.add_resource(
"person", contributor,
"{} {}".format(contributors[contributor]["hasFirstName"],
contributors[contributor]["hasLastName"]),
contributors[contributor])
for book in books:
ww_bulk_object.add_resource("book", book,
books[book]["hasBookTitle"][:16],
books[book])
for passage in passages:
ww_bulk_object.add_resource("passage", passage, "passage",
passages[passage])
for lexia in lexias:
ww_bulk_object.add_resource(
"lexia", lexia, "L: " + lexias[lexia]["hasLexiaTitle"][:16],
lexias[lexia])
for company in companies:
ww_bulk_object.add_resource("company", company, "company",
companies[company])
for venue in venues:
ww_bulk_object.add_resource("venue", venue, "venue", venues[venue])
BULKIMPORT_API_ENDPOINT = "http://localhost:3333/v1/resources/xmlimport/http%3A%2F%2Frdfh.ch%2Fprojects%2F0826"
headers = {"Content-Type": "application/xml"}
ww_bulk_object.write_xml(ww_bulk_xml)
ww_bulk_xml_string = open(ww_bulk_xml).read().encode("utf-8")
r = requests.post(BULKIMPORT_API_ENDPOINT,
data=ww_bulk_xml_string,
headers=headers,
auth=(args.user, args.password))
pprint(r.json())
