def do_import(input_dir_path):
temp_dir_path = tempfile.mkdtemp()
print(f"Using temporary directory {temp_dir_path}")
input_filenames = [
file_path for file_path in listdir(input_dir_path)
if isfile(join(input_dir_path, file_path))
and file_path[len(file_path) - 4:] == ".txt"
]
con = Knora("http://0.0.0.0:3333")
con.login("*****@*****.**", "test")
schema = con.create_schema("00FD", "books")
for input_filename in input_filenames:
print(f"Processing {input_filename}...")
input_filename_without_ext, _ = splitext(input_filename)
input_file_path = join(input_dir_path, input_filename)
output_file_base_path = join(temp_dir_path,
f"{input_filename_without_ext}")
author, title, fragment_paths = add_markup(input_file_path,
output_file_base_path)
fragment_number = 0
fragment_iris = []
for fragment_path in fragment_paths:
with open(fragment_path, "r", encoding="utf-8") as xml_file:
xml_content = xml_file.read()
resource_info = con.create_resource(
schema, "BookFragment",
f"{input_filename_without_ext}_{fragment_number}", {
"seqnum": fragment_number,
"hasText": {
"value":
KnoraStandoffXml(xml_content),
"mapping":
"http://rdfh.ch/projects/00FD/mappings/LinguisticMapping"
}
})
fragment_iri = resource_info['iri']
print(f"Created BookFragment resource {fragment_iri}")
fragment_iris.append(fragment_iri)
fragment_number += 1
resource_info = con.create_resource(schema, "Book",
f"{input_filename_without_ext}", {
"hasAuthor": author,
"hasTitle": title,
"hasFragment": fragment_iris
})
print(f"Created Book resource {resource_info['iri']}")
def test_create_resource(create_test_ontology_fixture):
server = "http://0.0.0.0:3333"
sipi = "http://0.0.0.0:1024",
user = "******",
password = "******"
projectcode = "00FE"
ontoname = "KPT"
con = Knora(server, user, password)
graph = con.get_ontology_graph('00FE', 'kpt')
# print(graph)
# exit(0)
schema = con.create_schema(projectcode, ontoname)
# pprint(schema)
# exit(0)
inst1_info = con.create_resource(
schema, "object1", "obj1_inst1", {
"textprop": "Dies ist ein Text!",
"intprop": 7,
"listprop": "options:opt2",
"dateprop": "1966:CE:1967-05-21",
"decimalprop": {
'value': "3.14159",
'comment': "Die Zahl PI"
},
"geonameprop": "2661604",
"richtextprop":
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<text><p><strong>this is</strong> text</p> with standoff</text>",
"intervalprop": "13.57:15.88"
})
pprint(inst1_info)
# first upload image to SIPI
sipi = Sipi(sipi, con.get_token())
res = sipi.upload_image('test.tif')
pprint(res)
fileref = res['uploadedFiles'][0]['internalFilename']
inst2_info = con.create_resource(schema, "object2", "obj2_inst1", {
"titleprop": "Stained glass",
"linkprop": inst1_info['iri']
}, fileref)
pprint(inst2_info)
def do_import(input_dir_path, upload):
temp_dir_path = tempfile.mkdtemp()
print(f"Using temporary directory {temp_dir_path}")
input_filenames = [
file_path for file_path in listdir(input_dir_path)
if isfile(join(input_dir_path, file_path))
and file_path[len(file_path) - 4:] == ".txt"
]
con = None
schema = None
if upload:
con = Knora("http://0.0.0.0:3333")
con.login("*****@*****.**", "test")
schema = con.create_schema("00FD", "books")
for input_filename in input_filenames:
print(f"Processing {input_filename}...")
input_filename_without_ext, _ = splitext(input_filename)
input_file_path = join(input_dir_path, input_filename)
output_file_path = join(temp_dir_path,
f"{input_filename_without_ext}.xml")
author, title = add_markup(input_file_path, output_file_path)
print(f"Wrote {output_file_path}")
if upload:
with open(output_file_path, "r", encoding="utf-8") as xml_file:
xml_content = xml_file.read()
resource_info = con.create_resource(
schema, "Book", f"{input_filename_without_ext}", {
"hasAuthor": author,
"hasTitle": title,
"hasText": {
"value":
KnoraStandoffXml(xml_content),
"mapping":
"http://rdfh.ch/projects/00FD/mappings/LinguisticMapping"
}
})
print(f"Created resource {resource_info['iri']}")
default="test",
help="The password for login")
parser.add_argument("-P",
"--projectcode",
default="0826",
help="Project short code")
parser.add_argument("-O",
"--ontoname",
default="teimww",
help="Shortname of ontology")
args = parser.parse_args()
con = Knora(args.server)
con.login(args.user, args.password)
schema = con.create_schema(args.projectcode, args.ontoname)
#res_info1 = con.create_resource(schema, "book", "test-book", {
# "title": "Romeo und Julia"
#})
#pprint(res_info1)
#
# res_info2 = con.create_resource(schema, "person", "test-person", {
# "internalID": "&000001",
# "firstName": "William",
# "lastName": "Shakespeare",
# "description": "English Dramatist",
# "birthDate": "GREGORIAN:1564",
# "deathDate": "GREGORIAN:1616",
# "isAuthorOf": "http://rdfh.ch/0826/I2xQrsXYSnuDARYIH772Eg"
import csv
from pprint import pprint
import os
import requests
from knora import Knora, Sipi
server = "http://0.0.0.0:3333"
#user = "******"
user = "******"
password = "******"
projectcode = "0805"
ontoname = "tdk_onto"
con = Knora(server)
con.login(user, password)
sipi = Sipi("http://0.0.0.0:1024", con.get_token())
graph = con.get_ontology_graph(projectcode, ontoname)
schema = con.create_schema(projectcode, ontoname)
json = {
"lageNr": "1234",
"lageGrab": 10,
"lageUmgebung": "Umgebung",
"lageAreal": "Areal",
"lageRaum": "Raum",
"lageSchnitt": "Schnitt"
}
result = con.create_resource(schema, "Lage", "test_resource", json)
pprint(result)
def program(args) -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
"inproject", help="Shortname of project the data should be added to")
parser.add_argument("-s",
"--server",
type=str,
default="http://0.0.0.0:3333",
help="URL of the Knora server")
parser.add_argument("-S",
"--sipi",
type=str,
default="http://0.0.0.0:1024",
help="URL of SIPI server")
parser.add_argument("-u",
"--user",
type=str,
default="*****@*****.**",
help="Username for Knora")
parser.add_argument("-p",
"--password",
type=str,
default="test",
help="The password for login")
parser.add_argument("-F", "--folder", default="-", help="Input folder.")
parser.add_argument("-i", "--infile", default="-", help="Input file.")
parser.add_argument("-a", "--assets", default="-", help="Assets folder.")
parser.add_argument("-I", "--images", default="-", help="images folder.")
parser.add_argument(
"-V",
"--validate",
action='store_true',
help="Do only validation of JSON, no upload of the ontology")
args = parser.parse_args(args)
current_dir = os.path.dirname(os.path.realpath(__file__))
if args.folder == '-':
folder = args.inproject + ".dir"
else:
folder = args.folder
if args.assets == '-':
assets_path = os.path.join(folder, 'assets')
else:
assets_path = args.assets
if args.images == '-':
images_path = os.path.join(folder, 'images')
else:
images_path = args.images
if args.infile == '-':
infile_path = os.path.join(folder, args.inproject) + '.xml'
else:
infile_path = args.infile
xmlschema_doc = etree.parse(
os.path.join(current_dir, 'knora-data-schema.xsd'))
xmlschema = etree.XMLSchema(xmlschema_doc)
doc = etree.parse(infile_path)
xmlschema.assertValid(doc)
del xmlschema
del doc
del xmlschema_doc
print(
"The imput data file is syntactically correct and passed validation!")
if args.validate:
exit(0)
#
# read the XML file containing the data, including project shortcode
#
context: etree.iterparse = etree.iterparse(infile_path,
events=("start", "end"))
resources: List[KnoraResource] = []
permissions: Dict[str, Permission] = {}
while True:
event, node = next(context)
if event == 'start':
if node.tag == 'knora':
ontology = node.attrib['ontology']
shortcode = node.attrib['shortcode']
elif event == 'start' and node.tag == 'resource':
resources.append(KnoraResource(context, node))
elif event == 'start' and node.tag == 'permissions':
permission = Permission(context, node)
permissions[permission.id] = permission
elif event == 'end':
if node.tag == 'knora':
break
context = None # delete XML tree tto save memory
#
# sort the resources so that resources which do not likt to others come first
#
resources = do_sortorder(resources)
#
# connect to Knora
#
con = Knora(args.server)
con.login(args.user, args.password)
sipi = Sipi(args.sipi, con.get_token())
graph = con.get_ontology_graph(shortcode, ontology)
schema = con.create_schema(shortcode, ontology)
permissions_lookup: StrDict = {}
for p in permissions.items():
permissions_lookup[p[0]] = create_permission(con, p[1])
resiri_lookup: StrDict = {}
# cnt: int = 0
for resource in resources:
if resource.image is not None:
print('Uploading ' + resource.image + '...')
imgres = sipi.upload_image(resource.image)
print('Upload done: ' +
imgres['uploadedFiles'][0]['internalFilename'])
fileref = imgres['uploadedFiles'][0]['internalFilename']
print('Uploading data...')
resinfo = con.create_resource(
schema=schema,
res_class=resource.restype,
label=resource.label,
values=resource.get_propvals(resiri_lookup,
permissions_lookup),
permissions=permissions_lookup.get(resource.permissions),
stillimage=fileref)
else:
resinfo = con.create_resource(
schema=schema,
res_class=resource.restype,
label=resource.label,
values=resource.get_propvals(resiri_lookup,
permissions_lookup),
permissions=permissions_lookup.get(resource.permissions))
resiri_lookup[resource.unique_id] = resinfo['iri']
def start():
try:
with open('00_data_as_json/author.json') as authors_file:
authors = json.load(authors_file)
with open('00_data_as_json/book.json') as books_file:
books = json.load(books_file)
with open('00_data_as_json/passage.json') as passages_file:
passages = json.load(passages_file)
with open('00_data_as_json/contributor.json') as contributors_file:
contributors = json.load(contributors_file)
with open('00_data_as_json/lexia.json') as lexias_files:
lexias = json.load(lexias_files)
with open('00_data_as_json/company.json') as company_files:
companies = json.load(company_files)
with open('00_data_as_json/venue.json') as venue_files:
venues = json.load(venue_files)
except Exception as err:
print(err, "Import Fail")
raise SystemExit(0)
parser = argparse.ArgumentParser()
parser.add_argument("-s",
"--server",
type=str,
default="http://0.0.0.0:3333",
help="URL of the Knora server")
parser.add_argument("-u",
"--user",
default="*****@*****.**",
help="Username for Knora")
parser.add_argument("-p",
"--password",
default="test",
help="The password for login")
parser.add_argument("-P",
"--projectcode",
default="0826",
help="Project short code")
parser.add_argument("-O",
"--ontoname",
default="teimww",
help="Shortname of ontology")
args = parser.parse_args()
con = Knora(args.server)
con.login(args.user, args.password)
schema = con.create_schema(args.projectcode, args.ontoname)
ww_bulk_xml = "./test-bulk-output.xml"
ww_bulk_object = BulkImport(schema)
for author in authors:
ww_bulk_object.add_resource(
"person", author, "{} {}".format(authors[author]["hasFirstName"],
authors[author]["hasLastName"]),
authors[author])
for contributor in contributors:
ww_bulk_object.add_resource(
"person", contributor,
"{} {}".format(contributors[contributor]["hasFirstName"],
contributors[contributor]["hasLastName"]),
contributors[contributor])
for book in books:
ww_bulk_object.add_resource("book", book,
books[book]["hasBookTitle"][:16],
books[book])
for passage in passages:
ww_bulk_object.add_resource("passage", passage, "passage",
passages[passage])
for lexia in lexias:
ww_bulk_object.add_resource(
"lexia", lexia, "L: " + lexias[lexia]["hasLexiaTitle"][:16],
lexias[lexia])
for company in companies:
ww_bulk_object.add_resource("company", company, "company",
companies[company])
for venue in venues:
ww_bulk_object.add_resource("venue", venue, "venue", venues[venue])
BULKIMPORT_API_ENDPOINT = "http://localhost:3333/v1/resources/xmlimport/http%3A%2F%2Frdfh.ch%2Fprojects%2F0826"
headers = {"Content-Type": "application/xml"}
ww_bulk_object.write_xml(ww_bulk_xml)
ww_bulk_xml_string = open(ww_bulk_xml).read().encode("utf-8")
r = requests.post(BULKIMPORT_API_ENDPOINT,
data=ww_bulk_xml_string,
headers=headers,
auth=(args.user, args.password))
pprint(r.json())
