Exemple #1
0
def do_import(input_dir_path):
    temp_dir_path = tempfile.mkdtemp()
    print(f"Using temporary directory {temp_dir_path}")
    input_filenames = [
        file_path for file_path in listdir(input_dir_path)
        if isfile(join(input_dir_path, file_path))
        and file_path[len(file_path) - 4:] == ".txt"
    ]

    con = Knora("http://0.0.0.0:3333")
    con.login("*****@*****.**", "test")
    schema = con.create_schema("00FD", "books")

    for input_filename in input_filenames:
        print(f"Processing {input_filename}...")
        input_filename_without_ext, _ = splitext(input_filename)
        input_file_path = join(input_dir_path, input_filename)
        output_file_base_path = join(temp_dir_path,
                                     f"{input_filename_without_ext}")
        author, title, fragment_paths = add_markup(input_file_path,
                                                   output_file_base_path)
        fragment_number = 0
        fragment_iris = []

        for fragment_path in fragment_paths:
            with open(fragment_path, "r", encoding="utf-8") as xml_file:
                xml_content = xml_file.read()

                resource_info = con.create_resource(
                    schema, "BookFragment",
                    f"{input_filename_without_ext}_{fragment_number}", {
                        "seqnum": fragment_number,
                        "hasText": {
                            "value":
                            KnoraStandoffXml(xml_content),
                            "mapping":
                            "http://rdfh.ch/projects/00FD/mappings/LinguisticMapping"
                        }
                    })

                fragment_iri = resource_info['iri']
                print(f"Created BookFragment resource {fragment_iri}")
                fragment_iris.append(fragment_iri)
                fragment_number += 1

        resource_info = con.create_resource(schema, "Book",
                                            f"{input_filename_without_ext}", {
                                                "hasAuthor": author,
                                                "hasTitle": title,
                                                "hasFragment": fragment_iris
                                            })

        print(f"Created Book resource {resource_info['iri']}")
Exemple #2
0
def test_create_resource(create_test_ontology_fixture):
    server = "http://0.0.0.0:3333"
    sipi = "http://0.0.0.0:1024",
    user = "******",
    password = "******"
    projectcode = "00FE"
    ontoname = "KPT"

    con = Knora(server, user, password)
    graph = con.get_ontology_graph('00FE', 'kpt')
    # print(graph)
    # exit(0)
    schema = con.create_schema(projectcode, ontoname)
    # pprint(schema)
    # exit(0)

    inst1_info = con.create_resource(
        schema, "object1", "obj1_inst1", {
            "textprop": "Dies ist ein Text!",
            "intprop": 7,
            "listprop": "options:opt2",
            "dateprop": "1966:CE:1967-05-21",
            "decimalprop": {
                'value': "3.14159",
                'comment': "Die Zahl PI"
            },
            "geonameprop": "2661604",
            "richtextprop":
            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<text><p><strong>this is</strong> text</p> with standoff</text>",
            "intervalprop": "13.57:15.88"
        })
    pprint(inst1_info)

    # first upload image to SIPI
    sipi = Sipi(sipi, con.get_token())
    res = sipi.upload_image('test.tif')
    pprint(res)

    fileref = res['uploadedFiles'][0]['internalFilename']
    inst2_info = con.create_resource(schema, "object2", "obj2_inst1", {
        "titleprop": "Stained glass",
        "linkprop": inst1_info['iri']
    }, fileref)
    pprint(inst2_info)
Exemple #3
0
def do_import(input_dir_path, upload):
    temp_dir_path = tempfile.mkdtemp()
    print(f"Using temporary directory {temp_dir_path}")
    input_filenames = [
        file_path for file_path in listdir(input_dir_path)
        if isfile(join(input_dir_path, file_path))
        and file_path[len(file_path) - 4:] == ".txt"
    ]

    con = None
    schema = None

    if upload:
        con = Knora("http://0.0.0.0:3333")
        con.login("*****@*****.**", "test")
        schema = con.create_schema("00FD", "books")

    for input_filename in input_filenames:
        print(f"Processing {input_filename}...")
        input_filename_without_ext, _ = splitext(input_filename)
        input_file_path = join(input_dir_path, input_filename)
        output_file_path = join(temp_dir_path,
                                f"{input_filename_without_ext}.xml")
        author, title = add_markup(input_file_path, output_file_path)
        print(f"Wrote {output_file_path}")

        if upload:
            with open(output_file_path, "r", encoding="utf-8") as xml_file:
                xml_content = xml_file.read()

                resource_info = con.create_resource(
                    schema, "Book", f"{input_filename_without_ext}", {
                        "hasAuthor": author,
                        "hasTitle": title,
                        "hasText": {
                            "value":
                            KnoraStandoffXml(xml_content),
                            "mapping":
                            "http://rdfh.ch/projects/00FD/mappings/LinguisticMapping"
                        }
                    })

                print(f"Created resource {resource_info['iri']}")
                    default="test",
                    help="The password for login")
parser.add_argument("-P",
                    "--projectcode",
                    default="0826",
                    help="Project short code")
parser.add_argument("-O",
                    "--ontoname",
                    default="teimww",
                    help="Shortname of ontology")

args = parser.parse_args()

con = Knora(args.server)
con.login(args.user, args.password)
schema = con.create_schema(args.projectcode, args.ontoname)

#res_info1 = con.create_resource(schema, "book", "test-book", {
#    "title": "Romeo und Julia"
#})

#pprint(res_info1)
#
# res_info2 = con.create_resource(schema, "person", "test-person", {
#     "internalID": "&000001",
#     "firstName": "William",
#     "lastName": "Shakespeare",
#     "description": "English Dramatist",
#     "birthDate": "GREGORIAN:1564",
#     "deathDate": "GREGORIAN:1616",
#     "isAuthorOf": "http://rdfh.ch/0826/I2xQrsXYSnuDARYIH772Eg"
Exemple #5
0
import csv
from pprint import pprint
import os
import requests
from knora import Knora, Sipi
server = "http://0.0.0.0:3333"
#user = "******"
user = "******"
password = "******"
projectcode = "0805"
ontoname = "tdk_onto"

con = Knora(server)
con.login(user, password)
sipi = Sipi("http://0.0.0.0:1024", con.get_token())

graph = con.get_ontology_graph(projectcode, ontoname)
schema = con.create_schema(projectcode, ontoname)
json = {
    "lageNr": "1234",
    "lageGrab": 10,
    "lageUmgebung": "Umgebung",
    "lageAreal": "Areal",
    "lageRaum": "Raum",
    "lageSchnitt": "Schnitt"
}
result = con.create_resource(schema, "Lage", "test_resource", json)
pprint(result)
def program(args) -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "inproject", help="Shortname of project the data should be added to")
    parser.add_argument("-s",
                        "--server",
                        type=str,
                        default="http://0.0.0.0:3333",
                        help="URL of the Knora server")
    parser.add_argument("-S",
                        "--sipi",
                        type=str,
                        default="http://0.0.0.0:1024",
                        help="URL of SIPI server")
    parser.add_argument("-u",
                        "--user",
                        type=str,
                        default="*****@*****.**",
                        help="Username for Knora")
    parser.add_argument("-p",
                        "--password",
                        type=str,
                        default="test",
                        help="The password for login")
    parser.add_argument("-F", "--folder", default="-", help="Input folder.")
    parser.add_argument("-i", "--infile", default="-", help="Input file.")
    parser.add_argument("-a", "--assets", default="-", help="Assets folder.")
    parser.add_argument("-I", "--images", default="-", help="images folder.")
    parser.add_argument(
        "-V",
        "--validate",
        action='store_true',
        help="Do only validation of JSON, no upload of the ontology")
    args = parser.parse_args(args)

    current_dir = os.path.dirname(os.path.realpath(__file__))

    if args.folder == '-':
        folder = args.inproject + ".dir"
    else:
        folder = args.folder

    if args.assets == '-':
        assets_path = os.path.join(folder, 'assets')
    else:
        assets_path = args.assets

    if args.images == '-':
        images_path = os.path.join(folder, 'images')
    else:
        images_path = args.images

    if args.infile == '-':
        infile_path = os.path.join(folder, args.inproject) + '.xml'
    else:
        infile_path = args.infile

    xmlschema_doc = etree.parse(
        os.path.join(current_dir, 'knora-data-schema.xsd'))
    xmlschema = etree.XMLSchema(xmlschema_doc)
    doc = etree.parse(infile_path)
    xmlschema.assertValid(doc)

    del xmlschema
    del doc
    del xmlschema_doc

    print(
        "The imput data file is syntactically correct and passed validation!")

    if args.validate:
        exit(0)

    #
    # read the XML file containing the data, including project shortcode
    #
    context: etree.iterparse = etree.iterparse(infile_path,
                                               events=("start", "end"))
    resources: List[KnoraResource] = []
    permissions: Dict[str, Permission] = {}
    while True:
        event, node = next(context)
        if event == 'start':
            if node.tag == 'knora':
                ontology = node.attrib['ontology']
                shortcode = node.attrib['shortcode']
            elif event == 'start' and node.tag == 'resource':
                resources.append(KnoraResource(context, node))
            elif event == 'start' and node.tag == 'permissions':
                permission = Permission(context, node)
                permissions[permission.id] = permission
        elif event == 'end':
            if node.tag == 'knora':
                break

    context = None  # delete XML tree tto save memory

    #
    # sort the resources so that resources which do not likt to others come first
    #
    resources = do_sortorder(resources)

    #
    # connect to Knora
    #
    con = Knora(args.server)
    con.login(args.user, args.password)

    sipi = Sipi(args.sipi, con.get_token())

    graph = con.get_ontology_graph(shortcode, ontology)
    schema = con.create_schema(shortcode, ontology)

    permissions_lookup: StrDict = {}
    for p in permissions.items():
        permissions_lookup[p[0]] = create_permission(con, p[1])

    resiri_lookup: StrDict = {}

    # cnt: int = 0

    for resource in resources:
        if resource.image is not None:
            print('Uploading ' + resource.image + '...')
            imgres = sipi.upload_image(resource.image)
            print('Upload done: ' +
                  imgres['uploadedFiles'][0]['internalFilename'])
            fileref = imgres['uploadedFiles'][0]['internalFilename']
            print('Uploading data...')
            resinfo = con.create_resource(
                schema=schema,
                res_class=resource.restype,
                label=resource.label,
                values=resource.get_propvals(resiri_lookup,
                                             permissions_lookup),
                permissions=permissions_lookup.get(resource.permissions),
                stillimage=fileref)
        else:
            resinfo = con.create_resource(
                schema=schema,
                res_class=resource.restype,
                label=resource.label,
                values=resource.get_propvals(resiri_lookup,
                                             permissions_lookup),
                permissions=permissions_lookup.get(resource.permissions))
        resiri_lookup[resource.unique_id] = resinfo['iri']
def start():
    try:
        with open('00_data_as_json/author.json') as authors_file:
            authors = json.load(authors_file)

        with open('00_data_as_json/book.json') as books_file:
            books = json.load(books_file)

        with open('00_data_as_json/passage.json') as passages_file:
            passages = json.load(passages_file)

        with open('00_data_as_json/contributor.json') as contributors_file:
            contributors = json.load(contributors_file)

        with open('00_data_as_json/lexia.json') as lexias_files:
            lexias = json.load(lexias_files)

        with open('00_data_as_json/company.json') as company_files:
            companies = json.load(company_files)

        with open('00_data_as_json/venue.json') as venue_files:
            venues = json.load(venue_files)

    except Exception as err:
        print(err, "Import Fail")
        raise SystemExit(0)

    parser = argparse.ArgumentParser()
    parser.add_argument("-s",
                        "--server",
                        type=str,
                        default="http://0.0.0.0:3333",
                        help="URL of the Knora server")
    parser.add_argument("-u",
                        "--user",
                        default="*****@*****.**",
                        help="Username for Knora")
    parser.add_argument("-p",
                        "--password",
                        default="test",
                        help="The password for login")
    parser.add_argument("-P",
                        "--projectcode",
                        default="0826",
                        help="Project short code")
    parser.add_argument("-O",
                        "--ontoname",
                        default="teimww",
                        help="Shortname of ontology")

    args = parser.parse_args()

    con = Knora(args.server)
    con.login(args.user, args.password)
    schema = con.create_schema(args.projectcode, args.ontoname)

    ww_bulk_xml = "./test-bulk-output.xml"
    ww_bulk_object = BulkImport(schema)

    for author in authors:
        ww_bulk_object.add_resource(
            "person", author, "{} {}".format(authors[author]["hasFirstName"],
                                             authors[author]["hasLastName"]),
            authors[author])

    for contributor in contributors:
        ww_bulk_object.add_resource(
            "person", contributor,
            "{} {}".format(contributors[contributor]["hasFirstName"],
                           contributors[contributor]["hasLastName"]),
            contributors[contributor])

    for book in books:
        ww_bulk_object.add_resource("book", book,
                                    books[book]["hasBookTitle"][:16],
                                    books[book])

    for passage in passages:
        ww_bulk_object.add_resource("passage", passage, "passage",
                                    passages[passage])

    for lexia in lexias:
        ww_bulk_object.add_resource(
            "lexia", lexia, "L: " + lexias[lexia]["hasLexiaTitle"][:16],
            lexias[lexia])

    for company in companies:
        ww_bulk_object.add_resource("company", company, "company",
                                    companies[company])

    for venue in venues:
        ww_bulk_object.add_resource("venue", venue, "venue", venues[venue])

    BULKIMPORT_API_ENDPOINT = "http://localhost:3333/v1/resources/xmlimport/http%3A%2F%2Frdfh.ch%2Fprojects%2F0826"
    headers = {"Content-Type": "application/xml"}

    ww_bulk_object.write_xml(ww_bulk_xml)
    ww_bulk_xml_string = open(ww_bulk_xml).read().encode("utf-8")
    r = requests.post(BULKIMPORT_API_ENDPOINT,
                      data=ww_bulk_xml_string,
                      headers=headers,
                      auth=(args.user, args.password))
    pprint(r.json())