def __init__(self, c):
"""This function inits the import module and sets the config."""
from modules.Helpers import Helpers
from modules.Weaviate import Weaviate
self.config = c
self.helpers = Helpers(c)
self.weaviate = Weaviate(c)
def __init__(self, c):
"""This function inits the Helpers and sets the config."""
# make the config available in this class
from modules.Weaviate import Weaviate
self.config = c
self.weaviate = Weaviate(c)
class Empty:
"""This module processes the data import."""
def __init__(self, c):
"""This function inits the module and sets the config."""
# make the config available in this class
from modules.Helpers import Helpers
from modules.Weaviate import Weaviate
self.config = c
self.helpers = Helpers(c)
self.weaviate = Weaviate(c)
def emptyWeaviate(self, conceptType):
counter = 0
# Get all concepts
_, result = self.weaviate.Get("/" + conceptType)
# check if there are concepts
if "totalResults" in result:
# loop and delete
for concept in result[conceptType]:
# delete the concept
statusCode = self.weaviate.Delete("/" + conceptType + "/" + concept["id"])
# validate if valid
if statusCode != 204:
self.helpers.Error(Messages().Get(207))
else:
self.helpers.Info(Messages().Get(125) + concept["id"])
# restart the function
self.emptyWeaviate(conceptType)
def Run(self, force):
"""This public function is the main Run command which runs this module."""
if force == True:
self.emptyWeaviate("things")
self.emptyWeaviate("actions")
else:
var = input(Messages().Get(124))
if var == "y" or var == "Y":
self.emptyWeaviate("things")
self.emptyWeaviate("actions")
else:
self.helpers.Info(Messages().Get(123))
class Truncate:
"""This module processes the data import."""
def __init__(self, c):
"""This function inits the module and sets the config."""
# make the config available in this class
from modules.Helpers import Helpers
from modules.Weaviate import Weaviate
self.config = c
self.helpers = Helpers(c)
self.weaviate = Weaviate(c)
def truncatePropsAndClassses(self, propType):
_, schema = self.weaviate.Get("/schema")
self.helpers.Info("Deleting properties and " + propType)
for singletonClass in schema[propType]["classes"]:
statusCode = self.weaviate.Delete("/schema/" + propType + "/" +
singletonClass["class"])
if statusCode != 200:
self.helpers.Error("Error while deleting property `" +
singletonClass["class"] +
"`. Error code: " + str(statusCode))
def truncateSchema(self):
# check if there is a schema
thingCount, actionsCount = self.helpers.SchemaCount()
if thingCount == 0 and actionsCount == 0:
self.helpers.Info("No schema found, done")
return
# Delete properties
self.truncatePropsAndClassses("actions")
self.truncatePropsAndClassses("things")
def Run(self, force):
"""This public function is the main Run command which runs this module."""
if force == True:
self.truncateSchema()
else:
var = input(Messages().Get(128))
if var == "y" or var == "Y":
self.truncateSchema()
else:
self.helpers.Info(Messages().Get(129))
def main():
"""This class reads the command line arguments and loads the correct modules."""
# Get parsed arguments
args = argparse.ArgumentParser(description=Messages().Get(112))
# Get the arguments for sinit
args.add_argument('--init', help=Messages().Get(100), action="store_true")
args.add_argument('--init-url', default="http://localhost", help=Messages().Get(101))
# Get the arguments for schema import
args.add_argument('--schema-import', help=Messages().Get(104), action="store_true")
args.add_argument('--schema-import-ontology', default="./ontology.json", help=Messages().Get(104))
args.add_argument('--schema-import-overwrite', help=Messages().Get(107), action="store_true")
# Get the arguments for schema export
args.add_argument('--schema-export', help=Messages().Get(108), action="store_true")
args.add_argument('--schema-export-things', default="./things.json", help=Messages().Get(109))
args.add_argument('--schema-export-actions', default="/actions.json", help=Messages().Get(110))
# truncate the schema
args.add_argument('--schema-truncate', help=Messages().Get(126), action="store_true")
args.add_argument('--schema-truncate-force', help=Messages().Get(127), action="store_true")
# Empty a weaviate
args.add_argument('--empty', help=Messages().Get(121), action="store_true")
args.add_argument('--empty-force', help=Messages().Get(122), action="store_true")
options = args.parse_args()
# Check init and validate if set
if options.init is True:
Init().setConfig(options)
# Set the config
config = Init().loadConfig()
# Ping Weaviate to validate the connection
Weaviate(config).Ping()
# Check which items to load
if options.schema_import is True:
from modules.SchemaImport import SchemaImport
SchemaImport(config).Run(options.schema_import_ontology, options.schema_import_overwrite)
elif options.schema_export is True:
from modules.SchemaExport import SchemaExport
SchemaExport(config).Run()
elif options.empty is True:
from modules.Empty import Empty
Empty(config).Run(options.empty_force)
elif options.schema_truncate is True:
from modules.Truncate import Truncate
Truncate(config).Run(options.schema_truncate_force)
else:
exit(0)
#!/usr/bin/env python3
import uuid, os, json, sys, time, weaviate, csv
from datetime import datetime
from modules.Weaviate import Weaviate
from utils.helper import *
import csv
DATADIR = sys.argv[2]
WEAVIATE = Weaviate(sys.argv[1])
# CACHEDIR = sys.argv[2]
CLIENT = weaviate.Client(sys.argv[1])
journals = [];
##
# Function to clean up data
##
def processInput(k, v):
if k == 'Author':
v = v.replace(' Wsj.Com', '')
v = v.replace('.', ' ')
return v
elif k == 'Summary':
v = v.replace('\n', ' ')
return v
return v
##
# Import the publications without refs except for cities
##
def main():
"""This class reads the command line arguments and loads the correct modules."""
# Get parsed arguments
args = argparse.ArgumentParser(description=Messages().Get(112))
# Get the arguments for sinit
args.add_argument('--init', help=Messages().Get(100), action="store_true")
args.add_argument('--init-url',
default="http://localhost",
help=Messages().Get(101))
args.add_argument('--init-key', default="UNSET", help=Messages().Get(102))
args.add_argument('--init-token',
default="UNSET",
help=Messages().Get(103))
# Get the arguments for schema import
args.add_argument('--schema-import',
help=Messages().Get(104),
action="store_true")
args.add_argument('--schema-import-things',
default="./things.json",
help=Messages().Get(105))
args.add_argument('--schema-import-actions',
default="/actions.json",
help=Messages().Get(106))
args.add_argument('--schema-import-delete',
help=Messages().Get(107),
action="store_true")
# Get the arguments for schema export
args.add_argument('--schema-export',
help=Messages().Get(108),
action="store_true")
args.add_argument('--schema-export-things',
default="./things.json",
help=Messages().Get(109))
args.add_argument('--schema-export-actions',
default="/actions.json",
help=Messages().Get(110))
# Get the arguments for bulk import
args.add_argument('--data-import',
default="UNSET",
help=Messages().Get(111))
options = args.parse_args()
# Check init and validate if set
if options.init is True:
Init().setConfig(options)
# Set the config
config = Init().loadConfig()
# Ping Weaviate to validate the connection
Weaviate(config).Ping()
# Check which items to load
if options.schema_import is True:
from modules.SchemaImport import SchemaImport
SchemaImport(config).Run(options.schema_import_things, \
options.schema_import_actions, \
options.schema_import_delete)
elif options.schema_export is True:
from modules.SchemaExport import SchemaExport
SchemaExport(config).Run()
elif options.data_import is True:
from modules.DataImport import DataImport
DataImport(config).Run()
else:
exit(0)
class SchemaImport:
"""This class handles the import of a schema."""
def __init__(self, c):
"""This function inits the import module and sets the config."""
from modules.Helpers import Helpers
from modules.Weaviate import Weaviate
self.config = c
self.helpers = Helpers(c)
self.weaviate = Weaviate(c)
def checkIfThereIsData(self, i):
"""This functions checks if there is data, returns True if there is."""
_, amountOfThings = self.weaviate.Get("/" + i)
if len(amountOfThings[i]) == 0:
return False
else:
return True
def downloadSchemaFiles(self, outputFile, url):
"""This functions downloads a schema file."""
thingsFileFromUrl = urllib.request.urlopen(url)
data = thingsFileFromUrl.read()
with open(outputFile, 'w+') as output:
output.write(data.decode('utf-8'))
return outputFile
def Run(self, thingsFile, actionsFile, deleteIfFound):
"""This functions runs the import module."""
# start the import
self.helpers.Info(Messages().Get(113))
# check if things files is url
if validators.url(thingsFile) is True:
thingsFile = self.downloadSchemaFiles('./things.json', thingsFile)
# open the thingsfile
try:
with open(thingsFile, 'r') as file:
things = json.load(file)
except IOError:
self.helpers.Error(Messages().Get(201) + thingsFile)
# check actions files
if validators.url(actionsFile) is True:
actionsFile = self.downloadSchemaFiles('./things.json',
actionsFile)
# open the actionsfile
try:
with open(actionsFile, 'r') as file:
actions = json.load(file)
except IOError:
self.helpers.Error(Messages().Get(202) + actionsFile)
# Validate if delete function would work
if deleteIfFound is True:
self.helpers.Info(Messages().Get(114))
# check if there is data
if self.checkIfThereIsData("things") is True \
or self.checkIfThereIsData("actions") is True:
self.helpers.Error(Messages().Get(203))
# Render and create things
self.helpers.Info(Messages().Get(115) + "things")
self.helpers.CreateConceptClasses("things", things["classes"],
deleteIfFound)
# Render and create actions
self.helpers.Info(Messages().Get(115) + "actions")
self.helpers.CreateConceptClasses("actions", actions["classes"],
deleteIfFound)
# Add properties to things (needs to run after CreateConceptClasses()!)
self.helpers.Info(Messages().Get(116) + "things")
self.helpers.AddPropsToConceptClasses("things", things["classes"])
# Add properties to things (needs to run after CreateConceptClasses()!)
self.helpers.Info(Messages().Get(116) + "actions")
self.helpers.AddPropsToConceptClasses("actions", actions["classes"])
# Validate Things & Actions
self.helpers.Info(Messages().Get(117))
if self.helpers.ValidateConceptClasses(things["classes"],
actions["classes"]) is True:
self.helpers.Info(Messages().Get(118))
exit(0)
else:
self.helpers.Error(Messages().Get(204))
class Helpers:
"""This class produces the Helpers."""
def __init__(self, c):
"""This function inits the Helpers and sets the config."""
# make the config available in this class
from modules.Weaviate import Weaviate
self.config = c
self.weaviate = Weaviate(c)
def ValidateAndGet(self, thing, value, context):
"""This function validates if a key is available and throws an error if not."""
# check if thing (t) has this value (v)
if value in thing:
return thing[value]
else:
self.Error("Can't locate: " + value + \
" in the context of: [" + context + "]. Are your files correctly formated?")
def CreateConceptClasses(self, name, concepts, deleteIfFound):
"""This function creates a concept in Weaviate style"""
# Loop over all concepts and add them without properties
for concept in concepts:
sendObject = {
"class": self.ValidateAndGet(concept, "class", "classname of " + name),
"description": self.ValidateAndGet(concept, "description", \
"description of " + name),
"properties": [],
"keywords": []
}
# Create empty schema first, this is done to avoid loops in the schema.
noPropertySendObject = sendObject
noPropertySendObject["properties"] = []
code, _ = self.weaviate.Post("/schema/" + name,
noPropertySendObject)
if code is 0 and deleteIfFound is True:
self.weaviate.Delete("/schema/" + name +
noPropertySendObject["class"])
# Add the item
self.weaviate.Post("/schema/" + name, sendObject)
def AddPropsToConceptClasses(self, name, concepts, deleteIfFound):
"""This function adds properties to a concept object. \
needs to run after CreateConceptClasses()."""
# Loop over all concepts and add the properties
for concept in concepts:
# loop over the properties
self.ValidateAndGet(concept, "properties",
"properties of root " + name)
for prperty in concept["properties"]:
# create the property object
propertyObject = {
"dataType": [],
"cardinality":
self.ValidateAndGet(prperty, "cardinality",
"cardinality of " + name),
"description":
self.ValidateAndGet(prperty, "description",
"description of " + name),
"name":
self.ValidateAndGet(prperty, "name", "name of " + name)
}
# validate if dataType is formatted correctly.
self.ValidateAndGet(prperty, "dataType", "dataType of" + name)
if len(prperty["dataType"]) == 0:
self.Error("There is no dataType for the Thing with class: " \
+ self.ValidateAndGet(prperty, "name", "root: " + name))
# check if the dataTypes are set correctly (with multiple crefs, only crefs)
if len(prperty["dataType"]) > 1:
# check if they are all crefs
correctlyFormatted = True
for datatype in prperty["dataType"]:
if datatype[0] != datatype[0].capitalize():
correctlyFormatted = False
if correctlyFormatted is False:
self.Error("There is an incorrect dataType for the Thing with class: " + \
self.ValidateAndGet(prperty, "name", "root dataType: " + name))
# add the dataType(s)
for datatype in prperty["dataType"]:
propertyObject["dataType"].append(datatype)
# add the Keywords
if "keywords" in propertyObject:
self.ValidateAndGet(prperty, "keywords", "keywords of the root " \
+ name + " => " + prperty["name"])
for keyword in prperty["keywords"]:
propertyObject["keywords"].append({
"keyword":
self.ValidateAndGet(keyword, "keyword",
"keyword" + name),
"weight":
self.ValidateAndGet(keyword, "weight",
"weight: " + name)
})
# Delete if deleteIfFound is set
if deleteIfFound == True:
self.Info("Delete: " +
self.ValidateAndGet(prperty, "name", "name of " +
name))
self.weaviate.Delete("/schema/" + name + "/" + \
self.ValidateAndGet(concept, "class", "classname of " + name) + \
"/properties/" + \
self.ValidateAndGet(prperty, "name", "name of " + name))
# Update the class with the schema
status, result = self.weaviate.Post("/schema/" + name + "/" + \
self.ValidateAndGet(concept, "class", "classname of " + name) + \
"/properties", propertyObject)
if status != 200:
self.Error(str(result))
def compareJSON(self, className, objectNeedle, objectStack):
"""This function compares the JSON of a remote Weaviate \
agains local Things and Action files."""
##
# https://github.com/semi-technologies/weaviate-cli/issues/9
##
# loop over stack
# for concept in objectStack:
# if concept["class"] == className:
# print(concept)
# print(objectNeedle)
# exit(0)
return True
def ValidateConceptClasses(self, things, actions):
"""This function validates if a concept is added correctly."""
# On success return True
success = True
# Get the meta tags
status, results = self.weaviate.Get("/meta")
if status != 200:
self.Error("Connection to Weaviate is lost.")
# Loop over the results for things
for remote in results["thingsSchema"]["classes"]:
success = self.compareJSON(remote["class"], remote["properties"],
things)
# Loop over the results for actions
for remote in results["actionsSchema"]["classes"]:
success = self.compareJSON(remote["class"], remote["properties"],
actions)
#
# COMPARRE result TO concepts
#
return success
def SchemaCount(self):
"""Counts the things and actions in the schema"""
_, conceptCount = self.weaviate.Get("/schema")
return len(conceptCount["things"]["classes"]), len(
conceptCount["actions"]["classes"])
def Error(self, i):
"""This function produces an error and throws an exit 1."""
print(datetime.datetime.now().isoformat() + " ERROR: " + i)
exit(1)
def Info(self, i):
"""This function produces an INFO statement."""
print(datetime.datetime.now().isoformat() + " INFO: " + i)