def run():
csvpath = "equinox.csv"
key = os.environ['TERMINUSDB_ACCESS_TOKEN']
endpoint = os.environ['TERMINUSDB_ENDPOINT']
team = os.environ['TERMINUSDB_TEAM']
team_quoted = urllib.parse.quote(team)
#client = WOQLClient(f"https://cloud.terminusdb.com/{team_quoted}/")
if endpoint == 'http://127.0.0.1:6363':
client = WOQLClient(f"{endpoint}")
else:
client = WOQLClient(f"{endpoint}/{team_quoted}/")
# make sure you have put the token in environment variable
# https://docs.terminusdb.com/v10.0/#/terminusx/get-your-api-key
# print(f"key: {key}")
use_token = True
if key == 'false':
use_token = False
client.connect(user='******', team=team, use_token=use_token)
else:
client.connect(team=team, use_token=use_token)
# use when not recreating
# client.connect(db=dbid,team=team,use_token=use_token)
exists = client.get_database(dbid)
if exists:
client.delete_database(dbid, team=team)
client.create_database(dbid,
team,
label=label,
description=description,
prefixes=prefixes)
schema = infer_schema(csvpath)
schema_objects = basic_schema + schema.dump_schema()
#print(json.dumps(schema_objects, indent=4))
import_schema(client,schema_objects)
objects = load_data(csvpath,schema)
# print(json.dumps(objects, indent=4))
import_data(client,objects)
# If you need to start over with connections...
# delete_relationships(client)
connect_polities(client,csvpath)
label="SpaCy Tokens").property("lemma", "string").property(
"pos", "string").property("tag", "string").property(
"dep", "string").property("shape", "string").property(
"is_alpha",
"boolean").property("is_stop", "boolean").property(
"head", "SpaCyToken").property("doc", "SpaCyDoc"))
schema = spacy_doc + token_dt
return schema.execute(client, "Create a schema for SpaCy Tokens")
if __name__ == "__main__":
db_id = "nlp_spacy"
client = WOQLClient(server_url="http://localhost:6363")
client.connect(key="root", account="admin", user="******")
existing = client.get_metadata(db_id, client.uid())
if not existing:
client.create_database(
db_id,
accountid="admin",
label="Spacy Tokens",
description="Storing tokenization result form SpaCy")
else:
client.delete_database(
db_id) # delete the original database if the sechema is updated
client.create_database(
db_id,
accountid="admin",
label="Spacy Tokens",
description="Storing tokenization result form SpaCy")
#client.db(db_id)
build_schema(client)
db = "schemaless_to_schema"
user = "******"
account = "admin"
key = "root"
repository = "local"
client = WOQLClient(server_url)
client.connect(user=user, account=account, key=key, db=db)
try:
client.delete_database(db)
except Exception as E:
pass
client.create_database(db,
account,
label="Schemaless to Schema",
description="Two branch database one with schema",
include_schema=False)
# Add some data which meets the schema
WQ().woql_and(WQ().insert("doc:mike", "scm:BankAccount").property(
"scm:owner", "mike").property("scm:balance",
123)).execute(client, "Add mike")
# Create the schema graph
client.create_graph("schema", "main", "Adding schema")
# Add the schema
WQ().woql_and(WQ().doctype("scm:BankAccount").label(
"Bank Account").description("A bank account").property(
"scm:owner", "xsd:string").label("owner").cardinality(1).property(
db_label = "Places"
db_comment = "All DBPedia places"
directory = 'geonames_100k'
user = "******"
account = "admin"
key = "root"
client = WOQLClient(server_url)
client.connect(user=user, account=account, key=key, db=db)
try:
client.delete_database(db)
except Exception as E:
print(E)
client.create_database(db,account,label=f"{db_label}",
include_schema=False,
description=f"All DBPedia {db} data")
times = []
for f in os.listdir(directory):
filename = f'{directory}/{f}'
ttl_file = open(filename)
contents = ttl_file.read()
ttl_file.close()
before = time.time()
client.insert_triples(
"instance","main",
contents,
f"Adding persondata in 100k chunk ({f})")
after = time.time()
total = (after - before)
#!/usr/bin/python3
from terminusdb_client import WOQLClient
from terminusdb_client import WOQLQuery
server_url = "https://127.0.0.1:6363"
client = WOQLClient(server_url)
client.connect(user="******", account="admin", key="root", db="bike")
client.delete_database("bike")
client.create_database("bike","admin",label="Bikes", description="description")
query = WOQLQuery().get(
WOQLQuery().woql_as("Start station","v:Start_Station")
.woql_as("End station", "v:End_Station")
.woql_as("Start date", "v:Start_Time")
.woql_as("End date", "v:End_Time")
.woql_as("Duration", "v:Duration")
.woql_as("Start station number", "v:Start_ID")
.woql_as("End station number", "v:End_ID")
.woql_as("Bike number", "v:Bike")
.woql_as("Member type", "v:Member_Type")
).post("bike_csv")
result = client.query(query,
"This is a commit string",
{'bike_csv' : '/home/gavin/tmp/bike_tutorial.csv'})
from terminusdb_client import WOQLClient
from terminusdb_client import WOQLQuery as WQ
server_url = "https://127.0.0.1:6363"
db = "wordnet"
user = "******"
account = "admin"
key = "root"
client = WOQLClient(server_url)
client.connect(user=user, account=account, key=key, db=db)
#client.delete_database(db)
try:
client.create_database(db,
account,
label="WordNet",
description="WordNet in TerminusDB",
include_schema=False)
except:
pass
times = []
directory = 'wordnet_chunks'
for f in os.listdir(directory):
filename = f'{directory}/{f}'
ttl_file = open(filename)
contents = ttl_file.read()
ttl_file.close()
before = time.time()
print(f"Loading WordNet in chunks ({f})")
client.insert_triples("instance", "main", contents,
# Rebase the current branch onto the specified remote branch
# more info: https://terminusdb.github.io/terminusdb-client-python/woqlClient.html#terminusdb_client.WOQLClient.rebase
client.rebase("main")
if __name__ == "__main__":
db_id = "Netflix"
url = "netflix.csv"
# TODO: change the team name
team = "<TEAM_NAME>"
client = WOQLClient("https://cloud.terminusdb.com/"+team)
try:
client.connect(team=team, use_token=True)
client.create_database(db_id, label = "Netflix Graph", description = "Create a graph with Netflix data")
except Exception:
client.connect(db=db_id, team=team, use_token=True)
schema.commit(client, commit_msg = "Adding Netflix Schema")
insert_content_data(client, url)
contents = client.query_document({"@type" : "Content"}, count=50)
insert_user_data(list(contents))
print("\nQUERING DOCUMENTS\n")
query_documents(client)
print("\nBranches\n")
user = "******"
account = "admin"
key = "root"
branches = ["literals", "types", "objects", "geo"]
client = WOQLClient(server_url, insecure=True)
client.connect(user=user, account=account, key=key, db=db, insecure=True)
try:
client.delete_database(db)
except Exception as E:
print(E)
client.create_database(db,
account,
label=db_label,
description=db_comment,
include_schema=False)
for branch in branches:
# Load the perties branch
client.branch(branch, empty=True)
client.checkout(branch)
print(f"Importing from {branch}")
for f in os.listdir(branch):
filename = f'{branch}/{f}'
ttl_file = open(filename)
contents = ttl_file.read()
ttl_file.close()
server_url = "https://127.0.0.1:6363"
user = "******"
account = "admin"
key = "root"
db = "roster"
repository = "local"
label = "Roster CSV Example"
description = "An example database for playing with bank accounts"
client = WOQLClient(server_url)
client.connect(user=user, account=account, key=key, db=db)
#client.delete_database(db)
try:
client.create_database(db,account,label=label,
description=description,
include_schema=None)
except Exception as E:
pass
query = WQ().woql_and(
WQ().get(WQ().woql_as("Name","v:Name")
.woql_as("Registration_Date", "v:Date")
.woql_as("Paid", "v:Paid")
).post('roster.csv'),
WQ().idgen("doc:RosterRecord",["v:Name","v:Date","v:Paid"],"v:ID"),
WQ().add_triple("v:ID","scm:name","v:Name"),
WQ().add_triple("v:ID","scm:date","v:Date"),
WQ().add_triple("v:ID","scm:paid","v:Paid"))
client.query(query,
"Adding Roster Data",
#!/usr/bin/python3
from terminusdb_client import WOQLClient
from terminusdb_client import WOQLQuery as WQ
server_url = "https://127.0.0.1:6363"
db = "foo"
user = "******"
account = "admin"
key = "root"
client = WOQLClient(server_url)
client.connect(user=user, account=account, key=key, db=db)
try:
client.create_database(db,
account,
label="foo",
description="foo and bar, together again")
except Exception as E:
print(E)
[x, y, z] = WQ().vars("x", "y", "z")
query = WQ().get(WQ().woql_as(x).woql_as(y).woql_as(z)).file(
"/home/gavin/dev/terminus-server/terminus-schema/api.owl.ttl",
{"type": "turtle"})
client.query(query)
import csv
# Place the snippet from TerminusX here:
# TerminusX
db = "stock_index"
client = WOQLClient(endpoint)
client.connect(account=team,user=user,key=key)
exists = client.get_database(db)
if not exists:
client.create_database(db,
team,
"Stock Exchange Index Data",
"Data From Indexes",
{ '@base' : "terminusdb:///stock_index/",
'@schema' : "terminusdb:///stock_index/schema#" },
True)
schema = [
{ '@type' : 'Class',
'@id' : 'IndexRecord',
'@key' : { '@type' : 'ValueHash' },
'index' : 'xsd:string',
'date' : 'xsd:date',
'open' : 'xsd:decimal',
'high' : 'xsd:decimal',
'low' : 'xsd:decimal',
'close' : 'xsd:decimal',
'adjusted_close' : 'xsd:decimal',
'volume' : 'xsd:decimal'
data_prep = WOQLQuery().woql_and(read_csv, prepare_genre_obj,
prepare_actor_obj, prepare_director_obj,
wangles)
return WOQLQuery().woql_and(data_prep, insert).execute(
client, "Loading data for the movie graph")
db_id = "movie_graph"
client = WOQLClient("https://localhost:6363", insecure=True)
client.connect(key="root", account="admin", user="******")
existing = client.get_metadata(db_id, client.uid())
if not existing:
client.create_database(db_id,
accountid="admin",
label="Graph of IMDB Movies Data",
description="Create a graph with IMDB movies data")
else:
client.set_db(db_id)
create_schema(client)
result = loading_data(
client,
"https://raw.githack.com/terminusdb/terminus-tutorials/master/movies-data/IMDB-Movie-Data.csv"
)
# result_df = query_to_df(result)
# print(result)
# print(result_df.columns)
# print("----------------")
# print(result_df["genre_objid"])
#!/usr/bin/python3
from terminusdb_client import WOQLClient
from terminusdb_client import WOQLQuery as WQ
server_url = "https://127.0.0.1:6363"
db = "places"
db_label = "Places"
db_comment = "All DBPedia places (GeoNames)"
user = "******"
account = "admin"
key = "root"
filename = "geonames_links_en.ttl"
commit_comment = "Adding all geonames"
client = WOQLClient(server_url)
client.connect(user=user, account=account, key=key, db=db)
try:
client.delete_database(db)
except Exception as E:
print(E)
client.create_database(db, account, label=db_label, description=db_comment)
ttl_file = open(filename)
contents = ttl_file.read()
ttl_file.close()
client.insert_triples("instance", "main", contents, commit_comment)
index = 0
for code in codes:
matches.append(generateMatchClause(code, type, index))
inserts.append(generateInsertClause(code, type, index))
index = index + 1
return WOQLQuery().when(WOQLQuery().woql_and(*matches),
WOQLQuery().woql_and(*inserts))
def load_data(client):
"""Load the Sample Data
Parameters
==========
client : a WOQLClient() connection
"""
codes = ["DUB", "LHR", "ETC", "XXX"]
q = generateMultiInsertQuery(codes, "Airport")
#print(json.dumps(q.json(), indent=4))
q.execute(client)
client = WOQLClient(server_url="https://127.0.0.1:6363")
client.connect(key="root", account="admin", user="******")
existing = client.get_metadata(dbId, client.uid())
if not existing:
client.create_database(dbId, "admin", label="Airplane Graph")
else:
client.db(dbId)
create_schema(client)
load_data(client)
brewery = Brewery()
brewery.name = row['name']
brewery.type_of = Brewery_Type[row['brewery_type']]
brewery.address_of = address
brewery.phone = str(row['phone'])
brewery.website_url = row['website_url']
breweries.append(brewery)
return breweries
if __name__ == "__main__":
db_id = "Brewery"
url = "https://raw.githubusercontent.com/openbrewerydb/openbrewerydb/master/breweries.csv"
client = WOQLClient("http://127.0.0.1:6363")
client.connect()
try:
client.create_database(db_id,
team="admin",
label="Open Brewery Graph",
description="Create a graph with brewery data")
except Exception:
client.set_db(db_id)
client.insert_document(my_schema.to_dict(),
graph_type="schema",
commit_msg="I am checking in the schema")
csv_info(url)
insert_data(client, url)
results = client.get_all_documents(graph_type="instance", count=10)
print("\nRESULTS\n", list(results))
from terminusdb_client import WOQLClient, WOQLQuery
import pytest
import time
import sys
import random
db = 'testdb_python_insert_test'
client = WOQLClient("https://127.0.0.1:6363")
client.connect(user="******", account="admin", key="root")
try:
client.delete_database(db, "admin")
except:
pass
client.create_database(db, "admin", include_schema=False)
schema_db = 'testdb_python_insert_schema_test'
schema_client = WOQLClient("https://127.0.0.1:6363")
schema_client.connect(user="******", account="admin", key="root")
try:
schema_client.delete_database(schema_db, "admin")
except:
pass
schema_client.create_database(schema_db, "admin")
def random_string_number():
return str(random.randint(1, 1000000))
def insert_triple():
subj = random_string_number()