def load_csvs(client, csvlist, wrangl, insert):
"""Load the CSVs as input
Parameters
==========
client : a WOQLClient() connection
csvs : a dict of all csvs to be input
"""
for url in csvlist:
csv = get_csv_variables(url)
inputs = WOQLQuery().woql_and(csv, *wrangl)
answer = WOQLQuery().when(inputs, insert)
answer.execute(client)
def load_csvs(client, csvs):
"""Load the CSVs as input
Parameters
==========
client : a WOQLClient() connection
csvs : a dict of all csvs to be input
"""
for key, url in csvs.items():
csv = get_csv_variables(url)
wrangles = get_wrangles()
inputs = WOQLQuery().woql_and(csv, *wrangles)
inserts = get_inserts()
answer = WOQLQuery().when(inputs, inserts)
answer.execute(client)
def create_schema(client):
#
# We first create an abstract class to represent ephemeral entities - things that have lifespans
#
schema = WOQLQuery().when(True).woql_and(
WOQLQuery().doctype("EphemeralEntity").label("Ephemeral Entity").
description("An entity that has a lifespan").abstract().property(
"lifespan_start", "dateTime").label("Existed From").property(
"lifespan_end", "dateTime").label("Existed To"),
#
# This allows us to attach existence start and end times to entities, in a consistent way.
# Then we create our four actual concrete classes, each as a subclass of this one:
#
# That gives you the meta-structure - the major relationships between airports, airlines, countries and flights -
# you can then add whatever other properties you want. All the entities also have lifespan_start and lifespan_end
# properties inherited from EphemeralEntity
WOQLQuery().add_class("Country").label("Country").description(
"A nation state").parent("EphemeralEntity").property(
"country_id", "string").label("Id").property(
"country_name", "string").label("Name").property(
"country_iso_code",
"string").label("ISO Code").property(
"country_fips_code",
"string").label("FIPS Code"), #primary key
WOQLQuery().add_class("Airline").label("Airline").description(
"An operator of airplane flights").parent(
"EphemeralEntity").property("icao_code", "string").label(
"ICAO Code") # primary key
.property("registered_in", "Country").label("Registered In"),
WOQLQuery().add_class("Airport").label("Airport").description(
"An airport where flights terminate").parent(
"EphemeralEntity").property(
"situated_in", "Country").label("Situated In").property(
"icao_code",
"string").label("ICAO Code") # primary key
.property("iata_code", "string").label("IATA Code").property(
"name", "string").label("Name"),
WOQLQuery().add_class("Flight").label("Flight").description(
"A flight between airports").parent("EphemeralEntity").property(
"departs", "Airport").label("Departs").property(
"arrives", "Airport").label("Arrives").property(
"operated_by", "Airline").label("Operated By"))
try:
print("[Building schema..]")
with wary.suppress_Terminus_diagnostics():
schema.execute(client)
except Exception as e:
wary.diagnose(e)
def insert_countries(countries):
c_generator = generate_bulk_insert(countries, country_query)
for Matches, Inserts in c_generator:
query = WOQLQuery().when(WOQLQuery().woql_and(*Matches),
WOQLQuery().woql_and(*Inserts))
try:
#with wary.suppress_Terminus_diagnostics():
query.execute(client)
except Exception as e:
print("EXCEPTION")
wary.diagnose(e)
def create_schema(client):
"""The query which creates the schema
Parameters - it uses variables rather than the fluent style as an example
==========
client : a WOQLClient() connection
"""
base = WOQLQuery().doctype("EphemeralEntity").label("Ephemeral Entity").description("An entity that has a lifespan")
base.property("lifespan_start", "dateTime").label("Existed From")
base.property("lifespan_end", "dateTime").label("Existed To")
country = WOQLQuery().add_class("Country").label("Country").description("A nation state").parent("EphemeralEntity")
country.property("iso_code", "string").label("ISO Code")
country.property("fip_code", "string").label("FIP Code")
airline = WOQLQuery().add_class("Airline").label("Airline").description("An operator of airplane flights").parent("EphemeralEntity")
airline.property("registered_in", "Country").label("Registered In"),
airport = WOQLQuery().add_class("Airport").label("Airport").description("An airport where flights terminate").parent("EphemeralEntity")
airport.property("situated_in", "Country").label("Situated In"),
flight = WOQLQuery().add_class("Flight").label("Flight").description("A flight between airports").parent("EphemeralEntity")
flight.property("departs", "Airport").label("Departs")
flight.property("arrives", "Airport").label("Arrives")
flight .property("operated_by", "Airline").label("Operated By")
schema = WOQLQuery().when(True).woql_and(base, country, airline, airport, flight)
return schema.execute(client)
def exec_bulk_country_query(country_queries, priority):
airport, country_queries_unzip = list(zip(*country_queries))
query = WOQLQuery().woql_and(*country_queries_unzip)
try:
# with wary.suppress_Terminus_diagnostics():
result = query.execute(client)
df_result = pd.DataFrame() if is_empty(result) else wdf.query_to_df(
result)
if not df_result.empty:
df_result['id'] = df_result.index
columns = [
"All", "Paren1", "Country", "country_id",
getPriority(priority), "country_name"
]
df_result = pd.wide_to_long(df_result, columns, i="id",
j="val").reset_index()
return (zip(airport, df_result["country_id"]))
## TODO: Waiting that opt() work to catch bad query and rerun them.
except Exception as e:
print("EXCEPTION")
wary.diagnose(e)
def create_schema_from_addon(client, queries):
new_queries = []
for query in queries:
if len(query) > 0:
new_queries.append(WOQLQuery().woql_and(*query))
schema = WOQLQuery().when(True).woql_and(*new_queries)
output_json(schema, filename)
return schema.execute(client)
def insert_airports(airports):
c_generator = generate_bulk_insert(airports, airport_query)
for Matches, Inserts in c_generator:
query = WOQLQuery().when(
WOQLQuery().woql_and(*Matches),
WOQLQuery().woql_and(*Inserts),
)
try:
print("INSERT NEXT TEN")
with wary.suppress_Terminus_diagnostics():
query.execute(client)
except Exception as e:
print("EXCEPTION")
wary.diagnose(e)
def create_schema(client):
'''
Build the schema.
For this example, it is very simple:
just a doctype for a Person with various attributes
:param client: TerminusDB server handle
'''
schema = WOQLQuery().when(True).woql_and(WOQLQuery().doctype(
"Person").label("Person").description("Somebody").property(
"Name", "string").property("Sex", "string").property(
"Parent1", "string").property("Parent2", "string"))
try:
print("[Building schema..]")
with wary.suppress_Terminus_diagnostics():
schema.execute(client)
except Exception as e:
wary.diagnose(e)
def load_csv(client, url, voyages):
'''
Read a .csv file and use its raw data to initialise a graph in the TerminusDB server.
In the case of a local file, it should be the file path relative to the value of the
TERMINUS_LOCAL environment variable set when the TerminusDB server was started...
:param client: handle on the TerminusDB server
:param url: string, eiher a local file name or http-style url
:param voyages: boolean, whether a Voyage or Berth document set are to be created
:return: None
'''
csv = get_csv_variables(url, voyages)
wrangles = get_wrangles(voyages)
inputs = WOQLQuery().woql_and(csv, *wrangles)
inserts = get_inserts(voyages)
answer = WOQLQuery().when(inputs, inserts)
try:
print("[Loading raw data from '{}'..]".format(url))
with wary.suppress_Terminus_diagnostics():
answer.execute(client)
except woqlError.APIError as e:
wary.diagnose(e)
def create_schema(client):
'''
Build the schema.
For this demo, there is a base document to capture ephemeral events.
There is then a derived document for Voyage events, and Berth events.
Note especially that ANY property common to all the derived documents MUST be put in the
base:
Here, the 'ship' property is common to both 'Voyage' and 'Berth' and so must be in the
'Ship_Event' document. If instead it is placed in 'Voyage' and also in 'Base', then
TerminusDB will complain with a "class subsumption" error...
:param client: TerminusDB server handle
'''
base = WOQLQuery().doctype("Ship_Event").label("Ship Event").description("An ephemeral")
base.property("ship", "string").label("Ship Name")
base.property("start", "dateTime").label("Existed From") # try "dateTime rather than string?
base.property("end", "dateTime").label("Existed To")
voyage = WOQLQuery().add_class("Voyage").label("Voyage").description("Ship movement").parent("Ship_Event")
voyage.property("route", "string").label("Route")
docking = WOQLQuery().add_class("Docking").label("Docking").description("A ship docked at a berth").parent("Ship_Event")
docking.property("berth", "string").label("Berth")
schema = WOQLQuery().when(True).woql_and(
base,
docking,
voyage
)
try:
print("[Building schema..]")
with wary.suppress_Terminus_diagnostics():
schema.execute(client)
except Exception as e:
wary.diagnose(e)
def create_schema(client):
"""The query which creates the schema
Parameters
==========
client : a WOQLClient() connection
"""
schema = WOQLQuery().when(True).woql_and(
WOQLQuery().doctype("Party").label("Party").description(
"Political Party"),
WOQLQuery().doctype("Representative").label("Representative").
description("An elected member Dublin city council").property(
"member_of", "Party").label("Member of").cardinality(1),
WOQLQuery().doctype("Similarity").label("Similarity").property(
"similarity", "decimal").label("Similarity").property(
"similar_to",
"Representative").label("Similar To").cardinality(2))
return schema.execute(client)
def create_schema(client):
"""The query which creates the schema
Parameters - it uses variables rather than the fluent style as an example
==========
client : a WOQLClient() connection
"""
schema = WOQLQuery().when(True).woql_and(
WOQLQuery().doctype("Station").
label("Bike Station").
description("A station where bikes are deposited"),
WOQLQuery().doctype("Bicycle").label("Bicycle"),
WOQLQuery().doctype("Journey").label("Journey").
property("start_station", "Station").label("Start Station").
property("end_station", "Station").label("End Station").
property("duration", "integer").label("Journey Duration").
property("start_time", "dateTime").label("Time Started").
property("end_time", "dateTime").label("Time Ended").
property("journey_bicycle", "Bicycle").label("Bicycle Used")
)
return schema.execute(client)
def create_schema_from_queries(client, queries):
schema = WOQLQuery().when(True).woql_and(*queries)
return schema.execute(client)
'@value': value.father
})))
#
# We are inserting each Person one at a time into Terminus: each insertion
# involves a call out to the database, and would thus be slow if there were a lot of data..
#
# We could instead change the woql when clause and both each woql_ands to bundle a number of
# idgens and insertions for different documents together. This then would be more efficient.
#
# For you to try if you wish.. :-)
#
try:
print("[Inserting {}..]".format(person))
with wary.suppress_Terminus_diagnostics():
answer.execute(client)
except Exception as e:
wary.diagnose(e)
#
# Some sample queries..
#
print("\nList people....")
df = list_people()
print("{:,} people found".format(df.shape[0]))
print("\nList all parents....")
df = list_parents_of()
print(df.to_string(index=False))
