def load_csvs(client, csvlist, wrangl, insert):
"""Load the CSVs as input
Parameters
==========
client : a WOQLClient() connection
csvs : a dict of all csvs to be input
"""
for url in csvlist:
csv = get_csv_variables(url)
inputs = WOQLQuery().woql_and(csv, *wrangl)
answer = WOQLQuery().when(inputs, insert)
answer.execute(client, f"loading {url} into the graph")
def create_schema(client):
"""The query which creates the schema
Parameters - it uses variables rather than the fluent style as an example
==========
client : a WOQLClient() connection
"""
base = WOQLQuery().doctype("EphemeralEntity").label(
"Ephemeral Entity").description("An entity that has a lifespan")
base.property("lifespan_start", "dateTime").label("Existed From")
base.property("lifespan_end", "dateTime").label("Existed To")
country = WOQLQuery().add_class("Country").label("Country").description(
"A nation state").parent("EphemeralEntity")
country.property("iso_code", "string").label("ISO Code")
country.property("fip_code", "string").label("FIP Code")
airline = WOQLQuery().add_class("Airline").label("Airline").description(
"An operator of airplane flights").parent("EphemeralEntity")
airline.property("registered_in", "Country").label("Registered In"),
airport = WOQLQuery().add_class("Airport").label("Airport").description(
"An airport where flights terminate").parent("EphemeralEntity")
airport.property("situated_in", "Country").label("Situated In"),
flight = WOQLQuery().add_class("Flight").label("Flight").description(
"A flight between airports").parent("EphemeralEntity")
flight.property("departs", "Airport").label("Departs")
flight.property("arrives", "Airport").label("Arrives")
flight.property("operated_by", "Airline").label("Operated By")
schema = WOQLQuery().when(True).woql_and(base, country, airline, airport,
flight)
return schema.execute(client)
def execute_commit(qlist):
global client, polity_query_name, polity_query
global total_assertions, total_inserts, total_deletes, total_commit_failures
msg = f"Committing data for {polity_query_name}"
try:
q = WOQLQuery().woql_and(*qlist)
execute_start_time = time.time()
result = q.execute(client, commit_msg=msg)
# Makes no difference to timing: client.squash(msg) # per Kevin: flatten the 'journal'
execute_elapsed_s = "%.2fs" % (time.time() - execute_start_time)
total_assertions += 1
if type(result) is dict:
inserts = result['inserts']
deletes = result['deletes']
total_inserts += inserts
total_deletes += deletes
print(f"{msg} i:{inserts} d:{deletes} {execute_elapsed_s}")
else:
# Sometimes we get a <Response> object that is not subscriptable?
# result.status_code is the HTTP status code, 200 is successful but we don't have bindings?
print(f"{msg} {execute_elapsed_s}")
if dump_results:
pprint.pprint(result, indent=4)
except Exception as exception: # API error or whatever
execute_elapsed_s = "%.2fs" % (time.time() - execute_start_time)
print(
f"Execution ERROR while {msg} after {execute_elapsed_s} -- skipped"
)
print(f"{exception.msg}")
total_commit_failures += 1
def create_schema(client):
"""The query which creates the schema
Parameters - it uses variables rather than the fluent style as an example
==========
client : a WOQLClient() connection
"""
schema = WOQLQuery().woql_and(
WOQLQuery().doctype("Station",
label="Bike Station",
description="A station where bikes are deposited"),
WOQLQuery().doctype("Bicycle", label="Bicycle"),
WOQLQuery().doctype("Journey", label="Journey").
property("start_station", "Station", label="Start Station").
property("end_station", "Station", label="End Station").
property("duration", "integer", label="Journey Duration").
property("start_time", "dateTime", label="Time Started").
property("end_time", "dateTime", label="Time Ended").
property("journey_bicycle", "Bicycle", label="Bicycle Used")
)
return schema.execute(client)
