def load_csvs(client, csvlist, wrangl, insert): """Load the CSVs as input Parameters ========== client : a WOQLClient() connection csvs : a dict of all csvs to be input """ for url in csvlist: csv = get_csv_variables(url) inputs = WOQLQuery().woql_and(csv, *wrangl) answer = WOQLQuery().when(inputs, insert) answer.execute(client)
def load_csvs(client, csvs): """Load the CSVs as input Parameters ========== client : a WOQLClient() connection csvs : a dict of all csvs to be input """ for key, url in csvs.items(): csv = get_csv_variables(url) wrangles = get_wrangles() inputs = WOQLQuery().woql_and(csv, *wrangles) inserts = get_inserts() answer = WOQLQuery().when(inputs, inserts) answer.execute(client)
def create_schema(client): # # We first create an abstract class to represent ephemeral entities - things that have lifespans # schema = WOQLQuery().when(True).woql_and( WOQLQuery().doctype("EphemeralEntity").label("Ephemeral Entity"). description("An entity that has a lifespan").abstract().property( "lifespan_start", "dateTime").label("Existed From").property( "lifespan_end", "dateTime").label("Existed To"), # # This allows us to attach existence start and end times to entities, in a consistent way. # Then we create our four actual concrete classes, each as a subclass of this one: # # That gives you the meta-structure - the major relationships between airports, airlines, countries and flights - # you can then add whatever other properties you want. All the entities also have lifespan_start and lifespan_end # properties inherited from EphemeralEntity WOQLQuery().add_class("Country").label("Country").description( "A nation state").parent("EphemeralEntity").property( "country_id", "string").label("Id").property( "country_name", "string").label("Name").property( "country_iso_code", "string").label("ISO Code").property( "country_fips_code", "string").label("FIPS Code"), #primary key WOQLQuery().add_class("Airline").label("Airline").description( "An operator of airplane flights").parent( "EphemeralEntity").property("icao_code", "string").label( "ICAO Code") # primary key .property("registered_in", "Country").label("Registered In"), WOQLQuery().add_class("Airport").label("Airport").description( "An airport where flights terminate").parent( "EphemeralEntity").property( "situated_in", "Country").label("Situated In").property( "icao_code", "string").label("ICAO Code") # primary key .property("iata_code", "string").label("IATA Code").property( "name", "string").label("Name"), WOQLQuery().add_class("Flight").label("Flight").description( "A flight between airports").parent("EphemeralEntity").property( "departs", "Airport").label("Departs").property( "arrives", "Airport").label("Arrives").property( "operated_by", "Airline").label("Operated By")) try: print("[Building schema..]") with wary.suppress_Terminus_diagnostics(): schema.execute(client) except Exception as e: wary.diagnose(e)
def insert_countries(countries): c_generator = generate_bulk_insert(countries, country_query) for Matches, Inserts in c_generator: query = WOQLQuery().when(WOQLQuery().woql_and(*Matches), WOQLQuery().woql_and(*Inserts)) try: #with wary.suppress_Terminus_diagnostics(): query.execute(client) except Exception as e: print("EXCEPTION") wary.diagnose(e)
def create_schema(client): """The query which creates the schema Parameters - it uses variables rather than the fluent style as an example ========== client : a WOQLClient() connection """ base = WOQLQuery().doctype("EphemeralEntity").label("Ephemeral Entity").description("An entity that has a lifespan") base.property("lifespan_start", "dateTime").label("Existed From") base.property("lifespan_end", "dateTime").label("Existed To") country = WOQLQuery().add_class("Country").label("Country").description("A nation state").parent("EphemeralEntity") country.property("iso_code", "string").label("ISO Code") country.property("fip_code", "string").label("FIP Code") airline = WOQLQuery().add_class("Airline").label("Airline").description("An operator of airplane flights").parent("EphemeralEntity") airline.property("registered_in", "Country").label("Registered In"), airport = WOQLQuery().add_class("Airport").label("Airport").description("An airport where flights terminate").parent("EphemeralEntity") airport.property("situated_in", "Country").label("Situated In"), flight = WOQLQuery().add_class("Flight").label("Flight").description("A flight between airports").parent("EphemeralEntity") flight.property("departs", "Airport").label("Departs") flight.property("arrives", "Airport").label("Arrives") flight .property("operated_by", "Airline").label("Operated By") schema = WOQLQuery().when(True).woql_and(base, country, airline, airport, flight) return schema.execute(client)
def exec_bulk_country_query(country_queries, priority): airport, country_queries_unzip = list(zip(*country_queries)) query = WOQLQuery().woql_and(*country_queries_unzip) try: # with wary.suppress_Terminus_diagnostics(): result = query.execute(client) df_result = pd.DataFrame() if is_empty(result) else wdf.query_to_df( result) if not df_result.empty: df_result['id'] = df_result.index columns = [ "All", "Paren1", "Country", "country_id", getPriority(priority), "country_name" ] df_result = pd.wide_to_long(df_result, columns, i="id", j="val").reset_index() return (zip(airport, df_result["country_id"])) ## TODO: Waiting that opt() work to catch bad query and rerun them. except Exception as e: print("EXCEPTION") wary.diagnose(e)
def create_schema_from_addon(client, queries): new_queries = [] for query in queries: if len(query) > 0: new_queries.append(WOQLQuery().woql_and(*query)) schema = WOQLQuery().when(True).woql_and(*new_queries) output_json(schema, filename) return schema.execute(client)
def insert_airports(airports): c_generator = generate_bulk_insert(airports, airport_query) for Matches, Inserts in c_generator: query = WOQLQuery().when( WOQLQuery().woql_and(*Matches), WOQLQuery().woql_and(*Inserts), ) try: print("INSERT NEXT TEN") with wary.suppress_Terminus_diagnostics(): query.execute(client) except Exception as e: print("EXCEPTION") wary.diagnose(e)
def create_schema(client): ''' Build the schema. For this example, it is very simple: just a doctype for a Person with various attributes :param client: TerminusDB server handle ''' schema = WOQLQuery().when(True).woql_and(WOQLQuery().doctype( "Person").label("Person").description("Somebody").property( "Name", "string").property("Sex", "string").property( "Parent1", "string").property("Parent2", "string")) try: print("[Building schema..]") with wary.suppress_Terminus_diagnostics(): schema.execute(client) except Exception as e: wary.diagnose(e)
def load_csv(client, url, voyages): ''' Read a .csv file and use its raw data to initialise a graph in the TerminusDB server. In the case of a local file, it should be the file path relative to the value of the TERMINUS_LOCAL environment variable set when the TerminusDB server was started... :param client: handle on the TerminusDB server :param url: string, eiher a local file name or http-style url :param voyages: boolean, whether a Voyage or Berth document set are to be created :return: None ''' csv = get_csv_variables(url, voyages) wrangles = get_wrangles(voyages) inputs = WOQLQuery().woql_and(csv, *wrangles) inserts = get_inserts(voyages) answer = WOQLQuery().when(inputs, inserts) try: print("[Loading raw data from '{}'..]".format(url)) with wary.suppress_Terminus_diagnostics(): answer.execute(client) except woqlError.APIError as e: wary.diagnose(e)
def create_schema(client): ''' Build the schema. For this demo, there is a base document to capture ephemeral events. There is then a derived document for Voyage events, and Berth events. Note especially that ANY property common to all the derived documents MUST be put in the base: Here, the 'ship' property is common to both 'Voyage' and 'Berth' and so must be in the 'Ship_Event' document. If instead it is placed in 'Voyage' and also in 'Base', then TerminusDB will complain with a "class subsumption" error... :param client: TerminusDB server handle ''' base = WOQLQuery().doctype("Ship_Event").label("Ship Event").description("An ephemeral") base.property("ship", "string").label("Ship Name") base.property("start", "dateTime").label("Existed From") # try "dateTime rather than string? base.property("end", "dateTime").label("Existed To") voyage = WOQLQuery().add_class("Voyage").label("Voyage").description("Ship movement").parent("Ship_Event") voyage.property("route", "string").label("Route") docking = WOQLQuery().add_class("Docking").label("Docking").description("A ship docked at a berth").parent("Ship_Event") docking.property("berth", "string").label("Berth") schema = WOQLQuery().when(True).woql_and( base, docking, voyage ) try: print("[Building schema..]") with wary.suppress_Terminus_diagnostics(): schema.execute(client) except Exception as e: wary.diagnose(e)
def create_schema(client): """The query which creates the schema Parameters ========== client : a WOQLClient() connection """ schema = WOQLQuery().when(True).woql_and( WOQLQuery().doctype("Party").label("Party").description( "Political Party"), WOQLQuery().doctype("Representative").label("Representative"). description("An elected member Dublin city council").property( "member_of", "Party").label("Member of").cardinality(1), WOQLQuery().doctype("Similarity").label("Similarity").property( "similarity", "decimal").label("Similarity").property( "similar_to", "Representative").label("Similar To").cardinality(2)) return schema.execute(client)
def create_schema(client): """The query which creates the schema Parameters - it uses variables rather than the fluent style as an example ========== client : a WOQLClient() connection """ schema = WOQLQuery().when(True).woql_and( WOQLQuery().doctype("Station"). label("Bike Station"). description("A station where bikes are deposited"), WOQLQuery().doctype("Bicycle").label("Bicycle"), WOQLQuery().doctype("Journey").label("Journey"). property("start_station", "Station").label("Start Station"). property("end_station", "Station").label("End Station"). property("duration", "integer").label("Journey Duration"). property("start_time", "dateTime").label("Time Started"). property("end_time", "dateTime").label("Time Ended"). property("journey_bicycle", "Bicycle").label("Bicycle Used") ) return schema.execute(client)
def create_schema_from_queries(client, queries): schema = WOQLQuery().when(True).woql_and(*queries) return schema.execute(client)
'@value': value.father }))) # # We are inserting each Person one at a time into Terminus: each insertion # involves a call out to the database, and would thus be slow if there were a lot of data.. # # We could instead change the woql when clause and both each woql_ands to bundle a number of # idgens and insertions for different documents together. This then would be more efficient. # # For you to try if you wish.. :-) # try: print("[Inserting {}..]".format(person)) with wary.suppress_Terminus_diagnostics(): answer.execute(client) except Exception as e: wary.diagnose(e) # # Some sample queries.. # print("\nList people....") df = list_people() print("{:,} people found".format(df.shape[0])) print("\nList all parents....") df = list_parents_of() print(df.to_string(index=False))