def add_link(self, variable: str, event: str, link_name: str):
"""
Adds a link between a variable and an event.
:param variable: The name of the variable
:param event: The name of the event
:param link_name: The name of the link
"""
e = self.get_event(event)
v = self.__get_variable(variable)
if e is None:
raise Exception(
"Trying to link [", e, "], but there's no such event!")
if v is None:
raise Exception(
"Trying to link to [", v, "], but there's no such variable!")
e.variables.append((link_name, v))
verbose(
" › New link [",
link_name,
"] from [",
event,
"] to [",
variable,
"]")
def add_variable(self, variable: str):
"""
Adds a variable, if it doesn't already exist.
:param variable: The name of the variable.
"""
if variable not in [var.id for var in self.__variables]:
self.__variables.append(Variable(variable))
verbose(" › New variable [", variable, "]")
def add_event(self, event: str):
"""
Adds an event, if it doesn't already exist.
:param event: The name of the event.
"""
if event not in [event.id for event in self.events]:
self.events.append(Event(event))
verbose(" › New event [", event, "]")
def register_command(self, name: str, callback):
"""
Registers a command, that can be called by the server.
:param name: The name of the command.
:param callback: A function that will be ran whenever the server invokes this command. Its first parameter
should be the this client object, and any other parameters will be treated as varargs.
"""
self.__commands[name] = callback
verbose("Client: Registered command [", name, "]")
def register_command(self, name: str, callback):
"""
Registers a command that will be used by the server.
:param name: The name of the command
:param callback: The code to execute when that command is found.
"""
self.commands[name] = callback
verbose("Server: Registered command [", name, "]")
def send(self, message, *args):
"""
Sends a message to the server.
:param message: The name of the command.
:param args:
"""
msg = Message(message, *args)
verbose("Client: sending [", msg, "] to", self.socket.getpeername())
msg.send(self.socket)
def send(self, command, *args):
"""
Sends a message to the client.
:param command: The name of the command to call.
:param args: The args that are sent to that client.
"""
msg = Message(command, *args)
verbose("Server: sending [", msg, "] to", self.coords)
msg.send(self.__socket)
def build(self):
"""
Add triples to the basic query template (sparql/queryBuilder/query_template.sparql) and sends the request
:return: the result of the request.
"""
verbose("Filling variables dictionary...")
self.__fill_dictionary()
for triple in self.__triples:
self.__query += ' ' + triple[0] + ' ' + triple[1] + \
' ' + triple[2] + ' .'
announce("Sending query: " + self.__query)
info("The answer is: " + self.__request())
def __connect_to_client(self, client):
"""
Connects to a client.
:param client: the client to connect to
"""
c = Client(self, client)
self.clients.append(c)
verbose("Added", c, "to the list of connected clients.")
c.listen()
self.clients.remove(c)
verbose("Removed", c, "from the list of connected clients.")
def nltk_to_query_objects(nltk_output: List[Expression]) -> List[Sentence]:
"""
Converts objects created by NLTK after the ccg2lambda pipeline into Query Objects.
:param nltk_output: The output from the ccg2lambda pipeline.
:return: A list of Sentences, as Query Objects.
"""
info(" Рђ║ nltk2qo: nltk Python Objects Ъад Query Objects")
sentences = [__parse(sentence) for sentence in nltk_output]
verbose("Conversion to Query Objects done.")
# display some stats about the conversion
return sentences
def __init__(self, address: str, port: int):
"""
Initializes a Client. It will connect to the given server, but will not send anything yet.
:param address: The IP address of the server to connect to
:param port: The port the server is listening on
"""
verbose("Client: Starting...")
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect((address, port))
self.__is_running = True
self.__commands = {}
verbose("Client: Connected.")
def server_logs(client, severity: str, *args: str):
if severity == "verbose":
verbose("Server:", *args)
elif severity == "info":
info(*args)
elif severity == "warning":
warning('Server:', *args)
elif severity == "error":
error(*args)
else:
verbose("Server: unknown severity:", *args)
warning("Received a message from the server, with unknown severity [",
severity, "]")
def test_file(path):
"""
Checks if a file exists.
:param path: The path to that file (absolute or relative)
:return: True if the file exists, False otherwise.
"""
path = os.path.abspath(path)
if os.path.isfile(path):
verbose("Found file [", path, "]")
return True
else:
warning("File not found [", path, "]")
return False
def run(self):
"""
Starts a loop in which the client will listen to the server. This loops stops when the server or the client
disconnects.
"""
verbose("Client: Listening...")
while self.__is_running:
try:
msg = self.socket.recv(8000)
except ConnectionResetError:
break
if msg == b'':
verbose("Client: The server closed the connection...")
break
for message in byte_to_message(msg):
if message.name in self.__commands:
verbose("Client:", self.socket.getpeername(), "sent [",
message, "]")
self.__commands[message.name](self, *message.args)
else:
warning("Client: Unknown command", message.name)
self.socket.close()
verbose("Client: Disconnected.")
def receive_message(self):
# Receive maximum 8000 bytes from the socket
msg = self.__socket.recv(8000)
# Nothing was received
if msg == b'':
return
for message in byte_to_message(msg):
if message.name in self.__server.commands:
verbose("Server:", self.coords, "sent [", message, "]")
self.__server.commands[message.name](
self.__server, self, *message.args)
else:
warning("Server: Unknown command [", message.name, "]")
def send(self, socket):
"""
Encodes this object and sends it in a Socket.
:param socket: The socket in which to send this Message.
"""
s = self.name.encode()
for arg in self.args:
s += ESCAPE_CHARACTER + arg.encode()
s += b"\n"
try:
verbose("Internal: Sending [", s.decode().replace('\n', ''), "]")
socket.send(s)
except OSError:
error("Cannot send message, invalid connection.")
def __parse(nltk_output: Expression) -> Sentence:
"""
Parses an Expression into a Sentence.
:param nltk_output: A query object representing a Sentence, fully built.
"""
verbose("Now parsing [", nltk_output, "]")
sentence = Sentence()
read_expressions(nltk_output, sentence)
sentence.fix()
verbose("Done with this sentence.")
sentence.pretty_print()
if sentence.main is None:
warning("No Question Marker found! This sentence is not a question.")
return sentence
def choose(client, *args: str):
verbose('The server is asking:', *args)
announce('\nWhat did you mean?')
options = [arg.replace('~', ' ').split('|') for arg in args]
for option in options:
print(' › ' + option[0] + '\t\x1b[32m' + option[1] + '\x1b[0m\t' +
option[2] + ' \x1b[1;35mhttps:' + option[3] + '\x1b[0m')
while True:
user: str = input("Type the number of your choice: ")
user_i: int = int(user)
if 0 <= user_i < len(options):
break
client.send('choice', str(user))
def add_tag(self, tag: str, entity: str):
"""
Adds a tag to a specific entity.
:param tag: The name of the tag
:param entity: The entity on which the tag should be added
"""
if tag == "QM":
if self.main is None:
self.main = self.__get_variable(entity)
verbose(" › Main subject of the sentence: [", entity, "]")
else:
warning(
"Already found the main subject! It looks like there are 2 QM markers. Keeping the first one, "
"and ignoring this one on [", entity, "]")
else:
self.get_entity(entity).tags.append(tag)
verbose(" › New tag [", tag, "] on [", entity, "]")
def byte_to_message(contents) -> List[Message]:
"""
Converts bytes received in the socket, into Message objects.
:param contents: the bytes received by the server
:return: A list of the messages that were received, after parsing. May be empty if there were no messages.
"""
messages = []
if contents != b'':
for command in contents.split(b'\n'):
if command != b'':
verbose("Internal: Received [", command.decode(), "]")
parts = command.split(ESCAPE_CHARACTER)
name = parts[0].decode()
params = [parts[i].decode() for i in range(1, len(parts))]
messages.append(Message(name, *params))
return messages
def test_directory(path, allow_empty=False):
"""
Checks if a directory exists, and is not empty.
:param allow_empty: Report a success even if the directory is empty
:param path: The path to that directory (absolute or relative)
:return: True if the directory exists & is not empty, False otherwise.
"""
path = os.path.abspath(path)
if os.path.isdir(path):
if len(os.listdir(path)) > 0:
verbose("Found [", path, "], and it is not empty.")
return True
else:
verbose("Found [", path, "], but it is empty!")
return allow_empty
else:
warning("Directory not found [", path, "]")
return False
def __fill_dictionary(self) -> None:
"""
gives a var name for each determined object of the sentence.
"""
for event in self.__sentence.events:
if not self.__qm_found:
for var in event.variables:
if var[1] == self.__sentence.main:
verbose("Found question marker.")
self.__qm_found = True
if self.__qm_found:
self.__manage_qm(event)
self.__sentence.events.remove(event)
# as the qm event has to be the first event to be worked with, we need to iterate on the
# remaining events
self.__fill_dictionary()
break
else:
self.__manage_event(event)
def read_expressions(value: Expression, sentence: Sentence):
"""
Recursively reads an Expression and modifies the given Sentence to reflect what is seen.
:param value: The Expression that needs to be parsed.
:param sentence: The Sentence output.
"""
t = type(value)
if t == ExistsExpression:
sentence.add(value.variable.name)
read_expressions(value.term, sentence)
elif t == AndExpression:
read_expressions(value.first, sentence)
read_expressions(value.second, sentence)
elif t == ApplicationExpression:
sentence.add_tag(
value.function.variable.name,
value.argument.variable.name)
elif t == EqualityExpression:
first = type(value.first)
if first == IndividualVariableExpression: # (x1 = x2)
sentence.mark_as_equal(
value.first.variable.name,
value.second.variable.name)
elif first == ApplicationExpression: # (link(e) = x)
sentence.add_link(value.second.variable.name,
value.first.argument.variable.name,
value.first.function.variable.name)
else:
error(
"Found an EqualityExpression but the first argument is a [",
first,
"], which is unforeseen!")
raise Exception(
"I do not know how to handle the type [", first, "] in a [", t, "] !")
elif t == AllExpression:
verbose("nltk2qo: Found a AllExpression")
warning(
"It looks like this sentence is a yes/no question, and they are not supported currently.")
else:
error("Type [", t, "] unknown, aborting.")
raise Exception("I do not know how to handle the type [", t, "] !")
def __wikipedia_suggestion(words: str) -> str:
"""
returns title of wikipedia page resulting of a search on words var
:param words: the searched words
:return: wikipedia page URL, None if not found
"""
# ignores warning from Wikipedia package about its use of bs4
warnings.filterwarnings("ignore", category=UserWarning, module='wikipedia')
try:
page = wikipedia.page(words)
except wikipedia.exceptions.DisambiguationError as e:
page = wikipedia.page(e.options[0])
except wikipedia.exceptions.PageError:
try:
suggestion = wikipedia.suggest(words)
page = wikipedia.page(suggestion)
except (wikipedia.exceptions.PageError, IndexError, ValueError):
return "[ERROR]: Item \"" + words + \
"\" does not match any pages. Try another id!"
verbose("Wikipedia page found: " + page.title)
return page.title
def fix(self):
for couple in self.__couples:
variables = [self.__get_variable(variable) for variable in couple]
first = variables[0]
second = variables[1]
verbose(
" › Fixing equal variables [",
first.id,
"] and [",
second.id,
"]...")
[first.tags.append(tag) for tag in second.tags]
second.tags = first.tags
second.id = first.id
if first != second:
error(
"The two variables [",
first,
"] and [",
second,
"] should be equal, but they are not!")
raise Exception("Variables that should be equal are not!")
self.__couples.clear()
def convert(sentences: List[str], output_file=False) -> List[Expression]:
"""
Converts a list of questions to an Abstract Syntax Tree (AST), using:
- spaCy to annotate each word with its grammatical class,
- depccg to convert the natural language to a CCG tree,
- ccg2lambda to convert the CCG tree to a λ expressions,
- nltk to parse that λ-expressions into an AST represented in Python objects.
:param output_file: if True, will output an xml file of the parsed sentences
:param sentences: a list of questions in English
:exception if less than 1 sentence is provided
:return a list of ASTs that each correspond to one of the given sentences (in the same order).
"""
if len(sentences) < 1:
raise Exception("Cannot run the pipeline with less than 1 sentence: " +
str(len(sentences)))
announce("Beginning conversion of", len(sentences),
"sentences, the first one is [", sentences[0], "]")
annotated_sentences, split_sentences = __annotate_spacy(sentences)
ccg_of_each_sentence = __convert_to_ccg(split_sentences)
lambda_expressions = __ccg_to_jigg_xml(ccg_of_each_sentence,
annotated_sentences)
formulas = __ccg_jigg_xml_to_lambda(lambda_expressions)
# Creates an XML file to be used
if output_file:
# Can be used in ccg2lambda python script visualize.py to output
# sentences.html to give better overview
info("Creating visualisation in file sentences.sem.xml")
visualisation.visualize(lambda_expressions, "sentences.html")
expr = __lambda_to_python(formulas)
verbose("Conversion done.")
return expr
def ask_client(options: List[Tuple[str, str, str, str]]) -> str:
"""
Asks the client to choose between multiple Wikidata objects.
:param options: A list of quadruplets: (q-code, name, description, URL on Wikidata)
:return The one the user chose
"""
if len(options) == 0:
error(
"The server just requested to ask a question to the client, but didn't provide any possible answer! "
"Skipping.")
verbose("Requested a question without providing an answer")
traceback.print_stack()
return ""
elif len(options) == 1:
warning(
'Asking a question with only one possible answer, the client will not be prompted.'
)
return options[0][0]
from network.server import server
tmp = [
str(i) + "|" + "|".join(
[o.replace('|', '~').replace(' ', '~') for o in options[i][1:4]])
for i in range(len(options))
]
verbose("I'm going to ask the client to choose between [", tmp, "]")
if len(server.clients) == 0:
error("No clients are connected at this point.")
raise Exception("Trying to ask the client, but there's no client.")
elif len(server.clients) > 1:
warning("Currently,", len(server.clients),
"are connected; the first one will be selected.")
client = server.clients[0]
verbose("The question will be asked to client [", client, "]")
client.send("choose", *tmp)
client.receive_message()
info('The client chose [', user_choice, ']')
user = int(user_choice)
return options[user][0]
# this package manages the connection with the wiki services.
# It consists in:
# - Fetching the closest result possible of the user input
# - Using this result, fetching up to 7 possible wikidata pages
# - For each page, fetch the most important data: its code, its name, a short description and the page link.
from qalogging import verbose
verbose("Loaded package 'wikidata'")
def ast(client, ast_tree: str):
verbose('AST:', ast_tree)
# - ccg2lambda :
# https://github.com/mynlp/ccg2lambda
#
# All installation guides are described on their respective link.
#
from pathlib import Path
from depccg.parser import EnglishCCGParser
from conversion.paths import init_paths, MODEL
from qalogging import verbose
init_paths()
verbose('Loading depccg...')
depccg_options = dict(
# A list of binary rules
# By default: depccg.combinator.en_default_binary_rules
binary_rules=None,
# Penalize an application of a unary rule by adding this value (negative
# log probability)
unary_penalty=0.1,
# Prune supertags with low probabilities using this value
beta=0.00001,
# Set False if not prune
use_beta=True,
# Use category dictionary
use_category_dict=True,
# Use seen rules
