def grammar(**kwargs):
return adjective.grammar(**kwargs) | preposition.grammar(**kwargs) | {
# Raw nouns
Rule("NOUN^", [NounParser(is_plural=False)], passthru),
Rule("NOUNS^", [NounParser(is_plural=True)], passthru),
# Nouns without prepositions
Rule("NOUN", [RuleRef('NOUN^')], passthru),
Rule("NOUNS", [RuleRef('NOUNS^')], passthru),
# Nouns with prepositions (the car of (the owner of the building))
Rule("NOUN", [RuleRef('NOUN^'), RuleRef('PP')],
lambda state, data: data[1] + Interpretation(local=data[0].local.with_preposition_phrase(data[1].local))),
Rule("NOUNS", [RuleRef('NOUNS^'), RuleRef('PP')],
lambda state, data: data[1] + Interpretation(local=data[0].local.with_preposition_phrase(data[1].local))),
Rule("NOUN", [RuleRef('ADJECTIVE'), RuleRef('NOUN')],
lambda state, data: data[1] + Interpretation(local=data[1].local.with_adjective(data[0].local))),
Rule("NOUNS", [RuleRef('ADJECTIVE'), RuleRef('NOUNS')],
lambda state, data: data[1] + Interpretation(local=data[1].local.with_adjective(data[0].local))),
Rule("NOUN*", [RuleRef('NOUN')], passthru),
Rule("NOUN*", [RuleRef('NOUNS')], passthru),
}
def grammar(assumptions=True, **kwargs):
return and_rules('EXPANDED_SPECIFIC_CLAIMS', 'EXPANDED_SPECIFIC_CLAIM', accept_singular=True) \
| and_rules('EXPANDED_SPECIFIC_CLAIMS_GENERAL_FIRST', 'EXPANDED_SPECIFIC_CLAIM', first_singleton='EXPANDED_GENERAL_CLAIM') \
| and_rules('EXPANDED_SPECIFIC_CLAIMS_GENERAL_LAST', 'EXPANDED_SPECIFIC_CLAIM', last_singleton='EXPANDED_GENERAL_CLAIM') \
| and_rules('SUPPORTS', 'SUPPORT', accept_singular=True) \
| and_rules('ATTACKS', 'ATTACK', accept_singular=True) \
| {
Rule('ARGUMENT', [RuleRef('SENTENCE')],
lambda state, data: data[0]),
Rule('ARGUMENT', [RuleRef('ARGUMENT'), RuleRef('SENTENCE')],
lambda state, data: data[0] + data[1]),
Rule('SENTENCE', [RuleRef('EXPANDED_SPECIFIC_CLAIM'), Literal('.')],
lambda state, data: data[0]),
Rule('EXPANDED_SPECIFIC_CLAIM', [RuleRef('SPECIFIC_CLAIM'), RuleRef('SUPPORTS'), RuleRef('ATTACKS')],
expanded_claim),
# We use CONDITIONAL_CLAIM instead of GENERAL_CLAIM here because a conditional claim is a more specific
# structure that can contain a general claim, and that is why it has to be above general claim in the
# hierarchy. And hey, in a sense a general claim is also a conditional claim because for a general claim
# to be applicable the subject has to match the group of the subject of the general claim, making it
# more or less a conditional claim, right?
Rule('EXPANDED_GENERAL_CLAIM', [RuleRef('CONDITIONAL_CLAIM'), RuleRef('SUPPORTS'), RuleRef('ATTACKS')],
expanded_claim),
Rule('SUPPORT', [Literal('because'), RuleRef('EXPANDED_SPECIFIC_CLAIMS')],
lambda state, data: data[1] + Interpretation(local=PartialRelation(Relation.SUPPORT, specifics=data[1].local, make_assumptions=assumptions))),
Rule('SUPPORT', [Literal('because'), RuleRef('EXPANDED_SPECIFIC_CLAIMS_GENERAL_FIRST')],
lambda state, data: data[1] + Interpretation(local=PartialRelation(Relation.SUPPORT, conditional=data[1].local[0], specifics=data[1].local[1:], make_assumptions=assumptions))),
Rule('SUPPORT', [Literal('because'), RuleRef('EXPANDED_SPECIFIC_CLAIMS_GENERAL_LAST')],
lambda state, data: data[1] + Interpretation(local=PartialRelation(Relation.SUPPORT, conditional=data[1].local[-1], specifics=data[1].local[0:-1], make_assumptions=assumptions))),
Rule('SUPPORT', [Literal('because'), RuleRef('EXPANDED_GENERAL_CLAIM')],
lambda state, data: data[1] + Interpretation(local=PartialRelation(Relation.SUPPORT, conditional=data[1].local, make_assumptions=assumptions))),
Rule('SUPPORTS', [],
lambda state, data: Interpretation()),
Rule('ATTACK', [Expression(r'^except|but$'), RuleRef('EXPANDED_SPECIFIC_CLAIMS')],
lambda state, data: data[1] + Interpretation(local=PartialRelation(Relation.ATTACK, specifics=data[1].local, make_assumptions=assumptions))),
Rule('ATTACK', [Expression(r'^except|but$'), RuleRef('EXPANDED_SPECIFIC_CLAIMS_GENERAL_FIRST')],
lambda state, data: data[1] + Interpretation(local=PartialRelation(Relation.ATTACK, conditional=data[1].local[0], specifics=data[1].local[1:], make_assumptions=assumptions))),
Rule('ATTACK', [Expression(r'^except|but$'), RuleRef('EXPANDED_SPECIFIC_CLAIMS_GENERAL_LAST')],
lambda state, data: data[1] + Interpretation(local=PartialRelation(Relation.ATTACK, conditional=data[1].local[-1], specifics=data[1].local[0:-1], make_assumptions=assumptions))),
Rule('ATTACK', [Expression(r'^except|but$'), RuleRef('EXPANDED_GENERAL_CLAIM')],
lambda state, data: data[1] + Interpretation(local=PartialRelation(Relation.ATTACK, conditional=data[1].local, make_assumptions=assumptions))),
Rule('ATTACKS', [],
lambda state, data: Interpretation()),
}
def interpret():
model = request.args.get('model')
text = request.args.get('text')
if model == "sentiment":
interpretation = Interpretation(sentiment_clf, model, text)
elif model == "sarcasm":
interpretation = Interpretation(sarcasm_clf, model, text)
else:
print("ERROR: {} model not recognized".format(model))
return render_template('interpret.html', interpretation=interpretation)
def blob_conditional_claim(state, data):
subject = Instance()
verb = None
object = None
claim = GeneralBlob(subject, verb, object, data[0])
argument = Argument(instances={subject: {subject}},
claims={claim: {claim}})
return Interpretation(argument=argument, local=claim)
def grammar(**kwargs):
return and_rules('SPECIFIC_CLAIMS', 'SPECIFIC_CLAIM', accept_singular=True) \
| and_rules('SPECIFIC_CLAIMS_CONDITIONAL_FIRST', 'SPECIFIC_CLAIM', first_singleton='CONDITIONAL_CLAIM') \
| and_rules('SPECIFIC_CLAIMS_CONDITIONAL_LAST', 'SPECIFIC_CLAIM', last_singleton='CONDITIONAL_CLAIM') \
| {
Rule('ARGUMENT', [RuleRef('SENTENCE')],
lambda state, data: data[0]),
Rule('ARGUMENT', [RuleRef('ARGUMENT'), RuleRef('SENTENCE')],
lambda state, data: data[0] + data[1]),
Rule('SENTENCE', [RuleRef('SUPPORTED_CLAIM'), Literal('.')],
lambda state, data: data[0]),
Rule('SENTENCE', [RuleRef('ATTACKED_CLAIM'), Literal('.')],
lambda state, data: data[0]),
Rule('SUPPORTED_CLAIM', [RuleRef('SPECIFIC_CLAIM'), Literal('because'), RuleRef('SPECIFIC_CLAIMS')],
lambda state, data: data[0] + data[2] + Interpretation(argument=relation(Relation.SUPPORT, data[0].local, specifics=data[2].local), local=data[0].local)),
Rule('SUPPORTED_CLAIM', [RuleRef('SUPPORTED_CLAIM_WITH_WARRANT')],
lambda state, data: data[0]),
Rule('SUPPORTED_CLAIM_WITH_WARRANT', [RuleRef('SPECIFIC_CLAIM'), Literal('because'), RuleRef('SPECIFIC_CLAIMS_CONDITIONAL_FIRST')],
lambda state, data: data[0] + data[2] + Interpretation(argument=relation(Relation.SUPPORT, data[0].local, general=data[2].local[0], specifics=data[2].local[1:]), local=data[0].local)),
Rule('SUPPORTED_CLAIM_WITH_WARRANT', [RuleRef('SPECIFIC_CLAIM'), Literal('because'), RuleRef('SPECIFIC_CLAIMS_CONDITIONAL_LAST')],
lambda state, data: data[0] + data[2] + Interpretation(argument=relation(Relation.SUPPORT, data[0].local, general=data[2].local[-1], specifics=data[2].local[:-1]), local=data[0].local)),
Rule('ATTACKED_CLAIM', [RuleRef('SPECIFIC_CLAIM'), Expression(r'^but|except$'), RuleRef('SPECIFIC_CLAIMS')],
lambda state, data: data[0] + data[2] + Interpretation(argument=relation(Relation.ATTACK, data[0].local, specifics=data[2].local), local=data[0].local)),
Rule('ATTACKED_CLAIM', [RuleRef('ATTACKED_CLAIM_WITH_WARRANT')],
lambda state, data: data[0]),
Rule('ATTACKED_CLAIM_WITH_WARRANT', [RuleRef('SPECIFIC_CLAIM'), Expression(r'^but|except$'), RuleRef('SPECIFIC_CLAIMS_CONDITIONAL_FIRST')],
lambda state, data: data[0] + data[2] + Interpretation(argument=relation(Relation.ATTACK, data[0].local, general=data[2].local[0], specifics=data[2].local[1:]), local=data[0].local)),
Rule('ATTACKED_CLAIM_WITH_WARRANT', [RuleRef('SPECIFIC_CLAIM'), Expression(r'^but|except$'), RuleRef('SPECIFIC_CLAIMS_CONDITIONAL_LAST')],
lambda state, data: data[0] + data[2] + Interpretation(argument=relation(Relation.ATTACK, data[0].local, general=data[2].local[-1], specifics=data[2].local[:-1]), local=data[0].local)),
# Experimental, don't know if I want this
# Rule('SUPPORTED_CLAIM', [RuleRef('SPECIFIC_CLAIM'), Literal('because'), RuleRef('CONDITIONAL_CLAIM')],
# lambda state, data: data[0] + data[2] + assume(data[0].local, data[2].local)),
# Attacking a warrant?
Rule('SUPPORTED_CLAIM', [RuleRef('SUPPORTED_CLAIM'), Expression(r'^but|except$'), RuleRef('SPECIFIC_CLAIMS')],
lambda state, data: data[0] + data[2] + Interpretation(argument=relation(Relation.ATTACK, data[0].local, specifics=data[2].local))),
Rule('SUPPORTED_CLAIM', [RuleRef('SUPPORTED_CLAIM_WITH_WARRANT'), Expression(r'^but|except$'), RuleRef('SPECIFIC_CLAIMS')],
lambda state, data: data[0] + data[2] + Interpretation(argument=relation(Relation.ATTACK, warrant_relation(data[0]), specifics=data[2].local)))
}
def grammar(**kwargs):
return noun.grammar(**kwargs) | {
Rule(
"PROTOTYPE", [Expression(r'^every|an?$'),
RuleRef("NOUN")],
lambda state, data: data[1] + Interpretation(local=Prototype(
data[1].local, article=data[0].local))),
Rule(
"PROTOTYPES", [RuleRef("NOUNS")], lambda state, data: data[0] +
Interpretation(local=Prototype(data[0].local))),
Rule(
"PROTOTYPES",
[Expression(r'^all|most|many|some$'),
RuleRef("NOUNS")], lambda state, data: data[0] + Interpretation(
local=Prototype(data[1].local, article=data[0].local))),
Rule("PROTOTYPE*", [RuleRef("PROTOTYPE")], passthru),
Rule("PROTOTYPE*", [RuleRef("PROTOTYPES")], passthru),
}
def and_rules(name: str, singleton: str, accept_singular: bool = False, first_singleton: Optional[str] = None, last_singleton: Optional[str] = None) -> Set[Rule]:
"""
Creates a mini-grammar of rules that are needed to parse 'A and B',
'A, B and C', 'A, B, C and D', etc. where A, B, C and D all are parseable
using the rule name passed using the singleton argument.
Grammar:
<As> ::= A_helper `and' A_last
<A_helper> ::= A_helper `,' A
<A_helper> ::= A_first
<As> ::= A
"""
if last_singleton is None:
last_singleton = singleton
if first_singleton is None:
first_singleton = singleton
helper = name + "_"
rules = {
# _ and C
Rule(name, [RuleRef(helper), Literal('and'), RuleRef(last_singleton)],
lambda state, data: data[0] + data[2] + Interpretation(local=data[0].local | OrderedSet([data[2].local]))),
# A, B # (allows for 'A, B and C')
Rule(helper, [RuleRef(helper), Literal(','), RuleRef(singleton)],
lambda state, data: data[0] + data[2] + Interpretation(local=data[0].local | OrderedSet([data[2].local]))),
# A (allows for 'A and B')
Rule(helper, [RuleRef(first_singleton)],
lambda state, data: data[0] + Interpretation(local=OrderedSet([data[0].local])))
}
if accept_singular:
rules |= {
Rule(name, [RuleRef(singleton)],
lambda state, data: data[0] + Interpretation(local=OrderedSet([data[0].local])))
}
return rules
def grammar(**kwargs):
return {
Rule('PREPOSITION', [PrepositionSymbol()], passthru),
Rule(
'PP', [RuleRef('PREPOSITION'),
RuleRef('INSTANCE*')],
lambda state, data: data[1] + Interpretation(
local=PrepositionPhrase(data[0].local, data[1].local))),
Rule(
'PP', [RuleRef('PREPOSITION'),
RuleRef('PROTOTYPE*')],
lambda state, data: data[1] + Interpretation(
local=PrepositionPhrase(data[0].local, data[1].local))),
Rule(
'PP', [RuleRef('PREPOSITION'),
RuleRef('CATEGORY')],
lambda state, data: data[1] + Interpretation(
local=PrepositionPhrase(data[0].local, data[1].local))),
}
def assume(claim, general):
assumption = SpecificClaim(claim.subject,
'is',
general.subject.singular,
assumption=True)
argument = Argument(claims={assumption: {assumption}})
argument += relation(Relation.SUPPORT,
claim,
general=general,
specifics={assumption})
return Interpretation(argument=argument, local=claim)
class Solver:
def __init__(self, input_data):
Interpretation.problem = input_data.clauses # Set static variable problem for Interpretation class
self.best_interpretation = Interpretation(num_vars=input_data.num_vars)
self.best_cost = self.best_interpretation.cost()
def solve(
self
): # Greedy search with random initialisation, finding the best neighbour every time
for i in range(max_tries):
if self.best_cost != 0:
neighbour = self.best_interpretation.best_neighbour()
if neighbour.cost() < self.best_cost:
self.best_interpretation = neighbour
self.best_cost = self.best_interpretation.cost()
return self.best_interpretation
def show(self):
print("c SAT")
print("s SATISFIABLE")
self.best_interpretation.show()
def general_claim_singular(state, data):
scope = Scope()
subj = Instance(pronoun=Span('something'), scope=scope)
condition = SpecificClaim(subj, Verb('is'), data[0].local.singular, scope=scope)
claim = GeneralClaim(subj, data[1].local, data[2].local, scope=scope)
relation = Relation({condition}, claim, Relation.CONDITION)
return data[0] + data[2] + Interpretation(
argument=Argument(
claims={claim: {claim}, condition: {condition}},
relations={relation},
instances={subj: {subj}}
),
local=claim)
def expanded_general_claim(state, data):
assert len(data[0].argument.claims) == 2
assert len(data[0].argument.relations) == 1
claim = data[0].local
conditions = set(claim.update(scope=data[0].local.scope) for claim in data[2].local)
remaining_claims = set(data[0].argument.claims.keys()) ^ {claim}
relation = Relation(remaining_claims | conditions , claim, Relation.CONDITION)
return data[0] + data[2] + Interpretation(
argument=Argument(
claims={condition: {condition} for condition in conditions},
relations={relation},
instances={condition.subject: {condition.subject} for condition in conditions}
),
local=claim)
def undetermined_claim(state, data):
scope = Scope()
conditions = set(claim.update(scope=scope) for claim in data[2].local)
claim = GeneralClaim.from_claim(data[0].local, scope=scope, conj=data[1].local)
relation = Relation(conditions, claim, Relation.CONDITION)
return data[0] + data[2] + Interpretation(
argument=Argument(
claims={
claim: {claim},
**{condition: {condition} for condition in conditions}
},
relations={relation},
instances={
claim.subject: {claim.subject},
**{condition.subject: {condition.subject} for condition in conditions}
}
),
local=claim)
def from_names_rule(cls, state, data):
instances = {Instance(name=name) for name in data[0].local}
instance=cls(instances=instances)
return data[0] + Interpretation(local=instance, argument=Argument(instances={instance: {instance}}))
def from_pronoun_rule(cls, state, data):
instance = cls(pronoun=data[0].local)
return data[0] + Interpretation(local=instance, argument=Argument(instances={instance: {instance}}))
def from_noun_rule(cls, state, data):
instance = cls(noun=data[1].local) # because 'the'
return data[1] + Interpretation(argument=Argument(instances={instance: {instance}}), local=instance)
def from_name_rule(cls, state, data):
instance = cls(name=data[0].local)
return data[0] + Interpretation(argument=Argument(instances={instance: {instance}}), local=instance)
def from_rule(cls, state, data):
claim = cls(data[0].local, data[1].local, data[2].local)
return data[0] + data[1] + data[2] + Interpretation(
argument=Argument(claims={claim: {claim}}), local=claim)
def instantiate(self, claim, context):
relation = Relation(sources={}, target=claim, type=self.type)
argument = Argument(relations={relation})
if self.specifics is not None:
instances = OrderedDict()
claims = OrderedDict()
for specific in self.specifics:
# If we're connecting to a scoped statement, make sure we have the right scope
if claim.scope and specific.scope != claim.scope:
scoped_specific = specific.update(scope=claim.scope)
relation.sources.add(scoped_specific)
instances[scoped_specific.subject] = {
scoped_specific.subject, specific.subject
}
claims[scoped_specific] = {scoped_specific, specific}
else:
relation.sources.add(specific)
argument = argument + Argument(instances=instances, claims=claims)
if self.make_assumptions and self.conditional is not None:
argument = argument + Argument(relations={
Relation([self.conditional], relation, Relation.SUPPORT)
})
conditions = find_conditions(self.conditional, context)
if len(conditions) > 0:
assumptions = []
for condition in conditions:
params = {
'subject': claim.subject,
'verb': condition.verb.for_subject(claim.subject),
}
if condition.verb.literal in ('is', 'are'):
params['object'] = getattr(
condition.object, claim.subject.grammatical_number)
assumptions.append(condition.assume(**params))
argument = argument + Argument(claims=dict(
(assumption, {assumption}) for assumption in assumptions))
relation.sources.update(assumptions)
if self.make_assumptions and self.conditional is None and self.specifics is not None:
"""
Make a general assumption in the trend of 'When Tweety can fly
because she is a bird, anything that is a bird can fly.'
"""
scope = Scope()
subject = Instance(pronoun=Span('something'), scope=scope)
if self.type == Relation.SUPPORT:
object = claim.object
elif self.type == Relation.ATTACK:
object = Negation(
claim.object
) #is this ok? Or are we jumping to conclusions?
else:
assert False, "Can't deal with this kind of relation"
conditional = GeneralClaim(subject,
claim.verb,
object,
scope=scope,
assumption=True)
assumptions = []
for specific in self.specifics:
if claim.subject.could_be(specific.subject):
assumptions.append(
specific.update(id=None,
subject=conditional.subject,
scope=scope))
if len(assumptions) > 0:
argument = argument + Argument(
instances={subject: {subject}},
claims={
conditional: {conditional},
**{
assumption: {assumption}
for assumption in assumptions
}
},
relations={
Relation([conditional], relation, Relation.SUPPORT),
Relation(assumptions, conditional, Relation.CONDITION)
})
if len(relation.sources) == 0:
raise Exception(
"No specific claim or assumption to link the conditional claim to"
)
return Interpretation(argument=argument, local=claim)
def __init__(self, input_data):
Interpretation.problem = input_data.clauses # Set static variable problem for Interpretation class
self.best_interpretation = Interpretation(num_vars=input_data.num_vars)
self.best_cost = self.best_interpretation.cost()
"""
The interpretation phase, although closely related to the evaluation phase, includes a more fine-grained understanding of the latent variables. The main goal of the interpretation
phase is to go beyond the latent variables and understand the latent variables in the context of the domain under study. This phase is highly depending on the research question
that we would want to have answered. What topics are present, how are they distributed over time, and how are they related to other topics are possible ways to explore the output
of the LDA analysis. Similarly to the evaluation phase, aiming for a deeper understanding of the topics might also result in flaws in the analysis. For example, a visualization of
the topics that places two very distinct topics in close proximity, high probability of a topic in a document that does not cover aspects of that topic, or topics that should not
co-occur together are indicators of flaws or areas of improvements. In such cases, it would be wise to revisit the pre-processing phase and to re-run the analysis with, for instance,
different model parameters or pre-processing steps.
IMPORTANT
Remember to label the words in the topics that were created during the evaluation phase. Update the function get_topic_label in helper_functions to reflect the correct number of labels.
"""
# instantiate interpretation class
interpretation = Interpretation()
# infer the documen topic distribution per publication
interpretation.infer_document_topic_distribution()
# # obtain list of document titles per topic
interpretation.get_document_title_per_topic()
# # plot topics over time
interpretation.plot_topics_over_time()
# # plot topics over time stacked
interpretation.plot_topics_over_time_stacked()
# plot topic co-occurrence
interpretation.plot_topic_co_occurrence()
def from_noun_rule(cls, state, data):
instance = cls(noun=data[1].local) # 1 because of the 'the' at pos 0.
return data[1] + Interpretation(local=instance, argument=Argument(instances={instance: {instance}}))
def finish(self, literal: str, position: int, state: 'State'):
span = super().finish(literal, position, state)
return Interpretation(local=Noun(span, self.is_plural))
def finish(self, literal: str, position: int, state: State):
return Interpretation(local=Verb(literal))
def from_rule(cls, state, data):
if isinstance(data[1].local, cls):
neg = data[1].local.object # Unwrap double negations
else:
neg = cls(data[1].local)
return data[1] + Interpretation(local=neg)