def digest(self):
if self.sentences is not None:
return
# Digest the problem into sentences
tokenizer = data.load("tokenizers/punkt/english.pickle")
self.sentences = tokenizer.tokenize(self.text.strip())
# Digest each sentence into words and part-of-speech tags
if self.sentence_tags is None:
sentence_tags = []
all_tags = []
all_words = []
for s in self.sentences:
all_words.append(s)
tags = pos_tag(word_tokenize(s))
sentence_tags.append(tags)
for t in tags:
l = len(t[0])
if not self.longest_word or self.longest_word < l:
self.longest_word = l
all_tags.append(t[1])
self.sentence_tags = sentence_tags
self.all_tags = uniq(all_tags)
self.all_words = uniq(all_words)
def digest(self):
if self.sentences is not None:
return
# Digest the problem into sentences
tokenizer = data.load("tokenizers/punkt/english.pickle")
self.sentences = tokenizer.tokenize(self.text.strip())
# Digest each sentence into words and part-of-speech tags
if self.sentence_tags is None:
sentence_tags = []
all_tags = []
all_words = []
for s in self.sentences:
all_words.append(s)
tags = pos_tag(word_tokenize(s))
sentence_tags.append(tags)
for t in tags:
l = len(t[0])
if not self.longest_word or self.longest_word < l:
self.longest_word = l
all_tags.append(t[1])
self.sentence_tags = sentence_tags
self.all_tags = uniq(all_tags)
self.all_words = uniq(all_words)
def condense(self, itemList):
if not "word_forms" in self.raw:
self.raw["word_forms"] = {}
if not "single" in self.raw["word_forms"]:
self.raw["word_forms"]["single"] = {}
if not "plural" in self.raw["word_forms"]:
self.raw["word_forms"]["plural"] = {}
if not "solved" in self.raw["word_forms"]:
self.raw["word_forms"]["solved"] = []
outp = []
if len(itemList) < 2:
outp = itemList
else:
comps = [] + itemList
for item in itemList:
for comp in comps:
if item == comp:
continue
else:
cond = self.condensor(item, comp)
if cond is None:
outp.append(item)
else:
outp.append(cond)
outp = uniq(outp)
return outp
def condense(self, itemList):
if not "word_forms" in self.raw:
self.raw["word_forms"] = {}
if not "single" in self.raw["word_forms"]:
self.raw["word_forms"]["single"] = {}
if not "plural" in self.raw["word_forms"]:
self.raw["word_forms"]["plural"] = {}
if not "solved" in self.raw["word_forms"]:
self.raw["word_forms"]["solved"] = []
outp = []
if len(itemList) < 2:
outp = itemList
else:
comps = [] + itemList
for item in itemList:
for comp in comps:
if item == comp:
continue
else:
cond = self.condensor(item, comp)
if cond is None:
outp.append(item)
else:
outp.append(cond)
outp = uniq(outp)
return outp
def run_sim(self):
""" Run model simulation! """
project_day = 0
# figure out the number of years for simulation and the number of
# days in each year
years = uniq(self.met_data["year"])
days_in_year = [self.met_data["year"].count(yr) for yr in years]
# =============== #
# YEAR LOOP #
# =============== #
for i, yr in enumerate(years):
self.day_output = [] # empty daily storage list for outputs
daylen = calculate_daylength(days_in_year[i], self.params.latitude)
if self.control.deciduous_model:
self.P.calculate_phenology_flows(daylen, self.met_data,
days_in_year[i], project_day)
self.zero_stuff()
# =============== #
# DAY LOOP #
# =============== #
for doy in xrange(days_in_year[i]):
# litterfall rate: C and N fluxes
(fdecay, rdecay) = self.lf.calculate_litter(doy)
# co2 assimilation, N uptake and loss
self.pg.calc_day_growth(project_day, fdecay, rdecay,
daylen[doy], doy,
float(days_in_year[i]), i)
# soil C & N model fluxes
self.cs.calculate_csoil_flows(project_day)
self.ns.calculate_nsoil_flows()
# calculate C:N ratios and increment annual flux sums
self.day_end_calculations(project_day, days_in_year[i])
#print self.state.shoot, self.state.lai
#print self.fluxes.gpp * 100, self.state.lai
# =============== #
# END OF DAY #
# =============== #
self.save_daily_outputs(yr, doy+1)
project_day += 1
# =============== #
# END OF YEAR #
# =============== #
if self.control.deciduous_model:
self.pg.allocate_stored_c_and_n(init=False)
if self.control.print_options == "DAILY" and self.spin_up == False:
self.print_output_file()
# close output files
if self.control.print_options == "END" and self.spin_up == False:
self.print_output_file()
self.pr.clean_up()
(control, params, state, files, fluxes, met_data, print_opts) = initialise_model_data(fname, met_header, DUMP=False)
control.ps_pathway = "C4"
if control.ps_pathway == "C3":
M = MateC3(control, params, state, fluxes, met_data)
else:
M = MateC4(control, params, state, fluxes, met_data)
# flai = "/Users/mdekauwe/research/NCEAS_face/GDAY_ornl_simulation/experiments/silvias_LAI.txt"
# lai_data = np.loadtxt(flai)
project_day = 0
# figure out the number of years for simulation and the number of
# days in each year
years = uniq(met_data["year"])
# years = years[:-1] # dump last year as missing "site" LAI
days_in_year = [met_data["year"].count(yr) for yr in years]
for i, yr in enumerate(years):
daylen = calculate_daylength(days_in_year[i], params.latitude)
for doy in xrange(days_in_year[i]):
state.wtfac_root = 1.0
# state.lai = lai_data[project_day]
state.lai = 2.0
if state.lai > 0.0:
state.shootnc = 0.03 # state.shootn / state.shoot
state.ncontent = state.shootnc * params.cfracts / params.sla * const.KG_AS_G
else:
state.ncontent = 0.0
def __init__(self, fname=None, DUMP=False, spin_up=False, met_header=4):
""" Set up model
* Read meterological forcing file
* Read user config file and adjust the model parameters, control or
initial state attributes that are used within the code.
* Setup all class instances...perhaps this isn't the tidyest place for
this?
* Initialise things, zero stores etc.
Parameters:
----------
fname : string
filename of model parameters, including path
chk_cmd_line : logical
parse the cmd line?
DUMP : logical
dump a the default parameters to a file
met_header : int
row number of met file header with variable names
Returns:
-------
Nothing
Controlling class of the model, runs things.
"""
self.day_output = [] # store daily outputs
# initialise model structures and read met data
(self.control, self.params, self.state, self.files, self.fluxes,
self.met_data, self.print_opts) = initialise_model_data(fname,
met_header,
DUMP=DUMP)
# params are defined in per year, needs to be per day
# Important this is done here as rate constants elsewhere in the code
# are assumed to be in units of days not years!
self.correct_rate_constants(output=False)
# class instances
self.cs = CarbonSoilFlows(self.control, self.params, self.state,
self.fluxes, self.met_data)
self.ns = NitrogenSoilFlows(self.control, self.params, self.state,
self.fluxes, self.met_data)
self.lf = Litter(self.control, self.params, self.state, self.fluxes)
self.pg = PlantGrowth(self.control, self.params, self.state,
self.fluxes, self.met_data)
self.cb = CheckBalance(self.control, self.params, self.state,
self.fluxes, self.met_data)
if self.control.deciduous_model:
self.pg.calc_carbon_allocation_fracs(0.0, 0, 0) #comment this!!
self.pg.allocate_stored_c_and_n(init=True)
self.P = Phenology(
self.fluxes,
self.state,
self.control,
self.params.previous_ncd,
store_transfer_len=self.params.store_transfer_len)
self.pr = PrintOutput(self.params, self.state, self.fluxes,
self.control, self.files, self.print_opts)
# build list of variables to print
(self.print_state, self.print_fluxes) = self.pr.get_vars_to_print()
# print model defaults
if DUMP == True:
self.pr.save_default_parameters()
sys.exit(0)
# calculate initial stuff, e.g. C:N ratios and zero annual flux sums
self.day_end_calculations(0, INIT=True)
self.state.pawater_root = self.params.wcapac_root
self.state.pawater_topsoil = self.params.wcapac_topsoil
self.spin_up = spin_up
self.state.sla = self.params.slainit # Specific leaf area (m2/kg DW)
self.state.lai = (self.params.slainit * const.M2_AS_HA /
const.KG_AS_TONNES / self.params.cfracts *
self.state.shoot)
# figure out the number of years for simulation and the number of
# days in each year
self.years = uniq(self.met_data["year"])
self.days_in_year = [self.met_data["year"].count(yr) \
for yr in self.years]
if self.control.water_stress == False:
sys.stderr.write("**** You have turned off the drought stress")
sys.stderr.write(", I assume you're debugging??!\n")
def execute(self): p = self.problem raw_actions, raw_operators, raw_subordinates = [], [], [] index = 0 for s_tags in p.sentence_tags: s = SentenceParser(s_tags, p, p.sentences[index]) self.sentences.append(s) self.track_longer(s.longest_phrase) if s.is_about_requirements: self.is_requirement_problem = True raw_operators += s.operators raw_actions += s.actions raw_subordinates += s.subordinates self.contexts += s.contexts self.units += s.units if len(s.subordinate_strings) > 0: sd = dict(s.subordinate_strings) if len(self.subordinate_strings) > 0: sd.update(self.subordinate_strings) self.subordinate_strings = sd if s.question: self.queries.append(index) index += 1 # Ensure uniqueness in our data raw_operators = uniq(raw_operators) raw_actions = uniq(raw_actions) raw_subordinates = uniq(raw_subordinates) self.contexts = uniq(self.contexts) self.units = uniq(self.units) self.condense() # Set our phrasings based on the contexts in play if len(self.contexts) > 1: op_key = OPERATOR_STR["multiple_contexts"] elif len(self.contexts) > 0: op_key = OPERATOR_STR["single_context"] else: op_key = OPERATOR_STR["no_context"] for sub in raw_subordinates: word, subtype = sub if subtype is not None and subtype[0:4] != "time": if "grouping" not in subtype: self.subordinates.append(self.subordinate_strings[word]) self.subordinates = uniq(self.subordinates) format_operators = True for act in raw_actions: self.actions.append(act) self.actions = uniq(self.actions) pre_ops = False if len(self.units) == 0 and len(self.actions) > 0: pre_ops = True # We have special phrasing which is mucked up by the operators # if we're only dealing with op_key = OPERATOR_STR["context_actions_nounit"] for op in raw_operators: if op != "eq" or len(raw_operators) == 1: try: self.operators.append(op_key[op]) except: pass self.operators = uniq(self.operators) if pre_ops: self.preops = self.operators self.operators = []
def __init__(self, fname=None, DUMP=False, spin_up=False, met_header=4):
""" Set up model
* Read meterological forcing file
* Read user config file and adjust the model parameters, control or
initial state attributes that are used within the code.
* Setup all class instances...perhaps this isn't the tidyest place for
this?
* Initialise things, zero stores etc.
Parameters:
----------
fname : string
filename of model parameters, including path
chk_cmd_line : logical
parse the cmd line?
DUMP : logical
dump a the default parameters to a file
met_header : in
row number of met file header with variable name
Returns:
-------
Nothing
Controlling class of the model, runs things.
"""
self.day_output = [] # store daily output
# initialise model structures and read met data
(self.control, self.params,
self.state, self.files,
self.fluxes, self.met_data,
self.print_opts) = initialise_model_data(fname, met_header, DUMP=DUMP)
# params are defined in per year, needs to be per day
# Important this is done here as rate constants elsewhere in the code
# are assumed to be in units of days not years!
self.correct_rate_constants(output=False)
# class instance
self.cs = CarbonSoilFlows(self.control, self.params, self.state,
self.fluxes, self.met_data)
self.ns = NitrogenSoilFlows(self.control, self.params, self.state,
self.fluxes, self.met_data)
self.lf = Litter(self.control, self.params, self.state, self.fluxes)
self.pg = PlantGrowth(self.control, self.params, self.state,
self.fluxes, self.met_data)
self.cb = CheckBalance(self.control, self.params, self.state,
self.fluxes, self.met_data)
self.db = Disturbance(self.control, self.params, self.state,
self.fluxes, self.met_data)
if self.control.deciduous_model:
if self.state.max_lai is None:
self.state.max_lai = 0.01 # initialise to something really low
self.state.max_shoot = 0.01 # initialise to something really low
# Are we reading in last years average growing season?
if (float_eq(self.state.avg_alleaf, 0.0) and
float_eq(self.state.avg_alstem, 0.0) and
float_eq(self.state.avg_albranch, 0.0) and
float_eq(self.state.avg_alleaf, 0.0) and
float_eq(self.state.avg_alroot, 0.0) and
float_eq(self.state.avg_alcroot, 0.0)):
self.pg.calc_carbon_allocation_fracs(0.0) #comment this!!
else:
self.fluxes.alleaf = self.state.avg_alleaf
self.fluxes.alstem = self.state.avg_alstem
self.fluxes.albranch = self.state.avg_albranch
self.fluxes.alroot = self.state.avg_alroot
self.fluxes.alcroot = self.state.avg_alcroot
self.pg.allocate_stored_c_and_n(init=True)
#self.pg.enforce_sensible_nstore()
self.P = Phenology(self.fluxes, self.state, self.control,
self.params.previous_ncd,
store_transfer_len=self.params.store_transfer_len)
self.pr = PrintOutput(self.params, self.state, self.fluxes,
self.control, self.files, self.print_opts)
# build list of variables to prin
(self.print_state, self.print_fluxes) = self.pr.get_vars_to_print()
# print model defaul
if DUMP == True:
self.pr.save_default_parameters()
sys.exit(0)
self.dead = False # johnny 5 is alive
# calculate initial stuff, e.g. C:N ratios and zero annual flux sum
self.day_end_calculations(INIT=True)
self.state.pawater_root = self.params.wcapac_root
self.state.pawater_topsoil = self.params.wcapac_topsoil
self.spin_up = spin_up
self.state.lai = max(0.01, (self.params.sla * const.M2_AS_HA /
const.KG_AS_TONNES / self.params.cfracts *
self.state.shoot))
# figure out the number of years for simulation and the number of
# days in each year
self.years = uniq(self.met_data["year"])
self.days_in_year = [self.met_data["year"].count(yr)
for yr in self.years]
if self.control.water_stress == False:
sys.stderr.write("**** You have turned off the drought stress")
sys.stderr.write(", I assume you're debugging??!\n")
else:
M = MateC4(control, params, state, fluxes, met_data)
#flai = "/Users/mdekauwe/research/NCEAS_face/GDAY_ornl_simulation/experiments/silvias_LAI.txt"
#lai_data = np.loadtxt(flai)
# Specific LAI (m2 onesided/kg DW)
state.sla = params.slainit
project_day = 0
# figure out the number of years for simulation and the number of
# days in each year
years = uniq(met_data["year"])
#years = years[:-1] # dump last year as missing "site" LAI
days_in_year = [met_data["year"].count(yr) for yr in years]
for i, yr in enumerate(years):
daylen = calculate_daylength(days_in_year[i], params.latitude)
for doy in xrange(days_in_year[i]):
state.wtfac_root = 1.0
#state.lai = lai_data[project_day]
state.lai = 2.0
if state.lai > 0.0:
state.shootnc = 0.03 #state.shootn / state.shoot
state.ncontent = (state.shootnc * params.cfracts /
state.sla * const.KG_AS_G)
else:
def execute(self):
p = self.problem
def process(word, tag):
did_something = False
self.subtype = None
if self.index == 0 and tag not in ["NNP", "NNPS"]:
# Fix capitalization for non proper nouns
word = word.lower()
word, tag = p.brain.retag(word, tag)
if tag == "!!!":
print "Missing retag converter in the brain"
exit(1)
if tag in ["NNP", "NNPS"]:
word = word.capitalize()
if self.last_conjunction is not None:
self.conjunction_parts.append(word)
# Fix the parser as we go on brain retags
def do_track(subtype, tag):
if subtype[0] is None and subtype[1] is None:
subtype = None
tag = p.brain.retag(word, tag)
process(word, tag)
return (False, subtype, tag)
return (True, subtype, tag)
if tag in ["NN", "NNS"]:
if self.last_tag in ["PRP$"]:
# Detects Jane's friends; probably context
context = " ".join([self.partial_context, word])
self.parsed.pop()
self.last_context = context
self.contexts.append(context)
did_something = True
self.subtype = self.get_subtype(word, tag)
self.partial_context = None
self.last_partial_context = None
self.partial_subtype = None
track, self.subtype, tag = do_track(self.subtype, tag)
if track:
if self.problem.involves_acting:
conjunction = ((word, tag), self.last_conjunction)
self.subordinate_strings[word] = " ".join(self.conjunction_parts)
self.conjunction_parts = []
self.conjunctions.append(conjunction)
did_something = True
self.track(conjunction[0], "subordinate", self.subtype)
else:
if self.subtype[0] == "plural" and self.subtype[1] == "mixed":
# PRP$ will always have a noun like last_subtype
last_plural, last_gender = self.last_subtype
# Last Plural Context
lpc = p.last_contexts["plurality"][last_plural]
# Last Gender Context
lgc = p.last_contexts["gender"][last_gender]
self.last_context = None
if lgc == lpc:
self.last_context = lgc
elif lgc is not None:
self.last_context = lgc
elif lpc is not None:
self.last_context = lpc
if self.last_context is not None:
# Last context string
inc = p.brain.inclusive(word,
"'{0}' ({1}) in '{2}'".format(self.last_word,
self.last_context[0],
context))
# Add the context by the word for easier lookup
# this will capture 'her friends' and define
# the concept of inclusiveness for friends then
# later auto-apply this to the concept of say,
# 'his friends'
p.brain.add("inclusive", context, inc)
# Assume the first context after a comparison is the
# comparator context
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype, conv=True)
else:
self.main_context = context
self.track(context, "context", self.subtype)
self.context_subtypes[context] = self.subtype
else:
do_reg_unit = True
if self.last_conjunction is not None:
do_reg_unit = False
do_conj_track = False
if tag == "NN" and self.last_tag in ["IN"]:
self.subtype = self.get_subtype(word, tag)
track, self.subtype, tag = do_track(self.subtype, tag)
did_something = True
if track:
unit = " ".join([self.last_unit, self.last_word, word])
self.problem.descriptive_units.append(unit)
self.problem.refined_units[self.last_unit] = unit
self.units.pop()
self.parsed.pop()
self.parsed.pop()
self.last_unit = unit
self.last_unit_tag = tag
self.last_unit_index = len(self.parsed)
unit, uidx = self.fix_unit(unit)
self.units.append(unit)
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
self.track(unit, "unit", self.subtype, uidx)
elif tag in ["NN", "NNS"] and self.last_conjunction_tag in ["IN"]:
self.subtype = self.get_subtype(word, tag)
track, self.subtype, tag = do_track(self.subtype, tag)
did_something = True
if track:
if self.last_variable is not None:
unit = word
self.problem.descriptive_units.append(unit)
self.problem.refined_units[self.last_unit] = unit
self.last_unit = unit
self.last_unit_tag = tag
self.last_unit_index = len(self.parsed)
unit, uidx = self.fix_unit(unit)
self.units.append(unit)
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
c_unit = " ".join([word, "owned by", p.previous_contexts["last"][0]])
self.track((c_unit, word, p.previous_contexts["last"][0], p.last_contexts["last"][0]), "context_unit", self.subtype, uidx)
else:
do_conj_track = True
else:
do_conj_track = True
do_unset_conj = True
if do_conj_track:
unit = word
do_std = True
if tag in ["NN", "NNS"]:
if self.last_tag in ["NN", "NNS"]:
if self.last_unit is None and self.last_context is not None:
self.last_context = None
lc = self.contexts.pop()
unit = " ".join([lc, word])
else:
unit = " ".join([self.last_unit, word])
self.units.pop()
self.used_unit_adjectives.append(self.last_word)
self.parsed.pop()
ptest = self.parsed.pop()
self.parsed.append(ptest)
if ptest[1] == "constant" and self.last_tag != "DT" and self.problem.exestential:
unit, uidx = self.fix_unit(unit)
self.last_unit = unit
self.last_unit_tag = tag
self.last_unit_index = len(self.parsed)
self.units.append(unit)
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
self.track(unit, "unit", self.subtype, uidx)
do_unset_conj = False
do_reg_unit = False
do_std = False
if do_std:
if word in self.used_unit_adjectives:
do_unset_conj = False
do_reg_unit = True
else:
self.used_unit_adjectives.append(word)
conjunction = ((unit, tag), self.last_conjunction)
self.subordinate_strings[unit] = " ".join(self.conjunction_parts)
self.conjunction_parts = []
self.conjunctions.append(conjunction)
did_something = True
self.track(conjunction[0], "subordinate", self.subtype)
if do_unset_conj:
self.last_conjunction = None
if do_reg_unit:
self.subtype = self.get_subtype(word, tag)
track, self.subtype, tag = do_track(self.subtype, tag)
did_something = True
if track:
#rint "here for", word, tag, self.last_tag, self.last_unit, self.last_word
unit = word
do_std_unit = True
if tag in ["NN", "NNS"] and self.last_tag in ["NN", "NNS"]:
# rint self.last_word, self.last_tag
# rint word, tag
# rint self.parsed
# rint "----"
ptest = self.parsed.pop()
if ptest[1] == "subordinate" and self.last_unit is None:
cst = self.conjunctions.pop()
subostr = " ".join([self.subordinate_strings[ptest[0][0]], word])
del self.subordinate_strings[ptest[0][0]]
unit = " ".join([ptest[0][0], word])
self.subordinate_strings[unit] = subostr
self.used_unit_adjectives.append(ptest[0][0])
conjunction = ((unit, tag), cst[1])
self.conjunctions.append(conjunction)
self.track(conjunction[0], "subordinate", self.subtype)
do_std_unit = False
else:
if self.last_unit is None and self.last_context is not None:
self.last_context = None
lc = self.contexts.pop()
unit = " ".join([lc, word])
else:
unit = " ".join([self.last_unit, word])
self.units.pop()
self.used_unit_adjectives.append(self.last_word)
if do_std_unit:
unit, uidx = self.fix_unit(unit)
if not unit in p.units_acting_as_context or not p.units_acting_as_context[unit]:
if self.last_unit and self.framing_question and len(self.operators) == 0:
context = unit
if self.subtype[0] == "self":
context = self.problem.brain.self_reflexive(context, True)
# @TODO: This needs to be a subroutine or something
self.last_context = context
self.contexts.append(context)
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype)
else:
self.main_context = context
self.track(context, "context", self.subtype)
self.context_subtypes[context] = self.subtype
else:
if len(self.operators) > 0 and self.last_unit and not self.main_context:
if self.last_unit != unit:
context = self.units.pop()
self.last_context = context
self.contexts.append(context)
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype, self.last_unit_index)
else:
self.main_context = context
self.track(context, "context", self.subtype, self.last_unit_index)
self.context_subtypes[context] = self.subtype
self.last_unit = unit
self.last_unit_tag = tag
self.last_unit_index = len(self.parsed)
self.units.append(unit)
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
self.track(unit, "unit", self.subtype, uidx)
else:
context = unit
if self.subtype[0] == "self":
context = self.problem.brain.self_reflexive(context, True)
# @TODO: This needs to be a subroutine or something
self.last_context = context
self.contexts.append(context)
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype, uidx)
else:
self.main_context = context
self.track(context, "context", self.subtype, uidx)
self.context_subtypes[context] = self.subtype
if not did_something and self.subtype == None:
self.subtype = self.get_subtype(word, tag)
if tag == "SUB":
conjunction = ((word, tag), self.last_conjunction)
self.subordinate_strings[word] = " ".join(self.conjunction_parts)
self.conjunction_parts = []
self.conjunctions.append(conjunction)
did_something = True
self.track(conjunction[0], "subordinate", self.subtype)
if tag == "PRP":
if self.subtype and (self.subtype[0] == "self" or self.subtype[1] == "self"):
context = word
if self.subtype[0] == "self":
context = self.problem.brain.self_reflexive(context, True)
self.last_context = context
self.contexts.append(context)
did_something = True
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype)
else:
self.main_context = context
self.track(context, "context", self.subtype)
self.context_subtypes[context] = self.subtype
else:
c = self.resolve_context(self.subtype)
if c is None:
c = self.resolve_context(self.subtype, compx=True)
if c is not None:
# If we're setting a relative quantity and the contexts are
# the same we're not actually setting a relative quantity
# we are simply indicating a mathematical operand
if self.is_relative_quantity and c[0] == self.last_context:
c2 = self.resolve_context(self.subtype, compx=True, failTest=True)
if c2 is not None and c2[0] != self.last_context:
c = c2
else:
self.is_relative_quantity = False
did_something = True
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = c[0]
self.track(c[0], "comparator_context", self.subtype, conv=True)
else:
self.main_context = c[0]
self.track(c[0], "context", self.subtype)
self.context_subtypes[c[0]] = self.subtype
# Unset hanging conjunctions when we set a context
self.last_conjunction = None
else:
if self.last_tag in ["IN"]:
# remove the "of" in "of them"
self.parsed.pop()
# Assume this is a unit
if not p.involves_acting or self.last_unit:
unit, uidx = self.fix_unit(self.last_unit)
self.last_unit = unit
self.last_unit_tag = self.last_unit_tag
self.last_unit_index = len(self.parsed)
self.units.append(unit)
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
self.track(unit, "unit", self.subtype, uidx)
if tag == "PRP$":
if self.last_conjunction is not None:
if self.is_relative_quantity and not self.comparator_context and self.main_context:
c = self.resolve_context(self.subtype)
ms = self.context_subtypes[self.main_context]
if (not c or self.main_context != c[0]) and (not ms or self.subtype[0] != ms[0]):
context = word
if self.subtype[0] == "self":
context = self.problem.brain.self_reflexive(context, True)
did_something = True
self.comparator_context = context
self.track(word, "comparator_context", self.subtype, conv=True)
self.last_conjunction = None
self.conjunction_parts = []
self.context_subtypes[word] = self.subtype
if not did_something:
did_something = True
term = word
st = self.subtype
conval = self.resolve_context(self.subtype)
if conval:
term, st = conval
term = ownerize(term)
if self.subtype[0] == "self":
term = self.problem.brain.self_reflexive(term, True)
self.partial_context = term
self.partial_subtype = st
self.last_partial_context = self.track(term, "partial_context", st)
if tag in ["NNP", "NNPS"]:
if self.last_tag in ["NNP", "NNPS"]:
# Last was partial context; the "Mrs." in "Mrs. Jones"
self.last_context = None
lc = self.contexts.pop()
self.parsed.pop()
context = " ".join([self.last_word, word])
if self.is_relative_quantity and self.comparator_context == lc:
self.comparator_context = context
if self.subtype is not None and self.subtype[1] == "ambiguous":
old = self.get_subtype(self.last_word, self.last_tag)
self.subtype = (self.subtype[0], old[1])
else:
context = word
if self.subtype[0] == "self":
context = self.problem.brain.self_reflexive(context, True)
self.last_context = context
self.contexts.append(context)
did_something = True
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype)
else:
self.main_context = context
self.track(context, "context", self.subtype)
self.context_subtypes[context] = self.subtype
if tag[:2] == "VB":
if tag == "VB" and self.question:
if self.last_verb_tag is not None:
# VB*...VB indicates a question not an operation
self.last_operator = None
q_start = self.raw_operators.pop()
st = self.get_subtype(self.last_verb, self.last_verb_tag)
self.track(q_start, "q_start", st, self.last_verb)
self.framing_question = False
self.did_frame_question = True
did_something = True
self.track(word, "q_stop", self.subtype)
elif self.framing_question:
did_something = True
self.framing_question = False
self.did_frame_question = True
self.track(word, "q_stop", self.subtype)
op = p.brain.operator(word, self.sentence_text)
if op == "ex":
self.is_exchange = True
elif tag != "VBG":
if self.framing_question:
did_something = True
self.framing_question = False
self.did_frame_question = True
self.track(word, "q_stop", self.subtype)
else:
if self.question and not self.did_frame_question:
# We are asking a question and likely don't
# need to invoke all the operator logic; this is
# likely a "start the question" verb
if not self.framing_question:
did_something = True
self.framing_question = True
self.track(word, "q_start", self.subtype)
else:
op = p.brain.operator(word, self.sentence_text)
if op not in [None, False]: # Retagged
if op == "re":
self.is_about_requirements = True
if op == "ex":
self.is_exchange = True
self.last_verb_tag = tag
self.last_verb = len(self.parsed)
self.last_operator = word
self.raw_operators.append(word)
did_something = True
self.operators.append(op)
self.operator[word] = op
self.subtype = self.get_subtype(word, tag)
self.track(word, "operator", self.subtype)
else:
did_something = True
process(word, tag)
else:
did_something = True
gtype = p.brain.gerund(word, self.sentence_text)
if gtype == "acting":
self.acting = False
self.actions.append(word)
if self.main_context:
p.context_actions[self.main_context] = (word, gtype, self.subtype)
self.track(word, gtype, self.subtype)
if tag in ["JJR", "COMP"]:
did_something = True
adj = p.brain.relative(word, self.sentence_text)
self.is_relative_quantity = adj != "noise"
self.track(word, adj, self.subtype)
if tag in ["IN", "TO"]:
self.last_conjunction = word
self.last_conjunction_tag = tag
self.conjunction_parts.append(word)
did_something = True
self.track(word, "conjunction", self.subtype)
if tag in [".", ","]:
did_something = True
self.track(word, "punctuation", self.subtype)
if tag == "EX":
did_something = True
self.track(word, "exestential", self.subtype)
self.problem.exestential = True
if tag in ["CD", "LS"]: # A cardinal number
if self.last_tag == "DT":
if self.last_determiner == "constant":
# should safely be able to ignore detemriner constant
self.parsed.pop()
self.last_determiner = None
if self.last_conjunction is not None:
self.conjunction_parts.pop()
did_something = True
self.track(word, "constant", self.subtype)
if tag == "DT": # A determiner (the)
did_something = True
dtype = p.brain.determiner(word, self.sentence_text)
self.last_determiner = dtype
if dtype == "variable":
vtype = p.brain.variable(word, self.sentence_text)
self.last_variable = word
self.last_variable_type = vtype
if vtype == "dynamic_variable":
self.track(word, vtype, self.subtype)
else:
self.track(vtype, "variable_relationship", self.subtype)
elif dtype == "constant":
self.track(p.brain.number(word, self.sentence_text), dtype, self.subtype)
else:
self.track(word, dtype, self.subtype)
if tag == "RB": # An adverb, probably a subordinate?
if self.last_conjunction is not None:
conjunction = ((word, tag), self.last_conjunction)
self.last_conjunction = None
self.subordinate_strings[word] = " ".join(self.conjunction_parts)
self.conjunction_parts = []
else:
conjunction = ((word, tag), None)
self.subordinate_strings[word] = word
self.conjunctions.append(conjunction)
did_something = True
self.track(conjunction[0], "subordinate", self.subtype)
# Anything about phrasing must come before the wh-determiner block
if tag == "JJ": # Adjective
if self.phrasing_question:
self.parsed.pop()
did_something = True
self.track(" ".join([self.last_word, word]), "asking", self.subtype)
elif self.last_tag in ["JJ"]:
self.used_unit_adjectives.append(word)
did_something = True
self.last_adjective = " ".join([self.last_adjective, word])
self.track(self.last_adjective, "adjective", self.subtype, self.last_adjective_index)
elif self.last_tag in ["IN"]:
did_something = True
unit = " ".join([self.last_unit, self.last_word, word])
self.problem.descriptive_units.append(unit)
self.problem.refined_units[self.last_unit] = unit
self.units.pop()
self.parsed.pop()
self.parsed.pop()
self.last_unit = unit
self.last_unit_tag = tag
self.last_unit_index = len(self.parsed)
unit, uidx = self.fix_unit(unit)
self.units.append(unit)
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
self.track(unit, "unit", self.subtype, uidx)
else:
self.used_unit_adjectives.append(word)
did_something = True
self.last_adjective = word
self.last_adj_index = len(self.parsed)
self.track(word, "adjective", self.subtype)
if tag == "PIP":
did_something = True
self.track(word, "pre_ind_plu", self.subtype)
if not self.problem.exestential:
self.acting = True
self.problem.involves_acting = True
if self.last_unit is not None and self.last_context is None:
# If we have a present indicitive plural with no context
# it is likely that we're dealing with a context which has
# been misinterpreted as a unit, so switch that
# mistake here.
p.units_acting_as_context[self.last_unit] = True
# Remove the last unit and make it a context
context = self.units.pop()
self.last_context = context
self.contexts.append(context)
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype, self.last_unit_index)
else:
self.main_context = context
self.track(context, "context", self.subtype, self.last_unit_index)
self.context_subtypes[context] = self.subtype
# @TODO: This needs a much better tracking system
self.last_unit = None
self.last_unit_tag = None
self.last_unit_index = -1
elif not self.phrasing_question:
self.new_units_as_context = True
if tag in ["WRB", "WDT", "WP", "WP$"]: # Wh- determiner
self.question = True
self.phrasing_question = True
did_something = True
self.track(word, "asking", self.subtype)
elif self.phrasing_question:
self.phrasing_question = False
if tag == "CC":
did_something = True
self.track(word, "coordinating_conjunction")
if self.subtype is not None:
did_something = True
if not did_something:
#rint "Answer Unknown", word, tag
p.brain.unknown(word, tag, self.subtype, self.sentence_text)
process(word, tag)
self.last_word = word
self.last_tag = tag
self.last_subtype = self.subtype
for s_tag in self.sentence:
process(*s_tag)
self.index += 1
if self.partial_context:
context = self.partial_context
idx = self.last_partial_context
stype = self.partial_subtype
self.partial_context = None
self.last_partial_context = None
self.partial_subtype = None
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", stype, idx, True)
# Make all the inferred items unique
self.raw_operators = uniq(self.raw_operators)
self.contexts = uniq(self.contexts)
self.units = uniq(self.units)
self.actions = uniq(self.actions)
self.operators = uniq(self.operators)
text = self.sentence_text
nunits = []
for unit in self.units:
acting_as_context = False
if unit in p.units_acting_as_context:
acting_as_context = p.units_acting_as_context[unit]
# Units can be complicated by adjectives which break up the unit
# into important subdivisions. However, these subdivisions may
# also be simple detractors and not relevant to the actual string
# match for the unit. As such, we split by any spaces and then
# compile all possible formulations of the unit
if not unit in self.problem.descriptive_units and " " in unit:
parts = unit.split(" ")
# The item is the last part of the unit
item = parts.pop()
for part in parts:
nu = " ".join([part, item])
nunits.append(nu)
# if acting_as_context:
# p.units_acting_as_context[nu] = True
self.units = uniq(self.units + nunits)
for unit in self.units:
if unit in self.problem.refined_units:
uxd = self.parsed[self.unit_idx[unit]]
self.parsed[self.unit_idx[unit]] = (self.problem.refined_units[unit], uxd[1], uxd[2])
# Resolve the subordinates
for o in self.conjunctions:
subordinate, conjunction = o
outp = p.brain.subordinate(subordinate, text)
self.subordinate_lookup[subordinate[0]] = outp
if outp == "object":
continue
self.subordinates.append((subordinate[0], outp))
if len(self.subordinate_strings[subordinate[0]]) == 0 and outp != "place_noun":
self.subordinate_strings[subordinate[0]] = ANSWER_SUBORDINATE[outp]
if outp == "context_grouping":
csub = self.get_subtype(subordinate[0], subordinate[1])
if csub:
c = self.resolve_context(csub)
if c:
self.problem.subordinate_adaptive_contexts.append(c[0])
self.subordinates = uniq(self.subordinates)
self.problem.units += self.units
self.problem.units = uniq(self.problem.units)
if len(self.subordinates) > 0:
if not self.main_context in self.problem.context_subordinates:
self.problem.context_subordinates[self.main_context] = (self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup)
elif self.main_context in self.problem.context_subordinates:
# This is an inferred context
self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup = self.problem.context_subordinates[self.main_context]
for sub in self.subordinates:
subord, subt = sub
self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord])
if not self.main_context and self.operator:
if self.problem.last_contexts["last"]:
context, subtype = self.problem.last_contexts["last"]
self.main_context = context
self.track(context, "context_inferred", subtype)
#rint self.subordinates, self.main_context, self.problem.context_subordinates
if self.main_context in self.problem.context_subordinates:
if len(self.subordinates) == 0:
# This is an inferred context
self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup = self.problem.context_subordinates[self.main_context]
for sub in self.subordinates:
subord, subt = sub
self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord])
elif len(self.subordinates) > 0:
subs, sub_strs, sub_subs, sub_look = self.problem.context_subordinates[self.main_context]
for sub in subs:
subord, subt = sub
if subord in self.subordinate_lookup:
continue
self.subordinate_subtypes[subord] = sub_subs[subord]
self.subordinate_lookup[subord] = sub_look[subord]
if subt != "refiner":
self.subordinates.append(sub)
self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord])
if self.main_context in self.problem.context_actions and len(self.actions) == 0:
word, gtype, subtype = self.problem.context_actions[self.main_context]
self.track(word, gtype + "_inferred", subtype)
def execute(self):
p = self.problem
def process(word, tag):
did_something = False
self.subtype = None
if self.index == 0 and tag not in ["NNP", "NNPS"]:
# Fix capitalization for non proper nouns
word = word.lower()
word, tag = p.brain.retag(word, tag)
if tag == "!!!":
print "Missing retag converter in the brain"
exit(1)
if tag in ["NNP", "NNPS"]:
word = word.capitalize()
if self.last_conjunction is not None:
self.conjunction_parts.append(word)
# Fix the parser as we go on brain retags
def do_track(subtype, tag):
if subtype[0] is None and subtype[1] is None:
subtype = None
tag = p.brain.retag(word, tag)
process(word, tag)
return (False, subtype, tag)
return (True, subtype, tag)
if tag in ["NN", "NNS"]:
if self.last_tag in ["PRP$"]:
# Detects Jane's friends; probably context
context = " ".join([self.partial_context, word])
self.parsed.pop()
self.last_context = context
self.contexts.append(context)
did_something = True
self.subtype = self.get_subtype(word, tag)
self.partial_context = None
self.last_partial_context = None
self.partial_subtype = None
track, self.subtype, tag = do_track(self.subtype, tag)
if track:
if self.problem.involves_acting:
conjunction = ((word, tag), self.last_conjunction)
self.subordinate_strings[word] = " ".join(self.conjunction_parts)
self.conjunction_parts = []
self.conjunctions.append(conjunction)
did_something = True
self.track(conjunction[0], "subordinate", self.subtype)
else:
if self.subtype[0] == "plural" and self.subtype[1] == "mixed":
# PRP$ will always have a noun like last_subtype
last_plural, last_gender = self.last_subtype
# Last Plural Context
lpc = p.last_contexts["plurality"][last_plural]
# Last Gender Context
lgc = p.last_contexts["gender"][last_gender]
self.last_context = None
if lgc == lpc:
self.last_context = lgc
elif lgc is not None:
self.last_context = lgc
elif lpc is not None:
self.last_context = lpc
if self.last_context is not None:
# Last context string
inc = p.brain.inclusive(word,
"'{0}' ({1}) in '{2}'".format(self.last_word,
self.last_context[0],
context))
# Add the context by the word for easier lookup
# this will capture 'her friends' and define
# the concept of inclusiveness for friends then
# later auto-apply this to the concept of say,
# 'his friends'
p.brain.add("inclusive", context, inc)
# Assume the first context after a comparison is the
# comparator context
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype, conv=True)
else:
self.main_context = context
self.track(context, "context", self.subtype)
self.context_subtypes[context] = self.subtype
else:
do_reg_unit = True
if self.last_conjunction is not None:
do_reg_unit = False
do_conj_track = False
if tag == "NN" and self.last_tag in ["IN"]:
self.subtype = self.get_subtype(word, tag)
track, self.subtype, tag = do_track(self.subtype, tag)
did_something = True
if track:
unit = " ".join([self.last_unit, self.last_word, word])
self.problem.descriptive_units.append(unit)
self.problem.refined_units[self.last_unit] = unit
self.units.pop()
del p.unit_subtypes[self.last_unit]
self.parsed.pop()
self.parsed.pop()
self.last_unit = unit
self.last_unit_tag = tag
self.last_unit_index = len(self.parsed)
unit, uidx = self.fix_unit(unit)
self.units.append(unit)
p.unit_subtypes[unit] = self.subtype
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
self.track(unit, "unit", self.subtype, uidx)
elif tag in ["NN", "NNS"] and self.last_conjunction_tag in ["IN"]:
self.subtype = self.get_subtype(word, tag)
track, self.subtype, tag = do_track(self.subtype, tag)
did_something = True
if track:
if self.last_variable is not None:
unit = word
self.problem.descriptive_units.append(unit)
self.problem.refined_units[self.last_unit] = unit
self.last_unit = unit
self.last_unit_tag = tag
self.last_unit_index = len(self.parsed)
unit, uidx = self.fix_unit(unit)
self.units.append(unit)
p.unit_subtypes[unit] = self.subtype
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
c_unit = " ".join([word, "owned by", p.previous_contexts["last"][0]])
self.track((c_unit, word, p.previous_contexts["last"][0], p.last_contexts["last"][0]), "context_unit", self.subtype, uidx)
else:
do_conj_track = True
else:
do_conj_track = True
do_unset_conj = True
if do_conj_track:
unit = word
do_std = True
if tag in ["NN", "NNS"]:
if self.last_tag in ["NN", "NNS"]:
if self.last_unit is None and self.last_context is not None:
self.last_context = None
lc = self.contexts.pop()
unit = " ".join([lc, word])
else:
unit = " ".join([self.last_unit, word])
self.units.pop()
self.used_unit_adjectives.append(self.last_word)
self.parsed.pop()
ptest = self.parsed.pop()
self.parsed.append(ptest)
if ptest[1] == "constant" and self.last_tag != "DT" and self.problem.exestential:
unit, uidx = self.fix_unit(unit)
self.last_unit = unit
self.last_unit_tag = tag
self.last_unit_index = len(self.parsed)
self.units.append(unit)
p.unit_subtypes[unit] = self.subtype
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
self.track(unit, "unit", self.subtype, uidx)
do_unset_conj = False
do_reg_unit = False
do_std = False
if do_std:
if word in self.used_unit_adjectives:
do_unset_conj = False
do_reg_unit = True
else:
conjunction = ((unit, tag), self.last_conjunction)
self.subordinate_strings[unit] = " ".join(self.conjunction_parts)
self.conjunction_parts = []
self.conjunctions.append(conjunction)
did_something = True
self.track(conjunction[0], "subordinate", self.subtype)
if do_unset_conj:
self.last_conjunction = None
if do_reg_unit:
self.subtype = self.get_subtype(word, tag)
track, self.subtype, tag = do_track(self.subtype, tag)
did_something = True
if track:
#rint "here for", word, tag, self.last_tag, self.last_unit, self.last_word
unit = word
do_std_unit = True
if tag in ["NN", "NNS"] and self.last_tag in ["NN", "NNS"]:
# rint self.last_word, self.last_tag
# rint word, tag
# rint self.parsed
# rint "----"
ptest = self.parsed.pop()
if ptest[1] == "subordinate" and self.last_unit is None:
cst = self.conjunctions.pop()
subostr = " ".join([self.subordinate_strings[ptest[0][0]], word])
del self.subordinate_strings[ptest[0][0]]
unit = " ".join([ptest[0][0], word])
self.subordinate_strings[unit] = subostr
self.used_unit_adjectives.append(ptest[0][0])
conjunction = ((unit, tag), cst[1])
self.conjunctions.append(conjunction)
self.track(conjunction[0], "subordinate", self.subtype)
do_std_unit = False
else:
if self.last_unit is None and self.last_context is not None:
self.last_context = None
lc = self.contexts.pop()
unit = " ".join([lc, word])
else:
unit = " ".join([self.last_unit, word])
self.units.pop()
del p.unit_subtypes[self.last_unit]
self.used_unit_adjectives.append(self.last_word)
if do_std_unit:
unit, uidx = self.fix_unit(unit)
if not unit in p.units_acting_as_context or not p.units_acting_as_context[unit]:
if self.last_unit and self.framing_question and len(self.operators) == 0 and self.last_unit != unit:
context = unit
if self.subtype[0] == "self":
context = self.problem.brain.self_reflexive(context, True)
# @TODO: This needs to be a subroutine or something
self.last_context = context
self.contexts.append(context)
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype)
else:
self.main_context = context
self.track(context, "context", self.subtype)
self.context_subtypes[context] = self.subtype
else:
if len(self.operators) > 0 and self.last_unit and not self.main_context:
if self.last_unit != unit:
context = self.units.pop()
del p.unit_subtypes[context]
self.last_context = context
self.contexts.append(context)
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype, self.last_unit_index)
else:
self.main_context = context
self.track(context, "context", self.subtype, self.last_unit_index)
self.context_subtypes[context] = self.subtype
self.last_unit = unit
self.last_unit_tag = tag
self.last_unit_index = len(self.parsed)
self.units.append(unit)
p.unit_subtypes[unit] = self.subtype
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
self.track(unit, "unit", self.subtype, uidx)
else:
context = unit
if self.subtype[0] == "self":
context = self.problem.brain.self_reflexive(context, True)
# @TODO: This needs to be a subroutine or something
self.last_context = context
self.contexts.append(context)
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype, uidx)
else:
self.main_context = context
self.track(context, "context", self.subtype, uidx)
self.context_subtypes[context] = self.subtype
if not did_something and self.subtype == None:
self.subtype = self.get_subtype(word, tag)
if tag == "SUB":
conjunction = ((word, tag), self.last_conjunction)
self.subordinate_strings[word] = " ".join(self.conjunction_parts)
self.conjunction_parts = []
self.conjunctions.append(conjunction)
did_something = True
self.track(conjunction[0], "subordinate", self.subtype)
if tag == "PRP":
if self.subtype and (self.subtype[0] == "self" or self.subtype[1] == "self"):
context = word
if self.subtype[0] == "self":
context = self.problem.brain.self_reflexive(context, True)
self.last_context = context
self.contexts.append(context)
did_something = True
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype)
else:
self.main_context = context
self.track(context, "context", self.subtype)
self.context_subtypes[context] = self.subtype
else:
c = self.resolve_context(self.subtype)
if c is None:
c = self.resolve_context(self.subtype, compx=True)
if c is not None:
# If we're setting a relative quantity and the contexts are
# the same we're not actually setting a relative quantity
# we are simply indicating a mathematical operand
if self.is_relative_quantity and c[0] == self.last_context:
c2 = self.resolve_context(self.subtype, compx=True, failTest=True)
if c2 is not None and c2[0] != self.last_context:
c = c2
else:
self.is_relative_quantity = False
did_something = True
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = c[0]
self.track(c[0], "comparator_context", self.subtype, conv=True)
else:
self.main_context = c[0]
self.track(c[0], "context", self.subtype)
self.context_subtypes[c[0]] = self.subtype
# Unset hanging conjunctions when we set a context
self.last_conjunction = None
else:
if self.last_tag in ["IN"]:
# remove the "of" in "of them"
self.parsed.pop()
# Assume this is a unit
if not p.involves_acting or self.last_unit:
unit, uidx = self.fix_unit(self.last_unit)
self.last_unit = unit
self.last_unit_tag = self.last_unit_tag
self.last_unit_index = len(self.parsed)
self.units.append(unit)
p.unit_subtypes[unit] = self.subtype
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
self.track(unit, "unit", self.subtype, uidx)
if tag == "PRP$":
if self.last_conjunction is not None:
if self.is_relative_quantity and not self.comparator_context and self.main_context:
c = self.resolve_context(self.subtype)
ms = self.context_subtypes[self.main_context]
if (not c or self.main_context != c[0]) and (not ms or self.subtype[0] != ms[0]):
context = word
if self.subtype[0] == "self":
context = self.problem.brain.self_reflexive(context, True)
did_something = True
self.comparator_context = context
self.track(word, "comparator_context", self.subtype, conv=True)
self.last_conjunction = None
self.conjunction_parts = []
self.context_subtypes[word] = self.subtype
if not did_something:
did_something = True
term = word
st = self.subtype
conval = self.resolve_context(self.subtype)
if conval:
term, st = conval
term = ownerize(term)
if self.subtype[0] == "self":
term = self.problem.brain.self_reflexive(term, True)
self.partial_context = term
self.partial_subtype = st
self.last_partial_context = self.track(term, "partial_context", st)
if tag in ["NNP", "NNPS"]:
if self.last_tag in ["NNP", "NNPS"]:
# Last was partial context; the "Mrs." in "Mrs. Jones"
self.last_context = None
lc = self.contexts.pop()
self.parsed.pop()
if self.main_context == lc:
self.main_context = None
context = " ".join([lc, word])
if self.is_relative_quantity and self.comparator_context == lc:
self.comparator_context = context
if self.subtype is not None and self.subtype[1] == "ambiguous":
old = self.get_subtype(self.last_word, self.last_tag)
self.subtype = (self.subtype[0], old[1])
else:
context = word
if self.subtype[0] == "self":
context = self.problem.brain.self_reflexive(context, True)
self.last_context = context
self.contexts.append(context)
did_something = True
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype)
else:
if not self.main_context:
self.main_context = context
self.track(context, "context", self.subtype)
self.context_subtypes[context] = self.subtype
if tag[:2] == "VB":
if tag == "VB" and self.question:
if self.last_verb_tag is not None:
# VB*...VB indicates a question not an operation
self.last_operator = None
q_start = self.raw_operators.pop()
st = self.get_subtype(self.last_verb, self.last_verb_tag)
self.track(q_start, "q_start", st, self.last_verb)
self.framing_question = False
self.did_frame_question = True
did_something = True
self.track(word, "q_stop", self.subtype)
elif self.framing_question:
did_something = True
self.framing_question = False
self.did_frame_question = True
self.track(word, "q_stop", self.subtype)
op = p.brain.operator(word, self.sentence_text)
if op == "ex":
self.is_exchange = True
elif tag != "VBG":
if self.framing_question:
did_something = True
self.framing_question = False
self.did_frame_question = True
self.track(word, "q_stop", self.subtype)
else:
if not self.question and self.is_question and not self.did_frame_question:
self.verb_question = True
self.question = True
self.phrasing_question = True
self.framing_question = True
did_something = True
self.track(word, "asking", self.subtype)
elif self.question and not self.did_frame_question:
# We are asking a question and likely don't
# need to invoke all the operator logic; this is
# likely a "start the question" verb
if not self.framing_question:
did_something = True
self.framing_question = True
self.track(word, "q_start", self.subtype)
else:
op = p.brain.operator(word, self.sentence_text)
if op not in [None, False]: # Retagged
if op == "re":
self.is_about_requirements = True
if op == "ex":
self.is_exchange = True
self.last_verb_tag = tag
self.last_verb = len(self.parsed)
self.last_operator = word
self.raw_operators.append(word)
did_something = True
self.operators.append(op)
self.operator[word] = op
self.subtype = self.get_subtype(word, tag)
self.track(word, "operator", self.subtype)
else:
did_something = True
process(word, tag)
else:
did_something = True
gtype = p.brain.gerund(word, self.sentence_text)
if gtype == "acting":
self.acting = False
self.actions.append(word)
if self.main_context:
p.context_actions[self.main_context] = (word, gtype, self.subtype)
self.track(word, gtype, self.subtype)
if tag in ["JJR", "COMP"]:
did_something = True
adj = p.brain.relative(word, self.sentence_text)
self.is_relative_quantity = adj != "noise"
self.track(word, adj, self.subtype)
if tag in ["IN", "TO"]:
self.last_conjunction = word
self.last_conjunction_tag = tag
self.conjunction_parts.append(word)
did_something = True
self.track(word, "conjunction", self.subtype)
if tag in ["POS"]:
did_something = True
self.track(word, "possessive", self.subtype)
if tag in [".", ","]:
did_something = True
self.track(word, "punctuation", self.subtype)
if tag == "EX":
did_something = True
self.track(word, "exestential", self.subtype)
self.problem.exestential = True
if tag in ["CD", "LS"]: # A cardinal number
if self.last_tag == "DT":
if self.last_determiner == "constant":
# should safely be able to ignore detemriner constant
self.parsed.pop()
self.last_determiner = None
if self.last_conjunction is not None:
self.conjunction_parts.pop()
did_something = True
if is_number(word):
self.track(word, "constant", self.subtype)
else:
self.track(p.brain.number(word, self.sentence_text), "constant", self.subtype)
if tag == "DT": # A determiner (the)
did_something = True
dtype = p.brain.determiner(word, self.sentence_text)
self.last_determiner = dtype
if dtype == "variable":
vtype = p.brain.variable(word, self.sentence_text)
if vtype == "dynamic_variable":
do_dyn_var_trk = True
if self.last_tag == "IN":
do_dyn_var_trk = False
process(word, "RB")
return
if do_dyn_var_trk:
self.track(word, vtype, self.subtype)
else:
self.track(vtype, "variable_relationship", self.subtype)
self.last_variable = word
self.last_variable_type = vtype
elif dtype == "constant":
self.track(p.brain.number(word, self.sentence_text), dtype, self.subtype)
elif dtype == "noise":
if self.is_question and self.phrasing_question:
self.parsed.pop()
did_something = True
self.track(" ".join([self.last_word, word]), "asking", self.subtype)
else:
self.track(word, dtype, self.subtype)
if tag == "RB": # An adverb, probably a subordinate?
if self.verb_question and self.framing_question:
tag = "JJ" # This is almost certainly not an adverb at this point
else:
if self.last_conjunction is not None:
conjunction = ((word, tag), self.last_conjunction)
self.last_conjunction = None
self.subordinate_strings[word] = " ".join(self.conjunction_parts)
self.conjunction_parts = []
else:
conjunction = ((word, tag), None)
self.subordinate_strings[word] = word
self.conjunctions.append(conjunction)
did_something = True
self.track(conjunction[0], "subordinate", self.subtype)
# Anything about phrasing must come before the wh-determiner block
if tag == "JJ": # Adjective
if self.phrasing_question:
self.parsed.pop()
did_something = True
self.track(" ".join([self.last_word, word]), "asking", self.subtype)
elif self.last_tag in ["JJ"]:
self.used_unit_adjectives.append(word)
did_something = True
self.last_adjective = " ".join([self.last_adjective, word])
self.track(self.last_adjective, "adjective", self.subtype, self.last_adj_index)
elif self.last_tag in ["IN"]:
did_something = True
unit = " ".join([self.last_unit, self.last_word, word])
self.problem.descriptive_units.append(unit)
self.problem.refined_units[self.last_unit] = unit
self.units.pop()
del p.unit_subtypes[self.last_unit]
self.parsed.pop()
self.parsed.pop()
self.last_unit = unit
self.last_unit_tag = tag
self.last_unit_index = len(self.parsed)
unit, uidx = self.fix_unit(unit)
self.units.append(unit)
p.unit_subtypes[unit] = self.subtype
self.unit_idx[unit] = self.last_unit_index
if self.new_units_as_context:
p.units_acting_as_context[unit] = True
self.track(unit, "unit", self.subtype, uidx)
else:
self.used_unit_adjectives.append(word)
did_something = True
self.last_adjective = word
self.last_adj_index = len(self.parsed)
self.track(word, "adjective", self.subtype)
if tag == "PIP":
did_something = True
self.track(word, "pre_ind_plu", self.subtype)
if not self.problem.exestential:
self.acting = True
self.problem.involves_acting = True
if self.last_unit is not None and self.last_context is None:
# If we have a present indicitive plural with no context
# it is likely that we're dealing with a context which has
# been misinterpreted as a unit, so switch that
# mistake here.
p.units_acting_as_context[self.last_unit] = True
# Remove the last unit and make it a context
context = self.units.pop()
del p.unit_subtypes[context]
self.last_context = context
self.contexts.append(context)
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", self.subtype, self.last_unit_index)
else:
self.main_context = context
self.track(context, "context", self.subtype, self.last_unit_index)
self.context_subtypes[context] = self.subtype
# @TODO: This needs a much better tracking system
self.last_unit = None
self.last_unit_tag = None
self.last_unit_index = -1
elif not self.phrasing_question:
self.new_units_as_context = True
if tag in ["WRB", "WDT", "WP", "WP$"]: # Wh- determiner
self.question = True
self.phrasing_question = True
did_something = True
self.track(word, "asking", self.subtype)
elif self.phrasing_question and not (tag[:2] == "VB" and self.is_question):
self.phrasing_question = False
if tag == "CC":
did_something = True
if self.verb_question and self.framing_question:
self.track(word, "eval_option_sep")
else:
self.track(word, "coordinating_conjunction")
if self.subtype is not None:
did_something = True
if not did_something:
#rint "Answer Unknown", word, tag
p.brain.unknown(word, tag, self.subtype, self.sentence_text)
process(word, tag)
self.last_word = word
self.last_tag = tag
self.last_subtype = self.subtype
# Look before you leap
for s_tag in self.sentence:
word, tag = s_tag
if tag == "." and word == "?":
self.is_question = True
for s_tag in self.sentence:
process(*s_tag)
self.index += 1
if self.partial_context:
context = self.partial_context
idx = self.last_partial_context
stype = self.partial_subtype
self.partial_context = None
self.last_partial_context = None
self.partial_subtype = None
if self.is_relative_quantity and not self.comparator_context and self.main_context:
self.comparator_context = context
self.track(context, "comparator_context", stype, idx, True)
last_context = None
last_possessive = None
is_possessive = False
was_possessive = False
did_swap_context_for_unit = False
reparsed = []
# Walk this to combine context with their unit possessives
for part in self.parsed:
word, role, subtype = part
if role == "context":
if word in self.problem.units or (word != self.main_context and len(self.units) == 0):
self.contexts.remove(word)
role = "unit"
self.units.append(word)
did_swap_context_for_unit = True
else:
last_context = part
elif last_context is not None and role == "possessive":
last_possessive = word
is_possessive = True
elif role == "unit":
if is_possessive:
was_possessive = True
cidx = self.contexts.index(last_context[0])
self.units.remove(word)
word = " ".join([last_context[0] + last_possessive, word])
subtype = last_context[2]
role = "context"
self.contexts[cidx] = word
last_context = None
last_possessive = None
is_possessive = False
reparsed.pop()
reparsed.pop()
elif did_swap_context_for_unit:
did_swap_context_for_unit = False
self.units.remove(word)
role = "context"
self.contexts.append(word)
reparsed.append((word, role, subtype))
#rint self.sentence_text, "::::", self.main_context
self.parsed = reparsed
# Make all the inferred items unique
self.raw_operators = uniq(self.raw_operators)
self.contexts = uniq(self.contexts)
self.units = uniq(self.units)
self.actions = uniq(self.actions)
self.operators = uniq(self.operators)
text = self.sentence_text
nunits = []
for unit in self.units:
acting_as_context = False
if unit in p.units_acting_as_context:
acting_as_context = p.units_acting_as_context[unit]
# Units can be complicated by adjectives which break up the unit
# into important subdivisions. However, these subdivisions may
# also be simple detractors and not relevant to the actual string
# match for the unit. As such, we split by any spaces and then
# compile all possible formulations of the unit
if not unit in self.problem.descriptive_units and " " in unit:
parts = unit.split(" ")
# The item is the last part of the unit
item = parts.pop()
for part in parts:
nu = " ".join([part, item])
nunits.append(nu)
# if acting_as_context:
# p.units_acting_as_context[nu] = True
self.units = uniq(self.units + nunits)
for unit in self.units:
if unit in self.problem.refined_units:
uxd = self.parsed[self.unit_idx[unit]]
self.parsed[self.unit_idx[unit]] = (self.problem.refined_units[unit], uxd[1], uxd[2])
# Resolve the subordinates
for o in self.conjunctions:
subordinate, conjunction = o
outp = p.brain.subordinate(subordinate, text)
self.subordinate_lookup[subordinate[0]] = outp
if outp == "object":
continue
self.subordinates.append((subordinate[0], outp))
if len(self.subordinate_strings[subordinate[0]]) == 0 and outp != "place_noun":
self.subordinate_strings[subordinate[0]] = ANSWER_SUBORDINATE[outp]
if outp == "context_grouping":
csub = self.get_subtype(subordinate[0], subordinate[1])
if csub:
c = self.resolve_context(csub)
if c:
self.problem.subordinate_adaptive_contexts.append(c[0])
self.subordinates = uniq(self.subordinates)
self.problem.running_units += self.units
self.problem.units += self.units
self.problem.units = uniq(self.problem.units)
if len(self.subordinates) > 0:
if not self.main_context in self.problem.context_subordinates:
self.problem.context_subordinates[self.main_context] = (self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup)
elif self.main_context in self.problem.context_subordinates:
# This is an inferred context
self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup = self.problem.context_subordinates[self.main_context]
for sub in self.subordinates:
subord, subt = sub
self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord])
if not self.main_context and self.operator:
if self.problem.last_contexts["last"]:
context, subtype = self.problem.last_contexts["last"]
self.main_context = context
self.track(context, "context_inferred", subtype)
#rint self.subordinates, self.main_context, self.problem.context_subordinates
has_unit_proxy = False
if self.main_context in self.problem.context_subordinates:
if len(self.subordinates) == 0:
# This is an inferred context
self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup = self.problem.context_subordinates[self.main_context]
for sub in self.subordinates:
subord, subt = sub
self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord])
elif len(self.subordinates) > 0:
subs, sub_strs, sub_subs, sub_look = self.problem.context_subordinates[self.main_context]
handeled_subtypes = []
for s in self.subordinate_lookup:
st = self.subordinate_lookup[s]
if st is not None:
if st[0:4] == "time":
st = st[0:4]
handeled_subtypes.append(st)
has_unit_proxy = "unit_grouping" in handeled_subtypes
for sub in subs:
subord, subt = sub
st = subt
if st is not None:
if st == "costpay":
continue
if st[0:4] == "time":
st = st[0:4]
if st in handeled_subtypes and st == "time":
continue
if subord in self.subordinate_lookup:
continue
self.subordinate_subtypes[subord] = sub_subs[subord]
self.subordinate_lookup[subord] = sub_look[subord]
if subt != "refiner":
self.subordinates.append(sub)
self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord])
if self.main_context in self.problem.context_actions and len(self.actions) == 0:
word, gtype, subtype = self.problem.context_actions[self.main_context]
self.track(word, gtype + "_inferred", subtype)
if len(self.units) == 0 and len(self.problem.running_units) > 0 and not has_unit_proxy:
iu = self.problem.running_units[-1]
self.track(iu, "unit_inferred", self.problem.unit_subtypes[iu])
self.problem.contexts += self.contexts
self.problem.contexts = uniq(self.problem.contexts)
