def digest(self): if self.sentences is not None: return # Digest the problem into sentences tokenizer = data.load("tokenizers/punkt/english.pickle") self.sentences = tokenizer.tokenize(self.text.strip()) # Digest each sentence into words and part-of-speech tags if self.sentence_tags is None: sentence_tags = [] all_tags = [] all_words = [] for s in self.sentences: all_words.append(s) tags = pos_tag(word_tokenize(s)) sentence_tags.append(tags) for t in tags: l = len(t[0]) if not self.longest_word or self.longest_word < l: self.longest_word = l all_tags.append(t[1]) self.sentence_tags = sentence_tags self.all_tags = uniq(all_tags) self.all_words = uniq(all_words)
def condense(self, itemList): if not "word_forms" in self.raw: self.raw["word_forms"] = {} if not "single" in self.raw["word_forms"]: self.raw["word_forms"]["single"] = {} if not "plural" in self.raw["word_forms"]: self.raw["word_forms"]["plural"] = {} if not "solved" in self.raw["word_forms"]: self.raw["word_forms"]["solved"] = [] outp = [] if len(itemList) < 2: outp = itemList else: comps = [] + itemList for item in itemList: for comp in comps: if item == comp: continue else: cond = self.condensor(item, comp) if cond is None: outp.append(item) else: outp.append(cond) outp = uniq(outp) return outp
def run_sim(self): """ Run model simulation! """ project_day = 0 # figure out the number of years for simulation and the number of # days in each year years = uniq(self.met_data["year"]) days_in_year = [self.met_data["year"].count(yr) for yr in years] # =============== # # YEAR LOOP # # =============== # for i, yr in enumerate(years): self.day_output = [] # empty daily storage list for outputs daylen = calculate_daylength(days_in_year[i], self.params.latitude) if self.control.deciduous_model: self.P.calculate_phenology_flows(daylen, self.met_data, days_in_year[i], project_day) self.zero_stuff() # =============== # # DAY LOOP # # =============== # for doy in xrange(days_in_year[i]): # litterfall rate: C and N fluxes (fdecay, rdecay) = self.lf.calculate_litter(doy) # co2 assimilation, N uptake and loss self.pg.calc_day_growth(project_day, fdecay, rdecay, daylen[doy], doy, float(days_in_year[i]), i) # soil C & N model fluxes self.cs.calculate_csoil_flows(project_day) self.ns.calculate_nsoil_flows() # calculate C:N ratios and increment annual flux sums self.day_end_calculations(project_day, days_in_year[i]) #print self.state.shoot, self.state.lai #print self.fluxes.gpp * 100, self.state.lai # =============== # # END OF DAY # # =============== # self.save_daily_outputs(yr, doy+1) project_day += 1 # =============== # # END OF YEAR # # =============== # if self.control.deciduous_model: self.pg.allocate_stored_c_and_n(init=False) if self.control.print_options == "DAILY" and self.spin_up == False: self.print_output_file() # close output files if self.control.print_options == "END" and self.spin_up == False: self.print_output_file() self.pr.clean_up()
(control, params, state, files, fluxes, met_data, print_opts) = initialise_model_data(fname, met_header, DUMP=False) control.ps_pathway = "C4" if control.ps_pathway == "C3": M = MateC3(control, params, state, fluxes, met_data) else: M = MateC4(control, params, state, fluxes, met_data) # flai = "/Users/mdekauwe/research/NCEAS_face/GDAY_ornl_simulation/experiments/silvias_LAI.txt" # lai_data = np.loadtxt(flai) project_day = 0 # figure out the number of years for simulation and the number of # days in each year years = uniq(met_data["year"]) # years = years[:-1] # dump last year as missing "site" LAI days_in_year = [met_data["year"].count(yr) for yr in years] for i, yr in enumerate(years): daylen = calculate_daylength(days_in_year[i], params.latitude) for doy in xrange(days_in_year[i]): state.wtfac_root = 1.0 # state.lai = lai_data[project_day] state.lai = 2.0 if state.lai > 0.0: state.shootnc = 0.03 # state.shootn / state.shoot state.ncontent = state.shootnc * params.cfracts / params.sla * const.KG_AS_G else: state.ncontent = 0.0
def __init__(self, fname=None, DUMP=False, spin_up=False, met_header=4): """ Set up model * Read meterological forcing file * Read user config file and adjust the model parameters, control or initial state attributes that are used within the code. * Setup all class instances...perhaps this isn't the tidyest place for this? * Initialise things, zero stores etc. Parameters: ---------- fname : string filename of model parameters, including path chk_cmd_line : logical parse the cmd line? DUMP : logical dump a the default parameters to a file met_header : int row number of met file header with variable names Returns: ------- Nothing Controlling class of the model, runs things. """ self.day_output = [] # store daily outputs # initialise model structures and read met data (self.control, self.params, self.state, self.files, self.fluxes, self.met_data, self.print_opts) = initialise_model_data(fname, met_header, DUMP=DUMP) # params are defined in per year, needs to be per day # Important this is done here as rate constants elsewhere in the code # are assumed to be in units of days not years! self.correct_rate_constants(output=False) # class instances self.cs = CarbonSoilFlows(self.control, self.params, self.state, self.fluxes, self.met_data) self.ns = NitrogenSoilFlows(self.control, self.params, self.state, self.fluxes, self.met_data) self.lf = Litter(self.control, self.params, self.state, self.fluxes) self.pg = PlantGrowth(self.control, self.params, self.state, self.fluxes, self.met_data) self.cb = CheckBalance(self.control, self.params, self.state, self.fluxes, self.met_data) if self.control.deciduous_model: self.pg.calc_carbon_allocation_fracs(0.0, 0, 0) #comment this!! self.pg.allocate_stored_c_and_n(init=True) self.P = Phenology( self.fluxes, self.state, self.control, self.params.previous_ncd, store_transfer_len=self.params.store_transfer_len) self.pr = PrintOutput(self.params, self.state, self.fluxes, self.control, self.files, self.print_opts) # build list of variables to print (self.print_state, self.print_fluxes) = self.pr.get_vars_to_print() # print model defaults if DUMP == True: self.pr.save_default_parameters() sys.exit(0) # calculate initial stuff, e.g. C:N ratios and zero annual flux sums self.day_end_calculations(0, INIT=True) self.state.pawater_root = self.params.wcapac_root self.state.pawater_topsoil = self.params.wcapac_topsoil self.spin_up = spin_up self.state.sla = self.params.slainit # Specific leaf area (m2/kg DW) self.state.lai = (self.params.slainit * const.M2_AS_HA / const.KG_AS_TONNES / self.params.cfracts * self.state.shoot) # figure out the number of years for simulation and the number of # days in each year self.years = uniq(self.met_data["year"]) self.days_in_year = [self.met_data["year"].count(yr) \ for yr in self.years] if self.control.water_stress == False: sys.stderr.write("**** You have turned off the drought stress") sys.stderr.write(", I assume you're debugging??!\n")
def execute(self): p = self.problem raw_actions, raw_operators, raw_subordinates = [], [], [] index = 0 for s_tags in p.sentence_tags: s = SentenceParser(s_tags, p, p.sentences[index]) self.sentences.append(s) self.track_longer(s.longest_phrase) if s.is_about_requirements: self.is_requirement_problem = True raw_operators += s.operators raw_actions += s.actions raw_subordinates += s.subordinates self.contexts += s.contexts self.units += s.units if len(s.subordinate_strings) > 0: sd = dict(s.subordinate_strings) if len(self.subordinate_strings) > 0: sd.update(self.subordinate_strings) self.subordinate_strings = sd if s.question: self.queries.append(index) index += 1 # Ensure uniqueness in our data raw_operators = uniq(raw_operators) raw_actions = uniq(raw_actions) raw_subordinates = uniq(raw_subordinates) self.contexts = uniq(self.contexts) self.units = uniq(self.units) self.condense() # Set our phrasings based on the contexts in play if len(self.contexts) > 1: op_key = OPERATOR_STR["multiple_contexts"] elif len(self.contexts) > 0: op_key = OPERATOR_STR["single_context"] else: op_key = OPERATOR_STR["no_context"] for sub in raw_subordinates: word, subtype = sub if subtype is not None and subtype[0:4] != "time": if "grouping" not in subtype: self.subordinates.append(self.subordinate_strings[word]) self.subordinates = uniq(self.subordinates) format_operators = True for act in raw_actions: self.actions.append(act) self.actions = uniq(self.actions) pre_ops = False if len(self.units) == 0 and len(self.actions) > 0: pre_ops = True # We have special phrasing which is mucked up by the operators # if we're only dealing with op_key = OPERATOR_STR["context_actions_nounit"] for op in raw_operators: if op != "eq" or len(raw_operators) == 1: try: self.operators.append(op_key[op]) except: pass self.operators = uniq(self.operators) if pre_ops: self.preops = self.operators self.operators = []
def __init__(self, fname=None, DUMP=False, spin_up=False, met_header=4): """ Set up model * Read meterological forcing file * Read user config file and adjust the model parameters, control or initial state attributes that are used within the code. * Setup all class instances...perhaps this isn't the tidyest place for this? * Initialise things, zero stores etc. Parameters: ---------- fname : string filename of model parameters, including path chk_cmd_line : logical parse the cmd line? DUMP : logical dump a the default parameters to a file met_header : in row number of met file header with variable name Returns: ------- Nothing Controlling class of the model, runs things. """ self.day_output = [] # store daily output # initialise model structures and read met data (self.control, self.params, self.state, self.files, self.fluxes, self.met_data, self.print_opts) = initialise_model_data(fname, met_header, DUMP=DUMP) # params are defined in per year, needs to be per day # Important this is done here as rate constants elsewhere in the code # are assumed to be in units of days not years! self.correct_rate_constants(output=False) # class instance self.cs = CarbonSoilFlows(self.control, self.params, self.state, self.fluxes, self.met_data) self.ns = NitrogenSoilFlows(self.control, self.params, self.state, self.fluxes, self.met_data) self.lf = Litter(self.control, self.params, self.state, self.fluxes) self.pg = PlantGrowth(self.control, self.params, self.state, self.fluxes, self.met_data) self.cb = CheckBalance(self.control, self.params, self.state, self.fluxes, self.met_data) self.db = Disturbance(self.control, self.params, self.state, self.fluxes, self.met_data) if self.control.deciduous_model: if self.state.max_lai is None: self.state.max_lai = 0.01 # initialise to something really low self.state.max_shoot = 0.01 # initialise to something really low # Are we reading in last years average growing season? if (float_eq(self.state.avg_alleaf, 0.0) and float_eq(self.state.avg_alstem, 0.0) and float_eq(self.state.avg_albranch, 0.0) and float_eq(self.state.avg_alleaf, 0.0) and float_eq(self.state.avg_alroot, 0.0) and float_eq(self.state.avg_alcroot, 0.0)): self.pg.calc_carbon_allocation_fracs(0.0) #comment this!! else: self.fluxes.alleaf = self.state.avg_alleaf self.fluxes.alstem = self.state.avg_alstem self.fluxes.albranch = self.state.avg_albranch self.fluxes.alroot = self.state.avg_alroot self.fluxes.alcroot = self.state.avg_alcroot self.pg.allocate_stored_c_and_n(init=True) #self.pg.enforce_sensible_nstore() self.P = Phenology(self.fluxes, self.state, self.control, self.params.previous_ncd, store_transfer_len=self.params.store_transfer_len) self.pr = PrintOutput(self.params, self.state, self.fluxes, self.control, self.files, self.print_opts) # build list of variables to prin (self.print_state, self.print_fluxes) = self.pr.get_vars_to_print() # print model defaul if DUMP == True: self.pr.save_default_parameters() sys.exit(0) self.dead = False # johnny 5 is alive # calculate initial stuff, e.g. C:N ratios and zero annual flux sum self.day_end_calculations(INIT=True) self.state.pawater_root = self.params.wcapac_root self.state.pawater_topsoil = self.params.wcapac_topsoil self.spin_up = spin_up self.state.lai = max(0.01, (self.params.sla * const.M2_AS_HA / const.KG_AS_TONNES / self.params.cfracts * self.state.shoot)) # figure out the number of years for simulation and the number of # days in each year self.years = uniq(self.met_data["year"]) self.days_in_year = [self.met_data["year"].count(yr) for yr in self.years] if self.control.water_stress == False: sys.stderr.write("**** You have turned off the drought stress") sys.stderr.write(", I assume you're debugging??!\n")
else: M = MateC4(control, params, state, fluxes, met_data) #flai = "/Users/mdekauwe/research/NCEAS_face/GDAY_ornl_simulation/experiments/silvias_LAI.txt" #lai_data = np.loadtxt(flai) # Specific LAI (m2 onesided/kg DW) state.sla = params.slainit project_day = 0 # figure out the number of years for simulation and the number of # days in each year years = uniq(met_data["year"]) #years = years[:-1] # dump last year as missing "site" LAI days_in_year = [met_data["year"].count(yr) for yr in years] for i, yr in enumerate(years): daylen = calculate_daylength(days_in_year[i], params.latitude) for doy in xrange(days_in_year[i]): state.wtfac_root = 1.0 #state.lai = lai_data[project_day] state.lai = 2.0 if state.lai > 0.0: state.shootnc = 0.03 #state.shootn / state.shoot state.ncontent = (state.shootnc * params.cfracts / state.sla * const.KG_AS_G) else:
def execute(self): p = self.problem def process(word, tag): did_something = False self.subtype = None if self.index == 0 and tag not in ["NNP", "NNPS"]: # Fix capitalization for non proper nouns word = word.lower() word, tag = p.brain.retag(word, tag) if tag == "!!!": print "Missing retag converter in the brain" exit(1) if tag in ["NNP", "NNPS"]: word = word.capitalize() if self.last_conjunction is not None: self.conjunction_parts.append(word) # Fix the parser as we go on brain retags def do_track(subtype, tag): if subtype[0] is None and subtype[1] is None: subtype = None tag = p.brain.retag(word, tag) process(word, tag) return (False, subtype, tag) return (True, subtype, tag) if tag in ["NN", "NNS"]: if self.last_tag in ["PRP$"]: # Detects Jane's friends; probably context context = " ".join([self.partial_context, word]) self.parsed.pop() self.last_context = context self.contexts.append(context) did_something = True self.subtype = self.get_subtype(word, tag) self.partial_context = None self.last_partial_context = None self.partial_subtype = None track, self.subtype, tag = do_track(self.subtype, tag) if track: if self.problem.involves_acting: conjunction = ((word, tag), self.last_conjunction) self.subordinate_strings[word] = " ".join(self.conjunction_parts) self.conjunction_parts = [] self.conjunctions.append(conjunction) did_something = True self.track(conjunction[0], "subordinate", self.subtype) else: if self.subtype[0] == "plural" and self.subtype[1] == "mixed": # PRP$ will always have a noun like last_subtype last_plural, last_gender = self.last_subtype # Last Plural Context lpc = p.last_contexts["plurality"][last_plural] # Last Gender Context lgc = p.last_contexts["gender"][last_gender] self.last_context = None if lgc == lpc: self.last_context = lgc elif lgc is not None: self.last_context = lgc elif lpc is not None: self.last_context = lpc if self.last_context is not None: # Last context string inc = p.brain.inclusive(word, "'{0}' ({1}) in '{2}'".format(self.last_word, self.last_context[0], context)) # Add the context by the word for easier lookup # this will capture 'her friends' and define # the concept of inclusiveness for friends then # later auto-apply this to the concept of say, # 'his friends' p.brain.add("inclusive", context, inc) # Assume the first context after a comparison is the # comparator context if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype, conv=True) else: self.main_context = context self.track(context, "context", self.subtype) self.context_subtypes[context] = self.subtype else: do_reg_unit = True if self.last_conjunction is not None: do_reg_unit = False do_conj_track = False if tag == "NN" and self.last_tag in ["IN"]: self.subtype = self.get_subtype(word, tag) track, self.subtype, tag = do_track(self.subtype, tag) did_something = True if track: unit = " ".join([self.last_unit, self.last_word, word]) self.problem.descriptive_units.append(unit) self.problem.refined_units[self.last_unit] = unit self.units.pop() self.parsed.pop() self.parsed.pop() self.last_unit = unit self.last_unit_tag = tag self.last_unit_index = len(self.parsed) unit, uidx = self.fix_unit(unit) self.units.append(unit) self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True self.track(unit, "unit", self.subtype, uidx) elif tag in ["NN", "NNS"] and self.last_conjunction_tag in ["IN"]: self.subtype = self.get_subtype(word, tag) track, self.subtype, tag = do_track(self.subtype, tag) did_something = True if track: if self.last_variable is not None: unit = word self.problem.descriptive_units.append(unit) self.problem.refined_units[self.last_unit] = unit self.last_unit = unit self.last_unit_tag = tag self.last_unit_index = len(self.parsed) unit, uidx = self.fix_unit(unit) self.units.append(unit) self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True c_unit = " ".join([word, "owned by", p.previous_contexts["last"][0]]) self.track((c_unit, word, p.previous_contexts["last"][0], p.last_contexts["last"][0]), "context_unit", self.subtype, uidx) else: do_conj_track = True else: do_conj_track = True do_unset_conj = True if do_conj_track: unit = word do_std = True if tag in ["NN", "NNS"]: if self.last_tag in ["NN", "NNS"]: if self.last_unit is None and self.last_context is not None: self.last_context = None lc = self.contexts.pop() unit = " ".join([lc, word]) else: unit = " ".join([self.last_unit, word]) self.units.pop() self.used_unit_adjectives.append(self.last_word) self.parsed.pop() ptest = self.parsed.pop() self.parsed.append(ptest) if ptest[1] == "constant" and self.last_tag != "DT" and self.problem.exestential: unit, uidx = self.fix_unit(unit) self.last_unit = unit self.last_unit_tag = tag self.last_unit_index = len(self.parsed) self.units.append(unit) self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True self.track(unit, "unit", self.subtype, uidx) do_unset_conj = False do_reg_unit = False do_std = False if do_std: if word in self.used_unit_adjectives: do_unset_conj = False do_reg_unit = True else: self.used_unit_adjectives.append(word) conjunction = ((unit, tag), self.last_conjunction) self.subordinate_strings[unit] = " ".join(self.conjunction_parts) self.conjunction_parts = [] self.conjunctions.append(conjunction) did_something = True self.track(conjunction[0], "subordinate", self.subtype) if do_unset_conj: self.last_conjunction = None if do_reg_unit: self.subtype = self.get_subtype(word, tag) track, self.subtype, tag = do_track(self.subtype, tag) did_something = True if track: #rint "here for", word, tag, self.last_tag, self.last_unit, self.last_word unit = word do_std_unit = True if tag in ["NN", "NNS"] and self.last_tag in ["NN", "NNS"]: # rint self.last_word, self.last_tag # rint word, tag # rint self.parsed # rint "----" ptest = self.parsed.pop() if ptest[1] == "subordinate" and self.last_unit is None: cst = self.conjunctions.pop() subostr = " ".join([self.subordinate_strings[ptest[0][0]], word]) del self.subordinate_strings[ptest[0][0]] unit = " ".join([ptest[0][0], word]) self.subordinate_strings[unit] = subostr self.used_unit_adjectives.append(ptest[0][0]) conjunction = ((unit, tag), cst[1]) self.conjunctions.append(conjunction) self.track(conjunction[0], "subordinate", self.subtype) do_std_unit = False else: if self.last_unit is None and self.last_context is not None: self.last_context = None lc = self.contexts.pop() unit = " ".join([lc, word]) else: unit = " ".join([self.last_unit, word]) self.units.pop() self.used_unit_adjectives.append(self.last_word) if do_std_unit: unit, uidx = self.fix_unit(unit) if not unit in p.units_acting_as_context or not p.units_acting_as_context[unit]: if self.last_unit and self.framing_question and len(self.operators) == 0: context = unit if self.subtype[0] == "self": context = self.problem.brain.self_reflexive(context, True) # @TODO: This needs to be a subroutine or something self.last_context = context self.contexts.append(context) if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype) else: self.main_context = context self.track(context, "context", self.subtype) self.context_subtypes[context] = self.subtype else: if len(self.operators) > 0 and self.last_unit and not self.main_context: if self.last_unit != unit: context = self.units.pop() self.last_context = context self.contexts.append(context) if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype, self.last_unit_index) else: self.main_context = context self.track(context, "context", self.subtype, self.last_unit_index) self.context_subtypes[context] = self.subtype self.last_unit = unit self.last_unit_tag = tag self.last_unit_index = len(self.parsed) self.units.append(unit) self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True self.track(unit, "unit", self.subtype, uidx) else: context = unit if self.subtype[0] == "self": context = self.problem.brain.self_reflexive(context, True) # @TODO: This needs to be a subroutine or something self.last_context = context self.contexts.append(context) if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype, uidx) else: self.main_context = context self.track(context, "context", self.subtype, uidx) self.context_subtypes[context] = self.subtype if not did_something and self.subtype == None: self.subtype = self.get_subtype(word, tag) if tag == "SUB": conjunction = ((word, tag), self.last_conjunction) self.subordinate_strings[word] = " ".join(self.conjunction_parts) self.conjunction_parts = [] self.conjunctions.append(conjunction) did_something = True self.track(conjunction[0], "subordinate", self.subtype) if tag == "PRP": if self.subtype and (self.subtype[0] == "self" or self.subtype[1] == "self"): context = word if self.subtype[0] == "self": context = self.problem.brain.self_reflexive(context, True) self.last_context = context self.contexts.append(context) did_something = True if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype) else: self.main_context = context self.track(context, "context", self.subtype) self.context_subtypes[context] = self.subtype else: c = self.resolve_context(self.subtype) if c is None: c = self.resolve_context(self.subtype, compx=True) if c is not None: # If we're setting a relative quantity and the contexts are # the same we're not actually setting a relative quantity # we are simply indicating a mathematical operand if self.is_relative_quantity and c[0] == self.last_context: c2 = self.resolve_context(self.subtype, compx=True, failTest=True) if c2 is not None and c2[0] != self.last_context: c = c2 else: self.is_relative_quantity = False did_something = True if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = c[0] self.track(c[0], "comparator_context", self.subtype, conv=True) else: self.main_context = c[0] self.track(c[0], "context", self.subtype) self.context_subtypes[c[0]] = self.subtype # Unset hanging conjunctions when we set a context self.last_conjunction = None else: if self.last_tag in ["IN"]: # remove the "of" in "of them" self.parsed.pop() # Assume this is a unit if not p.involves_acting or self.last_unit: unit, uidx = self.fix_unit(self.last_unit) self.last_unit = unit self.last_unit_tag = self.last_unit_tag self.last_unit_index = len(self.parsed) self.units.append(unit) self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True self.track(unit, "unit", self.subtype, uidx) if tag == "PRP$": if self.last_conjunction is not None: if self.is_relative_quantity and not self.comparator_context and self.main_context: c = self.resolve_context(self.subtype) ms = self.context_subtypes[self.main_context] if (not c or self.main_context != c[0]) and (not ms or self.subtype[0] != ms[0]): context = word if self.subtype[0] == "self": context = self.problem.brain.self_reflexive(context, True) did_something = True self.comparator_context = context self.track(word, "comparator_context", self.subtype, conv=True) self.last_conjunction = None self.conjunction_parts = [] self.context_subtypes[word] = self.subtype if not did_something: did_something = True term = word st = self.subtype conval = self.resolve_context(self.subtype) if conval: term, st = conval term = ownerize(term) if self.subtype[0] == "self": term = self.problem.brain.self_reflexive(term, True) self.partial_context = term self.partial_subtype = st self.last_partial_context = self.track(term, "partial_context", st) if tag in ["NNP", "NNPS"]: if self.last_tag in ["NNP", "NNPS"]: # Last was partial context; the "Mrs." in "Mrs. Jones" self.last_context = None lc = self.contexts.pop() self.parsed.pop() context = " ".join([self.last_word, word]) if self.is_relative_quantity and self.comparator_context == lc: self.comparator_context = context if self.subtype is not None and self.subtype[1] == "ambiguous": old = self.get_subtype(self.last_word, self.last_tag) self.subtype = (self.subtype[0], old[1]) else: context = word if self.subtype[0] == "self": context = self.problem.brain.self_reflexive(context, True) self.last_context = context self.contexts.append(context) did_something = True if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype) else: self.main_context = context self.track(context, "context", self.subtype) self.context_subtypes[context] = self.subtype if tag[:2] == "VB": if tag == "VB" and self.question: if self.last_verb_tag is not None: # VB*...VB indicates a question not an operation self.last_operator = None q_start = self.raw_operators.pop() st = self.get_subtype(self.last_verb, self.last_verb_tag) self.track(q_start, "q_start", st, self.last_verb) self.framing_question = False self.did_frame_question = True did_something = True self.track(word, "q_stop", self.subtype) elif self.framing_question: did_something = True self.framing_question = False self.did_frame_question = True self.track(word, "q_stop", self.subtype) op = p.brain.operator(word, self.sentence_text) if op == "ex": self.is_exchange = True elif tag != "VBG": if self.framing_question: did_something = True self.framing_question = False self.did_frame_question = True self.track(word, "q_stop", self.subtype) else: if self.question and not self.did_frame_question: # We are asking a question and likely don't # need to invoke all the operator logic; this is # likely a "start the question" verb if not self.framing_question: did_something = True self.framing_question = True self.track(word, "q_start", self.subtype) else: op = p.brain.operator(word, self.sentence_text) if op not in [None, False]: # Retagged if op == "re": self.is_about_requirements = True if op == "ex": self.is_exchange = True self.last_verb_tag = tag self.last_verb = len(self.parsed) self.last_operator = word self.raw_operators.append(word) did_something = True self.operators.append(op) self.operator[word] = op self.subtype = self.get_subtype(word, tag) self.track(word, "operator", self.subtype) else: did_something = True process(word, tag) else: did_something = True gtype = p.brain.gerund(word, self.sentence_text) if gtype == "acting": self.acting = False self.actions.append(word) if self.main_context: p.context_actions[self.main_context] = (word, gtype, self.subtype) self.track(word, gtype, self.subtype) if tag in ["JJR", "COMP"]: did_something = True adj = p.brain.relative(word, self.sentence_text) self.is_relative_quantity = adj != "noise" self.track(word, adj, self.subtype) if tag in ["IN", "TO"]: self.last_conjunction = word self.last_conjunction_tag = tag self.conjunction_parts.append(word) did_something = True self.track(word, "conjunction", self.subtype) if tag in [".", ","]: did_something = True self.track(word, "punctuation", self.subtype) if tag == "EX": did_something = True self.track(word, "exestential", self.subtype) self.problem.exestential = True if tag in ["CD", "LS"]: # A cardinal number if self.last_tag == "DT": if self.last_determiner == "constant": # should safely be able to ignore detemriner constant self.parsed.pop() self.last_determiner = None if self.last_conjunction is not None: self.conjunction_parts.pop() did_something = True self.track(word, "constant", self.subtype) if tag == "DT": # A determiner (the) did_something = True dtype = p.brain.determiner(word, self.sentence_text) self.last_determiner = dtype if dtype == "variable": vtype = p.brain.variable(word, self.sentence_text) self.last_variable = word self.last_variable_type = vtype if vtype == "dynamic_variable": self.track(word, vtype, self.subtype) else: self.track(vtype, "variable_relationship", self.subtype) elif dtype == "constant": self.track(p.brain.number(word, self.sentence_text), dtype, self.subtype) else: self.track(word, dtype, self.subtype) if tag == "RB": # An adverb, probably a subordinate? if self.last_conjunction is not None: conjunction = ((word, tag), self.last_conjunction) self.last_conjunction = None self.subordinate_strings[word] = " ".join(self.conjunction_parts) self.conjunction_parts = [] else: conjunction = ((word, tag), None) self.subordinate_strings[word] = word self.conjunctions.append(conjunction) did_something = True self.track(conjunction[0], "subordinate", self.subtype) # Anything about phrasing must come before the wh-determiner block if tag == "JJ": # Adjective if self.phrasing_question: self.parsed.pop() did_something = True self.track(" ".join([self.last_word, word]), "asking", self.subtype) elif self.last_tag in ["JJ"]: self.used_unit_adjectives.append(word) did_something = True self.last_adjective = " ".join([self.last_adjective, word]) self.track(self.last_adjective, "adjective", self.subtype, self.last_adjective_index) elif self.last_tag in ["IN"]: did_something = True unit = " ".join([self.last_unit, self.last_word, word]) self.problem.descriptive_units.append(unit) self.problem.refined_units[self.last_unit] = unit self.units.pop() self.parsed.pop() self.parsed.pop() self.last_unit = unit self.last_unit_tag = tag self.last_unit_index = len(self.parsed) unit, uidx = self.fix_unit(unit) self.units.append(unit) self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True self.track(unit, "unit", self.subtype, uidx) else: self.used_unit_adjectives.append(word) did_something = True self.last_adjective = word self.last_adj_index = len(self.parsed) self.track(word, "adjective", self.subtype) if tag == "PIP": did_something = True self.track(word, "pre_ind_plu", self.subtype) if not self.problem.exestential: self.acting = True self.problem.involves_acting = True if self.last_unit is not None and self.last_context is None: # If we have a present indicitive plural with no context # it is likely that we're dealing with a context which has # been misinterpreted as a unit, so switch that # mistake here. p.units_acting_as_context[self.last_unit] = True # Remove the last unit and make it a context context = self.units.pop() self.last_context = context self.contexts.append(context) if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype, self.last_unit_index) else: self.main_context = context self.track(context, "context", self.subtype, self.last_unit_index) self.context_subtypes[context] = self.subtype # @TODO: This needs a much better tracking system self.last_unit = None self.last_unit_tag = None self.last_unit_index = -1 elif not self.phrasing_question: self.new_units_as_context = True if tag in ["WRB", "WDT", "WP", "WP$"]: # Wh- determiner self.question = True self.phrasing_question = True did_something = True self.track(word, "asking", self.subtype) elif self.phrasing_question: self.phrasing_question = False if tag == "CC": did_something = True self.track(word, "coordinating_conjunction") if self.subtype is not None: did_something = True if not did_something: #rint "Answer Unknown", word, tag p.brain.unknown(word, tag, self.subtype, self.sentence_text) process(word, tag) self.last_word = word self.last_tag = tag self.last_subtype = self.subtype for s_tag in self.sentence: process(*s_tag) self.index += 1 if self.partial_context: context = self.partial_context idx = self.last_partial_context stype = self.partial_subtype self.partial_context = None self.last_partial_context = None self.partial_subtype = None if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", stype, idx, True) # Make all the inferred items unique self.raw_operators = uniq(self.raw_operators) self.contexts = uniq(self.contexts) self.units = uniq(self.units) self.actions = uniq(self.actions) self.operators = uniq(self.operators) text = self.sentence_text nunits = [] for unit in self.units: acting_as_context = False if unit in p.units_acting_as_context: acting_as_context = p.units_acting_as_context[unit] # Units can be complicated by adjectives which break up the unit # into important subdivisions. However, these subdivisions may # also be simple detractors and not relevant to the actual string # match for the unit. As such, we split by any spaces and then # compile all possible formulations of the unit if not unit in self.problem.descriptive_units and " " in unit: parts = unit.split(" ") # The item is the last part of the unit item = parts.pop() for part in parts: nu = " ".join([part, item]) nunits.append(nu) # if acting_as_context: # p.units_acting_as_context[nu] = True self.units = uniq(self.units + nunits) for unit in self.units: if unit in self.problem.refined_units: uxd = self.parsed[self.unit_idx[unit]] self.parsed[self.unit_idx[unit]] = (self.problem.refined_units[unit], uxd[1], uxd[2]) # Resolve the subordinates for o in self.conjunctions: subordinate, conjunction = o outp = p.brain.subordinate(subordinate, text) self.subordinate_lookup[subordinate[0]] = outp if outp == "object": continue self.subordinates.append((subordinate[0], outp)) if len(self.subordinate_strings[subordinate[0]]) == 0 and outp != "place_noun": self.subordinate_strings[subordinate[0]] = ANSWER_SUBORDINATE[outp] if outp == "context_grouping": csub = self.get_subtype(subordinate[0], subordinate[1]) if csub: c = self.resolve_context(csub) if c: self.problem.subordinate_adaptive_contexts.append(c[0]) self.subordinates = uniq(self.subordinates) self.problem.units += self.units self.problem.units = uniq(self.problem.units) if len(self.subordinates) > 0: if not self.main_context in self.problem.context_subordinates: self.problem.context_subordinates[self.main_context] = (self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup) elif self.main_context in self.problem.context_subordinates: # This is an inferred context self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup = self.problem.context_subordinates[self.main_context] for sub in self.subordinates: subord, subt = sub self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord]) if not self.main_context and self.operator: if self.problem.last_contexts["last"]: context, subtype = self.problem.last_contexts["last"] self.main_context = context self.track(context, "context_inferred", subtype) #rint self.subordinates, self.main_context, self.problem.context_subordinates if self.main_context in self.problem.context_subordinates: if len(self.subordinates) == 0: # This is an inferred context self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup = self.problem.context_subordinates[self.main_context] for sub in self.subordinates: subord, subt = sub self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord]) elif len(self.subordinates) > 0: subs, sub_strs, sub_subs, sub_look = self.problem.context_subordinates[self.main_context] for sub in subs: subord, subt = sub if subord in self.subordinate_lookup: continue self.subordinate_subtypes[subord] = sub_subs[subord] self.subordinate_lookup[subord] = sub_look[subord] if subt != "refiner": self.subordinates.append(sub) self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord]) if self.main_context in self.problem.context_actions and len(self.actions) == 0: word, gtype, subtype = self.problem.context_actions[self.main_context] self.track(word, gtype + "_inferred", subtype)
def execute(self): p = self.problem def process(word, tag): did_something = False self.subtype = None if self.index == 0 and tag not in ["NNP", "NNPS"]: # Fix capitalization for non proper nouns word = word.lower() word, tag = p.brain.retag(word, tag) if tag == "!!!": print "Missing retag converter in the brain" exit(1) if tag in ["NNP", "NNPS"]: word = word.capitalize() if self.last_conjunction is not None: self.conjunction_parts.append(word) # Fix the parser as we go on brain retags def do_track(subtype, tag): if subtype[0] is None and subtype[1] is None: subtype = None tag = p.brain.retag(word, tag) process(word, tag) return (False, subtype, tag) return (True, subtype, tag) if tag in ["NN", "NNS"]: if self.last_tag in ["PRP$"]: # Detects Jane's friends; probably context context = " ".join([self.partial_context, word]) self.parsed.pop() self.last_context = context self.contexts.append(context) did_something = True self.subtype = self.get_subtype(word, tag) self.partial_context = None self.last_partial_context = None self.partial_subtype = None track, self.subtype, tag = do_track(self.subtype, tag) if track: if self.problem.involves_acting: conjunction = ((word, tag), self.last_conjunction) self.subordinate_strings[word] = " ".join(self.conjunction_parts) self.conjunction_parts = [] self.conjunctions.append(conjunction) did_something = True self.track(conjunction[0], "subordinate", self.subtype) else: if self.subtype[0] == "plural" and self.subtype[1] == "mixed": # PRP$ will always have a noun like last_subtype last_plural, last_gender = self.last_subtype # Last Plural Context lpc = p.last_contexts["plurality"][last_plural] # Last Gender Context lgc = p.last_contexts["gender"][last_gender] self.last_context = None if lgc == lpc: self.last_context = lgc elif lgc is not None: self.last_context = lgc elif lpc is not None: self.last_context = lpc if self.last_context is not None: # Last context string inc = p.brain.inclusive(word, "'{0}' ({1}) in '{2}'".format(self.last_word, self.last_context[0], context)) # Add the context by the word for easier lookup # this will capture 'her friends' and define # the concept of inclusiveness for friends then # later auto-apply this to the concept of say, # 'his friends' p.brain.add("inclusive", context, inc) # Assume the first context after a comparison is the # comparator context if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype, conv=True) else: self.main_context = context self.track(context, "context", self.subtype) self.context_subtypes[context] = self.subtype else: do_reg_unit = True if self.last_conjunction is not None: do_reg_unit = False do_conj_track = False if tag == "NN" and self.last_tag in ["IN"]: self.subtype = self.get_subtype(word, tag) track, self.subtype, tag = do_track(self.subtype, tag) did_something = True if track: unit = " ".join([self.last_unit, self.last_word, word]) self.problem.descriptive_units.append(unit) self.problem.refined_units[self.last_unit] = unit self.units.pop() del p.unit_subtypes[self.last_unit] self.parsed.pop() self.parsed.pop() self.last_unit = unit self.last_unit_tag = tag self.last_unit_index = len(self.parsed) unit, uidx = self.fix_unit(unit) self.units.append(unit) p.unit_subtypes[unit] = self.subtype self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True self.track(unit, "unit", self.subtype, uidx) elif tag in ["NN", "NNS"] and self.last_conjunction_tag in ["IN"]: self.subtype = self.get_subtype(word, tag) track, self.subtype, tag = do_track(self.subtype, tag) did_something = True if track: if self.last_variable is not None: unit = word self.problem.descriptive_units.append(unit) self.problem.refined_units[self.last_unit] = unit self.last_unit = unit self.last_unit_tag = tag self.last_unit_index = len(self.parsed) unit, uidx = self.fix_unit(unit) self.units.append(unit) p.unit_subtypes[unit] = self.subtype self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True c_unit = " ".join([word, "owned by", p.previous_contexts["last"][0]]) self.track((c_unit, word, p.previous_contexts["last"][0], p.last_contexts["last"][0]), "context_unit", self.subtype, uidx) else: do_conj_track = True else: do_conj_track = True do_unset_conj = True if do_conj_track: unit = word do_std = True if tag in ["NN", "NNS"]: if self.last_tag in ["NN", "NNS"]: if self.last_unit is None and self.last_context is not None: self.last_context = None lc = self.contexts.pop() unit = " ".join([lc, word]) else: unit = " ".join([self.last_unit, word]) self.units.pop() self.used_unit_adjectives.append(self.last_word) self.parsed.pop() ptest = self.parsed.pop() self.parsed.append(ptest) if ptest[1] == "constant" and self.last_tag != "DT" and self.problem.exestential: unit, uidx = self.fix_unit(unit) self.last_unit = unit self.last_unit_tag = tag self.last_unit_index = len(self.parsed) self.units.append(unit) p.unit_subtypes[unit] = self.subtype self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True self.track(unit, "unit", self.subtype, uidx) do_unset_conj = False do_reg_unit = False do_std = False if do_std: if word in self.used_unit_adjectives: do_unset_conj = False do_reg_unit = True else: conjunction = ((unit, tag), self.last_conjunction) self.subordinate_strings[unit] = " ".join(self.conjunction_parts) self.conjunction_parts = [] self.conjunctions.append(conjunction) did_something = True self.track(conjunction[0], "subordinate", self.subtype) if do_unset_conj: self.last_conjunction = None if do_reg_unit: self.subtype = self.get_subtype(word, tag) track, self.subtype, tag = do_track(self.subtype, tag) did_something = True if track: #rint "here for", word, tag, self.last_tag, self.last_unit, self.last_word unit = word do_std_unit = True if tag in ["NN", "NNS"] and self.last_tag in ["NN", "NNS"]: # rint self.last_word, self.last_tag # rint word, tag # rint self.parsed # rint "----" ptest = self.parsed.pop() if ptest[1] == "subordinate" and self.last_unit is None: cst = self.conjunctions.pop() subostr = " ".join([self.subordinate_strings[ptest[0][0]], word]) del self.subordinate_strings[ptest[0][0]] unit = " ".join([ptest[0][0], word]) self.subordinate_strings[unit] = subostr self.used_unit_adjectives.append(ptest[0][0]) conjunction = ((unit, tag), cst[1]) self.conjunctions.append(conjunction) self.track(conjunction[0], "subordinate", self.subtype) do_std_unit = False else: if self.last_unit is None and self.last_context is not None: self.last_context = None lc = self.contexts.pop() unit = " ".join([lc, word]) else: unit = " ".join([self.last_unit, word]) self.units.pop() del p.unit_subtypes[self.last_unit] self.used_unit_adjectives.append(self.last_word) if do_std_unit: unit, uidx = self.fix_unit(unit) if not unit in p.units_acting_as_context or not p.units_acting_as_context[unit]: if self.last_unit and self.framing_question and len(self.operators) == 0 and self.last_unit != unit: context = unit if self.subtype[0] == "self": context = self.problem.brain.self_reflexive(context, True) # @TODO: This needs to be a subroutine or something self.last_context = context self.contexts.append(context) if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype) else: self.main_context = context self.track(context, "context", self.subtype) self.context_subtypes[context] = self.subtype else: if len(self.operators) > 0 and self.last_unit and not self.main_context: if self.last_unit != unit: context = self.units.pop() del p.unit_subtypes[context] self.last_context = context self.contexts.append(context) if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype, self.last_unit_index) else: self.main_context = context self.track(context, "context", self.subtype, self.last_unit_index) self.context_subtypes[context] = self.subtype self.last_unit = unit self.last_unit_tag = tag self.last_unit_index = len(self.parsed) self.units.append(unit) p.unit_subtypes[unit] = self.subtype self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True self.track(unit, "unit", self.subtype, uidx) else: context = unit if self.subtype[0] == "self": context = self.problem.brain.self_reflexive(context, True) # @TODO: This needs to be a subroutine or something self.last_context = context self.contexts.append(context) if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype, uidx) else: self.main_context = context self.track(context, "context", self.subtype, uidx) self.context_subtypes[context] = self.subtype if not did_something and self.subtype == None: self.subtype = self.get_subtype(word, tag) if tag == "SUB": conjunction = ((word, tag), self.last_conjunction) self.subordinate_strings[word] = " ".join(self.conjunction_parts) self.conjunction_parts = [] self.conjunctions.append(conjunction) did_something = True self.track(conjunction[0], "subordinate", self.subtype) if tag == "PRP": if self.subtype and (self.subtype[0] == "self" or self.subtype[1] == "self"): context = word if self.subtype[0] == "self": context = self.problem.brain.self_reflexive(context, True) self.last_context = context self.contexts.append(context) did_something = True if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype) else: self.main_context = context self.track(context, "context", self.subtype) self.context_subtypes[context] = self.subtype else: c = self.resolve_context(self.subtype) if c is None: c = self.resolve_context(self.subtype, compx=True) if c is not None: # If we're setting a relative quantity and the contexts are # the same we're not actually setting a relative quantity # we are simply indicating a mathematical operand if self.is_relative_quantity and c[0] == self.last_context: c2 = self.resolve_context(self.subtype, compx=True, failTest=True) if c2 is not None and c2[0] != self.last_context: c = c2 else: self.is_relative_quantity = False did_something = True if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = c[0] self.track(c[0], "comparator_context", self.subtype, conv=True) else: self.main_context = c[0] self.track(c[0], "context", self.subtype) self.context_subtypes[c[0]] = self.subtype # Unset hanging conjunctions when we set a context self.last_conjunction = None else: if self.last_tag in ["IN"]: # remove the "of" in "of them" self.parsed.pop() # Assume this is a unit if not p.involves_acting or self.last_unit: unit, uidx = self.fix_unit(self.last_unit) self.last_unit = unit self.last_unit_tag = self.last_unit_tag self.last_unit_index = len(self.parsed) self.units.append(unit) p.unit_subtypes[unit] = self.subtype self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True self.track(unit, "unit", self.subtype, uidx) if tag == "PRP$": if self.last_conjunction is not None: if self.is_relative_quantity and not self.comparator_context and self.main_context: c = self.resolve_context(self.subtype) ms = self.context_subtypes[self.main_context] if (not c or self.main_context != c[0]) and (not ms or self.subtype[0] != ms[0]): context = word if self.subtype[0] == "self": context = self.problem.brain.self_reflexive(context, True) did_something = True self.comparator_context = context self.track(word, "comparator_context", self.subtype, conv=True) self.last_conjunction = None self.conjunction_parts = [] self.context_subtypes[word] = self.subtype if not did_something: did_something = True term = word st = self.subtype conval = self.resolve_context(self.subtype) if conval: term, st = conval term = ownerize(term) if self.subtype[0] == "self": term = self.problem.brain.self_reflexive(term, True) self.partial_context = term self.partial_subtype = st self.last_partial_context = self.track(term, "partial_context", st) if tag in ["NNP", "NNPS"]: if self.last_tag in ["NNP", "NNPS"]: # Last was partial context; the "Mrs." in "Mrs. Jones" self.last_context = None lc = self.contexts.pop() self.parsed.pop() if self.main_context == lc: self.main_context = None context = " ".join([lc, word]) if self.is_relative_quantity and self.comparator_context == lc: self.comparator_context = context if self.subtype is not None and self.subtype[1] == "ambiguous": old = self.get_subtype(self.last_word, self.last_tag) self.subtype = (self.subtype[0], old[1]) else: context = word if self.subtype[0] == "self": context = self.problem.brain.self_reflexive(context, True) self.last_context = context self.contexts.append(context) did_something = True if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype) else: if not self.main_context: self.main_context = context self.track(context, "context", self.subtype) self.context_subtypes[context] = self.subtype if tag[:2] == "VB": if tag == "VB" and self.question: if self.last_verb_tag is not None: # VB*...VB indicates a question not an operation self.last_operator = None q_start = self.raw_operators.pop() st = self.get_subtype(self.last_verb, self.last_verb_tag) self.track(q_start, "q_start", st, self.last_verb) self.framing_question = False self.did_frame_question = True did_something = True self.track(word, "q_stop", self.subtype) elif self.framing_question: did_something = True self.framing_question = False self.did_frame_question = True self.track(word, "q_stop", self.subtype) op = p.brain.operator(word, self.sentence_text) if op == "ex": self.is_exchange = True elif tag != "VBG": if self.framing_question: did_something = True self.framing_question = False self.did_frame_question = True self.track(word, "q_stop", self.subtype) else: if not self.question and self.is_question and not self.did_frame_question: self.verb_question = True self.question = True self.phrasing_question = True self.framing_question = True did_something = True self.track(word, "asking", self.subtype) elif self.question and not self.did_frame_question: # We are asking a question and likely don't # need to invoke all the operator logic; this is # likely a "start the question" verb if not self.framing_question: did_something = True self.framing_question = True self.track(word, "q_start", self.subtype) else: op = p.brain.operator(word, self.sentence_text) if op not in [None, False]: # Retagged if op == "re": self.is_about_requirements = True if op == "ex": self.is_exchange = True self.last_verb_tag = tag self.last_verb = len(self.parsed) self.last_operator = word self.raw_operators.append(word) did_something = True self.operators.append(op) self.operator[word] = op self.subtype = self.get_subtype(word, tag) self.track(word, "operator", self.subtype) else: did_something = True process(word, tag) else: did_something = True gtype = p.brain.gerund(word, self.sentence_text) if gtype == "acting": self.acting = False self.actions.append(word) if self.main_context: p.context_actions[self.main_context] = (word, gtype, self.subtype) self.track(word, gtype, self.subtype) if tag in ["JJR", "COMP"]: did_something = True adj = p.brain.relative(word, self.sentence_text) self.is_relative_quantity = adj != "noise" self.track(word, adj, self.subtype) if tag in ["IN", "TO"]: self.last_conjunction = word self.last_conjunction_tag = tag self.conjunction_parts.append(word) did_something = True self.track(word, "conjunction", self.subtype) if tag in ["POS"]: did_something = True self.track(word, "possessive", self.subtype) if tag in [".", ","]: did_something = True self.track(word, "punctuation", self.subtype) if tag == "EX": did_something = True self.track(word, "exestential", self.subtype) self.problem.exestential = True if tag in ["CD", "LS"]: # A cardinal number if self.last_tag == "DT": if self.last_determiner == "constant": # should safely be able to ignore detemriner constant self.parsed.pop() self.last_determiner = None if self.last_conjunction is not None: self.conjunction_parts.pop() did_something = True if is_number(word): self.track(word, "constant", self.subtype) else: self.track(p.brain.number(word, self.sentence_text), "constant", self.subtype) if tag == "DT": # A determiner (the) did_something = True dtype = p.brain.determiner(word, self.sentence_text) self.last_determiner = dtype if dtype == "variable": vtype = p.brain.variable(word, self.sentence_text) if vtype == "dynamic_variable": do_dyn_var_trk = True if self.last_tag == "IN": do_dyn_var_trk = False process(word, "RB") return if do_dyn_var_trk: self.track(word, vtype, self.subtype) else: self.track(vtype, "variable_relationship", self.subtype) self.last_variable = word self.last_variable_type = vtype elif dtype == "constant": self.track(p.brain.number(word, self.sentence_text), dtype, self.subtype) elif dtype == "noise": if self.is_question and self.phrasing_question: self.parsed.pop() did_something = True self.track(" ".join([self.last_word, word]), "asking", self.subtype) else: self.track(word, dtype, self.subtype) if tag == "RB": # An adverb, probably a subordinate? if self.verb_question and self.framing_question: tag = "JJ" # This is almost certainly not an adverb at this point else: if self.last_conjunction is not None: conjunction = ((word, tag), self.last_conjunction) self.last_conjunction = None self.subordinate_strings[word] = " ".join(self.conjunction_parts) self.conjunction_parts = [] else: conjunction = ((word, tag), None) self.subordinate_strings[word] = word self.conjunctions.append(conjunction) did_something = True self.track(conjunction[0], "subordinate", self.subtype) # Anything about phrasing must come before the wh-determiner block if tag == "JJ": # Adjective if self.phrasing_question: self.parsed.pop() did_something = True self.track(" ".join([self.last_word, word]), "asking", self.subtype) elif self.last_tag in ["JJ"]: self.used_unit_adjectives.append(word) did_something = True self.last_adjective = " ".join([self.last_adjective, word]) self.track(self.last_adjective, "adjective", self.subtype, self.last_adj_index) elif self.last_tag in ["IN"]: did_something = True unit = " ".join([self.last_unit, self.last_word, word]) self.problem.descriptive_units.append(unit) self.problem.refined_units[self.last_unit] = unit self.units.pop() del p.unit_subtypes[self.last_unit] self.parsed.pop() self.parsed.pop() self.last_unit = unit self.last_unit_tag = tag self.last_unit_index = len(self.parsed) unit, uidx = self.fix_unit(unit) self.units.append(unit) p.unit_subtypes[unit] = self.subtype self.unit_idx[unit] = self.last_unit_index if self.new_units_as_context: p.units_acting_as_context[unit] = True self.track(unit, "unit", self.subtype, uidx) else: self.used_unit_adjectives.append(word) did_something = True self.last_adjective = word self.last_adj_index = len(self.parsed) self.track(word, "adjective", self.subtype) if tag == "PIP": did_something = True self.track(word, "pre_ind_plu", self.subtype) if not self.problem.exestential: self.acting = True self.problem.involves_acting = True if self.last_unit is not None and self.last_context is None: # If we have a present indicitive plural with no context # it is likely that we're dealing with a context which has # been misinterpreted as a unit, so switch that # mistake here. p.units_acting_as_context[self.last_unit] = True # Remove the last unit and make it a context context = self.units.pop() del p.unit_subtypes[context] self.last_context = context self.contexts.append(context) if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", self.subtype, self.last_unit_index) else: self.main_context = context self.track(context, "context", self.subtype, self.last_unit_index) self.context_subtypes[context] = self.subtype # @TODO: This needs a much better tracking system self.last_unit = None self.last_unit_tag = None self.last_unit_index = -1 elif not self.phrasing_question: self.new_units_as_context = True if tag in ["WRB", "WDT", "WP", "WP$"]: # Wh- determiner self.question = True self.phrasing_question = True did_something = True self.track(word, "asking", self.subtype) elif self.phrasing_question and not (tag[:2] == "VB" and self.is_question): self.phrasing_question = False if tag == "CC": did_something = True if self.verb_question and self.framing_question: self.track(word, "eval_option_sep") else: self.track(word, "coordinating_conjunction") if self.subtype is not None: did_something = True if not did_something: #rint "Answer Unknown", word, tag p.brain.unknown(word, tag, self.subtype, self.sentence_text) process(word, tag) self.last_word = word self.last_tag = tag self.last_subtype = self.subtype # Look before you leap for s_tag in self.sentence: word, tag = s_tag if tag == "." and word == "?": self.is_question = True for s_tag in self.sentence: process(*s_tag) self.index += 1 if self.partial_context: context = self.partial_context idx = self.last_partial_context stype = self.partial_subtype self.partial_context = None self.last_partial_context = None self.partial_subtype = None if self.is_relative_quantity and not self.comparator_context and self.main_context: self.comparator_context = context self.track(context, "comparator_context", stype, idx, True) last_context = None last_possessive = None is_possessive = False was_possessive = False did_swap_context_for_unit = False reparsed = [] # Walk this to combine context with their unit possessives for part in self.parsed: word, role, subtype = part if role == "context": if word in self.problem.units or (word != self.main_context and len(self.units) == 0): self.contexts.remove(word) role = "unit" self.units.append(word) did_swap_context_for_unit = True else: last_context = part elif last_context is not None and role == "possessive": last_possessive = word is_possessive = True elif role == "unit": if is_possessive: was_possessive = True cidx = self.contexts.index(last_context[0]) self.units.remove(word) word = " ".join([last_context[0] + last_possessive, word]) subtype = last_context[2] role = "context" self.contexts[cidx] = word last_context = None last_possessive = None is_possessive = False reparsed.pop() reparsed.pop() elif did_swap_context_for_unit: did_swap_context_for_unit = False self.units.remove(word) role = "context" self.contexts.append(word) reparsed.append((word, role, subtype)) #rint self.sentence_text, "::::", self.main_context self.parsed = reparsed # Make all the inferred items unique self.raw_operators = uniq(self.raw_operators) self.contexts = uniq(self.contexts) self.units = uniq(self.units) self.actions = uniq(self.actions) self.operators = uniq(self.operators) text = self.sentence_text nunits = [] for unit in self.units: acting_as_context = False if unit in p.units_acting_as_context: acting_as_context = p.units_acting_as_context[unit] # Units can be complicated by adjectives which break up the unit # into important subdivisions. However, these subdivisions may # also be simple detractors and not relevant to the actual string # match for the unit. As such, we split by any spaces and then # compile all possible formulations of the unit if not unit in self.problem.descriptive_units and " " in unit: parts = unit.split(" ") # The item is the last part of the unit item = parts.pop() for part in parts: nu = " ".join([part, item]) nunits.append(nu) # if acting_as_context: # p.units_acting_as_context[nu] = True self.units = uniq(self.units + nunits) for unit in self.units: if unit in self.problem.refined_units: uxd = self.parsed[self.unit_idx[unit]] self.parsed[self.unit_idx[unit]] = (self.problem.refined_units[unit], uxd[1], uxd[2]) # Resolve the subordinates for o in self.conjunctions: subordinate, conjunction = o outp = p.brain.subordinate(subordinate, text) self.subordinate_lookup[subordinate[0]] = outp if outp == "object": continue self.subordinates.append((subordinate[0], outp)) if len(self.subordinate_strings[subordinate[0]]) == 0 and outp != "place_noun": self.subordinate_strings[subordinate[0]] = ANSWER_SUBORDINATE[outp] if outp == "context_grouping": csub = self.get_subtype(subordinate[0], subordinate[1]) if csub: c = self.resolve_context(csub) if c: self.problem.subordinate_adaptive_contexts.append(c[0]) self.subordinates = uniq(self.subordinates) self.problem.running_units += self.units self.problem.units += self.units self.problem.units = uniq(self.problem.units) if len(self.subordinates) > 0: if not self.main_context in self.problem.context_subordinates: self.problem.context_subordinates[self.main_context] = (self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup) elif self.main_context in self.problem.context_subordinates: # This is an inferred context self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup = self.problem.context_subordinates[self.main_context] for sub in self.subordinates: subord, subt = sub self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord]) if not self.main_context and self.operator: if self.problem.last_contexts["last"]: context, subtype = self.problem.last_contexts["last"] self.main_context = context self.track(context, "context_inferred", subtype) #rint self.subordinates, self.main_context, self.problem.context_subordinates has_unit_proxy = False if self.main_context in self.problem.context_subordinates: if len(self.subordinates) == 0: # This is an inferred context self.subordinates, self.subordinate_strings, self.subordinate_subtypes, self.subordinate_lookup = self.problem.context_subordinates[self.main_context] for sub in self.subordinates: subord, subt = sub self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord]) elif len(self.subordinates) > 0: subs, sub_strs, sub_subs, sub_look = self.problem.context_subordinates[self.main_context] handeled_subtypes = [] for s in self.subordinate_lookup: st = self.subordinate_lookup[s] if st is not None: if st[0:4] == "time": st = st[0:4] handeled_subtypes.append(st) has_unit_proxy = "unit_grouping" in handeled_subtypes for sub in subs: subord, subt = sub st = subt if st is not None: if st == "costpay": continue if st[0:4] == "time": st = st[0:4] if st in handeled_subtypes and st == "time": continue if subord in self.subordinate_lookup: continue self.subordinate_subtypes[subord] = sub_subs[subord] self.subordinate_lookup[subord] = sub_look[subord] if subt != "refiner": self.subordinates.append(sub) self.track(sub, "subordinate_inferred", self.subordinate_subtypes[subord]) if self.main_context in self.problem.context_actions and len(self.actions) == 0: word, gtype, subtype = self.problem.context_actions[self.main_context] self.track(word, gtype + "_inferred", subtype) if len(self.units) == 0 and len(self.problem.running_units) > 0 and not has_unit_proxy: iu = self.problem.running_units[-1] self.track(iu, "unit_inferred", self.problem.unit_subtypes[iu]) self.problem.contexts += self.contexts self.problem.contexts = uniq(self.problem.contexts)