def eval_sequence(commands, start=cards.I, debug=False):
get = AbstractFunction.create('get', IntValue)
context = Context(None)
state = start
for cmd, side in commands:
if cmd == cards.get:
cmd = get
if side == 'r':
state = apply(state, cmd, context)
else:
state = apply(cmd, state, context)
if debug:
print cmd, side, ':', state
return state
def rec(cur, depth):
global cnt
scur = str(cur)
if scur.isdigit() and scur not in optimized_numbers:
t = terms.number_term(int(scur))
cost = terms.sequential_cost(t)
if cost > len(steps):
print len(steps), steps_to_str(steps), "->", scur
optimized_numbers.add(scur)
if scur in desired:
print len(steps), steps_to_str(steps), "->", scur
desired.remove(scur)
if len(desired) == 0:
print "Done. Press enter to exit"
raw_input()
exit()
if depth == 0:
return
cnt += 1
if cnt % 500000 == 0:
print cnt
for i, (f, side) in enumerate(possible_steps):
if len(steps) == 0 and i % num_parts != our_part:
continue
try:
context.app_limit = 30
if side == "r":
next = apply(cur, f, context)
else:
next = apply(f, cur, context)
except Error as e:
continue
# except Exception as e:
# print steps_to_str(steps+[(f, side)])
# print e
# raise e
# exit()
steps.append((f, side))
rec(next, depth - 1)
steps.pop()
def apply(self, slot, card, direction):
"return None or error"
context = Context(self)
try:
if self.proponent.vitalities[slot] <= 0:
raise Error("application involving dead slot")
s = self.proponent.values[slot]
if direction is LEFT_APP:
result = apply(card, s, context)
elif direction is RIGHT_APP:
result = apply(s, card, context)
else:
assert False
self.proponent.values[slot] = result
except Error as e:
if self.output_level == 1:
print "Exception: Native.Error"
if self.output_level == 2:
print "error:", str(e)
if self.output_level > 0:
print "slot {0} reset to I".format(slot)
self.proponent.values[slot] = cards.I
def update_bank_account_transactions(db, bank_config, account_config,
from_date, to_date):
logger.info(
'Updating {bank.name} account {account.id} transactions from {from_date} to {to_date}'
.format(bank=bank_config,
account=account_config,
from_date=from_date.strftime('%d/%m/%Y'),
to_date=to_date.strftime('%d/%m/%Y')))
bank_module = load_module(bank_config.id)
raw_transactions = scrap_bank_account_transactions(bank_module,
bank_config,
account_config,
from_date, to_date)
logger.info('{} transactions fetched'.format(len(raw_transactions)))
parsed_transactions = parse_account_transactions(bank_module, bank_config,
account_config,
raw_transactions)
filtered_transactions = list(
filter(
lambda transaction: transaction.transaction_date >= from_date and
transaction.transaction_date <= to_date, parsed_transactions))
discarded_transactions_count = len(parsed_transactions) - len(
filtered_transactions)
filtered_info = ' ({} out of date range got filtered)'.format(
discarded_transactions_count
) if discarded_transactions_count > 0 else ''
logger.info('{} transactions parsed{}'.format(len(parsed_transactions),
filtered_info))
processed_transactions = rules.apply(rules.load(), filtered_transactions)
logger.info('Rules applied to {} transactions'.format(
len(processed_transactions)))
removed, added, _ = database.update_account_transactions(
db, account_config.id, processed_transactions)
if added:
logger.info(
'Successfully added {} account transactions to the database.'.
format(added))
else:
logger.info('There are no new account transactions to add'.format(
len(raw_transactions)))
if removed:
logger.info(
'Successfully removed {} transactions from the database'.format(
removed))
return (removed, added)
def get_pp_attachments(s, format=[WORD, POS, CHUNK, PNP, LEMMA], timeout=None):
""" Takes a parsed string and returns a tuple of tuples ((anchor index, PP index), ...)
and a tuple with info about where the anchor came from (TiMBL/lowest_entropy/baseline).
- Lowest entropy is used when different anchor candidates have the same score.
- Baseline is used when no candidates where found with TiMBL.
The given sentence can also be a Sentence object (see tree.py).
"""
# Create a parse tree from the parsed string.
# We need POS, CHUNK and PNP tags, but it works up to 5x faster with LEMMA given.
if isinstance(s, (str, unicode)):
s = Sentence(s, token=format)
# Generate instances from the parse tree.
# Send the instances to the TiMBL server.
# The batch() function in the client module takes care of managing server clients,
# we simply pass it all the tagging jobs and a definition of the client we need.
instances = PP_instances(s)
instances2 = [x.encode(config.encoding) + ' ?' for x in instances]
tags = batch(instances2,
client=(TimblPP, HOST, PORT, config.PREPOSITION, config.log),
retries=1)
# Tag and group the instances by PP.
grouped = {}
for i, x in enumerate(instances):
grouped.setdefault(x.pp, []).append(
instance.TaggedInstance(x, tags[i][0], tags[i][1], tags[i][2]))
# Revote the instance groups that do not have unique anchors.
attachments = []
for instances in grouped.values():
type = _typeof(instances)
if type == 0:
# TiMBL determined the best anchor candidate.
attachments.append((_tuplify(instances), 'timbl'))
elif type == +1:
# There is more than one possible candidate and extra voting is needed.
# Example: "She added: 'I was a little naive as to the impact it would have
# because I really didn't have any idea it would be like that.'"
attachments.append(
(_tuplify(voting.lowest_entropy(instances)), 'lowest_entropy'))
elif type == -1:
# There are no candidates so we are going to make a calculated guess.
# Example: "The red cat at the left is sleeping."
attachments.append(
(_tuplify(voting.base_candidate(instances)), 'baseline'))
# Rules can be applied here (e.g. to correct things like "as for me").
attachments = rules.apply(attachments, s)
attachments = tuple([x[0] for x in attachments
]), tuple([x[1] for x in attachments])
return attachments
def zombie_phase(self):
prop = self.proponent
for i in range(SLOTS):
if prop.vitalities[i] != -1:
continue
if self.output_level > 0:
print "applying zombie slot {0}={{-1,{1}}} to I".format(i, prop.values[i])
z = prop.values[i]
context = Context(self, zombie=True)
try:
_ = apply(z, cards.I, context) # not interested in result
except Error as e:
if self.output_level == 1:
print "Exception: Native.Error"
if self.output_level == 2:
print "error:", str(e)
prop.values[i] = cards.I
prop.vitalities[i] = 0
def get_pp_attachments(s, format=[WORD, POS, CHUNK, PNP, LEMMA], timeout=None):
""" Takes a parsed string and returns a tuple of tuples ((anchor index, PP index), ...)
and a tuple with info about where the anchor came from (TiMBL/lowest_entropy/baseline).
- Lowest entropy is used when different anchor candidates have the same score.
- Baseline is used when no candidates where found with TiMBL.
The given sentence can also be a Sentence object (see tree.py).
"""
# Create a parse tree from the parsed string.
# We need POS, CHUNK and PNP tags, but it works up to 5x faster with LEMMA given.
if isinstance(s, (str, unicode)):
s = Sentence(s, token=format)
# Generate instances from the parse tree.
# Send the instances to the TiMBL server.
# The batch() function in the client module takes care of managing server clients,
# we simply pass it all the tagging jobs and a definition of the client we need.
instances = PP_instances(s)
instances2= [x.encode(config.encoding)+' ?' for x in instances]
tags = batch(instances2, client=(TimblPP, HOST, PORT, config.PREPOSITION, config.log), retries=1)
# Tag and group the instances by PP.
grouped = {}
for i, x in enumerate(instances):
grouped.setdefault(x.pp, []).append(instance.TaggedInstance(x, tags[i][0], tags[i][1], tags[i][2]))
# Revote the instance groups that do not have unique anchors.
attachments = []
for instances in grouped.values():
type = _typeof(instances)
if type == 0:
# TiMBL determined the best anchor candidate.
attachments.append((_tuplify(instances), 'timbl'))
elif type == +1:
# There is more than one possible candidate and extra voting is needed.
# Example: "She added: 'I was a little naive as to the impact it would have
# because I really didn't have any idea it would be like that.'"
attachments.append((_tuplify(voting.lowest_entropy(instances)), 'lowest_entropy'))
elif type == -1:
# There are no candidates so we are going to make a calculated guess.
# Example: "The red cat at the left is sleeping."
attachments.append((_tuplify(voting.base_candidate(instances)), 'baseline'))
# Rules can be applied here (e.g. to correct things like "as for me").
attachments = rules.apply(attachments, s)
attachments = tuple([x[0] for x in attachments]), tuple([x[1] for x in attachments])
return attachments
self.__rules[key] = rule
def tokenize(self, content):
for key, rule in self.__rules.items():
content = rule(content)
return content
def __normalize(content, match):
return content[:match.regs[1][0]] + match.group(1).replace(
'.', '') + content[match.regs[1][1]:]
simple_tokenizer = Tokenizer()
simple_tokenizer.add_rule(rules.replace(r'[^a-zA-Z]+', ' '))
simple_tokenizer.add_rule(rules.apply(str.lower))
simple_tokenizer.add_rule(rules.apply(str.split, ' '))
simple_tokenizer.add_rule(rules.filter(rules.comparator(len) >= 3))
tokenizer = Tokenizer()
tokenizer.add_rule(rules.replace(r'(\n)+', ' '))
tokenizer.add_rule(rules.apply(str.lower))
tokenizer.add_rule(
rules.modify(r'(?:[^a-z]|^)([a-z](?:\.[a-z])+\.?)(?:[^a-z]|$)',
__normalize))
tokenizer.add_rule(
rules.find(
r'([a-z]+(?:-[a-z]+)+)|(?:.*?([.,\-!?;*/=()\[\]:"\'\\]* +|[.,\-!?;*/=()\[\]:"\'\\]+ *))|(.+)'
))
tokenizer.add_rule(
rules.map(lambda match: match
def transform(p, x): res = [] for rule in rules.Rules: res.extend(rules.apply(rule, p)) return res
parsed_filters = list(
map(lambda f: f.lower(), filters_argument.split(',')))
print(table_all(all_transactions, filters=parsed_filters))
sys.exit(0)
if action == 'apply' and arguments['rules']:
db = database.load(bank.env()['database_folder'])
account_transactions = database.io.find_account_transactions(db)
credit_card_transactions = database.io.find_credit_card_transactions(
db)
all_transactions = sorted(chain(account_transactions,
credit_card_transactions),
key=attrgetter('transaction_date'))
processed_transactions = rules.apply(rules.load(), all_transactions)
changed = 0
for old_transaction, new_transaction in zip(all_transactions,
processed_transactions):
if old_transaction != new_transaction:
database.update_transaction(db, new_transaction)
changed += 1
print('Updated {} of {} transactions'.format(
changed, len(processed_transactions)))
sys.exit(1)
bank_id = arguments['<bank>']
def transform(p, x):
res = []
for rule in rules.Rules:
res.extend(rules.apply(rule, p))
return res
