def domain_state(tconf):
state = State()
state.txn_list = {}
state.get = lambda key, isCommitted=False: state.txn_list.get(key, None)
state.set = lambda key, value, isCommitted=False: state.txn_list.update(
{key: value})
return state
def on_chat(self, t, f, msg):
knowers = self.adapter.get_users()
knowers.remove(self.adapter.nickname)
context = {'author': f,
'recipient': t,
'msg': msg,
'knowers': knowers}
State.forceState(FindGossip, context)
def txn_author_agreement_handler(db_manager):
f = FakeSomething()
f.validate = lambda request, action_list: True
handler = TxnAuthorAgreementHandler(db_manager, FakeSomething(), f)
state = State()
state.txn_list = {}
state.get = lambda key, isCommitted=False: state.txn_list.get(key, None)
state.set = lambda key, value, isCommitted=False: state.txn_list.update({key: value})
db_manager.register_new_database(handler.ledger_id, FakeSomething(), state)
return handler
def nym_handler(tconf):
data_manager = DatabaseManager()
handler = NymHandler(tconf, data_manager)
state = State()
state.txn_list = {}
state.get = lambda key, isCommitted: state.txn_list.get(key, None)
state.set = lambda key, value: state.txn_list.update({key: value})
data_manager.register_new_database(handler.ledger_id, FakeSomething(),
state)
return handler
def node_handler():
data_manager = DatabaseManager()
bls = FakeSomething()
handler = NodeHandler(data_manager, bls)
state = State()
state.txn_list = {}
state.get = lambda key, is_committed: state.txn_list.get(key, None)
state.set = lambda key, value: state.txn_list.update({key: value})
data_manager.register_new_database(handler.ledger_id, FakeSomething(),
state)
return handler
def init_state_from_ledger(self, state: State, ledger: Ledger):
"""
If the trie is empty then initialize it by applying
txns from ledger.
"""
if state.isEmpty:
logger.info('{} found state to be empty, recreating from '
'ledger'.format(self))
for seq_no, txn in ledger.getAllTxn():
txn = self.node.update_txn_with_extra_data(txn)
self.node.write_manager.update_state(txn, isCommitted=True)
state.commit(rootHash=state.headHash)
def forget(self):
for timer in self.idle.values():
timer.cancel()
self.idle = {}
self.resumeState = {}
State.forget()
db = Database()
db.drop_tables()
db.close_conn()
self.on_join()
def test_main_page(self):
response = self.app.get('/', follow_redirects=True)
cookies = response.headers.getlist('Set-Cookie')
name = 'state'
for cookie in cookies:
for c_key, c_value in parse_cookie(cookie).items():
if c_key == name:
state = State(c_value)
self.assertEqual(c_value, state.state)
state.delete()
print(c_value)
self.assertEqual(response.status_code, 200)
def txn_author_agreement_aml_handler(tconf, domain_state):
data_manager = DatabaseManager()
handler = TxnAuthorAgreementAmlHandler(data_manager, FakeSomething())
state = State()
state.txn_list = {}
state.get = lambda key, isCommitted=False: state.txn_list.get(key, None)
state.set = lambda key, value, isCommitted=False: state.txn_list.update(
{key: value})
data_manager.register_new_database(handler.ledger_id, FakeSomething(),
state)
data_manager.register_new_database(DOMAIN_LEDGER_ID, FakeSomething(),
domain_state)
return handler
def __init__(self, server=None):
State.__init__(self, server)
self.matchIndex = defaultdict(int)
self.nextIndex = defaultdict(int)
for node in self.server.connectedNode:
# For each server, index of the next log entry to send to that server
self.nextIndex[node] = self.server.lastLogIndex() + 1
# For each server, index of highest log entry known to be replicated on server
self.matchIndex[node] = -1
if self.server.timer:
self.server.timer.cancel()
# heartbeat
self.hThread = threading.Thread(target=self.heartbeat)
self.hThread.start()
def on_message(self, user, timestamp, msg):
self.idle[GustafoBot.CHAT].cancel()
if self.idle.get(user, None) is not None:
self.idle[user].cancel()
if State.userState[user] is SolicitResponse:
State.userState[user] = self.resumeState[user]
it = time.time()
print self.adapter.get_users()
State.users = self.adapter.get_users()
res = State.query(user, msg)
rt = time.time()
if rt - it < 3.0:
print "Sleep:", 3.0 - (rt - it)
time.sleep(3.0 - (rt - it))
if res is not None:
self.send_message(user, res)
print user
self.idle[user] = Timer(GustafoBot.TIMEOUT, self.on_user_inactive, [user])
self.idle[user].start()
def naive_ego_to_flag(state: State) -> float:
ego_ball = state.get_ego_ball()
combined_radius = ego_ball.radius + state.flag.radius
flag_ego_dist = math.dist(ego_ball.x, ego_ball.y, state.flag.x,
state.flag.y)
return max(0, flag_ego_dist - combined_radius)
def wait_for_move(prevboard: Board, state: State, expected_move):
pboard = prevboard
board = state.board()
while True:
d, diffs = pboard.diff(board)
if d == 1:
dr, dc = diffs[0]
if pboard(dr, dc) != 0:
raise MoveError("You can\'t put sign on busy field")
if board(dr, dc) != expected_move:
raise MoveError("It's not your sign")
return dr, dc
elif d > 1:
raise MoveError("You can do only one move")
pboard = board
board = state.board()
def move(self, state: State, opponent: Player):
correct_input = False
board = state.board()
while not correct_input:
movement = input()
if len(movement) > 1:
raise TypeError("Wrong input!\n")
if ord(movement) < 48 or ord(movement) > 56:
raise TypeError("Wrong input!\n")
movement = int(movement, 10)
if board(movement // len(board), movement % len(board)) != 0:
print("This field is busy!\nWrite correct number of field.\n")
print(state)
else:
correct_input = True
state.move(movement // len(board), movement % len(board), self.symbol)
def evaluate_keys(state: State, keypresses: Keys, dt: int) -> float:
ego_ball = state.get_ego_ball()
future_ball = ego_ball.simulate_input(keypresses)
future_ball.update(dt)
possible_world = copy.deepcopy(state)
possible_world.balls[ego_ball.id] = future_ball
return score.naive_ego_to_flag(possible_world)
def on_user_exit(self, nick, timestamp):
print "##### EXIT #####"
knowers = self.adapter.get_users()
knowers.remove(self.adapter.nickname)
timestamp = time.strftime("%X", time.localtime())
msg = "left at " + timestamp
context = {'author': nick,
'recipient': "",
'msg': msg,
'knowers': knowers}
State.forceState(FindGossip, context)
def db_manager(tconf, tdir):
db_manager = DatabaseManager()
state = State()
state.txn_list = {}
state.get = lambda key, isCommitted=True: state.txn_list.get(key, None)
state.set = lambda key, value: state.txn_list.update({key: value})
name = 'name'
idr_cache = IdrCache(name,
initKeyValueStorage(KeyValueStorageType.Rocksdb,
tdir,
tconf.idrCacheDbName,
db_config=tconf.db_idr_cache_db_config))
db_manager.register_new_store(IDR_CACHE_LABEL, idr_cache)
db_manager.register_new_database(DOMAIN_LEDGER_ID, get_fake_ledger(), state)
return db_manager
def naive_ego_to_ball(state: State, target_ball_id: UUID) -> float:
ego_ball = state.get_ego_ball()
target_ball = next(b for b in state.balls if b.id == target_ball_id)
combined_radius = ego_ball.radius + target_ball.radius
target_ego_dist = math.dist(ego_ball.x, ego_ball.y, target_ball.x,
target_ball.y)
return max(0, target_ego_dist - combined_radius)
def test_dispatches_to_hardware(self):
with MockHardware() as hardware_service:
dispatcher = Dispatcher(hardware_service)
current_state = State(color=Color(100, 100, 100),
buzzer_pattern=BuzzerPattern.NONE)
command = Command(color=Color(110, 110, 110),
duration=100,
buzzer_pattern=BuzzerPattern.NONE)
now = 0
next_state = dispatcher.dispatch(current_state, command, now)
self.assertEqual(next_state.color.r, 100)
self.assertEqual(next_state.color.g, 100)
self.assertEqual(next_state.color.b, 100)
self.assertEqual(hardware_service.color_last_called_with.r, 100)
self.assertEqual(hardware_service.color_last_called_with.g, 100)
self.assertEqual(hardware_service.color_last_called_with.b, 100)
self.assertEqual(hardware_service.color_called_count, 1)
now = 10
next_state = dispatcher.dispatch(next_state, command, now)
self.assertEqual(next_state.color.r, 101)
self.assertEqual(next_state.color.g, 101)
self.assertEqual(next_state.color.b, 101)
self.assertEqual(hardware_service.color_last_called_with.r, 101)
self.assertEqual(hardware_service.color_last_called_with.g, 101)
self.assertEqual(hardware_service.color_last_called_with.b, 101)
self.assertEqual(hardware_service.color_called_count, 2)
now = 90
next_state = dispatcher.dispatch(next_state, command, now)
self.assertEqual(next_state.color.r, 109)
self.assertEqual(next_state.color.g, 109)
self.assertEqual(next_state.color.b, 109)
self.assertEqual(hardware_service.color_last_called_with.r, 109)
self.assertEqual(hardware_service.color_last_called_with.g, 109)
self.assertEqual(hardware_service.color_last_called_with.b, 109)
self.assertEqual(hardware_service.color_called_count, 3)
now = 100
next_state = dispatcher.dispatch(next_state, command, now)
self.assertEqual(next_state.color.r, 110)
self.assertEqual(next_state.color.g, 110)
self.assertEqual(next_state.color.b, 110)
self.assertEqual(hardware_service.color_last_called_with.r, 110)
self.assertEqual(hardware_service.color_last_called_with.g, 110)
self.assertEqual(hardware_service.color_last_called_with.b, 110)
self.assertEqual(hardware_service.color_called_count, 4)
def on_user_inactive(self, nick):
State.users = self.adapter.get_users()
if State.userState[nick] is not SolicitResponse:
self.resumeState[nick] = State.userState[nick]
res = State.forceState(SolicitResponse, {'_nick': nick})
#res = State.forceState(SolicitUser,{'_nick': nick})
self.idle[nick] = Timer(GustafoBot.TIMEOUT, self.on_user_inactive, [nick])
self.idle[nick].start()
else:
res = State.forceState(GiveUpState, {'_nick': nick})
del(State.userState[nick])
del(self.idle[nick])
if len(State.userState) == 0:
self.idle[GustafoBot.CHAT] = Timer(10.0, self.on_chat_inactive)
self.idle[GustafoBot.CHAT].start()
if res is not None:
self.send_message(nick, res)
def main():
black = (0, 0, 0)
pygame.init()
screen = pygame.display.set_mode((1000, 1000))
done = False
foe_ball = Ball(200, 200, uuid4(), Team.FOE, 50, 50)
friend_ball = Ball(100, 200, uuid4(), Team.FRIEND, 50, -50)
ego_ball = Ball(500, 500, uuid4(), Team.EGO, 50, 50)
flag = Flag(100, 100)
world_state = State([foe_ball] + [friend_ball] + [ego_ball], flag)
clock = pygame.time.Clock()
while not done:
for event in pygame.event.get():
if event.type == pygame.QUIT:
done = True
screen.fill(black) # reset the screen before we redraw the state
clock.tick(60) # tagpro runs at 60hz
delta_t_ms: int = clock.get_time()
pygame_pressed = pygame.key.get_pressed()
if pygame_pressed[pygame.K_k]:
ego_ball.handle_pop()
elif pygame_pressed[pygame.K_r]:
ego_ball.is_popped = False
elif pygame_pressed[pygame.K_SPACE]:
# Of all sensible keypress combinations, choose the one with the lowest score in dt.
best_keypresses = myopic.best_keypresses(world_state, delta_t_ms)
current_input = Input({ego_ball.id: best_keypresses})
else:
current_input = Input(
{ego_ball.id: Keys.from_pygame_pressed(pygame_pressed)})
world_state = world_state.next_state(current_input, delta_t_ms)
world_state.draw(screen)
pygame.display.flip()
def __init__(self, dispatcher_service: Dispatcher,
network_service: INetwork):
self.dispatcher_service = dispatcher_service
self.network_service = network_service
self.commands = []
self.commands.append(DefaultCommand)
self.current_command = DefaultCommand
self.state = State(Color(255, 255, 255), BuzzerPattern.NONE)
self.last_run = time()
self.last_poll = 0
def db_manager(tconf, tdir):
db_manager = DatabaseManager()
name = 'name'
idr_cache = IdrCache(
name,
initKeyValueStorage(KeyValueStorageType.Rocksdb,
tdir,
tconf.idrCacheDbName,
db_config=tconf.db_idr_cache_db_config))
db_manager.register_new_store('idr', idr_cache)
db_manager.register_new_database(DOMAIN_LEDGER_ID, get_fake_ledger(),
State())
return db_manager
def dispatch(self, command: Command, now: int):
start_time = command.start_time
end_time = command.start_time + command.duration
next_color = self.determine_color(command.start_color, command.color,
start_time, end_time, now)
next_buzzer = self.determine_buzzer(command.buzzer_pattern, start_time,
end_time, now)
next_state = State(next_color, next_buzzer)
self.state_to_hardware(next_state)
return next_state
def parse_input(filename):
"""Read input from filename."""
with open(filename, 'r') as f:
# create state while extracting first line
state = State(*[int(x) for x in next(f).split()])
# save info in state
for i in range(state.num_pizzas):
line = next(f).split()
state.pizzas.append(Pizza(i, int(line[0]), line[1:]))
return state
def move(self, state: State, opponent):
board = state.board()
self.say("Now it's my turn! You will lose!")
movement = optimum_move(board, opponent.symbol, self.symbol, self.symbol).index
r, c = movement
self.say("Please put {0} in {1} row and {2} column".format(
symbol_name(self.symbol), number_name(r+1), number_name(c + 1)))
try:
dr, dc = wait_for_move(board, state, self.symbol)
if dr != r or dc != c:
raise MoveError("Sign was putted not in requested place")
except MoveError as e:
self.say("You tried to lie me. I won't play with you, because {0}".format(e))
raise MoveError("Computer move error")
self.say("Thank you")
def parse_input(filename):
"""Read input from filename."""
with open(filename, 'r') as f:
# create state while extracting first line
state = State(*[int(x) for x in next(f).split()])
# save info in state
for _ in range(state.a):
x, y, z = [int(i) for i in next(f).split()]
state.a_values.append((x, y, z))
for _ in range(state.b):
x, y = [int(i) for i in next(f).split()]
state.b_values.append((x, y))
return state
def get_current_command(self, now):
if self.current_command.is_expired(now):
# Make sure our state is up to date
if not self.current_command.color.is_no_change():
self.state = State(self.current_command.color,
BuzzerPattern.NONE)
if len(self.commands) > 0:
self.current_command = self.commands[0]
del self.commands[0]
self.current_command.set_start_values(now, self.state.color)
else:
# We have no commands, use the default, but update the
# start values
self.current_command = DefaultCommand
self.current_command.set_start_values(now, self.state.color)
return self.current_command
def test_dispatches_to_hardware_with_buzzer(self):
with MockHardware() as hardware_service:
dispatcher = Dispatcher(hardware_service)
current_state = State(color=Color(100, 100, 100),
buzzer_pattern=BuzzerPattern.NONE)
command = Command(color=Color.no_change(),
duration=100,
buzzer_pattern=BuzzerPattern(duration=10,
strength=1))
now = 0
next_state = dispatcher.dispatch(current_state, command, now)
self.assertEqual(next_state.color.r, 100)
self.assertEqual(next_state.color.g, 100)
self.assertEqual(next_state.color.b, 100)
self.assertEqual(next_state.buzzer_pattern.strength, 1)
self.assertEqual(hardware_service.motor_called_count, 1)
self.assertEqual(hardware_service.motor_state, 1)
now = 10
next_state = dispatcher.dispatch(next_state, command, now)
self.assertEqual(next_state.color.r, 100)
self.assertEqual(next_state.color.g, 100)
self.assertEqual(next_state.color.b, 100)
self.assertEqual(next_state.buzzer_pattern.strength, 1)
self.assertEqual(hardware_service.motor_called_count, 2)
self.assertEqual(hardware_service.motor_state, 1)
now = 90
next_state = dispatcher.dispatch(next_state, command, now)
self.assertEqual(next_state.color.r, 100)
self.assertEqual(next_state.color.g, 100)
self.assertEqual(next_state.color.b, 100)
self.assertEqual(next_state.buzzer_pattern.strength, 0)
self.assertEqual(hardware_service.motor_stop_called_count, 1)
self.assertEqual(hardware_service.motor_state, 0)
def on_chat_inactive(self):
State.users = self.adapter.get_users()
users = self.adapter.get_users()
users.remove(self.adapter.nickname)
if "foaad" in users:
user = "******"
elif len(users) > 0:
random.shuffle(users)
user = users[0]
else:
self.idle[GustafoBot.CHAT] = Timer(GustafoBot.TIMEOUT, self.on_chat_inactive)
res = State.forceState(InitialOutreach, {'_nick': user})
if res is not None:
self.send_message(user, res)
self.idle[user] = Timer(GustafoBot.TIMEOUT, self.on_user_inactive, [user])
self.idle[user].start()
def index():
token = Token(request)
if token.status:
HandleCreds(token, credentials, UserData, session)
access_token = session.get('access_token', False)
if access_token:
logging.debug('access_token: {}'.format(access_token))
resp = make_response(render_template('index.html'))
session_state = session.get('state', 0)
if session_state == 0:
state = State(uuid.uuid4())
state.insert()
resp.set_cookie('state', state.state)
session['state'] = state.state
else:
state = State(session_state)
state.insert()
resp.set_cookie('state', state.state)
return resp
def parseState(str):
"""文字列を読み取って状態と ID の組 (state(ID, name) の形で表される) を取得し、ID をキーとする辞書を返します
Args:
str (string): 入力文字列
Returns:
dictionary: ID をキーとし、値に State オブジェクトをもつ辞書
"""
states = {}
state_group = re.findall("state\([0-9]+,\{.*?\}\)", str)
for state in state_group:
id = re.search("[0-9]+", state).group(0)
name = re.search("\{.*?\}", state).group(0)
lastPos = 0
# 状態に適宜改行を入れて読みやすくする
while name.find(" ", lastPos + 15) != -1:
lastPos = name.find(" ", lastPos + 15)
if name[lastPos + 1] != "}":
name = name[:lastPos] + "\n" + name[lastPos+1:]
states[id] = State(name)
return states
def die(self):
State.die()
for timer in self.idle.values():
timer.cancel()
def parse(self, passages, mode="test"):
"""
Parse given passages
:param passages: iterable of passages to parse
:param mode: "train", "test" or "dev".
If "train", use oracle to train on given passages.
Otherwise, just parse with classifier.
:return: generator of pairs of (parsed passage, original passage)
"""
train = mode == "train"
dev = mode == "dev"
test = mode == "test"
assert train or dev or test, "Invalid parse mode: %s" % mode
passage_word = "sentence" if Config().sentences else \
"paragraph" if Config().paragraphs else \
"passage"
self.total_actions = 0
self.total_correct = 0
total_duration = 0
total_tokens = 0
num_passages = 0
for passage in passages:
l0 = passage.layer(layer0.LAYER_ID)
num_tokens = len(l0.all)
total_tokens += num_tokens
l1 = passage.layer(layer1.LAYER_ID)
labeled = len(l1.all) > 1
assert not train or labeled, "Cannot train on unannotated passage"
print("%s %-7s" % (passage_word, passage.ID), end=Config().line_end, flush=True)
started = time.time()
self.action_count = 0
self.correct_count = 0
self.state = State(passage, callback=self.pos_tag)
self.state_hash_history = set()
self.oracle = Oracle(passage) if train else None
failed = False
try:
self.parse_passage(train) # This is where the actual parsing takes place
except ParserException as e:
if train:
raise
Config().log("%s %s: %s" % (passage_word, passage.ID, e))
if not test:
print("failed")
failed = True
predicted_passage = passage
if not train or Config().verify:
predicted_passage = self.state.create_passage(assert_proper=Config().verify)
duration = time.time() - started
total_duration += duration
if train: # We have an oracle to verify by
if not failed and Config().verify:
self.verify_passage(passage, predicted_passage, train)
if self.action_count:
print("%-16s" % ("%d%% (%d/%d)" %
(100 * self.correct_count / self.action_count,
self.correct_count, self.action_count)), end=Config().line_end)
print("%0.3fs" % duration, end="")
print("%-15s" % ("" if failed else " (%d tokens/s)" % (num_tokens / duration)), end="")
print(Config().line_end, end="")
if train:
print(Config().line_end, flush=True)
self.total_correct += self.correct_count
self.total_actions += self.action_count
num_passages += 1
yield predicted_passage, passage
if num_passages > 1:
print("Parsed %d %ss" % (num_passages, passage_word))
if self.oracle and self.total_actions:
print("Overall %d%% correct transitions (%d/%d) on %s" %
(100 * self.total_correct / self.total_actions,
self.total_correct, self.total_actions,
mode))
print("Total time: %.3fs (average time/%s: %.3fs, average tokens/s: %d)" % (
total_duration, passage_word, total_duration / num_passages,
total_tokens / total_duration), flush=True)
class Parser(object):
"""
Main class to implement transition-based UCCA parser
"""
def __init__(self, model_file=None, model_type="sparse"):
self.state = None # State object created at each parse
self.oracle = None # Oracle object created at each parse
self.scores = None # NumPy array of action scores at each action
self.action_count = 0
self.correct_count = 0
self.total_actions = 0
self.total_correct = 0
self.feature_extractor, self.model = models.create_model(model_type, Actions().all)
self.model_file = model_file
self.learning_rate = Config().learning_rate
self.decay_factor = Config().decay_factor
self.state_hash_history = None # For loop checking
# Used in verify_passage to optionally ignore a mismatch in linkage nodes:
self.ignore_node = lambda n: n.tag == layer1.NodeTags.Linkage if Config().no_linkage else None
def train(self, passages, dev=None, iterations=1, folds=None):
"""
Train parser on given passages
:param passages: iterable of passages to train on
:param dev: iterable of passages to tune on
:param iterations: number of iterations to perform
:param folds: whether we are inside cross-validation with this many folds
:return: trained model
"""
if not passages:
if self.model_file is not None: # Nothing to train on; pre-trained model given
self.model.load(self.model_file, util)
Actions().all = self.model.labels
return self.model
best_score = 0
best_model = None
save_model = True
last = False
if Config().dev_scores:
with open(Config().dev_scores, "w") as f:
print(",".join(["iteration"] + evaluation.Scores.field_titles()), file=f)
for iteration in range(iterations):
if last:
break
last = iteration == iterations - 1
print("Training iteration %d of %d: " % (iteration + 1, iterations))
passages = [passage for _, passage in self.parse(passages, mode="train")]
model = self.model # Save non-finalize model
self.model = self.model.finalize() # To evaluate finalized model on dev
if last:
if folds is None: # Free some memory, as these are not needed any more
del passages[:]
else:
self.learning_rate *= self.decay_factor
Config().random.shuffle(passages)
if dev:
print("Evaluating on dev passages")
dev, scores = zip(*[(passage, evaluate_passage(predicted_passage, passage))
for predicted_passage, passage in
self.parse(dev, mode="dev")])
dev = list(dev)
scores = evaluation.Scores.aggregate(scores)
score = scores.average_unlabeled_f1()
print("Average unlabeled F1 score on dev: %.3f" % score)
if Config().dev_scores:
with open(Config().dev_scores, "a") as f:
print(",".join([str(iteration)] + scores.fields()), file=f)
if score >= best_score:
print("Better than previous best score (%.3f)" % best_score)
best_score = score
save_model = True
else:
print("Not better than previous best score (%.3f)" % best_score)
save_model = False
if score >= 1: # Score cannot go any better, so no point in more training
last = True
if last and folds is None: # Free more memory
del dev[:]
if save_model or best_model is None:
best_model = self.model # This is the finalized model
if self.model_file is not None:
best_model.save(self.model_file, util)
if not last:
self.model = model # Restore non-finalized model
print("Trained %d iterations" % iterations)
self.model = best_model
return self.model
def parse(self, passages, mode="test"):
"""
Parse given passages
:param passages: iterable of passages to parse
:param mode: "train", "test" or "dev".
If "train", use oracle to train on given passages.
Otherwise, just parse with classifier.
:return: generator of pairs of (parsed passage, original passage)
"""
train = mode == "train"
dev = mode == "dev"
test = mode == "test"
assert train or dev or test, "Invalid parse mode: %s" % mode
passage_word = "sentence" if Config().sentences else \
"paragraph" if Config().paragraphs else \
"passage"
self.total_actions = 0
self.total_correct = 0
total_duration = 0
total_tokens = 0
num_passages = 0
for passage in passages:
l0 = passage.layer(layer0.LAYER_ID)
num_tokens = len(l0.all)
total_tokens += num_tokens
l1 = passage.layer(layer1.LAYER_ID)
labeled = len(l1.all) > 1
assert not train or labeled, "Cannot train on unannotated passage"
print("%s %-7s" % (passage_word, passage.ID), end=Config().line_end, flush=True)
started = time.time()
self.action_count = 0
self.correct_count = 0
self.state = State(passage, callback=self.pos_tag)
self.state_hash_history = set()
self.oracle = Oracle(passage) if train else None
failed = False
try:
self.parse_passage(train) # This is where the actual parsing takes place
except ParserException as e:
if train:
raise
Config().log("%s %s: %s" % (passage_word, passage.ID, e))
if not test:
print("failed")
failed = True
predicted_passage = passage
if not train or Config().verify:
predicted_passage = self.state.create_passage(assert_proper=Config().verify)
duration = time.time() - started
total_duration += duration
if train: # We have an oracle to verify by
if not failed and Config().verify:
self.verify_passage(passage, predicted_passage, train)
if self.action_count:
print("%-16s" % ("%d%% (%d/%d)" %
(100 * self.correct_count / self.action_count,
self.correct_count, self.action_count)), end=Config().line_end)
print("%0.3fs" % duration, end="")
print("%-15s" % ("" if failed else " (%d tokens/s)" % (num_tokens / duration)), end="")
print(Config().line_end, end="")
if train:
print(Config().line_end, flush=True)
self.total_correct += self.correct_count
self.total_actions += self.action_count
num_passages += 1
yield predicted_passage, passage
if num_passages > 1:
print("Parsed %d %ss" % (num_passages, passage_word))
if self.oracle and self.total_actions:
print("Overall %d%% correct transitions (%d/%d) on %s" %
(100 * self.total_correct / self.total_actions,
self.total_correct, self.total_actions,
mode))
print("Total time: %.3fs (average time/%s: %.3fs, average tokens/s: %d)" % (
total_duration, passage_word, total_duration / num_passages,
total_tokens / total_duration), flush=True)
def parse_passage(self, train=False):
"""
Internal method to parse a single passage
:param train: use oracle to train on given passages, or just parse with classifier?
"""
if Config().verbose:
print(" initial state: %s" % self.state)
while True:
if Config().check_loops:
self.check_loop(print_oracle=train)
true_actions = []
if self.oracle is not None:
try:
true_actions = self.oracle.get_actions(self.state)
except (AttributeError, AssertionError) as e:
if train:
raise ParserException("Error in oracle during training") from e
features = self.feature_extractor.extract_features(self.state)
predicted_action = self.predict_action(features, true_actions) # sets self.scores
action = predicted_action
if not true_actions:
true_actions = "?"
elif predicted_action in true_actions:
self.correct_count += 1
elif train:
best_true_action = true_actions[0] if len(true_actions) == 1 else \
true_actions[self.scores[[a.id for a in true_actions]].argmax()]
rate = self.learning_rate
if best_true_action.is_swap:
rate *= Config().importance
self.model.update(features, predicted_action.id, best_true_action.id, rate)
action = Config().random.choice(true_actions)
self.action_count += 1
try:
self.state.transition(action)
except AssertionError as e:
raise ParserException("Invalid transition (%s): %s" % (action, e)) from e
if Config().verbose:
if self.oracle is None:
print(" action: %-15s %s" % (action, self.state))
else:
print(" predicted: %-15s true: %-15s taken: %-15s %s" % (
predicted_action, "|".join(str(true_action) for true_action in true_actions),
action, self.state))
for line in self.state.log:
print(" " + line)
if self.state.finished:
return # action is FINISH
def check_loop(self, print_oracle):
"""
Check if the current state has already occurred, indicating a loop
:param print_oracle: whether to print the oracle in case of an assertion error
"""
h = hash(self.state)
assert h not in self.state_hash_history,\
"\n".join(["Transition loop", self.state.str("\n")] +
[self.oracle.str("\n")] if print_oracle else ())
self.state_hash_history.add(h)
def predict_action(self, features, true_actions):
"""
Choose action based on classifier
:param features: extracted feature values
:param true_actions: from the oracle, to copy orig_node if the same action is selected
:return: valid action with maximum probability according to classifier
"""
self.scores = self.model.score(features) # Returns a NumPy array
best_action = self.select_action(self.scores.argmax(), true_actions)
if self.state.is_valid(best_action):
return best_action
# Usually the best action is valid, so max is enough to choose it in O(n) time
# Otherwise, sort all the other scores to choose the best valid one in O(n lg n)
sorted_ids = self.scores.argsort()[-2::-1]
actions = (self.select_action(i, true_actions) for i in sorted_ids)
try:
return next(a for a in actions if self.state.is_valid(a))
except StopIteration as e:
raise ParserException("No valid actions available\n" +
("True actions: %s" % true_actions if true_actions
else self.oracle.log if self.oracle is not None
else "")) from e
@staticmethod
def select_action(i, true_actions):
"""
Find action with the given ID in true actions (if exists) or in all actions
:param i: ID to lookup
:param true_actions: preferred set of actions to look in first
:return: Action with id=i
"""
try:
return next(a for a in true_actions if a.id == i)
except StopIteration:
return Actions().all[i]
def verify_passage(self, passage, predicted_passage, show_diff):
"""
Compare predicted passage to true passage and die if they differ
:param passage: true passage
:param predicted_passage: predicted passage to compare
:param show_diff: if passages differ, show the difference between them?
Depends on predicted_passage having the original node IDs annotated
in the "remarks" field for each node.
"""
assert passage.equals(predicted_passage, ignore_node=self.ignore_node),\
"Failed to produce true passage" + \
(diffutil.diff_passages(
passage, predicted_passage) if show_diff else "")
@staticmethod
def pos_tag(state):
"""
Function to pass to State to POS tag the tokens when created
:param state: State object to modify
"""
tokens = [token for tokens in state.tokens for token in tokens]
tokens, tags = zip(*pos_tag(tokens))
if Config().verbose:
print(" ".join("%s/%s" % (token, tag) for (token, tag) in zip(tokens, tags)))
for node, tag in zip(state.nodes, tags):
node.pos_tag = tag
def move(self, state: State, opponent: Player):
movement = optimum_move(state.board(), opponent.symbol, self.symbol, self.symbol).index
r, c = movement
state.move(r, c, self.symbol)
def __init__(self, server=None):
State.__init__(self, server)
self.voteReceived = {self.server.id: 1}
self.votedFor = self.server.id
self.server.setElectionTimer()
self.requestElection()
class Parser(object):
"""
Main class to implement transition-based UCCA parser
"""
def __init__(self, model_file=None, model_type="sparse"):
self.state = None # State object created at each parse
self.oracle = None # Oracle object created at each parse
self.scores = None # NumPy array of action scores at each action
self.action_count = 0
self.correct_count = 0
self.total_actions = 0
self.total_correct = 0
self.feature_extractor, self.model = models.create_model(
model_type,
Actions().all)
self.model_file = model_file
self.learning_rate = Config().learning_rate
self.decay_factor = Config().decay_factor
self.state_hash_history = None # For loop checking
# Used in verify_passage to optionally ignore a mismatch in linkage nodes:
self.ignore_node = lambda n: n.tag == layer1.NodeTags.Linkage if Config(
).no_linkage else None
def train(self, passages, dev=None, iterations=1, folds=None):
"""
Train parser on given passages
:param passages: iterable of passages to train on
:param dev: iterable of passages to tune on
:param iterations: number of iterations to perform
:param folds: whether we are inside cross-validation with this many folds
:return: trained model
"""
if not passages:
if self.model_file is not None: # Nothing to train on; pre-trained model given
self.model.load(self.model_file, util)
Actions().all = self.model.labels
return self.model
best_score = 0
best_model = None
save_model = True
last = False
if Config().dev_scores:
with open(Config().dev_scores, "w") as f:
print(",".join(["iteration"] +
evaluation.Scores.field_titles()),
file=f)
for iteration in range(iterations):
if last:
break
last = iteration == iterations - 1
print("Training iteration %d of %d: " %
(iteration + 1, iterations))
passages = [
passage for _, passage in self.parse(passages, mode="train")
]
model = self.model # Save non-finalize model
self.model = self.model.finalize(
) # To evaluate finalized model on dev
if last:
if folds is None: # Free some memory, as these are not needed any more
del passages[:]
else:
self.learning_rate *= self.decay_factor
Config().random.shuffle(passages)
if dev:
print("Evaluating on dev passages")
dev, scores = zip(
*[(passage, evaluate_passage(predicted_passage, passage))
for predicted_passage, passage in self.parse(
dev, mode="dev")])
dev = list(dev)
scores = evaluation.Scores.aggregate(scores)
score = scores.average_unlabeled_f1()
print("Average unlabeled F1 score on dev: %.3f" % score)
if Config().dev_scores:
with open(Config().dev_scores, "a") as f:
print(",".join([str(iteration)] + scores.fields()),
file=f)
if score >= best_score:
print("Better than previous best score (%.3f)" %
best_score)
best_score = score
save_model = True
else:
print("Not better than previous best score (%.3f)" %
best_score)
save_model = False
if score >= 1: # Score cannot go any better, so no point in more training
last = True
if last and folds is None: # Free more memory
del dev[:]
if save_model or best_model is None:
best_model = self.model # This is the finalized model
if self.model_file is not None:
best_model.save(self.model_file, util)
if not last:
self.model = model # Restore non-finalized model
print("Trained %d iterations" % iterations)
self.model = best_model
return self.model
def parse(self, passages, mode="test"):
"""
Parse given passages
:param passages: iterable of passages to parse
:param mode: "train", "test" or "dev".
If "train", use oracle to train on given passages.
Otherwise, just parse with classifier.
:return: generator of pairs of (parsed passage, original passage)
"""
train = mode == "train"
dev = mode == "dev"
test = mode == "test"
assert train or dev or test, "Invalid parse mode: %s" % mode
passage_word = "sentence" if Config().sentences else \
"paragraph" if Config().paragraphs else \
"passage"
self.total_actions = 0
self.total_correct = 0
total_duration = 0
total_tokens = 0
num_passages = 0
for passage in passages:
l0 = passage.layer(layer0.LAYER_ID)
num_tokens = len(l0.all)
total_tokens += num_tokens
l1 = passage.layer(layer1.LAYER_ID)
labeled = len(l1.all) > 1
assert not train or labeled, "Cannot train on unannotated passage"
print("%s %-7s" % (passage_word, passage.ID),
end=Config().line_end,
flush=True)
started = time.time()
self.action_count = 0
self.correct_count = 0
self.state = State(passage, callback=self.pos_tag)
self.state_hash_history = set()
self.oracle = Oracle(passage) if train else None
failed = False
try:
self.parse_passage(
train) # This is where the actual parsing takes place
except ParserException as e:
if train:
raise
Config().log("%s %s: %s" % (passage_word, passage.ID, e))
if not test:
print("failed")
failed = True
predicted_passage = passage
if not train or Config().verify:
predicted_passage = self.state.create_passage(
assert_proper=Config().verify)
duration = time.time() - started
total_duration += duration
if train: # We have an oracle to verify by
if not failed and Config().verify:
self.verify_passage(passage, predicted_passage, train)
if self.action_count:
print("%-16s" %
("%d%% (%d/%d)" %
(100 * self.correct_count / self.action_count,
self.correct_count, self.action_count)),
end=Config().line_end)
print("%0.3fs" % duration, end="")
print("%-15s" % ("" if failed else " (%d tokens/s)" %
(num_tokens / duration)),
end="")
print(Config().line_end, end="")
if train:
print(Config().line_end, flush=True)
self.total_correct += self.correct_count
self.total_actions += self.action_count
num_passages += 1
yield predicted_passage, passage
if num_passages > 1:
print("Parsed %d %ss" % (num_passages, passage_word))
if self.oracle and self.total_actions:
print("Overall %d%% correct transitions (%d/%d) on %s" %
(100 * self.total_correct / self.total_actions,
self.total_correct, self.total_actions, mode))
print(
"Total time: %.3fs (average time/%s: %.3fs, average tokens/s: %d)"
% (total_duration, passage_word, total_duration / num_passages,
total_tokens / total_duration),
flush=True)
def parse_passage(self, train=False):
"""
Internal method to parse a single passage
:param train: use oracle to train on given passages, or just parse with classifier?
"""
if Config().verbose:
print(" initial state: %s" % self.state)
while True:
if Config().check_loops:
self.check_loop(print_oracle=train)
true_actions = []
if self.oracle is not None:
try:
true_actions = self.oracle.get_actions(self.state)
except (AttributeError, AssertionError) as e:
if train:
raise ParserException(
"Error in oracle during training") from e
features = self.feature_extractor.extract_features(self.state)
predicted_action = self.predict_action(
features, true_actions) # sets self.scores
action = predicted_action
if not true_actions:
true_actions = "?"
elif predicted_action in true_actions:
self.correct_count += 1
elif train:
best_true_action = true_actions[0] if len(true_actions) == 1 else \
true_actions[self.scores[[a.id for a in true_actions]].argmax()]
rate = self.learning_rate
if best_true_action.is_swap:
rate *= Config().importance
self.model.update(features, predicted_action.id,
best_true_action.id, rate)
action = Config().random.choice(true_actions)
self.action_count += 1
try:
self.state.transition(action)
except AssertionError as e:
raise ParserException("Invalid transition (%s): %s" %
(action, e)) from e
if Config().verbose:
if self.oracle is None:
print(" action: %-15s %s" % (action, self.state))
else:
print(" predicted: %-15s true: %-15s taken: %-15s %s" %
(predicted_action, "|".join(
str(true_action)
for true_action in true_actions), action,
self.state))
for line in self.state.log:
print(" " + line)
if self.state.finished:
return # action is FINISH
def check_loop(self, print_oracle):
"""
Check if the current state has already occurred, indicating a loop
:param print_oracle: whether to print the oracle in case of an assertion error
"""
h = hash(self.state)
assert h not in self.state_hash_history,\
"\n".join(["Transition loop", self.state.str("\n")] +
[self.oracle.str("\n")] if print_oracle else ())
self.state_hash_history.add(h)
def predict_action(self, features, true_actions):
"""
Choose action based on classifier
:param features: extracted feature values
:param true_actions: from the oracle, to copy orig_node if the same action is selected
:return: valid action with maximum probability according to classifier
"""
self.scores = self.model.score(features) # Returns a NumPy array
best_action = self.select_action(self.scores.argmax(), true_actions)
if self.state.is_valid(best_action):
return best_action
# Usually the best action is valid, so max is enough to choose it in O(n) time
# Otherwise, sort all the other scores to choose the best valid one in O(n lg n)
sorted_ids = self.scores.argsort()[-2::-1]
actions = (self.select_action(i, true_actions) for i in sorted_ids)
try:
return next(a for a in actions if self.state.is_valid(a))
except StopIteration as e:
raise ParserException("No valid actions available\n" + (
"True actions: %s" % true_actions if true_actions else self.
oracle.log if self.oracle is not None else "")) from e
@staticmethod
def select_action(i, true_actions):
"""
Find action with the given ID in true actions (if exists) or in all actions
:param i: ID to lookup
:param true_actions: preferred set of actions to look in first
:return: Action with id=i
"""
try:
return next(a for a in true_actions if a.id == i)
except StopIteration:
return Actions().all[i]
def verify_passage(self, passage, predicted_passage, show_diff):
"""
Compare predicted passage to true passage and die if they differ
:param passage: true passage
:param predicted_passage: predicted passage to compare
:param show_diff: if passages differ, show the difference between them?
Depends on predicted_passage having the original node IDs annotated
in the "remarks" field for each node.
"""
assert passage.equals(predicted_passage, ignore_node=self.ignore_node),\
"Failed to produce true passage" + \
(diffutil.diff_passages(
passage, predicted_passage) if show_diff else "")
@staticmethod
def pos_tag(state):
"""
Function to pass to State to POS tag the tokens when created
:param state: State object to modify
"""
tokens = [token for tokens in state.tokens for token in tokens]
tokens, tags = zip(*pos_tag(tokens))
if Config().verbose:
print(" ".join("%s/%s" % (token, tag)
for (token, tag) in zip(tokens, tags)))
for node, tag in zip(state.nodes, tags):
node.pos_tag = tag
def parse(self, passages, mode="test"):
"""
Parse given passages
:param passages: iterable of passages to parse
:param mode: "train", "test" or "dev".
If "train", use oracle to train on given passages.
Otherwise, just parse with classifier.
:return: generator of pairs of (parsed passage, original passage)
"""
train = mode == "train"
dev = mode == "dev"
test = mode == "test"
assert train or dev or test, "Invalid parse mode: %s" % mode
passage_word = "sentence" if Config().sentences else \
"paragraph" if Config().paragraphs else \
"passage"
self.total_actions = 0
self.total_correct = 0
total_duration = 0
total_tokens = 0
num_passages = 0
for passage in passages:
l0 = passage.layer(layer0.LAYER_ID)
num_tokens = len(l0.all)
total_tokens += num_tokens
l1 = passage.layer(layer1.LAYER_ID)
labeled = len(l1.all) > 1
assert not train or labeled, "Cannot train on unannotated passage"
print("%s %-7s" % (passage_word, passage.ID),
end=Config().line_end,
flush=True)
started = time.time()
self.action_count = 0
self.correct_count = 0
self.state = State(passage, callback=self.pos_tag)
self.state_hash_history = set()
self.oracle = Oracle(passage) if train else None
failed = False
try:
self.parse_passage(
train) # This is where the actual parsing takes place
except ParserException as e:
if train:
raise
Config().log("%s %s: %s" % (passage_word, passage.ID, e))
if not test:
print("failed")
failed = True
predicted_passage = passage
if not train or Config().verify:
predicted_passage = self.state.create_passage(
assert_proper=Config().verify)
duration = time.time() - started
total_duration += duration
if train: # We have an oracle to verify by
if not failed and Config().verify:
self.verify_passage(passage, predicted_passage, train)
if self.action_count:
print("%-16s" %
("%d%% (%d/%d)" %
(100 * self.correct_count / self.action_count,
self.correct_count, self.action_count)),
end=Config().line_end)
print("%0.3fs" % duration, end="")
print("%-15s" % ("" if failed else " (%d tokens/s)" %
(num_tokens / duration)),
end="")
print(Config().line_end, end="")
if train:
print(Config().line_end, flush=True)
self.total_correct += self.correct_count
self.total_actions += self.action_count
num_passages += 1
yield predicted_passage, passage
if num_passages > 1:
print("Parsed %d %ss" % (num_passages, passage_word))
if self.oracle and self.total_actions:
print("Overall %d%% correct transitions (%d/%d) on %s" %
(100 * self.total_correct / self.total_actions,
self.total_correct, self.total_actions, mode))
print(
"Total time: %.3fs (average time/%s: %.3fs, average tokens/s: %d)"
% (total_duration, passage_word, total_duration / num_passages,
total_tokens / total_duration),
flush=True)
