def all_features(parse_dict, constituent, i, constituents):
syntax_tree = constituent.syntax_tree
conn_category = Connectives_dict().conn_category
connective = constituent.connective
''' feat dict '''
feat_dict_CON_Str = {}
feat_dict_CON_LStr = {}
feat_dict_NT_Ctx = {}
feat_dict_CON_NT_Path = {}
feat_dict_CON_NT_Path_iLsib = {}
''' load dict '''
dict_CON_Str = NT_dict().dict_CON_Str
dict_CON_LStr = NT_dict().dict_CON_LStr
dict_NT_Ctx = NT_dict().dict_NT_Ctx
dict_CON_NT_Path = NT_dict().dict_CON_NT_Path
dict_CON_NT_Path_iLsib = NT_dict().dict_CON_NT_Path_iLsib
''' feature '''
conn_indices = connective.token_indices
DocID = connective.DocID
sent_index = connective.sent_index
conn_node = dict_util.get_conn_node(syntax_tree, conn_indices)
CON_Str = dict_util.get_CON_Str(parse_dict, DocID, sent_index,
conn_indices)
CON_LStr = CON_Str.lower()
CON_Cat = conn_category[connective.name]
CON_iLSib = dict_util.get_CON_iLSib(syntax_tree, conn_node)
CON_iRSib = dict_util.get_CON_iRSib(syntax_tree, conn_node)
NT_Ctx = dict_util.get_NT_Ctx(constituent)
CON_NT_Path = dict_util.get_CON_NT_Path(conn_node, constituent)
CON_NT_Position = dict_util.get_CON_NT_Position(conn_node, constituent)
if CON_iLSib > 1:
CON_NT_Path_iLsib = CON_NT_Path + ":>1"
else:
CON_NT_Path_iLsib = CON_NT_Path + ":<=1"
features = []
features.append(get_feature(feat_dict_CON_Str, dict_CON_Str, CON_Str))
features.append(get_feature(feat_dict_CON_LStr, dict_CON_LStr, CON_LStr))
features.append(get_feature(feat_dict_NT_Ctx, dict_NT_Ctx, NT_Ctx))
features.append(
get_feature(feat_dict_CON_NT_Path, dict_CON_NT_Path, CON_NT_Path))
features.append(
get_feature(feat_dict_CON_NT_Path_iLsib, dict_CON_NT_Path_iLsib,
CON_NT_Path_iLsib))
# cat
dict_category = {"subordinator": 1, "coordinator": 2, "adverbial": 3}
features.append(get_feature({}, dict_category, CON_Cat))
#number
features.append(Feature("", 1, {1: CON_iLSib}))
features.append(Feature("", 1, {1: CON_iRSib}))
#position
dict_position = {"right": 1, "left": 2}
features.append(get_feature({}, dict_position, CON_NT_Position))
return util.mergeFeatures(features)
def CON_Cat(parse_dict, constituent, i, constituents):
# load dict
dict_category = {"subordinator": 1, "coordinator": 2, "adverbial": 3}
# feature
conn_category = Connectives_dict().conn_category
connective = constituent.connective
CON_Cat = conn_category[connective.name]
return get_feature_by_feat(dict_category, CON_Cat)
def _check_connective_names(sent_tokens):
sent_tokens = [word.lower() for word in sent_tokens]
indices = []
conn_names = []
tagged = set([])
sortedConn = Connectives_dict().sorted_conns_list
for conn in sortedConn:
if '..' in conn:
c1, c2 = conn.split('..')
c1_indice = util.getSpanIndecesInSent(c1.split(),
sent_tokens) #[[7]]
c2_indice = util.getSpanIndecesInSent(c2.split(),
sent_tokens) #[[10]]
if c1_indice != [] and c2_indice != []:
if c1_indice[0][0] < c2_indice[0][0]:
temp = set([t for t in (c1_indice[0] + c2_indice[0])])
if tagged & temp == set([]):
indices.append(c1_indice[0] +
c2_indice[0]) # [[7], [10]]
conn_names.append(conn)
tagged = tagged.union(temp)
else:
c_indice = util.getSpanIndecesInSent(
conn.split(), sent_tokens) #[[2,6],[1,3],...]
if c_indice != []:
tt = []
for item in c_indice:
if set(item) & tagged == set([]):
tt.append(item)
c_indice = tt
if c_indice != []:
indices.extend([item for item in c_indice
]) #[([2,6], 'for instance'), ....]
tagged = tagged.union(set([r for t in c_indice
for r in t]))
conn_names.append(conn)
return conn_names, indices
def get_con_cat(arg_clauses, clause_index, parse_dict):
conn_category = Connectives_dict().conn_category
return conn_category[arg_clauses.conn_head_name]
def all_features(parse_dict, DocID, sent_index, conn_indices):
# feat dict
'''Z.Lin'''
feat_dict_CPOS_dict = {}
feat_dict_prev_C_dict = {}
feat_dict_prevPOS_dict = {}
feat_dict_prevPOS_CPOS_dict = {}
feat_dict_C_next_dict = {}
feat_dict_nextPOS_dict = {}
feat_dict_CPOS_nextPOS_dict = {}
feat_dict_CParent_to_root_path_dict = {}
feat_dict_compressed_CParent_to_root_path_dict = {}
'''Pitler'''
feat_dict_self_category_dict = {}
feat_dict_parent_category_dict = {}
feat_dict_left_sibling_category_dict = {}
feat_dict_right_sibling_category_dict = {}
''' conn_syn '''
feat_dict_conn_self_category_dict = {}
feat_dict_conn_parent_category_dict = {}
feat_dict_conn_left_sibling_category_dict = {}
feat_dict_conn_right_sibling_category_dict = {}
''' syn_syn '''
feat_dict_self_parent = {}
feat_dict_self_right = {}
feat_dict_self_left = {}
feat_dict_parent_left = {}
feat_dict_parent_right = {}
feat_dict_left_right = {}
#dict
'''Z.Lin'''
CPOS_dict = Connectives_dict().cpos_dict
prev_C_dict = Connectives_dict().prev_C_dict
prevPOS_dict = Connectives_dict().prevPOS_dict
prevPOS_CPOS_dict = Connectives_dict().prevPOS_CPOS_dict
C_next_dict = Connectives_dict().C_next_dict
nextPOS_dict = Connectives_dict().nextPOS_dict
CPOS_nextPOS_dict = Connectives_dict().CPOS_nextPOS_dict
CParent_to_root_path_dict = Connectives_dict().CParent_to_root_path_dict
compressed_CParent_to_root_path_dict = Connectives_dict(
).compressed_CParent_to_root_path_dict
'''Pitler'''
self_category_dict = Connectives_dict().self_category_dict
parent_category_dict = Connectives_dict().parent_category_dict
left_sibling_category_dict = Connectives_dict().left_sibling_category_dict
right_sibling_category_dict = Connectives_dict(
).right_sibling_category_dict
''' conn_syn '''
conn_self_category_dict = Connectives_dict().conn_self_category_dict
conn_parent_category_dict = Connectives_dict().conn_parent_category_dict
conn_left_sibling_category_dict = Connectives_dict(
).conn_left_sibling_category_dict
conn_right_sibling_category_dict = Connectives_dict(
).conn_right_sibling_category_dict
''' syn_syn '''
self_parent_dict = Connectives_dict().self_parent_dict
self_right_dict = Connectives_dict().self_right_dict
self_left_dict = Connectives_dict().self_left_dict
parent_left_dict = Connectives_dict().parent_left_dict
parent_right_dict = Connectives_dict().parent_right_dict
left_right_dict = Connectives_dict().left_right_dict
''' mine '''
dict_conn_lower_case = Connectives_dict().dict_conn_lower_case
dict_conn = Connectives_dict().dict_conn
dict_CParent_to_root_path_node_names = Connectives_dict(
).dict_CParent_to_root_path_node_names
dict_conn_rightSiblingCtx = Connectives_dict().dict_conn_rightSiblingCtx
dict_conn_parent_category_Ctx = Connectives_dict(
).dict_conn_parent_category_Ctx
''' c pos '''
pos_tag_list = []
for conn_index in conn_indices:
pos_tag_list.append(parse_dict[DocID]["sentences"][sent_index]["words"]
[conn_index][1]["PartOfSpeech"])
CPOS = "_".join(pos_tag_list)
''' prev '''
flag = 0
prev_index = conn_indices[0] - 1
prev_sent_index = sent_index
if prev_index < 0:
prev_index = -1
prev_sent_index -= 1
if prev_sent_index < 0:
flag = 1
if flag == 1:
prev = "NONE"
else:
prev = parse_dict[DocID]["sentences"][prev_sent_index]["words"][
prev_index][0]
''' conn_name '''
conn_name = " ".join([parse_dict[DocID]["sentences"][sent_index]["words"][word_token][0] \
for word_token in conn_indices ])
'''prevPOS'''
if prev == "NONE":
prevPOS = "NONE"
else:
prevPOS = parse_dict[DocID]["sentences"][prev_sent_index]["words"][
prev_index][1]["PartOfSpeech"]
'''next'''
sent_count = len(parse_dict[DocID]["sentences"])
sent_length = len(parse_dict[DocID]["sentences"][sent_index]["words"])
flag = 0
next_index = conn_indices[-1] + 1
next_sent_index = sent_index
if next_index >= sent_length:
next_sent_index += 1
next_index = 0
if next_sent_index >= sent_count:
flag = 1
if flag == 1:
next = "NONE"
else:
next = parse_dict[DocID]["sentences"][next_sent_index]["words"][
next_index][0]
''' next pos '''
if next == "NONE":
nextPOS = "NONE"
else:
nextPOS = parse_dict[DocID]["sentences"][next_sent_index]["words"][
next_index][1]["PartOfSpeech"]
parse_tree = parse_dict[DocID]["sentences"][sent_index]["parsetree"].strip(
)
syntax_tree = Syntax_tree(parse_tree)
''' c parent to root '''
if syntax_tree.tree == None:
cparent_to_root_path = "NONE_TREE"
else:
cparent_to_root_path = ""
for conn_index in conn_indices:
conn_node = syntax_tree.get_leaf_node_by_token_index(conn_index)
conn_parent_node = conn_node.up
cparent_to_root_path += syntax_tree.get_node_path_to_root(
conn_parent_node) + "&"
if cparent_to_root_path[-1] == "&":
cparent_to_root_path = cparent_to_root_path[:-1]
''' compressed c parent to root '''
if syntax_tree.tree == None:
compressed_path = "NONE_TREE"
else:
compressed_path = ""
for conn_index in conn_indices:
conn_node = syntax_tree.get_leaf_node_by_token_index(conn_index)
conn_parent_node = conn_node.up
path = syntax_tree.get_node_path_to_root(conn_parent_node)
compressed_path += util.get_compressed_path(path) + "&"
if compressed_path[-1] == "&":
compressed_path = compressed_path[:-1]
''' Pitler '''
if syntax_tree.tree == None:
self_category = "NONE_TREE"
else:
self_category = syntax_tree.get_self_category_node_by_token_indices(
conn_indices).name
if syntax_tree.tree == None:
parent_category = "NONE_TREE"
else:
parent_category_node = syntax_tree.get_parent_category_node_by_token_indices(
conn_indices)
if parent_category_node == None:
parent_category = "ROOT"
else:
parent_category = parent_category_node.name
if syntax_tree.tree == None:
left_sibling_category = "NONE_TREE"
else:
left_sibling_category_node = syntax_tree.get_left_sibling_category_node_by_token_indices(
conn_indices)
if left_sibling_category_node == None:
left_sibling_category = "NONE"
else:
left_sibling_category = left_sibling_category_node.name
if syntax_tree.tree == None:
right_sibling_category = "NONE_TREE"
else:
right_sibling_category_node = syntax_tree.get_right_sibling_category_node_by_token_indices(
conn_indices)
if right_sibling_category_node == None:
right_sibling_category = "NONE"
else:
right_sibling_category = right_sibling_category_node.name
prev_C = "%s|%s" % (prev, conn_name)
prePOS_CPOS = "%s|%s" % (prevPOS, CPOS)
C_next = "%s|%s" % (conn_name, next)
CPOS_nextPOS = "%s|%s" % (CPOS, nextPOS)
conn_self_category = "%s|%s" % (conn_name, self_category)
conn_parent_category = "%s|%s" % (conn_name, parent_category)
conn_left_sibling_category = "%s|%s" % (conn_name, left_sibling_category)
conn_right_sibling_category = "%s|%s" % (conn_name, right_sibling_category)
self_parent = "%s|%s" % (self_category, parent_category)
self_right = "%s|%s" % (self_category, right_sibling_category)
self_left = "%s|%s" % (self_category, left_sibling_category)
parent_left = "%s|%s" % (parent_category, left_sibling_category)
parent_right = "%s|%s" % (parent_category, right_sibling_category)
left_right = "%s|%s" % (left_sibling_category, right_sibling_category)
'''--- mine ---'''
conn_lower_case = conn_name.lower()
# prevPOS_C = "%s|%s" % (prevPOS, conn_name.lower())
if syntax_tree.tree == None:
_path = "NONE_TREE"
else:
_path = ""
for conn_index in conn_indices:
conn_node = syntax_tree.get_leaf_node_by_token_index(conn_index)
conn_parent_node = conn_node.up
_path += syntax_tree.get_node_path_to_root(
conn_parent_node) + "-->"
if _path[-3:] == "-->":
_path = _path[:-3]
# conn + connCtx
if syntax_tree.tree == None:
connCtx = "NONE_TREE"
else:
conn_node = syntax_tree.get_self_category_node_by_token_indices(
conn_indices)
connCtx = dict_util.get_node_Ctx(conn_node, syntax_tree)
conn_connCtx = "%s|%s" % (conn_name, connCtx)
# conn + right sibling ctx
if syntax_tree.tree == None:
rightSiblingCtx = "NONE_TREE"
else:
rightSibling_node = syntax_tree.get_right_sibling_category_node_by_token_indices(
conn_indices)
rightSiblingCtx = dict_util.get_node_linked_Ctx(
rightSibling_node, syntax_tree)
conn_rightSiblingCtx = "%s|%s" % (conn_name, rightSiblingCtx)
# conn _ left sibling ctx
if syntax_tree.tree == None:
leftSiblingCtx = "NONE_TREE"
else:
leftSibling_node = syntax_tree.get_left_sibling_category_node_by_token_indices(
conn_indices)
leftSiblingCtx = dict_util.get_node_linked_Ctx(leftSibling_node,
syntax_tree)
# conn parent category ctx
if syntax_tree.tree == None:
parent_categoryCtx = "NONE_TREE"
else:
parent_category_node = syntax_tree.get_parent_category_node_by_token_indices(
conn_indices)
parent_categoryCtx = dict_util.get_node_linked_Ctx(
parent_category_node, syntax_tree)
conn_parent_categoryCtx = "%s|%s" % (conn_name, parent_categoryCtx)
features = []
'''Z.Lin'''
features.append(get_feature(feat_dict_CPOS_dict, CPOS_dict, CPOS))
features.append(get_feature(feat_dict_prev_C_dict, prev_C_dict, prev_C))
features.append(get_feature(feat_dict_prevPOS_dict, prevPOS_dict, prevPOS))
features.append(
get_feature(feat_dict_prevPOS_CPOS_dict, prevPOS_CPOS_dict,
prePOS_CPOS))
features.append(get_feature(feat_dict_C_next_dict, C_next_dict, C_next))
features.append(get_feature(feat_dict_nextPOS_dict, nextPOS_dict, nextPOS))
features.append(
get_feature(feat_dict_CPOS_nextPOS_dict, CPOS_nextPOS_dict,
CPOS_nextPOS))
features.append(
get_feature(feat_dict_CParent_to_root_path_dict,
CParent_to_root_path_dict, cparent_to_root_path))
features.append(
get_feature(feat_dict_compressed_CParent_to_root_path_dict,
compressed_CParent_to_root_path_dict, compressed_path))
''' pitler '''
features.append(
get_feature(feat_dict_self_category_dict, self_category_dict,
self_category))
features.append(
get_feature(feat_dict_parent_category_dict, parent_category_dict,
parent_category))
features.append(
get_feature(feat_dict_left_sibling_category_dict,
left_sibling_category_dict, left_sibling_category))
features.append(
get_feature(feat_dict_right_sibling_category_dict,
right_sibling_category_dict, right_sibling_category))
feat_dict_is_right_sibling_contains_VP = {}
if syntax_tree.tree != None and right_sibling_category_node != None:
T = right_sibling_category_node.get_descendants()
T.append(right_sibling_category_node)
for node in T:
if node.name == "VP" or node.name == "S":
feat_dict_is_right_sibling_contains_VP[1] = 1
break
features.append(Feature("", 1, feat_dict_is_right_sibling_contains_VP))
''' conn-syn '''
features.append(
get_feature(feat_dict_conn_self_category_dict, conn_self_category_dict,
conn_self_category))
features.append(
get_feature(feat_dict_conn_parent_category_dict,
conn_parent_category_dict, conn_parent_category))
features.append(
get_feature(feat_dict_conn_left_sibling_category_dict,
conn_left_sibling_category_dict,
conn_left_sibling_category))
features.append(
get_feature(feat_dict_conn_right_sibling_category_dict,
conn_right_sibling_category_dict,
conn_right_sibling_category))
''' syn-syn '''
features.append(
get_feature(feat_dict_self_parent, self_parent_dict, self_parent))
features.append(
get_feature(feat_dict_self_right, self_right_dict, self_right))
features.append(get_feature(feat_dict_self_left, self_left_dict,
self_left))
features.append(
get_feature(feat_dict_parent_left, parent_left_dict, parent_left))
features.append(
get_feature(feat_dict_parent_right, parent_right_dict, parent_right))
features.append(
get_feature(feat_dict_left_right, left_right_dict, left_right))
''' mine '''
features.append(get_feature_by_feat(dict_conn_lower_case, conn_lower_case))
features.append(get_feature_by_feat(dict_conn, conn_name))
features.append(
get_feature_by_feat_list(dict_CParent_to_root_path_node_names,
_path.split("-->")))
features.append(
get_feature_by_feat(dict_conn_rightSiblingCtx, conn_rightSiblingCtx))
features.append(
get_feature_by_feat(dict_conn_parent_category_Ctx,
conn_parent_categoryCtx))
return util.mergeFeatures(features)