def extract(
mention_id="text",
doc_begin_index="int",
doc_end_index="int",
doc_id="text",
position="text",
sentence_index="int",
tokens="text[]",
pos_tags="text[]",
):
# Constant
# WINDOW_SIZE = 10
# Load keyword dictionaries using ddlib, for domain-specific features
# Words in "legal_penalty" dictionary are indicative of marriage
# Words in "non_legal_penalty" dictionary are indicative of non_marriage
APP_HOME = os.environ['APP_HOME']
ddlib.load_dictionary(APP_HOME + "/udf/dicts/kw_crime.txt",
dict_id="crime")
ddlib.load_dictionary(APP_HOME + "/udf/dicts/kw_non_crime.txt",
dict_id="non_crime")
# kw_non_legal_penalty = map(lambda word: word.strip(), open(APP_HOME + "/udf/dicts/kw_non_legal_penalty.txt", 'r').readlines())
# kw_legal_penalty = map(lambda word: word.strip(), open(APP_HOME + "/udf/dicts/kw_legal_penalty.txt", 'r').readlines())
# Non penalty signals on the left of candidate mention
# NON_PENAL_SIGNALS_LEFT = frozenset(kw_non_legal_penalty)
# Penalty signals on the right of candidate mention
# PENAL_SIGNALS_LEFT = frozenset(kw_legal_penalty)
WINDOW_SIZE = 10
MAX_PHRASE_LENGTH = 5
# Get all subsequences of left sentence with WINDOW_SIZE = 10
low_tokens = map(lambda token: token.lower(), tokens)
left_window = get_left_window(doc_begin_index, low_tokens, WINDOW_SIZE)
phrases_in_sentence_left = list(
get_all_phrases_in_sentence(left_window, MAX_PHRASE_LENGTH))
# Create a DDLIB sentence object, which is just a list of DDLIB Word objects
sent = []
for i, t in enumerate(tokens):
sent.append(
ddlib.Word(
begin_char_offset=None,
end_char_offset=None,
word=t,
lemma=tokens[i], # lemma for vietnamese: lowercase
pos=pos_tags[i],
ner=None,
dep_par=
-1, # Note that as stored from CoreNLP 0 is ROOT, but for DDLIB -1 is ROOT
dep_label=None))
# Create DDLIB Span for penalty candidate
penalty_span = ddlib.Span(begin_word_id=doc_begin_index,
length=(doc_end_index - doc_begin_index + 1))
# Generate the generic features using DDLIB on left and right window
for feature in ddlib.get_generic_features_mention(sent, penalty_span):
yield [mention_id, feature]
def extract(
organization_id="text",
begin_index="int",
end_index="int",
doc_id="text",
sentence_index="int",
tokens="text[]",
pos_tags="text[]",
dep_types="text[]",
dep_heads="int[]",
):
sent = []
for i, t in enumerate(tokens):
sent.append(
ddlib.Word(
begin_char_offset=None,
end_char_offset=None,
word=t,
lemma=tokens[i],
pos=pos_tags[i],
ner=None,
dep_par=dep_heads[i] -
1, # Note that as stored from CoreNLP 0 is ROOT, but for DDLIB -1 is ROOT
dep_label=dep_types[i]))
####
org_span = ddlib.Span(begin_word_id=begin_index,
length=(end_index - begin_index + 1))
for feature in ddlib.get_generic_features_mention(sent, org_span):
yield [organization_id, feature]
def create_ddlib_sentence(row):
"""Create a list of ddlib.Word objects from input row."""
sentence = []
for i, word in enumerate(row.words):
sentence.append(
ddlib.Word(begin_char_offset=None,
end_char_offset=None,
word=word,
lemma=row.lemmas[i],
pos=row.poses[i],
ner=row.ners[i],
dep_par=row.dep_parents[i],
dep_label=row.dep_paths[i]))
return sentence
def unpack_(begin_char_offsets, end_char_offsets, words, lemmas, poses,
ners, dep_parents, dep_paths):
wordobjs = []
for i in range(0, len(words)):
wordobjs.append(
ddlib.Word(
begin_char_offset=None,
end_char_offset=None,
word=words[i],
lemma=lemmas[i],
pos=poses[i],
ner='', # NER is noisy on medical docs
dep_par=dep_parents[i],
dep_label=dep_paths[i]))
return wordobjs
def extract(
p_id="text",
e_id="text",
p_begin_index="int",
p_end_index="int",
e_begin_index="int",
e_end_index="int",
doc_id="text",
sent_index="int",
tokens="text[]",
lemmas="text[]",
pos_tags="text[]",
ner_tags="text[]",
dep_types="text[]",
dep_parents="int[]",
):
"""
Uses DDLIB to generate features for the spouse relation.
"""
ddlib.load_dictionary(os.path.abspath("../../../job_employ_keyword.txt"),
dict_id="has_employment")
ddlib.load_dictionary(
os.path.abspath("../../../job_no_employ_keyword.txt"),
dict_id="no_employment")
# Create a DDLIB sentence object, which is just a list of DDLIB Word objects
sent = []
for i, t in enumerate(tokens):
sent.append(
ddlib.Word(
begin_char_offset=None,
end_char_offset=None,
word=t,
lemma=lemmas[i],
pos=pos_tags[i],
ner=ner_tags[i],
dep_par=dep_parents[i] -
1, # Note that as stored from CoreNLP 0 is ROOT, but for DDLIB -1 is ROOT
dep_label=dep_types[i]))
# Create DDLIB Spans for the two mentions
p_span = ddlib.Span(begin_word_id=p_begin_index,
length=(p_end_index - p_begin_index + 1))
e_span = ddlib.Span(begin_word_id=e_begin_index,
length=(e_end_index - e_begin_index + 1))
# Generate the generic features using DDLIB
for feature in ddlib.get_generic_features_relation(sent, p_span, e_span):
yield [p_id, e_id, feature]
def extract(
chemical_id = "text",
disease_id = "text",
chemical_begin_index = "int",
chemical_end_index = "int",
disease_begin_index = "int",
disease_end_index = "int",
doc_id = "text",
sent_index = "int",
tokens = "text[]",
lemmas = "text[]",
pos_tags = "text[]",
ner_tags = "text[]",
my_ner_tags = "text[]",
my_ner_tags_token_ids = "int[]",
dep_types = "text[]",
dep_parents = "int[]",
):
"""
Uses DDLIB to generate features for the chemical-disease relation candidates.
"""
# creates a dictionary of tags from the sparse my_ner_tags array
my_ner_tags_dict = { i:tag for i,tag in zip(my_ner_tags_token_ids, my_ner_tags) }
sent = []
for i,t in enumerate(tokens):
sent.append(ddlib.Word(
begin_char_offset=None,
end_char_offset=None,
word=t,
lemma=lemmas[i],
pos=pos_tags[i],
# replace NER tag if one is found for that token in my_ner_tags:
ner=my_ner_tags_dict[i] if i in my_ner_tags_dict else ner_tags[i],
dep_par=dep_parents[i] - 1, # Note that as stored from CoreNLP 0 is ROOT, but for DDLIB -1 is ROOT
dep_label=dep_types[i]))
# Create DDLIB Spans for the two person mentions
chemical_span = ddlib.Span(begin_word_id=chemical_begin_index, length=(chemical_end_index-chemical_begin_index+1))
disease_span = ddlib.Span(begin_word_id=disease_begin_index, length=(disease_end_index-disease_begin_index+1))
# Generate the generic features using DDLIB
for feature in ddlib.get_generic_features_relation(sent, chemical_span, disease_span):
yield [chemical_id, disease_id, feature]
def extract(
gene_id="text",
variation_id="text",
gene_begin_index="int",
gene_end_index="int",
var_begin_index="int",
var_end_index="int",
doc_id="text",
sent_index="int",
tokens="text[]",
lemmas="text[]",
pos_tags="text[]",
ner_tags="text[]",
dep_types="text[]",
dep_parents="int[]",
):
"""
Uses DDLIB to generate features for the spouse relation.
"""
# Create a DDLIB sentence object, which is just a list of DDLIB Word objects
sent = []
for i, t in enumerate(tokens):
sent.append(
ddlib.Word(
begin_char_offset=None,
end_char_offset=None,
word=t,
lemma=lemmas[i],
pos=pos_tags[i],
ner=ner_tags[i],
dep_par=dep_parents[i] -
1, # Note that as stored from CoreNLP 0 is ROOT, but for DDLIB -1 is ROOT
dep_label=dep_types[i]))
# Create DDLIB Spans for the gene and variation mentions
gene_span = ddlib.Span(begin_word_id=gene_begin_index,
length=gene_end_index - gene_begin_index)
variation_span = ddlib.Span(begin_word_id=var_begin_index,
length=var_end_index - var_begin_index)
# Generate the generic features using DDLIB
for feature in ddlib.get_generic_features_relation(sent, gene_span,
variation_span):
yield [gene_id, variation_id, feature]
def extract(
p1_id="text",
p2_id="text",
p1_begin_index="int",
p1_end_index="int",
p2_begin_index="int",
p2_end_index="int",
doc_id="text",
sent_index="int",
tokens="text[]",
lemmas="text[]",
pos_tags="text[]",
ner_tags="text[]",
dep_types="text[]",
dep_parents="int[]",
):
"""
Uses DDLIB to generate features for the relation of MED and ARD.
"""
# Create a DDLIB sentence object, which is just a list of DDLIB Word objects
sent = []
for i, t in enumerate(tokens):
sent.append(
ddlib.Word(
begin_char_offset=None,
end_char_offset=None,
word=t,
lemma=lemmas[i],
pos=pos_tags[i],
ner=ner_tags[i],
dep_par=dep_parents[i] -
1, # Note that as stored from CoreNLP 0 is ROOT, but for DDLIB -1 is ROOT
dep_label=dep_types[i]))
# Create DDLIB Spans for the two person mentions
p1_span = ddlib.Span(begin_word_id=p1_begin_index,
length=(p1_end_index - p1_begin_index + 1))
p2_span = ddlib.Span(begin_word_id=p2_begin_index,
length=(p2_end_index - p2_begin_index + 1))
# Generate the generic features using DDLIB
for feature in ddlib.get_generic_features_relation(sent, p1_span, p2_span):
yield [p1_id, p2_id, feature]
def extract(S_id="text",
O_id="text",
S_begin_index="int",
S_end_index="int",
O_begin_index="int",
O_end_index="int",
sent_id="text",
tokens="text[]",
pos_tags="text[]",
ner_tags="text[]",
dep_types="text[]",
dep_tokens="int[]"):
"""
Uses DDLIB to generate features for relation.
"""
# Create a DDLIB sentence object, which is just a list of DDLIB Word objects
sent = []
if len(tokens) != len(pos_tags):
print >> sys.stderr, '===>>>', sent_id, len(tokens), len(pos_tags)
for i, t in enumerate(tokens):
sent.append(
ddlib.Word(
begin_char_offset=None,
end_char_offset=None,
word=t,
lemma=tokens[i],
pos=pos_tags[i],
ner=ner_tags[i],
dep_par=dep_tokens[i] -
1, # Note that as stored from CoreNLP 0 is ROOT, but for DDLIB -1 is ROOT
dep_label=dep_types[i]))
# Create DDLIB Spans for the two person mentions
S_span = ddlib.Span(begin_word_id=S_begin_index,
length=(S_begin_index - S_end_index + 1))
O_span = ddlib.Span(begin_word_id=O_begin_index,
length=(O_begin_index - O_end_index + 1))
# Generate the generic features using DDLIB
for feature in ddlib.get_generic_features_relation(sent, S_span, O_span):
yield [S_id, O_id, feature]
def extract(
p_id="text",
p_begin_index="int",
p_end_index="int",
doc_id="text",
sent_index="int",
tokens="text[]",
pos_tags="text[]",
ner_tags="text[]",
dep_types="text[]",
dep_parents="int[]",
):
"""
Uses DDLIB to generate features for the legal penalty mention
"""
# Constant
# WINDOW_SIZE = 10
# Load keyword dictionaries using ddlib, for domain-specific features
# Words in "legal_penalty" dictionary are indicative of marriage
# Words in "non_legal_penalty" dictionary are indicative of non_marriage
APP_HOME = os.environ['APP_HOME']
ddlib.load_dictionary(APP_HOME + "/udf/dicts/kw_legal_penalty.txt",
dict_id="legal_penalty")
ddlib.load_dictionary(APP_HOME + "/udf/dicts/kw_non_legal_penalty.txt",
dict_id="non_legal_penalty")
kw_non_legal_penalty = map(
lambda word: word.strip(),
open(APP_HOME + "/udf/dicts/kw_non_legal_penalty.txt",
'r').readlines())
# kw_legal_penalty = map(lambda word: word.strip(), open(APP_HOME + "/udf/dicts/kw_legal_penalty.txt", 'r').readlines())
# Non penalty signals on the left of candidate mention
NON_PENAL_SIGNALS_LEFT = frozenset(kw_non_legal_penalty)
# Penalty signals on the right of candidate mention
# PENAL_SIGNALS_LEFT = frozenset(kw_legal_penalty)
WINDOW_SIZE = 10
MAX_PHRASE_LENGTH = 5
# Get all subsequences of left sentence with WINDOW_SIZE = 10
low_tokens = map(lambda token: token.lower(), tokens)
left_window = get_left_window(p_begin_index, low_tokens, WINDOW_SIZE)
phrases_in_sentence_left = list(
get_all_phrases_in_sentence(left_window, MAX_PHRASE_LENGTH))
# Create a DDLIB sentence object, which is just a list of DDLIB Word objects
sent = []
for i, t in enumerate(tokens):
sent.append(
ddlib.Word(
begin_char_offset=None,
end_char_offset=None,
word=t,
lemma=t.lower(), # lemma for vietnamese: lowercase
pos=pos_tags[i],
ner=ner_tags[i],
dep_par=dep_parents[i] -
1, # Note that as stored from CoreNLP 0 is ROOT, but for DDLIB -1 is ROOT
dep_label=dep_types[i]))
# Create DDLIB Span for penalty candidate
penalty_span = ddlib.Span(begin_word_id=p_begin_index,
length=(p_end_index - p_begin_index + 1))
# Generate the generic features using DDLIB on left and right window
for feature in ddlib.get_generic_features_mention(sent, penalty_span):
yield [p_id, feature]
# Keywords represent non-legal_penalty appears on the left
if len(NON_PENAL_SIGNALS_LEFT.intersection(phrases_in_sentence_left)) > 0:
yield [p_id, 'APPEAR_LEFT_KW_NON_LEGAL_PENALTY']
# "phạt tù" appear on the left of mention
if "phạt tù" in phrases_in_sentence_left:
yield [p_id, 'APPEAR_LEFT_PHAT_TU']
