def project_dependencies(rel: Relation) -> None:
"""
Update the fds in the relation by projecting them.
"""
new_fds = []
supers = []
for single in rel.relation:
closure = rel.closure({single})
fd = FD(f'{single}->{"".join(closure)}')
if closure == rel.relation:
supers.append(fd)
if not closure.issubset({single}):
new_fds.append(fd)
for s in powerset(rel.relation):
redundant = False
subset = set(s)
for fd in supers:
if subset.issuperset(fd.determinants):
redundant = True
break
if redundant or subset == rel.relation or not subset:
continue
closure = rel.closure(subset)
fd = FD(f'{"".join(subset)}->{"".join(closure)}')
if closure == rel.relation:
supers.append(fd)
if not closure.issubset(subset) and fd not in new_fds:
new_fds.append(fd)
rel.fds = new_fds
def main():
"""Quick tests."""
a = Attribute('hour', ['0,...,23'])
a2 = Attribute('minute', ['0,...,59'])
r_ahead = Relation('R1(h1,m1,h2,m2) <=> h1 > h2 or (h1 = h2 and m1 > m2)',
['hour', 'minute', 'hour', 'minute'], 1)
r_behind = Relation('R2(h1,m1,h2,m2) <=> h1 < h2 or (h1 = h2 and m1 < m2)',
['hour', 'minute', 'hour', 'minute'], 2)
r_pm = Relation('R3(h1) <=> h1 > 12', ['hour'], 3)
r_am = Relation('R4(h1) <=> h1 < 12', ['hour'], 4)
attribute_structure = AttributeStructure(a, a2, r_ahead, r_behind, r_pm,
r_am)
ahead_rs = RelationSymbol('Ahead', 4)
behind_rs = RelationSymbol('Behind', 4)
pm_rs = RelationSymbol('PM', 1)
vocabulary = Vocabulary(['C1', 'C2'], [ahead_rs, behind_rs, pm_rs],
['V1', 'V2'])
profiles = [[
ahead_rs, ('hour', 1), ('minute', 1), ('hour', 2), ('minute', 2)
], [behind_rs, ('hour', 1), ('minute', 1), ('hour', 2), ('minute', 2)],
[pm_rs, ('hour', 1)]]
mapping = {ahead_rs: 1, behind_rs: 2, pm_rs: 3}
ai = AttributeInterpretation(vocabulary, attribute_structure, mapping,
profiles)
print ai == ai
def read_relations(rel_path):
rel_dict = defaultdict(list)
for x in open(rel_path):
rel = Relation()
rel.init_with_annotation(json.loads(x))
rel_dict[rel.doc_id].append(rel)
return rel_dict
def read_relations(rel_path):
rel_dict = defaultdict(list)
for x in open(rel_path):
rel = Relation()
rel.init_with_annotation(json.loads(x))
rel_dict[rel.doc_id].append(rel)
return rel_dict
def ReadNextRelation(self):
""" Read in the next relation, might
need to read from disk, might have it
in the buffer. """
if (self.filePointer == None):
return None
line = self.filePointer.readline()
#print 'Line:', line
if (len(line) > 0):
line = line.strip()
else:
return None
if (len(line) < 3):
return self.ReadNextRelation()
if (line.find('set ') == 0):
end = line.find('\t')
if (end == -1):
end == -2
self.currentArticle = (line[5:end]).strip()
return self.ReadNextRelation()
elif (line.find('endset;') == 0):
return self.ReadNextRelation()
elif (len(self.currentArticle) < 3):
return self.ReadNextRelation()
else:
#end = line.find('+')
r = Relation(line, self.currentArticle)
if (r.success):
return r
else:
return Relation('', '')
def relation(self,type):
relation = Relation()
relation.setId1(self.currentPersonId)
relation.setId2(self.CurrentEntityId)
relation.setType(type)
relation.setStrength(self.getRelationStrength(self.person.getReference()))
self.dao.insertRelation('t_relation', relation, self.cur, self.conn)
def get_matches(images, dist):
# Create the parameters for FLANN
index_params = {'algorithm': 1, 'trees': 5}
search_params = {'checks': 50}
# Create the FLANN object
flann = cv.FlannBasedMatcher(index_params, search_params)
# Loop through the images
for image_a, image_b in combinations(images, 2):
# Get the descriptions
desc_a = image_a.descriptions
desc_b = image_b.descriptions
# Get the matches
matches = flann.knnMatch(desc_a, desc_b, k=2)
# The matches indices
match_a = []
match_b = []
# Filter the matches
for m, n in matches:
if m.distance < dist * n.distance:
# Add the point
match_a.append(m.queryIdx)
match_b.append(m.trainIdx)
# Update the matches
match_a = np.array(match_a).reshape((-1, 1))
match_b = np.array(match_b).reshape((-1, 1))
# Set the matches
match_a, match_b = np.append(match_a, match_b,
axis=1), np.append(match_b,
match_a,
axis=1)
# Get the points
points_a = image_a.points[match_a[:, 0]]
points_b = image_b.points[match_b[:, 0]]
# Calculate the essential matrix
# F, mask = cv.findFundamentalMat(points_a, points_b)
# E = np.matmul(image_b.intrinsic.T, np.matmul(F, image_a.intrinsic))
E, mask = cv.findEssentialMat(points_a, points_b, image_a.intrinsic)
# E, mask = cv.findEssentialMat(points_a, points_b)
# Remove the points that don't match the mask
match_a = match_a[(mask == 1)[:, 0]]
match_b = match_b[(mask == 1)[:, 0]]
# Create the relations
rel_a = Relation(E, image_a, image_b, match_a, match_b)
rel_b = Relation(E, image_b, image_a, match_b, match_a)
# Add the relations
image_a.add_relation(rel_a)
image_b.add_relation(rel_b)
def extract_date_of_birth(sentence):
data = sentence['annotation']
predicate = "DateOfBirth"
results = []
string = ""
rel = Relation("", "", "")
for items in data:
word = items[1]
type = items[4]
new = word + "(" + type + ") "
string += new
quoteObj = re.search("(\"\(I-PERSON\)) (\w*)\(I-PERSON\) (\"\(I-PERSON\))",
string)
if quoteObj is not None:
quote1 = str(quoteObj.group(1))
quote1 = re.sub('\(I-PERSON\)', '', quote1)
name = str(quoteObj.group(2))
quote2 = str(quoteObj.group(3))
quote2 = re.sub('\(I-PERSON\)', '', quote2)
enclosedName = quote1 + name + quote2 + "(I-PERSON)"
string = re.sub("(\"\(I-PERSON\)) (\w*)\(I-PERSON\) (\"\(I-PERSON\))",
enclosedName, string)
fullDateObj = re.search(
r'(\w*\.*\(B-PERSON\)) (\"*\w*\.*\"*\(I-PERSON\))? ?(\"*\w*\.*\"*\(I-PERSON\))? ?(\"*\w*\.*\"*\(I-PERSON\))?.*born.*?(\w*\(B-DATE\)) ?(.*\(I-DATE\))?',
string)
if fullDateObj is not None:
firstName = str(fullDateObj.group(1))
secondName = str(fullDateObj.group(2))
thirdName = str(fullDateObj.group(3))
fourthName = str(fullDateObj.group(4))
month = str(fullDateObj.group(5))
day_Year = str(fullDateObj.group(6))
firstName = re.sub(r'\(B-PERSON\)', '', firstName)
secondName = re.sub(r'\(I-PERSON\)', '', secondName)
thirdName = re.sub(r'\(I-PERSON\)', '', thirdName)
fourthName = re.sub(r'\(I-PERSON\)', '', fourthName)
fullName = firstName + " " + secondName + " " + thirdName + " " + fourthName
fullName = fullName.replace(" None", "")
month = re.sub(r'\(B-DATE\)| ', '', month)
day_Year = re.sub(r'\(I-DATE\)', '', day_Year)
day_Year = re.sub(r' , ', ', ', day_Year)
day_Year = day_Year[0:20]
day_YearObj = re.search(r'^(.*?[0-9]{4}).*', day_Year)
if day_YearObj is not None:
day_Year = str(day_YearObj.group(1))
monthObj = re.search(r'(^[0-9]{4}).*', month)
if monthObj is not None:
day_Year = "None"
DOB = month + " " + day_Year
DOB = DOB.replace(" None", "")
rel = Relation(fullName, predicate, DOB)
results.append(rel)
return results
def recommend_subcatalog(self, node_id, subcatalog_obj):
rr = Relation(attrs={"relation_type":"catalog-%s" % \
subcatalog_obj.__class__.__name__})
rr.set_relation_set(self, subcatalog_obj)
tmp = self.get_node_list(node_id, 'catalogs')
tmp.push(rr._id)
self.lib.relations_list.push(rr._id)
self.get_node_dict(node_id)['subcatalog_count'] += 1
self.do_update()
return rr
def recommend_subcatalog(self, node_id, subcatalog_obj):
rr = Relation(attrs={"relation_type":"catalog-%s" % \
subcatalog_obj.__class__.__name__})
rr.set_relation_set(self, subcatalog_obj)
tmp = self.get_node_list(node_id, 'catalogs')
tmp.push(rr._id)
self.lib.relations_list.push(rr._id)
self.get_node_dict(node_id)['subcatalog_count'] += 1
self.do_update()
return rr
def main():
"""quick dev tests."""
from interval import Interval
from relationSymbol import RelationSymbol
from vocabulary import Vocabulary
from attribute_interpretation import AttributeInterpretation
from formula import Formula
from assumption_base import AssumptionBase
from attribute import Attribute
from relation import Relation
from attribute_structure import AttributeStructure
from attribute_system import AttributeSystem
from constant_assignment import ConstantAssignment
from named_state import NamedState
from context import Context
from variable_assignment import VariableAssignment
a = Attribute('hour', [Interval(0, 23)])
a2 = Attribute('minute', [Interval(0, 59)])
r_pm = Relation('R1(h1) <=> h1 > 11', ['hour'], 1)
r_am = Relation('R2(h1) <=> h1 <= 11', ['hour'], 2)
r_ahead = Relation('R3(h1,m1,h2,m2) <=> h1 > h2 or (h1 = h2 and m1 > m2)',
['hour', 'minute', 'hour', 'minute'], 3)
r_behind = Relation('R4(h1,m1,h2,m2) <=> h1 < h2 or (h1 = h2 and m1 < m2)',
['hour', 'minute', 'hour', 'minute'], 4)
attribute_structure = AttributeStructure(a, a2, r_ahead, r_behind, r_pm,
r_am)
pm_rs = RelationSymbol('PM', 1)
am_rs = RelationSymbol('AM', 1)
ahead_rs = RelationSymbol('Ahead', 4)
behind_rs = RelationSymbol('Behind', 4)
vocabulary = Vocabulary(['C1', 'C2'], [pm_rs, am_rs, ahead_rs, behind_rs],
['V1', 'V2'])
objs = ['s1', 's2', 's3']
asys = AttributeSystem(attribute_structure, objs)
const_mapping_2 = {'C1': 's1'}
p2 = ConstantAssignment(vocabulary, asys, const_mapping_2)
ascriptions_1 = {
("hour", "s1"): [13, 15, 17],
("minute", "s1"): [10],
("hour", "s2"): [1, 3, 5],
("minute", "s2"): [10],
("hour", "s3"): [1, 3, 5],
("minute", "s3"): [10]
}
named_state_4 = NamedState(asys, p2, ascriptions_1)
def recommend_article(self, node_id, article_obj):
if self.get_node_dict(node_id)['title'] is None:
return None
rr = Relation(attrs={"relation_type":"catalog-%s" % \
article_obj.cls_name})
rr.set_relation_set(self, article_obj)
tmp_list = self.get_node_list(node_id, 'articles')
tmp_list.push(rr._id)
self.lib.relations_list.push(rr._id)
self.get_node_dict(node_id)['article_count'] += 1
self.do_update()
return rr
def recommend_article(self, node_id, article_obj):
if self.get_node_dict(node_id)['title'] is None:
return None
rr = Relation(attrs={"relation_type":"catalog-%s" % \
article_obj.cls_name})
rr.set_relation_set(self, article_obj)
tmp_list = self.get_node_list(node_id, 'articles')
tmp_list.push(rr._id)
self.lib.relations_list.push(rr._id)
self.get_node_dict(node_id)['article_count'] += 1
self.do_update()
return rr
def getRelations(self, doc: Doc) -> [Relation]:
relations = []
matches = self._matcher(doc)
for match_id, start, end in matches:
span = doc[start:end]
hypernym = span.root.text
hyponym = span.text.split()[-1]
relations.append(Relation(hypernym, hyponym))
for right in span.rights:
if right.pos_ == "NOUN":
relations.append(Relation(hypernym, right.text))
return relations
def test_get_attributes_closure(self):
complete_attrs = set(list('ABCDEFGHI'))
dep_pairs = [('A', 'B'), ('A', 'C'), ('CG', 'H'), ('CG', 'I'),
('B', 'H')]
deps = dependency_generater(dep_pairs)
r = Relation(attrs=complete_attrs, deps=deps)
attrs = set(['A', 'G'])
expect_closure = set(['A', 'B', 'C', 'G', 'H', 'I'])
closure = r.get_attrs_closure(attrs)
self.assertEqual(expect_closure, closure)
def getRelation(table, fromName, toName):
""" 读取关系对照表
"""
relation = Relation(fromName, toName)
for line in sopen(table):
if line.startswith('#'):
continue
flds = line.rstrip().split()
chs = flds[0]
for cht in flds[1:]:
relation.add(chs, cht)
return relation
def getdatabaserelation(self):
reloid = 1262
if not self.pg_database_attr:
self.pg_database_attr = [
PgAttribute95( 1262, "datname", 19, -1, Catalog95.NAMEDATALEN, 1, 0, -1, -1, False, 'p', 'c', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "datdba", 26, -1, 4, 2, 0, -1, -1, True, 'p', 'i', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "encoding", 23, -1, 4, 3, 0, -1, -1, True, 'p', 'i', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "datcollate", 19, -1, Catalog95.NAMEDATALEN, 4, 0, -1, -1, False, 'p', 'c', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "datctype", 19, -1, Catalog95.NAMEDATALEN, 5, 0, -1, -1, False, 'p', 'c', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "datistemplate", 16, -1, 1, 6, 0, -1, -1, True, 'p', 'c', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "datallowconn", 16, -1, 1, 7, 0, -1, -1, True, 'p', 'c', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "datconnlimit", 23, -1, 4, 8, 0, -1, -1, True, 'p', 'i', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "datlastsysoid", 26, -1, 4, 9, 0, -1, -1, True, 'p', 'i', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "datfrozenxid", 28, -1, 4, 10, 0, -1, -1, True, 'p', 'i', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "datminmxid", 28, -1, 4, 11, 0, -1, -1, True, 'p', 'i', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "dattablespace", 26, -1, 4, 12, 0, -1, -1, True, 'p', 'i', True, False, False, True, 0, 0 ), \
PgAttribute95( 1262, "datacl", 1034, -1, -1, 13, 1, -1, -1, False, 'x', 'i', False, False, False, True, 0, 0 )
]
relfilenode = RelFileNode()
relfilenode.space_node = Catalog95.GLOBALTABLESPACE_OID
relfilenode.db_node = 0
relfilenode.rel_node = self.getsharerelmap()[reloid]
databaserelation = Relation(reloid, 'pg_database',
self.pg_database_attr, relfilenode)
return databaserelation
def post(self):
handler_para = UnSpecArticleFromBookPara(self)
handler_json = UnSpecArticleFromBookJson(self)
usr = self.current_user
book = Catalog.by_id(handler_para['book_id'])
relation_obj = Relation.by_id(handler_para['relation_id'])
if book is None:
handler_json.by_status(2)
handler_json.write()
return #book not exist
if relation_obj is None:
handler_json.by_status(1)
handler_json.write()
return #relation not exist
auth_tmp = book.authority_verify(usr, env=book)
if test_auth(auth_tmp, A_WRITE) is False:
handler_json.by_status(4)
handler_json.write()
return #permission denied
status = book.unspec_article_to(handler_para['node_id'], relation_obj)
if status is False:
handler_json.by_status(3)
handler_json.write()
return #set failed, node_id may not exist
handler_json.by_status(0)
handler_json.write()
return #0
def parse_compound_concept(expression: str, store: Store, label: str=None, probability: float=1.0):
s = expression.find('(')
e = expression.rfind(')')
rel = Relation.find(expression[:s])
args = [] # type: list[str]
d = 0
arg_start = s+1
for x in range(s+1, e):
c = expression[x]
if c == '(':
d += 1
elif c == ')':
d -= 1
elif c == ',':
if d == 0:
args.append(expression[arg_start:x])
arg_start = x+1
args.append(expression[arg_start:e])
arg_concepts = [] # type: list[Concept]
arg_probability = probability
if rel == Relation.Implication:
arg_probability = .5
for a in args:
arg = parse(a.strip(), store, probability=arg_probability)
arg_concepts.append(arg)
if label is None and e < len(expression)-1:
label = expression[e+1:].strip()
return Concept(label, rel, arg_concepts, probability=probability)
def getRelations(self, doc: Doc) -> [Relation]:
relations = []
matches = self._matcher(doc)
for match_id, start, end in matches:
span = doc[start:end]
firstToken = span.root.head
results = [firstToken]
while firstToken and firstToken.head.pos_ == "NOUN":
results.append(firstToken.head)
firstToken = firstToken.head
hypernym = span.text.split()[-1]
relations.append(Relation(hypernym, span.text.split()[0]))
if len(results) > 0:
for result in results:
relations.append(Relation(hypernym, result.text))
return relations
def getRelations(self, doc: Doc) -> [Relation]:
relations = []
matches = self._matcher(doc)
for match_id, start, end in matches:
span = doc[start:end]
for sent in doc.sents:
for token in sent:
# Find the relation
if token.text == "including" and token.head.i == span.root.i:
for token2 in sent:
# First hyponym
if token2.head.i == token.i:
results = set()
results.add(span.text.split()[-1])
# Other hyponyms
for token3 in sent:
startToken = token3
while startToken and startToken.head.i != sent.root.i and startToken.i != token2.i:
if startToken.pos_ == "NOUN":
results.add(startToken.text)
startToken = startToken.head
if len(results) > 0:
hypernym = span.text.split()[0].replace(
',', '')
for result in results:
relations.append(
Relation(hypernym, result))
return relations
def loader(curr):
# Intentionally load only current node and only one direction of edge
# nodes absent from graph is trigger for loading the node in graph and
# its neighbors
graph.add_node(curr)
for relative, relation in get_relatives(curr).items():
graph.add_edge(curr, relative, Relation.get_distance(relation), relation)
def get_relation_to_catalog(self, catalog_obj, node_id):
from relation import Relation
tmp_lib = self.lib.relation_catalogs
tmp = tmp_lib[catalog_obj.uid + '#' + node_id]
if tmp is not None:
return Relation.by_id(tmp)
return None
def addRelation(self, follower, target, existing):
found = False
for cur in existing:
if follower == cur.follower and target == cur.user:
found = True
break
pass
if not found:
relation = Relation(follower=follower, user=target)
relation.put()
query = Tag.all()
query.filter("userid = ", target)
tags = query.fetch(1000)
for tag in tags:
tag.addToStream(follower)
def create_relation_dictionary(config, document, rel_dict=None):
if not rel_dict:
rel_dict = {}
for r in document.getRelations().getRelations():
rel_type = r.getType()
matching_rel_configs = [rc for rc in config['relations'] if rel_type in rc["types"]]
if not len(matching_rel_configs):
continue
annotation_from = r.getAnnotationFrom()
annotation_to = r.getAnnotationTo()
for rel_conf in matching_rel_configs:
if (any(re.match(_from, annotation_from.getType()) for _from in rel_conf['from']) and any(re.match(_to, annotation_to.getType()) for _to in rel_conf['to'])) \
or ("allow_reversed" in rel_conf and rel_conf["allow_reversed"] and any(re.match(_from, annotation_from.getType()) for _from in rel_conf['to']) and any(
re.match(_to, annotation_to.getType()) for _to in rel_conf['from'])):
relation_position = get_relation_position(r)
# print(rel_type, annotation_from.getType(), annotation_to.getType())
# if relation_position in rel_dict:
# print(relation_position, "already in dict", rel_type, rel_dict[relation_position].getType())
rel_dict[relation_position] = Relation(rel_type, annotation_from, annotation_to)
continue
# if any(
# any(re.match(_from, annotation_from.getType()) for _from in rel_conf['from']) and
# any(re.match(_to, annotation_to.getType()) for _to in rel_conf['to'])
# for rel_conf in matching_rel_configs):
# relation_position = self.get_relation_position(r)
# true_rel_dict[relation_position] = Relation(rel_type, annotation_from, annotation_to)
return rel_dict
def getRelations(self, doc: Doc) -> [Relation]:
relations = []
matches = self._matcher(doc)
for match_id, start, end in matches:
span = doc[start:end]
candidates = set()
for sent in docs.sents:
for token in sent:
#find relation
if token.i == span.root.i:
for token2 in sent:
#first hyponym
if token2.head.i == token.i:
for token3 in sent:
startToken = token3
while startToken and startToken.head.i != sent.root.i and startToken.i != token2.i:
if startToken.pos_ == 'NOUN':
candidates.add(startToken)
startToken = startToken.head
if len(candidates) > 0:
hypernym = span.text.split()[0].replace(',', '')
for candidate in candidates:
relations.append(
Relation(
hypernym,
candidate.text,
self.__matcherId
)
)
return relations
def post(self):
handler_para = UnSpecArticleFromBookPara(self)
handler_json = UnSpecArticleFromBookJson(self)
usr = self.current_user
book = Catalog.by_id(handler_para['book_id'])
relation_obj = Relation.by_id(handler_para['relation_id'])
if book is None:
handler_json.by_status(2)
handler_json.write()
return #book not exist
if relation_obj is None:
handler_json.by_status(1)
handler_json.write()
return #relation not exist
auth_tmp = book.authority_verify(usr, env=book)
if test_auth(auth_tmp, A_WRITE) is False:
handler_json.by_status(4)
handler_json.write()
return #permission denied
status = book.unspec_article_to(handler_para['node_id'], relation_obj)
if status is False:
handler_json.by_status(3)
handler_json.write()
return #set failed, node_id may not exist
handler_json.by_status(0)
handler_json.write()
return #0
def get_relation_to_catalog(self, catalog_obj, node_id):
from relation import Relation
tmp_lib = self.lib.relation_catalogs
tmp = tmp_lib[catalog_obj.uid + "#" + node_id]
if tmp is not None:
return Relation.by_id(tmp)
return None
def __init__(self, lineGenerator, sep=None, index={}):
"""
lineGenerator :: generator(str)
CSV-like data.
sep :: str
separator for str.split().
index :: dict(tuple(columnName :: str), [recordIndex :: int]))
Currently unused.
"""
schemaLine = lineGenerator.next().rstrip()
schemaLine = re.sub('^#', '', schemaLine)
schema = schemaLine.split(sep)
def getRawRecGenerator():
for line in lineGenerator:
yield tuple(line.rstrip().split(sep))
Relation.__init__(self, schema, getRawRecGenerator(), reuse=True)
def main():
"""Main method; quick testing."""
a, b, c = Attribute("a", []), Attribute("b", []), Attribute("c", [])
r = Relation("R1(a,b) <=> ", ["a", "b"], 1)
astr = AttributeStructure()
print astr + a + b + r
def loader(curr):
# Intentionally load only current node and only one direction of edge
# nodes absent from graph is trigger for loading the node in graph and
# its neighbors
graph.add_node(curr)
for relative, relation in get_relatives(curr).items():
graph.add_edge(curr, relative, Relation.get_distance(relation),
relation)
def _process_parsed_conn(self, articles, which='test'):
"""
generate explicit relation for each true discourse connective
"""
connParser = Connective()
conn_feat_name = FILE_PATH + '/../tmp/conn.feat'
conn_feat_file = codecs.open(conn_feat_name, 'w', 'utf-8')
checked_conns = []
for art in articles:
checked_conns.append(
connParser.print_features(art, which, conn_feat_file))
conn_feat_file.close()
conn_pred_name = FILE_PATH + '/../tmp/conn.pred'
Corpus.test_with_opennlp(conn_feat_name, connParser.model_file,
conn_pred_name)
conn_res = [
l.strip().split()[-1]
for l in codecs.open(conn_pred_name, 'r', 'utf-8')
]
assert len(checked_conns) == len(articles), 'article size not match'
s = 0
for art, cand_conns in zip(articles, checked_conns):
length = len(cand_conns)
cand_res = conn_res[s:s + length]
s += length
for conn, label in zip(cand_conns, cand_res):
if label == '1':
rel = Relation()
rel.doc_id = art.id
rel.rel_type = 'Explicit'
rel.article = art
rel.conn_leaves = conn
rel.conn_addr = [n.leaf_id for n in conn]
art.exp_relations.append(rel)
assert s == len(conn_res), 'conn size not match'
def extract_features(features_filepath, dataset_filepath):
"""Extracts features from the given dataset writes to the features file in
MALLET input format, with one instance per line.
"""
with open(dataset_filepath) as dataset_file:
with open(features_filepath, 'w') as feat_file:
last_doc_name = None
sents = None
rel_id = 0
for line in dataset_file:
values = line.split()
if last_doc_name != values[1]:
last_doc_name = values[1]
sents = file_to_sents(last_doc_name)
sent_trees = file_to_trees(last_doc_name)
rel = Relation(values, sents, sent_trees, rel_id)
rel_id += 1
feat_file.write(rel.to_string() + '\n')
def add_to_catalog(self, catalog_obj, node_id):
'''relation_obj will be returned'''
tmp_lib = self.lib.parent_catalogs
tmp = tmp_lib[str(catalog_obj._id) + '#' + node_id]
if tmp is not None:
return Relation(_id=tmp)
rr = catalog_obj.recommend_subcatalog(node_id, self)
tmp_lib[str(catalog_obj._id) + '#' + node_id] = rr._id
return rr
def get_all_relations(self):
'''
return list of instances of class Relation
@rtype: generator
@return: generator of instances of class Relation
'''
path_to_rels = "SynsetRelations/SynsetRelation"
for relation_el in self.synset_el.iterfind(path_to_rels):
yield Relation(relation_el)
def __init__(self, lineGenerator, sep=None, schema=None, reuse=True):
"""
lineGenerator :: generator(str)
CSV-like data.
sep :: str
separator for str.split().
schema :: Schema
Use the schema.
Assume there is no header.
"""
if schema is None:
schemaLine = lineGenerator.next().rstrip()
schema = Schema.parse(schemaLine, sep=sep)
def getRawRecGenerator():
for line in lineGenerator:
yield schema.parseValues(line.rstrip().split(sep))
Relation.__init__(self, schema, getRawRecGenerator(), reuse=reuse)
def __init__(self, lineGenerator, sep=None, index={}):
"""
lineGenerator :: generator(str)
CSV-like data.
sep :: str
separator for str.split().
index :: dict(tuple(columnName :: str), [recordIndex :: int]))
Currently unused.
"""
schemaLine = lineGenerator.next().rstrip()
schemaLine = re.sub('^#', '', schemaLine)
schema = schemaLine.split(sep)
def getRawRecGenerator():
for line in lineGenerator:
yield tuple(line.rstrip().split(sep))
Relation.__init__(self, schema, getRawRecGenerator(), reuse=True)
def remove_node(self, node_id):
if not self.lib.node_lib.sub_dict(node_id).load_all():
return False #node not exist
tmp_rel_ids = self.get_node_list(node_id, 'articles').load_all()
self.lib.relations_list.pull(*tuple(tmp_rel_ids))
rels = Relation.by_ids(tmp_rel_ids)
for each in rels:
each.remove()
tmp_rel_ids = self.get_node_list(node_id, 'catalogs').load_all()
self.lib.relations_list.pull(*tuple(tmp_rel_ids))
self.remove_count += 1
if self.get_node_list(node_id, 'main').load_all() != []:
self.complete_count -= 1
rels = Relation.by_ids(tmp_rel_ids)
for each in rels:
each.remove()
del self.lib.node_lib[node_id]
del self.lib.node_info_lib[node_id]
self.do_update()
return True
def remove_node(self, node_id):
if not self.lib.node_lib.sub_dict(node_id).load_all():
return False #node not exist
tmp_rel_ids = self.get_node_list(node_id, 'articles').load_all()
self.lib.relations_list.pull(*tuple(tmp_rel_ids))
rels = Relation.by_ids(tmp_rel_ids)
for each in rels:
each.remove()
tmp_rel_ids = self.get_node_list(node_id, 'catalogs').load_all()
self.lib.relations_list.pull(*tuple(tmp_rel_ids))
self.remove_count += 1
if self.get_node_list(node_id, 'main').load_all() != []:
self.complete_count -= 1
rels = Relation.by_ids(tmp_rel_ids)
for each in rels:
each.remove()
del self.lib.node_lib[node_id]
del self.lib.node_info_lib[node_id]
self.do_update()
return True
def _process_parsed_conn(self, articles, which='test'):
"""
generate explicit relation for each true discourse connective
"""
connParser = Connective()
conn_feat_name = FILE_PATH + '/../tmp/conn.feat'
conn_feat_file = codecs.open(conn_feat_name, 'w', 'utf-8')
checked_conns = []
for art in articles:
checked_conns.append(connParser.print_features(art, which, conn_feat_file))
conn_feat_file.close()
conn_pred_name = FILE_PATH + '/../tmp/conn.pred'
Corpus.test_with_opennlp(conn_feat_name, connParser.model_file, conn_pred_name)
conn_res = [l.strip().split()[-1] for l in codecs.open(conn_pred_name, 'r', 'utf-8')]
assert len(checked_conns) == len(articles), 'article size not match'
s = 0
for art, cand_conns in zip(articles, checked_conns):
length = len(cand_conns)
cand_res = conn_res[s:s+length]
s += length
for conn, label in zip(cand_conns, cand_res):
if label == '1':
rel = Relation()
rel.doc_id = art.id
rel.rel_type = 'Explicit'
rel.article = art
rel.conn_leaves = conn
rel.conn_addr = [n.leaf_id for n in conn]
art.exp_relations.append(rel)
assert s == len(conn_res), 'conn size not match'
def main():
customer = Relation(Schema("Customer", "id, fname, lname, age, height"))
customer.add(id=18392, fname="Frank", lname="Smith", age=45, height="5'8")
customer.add(id=48921, fname="Jane", lname="Doe", age=42, height="5'6")
print(customer)
# output:
#
# Customer
# id fname lname age height
# 48921 Jane Doe 42 5'6
# 18392 Frank Smith 45 5'8
print()
print(Pi["fname", "id"](customer))
# output:
#
# Customer__fname_id
# fname id
# Frank 18392
# Jane 48921
print()
print(Sigma[lambda tup: tup.age > 43](customer))
# output:
#
# Customer
# id fname lname age height
# 18392 Frank Smith 45 5'8
print()
print(Rho["MySchema"](Pi["fname", "id"](customer)))
def post(self):
self.response.headers.add_header("Access-Control-Allow-Origin", "http://www.jonathonduerig.com");
postid = self.request.get("postid")
userid = self.request.get("userid")
posts = []
followed = []
if postid:
query = Post.all()
query.filter("postid = ", postid)
posts = query.fetch(1000)
elif userid:
query = Stream.all()
query.filter("userid = ", userid)
query.order("-created")
posts = query.fetch(30)
query = Relation.all()
query.filter("follower = ", userid)
followed = query.fetch(1000)
resultList = []
for p in posts:
resultTags = []
postkey = ""
outPostid = ""
if postid:
postkey = p.key()
outPostid = postid
else:
postkey = p.postkey.key()
outPostid = p.postkey.postid
query = Tag.all()
query.filter("postkey = ", postkey)
tags = query.fetch(1000)
for t in tags:
found = False
for f in followed:
if t.userid == f.user:
found = True
break
pass
if found or postid:
tag = {'user': t.userid,
'key': t.tag_key,
'value': t.tag_value}
resultTags.append(tag)
pass
resultList.append({'postid': outPostid, 'post': p.post,
'tags': resultTags})
resultStr = json.dumps(resultList, sort_keys=True,
indent=2)
self.response.out.write(resultStr)
class TestRelation(TestCase):
def setUp(self):
self.rel = Relation("parenthood", 0.3, 0.5)
def test_init(self):
self.assertTrue(self.rel.name == "parenthood")
self.assertTrue(len(self.rel.participants) == 0)
self.assertTrue(len(self.rel.aware) == 0)
def test_participation(self):
self.rel.add_participant("vader")
self.rel.add_participant("luke", False)
self.assertTrue(self.rel.is_in("vader"))
self.assertTrue(self.rel.is_in("luke"))
self.assertTrue(self.rel.is_aware("vader"))
self.assertFalse(self.rel.is_aware("luke"))
def from_existing(folder):
self = Database()
self.duncecap = jpype.JPackage('duncecap')
self.folder = folder #string
self.qc = self.duncecap.QueryCompiler.fromDisk(self.folder+"/schema.bin")
dbInstance = self.qc.getDBInstance()
self.folder = dbInstance.getFolder()
self.config = Config.java2python(dbInstance.getConfig())
num_relations = dbInstance.getNumRelations()
self.relations = []
for i in range(0,num_relations):
self.relations.append(
Relation.java2python(dbInstance.getRelation(i)))
self.backend = DB(self.folder)
self.dbhash = dbhash
return self
def post(self):
self.response.headers.add_header("Access-Control-Allow-Origin", "http://www.jonathonduerig.com");
postid = self.request.get("postid")
post = self.request.get("post")
user = self.request.get("user")
key = self.request.get("key")
value = self.request.get("value")
query = Post.all()
query.filter("postid = ", postid)
postResult = query.fetch(1000)
post = Post(postid=postid, post=post)
if len(postResult) > 0:
post = postResult[0]
else:
post.put()
tag = Tag(userid=user, postkey=post.key(), tag_key=key, tag_value=value)
tag.put()
query = Relation.all()
query.filter("user = "******"Tagged post " + postid + " with " + user + ":" +
key + "=" + value)
def generate_nonexp_relations(self, article):
for para in article.paragraphs:
for s1, s2 in zip(para.sentences[:-1], para.sentences[1:]):
if not article.has_exp_relation(s1.id):
# TODO: Add detail implementation
rel = Relation()
rel.article = article
rel.doc_id = article.id
rel.arg1s['parsed'] = [s1.tree.root] if not s1.tree.is_null() else []
rel.arg1_leaves = self.remove_leading_tailing_punc(s1.leaves)
rel.arg1_addr = [n.leaf_id for n in rel.arg1_leaves]
rel.arg1_sid = rel.arg1_leaves[-1].goto_tree().sent_id if len(rel.arg1_leaves) > 0 else -1
rel.arg1_text = ' '.join(n.value for n in rel.arg1_leaves)
rel.arg2s['parsed'] = [s2.tree.root] if not s2.tree.is_null() else []
rel.arg2_leaves = self.remove_leading_tailing_punc(s2.leaves)
rel.arg2_addr = [n.leaf_id for n in rel.arg2_leaves]
rel.arg2_sid = rel.arg2_leaves[0].goto_tree().sent_id if len(rel.arg2_leaves) > 0 else -1
rel.arg2_text = ' '.join(n.value for n in rel.arg2_leaves)
article.nonexp_relations.append(rel)
def make_relation(d):
r = Relation.from_dict(d)
# Add relationship from both sides
add_relation(r)
add_relation(r.flip_relation())
def prepare_data(self, parse_path, rel_path, which, to_file):
rel_dict = Corpus.read_relations(rel_path)
articles = []
dist = defaultdict(int)
for art in Corpus.read_parses(parse_path, rel_dict):
articles.append(art)
for rel in art.relations:
rel.article = art
rel.get_arg_leaves()
if rel.rel_type == 'Explicit':
continue
labels = {s.replace(' ','_') for s in rel.sense}
for l in labels:
dist[l] += 1
if which == 'test':
labels = ['|'.join(labels)]
self.print_features(rel, labels, to_file)
# add NoRel relations
for art in articles:
for s1, s2 in zip(art.sentences[:-1], art.sentences[1:]):
if not art.has_inter_relation(s1.id):
rel = Relation()
rel.article = art
rel.doc_id = art.id
rel.arg1s['parsed'] = [s1.tree.root] if not s1.tree.is_null() else []
rel.arg1_leaves = self.remove_leading_tailing_punc(s1.leaves)
rel.arg1_addr = [n.leaf_id for n in rel.arg1_leaves]
rel.arg1_sid = rel.arg1_leaves[-1].goto_tree().sent_id if len(rel.arg1_leaves) > 0 else -1
rel.arg1_text = ' '.join(n.value for n in rel.arg1_leaves)
rel.arg2s['parsed'] = [s2.tree.root] if not s2.tree.is_null() else []
rel.arg2_leaves = self.remove_leading_tailing_punc(s2.leaves)
rel.arg2_addr = [n.leaf_id for n in rel.arg2_leaves]
rel.arg2_sid = rel.arg2_leaves[0].goto_tree().sent_id if len(rel.arg2_leaves) > 0 else -1
rel.arg2_text = ' '.join(n.value for n in rel.arg2_leaves)
self.print_features(rel, ['NoRel'], to_file)
def own_data(self):
ans = list()
ans += [each for each in
Relation.by_ids(self.relations_list.load_all())]
return ans
def generate_nonexp_relations(self, article):
for s1, s2 in zip(article.sentences[:-1], article.sentences[1:]):
if not article.has_exp_inter_relation(s1.id):
# TODO: Add detail implementation
rel = Relation()
rel.article = article
rel.doc_id = article.id
rel.arg1s['parsed'] = [s1.tree.root] if not s1.tree.is_null() else []
rel.arg1_leaves = self.remove_leading_tailing_punc(s1.leaves)
rel.arg1_addr = [n.leaf_id for n in rel.arg1_leaves]
rel.arg1_sid = rel.arg1_leaves[-1].goto_tree().sent_id if len(rel.arg1_leaves) > 0 else -1
rel.arg1_text = ' '.join(n.value for n in rel.arg1_leaves)
rel.arg2s['parsed'] = [s2.tree.root] if not s2.tree.is_null() else []
rel.arg2_leaves = self.remove_leading_tailing_punc(s2.leaves)
rel.arg2_addr = [n.leaf_id for n in rel.arg2_leaves]
rel.arg2_sid = rel.arg2_leaves[0].goto_tree().sent_id if len(rel.arg2_leaves) > 0 else -1
rel.arg2_text = ' '.join(n.value for n in rel.arg2_leaves)
article.nonexp_relations.append(rel)
# sentence intra nonexp relation
for sen in article.sentences:
tree = sen.tree
if len(sen.clauses) <= 1 :
continue
for c1, c2 in zip(sen.clauses[:-1], sen.clauses[1:]):
if not article.has_exp_intra_relation(sen.id):
rel = Relation()
rel.article = article
rel.doc_id = article.id
rel.arg1s['parsed'] = tree.find_subtrees(c1)
rel.arg1_leaves = self.remove_leading_tailing_punc(c1)
rel.arg1_addr = [n.leaf_id for n in rel.arg1_leaves]
rel.arg1_sid = sen.id
rel.arg1_text = ' '.join(n.value for n in rel.arg1_leaves)
rel.arg2s['parsed'] = tree.find_subtrees(c2)
rel.arg2_leaves = self.remove_leading_tailing_punc(c2)
rel.arg2_addr = [n.leaf_id for n in rel.arg2_leaves]
rel.arg2_sid = sen.id
rel.arg2_text = ' '.join(n.value for n in rel.arg2_leaves)
article.nonexp_relations.append(rel)
followers = []
followed = []
try:
followers = json.loads(self.request.get("followers"))
except Exception, error:
pass
query = Relation.all()
query.filter("user = "******"followed"))
except Exception, error:
pass
query = Relation.all()
query.filter("follower = ", user)
existingFollowed = query.fetch(1000)
for f in followers:
self.addRelation(f, user, existingFollowers)
for f in followed:
self.addRelation(user, f, existingFollowed)
self.response.out.write("Success!")
# except Exception, error:
# pass
def addRelation(self, follower, target, existing):
found = False
for cur in existing:
if follower == cur.follower and target == cur.user:
def setUp(self):
self.rel = Relation("parenthood", 0.3, 0.5)
def get_relations_from_node(self, lib_type, node_id):
'''lib_type: articles, main, catalogs'''
from relation import Relation
rids = self.lib.node_info_lib.\
sub_dict(node_id).sub_dict(lib_type).load_all()
return Relation.by_ids(rids)
