def get_class(interaction):
if opts.training_csv is not None:
intid=jpath('interaction.id',interaction)
if intid in intid2class: # otherwise None is returned
return intid2class[intid]
else:
c=jpath(opts.classpath,interaction)
if isinstance(c,list): # eg if the class is set inside interaction.tags
return c[0]
else:
return c
def get_class(interaction):
if opts.training_csv is not None:
intid = jpath('interaction.id', interaction)
if intid in intid2class: # otherwise None is returned
return intid2class[intid]
else:
c = jpath(opts.classpath, interaction)
if isinstance(c,
list): # eg if the class is set inside interaction.tags
return c[0]
else:
return c
def is_satisfied_by(self,interaction):
text=jpath(self.path,interaction)
if text is None:
return False
else:
words=WordComboFeature.split(text.lower())
return has_subsequence(words,self.combo)
def report_confusion(interactions,targets,fvectors,title):
logging.info("Confusion report for: %s",title)
expectedvsactuals=zip(map(get_classname,targets),map(get_classname,clf.predict(fvectors)))
confusion_matrix(expectedvsactuals)
for i,(exp,act) in enumerate(expectedvsactuals):
if exp!=act:
logging.info("exp:act (%s,%s): %s |%s",exp,act, nvl(jpath(featurepath,interactions[i])),\
'|'.join([selected_features[idx].string() for idx,satisfied in enumerate(fvectors[i]) if satisfied]).encode('utf-8','ignore'))
def is_satisfied_by(self,interaction):
text=jpath(self.path,interaction)
if text is None:
return False
else:
words=wordsplitter.split(text.lower())
(w,x)=self.pair
return w in words and x in words
def report_confusion(interactions, targets, fvectors, title):
logging.info("Confusion report for: %s", title)
expectedvsactuals = zip(map(get_classname, targets),
map(get_classname, clf.predict(fvectors)))
confusion_matrix(expectedvsactuals)
for i, (exp, act) in enumerate(expectedvsactuals):
if exp != act:
logging.info("exp:act (%s,%s): %s |%s",exp,act, nvl(jpath(featurepath,interactions[i])),\
'|'.join([selected_features[idx].string() for idx,satisfied in enumerate(fvectors[i]) if satisfied]).encode('utf-8','ignore'))
def is_satisfied_by(self,interaction):
global chunk_cache
text=jpath(self.path,interaction)
if text is None:
return False
else:
text=text.lower()
if text not in chunk_cache:
chunk_cache[text]=chunk(text)
return contains(chunk_cache[text],(self.word[:-1],self.word[-1]),2)
def is_satisfied_by(self,interaction):
text=jpath(self.path,interaction)
if text is None:
return False
if not isinstance(text,list): # we need to handle the case when the path returns a list, eg for hashtags, so standardise on that
text=[text]
for t in text:
t=t.lower()
if self.term==t:
return True
return False
def is_satisfied_by(self,interaction):
global chunk_cache
text=jpath(self.path,interaction)
if text is None:
return False
if not isinstance(text,list): # we need to handle the case when the path returns a list, e.g. for shareurls, so standardise on that
text=[text]
for t in text:
t=t.lower()
if t not in chunk_cache:
chunk_cache[t]=chunk(t)
if contains(chunk_cache[t],self.chunks,2):
return True
return False
try:
if not ctr % 500 and ctr>0:
print >>sys.stderr, "Looking for candidate features in interaction %i" %ctr
j=json.loads(i)
C=get_class(j)
if C is not None:
if ctr % opts.test_period == 0:
test_interactions.append(j)
test_targets.append(get_classid(C))
else:
interactions.append(j)
targets.append(get_classid(C))
content=jpath(featurepath,j)
if content is not None:
content=content.lower()
normalised_content=re.sub(r'(https?)://t.co/\w+','',content) #http://t.co/rerJLq49gl
normalised_content=re.sub(r'@\w+','',normalised_content) # need to detect @mentions before chunking for unigrams!
listofngrams=list()
words=[w for w in wordsplitter.split(normalised_content) if w not in stopwords and len(w)>0]
if config.useunigrams:
listofngrams+=[(w,) for w in words]
if config.usebigrams:
listofngrams+=ngrams(2,content)
if config.usetrigrams:
listofngrams+=ngrams(3,content)
for ngram in listofngrams:
#if len(ngram)==1 and ngram[0] in stopwords: # ignore unigrams consisting of a stopword
try:
if not ctr % 500 and ctr > 0:
print >> sys.stderr, "Looking for candidate features in interaction %i" % ctr
j = json.loads(i)
C = get_class(j)
if C is not None:
if ctr % opts.test_period == 0:
test_interactions.append(j)
test_targets.append(get_classid(C))
else:
interactions.append(j)
targets.append(get_classid(C))
content = jpath(featurepath, j)
if content is not None:
content = content.lower()
normalised_content = re.sub(
r'(https?)://t.co/\w+', '',
content) #http://t.co/rerJLq49gl
normalised_content = re.sub(
r'@\w+', '', normalised_content
) # need to detect @mentions before chunking for unigrams!
listofngrams = list()
words = [
w for w in wordsplitter.split(normalised_content)
if w not in stopwords and len(w) > 0
]
if config.useunigrams:
def is_satisfied_by(self,interaction):
text=jpath(self.path,interaction)
if text is None:
return False
else:
return self.chars in text # NB CASE SENSITIVE
def is_satisfied_by(self,interaction):
x=jpath(self.path,interaction)
if x is None:
return False
return x==self.val
def is_satisfied_by(self,interaction):
text=jpath(self.path,interaction)
if text is None:
return False
else:
return re.search(self.regex,text) is not None
def is_satisfied_by(self,interaction): return jpath(self.path,interaction) is not None