def test_find_simple_md5(self):
# observer=MyObserver()
# log.addObserver(observer.observe)
text=file(SIMPLE_MD5_FILE_PATH,'rb').read()
#text=UGLY_HACK
hashwords=findHashesInRawPaste(text)
self.assertTrue(len(hashwords)>0,"No messages received")
for i in hashwords:
if "80a751fde577028640c419000e33eba6" in i and "md5" in i:
break
else:
self.fail("Expected md5 value not found :(")
log.msg(str(e))
raise e
try:
pageDetails['snippet']=rawPaste.split("\n")[0]
fixedPasteDetails={}
for (k,v) in pageDetails.items():
fixedPasteDetails[k]=v.encode('utf8','ignore')
except (UnicodeDecodeError,UnicodeEncodeError), e:
log.msg("The type of v for key %s is %s" % (k,str(type(v))))
log.msg(str(e))
raise e
for k in "author date title snippet".split(" "):
if not fixedPasteDetails.has_key(k):
fixedPasteDetails[k]='__Unknown__'
log.msg("%(author)s on %(date)s posted the following about `%(title)s': %(snippet)s" % fixedPasteDetails)
hashwords=findHashesInRawPaste(rawPaste,["confident"])
for h in hashwords:
log.msg("%s found in pastebin ID: %s" % (h,pasteID))
return storePaste(pasteID,fixedPasteDetails,rawPaste,hashwords)
def findUniquePastes(archiveList):
query={'pasteID' : { '$in' : archiveList}}
d=MongoPool().getMongoConnection().addCallback(lambda cnx: cnx.pastebin.pastes.find(query,fields=['pasteID']))
d.addCallback(lambda res: set(archiveList) - set([x["pasteID"] for x in res]))
return d
def storePaste(pasteID,details,raw,hashwords):
data=details.copy()
data['pasteID']=pasteID
data['raw']=raw
data['hashwords']=hashwords
