def generate_tags(request, feedback_pk):
context = {}
client = IODClient(APIURL, APIKEY)
feedback_email = Email.objects.get(pk=feedback_pk)
index = client.getIndex('mailsift')
r = client.post('findsimilar', {'text':feedback_email.message, 'indexes':'mailsift'})
similar_feedback = r.json()
if len(similar_feedback['documents']) > 0:
similar_email = Email.objects.get(pk=similar_feedback['documents'][0]['reference'])
feedback_email.tags = similar_email.tags
else:
feedback_email.tags = ['']
feedback_email.save()
doc1={'reference':feedback_email.pk,'title':feedback_email.subject, 'content':feedback_email.message}
docs = [doc1]
index.addDocs([doc1])
for doc in docs:
index.pushDoc(doc)
print index.size()
index.commit()
print index.size()
context['test'] = feedback_email
return render(request, 'parse/test.html', context)
def test(request):
context = {}
client = IODClient(APIURL, APIKEY)
r=client.post('analyzesentiment',{'text':'I like cats'})
analyzesentiment=r.json()
sentiment = analyzesentiment['aggregate']['sentiment']
context['sentiment']=analyzesentiment
hightlight_sentiment = ''
for word in analyzesentiment[sentiment]:
hightlight_sentiment += '{},'.format(word['topic'])
print hightlight_sentiment + " here"
r=client.post('highlighttext',{'text':'I like cats', 'highlight_expression':'{}'.format(hightlight_sentiment), 'start_tag':'<b>', 'end_tag':'</b>', })
context['highlight']=r.json()['text']
index = client.getIndex('mailsift')
doc1={'reference':'doc1','title':'title1','content':'this is my content'}
doc2={'reference':'doc2','title':'title2','content':'this is another content'}
doc3={'reference':'doc3','title':'title2','content':'this is another content'}
doc4={'reference':'doc2','title':'titleNew','content':'this is another content alksdjflkjasdfkljaslkdf'}
docs = [doc1, doc2, doc3, doc4]
index.addDocs([doc1, doc2, doc3, doc4])
for doc in docs:
index.pushDoc(doc)
print index.size()
index.commit()
print index.size()
return render(request, 'parse/test.html', context)
def main(input,delete,resume,config):
config=json.loads(config.read())
config.update(json.loads(input.read()))
client = IODClient("http://api.idolondemand.com/",
config["iodkey"])
if delete:
deleteIndex(config["iodindex"])
category=False
description=False
flavor="standard"
if config["type"]=="categories":
category=True
flavor="categorization"
if config.get("description",False):
description=True
try:
index=client.createIndex(config["iodindex"],flavor="categorization",index_fields=config.get("indexfields",[]), parametric_fields=config.get("parametricfields",[]))
except:
index=client.getIndex(config["iodindex"])
cursor=""
if resume:
cursor= open('cursor','rb').read()
query= config["freebasequery"]
freebaseUtil = FreebaseUtil(config["freebasekey"])
freebaseUtil.aliases=config["aliasfields"]
freebaseUtil.runQuery(index,query,category=category,description=description,cursor=cursor)
from alchemyapi import AlchemyAPI
from iodpython.iodindex import IODClient
import os
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
ALCHEMYAPI_KEY = os.environ["DOSSIER_ALCHEMY_KEY"]
ALCHEMY_RELEVANCE_THRESHOLD = 0.7
alchemyapi = AlchemyAPI()
client = IODClient("http://api.idolondemand.com/", os.environ["DOSSIER_IDOL_KEY"])
index = client.getIndex("conversations")
cardIndex = client.getIndex("cards")
#index a conversation
def dossierConversation(transcript):
information = extractInformation(transcript)
if "name" in information:
title = "Conversation with " + information["name"]
addCardToIndex(information)
else:
information["name"] = ""
title = "Conversation"
# try:
# index=client.deleteIndex('comic_series')
# except:
# pass
try:
fields=["created_by_name","publisher_name","source_type_name","genre_name","continues_name","continued_by_name"]
index=client.createIndex('comic_series',index_fields=fields, parametric_fields=fields)
# index=client.createIndex('quotesdb',index_fields=["spoken_by_character_*","author_*","source_*","incorrectly_attributed_to_*","addressee_*"],parametric_fields=["spoken_by_character_*","author_*","source_*","incorrectly_attributed_to_*","addressee_*"])
print "sleeping"
time.sleep(5)
print "resuming"
except:
index=client.getIndex('comic_series')
#except:
## print "getting instead"
#index= client.getIndex('comic_book_series')
#cursor = freebaseUtil.do_query(index)
cursor = sync.get("cursor","")
if not cursor:
cursor = freebaseUtil.do_query(index)
while(cursor):
cursor = freebaseUtil.do_query(index,cursor)
sync["cursor"]=cursor
import collections
from alchemyapi import AlchemyAPI
from iodpython.iodindex import IODClient
import os
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
ALCHEMYAPI_KEY = os.environ["DOSSIER_ALCHEMY_KEY"]
ALCHEMY_RELEVANCE_THRESHOLD = 0.7
alchemyapi = AlchemyAPI()
client = IODClient("http://api.idolondemand.com/",
os.environ["DOSSIER_IDOL_KEY"])
index = client.getIndex("conversations")
cardIndex = client.getIndex("cards")
#index a conversation
def dossierConversation(transcript):
information = extractInformation(transcript)
if "name" in information:
title = "Conversation with " + information["name"]
addCardToIndex(information)
else:
information["name"] = ""
title = "Conversation"
freebaseUtil = FreebaseUtil(freebase_key,query)
client = IODClient("http://api.idolondemand.com/",
iod_key)
# client.deleteIndex('quotesdb')
index=client.getIndex('newquotesdb')
#try:
# fields=["spoken_by_character_name","author_name","author_profession","source_type_name","source_name","incorrectly_attributed_to_name","addressee_name"]
# index=client.createIndex('quotesdb',index_fields=fields, parametric_fields=fields)
# index=client.createIndex('quotesdb',index_fields=["spoken_by_character_*","author_*","source_*","incorrectly_attributed_to_*","addressee_*"],parametric_fields=["spoken_by_character_*","author_*","source_*","incorrectly_attributed_to_*","addressee_*"])
#except:
# index=client.getIndex('quotesdb')
#except:
## print "getting instead"
#index= client.getIndex('comic_book_series')
cursor = freebaseUtil.do_query(index)
while(cursor):
print cursor
# Initiate Html stripper
sourcelist= json.loads(open('sources.json').read())
schedulesecs=config["schedulesecs"]
database=config["database"]
keyconf = shelve.open('../config.db')
iod_key=keyconf['iod-key']
client = IODClient("http://api.idolondemand.com/",
iod_key)
#try:
try:
index=client.createIndex(database)
print index
time.sleep(3)
except:
print "getting instead"
index=client.getIndex(database)
def main():
start()
if __name__ == "__main__":
main()
# time.sleep(5)
