def setUp(self):
self.api = TextApi(testConfiguration.client)
def gettokens(txt):
return api.getTokensForText(config.RETINA_NAME, txt)
def getslices(txt):
return api.getSlicesForText(config.RETINA_NAME, txt)
if __name__ == '__main__':
client = config.client
#api = TermsApi(client)
#terms = api.getTerm("en_associative", term="apple", get_fingerprint=True)
api = TextApi(client)
scpr = scraper()
kwords = []
tokens = []
targettxt = url2txt(scpr, TARGET_COMPANY_URL)
candidatetxts = PDF2txt("resume.pdf")
for i in range(len(candidatetxts)):
kwords += getkw(candidatetxts[i])
tokens += gettokens(candidatetxts[i])
print("Keywords extracted, Tokens ready")
#slices=[getslices(candidatetxts[i]) for i in range(len(candidatetxts))]
if testing_slate:
print([r.encode('utf-8') for r in kwords])
print([r.encode('utf-8') for r in tokens])
def summarize(text, len_sentences = 5, retina = retina, apiKey = apiKey,
min_length = min_length, similarity_measure = similarity_measure) :
sentences = [sentence for sentence in text.split(".") if len(sentence) >= min_length]
# Make request to cortical to compare sentence by sentence to get distance
# for graph
# graph representation, to use : graph[0][1] is edge of vertex 0 to 1
graph = nx.Graph()
request_body = []
for s in sentences:
request_body.append({"text":s})
request_body = json.dumps(request_body)
client = ApiClient(apiKey=apiKey, apiServer="http://api.cortical.io/rest")
api = TextApi(client)
fingerprints = api.getRepresentationsForBulkText(retina, request_body)
pos_fingerprints = zip(range(0,len(sentences)),fingerprints)
double_pos_fingerprints = itertools.combinations(pos_fingerprints, 2)
for (pos1,finger1),(pos2,finger2) in double_pos_fingerprints:
graph.add_weighted_edges_from([(pos1,pos2,
cosine_similarity(finger1.positions,
finger2.positions))])
print(pos1,pos2)
"""
request_body = []
for s in sentences:
for s2 in sentences:
request_body.append([{"text":s},{"text":s2}])
request_body = json.dumps(request_body)
client = ApiClient(apiKey=apiKey, apiServer="http://api.cortical.io/rest")
api = CompareApi(client)
print("Sending to compare API")
metrics = api.compareBulk(retina, request_body)
print("Receive compare API")
for i in range(len(sentences)):
for j in range(len(sentences)):
graph.add_weighted_edges_from([(i,j,metrics[i*len(sentences)+j])].cosineSimilarity)
"""
pos_fingerprints = zip(range(0,len(sentences)),fingerprints)
# Compute pagerank for all vertex in matrix, refer to https://en.wikipedia.org/wiki/PageRank
page_rank = nx.pagerank(graph)
sorted_rank = [(page_rank[key],key) for key in page_rank]
sorted_rank.sort(reverse = True)
summary_graph = nx.Graph()
summaries = [pos for weight,pos in sorted_rank[:len_sentences]]
for i in range(0,len(summaries)):
for j in range(i+1,len(summaries)):
summary_graph.add_weighted_edges_from([(summaries[i],summaries[j],
graph[i][j]['weight'])])
gr = nx.minimum_spanning_tree(summary_graph)
visited = {}
summary = []
for tup in list(nx.dfs_edges(gr)):
for first in tup:
if (visited.get(first) is None):
summary.append(sentences[first])
visited[first] = True
return summary
# Code to get fingerprints from a string
# Body = "Semantic fingerprints are cool."
# Code to get fingerprints from a .txt file put filename
file_name = "15_UTX.txt"
with open('Company_Descriptions/' + file_name, "r") as myfile:
body = myfile.read().replace('\n', '')
####################################################
######### Code for fingerprint (vector) ############
####################################################
# Chose either en_synonymous or en_associative retina
text = TextApi(client).getRepresentationForText("en_synonymous", body)
print text[0].positions
####################################################
############# Code for keywords list ###############
####################################################
# Chose either en_synonymous or en_associative retina
terms = TextApi(client).getKeywordsForText("en_synonymous", body)
print terms
#####################################################
########## Code for fingerprint (image) #############
#####################################################
body = '{"text":"%s"}' % body
