def monkey_learn(texts, article_dicts_list): if not texts: return None ml = MonkeyLearn(cfg.MONKEYLEARN_TOKEN) # keyword extraction extraction = ml.extractors.extract('ex_y7BPYzNG', texts, use_company_names=1, max_keywords=10) # sentiment analysis classification = ml.classifiers.classify('cl_MX2qQKNi', texts) ml_results = [] for i in range(0, len(extraction.result)): ml_result = { "id": article_dicts_list[i]['id'], "url": article_dicts_list[i]['url'], "classification": classification.result[i], "keyword_extraction": extraction.result[i] } # remove keyword position indicies for kw in ml_result['keyword_extraction']: if kw.get('positions_in_text'): del kw['positions_in_text'] # convert dictionaries to JSON strings ml_results.append(json.dumps(ml_result, separators=(',', ': '))) # Return a list of JSON strings of results, 1 string per article return ml_results
def createMasterDict(input_list, model, API_token='<type Your API Key Here>'): if model in ["MonkeyLearn", "ML", "ml", "monkeylearn"]: ml = MonkeyLearn(API_token) model_id = 'ex_YCya9nrn' consolidated = [] j = 1 for data in input_list: features = {} features['paragraph'] = data features['para_ID'] = j j += 1 result = ml.extractors.extract(model_id, [data]) extrac = result.body[0]['extractions'] keywords = [] for i in range(len(extrac)): keywords.append(extrac[i]['parsed_value']) features['keywords'] = keywords consolidated.append(features) return consolidated elif model in ["rake", "Rake"]: r = Rake( ) # Uses stopwords for english from NLTK, and all puntuation characters. consolidated = [] k = 1 for para in input_list: features_ = {} features_['para_ID'] = k features_['paragraph'] = para k += 1 r.extract_keywords_from_text(para) #print(r.get_ranked_phrases()) # To get keyword phrases ranked highest to lowest. features_['keywords'] = r.get_ranked_phrases() consolidated.append(features_) return consolidated
def monkey_learn(titles, urls): if not titles and urls: return None ml = MonkeyLearn(cfg.MONKEYLEARN_TOKEN) # keyword extraction extraction = ml.extractors.extract('ex_y7BPYzNG', titles, use_company_names=1, max_keywords=10) ml_results = [] for i in range(0, len(extraction.result)): ml_result = { "id": str(hashlib.md5(titles[i].encode('utf8')).hexdigest()), "url": urls[i], "keyword_extraction": extraction.result[i] } # remove keyword position indicies for kw in ml_result['keyword_extraction']: if kw.get('positions_in_text'): del kw['positions_in_text'] print ml_result # convert dictionaries to JSON strings ml_results.append(json.dumps(ml_result, separators=(',', ': '))) # Return a list of JSON strings of results, 1 string per article return ml_results
def sentiment(s): ml = MonkeyLearn('9ad4bee6b5e1a1f32ba54b9c4c68528e5d465442') data = [s] model_id = 'cl_pi3C7JiL' result = ml.classifiers.classify(model_id, data) li = (result.body) #li is in the below format: #[ # { # "text": "This is a great app!", # "external_id": null, # "error": false, # "classifications": [ # { # "tag_name": "Positive", # "tag_id": 33767179, # "confidence": 0.998 # } # ] # } # ] li = li[0]['classifications'][0] if li['tag_name'] == 'Positive': return li['confidence'] else: return 1 - li['confidence']
def keywords(): ml = MonkeyLearn('adc2b91f1e63d6140722f8adf37cae5f1610a27c') typ = request.form['type'] model_id = 'ex_YCya9nrn' print(type) output = request.form['output'] print(output) print(type(output)) if typ == "text": output = re.sub("[^a-zA-Z.,]", " ", output) print(output) keywordresult = ml.extractors.extract(model_id, [output]) print(keywordresult.body) a = keywordresult.body[0] print(a) print(type(a)) b = a['extractions'] print(b) print(type(b)) keywords = [] entities = [] for i in b: keywords.append(i['parsed_value']) print(i['parsed_value']) print(keywords) #return keywords return render_template('keywords.html', keyword=keywords)
def sentimentsAnalysis(): columns = defaultdict(list) with open('media/recording1/transcript.csv') as f: reader = csv.DictReader(f) for row in reader: for (k, v) in row.items(): columns[k].append(v) positiveCounter = 0 negativeCounter = 0 neutralCounter = 0 ml = MonkeyLearn('ab7ba9286a2c0793d287a35eef87b272db9eac8c') model_id = 'cl_pi3C7JiL' for s in columns['sentence']: li = [s] result = ml.classifiers.classify(model_id, li).body print(result[0]['classifications'][0]['tag_name']) if result[0]['classifications'][0]['tag_name'] == 'Positive': positiveCounter += 1 elif result[0]['classifications'][0]['tag_name'] == 'Neutral': neutralCounter += 1 elif result[0]['classifications'][0]['tag_name'] == 'Negative': negativeCounter += 1 print('Positive ' + positiveCounter) print('Negative ' + negativeCounter) print('Neutral ' + neutralCounter) f.close() return "Sentiments result => Neutral-" + neutralCounter + " Negative-" + negativeCounter + " Positive-" + positiveCounter
def app(): st.title('Named Entity Recognization') st.write( "Named Entity Recognition is the task of identifying named entities (people, locations, organizations, etc.) in the input text." ) text = st.text_input('Enter the text') st.write('or') uploaded_file = st.file_uploader("Upload a document", type='txt') if uploaded_file is not None: uploaded_file.seek(0) stringio = StringIO(uploaded_file.read().decode("utf-8")) file_text = stringio.read() st.write(file_text) if st.button('Execute'): ml = MonkeyLearn('b5c3a529126d072f0aa6d43003083ffdc3273a7f') models = ['ex_SmwSdZ3C', 'ex_vqBQ7V9B', 'ex_A9nCcXfn'] model_name = ['PERSON', 'LOCATION', 'COMPANY'] responses = [None for _ in range(len(models))] outputs = [None for _ in range(len(models))] with st.spinner('Extracting...'): if uploaded_file is not None: data = file_text else: data = text for idx, model_id in enumerate(models): responses[idx] = ml.extractors.extract(model_id, data=[data]) outputs[idx] = responses[idx].body[0]['extractions'] for idx, output in enumerate(outputs): st.write(model_name[idx], ' :') for item in output: st.write(item['extracted_text'])
def app(): st.title('Sentiment Analysis') st.write( 'Sentiment Analysis is the task of interpreting and classifying emotions (positive or negative) in the input text.' ) text = st.text_input('Enter the text') st.write('or') uploaded_file = st.file_uploader("Upload a document", type='txt') if uploaded_file is not None: uploaded_file.seek(0) stringio = StringIO(uploaded_file.read().decode("utf-8")) file_text = stringio.read() st.write(file_text) if st.button('Execute'): ml = MonkeyLearn('b5c3a529126d072f0aa6d43003083ffdc3273a7f') model_id = 'cl_pi3C7JiL' with st.spinner('Classifying...'): if uploaded_file is not None: response = ml.classifiers.classify(model_id, [file_text]) else: response = ml.classifiers.classify(model_id, [text]) outputs = response.body[0]['classifications'][0] st.write('The model is are quite sure the sentence is', outputs['tag_name'], '. (', outputs["confidence"] * 100, ' %)')
def sentimentsAnalysis(): columns = defaultdict(list) with open('media/recording1/transcript.csv') as f: reader = csv.DictReader(f) for row in reader: for (k,v) in row.items(): columns[k].append(v) positiveCounter = 0 negativeCounter = 0 neutralCounter = 0 #ab7ba9286a2c0793d287a35eef87b272db9eac8c ml = MonkeyLearn('c9fa1e093134b6bdc811a42f55849bd06a751482') model_id = 'cl_pi3C7JiL' for s in columns['sentence']: li = [s] result = ml.classifiers.classify(model_id, li).body if result[0]['classifications'][0]['tag_name'] == 'Positive': positiveCounter += 1 elif result[0]['classifications'][0]['tag_name'] == 'Neutral': neutralCounter += 1 elif result[0]['classifications'][0]['tag_name'] == 'Negative': negativeCounter += 1 f.close() return f'Sentiments result => Neutral- {neutralCounter} Negative- {negativeCounter} Positive- {positiveCounter}'
def sentimentValue(self, tweet): ml = MonkeyLearn('aeafaa0aa014ed56b663c49fa20ac81a68f2d194') model_id = 'cl_TWmMTdgQ' data = [] data.append(tweet) result = ml.classifiers.classify(model_id, data) return result.body[0]['classifications'][0]
def entity_extract(Id, text, news): from monkeylearn import MonkeyLearn ml = MonkeyLearn('f61694907b120433ddc66da1880d537c5f9d8f1e') text_list = [text] module_id = 'ex_isnnZRbS' res = ml.extractors.extract(module_id, text_list) for row in res.result[0]: if not db.session.query(Keyword).filter( Keyword.key_name == row['entity']).count(): key = Keyword(key_name=row["entity"]) db.session.add(key) db.session.commit() else: key = Keyword.query.filter_by(key_name=row["entity"]).first() if news: nk = NewsKeyword(news_id=Id, key_id=key.id) db.session.add(nk) db.session.commit() else: # if not UserKeyword.query.filter_by(key_id=key.id, user_id=Id).count(): #may not be needed uk = UserKeyword(user_id=Id, key_id=key.id, priority=1) db.session.add(uk) db.session.commit()
def request_sentiment(message): data = [message] ml = MonkeyLearn(API_KEY_MONKEYLEARN) model_id = 'cl_pi3C7JiL' result = ml.classifiers.classify(model_id, data) #result_json = json.loads(result.body) sentiment = result.body[0]['classifications'][0]['tag_name'] return sentiment
def get_keywords(data_): ml = MonkeyLearn('1c0950b2356a258e00b735ab4002679a9a1f3642') model_id = 'ex_YCya9nrn' result = ml.extractors.extract(model_id, data_) extraction = '' for ex in result.body[0]['extractions']: extraction += ex['parsed_value'] + " " return extraction.strip()
def __init__(self, data): ml = MonkeyLearn(MONKEY_API_KEY) model_id = 'cl_pi3C7JiL' result = ml.classifiers.classify(model_id, data) res = result.body res1 = res[0]['classifications'][0]['tag_name'] res2 = res[0]['classifications'][0]['confidence'] self.full_res = res1 + ": " + str(res2)
def extract_scammer_name(message): ml = MonkeyLearn('d31c9827434f85201fc8ba34de9fa26a2ff91936') message = re.sub('\n', '', message) message = re.sub('\r', '', message) data = [message] model_id = 'ex_SmwSdZ3C' result = ml.extractors.extract(model_id, data) scammer_name = result.body[0]["extractions"][0]["parsed_value"] return scammer_name
def sentiment_analysis_monkeylearn(text): ml = MonkeyLearn('1a703adc6e4c0261a67e5e52de43071042af92d6') data = list() data.append(text) model_id = 'cl_pi3C7JiL' result = ml.classifiers.classify(model_id, data) print(result.body) return (result.body)[0]['classifications'][0]['tag_name'], ( result.body)[0]['classifications'][0]['confidence']
def getVibe(data: list): ml = MonkeyLearn('8c2577ec1c3e63771337b22667d9d1c6f36c7675') model_id = 'cl_pi3C7JiL' result = ml.classifiers.classify(model_id, data) vibe = result.body[0].get('classifications')[0].get('tag_name') confidence = result.body[0].get('classifications')[0].get('confidence') if confidence >= 0.3: return vibe else: return 'Neutral'
def __init__(self, token, module_id, field_to_classify, field_classification_output, batch_size, use_sandbox): self.items = [] self.token = token self.module_id = module_id self.ml = MonkeyLearn(token) self.field_to_classify = field_to_classify self.field_classification_output = field_classification_output self.batch_size = batch_size self.use_sandbox = use_sandbox
def get_latest_messages(): api = tweepy.API(auth.auth) public_tweets = api.home_timeline() for tweet in public_tweets: ml = MonkeyLearn(monkeylearn_credentials.MONKEY_AUTH_TOKEN) data = [tweet.text] model_id = 'cl_pi3C7JiL' result = ml.classifiers.classify(model_id, data) return jsonify(result.body), 200
def news_with_ticker(request, Ticker): url = 'https://finance.yahoo.com/quote/{}'.format(str(Ticker)) r = requests.get(url) soup = BeautifulSoup(r.text, 'html.parser') if (soup): price = soup.find('table', {'class': 'W(100%)'}).find("tbody") price2 = price.find('tr', {'data-reactid': '40'}) price3 = price.find('tr', {'data-reactid': '45'}) if (price2 != None and price3 != None): url = ('https://newsapi.org/v2/everything?' 'q={}&' 'from=2021-05-02&' 'sortBy=popularity&' 'apiKey=e42cf979dc004ba6abfca80bb7fce05d'.format( str(Ticker))) # enter api key ml = MonkeyLearn( "3fa9643fa860ec3e9376994cd1fc534b850d4c7d") # enter api key model_id = 'cl_pi3C7JiL' # keep model id same response = requests.get(url) json_dict = (response.json()) sentiment = [] data = {} data['Details'] = [] for i in range(10): sentiment.append(json_dict['articles'][i]['description']) for i in range(len(sentiment)): result = ml.classifiers.classify(model_id, [sentiment[i]]) data['Details'].append({ 'Date': '2021-04-22', 'Tikr': '{}'.format(str(Ticker)), 'Text': str(result.body[0]['text']), 'Sentiment': str(result.body[0]['classifications'][0]['tag_name']), 'Previous Close': str(price2.text.replace("Previous Close", "")), 'Open': str(price3.text.replace("Open", "")) }) output = [] j = 1 for i in data['Details']: output.append({j: i['Text']}) j += 1 #response to be returned to frontend -> json format #returning as rest_framework response for react JS return Response(output) else: return HttpResponse("Not Found")
def tagallfb(request): fb = Feedback.objects.all() ml = MonkeyLearn('2996cb83b0f2c42cdd8d5785a11d5609b7db736d') model_id = 'cl_pi3C7JiL' for x in fb: data = [x.review] result = ml.classifiers.classify(model_id, data) x.sentiment = result.body[0]["classifications"][0]["tag_name"] print(x.review , ' ',x.sentiment) x.save() return HttpResponseRedirect('/hc')
def lambda_handler(event, context): # TODO implement res = requests.get( "https://m4fcmxys7i.execute-api.us-west-2.amazonaws.com/default/reddit_parse" ) json_data = json.loads(res.text) total_symbols = res['total_symbols'] posts_now = res['posts_now'] posts_hot = res['posts_hot'] posts_top = res['posts_top'] posts = pd.concat([posts_now, posts_hot, posts_top]) ##############RECEIVE FROM RES ml = MonkeyLearn('f8727eca4adb7979d592c3d905bfba05ff81863a') model_id = 'cl_pi3C7JiL' stock_sentiment_points = defaultdict(int) for i in range(len(posts)): curr_title = posts.iloc[i, 0] curr_body = posts.iloc[i, 6] curr_comments = " ".join(posts.iloc[i, 8]) for symbol in total_symbols: if symbol in curr_title or symbol in curr_body or symbol in curr_comments: data = [curr_title + " " + curr_body + " " + curr_comments] result = ml.classifiers.classify(model_id, data) if result.body[0]["classifications"][0][ "tag_name"] == "Positive": stock_sentiment_points[symbol] += 1 else: stock_sentiment_points[symbol] -= 1 stock_sentiment_points[symbol] = stock_sentiment_points[ symbol] + curr_title.count("buy") + curr_body.count( "buy") + curr_comments.count("buy") stock_sentiment_points[symbol] = stock_sentiment_points[ symbol] - curr_title.count("sell") + curr_body.count( "sell") + curr_comments.count("sell") for symbol in total_symbols: if symbol not in stock_sentiment_points: stock_sentiment_points[symbol] = 0 for key in stock_sentiment_points.keys(): pos = "Hold" if stock_sentiment_points[key] > 0: pos = "Buy" elif stock_sentiment_points[key] < 0: pos = "Sell" table.put_item(Item={'stock': key, 'action': pos}) response = {'message': 'Item added'} return {'statusCode': 200, 'body': response}
def get_keywords_from_description(description: str, number_keywords=5) -> List[str]: ml = MonkeyLearn('c3e523c989d56faa8e968f6881a0269fe6f5dc09') model_id_for_key_word_distraction = 'ex_YCya9nrn' response_body_str = ml.extractors.extract( model_id=model_id_for_key_word_distraction, data=[description]).body keywords = [] for res in response_body_str: keywords.extend([kw['parsed_value'] for kw in res['extractions']]) return keywords[:number_keywords]
def CallApis(text): os.environ[ "GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Usuario/SocialHackers/src/claves.json" #import os #from google.oauth2 import service_account texto = str(text) print(texto + 'el texto del funcionario') #gmaps.configure(api_key="AIzaSyDo-i5z39is5UaVtmic0aAdenVlf33fkow") #credentials = service_account.Credentials.from_service_account_file("C:/Users/Usuario/claves.json") #client = translate.TranslateServiceClient(credentials=credentials) ml = MonkeyLearn('04331f53b46d850f750509ec4bf92f17b0df803a') translate_client = translate.Client() result = translate_client.translate(text, target_language='en') print(u"Text: {}".format(result["input"])) print(u"Translation: {}".format(result["translatedText"])) print(u"Detected source language: {}".format( result["detectedSourceLanguage"])) texttraducido = result["translatedText"] response_type = ml.classifiers.classify(model_id='cl_o46qggZq', data=[texttraducido]) response_sentiment = ml.classifiers.classify(model_id='cl_pi3C7JiL', data=[texttraducido]) tag_type = response_type.body[0]['classifications'][0]['tag_name'] tag_sentiment = response_sentiment.body[0]['classifications'][0][ 'tag_name'] if tag_type == "Health & Medicine": return "Salud" elif tag_type == "Food & Drink": return "Comida" elif tag_type == "Environment": return "Medio Ambiente" elif tag_type == "Education": return "Educacion" elif tag_type == "Society": return "Trabajo" elif tag_sentiment == "Negative": return "Psicologia" else: return "Others" #fig = gmaps.figure() #fig #headers = {"Accept": "application/json", "Content-Type": "application/json"} #text = { "text": "'+translate+'" } #url = "https://sentim-api.herokuapp.com/api/v1/" #response = requests.post(url, data=json.dumps(text), headers=headers) #print(response) #print(u"Type: {}".format(response["type"]))
def classify_intent(msg_body): ml = MonkeyLearn('673dabe257b841679b142da12a8cf7683633e7a8') data = [msg_body] model_id = 'cl_VGoxtFhL' result = ml.classifiers.classify(model_id, data) # print(result.body) classification_result = result.body[0]['classifications'] confidence = classification_result[0]['confidence'] intent = classification_result[0]['tag_name'] # print("Customer query: ", query) # print("Intent: ", classification_result[0]['tag_name']) return {'confidence': confidence, 'intent': intent}
def ml(): ml = MonkeyLearn("f0662751f5792b646356ec5516580c239b71aefb") text_list = [ "Disney Pixar Cars Toon: Mater's Tall tales - Walmart (Wii) Cars Toon: Mater's Tall Tales [Disney Pixar], a Nintendo console exclusive, is a comedy themed mini-game(s) / party game with action racing elements." ] module_id = 'cl_oFKL5wft' res = ml.classifiers.classify(module_id, text_list, sandbox=True) list1 = res.result str1 = ''.join(str(e) for e in list1) str2 = str1.rstrip('\n') str3 = str2.replace("u'", "'") str4 = str3.replace("'", '"') parsed = json.loads(str4) return parsed
def verifyOTP(request): fbid = request.GET['fbid'] fb = get_object_or_404(Feedback,pk=fbid) otp = int(str(request.POST['d1'])+str(request.POST['d2'])+str(request.POST['d3'])+str(request.POST['d4'])+str(request.POST['d5'])+str(request.POST['d6'])) if otp==fb.otp: fb.verified=True ml = MonkeyLearn('2996cb83b0f2c42cdd8d5785a11d5609b7db736d') data = [fb.review] model_id = 'cl_pi3C7JiL' result = ml.classifiers.classify(model_id, data) fb.sentiment = result.body[0]["classifications"][0]["tag_name"] fb.save() return render(request,'healthcenter/thanks.html') else: return render(request,'healthcenter/verify.html',{'em':fb.email,'fbid':fb.id,'fl':2})
def get_context_data(self, **kwargs): cleanr = re.compile('<.*?>') ml = MonkeyLearn('3f5c2d1c62cfd67d36c38b31500b6d3f93fa9d9c') model_id = 'cl_pi3C7JiL' data = super().get_context_data(**kwargs) likes_connected = get_object_or_404(Article, id=self.kwargs['pk']) body = [re.sub(cleanr, '', likes_connected.body)] response = ml.classifiers.classify(model_id, body) liked = False if likes_connected.likes.filter(id=self.request.user.id).exists(): liked = True data['number_of_likes'] = likes_connected.number_of_likes() data['post_is_liked'] = liked data['sentimentalAnanlysis'] = response.body[0]['classifications'][0][ 'tag_name'] return data
def get_tweets(username): #This will extract the tweets auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) api = tweepy.API(auth) number_of_tweets = 1 tweets = api.user_timeline(screen_name=username) temp = [] tweets_for_csv = [tweet.text for tweet in tweets] for j in tweets_for_csv: temp.append(j) amt = len(temp) #This will get the total number of tweets retrieved rand_num = randrange(amt - 1) #This will randomly select a tweet from the list post = temp[rand_num] ml = MonkeyLearn('dd20ef03405ebdd8301a985f30db2b2e0b570d92') data = [post] model_id = 'cl_4keevyV9' result = ml.classifiers.classify(model_id, data) counts = Counter( chain.from_iterable(i.keys() for i in result.body[0]['classifications'])) if len( result.body[0]['classifications'] ) == 0: #If the tweet doesn't belong in any categories the system will exit print(post) sys.exit("The tweet did not match any exisiting topics") for key, value in counts.items(): tag_name = value break print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") print("Tweet:", result.body[0]['text']) for i in range(value): print("Tag:", result.body[0]['classifications'][i]['tag_name'], "\tConfidence:", result.body[0]['classifications'][i]['confidence'])
def run_pipeline(handle): ml = MonkeyLearn('') data = { "twitter_user_name": handle, "twitter_access_token_key": '', "twitter_consumer_key": '', "twitter_consumer_secret": '', "twitter_access_token_secret": '' } module_id = 'pi_JJ9JrKvk' res = ml.pipelines.run(module_id, data, sandbox=True) keyword_array = [] num = len(res.result['keywords'][0]['keywords']) for x in range(0,num): keyword_array.append(res.result['keywords'][0]['keywords'][x]['keyword']) arr_of_arr.append(keyword_array)