def on_status(self, status):
logging.info("writing tweet to couchdb")
data = None
try:
data = analysis.extract(status)
except Exception as e:
logging.info("exception in parsing tweet: {}".format(e))
return True
data["sentiment"] = analysis.sentiment(data["text"])
self.db.save(json.loads(json.dumps(data)))
def on_status(self, status):
logging.info("writing tweet to {}".format(self.file))
data = None
try:
data = analysis.extract(status)
except Exception as e:
logging.info("exception in parsing tweet: {}".format(e))
return True
data["sentiment"] = analysis.sentiment(data["text"])
print(data, file=self.file)
return True
def list_campaigns():
curl = None
try:
curl, buffer = generateRequest('http://twitris.knoesis.org/api/v1/campaigns')
curl.perform()
data = json.loads(buffer.getvalue())
resp = []
for c in data:
if "movie" in c['campaign_type']:
c_id = c['id']
name = c['event']
c['info'] = get_info(name, True)
c['sentiment'] = sentiment(c_id, internal=True)
c['emotions'] = emotions(c_id, internal=True)
resp.append(c)
return make_response(jsonify({ "campaigns": resp}), curl.getinfo(curl.RESPONSE_CODE))
except Exception, e:
return serverError("error %s" % e)
def main():
try:
first_sleep = randrange(INITIAL_SLEEP_MAX)
logger.info('Starting retweet script, but first sleeping %d minutes'
% first_sleep)
sleep(60*first_sleep)
# Get tweets
tweets = getTweets(config.TWEET_LIKES,
result_type='mixed')
# Analyze tweets
analyzed = []
db.init_db()
retweets = [tweet.tweet_id for
tweet in db.session.query(ReTweet).all()]
for tweet in tweets:
if tweet.get('id_str') not in retweets:
analyzed.append((analysis.sentiment(tweet.get('text')),
tweet.get('id_str')
)
)
analyzed = sorted(analyzed, key=itemgetter(0), reverse=True)
# Retweet the most positive tweets
retweet_count = randrange(RETWEET_COUNT_MAX)
logger.info('Retweeting %d tweets' % retweet_count)
for i in range(retweet_count):
tweet = analyzed[i]
reTweet(tweet[1])
new_retweet = ReTweet()
new_retweet.tweet_id = tweet[1]
new_retweet.sentiment = tweet[0]
new_retweet.retweet_date = datetime.datetime.now()
db.session.add(new_retweet)
db.session.commit()
sleep_time = randrange(TWEET_SLEEP_MAX)
logger.info(
'Retweeting %d/%d tweet_id: %s, with sentiment %s and sleeping'
' for %d minutes' % (i+1,
retweet_count,
analyzed[0][1],
analyzed[0][0],
sleep_time))
sleep(60*sleep_time)
except Exception, err:
logger.error('%s: %s' % (Exception, err))
logging.info("using query {}".format(query))
auth = credentials.authenticate(cred_user)
api = API(auth, wait_on_rate_limit=True)
search_results = Cursor(api.search,
q=query,
count=MAX_COUNT,
include_entities=True).items(5000000)
try:
for status in search_results:
logging.info("saving tweet with id {}".format(status.id))
try:
data = analysis.extract(status)
except KeyError as e:
logging.info("exception in extract: {}".format(e))
continue
if data is None:
continue
data["sentiment"] = analysis.sentiment(data["text"])
db.save(json.loads(json.dumps(data)))
# print(data)
except error.TweepError as e: # Should cover RateLimitException
logging.error("exception in search_results: {}".format(e))
logging.info("exiting...")
import analysis
# quote = 'hello world. This is C++'
# testing = analysis.sentiment(quote)
# print(testing)
quote = input("Please enter string: ")
res = analysis.sentiment(quote)
tipe = res[0]
percent = res[1]
if percent <= 0.8:
print("Neutral")
elif tipe == 'pos':
print("Positive")
elif tipe == 'neg':
print("Negative")
def predict_NER():
mydb, mycursor = connectDB()
sql_insert1 = "INSERT INTO `berita_crawling` SELECT * FROM `temp`"
mycursor.execute(sql_insert1)
mydb.commit()
sql_query = """SELECT * FROM `temp`"""
mycursor.execute(sql_query)
berita = mycursor.fetchall()
#Load model NER masing-masing entitas
nlp1 = spacy.load('Person')
nlp2 = spacy.load('Position')
nlp3 = spacy.load('Organization')
nlp4 = spacy.load('Location')
nlp5 = spacy.load('Indicator')
nlp6 = spacy.load('Quote')
for row in berita:
id = row[0]
sumber = row[1]
tanggal = row[6]
judul = row[4]
konten = row[7]
konten = re.sub(r"[#/?]", "", konten)
doc1 = nlp1(konten)
doc2 = nlp2(konten)
doc3 = nlp3(konten)
doc4 = nlp4(konten)
doc5 = nlp5(konten)
doc6 = nlp6(konten)
# mengambil teks hasil prediksi dari label
person_temp = [(e.text) for e in doc1.ents if e.label_ == 'person']
position_temp = [(e.text) for e in doc2.ents if e.label_ == 'position']
organization_temp = [(e.text) for e in doc3.ents
if e.label_ == 'organization']
location_temp = [(e.text) for e in doc4.ents if e.label_ == 'location']
indicator_temp = [(e.text) for e in doc5.ents
if e.label_ == 'indicator']
quote_temp = [(e.text) for e in doc6.ents if e.label_ == 'quote']
# memasukkkan hasil prediksi kedalam list
person = []
position = []
organization = []
location = []
indicator = []
quote = []
for row in person_temp:
if row not in person:
person.append(row)
for row in position_temp:
if row not in position:
position.append(row)
for row in organization_temp:
if row not in organization:
organization.append(row)
for row in location_temp:
if row not in location:
location.append(row)
for row in indicator_temp:
if row not in indicator:
indicator.append(row)
for row in quote_temp:
if row not in quote:
quote.append(row)
# konversi list to str
person = str(person)
position = str(position)
organization = str(organization)
location = str(location)
indicator = str(indicator)
quote = str(quote)
# insert ke DB
mydb, mycursor = connectDB()
sql_insert_query = """INSERT INTO `temp_output`(`id`, `sumber`, `tanggal`, `judul_berita`, `konten_berita`,
`nama_tokoh`, `jabatan`, `organisasi`, `lokasi`, `alias`, `kutipan`)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
insert_tuple = (id, sumber, tanggal, judul, konten, person, position,
organization, location, indicator, quote)
mycursor.execute(sql_insert_query, insert_tuple)
# menghapus DB sementara
mycursor.execute("DELETE FROM `temp` WHERE 1")
mydb.commit()
# memanggil fungsi lainnya
print("Mengklasifikasi alias ke indikator BPS")
klasifikasi_indikator() # mengklasifikasikan alias ke indikator BPS
print("Mengelompokkan Sentimen")
sentiment() # mengelompokkan sentimen berita dan sentimen kutipan
mycursor.execute("INSERT INTO `output` SELECT * FROM `temp_output`")
mycursor.execute("DELETE FROM `temp_output` WHERE 1")
mydb.commit()
print("Menghitung ulang sub_indikator")
hitung_sub_indikator(
) # menghitung value dari setiap sub indikator BPS untuk zoomable sunburst
zoomable_sunburst() # membangun data JSON untuk zoomable sunburst
donutchart() # membangun data csv untuk donut chart
if not hostile_pair_found and valid_line:
for dyad in non_hostile_dyads:
betrayer = dyad[0]
victim = dyad[1]
if betrayer == atom.speaker and victim in atom.audience:
arb_betrayer_lines.append(atom.content)
elif victim == atom.speaker and betrayer in atom.audience:
arb_victim_lines.append(atom.content)
betrayer_diag = ' '.join(betrayer_lines)
victim_diag = ' '.join(victim_lines)
arb_betrayer_diag = ' '.join(arb_betrayer_lines)
arb_victim_diag = ' '.join(arb_victim_lines)
betrayer_sents = analysis.sentiment(nlp, betrayer_diag)
victim_sents = analysis.sentiment(nlp, victim_diag)
arb_betrayer_sents = analysis.sentiment(nlp, arb_betrayer_diag, 17000)
arb_victim_sents = analysis.sentiment(nlp, arb_victim_diag, 17000)
if len(betrayer_sents) > 0:
p_b = sentiments_to_percent_positive(betrayer_sents)
bootstrap_b = analysis.bootstrap(betrayer_sents,
sentiments_to_percent_positive,
BOOTSTRAP_NUM_SAMPLES)
else:
p_b = 0
bootstrap_b = (0, 0, 0)
if len(victim_sents) > 0:
p_v = sentiments_to_percent_positive(victim_sents)
