def __getsize(self):
try:
rows, columns = func.getTerminalSize()
except ValueError:
rows = columns = 0
if int(columns) > self.max + 2 + 44 + (len(str(self.max))*2 + 2):
self.columns = self.max
else:
# note: -2 is for brackets, -44 for 'Fitting islands...' text, rest is for pos/max text
self.columns = int(columns) - 2 - 44 - (len(str(self.max))*2 + 2)
return
from bs4 import BeautifulSoup
import functions as fn
import logging
import os
import json
import psycopg2
import re
import requests
import settings
# set news portal name
newsportalname = 'Republika Online'
# read width and height of the terminal window
(width, height) = fn.getTerminalSize()
# [1] Initiating connection to PostgreSQL DB and create table
try:
conn = psycopg2.connect(user=settings.RIPPLEET_USER,
password=settings.RIPPLEET_PASS,
host=settings.PSQL_HOST,
port=settings.PSQL_PORT,
database=settings.PSQL_DB)
cursor = conn.cursor()
create_table_q = '''CREATE TABLE IF NOT EXISTS rippleet_news (ID SERIAL PRIMARY KEY NOT NULL, NEWSPORTAL TEXT NOT NULL, TITLE TEXT NOT NULL, CONTENT TEXT NOT NULL, NEWSURL TEXT NOT NULL, NEWSTAG TEXT NOT NULL, PUBDATE DATE NOT NULL);'''
cursor.execute(create_table_q)
conn.commit()
print " [+] " + "Table created successfully (or already exists) in PostgreSQL Database: %s" % (
def print_opts(grouped_opts_list, img, banner=None):
"""Print options to screen.
Options can be sorted by group (defined in opts.py) previously defined by
group_opts. Output of grouped items is suppressed if parent option is
False. The layout is as follows:
[20 spaces par name with ...] = [at least 49 spaces for value]
[at least 49 spaces for doc]
When more than one line is required for the doc, the next line is:
[25 blank spaces][at least 47 spaces for doc]
As in casapy, print non-defaults in blue, options with suboptions in
47m and suboptions in green. Option Values are printed in bold, to help
to distinguish them from the descriptions. NOTE: in iTerm, one needs
to set the bold color in the profiles to white, as it defaults to red,
which is a bit hard on the eyes in this case.
"""
from image import Image
import os
import functions as func
termy, termx = func.getTerminalSize() # note: returns row, col -> y, x
minwidth = 28 # minimum width for parameter names and values
# Define colors for output
dc = '\033[1;34m' # Blue: non-default option text color
ec = '\033[0;47m' # expandable option text color
sc = '\033[0;32m' # Green: suboption text color
nc = '\033[0m' # normal text color
ncb = '\033[1m' # normal text color bold
if banner is not None:
print banner
spcstr = ' ' * minwidth # spaces string for second or later lines
infix = nc + ': ' + nc # infix character used to separate values from comments
print '=' * termx # division string for top of parameter listing
for indx, o in enumerate(grouped_opts_list):
if isinstance(o, tuple):
# Print main options, which are always tuples, before printing
# suboptions (if any).
k = o[0]
v = o[1]
val = img.opts.__getattribute__(k)
v1 = v2 = ''
if val == v._default:
# value is default
v1 = ncb
v2 = nc
else:
# value is non-default
v1 = dc
v2 = nc
if isinstance(val, str):
valstr = v1 + repr(val) + v2
if k == 'filename':
# Since we can check whether filename is valid,
# do so here and print in red if not.
if not os.path.exists(val):
valstr = '\033[31;1m' + repr(val) + nc
width_par_val = max(minwidth, len(k) + len(str(val)) + 5)
else:
if isinstance(val, float):
val = round_float(val)
if isinstance(val, tuple):
val = round_tuple(val)
valstr = v1 + str(val) + v2
width_par_val = max(minwidth, len(k) + len(str(val)) + 4)
width_desc = max(termx - width_par_val - 3, 44)
# Get the option description text from the doc string, which
# is defined in opts.py. By convention, print_opts will only
# show the short description; help('option_name') will
# print both the short and long description. The versions
# are separated in the doc string by '\n', which is split
# on here:
desc_text = wrap(str(v.doc()).split('\n')[0], width_desc)
fmt = '%' + str(minwidth) + 's' + infix + '%44s'
# Now loop over lines of description
if indx < len(grouped_opts_list) - 1:
# Here we check if next entry in options list is a tuple or a
# list. If it is a list, then the current option has
# suboptions and should be in the ec color. Since we check the
# next option, we can't do this if we let indx go to the end.
if isinstance(grouped_opts_list[indx + 1], tuple):
parvalstr = nc + k + nc + ' ..'
else:
parvalstr = ec + k + nc + ' ..'
else:
# Since this is the last entry in the options list and is a
# tuple, it cannot be an expandable option, so make it nc color
parvalstr = nc + k + nc + ' ..'
if "'" in valstr:
len_without_formatting = len(k) + len(str(val)) + 5
else:
len_without_formatting = len(k) + len(str(val)) + 4
for i in range(len_without_formatting, minwidth):
parvalstr += '.'
parvalstr += ' ' + valstr
if "'" not in valstr:
parvalstr += ' '
for dt_indx, dt in enumerate(desc_text):
if dt_indx == 0:
print fmt % (parvalstr.ljust(minwidth), dt.ljust(44))
else:
print nc + spcstr + ' %44s' % dt.ljust(44)
else:
# Print suboptions, indented 2 spaces from main options in sc color
parent_opt = grouped_opts_list[indx - 1]
parent_val = img.opts.__getattribute__(parent_opt[0])
if parent_val == True:
for og in o:
k = og[0]
v = og[1]
val = img.opts.__getattribute__(k)
v1 = v2 = ''
if val == v._default:
# value is default
v1 = ncb
v2 = nc
else:
# value is non-default
v1 = dc
v2 = nc
if isinstance(val, str):
valstr = v1 + repr(val) + v2
width_par_val = max(minwidth,
len(k) + len(str(val)) + 7)
else:
if isinstance(val, float):
val = round_float(val)
if k == 'beam_spectrum' and val is not None:
val = round_list_of_tuples(val)
if k == 'frequency_sp' and val is not None:
val = round_list(val)
valstr = v1 + str(val) + v2
width_par_val = max(minwidth,
len(k) + len(str(val)) + 6)
width_desc = max(termx - width_par_val - 3, 44)
desc_text = wrap(str(v.doc()).split('\n')[0], width_desc)
fmt = ' ' + '%' + str(minwidth) + 's' + infix + '%44s'
parvalstr = sc + k + nc + ' ..'
if "'" in valstr:
len_without_formatting = len(k) + len(str(val)) + 7
else:
len_without_formatting = len(k) + len(str(val)) + 6
for i in range(len_without_formatting, minwidth):
parvalstr += '.'
parvalstr += ' ' + valstr
if "'" not in valstr:
parvalstr += ' '
for dt_indx, dt in enumerate(desc_text):
if dt_indx == 0:
print fmt % (parvalstr.ljust(minwidth - 2),
dt.ljust(44))
else:
print nc + spcstr + ' %44s' % dt.ljust(44)
def print_opts(grouped_opts_list, img, banner=None):
"""Print options to screen.
Options can be sorted by group (defined in opts.py) previously defined by
group_opts. Output of grouped items is suppressed if parent option is
False. The layout is as follows:
[20 spaces par name with ...] = [at least 49 spaces for value]
[at least 49 spaces for doc]
When more than one line is required for the doc, the next line is:
[25 blank spaces][at least 47 spaces for doc]
As in casapy, print non-defaults in blue, options with suboptions in
47m and suboptions in green. Option Values are printed in bold, to help
to distinguish them from the descriptions. NOTE: in iTerm, one needs
to set the bold color in the profiles to white, as it defaults to red,
which is a bit hard on the eyes in this case.
"""
from image import Image
import os
import functions as func
termy, termx = func.getTerminalSize() # note: returns row, col -> y, x
minwidth = 28 # minimum width for parameter names and values
# Define colors for output
dc = '\033[1;34m' # Blue: non-default option text color
ec = '\033[0;47m' # expandable option text color
sc = '\033[0;32m' # Green: suboption text color
nc = '\033[0m' # normal text color
ncb = '\033[1m' # normal text color bold
if banner is not None:
print banner
spcstr = ' ' * minwidth # spaces string for second or later lines
infix = nc + ': ' + nc # infix character used to separate values from comments
print '=' * termx # division string for top of parameter listing
for indx, o in enumerate(grouped_opts_list):
if isinstance(o, tuple):
# Print main options, which are always tuples, before printing
# suboptions (if any).
k = o[0]
v = o[1]
val = img.opts.__getattribute__(k)
v1 = v2 = ''
if val == v._default:
# value is default
v1 = ncb
v2 = nc
else:
# value is non-default
v1 = dc
v2 = nc
if isinstance(val, str):
valstr = v1 + repr(val) + v2
if k == 'filename':
# Since we can check whether filename is valid,
# do so here and print in red if not.
if not os.path.exists(val):
valstr = '\033[31;1m' + repr(val) + nc
width_par_val = max(minwidth, len(k) + len(str(val)) + 5)
else:
if isinstance(val, float):
val = round_float(val)
if isinstance(val, tuple):
val = round_tuple(val)
valstr = v1 + str(val) + v2
width_par_val = max(minwidth, len(k) + len(str(val)) + 4)
width_desc = max(termx - width_par_val - 3, 44)
# Get the option description text from the doc string, which
# is defined in opts.py. By convention, print_opts will only
# show the short description; help('option_name') will
# print both the short and long description. The versions
# are separated in the doc string by '\n', which is split
# on here:
desc_text = wrap(str(v.doc()).split('\n')[0], width_desc)
fmt = '%' + str(minwidth) + 's' + infix + '%44s'
# Now loop over lines of description
if indx < len(grouped_opts_list)-1:
# Here we check if next entry in options list is a tuple or a
# list. If it is a list, then the current option has
# suboptions and should be in the ec color. Since we check the
# next option, we can't do this if we let indx go to the end.
if isinstance(grouped_opts_list[indx+1], tuple):
parvalstr = nc + k + nc + ' ..'
else:
parvalstr = ec + k + nc + ' ..'
else:
# Since this is the last entry in the options list and is a
# tuple, it cannot be an expandable option, so make it nc color
parvalstr = nc + k + nc + ' ..'
if "'" in valstr:
len_without_formatting = len(k) + len(str(val)) + 5
else:
len_without_formatting = len(k) + len(str(val)) + 4
for i in range(len_without_formatting, minwidth):
parvalstr += '.'
parvalstr += ' ' + valstr
if "'" not in valstr:
parvalstr += ' '
for dt_indx, dt in enumerate(desc_text):
if dt_indx == 0:
print fmt % (parvalstr.ljust(minwidth), dt.ljust(44))
else:
print nc + spcstr + ' %44s' % dt.ljust(44)
else:
# Print suboptions, indented 2 spaces from main options in sc color
parent_opt = grouped_opts_list[indx-1]
parent_val = img.opts.__getattribute__(parent_opt[0])
if parent_val == True:
for og in o:
k = og[0]
v = og[1]
val = img.opts.__getattribute__(k)
v1 = v2 = ''
if val == v._default:
# value is default
v1 = ncb
v2 = nc
else:
# value is non-default
v1 = dc
v2 = nc
if isinstance(val, str):
valstr = v1 + repr(val) + v2
width_par_val = max(minwidth, len(k) + len(str(val)) + 7)
else:
if isinstance(val, float):
val = round_float(val)
if k == 'beam_spectrum' and val is not None:
val = round_list_of_tuples(val)
if k == 'frequency_sp' and val is not None:
val = round_list(val)
valstr = v1 + str(val) + v2
width_par_val = max(minwidth, len(k) + len(str(val)) + 6)
width_desc = max(termx - width_par_val - 3, 44)
desc_text = wrap(str(v.doc()).split('\n')[0], width_desc)
fmt = ' ' + '%' + str(minwidth) + 's' + infix + '%44s'
parvalstr = sc + k + nc + ' ..'
if "'" in valstr:
len_without_formatting = len(k) + len(str(val)) + 7
else:
len_without_formatting = len(k) + len(str(val)) + 6
for i in range(len_without_formatting, minwidth):
parvalstr += '.'
parvalstr += ' ' + valstr
if "'" not in valstr:
parvalstr += ' '
for dt_indx, dt in enumerate(desc_text):
if dt_indx == 0:
print fmt % (parvalstr.ljust(minwidth-2), dt.ljust(44))
else:
print nc + spcstr + ' %44s' % dt.ljust(44)
def on_data(self, data):
(width, height) = fn.getTerminalSize()
objects = []
try:
# split each line of data
objects = data.splitlines()
for line in objects:
# loads the data into json object
d = json.loads(line)
# processing the data .encode('utf-8',errors='ignore')
if d["lang"] == "id" or d["lang"] == "in":
if 'extended_tweet' in d and 'RT @' not in d[
'extended_tweet']['full_text']:
teks_twit = re.sub(r'[^a-zA-Z0-9_@ ,.-:/]', '',
d['extended_tweet']['full_text'])
teksclean = fn.multiwordReplace(teks_twit, fn.wordDic)
# print teksclean
# print console_colors.OKBLUE + d['user']['screen_name'].encode('utf-8') + ": " + console_colors.ENDC + teksclean + "\n"
# print("GMT: "+console_colors.WHITE+time.strftime("%a, %d %b %Y %I:%M:%S GMT"+console_colors.ENDC, time.gmtime()))
# print("Local: "+console_colors.WHITE+strftime("%a, %d %b %Y %I:%M:%S %p %Z (GMT+7)\r"+console_colors.ENDC))
# sentiment check
analisis = textblob.TextBlob(
d['extended_tweet']['full_text'])
an = analisis.translate(from_lang="id", to="en")
# print(an.sentiment)
# print(an.polarity)
elif 'extended_tweet' not in d and 'RT @' not in d['text']:
teks_twit = re.sub(r'[^a-zA-Z0-9_@ ,.-:/]', '',
d['text'])
teksclean = fn.multiwordReplace(teks_twit, fn.wordDic)
#print teksclean
# print console_colors.OKBLUE + d['user']['screen_name'].encode('utf-8') + ": " + console_colors.ENDC + teksclean + "\n"
# print("GMT: "+console_colors.WHITE+time.strftime("%a, %d %b %Y %I:%M:%S GMT"+console_colors.ENDC, time.gmtime()))
# print("Local: "+console_colors.WHITE+strftime("%a, %d %b %Y %I:%M:%S %p %Z (GMT+7)\r"+console_colors.ENDC))
# sentiment check
analisis = textblob.TextBlob(d['text'])
an = analisis.translate(from_lang="id", to="en")
# print(an.sentiment)
# print(an.polarity)
# capturing tweets which are retweet of other's tweet
else:
teks_twit = re.sub(r'[^a-zA-Z0-9_@ ,.-:/]', '',
d['text'])
teksclean = fn.multiwordReplace(teks_twit, fn.wordDic)
# print teksclean
# print console_colors.OKBLUE + d['user']['screen_name'].encode('utf-8') + ": " + console_colors.ENDC + teksclean + "\n"
# print("GMT: "+console_colors.WHITE+time.strftime("%a, %d %b %Y %I:%M:%S GMT"+console_colors.ENDC, time.gmtime()))
# print("Local: "+console_colors.WHITE+strftime("%a, %d %b %Y %I:%M:%S %p %Z (GMT+7)\r"+console_colors.ENDC))
# sentiment check
analisis = textblob.TextBlob(d['text'])
an = analisis.translate(from_lang="id", to="en")
# print(an.sentiment)
# print(an.polarity)
for keyword in settings.BLOCK_TERMS:
if keyword in teks_twit:
fn.ToRify()
print console_colors.WHITE + "Username: "******"@%s" % (
console_colors.OKBLUE +
d['user']['screen_name'] + console_colors.ENDC
) + " | " + "Followers= %s" % (str(
d['user']['followers_count']))
print console_colors.WHITE + "Tweet: " + console_colors.ENDC + "%s" % (
teks_twit)
print console_colors.WHITE + "Polarity: " + console_colors.ENDC + "%s" % (
an.polarity)
# report account
api.report_spam(
screen_name=d['user']['screen_name'])
print "Keywords %s found in tweet text.. >> " % (
keyword
) + console_colors.RED + " FOUND" + console_colors.ENDC + " << (Time elapsed: %d)" % (
fn.gettimeNow() - starttime)
print str(d['user']
['screen_name']) + ' is being reported.'
print console_colors.GREY + "_" * width + console_colors.ENDC
# with open(logfile,'a+') as records:
# records.write("Reported account: @%s (Follower: %s)" %(d['user']['screen_name'],str(d['user']['followers_count'])) + " | " + "Tweet: %s.\n" %(teks_twit))
# records.close()
with open(logfile, 'a+') as records:
records.write("@%s\n" %
(d['user']['screen_name']))
records.close()
else:
print console_colors.WHITE + "Username: "******"@%s" % (
console_colors.OKBLUE +
d['user']['screen_name'] + console_colors.ENDC
) + " | " + "Followers= %s" % (str(
d['user']['followers_count']))
print console_colors.WHITE + "Tweet: " + console_colors.ENDC + "%s" % (
teks_twit)
print console_colors.WHITE + "Polarity: " + console_colors.ENDC + "%s" % (
an.polarity)
print "Keywords %s not found in tweet text.. >> " % (
keyword
) + console_colors.GREY + "skip" + console_colors.ENDC + " << (Time elapsed: %d)" % (
fn.gettimeNow() - starttime)
print console_colors.GREY + "_" * width + console_colors.ENDC
# delay 2 secs
time.sleep(2)
else:
pass
except BaseException as e:
print("Error on_data: %s" % str(e))
if str(e) == "HTTP Error 429: Too Many Requests":
print "Sleep mode for 5 minutes."
time.sleep(300)
else:
print "Delay for 5 seconds."
time.sleep(5)
# record the data into postgresql data
def on_exception(self, exception):
print(exception)
return
def on_error(self, status):
print(status)
return True
def on_data(self, data):
(width, height) = fn.getTerminalSize()
objects = []
try:
# split each line of data
objects = data.splitlines()
# objects = re.split(r"[~\r\n]+", data)
for line in objects:
#print type(line)
# loads the data into json object
d = json.loads(line)
# processing the data
if d["lang"] == "id" or d["lang"] == "in":
if 'extended_tweet' in d and 'RT @' not in d[
'extended_tweet']['full_text']:
teks_twit = re.sub(r'[^a-zA-Z0-9_@ ,.-:/]', '',
d['extended_tweet']['full_text'])
teksclean = fn.multiwordReplace(teks_twit, fn.wordDic)
#print teksclean
print console_colors.OKBLUE + d['user'][
'screen_name'] + ": " + console_colors.ENDC + teksclean + "\n"
print(
"GMT: " + console_colors.WHITE + time.strftime(
"%a, %d %b %Y %I:%M:%S GMT" +
console_colors.ENDC, time.gmtime()))
print(
"Local: " + console_colors.WHITE +
strftime("%a, %d %b %Y %I:%M:%S %p %Z (GMT+7)\r" +
console_colors.ENDC))
# send data to socket for processing in spark
print d['user']['screen_name']
analisis = textblob.TextBlob(
d['extended_tweet']['full_text'])
an = analisis.translate(from_lang="id", to="en")
print(an.sentiment)
print(an.polarity)
# self.client_socket.send(data)
# indexing data to Elasticsearch
try:
ok = fn.connect_elasticsearch()
fn.create_index_tweet(ok, nameofindex)
fn.store_record(ok, nameofindex, d)
except Exception as ex:
print('Error creating index: ', str(ex))
traceback.print_exc(file=sys.stdout)
# insert record to PostgreSQL table
try:
conn = psycopg2.connect(
user=settings.RIPPLEET_USER,
password=settings.RIPPLEET_PASS,
host=settings.PSQL_HOST,
port=settings.PSQL_PORT,
database=settings.PSQL_DB)
cursor = conn.cursor()
# insert data into table
sqlquery = ''' INSERT INTO rippleet_tweet (TIMECOL, SCREENNAME, TWEET, WORDCOUNTS, REPLYCOUNTS, RETWEETCOUNTS, FAVCOUNTS, FOLLOWER, POLARITY) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) '''
cursor.execute(
sqlquery,
(fn.gettimeNow(), d['user']['screen_name'],
teksclean,
len(d['extended_tweet']['full_text'].split()),
d['reply_count'], d['retweet_count'],
d['favorite_count'],
d['user']['followers_count'], an.polarity))
conn.commit()
count = cursor.rowcount
print(count,
" Record inserted successfully into table")
except (Exception, psycopg2.Error) as error:
if (conn):
print("Failed to insert record into table",
error)
finally:
#closing database connection.
if (conn):
cursor.close()
conn.close()
print(" [*] PostgreSQL connection is closed")
print console_colors.GREY + "_" * width + console_colors.ENDC
elif 'extended_tweet' not in d and 'RT @' not in d['text']:
teks_twit = re.sub(r'[^a-zA-Z0-9_@ ,.-:/]', '',
d['text'])
teksclean = fn.multiwordReplace(teks_twit, fn.wordDic)
#print teksclean
print console_colors.OKBLUE + d['user'][
'screen_name'] + ": " + console_colors.ENDC + teksclean + "\n"
print(
"GMT: " + console_colors.WHITE + time.strftime(
"%a, %d %b %Y %I:%M:%S GMT" +
console_colors.ENDC, time.gmtime()))
print(
"Local: " + console_colors.WHITE +
strftime("%a, %d %b %Y %I:%M:%S %p %Z (GMT+7)\r" +
console_colors.ENDC))
# send data to socket for processing in spark
print d['user']['screen_name']
analisis = textblob.TextBlob(d['text'])
an = analisis.translate(from_lang="id", to="en")
print(an.sentiment)
print(an.polarity)
# self.client_socket.send(data)
# indexing data to Elasticsearch
try:
ok = fn.connect_elasticsearch()
fn.create_index_tweet(ok, nameofindex)
# create_index(ok, "rippleet_tweet")
fn.store_record(ok, nameofindex, d)
except Exception as ex:
print('Error creating index: ', str(ex))
traceback.print_exc(file=sys.stdout)
# insert record to table
try:
conn = psycopg2.connect(
user=settings.RIPPLEET_USER,
password=settings.RIPPLEET_PASS,
host=settings.PSQL_HOST,
port=settings.PSQL_PORT,
database=settings.PSQL_DB)
cursor = conn.cursor()
# insert data into table
sqlquery = ''' INSERT INTO rippleet_tweet (TIMECOL, SCREENNAME, TWEET, WORDCOUNTS, REPLYCOUNTS, RETWEETCOUNTS, FAVCOUNTS, FOLLOWER, POLARITY) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) '''
cursor.execute(
sqlquery,
(fn.gettimeNow(), d['user']['screen_name'],
teksclean, len(
d['text'].split()), d['reply_count'],
d['retweet_count'], d['favorite_count'],
d['user']['followers_count'], an.polarity))
conn.commit()
count = cursor.rowcount
print(count,
" Record inserted successfully into table")
except (Exception, psycopg2.Error) as error:
if (conn):
print("Failed to insert record into table",
error)
finally:
#closing database connection.
if (conn):
cursor.close()
conn.close()
print(" [*] PostgreSQL connection is closed")
print console_colors.GREY + "_" * width + console_colors.ENDC
# capturing tweets which are retweet of other's tweet
else:
teks_twit = re.sub(r'[^a-zA-Z0-9_@ ,.-:/]', '',
d['text'])
teksclean = fn.multiwordReplace(teks_twit, fn.wordDic)
#print teksclean
print console_colors.OKBLUE + d['user'][
'screen_name'] + ": " + console_colors.ENDC + teksclean + "\n"
print(
"GMT: " + console_colors.WHITE + time.strftime(
"%a, %d %b %Y %I:%M:%S GMT" +
console_colors.ENDC, time.gmtime()))
print(
"Local: " + console_colors.WHITE +
strftime("%a, %d %b %Y %I:%M:%S %p %Z (GMT+7)\r" +
console_colors.ENDC))
# send data to socket for processing in spark
print d['user']['screen_name']
analisis = textblob.TextBlob(d['text'])
an = analisis.translate(from_lang="id", to="en")
print(an.sentiment)
print(an.polarity)
# self.client_socket.send(data)
# indexing data to Elasticsearch
try:
ok = fn.connect_elasticsearch()
fn.create_index_tweet(ok, nameofindex)
# create_index(ok, "rippleet_tweet")
fn.store_record(ok, nameofindex, d)
except Exception as ex:
print('Error creating index: ', str(ex))
traceback.print_exc(file=sys.stdout)
pass
# insert record to table
try:
conn = psycopg2.connect(
user=settings.RIPPLEET_USER,
password=settings.RIPPLEET_PASS,
host=settings.PSQL_HOST,
port=settings.PSQL_PORT,
database=settings.PSQL_DB)
cursor = conn.cursor()
# insert data into table
sqlquery = ''' INSERT INTO rippleet_tweet (TIMECOL, SCREENNAME, TWEET, WORDCOUNTS, REPLYCOUNTS, RETWEETCOUNTS, FAVCOUNTS, FOLLOWER, POLARITY) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) '''
cursor.execute(
sqlquery,
(fn.gettimeNow(), d['user']['screen_name'],
teksclean, len(
d['text'].split()), d['reply_count'],
d['retweet_count'], d['favorite_count'],
d['user']['followers_count'], an.polarity))
conn.commit()
count = cursor.rowcount
print(count,
" Record inserted successfully into table")
except (Exception, psycopg2.Error) as error:
if (conn):
print("Failed to insert record into table",
error)
finally:
#closing database connection.
if (conn):
cursor.close()
conn.close()
print(" [*] PostgreSQL connection is closed")
print console_colors.GREY + "_" * width + console_colors.ENDC
# delay 2 secs
time.sleep(2)
else:
pass
except BaseException as e:
print("Error on_data: %s" % str(e))
if str(e) == "HTTP Error 429: Too Many Requests":
print "Sleep mode for 5 minutes."
time.sleep(300)
else:
print "Delay for 5 seconds."
time.sleep(5)
# record the data into postgresql data
def on_exception(self, exception):
print(exception)
return
def on_error(self, status):
print(status)
return True
