def init_run(self):
"""
Prints info message and performs application authentication with Twitter.
:return: None
"""
Helper.output("Spawning query thread [Q].")
self.avivore.twitter_auth()
def init_database(self):
"""
Prepares a sqlite3 database for data set storage. If the file specified in AvivoreConfig.database_path
doesn't exist a new sqlite3 database with table 'Data' is created. Otherwise the existing database is used to
store additional data sets.
:return: None
"""
if not Helper.is_string(self.database_path):
raise AvivoreConfigException('Invalid database path specified!')
table_data_exists = False
if os.path.isfile(self.database_path):
Helper.output('Using existing database to store results.')
try:
dbcon = lite.connect(self.database_path)
dbcur = dbcon.cursor()
dbcur.execute('SELECT Count(*) FROM Data')
Helper.output(
str(dbcur.fetchone()[0]) +
' entries in this database so far.')
table_data_exists = True
except lite.OperationalError:
Helper.output('[W] Table \'Data\' not found in database!')
if not table_data_exists: # If the database doesn't exist, we'll create it.
Helper.output(
'Creating new table \'Data\' in database \'%s\' to store data!'
% self.database_path)
dbcon = lite.connect(self.database_path)
dbcur = dbcon.cursor()
dbcur.execute(
'CREATE TABLE Data (TimeRecv int, Type int, User text, UserId text, Value text, TID int, Message text)'
)
def init_database(self):
"""
Prepares a sqlite3 database for data set storage. If the file specified in AvivoreConfig.database_path
doesn't exist a new sqlite3 database with table 'Data' is created. Otherwise the existing database is used to
store additional data sets.
:return: None
"""
if not Helper.is_string(self.database_path):
raise AvivoreConfigException('Invalid database path specified!')
table_data_exists = False
if os.path.isfile(self.database_path):
Helper.output('Using existing database to store results.')
try:
dbcon = lite.connect(self.database_path)
dbcur = dbcon.cursor()
dbcur.execute('SELECT Count(*) FROM Data')
Helper.output(str(dbcur.fetchone()[0]) + ' entries in this database so far.')
table_data_exists = True
except lite.OperationalError:
Helper.output('[W] Table \'Data\' not found in database!')
if not table_data_exists: # If the database doesn't exist, we'll create it.
Helper.output('Creating new table \'Data\' in database \'%s\' to store data!' % self.database_path)
dbcon = lite.connect(self.database_path)
dbcur = dbcon.cursor()
dbcur.execute(
'CREATE TABLE Data (TimeRecv int, Type int, User text, UserId text, Value text, TID int, Message text)')
def twitter_search(self, search_string):
try:
search_results = self.twitter_instance.search.tweets(q=search_string)
output = search_results['statuses']
except (TwitterHTTPError, URLError):
Helper.output("[!] Problem connecting to twitter...")
output = None # If this bombs out, we have the option of at least spitting out a result.
except Exception as e:
Helper.output("[!] Unknown problem querying twitter:\n%s" % e)
output = None # If this bombs out, we have the option of at least spitting out a result.
return output
def test_filepath_exists(self):
self.assertTrue(Helper.filepath_exists('/etc/passwd'))
self.assertTrue(Helper.filepath_exists('/etc/bullshit'))
self.assertFalse(Helper.filepath_exists('/complete_bullshit/bs'))
self.assertTrue(Helper.filepath_exists('/blubb'))
self.assertTrue(Helper.filepath_exists('test/__init__.py'))
self.assertTrue(Helper.filepath_exists('test/invalid_file'))
self.assertTrue(Helper.filepath_exists('test'))
self.assertTrue(Helper.filepath_exists('./test'))
self.assertTrue(Helper.filepath_exists('./test/'))
def test_filepath_exists(self):
self.assertTrue(Helper.filepath_exists('/etc/passwd'))
self.assertTrue(Helper.filepath_exists('/etc/bullshit'))
self.assertFalse(Helper.filepath_exists('/complete_bullshit/bs'))
self.assertTrue(Helper.filepath_exists('/blubb'))
self.assertTrue(Helper.filepath_exists('test/__init__.py'))
self.assertTrue(Helper.filepath_exists('test/invalid_file'))
self.assertTrue(Helper.filepath_exists('test'))
self.assertTrue(Helper.filepath_exists('./test'))
self.assertTrue(Helper.filepath_exists('./test/'))
def __init__(self, config_type, config_filename):
if not Helper.is_string(config_filename):
raise AvivoreConfigException('Invalid config file specified!')
self.config_type = config_type
self.config_filename = config_filename
self.config = ConfigParser()
self.twitter_search_types = []
self.twitter_search_terms = []
self.twitter_track_keywords = None
self.twitter_consumer_key = None
self.twitter_consumer_secret = None
self.twitter_search_interval = 30
self.credentials_file = None
self.database_path = None
self.mandatory_options = [
# Be sure to keep order of database table layout for options from 'config' database
# append type definitions for twitter_search_objects!
('database', 'dbpath'),
('twitter_auth', 'consumer_key'),
('twitter_auth', 'consumer_secret'),
('twitter_auth', 'credentials_file'),
('twitter_search', 'stream_tracking_keyword'),
('twitter_search', 'csv_search_term'),
('twitter_search_objects', '0'),
]
def __init__(self, config_type, config_filename):
if not Helper.is_string(config_filename):
raise AvivoreConfigException('Invalid config file specified!')
self.config_type = config_type
self.config_filename = config_filename
self.config = ConfigParser()
self.twitter_search_types = []
self.twitter_search_terms = []
self.twitter_track_keywords = None
self.twitter_consumer_key = None
self.twitter_consumer_secret = None
self.twitter_search_interval = 30
self.credentials_file = None
self.database_path = None
self.mandatory_options = [
# Be sure to keep order of database table layout for options from 'config' database
# append type definitions for twitter_search_objects!
('database', 'dbpath'),
('twitter_auth', 'consumer_key'),
('twitter_auth', 'consumer_secret'),
('twitter_auth', 'credentials_file'),
('twitter_search', 'stream_tracking_keyword'),
('twitter_search', 'csv_search_term'),
('twitter_search_objects', '0'),
]
def init_config_database(self, config_database_path):
"""
Prepares a sqlite3 database for use as a configuration database. Creates a database with the specified
filename and creates empty 'Config' and 'TypeDefs' database tables that are going to hold all configuration
settings.
:param config_database_path: Path to database file.
:return: Returns a cursor to the sqlite3 database connection that can be used for further database
operations.
"""
if not os.path.isfile(config_database_path):
if not Helper.filepath_exists(config_database_path):
raise AvivoreConfigException(
'Invalid path to configuration database file specified!')
Helper.output('[W] Configuration database file not found!')
Helper.output('[W] Creating a new configuration database.')
Helper.output(
'[W] Now please store your configuration in database file \'%s\', then try again!'
% config_database_path)
dbcon = lite.connect(config_database_path)
dbcur = dbcon.cursor()
dbcur.execute(
'CREATE TABLE IF NOT EXISTS Config (dbpath text, consumer_key text, consumer_secret text, \
credentials_file text, stream_tracking_keyword text, csv_search_term, interval int)'
)
dbcur.execute(
'CREATE TABLE IF NOT EXISTS TypeDefs (Id int, Regex text, Comment text)'
)
return dbcur
def extract_data_from_tweet(self, avivore, tweet):
"""
Scans data for extractable data sets (previously defined in the 'TypeDefs' section of the AvivoreXT
configuration) and if found stores them in the result database.
:param avivore: Previously initialized Avivore instance.
:param tweet: The tweet to analyzed that was returned from Twitter as a JSON object.
:return: None
"""
z = tweet['id'], tweet['created_at'], tweet['user']['screen_name'], tweet['text'], tweet['user']['id_str']
result = avivore.twitter_read_tweet(z[3])
if result[0] >= 0: # If something is found, then we'll process the tweet
self.stored = self.stored, int(z[0])
# result value, time, result itself, tweet ID, tweet itself, userId
string = result[0], z[2], result[1], z[0], z[3], z[4]
message = avivore.process_tweet(string)
if message is not None:
Helper.output("[Q] " + message)
def twitter_read_tweet(self, string):
i = -1
for x in self.avivore_config.twitter_search_types:
i += 1
findtype = re.compile(x)
result_raw = findtype.findall(string)
if not result_raw:
continue
else:
if Helper.is_sequence(result_raw[0]):
result = [x for x in result_raw[0] if x]
else:
result = result_raw[0]
if Helper.is_sequence(result):
result = result[0]
return i, result
# nothing found
return -1, 0
def init_config_database(self, config_database_path):
"""
Prepares a sqlite3 database for use as a configuration database. Creates a database with the specified
filename and creates empty 'Config' and 'TypeDefs' database tables that are going to hold all configuration
settings.
:param config_database_path: Path to database file.
:return: Returns a cursor to the sqlite3 database connection that can be used for further database
operations.
"""
if not os.path.isfile(config_database_path):
if not Helper.filepath_exists(config_database_path):
raise AvivoreConfigException('Invalid path to configuration database file specified!')
Helper.output('[W] Configuration database file not found!')
Helper.output('[W] Creating a new configuration database.')
Helper.output('[W] Now please store your configuration in database file \'%s\', then try again!'
% config_database_path)
dbcon = lite.connect(config_database_path)
dbcur = dbcon.cursor()
dbcur.execute(
'CREATE TABLE IF NOT EXISTS Config (dbpath text, consumer_key text, consumer_secret text, \
credentials_file text, stream_tracking_keyword text, csv_search_term, interval int)')
dbcur.execute('CREATE TABLE IF NOT EXISTS TypeDefs (Id int, Regex text, Comment text)')
return dbcur
def test_output(self):
self.assertTrue(None == Helper.output("Test"))
self.assertTrue(None == Helper.output("\u2031\u203c\u2049"))
self.assertTrue(None == Helper.output(self))
self.assertTrue(None == Helper.output(None))
self.assertTrue(None == Helper.output(True))
self.assertTrue(None == Helper.output(Helper.output))
self.assertTrue(None == Helper.output(3))
self.assertTrue(None == Helper.output(3.14))
self.assertTrue(None == Helper.output(['a1', 'b2', 'c3']))
self.assertTrue(None == Helper.output(('it1', 'it2')))
self.assertTrue(None == Helper.output(array('l', [1, 2, 3, 4, 5])))
self.assertTrue(None == Helper.output(array('B', [0x41, 0x42, 0x43, 0x44, 0x45])))
self.assertTrue(None == Helper.output(bytearray.fromhex("deadbeef")))
self.assertTrue(None == Helper.output(dict(one=1, two=2, three=3)))
def test_is_sequence(self):
# testing sequential types
self.assertTrue(Helper.is_sequence(['value1', 'value2', 'value3']))
self.assertTrue(Helper.is_sequence(('item1', 'item2')))
self.assertTrue(Helper.is_sequence(array('l', [1, 2, 3, 4, 5])))
self.assertTrue(
Helper.is_sequence(array('d', [1.0, 2.0, 3.0, 4.0, 5.0])))
self.assertTrue(Helper.is_sequence(bytearray.fromhex("deadbeef")))
self.assertTrue(Helper.is_sequence(dict(one=1, two=2, three=3)))
# testing non-sequential types
self.assertFalse(Helper.is_sequence("Hello World"))
self.assertFalse(Helper.is_sequence("Hello World\u203c"))
self.assertFalse(Helper.is_sequence(3))
self.assertFalse(Helper.is_sequence(3.14))
self.assertFalse(Helper.is_sequence(self))
self.assertFalse(Helper.is_sequence(None))
self.assertFalse(Helper.is_sequence(True))
self.assertFalse(Helper.is_sequence(Helper.is_sequence))
def test_is_sequence(self):
# testing sequential types
self.assertTrue(Helper.is_sequence(['value1', 'value2', 'value3']))
self.assertTrue(Helper.is_sequence(('item1', 'item2')))
self.assertTrue(Helper.is_sequence(array('l', [1, 2, 3, 4, 5])))
self.assertTrue(Helper.is_sequence(array('d', [1.0, 2.0, 3.0, 4.0, 5.0])))
self.assertTrue(Helper.is_sequence(bytearray.fromhex("deadbeef")))
self.assertTrue(Helper.is_sequence(dict(one=1, two=2, three=3)))
# testing non-sequential types
self.assertFalse(Helper.is_sequence("Hello World"))
self.assertFalse(Helper.is_sequence("Hello World\u203c"))
self.assertFalse(Helper.is_sequence(3))
self.assertFalse(Helper.is_sequence(3.14))
self.assertFalse(Helper.is_sequence(self))
self.assertFalse(Helper.is_sequence(None))
self.assertFalse(Helper.is_sequence(True))
self.assertFalse(Helper.is_sequence(Helper.is_sequence))
def test_output(self):
self.assertTrue(None == Helper.output("Test"))
self.assertTrue(None == Helper.output("\u2031\u203c\u2049"))
self.assertTrue(None == Helper.output(self))
self.assertTrue(None == Helper.output(None))
self.assertTrue(None == Helper.output(True))
self.assertTrue(None == Helper.output(Helper.output))
self.assertTrue(None == Helper.output(3))
self.assertTrue(None == Helper.output(3.14))
self.assertTrue(None == Helper.output(['a1', 'b2', 'c3']))
self.assertTrue(None == Helper.output(('it1', 'it2')))
self.assertTrue(None == Helper.output(array('l', [1, 2, 3, 4, 5])))
self.assertTrue(
None == Helper.output(array('B', [0x41, 0x42, 0x43, 0x44, 0x45])))
self.assertTrue(None == Helper.output(bytearray.fromhex("deadbeef")))
self.assertTrue(None == Helper.output(dict(one=1, two=2, three=3)))
def twitter_stream_main(self):
Helper.output("Beginning stream processing [S].")
stored = []
twitter_stream_inst = self.twitter_stream_auth()
try:
iterator = twitter_stream_inst.statuses.filter(track=self.avivore_config.twitter_track_keywords)
for y in iterator:
# tweet may be a delete or data msg
if y is None:
pass
# tweet may indicate a timeout
elif y is Timeout:
Helper.output("[S] Stream timeout...")
elif y is HeartbeatTimeout:
Helper.output("[S] Heartbeat timeout...")
# tweet may indicate a hung up stream conn.
elif y is Hangup:
Helper.output("[S] Stream hangup detected!")
# ok, tweet seems to contain text, process it
elif 'id' in y and 'created_at' in y and 'user' in y and 'text' in y:
# print y
z = y['id'], y['created_at'], y['user']['screen_name'], y['text'], y['user']['id_str']
result = self.twitter_read_tweet(z[3])
if result[0] < 0:
pass
else: # If something is found, then we'll process the tweet
stored = stored, int(z[0])
# result value, time, result itself, tweet ID, tweet itself, userId
string = result[0], z[2], result[1], z[0], z[3], z[4]
message = self.process_tweet(string)
if 0 != message:
Helper.output("[S] " + message)
except (TwitterHTTPError, URLError):
Helper.output("[!] Can't connect to twitter stream! Check your network connection!")
except Exception as e:
Helper.output("[!] Unknown stream processing error: %s\n" % (str(e)))
finally:
Helper.output("[S] Stream processing stopped.")
