def init_run(self): """ Prints info message and performs application authentication with Twitter. :return: None """ Helper.output("Spawning query thread [Q].") self.avivore.twitter_auth()
def init_database(self): """ Prepares a sqlite3 database for data set storage. If the file specified in AvivoreConfig.database_path doesn't exist a new sqlite3 database with table 'Data' is created. Otherwise the existing database is used to store additional data sets. :return: None """ if not Helper.is_string(self.database_path): raise AvivoreConfigException('Invalid database path specified!') table_data_exists = False if os.path.isfile(self.database_path): Helper.output('Using existing database to store results.') try: dbcon = lite.connect(self.database_path) dbcur = dbcon.cursor() dbcur.execute('SELECT Count(*) FROM Data') Helper.output( str(dbcur.fetchone()[0]) + ' entries in this database so far.') table_data_exists = True except lite.OperationalError: Helper.output('[W] Table \'Data\' not found in database!') if not table_data_exists: # If the database doesn't exist, we'll create it. Helper.output( 'Creating new table \'Data\' in database \'%s\' to store data!' % self.database_path) dbcon = lite.connect(self.database_path) dbcur = dbcon.cursor() dbcur.execute( 'CREATE TABLE Data (TimeRecv int, Type int, User text, UserId text, Value text, TID int, Message text)' )
def init_database(self): """ Prepares a sqlite3 database for data set storage. If the file specified in AvivoreConfig.database_path doesn't exist a new sqlite3 database with table 'Data' is created. Otherwise the existing database is used to store additional data sets. :return: None """ if not Helper.is_string(self.database_path): raise AvivoreConfigException('Invalid database path specified!') table_data_exists = False if os.path.isfile(self.database_path): Helper.output('Using existing database to store results.') try: dbcon = lite.connect(self.database_path) dbcur = dbcon.cursor() dbcur.execute('SELECT Count(*) FROM Data') Helper.output(str(dbcur.fetchone()[0]) + ' entries in this database so far.') table_data_exists = True except lite.OperationalError: Helper.output('[W] Table \'Data\' not found in database!') if not table_data_exists: # If the database doesn't exist, we'll create it. Helper.output('Creating new table \'Data\' in database \'%s\' to store data!' % self.database_path) dbcon = lite.connect(self.database_path) dbcur = dbcon.cursor() dbcur.execute( 'CREATE TABLE Data (TimeRecv int, Type int, User text, UserId text, Value text, TID int, Message text)')
def twitter_search(self, search_string): try: search_results = self.twitter_instance.search.tweets(q=search_string) output = search_results['statuses'] except (TwitterHTTPError, URLError): Helper.output("[!] Problem connecting to twitter...") output = None # If this bombs out, we have the option of at least spitting out a result. except Exception as e: Helper.output("[!] Unknown problem querying twitter:\n%s" % e) output = None # If this bombs out, we have the option of at least spitting out a result. return output
def test_filepath_exists(self): self.assertTrue(Helper.filepath_exists('/etc/passwd')) self.assertTrue(Helper.filepath_exists('/etc/bullshit')) self.assertFalse(Helper.filepath_exists('/complete_bullshit/bs')) self.assertTrue(Helper.filepath_exists('/blubb')) self.assertTrue(Helper.filepath_exists('test/__init__.py')) self.assertTrue(Helper.filepath_exists('test/invalid_file')) self.assertTrue(Helper.filepath_exists('test')) self.assertTrue(Helper.filepath_exists('./test')) self.assertTrue(Helper.filepath_exists('./test/'))
def __init__(self, config_type, config_filename): if not Helper.is_string(config_filename): raise AvivoreConfigException('Invalid config file specified!') self.config_type = config_type self.config_filename = config_filename self.config = ConfigParser() self.twitter_search_types = [] self.twitter_search_terms = [] self.twitter_track_keywords = None self.twitter_consumer_key = None self.twitter_consumer_secret = None self.twitter_search_interval = 30 self.credentials_file = None self.database_path = None self.mandatory_options = [ # Be sure to keep order of database table layout for options from 'config' database # append type definitions for twitter_search_objects! ('database', 'dbpath'), ('twitter_auth', 'consumer_key'), ('twitter_auth', 'consumer_secret'), ('twitter_auth', 'credentials_file'), ('twitter_search', 'stream_tracking_keyword'), ('twitter_search', 'csv_search_term'), ('twitter_search_objects', '0'), ]
def init_config_database(self, config_database_path): """ Prepares a sqlite3 database for use as a configuration database. Creates a database with the specified filename and creates empty 'Config' and 'TypeDefs' database tables that are going to hold all configuration settings. :param config_database_path: Path to database file. :return: Returns a cursor to the sqlite3 database connection that can be used for further database operations. """ if not os.path.isfile(config_database_path): if not Helper.filepath_exists(config_database_path): raise AvivoreConfigException( 'Invalid path to configuration database file specified!') Helper.output('[W] Configuration database file not found!') Helper.output('[W] Creating a new configuration database.') Helper.output( '[W] Now please store your configuration in database file \'%s\', then try again!' % config_database_path) dbcon = lite.connect(config_database_path) dbcur = dbcon.cursor() dbcur.execute( 'CREATE TABLE IF NOT EXISTS Config (dbpath text, consumer_key text, consumer_secret text, \ credentials_file text, stream_tracking_keyword text, csv_search_term, interval int)' ) dbcur.execute( 'CREATE TABLE IF NOT EXISTS TypeDefs (Id int, Regex text, Comment text)' ) return dbcur
def extract_data_from_tweet(self, avivore, tweet): """ Scans data for extractable data sets (previously defined in the 'TypeDefs' section of the AvivoreXT configuration) and if found stores them in the result database. :param avivore: Previously initialized Avivore instance. :param tweet: The tweet to analyzed that was returned from Twitter as a JSON object. :return: None """ z = tweet['id'], tweet['created_at'], tweet['user']['screen_name'], tweet['text'], tweet['user']['id_str'] result = avivore.twitter_read_tweet(z[3]) if result[0] >= 0: # If something is found, then we'll process the tweet self.stored = self.stored, int(z[0]) # result value, time, result itself, tweet ID, tweet itself, userId string = result[0], z[2], result[1], z[0], z[3], z[4] message = avivore.process_tweet(string) if message is not None: Helper.output("[Q] " + message)
def twitter_read_tweet(self, string): i = -1 for x in self.avivore_config.twitter_search_types: i += 1 findtype = re.compile(x) result_raw = findtype.findall(string) if not result_raw: continue else: if Helper.is_sequence(result_raw[0]): result = [x for x in result_raw[0] if x] else: result = result_raw[0] if Helper.is_sequence(result): result = result[0] return i, result # nothing found return -1, 0
def init_config_database(self, config_database_path): """ Prepares a sqlite3 database for use as a configuration database. Creates a database with the specified filename and creates empty 'Config' and 'TypeDefs' database tables that are going to hold all configuration settings. :param config_database_path: Path to database file. :return: Returns a cursor to the sqlite3 database connection that can be used for further database operations. """ if not os.path.isfile(config_database_path): if not Helper.filepath_exists(config_database_path): raise AvivoreConfigException('Invalid path to configuration database file specified!') Helper.output('[W] Configuration database file not found!') Helper.output('[W] Creating a new configuration database.') Helper.output('[W] Now please store your configuration in database file \'%s\', then try again!' % config_database_path) dbcon = lite.connect(config_database_path) dbcur = dbcon.cursor() dbcur.execute( 'CREATE TABLE IF NOT EXISTS Config (dbpath text, consumer_key text, consumer_secret text, \ credentials_file text, stream_tracking_keyword text, csv_search_term, interval int)') dbcur.execute('CREATE TABLE IF NOT EXISTS TypeDefs (Id int, Regex text, Comment text)') return dbcur
def test_output(self): self.assertTrue(None == Helper.output("Test")) self.assertTrue(None == Helper.output("\u2031\u203c\u2049")) self.assertTrue(None == Helper.output(self)) self.assertTrue(None == Helper.output(None)) self.assertTrue(None == Helper.output(True)) self.assertTrue(None == Helper.output(Helper.output)) self.assertTrue(None == Helper.output(3)) self.assertTrue(None == Helper.output(3.14)) self.assertTrue(None == Helper.output(['a1', 'b2', 'c3'])) self.assertTrue(None == Helper.output(('it1', 'it2'))) self.assertTrue(None == Helper.output(array('l', [1, 2, 3, 4, 5]))) self.assertTrue(None == Helper.output(array('B', [0x41, 0x42, 0x43, 0x44, 0x45]))) self.assertTrue(None == Helper.output(bytearray.fromhex("deadbeef"))) self.assertTrue(None == Helper.output(dict(one=1, two=2, three=3)))
def test_is_sequence(self): # testing sequential types self.assertTrue(Helper.is_sequence(['value1', 'value2', 'value3'])) self.assertTrue(Helper.is_sequence(('item1', 'item2'))) self.assertTrue(Helper.is_sequence(array('l', [1, 2, 3, 4, 5]))) self.assertTrue( Helper.is_sequence(array('d', [1.0, 2.0, 3.0, 4.0, 5.0]))) self.assertTrue(Helper.is_sequence(bytearray.fromhex("deadbeef"))) self.assertTrue(Helper.is_sequence(dict(one=1, two=2, three=3))) # testing non-sequential types self.assertFalse(Helper.is_sequence("Hello World")) self.assertFalse(Helper.is_sequence("Hello World\u203c")) self.assertFalse(Helper.is_sequence(3)) self.assertFalse(Helper.is_sequence(3.14)) self.assertFalse(Helper.is_sequence(self)) self.assertFalse(Helper.is_sequence(None)) self.assertFalse(Helper.is_sequence(True)) self.assertFalse(Helper.is_sequence(Helper.is_sequence))
def test_is_sequence(self): # testing sequential types self.assertTrue(Helper.is_sequence(['value1', 'value2', 'value3'])) self.assertTrue(Helper.is_sequence(('item1', 'item2'))) self.assertTrue(Helper.is_sequence(array('l', [1, 2, 3, 4, 5]))) self.assertTrue(Helper.is_sequence(array('d', [1.0, 2.0, 3.0, 4.0, 5.0]))) self.assertTrue(Helper.is_sequence(bytearray.fromhex("deadbeef"))) self.assertTrue(Helper.is_sequence(dict(one=1, two=2, three=3))) # testing non-sequential types self.assertFalse(Helper.is_sequence("Hello World")) self.assertFalse(Helper.is_sequence("Hello World\u203c")) self.assertFalse(Helper.is_sequence(3)) self.assertFalse(Helper.is_sequence(3.14)) self.assertFalse(Helper.is_sequence(self)) self.assertFalse(Helper.is_sequence(None)) self.assertFalse(Helper.is_sequence(True)) self.assertFalse(Helper.is_sequence(Helper.is_sequence))
def test_output(self): self.assertTrue(None == Helper.output("Test")) self.assertTrue(None == Helper.output("\u2031\u203c\u2049")) self.assertTrue(None == Helper.output(self)) self.assertTrue(None == Helper.output(None)) self.assertTrue(None == Helper.output(True)) self.assertTrue(None == Helper.output(Helper.output)) self.assertTrue(None == Helper.output(3)) self.assertTrue(None == Helper.output(3.14)) self.assertTrue(None == Helper.output(['a1', 'b2', 'c3'])) self.assertTrue(None == Helper.output(('it1', 'it2'))) self.assertTrue(None == Helper.output(array('l', [1, 2, 3, 4, 5]))) self.assertTrue( None == Helper.output(array('B', [0x41, 0x42, 0x43, 0x44, 0x45]))) self.assertTrue(None == Helper.output(bytearray.fromhex("deadbeef"))) self.assertTrue(None == Helper.output(dict(one=1, two=2, three=3)))
def twitter_stream_main(self): Helper.output("Beginning stream processing [S].") stored = [] twitter_stream_inst = self.twitter_stream_auth() try: iterator = twitter_stream_inst.statuses.filter(track=self.avivore_config.twitter_track_keywords) for y in iterator: # tweet may be a delete or data msg if y is None: pass # tweet may indicate a timeout elif y is Timeout: Helper.output("[S] Stream timeout...") elif y is HeartbeatTimeout: Helper.output("[S] Heartbeat timeout...") # tweet may indicate a hung up stream conn. elif y is Hangup: Helper.output("[S] Stream hangup detected!") # ok, tweet seems to contain text, process it elif 'id' in y and 'created_at' in y and 'user' in y and 'text' in y: # print y z = y['id'], y['created_at'], y['user']['screen_name'], y['text'], y['user']['id_str'] result = self.twitter_read_tweet(z[3]) if result[0] < 0: pass else: # If something is found, then we'll process the tweet stored = stored, int(z[0]) # result value, time, result itself, tweet ID, tweet itself, userId string = result[0], z[2], result[1], z[0], z[3], z[4] message = self.process_tweet(string) if 0 != message: Helper.output("[S] " + message) except (TwitterHTTPError, URLError): Helper.output("[!] Can't connect to twitter stream! Check your network connection!") except Exception as e: Helper.output("[!] Unknown stream processing error: %s\n" % (str(e))) finally: Helper.output("[S] Stream processing stopped.")