def __init__(self, queue: str = 'redis'):
"""
Initialise logger, redis connection and cursor, and the update query.
Note we close the connection in __del__.
"""
self.logger = logging.getLogger('rmqtest')
self.queue_name = queue
self.redis = get_redis_connection()
self.mysql_conn = get_db_connection()
self.cursor = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
self.cursor.execute('SELECT * FROM crowdsourcer;')
crowdsourcer = self.cursor.fetchall()
for d in crowdsourcer:
self.redis.hset('crowdsourcer', d['id'], d['name'])
self.redis.hset('output', str(d['id']) + '_minor', 0)
self.redis.hset('output', str(d['id']) + '_medium', 0)
self.redis.hset('output', str(d['id']) + '_high', 0)
self.redis.hset('output', str(d['id']) + '_critical', 0)
self.cursor.execute('SELECT * FROM severity;')
severity = self.cursor.fetchall()
for d in severity:
self.redis.hset('severity_name', d['id'], d['severity'])
self.redis.hset('severity', d['id'], d['karma'])
self.cursor.execute('SELECT * FROM vulnerability;')
vulnerability = self.cursor.fetchall()
for d in vulnerability:
self.redis.hset('vulnerability_cs', d['id'], d['cs_id'])
self.redis.hset('vulnerability_sev', d['id'], d['sev_id'])
self.cursor.close()
self.mysql_conn.close()
def get_latest_tweet(q):
with connections.get_db_connection() as client:
collection = client['tweets'][q]
try:
return collection.find({}).sort('id', -1)[0].get('id_str')
except IndexError:
return None
def __init__(self, queue: str = 'mariadb'):
"""
Initialise logger, MariaDB connection and cursor, and the update query.
Note we close the connection in __del__.
"""
self.logger = logging.getLogger('rmqtest')
self.queue_name = queue
self.mysql_conn = get_db_connection()
self.cursor = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
with open('update_karma.sql', 'r') as query_file:
self.query = query_file.read()
def search(q):
with connections.get_db_connection() as client:
tweets = get_latests_tweets(q)
if not tweets:
print "There are NO new tweets"
else:
print "{0} new tweets fetched!".format(len(tweets))
for tweet in tweets:
add_classification_information(tweet)
db = client["tweets"]
collection = db[q]
result = collection.insert_many(tweets)
print "{0} tweets classified".format(len(result.inserted_ids))
return
def reclassify(classify=True, add_time=True):
start_time = time.time()
for collection in COLLECTIONS:
with connections.get_db_connection() as client:
mongo_collection = client["tweets"][collection]
tweets = mongo_collection.find({})
for tweet in tweets:
tweet_classified = tweet
if classify:
tweet_classified = add_classification_information(tweet)
if add_time:
tweet_classified = add_date_fields(tweet)
mongo_collection.replace_one({"_id": tweet["_id"]}, tweet_classified)
elapsed_time = time.time() - start_time
print "Time elapsed: {} seconds".format(elapsed_time)
return
def get_data(self, hashtag):
with connections.get_db_connection() as client:
tweets = client['tweets'][hashtag].find({'retweeted': False})
return tweets