Exemplo n.º 1
0
    def __init__(self, queue: str = 'redis'):
        """
        Initialise logger, redis connection and cursor, and the update query.

        Note we close the connection in __del__.
        """
        self.logger = logging.getLogger('rmqtest')
        self.queue_name = queue
        self.redis = get_redis_connection()
        self.mysql_conn = get_db_connection()
        self.cursor = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)

        self.cursor.execute('SELECT * FROM crowdsourcer;')
        crowdsourcer = self.cursor.fetchall()
        for d in crowdsourcer:
            self.redis.hset('crowdsourcer', d['id'], d['name'])
            self.redis.hset('output', str(d['id']) + '_minor', 0)
            self.redis.hset('output', str(d['id']) + '_medium', 0)
            self.redis.hset('output', str(d['id']) + '_high', 0)
            self.redis.hset('output', str(d['id']) + '_critical', 0)

        self.cursor.execute('SELECT * FROM severity;')
        severity = self.cursor.fetchall()
        for d in severity:
            self.redis.hset('severity_name', d['id'], d['severity'])
            self.redis.hset('severity', d['id'], d['karma'])

        self.cursor.execute('SELECT * FROM vulnerability;')
        vulnerability = self.cursor.fetchall()
        for d in vulnerability:
            self.redis.hset('vulnerability_cs', d['id'], d['cs_id'])
            self.redis.hset('vulnerability_sev', d['id'], d['sev_id'])

        self.cursor.close()
        self.mysql_conn.close()
Exemplo n.º 2
0
def get_latest_tweet(q):
    with connections.get_db_connection() as client:
        collection = client['tweets'][q]
        try:
            return collection.find({}).sort('id', -1)[0].get('id_str')
        except IndexError:
            return None
    def __init__(self, queue: str = 'mariadb'):
        """
        Initialise logger, MariaDB connection and cursor, and the update query.

        Note we close the connection in __del__.
        """
        self.logger = logging.getLogger('rmqtest')
        self.queue_name = queue
        self.mysql_conn = get_db_connection()
        self.cursor = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
        with open('update_karma.sql', 'r') as query_file:
            self.query = query_file.read()
Exemplo n.º 4
0
def search(q):
    with connections.get_db_connection() as client:
        tweets = get_latests_tweets(q)

        if not tweets:
            print "There are NO new tweets"
        else:
            print "{0} new tweets fetched!".format(len(tweets))

            for tweet in tweets:
                add_classification_information(tweet)

            db = client["tweets"]
            collection = db[q]

            result = collection.insert_many(tweets)
            print "{0} tweets classified".format(len(result.inserted_ids))

        return
Exemplo n.º 5
0
def reclassify(classify=True, add_time=True):
    start_time = time.time()
    for collection in COLLECTIONS:

        with connections.get_db_connection() as client:
            mongo_collection = client["tweets"][collection]
            tweets = mongo_collection.find({})

            for tweet in tweets:
                tweet_classified = tweet

                if classify:
                    tweet_classified = add_classification_information(tweet)
                if add_time:
                    tweet_classified = add_date_fields(tweet)

                mongo_collection.replace_one({"_id": tweet["_id"]}, tweet_classified)
    elapsed_time = time.time() - start_time

    print "Time elapsed: {} seconds".format(elapsed_time)
    return
Exemplo n.º 6
0
 def get_data(self, hashtag):
     with connections.get_db_connection() as client:
         tweets = client['tweets'][hashtag].find({'retweeted': False})
         return tweets