Esempio n. 1
0
    def get_friends(user, page=1):
        """
        Return List of friends of a specific user
        :param user: User Screen Name
        :param page: Page Number
        :return:
        """
        total_result = Connection.Instance().db.twitter.find({
            "user":
            user,
            "source":
            "friends"
        }).count()
        try:
            page = int(page) if int(page) > 0 else 1
        except:
            page = 1
        n_result = 10
        result = list(Connection.Instance().db.twitter.find({
            "source": "friends",
            "user": user
        }).skip((page - 1) * n_result).limit(n_result))

        page_size = len(result)

        return jsonify(
            total_result=total_result,
            status=200,
            page_size=page_size,
            page=page,
            result=[data['data'] for data in result],
            next="/api/v1/twitter/friends/%s/%s" %
            (user, str(page + 1)) if page_size == n_result else None,
            before="/api/v1/twitter/friends/%s/%s" %
            (user, str(page - 1)) if page > 1 else None), 200
 def fetcher(self):
     """
     Tweets loader
     :return:
     """
     for page in self.fetcherInstance.get_tweets():
         for tweet in page:
             try:
                 if not Connection.Instance().db.twitter.find_one({'user': tweet.user.screen_name,
                                                                   'source': 'collector',
                                                                   'data.id': tweet.id}):
                     Connection.Instance().db.twitter.insert_one({
                         'source': 'collector',
                         'data': {
                             'created_at': tweet.created_at,
                             'favorite_count': tweet.favorite_count,
                             'geo': tweet.geo,
                             'id': tweet.id,
                             'source': tweet.source,
                             'in_reply_to_screen_name': tweet.in_reply_to_screen_name,
                             'in_reply_to_status_id': tweet.in_reply_to_status_id,
                             'in_reply_to_user_id': tweet.in_reply_to_user_id,
                             'retweet_count': tweet.retweet_count,
                             'retweeted': tweet.retweeted,
                             'text': tweet.text,
                             'entities': tweet.entities
                         },
                         'user': tweet.user.screen_name,
                         'created': what_time_is_it()
                     })
             except Exception as genericException:
                 logging.error("MongoDB Insert Error in collector: %s" % genericException)
     import multiprocessing
     ProcessManager.terminate_process(multiprocessing.current_process().pid, True)
 def on_status(self, tweet):
     """
     If new tweet is received, this function will be called.
     For each tweet will create a new Mongo Insert
     :param tweet:
     :return:
     """
     try:
         Connection.Instance().db.twitter.insert_one({
             'source':
             'collector',
             'data': {
                 'created_at': tweet.created_at,
                 'favorite_count': tweet.favorite_count,
                 'geo': tweet.geo,
                 'id': tweet.id,
                 'source': tweet.source,
                 'in_reply_to_screen_name': tweet.in_reply_to_screen_name,
                 'in_reply_to_status_id': tweet.in_reply_to_status_id,
                 'in_reply_to_user_id': tweet.in_reply_to_user_id,
                 'retweet_count': tweet.retweet_count,
                 'retweeted': tweet.retweeted,
                 'text': tweet.text,
                 'entities': tweet.entities
             },
             'user':
             self.user,
             'created':
             what_time_is_it()
         })
     except Exception as e:
         logging.error("MongoDB Insert Error in listener: " + e)
Esempio n. 4
0
 def get_single_process(pid):
     """
     Get a Single Process given the PID
     :param pid:  Process ID
     :return:
     """
     return Connection.Instance().db.manager.find_one({'pid': pid})
Esempio n. 5
0
 def insert_process(process):
     """
     Insert new process
     :param process: Process object
     :return:
     """
     return Connection.Instance().db.manager.insert_one(process)
Esempio n. 6
0
 def get_all_processes_with_condition(condition):
     """
     Return the list of the processes
     :return:
     """
     return list(Connection.Instance().db.manager.find(condition).sort([
         ('last_update', -1)
     ]))
Esempio n. 7
0
 def update_process(pid, newobj):
     """
     Update a single process
     :param pid: PID
     :param newobj: New Object to Set
     :return:
     """
     return Connection.Instance().db.manager.update({'pid': pid},
                                                    {'$set': newobj})
Esempio n. 8
0
 def get_keywords():
     """
     Return the list of all available keywords
     :return: Json
     """
     return jsonify(status=200,
                    keywords=Connection.Instance().db.twitter.find({
                        "source":
                        "listener"
                    }).distinct("keywords")), 200
 def fetcher(self):
     """
     Followers loader
     :return:
     """
     for page in self.fetcherInstance.get_followers():
         for follower in page:
             try:
                 if not Connection.Instance().db.twitter.find_one(
                     {
                         'source': 'follower',
                         'user': self.user,
                         'data.userid': follower.id
                     }):
                     Connection.Instance().db.twitter.insert_one({
                         'source':
                         'follower',
                         'data': {
                             'userid': follower.id,
                             'description': follower.description,
                             'favourites_count': follower.favourites_count,
                             'followers_count': follower.followers_count,
                             'friends_count': follower.friends_count,
                             'lang': follower.lang,
                             'location': follower.location,
                             'name': follower.name,
                             'screen_name': follower.screen_name,
                             'geo_enabled': follower.geo_enabled,
                             'url': follower.url,
                             'time_zone': follower.time_zone,
                             'statuses_count': follower.statuses_count
                         },
                         'user':
                         self.user,
                         'created':
                         what_time_is_it()
                     })
             except Exception as e:
                 logging.error("MongoDB Insert Error in get followers: " +
                               e)
     import multiprocessing
     ProcessManager.terminate_process(multiprocessing.current_process().pid,
                                      True)
Esempio n. 10
0
 def get_users():
     """
     Return list of users fetched
     :return: Json
     """
     return jsonify(status=200,
                    users=Connection.Instance().db.twitter.find({
                        "user": {
                            "$ne": None
                        }
                    }).distinct("user")), 200
Esempio n. 11
0
    def search(keyword, page=1):
        """
        Get all tweets from a given keyword with pagination support.
        :param keyword: Keyword to search
        :param page: Page Number
        :return:
        """
        try:
            page = int(page) if int(page) > 0 else 1
        except:
            page = 1
        n_result = 10
        total_result = Connection.Instance().db.twitter.find({
            "keywords":
            keyword,
            "source":
            "listener"
        }).count()
        result = list(Connection.Instance().db.twitter.find(
            {
                "keywords": keyword,
                "source": "listener"
            }, {
                "_id": False,
                "created": False,
                "keywords": False,
                "source": False
            }).skip((page - 1) * n_result).limit(n_result))

        page_size = len(result)

        return jsonify(total_result=total_result,
                       status=200,
                       page_size=page_size,
                       page=page,
                       result=[data['data'] for data in result],
                       next="/api/v1/twitter/search/python/%s" %
                       str(page + 1) if page_size == n_result else None,
                       before="/api/v1/twitter/search/python/%s" %
                       str(page - 1) if page > 1 else None), 200
Esempio n. 12
0
 def on_status(self, tweet):
     """
     If new tweet is received, this function will be called.
     For each tweet will create a new Mongo Insert
     :param tweet:
     :return:
     """
     try:
         Connection.Instance().db.twitter.insert_one({
             'source':
             'listener',
             'data':
             tweet._json,
             'keywords':
             self.keywords,
             'created':
             what_time_is_it()
         })
     except Exception as e:
         logging.error("MongoDB Insert Error in listener: " + e)
Esempio n. 13
0
 def refresh_status(self):
     """
     Refresh process status.
     If a process is dead or in zombie status a flag 'is_alive' will be setted to False
     :return:
     """
     what_time_is_now = what_time_is_it()
     for process in Connection.Instance().db.manager.find(
         {"is_alive": True}):
         if (not psutil.pid_exists(process['pid'])) or \
                 (psutil.pid_exists(process['pid']) and psutil.Process(process['pid']).status() ==
                     psutil.STATUS_ZOMBIE):
             self.update_process(process['pid'], {
                 'is_alive': False,
                 'last_update': what_time_is_now
             })
         elif psutil.pid_exists(process['pid']):
             self.update_process(process['pid'],
                                 {'last_update': what_time_is_now})
     logging.info('Refresh Done!')