Exemple #1
0
    def test_history(self):
        """
        test history parsing
        :return:
        """
        self.assertRaisesRegexp(TypeError, "None", bingHistory.parse, None)

        output = bingHistory.parse("")
        self.assertIsNotNone(output, "missing output " + str(output))

        page = '<span class="query_t">'
        page += '<div id="results_area"></div><div id="sidebar"></div>'
        output = bingHistory.parse(page)
        self.assertIsNotNone(output, "missing output " + str(output))

        page = '<span class="sh_item_qu_query">'
        page += '<ul class="sh_dayul"></ul>'
        page += ' value == 0'
        page += '</span>'
        output = bingHistory.parse(page)
        self.assertIsNotNone(output, "missing output " + str(output))

        page = '<ul class="sh_dayul"> </ul>'
        output = bingHistory.parse(page)
        self.assertIsNotNone(output, "missing output " + str(output))

        output = bingHistory.getBingHistoryTodayURL()
        self.assertRegexpMatches(output, "https", "missing url " + str(output))
Exemple #2
0
    def __processSearch(self, reward, verbose, searchType):
        """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""

        BING_QUERY_URL = 'http://www.bing.com/search?q='
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
        IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
        IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
        IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')

        res = self.RewardResult(reward)
        if reward.isAchieved():
            res.message = "This reward has been already achieved"
            return res

        indCol = bfp.Reward.Type.Col.INDEX

# get a set of queries from today's Bing! history
        url = bingHistory.getBingHistoryTodayURL()
        request = urllib2.Request(url = url, headers = self.httpHeaders)
        with self.opener.open(request) as response:
            page = helpers.getResponseBody(response)
        history = bingHistory.parse(page)

# find out how many searches need to be performed
        #matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
        #if matches is None:
        #    print "No RegEx matches found for this search and earn"
        #    res.isError = True
        #    res.message = "No RegEx matches found for this search and earn"
        #    return res
        #maxRewardsCount = int(matches.group(1))
        #rewardsCount    = int(matches.group(2))
        #rewardCost      = 1 # Looks like it's now always X points per one search
        #searchesCount = maxRewardsCount * rewardCost / rewardsCount

# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
        #searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount

        headers = self.httpHeaders
        if reward.tp == bfp.Reward.Type.SEARCH_PC or reward.tp == bfp.Reward.Type.SEARCH_AND_EARN or searchType == 0:
            headers["User-Agent"] = self.userAgents.pc

	searchesCount = 30
        searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
        print
        print "Running PC searches"
        print
        else: reward.tp == bfp.Reward.Type.SEARCH_MOBILE or searchType == 1:
            headers["User-Agent"] = self.userAgents.mobile
            searchesCount = 20
            searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
            print
            print "Running mobile searches"
            print
    def __processSearch(self, reward):
        """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""

        BING_QUERY_URL = 'http://www.bing.com/search?q='
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'

        res = self.RewardResult(reward)
        if reward.isAchieved():
            res.message = "This reward has been already achieved"
            return res

        indCol = bfp.Reward.Type.Col.INDEX

        # get a set of queries from today's Bing! history
        url = bingHistory.getBingHistoryTodayURL()
        request = urllib2.Request(url=url, headers=self.httpHeaders)
        with self.opener.open(request) as response:
            page = helpers.getResponseBody(response)
        history = bingHistory.parse(page)

        # find out how many searches need to be performed
        matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
            reward.description)
        rewardsCount = int(matches.group(1))
        rewardCost = int(matches.group(2))
        maxRewardsCount = int(matches.group(4))
        searchesCount = maxRewardsCount * rewardCost / rewardsCount

        # adjust to the current progress
        searchesCount -= reward.progressCurrent * rewardCost

        headers = self.httpHeaders

        if reward.tp == bfp.Reward.Type.SEARCH_PC:
            headers["User-Agent"] = self.userAgents.pc
            print
            print "Running PC searches"
            print
        elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
            headers["User-Agent"] = self.userAgents.mobile
            print
            print "Running mobile searches"
            print
        else:
            res.isError = True
            res.message = "Don't know how to process this search"
            return res

        # Import the query generator
        try:
            qg = importlib.import_module(self.queryGenerator, package=None)
            queryGenerator = qg.queryGenerator(self)
        except ImportError:
            raise TypeError("{0} is not a module".format(self.queryGenerator))
        # generate a set of queries to run
        queries = queryGenerator.generateQueries(searchesCount, history)

        if len(queries) < searchesCount:
            print "Warning: not enough queries to run were generated!"
            print "Requested:", searchesCount
            print "Generated:", len(queries)

        successfullQueries = 0
        i = 1
        totalQueries = len(queries)

        for query in queries:
            if i > 1:
                # sleep some time between queries (don't worry Bing! ;) )
                t = self.betweenQueriesInterval + random.uniform(
                    0, self.betweenQueriesSalt)
                time.sleep(t)

            url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))

            print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
                                                 totalQueries, query)

            request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
            with self.opener.open(request) as response:
                page = helpers.getResponseBody(response)

# check for the successfull marker
            found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
                 or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1

            if not found:
                filename = helpers.dumpErrorPage(page)
                print "Warning! Query:"
                print "\t" + query
                print "returned no results, check " + filename + " file for more information"

            else:
                successfullQueries += 1

            i += 1

        if successfullQueries < searchesCount:
            res.message = str(successfullQueries) + " out of " + str(
                searchesCount) + " requests were successfully processed"
        else:
            res.message = "All " + str(
                successfullQueries) + " requests were successfully processed"

        # reset header to pc so pc pages return in getting life time points
        headers["User-Agent"] = self.userAgents.pc

        return res
    def __processSearch(self, reward, verbose):
        """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""

        BING_QUERY_URL = 'http://www.bing.com/search?q='
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content"'
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content"'
        IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
        IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"')
        IG_SEARCH_RESULTS_PATTERN = re.compile(r'<ol\s.*?id="b_results"(.+?)</ol>')
        IG_SEARCHS_PATTERN = re.compile(r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"')

        res = self.RewardResult(reward)
        if reward.isAchieved():
            res.message = "This reward has been already achieved"
            return res

        indCol = bfp.Reward.Type.Col.INDEX

# get a set of queries from today's Bing! history
        url = bingHistory.getBingHistoryTodayURL()
        request = urllib2.Request(url = url, headers = self.httpHeaders)
        with self.opener.open(request) as response:
            page = helpers.getResponseBody(response)
        history = bingHistory.parse(page)

# find out how many searches need to be performed
        matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
        rewardsCount    = int(matches.group(1))
        rewardCost      = int(matches.group(2))
        maxRewardsCount = int(matches.group(4))
        searchesCount = maxRewardsCount * rewardCost / rewardsCount

# adjust to the current progress
        searchesCount -= reward.progressCurrent * rewardCost

        headers = self.httpHeaders

        if reward.tp == bfp.Reward.Type.SEARCH_PC:
            headers["User-Agent"] = self.userAgents.pc
            searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
            print
            print "Running PC searches"
            print
        elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
            headers["User-Agent"] = self.userAgents.mobile
            searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
            print
            print "Running mobile searches"
            print
        else:
            res.isError = True
            res.message = "Don't know how to process this search"
            return res

        if verbose:
            print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
            print

        # Import the query generator
        try:
            qg = importlib.import_module(self.queryGenerator, package=None)
            queryGenerator = qg.queryGenerator(self)
        except ImportError:
            raise TypeError("{0} is not a module".format(self.queryGenerator))
        # generate a set of queries to run
        queries = queryGenerator.generateQueries(searchesCount, history)

        if len(queries) < searchesCount:
            print "Warning: not enough queries to run were generated!"
            print "Requested:", searchesCount
            print "Generated:", len(queries)

        successfullQueries = 0
        i = 1
        totalQueries = len(queries)

        for query in queries:
            if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
                t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
                time.sleep(t)

            url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))

            print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)

            request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
            with self.opener.open(request) as response:
                page = helpers.getResponseBody(response)

# check for the successfull marker
            found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
                 or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1

            if not found:
                filename = helpers.dumpErrorPage(page)
                print "Warning! Query:"
                print "\t" + query
                print "returned no results, check " + filename + " file for more information"

            else:
                successfullQueries += 1

                # randomly open a link
                if self.openLinkChance > random.random():
                    # get IG number
                    ig_number = IG_NUMBER_PATTERN.search(page)
                    if ig_number != None:
                        ig_number = ig_number.group(1)
                        # get search results
                        ig_results = IG_SEARCH_RESULTS_PATTERN.search(page)
                        if ig_results != None:
                            ig_results = ig_results.group(1)
                            # seperate search results
                            ig_searches = IG_SEARCHS_PATTERN.findall(ig_results)
                            # make sure we have at least 1 search
                            if len(ig_searches) > 0:
                                # get a random link to open
                                ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
                                ig_link_num = random.randint(0, ig_max_rand)
                                # number of the link we will use
                                ig_link = "{0}?IG={1}&{2}".format(
                                    IG_PING_LINK,
                                    urllib.quote_plus(ig_number),
                                    urllib.quote_plus(ig_searches[ig_link_num][1])
                                )

                                # sleep a reasonable amount of time before clicking the link
                                # use defaults to save space in config
                                t = random.uniform(0.75, 3.0)
                                time.sleep(t)

                                # open the random link
                                request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
                                request.headers["Referer"] = response.url
                                self.opener.open(request)

                                if verbose:
                                    print("Followed Link {}".format(ig_link_num + 1))
                            else:
                                filename = helpers.dumpErrorPage(page)
                                print "Warning! No searches were found on search results page"
                                print "Check {0} file for more information".format(filename)
                    else:
                        filename = helpers.dumpErrorPage(page)
                        print "Warning! Could not find search result IG number"
                        print "Check {0} file for more information".format(filename)

            i += 1

        if successfullQueries < searchesCount:
            res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
        else:
            res.message = "All " + str(successfullQueries) + " requests were successfully processed"

        # reset header to pc so pc pages return in getting life time points
        headers["User-Agent"] = self.userAgents.pc

        return res
Exemple #5
0
    def __processSearch(self, reward, verbose):
        """Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""

        BING_QUERY_URL = 'http://www.bing.com/search?q='
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
        IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
        IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
        IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')

        res = self.RewardResult(reward)
        if reward.isAchieved():
            res.message = "This reward has been already achieved"
            return res

        indCol = bdp.Reward.Type.Col.INDEX

# get a set of queries from today's Bing history
        url = bingHistory.getBingHistoryTodayURL()
        request = urllib2.Request(url = url, headers = self.httpHeaders)
        with self.opener.open(request) as response:
            page = helpers.getResponseBody(response)
        history = bingHistory.parse(page)

# find out how many searches need to be performed
        matchesMobile = False
        matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
        #Mobile description changed, so check that one too
        if matches is None:
            matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE_MOBILE.search(reward.description)
            matchesMobile = True
        if matches is None: 
            print "No RegEx matches found for this search and earn"
            res.isError = True
            res.message = "No RegEx matches found for this search and earn"
            return res
        maxRewardsCount = int(matches.group(1))
        rewardsCount    = int(matches.group(2))
        rewardCost      = 1 # Looks like it's now always X points per one search
        searchesCount = maxRewardsCount * rewardCost / rewardsCount

# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
        searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount

        if matchesMobile == True:
            #new mobile search description gives total search count + points per search for edge/non-edge
            edgeValue = int(matches.group(1))
            nonEdgeValue = int(matches.group(2))
            searchesCount = int(matches.group(3))
            #apparently ios uses EdgiOS, so we only check the first 3 letters not the full word 'edge'
            if self.userAgents.mobile.lower().find("edg") != -1:
                #we are searching on edge so points go to 200
                searchesCount -= reward.progressCurrent / edgeValue
            else:
                #non-edge search so 100 is the max
                searchesCount -= reward.progressCurrent / nonEdgeValue

        headers = self.httpHeaders

        if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
            headers["User-Agent"] = self.userAgents.pc
            searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
            print
            print "Running PC searches"
            print
        elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
            headers["User-Agent"] = self.userAgents.mobile
            searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
            print
            print "Running mobile searches"
            print
        else:
            res.isError = True
            res.message = "Don't know how to process this search"
            return res

        if verbose:
            print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
            print

        # Import the query generator
        try:
            qg = importlib.import_module(self.queryGenerator, package=None)
            queryGenerator = qg.queryGenerator(self)
        except ImportError:
            raise TypeError("{0} is not a module".format(self.queryGenerator))

        # generate a set of queries to run
        queries = queryGenerator.generateQueries(searchesCount, history)

        if len(queries) < searchesCount:
            print "Warning: not enough queries to run were generated!"
            print "Requested:", searchesCount
            print "Generated:", len(queries)

        successfullQueries = 0
        i = 1
        totalQueries = len(queries)

        for query in queries:
            if i > 1:
# sleep some time between queries (don't worry Bing ;) )
                t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
                time.sleep(t)

            url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))

            print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)

            request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
            with self.opener.open(request) as response:
                page = helpers.getResponseBody(response)

# check for the successfull marker
            found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
                 or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1

            if not found:
                filename = helpers.dumpErrorPage(page)
                print "Warning! Query:"
                print "\t" + query
                print "returned no results, check " + filename + " file for more information"

            else:
                successfullQueries += 1

                # randomly open a link
                if self.openLinkChance > random.random():
                    # get IG number
                    ig_number = IG_NUMBER_PATTERN.search(page)
                    if ig_number is not None:
                        ig_number = ig_number.group(1)
                        ig_searches = IG_SEARCHES_PATTERN.findall(page)
                        # make sure we have at least 1 search
                        if len(ig_searches) > 0:
                            # get a random link to open
                            ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
                            # number of the link we will use
                            ig_link_num = random.randint(0, ig_max_rand)
                            ig_link = "{0}?IG={1}&{2}".format(
                                IG_PING_LINK,
                                urllib.quote_plus(ig_number),
                                urllib.quote_plus(ig_searches[ig_link_num][1])
                            )

                            # sleep a reasonable amount of time before clicking the link
                            # use defaults to save space in config
                            t = random.uniform(0.75, 3.0)
                            time.sleep(t)

                            # open the random link
                            request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
                            request.headers["Referer"] = response.url
                            self.opener.open(request)

                            if verbose:
                                print("Followed Link {}".format(ig_link_num + 1))
                        else:
                            filename = helpers.dumpErrorPage(page)
                            print "Warning! No searches were found on search results page"
                            print "Check {0} file for more information".format(filename)
                    else:
                        filename = helpers.dumpErrorPage(page)
                        print "Warning! Could not find search result IG number"
                        print "Check {0} file for more information".format(filename)

            i += 1

        if successfullQueries < searchesCount:
            res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
        else:
            res.message = "All " + str(successfullQueries) + " requests were successfully processed"

        # reset header to pc so pc pages return in getting life time points
        headers["User-Agent"] = self.userAgents.pc

        return res
Exemple #6
0
    def __processSearch(self, reward, verbose):
        """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""

        BING_QUERY_URL = 'http://www.bing.com/search?q='
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
        IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
        IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"')
        IG_SEARCH_RESULTS_PATTERN = re.compile(
            r'<ol\s.*?id="b_results"(.+?)</ol>')
        IG_SEARCHS_PATTERN = re.compile(
            r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"')

        res = self.RewardResult(reward)
        if reward.isAchieved():
            res.message = "This reward has been already achieved"
            return res

        indCol = bfp.Reward.Type.Col.INDEX

        # get a set of queries from today's Bing! history
        url = bingHistory.getBingHistoryTodayURL()
        request = urllib2.Request(url=url, headers=self.httpHeaders)
        with self.opener.open(request) as response:
            page = helpers.getResponseBody(response)
        history = bingHistory.parse(page)

        # find out how many searches need to be performed
        matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
            reward.description)
        rewardsCount = int(matches.group(1))
        rewardCost = int(matches.group(2))
        maxRewardsCount = int(matches.group(4))
        searchesCount = maxRewardsCount * rewardCost / rewardsCount

        # adjust to the current progress
        searchesCount -= reward.progressCurrent * rewardCost

        headers = self.httpHeaders

        if reward.tp == bfp.Reward.Type.SEARCH_PC:
            headers["User-Agent"] = self.userAgents.pc
            searchesCount += self.addSearchesDesktop + random.randint(
                0, self.addSearchesDesktopSalt)
            print
            print "Running PC searches"
            print
        elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
            headers["User-Agent"] = self.userAgents.mobile
            searchesCount += self.addSearchesMobile + random.randint(
                0, self.addSearchesMobileSalt)
            print
            print "Running mobile searches"
            print
        else:
            res.isError = True
            res.message = "Don't know how to process this search"
            return res

        # Import the query generator
        try:
            qg = importlib.import_module(self.queryGenerator, package=None)
            queryGenerator = qg.queryGenerator(self)
        except ImportError:
            raise TypeError("{0} is not a module".format(self.queryGenerator))
        # generate a set of queries to run
        queries = queryGenerator.generateQueries(searchesCount, history)

        if len(queries) < searchesCount:
            print "Warning: not enough queries to run were generated!"
            print "Requested:", searchesCount
            print "Generated:", len(queries)

        successfullQueries = 0
        i = 1
        totalQueries = len(queries)

        for query in queries:
            if i > 1:
                # sleep some time between queries (don't worry Bing! ;) )
                t = self.betweenQueriesInterval + random.uniform(
                    0, self.betweenQueriesSalt)
                time.sleep(t)

            url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))

            print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
                                                 totalQueries, query)

            request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
            with self.opener.open(request) as response:
                page = helpers.getResponseBody(response)

# check for the successfull marker
            found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
                 or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1

            if not found:
                filename = helpers.dumpErrorPage(page)
                print "Warning! Query:"
                print "\t" + query
                print "returned no results, check " + filename + " file for more information"

            else:
                successfullQueries += 1

                # randomly open a link
                if self.openLinkChance > random.random():
                    # get IG number
                    ig_number = IG_NUMBER_PATTERN.search(page)
                    if ig_number != None:
                        ig_number = ig_number.group(1)
                        # get search results
                        ig_results = IG_SEARCH_RESULTS_PATTERN.search(page)
                        if ig_results != None:
                            ig_results = ig_results.group(1)
                            # seperate search results
                            ig_searches = IG_SEARCHS_PATTERN.findall(
                                ig_results)
                            # get a random link to open
                            ig_max_rand = min(self.openTopLinkRange,
                                              len(ig_searches) - 1)
                            ig_link_num = random.randint(0, ig_max_rand)
                            # number of the link we will use
                            ig_link = "{0}?IG={1}&{2}".format(
                                IG_PING_LINK, urllib.quote_plus(ig_number),
                                urllib.quote_plus(ig_searches[ig_link_num][1]))

                            # sleep a reasonable amount of time before clicking the link
                            # use defaults to save space in config
                            t = random.uniform(0.75, 3.0)
                            time.sleep(t)

                            # open the random link
                            request = urllib2.Request(
                                url=ig_link, headers=bingCommon.HEADERS)
                            request.headers["Referer"] = response.url
                            self.opener.open(request)

                            if verbose:
                                print("Followed Link {}".format(ig_link_num +
                                                                1))
                    else:
                        filename = helpers.dumpErrorPage(page)
                        print "Warning! Could not find search result IG number"
                        print "Check {0} file for more information".format(
                            filename)

            i += 1

        if successfullQueries < searchesCount:
            res.message = str(successfullQueries) + " out of " + str(
                searchesCount) + " requests were successfully processed"
        else:
            res.message = "All " + str(
                successfullQueries) + " requests were successfully processed"

        # reset header to pc so pc pages return in getting life time points
        headers["User-Agent"] = self.userAgents.pc

        return res
    def __processSearch(self, reward):
        """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""

        BING_QUERY_URL = 'http://www.bing.com/search?q='
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'

        res = self.RewardResult(reward)
        if reward.isAchieved():
            res.message = "This reward has been already achieved"
            return res

        indCol = bfp.Reward.Type.Col.INDEX

# get a set of queries from today's Bing! history
        url = bingHistory.getBingHistoryTodayURL()
        request = urllib2.Request(url = url, headers = self.httpHeaders)
        with self.opener.open(request) as response:
            page = helpers.getResponseBody(response)
        history = bingHistory.parse(page)

# find out how many searches need to be performed
        matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
        rewardsCount    = int(matches.group(1))
        rewardCost      = int(matches.group(2))
        maxRewardsCount = int(matches.group(4))
        searchesCount = maxRewardsCount * rewardCost / rewardsCount

# adjust to the current progress
        searchesCount -= reward.progressCurrent * rewardCost

        headers = self.httpHeaders

        if reward.tp == bfp.Reward.Type.SEARCH_PC:
            headers["User-Agent"] = self.userAgents.pc
            print
            print "Running PC searches"
            print
        elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
            headers["User-Agent"] = self.userAgents.mobile
            print
            print "Running mobile searches"
            print
        else:
            res.isError = True
            res.message = "Don't know how to process this search"
            return res

        request = urllib2.Request(url = BING_NEWS_URL, headers = headers)
        with self.opener.open(request) as response:
            page = helpers.getResponseBody(response)

# generate a set of queries to run
        bingQueriesGenerator = BingQueriesGenerator(searchesCount, history, reward.tp)
        queries = bingQueriesGenerator.parseBingNews(page)
        if len(queries) < searchesCount:
            print "Warning: not enough queries to run were generated !"
            print "Requested:", searchesCount
            print "Generated:", len(bingQueriesGenerator.queries)

        successfullQueries = 0
        i = 1
        totalQueries = len(queries)

        for query in queries:
            if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
                t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesInterval)
                time.sleep(t)

            url = BING_QUERY_URL + urllib.quote_plus(query)

            print "%s - %2d/%2d - Requesting: %s" % (helpers.getLoggingTime(), i, totalQueries, url)

            request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
            with self.opener.open(request) as response:
                page = helpers.getResponseBody(response)

# check for the successfull marker
            found = ( reward.tp == bfp.Reward.Type.SEARCH_PC and
                      page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 ) \
                 or ( reward.tp == bfp.Reward.Type.SEARCH_MOBILE and
                      page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 )

            if not found:
                filename = helpers.dumpErrorPage(page)
                print "Warning! Query:"
                print "\t" + query
                print "returned no results, check " + filename + " file for more information"

            else:
                successfullQueries += 1

            i += 1

        if successfullQueries < searchesCount:
            res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
        else:
            res.message = "All " + str(successfullQueries) + " requests were successfully processed"

        return res
Exemple #8
0
    def __processSearch(self, reward, verbose):
        """Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""

        BING_QUERY_URL = 'http://www.bing.com/search?q='
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
        IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
        IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
        IG_SEARCHES_PATTERN = re.compile(
            r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"'
        )

        res = self.RewardResult(reward)
        if reward.isAchieved():
            res.message = "This reward has been already achieved"
            return res

        indCol = bdp.Reward.Type.Col.INDEX

        # get a set of queries from today's Bing! history
        url = bingHistory.getBingHistoryTodayURL()
        request = urllib2.Request(url=url, headers=self.httpHeaders)
        with self.opener.open(request) as response:
            page = helpers.getResponseBody(response)
        history = bingHistory.parse(page)

        # find out how many searches need to be performed
        matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
            reward.description)
        if matches is None:
            print "No RegEx matches found for this search and earn"
            res.isError = True
            res.message = "No RegEx matches found for this search and earn"
            return res
        maxRewardsCount = int(matches.group(1))
        rewardsCount = int(matches.group(2))
        rewardCost = 1  # Looks like it's now always X points per one search
        searchesCount = maxRewardsCount * rewardCost / rewardsCount

        # adjust to the current progress
        # reward.progressCurrent is now returning current points, not current searches
        # so divide it by points per search (rewardsCount) to get correct search count needed
        searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount

        headers = self.httpHeaders

        if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
            headers["User-Agent"] = self.userAgents.pc
            searchesCount += self.addSearchesDesktop + random.randint(
                0, self.addSearchesDesktopSalt)
            print
            print "Running PC searches"
            print
        elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
            headers["User-Agent"] = self.userAgents.mobile
            searchesCount += self.addSearchesMobile + random.randint(
                0, self.addSearchesMobileSalt)
            print
            print "Running mobile searches"
            print

        if verbose:
            print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
            print

        # Import the query generator
        try:
            qg = importlib.import_module(self.queryGenerator, package=None)
            queryGenerator = qg.queryGenerator(self)
        except ImportError:
            raise TypeError("{0} is not a module".format(self.queryGenerator))

        # generate a set of queries to run
        queries = queryGenerator.generateQueries(searchesCount, history)

        if len(queries) < searchesCount:
            print "Warning: not enough queries to run were generated!\nRequested: {}\nGenerated: {}\n"\
                        .format(searchesCount, len(queries))

        successfullQueries = 0
        i = 1
        totalQueries = len(queries)

        for query in queries:
            if i > 1:
                # sleep some time between queries (don't worry Bing! ;) )
                t = self.betweenQueriesInterval + random.uniform(
                    0, self.betweenQueriesSalt)
                time.sleep(t)

            url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))

            print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
                                                 totalQueries, query)

            request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
            with self.opener.open(request) as response:
                page = helpers.getResponseBody(response)

            # check for the successfull marker
            found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
                 or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1

            if not found:
                print "Warning! Query:\n\t {}\n returned no results, check file for more information {}".format(
                    query, helpers.dumpErrorPage(page))
            else:
                successfullQueries += 1
                self.randomClick(IG_NUMBER_PATTERN, IG_PING_LINK,
                                 IG_SEARCHES_PATTERN, page, response, verbose)

            i += 1

        if successfullQueries < searchesCount:
            res.message = str(successfullQueries) + " out of " + str(
                searchesCount) + " requests were successfully processed"
        else:
            res.message = "All " + str(
                successfullQueries) + " requests were successfully processed"

        # reset header to pc so pc pages return in getting life time points
        headers["User-Agent"] = self.userAgents.pc

        return res
    def __processSearch(self, reward, verbose):
        """Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""

        BING_QUERY_URL = 'http://www.bing.com/search?q='
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
        BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
        IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
        IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
        IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')

        res = self.RewardResult(reward)
        if reward.isAchieved():
            res.message = "This reward has been already achieved"
            return res

        indCol = bdp.Reward.Type.Col.INDEX

# get a set of queries from today's Bing history
        url = bingHistory.getBingHistoryTodayURL()
        request = urllib2.Request(url = url, headers = self.httpHeaders)
        with self.opener.open(request) as response:
            page = helpers.getResponseBody(response)
        history = bingHistory.parse(page)

# find out how many searches need to be performed
        matchesMobile = False
        matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
        #Mobile description changed, so check that one too
        if matches is None:
            matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE_MOBILE.search(reward.description)
            matchesMobile = True
        if matches is None: 
            print "No RegEx matches found for this search and earn"
            res.isError = True
            res.message = "No RegEx matches found for this search and earn"
            return res
        maxRewardsCount = int(matches.group(1))
        rewardsCount    = int(matches.group(2))
        rewardCost      = 1 # Looks like it's now always X points per one search
        searchesCount = maxRewardsCount * rewardCost / rewardsCount

# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
        searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount

        if matchesMobile == True:
            #new mobile search description gives total search count + points per search for edge/non-edge
            edgeValue = int(matches.group(1))
            nonEdgeValue = int(matches.group(2))
            searchesCount = int(matches.group(3))
            #apparently ios uses EdgiOS, so we only check the first 3 letters not the full word 'edge'
            if self.userAgents.mobile.lower().find("edg") != -1:
                #we are searching on edge so points go to 200
                searchesCount -= reward.progressCurrent / edgeValue
            else:
                #non-edge search so 100 is the max
                searchesCount -= reward.progressCurrent / nonEdgeValue

        headers = self.httpHeaders

        if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
            headers["User-Agent"] = self.userAgents.pc
            searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
            print
            print "Running PC searches"
            print
        elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
            headers["User-Agent"] = self.userAgents.mobile
            searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
            print
            print "Running mobile searches"
            print
        else:
            res.isError = True
            res.message = "Don't know how to process this search"
            return res

        if verbose:
            print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
            print

        # Import the query generator
        try:
            qg = importlib.import_module(self.queryGenerator, package=None)
            queryGenerator = qg.queryGenerator(self)
        except ImportError:
            raise TypeError("{0} is not a module".format(self.queryGenerator))

        # generate a set of queries to run
        queries = queryGenerator.generateQueries(searchesCount, history)

        if len(queries) < searchesCount:
            print "Warning: not enough queries to run were generated!"
            print "Requested:", searchesCount
            print "Generated:", len(queries)

        successfullQueries = 0
        i = 1
        totalQueries = len(queries)

        for query in queries:
            if i > 1:
# sleep some time between queries (don't worry Bing ;) )
                t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
                time.sleep(t)

            url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))

            print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)

            request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
            with self.opener.open(request) as response:
                page = helpers.getResponseBody(response)

# check for the successfull marker
            found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
                 or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1

            if not found:
                filename = helpers.dumpErrorPage(page)
                print "Warning! Query:"
                print "\t" + query
                print "returned no results, check " + filename + " file for more information"

            else:
                successfullQueries += 1

                # randomly open a link
                if self.openLinkChance > random.random():
                    # get IG number
                    ig_number = IG_NUMBER_PATTERN.search(page)
                    if ig_number is not None:
                        ig_number = ig_number.group(1)
                        ig_searches = IG_SEARCHES_PATTERN.findall(page)
                        # make sure we have at least 1 search
                        if len(ig_searches) > 0:
                            # get a random link to open
                            ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
                            # number of the link we will use
                            ig_link_num = random.randint(0, ig_max_rand)
                            ig_link = "{0}?IG={1}&{2}".format(
                                IG_PING_LINK,
                                urllib.quote_plus(ig_number),
                                urllib.quote_plus(ig_searches[ig_link_num][1])
                            )

                            # sleep a reasonable amount of time before clicking the link
                            # use defaults to save space in config
                            t = random.uniform(0.75, 3.0)
                            time.sleep(t)

                            # open the random link
                            request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
                            request.headers["Referer"] = response.url
                            self.opener.open(request)

                            if verbose:
                                print("Followed Link {}".format(ig_link_num + 1))
                        else:
                            filename = helpers.dumpErrorPage(page)
                            print "Warning! No searches were found on search results page"
                            print "Check {0} file for more information".format(filename)
                    else:
                        filename = helpers.dumpErrorPage(page)
                        print "Warning! Could not find search result IG number"
                        print "Check {0} file for more information".format(filename)

            i += 1

        if successfullQueries < searchesCount:
            res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
        else:
            res.message = "All " + str(successfullQueries) + " requests were successfully processed"

        # reset header to pc so pc pages return in getting life time points
        headers["User-Agent"] = self.userAgents.pc

        return res