def test_timestamp(self): """ test getlogtime :return: """ stamp = helpers.getLoggingTime() self.assertRegexpMatches(stamp, "\d{4}-\d{2}-\d{2}", "should have time stamp,\n" + stamp)
def __run(config): report = list() doSleep = False for key, account in config.accounts.iteritems(): if account.disabled: continue # sleep between two accounts logins if doSleep: extra = config.general.betweenAccountsInterval + random.uniform( 0, config.general.betweenAccountsSalt) if verbose: print("\nPausing between accounts for {0} seconds".format( int(extra))) time.sleep(extra) reportItem = BingRewardsReportItem() reportItem.accountType = account.accountType reportItem.accountLogin = account.accountLogin agents = bingCommon.UserAgents.generate(account) httpHeaders = bingCommon.HEADERS httpHeaders["User-Agent"] = agents.pc __processAccount(config, httpHeaders, agents, reportItem, account.password) report.append(reportItem) doSleep = True # # trigger full report if needed # if showFullReport or totalPoints > 0 and len(report) > 1: print print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= FINAL REPORT =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" print print " Account | Before | After | Earned | Lifetime | Retries " print "------------------------------+--------+--------+--------+------------+---------" for r in report: print " %-28s | %6d | %6d | %6d | %10d | %7d" % ( __stringifyAccount(r, 28), r.oldPoints, r.newPoints, r.pointsEarnedRetrying, r.lifetimeCredits, r.retries) print # # print footer # print "Total points earned: %d" % totalPoints print print "%s - script ended" % helpers.getLoggingTime() EventsProcessor.onScriptComplete(config)
def __run(config): report = list() doSleep = False for key, account in config.accounts.iteritems(): if account.disabled: continue # sleep between two accounts logins if doSleep: extra = config.general.betweenAccountsInterval + random.uniform(0, config.general.betweenAccountsSalt) if verbose: print("\nPausing between accounts for {0} seconds".format(int(extra))) time.sleep(extra) reportItem = BingRewardsReportItem() reportItem.accountType = account.accountType reportItem.accountLogin = account.accountLogin agents = bingCommon.UserAgents.generate(account) httpHeaders = bingCommon.HEADERS httpHeaders["User-Agent"] = agents.pc __processAccount(config, httpHeaders, agents, reportItem, account.password) report.append(reportItem) doSleep = True # # trigger full report if needed # if showFullReport or totalPoints > 0 and len(report) > 1: print print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= FINAL REPORT =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" print print " Account | Before | After | Earned | Lifetime | Retries " print "------------------------------+--------+--------+--------+------------+---------" for r in report: print " %-28s | %6d | %6d | %6d | %10d | %7d" % (__stringifyAccount(r, 28), r.oldPoints, r.newPoints, r.pointsEarnedRetrying, r.lifetimeCredits, r.retries) print # # print footer # print "Total points earned: %d" % totalPoints print print "%s - script ended" % helpers.getLoggingTime() EventsProcessor.onScriptComplete(config)
def __run(config): report = list() accountIndex = 0 for key, account in config.accounts.iteritems(): if account.disabled: continue # sleep between two accounts logins if accountIndex > 0: isFirstAccount = False extra = config.general.betweenAccountsInterval + random.uniform(0, config.general.betweenAccountsSalt) time.sleep(extra) reportItem = BingRewardsReportItem() reportItem.accountType = account.accountType reportItem.accountLogin = account.accountLogin __processAccount(config, reportItem, account.password) report.append(reportItem) accountIndex += 1 EventsProcessor.onScriptComplete(config) # # trigger full report if needed # if showFullReport or totalPoints > 0 and len(report) > 1: print print " -=-=-=-=-=-=-=-=-=-=--=-=- FULL REPORT -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=" print print " Account | Before | After | Earned | Retries | Lifetime Credits" print "---------------------------+--------+--------+--------+---------+-----------------" for r in report: print " %25s | %6d | %6d | %6d | %7d | %16d" % (__stringifyAccount(r, 25), r.oldPoints, r.newPoints, r.pointsEarnedRetrying, r.retries, r.lifetimeCredits) print # # print footer # print "Total points earned: %d" % totalPoints print print "%s - script ended" % helpers.getLoggingTime()
def __processQuiz(self, reward): """Processes bdp.Reward.Type.Action.QUIZ and returns self.RewardResult""" res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res pointsEarned = self.getRewardsPoints() currPage = self.getDashboardPage() startIndex = currPage.find('__RequestVerificationToken') endIndex = currPage[startIndex:].find('/>') #pad here to get to the correct spot verificationAttr = currPage[startIndex + 49:startIndex + endIndex - 2] #TODO: last parameter is ', "Timezone" : 240'. Is it needed? verificationData = '{"ActivitySubType" : "quiz", "ActivityType" : "notification", "OfferId" : "' + reward.hitId + '", "Channel" : "Bing.Com", "PartnerId" : "BingTrivia"}' verificationUrl = 'https://www.bing.com/msrewards/api/v1/ReportActivity' print print "Running activity: %s" % reward.name print request = urllib2.Request(url=verificationUrl, headers=self.httpHeaders) for i in range(reward.progressCurrent, reward.progressMax, 10): print "%s - %2d/%2d - Activity: %s" % (helpers.getLoggingTime(), i + 10, reward.progressMax, reward.name) with self.opener.open(request, verificationData) as response: page = helpers.getResponseBody(response) #default pause between guesses t = self.betweenQueriesInterval + random.uniform( 0, self.betweenQueriesSalt) time.sleep(t) pointsEarned = self.getRewardsPoints() - pointsEarned # if QUIZ is against bdp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to # pointsExpected indCol = bdp.Reward.Type.Col.INDEX if reward.tp[indCol] == bdp.Reward.Type.RE_EARN_CREDITS[indCol]: pointsExpected = reward.progressMax - reward.progressCurrent if pointsExpected != pointsEarned: filename = helpers.dumpErrorPage(page) res.isError = True res.message = "Expected to earn " + str(pointsExpected) + " points, but earned " + \ str(pointsEarned) + " points. Check " + filename + " for further information" return res
def __run(config): report = list() doSleep = False for key, account in config.accounts.iteritems(): if account.disabled: continue reportItem = __processAccountUserAgent(config, account, bingCommon.USER_AGENTS_PC, doSleep) report.append(reportItem) doSleep = True # # trigger full report if needed # if showFullReport or totalPoints > 0 and len(report) > 1: print print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= FINAL REPORT =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" print print " Account | Before | After | Earned | Lifetime | Retries " print "------------------------------+--------+--------+--------+------------+---------" for r in report: print " %-28s | %6d | %6d | %6d | %10d | %7d" % ( __stringifyAccount(r, 28), r.oldPoints, r.newPoints, r.pointsEarnedRetrying, r.lifetimeCredits, r.retries) print # # print footer # print "Total points earned: %d" % totalPoints print print "%s - script ended" % helpers.getLoggingTime() EventsProcessor.onScriptComplete(config)
def __run(config): report = list() doSleep = False #for key, account in config.accounts.iteritems(): for key, account in config.accounts.items(): if account.disabled: continue reportItem = __processAccountUserAgent(config, account, bingCommon.USER_AGENTS_PC, doSleep) report.append(reportItem) doSleep = True # # trigger full report if needed # if (showFullReport or totalPoints > 0 and len(report) > 1): print print (" -=-=-=-=-=-=-=-=-=-=--=-=- FULL REPORT -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-") print print (" Account | Before | After | Earned | Retries | Lifetime Credits") print ("---------------------------+--------+--------+--------+---------+-----------------") for r in report: print (" %25s | %6d | %6d | %6d | %7d | %16d" % (__stringifyAccount(r, 25), r.oldPoints, r.newPoints, r.pointsEarnedRetrying, r.retries, r.lifetimeCredits)) print # # print footer # print ("Total points earned: %d" % totalPoints) print print ("%s - script ended" % helpers.getLoggingTime()) EventsProcessor.onScriptComplete(config)
def __processSearch(self, reward, verbose): """Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">' IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx" IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"') IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"') res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bdp.Reward.Type.Col.INDEX # get a set of queries from today's Bing history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url = url, headers = self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matchesMobile = False matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description) #Mobile description changed, so check that one too if matches is None: matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE_MOBILE.search(reward.description) matchesMobile = True if matches is None: print "No RegEx matches found for this search and earn" res.isError = True res.message = "No RegEx matches found for this search and earn" return res maxRewardsCount = int(matches.group(1)) rewardsCount = int(matches.group(2)) rewardCost = 1 # Looks like it's now always X points per one search searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress # reward.progressCurrent is now returning current points, not current searches # so divide it by points per search (rewardsCount) to get correct search count needed searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount if matchesMobile == True: #new mobile search description gives total search count + points per search for edge/non-edge edgeValue = int(matches.group(1)) nonEdgeValue = int(matches.group(2)) searchesCount = int(matches.group(3)) #apparently ios uses EdgiOS, so we only check the first 3 letters not the full word 'edge' if self.userAgents.mobile.lower().find("edg") != -1: #we are searching on edge so points go to 200 searchesCount -= reward.progressCurrent / edgeValue else: #non-edge search so 100 is the max searchesCount -= reward.progressCurrent / nonEdgeValue headers = self.httpHeaders if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN: headers["User-Agent"] = self.userAgents.pc searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt) print print "Running PC searches" print elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt) print print "Running mobile searches" print else: res.isError = True res.message = "Don't know how to process this search" return res if verbose: print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"])) print # Import the query generator try: qg = importlib.import_module(self.queryGenerator, package=None) queryGenerator = qg.queryGenerator(self) except ImportError: raise TypeError("{0} is not a module".format(self.queryGenerator)) # generate a set of queries to run queries = queryGenerator.generateQueries(searchesCount, history) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated!" print "Requested:", searchesCount print "Generated:", len(queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing ;) ) t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url = url, headers = bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 # randomly open a link if self.openLinkChance > random.random(): # get IG number ig_number = IG_NUMBER_PATTERN.search(page) if ig_number is not None: ig_number = ig_number.group(1) ig_searches = IG_SEARCHES_PATTERN.findall(page) # make sure we have at least 1 search if len(ig_searches) > 0: # get a random link to open ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1) # number of the link we will use ig_link_num = random.randint(0, ig_max_rand) ig_link = "{0}?IG={1}&{2}".format( IG_PING_LINK, urllib.quote_plus(ig_number), urllib.quote_plus(ig_searches[ig_link_num][1]) ) # sleep a reasonable amount of time before clicking the link # use defaults to save space in config t = random.uniform(0.75, 3.0) time.sleep(t) # open the random link request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS) request.headers["Referer"] = response.url self.opener.open(request) if verbose: print("Followed Link {}".format(ig_link_num + 1)) else: filename = helpers.dumpErrorPage(page) print "Warning! No searches were found on search results page" print "Check {0} file for more information".format(filename) else: filename = helpers.dumpErrorPage(page) print "Warning! Could not find search result IG number" print "Check {0} file for more information".format(filename) i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed" else: res.message = "All " + str(successfullQueries) + " requests were successfully processed" # reset header to pc so pc pages return in getting life time points headers["User-Agent"] = self.userAgents.pc return res
if o in ("-h", "--help"): usage() sys.exit() elif o in ("-f", "--configFile"): configFile = a elif o in ("-r", "--full-report"): showFullReport = True elif o in ("-v", "--verbose"): verbose = True elif o == "--version": printVersion() sys.exit() else: raise NotImplementedError("option '" + o + "' is not implemented") print "%s - script started" % helpers.getLoggingTime() print "-" * 80 print helpers.createResultsDir(__file__) config = Config() try: config.parseFromFile(configFile) except IOError, e: print "IOError: %s" % e sys.exit(2) except ConfigError, e: print "ConfigError: %s" % e sys.exit(2)
def __processSearch(self, reward): """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">' res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bfp.Reward.Type.Col.INDEX # get a set of queries from today's Bing! history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url=url, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search( reward.description) rewardsCount = int(matches.group(1)) rewardCost = int(matches.group(2)) maxRewardsCount = int(matches.group(4)) searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress searchesCount -= reward.progressCurrent * rewardCost headers = self.httpHeaders if reward.tp == bfp.Reward.Type.SEARCH_PC: headers["User-Agent"] = self.userAgents.pc print print "Running PC searches" print elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile print print "Running mobile searches" print else: res.isError = True res.message = "Don't know how to process this search" return res # Import the query generator try: qg = importlib.import_module(self.queryGenerator, package=None) queryGenerator = qg.queryGenerator(self) except ImportError: raise TypeError("{0} is not a module".format(self.queryGenerator)) # generate a set of queries to run queries = queryGenerator.generateQueries(searchesCount, history) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated!" print "Requested:", searchesCount print "Generated:", len(queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform( 0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url=url, headers=bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str( searchesCount) + " requests were successfully processed" else: res.message = "All " + str( successfullQueries) + " requests were successfully processed" # reset header to pc so pc pages return in getting life time points headers["User-Agent"] = self.userAgents.pc return res
print "Requested:", searchesCount print "Generated:", len(queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url = url, headers = bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information"
def __processSearch(self, reward, verbose): """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content"' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content"' IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx" IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"') IG_SEARCH_RESULTS_PATTERN = re.compile(r'<ol\s.*?id="b_results"(.+?)</ol>') IG_SEARCHS_PATTERN = re.compile(r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"') res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bfp.Reward.Type.Col.INDEX # get a set of queries from today's Bing! history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url = url, headers = self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description) rewardsCount = int(matches.group(1)) rewardCost = int(matches.group(2)) maxRewardsCount = int(matches.group(4)) searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress searchesCount -= reward.progressCurrent * rewardCost headers = self.httpHeaders if reward.tp == bfp.Reward.Type.SEARCH_PC: headers["User-Agent"] = self.userAgents.pc searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt) print print "Running PC searches" print elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt) print print "Running mobile searches" print else: res.isError = True res.message = "Don't know how to process this search" return res if verbose: print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"])) print # Import the query generator try: qg = importlib.import_module(self.queryGenerator, package=None) queryGenerator = qg.queryGenerator(self) except ImportError: raise TypeError("{0} is not a module".format(self.queryGenerator)) # generate a set of queries to run queries = queryGenerator.generateQueries(searchesCount, history) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated!" print "Requested:", searchesCount print "Generated:", len(queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url = url, headers = bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 # randomly open a link if self.openLinkChance > random.random(): # get IG number ig_number = IG_NUMBER_PATTERN.search(page) if ig_number != None: ig_number = ig_number.group(1) # get search results ig_results = IG_SEARCH_RESULTS_PATTERN.search(page) if ig_results != None: ig_results = ig_results.group(1) # seperate search results ig_searches = IG_SEARCHS_PATTERN.findall(ig_results) # make sure we have at least 1 search if len(ig_searches) > 0: # get a random link to open ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1) ig_link_num = random.randint(0, ig_max_rand) # number of the link we will use ig_link = "{0}?IG={1}&{2}".format( IG_PING_LINK, urllib.quote_plus(ig_number), urllib.quote_plus(ig_searches[ig_link_num][1]) ) # sleep a reasonable amount of time before clicking the link # use defaults to save space in config t = random.uniform(0.75, 3.0) time.sleep(t) # open the random link request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS) request.headers["Referer"] = response.url self.opener.open(request) if verbose: print("Followed Link {}".format(ig_link_num + 1)) else: filename = helpers.dumpErrorPage(page) print "Warning! No searches were found on search results page" print "Check {0} file for more information".format(filename) else: filename = helpers.dumpErrorPage(page) print "Warning! Could not find search result IG number" print "Check {0} file for more information".format(filename) i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed" else: res.message = "All " + str(successfullQueries) + " requests were successfully processed" # reset header to pc so pc pages return in getting life time points headers["User-Agent"] = self.userAgents.pc return res
def __processSearch(self, reward, verbose): """Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">' IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx" IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"') IG_SEARCHES_PATTERN = re.compile( r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"' ) res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bdp.Reward.Type.Col.INDEX # get a set of queries from today's Bing! history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url=url, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search( reward.description) if matches is None: print "No RegEx matches found for this search and earn" res.isError = True res.message = "No RegEx matches found for this search and earn" return res maxRewardsCount = int(matches.group(1)) rewardsCount = int(matches.group(2)) rewardCost = 1 # Looks like it's now always X points per one search searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress # reward.progressCurrent is now returning current points, not current searches # so divide it by points per search (rewardsCount) to get correct search count needed searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount headers = self.httpHeaders if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN: headers["User-Agent"] = self.userAgents.pc searchesCount += self.addSearchesDesktop + random.randint( 0, self.addSearchesDesktopSalt) print print "Running PC searches" print elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile searchesCount += self.addSearchesMobile + random.randint( 0, self.addSearchesMobileSalt) print print "Running mobile searches" print if verbose: print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"])) print # Import the query generator try: qg = importlib.import_module(self.queryGenerator, package=None) queryGenerator = qg.queryGenerator(self) except ImportError: raise TypeError("{0} is not a module".format(self.queryGenerator)) # generate a set of queries to run queries = queryGenerator.generateQueries(searchesCount, history) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated!\nRequested: {}\nGenerated: {}\n"\ .format(searchesCount, len(queries)) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform( 0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url=url, headers=bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: print "Warning! Query:\n\t {}\n returned no results, check file for more information {}".format( query, helpers.dumpErrorPage(page)) else: successfullQueries += 1 self.randomClick(IG_NUMBER_PATTERN, IG_PING_LINK, IG_SEARCHES_PATTERN, page, response, verbose) i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str( searchesCount) + " requests were successfully processed" else: res.message = "All " + str( successfullQueries) + " requests were successfully processed" # reset header to pc so pc pages return in getting life time points headers["User-Agent"] = self.userAgents.pc return res
def __processSearch(self, reward): """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">' res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bfp.Reward.Type.Col.INDEX # get a set of queries from today's Bing! history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url = url, headers = self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description) rewardsCount = int(matches.group(1)) rewardCost = int(matches.group(2)) maxRewardsCount = int(matches.group(4)) searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress searchesCount -= reward.progressCurrent * rewardCost headers = self.httpHeaders if reward.tp == bfp.Reward.Type.SEARCH_PC: headers["User-Agent"] = self.userAgents.pc print print "Running PC searches" print elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile print print "Running mobile searches" print else: res.isError = True res.message = "Don't know how to process this search" return res request = urllib2.Request(url = BING_NEWS_URL, headers = headers) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # generate a set of queries to run bingQueriesGenerator = BingQueriesGenerator(searchesCount, history, reward.tp) queries = bingQueriesGenerator.parseBingNews(page) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated !" print "Requested:", searchesCount print "Generated:", len(bingQueriesGenerator.queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesInterval) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query) print "%s - %2d/%2d - Requesting: %s" % (helpers.getLoggingTime(), i, totalQueries, url) request = urllib2.Request(url = url, headers = bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = ( reward.tp == bfp.Reward.Type.SEARCH_PC and page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 ) \ or ( reward.tp == bfp.Reward.Type.SEARCH_MOBILE and page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 ) if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed" else: res.message = "All " + str(successfullQueries) + " requests were successfully processed" return res
def __processSearch(self, reward, verbose): """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">' IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx" IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"') IG_SEARCH_RESULTS_PATTERN = re.compile( r'<ol\s.*?id="b_results"(.+?)</ol>') IG_SEARCHS_PATTERN = re.compile( r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"') res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bfp.Reward.Type.Col.INDEX # get a set of queries from today's Bing! history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url=url, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search( reward.description) rewardsCount = int(matches.group(1)) rewardCost = int(matches.group(2)) maxRewardsCount = int(matches.group(4)) searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress searchesCount -= reward.progressCurrent * rewardCost headers = self.httpHeaders if reward.tp == bfp.Reward.Type.SEARCH_PC: headers["User-Agent"] = self.userAgents.pc searchesCount += self.addSearchesDesktop + random.randint( 0, self.addSearchesDesktopSalt) print print "Running PC searches" print elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile searchesCount += self.addSearchesMobile + random.randint( 0, self.addSearchesMobileSalt) print print "Running mobile searches" print else: res.isError = True res.message = "Don't know how to process this search" return res # Import the query generator try: qg = importlib.import_module(self.queryGenerator, package=None) queryGenerator = qg.queryGenerator(self) except ImportError: raise TypeError("{0} is not a module".format(self.queryGenerator)) # generate a set of queries to run queries = queryGenerator.generateQueries(searchesCount, history) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated!" print "Requested:", searchesCount print "Generated:", len(queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform( 0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url=url, headers=bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 # randomly open a link if self.openLinkChance > random.random(): # get IG number ig_number = IG_NUMBER_PATTERN.search(page) if ig_number != None: ig_number = ig_number.group(1) # get search results ig_results = IG_SEARCH_RESULTS_PATTERN.search(page) if ig_results != None: ig_results = ig_results.group(1) # seperate search results ig_searches = IG_SEARCHS_PATTERN.findall( ig_results) # get a random link to open ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1) ig_link_num = random.randint(0, ig_max_rand) # number of the link we will use ig_link = "{0}?IG={1}&{2}".format( IG_PING_LINK, urllib.quote_plus(ig_number), urllib.quote_plus(ig_searches[ig_link_num][1])) # sleep a reasonable amount of time before clicking the link # use defaults to save space in config t = random.uniform(0.75, 3.0) time.sleep(t) # open the random link request = urllib2.Request( url=ig_link, headers=bingCommon.HEADERS) request.headers["Referer"] = response.url self.opener.open(request) if verbose: print("Followed Link {}".format(ig_link_num + 1)) else: filename = helpers.dumpErrorPage(page) print "Warning! Could not find search result IG number" print "Check {0} file for more information".format( filename) i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str( searchesCount) + " requests were successfully processed" else: res.message = "All " + str( successfullQueries) + " requests were successfully processed" # reset header to pc so pc pages return in getting life time points headers["User-Agent"] = self.userAgents.pc return res