def __processHit(self, reward): """Processes bdp.Reward.Type.Action.HIT and returns self.RewardResult""" res = self.RewardResult(reward) pointsEarned = self.getRewardsPoints() currPage = self.getDashboardPage() startIndex = currPage.find('__RequestVerificationToken') endIndex = currPage[startIndex:].find('/>') #pad here to get to the correct spot verificationAttr = currPage[startIndex+49:startIndex+endIndex-2] verificationData = [ ('id', reward.hitId), ('hash', reward.hitHash), ('timeZone', '-300'), ('activityAmount', '1'), ('__RequestVerificationToken', verificationAttr) #there was a comma here, removed it. Monitor to make sure shit still works ] verificationUrl = 'https://account.microsoft.com/rewards/api/reportactivity?refd=www.bing.com&X-Requested-With=XMLHttpRequest' request = urllib2.Request(url = verificationUrl, headers = self.httpHeaders) with self.opener.open(request, urllib.urlencode(verificationData)) as response: page = helpers.getResponseBody(response) pointsEarned = self.getRewardsPoints() - pointsEarned # if HIT is against bdp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to # pointsExpected indCol = bdp.Reward.Type.Col.INDEX if reward.tp[indCol] == bdp.Reward.Type.RE_EARN_CREDITS[indCol]: pointsExpected = reward.progressMax - reward.progressCurrent if pointsExpected != pointsEarned: filename = helpers.dumpErrorPage(page) res.isError = True res.message = "Expected to earn " + str(pointsExpected) + " points, but earned " + \ str(pointsEarned) + " points. Check " + filename + " for further information" return res
def test_dump(self): """ test dump page to a file :return: """ filename = helpers.dumpErrorPage(self.mockdate) output = "" with open("result/" + filename, "r") as fd: output += fd.readline() self.assertRegexpMatches(output, "\d{4}-\d{2}-\d{2}", "should have time stamp,\n" + output)
def randomClick(self, IG_NUMBER_PATTERN, IG_PING_LINK, IG_SEARCHES_PATTERN, page, response, verbose): # randomly open a link if self.openLinkChance > random.random(): # get IG number ig_number = IG_NUMBER_PATTERN.search(page) if ig_number is not None: ig_number = ig_number.group(1) ig_searches = IG_SEARCHES_PATTERN.findall(page) # make sure we have at least 1 search if len(ig_searches) > 0: # get a random link to open ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1) # number of the link we will use ig_link_num = random.randint(0, ig_max_rand) ig_link = "{0}?IG={1}&{2}".format( IG_PING_LINK, urllib.quote_plus(ig_number), urllib.quote_plus(ig_searches[ig_link_num][1])) # sleep a reasonable amount of time before clicking the link # use defaults to save space in config t = random.uniform(0.75, 3.0) time.sleep(t) # open the random link request = urllib2.Request(url=ig_link, headers=bingCommon.HEADERS) request.headers["Referer"] = response.url self.opener.open(request) if verbose: print("Followed Link {}".format(ig_link_num + 1)) else: print "Warning! No searches were found on search results page Check {0} file for more information".format( helpers.dumpErrorPage(page)) else: print "Warning! Could not find search result IG number Check {0} file for more information".format( helpers.dumpErrorPage(page))
def __processQuiz(self, reward): """Processes bdp.Reward.Type.Action.QUIZ and returns self.RewardResult""" res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res pointsEarned = self.getRewardsPoints() currPage = self.getDashboardPage() startIndex = currPage.find('__RequestVerificationToken') endIndex = currPage[startIndex:].find('/>') #pad here to get to the correct spot verificationAttr = currPage[startIndex + 49:startIndex + endIndex - 2] #TODO: last parameter is ', "Timezone" : 240'. Is it needed? verificationData = '{"ActivitySubType" : "quiz", "ActivityType" : "notification", "OfferId" : "' + reward.hitId + '", "Channel" : "Bing.Com", "PartnerId" : "BingTrivia"}' verificationUrl = 'https://www.bing.com/msrewards/api/v1/ReportActivity' print print "Running activity: %s" % reward.name print request = urllib2.Request(url=verificationUrl, headers=self.httpHeaders) for i in range(reward.progressCurrent, reward.progressMax, 10): print "%s - %2d/%2d - Activity: %s" % (helpers.getLoggingTime(), i + 10, reward.progressMax, reward.name) with self.opener.open(request, verificationData) as response: page = helpers.getResponseBody(response) #default pause between guesses t = self.betweenQueriesInterval + random.uniform( 0, self.betweenQueriesSalt) time.sleep(t) pointsEarned = self.getRewardsPoints() - pointsEarned # if QUIZ is against bdp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to # pointsExpected indCol = bdp.Reward.Type.Col.INDEX if reward.tp[indCol] == bdp.Reward.Type.RE_EARN_CREDITS[indCol]: pointsExpected = reward.progressMax - reward.progressCurrent if pointsExpected != pointsEarned: filename = helpers.dumpErrorPage(page) res.isError = True res.message = "Expected to earn " + str(pointsExpected) + " points, but earned " + \ str(pointsEarned) + " points. Check " + filename + " for further information" return res
def __processHit(self, reward): """Processes bdp.Reward.Type.Action.HIT and returns self.RewardResult""" res = self.RewardResult(reward) pointsEarned = self.getRewardsPoints() request = urllib2.Request(url=reward.url, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) pointsEarned = self.getRewardsPoints() - pointsEarned # if HIT is against bdp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to # pointsExpected indCol = bdp.Reward.Type.Col.INDEX if reward.tp[indCol] == bdp.Reward.Type.RE_EARN_CREDITS[indCol]: pointsExpected = reward.progressMax - reward.progressCurrent if pointsExpected != pointsEarned: filename = helpers.dumpErrorPage(page) res.isError, res.message = True, "Expected to earn " + str(pointsExpected) + " points, but earned " + \ str(pointsEarned) + " points. Check " + filename + " for further information" return res
def __processHit(self, reward): """Processes bfp.Reward.Type.Action.HIT and returns self.RewardResult""" res = self.RewardResult(reward) pointsEarned = self.getRewardsPoints() request = urllib2.Request(url = reward.url, headers = self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) pointsEarned = self.getRewardsPoints() - pointsEarned # if HIT is against bfp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to # pointsExpected indCol = bfp.Reward.Type.Col.INDEX if reward.tp[indCol] == bfp.Reward.Type.RE_EARN_CREDITS[indCol]: matches = bfp.Reward.Type.RE_EARN_CREDITS[bfp.Reward.Type.Col.NAME].search(reward.name) pointsExpected = int(matches.group(1)) if pointsExpected != pointsEarned: filename = helpers.dumpErrorPage(page) res.isError = True res.message = "Expected to earn " + str(pointsExpected) + " points, but earned " + \ str(pointsEarned) + " points. Check " + filename + " for further information" return res
def __authenticateLive(self, login, password): """ Authenticates a user on bing.com with his/her Live account. throws AuthenticationError if authentication can not be passed throws urllib2.HTTPError if the server couldn't fulfill the request throws urllib2.URLError if failed to reach the server """ # print "Requesting bing.com" # request http://www.bing.com request = urllib2.Request(url = bingCommon.BING_URL, headers = self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # get connection URL for provider Live s = page.index('"WindowsLiveId":"') s += len('"WindowsLiveId":"') e = page.index('"', s) url = BingAuth._escapeString(page[s:e]) request = urllib2.Request(url = url, headers = self.httpHeaders) request.add_header("Referer", bingCommon.BING_URL) with self.opener.open(request) as response: referer = response.geturl() # get Facebook authenctication form action url page = helpers.getResponseBody(response) # get PPFT parameter s = page.index("sFTTag") s = page.index('value="', s) s += len('value="') e = page.index('"', s) PPFT = page[s:e] # get PPSX parameter ppsxParam = "" if ",M:" in page: ppsxParam = ",M:" elif ",g:" in page: ppsxParam = ",g:" elif ",j:" in page: ppsxParam = ",j:" else: ppsxParam = ",d:" s = page.index(ppsxParam) s += len(ppsxParam) e = page.index(",", s) PPSX = page[s:e] if PPSX[0] == "'": PPSX = PPSX[1:-1] # generate ClientLoginTime clt = 20000 + int(random.uniform(0, 1000)) # generate RenderCompleteTime renderTime = 130 + int(random.uniform(0, 100)) # generate ResourcesCompleteTime resourcesTime = renderTime + int(random.uniform(2, 5)) # generate ResourcesCompleteTime PLT = 870 + int(random.uniform(0, 250)) # get url to post data to s = page.index(",urlPost:'") s += len(",urlPost:'") e = page.index("'", s) url = page[s:e] postFields = urllib.urlencode({ "login" : login, "passwd" : password, "SI" : "Sign in", "type" : "11", "PPFT" : PPFT, "PPSX" : str(PPSX), "idsbho" : "1", "LoginOptions" : "3", "NewUser" : "1", "i1" : "0", # ClientUserSaved "i2" : "1", # ClientMode "i3" : str(clt), # ClientLoginTime "i4" : "0", # ClientExplore "i7" : "0", # ClientOTPRequest "i12" : "1", # LoginUsedSSL "i13" : "0", # ClientUsedKMSI "i14" : str(renderTime), # RenderCompleteTime "i15" : str(resourcesTime), # RenderCompleteTime "i16" : str(PLT), # PLT "i17" : "0", # SRSFailed "i18" : "__Login_Strings|1,__Login_Core|1," # SRSSuccess }) # get Passport page request = urllib2.Request(url, postFields, self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # Checking for bad usernames and password helpers.errorOnText(page, 'That password is incorrect.', 'Authentication has not been passed: Invalid password') helpers.errorOnText(page, "That Microsoft account doesn\\'t exist", 'Authentication has not been passed: Invalid username') # check if there is a new terms of use helpers.errorOnText(page, '//account.live.com/tou/accrue', 'Please log in (log out first if necessary) through a browser and accept the Terms Of Use') # get auth form url and contents authForm = re.search(r"<form.+action=\"([^\"]+)\".*?>(.+)?</form>", page) if authForm == None: filename = helpers.dumpErrorPage(page) raise AuthenticationError("Could not find login form:\ncheck " + filename + " file for more information") parser = HTMLFormInputsParser() parser.feed(authForm.group(2).decode("utf-8")) parser.close() postFields = urllib.urlencode(parser.inputs) # finish passing authentication url = authForm.group(1) request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header("Origin", "https://login.live.com") with self.opener.open(request) as response: page = helpers.getResponseBody(response) url = bingCommon.BING_URL request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header("Referer", authForm.group(1)) with self.opener.open(request) as response: url = response.geturl() # if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report if url.find(bingCommon.BING_URL) == -1: try: filename = helpers.dumpErrorPage(helpers.getResponseBody(response)) s = "check " + filename + " file for more information" except IOError: s = "no further information could be provided - failed to write a file into " + \ helpers.RESULTS_DIR + " subfolder" raise AuthenticationError("Authentication has not been passed:\n" + s)
def __processSearch(self, reward, verbose): """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content"' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content"' IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx" IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"') IG_SEARCH_RESULTS_PATTERN = re.compile(r'<ol\s.*?id="b_results"(.+?)</ol>') IG_SEARCHS_PATTERN = re.compile(r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"') res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bfp.Reward.Type.Col.INDEX # get a set of queries from today's Bing! history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url = url, headers = self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description) rewardsCount = int(matches.group(1)) rewardCost = int(matches.group(2)) maxRewardsCount = int(matches.group(4)) searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress searchesCount -= reward.progressCurrent * rewardCost headers = self.httpHeaders if reward.tp == bfp.Reward.Type.SEARCH_PC: headers["User-Agent"] = self.userAgents.pc searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt) print print "Running PC searches" print elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt) print print "Running mobile searches" print else: res.isError = True res.message = "Don't know how to process this search" return res if verbose: print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"])) print # Import the query generator try: qg = importlib.import_module(self.queryGenerator, package=None) queryGenerator = qg.queryGenerator(self) except ImportError: raise TypeError("{0} is not a module".format(self.queryGenerator)) # generate a set of queries to run queries = queryGenerator.generateQueries(searchesCount, history) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated!" print "Requested:", searchesCount print "Generated:", len(queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url = url, headers = bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 # randomly open a link if self.openLinkChance > random.random(): # get IG number ig_number = IG_NUMBER_PATTERN.search(page) if ig_number != None: ig_number = ig_number.group(1) # get search results ig_results = IG_SEARCH_RESULTS_PATTERN.search(page) if ig_results != None: ig_results = ig_results.group(1) # seperate search results ig_searches = IG_SEARCHS_PATTERN.findall(ig_results) # make sure we have at least 1 search if len(ig_searches) > 0: # get a random link to open ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1) ig_link_num = random.randint(0, ig_max_rand) # number of the link we will use ig_link = "{0}?IG={1}&{2}".format( IG_PING_LINK, urllib.quote_plus(ig_number), urllib.quote_plus(ig_searches[ig_link_num][1]) ) # sleep a reasonable amount of time before clicking the link # use defaults to save space in config t = random.uniform(0.75, 3.0) time.sleep(t) # open the random link request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS) request.headers["Referer"] = response.url self.opener.open(request) if verbose: print("Followed Link {}".format(ig_link_num + 1)) else: filename = helpers.dumpErrorPage(page) print "Warning! No searches were found on search results page" print "Check {0} file for more information".format(filename) else: filename = helpers.dumpErrorPage(page) print "Warning! Could not find search result IG number" print "Check {0} file for more information".format(filename) i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed" else: res.message = "All " + str(successfullQueries) + " requests were successfully processed" # reset header to pc so pc pages return in getting life time points headers["User-Agent"] = self.userAgents.pc return res
def __authenticateFacebook(self, login, password): """ Authenticates a user on bing.com with his/her Facebook account. throws AuthenticationError if authentication can not be passed throws HTMLParser.HTMLParseError throws urllib2.HTTPError if the server couldn't fulfill the request throws urllib2.URLError if failed to reach the server """ BING_REQUEST_PERMISSIONS = "http://www.bing.com/fd/auth/signin?action=interactive&provider=facebook&return_url=http%3a%2f%2fwww.bing.com%2f&src=EXPLICIT&perms=read_stream%2cuser_photos%2cfriends_photos&sig=" # print "Requesting bing.com" # request http://www.bing.com request = urllib2.Request(url=bingCommon.BING_URL, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # get connection URL for provider Facebook s = page.index('"Facebook":"') s += len('"Facebook":"') e = page.index('"', s) # the URL contains escape characters (http:// is http\x3a\x2f\x2f, for example) # this needs to be decoded back to normal URL # see (http://docs.python.org/2/library/codecs.html#python-specific-encodings) url = page[s:e].decode('string-escape') s = url.index('sig=') s += len('sig=') e = url.find('&', s) if e == -1: e = len(url) url = BING_REQUEST_PERMISSIONS + url[s:e] # print "Now requesting facebook authentication page" # request FACEBOOK_CONNECT_ORIGINAL_URL request = urllib2.Request(url=url, headers=self.httpHeaders) request.add_header("Referer", bingCommon.BING_URL) with self.opener.open(request) as response: referer = response.geturl() # get Facebook authenctication form action url page = helpers.getResponseBody(response) s = page.index('<form id="login_form"') s = page.index('action="', s) s += len('action="') e = page.index('"', s) url = page[s:e] # relative url? add url from the previous response if url[0:1] == "/": url = referer + url # find all html elements which need to be sent to the server s = page.index('>', s) s += 1 e = page.index('</form>') parser = HTMLFormInputsParser() parser.feed(page[s:e].decode("utf-8")) parser.close() parser.inputs["email"] = login parser.inputs["pass"] = password # print "Now passing facebook authentication" # pass facebook authentication postFields = urllib.urlencode(parser.inputs) request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header("Referer", referer) with self.opener.open(request) as response: url = response.geturl() # if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report if url.find(bingCommon.BING_URL) == -1: try: filename = helpers.dumpErrorPage( helpers.getResponseBody(response)) s = "check " + filename + " file for more information" except IOError: s = "no further information could be provided - failed to write a file into " + \ helpers.RESULTS_DIR + " subfolder" raise AuthenticationError( "Authentication has not been passed:\n" + s)
def __authenticateLive(self, login, password): """ Authenticates a user on bing.com with his/her Live account. throws AuthenticationError if authentication can not be passed throws urllib2.HTTPError if the server couldn't fulfill the request throws urllib2.URLError if failed to reach the server """ # print "Requesting bing.com" # request http://www.bing.com request = urllib2.Request(url=bingCommon.BING_URL, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # get connection URL for provider Live s = page.index('"WindowsLiveId":"') s += len('"WindowsLiveId":"') e = page.index('"', s) # the URL contains escape characters (http:// is http\x3a\x2f\x2f, for example) # this needs to be decoded back to normal URL # see (http://docs.python.org/2/library/codecs.html#python-specific-encodings) url = page[s:e].decode('string-escape') request = urllib2.Request(url=url, headers=self.httpHeaders) request.add_header("Referer", bingCommon.BING_URL) with self.opener.open(request) as response: referer = response.geturl() # get Facebook authenctication form action url page = helpers.getResponseBody(response) # get PPFT parameter s = page.index("sFTTag") s = page.index('value="', s) s += len('value="') e = page.index('"', s) PPFT = page[s:e] # get PPSX parameter s = page.index(",g:") s += len(",g:") e = page.index(",", s) PPSX = page[s:e] if PPSX[0] == "'": PPSX = PPSX[1:-1] # get sso parameter s = page.index(",W:") s += len(",W:") e = page.index(",", s) sso = page[s:e] # generate ClientLoginTime clt = 20000 + int(random.uniform(0, 1000)) # generate RenderCompleteTime renderTime = 130 + int(random.uniform(0, 100)) # generate ResourcesCompleteTime resourcesTime = renderTime + int(random.uniform(2, 5)) # generate ResourcesCompleteTime PLT = 870 + int(random.uniform(0, 250)) # get url to post data to s = page.index(",urlPost:'") s += len(",urlPost:'") e = page.index("'", s) url = page[s:e] postFields = urllib.urlencode({ "login": login, "passwd": password, "SI": "Sign in", "type": "11", "PPFT": PPFT, "PPSX": str(PPSX), "idsbho": "1", "LoginOptions": "3", "sso": sso, "NewUser": "******", "i1": "0", # ClientUserSaved "i2": "1", # ClientMode "i3": str(clt), # ClientLoginTime "i4": "0", # ClientExplore "i7": "0", # ClientOTPRequest "i12": "1", # LoginUsedSSL "i13": "0", # ClientUsedKMSI "i14": str(renderTime), # RenderCompleteTime "i15": str(resourcesTime), # RenderCompleteTime "i16": str(PLT), # PLT "i17": "0", # SRSFailed "i18": "__Login_Strings|1,__Login_Core|1," # SRSSuccess }) # get Passport page request = urllib2.Request(url, postFields, self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) s = page.index("<form ") e = page.index("</form>", s) e += len("</form>") parser = HTMLFormInputsParser() parser.feed(page[s:e].decode("utf-8")) parser.close() postFields = urllib.urlencode(parser.inputs) # finish passing authentication url = "http://www.bing.com/Passport.aspx?requrl=http%3a%2f%2fwww.bing.com%2f&wa=wsignin1.0" request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header("Origin", "https://login.live.com") with self.opener.open(request) as response: page = helpers.getResponseBody(response) url = bingCommon.BING_URL request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header( "Referer", "http://www.bing.com/Passport.aspx?requrl=http%3a%2f%2fwww.bing.com%2f&wa=wsignin1.0" ) with self.opener.open(request) as response: url = response.geturl() # if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report if url.find(bingCommon.BING_URL) == -1: try: filename = helpers.dumpErrorPage( helpers.getResponseBody(response)) s = "check " + filename + " file for more information" except IOError: s = "no further information could be provided - failed to write a file into " + \ helpers.RESULTS_DIR + " subfolder" raise AuthenticationError( "Authentication has not been passed:\n" + s)
def __authenticateLive(self, login, password): """ Authenticates a user on bing.com with his/her Live account. throws AuthenticationError if authentication can not be passed throws urllib2.HTTPError if the server couldn't fulfill the request throws urllib2.URLError if failed to reach the server """ # print "Requesting bing.com" # request http://www.bing.com request = urllib2.Request(url = bingCommon.BING_URL, headers = bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # get connection URL for provider Live s = page.index('"WindowsLiveId":"') s += len('"WindowsLiveId":"') e = page.index('"', s) url = page[s:e] request = urllib2.Request(url = url, headers = bingCommon.HEADERS) request.add_header("Referer", bingCommon.BING_URL) with self.opener.open(request) as response: referer = response.geturl() # get Facebook authenctication form action url page = helpers.getResponseBody(response) # get PPFT parameter s = page.index("sFTTag") s = page.index('value="', s) s += len('value="') e = page.index('"', s) PPFT = page[s:e] # get PPSX parameter s = page.index(",g:'") s += len(",g:'") e = page.index("'", s) PPSX = page[s:e] # get sso parameter s = page.index(",W:") s += len(",W:") e = page.index(",", s) sso = int(page[s:e]) # generate ClientLoginTime clt = 20000 + int(random.uniform(0, 1000)) # generate RenderCompleteTime renderTime = 130 + int(random.uniform(0, 100)) # generate ResourcesCompleteTime resourcesTime = renderTime + int(random.uniform(2, 5)) # generate ResourcesCompleteTime PLT = 870 + int(random.uniform(0, 250)) # get url to post data to s = page.index(",urlPost:'") s += len(",urlPost:'") e = page.index("'", s) url = page[s:e] postFields = urllib.urlencode({ "login" : login, "passwd" : password, "SI" : "Sign in", "type" : "11", "PPFT" : PPFT, "PPSX" : PPSX, "idsbho" : "1", "LoginOptions" : "3", "sso" : str(sso), "NewUser" : "1", "i1" : "0", # ClientUserSaved "i2" : "1", # ClientMode "i3" : str(clt), # ClientLoginTime "i4" : "0", # ClientExplore "i7" : "0", # ClientOTPRequest "i12" : "1", # LoginUsedSSL "i13" : "0", # ClientUsedKMSI "i14" : str(renderTime), # RenderCompleteTime "i15" : str(resourcesTime), # RenderCompleteTime "i16" : str(PLT), # PLT "i17" : "0", # SRSFailed "i18" : "__Login_Strings|1,__Login_Core|1," # SRSSuccess }) # get Passport page request = urllib2.Request(url, postFields, bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) s = page.index("<form ") e = page.index("</form>", s) e += len("</form>") parser = HTMLFormInputsParser() parser.feed(page[s:e].decode("utf-8")) parser.close() postFields = urllib.urlencode(parser.inputs) # finish passing authentication url = "http://www.bing.com/Passport.aspx?requrl=http%3a%2f%2fwww.bing.com%2f&wa=wsignin1.0" request = urllib2.Request(url, postFields, bingCommon.HEADERS) request.add_header("Origin", "https://login.live.com") with self.opener.open(request) as response: page = helpers.getResponseBody(response) url = bingCommon.BING_URL request = urllib2.Request(url, postFields, bingCommon.HEADERS) request.add_header("Referer", "http://www.bing.com/Passport.aspx?requrl=http%3a%2f%2fwww.bing.com%2f&wa=wsignin1.0") with self.opener.open(request) as response: url = response.geturl() # if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report if url.find(bingCommon.BING_URL) == -1: try: filename = helpers.dumpErrorPage(helpers.getResponseBody(response)) s = "check " + filename + " file for more information" except IOError: s = "no further information could be provided - failed to write a file into " + \ helpers.RESULTS_DIR + " subfolder" raise AuthenticationError("Authentication has not been passed:\n" + s)
def __processSearch(self, reward, verbose): """Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">' IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx" IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"') IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"') res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bdp.Reward.Type.Col.INDEX # get a set of queries from today's Bing history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url = url, headers = self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matchesMobile = False matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description) #Mobile description changed, so check that one too if matches is None: matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE_MOBILE.search(reward.description) matchesMobile = True if matches is None: print "No RegEx matches found for this search and earn" res.isError = True res.message = "No RegEx matches found for this search and earn" return res maxRewardsCount = int(matches.group(1)) rewardsCount = int(matches.group(2)) rewardCost = 1 # Looks like it's now always X points per one search searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress # reward.progressCurrent is now returning current points, not current searches # so divide it by points per search (rewardsCount) to get correct search count needed searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount if matchesMobile == True: #new mobile search description gives total search count + points per search for edge/non-edge edgeValue = int(matches.group(1)) nonEdgeValue = int(matches.group(2)) searchesCount = int(matches.group(3)) #apparently ios uses EdgiOS, so we only check the first 3 letters not the full word 'edge' if self.userAgents.mobile.lower().find("edg") != -1: #we are searching on edge so points go to 200 searchesCount -= reward.progressCurrent / edgeValue else: #non-edge search so 100 is the max searchesCount -= reward.progressCurrent / nonEdgeValue headers = self.httpHeaders if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN: headers["User-Agent"] = self.userAgents.pc searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt) print print "Running PC searches" print elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt) print print "Running mobile searches" print else: res.isError = True res.message = "Don't know how to process this search" return res if verbose: print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"])) print # Import the query generator try: qg = importlib.import_module(self.queryGenerator, package=None) queryGenerator = qg.queryGenerator(self) except ImportError: raise TypeError("{0} is not a module".format(self.queryGenerator)) # generate a set of queries to run queries = queryGenerator.generateQueries(searchesCount, history) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated!" print "Requested:", searchesCount print "Generated:", len(queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing ;) ) t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url = url, headers = bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 # randomly open a link if self.openLinkChance > random.random(): # get IG number ig_number = IG_NUMBER_PATTERN.search(page) if ig_number is not None: ig_number = ig_number.group(1) ig_searches = IG_SEARCHES_PATTERN.findall(page) # make sure we have at least 1 search if len(ig_searches) > 0: # get a random link to open ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1) # number of the link we will use ig_link_num = random.randint(0, ig_max_rand) ig_link = "{0}?IG={1}&{2}".format( IG_PING_LINK, urllib.quote_plus(ig_number), urllib.quote_plus(ig_searches[ig_link_num][1]) ) # sleep a reasonable amount of time before clicking the link # use defaults to save space in config t = random.uniform(0.75, 3.0) time.sleep(t) # open the random link request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS) request.headers["Referer"] = response.url self.opener.open(request) if verbose: print("Followed Link {}".format(ig_link_num + 1)) else: filename = helpers.dumpErrorPage(page) print "Warning! No searches were found on search results page" print "Check {0} file for more information".format(filename) else: filename = helpers.dumpErrorPage(page) print "Warning! Could not find search result IG number" print "Check {0} file for more information".format(filename) i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed" else: res.message = "All " + str(successfullQueries) + " requests were successfully processed" # reset header to pc so pc pages return in getting life time points headers["User-Agent"] = self.userAgents.pc return res
def __authenticateLive(self, login, password): """ Authenticates a user on bing.com with his/her Live account. throws AuthenticationError if authentication can not be passed throws urllib2.HTTPError if the server couldn't fulfill the request throws urllib2.URLError if failed to reach the server """ # print "Requesting bing.com" # request http://www.bing.com request = urllib2.Request(url=bingCommon.BING_URL, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # get connection URL for provider Live s = page.index('"WindowsLiveId":"') s += len('"WindowsLiveId":"') e = page.index('"', s) url = BingAuth._escapeString(page[s:e]) request = urllib2.Request(url=url, headers=self.httpHeaders) request.add_header("Referer", bingCommon.BING_URL) with self.opener.open(request) as response: referer = response.geturl() # get Facebook authenctication form action url page = helpers.getResponseBody(response) # get PPFT parameter s = page.index("sFTTag") s = page.index('value="', s) s += len('value="') e = page.index('"', s) PPFT = page[s:e] # get PPSX parameter ppsxSearch = self.ppsxValue.search(page) if ppsxSearch == None: raise AuthenticationError( "Could not find variable 't' on Live login page") PPSX = ppsxSearch.group(1) # generate ClientLoginTime clt = 20000 + int(random.uniform(0, 1000)) # get url to post data to s = page.index(",urlPost:'") s += len(",urlPost:'") e = page.index("'", s) url = page[s:e] timestamp = int(round(time.time() * 1000)) # TODO: randomize times a bit? i16 = json.dumps({ "navigationStart": timestamp, "unloadEventStart": timestamp + 209, "unloadEventEnd": timestamp + 210, "redirectStart": 0, "redirectEnd": 0, "fetchStart": timestamp + 73, "domainLookupStart": timestamp + 73, "domainLookupEnd": timestamp + 130, "connectStart": timestamp + 130, "connectEnd": timestamp + 130, "secureConnectionStart": timestamp + 210, "requestStart": timestamp + 183, "responseStart": timestamp + 205, "responseEnd": timestamp + 205, "domLoading": timestamp + 208, "domInteractive": timestamp + 406, "domContentLoadedEventStart": timestamp + 420, "domContentLoadedEventEnd": timestamp + 420, "domComplete": timestamp + 422, "loadEventStart": timestamp + 422, "loadEventEnd": 0 }) postFields = urllib.urlencode({ "loginfmt": login, "login": login, "passwd": password, "type": "11", "PPFT": PPFT, "PPSX": str(PPSX), "LoginOptions": "3", "FoundMSAs": "", "fspost": "0", "NewUser": "******", "i2": "1", # ClientMode "i13": "0", # ClientUsedKMSI "i16": i16, "i19": str(clt), # ClientLoginTime "i21": "0", "i22": "0", "i17": "0", # SRSFailed "i18": "__DefaultLogin_Strings|1,__DefaultLogin_Core|1," # SRSSuccess }) # get Passport page request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header("Referer", referer) with self.opener.open(request) as response: referer = response.geturl() page = helpers.getResponseBody(response) # Checking for bad usernames and password helpers.errorOnText( page, 'That password is incorrect.', 'Authentication has not been passed: Invalid password') helpers.errorOnText( page, "That Microsoft account doesn\\'t exist", 'Authentication has not been passed: Invalid username') # check if there is a new terms of use helpers.errorOnText( page, '//account.live.com/tou/accrue', 'Please log in (log out first if necessary) through a browser and accept the Terms Of Use' ) contSubmitUrl = self.formAction.search(page) if contSubmitUrl == None: raise AuthenticationError( "Could not find form action for continue page") url = contSubmitUrl.group(1) # get all form inputs formFields = self.inputNameValue.findall(page) postFields = {} for field in formFields: postFields[field[0]] = field[1] postFields = urllib.urlencode(postFields) # submit continue page request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header("Referer", referer) with self.opener.open(request) as response: referer = response.geturl() page = helpers.getResponseBody(response) request = urllib2.Request(url=bingCommon.BING_URL, headers=self.httpHeaders) request.add_header("Referer", referer) with self.opener.open(request) as response: referer = response.geturl() # if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report if referer.find(bingCommon.BING_URL) == -1: try: filename = helpers.dumpErrorPage( helpers.getResponseBody(response)) s = "check " + filename + " file for more information" except IOError: s = "no further information could be provided - failed to write a file into " + \ helpers.RESULTS_DIR + " subfolder" raise AuthenticationError( "Authentication has not been passed:\n" + s)
def __authenticateLive(self, login, password): """ Authenticates a user on bing.com with his/her Live account. throws AuthenticationError if authentication can not be passed throws urllib2.HTTPError if the server couldn't fulfill the request throws urllib2.URLError if failed to reach the server """ # print "Requesting bing.com" # request http://www.bing.com request = urllib2.Request(url = bingCommon.BING_URL, headers = self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # get connection URL for provider Live s = page.index('"WindowsLiveId":"') s += len('"WindowsLiveId":"') e = page.index('"', s) url = BingAuth._escapeString(page[s:e]) request = urllib2.Request(url = url, headers = self.httpHeaders) request.add_header("Referer", bingCommon.BING_URL) with self.opener.open(request) as response: referer = response.geturl() # get Facebook authenctication form action url page = helpers.getResponseBody(response) # get PPFT parameter s = page.index("sFTTag") s = page.index('value="', s) s += len('value="') e = page.index('"', s) PPFT = page[s:e] # get PPSX parameter ppsxSearch = self.ppsxValue.search(page) if ppsxSearch == None: raise AuthenticationError("Could not find variable 't' on Live login page") PPSX = ppsxSearch.group(1) # generate ClientLoginTime clt = 20000 + int(random.uniform(0, 1000)) # get url to post data to s = page.index(",urlPost:'") s += len(",urlPost:'") e = page.index("'", s) url = page[s:e] timestamp = int(round(time.time() * 1000)) # TODO: randomize times a bit? i16 = json.dumps({ "navigationStart": timestamp, "unloadEventStart": timestamp + 209, "unloadEventEnd": timestamp + 210, "redirectStart": 0, "redirectEnd": 0, "fetchStart": timestamp + 73, "domainLookupStart": timestamp + 73, "domainLookupEnd": timestamp + 130, "connectStart": timestamp + 130, "connectEnd": timestamp + 130, "secureConnectionStart": timestamp + 210, "requestStart": timestamp + 183, "responseStart": timestamp + 205, "responseEnd": timestamp + 205, "domLoading": timestamp + 208, "domInteractive": timestamp + 406, "domContentLoadedEventStart": timestamp + 420, "domContentLoadedEventEnd": timestamp + 420, "domComplete": timestamp + 422, "loadEventStart": timestamp + 422, "loadEventEnd": 0 }) postFields = urllib.urlencode({ "loginfmt" : login, "login" : login, "passwd" : password, "type" : "11", "PPFT" : PPFT, "PPSX" : str(PPSX), "LoginOptions" : "3", "FoundMSAs" : "", "fspost" : "0", "NewUser" : "1", "i2" : "1", # ClientMode "i13" : "0", # ClientUsedKMSI "i16" : i16, "i19" : str(clt), # ClientLoginTime "i21" : "0", "i22" : "0", "i17" : "0", # SRSFailed "i18" : "__DefaultLogin_Strings|1,__DefaultLogin_Core|1," # SRSSuccess }) # get Passport page request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header("Referer", referer) with self.opener.open(request) as response: referer = response.geturl() page = helpers.getResponseBody(response) # Checking for bad usernames and password helpers.errorOnText(page, 'That password is incorrect.', 'Authentication has not been passed: Invalid password') helpers.errorOnText(page, "That Microsoft account doesn\\'t exist", 'Authentication has not been passed: Invalid username') # check if there is a new terms of use helpers.errorOnText(page, '//account.live.com/tou/accrue', 'Please log in (log out first if necessary) through a browser and accept the Terms Of Use') contSubmitUrl = self.formAction.search(page) if contSubmitUrl == None: raise AuthenticationError("Could not find form action for continue page") url = contSubmitUrl.group(1) # get all form inputs formFields = self.inputNameValue.findall(page) postFields = {} for field in formFields: postFields[field[0]] = field[1] postFields = urllib.urlencode(postFields) # submit continue page request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header("Referer", referer) with self.opener.open(request) as response: referer = response.geturl() page = helpers.getResponseBody(response) request = urllib2.Request(url = bingCommon.BING_URL, headers = self.httpHeaders) request.add_header("Referer", referer) with self.opener.open(request) as response: referer = response.geturl() # if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report if referer.find(bingCommon.BING_URL) == -1: try: filename = helpers.dumpErrorPage(helpers.getResponseBody(response)) s = "check " + filename + " file for more information" except IOError: s = "no further information could be provided - failed to write a file into " + \ helpers.RESULTS_DIR + " subfolder" raise AuthenticationError("Authentication has not been passed:\n" + s)
def __processSearch(self, reward, verbose): """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">' IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx" IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"') IG_SEARCH_RESULTS_PATTERN = re.compile( r'<ol\s.*?id="b_results"(.+?)</ol>') IG_SEARCHS_PATTERN = re.compile( r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"') res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bfp.Reward.Type.Col.INDEX # get a set of queries from today's Bing! history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url=url, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search( reward.description) rewardsCount = int(matches.group(1)) rewardCost = int(matches.group(2)) maxRewardsCount = int(matches.group(4)) searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress searchesCount -= reward.progressCurrent * rewardCost headers = self.httpHeaders if reward.tp == bfp.Reward.Type.SEARCH_PC: headers["User-Agent"] = self.userAgents.pc searchesCount += self.addSearchesDesktop + random.randint( 0, self.addSearchesDesktopSalt) print print "Running PC searches" print elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile searchesCount += self.addSearchesMobile + random.randint( 0, self.addSearchesMobileSalt) print print "Running mobile searches" print else: res.isError = True res.message = "Don't know how to process this search" return res # Import the query generator try: qg = importlib.import_module(self.queryGenerator, package=None) queryGenerator = qg.queryGenerator(self) except ImportError: raise TypeError("{0} is not a module".format(self.queryGenerator)) # generate a set of queries to run queries = queryGenerator.generateQueries(searchesCount, history) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated!" print "Requested:", searchesCount print "Generated:", len(queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform( 0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url=url, headers=bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 # randomly open a link if self.openLinkChance > random.random(): # get IG number ig_number = IG_NUMBER_PATTERN.search(page) if ig_number != None: ig_number = ig_number.group(1) # get search results ig_results = IG_SEARCH_RESULTS_PATTERN.search(page) if ig_results != None: ig_results = ig_results.group(1) # seperate search results ig_searches = IG_SEARCHS_PATTERN.findall( ig_results) # get a random link to open ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1) ig_link_num = random.randint(0, ig_max_rand) # number of the link we will use ig_link = "{0}?IG={1}&{2}".format( IG_PING_LINK, urllib.quote_plus(ig_number), urllib.quote_plus(ig_searches[ig_link_num][1])) # sleep a reasonable amount of time before clicking the link # use defaults to save space in config t = random.uniform(0.75, 3.0) time.sleep(t) # open the random link request = urllib2.Request( url=ig_link, headers=bingCommon.HEADERS) request.headers["Referer"] = response.url self.opener.open(request) if verbose: print("Followed Link {}".format(ig_link_num + 1)) else: filename = helpers.dumpErrorPage(page) print "Warning! Could not find search result IG number" print "Check {0} file for more information".format( filename) i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str( searchesCount) + " requests were successfully processed" else: res.message = "All " + str( successfullQueries) + " requests were successfully processed" # reset header to pc so pc pages return in getting life time points headers["User-Agent"] = self.userAgents.pc return res
def __processSearch(self, reward): """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">' res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bfp.Reward.Type.Col.INDEX # get a set of queries from today's Bing! history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url = url, headers = self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description) rewardsCount = int(matches.group(1)) rewardCost = int(matches.group(2)) maxRewardsCount = int(matches.group(4)) searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress searchesCount -= reward.progressCurrent * rewardCost headers = self.httpHeaders if reward.tp == bfp.Reward.Type.SEARCH_PC: headers["User-Agent"] = self.userAgents.pc print print "Running PC searches" print elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile print print "Running mobile searches" print else: res.isError = True res.message = "Don't know how to process this search" return res request = urllib2.Request(url = BING_NEWS_URL, headers = headers) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # generate a set of queries to run bingQueriesGenerator = BingQueriesGenerator(searchesCount, history, reward.tp) queries = bingQueriesGenerator.parseBingNews(page) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated !" print "Requested:", searchesCount print "Generated:", len(bingQueriesGenerator.queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesInterval) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query) print "%s - %2d/%2d - Requesting: %s" % (helpers.getLoggingTime(), i, totalQueries, url) request = urllib2.Request(url = url, headers = bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = ( reward.tp == bfp.Reward.Type.SEARCH_PC and page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 ) \ or ( reward.tp == bfp.Reward.Type.SEARCH_MOBILE and page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 ) if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed" else: res.message = "All " + str(successfullQueries) + " requests were successfully processed" return res
def __processSearch(self, reward, verbose): """Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">' IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx" IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"') IG_SEARCHES_PATTERN = re.compile( r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"' ) res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bdp.Reward.Type.Col.INDEX # get a set of queries from today's Bing! history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url=url, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search( reward.description) if matches is None: print "No RegEx matches found for this search and earn" res.isError = True res.message = "No RegEx matches found for this search and earn" return res maxRewardsCount = int(matches.group(1)) rewardsCount = int(matches.group(2)) rewardCost = 1 # Looks like it's now always X points per one search searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress # reward.progressCurrent is now returning current points, not current searches # so divide it by points per search (rewardsCount) to get correct search count needed searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount headers = self.httpHeaders if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN: headers["User-Agent"] = self.userAgents.pc searchesCount += self.addSearchesDesktop + random.randint( 0, self.addSearchesDesktopSalt) print print "Running PC searches" print elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile searchesCount += self.addSearchesMobile + random.randint( 0, self.addSearchesMobileSalt) print print "Running mobile searches" print if verbose: print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"])) print # Import the query generator try: qg = importlib.import_module(self.queryGenerator, package=None) queryGenerator = qg.queryGenerator(self) except ImportError: raise TypeError("{0} is not a module".format(self.queryGenerator)) # generate a set of queries to run queries = queryGenerator.generateQueries(searchesCount, history) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated!\nRequested: {}\nGenerated: {}\n"\ .format(searchesCount, len(queries)) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform( 0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url=url, headers=bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: print "Warning! Query:\n\t {}\n returned no results, check file for more information {}".format( query, helpers.dumpErrorPage(page)) else: successfullQueries += 1 self.randomClick(IG_NUMBER_PATTERN, IG_PING_LINK, IG_SEARCHES_PATTERN, page, response, verbose) i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str( searchesCount) + " requests were successfully processed" else: res.message = "All " + str( successfullQueries) + " requests were successfully processed" # reset header to pc so pc pages return in getting life time points headers["User-Agent"] = self.userAgents.pc return res
time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url = url, headers = bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 # randomly open a link if self.openLinkChance > random.random(): # get IG number ig_number = IG_NUMBER_PATTERN.search(page) if ig_number is not None: ig_number = ig_number.group(1) ig_searches = IG_SEARCHES_PATTERN.findall(page) # make sure we have at least 1 search
def __processSearch(self, reward): """Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult""" BING_QUERY_URL = 'http://www.bing.com/search?q=' BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">' BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">' res = self.RewardResult(reward) if reward.isAchieved(): res.message = "This reward has been already achieved" return res indCol = bfp.Reward.Type.Col.INDEX # get a set of queries from today's Bing! history url = bingHistory.getBingHistoryTodayURL() request = urllib2.Request(url=url, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) history = bingHistory.parse(page) # find out how many searches need to be performed matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search( reward.description) rewardsCount = int(matches.group(1)) rewardCost = int(matches.group(2)) maxRewardsCount = int(matches.group(4)) searchesCount = maxRewardsCount * rewardCost / rewardsCount # adjust to the current progress searchesCount -= reward.progressCurrent * rewardCost headers = self.httpHeaders if reward.tp == bfp.Reward.Type.SEARCH_PC: headers["User-Agent"] = self.userAgents.pc print print "Running PC searches" print elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE: headers["User-Agent"] = self.userAgents.mobile print print "Running mobile searches" print else: res.isError = True res.message = "Don't know how to process this search" return res # Import the query generator try: qg = importlib.import_module(self.queryGenerator, package=None) queryGenerator = qg.queryGenerator(self) except ImportError: raise TypeError("{0} is not a module".format(self.queryGenerator)) # generate a set of queries to run queries = queryGenerator.generateQueries(searchesCount, history) if len(queries) < searchesCount: print "Warning: not enough queries to run were generated!" print "Requested:", searchesCount print "Generated:", len(queries) successfullQueries = 0 i = 1 totalQueries = len(queries) for query in queries: if i > 1: # sleep some time between queries (don't worry Bing! ;) ) t = self.betweenQueriesInterval + random.uniform( 0, self.betweenQueriesSalt) time.sleep(t) url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8')) print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query) request = urllib2.Request(url=url, headers=bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # check for the successfull marker found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \ or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 if not found: filename = helpers.dumpErrorPage(page) print "Warning! Query:" print "\t" + query print "returned no results, check " + filename + " file for more information" else: successfullQueries += 1 i += 1 if successfullQueries < searchesCount: res.message = str(successfullQueries) + " out of " + str( searchesCount) + " requests were successfully processed" else: res.message = "All " + str( successfullQueries) + " requests were successfully processed" # reset header to pc so pc pages return in getting life time points headers["User-Agent"] = self.userAgents.pc return res
def __authenticateFacebook(self, login, password): """ Authenticates a user on bing.com with his/her Facebook account. throws AuthenticationError if authentication can not be passed throws HTMLParser.HTMLParseError throws urllib2.HTTPError if the server couldn't fulfill the request throws urllib2.URLError if failed to reach the server """ BING_REQUEST_PERMISSIONS = "http://www.bing.com/fd/auth/signin?action=interactive&provider=facebook&return_url=http%3a%2f%2fwww.bing.com%2f&src=EXPLICIT&perms=read_stream%2cuser_photos%2cfriends_photos&sig=" # print "Requesting bing.com" # request http://www.bing.com request = urllib2.Request(url = bingCommon.BING_URL, headers = bingCommon.HEADERS) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # get connection URL for provider Facebook s = page.index('"Facebook":"') s += len('"Facebook":"') e = page.index('"', s) url = page[s:e] s = url.index('sig=') s += len('sig=') e = url.find('&', s) if e == -1: e = len(url) url = BING_REQUEST_PERMISSIONS + url[s:e] # print "Now requesting facebook authentication page" # request FACEBOOK_CONNECT_ORIGINAL_URL request = urllib2.Request(url = url, headers = bingCommon.HEADERS) request.add_header("Referer", bingCommon.BING_URL) with self.opener.open(request) as response: referer = response.geturl() # get Facebook authenctication form action url page = helpers.getResponseBody(response) s = page.index('<form id="login_form"') s = page.index('action="', s) s += len('action="') e = page.index('"', s) url = page[s:e] # relative url? add url from the previous response if url[0:1] == "/": url = referer + url # find all html elements which need to be sent to the server s = page.index('>', s) s += 1 e = page.index('</form>') parser = HTMLFormInputsParser() parser.feed(page[s:e].decode("utf-8")) parser.close() parser.inputs["email"] = login parser.inputs["pass"] = password # print "Now passing facebook authentication" # pass facebook authentication postFields = urllib.urlencode(parser.inputs) request = urllib2.Request(url, postFields, bingCommon.HEADERS) request.add_header("Referer", referer) with self.opener.open(request) as response: url = response.geturl() # if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report if url.find(bingCommon.BING_URL) == -1: try: filename = helpers.dumpErrorPage(helpers.getResponseBody(response)) s = "check " + filename + " file for more information" except IOError: s = "no further information could be provided - failed to write a file into " + \ helpers.RESULTS_DIR + " subfolder" raise AuthenticationError("Authentication has not been passed:\n" + s)
def __authenticateLive(self, login, password): """ Authenticates a user on bing.com with his/her Live account. throws AuthenticationError if authentication can not be passed throws urllib2.HTTPError if the server couldn't fulfill the request throws urllib2.URLError if failed to reach the server """ # print "Requesting bing.com" # request http://www.bing.com request = urllib2.Request(url=bingCommon.BING_URL, headers=self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # get connection URL for provider Live s = page.index('"WindowsLiveId":"') s += len('"WindowsLiveId":"') e = page.index('"', s) url = BingAuth._escapeString(page[s:e]) request = urllib2.Request(url=url, headers=self.httpHeaders) request.add_header("Referer", bingCommon.BING_URL) with self.opener.open(request) as response: referer = response.geturl() # get Facebook authenctication form action url page = helpers.getResponseBody(response) # get PPFT parameter s = page.index("sFTTag") s = page.index('value="', s) s += len('value="') e = page.index('"', s) PPFT = page[s:e] # get PPSX parameter ppsxParam = "" if ",M:" in page: ppsxParam = ",M:" elif ",g:" in page: ppsxParam = ",g:" elif ",j:" in page: ppsxParam = ",j:" else: ppsxParam = ",d:" s = page.index(ppsxParam) s += len(ppsxParam) e = page.index(",", s) PPSX = page[s:e] if PPSX[0] == "'": PPSX = PPSX[1:-1] # generate ClientLoginTime clt = 20000 + int(random.uniform(0, 1000)) # generate RenderCompleteTime renderTime = 130 + int(random.uniform(0, 100)) # generate ResourcesCompleteTime resourcesTime = renderTime + int(random.uniform(2, 5)) # generate ResourcesCompleteTime PLT = 870 + int(random.uniform(0, 250)) # get url to post data to s = page.index(",urlPost:'") s += len(",urlPost:'") e = page.index("'", s) url = page[s:e] postFields = urllib.urlencode({ "login": login, "passwd": password, "SI": "Sign in", "type": "11", "PPFT": PPFT, "PPSX": str(PPSX), "idsbho": "1", "LoginOptions": "3", "NewUser": "******", "i1": "0", # ClientUserSaved "i2": "1", # ClientMode "i3": str(clt), # ClientLoginTime "i4": "0", # ClientExplore "i7": "0", # ClientOTPRequest "i12": "1", # LoginUsedSSL "i13": "0", # ClientUsedKMSI "i14": str(renderTime), # RenderCompleteTime "i15": str(resourcesTime), # RenderCompleteTime "i16": str(PLT), # PLT "i17": "0", # SRSFailed "i18": "__Login_Strings|1,__Login_Core|1," # SRSSuccess }) # get Passport page request = urllib2.Request(url, postFields, self.httpHeaders) with self.opener.open(request) as response: page = helpers.getResponseBody(response) # Checking for bad usernames and password helpers.errorOnText( page, 'That password is incorrect.', 'Authentication has not been passed: Invalid password') helpers.errorOnText( page, "That Microsoft account doesn\\'t exist", 'Authentication has not been passed: Invalid username') # check if there is a new terms of use helpers.errorOnText( page, '//account.live.com/tou/accrue', 'Please log in (log out first if necessary) through a browser and accept the Terms Of Use' ) # get auth form url and contents authForm = re.search(r"<form.+action=\"([^\"]+)\".*?>(.+)?</form>", page) if authForm == None: filename = helpers.dumpErrorPage(page) raise AuthenticationError("Could not find login form:\ncheck " + filename + " file for more information") parser = HTMLFormInputsParser() parser.feed(authForm.group(2).decode("utf-8")) parser.close() postFields = urllib.urlencode(parser.inputs) # finish passing authentication url = authForm.group(1) request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header("Origin", "https://login.live.com") with self.opener.open(request) as response: page = helpers.getResponseBody(response) url = bingCommon.BING_URL request = urllib2.Request(url, postFields, self.httpHeaders) request.add_header("Referer", authForm.group(1)) with self.opener.open(request) as response: url = response.geturl() # if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report if url.find(bingCommon.BING_URL) == -1: try: filename = helpers.dumpErrorPage( helpers.getResponseBody(response)) s = "check " + filename + " file for more information" except IOError: s = "no further information could be provided - failed to write a file into " + \ helpers.RESULTS_DIR + " subfolder" raise AuthenticationError( "Authentication has not been passed:\n" + s)