def getDashboardPage(self):
"""
Returns https://account.microsoft.com/rewards Lifetime Credits
The number of credits earned since day one of the account
"""
url = "https://account.microsoft.com/rewards/dashboard"
request = urllib2.Request(url = url, headers = self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
page = helpers.getResponseBody(response)
# If we have already gone through the sign in process once, we don't need to do it again, just return the page
if page.find('JavaScript required to sign in') == -1:
return page
# get form data
s = page.index('action="')
s += len('action="')
e = page.index('"', s)
action = page[s:e]
s = page.index("NAP")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
nap = page[s:e]
s = page.index("ANON")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
anon = page[s:e]
s = page.index('id="t"')
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
t = page[s:e]
s = page.index('id="pprid"')
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
pprid = page[s:e]
postFields = urllib.urlencode({
"NAP" : nap,
"ANON" : anon,
"t" : t,
"pprid" : pprid
})
request = urllib2.Request(action, postFields, self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
return page
def getDashboardPage(self):
"""
Returns https://account.microsoft.com/rewards Lifetime Credits
The number of credits earned since day one of the account
"""
url = "https://account.microsoft.com/rewards/dashboard"
request = urllib2.Request(url = url, headers = self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
page = helpers.getResponseBody(response)
#If we have already gone through the sign in process once, we don't need to do it again, just return the page
if page.find('JavaScript required to sign in') == -1:
return page
# get form data
s = page.index('action="')
s += len('action="')
e = page.index('"', s)
action = page[s:e]
s = page.index("NAP")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
nap = page[s:e]
s = page.index("ANON")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
anon = page[s:e]
s = page.index('id="t"')
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
t = page[s:e]
s = page.index('id="pprid"')
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
pprid = page[s:e]
postFields = urllib.urlencode({
"NAP" : nap,
"ANON" : anon,
"t" : t,
"pprid" : pprid
})
request = urllib2.Request(action, postFields, self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
return page
def getLifetimeCredits(self):
"""
Returns https://account.microsoft.com/rewards Lifetime Credits
The number of credits earned since day one of the account
"""
url = "https://account.microsoft.com/rewards"
request = urllib2.Request(url = url, headers = self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
page = helpers.getResponseBody(response)
# get form data
s = page.index('action="')
s += len('action="')
e = page.index('"', s)
action = page[s:e]
s = page.index("NAP")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
nap = page[s:e]
s = page.index("ANON")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
anon = page[s:e]
s = page.index('id="t"')
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
t = page[s:e]
postFields = urllib.urlencode({
"NAP" : nap,
"ANON" : anon,
"t" : t
})
request = urllib2.Request(action, postFields, self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# find lifetime points
s = page.find(' lifetime points</div>') - 20
s = page.find('>', s) + 1
e = page.find(' ', s)
points = page[s:e]
return int(points.replace(",", "")) # remove commas so we can cast as int
def getLifetimeCredits(self):
"""
Returns https://account.microsoft.com/rewards Lifetime Credits
The number of credits earned since day one of the account
"""
url = "https://account.microsoft.com/rewards"
request = urllib2.Request(url = url, headers = self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
page = helpers.getResponseBody(response)
# get form data
s = page.index('action="')
s += len('action="')
e = page.index('"', s)
action = page[s:e]
s = page.index("NAP")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
nap = page[s:e]
s = page.index("ANON")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
anon = page[s:e]
s = page.index('id="t"')
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
t = page[s:e]
postFields = urllib.urlencode({
"NAP" : nap,
"ANON" : anon,
"t" : t
})
request = urllib2.Request(action, postFields, self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# find lifetime points
s = page.find(' lifetime points</div>') - 20
s = page.find('>', s) + 1
e = page.find(' ', s)
points = page[s:e]
return int(points.replace(",", "")) # remove commas so we can cast as int
def getLifetimeCredits(self):
"""
Returns http://www.bing.com/rewards/dashboard Lifetime Credits
The number of credits earned since day one of the account
"""
url = "http://www.bing.com/rewards/dashboard"
request = urllib2.Request(url = url, headers = self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# parse dashboard page
s = page.find('<div class="credits-right')
if s != -1:
s += len('<div class="credits-right')
s = page.index('<div class="credits', s)
s += len('<div class="credits')
else:
s = page.index('<div class="data-lifetime')
s += len('<div class="data-lifetime')
s = page.index('<div class="data-value-text', s)
s += len('<div class="data-value-text')
s = page.index(">", s) + 1
e = page.index('</div>', s)
result = int(page[s:e])
return result
def getRewardsPoints(self):
"""
Returns rewards points as int
"""
# report activity
postFields = urllib.urlencode( { "url" : bingCommon.BING_URL, "V" : "web" } )
url = "http://www.bing.com/rewardsapp/reportActivity"
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
if len(page) == 0:
raise Exception("Rewards points page is empty. That could mean you are not signed up for rewards with this account")
# There are instances where the account appears to be signed in, but really is not
helpers.errorOnText(page, "You are not signed", "Temporary account ban: User was not successfully signed in.\n")
# parse activity page
s = page.index("t.innerHTML='")
s += len("t.innerHTML='")
e = page.index("'", s)
rewardsText = page[s:e]
if rewardsText == 'Rewards': # The account is banned
raise helpers.BingAccountError("Banned from BingRewards: Could not get the number of rewards.")
else:
return int(rewardsText)
def getLifetimeCredits(self):
"""
Returns http://www.bing.com/rewards/dashboard Lifetime Credits
The number of credits earned since day one of the account
"""
url = "http://www.bing.com/rewards/dashboard"
request = urllib2.Request(url=url, headers=self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# parse dashboard page
s = page.find('<div class="credits-right')
if s != -1:
s += len('<div class="credits-right')
s = page.index('<div class="credits', s)
s += len('<div class="credits')
else:
s = page.index('<div class="data-lifetime')
s += len('<div class="data-lifetime')
s = page.index('<div class="data-value-text', s)
s += len('<div class="data-value-text')
s = page.index(">", s) + 1
e = page.index('</div>', s)
result = int(page[s:e])
return result
def getRewardsPoints(self):
"""
Returns rewards points as int
"""
# report activity
postFields = urllib.urlencode( { "url" : bingCommon.BING_URL, "V" : "web" } )
url = "http://www.bing.com/rewardsapp/reportActivity"
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
if len(page) == 0:
raise Exception("Rewards points page is empty. That could mean you are not signed up for rewards with this account")
# There are instances where the account appears to be signed in, but really is not
helpers.errorOnText(page, "You are not signed", "Temporary account ban: User was not successfully signed in.\n")
# parse activity page
s = page.index("t.innerHTML='")
s += len("t.innerHTML='")
e = page.index("'", s)
rewardsText = page[s:e]
if rewardsText == 'Rewards': # The account is banned
raise helpers.BingAccountError("Banned from BingRewards: Could not get the number of rewards.")
else:
return int(rewardsText)
def __processHit(self, reward):
"""Processes bdp.Reward.Type.Action.HIT and returns self.RewardResult"""
res = self.RewardResult(reward)
pointsEarned = self.getRewardsPoints()
currPage = self.getDashboardPage()
startIndex = currPage.find('__RequestVerificationToken')
endIndex = currPage[startIndex:].find('/>')
#pad here to get to the correct spot
verificationAttr = currPage[startIndex+49:startIndex+endIndex-2]
verificationData = [
('id', reward.hitId),
('hash', reward.hitHash),
('timeZone', '-300'),
('activityAmount', '1'),
('__RequestVerificationToken', verificationAttr) #there was a comma here, removed it. Monitor to make sure shit still works
]
verificationUrl = 'https://account.microsoft.com/rewards/api/reportactivity?refd=www.bing.com&X-Requested-With=XMLHttpRequest'
request = urllib2.Request(url = verificationUrl, headers = self.httpHeaders)
with self.opener.open(request, urllib.urlencode(verificationData)) as response:
page = helpers.getResponseBody(response)
pointsEarned = self.getRewardsPoints() - pointsEarned
# if HIT is against bdp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to
# pointsExpected
indCol = bdp.Reward.Type.Col.INDEX
if reward.tp[indCol] == bdp.Reward.Type.RE_EARN_CREDITS[indCol]:
pointsExpected = reward.progressMax - reward.progressCurrent
if pointsExpected != pointsEarned:
filename = helpers.dumpErrorPage(page)
res.isError = True
res.message = "Expected to earn " + str(pointsExpected) + " points, but earned " + \
str(pointsEarned) + " points. Check " + filename + " for further information"
return res
def __processHit(self, reward):
"""Processes bdp.Reward.Type.Action.HIT and returns self.RewardResult"""
res = self.RewardResult(reward)
pointsEarned = self.getRewardsPoints()
currPage = self.getDashboardPage()
startIndex = currPage.find('__RequestVerificationToken')
endIndex = currPage[startIndex:].find('/>')
#pad here to get to the correct spot
verificationAttr = currPage[startIndex+49:startIndex+endIndex-2]
verificationData = [
('id', reward.hitId),
('hash', reward.hitHash),
('timeZone', '-300'),
('activityAmount', '1'),
('__RequestVerificationToken', verificationAttr) #there was a comma here, removed it. Monitor to make sure shit still works
]
verificationUrl = 'https://account.microsoft.com/rewards/api/reportactivity?refd=www.bing.com&X-Requested-With=XMLHttpRequest'
request = urllib2.Request(url = verificationUrl, headers = self.httpHeaders)
with self.opener.open(request, urllib.urlencode(verificationData)) as response:
page = helpers.getResponseBody(response)
pointsEarned = self.getRewardsPoints() - pointsEarned
# if HIT is against bdp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to
# pointsExpected
indCol = bdp.Reward.Type.Col.INDEX
if reward.tp[indCol] == bdp.Reward.Type.RE_EARN_CREDITS[indCol]:
pointsExpected = reward.progressMax - reward.progressCurrent
if pointsExpected != pointsEarned:
filename = helpers.dumpErrorPage(page)
res.isError = True
res.message = "Expected to earn " + str(pointsExpected) + " points, but earned " + \
str(pointsEarned) + " points. Check " + filename + " for further information"
return res
def generateQueries(self,
queriesToGenerate,
history,
maxQueryLen=MAX_QUERY_LEN):
"""
parses Bing! news page and generates a set of unique queries to run on Bing!
A query is considered to be unique if it distingueshes from any other query at
least in a meaning letter. Where meaning letter is any ASCII charecter, but
terminators (space, comma, colon, etc.)
Url good enough to get Bing! news
http://www.bing.com/news?q=world+news
param queriesToGenerate the number of queries to return
param history a set of previous searches
param maxQueryLen the maximum query length
returns a set of queries - self.queries
"""
if queriesToGenerate <= 0:
raise ValueError(
"numberOfQueries should be more than 0, but it is %d" %
queriesToGenerate)
self.numberOfQueries = queriesToGenerate
if history is None or not isinstance(history, set):
raise ValueError("history is not set or not an instance of set")
self.history = history
request = urllib2.Request(url=BING_NEWS_URL,
headers=self.bingRewards.httpHeaders)
with self.bingRewards.opener.open(request) as response:
newsPage = helpers.getResponseBody(response)
if newsPage is None: raise TypeError("newsPage is None")
if newsPage.strip() == "": raise ValueError("newsPage is empty")
self.rewardType = bfp.Reward.Type.SEARCH_PC
startMarker = '<div class="NewsResultSet'
s = newsPage.find(startMarker)
if s == -1:
self.rewardType = bfp.Reward.Type.SEARCH_MOBILE
startMarker = '<div class="mpage'
s = newsPage.index(startMarker)
s += len(startMarker)
s = newsPage.index(">", s)
s += 1
if self.rewardType == bfp.Reward.Type.SEARCH_MOBILE:
endMarker = '<div id="CntFtr"'
else:
endMarker = '<div class="RightRail"'
e = newsPage.index(endMarker, s)
self.__generateQueries(newsPage[s:e], maxQueryLen)
return self.queries
def __processSearch(self, reward, verbose, searchType):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
#matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
#if matches is None:
# print "No RegEx matches found for this search and earn"
# res.isError = True
# res.message = "No RegEx matches found for this search and earn"
# return res
#maxRewardsCount = int(matches.group(1))
#rewardsCount = int(matches.group(2))
#rewardCost = 1 # Looks like it's now always X points per one search
#searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
#searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC or reward.tp == bfp.Reward.Type.SEARCH_AND_EARN or searchType == 0:
headers["User-Agent"] = self.userAgents.pc
searchesCount = 30
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
else: reward.tp == bfp.Reward.Type.SEARCH_MOBILE or searchType == 1:
headers["User-Agent"] = self.userAgents.mobile
searchesCount = 20
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
def requestFlyoutPage(self):
"""
Returns bing.com flyout page
This page shows what rewarding activity can be performed in
order to earn Bing points
"""
url = self.BING_FLYOUT_PAGE
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
request.add_header("Referer", "http://www.bing.com/rewards/dashboard")
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
return page
def __processHit(self, reward):
"""Processes bdp.Reward.Type.Action.HIT and returns self.RewardResult"""
res = self.RewardResult(reward)
pointsEarned = self.getRewardsPoints()
request = urllib2.Request(url=reward.url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
pointsEarned = self.getRewardsPoints() - pointsEarned
indCol = bdp.Reward.Type.Col.INDEX
if reward.tp[indCol] == bdp.Reward.Type.RE_EARN_CREDITS[indCol]:
pointsExpected = reward.progressMax - reward.progressCurrent
return res
def generateQueries(self, queriesToGenerate, history, maxQueryLen = MAX_QUERY_LEN):
"""
parses Bing! news page and generates a set of unique queries to run on Bing!
A query is considered to be unique if it distingueshes from any other query at
least in a meaning letter. Where meaning letter is any ASCII charecter, but
terminators (space, comma, colon, etc.)
Url good enough to get Bing! news
http://www.bing.com/news?q=world+news
param queriesToGenerate the number of queries to return
param history a set of previous searches
param maxQueryLen the maximum query length
returns a set of queries - self.queries
"""
if queriesToGenerate <= 0:
raise ValueError("numberOfQueries should be more than 0, but it is %d" % queriesToGenerate)
self.numberOfQueries = queriesToGenerate
if history is None or not isinstance(history, set):
raise ValueError("history is not set or not an instance of set")
self.history = history
request = urllib2.Request(url = BING_NEWS_URL, headers = self.bingRewards.httpHeaders)
with self.bingRewards.opener.open(request) as response:
newsPage = helpers.getResponseBody(response)
if newsPage is None: raise TypeError("newsPage is None")
if newsPage.strip() == "": raise ValueError("newsPage is empty")
self.rewardType = bfp.Reward.Type.SEARCH_PC
startMarker = '<div class="NewsResultSet'
s = newsPage.find(startMarker)
if s == -1:
self.rewardType = bfp.Reward.Type.SEARCH_MOBILE
startMarker = '<div class="mpage'
s = newsPage.index(startMarker)
s += len(startMarker)
s = newsPage.index(">", s)
s += 1
if self.rewardType == bfp.Reward.Type.SEARCH_MOBILE:
endMarker = '<div id="CntFtr"'
else:
endMarker = '<div class="RightRail"'
e = newsPage.index(endMarker, s)
self.__generateQueries(newsPage[s:e], maxQueryLen)
return self.queries
def generateQueries(self, queriesToGenerate, history, maxQueryLen=None):
"""
Parses the current days wikipedia.com page and generates queries
from the links on the page.
param queriesToGenerate the number of queries to return
param history a set of previous searches
param maxQueryLen the maximum query length
returns a set of queries - self.queries
"""
if queriesToGenerate <= 0:
raise ValueError(
"numberOfQueries should be more than 0, but it is %d" %
queriesToGenerate)
if history is None or not isinstance(history, set):
raise ValueError("history is not set or not an instance of set")
request = urllib2.Request(url=QUERY_URL,
headers=self.bingRewards.httpHeaders)
with self.bingRewards.opener.open(request) as response:
page = helpers.getResponseBody(response)
if page.strip() == "": raise ValueError("wikipedia page is empty")
# get rid of new lines
page = page.replace("\n", "")
# seperate out the guts of the article
page = re.search('id="Events"(.+?)id="External_links"', page, re.I)
if page == None:
raise ValueError("wiki page has no contents")
# seperate out each result
searchTerms = re.findall(
r'<li.+?<a href="/wiki/.+?".*?>([a-zA-Z\s]+?)</a>.*?</li>',
page.group(1))
queries = set()
queriesNeeded = queriesToGenerate
while queriesNeeded > 0 and len(searchTerms) > 0:
ri = randint(0, len(searchTerms) - 1)
# ignore things in your history
if searchTerms[ri] in history:
del searchTerms[ri]
continue
queries.add(searchTerms[ri])
del searchTerms[ri]
queriesNeeded -= 1
return queries
def requestFlyoutPage(self):
"""
Returns bing.com flyout page
This page shows what rewarding activity can be performed in
order to earn Bing points
"""
url = self.BING_FLYOUT_PAGE
request = urllib2.Request(url=url, headers=self.httpHeaders)
# commenting the line below, because on 10/25/2013 Bing! started to return an empty flyout page if referer is other than http://www.bing.com
#request.add_header("Referer", "http://www.bing.com/rewards/dashboard")
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
return page
def requestFlyoutPage(self):
"""
Returns bing.com flyout page
This page shows what rewarding activity can be performed in
order to earn Bing points
"""
url = self.BING_FLYOUT_PAGE
request = urllib2.Request(url = url, headers = self.httpHeaders)
# commenting the line below, because on 10/25/2013 Bing! started to return an empty flyout page if referer is other than http://www.bing.com
#request.add_header("Referer", "http://www.bing.com/rewards/dashboard")
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
return page
def __processQuiz(self, reward):
"""Processes bdp.Reward.Type.Action.QUIZ and returns self.RewardResult"""
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
pointsEarned = self.getRewardsPoints()
currPage = self.getDashboardPage()
startIndex = currPage.find('__RequestVerificationToken')
endIndex = currPage[startIndex:].find('/>')
#pad here to get to the correct spot
verificationAttr = currPage[startIndex + 49:startIndex + endIndex - 2]
#TODO: last parameter is ', "Timezone" : 240'. Is it needed?
verificationData = '{"ActivitySubType" : "quiz", "ActivityType" : "notification", "OfferId" : "' + reward.hitId + '", "Channel" : "Bing.Com", "PartnerId" : "BingTrivia"}'
verificationUrl = 'https://www.bing.com/msrewards/api/v1/ReportActivity'
print
print "Running activity: %s" % reward.name
print
request = urllib2.Request(url=verificationUrl,
headers=self.httpHeaders)
for i in range(reward.progressCurrent, reward.progressMax, 10):
print "%s - %2d/%2d - Activity: %s" % (helpers.getLoggingTime(),
i + 10, reward.progressMax,
reward.name)
with self.opener.open(request, verificationData) as response:
page = helpers.getResponseBody(response)
#default pause between guesses
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
pointsEarned = self.getRewardsPoints() - pointsEarned
# if QUIZ is against bdp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to
# pointsExpected
indCol = bdp.Reward.Type.Col.INDEX
if reward.tp[indCol] == bdp.Reward.Type.RE_EARN_CREDITS[indCol]:
pointsExpected = reward.progressMax - reward.progressCurrent
if pointsExpected != pointsEarned:
filename = helpers.dumpErrorPage(page)
res.isError = True
res.message = "Expected to earn " + str(pointsExpected) + " points, but earned " + \
str(pointsEarned) + " points. Check " + filename + " for further information"
return res
def __processWarn(self, reward):
"""Processes bfp.Reward.Type.Action.WARN and returns self.RewardResult"""
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
pointsEarned = self.getRewardsPoints()
request = urllib2.Request(url = reward.url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
pointsEarned = self.getRewardsPoints() - pointsEarned
# check if we earned any points
if pointsEarned < 1:
res.isError = True
res.message = "Didn't earn any points for click"
return res
def __processHit(self, reward):
"""Processes bdp.Reward.Type.Action.HIT and returns self.RewardResult"""
res = self.RewardResult(reward)
pointsEarned = self.getRewardsPoints()
request = urllib2.Request(url=reward.url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
pointsEarned = self.getRewardsPoints() - pointsEarned
# if HIT is against bdp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to
# pointsExpected
indCol = bdp.Reward.Type.Col.INDEX
if reward.tp[indCol] == bdp.Reward.Type.RE_EARN_CREDITS[indCol]:
pointsExpected = reward.progressMax - reward.progressCurrent
if pointsExpected != pointsEarned:
filename = helpers.dumpErrorPage(page)
res.isError, res.message = True, "Expected to earn " + str(pointsExpected) + " points, but earned " + \
str(pointsEarned) + " points. Check " + filename + " for further information"
return res
def __processWarn(self, reward):
"""Processes bfp.Reward.Type.Action.WARN and returns self.RewardResult"""
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
pointsEarned = self.getRewardsPoints()
request = urllib2.Request(url=reward.url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
pointsEarned = self.getRewardsPoints() - pointsEarned
# check if we earned any points
if pointsEarned < 1:
res.isError = True
res.message = "Didn't earn any points for click"
return res
def generateQueries(self, queriesToGenerate, history, maxQueryLen = None):
"""
Parses the current days wikipedia.com page and generates queries
from the links on the page.
param queriesToGenerate the number of queries to return
param history a set of previous searches
param maxQueryLen the maximum query length
returns a set of queries - self.queries
"""
if queriesToGenerate <= 0:
raise ValueError("numberOfQueries should be more than 0, but it is %d" % queriesToGenerate)
if history is None or not isinstance(history, set):
raise ValueError("history is not set or not an instance of set")
request = urllib.request.Request(url = QUERY_URL, headers = self.bingRewards.httpHeaders)
with self.bingRewards.opener.open(request) as response:
page = helpers.getResponseBody(response)
if page.strip() == "": raise ValueError("wikipedia page is empty")
# get rid of new lines
page = page.replace("\n", "")
# seperate out the guts of the article
page = re.search('id="Events"(.+?)id="External_links"', page, re.I)
if page == None:
raise ValueError("wiki page has no contents")
# seperate out each result
searchTerms = re.findall(r'<li.+?<a href="/wiki/.+?".*?>([a-zA-Z\s]+?)</a>.*?</li>', page.group(1))
queries = set()
queriesNeeded = queriesToGenerate
while queriesNeeded > 0 and len(searchTerms) > 0:
ri = randint(0, len(searchTerms) - 1)
# ignore things in your history
if searchTerms[ri] in history:
del searchTerms[ri]
continue
queries.add(searchTerms[ri])
del searchTerms[ri]
queriesNeeded -= 1
return queries
def getLifetimeCredits(self):
"""
Returns http://www.bing.com/rewards/dashboard Lifetime Credits
The number of credits earned since day one of the account
"""
url = "http://www.bing.com/rewards/dashboard"
request = urllib2.Request(url=url, headers=self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# parse dashboard page
s = page.find('<div class="credits-right')
d = page.find('<span class="credits-right')
# There are instances where the account appears to be signed in, but really is not
helpers.errorOnText(
page, "You are not signed",
"Temporary account ban: User was not successfully signed in.\n")
if s != -1:
s += len('<div class="credits-right')
s = page.index('<div class="credits', s)
s += len('<div class="credits')
elif d != -1:
d += len('<span class="credits-right')
d = page.index('<div class="credits', d)
d += len('<div class="credits')
s = d
else:
s = page.index('<div class="data-lifetime')
s += len('<div class="data-lifetime')
s = page.index('<div class="data-value-text', s)
s += len('<div class="data-value-text')
s = page.index(">", s) + 1
e = page.index('</div>', s)
result = int(page[s:e])
return result
def __processHit(self, reward):
"""Processes bfp.Reward.Type.Action.HIT and returns self.RewardResult"""
res = self.RewardResult(reward)
pointsEarned = self.getRewardsPoints()
request = urllib2.Request(url = reward.url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
pointsEarned = self.getRewardsPoints() - pointsEarned
# if HIT is against bfp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to
# pointsExpected
indCol = bfp.Reward.Type.Col.INDEX
if reward.tp[indCol] == bfp.Reward.Type.RE_EARN_CREDITS[indCol]:
matches = bfp.Reward.Type.RE_EARN_CREDITS[bfp.Reward.Type.Col.NAME].search(reward.name)
pointsExpected = int(matches.group(1))
if pointsExpected != pointsEarned:
filename = helpers.dumpErrorPage(page)
res.isError = True
res.message = "Expected to earn " + str(pointsExpected) + " points, but earned " + \
str(pointsEarned) + " points. Check " + filename + " for further information"
return res
def getRewardsPoints(self):
"""
Returns rewards points as int
"""
# report activity
postFields = urllib.urlencode( { "url" : bingCommon.BING_URL, "V" : "web" } )
url = "http://www.bing.com/rewardsapp/reportActivity"
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
if len(page) == 0:
raise Exception("Rewards points page is empty. That could mean you are not signed up for rewards with this account")
# parse activity page
s = page.index("t.innerHTML='")
s += len("t.innerHTML='")
e = page.index("'", s)
return int(page[s:e])
def getRewardsPoints(self):
"""
Returns rewards points as int
"""
# report activity
postFields = urllib.urlencode( { "url" : bingCommon.BING_URL, "V" : "web" } )
url = "http://www.bing.com/rewardsapp/reportActivity"
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
if len(page) == 0:
raise Exception("Rewards points page is empty. That could mean you are not signed up for rewards with this account")
# parse activity page
s = page.index("t.innerHTML='")
s += len("t.innerHTML='")
e = page.index("'", s)
return int(page[s:e])
def getLifetimeCredits(self):
"""
Returns http://www.bing.com/rewards/dashboard Lifetime Credits
The number of credits earned since day one of the account
"""
url = "http://www.bing.com/rewards/dashboard"
request = urllib2.Request(url = url, headers = self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# parse dashboard page
s = page.find('<div class="credits-right')
d = page.find('<span class="credits-right')
# There are instances where the account appears to be signed in, but really is not
helpers.errorOnText(page, "You are not signed", "Temporary account ban: User was not successfully signed in.\n")
if s != -1:
s += len('<div class="credits-right')
s = page.index('<div class="credits', s)
s += len('<div class="credits')
elif d != -1:
d += len('<span class="credits-right')
d = page.index('<div class="credits', d)
d += len('<div class="credits')
s = d
else:
s = page.index('<div class="data-lifetime')
s += len('<div class="data-lifetime')
s = page.index('<div class="data-value-text', s)
s += len('<div class="data-value-text')
s = page.index(">", s) + 1
e = page.index('</div>', s)
result = int(page[s:e])
return result
def getLifetimeCredits(self):
"""
Returns http://www.bing.com/rewards/dashboard Lifetime Credits
The number of credits earned since day one of the account
"""
url = "http://www.bing.com/rewards/dashboard"
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# parse dashboard page
s = page.index('<div class="user-balance')
s += len('<div class="user-balance')
s = page.index("Lifetime Credits</span>", s)
s += len("Lifetime Credits</span>")
s = page.index('<span class="data-value-text', s)
s += len('<span class="data-value-text')
s = page.index(">", s) + 1
e = page.index("</span>", s)
result = int(page[s:e])
return result
def __processSearch(self, reward, verbose):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"')
IG_SEARCH_RESULTS_PATTERN = re.compile(
r'<ol\s.*?id="b_results"(.+?)</ol>')
IG_SEARCHS_PATTERN = re.compile(
r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url=url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(
0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(
0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
totalQueries, query)
request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number != None:
ig_number = ig_number.group(1)
# get search results
ig_results = IG_SEARCH_RESULTS_PATTERN.search(page)
if ig_results != None:
ig_results = ig_results.group(1)
# seperate search results
ig_searches = IG_SEARCHS_PATTERN.findall(
ig_results)
# get a random link to open
ig_max_rand = min(self.openTopLinkRange,
len(ig_searches) - 1)
ig_link_num = random.randint(0, ig_max_rand)
# number of the link we will use
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK, urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1]))
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(
url=ig_link, headers=bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num +
1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(
filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(
searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(
successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def authenticate(self, authType, login, password):
"""
throws ValueError if login or password is None
throws AuthenticationError
"""
if login is None: raise ValueError("login is None")
if password is None: raise ValueError("password is None")
"""
Authenticates a user on bing.com with his/her Live account.
throws AuthenticationError if authentication can not be passed
throws urllib2.HTTPError if the server couldn't fulfill the request
throws urllib2.URLError if failed to reach the server
"""
# request http://www.bing.com
request = urllib2.Request(url = bingCommon.BING_URL, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# get connection URL for provider Live
urlSearch = self.winLiveId.search(page)
if urlSearch == None:
raise AuthenticationError("Could not find variable 'WindowsLiveId' on Live login page")
url = urlSearch.group(1).decode("unicode_escape")
request = urllib2.Request(url = url, headers = self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
page, referer = helpers.getResponses(self, request)
# get PPFT parameter
PPFTSearch = self.ppftValue.search(page)
if PPFTSearch == None:
raise AuthenticationError("Could not find variable 'PPFT' on Live login page")
PPFT = PPFTSearch.group(1)
# get PPSX parameter
ppsxSearch = self.ppsxValue.search(page)
if ppsxSearch == None:
raise AuthenticationError("Could not find PassportRN variable on Live login page")
PPSX = ppsxSearch.group(1)
# generate ClientLoginTime
clt = 20000 + int(random.uniform(0, 1000))
# get url to post data to
urlSearch = self.urlPostValue.search(page)
if urlSearch == None:
raise AuthenticationError("Could not find variable 'urlPost' on Live login page")
url = urlSearch.group(1)
timestamp = int(round(time.time() * 1000))
# TODO: randomize times a bit?
i16 = json.dumps({
"navigationStart": timestamp,
"unloadEventStart": timestamp + 209,
"unloadEventEnd": timestamp + 210,
"redirectStart": 0,
"redirectEnd": 0,
"fetchStart": timestamp + 73,
"domainLookupStart": timestamp + 73,
"domainLookupEnd": timestamp + 130,
"connectStart": timestamp + 130,
"connectEnd": timestamp + 130,
"secureConnectionStart": timestamp + 210,
"requestStart": timestamp + 183,
"responseStart": timestamp + 205,
"responseEnd": timestamp + 205,
"domLoading": timestamp + 208,
"domInteractive": timestamp + 406,
"domContentLoadedEventStart": timestamp + 420,
"domContentLoadedEventEnd": timestamp + 420,
"domComplete": timestamp + 422,
"loadEventStart": timestamp + 422,
"loadEventEnd": 0
})
postFields = urllib.urlencode({
"loginfmt" : login,
"login" : login,
"passwd" : password,
"type" : "11",
"PPFT" : PPFT,
"PPSX" : str(PPSX),
"LoginOptions" : "3",
"FoundMSAs" : "",
"fspost" : "0",
"NewUser" : "1",
"i2" : "1", # ClientMode
"i13" : "0", # ClientUsedKMSI
"i16" : i16,
"i19" : str(clt), # ClientLoginTime
"i21" : "0",
"i22" : "0",
"i17" : "0", # SRSFailed
"i18" : "__DefaultLogin_Strings|1,__DefaultLogin_Core|1," # SRSSuccess
})
# get Passport page
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", referer)
page, referer = helpers.getResponses(self, request)
# Checking for bad usernames and password
helpers.errorOnText(page, "That password is incorrect.", "Authentication has not been passed: Invalid password")
helpers.errorOnText(page, "That Microsoft account doesn't exist", "Authentication has not been passed: Invalid username")
# check if there is a new terms of use
helpers.errorOnText(page, "//account.live.com/tou/accrue", "Please log in (log out first if necessary) through a browser and accept the Terms Of Use")
contSubmitUrl = self.formAction.search(page)
if contSubmitUrl == None:
raise AuthenticationError("Could not find form action for continue page")
url = contSubmitUrl.group(1)
# get all form inputs
formFields = self.inputNameValue.findall(page)
postFields = {}
for field in formFields:
postFields[field[0]] = field[1]
postFields = urllib.urlencode(postFields)
# submit continue page
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", referer)
page, referer = helpers.getResponses(self, request)
request = urllib2.Request(url = bingCommon.BING_URL, headers = self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
referer = response.geturl()
# if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report
if referer.find(bingCommon.BING_URL) == -1:
raise AuthenticationError("Authentication has not been passed:\n")
def __processSearch(self, reward, verbose):
"""Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bdp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matchesMobile = False
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
#Mobile description changed, so check that one too
if matches is None:
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE_MOBILE.search(reward.description)
matchesMobile = True
if matches is None:
print "No RegEx matches found for this search and earn"
res.isError = True
res.message = "No RegEx matches found for this search and earn"
return res
maxRewardsCount = int(matches.group(1))
rewardsCount = int(matches.group(2))
rewardCost = 1 # Looks like it's now always X points per one search
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
if matchesMobile == True:
#new mobile search description gives total search count + points per search for edge/non-edge
edgeValue = int(matches.group(1))
nonEdgeValue = int(matches.group(2))
searchesCount = int(matches.group(3))
#apparently ios uses EdgiOS, so we only check the first 3 letters not the full word 'edge'
if self.userAgents.mobile.lower().find("edg") != -1:
#we are searching on edge so points go to 200
searchesCount -= reward.progressCurrent / edgeValue
else:
#non-edge search so 100 is the max
searchesCount -= reward.progressCurrent / nonEdgeValue
headers = self.httpHeaders
if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number is not None:
ig_number = ig_number.group(1)
ig_searches = IG_SEARCHES_PATTERN.findall(page)
# make sure we have at least 1 search
if len(ig_searches) > 0:
# get a random link to open
ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
# number of the link we will use
ig_link_num = random.randint(0, ig_max_rand)
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK,
urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1])
)
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num + 1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! No searches were found on search results page"
print "Check {0} file for more information".format(filename)
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __authenticateLive(self, login, password):
"""
Authenticates a user on bing.com with his/her Live account.
throws AuthenticationError if authentication can not be passed
throws urllib2.HTTPError if the server couldn't fulfill the request
throws urllib2.URLError if failed to reach the server
"""
# print "Requesting bing.com"
# request http://www.bing.com
request = urllib2.Request(url = bingCommon.BING_URL, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# get connection URL for provider Live
s = page.index('"WindowsLiveId":"')
s += len('"WindowsLiveId":"')
e = page.index('"', s)
url = BingAuth._escapeString(page[s:e])
request = urllib2.Request(url = url, headers = self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
# get Facebook authenctication form action url
page = helpers.getResponseBody(response)
# get PPFT parameter
s = page.index("sFTTag")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
PPFT = page[s:e]
# get PPSX parameter
ppsxSearch = self.ppsxValue.search(page)
if ppsxSearch == None:
raise AuthenticationError("Could not find variable 't' on Live login page")
PPSX = ppsxSearch.group(1)
# generate ClientLoginTime
clt = 20000 + int(random.uniform(0, 1000))
# get url to post data to
s = page.index(",urlPost:'")
s += len(",urlPost:'")
e = page.index("'", s)
url = page[s:e]
timestamp = int(round(time.time() * 1000))
# TODO: randomize times a bit?
i16 = json.dumps({
"navigationStart": timestamp,
"unloadEventStart": timestamp + 209,
"unloadEventEnd": timestamp + 210,
"redirectStart": 0,
"redirectEnd": 0,
"fetchStart": timestamp + 73,
"domainLookupStart": timestamp + 73,
"domainLookupEnd": timestamp + 130,
"connectStart": timestamp + 130,
"connectEnd": timestamp + 130,
"secureConnectionStart": timestamp + 210,
"requestStart": timestamp + 183,
"responseStart": timestamp + 205,
"responseEnd": timestamp + 205,
"domLoading": timestamp + 208,
"domInteractive": timestamp + 406,
"domContentLoadedEventStart": timestamp + 420,
"domContentLoadedEventEnd": timestamp + 420,
"domComplete": timestamp + 422,
"loadEventStart": timestamp + 422,
"loadEventEnd": 0
})
postFields = urllib.urlencode({
"loginfmt" : login,
"login" : login,
"passwd" : password,
"type" : "11",
"PPFT" : PPFT,
"PPSX" : str(PPSX),
"LoginOptions" : "3",
"FoundMSAs" : "",
"fspost" : "0",
"NewUser" : "1",
"i2" : "1", # ClientMode
"i13" : "0", # ClientUsedKMSI
"i16" : i16,
"i19" : str(clt), # ClientLoginTime
"i21" : "0",
"i22" : "0",
"i17" : "0", # SRSFailed
"i18" : "__DefaultLogin_Strings|1,__DefaultLogin_Core|1," # SRSSuccess
})
# get Passport page
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
referer = response.geturl()
page = helpers.getResponseBody(response)
# Checking for bad usernames and password
helpers.errorOnText(page, 'That password is incorrect.', 'Authentication has not been passed: Invalid password')
helpers.errorOnText(page, "That Microsoft account doesn\\'t exist", 'Authentication has not been passed: Invalid username')
# check if there is a new terms of use
helpers.errorOnText(page, '//account.live.com/tou/accrue', 'Please log in (log out first if necessary) through a browser and accept the Terms Of Use')
contSubmitUrl = self.formAction.search(page)
if contSubmitUrl == None:
raise AuthenticationError("Could not find form action for continue page")
url = contSubmitUrl.group(1)
# get all form inputs
formFields = self.inputNameValue.findall(page)
postFields = {}
for field in formFields:
postFields[field[0]] = field[1]
postFields = urllib.urlencode(postFields)
# submit continue page
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
referer = response.geturl()
page = helpers.getResponseBody(response)
request = urllib2.Request(url = bingCommon.BING_URL, headers = self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
referer = response.geturl()
# if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report
if referer.find(bingCommon.BING_URL) == -1:
try:
filename = helpers.dumpErrorPage(helpers.getResponseBody(response))
s = "check " + filename + " file for more information"
except IOError:
s = "no further information could be provided - failed to write a file into " + \
helpers.RESULTS_DIR + " subfolder"
raise AuthenticationError("Authentication has not been passed:\n" + s)
def __authenticateFacebook(self, login, password):
"""
Authenticates a user on bing.com with his/her Facebook account.
throws AuthenticationError if authentication can not be passed
throws HTMLParser.HTMLParseError
throws urllib2.HTTPError if the server couldn't fulfill the request
throws urllib2.URLError if failed to reach the server
"""
BING_REQUEST_PERMISSIONS = "http://www.bing.com/fd/auth/signin?action=interactive&provider=facebook&return_url=http%3a%2f%2fwww.bing.com%2f&src=EXPLICIT&perms=read_stream%2cuser_photos%2cfriends_photos&sig="
# print "Requesting bing.com"
# request http://www.bing.com
request = urllib2.Request(url=bingCommon.BING_URL,
headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# get connection URL for provider Facebook
s = page.index('"Facebook":"')
s += len('"Facebook":"')
e = page.index('"', s)
# the URL contains escape characters (http:// is http\x3a\x2f\x2f, for example)
# this needs to be decoded back to normal URL
# see (http://docs.python.org/2/library/codecs.html#python-specific-encodings)
url = page[s:e].decode('string-escape')
s = url.index('sig=')
s += len('sig=')
e = url.find('&', s)
if e == -1:
e = len(url)
url = BING_REQUEST_PERMISSIONS + url[s:e]
# print "Now requesting facebook authentication page"
# request FACEBOOK_CONNECT_ORIGINAL_URL
request = urllib2.Request(url=url, headers=self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
# get Facebook authenctication form action url
page = helpers.getResponseBody(response)
s = page.index('<form id="login_form"')
s = page.index('action="', s)
s += len('action="')
e = page.index('"', s)
url = page[s:e]
# relative url? add url from the previous response
if url[0:1] == "/":
url = referer + url
# find all html elements which need to be sent to the server
s = page.index('>', s)
s += 1
e = page.index('</form>')
parser = HTMLFormInputsParser()
parser.feed(page[s:e].decode("utf-8"))
parser.close()
parser.inputs["email"] = login
parser.inputs["pass"] = password
# print "Now passing facebook authentication"
# pass facebook authentication
postFields = urllib.urlencode(parser.inputs)
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
url = response.geturl()
# if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report
if url.find(bingCommon.BING_URL) == -1:
try:
filename = helpers.dumpErrorPage(
helpers.getResponseBody(response))
s = "check " + filename + " file for more information"
except IOError:
s = "no further information could be provided - failed to write a file into " + \
helpers.RESULTS_DIR + " subfolder"
raise AuthenticationError(
"Authentication has not been passed:\n" + s)
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
def __processSearch(self, reward, verbose):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content"'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content"'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"')
IG_SEARCH_RESULTS_PATTERN = re.compile(r'<ol\s.*?id="b_results"(.+?)</ol>')
IG_SEARCHS_PATTERN = re.compile(r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number != None:
ig_number = ig_number.group(1)
# get search results
ig_results = IG_SEARCH_RESULTS_PATTERN.search(page)
if ig_results != None:
ig_results = ig_results.group(1)
# seperate search results
ig_searches = IG_SEARCHS_PATTERN.findall(ig_results)
# make sure we have at least 1 search
if len(ig_searches) > 0:
# get a random link to open
ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
ig_link_num = random.randint(0, ig_max_rand)
# number of the link we will use
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK,
urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1])
)
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num + 1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! No searches were found on search results page"
print "Check {0} file for more information".format(filename)
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __authenticateLive(self, login, password):
"""
Authenticates a user on bing.com with his/her Live account.
throws AuthenticationError if authentication can not be passed
throws urllib2.HTTPError if the server couldn't fulfill the request
throws urllib2.URLError if failed to reach the server
"""
# print "Requesting bing.com"
# request http://www.bing.com
request = urllib2.Request(url=bingCommon.BING_URL,
headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# get connection URL for provider Live
s = page.index('"WindowsLiveId":"')
s += len('"WindowsLiveId":"')
e = page.index('"', s)
url = BingAuth._escapeString(page[s:e])
request = urllib2.Request(url=url, headers=self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
# get Facebook authenctication form action url
page = helpers.getResponseBody(response)
# get PPFT parameter
s = page.index("sFTTag")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
PPFT = page[s:e]
# get PPSX parameter
ppsxSearch = self.ppsxValue.search(page)
if ppsxSearch == None:
raise AuthenticationError(
"Could not find variable 't' on Live login page")
PPSX = ppsxSearch.group(1)
# generate ClientLoginTime
clt = 20000 + int(random.uniform(0, 1000))
# get url to post data to
s = page.index(",urlPost:'")
s += len(",urlPost:'")
e = page.index("'", s)
url = page[s:e]
timestamp = int(round(time.time() * 1000))
# TODO: randomize times a bit?
i16 = json.dumps({
"navigationStart": timestamp,
"unloadEventStart": timestamp + 209,
"unloadEventEnd": timestamp + 210,
"redirectStart": 0,
"redirectEnd": 0,
"fetchStart": timestamp + 73,
"domainLookupStart": timestamp + 73,
"domainLookupEnd": timestamp + 130,
"connectStart": timestamp + 130,
"connectEnd": timestamp + 130,
"secureConnectionStart": timestamp + 210,
"requestStart": timestamp + 183,
"responseStart": timestamp + 205,
"responseEnd": timestamp + 205,
"domLoading": timestamp + 208,
"domInteractive": timestamp + 406,
"domContentLoadedEventStart": timestamp + 420,
"domContentLoadedEventEnd": timestamp + 420,
"domComplete": timestamp + 422,
"loadEventStart": timestamp + 422,
"loadEventEnd": 0
})
postFields = urllib.urlencode({
"loginfmt":
login,
"login":
login,
"passwd":
password,
"type":
"11",
"PPFT":
PPFT,
"PPSX":
str(PPSX),
"LoginOptions":
"3",
"FoundMSAs":
"",
"fspost":
"0",
"NewUser":
"******",
"i2":
"1", # ClientMode
"i13":
"0", # ClientUsedKMSI
"i16":
i16,
"i19":
str(clt), # ClientLoginTime
"i21":
"0",
"i22":
"0",
"i17":
"0", # SRSFailed
"i18":
"__DefaultLogin_Strings|1,__DefaultLogin_Core|1," # SRSSuccess
})
# get Passport page
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
referer = response.geturl()
page = helpers.getResponseBody(response)
# Checking for bad usernames and password
helpers.errorOnText(
page, 'That password is incorrect.',
'Authentication has not been passed: Invalid password')
helpers.errorOnText(
page, "That Microsoft account doesn\\'t exist",
'Authentication has not been passed: Invalid username')
# check if there is a new terms of use
helpers.errorOnText(
page, '//account.live.com/tou/accrue',
'Please log in (log out first if necessary) through a browser and accept the Terms Of Use'
)
contSubmitUrl = self.formAction.search(page)
if contSubmitUrl == None:
raise AuthenticationError(
"Could not find form action for continue page")
url = contSubmitUrl.group(1)
# get all form inputs
formFields = self.inputNameValue.findall(page)
postFields = {}
for field in formFields:
postFields[field[0]] = field[1]
postFields = urllib.urlencode(postFields)
# submit continue page
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
referer = response.geturl()
page = helpers.getResponseBody(response)
request = urllib2.Request(url=bingCommon.BING_URL,
headers=self.httpHeaders)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
referer = response.geturl()
# if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report
if referer.find(bingCommon.BING_URL) == -1:
try:
filename = helpers.dumpErrorPage(
helpers.getResponseBody(response))
s = "check " + filename + " file for more information"
except IOError:
s = "no further information could be provided - failed to write a file into " + \
helpers.RESULTS_DIR + " subfolder"
raise AuthenticationError(
"Authentication has not been passed:\n" + s)
def __processSearch(self, reward, verbose):
"""Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bdp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matchesMobile = False
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
#Mobile description changed, so check that one too
if matches is None:
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE_MOBILE.search(reward.description)
matchesMobile = True
if matches is None:
print "No RegEx matches found for this search and earn"
res.isError = True
res.message = "No RegEx matches found for this search and earn"
return res
maxRewardsCount = int(matches.group(1))
rewardsCount = int(matches.group(2))
rewardCost = 1 # Looks like it's now always X points per one search
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
if matchesMobile == True:
#new mobile search description gives total search count + points per search for edge/non-edge
edgeValue = int(matches.group(1))
nonEdgeValue = int(matches.group(2))
searchesCount = int(matches.group(3))
#apparently ios uses EdgiOS, so we only check the first 3 letters not the full word 'edge'
if self.userAgents.mobile.lower().find("edg") != -1:
#we are searching on edge so points go to 200
searchesCount -= reward.progressCurrent / edgeValue
else:
#non-edge search so 100 is the max
searchesCount -= reward.progressCurrent / nonEdgeValue
headers = self.httpHeaders
if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number is not None:
ig_number = ig_number.group(1)
ig_searches = IG_SEARCHES_PATTERN.findall(page)
# make sure we have at least 1 search
if len(ig_searches) > 0:
# get a random link to open
ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
# number of the link we will use
ig_link_num = random.randint(0, ig_max_rand)
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK,
urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1])
)
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num + 1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! No searches were found on search results page"
print "Check {0} file for more information".format(filename)
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __authenticateLive(self, login, password):
"""
Authenticates a user on bing.com with his/her Live account.
throws AuthenticationError if authentication can not be passed
throws urllib2.HTTPError if the server couldn't fulfill the request
throws urllib2.URLError if failed to reach the server
"""
# print "Requesting bing.com"
# request http://www.bing.com
request = urllib2.Request(url=bingCommon.BING_URL,
headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# get connection URL for provider Live
s = page.index('"WindowsLiveId":"')
s += len('"WindowsLiveId":"')
e = page.index('"', s)
url = BingAuth._escapeString(page[s:e])
request = urllib2.Request(url=url, headers=self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
# get Facebook authenctication form action url
page = helpers.getResponseBody(response)
# get PPFT parameter
s = page.index("sFTTag")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
PPFT = page[s:e]
# get PPSX parameter
ppsxParam = ""
if ",M:" in page:
ppsxParam = ",M:"
elif ",g:" in page:
ppsxParam = ",g:"
elif ",j:" in page:
ppsxParam = ",j:"
else:
ppsxParam = ",d:"
s = page.index(ppsxParam)
s += len(ppsxParam)
e = page.index(",", s)
PPSX = page[s:e]
if PPSX[0] == "'":
PPSX = PPSX[1:-1]
# generate ClientLoginTime
clt = 20000 + int(random.uniform(0, 1000))
# generate RenderCompleteTime
renderTime = 130 + int(random.uniform(0, 100))
# generate ResourcesCompleteTime
resourcesTime = renderTime + int(random.uniform(2, 5))
# generate ResourcesCompleteTime
PLT = 870 + int(random.uniform(0, 250))
# get url to post data to
s = page.index(",urlPost:'")
s += len(",urlPost:'")
e = page.index("'", s)
url = page[s:e]
postFields = urllib.urlencode({
"login":
login,
"passwd":
password,
"SI":
"Sign in",
"type":
"11",
"PPFT":
PPFT,
"PPSX":
str(PPSX),
"idsbho":
"1",
"LoginOptions":
"3",
"NewUser":
"******",
"i1":
"0", # ClientUserSaved
"i2":
"1", # ClientMode
"i3":
str(clt), # ClientLoginTime
"i4":
"0", # ClientExplore
"i7":
"0", # ClientOTPRequest
"i12":
"1", # LoginUsedSSL
"i13":
"0", # ClientUsedKMSI
"i14":
str(renderTime), # RenderCompleteTime
"i15":
str(resourcesTime), # RenderCompleteTime
"i16":
str(PLT), # PLT
"i17":
"0", # SRSFailed
"i18":
"__Login_Strings|1,__Login_Core|1," # SRSSuccess
})
# get Passport page
request = urllib2.Request(url, postFields, self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# Checking for bad usernames and password
helpers.errorOnText(
page, 'That password is incorrect.',
'Authentication has not been passed: Invalid password')
helpers.errorOnText(
page, "That Microsoft account doesn\\'t exist",
'Authentication has not been passed: Invalid username')
# check if there is a new terms of use
helpers.errorOnText(
page, '//account.live.com/tou/accrue',
'Please log in (log out first if necessary) through a browser and accept the Terms Of Use'
)
# get auth form url and contents
authForm = re.search(r"<form.+action=\"([^\"]+)\".*?>(.+)?</form>",
page)
if authForm == None:
filename = helpers.dumpErrorPage(page)
raise AuthenticationError("Could not find login form:\ncheck " +
filename + " file for more information")
parser = HTMLFormInputsParser()
parser.feed(authForm.group(2).decode("utf-8"))
parser.close()
postFields = urllib.urlencode(parser.inputs)
# finish passing authentication
url = authForm.group(1)
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Origin", "https://login.live.com")
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
url = bingCommon.BING_URL
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", authForm.group(1))
with self.opener.open(request) as response:
url = response.geturl()
# if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report
if url.find(bingCommon.BING_URL) == -1:
try:
filename = helpers.dumpErrorPage(
helpers.getResponseBody(response))
s = "check " + filename + " file for more information"
except IOError:
s = "no further information could be provided - failed to write a file into " + \
helpers.RESULTS_DIR + " subfolder"
raise AuthenticationError(
"Authentication has not been passed:\n" + s)
def __authenticateLive(self, login, password):
"""
Authenticates a user on bing.com with his/her Live account.
throws AuthenticationError if authentication can not be passed
throws urllib2.HTTPError if the server couldn't fulfill the request
throws urllib2.URLError if failed to reach the server
"""
# print "Requesting bing.com"
# request http://www.bing.com
request = urllib2.Request(url=bingCommon.BING_URL,
headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# get connection URL for provider Live
s = page.index('"WindowsLiveId":"')
s += len('"WindowsLiveId":"')
e = page.index('"', s)
# the URL contains escape characters (http:// is http\x3a\x2f\x2f, for example)
# this needs to be decoded back to normal URL
# see (http://docs.python.org/2/library/codecs.html#python-specific-encodings)
url = page[s:e].decode('string-escape')
request = urllib2.Request(url=url, headers=self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
# get Facebook authenctication form action url
page = helpers.getResponseBody(response)
# get PPFT parameter
s = page.index("sFTTag")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
PPFT = page[s:e]
# get PPSX parameter
s = page.index(",g:")
s += len(",g:")
e = page.index(",", s)
PPSX = page[s:e]
if PPSX[0] == "'":
PPSX = PPSX[1:-1]
# get sso parameter
s = page.index(",W:")
s += len(",W:")
e = page.index(",", s)
sso = page[s:e]
# generate ClientLoginTime
clt = 20000 + int(random.uniform(0, 1000))
# generate RenderCompleteTime
renderTime = 130 + int(random.uniform(0, 100))
# generate ResourcesCompleteTime
resourcesTime = renderTime + int(random.uniform(2, 5))
# generate ResourcesCompleteTime
PLT = 870 + int(random.uniform(0, 250))
# get url to post data to
s = page.index(",urlPost:'")
s += len(",urlPost:'")
e = page.index("'", s)
url = page[s:e]
postFields = urllib.urlencode({
"login":
login,
"passwd":
password,
"SI":
"Sign in",
"type":
"11",
"PPFT":
PPFT,
"PPSX":
str(PPSX),
"idsbho":
"1",
"LoginOptions":
"3",
"sso":
sso,
"NewUser":
"******",
"i1":
"0", # ClientUserSaved
"i2":
"1", # ClientMode
"i3":
str(clt), # ClientLoginTime
"i4":
"0", # ClientExplore
"i7":
"0", # ClientOTPRequest
"i12":
"1", # LoginUsedSSL
"i13":
"0", # ClientUsedKMSI
"i14":
str(renderTime), # RenderCompleteTime
"i15":
str(resourcesTime), # RenderCompleteTime
"i16":
str(PLT), # PLT
"i17":
"0", # SRSFailed
"i18":
"__Login_Strings|1,__Login_Core|1," # SRSSuccess
})
# get Passport page
request = urllib2.Request(url, postFields, self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
s = page.index("<form ")
e = page.index("</form>", s)
e += len("</form>")
parser = HTMLFormInputsParser()
parser.feed(page[s:e].decode("utf-8"))
parser.close()
postFields = urllib.urlencode(parser.inputs)
# finish passing authentication
url = "http://www.bing.com/Passport.aspx?requrl=http%3a%2f%2fwww.bing.com%2f&wa=wsignin1.0"
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Origin", "https://login.live.com")
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
url = bingCommon.BING_URL
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header(
"Referer",
"http://www.bing.com/Passport.aspx?requrl=http%3a%2f%2fwww.bing.com%2f&wa=wsignin1.0"
)
with self.opener.open(request) as response:
url = response.geturl()
# if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report
if url.find(bingCommon.BING_URL) == -1:
try:
filename = helpers.dumpErrorPage(
helpers.getResponseBody(response))
s = "check " + filename + " file for more information"
except IOError:
s = "no further information could be provided - failed to write a file into " + \
helpers.RESULTS_DIR + " subfolder"
raise AuthenticationError(
"Authentication has not been passed:\n" + s)
def __processSearch(self, reward):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
request = urllib2.Request(url = BING_NEWS_URL, headers = headers)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# generate a set of queries to run
bingQueriesGenerator = BingQueriesGenerator(searchesCount, history, reward.tp)
queries = bingQueriesGenerator.parseBingNews(page)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated !"
print "Requested:", searchesCount
print "Generated:", len(bingQueriesGenerator.queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesInterval)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query)
print "%s - %2d/%2d - Requesting: %s" % (helpers.getLoggingTime(), i, totalQueries, url)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = ( reward.tp == bfp.Reward.Type.SEARCH_PC and
page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 ) \
or ( reward.tp == bfp.Reward.Type.SEARCH_MOBILE and
page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 )
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
return res
def __authenticateLive(self, login, password):
"""
Authenticates a user on bing.com with his/her Live account.
throws AuthenticationError if authentication can not be passed
throws urllib2.HTTPError if the server couldn't fulfill the request
throws urllib2.URLError if failed to reach the server
"""
# print "Requesting bing.com"
# request http://www.bing.com
request = urllib2.Request(url = bingCommon.BING_URL, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# get connection URL for provider Live
s = page.index('"WindowsLiveId":"')
s += len('"WindowsLiveId":"')
e = page.index('"', s)
url = page[s:e]
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
# get Facebook authenctication form action url
page = helpers.getResponseBody(response)
# get PPFT parameter
s = page.index("sFTTag")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
PPFT = page[s:e]
# get PPSX parameter
s = page.index(",g:'")
s += len(",g:'")
e = page.index("'", s)
PPSX = page[s:e]
# get sso parameter
s = page.index(",W:")
s += len(",W:")
e = page.index(",", s)
sso = int(page[s:e])
# generate ClientLoginTime
clt = 20000 + int(random.uniform(0, 1000))
# generate RenderCompleteTime
renderTime = 130 + int(random.uniform(0, 100))
# generate ResourcesCompleteTime
resourcesTime = renderTime + int(random.uniform(2, 5))
# generate ResourcesCompleteTime
PLT = 870 + int(random.uniform(0, 250))
# get url to post data to
s = page.index(",urlPost:'")
s += len(",urlPost:'")
e = page.index("'", s)
url = page[s:e]
postFields = urllib.urlencode({
"login" : login,
"passwd" : password,
"SI" : "Sign in",
"type" : "11",
"PPFT" : PPFT,
"PPSX" : PPSX,
"idsbho" : "1",
"LoginOptions" : "3",
"sso" : str(sso),
"NewUser" : "1",
"i1" : "0", # ClientUserSaved
"i2" : "1", # ClientMode
"i3" : str(clt), # ClientLoginTime
"i4" : "0", # ClientExplore
"i7" : "0", # ClientOTPRequest
"i12" : "1", # LoginUsedSSL
"i13" : "0", # ClientUsedKMSI
"i14" : str(renderTime), # RenderCompleteTime
"i15" : str(resourcesTime), # RenderCompleteTime
"i16" : str(PLT), # PLT
"i17" : "0", # SRSFailed
"i18" : "__Login_Strings|1,__Login_Core|1," # SRSSuccess
})
# get Passport page
request = urllib2.Request(url, postFields, bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
s = page.index("<form ")
e = page.index("</form>", s)
e += len("</form>")
parser = HTMLFormInputsParser()
parser.feed(page[s:e].decode("utf-8"))
parser.close()
postFields = urllib.urlencode(parser.inputs)
# finish passing authentication
url = "http://www.bing.com/Passport.aspx?requrl=http%3a%2f%2fwww.bing.com%2f&wa=wsignin1.0"
request = urllib2.Request(url, postFields, bingCommon.HEADERS)
request.add_header("Origin", "https://login.live.com")
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
url = bingCommon.BING_URL
request = urllib2.Request(url, postFields, bingCommon.HEADERS)
request.add_header("Referer", "http://www.bing.com/Passport.aspx?requrl=http%3a%2f%2fwww.bing.com%2f&wa=wsignin1.0")
with self.opener.open(request) as response:
url = response.geturl()
# if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report
if url.find(bingCommon.BING_URL) == -1:
try:
filename = helpers.dumpErrorPage(helpers.getResponseBody(response))
s = "check " + filename + " file for more information"
except IOError:
s = "no further information could be provided - failed to write a file into " + \
helpers.RESULTS_DIR + " subfolder"
raise AuthenticationError("Authentication has not been passed:\n" + s)
def __processSearch(self, reward):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url=url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
totalQueries, query)
request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(
searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(
successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __authenticateLive(self, login, password):
"""
Authenticates a user on bing.com with his/her Live account.
throws AuthenticationError if authentication can not be passed
throws urllib2.HTTPError if the server couldn't fulfill the request
throws urllib2.URLError if failed to reach the server
"""
# print "Requesting bing.com"
# request http://www.bing.com
request = urllib2.Request(url = bingCommon.BING_URL, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# get connection URL for provider Live
s = page.index('"WindowsLiveId":"')
s += len('"WindowsLiveId":"')
e = page.index('"', s)
url = BingAuth._escapeString(page[s:e])
request = urllib2.Request(url = url, headers = self.httpHeaders)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
# get Facebook authenctication form action url
page = helpers.getResponseBody(response)
# get PPFT parameter
s = page.index("sFTTag")
s = page.index('value="', s)
s += len('value="')
e = page.index('"', s)
PPFT = page[s:e]
# get PPSX parameter
ppsxParam = ""
if ",M:" in page:
ppsxParam = ",M:"
elif ",g:" in page:
ppsxParam = ",g:"
elif ",j:" in page:
ppsxParam = ",j:"
else:
ppsxParam = ",d:"
s = page.index(ppsxParam)
s += len(ppsxParam)
e = page.index(",", s)
PPSX = page[s:e]
if PPSX[0] == "'":
PPSX = PPSX[1:-1]
# generate ClientLoginTime
clt = 20000 + int(random.uniform(0, 1000))
# generate RenderCompleteTime
renderTime = 130 + int(random.uniform(0, 100))
# generate ResourcesCompleteTime
resourcesTime = renderTime + int(random.uniform(2, 5))
# generate ResourcesCompleteTime
PLT = 870 + int(random.uniform(0, 250))
# get url to post data to
s = page.index(",urlPost:'")
s += len(",urlPost:'")
e = page.index("'", s)
url = page[s:e]
postFields = urllib.urlencode({
"login" : login,
"passwd" : password,
"SI" : "Sign in",
"type" : "11",
"PPFT" : PPFT,
"PPSX" : str(PPSX),
"idsbho" : "1",
"LoginOptions" : "3",
"NewUser" : "1",
"i1" : "0", # ClientUserSaved
"i2" : "1", # ClientMode
"i3" : str(clt), # ClientLoginTime
"i4" : "0", # ClientExplore
"i7" : "0", # ClientOTPRequest
"i12" : "1", # LoginUsedSSL
"i13" : "0", # ClientUsedKMSI
"i14" : str(renderTime), # RenderCompleteTime
"i15" : str(resourcesTime), # RenderCompleteTime
"i16" : str(PLT), # PLT
"i17" : "0", # SRSFailed
"i18" : "__Login_Strings|1,__Login_Core|1," # SRSSuccess
})
# get Passport page
request = urllib2.Request(url, postFields, self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# Checking for bad usernames and password
helpers.errorOnText(page, 'That password is incorrect.', 'Authentication has not been passed: Invalid password')
helpers.errorOnText(page, "That Microsoft account doesn\\'t exist", 'Authentication has not been passed: Invalid username')
# check if there is a new terms of use
helpers.errorOnText(page, '//account.live.com/tou/accrue', 'Please log in (log out first if necessary) through a browser and accept the Terms Of Use')
# get auth form url and contents
authForm = re.search(r"<form.+action=\"([^\"]+)\".*?>(.+)?</form>", page)
if authForm == None:
filename = helpers.dumpErrorPage(page)
raise AuthenticationError("Could not find login form:\ncheck " + filename + " file for more information")
parser = HTMLFormInputsParser()
parser.feed(authForm.group(2).decode("utf-8"))
parser.close()
postFields = urllib.urlencode(parser.inputs)
# finish passing authentication
url = authForm.group(1)
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Origin", "https://login.live.com")
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
url = bingCommon.BING_URL
request = urllib2.Request(url, postFields, self.httpHeaders)
request.add_header("Referer", authForm.group(1))
with self.opener.open(request) as response:
url = response.geturl()
# if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report
if url.find(bingCommon.BING_URL) == -1:
try:
filename = helpers.dumpErrorPage(helpers.getResponseBody(response))
s = "check " + filename + " file for more information"
except IOError:
s = "no further information could be provided - failed to write a file into " + \
helpers.RESULTS_DIR + " subfolder"
raise AuthenticationError("Authentication has not been passed:\n" + s)
def __processSearch(self, reward, verbose):
"""Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(
r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"'
)
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bdp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url=url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
reward.description)
if matches is None:
print "No RegEx matches found for this search and earn"
res.isError = True
res.message = "No RegEx matches found for this search and earn"
return res
maxRewardsCount = int(matches.group(1))
rewardsCount = int(matches.group(2))
rewardCost = 1 # Looks like it's now always X points per one search
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
headers = self.httpHeaders
if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(
0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(
0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!\nRequested: {}\nGenerated: {}\n"\
.format(searchesCount, len(queries))
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
totalQueries, query)
request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
print "Warning! Query:\n\t {}\n returned no results, check file for more information {}".format(
query, helpers.dumpErrorPage(page))
else:
successfullQueries += 1
self.randomClick(IG_NUMBER_PATTERN, IG_PING_LINK,
IG_SEARCHES_PATTERN, page, response, verbose)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(
searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(
successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def generateQueries(self, queriesToGenerate, history, maxQueryLen=None):
"""
Parses the current days wikipedia.com page and generates queries
from the links on the page.
param queriesToGenerate the number of queries to return
param history a set of previous searches
param maxQueryLen the maximum query length
returns a set of queries - self.queries
"""
if queriesToGenerate <= 0:
raise ValueError(
"numberOfQueries should be more than 0, but it is %d" %
queriesToGenerate)
if history is None or not isinstance(history, set):
raise ValueError("history is not set or not an instance of set")
request = urllib2.Request(url=QUERY_URL,
headers=self.bingRewards.httpHeaders)
with self.bingRewards.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check that the page has content
if page.strip() == "":
raise ValueError("Wikipedia page is empty")
# convert history to lowercase
history = [x.strip().lower() for x in history]
# get sections of the page (ie. Events, Births, Deaths, Holidays)
rawSections = WIKIPEDIA_SECTION_PATTERN.findall(page)
if len(rawSections) == 0:
raise ValueError("Wikipedia page is empty")
# a list of search terms
searchTerms = []
for sectionName, conts in rawSections:
section = sectionName.lower()
# skip unwanted sections
if section in ["external links"]:
continue
# extract search terms
rawTerms = WIKIPEDIA_LINK_PATTERN.findall(conts)
# skip empty sections
if len(rawTerms) == 0:
continue
terms = []
# check each term against history
for term in rawTerms:
# humans search in lowercase
term = term.lower()
# check if the term was searched for before
if term not in history:
terms.append(term)
# entire section is in history... skip it
if len(terms) == 0:
continue
# section search weight
weight = DEFAULT_SECTION_WEIGHT
if section in DEFAULT_SECTION_WEIGHTS:
weight = DEFAULT_SECTION_WEIGHTS[section]
# add each search term list the number of weighted times
for i in range(weight):
searchTerms.extend(terms)
# randomize the search terms for good measure
shuffle(searchTerms)
queries = set()
queriesNeeded = queriesToGenerate
# loop until we have enough queries or run out of things to search for
while queriesNeeded > 0 and len(searchTerms) > 0:
ri = randint(0, len(searchTerms) - 1)
# add current term to queries
queries.add(searchTerms[ri])
# remove each instance of current term from searchTerms
searchTerms = filter(lambda x: x != searchTerms[ri], searchTerms)
queriesNeeded -= 1
return queries
def __authenticateFacebook(self, login, password):
"""
Authenticates a user on bing.com with his/her Facebook account.
throws AuthenticationError if authentication can not be passed
throws HTMLParser.HTMLParseError
throws urllib2.HTTPError if the server couldn't fulfill the request
throws urllib2.URLError if failed to reach the server
"""
BING_REQUEST_PERMISSIONS = "http://www.bing.com/fd/auth/signin?action=interactive&provider=facebook&return_url=http%3a%2f%2fwww.bing.com%2f&src=EXPLICIT&perms=read_stream%2cuser_photos%2cfriends_photos&sig="
# print "Requesting bing.com"
# request http://www.bing.com
request = urllib2.Request(url = bingCommon.BING_URL, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# get connection URL for provider Facebook
s = page.index('"Facebook":"')
s += len('"Facebook":"')
e = page.index('"', s)
url = page[s:e]
s = url.index('sig=')
s += len('sig=')
e = url.find('&', s)
if e == -1:
e = len(url)
url = BING_REQUEST_PERMISSIONS + url[s:e]
# print "Now requesting facebook authentication page"
# request FACEBOOK_CONNECT_ORIGINAL_URL
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
request.add_header("Referer", bingCommon.BING_URL)
with self.opener.open(request) as response:
referer = response.geturl()
# get Facebook authenctication form action url
page = helpers.getResponseBody(response)
s = page.index('<form id="login_form"')
s = page.index('action="', s)
s += len('action="')
e = page.index('"', s)
url = page[s:e]
# relative url? add url from the previous response
if url[0:1] == "/":
url = referer + url
# find all html elements which need to be sent to the server
s = page.index('>', s)
s += 1
e = page.index('</form>')
parser = HTMLFormInputsParser()
parser.feed(page[s:e].decode("utf-8"))
parser.close()
parser.inputs["email"] = login
parser.inputs["pass"] = password
# print "Now passing facebook authentication"
# pass facebook authentication
postFields = urllib.urlencode(parser.inputs)
request = urllib2.Request(url, postFields, bingCommon.HEADERS)
request.add_header("Referer", referer)
with self.opener.open(request) as response:
url = response.geturl()
# if that's not bingCommon.BING_URL => authentication wasn't pass => write the page to the file and report
if url.find(bingCommon.BING_URL) == -1:
try:
filename = helpers.dumpErrorPage(helpers.getResponseBody(response))
s = "check " + filename + " file for more information"
except IOError:
s = "no further information could be provided - failed to write a file into " + \
helpers.RESULTS_DIR + " subfolder"
raise AuthenticationError("Authentication has not been passed:\n" + s)
def generateQueries(self, queriesToGenerate, history, maxQueryLen = None):
"""
Parses the current days wikipedia.com page and generates queries
from the links on the page.
param queriesToGenerate the number of queries to return
param history a set of previous searches
param maxQueryLen the maximum query length
returns a set of queries - self.queries
"""
if queriesToGenerate <= 0:
raise ValueError("numberOfQueries should be more than 0, but it is %d" % queriesToGenerate)
if history is None or not isinstance(history, set):
raise ValueError("history is not set or not an instance of set")
request = urllib2.Request(url = QUERY_URL, headers = self.bingRewards.httpHeaders)
with self.bingRewards.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check that the page has content
if page.strip() == "":
raise ValueError("Wikipedia page is empty")
# convert history to lowercase
history = [x.strip().lower() for x in history]
# get sections of the page (ie. Events, Births, Deaths, Holidays)
rawSections = WIKIPEDIA_SECTION_PATTERN.findall(page)
if len(rawSections) == 0:
raise ValueError("Wikipedia page is empty")
# a list of search terms
searchTerms = []
for sectionName, conts in rawSections:
section = sectionName.lower()
# skip unwanted sections
if section in ["external links"]:
continue
# extract search terms
rawTerms = WIKIPEDIA_LINK_PATTERN.findall(conts)
# skip empty sections
if len(rawTerms) == 0:
continue
terms = []
# check each term against history
for term in rawTerms:
# humans search in lowercase
term = term.lower()
# check if the term was searched for before
if term not in history:
terms.append(term)
# entire section is in history... skip it
if len(terms) == 0:
continue
# section search weight
weight = DEFAULT_SECTION_WEIGHT
if section in DEFAULT_SECTION_WEIGHTS:
weight = DEFAULT_SECTION_WEIGHTS[section]
# add each search term list the number of weighted times
for i in range(weight):
searchTerms.extend(terms)
# randomize the search terms for good measure
shuffle(searchTerms)
queries = set()
queriesNeeded = queriesToGenerate
# loop until we have enough queries or run out of things to search for
while queriesNeeded > 0 and len(searchTerms) > 0:
ri = randint(0, len(searchTerms) - 1)
# add current term to queries
queries.add(searchTerms[ri])
# remove each instance of current term from searchTerms
searchTerms = filter(lambda x: x != searchTerms[ri], searchTerms)
queriesNeeded -= 1
return queries
