def test_history(self):
"""
test history parsing
:return:
"""
self.assertRaisesRegexp(TypeError, "None", bingHistory.parse, None)
output = bingHistory.parse("")
self.assertIsNotNone(output, "missing output " + str(output))
page = '<span class="query_t">'
page += '<div id="results_area"></div><div id="sidebar"></div>'
output = bingHistory.parse(page)
self.assertIsNotNone(output, "missing output " + str(output))
page = '<span class="sh_item_qu_query">'
page += '<ul class="sh_dayul"></ul>'
page += ' value == 0'
page += '</span>'
output = bingHistory.parse(page)
self.assertIsNotNone(output, "missing output " + str(output))
page = '<ul class="sh_dayul"> </ul>'
output = bingHistory.parse(page)
self.assertIsNotNone(output, "missing output " + str(output))
output = bingHistory.getBingHistoryTodayURL()
self.assertRegexpMatches(output, "https", "missing url " + str(output))
def __processSearch(self, reward, verbose, searchType):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
#matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
#if matches is None:
# print "No RegEx matches found for this search and earn"
# res.isError = True
# res.message = "No RegEx matches found for this search and earn"
# return res
#maxRewardsCount = int(matches.group(1))
#rewardsCount = int(matches.group(2))
#rewardCost = 1 # Looks like it's now always X points per one search
#searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
#searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC or reward.tp == bfp.Reward.Type.SEARCH_AND_EARN or searchType == 0:
headers["User-Agent"] = self.userAgents.pc
searchesCount = 30
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
else: reward.tp == bfp.Reward.Type.SEARCH_MOBILE or searchType == 1:
headers["User-Agent"] = self.userAgents.mobile
searchesCount = 20
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
def __processSearch(self, reward):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url=url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
totalQueries, query)
request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(
searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(
successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __processSearch(self, reward, verbose):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content"'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content"'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"')
IG_SEARCH_RESULTS_PATTERN = re.compile(r'<ol\s.*?id="b_results"(.+?)</ol>')
IG_SEARCHS_PATTERN = re.compile(r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number != None:
ig_number = ig_number.group(1)
# get search results
ig_results = IG_SEARCH_RESULTS_PATTERN.search(page)
if ig_results != None:
ig_results = ig_results.group(1)
# seperate search results
ig_searches = IG_SEARCHS_PATTERN.findall(ig_results)
# make sure we have at least 1 search
if len(ig_searches) > 0:
# get a random link to open
ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
ig_link_num = random.randint(0, ig_max_rand)
# number of the link we will use
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK,
urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1])
)
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num + 1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! No searches were found on search results page"
print "Check {0} file for more information".format(filename)
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __processSearch(self, reward, verbose):
"""Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bdp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matchesMobile = False
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
#Mobile description changed, so check that one too
if matches is None:
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE_MOBILE.search(reward.description)
matchesMobile = True
if matches is None:
print "No RegEx matches found for this search and earn"
res.isError = True
res.message = "No RegEx matches found for this search and earn"
return res
maxRewardsCount = int(matches.group(1))
rewardsCount = int(matches.group(2))
rewardCost = 1 # Looks like it's now always X points per one search
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
if matchesMobile == True:
#new mobile search description gives total search count + points per search for edge/non-edge
edgeValue = int(matches.group(1))
nonEdgeValue = int(matches.group(2))
searchesCount = int(matches.group(3))
#apparently ios uses EdgiOS, so we only check the first 3 letters not the full word 'edge'
if self.userAgents.mobile.lower().find("edg") != -1:
#we are searching on edge so points go to 200
searchesCount -= reward.progressCurrent / edgeValue
else:
#non-edge search so 100 is the max
searchesCount -= reward.progressCurrent / nonEdgeValue
headers = self.httpHeaders
if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number is not None:
ig_number = ig_number.group(1)
ig_searches = IG_SEARCHES_PATTERN.findall(page)
# make sure we have at least 1 search
if len(ig_searches) > 0:
# get a random link to open
ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
# number of the link we will use
ig_link_num = random.randint(0, ig_max_rand)
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK,
urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1])
)
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num + 1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! No searches were found on search results page"
print "Check {0} file for more information".format(filename)
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __processSearch(self, reward, verbose):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"')
IG_SEARCH_RESULTS_PATTERN = re.compile(
r'<ol\s.*?id="b_results"(.+?)</ol>')
IG_SEARCHS_PATTERN = re.compile(
r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url=url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(
0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(
0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
totalQueries, query)
request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number != None:
ig_number = ig_number.group(1)
# get search results
ig_results = IG_SEARCH_RESULTS_PATTERN.search(page)
if ig_results != None:
ig_results = ig_results.group(1)
# seperate search results
ig_searches = IG_SEARCHS_PATTERN.findall(
ig_results)
# get a random link to open
ig_max_rand = min(self.openTopLinkRange,
len(ig_searches) - 1)
ig_link_num = random.randint(0, ig_max_rand)
# number of the link we will use
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK, urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1]))
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(
url=ig_link, headers=bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num +
1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(
filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(
searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(
successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __processSearch(self, reward):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
request = urllib2.Request(url = BING_NEWS_URL, headers = headers)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# generate a set of queries to run
bingQueriesGenerator = BingQueriesGenerator(searchesCount, history, reward.tp)
queries = bingQueriesGenerator.parseBingNews(page)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated !"
print "Requested:", searchesCount
print "Generated:", len(bingQueriesGenerator.queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesInterval)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query)
print "%s - %2d/%2d - Requesting: %s" % (helpers.getLoggingTime(), i, totalQueries, url)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = ( reward.tp == bfp.Reward.Type.SEARCH_PC and
page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 ) \
or ( reward.tp == bfp.Reward.Type.SEARCH_MOBILE and
page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 )
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
return res
def __processSearch(self, reward, verbose):
"""Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(
r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"'
)
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bdp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url=url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
reward.description)
if matches is None:
print "No RegEx matches found for this search and earn"
res.isError = True
res.message = "No RegEx matches found for this search and earn"
return res
maxRewardsCount = int(matches.group(1))
rewardsCount = int(matches.group(2))
rewardCost = 1 # Looks like it's now always X points per one search
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
headers = self.httpHeaders
if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(
0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(
0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!\nRequested: {}\nGenerated: {}\n"\
.format(searchesCount, len(queries))
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
totalQueries, query)
request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
print "Warning! Query:\n\t {}\n returned no results, check file for more information {}".format(
query, helpers.dumpErrorPage(page))
else:
successfullQueries += 1
self.randomClick(IG_NUMBER_PATTERN, IG_PING_LINK,
IG_SEARCHES_PATTERN, page, response, verbose)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(
searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(
successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __processSearch(self, reward, verbose):
"""Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bdp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matchesMobile = False
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
#Mobile description changed, so check that one too
if matches is None:
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE_MOBILE.search(reward.description)
matchesMobile = True
if matches is None:
print "No RegEx matches found for this search and earn"
res.isError = True
res.message = "No RegEx matches found for this search and earn"
return res
maxRewardsCount = int(matches.group(1))
rewardsCount = int(matches.group(2))
rewardCost = 1 # Looks like it's now always X points per one search
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
if matchesMobile == True:
#new mobile search description gives total search count + points per search for edge/non-edge
edgeValue = int(matches.group(1))
nonEdgeValue = int(matches.group(2))
searchesCount = int(matches.group(3))
#apparently ios uses EdgiOS, so we only check the first 3 letters not the full word 'edge'
if self.userAgents.mobile.lower().find("edg") != -1:
#we are searching on edge so points go to 200
searchesCount -= reward.progressCurrent / edgeValue
else:
#non-edge search so 100 is the max
searchesCount -= reward.progressCurrent / nonEdgeValue
headers = self.httpHeaders
if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number is not None:
ig_number = ig_number.group(1)
ig_searches = IG_SEARCHES_PATTERN.findall(page)
# make sure we have at least 1 search
if len(ig_searches) > 0:
# get a random link to open
ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
# number of the link we will use
ig_link_num = random.randint(0, ig_max_rand)
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK,
urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1])
)
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num + 1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! No searches were found on search results page"
print "Check {0} file for more information".format(filename)
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res