def test_timestamp(self):
"""
test getlogtime
:return:
"""
stamp = helpers.getLoggingTime()
self.assertRegexpMatches(stamp, "\d{4}-\d{2}-\d{2}", "should have time stamp,\n" + stamp)
def __run(config):
report = list()
doSleep = False
for key, account in config.accounts.iteritems():
if account.disabled:
continue
# sleep between two accounts logins
if doSleep:
extra = config.general.betweenAccountsInterval + random.uniform(
0, config.general.betweenAccountsSalt)
if verbose:
print("\nPausing between accounts for {0} seconds".format(
int(extra)))
time.sleep(extra)
reportItem = BingRewardsReportItem()
reportItem.accountType = account.accountType
reportItem.accountLogin = account.accountLogin
agents = bingCommon.UserAgents.generate(account)
httpHeaders = bingCommon.HEADERS
httpHeaders["User-Agent"] = agents.pc
__processAccount(config, httpHeaders, agents, reportItem,
account.password)
report.append(reportItem)
doSleep = True
#
# trigger full report if needed
#
if showFullReport or totalPoints > 0 and len(report) > 1:
print
print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= FINAL REPORT =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
print
print " Account | Before | After | Earned | Lifetime | Retries "
print "------------------------------+--------+--------+--------+------------+---------"
for r in report:
print " %-28s | %6d | %6d | %6d | %10d | %7d" % (
__stringifyAccount(r, 28), r.oldPoints, r.newPoints,
r.pointsEarnedRetrying, r.lifetimeCredits, r.retries)
print
#
# print footer
#
print "Total points earned: %d" % totalPoints
print
print "%s - script ended" % helpers.getLoggingTime()
EventsProcessor.onScriptComplete(config)
def __run(config):
report = list()
doSleep = False
for key, account in config.accounts.iteritems():
if account.disabled:
continue
# sleep between two accounts logins
if doSleep:
extra = config.general.betweenAccountsInterval + random.uniform(0, config.general.betweenAccountsSalt)
if verbose:
print("\nPausing between accounts for {0} seconds".format(int(extra)))
time.sleep(extra)
reportItem = BingRewardsReportItem()
reportItem.accountType = account.accountType
reportItem.accountLogin = account.accountLogin
agents = bingCommon.UserAgents.generate(account)
httpHeaders = bingCommon.HEADERS
httpHeaders["User-Agent"] = agents.pc
__processAccount(config, httpHeaders, agents, reportItem, account.password)
report.append(reportItem)
doSleep = True
#
# trigger full report if needed
#
if showFullReport or totalPoints > 0 and len(report) > 1:
print
print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= FINAL REPORT =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
print
print " Account | Before | After | Earned | Lifetime | Retries "
print "------------------------------+--------+--------+--------+------------+---------"
for r in report:
print " %-28s | %6d | %6d | %6d | %10d | %7d" % (__stringifyAccount(r, 28), r.oldPoints, r.newPoints, r.pointsEarnedRetrying, r.lifetimeCredits, r.retries)
print
#
# print footer
#
print "Total points earned: %d" % totalPoints
print
print "%s - script ended" % helpers.getLoggingTime()
EventsProcessor.onScriptComplete(config)
def __run(config):
report = list()
accountIndex = 0
for key, account in config.accounts.iteritems():
if account.disabled:
continue
# sleep between two accounts logins
if accountIndex > 0:
isFirstAccount = False
extra = config.general.betweenAccountsInterval + random.uniform(0, config.general.betweenAccountsSalt)
time.sleep(extra)
reportItem = BingRewardsReportItem()
reportItem.accountType = account.accountType
reportItem.accountLogin = account.accountLogin
__processAccount(config, reportItem, account.password)
report.append(reportItem)
accountIndex += 1
EventsProcessor.onScriptComplete(config)
#
# trigger full report if needed
#
if showFullReport or totalPoints > 0 and len(report) > 1:
print
print " -=-=-=-=-=-=-=-=-=-=--=-=- FULL REPORT -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-="
print
print " Account | Before | After | Earned | Retries | Lifetime Credits"
print "---------------------------+--------+--------+--------+---------+-----------------"
for r in report:
print " %25s | %6d | %6d | %6d | %7d | %16d" % (__stringifyAccount(r, 25), r.oldPoints, r.newPoints, r.pointsEarnedRetrying, r.retries, r.lifetimeCredits)
print
#
# print footer
#
print "Total points earned: %d" % totalPoints
print
print "%s - script ended" % helpers.getLoggingTime()
def __processQuiz(self, reward):
"""Processes bdp.Reward.Type.Action.QUIZ and returns self.RewardResult"""
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
pointsEarned = self.getRewardsPoints()
currPage = self.getDashboardPage()
startIndex = currPage.find('__RequestVerificationToken')
endIndex = currPage[startIndex:].find('/>')
#pad here to get to the correct spot
verificationAttr = currPage[startIndex + 49:startIndex + endIndex - 2]
#TODO: last parameter is ', "Timezone" : 240'. Is it needed?
verificationData = '{"ActivitySubType" : "quiz", "ActivityType" : "notification", "OfferId" : "' + reward.hitId + '", "Channel" : "Bing.Com", "PartnerId" : "BingTrivia"}'
verificationUrl = 'https://www.bing.com/msrewards/api/v1/ReportActivity'
print
print "Running activity: %s" % reward.name
print
request = urllib2.Request(url=verificationUrl,
headers=self.httpHeaders)
for i in range(reward.progressCurrent, reward.progressMax, 10):
print "%s - %2d/%2d - Activity: %s" % (helpers.getLoggingTime(),
i + 10, reward.progressMax,
reward.name)
with self.opener.open(request, verificationData) as response:
page = helpers.getResponseBody(response)
#default pause between guesses
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
pointsEarned = self.getRewardsPoints() - pointsEarned
# if QUIZ is against bdp.Reward.Type.RE_EARN_CREDITS - check if pointsEarned is the same to
# pointsExpected
indCol = bdp.Reward.Type.Col.INDEX
if reward.tp[indCol] == bdp.Reward.Type.RE_EARN_CREDITS[indCol]:
pointsExpected = reward.progressMax - reward.progressCurrent
if pointsExpected != pointsEarned:
filename = helpers.dumpErrorPage(page)
res.isError = True
res.message = "Expected to earn " + str(pointsExpected) + " points, but earned " + \
str(pointsEarned) + " points. Check " + filename + " for further information"
return res
def __run(config):
report = list()
doSleep = False
for key, account in config.accounts.iteritems():
if account.disabled:
continue
reportItem = __processAccountUserAgent(config, account,
bingCommon.USER_AGENTS_PC,
doSleep)
report.append(reportItem)
doSleep = True
#
# trigger full report if needed
#
if showFullReport or totalPoints > 0 and len(report) > 1:
print
print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= FINAL REPORT =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
print
print " Account | Before | After | Earned | Lifetime | Retries "
print "------------------------------+--------+--------+--------+------------+---------"
for r in report:
print " %-28s | %6d | %6d | %6d | %10d | %7d" % (
__stringifyAccount(r, 28), r.oldPoints, r.newPoints,
r.pointsEarnedRetrying, r.lifetimeCredits, r.retries)
print
#
# print footer
#
print "Total points earned: %d" % totalPoints
print
print "%s - script ended" % helpers.getLoggingTime()
EventsProcessor.onScriptComplete(config)
def __run(config):
report = list()
doSleep = False
#for key, account in config.accounts.iteritems():
for key, account in config.accounts.items():
if account.disabled:
continue
reportItem = __processAccountUserAgent(config, account, bingCommon.USER_AGENTS_PC, doSleep)
report.append(reportItem)
doSleep = True
#
# trigger full report if needed
#
if (showFullReport or totalPoints > 0 and len(report) > 1):
print
print (" -=-=-=-=-=-=-=-=-=-=--=-=- FULL REPORT -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-")
print
print (" Account | Before | After | Earned | Retries | Lifetime Credits")
print ("---------------------------+--------+--------+--------+---------+-----------------")
for r in report:
print (" %25s | %6d | %6d | %6d | %7d | %16d" % (__stringifyAccount(r, 25), r.oldPoints, r.newPoints, r.pointsEarnedRetrying, r.retries, r.lifetimeCredits))
print
#
# print footer
#
print ("Total points earned: %d" % totalPoints)
print
print ("%s - script ended" % helpers.getLoggingTime())
EventsProcessor.onScriptComplete(config)
def __processSearch(self, reward, verbose):
"""Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bdp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matchesMobile = False
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
#Mobile description changed, so check that one too
if matches is None:
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE_MOBILE.search(reward.description)
matchesMobile = True
if matches is None:
print "No RegEx matches found for this search and earn"
res.isError = True
res.message = "No RegEx matches found for this search and earn"
return res
maxRewardsCount = int(matches.group(1))
rewardsCount = int(matches.group(2))
rewardCost = 1 # Looks like it's now always X points per one search
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
if matchesMobile == True:
#new mobile search description gives total search count + points per search for edge/non-edge
edgeValue = int(matches.group(1))
nonEdgeValue = int(matches.group(2))
searchesCount = int(matches.group(3))
#apparently ios uses EdgiOS, so we only check the first 3 letters not the full word 'edge'
if self.userAgents.mobile.lower().find("edg") != -1:
#we are searching on edge so points go to 200
searchesCount -= reward.progressCurrent / edgeValue
else:
#non-edge search so 100 is the max
searchesCount -= reward.progressCurrent / nonEdgeValue
headers = self.httpHeaders
if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number is not None:
ig_number = ig_number.group(1)
ig_searches = IG_SEARCHES_PATTERN.findall(page)
# make sure we have at least 1 search
if len(ig_searches) > 0:
# get a random link to open
ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
# number of the link we will use
ig_link_num = random.randint(0, ig_max_rand)
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK,
urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1])
)
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num + 1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! No searches were found on search results page"
print "Check {0} file for more information".format(filename)
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
if o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-f", "--configFile"):
configFile = a
elif o in ("-r", "--full-report"):
showFullReport = True
elif o in ("-v", "--verbose"):
verbose = True
elif o == "--version":
printVersion()
sys.exit()
else:
raise NotImplementedError("option '" + o + "' is not implemented")
print "%s - script started" % helpers.getLoggingTime()
print "-" * 80
print
helpers.createResultsDir(__file__)
config = Config()
try:
config.parseFromFile(configFile)
except IOError, e:
print "IOError: %s" % e
sys.exit(2)
except ConfigError, e:
print "ConfigError: %s" % e
sys.exit(2)
if o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-f", "--configFile"):
configFile = a
elif o in ("-r", "--full-report"):
showFullReport = True
elif o in ("-v", "--verbose"):
verbose = True
elif o == "--version":
printVersion()
sys.exit()
else:
raise NotImplementedError("option '" + o + "' is not implemented")
print "%s - script started" % helpers.getLoggingTime()
print "-" * 80
print
helpers.createResultsDir(__file__)
config = Config()
try:
config.parseFromFile(configFile)
except IOError, e:
print "IOError: %s" % e
sys.exit(2)
except ConfigError, e:
print "ConfigError: %s" % e
sys.exit(2)
def __processSearch(self, reward):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url=url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
totalQueries, query)
request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(
searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(
successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
def __processSearch(self, reward, verbose):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content"'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content"'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"')
IG_SEARCH_RESULTS_PATTERN = re.compile(r'<ol\s.*?id="b_results"(.+?)</ol>')
IG_SEARCHS_PATTERN = re.compile(r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number != None:
ig_number = ig_number.group(1)
# get search results
ig_results = IG_SEARCH_RESULTS_PATTERN.search(page)
if ig_results != None:
ig_results = ig_results.group(1)
# seperate search results
ig_searches = IG_SEARCHS_PATTERN.findall(ig_results)
# make sure we have at least 1 search
if len(ig_searches) > 0:
# get a random link to open
ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
ig_link_num = random.randint(0, ig_max_rand)
# number of the link we will use
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK,
urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1])
)
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num + 1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! No searches were found on search results page"
print "Check {0} file for more information".format(filename)
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __processSearch(self, reward, verbose):
"""Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(
r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"'
)
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bdp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url=url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
reward.description)
if matches is None:
print "No RegEx matches found for this search and earn"
res.isError = True
res.message = "No RegEx matches found for this search and earn"
return res
maxRewardsCount = int(matches.group(1))
rewardsCount = int(matches.group(2))
rewardCost = 1 # Looks like it's now always X points per one search
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
headers = self.httpHeaders
if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(
0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(
0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!\nRequested: {}\nGenerated: {}\n"\
.format(searchesCount, len(queries))
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
totalQueries, query)
request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
print "Warning! Query:\n\t {}\n returned no results, check file for more information {}".format(
query, helpers.dumpErrorPage(page))
else:
successfullQueries += 1
self.randomClick(IG_NUMBER_PATTERN, IG_PING_LINK,
IG_SEARCHES_PATTERN, page, response, verbose)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(
searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(
successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __processSearch(self, reward, verbose):
"""Processes bdp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"([^"]+)"')
IG_SEARCHES_PATTERN = re.compile(r'<li\s[^>]*class="b_algo"[^>]*><h2><a\s[^>]*href="(http[^"]+)"\s[^>]*h="([^"]+)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bdp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matchesMobile = False
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
#Mobile description changed, so check that one too
if matches is None:
matches = bdp.Reward.Type.SEARCH_AND_EARN_DESCR_RE_MOBILE.search(reward.description)
matchesMobile = True
if matches is None:
print "No RegEx matches found for this search and earn"
res.isError = True
res.message = "No RegEx matches found for this search and earn"
return res
maxRewardsCount = int(matches.group(1))
rewardsCount = int(matches.group(2))
rewardCost = 1 # Looks like it's now always X points per one search
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
# reward.progressCurrent is now returning current points, not current searches
# so divide it by points per search (rewardsCount) to get correct search count needed
searchesCount -= (reward.progressCurrent * rewardCost) / rewardsCount
if matchesMobile == True:
#new mobile search description gives total search count + points per search for edge/non-edge
edgeValue = int(matches.group(1))
nonEdgeValue = int(matches.group(2))
searchesCount = int(matches.group(3))
#apparently ios uses EdgiOS, so we only check the first 3 letters not the full word 'edge'
if self.userAgents.mobile.lower().find("edg") != -1:
#we are searching on edge so points go to 200
searchesCount -= reward.progressCurrent / edgeValue
else:
#non-edge search so 100 is the max
searchesCount -= reward.progressCurrent / nonEdgeValue
headers = self.httpHeaders
if reward.tp == bdp.Reward.Type.SEARCH_PC or reward.tp == bdp.Reward.Type.SEARCH_AND_EARN:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bdp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
if verbose:
print("User-Agent: {0}".format(bingCommon.HEADERS["User-Agent"]))
print
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i, totalQueries, query)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number is not None:
ig_number = ig_number.group(1)
ig_searches = IG_SEARCHES_PATTERN.findall(page)
# make sure we have at least 1 search
if len(ig_searches) > 0:
# get a random link to open
ig_max_rand = min(self.openTopLinkRange, len(ig_searches) - 1)
# number of the link we will use
ig_link_num = random.randint(0, ig_max_rand)
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK,
urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1])
)
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(url = ig_link, headers = bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num + 1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! No searches were found on search results page"
print "Check {0} file for more information".format(filename)
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
def __processSearch(self, reward):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url = url, headers = self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
request = urllib2.Request(url = BING_NEWS_URL, headers = headers)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# generate a set of queries to run
bingQueriesGenerator = BingQueriesGenerator(searchesCount, history, reward.tp)
queries = bingQueriesGenerator.parseBingNews(page)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated !"
print "Requested:", searchesCount
print "Generated:", len(bingQueriesGenerator.queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(0, self.betweenQueriesInterval)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query)
print "%s - %2d/%2d - Requesting: %s" % (helpers.getLoggingTime(), i, totalQueries, url)
request = urllib2.Request(url = url, headers = bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = ( reward.tp == bfp.Reward.Type.SEARCH_PC and
page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 ) \
or ( reward.tp == bfp.Reward.Type.SEARCH_MOBILE and
page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1 )
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(successfullQueries) + " requests were successfully processed"
return res
def __processSearch(self, reward, verbose):
"""Processes bfp.Reward.Type.Action.SEARCH and returns self.RewardResult"""
BING_QUERY_URL = 'http://www.bing.com/search?q='
BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC = '<div id="b_content">'
BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE = '<div id="content">'
IG_PING_LINK = "http://www.bing.com/fd/ls/GLinkPing.aspx"
IG_NUMBER_PATTERN = re.compile(r'IG:"(.+?)"')
IG_SEARCH_RESULTS_PATTERN = re.compile(
r'<ol\s.*?id="b_results"(.+?)</ol>')
IG_SEARCHS_PATTERN = re.compile(
r'<a\s.*?href="(http.+?)".*?\sh="(.+?)"')
res = self.RewardResult(reward)
if reward.isAchieved():
res.message = "This reward has been already achieved"
return res
indCol = bfp.Reward.Type.Col.INDEX
# get a set of queries from today's Bing! history
url = bingHistory.getBingHistoryTodayURL()
request = urllib2.Request(url=url, headers=self.httpHeaders)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
history = bingHistory.parse(page)
# find out how many searches need to be performed
matches = bfp.Reward.Type.SEARCH_AND_EARN_DESCR_RE.search(
reward.description)
rewardsCount = int(matches.group(1))
rewardCost = int(matches.group(2))
maxRewardsCount = int(matches.group(4))
searchesCount = maxRewardsCount * rewardCost / rewardsCount
# adjust to the current progress
searchesCount -= reward.progressCurrent * rewardCost
headers = self.httpHeaders
if reward.tp == bfp.Reward.Type.SEARCH_PC:
headers["User-Agent"] = self.userAgents.pc
searchesCount += self.addSearchesDesktop + random.randint(
0, self.addSearchesDesktopSalt)
print
print "Running PC searches"
print
elif reward.tp == bfp.Reward.Type.SEARCH_MOBILE:
headers["User-Agent"] = self.userAgents.mobile
searchesCount += self.addSearchesMobile + random.randint(
0, self.addSearchesMobileSalt)
print
print "Running mobile searches"
print
else:
res.isError = True
res.message = "Don't know how to process this search"
return res
# Import the query generator
try:
qg = importlib.import_module(self.queryGenerator, package=None)
queryGenerator = qg.queryGenerator(self)
except ImportError:
raise TypeError("{0} is not a module".format(self.queryGenerator))
# generate a set of queries to run
queries = queryGenerator.generateQueries(searchesCount, history)
if len(queries) < searchesCount:
print "Warning: not enough queries to run were generated!"
print "Requested:", searchesCount
print "Generated:", len(queries)
successfullQueries = 0
i = 1
totalQueries = len(queries)
for query in queries:
if i > 1:
# sleep some time between queries (don't worry Bing! ;) )
t = self.betweenQueriesInterval + random.uniform(
0, self.betweenQueriesSalt)
time.sleep(t)
url = BING_QUERY_URL + urllib.quote_plus(query.encode('utf-8'))
print "%s - %2d/%2d - Search: %s" % (helpers.getLoggingTime(), i,
totalQueries, query)
request = urllib2.Request(url=url, headers=bingCommon.HEADERS)
with self.opener.open(request) as response:
page = helpers.getResponseBody(response)
# check for the successfull marker
found = page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_PC) != -1 \
or page.find(BING_QUERY_SUCCESSFULL_RESULT_MARKER_MOBILE) != -1
if not found:
filename = helpers.dumpErrorPage(page)
print "Warning! Query:"
print "\t" + query
print "returned no results, check " + filename + " file for more information"
else:
successfullQueries += 1
# randomly open a link
if self.openLinkChance > random.random():
# get IG number
ig_number = IG_NUMBER_PATTERN.search(page)
if ig_number != None:
ig_number = ig_number.group(1)
# get search results
ig_results = IG_SEARCH_RESULTS_PATTERN.search(page)
if ig_results != None:
ig_results = ig_results.group(1)
# seperate search results
ig_searches = IG_SEARCHS_PATTERN.findall(
ig_results)
# get a random link to open
ig_max_rand = min(self.openTopLinkRange,
len(ig_searches) - 1)
ig_link_num = random.randint(0, ig_max_rand)
# number of the link we will use
ig_link = "{0}?IG={1}&{2}".format(
IG_PING_LINK, urllib.quote_plus(ig_number),
urllib.quote_plus(ig_searches[ig_link_num][1]))
# sleep a reasonable amount of time before clicking the link
# use defaults to save space in config
t = random.uniform(0.75, 3.0)
time.sleep(t)
# open the random link
request = urllib2.Request(
url=ig_link, headers=bingCommon.HEADERS)
request.headers["Referer"] = response.url
self.opener.open(request)
if verbose:
print("Followed Link {}".format(ig_link_num +
1))
else:
filename = helpers.dumpErrorPage(page)
print "Warning! Could not find search result IG number"
print "Check {0} file for more information".format(
filename)
i += 1
if successfullQueries < searchesCount:
res.message = str(successfullQueries) + " out of " + str(
searchesCount) + " requests were successfully processed"
else:
res.message = "All " + str(
successfullQueries) + " requests were successfully processed"
# reset header to pc so pc pages return in getting life time points
headers["User-Agent"] = self.userAgents.pc
return res
