def next_request(self, proxy_param, exec_time_param):
now = time()
if self.use_proxies:
# If there are still some URLs to be loaded with the same proxy
if len(self.urls_to_crawl) > 0:
# We load the next URL that has been assigned to the same proxy as the one used for the request
# that triggered this reply
# But first, let us compute whether this next request should be executed right now or not (in case
# the proxy would have been exceptionally faster than the between-requests delay)
delta = now - exec_time_param
l("delta=", delta)
if delta < PROXY_SPECIFIC_REQUEST_DELAY:
l("Adding a delay to respect between-requests delay for proxy", proxy_param)
# The difference between the required delta and the current delta
# Note: Randomness is in order not to be detected as bot by having a predictable behaviour
r = randu(0.5, 1.5)
l("Randomness factor r=", r)
exec_time_delay = r * (PROXY_SPECIFIC_REQUEST_DELAY - delta)
exec_time = now + exec_time_delay # Approximation of execution time
else:
exec_time = now
exec_time_delay = 0
# Registering the fact that this proxy is working
# (we are building a new request for it = it returned some request = it's working)
self.alive_proxy(proxy_param)
next_req = self.build_request_for_proxy(proxy_param, exec_time)
if exec_time_delay is 0:
l("Returning request ", next_req.url, "without a delay")
yield next_req
else:
l("Storing request", next_req.url, "for later")
self.pending_requests[next_req.url] = next_req
elif self.SERPS_PER_QUERY > 1:
yield self.build_request_for_proxy(proxy_param, now)
else:
yield None
# from CS1110 9B
"""
This module estimates pi by simulating a dart throw game.
A large number of darts are thrown at a square with vertices
(1,-1), (1,1),(-1,1) and (-1,-1). The darts land anywhere
in the square with equal probability. If the dart lands inside
the unit circle we call that a "hit". For a large number of throws
N, the the ratio of hits to throws should approximately equal
the ratio of the area of the circle to the area of the square.
Thus pi/4 should approximately equal hits/N.
"""
# What we need from the standard module random
from random import uniform as randu
from random import seed
N = 1000000 # Number of darts to throw
seed(0) # For repeatability of experiments
Hits = 0
for throws in range(N):
# Generate and check the k-th dart throw
x = randu(-1, 1)
y = randu(-1, 1)
if x ** 2 + y ** 2 <= 1:
# Inside the unit circle
Hits += 1
piEst = 4 * (float(Hits) / float(N))
print "\nTotal number of dart throws = %1d" % N
print "Pi Estimate = %7.5f" % piEst
Suppose n points are placed uniformly at random in the unit square.
We want to know the expected minimum distance between these points.
This program perfoms a Monte-Carlo simulation of this and graphs
the distance versus n.
"""
from random import uniform as randu
from scipy.spatial.distance import pdist
from matplotlib import pyplot as plt
nList = [100, 200, 300, 400, 500] # values for n
numTrials = 1000
expectedValues = []
for n in nList:
minDistances = []
for trial in range(numTrials):
points = [[randu(0, 1), randu(0, 1)] for x in range(n)]
minDistances.append(min(pdist(points)))
expectedValues.append(sum(minDistances) / n)
plt.plot(nList, expectedValues)
plt.show()
for count in range(len(nList)):
print("The expected value for n = " + str(nList[count]) + " is about " +
str(expectedValues[count]))
from random import randint as randi
from random import uniform as randu
N = 100000
#x in[10,20]or[45,70]
count = 0
for k in range(N):
x = randu(0, 100)
if 10 <= x <= 20 or 45 <= x <= 70:
count += 1
print(float(count) / float(N))
count = 0
for k in range(N):
x = randu(0, 100)
if 10 <= x <= 45 and 20 <= x <= 60:
count += 1
print(float(count) / float(N))
count = 0
for k in range(N):
x = randi(1, 20)
if x % 3 == 1 or x % 5 == 0:
count += 1
print(float(count) / float(N))
count = 0
for k in range(N):
x = randi(1, 20)
if 13 <= x < 17:
count += 1
