def get_proxy_list(valid_proxies):
from http_request_randomizer.requests.proxy.requestProxy import RequestProxy
import requests
import concurrent.futures
req_proxy = RequestProxy() #you may get different number of proxy when you run this at each time
proxies_list = req_proxy.get_proxy_list() #this will create proxy list
proxies = []
for prox in proxies_list:
proxies.append(prox.get_address())
print(proxies)
def extract(proxy):
try:
r = requests.get('https://httpbin.org/ip', proxies={'http':"//" + proxy,'https':"//" +proxy},timeout = 1)
print(r.json(),'-working')
valid_proxies.append(proxy)
except Exception as e:
pass
return proxy
extract("203.115.112.218:3128")
print("se llego")
with concurrent.futures.ThreadPoolExecutor() as exector:
exector.map(extract,proxies)
return valid_proxies
def botStartChrome():
print("give me a bottle of rum!")
req_proxy = RequestProxy() #you may get different number of proxy when you run this at each time
proxies = req_proxy.get_proxy_list()
number = random.randint(0,50)
print("rand int ==== " + str(number))
PROXY = proxies[number].get_address()
print(proxies[number].get_address())
print(proxies[number].country)
webdriver.DesiredCapabilities.CHROME['proxy']={
"httpProxy":PROXY,
"ftpProxy":PROXY,
"sslProxy":PROXY,
"proxyType":"MANUAL",
}
driver = webdriver.Firefox(executable_path='chromedriver')
try:
driver.get('https://temanpost.com/games/daftar-game-berbayar-yang-bisa-diklaim-gratis-pada-agustus-2020/')
time.sleep(30)
except:
print("error something wrong")
driver.quit()
def make_request():
req_proxy = RequestProxy(
) #you may get different number of proxy when you run this at each time
proxies = req_proxy.get_proxy_list() #this will create proxy list
ls = []
for prox in proxies:
ls.append(prox.get_address())
save_prox(ls)
class _Proxy:
ip = re.compile(r"\d+\.\d+\.\d+\.\d+")
@staticmethod
def get_ip():
return _Proxy.ip.search(
requests.get('http://icanhazip.com/').text).group(0)
def __init__(self, conf, file=None):
self.config = conf
self.req_proxy = RequestProxy(protocol=Protocol.HTTP)
if file is not None:
self.proxies_list = self.config["proxies"]
else:
self.proxies_list = self.req_proxy.get_proxy_list()
def get_proxy(self):
logger = logging.getLogger(f"pid={os.getpid()}")
while True:
p = self.proxies_list.pop(0).get_address()
try:
logger.info(f"Trying {p}")
proxy = {"http": f"http://{p}", "https": f"https://{p}"}
ip = _Proxy.ip.search(
requests.get("http://icanhazip.com/",
proxies=proxy,
timeout=2).text)
if ip.group(0) is None:
raise Exception()
if ip.group(0) == self.get_ip():
raise Exception()
if requests.get("http://google.com/", proxies=proxy,
timeout=5).status_code != 200:
raise Exception()
return p
except IndexError:
logger.info(f"Loading more proxies")
self.proxies_list = self.req_proxy.get_proxy_list()
def __init__(self, url='https://youtube.com', proxy=None, verbose=False):
""" init variables """
self.url = url
self.proxy = proxy
self.verbose = verbose
# All chrome options
# https://peter.sh/experiments/chromium-command-line-switches/
self.options = webdriver.ChromeOptions()
# Run in headless mode, without a UI or display server dependencies
self.options.add_argument('--headless')
# Disables GPU hardware acceleration. If software renderer is not in
# place, then the GPU process won't launch
self.options.add_argument('--disable-gpu')
# Disable audio
self.options.add_argument('--mute-audio')
# Runs the renderer and plugins in the same process as the browser
self.options.add_argument('--single-process')
# Autoplay policy
self.options.add_argument('--autoplay-policy=no-user-gesture-required')
if self.proxy:
# Uses a specified proxy server, overrides system settings. This
# switch only affects HTTP and HTTPS requests
self.options.add_argument('--proxy-server={0}'.format(self.proxy))
# A string used to override the default user agent with a custom one
req_proxy = RequestProxy(
) #you may get different number of proxy when you run this at each time
proxies = req_proxy.get_proxy_list() #this will create proxy list
PROXY = proxies[0].get_address()
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": PROXY,
"ftpProxy": PROXY,
"sslProxy": PROXY,
"proxyType": "MANUAL",
}
self.user_agent = utils.user_agent()
self.options.add_argument('--user-agent={0}'.format(self.user_agent))
self.browser = webdriver.Chrome(ChromeDriverManager().install(),
options=self.options)
self.default_timeout = 20
# Specifies the amount of time the driver should wait when trying to
# find any element (or elements) if it is not immediately available.
# The default setting is 0. Once set, the implicit wait is set for the
# life of the WebDriver object.
self.browser.implicitly_wait(self.default_timeout)
# Set the amount of time to wait for a page load to complete before
# throwing an error.
# self.browser.set_page_load_timeout(self.default_timeout)
# Set the amount of time that the script should wait during an
# execute_async_script call before throwing an error.
# self.browser.set_script_timeout(self.default_timeout)
# Sets the width and height of the current window$
self.browser.set_window_size(1920, 1080)
# Opens the page
self.open_url()
def proxies():
if os.path.isfile('proxies.txt'):
pass
else:
req_proxy = RequestProxy()
PROXIES = "{0}".format(
list(map(lambda x: x.get_address(), req_proxy.get_proxy_list())))
# Open the file for writing
F = open('proxies.txt', 'w')
F.writelines(PROXIES)
F.close()
pass
def __init__(self,
db,
path_to_webdriver,
config=None,
logger=None,
cookies=None):
self.logger = logger
self.logger.info("webdriver path: ".format(path_to_webdriver))
self.config = config
chrome_options = ChromeOption()
prefs = {"profile.default_content_setting_values.notifications": 2}
chrome_options.add_experimental_option("prefs", prefs)
# ignore error proxy
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--ignore-ssl-errors')
# automatically dismiss prompt
chrome_options.set_capability('unhandledPromptBehavior', 'dismiss')
self.driver = webdriver.Chrome(path_to_webdriver,
chrome_options=chrome_options)
# get PROXY
req_proxy = RequestProxy()
proxies = req_proxy.get_proxy_list()
# set PROXY
PROXY = proxies[0].get_address()
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": PROXY,
"ftpProxy": PROXY,
"sslProxy": PROXY,
"proxyType": "MANUAL",
}
if cookies is None:
self.cookies = self.driver.get_cookies()
else:
for cookie in cookies:
self.driver.add_cookie(cookie)
self.cookies = cookies
self.db = db
class Proxies:
def __init__(self):
self.req_proxy = RequestProxy()
self.proxies = self.req_proxy.get_proxy_list(
) # this will create proxy list
self.total = len(self.proxies)
self.getExtraProxies()
pass
def getExtraProxies(self):
addrs = []
with open("proxy_list.csv", 'r') as f:
lines = f.readlines()
for line in lines:
items = line.split()
if len(items) < 2:
continue
if len(items[0].split('.')) == 4:
addrs.append(items[0] + ":" + items[1])
self.addrs = addrs
def getProxy(self):
# if(self.pointer+1 > self.total):
# self.renewProxies()
# self.pointer = 0
rand_n = random.randint(0, self.total - 1)
prox = self.proxies[rand_n]
if self.isValid(prox) == False:
rand_n = random.randint(0, len(self.addrs) - 1)
return self.addrs[rand_n]
# while self.isValid(prox) == False:
# rand_n = random.randint(0, self.total-1)
# prox = self.proxies[rand_n]
# print(prox.get_address())
# pass
# self.pointer += 1
return prox
def isValid(self, prox):
checker = ProxyChecker()
return checker.check_proxy(prox.get_address())
def getAddresses(self):
addrs = [x.get_address() for x in self.proxies]
return addrs
def working_proxies(countries_list=None):
req_proxy = RequestProxy(
) #you may get different number of proxy when you run this at each time
proxies = req_proxy.get_proxy_list() #this will create proxy list
##################################preparation of proxy list###############################
proxies_address = list(map(lambda x: x.get_address(), proxies))
proxies_countries = list(map(lambda x: str(x.country), proxies))
df = pd.DataFrame({'countries': proxies_countries, 'IP': proxies_address})
#obtain the asia countries want to keep
#countries=Counter(proxies_countries)
#for key, value in countries.items():
# print(key, value)
if countries_list is None:
countries_list = [ #'Indonesia' # total 23, 8/20
#'Cambodia' # total 8, 3/8
#'India' # total 23, 7/20
'Hong Kong', # total 10, 6/10 chosen
'Thailand', # total 26, 10/20 chosen
#'Nepal' # total 11, 3/11
#'Myanmar', # total 4, 2/4
#'Bangladesh' # total 12, 2/12
#'Philippines' # total 3, 1/3
#'Singapore' # total 7, 2/7
#'Mongolia' # total 4, 0/4
'Vietnam', # total 11, 6/11 chosen
'Pakistan', # total 14, 7/14 chosen
#'Japan' # total 11, 1/11
#'Korea' # total 1, 0/1
#'China' # total 7, 0/7
'Macau' # total 3, 3/3 chosen
]
df = df[df['countries'].isin(
countries_list)] #a df, coulumn: countries, IP addresses
b = in_list(countries_list, df)
print_results(b)
return df
class RequestMaker:
def __init__(self):
self.req_proxy = RequestProxy()
def _generate_proxied_request(self, url, params=None):
if params is None:
params = {}
for _ in range(0, len(self.req_proxy.get_proxy_list())):
proxy_response = self.req_proxy.generate_proxied_request(
url, params=params)
if proxy_response is not None:
return proxy_response
return None
def get(self, url, params=None):
proxy_response = self._generate_proxied_request(url, params)
if proxy_response is None:
raise RuntimeError(
'Failed to generate proxied request for {}'.format(url))
return proxy_response
def Run(request):
if request.method == 'POST':
if request.POST.get('param'):
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import ElementClickInterceptedException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium.webdriver.support.ui import Select
import random
from http_request_randomizer.requests.proxy.requestProxy import RequestProxy
from Home.models import Emails
from selenium.webdriver.remote.webdriver import WebDriver
#webdriver local location
path = "C:\Program Files\chromedriver.exe"
#list of tags and urls
URLs = [
[
"http://snappysurveys.net/", "email", "started",
"first_name", "last_name", "dob_month", "dob_day",
"dob_year", "addr_full", "addr_city",
"addr_state_province", "addr_zip", "addr_phone", "offers1",
"offers2", "offers3", "offers4", "offers5", "submitBtn"
],
[
"https://youreducationfuture.com/", "firstname",
"lastname", "address", "city", "state", "email",
"areacode", "phone", "btn btn-large btn-primary"
],
[
"https://www.nationalpayday.com/", "first_name", "email",
"amount", "option", "submit"
],
[
"http://mycharityshopping.com/", "fname", "lname",
"exampleInputEmail1", "pwd1", "pwd2", "checkbox"
],
[
"http://mortgageloans101.com/index.php/mortgage-quote-form/",
"wpforms[fields][9]", "wpforms[fields][10]",
"wpforms[fields][18]", "wpforms[fields][14]",
"wpforms[fields][7]", "wpforms[fields][15]",
"wpforms[fields][0][first]", "wpforms[fields][0][last]",
"wpforms[fields][3]", "wpforms[fields][2]",
"wpforms-107-field_12_1", "wpforms[submit]"
],
[
"http://kidsdineforfree.com/", "fname", "lname", "email",
"pwd1", "pwd2", "newsletter", "frmaction"
],
[
"http://emortgagefinders.com/", "input_5", "input_6",
"input_50", "input_8", "input_10", "input_51",
"input_12.3", "input_12.6", "input_14",
"gform_next_button_6_1"
],
[
"http://consumerofferstore.com/", "fname", "lname",
"email", "contact", "state", "city", "country", "checkbox",
"checkbox1", "a-b3xqfy75bf3j", "Submit"
]
]
#lise of american fake identities
Identities = [
[
"Mary", "M.Pfister", "NewYork", "Huntington",
"4662 Duncan Avenue", "+1 610-934-1119", "11743"
],
[
"Raymond", "M.Gamboa", "Kentucky", "Owensboro",
"4072 Coffman Alley", "+1 270-691-3671", "42301"
],
[
"Pamela", "K.Smith", "Georgia", "Atlanta",
"1707 Musgrave Street", "+1 404-934-8171", "30303"
],
[
"Nadine", "B.Lowe", "Arizona", "Superstition",
"423 East Avenue", "+1 480-358-3654", "85207"
],
[
"Oscar", "L.Merrill", "Georgia", "Atlanta",
"411 Pine Garden Lane", "+1 770-741-7993", "30305"
],
[
"Theresa", "K.Johnson", "Florida", "Sunrise",
"1116 Ridenour Street", "+1 786-306-3113", "33323"
],
[
"Theodore", "J.Mejia", "Georgia", "Atlanta",
"2207 Edington Drive", "+1 678-799-9599", "30303"
],
[
"Kenneth", "E.Pabon", "Maryland", "Sykesville",
"15 Woodhill Avenue", "+1 410-795-2288", "21784"
],
[
"Juanita", "J.Perry", "Iowa", "Des Moines",
"4372 Southern Avenue", "+1 641-328-8365", "50309"
],
[
"Pamela", "J.Chancellor", "Iowa", "Westside",
"2497 Centennial Farm Road", "+1 712-663-4676", "51467"
],
[
"Mack", "P.King", "California", "Burbank",
"2181 Quiet Valley Lane", "+1 818-972-1068", "91502"
],
[
"Chris", "M.Bibb", "Ohio", "Dayton", "1580 College Avenue",
"+1 937-434-9264", "45459"
],
[
"Dorothy", "J.Honeycutt", "New Jersey", "Camden",
"939 Valley Street", "+1 856-885-6555", "08102"
],
[
"Scott", "E.Brown", "California", "Bakersfield",
"179 Atha Drive", "+1 661-586-6085", "93304"
],
[
"Barry", "L.Murchison", "Kentucky", "Pleasant Ridge",
"2210 Broaddus Avenue", "+1 270-275-3710", "40769"
],
[
"Maye", "L.Moseley", "Michigan", "Grand Rapids",
"916 Goff Avenue", "+1 269-589-1746", "49503"
],
[
"Jerry", "Y.Winn", "Tennessee", "Portland",
"422 Frum Street", "+1 615-325-8391", "37148"
],
[
"Andrew", "N.Jones", "Ohio", "Cincinnati",
"2576 Goldie Lane", "+1 513-374-9889", "45214"
],
[
"Timothy", "B.Frye", "California", "Sherman Oaks",
"3789 Par Drive", "+1 805-808-3371", "91403"
],
[
"Kevin", "D.Carrillo", "Alabama", "Opelika",
"1774 Fleming Street", "+1 334-364-1184", "36801"
]
]
#["fname","lname","state","city","adress","phone number"]
#list of e-mails
emails = [
1, 2, 3, 4, 5, 6, 7, 8, 9, 7, 8, 9, 7, 8, 9, 7, 8, 9, 7, 8, 9,
7, 8, 9, 7, 8, 9
]
#list of proxies
PROXIES = [
'0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0',
'0', '0', '0', '0', '0', '0', '0', '0'
]
req_proxy = RequestProxy(
) #you may get different number of proxy when you run this at each time
proxies = req_proxy.get_proxy_list() #this will create proxy list
for i in range(20):
PROXIES[i] = proxies[i].get_address()
count = Emails.objects.all().count()
emails[0] = Emails.objects.first().Email
k = Emails.objects.first().pk
k = k + 1
for j in range(1, count - 1):
if Emails.objects.get(pk=k):
emails[j] = Emails.objects.get(pk=k).Email
k = k + 1
else:
k = k + 1
#Registration in 1st website
for email in emails:
proxy = random.choice(PROXIES)
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"proxyType": "MANUAL",
}
i = random.randint(0, 20)
browser = webdriver.Chrome(path)
browser.maximize_window()
browser.get(URLs[0][0])
time.sleep(2)
element = browser.find_element_by_name(URLs[0][1])
element.send_keys(email)
element = browser.find_element_by_class_name(URLs[0][2])
element.click()
window_after = browser.window_handles[1]
browser.switch_to.window(window_after)
time.sleep(5)
element = browser.find_element_by_name(URLs[0][3])
element.send_keys(Identities[i][0])
element = browser.find_element_by_name(URLs[0][4])
element.send_keys(Identities[i][1])
browser.find_element_by_xpath(
"//select[@name='dob_month']/option[text()='December']"
).click()
browser.find_element_by_xpath(
"//select[@name='dob_day']/option[text()='1']").click()
browser.find_element_by_xpath(
"//select[@name='dob_year']/option[text()='2000']").click(
)
element = browser.find_element_by_name(URLs[0][8])
element.send_keys(Identities[i][4])
element = browser.find_element_by_name(URLs[0][9])
element.send_keys(Identities[i][3])
element = browser.find_element_by_name(URLs[0][10])
element.send_keys(Identities[i][2])
element = browser.find_element_by_name(URLs[0][11])
element.send_keys(Identities[i][6])
element = browser.find_element_by_name(URLs[0][12])
element.send_keys(Identities[i][5])
element = browser.find_element_by_name(URLs[0][13])
element.click()
element = browser.find_element_by_name(URLs[0][14])
element.click()
element = browser.find_element_by_name(URLs[0][15])
element.click()
element = browser.find_element_by_name(URLs[0][16])
element.click()
browser.find_element_by_xpath(
"//input[@name='offers4' and @value=5]").click()
element = browser.find_element_by_name(URLs[0][18])
element.click()
browser.close()
#Registration in 2nd website
for email in emails:
proxy = random.choice(PROXIES)
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"proxyType": "MANUAL",
}
i = random.randint(0, 20)
browser = webdriver.Chrome(path)
browser.get(URLs[1][0])
time.sleep(2)
element = browser.find_element_by_name(URLs[1][1])
element.send_keys(Identities[i][0])
element = browser.find_element_by_name(URLs[1][2])
element.send_keys(Identities[i][1])
element = browser.find_element_by_name(URLs[1][3])
element.send_keys(Identities[i][4])
element = browser.find_element_by_name(URLs[1][4])
element.send_keys(Identities[i][3])
element = browser.find_element_by_name(URLs[1][5])
element.send_keys(Identities[i][2])
element = browser.find_element_by_name(URLs[1][6])
element.send_keys(email)
element = browser.find_element_by_name(URLs[1][7])
element.send_keys('907')
element = browser.find_element_by_name(URLs[1][8])
element.send_keys(Identities[i][5])
element = browser.find_element_by_class_name(URLs[1][9])
element.click()
browser.close()
#Registration in 3nd website
for email in emails:
proxy = random.choice(PROXIES)
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"proxyType": "MANUAL",
}
i = random.randint(0, 20)
browser = webdriver.Chrome(path)
browser.get(URLs[2][0])
time.sleep(2)
element = browser.find_element_by_name(URLs[2][1])
element.send_keys(Identities[i][0])
element = browser.find_element_by_name(URLs[2][2])
element.send_keys(email)
browser.find_element_by_xpath(
"//select[@name='amount']/option[text()='$600']").click()
element = browser.find_element_by_name(URLs[2][5])
element.click()
browser.close()
#Registration in 4th website
for email in emails:
proxy = random.choice(PROXIES)
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"proxyType": "MANUAL",
}
i = random.randint(0, 20)
browser = webdriver.Chrome(path)
browser.get(URLs[3][0])
time.sleep(2)
element = browser.find_element_by_name(URLs[3][1])
#element.send_keys(Identities[i][0])
element = browser.find_element_by_name(URLs[3][2])
element.send_keys(Identities[i][1])
element = browser.find_element_by_id(URLs[3][3])
element.send_keys(email)
element = browser.find_element_by_name(URLs[3][4])
element.send_keys("password")
element = browser.find_element_by_name(URLs[3][5])
element.send_keys("password")
element = browser.find_element_by_class_name(URLs[3][6])
element.click()
element = browser.find_element_by_xpath(
"//button[@type='submit' and @class='btn btn-orange']")
element.click()
browser.close()
#Registration in 5th website
for email in emails:
proxy = random.choice(PROXIES)
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"proxyType": "MANUAL",
}
i = random.randint(0, 20)
browser = webdriver.Chrome(path)
browser.get(URLs[4][0])
time.sleep(2)
browser.find_element_by_xpath(
"//select[@name='wpforms[fields][9]']/option[text()='Refinance']"
).click()
browser.find_element_by_xpath(
"//select[@name='wpforms[fields][10]']/option[text()='Condo']"
).click()
browser.find_element_by_xpath(
"//select[@name='wpforms[fields][18]']/option[text()='Alabama']"
).click()
browser.find_element_by_xpath(
"//select[@name='wpforms[fields][14]']/option[text()='$50,000 - $60,000']"
).click()
browser.find_element_by_xpath(
"//select[@name='wpforms[fields][7]']/option[text()='Excellent (800+)']"
).click()
browser.find_element_by_xpath(
"//select[@name='wpforms[fields][15]']/option[text()='Yes']"
).click()
element = browser.find_element_by_name(URLs[4][7])
element.send_keys(Identities[i][0])
element = browser.find_element_by_name(URLs[4][8])
element.send_keys(Identities[i][1])
element = browser.find_element_by_name(URLs[4][9])
element.send_keys(Identities[i][5])
element = browser.find_element_by_name(URLs[4][10])
element.send_keys(email)
element = browser.find_element_by_xpath(
"//input[@type='checkbox' and @id='wpforms-107-field_12_1']"
)
element.click()
#element = browser.find_element_by_id(URLs[4][11])
#element.click()
element = browser.find_element_by_name(URLs[4][12])
element.click()
browser.close()
#Registration in 6th website
for email in emails:
proxy = random.choice(PROXIES)
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"proxyType": "MANUAL",
}
i = random.randint(0, 20)
browser = webdriver.Chrome(path)
browser.get(URLs[5][0])
time.sleep(2)
element = browser.find_element_by_name(URLs[5][1])
element.send_keys(Identities[i][0])
element = browser.find_element_by_name(URLs[5][2])
element.send_keys(Identities[i][1])
element = browser.find_element_by_name(URLs[5][3])
element.send_keys(email)
element = browser.find_element_by_name(URLs[5][4])
element.send_keys("password")
element = browser.find_element_by_name(URLs[5][5])
element.send_keys("password")
element = browser.find_element_by_name(URLs[5][6])
element.click()
element = browser.find_element_by_name(URLs[5][7])
element.click()
browser.close()
return render(request, 'Home/Home.html')
def loadProxies():
req_proxy = RequestProxy()
proxies = req_proxy.get_proxy_list() # this will create proxy list
data = {"proxies": [proxy.get_address() for proxy in proxies]}
return data
def Init_Proxy():
req_proxy = RequestProxy()
proxies = req_proxy.get_proxy_list()
proxy_cycle = cycle(proxies)
return proxy_cycle
def randomProxy():
listOfProxies = RequestProxy()
myProxy = listOfProxies.get_proxy_list()
return myProxy[0]
# Results are written to disk after each chunk. Order is preserved.
# To reduce slowdowns due to disk IO, set this to a large-ish number
CHUNKSIZE = 200
# Min and max backoff after each thread's scrape, in seconds
BACKOFF_MIN = 1
BACKOFF_MAX = 5
####################################
# GLOBAL OBJECTS REQUIRED BY SCRAPER
####################################
LOGGER = setup_logger(name=os.path.basename(__file__), log_to_file=True, level=logging.INFO, tqdm_support=True)
if USE_PROXY:
LOGGER.info("Initializing RequestProxy()")
REQ_PROXY = RequestProxy()
LOGGER.info("{0} proxies loaded.".format(len(REQ_PROXY.get_proxy_list())))
else:
UA = UserAgent()
####################################
# BASE FUNCTIONS REQUIRED BY SCRAPER
####################################
def get_content_from_url(url):
"""
Scrapes the page in two modes:
USE_PROXY=True: random proxies and user-agents
USE_PROXY=False: random user-agents and sleep time
Retries for connection problems including timeouts, up to NUM_ATTEMPTS
Will not retry for HTTP errors (e.g. 404, 500)
TOR_FOLDER = os.path.join(os.getcwd(), 'tor')
TOR_PATH = type('Enum', (), {
'WINDOWS': os.path.join(TOR_FOLDER, 'windows', 'tor.exe'),
'MAC': os.path.join(TOR_FOLDER, 'mac', 'tor.real'),
'LINUX': os.path.join(TOR_FOLDER, 'linux', 'tor'),
'NONE': ''
})
BROWSER_OPTIONS = type('Enum', (), {
'CHROME': ChromeOptions(),
'FIREFOX': FirefoxOptions()
})
request_proxy = RequestProxy()
request_proxy.set_logger_level(40)
proxies = request_proxy.get_proxy_list()
def hidden(browser_options=BROWSER_OPTIONS.FIREFOX):
if type(browser_options) == ChromeOptions:
browser_options.add_argument('--incognito')
browser_options.add_argument('--disable-blink-features=AutomationControlled')
elif type(browser_options) == FirefoxOptions:
browser_options.add_argument('--private')
browser_options.set_preference("dom.webdriver.enabled", False)
browser_options.set_preference('useAutomationExtension', False)
return browser_options
def simplify(browser_options=BROWSER_OPTIONS.FIREFOX):
if type(browser_options) == ChromeOptions:
def get_proxies():
req_proxy = RequestProxy(
) # you may get different number of proxy when you run this at each time
proxies = req_proxy.get_proxy_list() # this will create proxy list
return proxies
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
from http_request_randomizer.requests.proxy.requestProxy import RequestProxy
from webdriver_manager.chrome import ChromeDriverManager
import time
from time import sleep
import random
accNo = int(input("No of Accounts: "))
print("Requesting Proxies:")
req_proxy = RequestProxy()
proxies = req_proxy.get_proxy_list()
maxNum = len(proxies)
print("\n \n Max No of Proxies:" + str(maxNum) + "\n \n")
i = 0
while accNo > i:
k = random.randint(0, maxNum - 1)
l = random.randint(0, maxNum - 1)
PROXY = proxies[k].get_address()
PROXY_PLACE = proxies[k].country
first_name = (random.choice(open("Fnames.txt").read().split()))
last_name = (random.choice(open("Lnames.txt").read().split()))
full_name = (first_name + ' ' + last_name)
username = (first_name + last_name + '.' + str(random.randint(1, 100)) +
str(random.randint(1, 1000)))
urllib.request.install_opener(opener)
req = urllib.request.Request('http://www.google.com')
urllib.request.urlopen(req)
except urllib.error.HTTPError as error:
print('Error code: ', error.code)
return error.code
except Exception as detail:
print("ERROR:", detail)
return True
return False
SCHED = BlockingScheduler()
REQ_PROXY = RequestProxy()
PROXIES = REQ_PROXY.get_proxy_list()
NEXT_RUN = datetime.now() + timedelta(hours=3)
selenium_logger.setLevel(logging.WARNING)
class Parser:
"""
Main Parser class.
"""
def __init__(self, use_proxy=False):
"""
Parser initialisation by launching the browser.
"""
self.browser = self.browser_setup(use_proxy=use_proxy)
self.tg_sources_path = 'telegram_parsing/channels.txt'
def get_proxy():
req_proxy = RequestProxy()
proxies = req_proxy.get_proxy_list()
return _get_working_proxy(proxies)
from apscheduler.schedulers.background import BackgroundScheduler
from http_request_randomizer.requests.proxy.requestProxy import RequestProxy
from handlers import globalvars
def ipgrab():
print("started extraction ip")
globalvars.req_proxy_list = req_proxy.get_proxy_list()
print("done")
minutes = 5
interval = 60 * minutes
print("loading")
req_proxy = RequestProxy()
globalvars.req_proxy_list = req_proxy.get_proxy_list()
print("done initial load")
sched = BackgroundScheduler()
sched.add_job(ipgrab, 'interval', seconds=interval)
sched.start()
print("made it here")
#!/usr/bin/env python2.7
import time
from http_request_randomizer.requests.proxy.requestProxy import RequestProxy
if __name__ == '__main__':
start = time.time()
req_proxy = RequestProxy()
print "Initialization took: {0} sec".format((time.time() - start))
print "Size : ", len(req_proxy.get_proxy_list())
print " ALL = ", req_proxy.get_proxy_list()
test_url = 'http://ipv4.icanhazip.com'
while True:
start = time.time()
request = req_proxy.generate_proxied_request(test_url)
print "Proxied Request Took: {0} sec => Status: {1}".format(
(time.time() - start), request.__str__())
if request is not None:
print "\t Response: ip={0}".format(u''.join(
request.text).encode('utf-8'))
print "Proxy List Size: ", len(req_proxy.get_proxy_list())
print "-> Going to sleep.."
time.sleep(1)
"httpProxy":PROXY,
"ftpProxy":PROXY,
"sslProxy":PROXY,
"proxyType":"MANUAL",
'trustAllServers':'true',
}
browser = webdriver.Chrome(options=chrome_options)
return browser
#logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING)
#from selenium.webdriver.remote.remote_connection import LOGGER
#LOGGER.setLevel(logging.WARNING)
req_proxy = RequestProxy() #you may get different number of proxy when you run this at each time
proxies = req_proxy.get_proxy_list() #this will create proxy list
sp = [] #int is list of Indian proxy
for proxy in proxies:
if proxy.country == 'Spain':
sp.append(proxy)
proxies=sp
browser_categories=["popular","blocks","featured","beta"]
data=get_all_popular_plugins()
json_data=json.dumps(data)
current_date_and_time = datetime.datetime.now()
current_date_and_time_string = str(current_date_and_time)
extension = ".json"
import time
from http_request_randomizer.requests.proxy.requestProxy import RequestProxy
if __name__ == '__main__':
start = time.time()
req_proxy = RequestProxy()
print("Initialization took: {0} sec".format((time.time() - start)))
print("Size: {0}".format(len(req_proxy.get_proxy_list())))
print("ALL = {0} ".format(list(map(lambda x: x.get_address(), req_proxy.get_proxy_list()))))
test_url = 'http://ipv4.icanhazip.com'
while True:
start = time.time()
request = req_proxy.generate_proxied_request(test_url)
print("Proxied Request Took: {0} sec => Status: {1}".format((time.time() - start), request.__str__()))
if request is not None:
print("\t Response: ip={0}".format(u''.join(request.text).encode('utf-8')))
print("Proxy List Size: {0}".format(len(req_proxy.get_proxy_list())))
print("-> Going to sleep..")
time.sleep(10)
