class TruliaHelper():
def __init__(self):
self.url = 'https://www.trulia.com'
# need to set chrome path here.
tbpath = "/home/XX/XXXX/tor-browser-linux64-8.0.8_en-US/tor-browser_en-US"
self.driver = TorBrowserDriver(tbb_path=tbpath, tbb_logfile_path='test.log')
# self.driver = webdriver.Firefox(firefox_profile=profile, firefox_binary=binary)
# self.driver = webdriver.Chrome(executable_path='../utility/chromedriver.exe', chrome_options=chrome_options)
# method to get items from given link.
def getItems(self):
df=pd.read_excel("/home/XXXXX/XXXXX/XXXXXX.xlsx")
a=df['Site Address']
b=df['Site City']
c=df['Site State']
d=df['Site Zip']
items = []
# keywords = ['512 W 10th St Perris CA 92570', 'New York, NY', 'San Francisco, CA', 'Washington, CA']
for keyword in (pd.concat([a,b,c,d],axis=1)).values.tolist():
# keywords = ['512 W 10th St Perris CA 92570'] * 10
# for keyword in keywords:
self.driver.get(self.url)
search_box = self.driver.find_element_by_id("homepageSearchBoxTextInput")
search_box.clear()
search_box.send_keys(str(keyword))
search_btn = self.driver.find_element_by_xpath("//button[@data-auto-test-id='searchButton']")
if search_btn:
search_btn.click()
time.sleep(10)
items.append(self.getItemDetail())
# break
self.driver.close()
return items
def getItemDetail(self):
data = {}
try:
soup = BeautifulSoup(self.driver.page_source, u'html.parser')
#image = soup.find("div", attrs={"class": "Tiles__TileBackground-fk0fs3-0 cSObNX"}).find("img")["src"]
price = soup.find("div", attrs={"class": "Text__TextBase-sc-1cait9d-0-div Text__TextContainerBase-sc-1cait9d-1 hlvKRM"}).text
# container = soup.find("div", attrs={"class": "resultsColumn"}).find("ul")
# items = container.findAll("li", recursive=False)
print(price)
except:
pass
return data
# method to start process.
def start(self):
items = self.getItems()
print("Items : ",items)
class crearCorros:
def __init__(self):
self.urlProtocoe = 'http://3g2upl4pq6kufc4m.onion', 'https://mail.protonmail.com/create/new', 'https://singlelogin.org/registration.php'
print(self.urlProtocoe[2])
self.tbb_dir = "/usr/local/share/tor-browser_en-US"
self.protocoe = TorBrowserDriver(self.tbb_dir,
tbb_logfile_path='test.log')
self.dirNombre = '/home/dgc7/ejersiciosLibros/pyaton/ejemplos/scrapin/zlibrari/crearCorreos/nombre.txt'
self.nombre = open(self.dirNombre, 'r+')
self.dirapellido = '/home/dgc7/ejersiciosLibros/pyaton/ejemplos/scrapin/zlibrari/crearCorreos/apellidos.txt'
self.apellido = open(self.dirapellido, 'r+')
self.dirContrasenna = '/home/dgc7/ejersiciosLibros/pyaton/ejemplos/scrapin/zlibrari/crearCorreos/contraseña.txt'
self.contrasenna = open(self.dirContrasenna, 'r+')
self.dirCotrasenna2 = '/home/dgc7/ejersiciosLibros/pyaton/ejemplos/scrapin/zlibrari/crearCorreos/contraseña2.txt'
self.Contrasenna2 = open(self.dirCotrasenna2, 'r+')
self.datosContrasenna = []
self.lista = []
for self.d in range(0, 101):
self.lista.append(self.nombre.readline() + 'asdsdf')
self.datosContrasenna.append(self.contrasenna.readline() +
self.Contrasenna2.readline())
for self.d in range(0, 100):
self.lista[self.d] = re.sub('\n', 'asdaawderca',
self.lista[self.d])
self.datosContrasenna[self.d] = re.sub(
'\n', 'radabanals', self.datosContrasenna[self.d])
self.lista[self.d] = re.sub(
r"([^n\u0300-\u036f]|n(?!\u0303(?![\u0300-\u036f])))[\u0300-\u036f]+",
r"\1", normalize("NFD", self.lista[self.d]), 0, re.I)
self.lista[self.d] = normalize('NFC', self.lista[self.d])
self.datosContrasenna[self.d] = re.sub(
r"([^n\u0300-\u036f]|n(?!\u0303(?![\u0300-\u036f])))[\u0300-\u036f]+",
r"\1", normalize("NFD",
self.datosContrasenna[self.d]), 0, re.I)
self.datosContrasenna[self.d] = normalize(
'NFC', self.datosContrasenna[self.d])
def iniciarTor(self):
self.protocoe.load_url(self.urlProtocoe[2])
def ingresarDatos(self, fila):
self.pasword = self.protocoe.find_element_by_name("password")
self.pasword.click()
sleep(random.uniform(1.0, 4))
self.pasword.send_keys(self.datosContrasenna[fila])
self.pasword = self.protocoe.find_element_by_name("passwordc")
sleep(random.uniform(1.0, 3))
self.pasword.click()
self.pasword.send_keys(self.datosContrasenna[fila])
sleep(random.uniform(2.0, 5.7))
self.iframes = self.protocoe.find_element_by_tag_name("iframe")
self.protocoe.switch_to.frame(self.iframes)
self.usuario = self.protocoe.find_element_by_xpath('//input')
self.usuario.click()
self.usuario.send_keys(self.lista[fila])
sleep(random.uniform(0, 5))
self.usuario.send_keys(Keys.ENTER)
self.protocoe.switch_to.default_content()
sleep(20)
self.enter = self.protocoe.find_element_by_xpath(
'//button[@class="pm_button primary modal-footer-button"]')
self.enter.click()
def serrarTor(self):
self.protocoe.close()
def imprimirDatos(self):
for d in range(0, 100):
print(self.lista[d])
print(self.datosContrasenna[d])
cursor.execute(sql)
result = cursor.fetchall()
originalTerm = result[0]['term']
print(originalTerm)
term = originalTerm.replace("-", " ")
driver.implicitly_wait(10)
driver.get(
"https://www.udemy.com/instructor/marketplace-insights/?q=" +
term + "&lang=en")
sleep(random.randint(5, 8))
try:
demandEl = driver.find_element_by_xpath(
'//div[contains(@class,"panel-body")]/div[contains(@class,"course-label-metrics-opportunity")]/div[1]/div/div[2]'
)
print(demandEl.text)
except NoSuchElementException:
print("trying hyphenated...")
term = term.replace(" ", "-")
driver.get(
"https://www.udemy.com/instructor/marketplace-insights/?q=" +
term + "&lang=en")
sleep(random.randint(5, 8))
demandEl = driver.find_element_by_xpath(
'//div[contains(@class,"panel-body")]/div[contains(@class,"course-label-metrics-opportunity")]/div[1]/div/div[2]'
)
print(demandEl.text)
demandStr = demandEl.text
class crearCorros:
def __init__(self):
self.urlProtocoe = 'http://3g2upl4pq6kufc4m.onion', 'https://mail.protonmail.com/create/new', 'https://singlelogin.org/registration.php', 'https://singlelogin.org/?logoutAll'
print(self.urlProtocoe[2])
self.tbb_dir = "/usr/local/share/tor-browser_en-US"
self.protocoe = TorBrowserDriver(self.tbb_dir,
tbb_logfile_path='test.log')
self.dirNombre = '/home/dgc7/ejersiciosLibros/pyaton/ejemplos/scrapin/zlibrari/crearCorreos/nombre.txt'
self.nombre = open(self.dirNombre, 'r+')
self.dirapellido = '/home/dgc7/ejersiciosLibros/pyaton/ejemplos/scrapin/zlibrari/crearCorreos/apellidos.txt'
self.apellido = open(self.dirapellido, 'r+')
self.dirContrasenna = '/home/dgc7/ejersiciosLibros/pyaton/ejemplos/scrapin/zlibrari/crearCorreos/contraseña.txt'
self.contrasenna = open(self.dirContrasenna, 'r+')
self.dirCotrasenna2 = '/home/dgc7/ejersiciosLibros/pyaton/ejemplos/scrapin/zlibrari/crearCorreos/contraseña2.txt'
self.Contrasenna2 = open(self.dirCotrasenna2, 'r+')
self.datosContrasenna = []
self.lista = []
for self.d in range(0, 101):
self.lista.append(self.nombre.readline() + 'asdsdf')
self.datosContrasenna.append(self.contrasenna.readline() +
"blabal")
for self.d in range(0, 100):
self.lista[self.d] = re.sub('\n', 'asdaawderca',
self.lista[self.d])
self.datosContrasenna[self.d] = re.sub(
'\n', 'radabanals', self.datosContrasenna[self.d])
self.lista[self.d] = re.sub(
r"([^n\u0300-\u036f]|n(?!\u0303(?![\u0300-\u036f])))[\u0300-\u036f]+",
r"\1", normalize("NFD", self.lista[self.d]), 0, re.I)
self.lista[self.d] = normalize('NFC', self.lista[self.d])
self.datosContrasenna[self.d] = re.sub(
r"([^n\u0300-\u036f]|n(?!\u0303(?![\u0300-\u036f])))[\u0300-\u036f]+",
r"\1", normalize("NFD",
self.datosContrasenna[self.d]), 0, re.I)
self.datosContrasenna[self.d] = normalize(
'NFC', self.datosContrasenna[self.d])
self.lista[self.d] += '@maildrop.cc'
def iniciarTor(self):
self.protocoe.load_url(self.urlProtocoe[3])
def ingresarDatos(self, fila):
self.eamil = self.protocoe.find_element_by_name('email')
self.eamil.click()
sleep(random.uniform(1.0, 4))
self.eamil.send_keys(self.lista[fila])
self.pasword = self.protocoe.find_element_by_name("password")
self.pasword.click()
sleep(random.uniform(1.0, 5))
self.pasword.send_keys(self.datosContrasenna[fila])
sleep(random.uniform(2.0, 4.7))
self.pasword.send_keys(Keys.RETURN)
sleep(20)
self.protocoe.load_url('https://b-ok.cc/profile.php')
self.resen = self.protocoe.find_element_by_xpath(
'//a[@id="resendConfEmail"]')
self.resen.click()
sleep(4)
self.alerta = self.protocoe.switch_to_alert()
print(self.alerta.text)
self.alerta.accept()
def serrarTor(self):
self.protocoe.close()
def imprimirDatos(self):
#self.dirscv='/home/dgc7/ejersiciosLibros/pyaton/ejemplos/scrapin/zlibrari/emailFalsos/contraseñasYcorreos.csv'
#self.datos=csv.writer(open(self.dirscv,'w'))
for d in range(0, 100):
#self.datos.writerow([self.lista[d]])
#self.datos.writerow([self.datosContrasenna[d]])
print(self.lista[d])
print(self.datosContrasenna[d])
class TruliaHelper():
def __init__(self):
self.url = 'https://www.trulia.com'
# need to set Tor Browser path here.
tbpath = "/home/gc14/Documents/softwares/tor-browser_en-US"
self.driver = TorBrowserDriver(tbb_path=tbpath,
tbb_logfile_path='test.log')
# self.driver = webdriver.Firefox(firefox_profile=profile, firefox_binary=binary)
# self.driver = webdriver.Chrome(executable_path='../utility/chromedriver.exe', chrome_options=chrome_options)
# method to get items from given link.
def getItems(self):
items = []
# keywords = ['512 W 10th St Perris CA 92570', 'New York, NY', 'San Francisco, CA', 'Washington, CA']
keywords = ['512 W 10th St Perris CA 92570'] * 2
for keyword in keywords:
self.driver.get(self.url)
search_box = self.driver.find_element_by_id(
"homepageSearchBoxTextInput")
search_box.clear()
search_box.send_keys(keyword)
search_btn = self.driver.find_element_by_xpath(
"//button[@data-auto-test-id='searchButton']")
if search_btn:
print("Going to click")
search_btn.click()
time.sleep(10)
items.append(self.getItemDetail())
self.driver.close()
return items
def getItemDetail(self):
data = {}
try:
soup = BeautifulSoup(self.driver.page_source, u'html.parser')
image = soup.find("div",
attrs={
"class":
"Tiles__TileBackground-fk0fs3-0 cSObNX"
}).find("img")["src"]
price = soup.find(
"div",
attrs={
"class":
"Text__TextBase-sc-1cait9d-0-div Text__TextContainerBase-sc-1cait9d-1 hlvKRM"
}).text
# container = soup.find("div", attrs={"class": "resultsColumn"}).find("ul")
# items = container.findAll("li", recursive=False)
data.update({"image": image, "price": price})
except:
pass
return data
# method to write csv file
def writeCSVFile(self, data):
try:
with open(
'/home/gc14/Documents/fiverr/custom_scrapers/home/trulia.csv',
mode='w') as csv_file:
fieldnames = ['Image', 'Price']
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writeheader()
for d in data:
writer.writerow({'Image': d['image'], 'Price': d['price']})
csv_file.close()
print("File written successfully.")
except:
print(sys.exc_info())
pass
# method to start process.
def start(self):
items = self.getItems()
print("Items : ", len(items))
if items:
self.writeCSVFile(items)