def select_size_and_add_to_cart(self, size):
sizeBox = self.findElement(IDMODE.CLASS, 'exp-pdp-size-and-quantity-container')
select2 = sizeBox.find_element_by_tag_name('a')
select2.click()
time.sleep(2)
lis = sizeBox.find_elements_by_tag_name('li')
for li in lis:
if li.text.strip() == size:
li.click()
break
time.sleep(2)
button = self.findElement(IDMODE.CLASS, 'exp-pdp-save-container')
button.find_element_by_tag_name('button').click()
time.sleep(5)
try:
span = self.driver.find_element_by_class_name("smart-cart-header")
if "Out of Stock" in span.text:
logger.info("Out of Stock")
return False
except:
pass
return True
def scrape_all_results(self):
i = 1
while True:
# scrape results on a page
self.waitUntilElementIsPresent(IDMODE.XPATH, "//table[@class='list-font-head']")
self.scrape_results_on_page()
# go to next page
nextButtonTable = self.driver.find_element_by_xpath("//table[@class='list-font-head']")
if ">>" in nextButtonTable.text:
logger.info("Page " + str(i))
nextButton = nextButtonTable.find_elements_by_tag_name("td")[2]
nextButton.find_element_by_tag_name("a").click()
i += 1
else:
break
def doScrape():
browser = SiteEvents()
logger.info("Opening Browser...")
browser.navigate(config.baseURL)
logger.info("...Browser Opened")
logger.info("Entering Parameters...")
browser.fill_search_parameters()
logger.info("...Parameters Entered")
logger.info("Scraping Results...")
browser.scrape_all_results()
logger.info("... Results Scraped")
if len(config.Results) == 0:
logger.info("No Results")
return "NO_RESULTS"
logger.info("Writing CSV...")
inputContent.write_csv_results()
logger.info("...CSV Written")
return 1
def doScrape(Con, q):
try:
g = Gmail()
################ LOGIN #####################################
q.put(('Logging In', 5), )
logger.info("Logging In")
try:
g.login(Con.Username, Con.Password)
except AuthenticationError:
logger.exception(sys.exc_info())
q.put(('Login Failed', 100), )
return "AUTH_ERROR"
############################################################
################ GET LABEL MAILBOX #########################
mailbox = None
q.put(('Getting Mailbox', 10), )
logger.info("Getting Mailbox")
try:
if Con.Label.lower() == 'inbox':
mailbox = g.inbox()
else:
mailbox = g.mailbox(Con.Label)
except:
logger.exception(sys.exc_info())
q.put(('Problem in fetching Gmail Label', 100), )
return "LABEL_FETCH_ERROR"
if not mailbox:
q.put(('Gmail Label Not Found', 100), )
return "LABEL_NOT_FOUND"
############################################################
################ GET EMAILS ################################
mails = None
q.put(('Searching For Emails', 15), )
logger.info("Searching Emails")
try:
afterDate = Con.FromDate - timedelta(days=1)
beforeDate = Con.ToDate + timedelta(days=1)
mails = mailbox.mail(
subject='Fiverr: Congrats! You have a new order',
after=afterDate,
before=beforeDate)
mails.extend(
mailbox.mail(subject='just ordered an Extra',
after=afterDate,
before=beforeDate))
# mails = mailbox.mail(after=Con.FromDate, before=Con.ToDate)
except:
logger.exception(sys.exc_info())
q.put(('Problem in searching for emails', 100), )
return "EMAILS_FETCH_ERROR"
if len(mails) == 0:
q.put(('No Emails Found with search criteria', 100), )
return "NO_EMAIL_FOUND"
############################################################
################ FETCH EMAILS ##############################
q.put(('Fetching Emails', 20), )
logger.info("Scraping Order Data From Emails")
Con.Orders = []
logger.info("Num of Emails found: " + str(len(mails)))
try:
for mail in mails:
msg = "Fetching Email " + str(mails.index(mail) +
1) + ' of ' + str(len(mails))
per = 20 + int((float(mails.index(mail) + 1) * 100.0 * 0.6 /
float(len(mails))))
q.put((msg, per), )
#logger.info(msg)
mail.fetch()
gmailEvents.extract_orders_from_email(mail, Con)
except:
logger.exception(sys.exc_info())
q.put(('Problem in fetching emails', 100), )
return "EMAIL_FETCH_ERROR"
############################################################
# return 'SUCCESS'
################ CALCULATE TOTAL AMOUNT ####################
q.put(('Calculating Total and Revenue', 85), )
logger.info("Calculating Total Amount")
gmailEvents.calculate_total_amount(Con)
############################################################
################ GENERATE XLS ##############################
q.put(('Generating XLS', 90), )
logger.info("Generating XLS")
gmailEvents.generate_xls(Con)
############################################################
q.put(('Logging Out of Gmail', 95), )
g.logout()
q.put(('SUCCESS', 100), )
return 'SUCCESS'
except:
if g:
g.logout()
logger.exception(sys.exc_info())
q.put(('Error Occurred', 100), )
return "ERROR"
def doScrape(Con, q):
try:
g = Gmail()
################ LOGIN #####################################
q.put(('Logging In', 5),)
logger.info("Logging In")
try:
g.login(Con.Username, Con.Password)
except AuthenticationError:
logger.exception(sys.exc_info())
q.put(('Login Failed', 100),)
return "AUTH_ERROR"
############################################################
################ GET LABEL MAILBOX #########################
mailbox = None
q.put(('Getting Mailbox', 10),)
logger.info("Getting Mailbox")
try:
if Con.Label.lower() == 'inbox':
mailbox = g.inbox()
else:
mailbox = g.mailbox(Con.Label)
except:
logger.exception(sys.exc_info())
q.put(('Problem in fetching Gmail Label', 100),)
return "LABEL_FETCH_ERROR"
if not mailbox:
q.put(('Gmail Label Not Found', 100),)
return "LABEL_NOT_FOUND"
############################################################
################ GET EMAILS ################################
mails = None
q.put(('Searching For Emails', 15),)
logger.info("Searching Emails")
try:
afterDate = Con.FromDate - timedelta(days=1)
beforeDate = Con.ToDate + timedelta(days=1)
mails = mailbox.mail(subject='Fiverr: Congrats! You have a new order',
after=afterDate, before=beforeDate)
mails.extend(mailbox.mail(subject='just ordered an Extra',
after=afterDate, before=beforeDate))
# mails = mailbox.mail(after=Con.FromDate, before=Con.ToDate)
except:
logger.exception(sys.exc_info())
q.put(('Problem in searching for emails', 100),)
return "EMAILS_FETCH_ERROR"
if len(mails) == 0:
q.put(('No Emails Found with search criteria', 100),)
return "NO_EMAIL_FOUND"
############################################################
################ FETCH EMAILS ##############################
q.put(('Fetching Emails', 20),)
logger.info("Scraping Order Data From Emails")
Con.Orders = []
logger.info("Num of Emails found: " + str(len(mails)))
try:
for mail in mails:
msg = "Fetching Email " + str(mails.index(mail)+1) + ' of ' + str(len(mails))
per = 20 + int((float(mails.index(mail)+1) * 100.0 * 0.6 / float(len(mails))))
q.put((msg, per),)
#logger.info(msg)
mail.fetch()
gmailEvents.extract_orders_from_email(mail, Con)
except:
logger.exception(sys.exc_info())
q.put(('Problem in fetching emails', 100),)
return "EMAIL_FETCH_ERROR"
############################################################
# return 'SUCCESS'
################ CALCULATE TOTAL AMOUNT ####################
q.put(('Calculating Total and Revenue', 85),)
logger.info("Calculating Total Amount")
gmailEvents.calculate_total_amount(Con)
############################################################
################ GENERATE XLS ##############################
q.put(('Generating XLS', 90),)
logger.info("Generating XLS")
gmailEvents.generate_xls(Con)
############################################################
q.put(('Logging Out of Gmail', 95),)
g.logout()
q.put(('SUCCESS', 100),)
return 'SUCCESS'
except:
if g:
g.logout()
logger.exception(sys.exc_info())
q.put(('Error Occurred', 100),)
return "ERROR"
contents = soup.find_all("div", class_="content")
for content in contents:
tweetText = content.p.text
if config.Keyword in tweetText and ('available' in tweetText or 'arrived' in tweetText):
urls = re.findall("http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+",
tweetText)
return urls[0]
time.sleep(4)
readProperties()
try:
nike = NikeEvents(config.ProxyIP)
logger.info("Opening Nike Site...")
nike.navigate("http://store.nike.com/us/en_us/?l=shop%2Clogin")
logger.info("...Opened Nike Site")
logger.info(nike.driver.title)
logger.info(nike.driver.current_url)
logger.info("Logging in to Nike...")
nike.loginToNike(config.Email, config.Password)
time.sleep(10)
logger.info("...Logged in to Nike")
logger.info(nike.driver.title)
logger.info(nike.driver.current_url)
logger.info("Waiting for tweet...")
shoeURL = get_desired_url()
