def redeem_courses(driver: WebDriver, settings: Settings, choice):
"""
Method to scrape courses from tutorialbar.com and enroll in them on udemy
:return:
"""
tb_scraper = TutorialBarScraper()
udemy_actions = UdemyActions(driver, settings)
udemy_actions.login() # login once outside while loop
while True:
udemy_course_links = tb_scraper.run(choice)
for course_link in udemy_course_links:
try:
udemy_actions.redeem(course_link)
if settings.is_ci_build:
return
except KeyboardInterrupt:
raise
except Exception:
if settings.is_ci_build:
return
print(
"Unable to enroll for this course either because you have already claimed it or the browser "
"window has been closed!")
print(
"Moving on to the next page of the course list on tutorialbar.com")
def redeem_courses(driver: WebDriver, settings: Settings):
"""
Method to scrape courses from tutorialbar.com and enroll in them on udemy
:return:
"""
tb_scraper = TutorialBarScraper()
udemy_actions = UdemyActions(driver, settings)
udemy_actions.login() # login once outside while loop
while True:
udemy_course_links = tb_scraper.run()
for course_link in udemy_course_links:
try:
udemy_actions.redeem(course_link)
except NoSuchElementException as e:
print(e)
except TimeoutException:
print(f"Timeout on link: {course_link}")
except WebDriverException as e:
print(f"Webdriver exception on link: {course_link}")
print(e)
except KeyboardInterrupt:
raise
except exceptions.RobotException as e:
print(e)
raise e
except Exception as e:
print(f"Unexpected exception: {e}")
finally:
if settings.is_ci_build:
return
print(
"Moving on to the next page of the course list on tutorialbar.com")
def getCourses(oldCourseFileName, newCourseFileName):
tb_scraper = TutorialBarScraper()
oldCourseNames, oldCourseUrls = get_old_course_details(oldCourseFileName)
flag = 0
courseCounter = 0
choice = "99"
while True:
b=0
newCourseUrls = tb_scraper.run(choice)
for newCourseUrl in newCourseUrls:
if "bit.ly" in newCourseUrl:
print("\n" + newCourseUrl + " is invalid\n")
elif newCourseUrl in oldCourseUrls:
print("\n" + newCourseUrl + " is repeated\n")
b=1
break
else:
flag, courseCounter, courseTitle, courseDescription = get_new_course_details(courseCounter, newCourseUrl)
if flag==1:
if courseTitle in oldCourseNames:
print("\n" + courseTitle + " is repeated")
courseCounter = courseCounter - 1
else:
save_new_course_details(courseCounter, newCourseUrl, courseTitle, courseDescription, newCourseFileName)
if b==1:
break
print("All new available courses are now successfully saved!")
def test_run(
mock_get_course_links,
mock_gather_udemy_course_links,
tutorialbar_course_page_link,
tutorialbar_links,
udemy_links,
):
mock_get_course_links.return_value = tutorialbar_links
mock_gather_udemy_course_links.return_value = udemy_links
tbs = TutorialBarScraper()
links = tbs.run()
mock_get_course_links.assert_called_with(tutorialbar_course_page_link)
mock_gather_udemy_course_links.assert_called_with(tutorialbar_links)
for link in links:
assert link in udemy_links
def test_get_course_links(mock_requests, tutorialbar_main_page):
url = "https://www.tutorialbar.com/main"
requests_response = mock.Mock()
requests_response.content = tutorialbar_main_page
mock_requests.get.return_value = requests_response
tbs = TutorialBarScraper()
tbs.current_page = 1
links = tbs.get_course_links(url)
assert tbs.last_page == "601"
assert links == [
"https://www.tutorialbar.com/mindfulness-meditation-for-pain-relief-stress-management/",
"https://www.tutorialbar.com/become-a-crm-manager-overview-for-email-marketing-starters/",
"https://www.tutorialbar.com/superminds-the-future-of-artificial-intelligence-ai/",
"https://www.tutorialbar.com/invade-your-classroom-with-digital-robot-teachers-in-2020/",
"https://www.tutorialbar.com/introduction-au-machine-learning-python/",
"https://www.tutorialbar.com/comic-creation-for-entrepreneurs-2020-edition/",
"https://www.tutorialbar.com/delicious-japanese-language-for-foodies-jlpt-n5-jlpt-n4/",
"https://www.tutorialbar.com/sparring-tai-chi-chen-new-frame-routine-2-for-fitness/",
"https://www.tutorialbar.com/active-learning-using-games-in-education/",
"https://www.tutorialbar.com/eiq2-coaching-for-improved-performance-and-superior-results/",
"https://www.tutorialbar.com/quickbooks-pro-desktop-bookkeeping-business-easy-way/",
"https://www.tutorialbar.com/quickbooks-online-bank-feeds-credit-card-feeds-2020/",
]
from selenium.webdriver.remote.webdriver import WebDriver
from core import TutorialBarScraper
links = []
tb_scraper = TutorialBarScraper()
print('1. IT software')
print('2. Development')
print('3. Finance Accounting')
print('4. Design')
print('5. Business')
print('6. Marketing')
print('7. Health and Fitness')
print('8. Office Productivity')
print('9. Photography')
print('10. Personal Development')
print('11. Teaching and Academics')
print('99. All Courses')
choice = input('Please select what category you want? :')
while True:
udemy_course_links = tb_scraper.run(choice)
links = links + udemy_course_links
print(links)
print(len(links))
def _redeem_courses(
driver: WebDriver,
settings: Settings,
max_pages: Union[int, None],
cache_hit_limit: int,
) -> None:
"""
Method to scrape courses from tutorialbar.com and enroll in them on udemy
:param WebDriver driver: Webdriver used to enroll in Udemy courses
:param Settings settings: Core settings used for Udemy
:param int max_pages: Max pages to scrape from tutorialbar.com
:param int cache_hit_limit: If we hit the cache this many times in a row we exit the script
:return:
"""
cache = CourseCache()
tb_scraper = TutorialBarScraper(max_pages)
udemy_actions = UdemyActions(driver, settings)
udemy_actions.login() # login once outside while loop
current_cache_hits = 0
while True:
# Check if we should exit the loop
if not tb_scraper.script_should_run():
break
udemy_course_links = tb_scraper.run()
for course_link in udemy_course_links:
try:
if course_link not in cache:
status = udemy_actions.redeem(course_link)
cache.add(course_link, status)
# Reset cache hit count as we haven't scraped this page before
current_cache_hits = 0
else:
logger.info(f"In cache: {course_link}")
# Increment the cache hit count since this link is in the cache
current_cache_hits += 1
# Exit the loop if we have reached the cache hit limit
if _reached_cache_hit_limit(cache_hit_limit,
current_cache_hits):
return
except NoSuchElementException as e:
logger.error(e)
except TimeoutException:
logger.error(f"Timeout on link: {course_link}")
except WebDriverException:
logger.error(f"Webdriver exception on link: {course_link}")
except KeyboardInterrupt:
logger.error("Exiting the script")
raise
except exceptions.RobotException as e:
logger.error(e)
raise
except Exception as e:
logger.error(f"Unexpected exception: {e}")
finally:
if settings.is_ci_build:
logger.info(
"We have attempted to subscribe to 1 udemy course")
logger.info("Ending test")
return
logger.info(
"Moving on to the next page of the course list on tutorialbar.com")
def test_check_page_number(page_number, is_first_page):
tbs = TutorialBarScraper()
tbs.current_page = page_number
assert tbs.is_first_loop() == is_first_page