def start_and_login():
while True:
opts = chromeOpts()
opts.add_argument(f"user-agent={ua.random}")
prefs = {
'profile.managed_default_content_settings.images': 2
} # Disallow images from loading
opts.add_experimental_option("prefs", prefs)
driver = helium.start_chrome(
"https://www.udemy.com/random_page_that_does_not_exist/",
headless=False,
options=opts)
driver.add_cookie({
'name': 'client_id',
'value': client_id,
'domain': udemy_domain
})
driver.add_cookie({
'name': 'access_token',
'value': access_token,
'domain': udemy_domain
})
helium.go_to("https://www.udemy.com/?persist_locale=&locale=en_US")
if 'upgrade-your-browser' in driver.current_url:
driver.quit()
else:
break
try:
helium.wait_until(helium.Button('Log in').exists, timeout_secs=3.5)
helium.kill_browser()
raise InvalidCookiesException()
except TimeoutException:
return driver
def main(dict_classes, schedule_sheet, hours_list, cells_list, days_list,
credentials):
take()
start_chrome("google.com")
# login into the Google account
if credentials[0] == "ita\n":
go_to(
"https://accounts.google.com/signin/v2/identifier?service=classroom&passive=1209600&continue=https%3A%2F%2Fclassroom.google.com%2Fu%2F0%2Fh%3Fhl%3Dit&followup=https%3A%2F%2Fclassroom.google.com%2Fu%2F0%2Fh%3Fhl%3Dit&hl=it&flowName=GlifWebSignIn&flowEntry=ServiceLogin"
)
write(
credentials[1].replace("\n", ""),
into="Indirizzo email o numero di telefono",
)
press(ENTER)
write(credentials[2], into="Inserisci la password")
press(ENTER)
elif credentials[0] == "eng\n,":
go_to(
"https://accounts.google.com/signin/v2/identifier?service=classroom&passive=1209600&continue=https%3A%2F%2Fclassroom.google.com%2Fu%2F0%2Fh%3Fhl%3Dit&followup=https%3A%2F%2Fclassroom.google.com%2Fu%2F0%2Fh%3Fhl%3Dit&hl=it&flowName=GlifWebSignIn&flowEntry=ServiceLogin"
)
write(credentials[1].replace("\n", ""), into="Email or phone")
press(ENTER)
write(credentials[2], into="Enter your password")
press(ENTER)
else:
print(colorama.Fore.RED + colorama.Style.BRIGHT +
"\n\nThe language is not supported (ita-eng)")
while A == 0:
k = giorno(days_list)
materia = ora(hours_list, cells_list, k, schedule_sheet)
meet(materia, dict_classes, k, credentials)
def search_single_pub(self, query):
"""restituisce il primo paper dalla ricerca"""
url = 'https://www.semanticscholar.org/search?q={}&sort=relevance'.format(
query)
helium.go_to(url)
self._wait('.result-page')
paper = self._chrm.find_elements_by_css_selector('.cl-paper-row')
if len(paper) == 0:
return None
paper = paper[0]
title = paper.find_element_by_class_name('cl-paper-title').text
author = paper.find_element_by_class_name('cl-paper-authors').text
year = paper.find_element_by_class_name('cl-paper-pubdates').text[-4:]
link = paper.find_element_by_css_selector(
'.cl-paper-row > a').get_attribute('href')
abs_l = paper.find_elements_by_css_selector(
'.cl-paper-abstract span span')
abstract = ""
if len(abs_l) > 0:
abstract = abs_l[0].text
pub = {
'title': title,
'author': author,
'pub_year': year,
'venue': 'venue',
'abstract': abstract,
'link': link
}
return pub
def login_connpass():
"""connpassにログインする（ユーザー名とパスワードは環境変数を想定）"""
go_to("connpass.com/login")
write(os.getenv("CONNPASS_USERNAME"), into="ユーザー名")
write(os.getenv("CONNPASS_PASSWORD"), into="パスワード")
click("ログインする")
wait_until(Text("あなたのイベント").exists)
def cited_by(self, pub):
pubs = []
helium.go_to(pub['link'])
#self._wait('.card-container')
page = 1
while page < 6:
papers = self._chrm.find_elements_by_css_selector(
'#citing-papers .cl-paper-row.citation-list__paper-row')
for p in papers:
title = p.find_elements_by_css_selector('.cl-paper-title')
author = p.find_elements_by_css_selector('.cl-paper-authors')
year = p.find_elements_by_css_selector('.cl-paper-pubdates')
link = p.find_elements_by_css_selector('.cl-paper-row > a')
if len(title) > 0:
title = title[0].text
else:
continue
if len(author) > 0:
author = author[0].text
else:
continue
if len(year) > 0:
year = year[0].text
else:
continue
if len(link) > 0:
link = link[0].get_attribute('href')
else:
continue
pubs.append({
'title': title,
'author': author,
'pub_year': year,
'venue': 'venue',
'abstract': "",
'link': link
})
#test next page
l = self._chrm.find_elements_by_css_selector(
'#citing-papers .cl-pager.cl-pager--has-next-enabled')
if len(l) == 0:
break
#click next page
elm = self._chrm.find_element_by_css_selector(
'#citing-papers .cl-pager__button.cl-pager__next')
self._chrm.execute_script("arguments[0].click();", elm)
page += 1
self._wait(
f'#citing-papers .cl-pager[data-curr-page-num="{page}"]')
return pubs
def get_user_info():
try:
helium.go_to("https://www.udemy.com/user/edit-profile/")
user_information = browser.execute_script('return UD.me')
mail = user_information["email"]
display_name = user_information["display_name"]
account_id = user_information['id']
return mail, display_name, account_id
except TimeoutException:
return None
def meet(materia, dict_classes, k, credentials):
try:
if materia:
print(colorama.Fore.GREEN + colorama.Style.BRIGHT +
"\n\nMeet found")
print(colorama.Fore.WHITE + f"Connecting to: {materia}")
timeout = threading.Event()
timeout.wait(timeout=5)
# opens the meet page
go_to(dict_classes[materia])
close = threading.Thread(target=closef)
close.start()
# join the meet
join = threading.Thread(target=joinf)
join.start()
except KeyError:
ora(k)
def run(self, dispatcher: CollectingDispatcher, tracker: Tracker,
domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:
web_name = tracker.get_slot('web_name')
url = tracker.get_slot('url')
if web_name:
search_url = f"https://www.dogedoge.com/results?q={web_name}官网"
r = session.get(search_url)
result_list = r.html.find(".result__a", first=True).absolute_links
web_url = list(result_list)[0]
go_to(web_url)
tracker.slots = {'web_name': ''}
dispatcher.utter_message("已打开")
elif url:
go_to(url)
tracker.slots = {'url': ''}
dispatcher.utter_message("已打开")
else:
dispatcher.utter_message("抱歉，无法识别要打开的网站")
return []
def setUpClass(cls):
super().setUpClass()
go_to(get_data_file_url('test_window/test_window.html'))
click("Click here to open a popup.")
wait_until(Window('test_window - popup').exists)
from contextlib import contextmanager
from helium import (
Text,
click,
go_to,
kill_browser,
start_firefox,
wait_until,
write,
)
@contextmanager
def using_firefox():
start_firefox()
try:
yield
finally:
kill_browser()
if __name__ == "__main__":
with using_firefox():
go_to("connpass.com/login")
write(os.getenv("CONNPASS_USERNAME"), into="ユーザー名")
write(os.getenv("CONNPASS_PASSWORD"), into="パスワード")
click("ログインする")
wait_until(Text("あなたのイベント").exists)
def setUp(self):
go_to(self.get_url())
rating_widget = content.find('div', class_='ipl-rating-widget')
summary_block = metascore_block.findNext(
'p'
) if metascore_block is not None else rating_widget.findNext('p')
summary = '\n\t'.join([
el.strip() for el in wrap(summary_block.text, width=wrap_width)
])
org_text = f"* [ ] {index} [[{title_link}][{title_txt}]] {rating}\t{ ':' + ':'.join([el.strip() for el in genre.split(',')]) + ':' }\n\t{year}\t{length}\t{metascore}\n\t{summary}\n" + f"\t[[file:./{str.lower(str(list_name))}/{title_txt}.jpg]]\n" if save_images else ""
f.write(org_text)
if __name__ == "__main__":
driver = helium.start_firefox(headless=True)
generated = list()
for genre in URLs:
helium.go_to(URLs[genre])
clen = 0
height = driver.execute_script("return document.body.scrollHeight")
while clen < height:
helium.scroll_down(num_pixels=400)
clen += 400
height = driver.execute_script("return document.body.scrollHeight")
time.sleep(1.)
res = driver.page_source
imdb_list_org(genre, res, save_images=True)
generated.append(genre)
with open('.generated.yaml', 'a') as f:
yaml.safe_dump(generated, f)
helium.kill_browser()
def search_helium_repository():
"""HeliumのGitHubリポジトリをGoogle検索して移動する"""
go_to("google.com")
write("helium selenium github")
press(ENTER)
click("Selenium-python but lighter: Helium - GitHub")
def test_go_to(self): self.driver = start_browser() go_to(self.url) self.assertUrlEquals(self.url, self.driver.current_url)
courses = get_free_courses()
print(f"{Fore.GREEN}[{tick}] Scraped {len(courses)} courses!")
browser = start_and_login()
user_info = get_user_info()
if user_info is None:
print(f"{Fore.GREEN}[{tick}] Successfully logged in!\n")
else:
print(f"{Fore.GREEN}[{tick}] Successfully logged in!")
print(f"{Fore.YELLOW}[!] Email: {user_info[0]}")
print(f"{Fore.YELLOW}[!] Display Name: {user_info[1]}\n")
atexit.register(browser.quit)
success_counter = 0
for url in courses:
helium.go_to(url)
if helium.S(
"//span[contains(text(),'Sorry, this course is no longer accepting enrollme')]"
).exists():
print(
f"{Fore.YELLOW}[!] Cannot enroll in the course at {url} because the course is not "
f"accepting enrollments anymore")
continue
elif helium.S(
"//span[contains(text(), ' is no longer available.')]").exists():
print(f"{Fore.YELLOW}[!] The course at {url} does not exist anymore")
continue
elif helium.S("//h1[@class='error__greeting']").exists():
print(f"{Fore.YELLOW}[!] The {url} does not exist")
continue
try:
def connect_to_local_runtime(user_choice):
try:
# import chromedriver_binary
if user_choice == "ClointFusion Labs (Public)":
colab_url = "https://accounts.google.com/signin/v2/identifier?authuser=0&hl=en&continue=https://colab.research.google.com/github/ClointFusion/ClointFusion/blob/master/ClointFusion_Labs.ipynb" #https://colab.research.google.com/github/ClointFusion/ClointFusion/blob/master/ClointFusion_Labs.ipynb"
# colab_url = "https://colab.research.google.com/github/ClointFusion/ClointFusion/blob/master/ClointFusion_Labs.ipynb"
# elif user_choice == "ClointFusion Lite (Interns Only)":
# #Extract encrypted version of ClointFusion_Lite to a specific folder and in Colab import that folder
# colab_url = 'https://accounts.google.com/signin/v2/identifier?authuser=0&hl=en&continue=https://colab.research.google.com/drive/11MvoQfNFXJqlXKcXV1LBVUE98Ks48M_a'
elif user_choice == "ClointFusion Starter (Hackathon)":
colab_url = 'https://accounts.google.com/signin/v2/identifier?authuser=0&hl=en&continue=https://colab.research.google.com/drive/1G9mh58z8AbWqBit2TC4Wgg6p_eHPvUJB'
user_data_path = "C:\\Users\\{}\\AppData\\Local\\Google\\Chrome\\User Data".format(os.getlogin())
modify_file_as_text(user_data_path + '\\Default\\Preferences', 'crashed', 'false')
options = Options()
options.add_argument("--start-maximized")
options.add_experimental_option('excludeSwitches', ['enable-logging'])
if os_name == "windows":
options.add_argument("user-data-dir=C:\\Users\\{}\\AppData\\Local\\Google\\Chrome\\User Data".format(os.getlogin()))
elif os_name == "darwin":
options.add_argument("user-data-dir=/Users/{}/Library/Application/Support/Google/Chrome/User Data".format(os.getlogin()))
options.add_argument(f"profile-directory=Default")
browser_driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
browser.set_driver(browser_driver)
browser.go_to(colab_url)
chrome = gw.getWindowsWithTitle('Google Chrome')[0]
chrome.activate()
# pg.doubleClick(pg.size()[0]/2,pg.size()[1]/2)
# kb.press_and_release('esc')
# kb.press_and_release('esc')
try:
browser.wait_until(browser.Text("Code").exists,timeout_secs=6)
except :#selenium_wrappers.common.exceptions.TimeoutException:
try:
browser.click(email)
except:
browser.write(email, into='Email or phone')
browser.click('Next')
time.sleep(0.5)
browser.write(passwd, into='Enter your password')
browser.click('Next')
time.sleep(0.5)
browser.wait_until(browser.Text("Code").exists,timeout_secs=240)
# kb.press_and_release('esc')
# time.sleep(0.2)
# pg.press(ESCAPE)
# time.sleep(0.2)
# press(ESCAPE)
# time.sleep(0.2)
if FIRST_TIME:
#create short-cut
browser.press(browser.CONTROL + 'mh')
time.sleep(1)
v = S("//input[@id='pref_shortcut_connectLocal']")
browser.write('',v)
browser.press(browser.CONTROL + '1')
time.sleep(0.5)
browser.click("SAVE")
time.sleep(1)
#use short-cut
browser.press(browser.CONTROL + '1')
time.sleep(1)
# pg.alert("HKHR")
pg.doubleClick(pg.size()[0]/2,pg.size()[1]/2)
time.sleep(1)
if FIRST_TIME:
# kb.press_and_release('SHIFT+TAB')
pg.hotkey('SHIFT', 'TAB')
time.sleep(0.5)
# kb.press_and_release('SHIFT+TAB')
pg.hotkey('SHIFT', 'TAB')
time.sleep(0.5)
# kb.press_and_release('SHIFT+TAB')
pg.hotkey('SHIFT', 'TAB')
time.sleep(0.5)
pg.write("http://localhost:8888")
# kb.write("http://localhost:8888")
time.sleep(2)
# click("CONNECT")
# kb.press_and_release('TAB')
pg.hotkey('TAB')
time.sleep(0.5)
# pg.alert(1)
# kb.press_and_release('TAB')
pg.hotkey('TAB')
time.sleep(0.5)
# pg.alert(2)
else:
# kb.press_and_release('SHIFT+TAB')
pg.hotkey('SHIFT', 'TAB')
time.sleep(0.5)
browser.press(browser.ENTER)
time.sleep(2)
# try:
# img = "Restore_Bubble.PNG"
# pos = pg.locateOnScreen(img, confidence=0.8) #region=
# pg.alert(pos)
# pg.click(*pos)
# except:
# pass
pg.alert("Ready ! Google Colab is now connected with your Local Runtime.\n\nPlease click 'OK' & you are all set to work on ClointFusion Colabs...")
except Exception as ex:
print("Error in connect_to_local_runtime="+str(ex))
exc_type, exc_value, exc_tb = sys.exc_info()
pg.alert(traceback.format_exception(exc_type, exc_value, exc_tb,limit=None, chain=True))
pg.alert("Error in connect_to_local_runtime="+str(ex))
connect_to_local_runtime()
/ args.slide_path_or_directory_name
)
slides = list(slide_directory.glob("*.html"))
assert len(slides) == 1, str(slides)
slide_html_path = slides[0]
else:
# Case: slide path
slide_html_path = Path(args.slide_path_or_directory_name).resolve()
slide_page_url = slide_html_path.as_uri()
# タイトルが長い場合は2枚めを使う運用をしている
if args.is_long_title:
slide_page_url += "#/1"
if not args.output_path:
args.output_path = (
project_root
/ "docs"
/ "_images"
/ "ogps"
/ f"{slide_html_path.stem}.png"
)
options = FirefoxOptions()
options.add_argument("--width=800")
options.add_argument("--height=485") # ブラウザウィンドウ上部が85px分ある
start_firefox(options=options, headless=True)
go_to(slide_page_url) # TODO: #/1 のようにページの指定を追加可能に
get_driver().save_screenshot(str(args.output_path))
kill_browser()