def parse(self, username, password):
driver = Chrome()
self.sign_in(username, password, driver)
input_data = self.get_input_data()
highlights = {
item['title']: item['id']
for item in input_data['highlights']
}
story_hashtags = {
item['hash_tag']: item['id']
for item in input_data['story_hashtags']
}
igtv_hashtags = {
item['hash_tag']: item['id']
for item in input_data['igtv_hashtags']
}
output_data = dict()
for user in input_data['users']:
driver.get(user['instagram'])
request = driver.wait_for_request('include_suggested_users')
output_data['stories'] = self.get_stories(
user['id'],
driver,
request,
highlights,
story_hashtags,
)
output_data['igtvs'] = self.get_igtv_data(
driver,
user,
igtv_hashtags,
)
self.send_to_app(output_data)
def test_set_proxy_config(self, chrome_super_kwargs):
Chrome()
options = chrome_super_kwargs['options']
assert options.capabilities['proxy']['proxyType'] == 'manual'
assert options.capabilities['proxy']['httpProxy'] == '127.0.0.1:12345'
assert options.capabilities['proxy']['sslProxy'] == '127.0.0.1:12345'
assert 'noProxy' not in options.capabilities['proxy']
assert options.capabilities['acceptInsecureCerts'] is True
def browser():
driver = Chrome()
driver.get(urls.SIGN_IN_URL)
driver.implicitly_wait(5)
yield driver
driver.quit()
import subprocess as sp
from pathlib import Path
import requests
from seleniumwire.webdriver import Chrome
# from selenium.webdriver import
VID_PAT = re.compile(r"https://vod\.video\.cornell\.edu/media/(?P<vid_id>[^/]+)")
HLS_HOST = "https://cdnapisec.kaltura.com/"
SUFFIX_PAT = r"/a.m3u8.*uiConfId=\d+$"
M3U8_PAT = HLS_HOST + ".*" + SUFFIX_PAT
SRT_HOST = "https://cdnsecakmi.kaltura.com/"
SRT_SUFFIX_PAT = r"\.srt$"
SRT_PAT = SRT_HOST + ".*" + SRT_SUFFIX_PAT
driver = Chrome()
driver.scopes = [M3U8_PAT, SRT_PAT]
driver.implicitly_wait(20)
def get_master_pls(vid_id):
driver.switch_to.frame("kplayer_ifp")
driver.find_element_by_class_name('largePlayBtn').click()
driver.wait_for_request(M3U8_PAT)
driver.execute_script('document.getElementsByClassName("mwEmbedPlayer")[0].click()')
# driver.find_element_by_class_name("mwEmbedPlayer").click()
driver.switch_to.parent_frame()
for req in driver.requests:
if re.search(HLS_HOST + rf".*{vid_id}.*" + SUFFIX_PAT , req.url):
return req.response.body
def __init__(
self,
url,
get_rate=0.5,
chat_per_refresh=100,
cli_login=True,
user_data=None,
timeout=10,
):
self.chatURL = url
self.chat_per_refresh = chat_per_refresh
self.get_rate = get_rate
self.timeout = timeout
self.on_chat = lambda x, y: []
self.on_ready = lambda: []
self.dict_user = {}
def response_interceptor(req, res):
if 'sync_chat_channel' in req.url and req.method == 'GET':
try:
dict_raw = gzip.decompress(res.body)
dict_usr = json.loads(dict_raw)
dict_usr = dict_usr['result_data']['users']
for usr in dict_usr:
self.dict_user[usr["user_no"]] = usr["name"]
print("Found usercode-username dictionary")
except:
print("Invalid dictionary")
caps = DesiredCapabilities.CHROME
caps['goog:loggingPrefs'] = {'performance': 'ALL'}
options = ChromeOptions()
options.add_experimental_option('perfLoggingPrefs', {
'enableNetwork': True,
'enablePage': False
})
options.add_argument('--disable-extensions')
options.add_argument("--no-sandbox")
if cli_login:
options.headless = True
if user_data is not None:
options.add_argument(f"--user-data-dir={user_data}")
print("Driver initializing...")
self.driver = Chrome(options=options)
self.driver.response_interceptor = response_interceptor
self.driver.implicitly_wait(timeout)
print("Driver initialized.")
if cli_login:
self.driver.get(self.chatURL)
print("Get login page completed.")
self._locate_by_css_selector(".uBtn.-icoType.-phone").click()
phone_box = self._locate_by_id("input_local_phone_number")
print("Get PhonenumberPage completed.")
phone = input("Phone number: +82")
phone_box.send_keys(phone)
self._locate_by_css_selector(".uBtn.-tcType.-confirm").click()
pw_box = self._locate_by_id("pw")
print("Get PasswordPage completed.")
pw = input("Password: "******"pw").send_keys(pw)
self._locate_by_css_selector(".uBtn.-tcType.-confirm").click()
print("Get SMSPage completed.")
try:
print("Trying to get hintNumberDiv...")
hint = self._locate_by_id("hintNumberDiv")
print("Hintnumber:", hint.text)
input("Press Enter to continue...")
except:
print("Retrieving SMS authcode...")
code_box = self._locate_by_id("code")
print("codebox grabbed")
pw_band = input("SMS authcode: ")
code_box.send_keys(str(pw_band))
self._locate_by_css_selector(
"button.uBtn.-tcType.-confirm").click()
print("Login completed.")
else:
self.driver.get(self.chatURL)
input("Please login from GUI.\nPress Enter to Continue...")
self._refresh()
self._clear_log()
def test_chrome_options(self, chrome_super_kwargs):
options = ChromeOptions()
Chrome(chrome_options=options, seleniumwire_options={})
assert 'chrome_options' not in chrome_super_kwargs
assert chrome_super_kwargs['options'] == options
def test_no_auto_config(self, chrome_super_kwargs):
Chrome(seleniumwire_options={'auto_config': False}, capabilities={'test': 'capability'})
assert 'proxy' not in chrome_super_kwargs['options'].capabilities
def test_existing_capability(self, chrome_super_kwargs):
Chrome(desired_capabilities={'test': 'capability'})
capabilties = chrome_super_kwargs['desired_capabilities']
assert capabilties['test'] == 'capability'
def test_no_proxy(self, chrome_super_kwargs):
Chrome(seleniumwire_options={'exclude_hosts': 'test_host'})
options = chrome_super_kwargs['options']
assert options.capabilities['proxy']['noProxy'] == 'test_host'
def test_accept_insecure_certs(self, chrome_super_kwargs):
Chrome()
chrome_options = chrome_super_kwargs['options']
assert 'acceptInsecureCerts' in chrome_options.capabilities
def test_proxy_bypass_list(self, chrome_super_kwargs):
Chrome()
chrome_options = chrome_super_kwargs['options']
assert '--proxy-bypass-list=<-loopback>' in chrome_options.arguments
def test_create_backend(self, mock_backend):
chrome = Chrome()
assert chrome.backend
mock_backend.create.assert_called_once_with(addr='127.0.0.1', port=0, options={})
mock_backend.create.return_value.address.assert_called_once_with()