def __init__(self): super(MyScrapper, self).__init__() self.chromedriver = ChromeDriver() self.driver = self.chromedriver.drivers() self.user = '******' self.password = '******' self.download_url = 'https://5f56f298ac5ad2bc0c9c79ab.imnuke.app/video/stock/' self.required_login() self.downloadVideos()
def __init__(self):
RemoteWebDriver.__init__(self,
command_executor=ChromeDriver(),
browser_name='chrome', platform='ANY', version='',
javascript_enabled=True)
def __init__(self):
RemoteWebDriver.__init__(
self,
command_executor=ChromeDriver(custom_profile=None,
untrusted_certificates=False),
desired_capabilities=DesiredCapabilities.CHROME)
class MyScrapper(object):
"""docstring for MyScrapper"""
def __init__(self):
super(MyScrapper, self).__init__()
self.chromedriver = ChromeDriver()
self.driver = self.chromedriver.drivers()
self.user = '******'
self.password = '******'
self.download_url = 'https://5f56f298ac5ad2bc0c9c79ab.imnuke.app/video/stock/'
self.required_login()
self.downloadVideos()
def get_xpath(self,path):
element = self.driver.find_element_by_name(path)
return element
def check_size(self,text):
val= text.split("-")[1].replace("MB",'').strip()
return float(val) if val else None
def required_login(self):
url = 'https://imnuke.app/auth/login'
self.driver.get(url)
sleep = self.chromedriver.wait_random()
name_elem = self.get_xpath('login')
password_elem = self.get_xpath('password')
name_elem.send_keys(self.user)
sleep = self.chromedriver.wait_random()
password_elem.send_keys(self.password)
sleep = self.chromedriver.wait_random()
login_box = self.driver.find_element_by_class_name('btn-second').click()
sleep = self.chromedriver.sleep_time(5)
paid_button = self.driver.find_element_by_xpath('//*[text()="Only Paid"]').click()
sleep = self.chromedriver.sleep_time(8)
video_box = self.driver.find_element_by_xpath('//img[contains(@src,"https://imgur.com/PO6GNDr.png")]').click()
sleep = self.chromedriver.sleep_time(7)
def downloadVideos(self):
csv_instance = ReadCsv()
links_list = csv_instance.getLinks()
sleep = self.chromedriver.sleep_time(7)
for link in links_list:
sleep = self.chromedriver.sleep_time(3)
self.driver.get(self.download_url+link)
sleep = self.chromedriver.sleep_time(10)
# 4K
min_quality = self.driver.find_element_by_id('4KMP4')
min_quality_int=self.check_size(min_quality.text)
# HDMOV
hd_button = self.driver.find_element_by_id('HDMOV')
hd_button_int=self.check_size(hd_button.text)
# HDMP4
hdmp4_button = self.driver.find_element_by_id('HDMOV')
hdmp4_button_int=self.check_size(hdmp4_button.text)
if min_quality_int <= 150:
min_quality.click()
elif hd_button_int < min_quality_int and hd_button_int < hdmp4_button_int:
hd_button.click()
elif hdmp4_button_int < min_quality_int and hdmp4_button_int < hd_button_int:
hdmp4_button.click()
sleep = self.chromedriver.sleep_time(2)
download_button = self.driver.find_element_by_class_name('PrimaryButton').click()
sleep = self.chromedriver.sleep_time(6)
self.driver.close()
class WebDriver:
name = "Chrome"
def __init__(self):
self._driver = ChromeDriver()
self._driver.start()
def _execute(self, request, **kw):
command = kw.copy()
command["request"] = request
response = self._driver.execute(command)
code = response["statusCode"]
if code != 0:
raise ErrorInResponseException(response)
return response.get("value", None)
def get(self, url):
return self._execute("get", url=url)
def back(self):
return self._execute("goBack")
def forward(self):
return self._execute("goForward")
def close(self):
return self._execute("close")
def get_current_url(self):
return self._execute("getCurrentUrl")
def get_title(self):
return self._execute("getTitle")
def get_page_source(self):
return self._execute("getPageSource")
def get_cookies(self):
return self._execute("getCookies")
def add_cookie(self, cookie):
return self._execute("addCookie", cookie=cookie)
def delete_cookie(self, name):
return self._execute("deleteCookie", name=name)
def delete_cookies(self):
return self._execute("deleteAllCookies")
def switch_to_active_element(self):
eid = self._execute("getActiveElement")
return WebElement(eid, self)
def switch_to_window(self, name):
self._execute("switchToWindow", windowName=name)
def switch_to_frame(self, name):
self._execute("switchToFrameByName", name=name)
def wait_for_load_complete(self):
# Interface compatibility
return
def get_window_handles(self):
return self._execute("getWindowHandles")
def get_current_window_handle(self):
return self._execute("getCurrentWindowHandle")
def save_screenshot(self, jpeg_file):
image = self._execute("screenshot")
with open(jpeg_file, "w") as fo:
fo.write(image.decode("base64"))
def quit(self):
self._driver.stop()
def execute_script(self, script, *args):
#script = "(function() { return function(){" + script + "};})();";
args = [self._wrap_argument(a) for a in args]
response = self._execute("executeScript", script=script, args=args)
if not response:
return
return self._unwrap_argument(response)
def _wrap_argument(self, arg):
if isinstance(arg, (int, long, float)):
return { "NUMBER" : arg }
elif isinstance(arg, (bool, NoneType)):
return { "BOOLEAN" : bool(arg) }
elif isinstance(arg, WebElement):
return { "ELEMENT" : arg._id }
elif isinstance(arg, basestring):
return { "STRING" : arg }
elif isinstance(arg, (list, tuple, set)):
return [self._wrap_argument(a) for a in arg]
raise ValueError("Unknown type - %s" % arg.__class__)
def _unwrap_argument(self, arg):
if isinstance(arg, list):
return [self._unwrap_argument(a) for a in arg]
argtype = arg["type"]
if argtype == "NULL":
return None
elif argtype == "ELEMENT":
return WebElement(self, element_id(arg["value"]))
elif argtype == "POINT":
return Point(arg["x"], arg["y"])
elif argtype == "DIMENSION":
return Dimension(arg["width"], arg["height"])
elif argtype == "COOKIE":
return { "name" : arg["name"], "value" : arg["value"] }
else:
return arg["value"]
def _find(self, name, using, what, plural, parent=None):
args = {
"using" : using,
"value" : what
}
if parent:
args["id"] = parent
result = self._execute(name, **args)
if not result:
raise NoSuchElementException
elements = [WebElement(element_id(eid), self) for eid in result]
return elements if plural else elements[0]
def __init__(self):
self._driver = ChromeDriver()
self._driver.start()
def main():
# parse command line arguments
argparser = argparse.ArgumentParser(
description="Create and sync Trading212 pies holdings allocations to a shared pie or external source"
)
argparser.add_argument(
"--fetch-available-equities",
type=argparse.FileType("w"),
help="Fetch the list of available equieties to trade on Trading212 invest and save it to this file. "
"when using this option, there's no need to supply email, password or pie name.",
)
if not "--fetch-available-equities" in sys.argv:
argparser.add_argument(
"username", help="The email to log into your Trading212 account"
)
argparser.add_argument(
"password", help="The password to log into your Trading212 account"
)
argparser.add_argument(
"pie", help="The name of the pie to update (case-sensitive)"
)
argparser.add_argument(
"--from-json",
type=argparse.FileType("r"),
help="Parse the list of holdings to update from this .json file "
"with the format { [ticker]: [percentage], ... }",
)
argparser.add_argument(
"--from-csv",
type=argparse.FileType("r"),
help="Parse the list of holdings to update from this .csv file "
"with the format [ticker],[percentage] for each line",
)
argparser.add_argument(
"--from-shared-pie",
help="Parse the list of instruments to update from the URL of a shared pie",
)
argparser.add_argument(
"--substitutions",
type=argparse.FileType("r"),
default="substitutions.json",
help="Parse a list of replacement tickers from this .json file, "
"To be used when a ticker is not found. The list format is "
"{ [original ticker]: [ticker to use if original not found], ... }",
)
argparser.add_argument(
"-c",
"--await-confirm",
action="store_true",
help="Do not commit changes automatically and wait for user to confirm",
)
argparser.add_argument(
"-v", "--verbose", action="store_true", help="Increase output log verbosity"
)
args = argparser.parse_args()
# configure logging for the application
log.setLevel(logging.INFO if not args.verbose else logging.DEBUG)
rich_handler = RichHandler()
rich_handler.setFormatter(logging.Formatter(fmt="%(message)s", datefmt="[%X]"))
log.addHandler(rich_handler)
log.propagate = False
# initialize chromedriver
try:
driver = ChromeDriver()
n = Navigator(driver)
except InvalidArgumentException as e:
log.error(
f"Error initalising ChromeDriver: {e}"
+ "Is another automated Chrome window still open?"
)
sys.exit(0)
if args.fetch_available_equities:
file = args.fetch_available_equities
instruments = n.get_available_instruments()
file.write(" ".join(instruments))
log.info(f"Exported {len(instruments)} instruments to {file.name}")
sys.exit(0)
# start the application
data = {"instruments": {}}
if args.from_json:
data = json.load(args.from_json)
elif args.from_csv:
reader = csv.reader(args.from_csv)
data = {rows[0]: rows[1] for rows in reader}
elif args.from_shared_pie:
data = n.parse_shared_pie(args.from_shared_pie)
n.open_dashboard(args.username, args.password)
n.select_pie(args.pie)
current_instruments = n.get_current_instruments_tickers()
unused = [
ticker
for ticker in current_instruments
if ticker not in data.keys() or round(float(data[ticker]), 1) < 0.5
]
substitutions = json.load(args.substitutions) if args.substitutions else {}
for ticker in unused:
n.remove_instrument(ticker)
for ticker, distribution in data.items():
n.rebalance_instrument(ticker, distribution, substitutions)
n.redistribute_pie()
if not args.await_confirm:
n.commit_pie_edits(name=args.pie)
else:
input("Confirm changes and then press Enter to close the browser...")