def push_file(self, destination_path, base64data=None, source_path=None):
"""Puts the data from the file at `source_path`, encoded as Base64, in the file specified as `path`.
Specify either `base64data` or `source_path`, if both specified default to `source_path`
:param destination_path: the location on the device/simulator where the local file contents should be saved
:param base64data: file contents, encoded as Base64, to be written to the file on the device/simulator
:param source_path: local file path for the file to be loaded on device
:return: WebDriver instance
"""
if source_path is None and base64data is None:
raise InvalidArgumentException(
'Must either pass base64 data or a local file path')
if source_path is not None:
try:
with open(source_path, 'rb') as f:
data = f.read()
except IOError:
message = 'source_path {} could not be found. Are you sure the file exists?'.format(
source_path)
raise InvalidArgumentException(message)
base64data = base64.b64encode(data).decode('utf-8')
data = {
'path': destination_path,
'data': base64data,
}
self.execute(Command.PUSH_FILE, data)
return self
def push_file(self: T, destination_path: str,
base64data: Optional[str] = None, source_path: Optional[str] = None) -> T:
"""Puts the data from the file at `source_path`, encoded as Base64, in the file specified as `path`.
Specify either `base64data` or `source_path`, if both specified default to `source_path`
Args:
destination_path: the location on the device/simulator where the local file contents should be saved
base64data: file contents, encoded as Base64, to be written
to the file on the device/simulator
source_path: local file path for the file to be loaded on device
Returns:
Union['WebDriver', 'RemoteFS']: Self instance
"""
if source_path is None and base64data is None:
raise InvalidArgumentException('Must either pass base64 data or a local file path')
if source_path is not None:
try:
with open(source_path, 'rb') as f:
file_data = f.read()
except IOError as e:
message = f'source_path "{source_path}" could not be found. Are you sure the file exists?'
raise InvalidArgumentException(message) from e
base64data = base64.b64encode(file_data).decode('utf-8')
data = {
'path': destination_path,
'data': base64data,
}
self.execute(Command.PUSH_FILE, data)
return self
def start_session(self, capabilities, browser_profile=None):
"""
Creates a new session with the desired capabilities.
:Args:
- browser_name - The name of the browser to request.
- version - Which browser version to request.
- platform - Which platform to request the browser on.
- javascript_enabled - Whether the new session should support JavaScript.
- browser_profile - A selenium.webdriver.firefox.firefox_profile.FirefoxProfile object. Only used if Firefox is requested.
"""
if not isinstance(capabilities, dict):
raise InvalidArgumentException("Capabilities must be a dictionary")
if browser_profile:
if "moz:firefoxOptions" in capabilities:
capabilities["moz:firefoxOptions"]["profile"] = browser_profile.encoded
else:
capabilities.update({'firefox_profile': browser_profile.encoded})
w3c_caps = _make_w3c_caps(capabilities)
parameters = {"capabilities": w3c_caps,
"desiredCapabilities": capabilities}
response = self.execute(Command.NEW_SESSION, parameters)
if 'sessionId' not in response:
response = response['value']
self.session_id = response['sessionId']
self.capabilities = response.get('value')
# if capabilities is none we are probably speaking to
# a W3C endpoint
if self.capabilities is None:
self.capabilities = response.get('capabilities')
# Double check to see if we have a W3C Compliant browser
self.w3c = response.get('status') is None
self.command_executor.w3c = self.w3c
def set_window_rect(self, x=None, y=None, width=None, height=None):
"""
Sets the x, y coordinates of the window as well as height and width of
the current window. This method is only supported for W3C compatible
browsers; other browsers should use `set_window_position` and
`set_window_size`.
:Usage:
::
driver.set_window_rect(x=10, y=10)
driver.set_window_rect(width=100, height=200)
driver.set_window_rect(x=10, y=10, width=100, height=200)
"""
if not self.w3c:
raise UnknownMethodException(
"set_window_rect is only supported for W3C compatible browsers"
)
if (x is None and y is None) and (height is None and width is None):
raise InvalidArgumentException(
"x and y or height and width need values")
return self.execute(Command.SET_WINDOW_RECT, {
"x": x,
"y": y,
"width": width,
"height": height
})['value']
def __init__(self, kind, name):
super(PointerInput, self).__init__()
if (kind not in POINTER_KINDS):
raise InvalidArgumentException("Invalid PointerInput kind '%s'" % kind)
self.type = POINTER
self.kind = kind
self.name = name
def parse_founded(year_loc):
# Formats the overview->founded data to be a list.
# [0-1] = year, location. Defaults to -1 if either is not available.
split = year_loc.split(", ")
founded_list = [-1, -1]
if (len(split) == 0 or year_loc == ""):
return founded_list
elif (len(split) == 1):
try:
# if only available info is year
int(split[0])
founded_list[0] = split[0]
except ValueError:
# if only available info is location
founded_list[1] = split[0]
return founded_list
elif (len(split) == 2):
return split
elif (len(split) == 3):
# year, city, state format
founded_list[0] = split[0]
founded_list[1] = split[1] + ", " + split[2]
else:
raise InvalidArgumentException("overview->founded contained 4 pieces")
return founded_list
async def set_window_rect2(self, x=None, y=None, width=None, height=None):
if (x is None and y is None) and (height is None and width is None):
raise InvalidArgumentException("x and y or height and width need values")
rect = {"x": x, "y": y, "width": width, "height": height}
result = await self.execute2(Command.SET_WINDOW_RECT, rect)
return result['value']
def start_session(self, capabilities: dict, browser_profile=None) -> None:
"""
Creates a new session with the desired capabilities.
:Args:
- capabilities - a capabilities dict to start the session with.
- browser_profile - A selenium.webdriver.firefox.firefox_profile.FirefoxProfile object. Only used if Firefox is requested.
"""
if not isinstance(capabilities, dict):
raise InvalidArgumentException("Capabilities must be a dictionary")
if browser_profile:
if "moz:firefoxOptions" in capabilities:
capabilities["moz:firefoxOptions"][
"profile"] = browser_profile.encoded
else:
capabilities.update(
{'firefox_profile': browser_profile.encoded})
w3c_caps = _make_w3c_caps(capabilities)
parameters = {"capabilities": w3c_caps}
response = self.execute(Command.NEW_SESSION, parameters)
if 'sessionId' not in response:
response = response['value']
self.session_id = response['sessionId']
self.caps = response.get('value')
# if capabilities is none we are probably speaking to
# a W3C endpoint
if not self.caps:
self.caps = response.get('capabilities')
def index_to_col(index):
# Returns the column letter corresponding to the index returned from founded_list
to_alpha = {26: "AA", 27: "AB", 28: "AC", 29: "AD", 30: "AE", 31: "AF"}
if (index < 26):
return str(chr(index + 65))
elif (index < 32):
return to_alpha[index]
else:
raise InvalidArgumentException("founded_list has too many elements")
def main():
# Parse command line arguments
args = parser_arguments()
global verbose, delay, archived, level, pages, hash
verbose = args.verbose
delay = args.delay
level = args.level
hash = args.hash
try:
driver = get_suitable_driver(args.browser)
try:
# For each url to crawl
for url in args.urls:
# Obtain the domain and crawl
domain = get_domain(url, args.subdomain)
# si el dominio ya ha sido rastreado, carga la lista de links obtenidas
if reuse_links(domain, args.force, args.update):
print(
"El dominio '{0}' ya ha sido rastreado. Se usará la lista previa de enlaces. "
"Use -f para evitarlo o -u para actualizarla.".format(
domain))
pages = load_links(domain)
else:
# En caso contrario rastrea y almacena los links encontrados
print(
"Rastreando el dominio '{0}'. Esto puede tardar un rato."
.format(domain))
crawl_web(domain, driver, url, level, args.subdomain)
store_links(domain, pages)
if args.force:
delete_log_file(domain)
if not verbose:
print()
if not args.only:
# Archivo las páginas encontradas
tqdm.ncols = 120
for link in tqdm(pages, dynamic_ncols=True):
if args.archive == IS:
archive_is_page(driver, domain, link)
else:
raise InvalidArgumentException(
'En estos momentos solo está permitida la web de archive.is.'
)
showStatistics(domain)
except InvalidArgumentException as ex:
if args.verbose:
traceback.print_exc()
else:
sys.stderr.write('ERROR DE ARGUMENTOS: {0}\n'.format(ex.msg))
driver.close()
except WebDriverException as e:
print(str(e), file=sys.stderr)
def send_keypress(self, key, number_to_send=1):
""" Send 1 or more keypresses to the element """
# If key argument contains more than one character, throw exception
if len(key) != 1:
raise InvalidArgumentException(
"'key' argument must be a single character; '{key}' is invalid."
)
self.click()
element = self.driver.find_element(*self.locator)
[element.send_keys(key) for _ in range(number_to_send)]
return True
def start_session(self, capabilities, browser_profile=None):
if not isinstance(capabilities, dict):
raise InvalidArgumentException("Capabilities must be a dictionary")
if browser_profile:
if "moz:firefoxOptions" in capabilities:
capabilities["moz:firefoxOptions"]["profile"] = browser_profile.encoded
else:
capabilities.update({'firefox_profile': browser_profile.encoded})
self.capabilities = options.Options().to_capabilities()
self.session_id = self.r_session_id
self.w3c = True
def find_by(self, wait=None):
wait = self.wait if wait is None else wait
if self.by is None:
self.by = By.CSS_SELECTOR
elif self.by == "xpath":
self.by = By.XPATH
parent = self.browser.driver if self.parent is None else self.parent.get_actual()
try:
self.__cached__ = self.browser.wait(parent, wait=wait).until(lambda x: x.find_elements(self.by, self.locator), message=f"{self}")
except InvalidArgumentException:
raise InvalidArgumentException(f"invalid locator {self}")
def start_session(self, capabilities, browser_profile=None):
# 重写start_session方法，不再创建新窗口
Options = Remote.FirefoxOptions
if not isinstance(capabilities, dict):
raise InvalidArgumentException("Capabilities must be a dictionary")
if browser_profile:
if "moz:firefoxOptions" in capabilities:
capabilities["moz:firefoxOptions"][
"profile"] = browser_profile.encoded
else:
capabilities.update(
{'firefox_profile': browser_profile.encoded})
self.capabilities = Options().to_capabilities()
def open(self, url: str = ""):
from selenium.common.exceptions import InvalidArgumentException
"""Open given URL"""
self.__init_browser()
if self.base_url and not _FULL_URL_RE.fullmatch(url):
if not url.startswith("/"):
url = f"/{url}"
url = f"{self.base_url}{url}"
try:
self._driver.get(url)
except InvalidArgumentException as e:
raise InvalidArgumentException(f"invalid url")
return self
def get_suitable_driver(browser):
try:
if browser == PHANTOM:
return webdriver.PhantomJS(get_driver_path('phantomjs'))
elif browser == FIREFOX:
return webdriver.Firefox(get_driver_path('geckodriver'))
elif browser == CHROME:
return webdriver.Chrome(get_driver_path('chormedriver'))
else:
raise InvalidArgumentException(
"The navegador '{0}' no está contemplado en esta versión".
format(browser))
except WebDriverException as e:
raise WebDriverException(
f'The specific driver has to be in the path "{get_driver_path("")}": {str(e)}'
)
def set_window_rect(self, x=None, y=None, width=None, height=None):
"""
Sets the x, y coordinates of the window as well as height and width of
the current window.
:Usage:
driver.set_window_rect(x=10, y=10)
driver.set_window_rect(width=100, height=200)
driver.set_window_rect(x=10, y=10, width=100, height=200)
"""
if (x is None and y is None) and (height is None and width is None):
raise InvalidArgumentException("x and y or height and width need values")
return self.execute(Command.SET_WINDOW_RECT, {"x": x, "y": y,
"width": width,
"height": height})['value']
def start_session(self,
capabilities: Dict,
browser_profile: Optional[str] = None) -> None:
"""Creates a new session with the desired capabilities.
Override for Appium
Args:
automation_name: The name of automation engine to use.
platform_name: The name of target platform.
platform_version: The kind of mobile device or emulator to use
app: The absolute local path or remote http URL to an .ipa or .apk file, or a .zip containing one of these.
Read https://github.com/appium/appium/blob/master/docs/en/writing-running-appium/caps.md for more details.
"""
if not isinstance(capabilities, dict):
raise InvalidArgumentException('Capabilities must be a dictionary')
if browser_profile:
if 'moz:firefoxOptions' in capabilities:
# encoded is defined in selenium's original codes
capabilities['moz:firefoxOptions'][
'profile'] = browser_profile.encoded # type: ignore
else:
# encoded is defined in selenium's original codes
capabilities.update(
{'firefox_profile':
browser_profile.encoded}) # type: ignore
parameters = self._merge_capabilities(capabilities)
response = self.execute(RemoteCommand.NEW_SESSION, parameters)
if 'sessionId' not in response:
response = response['value']
self.session_id = response['sessionId']
self.capabilities = response.get('value')
# if capabilities is none we are probably speaking to
# a W3C endpoint
if self.capabilities is None:
self.capabilities = response.get('capabilities')
# Double check to see if we have a W3C Compliant browser
self.w3c = response.get('status') is None
self.command_executor.w3c = self.w3c
def start_session(self,
capabilities: Dict,
browser_profile: Optional[str] = None) -> None:
"""Creates a new session with the desired capabilities.
Override for Appium
Args:
capabilities: Capabilities which have following keys like 'automation_name', 'platform_name', 'platform_version', 'app'.
Read https://github.com/appium/appium/blob/master/docs/en/writing-running-appium/caps.md for more details.
browser_profile: Browser profile
"""
if not isinstance(capabilities, dict):
raise InvalidArgumentException('Capabilities must be a dictionary')
if browser_profile:
if 'moz:firefoxOptions' in capabilities:
# encoded is defined in selenium's original codes
capabilities['moz:firefoxOptions'][
'profile'] = browser_profile.encoded # type: ignore
else:
# encoded is defined in selenium's original codes
capabilities.update(
{'firefox_profile':
browser_profile.encoded}) # type: ignore
parameters = self._merge_capabilities(capabilities)
response = self.execute(RemoteCommand.NEW_SESSION, parameters)
if 'sessionId' not in response:
response = response['value']
self.session_id = response['sessionId']
self.capabilities = response.get('value')
# if capabilities is none we are probably speaking to
# a W3C endpoint
if self.capabilities is None:
self.capabilities = response.get('capabilities')
# Double check to see if we have a W3C Compliant browser
self.w3c = response.get('status') is None
self.command_executor.w3c = self.w3c
def parse_attributes(attributes_table):
# Parses the detail-col-attribute element, translating icons to type. Returns as comma separated string
auditable = "https://www.smithandcrown.com/wp-content/uploads/2017/03/ico_transparency_large-2.png"
detailed = "https://www.smithandcrown.com/wp-content/uploads/2017/01/ident-200.png"
new_chain = "https://www.smithandcrown.com/wp-content/uploads/2017/01/chain-200.png"
code = "https://www.smithandcrown.com/wp-content/uploads/2017/01/code-600.png"
barred = "https://www.smithandcrown.com/wp-content/uploads/2017/06/ico_nous_large.png"
attributes = ""
for ele in attributes_table.find_elements_by_tag_name("li"):
icon = ele.find_element_by_tag_name("img").get_attribute("src")
if (icon == auditable):
attributes = easy_concat(attributes, "Auditable Raise Amount")
elif (icon == detailed):
attributes = easy_concat(attributes, "Detailed Founder Identities")
elif (icon == new_chain):
attributes = easy_concat(attributes, "New Blockchain")
elif (icon == code):
attributes = easy_concat(attributes, "Project Code Available")
elif (icon == barred):
attributes = easy_concat(attributes,
"Sale Barred to U.S. Participants")
else:
raise InvalidArgumentException("Unknown icon in attributes")
return attributes
def proxy(self, value):
if not isinstance(value, Proxy):
raise InvalidArgumentException(
"Only Proxy objects can be passed in.")
self._proxy = value
def index_to_col(index):
# Returns the column letter corresponding to the index returned from founded_list
if (index < 26):
return str(chr(index + 65))
else:
raise InvalidArgumentException("index shouldn't be this high")