def capture_url(port_tuple, project='project', timeout=10):
"""
This function is responsible to create a screen capture from ip and port.
The procedure of this function creates a URL which consists from ip:port,
If the url is valid, it opens headless browser and capture the page.
Finally, it returns tuple with the current details (host, port, details, url).
:param port_tuple: Tuple of ports.
:param project: Project name. Default is 'project'
:param timeout:How long to wait on page load. Default is 10 secs.
"""
# Extract The Port Tuple
host, port, details = port_tuple
# Crate New Port Details Dictionary
port_details = {
'state':
details['state'].upper(),
'service':
details['name'].upper(),
'product':
"{product_name} {product_version}".format(
product_name=details['product'],
product_version=details['version'],
).strip()
}
# Create Ghost Object And Set Size
ghost = Ghost(
wait_timeout=timeout,
download_images=True,
cache_dir=None,
)
ghost.webview.resize(QSize(1280, 720))
ghost.page.setViewportSize(QSize(1280, 720))
# Try To Open URL
page = None
for http_type in ('http', 'https'):
request_url = '%s://%s:%d/' % (http_type, host, port)
try:
page = ghost.open(request_url)[0]
except TimeoutError:
pass
if page:
# Make a Screen Capture
capture_name = '%s-%s-%d.png' % (http_type, host, port)
ghost.capture_to(path.join(project, capture_name))
return host, port, port_details, request_url
return host, port, port_details, 'Not HTTP/S Service'
def capture_url(port_tuple, project='project', timeout=10):
"""
This function is responsible to create a screen capture from ip and port.
The procedure of this function creates a URL which consists from ip:port,
If the url is valid, it opens headless browser and capture the page.
Finally, it returns tuple with the current details (host, port, details, url).
:param port_tuple: Tuple of ports.
:param project: Project name. Default is 'project'
:param timeout:How long to wait on page load. Default is 10 secs.
"""
# Extract The Port Tuple
host, port, details = port_tuple
# Crate New Port Details Dictionary
port_details = {
'state': details['state'].upper(),
'service': details['name'].upper(),
'product': "{product_name} {product_version}".format(
product_name=details['product'],
product_version=details['version'],
).strip()
}
# Create Ghost Object And Set Size
ghost = Ghost(
wait_timeout=timeout,
download_images=True,
cache_dir=None,
)
ghost.webview.resize(QSize(1280, 720))
ghost.page.setViewportSize(QSize(1280, 720))
# Try To Open URL
page = None
for http_type in ('http', 'https'):
request_url = '%s://%s:%d/' % (http_type, host, port)
try:
page = ghost.open(request_url)[0]
except TimeoutError:
pass
if page:
# Make a Screen Capture
capture_name = '%s-%s-%d.png' % (http_type, host, port)
ghost.capture_to(path.join(project, capture_name))
return host, port, port_details, request_url
return host, port, port_details, 'Not HTTP/S Service'
def add_game(self, gid: int, host: str) -> bool:
''' Creates a game in the engine.
Returns True if the game was successfully created '''
if gid in self.__games:
logging.warning(GhostEngine.ERR_GID_ALREADY_EXISTS)
return False
elif len(self.__games) >= self.__MAX_NUMBER_OF_GAMES:
logging.warning(GhostEngine.ERR_TOO_MANY_GAMES)
return False
new_game = Ghost()
self.__games[gid] = new_game
self.__host_to_gid[host] = gid
return True
def start_onmyoji(self):
section = self.ui.tabWidget.currentIndex()
# 读取配置
self.get_conf(section)
if section == 0:
# 御魂
if self.ui.mitama_single.isChecked():
# 单刷
self.fight = SingleFight()
elif self.ui.mitama_driver.isChecked():
# 司机
self.fight = DriverFighter()
elif self.ui.mitama_passenger.isChecked():
# 乘客
self.fight = FighterPassenger()
elif self.ui.mitama_dual.isChecked():
# 双开
self.fight = DualFighter()
elif section == 1:
# 探索
self.fight = ExploreFight()
elif section == 2:
# 百鬼夜行
self.fight = Ghost()
elif section == 3:
# 结界突破
if self.ui.individual.isChecked():
# 个人突破
self.fight = Breakthrough()
elif self.ui.shack.isChecked():
# 个人突破
self.fight = ShackBreakthrough()
elif section == 4:
self.fight = Task()
task = threading.Thread(target=self.fight.start)
task.start()
def browser_process(queue,
report_queue,
useragent,
proxy=None,
proxy_auth=None,
java_enabled=False,
project='project',
timeout=10,
verbose=False):
"""
This function is responsible to create a hidden browser with seperate process, navigating over URLS from queue and taking a screen captures.
The procedure of this function creates a URL which consists from ip:port,
If the url is valid, it opens headless browser and capture the page.
Finally, it push a tuple with: host, port, details, url to the report queue.
:param queue: Queue of Tuple of ports.
:param report_queue: report queue
:param useragent: Specify User Agent.
:param proxy:
:param proxy_auth:
:param java_enabled:
:param project: Project name. Default is 'project'
:param timeout: How long to wait on page load. Default is 10 secs.
:param verbose:
:return:
"""
configure_logger()
try:
# Create Ghost Object And Set Size
ghost = Ghost()
session = ghost.start(wait_timeout=int(timeout),
user_agent=useragent,
java_enabled=java_enabled,
ignore_ssl_errors=True)
session = configure_proxy(session, proxy, proxy_auth)
session.page.setViewportSize(QSize(1280, 720))
# Extract The Port Tuple
while not queue.empty():
port_tuple = queue.get()
host, port, details = port_tuple
# Crate New Port Details Dictionary
port_details = {
'state':
details['state'].upper(),
'service':
details['name'].upper(),
'product':
"{product_name} {product_version}".format(
product_name=details['product'],
product_version=details['version'],
).strip()
}
is_webapp = False
# Try To Open URL
for protocol_type in ['http', 'https']:
request_url = '%s://%s:%d/' % (protocol_type, host, port)
capture_name = '%s-%s-%d.png' % (protocol_type, host, port)
capture_name = path.join(project, capture_name)
try:
page = session.open(request_url)[0]
if isinstance(page, HttpResource):
# Make a Screen Capture
page_content = str(page.content)
session.capture_to(capture_name)
title = str(
re.findall(r"<title>(.*)</title>", page_content,
re.DOTALL)[0])
port_details['url'] = request_url
port_details['page_title'] = title.replace(
"\r", '').replace("\n", '')
page_title = title if title != '' else "No title"
LOGGER.info(
"[{url}] {product}({name}) - {title}".format(
product=port_details['product'],
name=port_details['service'],
url=request_url,
title=page_title))
is_webapp = True
break
except TimeoutError:
continue
except IndexError:
continue
except Exception as e:
LOGGER.error("Error: %s" % str(e))
continue
if not is_webapp:
port_details['url'] = "Not HTTP/S Service"
if verbose:
LOGGER.debug(
"Host: {host}:{port} is not HTTP/S Service".format(
host=host,
port=port,
))
report_queue.put((host, port, port_details))
queue.task_done()
ghost.exit()
except KeyboardInterrupt:
raise KeyboardInterruptError()
except Exception as e:
LOGGER.error("Error: %s" % str(e))
def __get_game_from_gid(self, gid: int) -> Ghost:
if not self.__is_game_exists(gid):
return Ghost()
return self.__games[gid]
def fetch_grades(year,
semester,
base_meyda_net_url,
id_number,
meyda_net_password,
timeout=None):
logging.info("Starting fetching process")
reload(sys)
sys.setdefaultencoding("utf-8")
ghost = Ghost().start()
page, extra_resources = ghost.open(base_meyda_net_url + "/fireflyweb.aspx",
timeout=timeout)
if not page:
return None
result, resources = ghost.set_field_value("input[name=R1C1]", id_number)
result, resources = ghost.set_field_value("input[name=R1C2]",
meyda_net_password)
page, resources = ghost.call("form", "submit", expect_loading=True)
page, extra_resources = ghost.open(
base_meyda_net_url +
"/fireflyweb.aspx?PRGname=MenuCall&ARGUMENTS=-N,-N,-N0013,",
timeout=timeout)
uniq = str(
ghost.evaluate('document.getElementsByName("UNIQ")[0].value')[0])
# R1C2=0 means "שנתי"
page, extra_resources = ghost.open(
(base_meyda_net_url + "/fireflyweb.aspx?PRGname=Bitsua_maarechet_shaot"
+ "&ARGUMENTS=TZ,UNIQ,MisparSheilta,R1C1,R1C2&TZ=" + id_number +
"&UNIQ=%s&MisparSheilta=13&R1C1=" + year + "&R1C2=" + semester) %
uniq,
timeout=timeout)
logging.info("Starting parsing process")
parser = BeautifulSoup(str(page.content), "lxml")
table = parser.find("table", class_="SortMe")
table_thead = table.find("thead")
HEADERS = ["סמסטר", "שם קורס", "מועד", "ציון", "סוג מקצוע"]
table_col_dict = {}
index = 0
for th in table_thead.find("tr").find_all("th"):
current_header = th.text.strip().encode("utf-8")
if current_header in HEADERS:
table_col_dict[current_header] = index
index += 1
curr_grades = {}
def extract_text_by_key(key):
return line_cells[table_col_dict[key]].text.strip().encode("utf-8")
for tr in table.find("tbody").find_all("tr"):
line_cells = tr.find_all("td")
if "שעור" in extract_text_by_key("סוג מקצוע"):
ukey = extract_text_by_key("שם קורס") + ";" + extract_text_by_key(
"סמסטר") + ";" + extract_text_by_key("מועד")
curr_grades[ukey] = extract_text_by_key("ציון")
return curr_grades
def browser_process(queue, report_queue, useragent, proxy=None, proxy_auth=None,
java_enabled=False, project='project', timeout=10, verbose=False):
"""
This function is responsible to create a hidden browser with seperate process, navigating over URLS from queue and taking a screen captures.
The procedure of this function creates a URL which consists from ip:port,
If the url is valid, it opens headless browser and capture the page.
Finally, it push a tuple with: host, port, details, url to the report queue.
:param queue: Queue of Tuple of ports.
:param report_queue: report queue
:param useragent: Specify User Agent.
:param proxy:
:param proxy_auth:
:param java_enabled:
:param project: Project name. Default is 'project'
:param timeout: How long to wait on page load. Default is 10 secs.
:param verbose:
:return:
"""
configure_logger()
try:
# Create Ghost Object And Set Size
ghost = Ghost()
session = ghost.start(wait_timeout=int(timeout), user_agent=useragent, java_enabled=java_enabled,
ignore_ssl_errors=True)
session = configure_proxy(session, proxy, proxy_auth)
session.page.setViewportSize(QSize(1280, 720))
# Extract The Port Tuple
while not queue.empty():
port_tuple = queue.get()
host, port, details = port_tuple
# Crate New Port Details Dictionary
port_details = {
'state': details['state'].upper(),
'service': details['name'].upper(),
'product': "{product_name} {product_version}".format(
product_name=details['product'],
product_version=details['version'],
).strip()
}
is_webapp = False
# Try To Open URL
for protocol_type in ['http', 'https']:
request_url = '%s://%s:%d/' % (protocol_type, host, port)
capture_name = '%s-%s-%d.png' % (protocol_type, host, port)
capture_name = path.join(project, capture_name)
try:
page = session.open(request_url)[0]
if isinstance(page, HttpResource):
# Make a Screen Capture
page_content = str(page.content)
session.capture_to(capture_name)
title = str(re.findall(r"<title>(.*)</title>", page_content, re.DOTALL)[0])
port_details['url'] = request_url
port_details['page_title'] = title.replace("\r", '').replace("\n", '')
page_title = title if title != '' else "No title"
LOGGER.info("[{url}] {product}({name}) - {title}".format(
product=port_details['product'],
name=port_details['service'],
url=request_url,
title=page_title
))
is_webapp = True
break
except TimeoutError:
continue
except IndexError:
continue
except Exception as e:
LOGGER.error("Error: %s" % str(e))
continue
if not is_webapp:
port_details['url'] = "Not HTTP/S Service"
if verbose:
LOGGER.debug("Host: {host}:{port} is not HTTP/S Service".format(
host=host,
port=port,
))
report_queue.put((host, port, port_details))
queue.task_done()
ghost.exit()
except KeyboardInterrupt:
raise KeyboardInterruptError()
except Exception as e:
LOGGER.error("Error: %s" % str(e))
