def _parse_link_from_db(self, link):
url = Url(link['url']['url'])
url.abstract_url = link['url']['abstract_url']
url.depth_of_finding = link['url']['depth_of_finding']
result = Link(url, link['dom_address'], link['html_id'],
link['html_class'])
return result
def add_link(self, board_id, pith, source, target):
link = Link(board_id=board_id, pith=pith, source=source, target=target)
link.id = "{}:{}".format(link.board_id, link.short_id)
self.gm.links.insert_one(link.to_mongo())
self._record_unit_update(board_id, source)
self._record_unit_update(board_id, target)
return {"link": self.gm._get_link(board_id, link.short_id)}
def post(self):
parsed_args = parser.parse_args()
link = Link(url=parsed_args['url'])
card = session.query(Card).filter(
Card.id == parsed_args['card_id']).first()
card.links.append(link)
session.add(link)
session.commit()
return link, 201
def post(self):
parsed_args = parser.parse_args()
card = Card(title=parsed_args['title'], content=parsed_args['content'])
if 'links' in parsed_args and parsed_args['links'] is not None:
for l in parsed_args['links']:
link = Link(url=l['url'])
card.links.append(link)
session.add(card)
session.commit()
return card, 201
def add_link(category: str, link: str,
uow: unit_of_work.AbstractUnitOfWork) -> Link:
with uow:
model = Link(link=link, category=category)
link_instance = uow.links.add(model)
uow.commit()
return link_instance
def extract_links_from_iframe(elements):
found_links = []
if len(elements) == 0:
return []
for element in elements:
src = element.attribute("src")
html_id = element.attribute("id")
html_class = element.attribute("class")
dom_address = element.evaluateJavaScript("getXPath(this)")
link = Link(src, dom_address, html_id, html_class)
found_links.append(link)
return found_links
def put(self, id):
parsed_args = parser.parse_args()
card = session.query(Card).filter(Card.id == id).first()
card.title = parsed_args['title']
card.content = parsed_args['content']
# TODO: use old links (now new ones are created)
if 'links' in parsed_args and parsed_args['links'] is not None:
for l in parsed_args['links']:
print(l['url'])
link = Link(url=l['url'])
card.links.append(link)
session.add(card)
session.commit()
return card, 201
def walk_folder_tree(self, entry_point: str):
root_link = Link(url=entry_point,
path=self.get_current_folder(entry_point),
name=self.get_current_folder(entry_point),
is_folder=True,
children=[])
stack = [root_link]
while stack:
link = stack.pop()
if link.is_folder:
links = self.list_children(link)
for link in links:
if link.is_folder:
stack.append(link)
self.notify_folder_updated(link)
def get_url_info(url, guess_and_check, max_depth):
# Create a new Link instance
link = Link()
# Save the extracted URL
link.url = url
# Remove markdown and other artifacts from the URL
link.url_clean = remove_markdown(url)
# Check if the clean URL is valid, if so continue with the next steps
link.url_clean_is_valid = check_if_valid_url(link.url_clean)
if link.url_clean_is_valid:
link.is_amp = check_if_amp(link.url_clean)
if link.is_amp:
link.is_cached = check_if_cached(link.url_clean)
link.domain = tldextract.extract(link.url_clean).domain
link = get_canonical(link, guess_and_check, max_depth)
return link
def get_url_info(url, use_gac, max_depth) -> Link:
link = Link(canonicals=[])
origin = UrlMeta(url=remove_markdown(url))
origin.is_valid = check_if_valid_url(origin.url)
origin.is_amp = check_if_amp(origin.url) and not any(
map(origin.url.__contains__, static.DENYLISTED_DOMAINS))
if origin.is_valid:
if origin.is_amp:
origin.is_cached = check_if_cached(origin.url)
origin.domain = tldextract.extract(origin.url).domain
link.origin = origin
link = get_canonicals(link=link,
max_depth=max_depth,
use_gac=use_gac)
return link
def list_children(self, link: Link) -> List[Link]:
parsed_url = urlparse(link.url)
url_prefix = "%s://%s%s" % (parsed_url.scheme, parsed_url.hostname,
parsed_url.path)
resp = requests.get(link.url)
soup = BeautifulSoup(resp.text, 'html.parser')
links = soup.find_all('a', class_="clearfix")
# Remove back to parent link
links = list(
filter(lambda x: x.get('data-name').strip() != '..', links))
return list(
map(
lambda x: Link(name=x.get('data-name').strip(),
path="%s/%s" %
(link.path, x.get('data-name').strip()),
url=url_prefix + x.get('href'),
is_folder=self.is_folder(x.get('href')),
children=[]), links))
def _extract_new_links_from_links(elements, requested_url):
found_links = []
new_clickables = []
if (len(elements) == 0):
#logging.debug("No links found...")
return [], []
else:
for elem in elements:
href = elem.attribute("href")
#logging.debug(str(type(elem)) + " href: " + str(href) + " Tagname: " + str(elem.tagName()))
if href == "/" or href == requested_url or href == "": #or href[0] == '#':
continue
elif "javascript:" in href: #We assume it as clickable
html_id = elem.attribute("id")
html_class = elem.attribute("class")
dom_address = elem.evaluateJavaScript("getXPath(this)")
event = href
tag = "a"
new_clickables.append(
Clickable(event, tag, dom_address, html_id, html_class,
None, None))
elif "#" in href:
html_id = elem.attribute("id")
html_class = elem.attribute("class")
dom_address = elem.evaluateJavaScript("getXPath(this)")
event = "click"
tag = "a"
new_clickables.append(
Clickable(event, tag, dom_address, html_id, html_class,
None, None))
elif len(href) > 0:
html_id = elem.attribute("id")
html_class = elem.attribute("class")
dom_address = elem.evaluateJavaScript("getXPath(this)")
url = href
link = Link(url, dom_address, html_id, html_class)
found_links.append(link)
else:
logging.debug(
"Elem has attribute href: " +
str(elem.attribute("href") + " and matches no criteria"))
return found_links, new_clickables
def get_nodes():
return_value = ""
threads = list()
for device in json.loads(config['TARGETS']['devices']):
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(run_cmd, device, "cisco_ios",
config['AUTH']['username'],
config['AUTH']['password'],
"show lldp neighbors detail")
return_value = future.result()
neighbors = json.loads(return_value,
object_hook=lambda d: SimpleNamespace(**d))
for i in neighbors:
nb = i.neighbor.split(".")[0].lower()
if nb not in nodes:
nodes[nb] = Node(i.neighbor, i.chassis_id, i.management_ip,
i.capabilities)
if i.local_interface != "" and i.neighbor_interface != "":
links[device + "<->" + nb] = Link([
Interface(i.local_interface, "", "", "", "", device),
Interface(i.neighbor_interface, "", "", "", "", nb)
])
def get(self):
template_params = {}
description = self.request.get('description')
url_link = self.request.get('url_link')
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
link = Link()
link.description = description
link.url_link = url_link
link.user = user.key
link.put()
self.response.set_cookie('our_token', str(user.key.id()))
self.response.write(json.dumps({'status': 'OK'}))
return
def get(self):
des = self.request.get('des')
url_link = self.request.get('url_link')
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
self.error(403)
self.response.write('No user')
self.response.write(html)
return
group = Group.get_by_id(int(self.request.get('groupid')))
link = Link()
link.description = des
link.url_link = url_link
link.user = user.key
link.ifInTheGroup = True
link.put()
group.links.append(link.key)
group.put()
self.response.write(json.dumps({"status": "OK"}))
Tag(name="React", folder_id=1),
Tag(name="SQL", folder_id=1),
Tag(name="Front end", folder_id=1),
Tag(name="Back end", folder_id=1),
Tag(name="Python", folder_id=1),
Tag(name="Design", folder_id=1),
Tag(name="Testing", folder_id=1),
Tag(name="News", folder_id=1)
]
#! Need to add tags for links in folders 2 and 3
list_links = [
Link(name="NPM Supertest",
description="SuperAgent driven library for testing HTTP servers",
url="https://www.npmjs.com/package/supertest",
image="TODO: update",
importance="Medium",
tags=[list_tags[0]],
folder_id=1),
Link(
name="Chai testing library",
description=
"Chai is a BDD / TDD assertion library for [node](http://nodejs.org) and the browser that can be delightfully paired with any javascript testing framework.",
url="https://www.chaijs.com/",
image="TODO: update",
importance="Low",
tags=[list_tags[1]],
folder_id=1),
Link(
name="Cloudinary",
description=
