def add_link(self, board_id, pith, source, target):
link = Link(board_id=board_id, pith=pith, source=source, target=target)
link.id = "{}:{}".format(link.board_id, link.short_id)
self.gm.links.insert_one(link.to_mongo())
self._record_unit_update(board_id, source)
self._record_unit_update(board_id, target)
return {"link": self.gm._get_link(board_id, link.short_id)}
def get(self):
template_params={}
user = None
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
#newlinks
linkslist=Link.getAllLinksPerUser(user)
newurls = []
template_params = {}
if linkslist:
for link in linkslist:
url = link.url_link
des = link.description
fromlink=link.from_link
if fromlink is not None:
urlandlink =[url,des,fromlink]
newurls.append(urlandlink)
template_params['newurls'] = newurls
#newlinks
template_params['useremail'] = user.email
contactlist= Contact.getAllContactsPerUser(user)
contacts= []
if contactlist:
for contact in contactlist:
contact_email= contact.contact_user_email
contact_nickname= contact.nick_name
email_and_nickname= [contact_email, contact_nickname]
contacts.append(email_and_nickname)
template_params['contacts']=contacts
html = template.render("web/templates/contactspage.html", template_params)
self.response.write(html)
def get(self):
template_params={}
user = None
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
#newlinks
linkslist=Link.getAllLinksPerUser(user)
newurls = []
template_params = {}
if linkslist:
for link in linkslist:
url = link.url_link
des = link.description
fromlink=link.from_link
if fromlink is not None:
urlandlink =[url,des,fromlink]
newurls.append(urlandlink)
template_params['newurls'] = newurls
#newlinks
template_params['useremail'] = user.email
grouplist= Group.getAllGroups(user)
groups= []
if grouplist:
for group in grouplist:
group_name= group['group_name']
groupid=group['id']
one=[group_name,groupid]
groups.append(one)
template_params['groupss']= groups
html = template.render("web/templates/mygroups.html", template_params)
self.response.write(html)
def get(self, shortened_url):
"""Define get method."""
redirect_url = Link.get_original_url(shortened_url)
if redirect_url:
self.redirect(redirect_url)
else:
self.error(404)
def get(self):
template_params={}
user = None
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
linkslist=Link.getAllLinksPerUser(user)
#sorted(linkslist, key=lambda link: link.time_of_enter_the_link') # sort by age
newurls = []
template_params = {}
if linkslist:
for link in linkslist:
url = link.url_link
des = link.description
fromlink=link.from_link
if fromlink is not None:
urlandlink =[url,des,fromlink]
newurls.append(urlandlink)
template_params['newurls'] = newurls
template_params['useremail'] = user.email
html = template.render("web/templates/newlinks.html", template_params)
self.response.write(html)
return
def _parse_link_from_db(self, link):
url = Url(link['url']['url'])
url.abstract_url = link['url']['abstract_url']
url.depth_of_finding = link['url']['depth_of_finding']
result = Link(url, link['dom_address'], link['html_id'],
link['html_class'])
return result
def get(self):
template_params = {}
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
linkslist = Link.getAllLinksPerUser(user)
#sorted(linkslist, key=lambda link: link.time_of_enter_the_link') # sort by age
newurls = []
template_params = {}
if linkslist:
for link in linkslist:
url = link.url_link
des = link.description
fromlink = link.from_link
if fromlink is not None:
urlandlink = [url, des, fromlink]
newurls.append(urlandlink)
template_params['newurls'] = newurls
template_params['useremail'] = user.email
html = template.render("web/templates/newlinks.html", template_params)
self.response.write(html)
return
def post(self):
# prepare the csv file for reading
upload_files = self.get_uploads('csv_file')
blob_info = upload_files[0]
blob_reader = blobstore.BlobReader(blob_info.key())
reader = csv.reader(blob_reader, delimiter=',')
# set up the csv file for download
self.response.headers['Content-Type'] = 'text/csv'
self.response.headers['Content-Disposition'] = 'attachment; filename=shorterned_urls.csv'
writer = csv.writer(self.response.out)
writer.writerow(['Original Url', 'Short Url'])
# maximum link to convert is 500,000
if len(list(reader)) > MAX_URLS_TO_CONVERT:
raise ValueError("The maximum number of links to be converted is 500,000")
# A better way to handle this would be with flash messages or alerts, etc
# convert links, save to DB and download CSV
for row in reader:
if len(row) > 1:
raise ValueError("Input must have a single url per line")
# better checks can be implemented
original_url = row[0]
shortened_url = '{}/{}'.format(
self.request.application_url,
Link.shorten_url(original_url))
writer.writerow([original_url, shortened_url])
def get(self):
template_params = {}
# user = User.checkUser()
# if not user:
# template_params['loginUrl'] = User.loginUrl()
# else:
# template_params['logoutUrl'] = User.logoutUrl()
# template_params['user'] = user.email
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if user:
#newlinks
linkslist = Link.getAllLinksPerUser(user)
newurls = []
template_params = {}
if linkslist:
for link in linkslist:
url = link.url_link
des = link.description
fromlink = link.from_link
if fromlink is not None:
urlandlink = [url, des, fromlink]
newurls.append(urlandlink)
template_params['newurls'] = newurls
#newlinks
template_params['useremail'] = user.email
html = template.render("web/templates/index.html", template_params)
self.response.write(html)
def get(self):
template_params = {}
# user = User.checkUser()
# if not user:
# template_params['loginUrl'] = User.loginUrl()
# else:
# template_params['logoutUrl'] = User.logoutUrl()
# template_params['user'] = user.email
user = None
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if user:
#newlinks
linkslist=Link.getAllLinksPerUser(user)
newurls = []
template_params = {}
if linkslist:
for link in linkslist:
url = link.url_link
des = link.description
fromlink=link.from_link
if fromlink is not None:
urlandlink =[url,des,fromlink]
newurls.append(urlandlink)
template_params['newurls'] = newurls
#newlinks
template_params['useremail'] = user.email
html = template.render("web/templates/details.html", template_params)
self.response.write(html)
def get(self, group_id):
template_params={}
user = None
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
#newlinks
linkslist=Link.getAllLinksPerUser(user)
newurls = []
template_params = {}
if linkslist:
for link in linkslist:
url = link.url_link
des = link.description
fromlink=link.from_link
if fromlink is not None:
urlandlink =[url,des,fromlink]
newurls.append(urlandlink)
template_params['newurls'] = newurls
#newlinks
template_params['useremail'] = user.email
grouplist= Group.getAllGroups(user)
groups= []
if grouplist:
for group in grouplist:
group_name= group['group_name']
groupid=group['id']
one=[group_name,groupid]
groups.append(one)
template_params['groupss']= groups
group = Group.get_by_id(int(group_id))
if group is None:
return
if group.admin != user.key and user.key not in group.members:
template_params['no_access'] = True
html = template.render('web/templates/mygroups.html', template_params)
self.response.write(html)
return
else:
template_params['group_name'] = group.group_name
template_params['group_id'] = group_id
linksInGroup=group.getLinks()
links= []
for link in linksInGroup:
link_user_name= link['email']
link_description= link['description']
link_url_link= link['url_link']
link_date= link['time_of_enter_the_link']
one=[link_user_name,link_description,link_url_link,link_date]
links.append(one)
links.reverse()
template_params['links']= links
html = template.render('web/templates/mygroupschat.html', template_params)
self.response.write(html)
def get(self):
"""Define get method."""
stats = Link.get_stats()
self.render(
'links/get_stats.html',
stats=stats,
url=self.request.application_url
)
def get_url_info(url, use_gac, max_depth) -> Link:
link = Link(canonicals=[])
origin = UrlMeta(url=remove_markdown(url))
origin.is_valid = check_if_valid_url(origin.url)
origin.is_amp = check_if_amp(origin.url) and not any(
map(origin.url.__contains__, static.DENYLISTED_DOMAINS))
if origin.is_valid:
if origin.is_amp:
origin.is_cached = check_if_cached(origin.url)
origin.domain = tldextract.extract(origin.url).domain
link.origin = origin
link = get_canonicals(link=link,
max_depth=max_depth,
use_gac=use_gac)
return link
def post(self):
original_url = self.request.get('original_url')
shortened_url = Link.shorten_url(original_url)
self.render(
'links/add_link.html',
original_url=original_url,
shortened_url=shortened_url,
url=self.request.application_url)
def get(self):
description = self.request.get('description')
url_link = self.request.get('url_link')
to_link = self.request.get('to_link')
user = None
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user :
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
userTo=User.checkIfUesr(to_link)
if userTo is None:
return
Link.addlinkfronuser(userTo,url_link,description,user)
self.response.set_cookie('our_token', str(user.key.id()))
self.response.write(json.dumps({'status':'OK'}))
return
def get(self):
description = self.request.get('description')
url_link = self.request.get('url_link')
from_link = self.request.get('from_link')
if from_link == 'None':
from_link=None
user = None
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user :
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
#link=Link.getLink(user.key,url_link,description,None)
Link.remove(user.key,url_link,description,from_link)
self.response.set_cookie('our_token', str(user.key.id()))
self.response.write(json.dumps({'status':'OK'}))
return
def post(self):
parsed_args = parser.parse_args()
link = Link(url=parsed_args['url'])
card = session.query(Card).filter(
Card.id == parsed_args['card_id']).first()
card.links.append(link)
session.add(link)
session.commit()
return link, 201
def get(self):
counts = {}
counts["content"] = Content.all(keys_only=True).count()
counts["image"] = Image.all(keys_only=True).count()
counts["image_data"] = ImageData.all(keys_only=True).count()
counts["link"] = Link.all(keys_only=True).count()
counts["style"] = Style.all(keys_only=True).count()
counts["workspace"] = Workspace.all(keys_only=True).count()
self.response.out.write(str(counts) + "<p><form method='POST' action='/cleanup'><input type=submit value='Clean up'></form>")
def post(self):
parsed_args = parser.parse_args()
card = Card(title=parsed_args['title'], content=parsed_args['content'])
if 'links' in parsed_args and parsed_args['links'] is not None:
for l in parsed_args['links']:
link = Link(url=l['url'])
card.links.append(link)
session.add(card)
session.commit()
return card, 201
def add_link(category: str, link: str,
uow: unit_of_work.AbstractUnitOfWork) -> Link:
with uow:
model = Link(link=link, category=category)
link_instance = uow.links.add(model)
uow.commit()
return link_instance
def get(self):
description = self.request.get('description')
url_link = self.request.get('url_link')
to_link = self.request.get('to_link')
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
userTo = User.checkIfUesr(to_link)
if userTo is None:
return
Link.addlinkfronuser(userTo, url_link, description, user)
self.response.set_cookie('our_token', str(user.key.id()))
self.response.write(json.dumps({'status': 'OK'}))
return
def get(self):
description = self.request.get('description')
url_link = self.request.get('url_link')
from_link = self.request.get('from_link')
if from_link == 'None':
from_link = None
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
#link=Link.getLink(user.key,url_link,description,None)
Link.remove(user.key, url_link, description, from_link)
self.response.set_cookie('our_token', str(user.key.id()))
self.response.write(json.dumps({'status': 'OK'}))
return
def extract_links_from_iframe(elements):
found_links = []
if len(elements) == 0:
return []
for element in elements:
src = element.attribute("src")
html_id = element.attribute("id")
html_class = element.attribute("class")
dom_address = element.evaluateJavaScript("getXPath(this)")
link = Link(src, dom_address, html_id, html_class)
found_links.append(link)
return found_links
def post(self):
user = users.get_current_user()
if user and user.nickname() == "coolcucumber":
# Deletes all Datastore data!!!
db.delete(Content.all(keys_only=True).fetch(None))
db.delete(Image.all(keys_only=True).fetch(None))
db.delete(ImageData.all(keys_only=True).fetch(None))
db.delete(Link.all(keys_only=True).fetch(None))
db.delete(Style.all(keys_only=True).fetch(None))
db.delete(Workspace.all(keys_only=True).fetch(None))
self.response.out.write("Cleaned up")
else:
self.response.out.write("Unauthorized user")
def put(self, id):
parsed_args = parser.parse_args()
card = session.query(Card).filter(Card.id == id).first()
card.title = parsed_args['title']
card.content = parsed_args['content']
# TODO: use old links (now new ones are created)
if 'links' in parsed_args and parsed_args['links'] is not None:
for l in parsed_args['links']:
print(l['url'])
link = Link(url=l['url'])
card.links.append(link)
session.add(card)
session.commit()
return card, 201
def walk_folder_tree(self, entry_point: str):
root_link = Link(url=entry_point,
path=self.get_current_folder(entry_point),
name=self.get_current_folder(entry_point),
is_folder=True,
children=[])
stack = [root_link]
while stack:
link = stack.pop()
if link.is_folder:
links = self.list_children(link)
for link in links:
if link.is_folder:
stack.append(link)
self.notify_folder_updated(link)
def get_url_info(url, guess_and_check, max_depth):
# Create a new Link instance
link = Link()
# Save the extracted URL
link.url = url
# Remove markdown and other artifacts from the URL
link.url_clean = remove_markdown(url)
# Check if the clean URL is valid, if so continue with the next steps
link.url_clean_is_valid = check_if_valid_url(link.url_clean)
if link.url_clean_is_valid:
link.is_amp = check_if_amp(link.url_clean)
if link.is_amp:
link.is_cached = check_if_cached(link.url_clean)
link.domain = tldextract.extract(link.url_clean).domain
link = get_canonical(link, guess_and_check, max_depth)
return link
def list_children(self, link: Link) -> List[Link]:
parsed_url = urlparse(link.url)
url_prefix = "%s://%s%s" % (parsed_url.scheme, parsed_url.hostname,
parsed_url.path)
resp = requests.get(link.url)
soup = BeautifulSoup(resp.text, 'html.parser')
links = soup.find_all('a', class_="clearfix")
# Remove back to parent link
links = list(
filter(lambda x: x.get('data-name').strip() != '..', links))
return list(
map(
lambda x: Link(name=x.get('data-name').strip(),
path="%s/%s" %
(link.path, x.get('data-name').strip()),
url=url_prefix + x.get('href'),
is_folder=self.is_folder(x.get('href')),
children=[]), links))
def _extract_new_links_from_links(elements, requested_url):
found_links = []
new_clickables = []
if (len(elements) == 0):
#logging.debug("No links found...")
return [], []
else:
for elem in elements:
href = elem.attribute("href")
#logging.debug(str(type(elem)) + " href: " + str(href) + " Tagname: " + str(elem.tagName()))
if href == "/" or href == requested_url or href == "": #or href[0] == '#':
continue
elif "javascript:" in href: #We assume it as clickable
html_id = elem.attribute("id")
html_class = elem.attribute("class")
dom_address = elem.evaluateJavaScript("getXPath(this)")
event = href
tag = "a"
new_clickables.append(
Clickable(event, tag, dom_address, html_id, html_class,
None, None))
elif "#" in href:
html_id = elem.attribute("id")
html_class = elem.attribute("class")
dom_address = elem.evaluateJavaScript("getXPath(this)")
event = "click"
tag = "a"
new_clickables.append(
Clickable(event, tag, dom_address, html_id, html_class,
None, None))
elif len(href) > 0:
html_id = elem.attribute("id")
html_class = elem.attribute("class")
dom_address = elem.evaluateJavaScript("getXPath(this)")
url = href
link = Link(url, dom_address, html_id, html_class)
found_links.append(link)
else:
logging.debug(
"Elem has attribute href: " +
str(elem.attribute("href") + " and matches no criteria"))
return found_links, new_clickables
def get(self):
des = self.request.get('des')
url_link = self.request.get('url_link')
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
self.error(403)
self.response.write('No user')
self.response.write(html)
return
group = Group.get_by_id(int(self.request.get('groupid')))
link = Link()
link.description = des
link.url_link = url_link
link.user = user.key
link.ifInTheGroup = True
link.put()
group.links.append(link.key)
group.put()
self.response.write(json.dumps({"status": "OK"}))
def get_nodes():
return_value = ""
threads = list()
for device in json.loads(config['TARGETS']['devices']):
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(run_cmd, device, "cisco_ios",
config['AUTH']['username'],
config['AUTH']['password'],
"show lldp neighbors detail")
return_value = future.result()
neighbors = json.loads(return_value,
object_hook=lambda d: SimpleNamespace(**d))
for i in neighbors:
nb = i.neighbor.split(".")[0].lower()
if nb not in nodes:
nodes[nb] = Node(i.neighbor, i.chassis_id, i.management_ip,
i.capabilities)
if i.local_interface != "" and i.neighbor_interface != "":
links[device + "<->" + nb] = Link([
Interface(i.local_interface, "", "", "", "", device),
Interface(i.neighbor_interface, "", "", "", "", nb)
])
def get(self):
template_params = {}
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
#newlinks
linkslist = Link.getAllLinksPerUser(user)
newurls = []
template_params = {}
if linkslist:
for link in linkslist:
url = link.url_link
des = link.description
fromlink = link.from_link
if fromlink is not None:
urlandlink = [url, des, fromlink]
newurls.append(urlandlink)
template_params['newurls'] = newurls
#newlinks
template_params['useremail'] = user.email
contactlist = Contact.getAllContactsPerUser(user)
contacts = []
if contactlist:
for contact in contactlist:
contact_email = contact.contact_user_email
contact_nickname = contact.nick_name
email_and_nickname = [contact_email, contact_nickname]
contacts.append(email_and_nickname)
template_params['contacts'] = contacts
html = template.render("web/templates/contactspage.html",
template_params)
self.response.write(html)
def get(self):
template_params = {}
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
#newlinks
linkslist = Link.getAllLinksPerUser(user)
newurls = []
template_params = {}
if linkslist:
for link in linkslist:
url = link.url_link
des = link.description
fromlink = link.from_link
if fromlink is not None:
urlandlink = [url, des, fromlink]
newurls.append(urlandlink)
template_params['newurls'] = newurls
#newlinks
template_params['useremail'] = user.email
grouplist = Group.getAllGroups(user)
groups = []
if grouplist:
for group in grouplist:
group_name = group['group_name']
groupid = group['id']
one = [group_name, groupid]
groups.append(one)
template_params['groupss'] = groups
html = template.render("web/templates/mygroups.html", template_params)
self.response.write(html)
def get(self):
template_params = {}
description = self.request.get('description')
url_link = self.request.get('url_link')
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
link = Link()
link.description = description
link.url_link = url_link
link.user = user.key
link.put()
self.response.set_cookie('our_token', str(user.key.id()))
self.response.write(json.dumps({'status': 'OK'}))
return
def get(self):
template_params = {}
description = self.request.get('description')
url_link = self.request.get('url_link')
user = None
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user :
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
link=Link()
link.description=description
link.url_link=url_link
link.user=user.key
link.put()
self.response.set_cookie('our_token', str(user.key.id()))
self.response.write(json.dumps({'status':'OK'}))
return
def post(self):
user = users.get_current_user()
if not user or user.email() != '*****@*****.**':
return
recordId = None
if "id" in self.request.POST:
recordId = self.request.POST["id"]
link = None
if recordId is not None and recordId.isdigit():
#logging.info(recordId)
#logging.info(long(recordId))
link = Link.get_by_id(long(recordId))
if link is None:
link = Link()
link.title = self.request.POST["title"]
link.description = self.request.POST["description"]
link.url = self.request.POST["url"]
newKey = link.put()
self.response.out.write(newKey.id())
Tag(name="React", folder_id=1),
Tag(name="SQL", folder_id=1),
Tag(name="Front end", folder_id=1),
Tag(name="Back end", folder_id=1),
Tag(name="Python", folder_id=1),
Tag(name="Design", folder_id=1),
Tag(name="Testing", folder_id=1),
Tag(name="News", folder_id=1)
]
#! Need to add tags for links in folders 2 and 3
list_links = [
Link(name="NPM Supertest",
description="SuperAgent driven library for testing HTTP servers",
url="https://www.npmjs.com/package/supertest",
image="TODO: update",
importance="Medium",
tags=[list_tags[0]],
folder_id=1),
Link(
name="Chai testing library",
description=
"Chai is a BDD / TDD assertion library for [node](http://nodejs.org) and the browser that can be delightfully paired with any javascript testing framework.",
url="https://www.chaijs.com/",
image="TODO: update",
importance="Low",
tags=[list_tags[1]],
folder_id=1),
Link(
name="Cloudinary",
description=
def get_canonicals(link: Link,
max_depth=static.MAX_DEPTH,
use_db=True,
use_gac=True,
use_mr=True) -> Link:
next_url = link.origin.url
depth = 0
while depth < max_depth:
log.info(f"\nNEXT UP = {next_url}")
# Get the HTML content of the URL
response = get_page(next_url)
if not response:
log.warning("Invalid response, the page could not be fetched!")
return link
# Try every soup method
for method in CanonicalType:
# Create a new canonical instance of the specified method
canonical = Canonical(type=method)
# Assign the found url and note if it is still amp
canonical = get_canonical_with_soup(
r=response,
url=next_url,
meta=canonical,
original_url=link.origin.url,
use_db=use_db,
use_gac=use_gac,
use_mr=use_mr,
)
if canonical and canonical.is_valid:
link.canonicals.append(canonical)
# Disable resource-heavy and untrustworthy methods if a canonical without AMP was found
if not canonical.is_amp:
use_db = False
use_gac = False
use_mr = False
# Return 'empty' link if no canonicals were found
if len(link.canonicals) == 0:
log.warning(f"No canonicals found")
return link
# Sort the canonicals based on their url similarity score
link.canonicals.sort(key=lambda c: c.url_similarity, reverse=True)
# Filter out the canonicals that are still amp
canonicals_solved = [c for c in link.canonicals if c.is_amp is False]
# If there are one or more solved canonicals, pick the best one and assign it to the canonical attribute
if len(canonicals_solved) > 0:
log.info(f"Solved canonicals: {len(canonicals_solved)}")
link.canonical = canonicals_solved[0]
log.info(f"Best canonical: {link.canonical.url}")
# Assign alternative canonicals if solved canonicals have different domains
if len(canonicals_solved) > 1:
c_alt: Canonical = next((c for c in canonicals_solved
if c.domain != link.canonical.domain),
None)
if c_alt is not None:
for c in link.canonicals:
if c.domain == c_alt.domain and c.url_similarity == c_alt.url_similarity:
c.is_alt = True
log.info(f"Alternative canonical: {c_alt.url}")
return link
# If there a no solved canonicals, return no canonical, the amp canonical or run again
else:
# If the found URL is the same as the one before, return no canonical or the amp canonical
if next_url == link.canonicals[0].url:
log.warning("Didn't find a new canonical!")
if len(link.canonicals) > 0:
amp_canonical = link.canonicals[0]
log.info(
f"Found a canonical, but it's still AMP: {amp_canonical.url}"
)
if link.origin.is_cached:
log.info("No longer cached, using it")
link.amp_canonical = amp_canonical
return link
# If the found URL is different from before, but still amp, run again with the found URL
else:
log.info("Canonical found, but still AMP! Running again\n")
next_url = link.canonicals[0].url
depth += 1
# If there are no solved canonicals and depth has maxed out, return the amp canonical or no canonical
if len([c for c in link.canonicals if c.is_amp is False]) == 0:
if len(link.canonicals) > 0:
amp_canonical = link.canonicals[0]
log.info(f"Canonical found, but still AMP: {amp_canonical}")
if link.origin.is_cached:
log.info("No longer cached, using it")
link.amp_canonical = amp_canonical
log.warning("\nMax depth reached, no canonicals found!")
return link
def get(self, group_id):
template_params={}
user = None
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
html = template.render("web/templates/index.html", {})
self.response.write(html)
return
#newlinks
linkslist=Link.getAllLinksPerUser(user)
newurls = []
template_params = {}
if linkslist:
for link in linkslist:
url = link.url_link
des = link.description
fromlink=link.from_link
if fromlink is not None:
urlandlink =[url,des,fromlink]
newurls.append(urlandlink)
template_params['newurls'] = newurls
template_params['useremail'] = user.email
grouplist= Group.getAllGroups(user)
groups= []
if grouplist:
for group in grouplist:
group_name= group['group_name']
groupid=group['id']
one=[group_name,groupid]
groups.append(one)
template_params['groupss']= groups
group = Group.get_by_id(int(group_id))
if group is None:
return
if group.admin != user.key and user.key not in group.members:
template_params['no_access'] = True
html = template.render('web/templates/mygroups.html', template_params)
self.response.write(html)
return
else:
template_params['admin_access'] =False
if group.admin == user.key:
template_params['admin_access'] =True
template_params['group_name'] = group.group_name
template_params['group_id'] = group_id
linksInGroup=group.getLinks()
links= []
for link in linksInGroup:
link_user_name= link['email']
link_description= link['description']
link_url_link= link['url_link']
link_date= link['time_of_enter_the_link']
one=[link_user_name,link_description,link_url_link,link_date]
links.append(one)
sorted(links, key=lambda links: one[3])
links.reverse()
template_params['links']= links
all_Member_In_The_Group=group.getMembers()
all_Member_In_The_Group.insert(0,group.getAdminOfTheGroup())
template_params['members']=all_Member_In_The_Group
html = template.render('web/templates/mygroupschat.html', template_params)
self.response.write(html)
def get(self):
links = Link.all()
self.response.out.write(simplejson.dumps([m.to_dict() for m in links]))
def save():
now = datetime.datetime.now()
for rank, (url, name) in enumerate(get_data()):
link = Link.get_by_url_and_name(url, name)
top_link = TopLink(link, rank, now)
top_link.save()
