def link_base(request):
from main.models import Link
from generic.views import get_unit_data
unit_mode, current_unit, unit_blank = get_unit_data(Link, request)
from units.models import Unit
main_unit = Unit.objects.get(pk=settings.ROOT_UNIT_PK)
main_unit.set_rights_can_select(lambda unit: Link.static_rights_can(
'SHOW_BASE', request.user, unit, None))
main_unit.set_rights_can_edit(lambda unit: Link.static_rights_can(
'SHOW_BASE', request.user, unit, None))
main_unit.check_if_can_use_hidden(request.user)
if Link.static_rights_can('SHOW_BASE', request.user, current_unit, None):
links = Link.objects.filter(deleted=False,
unit=current_unit,
leftmenu=None).order_by('title')
else:
links = []
return render(request, 'main/link/base.html', {
'unit_mode': unit_mode,
'main_unit': main_unit,
'links': links
})
def add_link(request):
context = RequestContext(request)
if request.method == 'POST':
url = request.POST.get("url", "")
tags = request.POST.get("tags", "")
title = request.POST.get("title", "")
tag = Tag(name=tags)
tag.save()
link = Link(title=title, url=url)
link.save()
link.tags.add(tag)
return redirect(index)
def add_link(request):
context = RequestContext(request)
if (request.method == 'POST'):
url = request.POST.get("url", "")
tags = request.POST.get("tags", "")
title = request.POST.get("title", "")
# Create and save tag
tag = Tag(name=tags)
tag.save()
# create and save link
link = Link(title=title, url=url)
link.save()
# add tag to link
link.tags.add(tag)
return redirect(index)
def checkMail():
"""
Celery task: checks mailbox for new emails with links.
If found, adds links to db.
"""
logger.debug("Checking for new mail")
boxes = Mailbox.objects.filter(name=MAIL_CONFIG["name"])
if boxes.count() == 0:
mailBox = Mailbox(name=MAIL_CONFIG["name"], uri=MAIL_CONFIG["uri"])
mailBox.save()
else:
mailBox = boxes[0]
logger.debug("Retrieving new mail from mailbox: " + mailBox.name)
newMessages = mailBox.get_new_mail()
if newMessages:
logger.info('[' + mailBox.name + '] Got new messages: ' + str(newMessages))
for msg in newMessages:
# find user for given email address
try:
user = User.objects.get(email=msg.from_address[0])
except User.DoesNotExist:
logger.warning('Got email from unknown user: '******'. Ignoring.')
continue
# extract url from message
url = re.search("(?P<url>https?://[^\s]+)", msg.get_text_body())
if url:
url = url.group("url")
else:
logger.warning('Got email without url from user: '******'. Ignoring.')
continue
# extract description from message
description = re.sub(r'https?:\/\/[^\s]+', '', msg.get_text_body(), flags=re.MULTILINE)
link = Link(user=user, title=msg.subject, link=url, description=description)
# don't let mezzanine screw up our description
link.gen_description = False
link.save()
logger.info('User ' + user.username + ' added new link by email: ' + url)
async def get(self):
session = await get_session(self.request)
if session.get('user'):
link = Link({'user_id': session.get('user')})
result = await link.link_list()
return {'links' : result }
else:
redirect(self.request,'login')
async def post(self):
data = await self.request.post()
session = await get_session(self.request)
user_id = session.get('user')
if user_id:
link = Link({'link': data.get('link'),'user_id': user_id})
result = await link.add_link()
if isinstance(result,str):
return { 'error' : result }
else:
redirect(self.request,'main')
def add_link(request) :
context = RequestContext(request)
if request.method == 'POST':
url = request.POST.get("url", "")
tags = request.POST.get("tags", "").split(' ')
title = request.POST.get("title", "")
# TODO: Your code here!
newLink = Link(title=title, url=url)
newLink.save()
for tag in tags:
try:
the_tag = Tag.objects.get(name=tag)
except ObjectDoesNotExist:
the_tag = Tag(name=tag)
the_tag.save()
newLink.tags.add(the_tag)
newLink.save()
return redirect(index)
def link_base(request):
from main.models import Link
from generic.views import get_unit_data
unit_mode, current_unit, unit_blank = get_unit_data(Link, request)
from units.models import Unit
main_unit = Unit.objects.get(pk=settings.ROOT_UNIT_PK)
main_unit.set_rights_can_select(lambda unit: Link.static_rights_can("SHOW_BASE", request.user, unit, None))
main_unit.set_rights_can_edit(lambda unit: Link.static_rights_can("SHOW_BASE", request.user, unit, None))
main_unit.check_if_can_use_hidden(request.user)
if Link.static_rights_can("SHOW_BASE", request.user, current_unit, None):
links = Link.objects.filter(deleted=False, unit=current_unit, leftmenu=None).order_by("title")
else:
links = []
return render(request, "main/link/base.html", {"unit_mode": unit_mode, "main_unit": main_unit, "links": links})
def minobrnauki_parse(url):
page_number = 1
type_grant = Type.objects.get(name='minobrnauki')
clean_grant(type_grant)
while True:
response = requests.get(
f'{url}?order_4=P_DATE&dir_4=DESC&page_4={page_number}',
verify=False,
timeout=120)
if response.status_code == 200:
soup = BS(response.text, features='html5lib')
li = soup.find("ul", class_="anons-list_docs").find_all("li")
if li:
for i in li:
text = i.find("a").text
time = i.find("time").text
time = datetime.datetime.strptime(time, '%d.%m.%Y').date()
link = 'https://www.minobrnauki.gov.ru' + i.find(
"a")['href']
if Grant.objects.filter(grant_name=type_grant,
text=text,
time=time,
link=link).exists():
return
else:
grant = Grant()
grant.grant_name = type_grant
grant.time = time
grant.link = link
grant.text = text
grant.save()
doc_links = i.find('div', class_='doc-links')
if doc_links is not None:
for i in doc_links.find_all('a'):
link = Link()
link.link = 'https://www.minobrnauki.gov.ru' + i[
'href']
link.grant_id = grant
link.save()
else:
break
else:
break
page_number += 1
def add_link(request):
"""Add a new link"""
# Get the context
context = RequestContext(request)
if request.method == "POST":
url = request.POST.get("url", "")
tags = request.POST.get("tags", "")
title = request.POST.get("title", "")
# TODO: Add the link to the tags
try:
link = Link(title=title, url=url)
link.save()
except IntegrityError:
messages.add_message(request, messages.ERROR, 'Error: That URL or title already exists.')
return redirect(index)
for tag_new in tags.split(" "):
try:
t1 = Tag.objects.get(name=tag_new)
except Tag.DoesNotExist:
t1 = Tag(name=tag_new)
t1.save()
finally:
link.tags.add(t1)
link.save()
return redirect(index)
def add_link(request):
context = RequestContext(request)
if request.method == 'POST':
url = request.POST.get("url", "")
input_tags = request.POST.get("tags", "")
input_tags = input_tags.split(' ')
title = request.POST.get("title", "")
new_link = Link(title=title, url=url)
try:
new_link.save()
except IntegrityError as e:
messages.error(request, e.message)
return redirect(index)
for tag in input_tags:
try:
found = Tag.objects.get(name=tag)
new_link.tags.add(found)
except:
new_tag = Tag(name=tag)
new_tag.save()
new_link.tags.add(new_tag)
try:
new_link.save()
except Error as e:
messages.error(request, e.message)
return redirect(index)
return redirect(index)
def update_link_table():
if Link.objects.all():
Link.objects.all().delete()
with open('main/data/links.csv') as f:
reader = csv.reader(f)
next(reader)
link_objects = [
Link(
movie_id=row[0],
imdb_id=row[1] if row[1] else None,
tmdb_id=row[2] if row[2] else None
)
for row in reader
]
Link.objects.bulk_create(link_objects)
def update_link():
itlist = []
for line in open(update_dir + '/link'):
flist = line.strip().split('\3')
if len(flist) < 3:
continue
it = Link()
it.mid = int(flist[0])
it.url = flist[1]
it.title = flist[2]
time = flist[3].split('-')
it.found_date = int(time[0]) * 10000 + int(time[1]) * 100 + int(
time[2])
itlist.append(it)
havelist = Link.objects.filter(mid__in=[it.mid for it in itlist])
linkmap = {i.url: i for i in havelist}
for it in itlist:
if it.url not in linkmap:
it.save()
def update_link():
itlist = []
for line in open(update_dir +'/link'):
flist = line.strip().split('\3')
if len(flist) <3:
continue
it = Link()
it.mid = int(flist[0])
it.url = flist[1]
it.title = flist[2]
time = flist[3].split('-')
it.found_date = int(time[0])*10000 +int(time[1])*100 +int(time[2])
itlist.append(it)
havelist = Link.objects.filter(mid__in=[it.mid for it in itlist])
linkmap = { i.url:i for i in havelist}
for it in itlist:
if it.url not in linkmap:
it.save()
def add_link(request):
context = RequestContext(request)
if request.method == 'POST':
url = request.POST.get("url", "")
tags = request.POST.get("tags", "")
title = request.POST.get("title", "")
link = Link()
link.url = url
link.title = title
link.save()
for tag in tags.split(','):
tag = tag.strip()
try:
link.tags.add(Tag.objects.get(name=tag))
except:
pass #Don't care if tag non-existent
link.save()
return redirect(index)
def add_link(request):
context = RequestContext(request)
if request.method == 'POST':
url = request.POST.get("url", "")
tags = request.POST.get("tags", "")
title = request.POST.get("title", "")
tags = tags.split(',')
l = Link()
l.url = url
l.title = title
l.save()
existing_tags = Tag.objects.all()
for tag in tags:
tag = tag.strip()
try:
link.tags.add(Tag.objects.create(name = "tag"))
except:
pass
l.save()
return redirect(index)
def add_link(request):
context= RequestContext(request)
if request.method =="POST":
url=request.POST.get('url','')
tags=request.POST.get('tags','')
title= request.POST.get('title','')
new_link= Link(title=title,url=url)
#Many to many relationship
new_link.save()
for tag in tags.split(','):
try:
new_link.tags.add(Tag.objects.get(name=tag))
except:
continue
#new_link.save()
#return redirect(index)
new_link.save()
return redirect(index)
# return render(request, 'main/index.html', {})
# Create your views here.
#http://127.0.0.1:8000/bookmarks/
def update_link(linklist):
itlist = []
for info in linklist:
it = Link()
try:
if len(info.mid) > 1:
it.mid = int(info.mid)
else:
it.mid = 0
if len(info.imdbid) > 1:
it.imdbid = int(info.imdbid)
else:
it.imdbid = 0
it.url = info.url
print "quality", info.quality
it.quality = info.quality
it.urlmd5 = utils.get_md5_value(info.url)
it.cname = info.cname
it.ename = info.ename
it.actors = info.actors
it.director = info.director
it.date = utils.get_date_from_string(info.date)
it.title = info.raw
it.content = info.content
if len(it.content) > 950:
it.content = it.content[0:950]
it.found_date = utils.get_date_now()
except Exception, e:
traceback.print_exc(sys.stdout)
print "update LINK ERROR:", e
print "update LINK ERROR:url=", info.url, info.raw
continue
itlist.append(it)
def update_link(linklist):
itlist = []
for info in linklist:
it = Link()
try:
if len(info.mid) >1 :
it.mid = int(info.mid)
else:
it.mid = 0
if len(info.imdbid) >1:
it.imdbid = int(info.imdbid)
else:
it.imdbid = 0
it.url = info.url
print "quality",info.quality
it.quality = info.quality
it.urlmd5 = utils.get_md5_value(info.url)
it.cname = info.cname
it.ename = info.ename
it.actors = info.actors
it.director = info.director
it.date = utils.get_date_from_string(info.date)
it.title = info.raw
it.content = info.content
if len(it.content)> 950:
it.content = it.content[0:950]
it.found_date = utils.get_date_now()
except Exception,e:
traceback.print_exc(sys.stdout)
print "update LINK ERROR:",e
print "update LINK ERROR:url=",info.url,info.raw
continue
itlist.append(it)
def update_link(linklist):
itlist = []
for info in linklist:
it = Link()
try:
it.mid = 0
it.url = info.url
it.urlmd5 = get_md5_value(info.url)
it.cname = info.cname
it.ename = info.ename
it.actors = info.actors
it.director = info.director
it.date = get_date_from_string(info.date)
it.title = info.raw
it.content = info.content
time = datetime.datetime.now()
it.found_date = int(time.strftime('%Y'))*10000+int(time.strftime('%m'))*100 +int(time.strftime('%d'))
except Exception,e:
print traceback.print_exc()
print "UPDATE LINK ERROR:",e
print "UPDATE LINK ERROR:url=",info.url,info.raw
continue
itlist.append(it)
def cbias_parse(url):
page_number = 1
type_grant = Type.objects.get(name='cbias')
clean_grant(type_grant)
while True:
try:
response = requests.get(f'{url}page/{page_number}/', timeout=120)
except requests.exceptions.RequestException as e:
print(e)
break
if response.status_code == 200:
soup = BS(response.text, features='html5lib')
posts = soup.find_all(class_='post_inner')
if posts:
for post in posts:
text = post.find_all('p')
text = ' '.join(i.text.strip() for i in text)
time = post.find('div', class_='post_date').text
time = datetime.datetime.strptime(time, '%d.%m.%Y').date()
if Grant.objects.filter(grant_name=type_grant,
text=text,
time=time).exists():
return
grant = Grant()
grant.grant_name = type_grant
grant.time = time
grant.text = text
grant.save()
doc_link = post.find_all('p')
doc_link = [i.find('a') for i in doc_link]
for doc in doc_link:
if doc is not None:
doc = doc['href']
if doc.endswith('.pdf') or doc.endswith('.doc'):
link = Link()
link.grant_id = grant
link.link = doc
link.save()
else:
break
else:
break
page_number += 1
def update_link(linklist):
itlist = []
for info in linklist:
it = Link()
try:
it.mid = 0
it.url = info.url
it.urlmd5 = get_md5_value(info.url)
it.cname = info.cname
it.ename = info.ename
it.actors = info.actors
it.director = info.director
it.date = get_date_from_string(info.date)
it.title = info.raw
it.content = info.content
time = datetime.datetime.now()
it.found_date = int(time.strftime('%Y')) * 10000 + int(
time.strftime('%m')) * 100 + int(time.strftime('%d'))
except Exception, e:
print traceback.print_exc()
print "UPDATE LINK ERROR:", e
print "UPDATE LINK ERROR:url=", info.url, info.raw
continue
itlist.append(it)
