def setUp(self):
super().setUp()
self.login()
category = Category(name='默认')
post = Post(title='post title', category=category, body='post body')
comment_admin = Comment(body='admin comment',
post=post,
from_admin=True,
reviewed=True)
comment_guest = Comment(author='guest',
post=post,
body='guest comment',
from_admin=False,
reviewed=True)
comment_unread = Comment(author='unread',
post=post,
from_admin=False,
body='unread comment',
reviewed=False)
link = Link(name='link', tag='other', url='https://www.google.com')
friend_link = Link(name='friend_link',
tag='friendLink',
url='https://github.com')
db.session.add_all([
category, post, comment_admin, comment_guest, comment_unread, link,
friend_link
])
db.session.commit()
def fake_links():
twitter = Link(name='Twitter', url='#')
facebook = Link(name='Facebook', url='#')
linkedin = Link(name='LinkedIn', url='#')
google = Link(name='Google+', url='#')
db.session.add_all([twitter, facebook, linkedin, google])
db.session.commit()
def new_link():
form = LinkForm()
if form.validate_on_submit():
with db.auto_commit():
link = Link()
link.set__attrs(form.data)
db.session.add(link)
flash('添加链接成功', 'success')
return redirect(url_for('web.manage_links'))
return render_template('admin/new_link.html', form=form)
def add_link(self, title, manager_page_id):
manager_page = self.page(page_id=manager_page_id)
if manager_page is not None and self.user and self.user.has_perm(
APP_NAME + '.add_link'):
profile = ProfileRepo(user=self.user).me
if profile is not None:
link = Link(title=title, icon_material='link', profile=profile)
link.save()
manager_page.links.add(link)
return True
return False
def catch_link(willie, trigger):
re_match = re.match(
r'(?u).*((http|https|ftp|www)(://\S+|\.\S+)).*', trigger)
url = re_match.group(1)
context = trigger
context = unicodedata.normalize('NFKD', context).encode('ascii', 'ignore')
if url.startswith('http'):
Link.add_or_increment(url, trigger.nick, trigger.sender, context)
else:
Link.add_or_increment(
'http://' + url, trigger.nick, trigger.sender, context)
def process():
form = LinkForm(request.form)
if form.validate():
page = make_page()
short_link = request.host_url + page
link = Link(page=page, short_link=short_link, **form.data)
if current_user.is_authenticated:
link.user = current_user
db.session.add(link)
db.session.commit()
return jsonify({'short_link': short_link})
return jsonify({'errors': form.full_link.errors[-1]})
def make_relation_back(user, link, relation_back):
if link is not None and relation_back != '':
new_link = Link(user=user)
new_link.node1 = link.node2
new_link.node2 = link.node1
new_link.provider1 = link.provider2
new_link.provider2 = link.provider1
new_link.relation = relation_back
new_link.save()
def link_add(request):
context = {
"profile": Profile.objects.get(user=request.user),
"bot_api_key": BotKey.objects.get(user=request.user).key
}
if request.method == "POST":
link_form = LinkAddForm(request.user, request.POST)
if link_form.is_valid():
try:
try:
folder = Folder.objects.get(
user=request.user, name=link_form.data.get("folder"))
except Folder.DoesNotExist:
messages.error(request, "Подборки не существует.")
context["form"] = LinkAddForm(request.user, request.POST)
return render(request, "link/add.html", context)
link = Link.objects.get(link=link_form.data.get("link"),
folder=folder)
messages.error(request, "Вы уже сохранили данную ссылку.")
return redirect('link_add')
except Link.DoesNotExist:
ok = check_blacklist(link_form.data['link'])
if ok is not None:
messages.error(request,
f"Ссылка не должна содержать \"{ok}\"")
return redirect(reverse("link_add"))
folder = Folder.objects.get(user=request.user,
name=link_form.data.get("folder"))
link = Link(link=link_form.data.get("link"), folder=folder)
link.save()
messages.success(request, "Ссылка добавлена")
return redirect(
reverse('folder_view', kwargs={"folder_id": folder.id}))
else:
messages.error(request, "Неправильные ссылка")
context["form"] = LinkAddForm(request.user, request.POST)
else:
folder_id = request.GET.get('folder_id', None)
context["form"] = LinkAddForm(request.user)
if folder_id is not None:
try:
folder = Folder.objects.get(id=folder_id, user=request.user)
context["form"] = LinkAddForm(
request.user, initial={"folder": [folder.name]})
except Folder.DoesNotExist:
pass
folders = Folder.objects.filter(user=request.user)
if len(folders) == 0:
messages.warning(request, "Для начала, создайте подборку.")
return redirect(reverse('folder_add'))
return render(request, "link/add.html", context)
def manage_link():
"""后台链接管理管理视图"""
form = NewLinkForm(request.form)
if form.validate_on_submit():
with db.auto_commit():
link = Link()
link.set_attr(form.data)
db.session.add(link)
return redirect_back()
fields_names, fields_errors = get_form_error_items(form)
return render_template('admin/manage_link.html', form=form,
fields_errors=fields_errors,
fields_names=fields_names)
def index():
form = ShortenURL()
if form.validate_on_submit():
original_url = form.original_url.data
link = form.custom_link.data
if not link:
link = generate_link()
if current_user.is_authenticated:
user = current_user
else:
user = None
new_entry = Link(link=link, original_url=original_url, user=user)
db.session.add(new_entry)
db.session.commit()
return redirect(url_for('main.index', shortened=link))
shortened = Link.query.filter_by(link=request.args.get('shortened')).first()
managed_links = []
if current_user.is_authenticated:
for entry in current_user.managed_links.order_by('id'):
managed_links.append(entry)
else:
if shortened:
managed_links.append(shortened)
managed_links.reverse()
return render_template('main/index.html', form=form, managed_links=managed_links)
def test_linking(self):
message = Message(tlg_chat_id=1,
tlg_msg_id=1,
text='test',
media_group_id=None,
from_user=1,
to_user=2,
file_ids=[],
from_claimant=True)
db.session.add(message)
db.session.commit()
b_m = namedtuple('b_m', ['message_id'])
bot_message = b_m(message_id=2)
link = Link(from_chat_id=message.tlg_chat_id,
source_message_id=message.tlg_msg_id,
current_message_id=bot_message.message_id,
claimant='test_claimant')
db.session.add(link)
db.session.commit()
cl_chat_id_raw = Link.query\
.filter_by(current_message_id=bot_message.message_id)\
.first()
old_message = Message.query\
.filter_by(tlg_chat_id=cl_chat_id_raw.source_message_id).first()
self.assertEqual(old_message, message)
def create_link():
""" Create a new link for a given user
:return: JSON of new link if successful, error response if not
"""
# TODO: validate user ID
try:
data = request.json
# See if this specific user has a link entity with this specific url
# This should prevent a link from being added twice but also run it through regardless
link = Link.query.filter_by(user_id=data['user_id'], url=data['url']).first()
if not link:
link = Link(
user_id=data['user_id'],
url=data['url'],
)
db.session.add(link)
db.session.commit()
process_link(link)
return APIResponseBuilder.success({
"link": link
})
except SQLAlchemyError as e:
return APIResponseBuilder.error(f"Issue running query: {e}")
except Exception as e:
return APIResponseBuilder.error(f"Error encountered: {e}")
def create_empty_link():
link = Link()
db.session.add(link)
db.session.commit()
return link
def test_link_model_creates(client):
with app.app_context():
new_link = Link(LinkId=None, Text="new", URL="https://newurl.com")
db.session.add(new_link)
db.session.commit()
link = Link.query.filter_by(Text="new").first()
assert link.Text == "new"
def index():
form = LinkForm()
if form.validate_on_submit():
link = Link(redirect_url=form.redirect_url.data,
title=form.title.data,
subtext=form.subtext.data,
image_url=form.image_url.data,
favicon_url=form.favicon_url.data)
link.slug = '-'.join(link.title.lower().split()[:8])
db.session.add(link)
db.session.commit()
return render_template('index.html',
url=url_for('do_redirect',
_external=True,
slug=link.slug))
return render_template('index.html', form=form)
def setUp(self) -> None:
"""启动测试前执行"""
app = create_app(config='test')
self.context = app.test_request_context()
self.context.push()
self.client = app.test_client()
self.runner = app.test_cli_runner()
self.runner.invoke(args=['initdb', '--_init'])
self.fake_data = FakeData()
self.fake_data.fake_admin()
# 测试前生成一些少部分测试数据以供使用
category = Category(
name='testCategory',
alias='test-alias',
)
link = Link(
name='testLink',
url='https://www.test.com',
)
post = Post(title='testTitle',
content='<p>testContent<p>',
content_markdown='testContent',
description='testDescription',
categories=[category])
comment = Comment(author='testAuthor',
content='testCommentContent',
post=post,
reviewed=True)
with db.auto_commit():
db.session.add_all([category, link, post, comment])
def save_link(message, bot_message):
link = Link(from_chat_id=message.chat.id,
source_message_id=message.message_id,
current_message_id=bot_message.message_id,
claimant=str(message.from_user.username).lower())
db.session.add(link)
db.session.commit()
def post(self, request):
url = request.POST.get('url')
link = Link.shorten(url)
if link is None:
return render(request, 'app/create.html', {'invalid_url': True}, status=422)
else:
return redirect('app:show', short_code=link.short_code)
return render(request, 'app/create.html')
def post(self, request):
url = request.POST["url"]
link = Link.shorten(url)
if link is None:
return render(request,
"app/create.html", {"invalid_url": True},
status=422)
else:
return redirect('app:show', link.short_code)
def query_add_links(process_id, products, link_type):
links = [
Link(process_id=process_id, product_id=product_id, link_type=link_type)
for product_id in products
]
db.session.add_all(links)
db.session.commit()
db.session.close()
return
def add_url():
original_url = request.form['original_url']
link = Link(original_url=original_url)
db.session.add(link)
db.session.commit()
return render_template('url_added.html',
new_link=link.short_url,
original_url=link.original_url)
def callback_handler(update, context):
lang = get_lang(update.callback_query, query=True)
query = update.callback_query
chat_id = str(query.from_user.id)
keyboard = None
try:
unsaved_link = BotUnsavedLinks.objects.get(chat_id=chat_id)
except BotUnsavedLinks.DoesNotExist:
query.answer("лЦлљ-лЦлљ-лЦлљ")
return
if query.data.startswith("folder_next_"):
keyboard = get_keyboard(chat_id, int(query.data.replace("folder_next_", "")), query)
answer = words[lang]['folder']["choose"].format(unsaved_link.link)
elif query.data.startswith("folder_choose_"):
user = BotKey.objects.get(chat_id=chat_id).user
try:
folder = Folder.objects.get(id=int(query.data.replace("folder_choose_", "")), user=user)
except Folder.DoesNotExist:
query.answer("лЦлюлюлюлюлюлюлю")
return
links = Link.objects.filter(folder=folder)
is_saved = False
for link in links:
if link.link == unsaved_link.link:
answer = words[lang]["links"]["error"]["already_saved"].format(link.link, folder.name)
keyboard = get_keyboard(chat_id, 0, query)
is_saved = True
if not is_saved:
link = Link(folder=folder, link=unsaved_link.link)
answer = words[lang]["links"]["saved"].format(link.link, link.folder.name)
link.save()
elif query.data.startswith("setlanguage_"):
language = query.data.replace("setlanguage_", "")
profile = BotKeyLanguage.objects.get(chat_id=chat_id)
profile.lang = language
profile.save()
answer = words[get_lang(query, True)]['success']
try:
query.edit_message_text(answer, reply_markup=keyboard)
except telegram.error.BadRequest:
query.edit_message_text(answer + ".", reply_markup=keyboard)
# CallbackQueries need to be answered, even if no notification to the user is needed
# Some clients may have trouble otherwise. See https://core.telegram.org/bots/api#callbackquery
query.answer()
def get_links():
# get table sorting preferences from request
r = request.args
sort_column = r['column']
if sort_column == 'long_link':
sort_column = 'link_name'
elif sort_column == 'short_link':
sort_column = 'link_token'
sort_desc = r['desc'] == 'true'
page_start = int(r['start'])
PAGE_COUNT = 10
if page_start == 0:
page_number = 1
else:
page_number = (page_start // PAGE_COUNT) + 1
query = Link.query
query = query.filter_by(owner=current_user)
if sort_desc:
records = query.order_by(desc(sort_column)).paginate(
page=page_number, per_page=PAGE_COUNT, error_out=False)
else:
records = query.order_by(sort_column).paginate(page=page_number,
per_page=PAGE_COUNT,
error_out=False)
link_records = records.items
# format links
formatted_links = []
for link_record in link_records:
new_link_record = dict()
new_link_record['long_link'] = dict()
# format record values
new_link_record['long_link']['name'] = link_record.link_name
new_link_record['long_link']['url'] = link_record.link_url
new_link_record['created'] = Link.format_date(link_record.created)
new_link_record['short_link'] = link_record.link_token
# also need to add dummy elements for the buttons and send the link_token
new_link_record['copy'] = link_record.link_token
new_link_record['delete'] = link_record.link_token
formatted_links.append(new_link_record)
return jsonify({
'data': formatted_links,
'recordsTotal': records.total,
'recordsFiltered': records.total
})
def delete_link():
"""Delete supplied link from the database."""
try:
link_token = request.form['link_token']
link_data = Link.delete_link(link_token=link_token)
except Exception as e:
print(e)
return redirect(url_for('index'))
def fake_links():
google = Link(name='Google', url='#')
baidu = Link(name='百度', url='#')
linkedin = Link(name='LinkedIn', url='#')
douban = Link(name='豆瓣', url='#')
shiguang = Link(name='时光网', url='#')
zhihu = Link(name='知乎', url='#')
hackernews = Link(name='Hacker News', url='#')
reddit = Link(name='Reddit', url='#')
db.session.add_all(
[google, baidu, linkedin, douban, shiguang, zhihu, hackernews, reddit])
db.session.commit()
def demo():
link = Link(url='https://www.w3schools.com/js/js_json_intro.asp',
user_id="aaabbbcccddd",
link_title="test",
link_description="test desc")
db.session.add(link)
db.session.commit()
print("Starting process")
tasks.process_link(link)
print("Ending process")
return "Initialized"
async def create(*, request: Request):
data = await request.json()
try:
link = Link(**data)
except ValidationError as e:
msg = e.errors()[0].get("msg")
if "Error" in msg or "Exception" in msg:
raise HTTPException(500)
raise HTTPException(400, detail=msg)
if link.key is None:
link.key = await db.get_free_key()
custom = True
else:
custom = False
if not await db.set_link(link.key, link.url, custom=custom):
raise HTTPException(400, "link in use")
return {"key": link.key, "url": link.url}
def setUp(self):
super().setUp()
self.login()
category = Category(name='默认')
post = Post(title='Post', category=category, body='post body')
comment = Comment(body='comment body', post=post, from_admin=True,
reviewed=True)
link = Link(name='link', tag='other', url='https://link.com')
db.session.add_all([category, post, comment, link])
db.session.commit()
def new_link():
form = LinkForm()
if form.validate_on_submit():
name = form.name.data
url = form.url.data
link = Link(name=name, url=url)
db.session.add(link)
db.session.commit()
flash('Link created.', 'success')
return redirect(url_for('.manage_link'))
return render_template('/admin/new_link.html', form=form)
def test_step2_get_existing_link_renders_app_show_with_link(self):
'''app:show with the short_code for an existing link
renders app/show.html with the short_code's link provided in
the context.'''
l = Link.shorten('https://www.basecampcodingacademy.org')
response = self.client.get(
reverse('app:show', kwargs={'short_code': l.short_code}), )
self.assertTemplateUsed(response, 'app/show.html')
self.assertEqual(response.context.get('link'), l)
def test_step2_app_goto_with_existing_link_redirects_to_link_original(
self):
'app:goto should redirect to short_code\'s link\'s original url'
l = Link.shorten('https://www.basecampcodingacademy.org')
response = self.client.get(
reverse('app:goto', kwargs={'short_code': l.short_code}))
self.assertRedirects(response,
l.original,
fetch_redirect_response=False)
def test_permanent_redirect(self, requests_get, requests_head):
url = "http://www.example.com/foo"
redirect_url = "http://www.example.com/bar"
headers = { 'location': redirect_url }
requests_head.return_value = MagicMock(status_code=301, headers=headers)
page = Page.create(url=url, content='', status_code=0)
add_page_info_to_page(page)
to_page = Page.select().where(Page.url == redirect_url).first()
self.assertTrue(to_page)
url_redirect_link = Link.select().where(
Link.from_page == page,
Link.to_page == to_page)
self.assertTrue(url_redirect_link.exists())
self.assertEqual(requests_head.call_count, 1)
self.assertFalse(requests_get.called)
self.assertEqual(page.content, redirect_url)
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
if __name__ == "__main__":
initialize('corpus.db')
page_count = Page.select().count()
crawled_count = Page.select().where(
(Page.status_code == 200) &
((Page.content_type == 'text/html') |
(Page.content_type == 'text/plain')))\
.count()
redirect_count = Page.select().where(Page.status_code == 301).count()
to_crawl_count = Page.select().where(Page.status_code == 0).count()
other_count = page_count - crawled_count - redirect_count - to_crawl_count
link_count = Link.select().count()
corpus_size = os.stat('corpus.db').st_size
corpus_size = sizeof_fmt(corpus_size)
print('crawled pages: {}'.format(crawled_count))
print('redirect pages: {}'.format(redirect_count))
print('pages to crawl: {}'.format(to_crawl_count))
print('other pages: {}'.format(other_count)) # could include temporary redirects, server errors, etc.
print('links: {}'.format(link_count))
print('db size: {}'.format(corpus_size))
print
