def _sync_github(user, default_list, state):
website = 'github'
user_social_auth = user.social_auth.get(provider=website)
github_username = user_social_auth.extra_data['login']
page = 1
while True:
resp = urllib2.urlopen('https://api.github.com/users/' + github_username + '/starred?page=' + str(page))
data = resp.read()
items = json.loads(data)
for item in items:
url = item['html_url']
desc = item['description']
language = item.get('language', '')
title = item['full_name']
try:
#: if one of the github project url is saved before, stop iterating and return.
Bookmark.objects.get(list=default_list, url=url, user=user)
return
except Bookmark.DoesNotExist:
bookmark = Bookmark(url=url, user=user, note=desc)
bookmark.domain = urlparse(url).netloc
bookmark.title = title
bookmark.list = default_list
if language:
bookmark.tags = language.lower()
bookmark.save()
#: if less than 30, there will no more items, stop fetching data from Github
if len(items) < 30:
break
page += 1
if state.state != 1:
state.state = 1
state.list = default_list
state.save()
def import_to_lianpeng(file_name):
f = open(file_name)
xml = f.read()
f.close()
soup = BeautifulSoup(xml)
posts = soup.findAll('post')
#import pdb;pdb.set_trace()
h = HTMLParser.HTMLParser()
for post in posts:
stamp = float(post.createtime.string) / 1000
created_time = datetime.datetime.utcfromtimestamp(stamp).replace(tzinfo=utc)
tags = " ".join([tag.string for tag in post.findAll('tag')])
title = post.find('title').string
if not title:
title = "No Title"
print created_time
print title
note = post.find('text')
if note and note.string:
note = h.unescape(note.string)
else:
note = ""
bookmark = Bookmark(list_id=1397, user_id=3, title=title, note=note, created_time=created_time, domain='note.lianpeng.me', tags=tags)
bookmark.save()
bookmark = Bookmark.objects.get(id=bookmark.id)
bookmark.url = 'http://lianpeng.me/note/{}/'.format(bookmark.id)
bookmark.save()
def post(self, request):
if not request.FILES:
message = "Select a correct HTML file"
response = json.dumps({"False": True, "message": message})
return HttpResponse(response)
f = request.FILES["file"]
html = ""
for chunk in f.chunks():
html += str(chunk)
bookmarks = HtmlBookmarks(html)
for bookmark in bookmarks.parse():
folder = None
if bookmark.folder:
try:
folder = Folder.objects.get(name=bookmark.folder,
owner=self.request.user)
except Folder.DoesNotExist:
folder = Folder(name=bookmark.folder,
owner=self.request.user)
folder.save()
mark = BookmarkModel(
url=bookmark.url,
title=bookmark.title or bookmark.url,
owner=self.request.user,
folder=folder,
)
try:
mark.save()
except DatabaseError:
message = "Opps, Something fail."
response = json.dumps({"False": True, "message": message})
return HttpResponse(response)
message = "Bookmarks saved successfully"
response = json.dumps({"status": True, "message": message})
return HttpResponse(response)
def import_to_lianpeng(file_name):
f = open(file_name)
xml = f.read()
f.close()
soup = BeautifulSoup(xml)
posts = soup.findAll('post')
#import pdb;pdb.set_trace()
h = HTMLParser.HTMLParser()
for post in posts:
stamp = float(post.createtime.string) / 1000
created_time = datetime.datetime.utcfromtimestamp(stamp).replace(
tzinfo=utc)
tags = " ".join([tag.string for tag in post.findAll('tag')])
title = post.find('title').string
if not title:
title = "No Title"
print created_time
print title
note = post.find('text')
if note and note.string:
note = h.unescape(note.string)
else:
note = ""
bookmark = Bookmark(list_id=1397,
user_id=3,
title=title,
note=note,
created_time=created_time,
domain='note.lianpeng.me',
tags=tags)
bookmark.save()
bookmark = Bookmark.objects.get(id=bookmark.id)
bookmark.url = 'http://lianpeng.me/note/{}/'.format(bookmark.id)
bookmark.save()
def test_get_others_bookmark(self):
bookmark = Bookmark(user=self.user, title="you cannot edit me", url="http://google.com", domain="google.com")
bookmark.save()
self.client.login(username="******", password="******")
response = self.client.get('/api/v1/bookmark/'+ str(bookmark.id), content_type="application/json")
print response.status_code
self.assertTrue(response.status_code == 401)
def postdata(request):
if request.method == 'POST':
b_title = request.POST.get('title')
b_url = request.POST.get('url')
sdata = Bookmark(title=b_title, url=b_url)
sdata.save()
return render(request, 'postdata.html')
def test_award_points(self):
l = self.user.list_set.all()[0] #: Inbox
bookmark = Bookmark(user=self.user, title="test page", url="http://example.me/", domain='example.com', list=l)
bookmark.save()
self.assertTrue(self.user.targetstat_targets.points == 1)
bookmark = Bookmark(user=self.user, title="test page", url="http://example.com/", domain='example.me', list=l)
bookmark.save()
self.assertTrue(self.user.targetstat_targets.points == 1)
def test_change_bookmark_list_that_is_not_mine(self):
l = List(name="My List", user=self.john)
l.save()
bookmark = Bookmark(user=self.user, title="you cannot edit me", url="http://google.com", domain="google.com")
bookmark.save()
self.client.login(username="******", password="******")
put_data = {'title': 'awesome website', 'url': 'http://google.com', 'list': '/api/v1/list/' + str(l.id) , 'tags': 'searchengine google', 'created_time': "2013-2-12"}
response = self.client.put('/api/v1/bookmark/'+ str(bookmark.id), json.dumps(put_data), content_type="application/json")
self.assertTrue(response.status_code == 401)
def get(self, request, url):
url = request.get_full_path()[1:]
title = url
bookmark = BookmarkModel(url=url,
title=title[:255],
owner=self.request.user)
bookmark.save()
return redirect(settings.LOGIN_REDIRECT_URL)
def test_get_others_bookmarks(self):
bookmark = Bookmark(user=self.user, title="you cannot edit me", url="http://google.com", domain="google.com")
bookmark.save()
l = List.objects.get(name="Inbox", user=self.user)
self.client.login(username="******", password="******")
response = self.client.get('/api/v1/list/' + str(l.id) + '/bookmarks', content_type="application/json")
print response
print response.status_code
data = json.loads(response.content)
self.assertTrue(len(data['objects']) == 0)
def test_edit_bookmark(self):
bookmark = Bookmark(user=self.user, title="you cannot edit me", url="http://google.com", domain="google.com")
bookmark.save()
self.client.login(username="******", password="******")
put_data = {'title': 'awesome website', 'url': 'http://google.com', 'tags': 'searchengine google', 'created_time': "2013-2-12"}
response = self.client.put('/api/v1/bookmark/'+ str(bookmark.id), json.dumps(put_data), content_type="application/json")
print response.status_code
self.assertTrue(response.status_code == 202)
bookmark = Bookmark.objects.get(id=bookmark.id)
self.assertTrue(bookmark.title == "awesome website")
def test_get_others_bookmark(self):
bookmark = Bookmark(user=self.user,
title="you cannot edit me",
url="http://google.com",
domain="google.com")
bookmark.save()
self.client.login(username="******", password="******")
response = self.client.get('/api/v1/bookmark/' + str(bookmark.id),
content_type="application/json")
print response.status_code
self.assertTrue(response.status_code == 401)
def test_delete_bookmark_by_other_user(self):
bookmark = Bookmark(user=self.user, title="you cannot edit me", url="http://google.com", domain="google.com")
bookmark.save()
self.client.login(username="******", password="******")
response = self.client.delete('/api/v1/bookmark/'+ str(bookmark.id), content_type="application/json")
print response, response.status_code
self.assertTrue(response.status_code == 401)
try:
bookmark = Bookmark.objects.get(id=bookmark.id)
except:
self.fail()
def index(request):
l, created = List.objects.get_or_create(kind=LIST_KIND_CHOICES.NOTE,
user=request.user, defaults={"name": _("Note")})
site = Site.objects.get_current()
bookmark = Bookmark(url="http://{}/note/empty/".format(site.domain),
domain="note.lianpeng.me", title="输入标题", note="",
user=request.user, charset="UTF-8", list=l)
bookmark.save()
bookmark.url = 'http://{}{}'.format(site.domain, reverse('note_detail', args=(bookmark.id, )))
bookmark.save()
return redirect('note_edit', id=bookmark.id)
def test_get_others_bookmarks(self):
bookmark = Bookmark(user=self.user,
title="you cannot edit me",
url="http://google.com",
domain="google.com")
bookmark.save()
l = List.objects.get(name="Inbox", user=self.user)
self.client.login(username="******", password="******")
response = self.client.get('/api/v1/list/' + str(l.id) + '/bookmarks',
content_type="application/json")
print response
print response.status_code
data = json.loads(response.content)
self.assertTrue(len(data['objects']) == 0)
def post(self, request):
form = BookmarkForm(request.POST)
if form.is_valid():
bookmark = Bookmark()
bookmark.title = form.cleaned_data.get("title")
bookmark.url = form.cleaned_data.get("url")
bookmark.save()
new_tags = form.cleaned_data.get("tags")
tags = [x.strip() for x in new_tags.split(',')]
for t in tags:
tag, created = Tag.objects.get_or_create(name=t)
bookmark_tag, created = \
BookmarkTag.objects.get_or_create(bookmark=bookmark,
tag=tag)
return HttpResponseRedirect(reverse('bookmark:home'))
def test_delete_bookmark_by_other_user(self):
bookmark = Bookmark(user=self.user,
title="you cannot edit me",
url="http://google.com",
domain="google.com")
bookmark.save()
self.client.login(username="******", password="******")
response = self.client.delete('/api/v1/bookmark/' + str(bookmark.id),
content_type="application/json")
print response, response.status_code
self.assertTrue(response.status_code == 401)
try:
bookmark = Bookmark.objects.get(id=bookmark.id)
except:
self.fail()
def test_award_points(self):
l = self.user.list_set.all()[0] #: Inbox
bookmark = Bookmark(user=self.user,
title="test page",
url="http://example.me/",
domain='example.com',
list=l)
bookmark.save()
self.assertTrue(self.user.targetstat_targets.points == 1)
bookmark = Bookmark(user=self.user,
title="test page",
url="http://example.com/",
domain='example.me',
list=l)
bookmark.save()
self.assertTrue(self.user.targetstat_targets.points == 1)
def index(request):
l, created = List.objects.get_or_create(kind=LIST_KIND_CHOICES.NOTE,
user=request.user,
defaults={"name": _("Note")})
site = Site.objects.get_current()
bookmark = Bookmark(url="http://{}/note/empty/".format(site.domain),
domain="note.lianpeng.me",
title="输入标题",
note="",
user=request.user,
charset="UTF-8",
list=l)
bookmark.save()
bookmark = Bookmark.objects.get(id=bookmark.id)
bookmark.url = 'http://{}{}'.format(
site.domain, reverse('note_detail', args=(bookmark.id, )))
bookmark.save()
return redirect('note_edit', id=bookmark.id)
def test_edit_bookmark(self):
bookmark = Bookmark(user=self.user,
title="you cannot edit me",
url="http://google.com",
domain="google.com")
bookmark.save()
self.client.login(username="******", password="******")
put_data = {
'title': 'awesome website',
'url': 'http://google.com',
'tags': 'searchengine google',
'created_time': "2013-2-12"
}
response = self.client.put('/api/v1/bookmark/' + str(bookmark.id),
json.dumps(put_data),
content_type="application/json")
print response.status_code
self.assertTrue(response.status_code == 202)
bookmark = Bookmark.objects.get(id=bookmark.id)
self.assertTrue(bookmark.title == "awesome website")
def test_change_bookmark_list_that_is_not_mine(self):
l = List(name="My List", user=self.john)
l.save()
bookmark = Bookmark(user=self.user,
title="you cannot edit me",
url="http://google.com",
domain="google.com")
bookmark.save()
self.client.login(username="******", password="******")
put_data = {
'title': 'awesome website',
'url': 'http://google.com',
'list': '/api/v1/list/' + str(l.id),
'tags': 'searchengine google',
'created_time': "2013-2-12"
}
response = self.client.put('/api/v1/bookmark/' + str(bookmark.id),
json.dumps(put_data),
content_type="application/json")
self.assertTrue(response.status_code == 401)
def bookmark(request):
try:
email = request.session['login_id']
except KeyError as e:
return render(request, 'login_form.html')
all_bookmark = None
if request.method == 'POST':
bookmark = Bookmark(
bookmark_name=request.POST["bookmark_name"], bookmark_url=request.POST["bookmark_url"],
bookmark_desc=request.POST["bookmark_desc"]
)
bookmark.save()
result = {"login_id": email,
"bookmark_name": request.POST["bookmark_name"],
"bookmark_url": request.POST["bookmark_url"]}
return render(request, 'bookmark.html', result)
else:
all_bookmark = Bookmark.objects.all()
result = {"login_id": email,
"all_bookmark": all_bookmark}
return render(request, 'bookmark.html', result)
def _sync_github(user, default_list, state):
website = 'github'
user_social_auth = user.social_auth.get(provider=website)
github_username = user_social_auth.extra_data['login']
page = 1
while True:
resp = urllib2.urlopen('https://api.github.com/users/' +
github_username + '/starred?page=' + str(page))
data = resp.read()
items = json.loads(data)
for item in items:
url = item['html_url']
desc = item['description']
language = item.get('language', '')
title = item['full_name']
try:
#: if one of the github project url is saved before, stop iterating and return.
Bookmark.objects.get(list=default_list, url=url, user=user)
return
except Bookmark.DoesNotExist:
bookmark = Bookmark(url=url, user=user, note=desc)
bookmark.domain = urlparse(url).netloc
bookmark.title = title
bookmark.list = default_list
if language:
bookmark.tags = language.lower()
bookmark.save()
#: if less than 30, there will no more items, stop fetching data from Github
if len(items) < 30:
break
page += 1
if state.state != 1:
state.state = 1
state.list = default_list
state.save()
def handle_imported_file(data, user, site, list_name):
soup = BeautifulSoup(data)
entries = soup.findAll('dt')
default_list, created = List.objects.get_or_create(name=list_name,
user=user)
for entry in entries:
link = entry.find('a')
bookmark = Bookmark()
l = None
date = None
if not link:
continue
for attr in link.attrs:
if attr[0] == 'href':
bookmark.url = attr[1]
if attr[0] == 'add_date':
stamp = int(attr[1])
scale = len(attr[1]) - 10
stamp = stamp / pow(10, scale)
date = datetime.datetime.utcfromtimestamp(stamp).replace(
tzinfo=utc)
bookmark.created_time = date
if attr[0] == 'tags':
tags = attr[1].replace(',', ' ')
bookmark.tags = tags
if attr[0] == 'list':
name = attr[1]
l, created = List.objects.get_or_create(name=name, user=user)
if site == 'kippt':
dd = entry.findNext('dd')
if dd:
tags = dd.string
if tags:
tags = re.findall('#([^ ]+)', tags)
tags = ' '.join(tags)
bookmark.tags = tags
if site == 'google':
tag_ele = entry.find('h3')
if tag_ele:
tag = tag_ele.string
continue
else:
try:
existed_bookmark = Bookmark.objects.get(url=bookmark.url,
user=user)
existed_bookmark.tags = existed_bookmark.tags + " " + tag
existed_bookmark.save()
continue
except Bookmark.DoesNotExist:
try:
bookmark.tags = bookmark.tags + " " + tag
except UnboundLocalError:
pass
except:
continue
bookmark.domain = urlparse(bookmark.url).netloc
if l:
bookmark.list = l
else:
bookmark.list = default_list
bookmark.title = link.string
bookmark.user = user
# we can not have two bookmark with the same url in the same list
try:
Bookmark.objects.get(url=bookmark.url, list=bookmark.list)
continue
except Bookmark.DoesNotExist:
pass
try:
bookmark.save()
except:
continue
def handle_imported_file(data, user, site, list_name):
soup = BeautifulSoup(data)
entries = soup.findAll('dt')
default_list, created = List.objects.get_or_create(name=list_name, user=user)
for entry in entries:
link = entry.find('a')
bookmark = Bookmark()
l = None
date = None
if not link:
continue
for attr in link.attrs:
if attr[0] == 'href':
bookmark.url = attr[1]
if attr[0] == 'add_date':
stamp = int(attr[1])
scale = len(attr[1]) - 10
stamp = stamp / pow(10, scale)
date = datetime.datetime.utcfromtimestamp(stamp).replace(tzinfo=utc)
bookmark.created_time = date
if attr[0] == 'tags':
tags = attr[1].replace(',', ' ')
bookmark.tags = tags
if attr[0] == 'list':
name = attr[1]
l, created = List.objects.get_or_create(name=name, user=user)
if site == 'kippt':
dd = entry.findNext('dd')
if dd:
tags = dd.string
if tags:
tags = re.findall('#([^ ]+)', tags)
tags = ' '.join(tags)
bookmark.tags = tags
if site == 'google':
tag_ele = entry.find('h3')
if tag_ele:
tag = tag_ele.string
continue
else:
try:
existed_bookmark = Bookmark.objects.get(url=bookmark.url, user=user)
existed_bookmark.tags = existed_bookmark.tags + " " + tag
existed_bookmark.save()
continue
except Bookmark.DoesNotExist:
try:
bookmark.tags = bookmark.tags + " " + tag
except UnboundLocalError:
pass
except:
continue
bookmark.domain = urlparse(bookmark.url).netloc
if l:
bookmark.list = l
else:
bookmark.list = default_list
bookmark.title = link.string
bookmark.user = user
# we can not have two bookmark with the same url in the same list
try:
Bookmark.objects.get(url=bookmark.url, list=bookmark.list)
continue
except Bookmark.DoesNotExist:
pass
try:
bookmark.save()
except:
continue
