def newTab(self, root):
"""
Inicijalizuje novi tab
Return:
Objekat book
"""
if len(root.getChildren()) == 0:
newName, ok = QInputDialog.getText(
None, "New Chapter name", "Enter desired first chapter name")
if not ok or newName == "":
while not ok or newName == "":
newName, ok = QInputDialog.getText(
None, "New Chapter name",
"Enter desired first chapter name")
root.addChild(Chapter(newName))
book = Book(root.getName())
book.setPath(root.getPath())
book.setParent(QApplication.instance().model)
rootModel = HierarchyTreeModel(book)
rootView = HierarchyTreeView(rootModel)
for chapter in root.getChildren():
tmpChapter = Chapter(chapter.getName())
book.addChild(tmpChapter)
for page in chapter.getChildren():
tmpPage = Page(page.getName())
tmpChapter.addChild(tmpPage)
for element in page.getChildren():
element.setParent(tmpPage)
tmpPage.addChild(element)
self.tabs.addTab(rootView, root.getName())
self.LeftButton.clicked.connect(rootView.leftButtonPressed)
self.RightButton.clicked.connect(rootView.rightButtonPressed)
rootView.SelectRoot()
return book
def get(self,page_name):
user = self.get_user()
if not user:
self.redirect('/wiki/signup')
return
edit_link = '/wiki/_edit' + page_name
view_link = '/wiki' + page_name
page = Page.get_page_by_name(page_name)
if not page:
self.redirect(edit_link)
return
editions = []
eds = list(page.editions)
eds.reverse()
num = 1
for edition_key in eds:
edition = db.get(edition_key)
editions.append( (edition.created.strftime('%d %b %Y %H:%M:%S'), escape(edition.content), view_link + '?v=' + str(num), edit_link + '?v=' + str(num) ) )
num+=1
self.render('wiki_history.html', editions = editions, view_page = view_link, edit_page = edit_link)
def post(self, page_name):
content = self.request.get('content')
user = self.get_user()
if not user:
self.redirect('/wiki/signup')
return
page = Page.get_page_by_name(page_name)
if not page:
page = Page(name = page_name, editions = [])
page_edition = PageEdition(content = content, author = user)
page_edition.put()
page.editions.append(page_edition.key())
page.put()
self.redirect('/wiki'+page_name)
def actionCalled(self):
"""
Dodaje sibling node, insertuje novi node izmedju selektovanog i njegovog narednog
"""
sibling = QApplication.instance().selectionModel
parent = sibling.getParent()
if isinstance(sibling, Chapter):
newName, ok = QInputDialog.getText(None, "New Chapter name",
"Enter desired new name")
if ok:
if parent.isValidName(newName):
parent.insertChild(sibling.getIndex() + 1,
Chapter(newName))
else:
while not parent.isValidName(newName):
dialog = QMessageBox()
dialog.setWindowTitle("Error")
dialog.setText("That name is not valid")
dialog.setWindowIcon(QIcon("src/notification.png"))
dialog.setModal(True)
dialog.exec_()
newName, cancel = QInputDialog.getText(
None, "New Chapter name", "Enter desired new name")
if not cancel:
break
else:
if parent.isValidName(newName):
parent.insertChild(sibling.getIndex() + 1,
Chapter(newName))
break
if isinstance(sibling, Page):
if len(sibling.getParent().getChildren()) > 0:
sibling.getParent().insertChild(
sibling.getIndex() + 1,
Page(sibling.getName()[:-1] +
str(int(sibling.getName()[-1:]) + 1)))
else:
sibling.getParent().addChild(Page("Strana1"))
def __init__(self, model):
"""
Konstruktor
Uključuje opciju prikazivanja kontekstnog menija.
"""
super(HierarchyTreeView, self).__init__()
self.tree = model
self.setModel(self.tree)
self.setSelectionMode(QAbstractItemView.SingleSelection)
self.tree.removedPage.connect(self.Clear)
self.tree.clearedSignal.connect(self.Clear)
# ukljucuje kontekstni meni
self.setContextMenuPolicy(Qt.CustomContextMenu)
self.customContextMenuRequested.connect(self.openMenu)
selModel = self.selectionModel()
selModel.selectionChanged.connect(self.updateTxt)
selModel.selectionChanged.connect(self.updateSelection)
self.lastSelectedPage = Page("tmp")
self.lastSelectedIndex = QModelIndex()
def get(self, page_name):
user = self.get_user()
v = self.request.get('v')
if v and v.isdigit():
v = int(v)
else:
v = 1
if not user:
self.redirect('/wiki/signup')
return
page = Page.get_page_by_name(page_name)
content = ''
if page:
content = db.get(page.editions[-v]).content
self.render('wiki_edit.html', content = escape(content) )
def get(self,page_name):
user = self.get_user()
v = self.request.get('v')
if v and v.isdigit():
v = int(v)
else:
v = 1
edit_link = '/wiki/_edit' + page_name
history_link = '/wiki/_history' + page_name
page = Page.get_page_by_name(page_name)
if not page:
logging.error('redirecting')
self.redirect(edit_link)
return
content = db.get(page.editions[-v]).content
self.render('wiki.html', content = content, logged = user, edit_page = edit_link, history_link = history_link)
def actionCalled(self):
"""
Dodaje child node
"""
parent = QApplication.instance().selectionModel
if isinstance(parent, Book):
newName, ok = QInputDialog.getText(None, "New Chapter name",
"Enter desired new name")
if ok:
if parent.isValidName(newName):
parent.addChild(Chapter(newName))
else:
while not parent.isValidName(newName):
dialog = QMessageBox()
dialog.setWindowTitle("Error")
dialog.setText("That name is not valid")
dialog.setWindowIcon(QIcon("src/notification.png"))
dialog.setModal(True)
dialog.exec_()
newName, cancel = QInputDialog.getText(
None, "New Chapter name", "Enter desired new name")
if not cancel:
break
else:
if parent.isValidName(newName):
parent.addChild(Chapter(newName))
break
if isinstance(parent, Chapter):
if len(parent.getChildren()) > 0:
parent.addChild(
Page(parent.getChildren()[-1].getName()[:-1] +
str(int(parent.getChildren()[-1].getName()[-1:]) +
1)))
else:
parent.addChild(Page("Strana1"))
if isinstance(parent, Page):
item, ok = QInputDialog.getItem(QInputDialog(), "Add an element",
"Choose one option:",
["Add text", "Add picture"], 0,
False)
if ok:
if item == "Add text":
tmpList = []
for child in parent.getChildren():
if isinstance(child, Text):
tmpList.append(child)
if len(tmpList) > 0:
parent.addChild(
Text(tmpList[-1].getName()[:-1] +
str(int(tmpList[-1].getName()[-1:]) + 1)))
else:
parent.addChild(Text("Text1"))
if item == "Add picture":
image = QFileDialog.getOpenFileName(None, 'OpenFile', '')
if image[1]:
path = image[0]
if path != None:
tmpPic = Picture(path.split("/")[-1])
tmpPic.setPicture(path)
parent.addChild(tmpPic)
class HierarchyTreeView(QTreeView):
"""
Graficki prikaz hijerarhijskog stabla uz implementiran kontekstni meni, i displayer za sadrzaj stranice
"""
def __init__(self, model):
"""
Konstruktor
Uključuje opciju prikazivanja kontekstnog menija.
"""
super(HierarchyTreeView, self).__init__()
self.tree = model
self.setModel(self.tree)
self.setSelectionMode(QAbstractItemView.SingleSelection)
self.tree.removedPage.connect(self.Clear)
self.tree.clearedSignal.connect(self.Clear)
# ukljucuje kontekstni meni
self.setContextMenuPolicy(Qt.CustomContextMenu)
self.customContextMenuRequested.connect(self.openMenu)
selModel = self.selectionModel()
selModel.selectionChanged.connect(self.updateTxt)
selModel.selectionChanged.connect(self.updateSelection)
self.lastSelectedPage = Page("tmp")
self.lastSelectedIndex = QModelIndex()
def getModel(self):
"""
Vraca tree
Return:
Tree
"""
return self.tree
def openMenu(self, position):
"""
Metoda povezana na customContextMenuRequested. Kreira kontekstni meni sa akcijama dodavanja, brisanja i promene naziva elemenata.
Kontekstni meni se prikazuje na poziciji na kojoj se nalazio kursor misa.
Args:
position(QPoint): pozicija kursora misa
"""
self.contextMenu = QMenu()
actionManager = QApplication.instance().actionManager
tmp = QApplication.instance().selectionModel
if not isinstance(tmp, Text) and not isinstance(tmp, Picture):
self.contextMenu.addAction(actionManager.addChildAction)
self.contextMenu.addAction(actionManager.addAtAction)
self.contextMenu.addAction(actionManager.addBefore)
self.contextMenu.addSeparator()
if not isinstance(tmp, Page):
self.contextMenu.addAction(actionManager.renameNodeAction)
#if isinstance(tmp, Picture):
#self.contextMenu.addAction(actionManager.renameNodeAction)
self.contextMenu.addAction(actionManager.removeChildAction)
# prikaz kontekstnog menija
self.contextMenu.exec_(self.viewport().mapToGlobal(position))
def mousePressEvent(self, event):
"""
Redefinisanje mouse pressed event-a.
Uradjeno jer default-na implementacija rukovanja ovim dogadjajem ne podrazumeva deselekciju elementa stabla prilikom klika na praznu povrsinu.
"""
if (self.selectionMode() == QAbstractItemView.SingleSelection):
self.selectionModel().clear()
self.clearSelection()
self.setCurrentIndex(QModelIndex())
super(HierarchyTreeView, self).mousePressEvent(event)
def updateSelection(self, new):
"""
Promena selekcije
"""
if new.empty():
self.SelectRoot()
else:
QApplication.instance().selectionModel = self.selectionModel(
).currentIndex().internalPointer()
def Clear(self):
"""
Refresh stranice
"""
try:
self.clearSelection()
self.setCurrentIndex(self.lastSelectedIndex)
except:
pass
def SelectRoot(self):
"""
Selektovanje koren node
"""
QApplication.instance().selectionModel = self.model().root
def updateTxt(self):
"""
Updateovanje status bara, labela za prikaz stranice, sadrzaja text i slika komponenti pri promeni selektovane strane
"""
try:
tmp = self.selectionModel().currentIndex().internalPointer()
if isinstance(tmp, Page):
self.lastSelectedIndex = self.selectionModel().currentIndex()
widgetList = []
j = 0
for i in reversed(range(QApplication.instance().page.count())):
QApplication.instance().page.takeAt(i).widget().setParent(
None)
for child in tmp.getChildren():
try:
if isinstance(child, Text):
widget = MyTextEdit(child)
QApplication.instance().page.addWidget(
widget, Qt.FramelessWindowHint)
widget.show()
if isinstance(child, Picture):
widget = MyPictureEdit(child)
QApplication.instance().page.addWidget(widget)
widget.show()
widgetList.append(widget)
for widget in widgetList:
widget.setPosition(widget.object.getPosition())
except Exception as e:
print(e)
pageLabel = QApplication.instance().pageLabel
pageLabel.setText(
str(tmp.getName()[:-1]) + ": " + tmp.getName()[-1])
self.lastSelectedPage = tmp
if isinstance(tmp, Page):
QApplication.instance().statusBar.setText(
str(tmp.getParent().getParent().getName()) + "->" +
str(tmp.getParent().getName()) + "->" + str(tmp.getName()))
if isinstance(tmp, Chapter):
QApplication.instance().statusBar.setText(
str(tmp.getParent().getName()) + "->" + str(tmp.getName()))
except:
pass
def leftButtonPressed(self):
"""
Prelazak na prethodnu stranu
"""
try:
tmp = self.lastSelectedIndex.sibling(
self.lastSelectedPage.getIndex() - 1, 0)
if tmp.isValid():
self.setCurrentIndex(tmp)
self.lastSelectedIndex = tmp
self.lastSelectedPage = self.selectionModel().currentIndex(
).internalPointer()
else:
parent = self.lastSelectedIndex.parent()
pageUncle = self.lastSelectedPage.getParent()
book = pageUncle.getParent()
index = pageUncle.getIndex()
i = -1
while len(book.childAt(index + i).getChildren()) == 0:
i -= 1
parent = parent.sibling(
self.lastSelectedPage.getParent().getIndex() + i, 0)
i = 0
isValid = True
while isValid:
if parent.child(i, 0).isValid():
i += 1
else:
isValid = False
parent = parent.child(i - 1, 0)
if parent.isValid():
self.setCurrentIndex(parent)
self.lastSelectedIndex = parent
self.lastSelectedPage = self.selectionModel().currentIndex(
).internalPointer()
except:
pass
def rightButtonPressed(self):
"""
Prelazak na narednu stranu
"""
try:
tmp = self.lastSelectedIndex.sibling(
self.lastSelectedPage.getIndex() + 1, 0)
if tmp.isValid():
self.setCurrentIndex(tmp)
self.lastSelectedIndex = tmp
self.lastSelectedPage = self.selectionModel().currentIndex(
).internalPointer()
else:
parent = self.lastSelectedIndex.parent()
pageUncle = self.lastSelectedPage.getParent()
book = pageUncle.getParent()
index = pageUncle.getIndex()
i = 1
while len(book.childAt(index + i).getChildren()) == 0:
i += 1
parent = parent.sibling(
self.lastSelectedPage.getParent().getIndex() + i, 0)
parent = parent.child(0, 0)
if parent.isValid():
self.setCurrentIndex(parent)
self.lastSelectedIndex = parent
self.lastSelectedPage = self.selectionModel().currentIndex(
).internalPointer()
except:
pass
def process_item(self, item, spider):
db = DBSession()
redis = confRedis
rule_id = spider.rule_id
url = item['url']
md5 = hashlib.md5()
md5.update(url)
urlmd5 = md5.hexdigest()
site_name = utils.get_site(item['url'])
# site_name = spider.rule['allow_domains']
html_title = item['html_title']
# html_body = item['html_body']
save_path = utils.md5dir(item['url'])
save_time = int(time.time())
title = item['title'] if 'title' in item else ""
body = item['body'] if 'body' in item else ""
thumb = item['thumb'] if 'thumb' in item else ""
img_list = item['img_list'] if 'img_list' in item else ""
# TODO 这里使用一个分析方法,分析抓取到数据的发布时间,然后转换成时间戳
publish_time = utils.smart2date(
item['publish_time']) if 'publish_time' in item else ""
source_site = item['source_site'] if 'source_site' in item else ""
flag = default_page_flag
page = Page(rule_id=rule_id,
url=item['url'],
urlmd5=urlmd5,
site_name=site_name,
html_title=html_title,
save_path=save_path,
save_time=save_time,
title=title,
thumb=thumb,
img_list=img_list,
body=body,
publish_time=publish_time,
source_site=source_site,
flag=flag)
has = db.query(Page).filter(Page.urlmd5 == urlmd5).first()
if has:
page = Page(rule_id=rule_id,
url=item['url'],
site_name=site_name,
html_title=html_title,
save_path=save_path,
save_time=save_time,
title=title,
thumb=thumb,
img_list=img_list,
body=body,
publish_time=publish_time,
source_site=source_site,
flag=flag)
db.add(page)
try:
db.commit()
utils.save_file('%s/%s' % (html_path, save_path),
item['html_body'])
redis.set('url:%s' % url, 1)
except exc.SQLAlchemyError, e:
raise DropItem("SaveDbError: %s,%s" % (url, format(e)))
