def newTab(self, root):
"""
Inicijalizuje novi tab
Return:
Objekat book
"""
if len(root.getChildren()) == 0:
newName, ok = QInputDialog.getText(
None, "New Chapter name", "Enter desired first chapter name")
if not ok or newName == "":
while not ok or newName == "":
newName, ok = QInputDialog.getText(
None, "New Chapter name",
"Enter desired first chapter name")
root.addChild(Chapter(newName))
book = Book(root.getName())
book.setPath(root.getPath())
book.setParent(QApplication.instance().model)
rootModel = HierarchyTreeModel(book)
rootView = HierarchyTreeView(rootModel)
for chapter in root.getChildren():
tmpChapter = Chapter(chapter.getName())
book.addChild(tmpChapter)
for page in chapter.getChildren():
tmpPage = Page(page.getName())
tmpChapter.addChild(tmpPage)
for element in page.getChildren():
element.setParent(tmpPage)
tmpPage.addChild(element)
self.tabs.addTab(rootView, root.getName())
self.LeftButton.clicked.connect(rootView.leftButtonPressed)
self.RightButton.clicked.connect(rootView.rightButtonPressed)
rootView.SelectRoot()
return book
def actionCalled(self):
"""
Dodaje sibling node, insertuje novi node izmedju selektovanog i njegovog narednog
"""
sibling = QApplication.instance().selectionModel
parent = sibling.getParent()
if isinstance(sibling, Chapter):
newName, ok = QInputDialog.getText(None, "New Chapter name",
"Enter desired new name")
if ok:
if parent.isValidName(newName):
parent.insertChild(sibling.getIndex() + 1,
Chapter(newName))
else:
while not parent.isValidName(newName):
dialog = QMessageBox()
dialog.setWindowTitle("Error")
dialog.setText("That name is not valid")
dialog.setWindowIcon(QIcon("src/notification.png"))
dialog.setModal(True)
dialog.exec_()
newName, cancel = QInputDialog.getText(
None, "New Chapter name", "Enter desired new name")
if not cancel:
break
else:
if parent.isValidName(newName):
parent.insertChild(sibling.getIndex() + 1,
Chapter(newName))
break
if isinstance(sibling, Page):
if len(sibling.getParent().getChildren()) > 0:
sibling.getParent().insertChild(
sibling.getIndex() + 1,
Page(sibling.getName()[:-1] +
str(int(sibling.getName()[-1:]) + 1)))
else:
sibling.getParent().addChild(Page("Strana1"))
def __init__(self, model):
"""
Konstruktor
Uključuje opciju prikazivanja kontekstnog menija.
"""
super(HierarchyTreeView, self).__init__()
self.tree = model
self.setModel(self.tree)
self.setSelectionMode(QAbstractItemView.SingleSelection)
self.tree.removedPage.connect(self.Clear)
self.tree.clearedSignal.connect(self.Clear)
# ukljucuje kontekstni meni
self.setContextMenuPolicy(Qt.CustomContextMenu)
self.customContextMenuRequested.connect(self.openMenu)
selModel = self.selectionModel()
selModel.selectionChanged.connect(self.updateTxt)
selModel.selectionChanged.connect(self.updateSelection)
self.lastSelectedPage = Page("tmp")
self.lastSelectedIndex = QModelIndex()
def actionCalled(self):
"""
Dodaje child node
"""
parent = QApplication.instance().selectionModel
if isinstance(parent, Book):
newName, ok = QInputDialog.getText(None, "New Chapter name",
"Enter desired new name")
if ok:
if parent.isValidName(newName):
parent.addChild(Chapter(newName))
else:
while not parent.isValidName(newName):
dialog = QMessageBox()
dialog.setWindowTitle("Error")
dialog.setText("That name is not valid")
dialog.setWindowIcon(QIcon("src/notification.png"))
dialog.setModal(True)
dialog.exec_()
newName, cancel = QInputDialog.getText(
None, "New Chapter name", "Enter desired new name")
if not cancel:
break
else:
if parent.isValidName(newName):
parent.addChild(Chapter(newName))
break
if isinstance(parent, Chapter):
if len(parent.getChildren()) > 0:
parent.addChild(
Page(parent.getChildren()[-1].getName()[:-1] +
str(int(parent.getChildren()[-1].getName()[-1:]) +
1)))
else:
parent.addChild(Page("Strana1"))
if isinstance(parent, Page):
item, ok = QInputDialog.getItem(QInputDialog(), "Add an element",
"Choose one option:",
["Add text", "Add picture"], 0,
False)
if ok:
if item == "Add text":
tmpList = []
for child in parent.getChildren():
if isinstance(child, Text):
tmpList.append(child)
if len(tmpList) > 0:
parent.addChild(
Text(tmpList[-1].getName()[:-1] +
str(int(tmpList[-1].getName()[-1:]) + 1)))
else:
parent.addChild(Text("Text1"))
if item == "Add picture":
image = QFileDialog.getOpenFileName(None, 'OpenFile', '')
if image[1]:
path = image[0]
if path != None:
tmpPic = Picture(path.split("/")[-1])
tmpPic.setPicture(path)
parent.addChild(tmpPic)
def process_item(self, item, spider):
db = DBSession()
redis = confRedis
rule_id = spider.rule_id
url = item['url']
md5 = hashlib.md5()
md5.update(url)
urlmd5 = md5.hexdigest()
site_name = utils.get_site(item['url'])
# site_name = spider.rule['allow_domains']
html_title = item['html_title']
# html_body = item['html_body']
save_path = utils.md5dir(item['url'])
save_time = int(time.time())
title = item['title'] if 'title' in item else ""
body = item['body'] if 'body' in item else ""
thumb = item['thumb'] if 'thumb' in item else ""
img_list = item['img_list'] if 'img_list' in item else ""
# TODO 这里使用一个分析方法,分析抓取到数据的发布时间,然后转换成时间戳
publish_time = utils.smart2date(
item['publish_time']) if 'publish_time' in item else ""
source_site = item['source_site'] if 'source_site' in item else ""
flag = default_page_flag
page = Page(rule_id=rule_id,
url=item['url'],
urlmd5=urlmd5,
site_name=site_name,
html_title=html_title,
save_path=save_path,
save_time=save_time,
title=title,
thumb=thumb,
img_list=img_list,
body=body,
publish_time=publish_time,
source_site=source_site,
flag=flag)
has = db.query(Page).filter(Page.urlmd5 == urlmd5).first()
if has:
page = Page(rule_id=rule_id,
url=item['url'],
site_name=site_name,
html_title=html_title,
save_path=save_path,
save_time=save_time,
title=title,
thumb=thumb,
img_list=img_list,
body=body,
publish_time=publish_time,
source_site=source_site,
flag=flag)
db.add(page)
try:
db.commit()
utils.save_file('%s/%s' % (html_path, save_path),
item['html_body'])
redis.set('url:%s' % url, 1)
except exc.SQLAlchemyError, e:
raise DropItem("SaveDbError: %s,%s" % (url, format(e)))
