def submit_url(self, urlinput):
site = SiteInfo(urlinput)
site_info = site.get_site_info()
section = """<h3>{0}</h3>""".format(urlinput)
for item in site_info:
section += """<p>
<b>{0} </b>""".format(item[0])
if item[0] == 'Keywords':
section += """
<textarea rows = "6" cols = "100">{0}</textarea>
</p>""".format(item[1])
elif item[0] == 'Description' or item[0] == 'Title':
section += """
<textarea rows = "3" cols = "100">{0}</textarea>
</p>""".format(item[1])
else:
section += """
<textarea rows = "1" cols= "100">{0}</textarea>
</p>""".format(item[1])
return self._template(section)
def create_siteinfo(self):
self.siteinfo = SiteInfo(settings, self.site_path)
self.siteinfo.refresh()
settings.CONTEXT['site'] = self.siteinfo.content_node
class Generator(object):
"""
Generates a deployable website from the templates. Can monitor the site for
"""
def __init__(self, site_path):
super(Generator, self).__init__()
self.site_path = os.path.abspath(os.path.expandvars(
os.path.expanduser(site_path)))
self.regenerate_request = Event()
self.regeneration_complete = Event()
self.queue = Queue()
self.watcher = Thread(target=self.__watch__)
self.regenerator = Thread(target=self.__regenerate__)
self.processor = Processor(settings)
self.quitting = False
def notify(self, title, message):
if hasattr(settings, "GROWL") and settings.GROWL and File(settings.GROWL).exists:
try:
subprocess.call([settings.GROWL, "-n", "Hyde", "-t", title, "-m", message])
except:
pass
def pre_process(self, node):
self.processor.pre_process(node)
def process(self, item, change="Added"):
if change in ("Added", "Modified"):
settings.CONTEXT['node'] = item.node
settings.CONTEXT['resource'] = item
return self.processor.process(item)
elif change in ("Deleted", "NodeRemoved"):
return self.processor.remove(item)
def build_siteinfo(self, deploy_path=None):
tmp_folder = Folder(settings.TMP_DIR)
deploy_folder = Folder(
(deploy_path, settings.DEPLOY_DIR)
[not deploy_path])
if deploy_folder.exists and settings.BACKUP:
backup_folder = Folder(settings.BACKUPS_DIR).make()
deploy_folder.backup(backup_folder)
tmp_folder.delete()
tmp_folder.make()
settings.DEPLOY_DIR = deploy_folder.path
if not deploy_folder.exists:
deploy_folder.make()
add_to_builtins('hydeengine.templatetags.hydetags')
add_to_builtins('hydeengine.templatetags.aym')
add_to_builtins('hydeengine.templatetags.typogrify')
self.create_siteinfo()
def create_siteinfo(self):
self.siteinfo = SiteInfo(settings, self.site_path)
self.siteinfo.refresh()
settings.CONTEXT['site'] = self.siteinfo.content_node
def post_process(self, node):
self.processor.post_process(node)
def process_all(self):
self.notify(self.siteinfo.name, "Website Generation Started")
try:
self.pre_process(self.siteinfo)
for resource in self.siteinfo.walk_resources():
self.process(resource)
self.complete_generation()
except:
print >> sys.stderr, "Generation Failed"
print >> sys.stderr, sys.exc_info()
self.notify(self.siteinfo.name, "Generation Failed")
return
self.notify(self.siteinfo.name, "Generation Complete")
def complete_generation(self):
self.post_process(self.siteinfo)
self.siteinfo.target_folder.copy_contents_of(
self.siteinfo.temp_folder, incremental=True)
if(hasattr(settings, "post_deploy")):
settings.post_deploy()
def __regenerate__(self):
pending = False
while True:
try:
if self.quit_event.isSet():
self.notify(self.siteinfo.name, "Exiting Regenerator")
print "Exiting regenerator..."
break
# Wait for the regeneration event to be set
self.regenerate_request.wait(5)
# Wait until there are no more requests
# Got a request, we dont want to process it
# immedietely since other changes may be under way.
# Another request coming in renews the initil request.
# When there are no more requests, we go ahead and process
# the event.
if not self.regenerate_request.isSet() and pending:
pending = False
self.process_all()
self.regeneration_complete.set()
elif self.regenerate_request.isSet():
self.regeneration_complete.clear()
pending = True
self.regenerate_request.clear()
except:
print >> sys.stderr, "Error during regeneration"
print >> sys.stderr, sys.exc_info()
self.notify(self.siteinfo.name, "Error during regeneration")
self.regeneration_complete.set()
self.regenerate_request.clear()
pending = False
def __watch__(self):
regenerating = False
while True:
try:
if self.quit_event.isSet():
print "Exiting watcher..."
self.notify(self.siteinfo.name, "Exiting Watcher")
break
try:
pending = self.queue.get(timeout=10)
except Empty:
continue
self.queue.task_done()
if pending.setdefault("exception", False):
self.quit_event.set()
print "Exiting watcher"
self.notify(self.siteinfo.name, "Exiting Watcher")
break
if 'resource' in pending:
resource = pending['resource']
if self.regeneration_complete.isSet():
regenerating = False
if pending['change'] == "Deleted":
self.process(resource, pending['change'])
elif pending['change'] == "NodeRemoved":
self.process(pending['node'], pending['change'])
if (pending['change'] in ("Deleted", "NodeRemoved") or
resource.is_layout or regenerating):
regenerating = True
self.regenerate_request.set()
continue
self.notify(self.siteinfo.name, "Processing " + resource.name)
if self.process(resource, pending['change']):
self.complete_generation()
self.notify(self.siteinfo.name, "Completed processing " + resource.name)
except:
print >> sys.stderr, "Error during regeneration"
print >> sys.stderr, sys.exc_info()
self.notify(self.siteinfo.name, "Error during regeneration")
self.regeneration_complete.set()
self.regenerate_request.clear()
regenerating = False
def generate(self, deploy_path=None,
keep_watching=False,
exit_listner=None):
self.exit_listner = exit_listner
self.quit_event = Event()
setup_env(self.site_path)
validate_settings()
self.build_siteinfo(deploy_path)
self.process_all()
self.siteinfo.temp_folder.delete()
if keep_watching:
try:
self.siteinfo.temp_folder.make()
self.watcher.start()
self.regenerator.start()
self.siteinfo.monitor(self.queue)
except (KeyboardInterrupt, IOError, SystemExit):
self.quit()
except:
self.quit()
raise
def block(self):
try:
while self.watcher.isAlive():
self.watcher.join(0.1)
while self.regenerator.isAlive():
self.regenerator.join(0.1)
self.siteinfo.dont_monitor()
except (KeyboardInterrupt, IOError, SystemExit):
self.quit()
except:
self.quit()
raise
def quit(self):
if self.quitting:
return
self.quitting = True
print "Shutting down..."
self.notify(self.siteinfo.name, "Shutting Down")
self.siteinfo.dont_monitor()
self.quit_event.set()
if self.exit_listner:
self.exit_listner()
def create_siteinfo(self):
self.siteinfo = SiteInfo(settings, self.site_path)
self.siteinfo.refresh()
settings.CONTEXT['site'] = self.siteinfo.content_node
class Generator(object):
"""
Generates a deployable website from the templates. Can monitor the site for
"""
def __init__(self, site_path):
super(Generator, self).__init__()
self.site_path = os.path.abspath(
os.path.expandvars(os.path.expanduser(site_path)))
self.regenerate_request = Event()
self.regeneration_complete = Event()
self.queue = Queue()
self.watcher = Thread(target=self.__watch__)
self.regenerator = Thread(target=self.__regenerate__)
self.processor = Processor(settings)
self.quitting = False
def notify(self, title, message):
if hasattr(settings, "GROWL") and settings.GROWL and File(
settings.GROWL).exists:
try:
subprocess.call(
[settings.GROWL, "-n", "Hyde", "-t", title, "-m", message])
except:
pass
def pre_process(self, node):
self.processor.pre_process(node)
def process(self, item, change="Added"):
if change in ("Added", "Modified"):
settings.CONTEXT['node'] = item.node
settings.CONTEXT['resource'] = item
return self.processor.process(item)
elif change in ("Deleted", "NodeRemoved"):
return self.processor.remove(item)
def build_siteinfo(self, deploy_path=None):
tmp_folder = Folder(settings.TMP_DIR)
deploy_folder = Folder(
(deploy_path, settings.DEPLOY_DIR)[not deploy_path])
if deploy_folder.exists and settings.BACKUP:
backup_folder = Folder(settings.BACKUPS_DIR).make()
deploy_folder.backup(backup_folder)
tmp_folder.delete()
tmp_folder.make()
settings.DEPLOY_DIR = deploy_folder.path
if not deploy_folder.exists:
deploy_folder.make()
self.create_siteinfo()
def create_siteinfo(self):
self.siteinfo = SiteInfo(settings, self.site_path)
self.siteinfo.refresh()
settings.CONTEXT['site'] = self.siteinfo.content_node
def post_process(self, node):
self.processor.post_process(node)
def process_all(self):
self.notify(self.siteinfo.name, "Website Generation Started")
self.pre_process(self.siteinfo)
for resource in self.siteinfo.walk_resources():
self.process(resource)
self.complete_generation()
#try:
# self.pre_process(self.siteinfo)
# for resource in self.siteinfo.walk_resources():
# self.process(resource)
# self.complete_generation()
#except:
# print >> sys.stderr, "Generation Failed"
# print >> sys.stderr, sys.exc_info()
# self.notify(self.siteinfo.name, "Generation Failed")
# return
self.notify(self.siteinfo.name, "Generation Complete")
def complete_generation(self):
self.post_process(self.siteinfo)
self.siteinfo.target_folder.copy_contents_of(self.siteinfo.temp_folder,
incremental=True)
if (hasattr(settings, "post_deploy")):
settings.post_deploy()
def __regenerate__(self):
pending = False
while True:
try:
if self.quit_event.isSet():
self.notify(self.siteinfo.name, "Exiting Regenerator")
print "Exiting regenerator..."
break
# Wait for the regeneration event to be set
self.regenerate_request.wait(5)
# Wait until there are no more requests
# Got a request, we dont want to process it
# immedietely since other changes may be under way.
# Another request coming in renews the initil request.
# When there are no more requests, we go ahead and process
# the event.
if not self.regenerate_request.isSet() and pending:
pending = False
self.process_all()
self.regeneration_complete.set()
elif self.regenerate_request.isSet():
self.regeneration_complete.clear()
pending = True
self.regenerate_request.clear()
except:
print >> sys.stderr, "Error during regeneration"
print >> sys.stderr, sys.exc_info()
self.notify(self.siteinfo.name, "Error during regeneration")
self.regeneration_complete.set()
self.regenerate_request.clear()
pending = False
def __watch__(self):
regenerating = False
while True:
try:
if self.quit_event.isSet():
print "Exiting watcher..."
self.notify(self.siteinfo.name, "Exiting Watcher")
break
try:
pending = self.queue.get(timeout=10)
except Empty:
continue
self.queue.task_done()
if pending.setdefault("exception", False):
self.quit_event.set()
print "Exiting watcher"
self.notify(self.siteinfo.name, "Exiting Watcher")
break
if 'resource' in pending:
resource = pending['resource']
if self.regeneration_complete.isSet():
regenerating = False
if pending['change'] == "Deleted":
self.process(resource, pending['change'])
elif pending['change'] == "NodeRemoved":
self.process(pending['node'], pending['change'])
if (pending['change'] in ("Deleted", "NodeRemoved")
or resource.is_layout or regenerating):
regenerating = True
self.regenerate_request.set()
continue
self.notify(self.siteinfo.name, "Processing " + resource.name)
if self.process(resource, pending['change']):
self.complete_generation()
self.notify(self.siteinfo.name,
"Completed processing " + resource.name)
except:
print >> sys.stderr, "Error during regeneration"
print >> sys.stderr, sys.exc_info()
self.notify(self.siteinfo.name, "Error during regeneration")
self.regeneration_complete.set()
self.regenerate_request.clear()
regenerating = False
def generate(self,
deploy_path=None,
keep_watching=False,
exit_listner=None):
self.exit_listner = exit_listner
self.quit_event = Event()
setup_env(self.site_path)
validate_settings()
self.build_siteinfo(deploy_path)
self.process_all()
self.siteinfo.temp_folder.delete()
if keep_watching:
try:
self.siteinfo.temp_folder.make()
self.watcher.start()
self.regenerator.start()
self.siteinfo.monitor(self.queue)
except (KeyboardInterrupt, IOError, SystemExit):
self.quit()
except:
self.quit()
raise
def block(self):
try:
while self.watcher.isAlive():
self.watcher.join(0.1)
while self.regenerator.isAlive():
self.regenerator.join(0.1)
self.siteinfo.dont_monitor()
except (KeyboardInterrupt, IOError, SystemExit):
self.quit()
except:
self.quit()
raise
def quit(self):
if self.quitting:
return
self.quitting = True
print "Shutting down..."
self.notify(self.siteinfo.name, "Shutting Down")
self.siteinfo.dont_monitor()
self.quit_event.set()
if self.exit_listner:
self.exit_listner()
from siteinfo import SiteInfo
# 笔趣阁(1)
www_xbiquge_la = SiteInfo(site_domain='https://www.xbiquge.la',
xpath_book_name='//div[@id="info"]/h1/text()',
xpath_book_author='//div[@id="info"]/p/text()',
xpath_chapter_title='//div[@id="list"]//dd/a/text()',
xpath_chapter_url='//div[@id="list"]//dd/a/@href',
xpath_chapter_content='//div[@id="content"]/text()')
# 我要看书
www_51kanshu_cc = SiteInfo(site_domain='https://www.51kanshu.cc',
xpath_book_name='//div[@id="info"]/h1/text()',
xpath_book_author='//div[@id="info"]/p/text()',
xpath_chapter_title='//div[@class="listmain"]//dd/a/text()',
xpath_chapter_url='//div[@class="listmain"]//dd/a/@href',
xpath_chapter_content='//div[@id="content"]/p/text()')
# 원주학술정보센터자치회
def rule_wlibfacebook(Soup):
items = []
url = "https://www.facebook.com/pg/ysbookmark/posts/?ref=page_internal"
mainContents = Soup.find('div', {'role': 'main'})
v = zip(mainContents.select('div.userContent'),
mainContents.select('span.timestampContent'))
for userContent, timestamp in v:
when = timestamp.contents[0]
title = userContent.select('p')
if title == []:
title = "(텍스트가 없는 소식)"
else:
title = title[0].contents[0].strip()
items.append((when + ':: ' + title, url))
return items
sites = [
SiteInfo("연세대학교",
"http://www.yonsei.ac.kr/wj/support/notice.jsp",
rule=rule_yonsei),
SiteInfo("컴퓨터정보통신공학부",
"http://it.yonsei.ac.kr/index.php?mid=board_notice",
rule=rule_ityonsei),
SiteInfo("원주학술정보센터 자치회 책갈피",
"https://www.facebook.com/pg/ysbookmark/posts/?ref=page_internal",
rule=rule_wlibfacebook)
]
