def main(argv):
global _IGNORE_KEYBOARD_INTERRUPTS
try:
import feedparser
except ImportError:
sys.stderr.write("RSSReader: Python feedparser module not available -- can't run RSS scanner.\n")
sys.exit(1)
if argv[0] == "run":
sys.path.append("/local/share/UpLib-1.7.9/code")
from uplib.plibUtil import set_verbosity, set_note_sink, uthread
from uplib.repository import Repository
uthread.initialize()
set_note_sink(sys.stderr)
set_verbosity(4)
_IGNORE_KEYBOARD_INTERRUPTS = False
if len(argv) > 1:
repo = Repository("1.7.9", argv[1], {})
else:
repo = None
_scan_rss_sites(repo)
elif argv[0] == "scan":
sys.path.append("/local/share/UpLib-1.7.9/code")
from uplib.plibUtil import write_metadata
for arg in argv[1:]:
for feed in find_feeds(arg):
print feed.feed.title, feed.href, len(feed.entries)
for entry in feed.entries:
d = process_entry(entry)
if d:
print (u'%s, by %s, at %s' % (d.get("title"), d.get("authors"), time.ctime(int(d.get("rss-timestamp"))))).encode("UTF-8", "strict")
if "'" in d.get("title"):
mdoutput = StringIO.StringIO()
write_metadata(mdoutput, d)
md = mdoutput.getvalue()
mdoutput.close()
for line in md.split("\n"):
line = line.strip()
print ' ' + line
else:
sys.exit(0)
###
############################################################
# we use Medusa's logger
if logfilename:
lgr = logger.file_logger (logfilename)
else:
# This will log to an overhead subdirectory of the repository
logfile_directory = os.path.join(directory, "overhead")
if not os.path.exists(logfile_directory):
os.makedirs(logfile_directory)
lgr = thread_safe_rotating_file_logger (os.path.join(logfile_directory, "angel.log"), "weekly", 10*1024*1024, true)
# make sure anything written with note() will go to the logfile
set_note_sink(lgr)
repo, use_ssl, ip_addr, conf = Repository.build_world(directory, portno, logfilename)
note(0, "repo is %s, use_ssl is %s, ip_addr is %s, conf is %s",
repo, use_ssl, ip_addr, conf)
############################################################
###
### check for pending documents, and start them
###
############################################################
retry_folders (repo)
############################################################
def start_angel(directory, portno, logfilename=None):
from uplib.plibUtil import note, configurator, set_note_sink
from uplib.tornadoHandler import ActionHandler, PingHandler, DocsHandler, LoginHandler
from uplib.repository import Repository
from uplib.newFolder import retry_folders
############################################################
###
### create the repo and run initializers
###
############################################################
# set up logging
rootlogger = logging.getLogger()
if logfilename:
rootlogger.addHandler(
logging.handlers.RotatingFileHandler(logfilename, "a", 10 * 1024 * 1024, 10 * 1024 * 1024)
)
else:
# This will log to an overhead subdirectory of the repository
logfile_directory = os.path.join(directory, "overhead")
if not os.path.exists(logfile_directory):
os.makedirs(logfile_directory)
rootlogger.addHandler(
logging.handlers.RotatingFileHandler(
os.path.join(logfile_directory, "angel.log"), "a", 10 * 1024 * 1024, 10 * 1024 * 1024
)
)
# filter out logging of repo_status_json calls, which occur frequently
class RepoStatusJsonFilter(logging.Filter):
def filter(self, record):
text = (record.args and str(record.msg) % record.args) or record.msg
return "/action/basic/repo_status_json" not in text
rootlogger.addFilter(RepoStatusJsonFilter())
# make sure anything written with note() will go to the logfile
set_note_sink(rootlogger)
repo, use_ssl, ip_addr, conf = Repository.build_world(directory, portno, logfilename)
note("using IP address %s, use_ssl is %s", ip_addr, use_ssl)
############################################################
###
### check for pending documents, and start them
###
############################################################
retry_folders(repo)
############################################################
###
### Now create the Tornado app...
###
############################################################
transforms = list()
transforms.append(tornado.web.GZipContentEncoding)
# see if we should use chunking in our server replies
use_chunking = not conf.get_bool("no-server-chunking", False)
if not use_chunking:
note("disabling server HTTP 1.1 reply chunking")
else:
transforms.append(tornado.web.ChunkedTransferEncoding)
handlers = list()
# This handler handles extension requests
handlers.append(("/action/(.*)", ActionHandler, {"repository": repo}))
# Redirect "/" to the top-level root
handlers.append(("/", ActionHandler, {"repository": repo}))
# This handler simply replys OK to a request for /ping, to indicate that
# the server is alive and OK, and for /login, to establish authorization
# credentials
handlers.append(("/ping", PingHandler, {"repository": repo}))
handlers.append(("/favicon.ico", PingHandler, {"repository": repo}))
handlers.append(("/html/signedreadup.jar", PingHandler, {"repository": repo}))
handlers.append(("/html/images/ReadUpJWS.gif", PingHandler, {"repository": repo}))
# The folder handler is the class which delivers documents from the docs
# subdirectory requested directly, such as thumbnails.
handlers.append(("/docs/(.*)", DocsHandler, {"repository": repo, "allow-caching": conf.get_bool("allow-caching")}))
# this handler serves up static HTML, jars, and images from under /html/
handlers.append(("/html/(.*)", tornado.web.StaticFileHandler, {"path": os.path.join(repo.root(), "html")}))
# use LoginHandler to handle login authorization
handlers.append(("/login", LoginHandler, {"repository": repo}))
# now set up server
if use_ssl:
import ssl
ssl_options = {"certfile": os.path.normpath(repo.certfilename())}
root_certs_file = conf.get("root-certificates-file")
if root_certs_file:
if not os.path.exists(root_certs_file):
sys.stderr.write("specified root-certificates-file %s does not exist!\n" % root_certs_file)
sys.exit(1)
else:
ssl_options["ca_certs"] = root_certs_file
ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
else:
ssl_options = None
try:
http_server = tornado.httpserver.HTTPServer(
tornado.web.Application(handlers=handlers, transforms=transforms), ssl_options=ssl_options
)
http_server.listen(repo.port(), ip_addr)
note("Now listening at %s://%s:%s" % ((ssl_options and "https") or "http", ip_addr, repo.port()))
except:
type, value, tb = sys.exc_info()
s = "".join(traceback.format_exception(type, value, tb))
note("Can't establish HTTP server: exception: " + s)
note("Abandoning attempt to start this daemon...")
os._exit(1)
# kill other subprocesses when exiting
if not sys.platform.lower().startswith("win"):
import signal
def kill_subprocs():
gid = os.getpgrp()
note(0, "killing subprocesses in process group %s...", gid)
os.killpg(-gid, signal.SIGTERM)
atexit.register(kill_subprocs)
# shutdown the server cleanly if things exit
atexit.register(lambda x=http_server: x.close())
# call the repo shutdown if the service exits
atexit.register(lambda x=repo: x.shutdown(0))
#
# Catch various termination signals and save state cleanly
#
if not sys.platform.lower().startswith("win"):
import signal
signal.signal(signal.SIGTERM, lambda signum, frame: (note(0, "SIGTERM received, exiting..."), sys.exit(0)))
signal.signal(signal.SIGUSR1, lambda signum, frame: (note(0, "SIGUSR1 received, saving..."), repo.save(true)))
note(0, "------- Restarted at %s. ----------", time.ctime())
# touch flag file to signal watchers...
open(os.path.join(repo.root(), "overhead", "angel.started"), "w").write(str(time.time()))
return repo
