def find_action_function (module_name, function_name, repo_actions_path):
# special case "basic" module
if module_name == "basic":
import uplib.basicPlugins
return uplib.basicPlugins.lookup_action(function_name)
# special case "externalAPI" module
if module_name == "externalAPI":
import uplib.externalAPI
return getattr(uplib.externalAPI, function_name)
# for others, we load the module and search for the function in it (eventually)
else:
import string, traceback
try:
module = find_and_load_extension (module_name, repo_actions_path, REPO_EXTENSION_DIRS,
allow_old_extensions=ALLOW_OLD_EXTENSIONS)
if module:
note(4, "module %s was found", module_name)
if hasattr(module, function_name):
value = getattr(module, function_name)
if callable(value):
return value
if hasattr(module, "lookup_action"):
return module.lookup_action(function_name)
else:
note(3, "module %s was not found. repo_actions_path = [%s]. REPO_EXTENSION_DIRS = [%s]", module_name, repo_actions_path, REPO_EXTENSION_DIRS)
except:
typ, value, tb = sys.exc_info()
note(2, "%s", ''.join(traceback.format_exception(typ, value, tb)))
return None
def _scan_rss_sites(repo):
global _ADDED_SITES, _REMOVED_SITES
try:
from uplib.plibUtil import configurator, note, write_metadata, id_to_time, create_new_id
from uplib.extensions import find_and_load_extension
conf = configurator.default_configurator()
if repo:
sys_inits_path = os.path.join(conf.get('uplib-lib'), 'site-extensions')
repo_inits_path = os.path.join(repo.root(), "overhead", "extensions", "active")
upload_m = find_and_load_extension("UploadDocument", "%s|%s" % (repo_inits_path, sys_inits_path), None, True)
if not upload_m:
note(0, "Can't load UploadDocument extension!")
sys.exit(1)
else:
note("UploadDocument extension is %s", upload_m)
scan_period = conf.get_int("rss-scan-period", 60 * 2)
startup_delay = conf.get_int("rss-startup-delay", 0)
del conf
import feedparser
if startup_delay > 0:
note(3, "startup delay is %d", startup_delay)
time.sleep(startup_delay)
except:
note(0, "RSSReader: exception starting RSS scan thread:\n%s",
''.join(traceback.format_exception(*sys.exc_info())))
return
rss_sites = -1
while True:
try:
conf = configurator() # re-read uplibrc file
old_rss_sites = rss_sites
rss_sites = conf.get("rss-sites")
if old_rss_sites == -1 or (old_rss_sites != rss_sites):
note(2, "rss_sites are %s", rss_sites)
scan_period = conf.get_int("rss-scan-period", scan_period)
expiration_period = conf.get_int("rss-expiration-period", 30 * 24 * 60 * 60) # 30 days
if rss_sites:
rss_sites = rss_sites.split() + _ADDED_SITES
else:
rss_sites = _ADDED_SITES[:]
if rss_sites:
for site in _REMOVED_SITES:
if site in rss_sites:
rss_sites.remove(site)
if rss_sites:
feeds = []
for site in rss_sites:
if site.startswith("feed:"):
feeds.append(feedparser.parse(site))
elif site.startswith("http:") or site.startswith("https:"):
feeds += find_feeds(site)
note("feeds are:\n%s", [(x.feed.title, x.href, len(x.entries)) for x in feeds])
for feed in feeds:
note("RSSReader: %s: %s entries in feed %s", time.ctime(), len(feed.entries), feed.feed.title)
for entry in feed.entries:
d = process_entry(entry)
if not d:
continue
id = d.get("rss-id")
hits = repo.do_query('+rss-id:"%s"' % id)
if hits:
# already in repo
continue
if repo:
response = FakeResponse(repo)
mdoutput = StringIO.StringIO()
write_metadata(mdoutput, d)
md = mdoutput.getvalue()
mdoutput.close()
upload_m.add(repo, response, { 'URL': d.get("original-url"),
'wait': "true",
'no-redirect': "true",
'metadata': md,
'md-categories': "RSSReader/%s" % feed.feed.title,
})
if response.thread:
while response.thread.isAlive():
response.thread.join(1.0)
note("RSSReader: %s: %s (%s: %s)", time.ctime(), repr(d.get("title")), response.code, response.message)
else:
note("RSSReader: %s: %s (%s)\n %s", time.ctime(), repr(d.get("title")), d.get("date"), d.get("summary"))
# now do expiries
old_id = create_new_id(time.time() - expiration_period)[:-5]
hits = repo.do_query("categories:RSSReader AND id:[00000-00-0000-000 TO %s] AND NOT categories:RSSReader/_noexpire_" % old_id)
for score, doc in hits:
# check to see if the user has looked at it
if os.path.exists(os.path.join(doc.folder(), "activity")):
doc.add_category("RSSReader/_noexpire_", True)
# and if not, remove it
else:
repo.delete_document(doc.id)
time.sleep(scan_period)
except KeyboardInterrupt:
if _IGNORE_KEYBOARD_INTERRUPTS:
note(0, "RSSReader: %s", ''.join(traceback.format_exception(*sys.exc_info())))
else:
sys.exit(0)
except:
note(0, "RSSReader: %s", ''.join(traceback.format_exception(*sys.exc_info())))
