def find_action_function (module_name, function_name, repo_actions_path): # special case "basic" module if module_name == "basic": import uplib.basicPlugins return uplib.basicPlugins.lookup_action(function_name) # special case "externalAPI" module if module_name == "externalAPI": import uplib.externalAPI return getattr(uplib.externalAPI, function_name) # for others, we load the module and search for the function in it (eventually) else: import string, traceback try: module = find_and_load_extension (module_name, repo_actions_path, REPO_EXTENSION_DIRS, allow_old_extensions=ALLOW_OLD_EXTENSIONS) if module: note(4, "module %s was found", module_name) if hasattr(module, function_name): value = getattr(module, function_name) if callable(value): return value if hasattr(module, "lookup_action"): return module.lookup_action(function_name) else: note(3, "module %s was not found. repo_actions_path = [%s]. REPO_EXTENSION_DIRS = [%s]", module_name, repo_actions_path, REPO_EXTENSION_DIRS) except: typ, value, tb = sys.exc_info() note(2, "%s", ''.join(traceback.format_exception(typ, value, tb))) return None
def _scan_rss_sites(repo): global _ADDED_SITES, _REMOVED_SITES try: from uplib.plibUtil import configurator, note, write_metadata, id_to_time, create_new_id from uplib.extensions import find_and_load_extension conf = configurator.default_configurator() if repo: sys_inits_path = os.path.join(conf.get('uplib-lib'), 'site-extensions') repo_inits_path = os.path.join(repo.root(), "overhead", "extensions", "active") upload_m = find_and_load_extension("UploadDocument", "%s|%s" % (repo_inits_path, sys_inits_path), None, True) if not upload_m: note(0, "Can't load UploadDocument extension!") sys.exit(1) else: note("UploadDocument extension is %s", upload_m) scan_period = conf.get_int("rss-scan-period", 60 * 2) startup_delay = conf.get_int("rss-startup-delay", 0) del conf import feedparser if startup_delay > 0: note(3, "startup delay is %d", startup_delay) time.sleep(startup_delay) except: note(0, "RSSReader: exception starting RSS scan thread:\n%s", ''.join(traceback.format_exception(*sys.exc_info()))) return rss_sites = -1 while True: try: conf = configurator() # re-read uplibrc file old_rss_sites = rss_sites rss_sites = conf.get("rss-sites") if old_rss_sites == -1 or (old_rss_sites != rss_sites): note(2, "rss_sites are %s", rss_sites) scan_period = conf.get_int("rss-scan-period", scan_period) expiration_period = conf.get_int("rss-expiration-period", 30 * 24 * 60 * 60) # 30 days if rss_sites: rss_sites = rss_sites.split() + _ADDED_SITES else: rss_sites = _ADDED_SITES[:] if rss_sites: for site in _REMOVED_SITES: if site in rss_sites: rss_sites.remove(site) if rss_sites: feeds = [] for site in rss_sites: if site.startswith("feed:"): feeds.append(feedparser.parse(site)) elif site.startswith("http:") or site.startswith("https:"): feeds += find_feeds(site) note("feeds are:\n%s", [(x.feed.title, x.href, len(x.entries)) for x in feeds]) for feed in feeds: note("RSSReader: %s: %s entries in feed %s", time.ctime(), len(feed.entries), feed.feed.title) for entry in feed.entries: d = process_entry(entry) if not d: continue id = d.get("rss-id") hits = repo.do_query('+rss-id:"%s"' % id) if hits: # already in repo continue if repo: response = FakeResponse(repo) mdoutput = StringIO.StringIO() write_metadata(mdoutput, d) md = mdoutput.getvalue() mdoutput.close() upload_m.add(repo, response, { 'URL': d.get("original-url"), 'wait': "true", 'no-redirect': "true", 'metadata': md, 'md-categories': "RSSReader/%s" % feed.feed.title, }) if response.thread: while response.thread.isAlive(): response.thread.join(1.0) note("RSSReader: %s: %s (%s: %s)", time.ctime(), repr(d.get("title")), response.code, response.message) else: note("RSSReader: %s: %s (%s)\n %s", time.ctime(), repr(d.get("title")), d.get("date"), d.get("summary")) # now do expiries old_id = create_new_id(time.time() - expiration_period)[:-5] hits = repo.do_query("categories:RSSReader AND id:[00000-00-0000-000 TO %s] AND NOT categories:RSSReader/_noexpire_" % old_id) for score, doc in hits: # check to see if the user has looked at it if os.path.exists(os.path.join(doc.folder(), "activity")): doc.add_category("RSSReader/_noexpire_", True) # and if not, remove it else: repo.delete_document(doc.id) time.sleep(scan_period) except KeyboardInterrupt: if _IGNORE_KEYBOARD_INTERRUPTS: note(0, "RSSReader: %s", ''.join(traceback.format_exception(*sys.exc_info()))) else: sys.exit(0) except: note(0, "RSSReader: %s", ''.join(traceback.format_exception(*sys.exc_info())))