Пример #1
0
def main(enc):
    conf_dir, log_file, conf_file, feed_dir, script_dir, optlist =\
        args.parse_common_args(enc,
            "hvVfdbi:s", ["help","version","verbose","force","daemon",\
                    "background", "interval=", "sysfp"], "canto-fetch")

    try:
        cfg = get_cfg(conf_file, log_file, feed_dir, script_dir)
        cfg.parse()
    except:
        traceback.print_exc()
        sys.exit(-1)

    cfg.log("Canto-fetch v %s (%s)" % \
            ("%d.%d.%d" % VERSION_TUPLE, GIT_SHA), "w")
    cfg.log("Time: %s" % time.asctime())
    cfg.log("Config parsed successfully.")

    def log_func(x):
        if verbose:
            print x
        cfg.log(x)

    #Defaults
    updateInterval = 60
    daemon = False
    background = False
    verbose = False
    force = False

    for opt, arg in optlist:
        if opt in ["-d", "--daemon"]:
            daemon = True
        if opt in ["-b", "--background"]:
            background = True
            daemon = True
        if opt in ["-i", "--interval"]:
            try:
                arg = unicode(arg, enc, "ignore")
                i = int(arg)
                if i < 60:
                    cfg.log("interval must be >= 60 (one minute)")
                else:
                    updateInterval = i
            except:
                cfg.log("%s isn't a valid interval" % arg)
            else:
                cfg.log("interval = %d seconds" % updateInterval)
        if opt in ["-s", "--sysfp"]:
            log_func("Using system feedparser")
            global feedparser
            try:
                import feedparser as feedparser_system
                feedparser = feedparser_system
            except:
                log_func("Import failed. Falling back on builtin.")
        if opt in ["-V", "--verbose"]:
            verbose = True
        elif opt in ["-f", "--force"]:
            force = True

    # Remove any crap out of the directory. This is mostly for
    # cleaning up when the user has removed a feed from the configuration.

    valid_names = [f.URL.replace("/", " ") for f in cfg.feeds]
    for file in os.listdir(cfg.feed_dir):
        if not file in valid_names:
            log_func("Deleted extraneous file: %s" % file)
            try:
                os.unlink(cfg.feed_dir + file)
            except:
                pass

    if background:
        # This is a pretty canonical way to do backgrounding.

        pid = os.fork()
        if not pid:
            # New terminal session
            os.setsid()

            os.chdir("/")
            os.umask(0)
            pid = os.fork()
            if pid:
                sys.exit(0)
        else:
            sys.exit(0)

        # Close all possible terminal output
        # file descriptors.

        os.close(0)
        os.close(1)
        os.close(2)

    if daemon:
        while 1:
            run(cfg, verbose, force)
            time.sleep(updateInterval)
            oldcfg = cfg
            try:
                cfg = get_cfg(conf_file, log_file, feed_dir, script_dir)
                self.cfg.parse()
            except:
                cfg = oldcfg
    else:
        sys.exit(run(cfg, verbose, force))
Пример #2
0
    def __init__(self, stdscr=None):
        if Main.instance:
            raise Exception("attempting to create a duplicate Main instance")
        Main.instance = self

        signal.signal(signal.SIGUSR2, self.debug_out)

        # Let locale figure itself out
        locale.setlocale(locale.LC_ALL, "")
        enc = locale.getpreferredencoding()

        # If we're canto-fetch, jump to that main function
        if sys.argv[0].endswith("canto-fetch"):
            canto_fetch.main(enc)

        # Parse arguments that canto shares with canto-fetch, return
        # a lot of file locations and an optlist that will contain the
        # parsed, but yet unused canto specific arguments.

        conf_dir, log_file, conf_file, feed_dir, script_dir, optlist = args.parse_common_args(
            enc,
            "hvulaor:t:i:n:",
            ["help", "version", "update", "list", "checkall", "opml", "import=", "url=", "checknew=", "tag="],
        )

        # Instantiate the config and start the log.
        try:
            self.cfg = get_cfg(conf_file, log_file, feed_dir, script_dir)
            self.cfg.parse()
        except:
            traceback.print_exc()
            upgrade_help()
            sys.exit(-1)

        self.cfg.log("Canto v %s (%s)" % ("%d.%d.%d" % VERSION_TUPLE, GIT_SHA), "w")
        self.cfg.log("Time: %s" % time.asctime())
        self.cfg.log("Config parsed successfully.")

        # If we were passed an existing curses screen (i.e. restart)
        # pass it through to the config.

        self.cfg.stdscr = stdscr
        if self.cfg.stdscr:
            self.restarting = True
        else:
            self.restarting = False
        self.restart = False

        # Default arguments.
        flags = 0
        feed_ct = None
        opml_file = None
        url = None
        newtag = None

        # Note that every single flag that takes an argument has its
        # argument converted to unicode. Saves a lot of bullshit later.

        for opt, arg in optlist:
            if opt in ["-u", "--update"]:
                flags |= UPDATE_FIRST
            elif opt in ["-n", "--checknew"]:
                flags |= CHECK_NEW
                feed_ct = unicode(arg, enc, "ignore")
            elif opt in ["-a", "--checkall"]:
                flags |= CHECK_NEW
            elif opt in ["-l", "--list"]:
                flags |= FEED_LIST
            elif opt in ["-o", "--opml"]:
                flags |= OUT_OPML
            elif opt in ["-i", "--import"]:
                flags |= IN_OPML
                opml_file = unicode(arg, enc, "ignore")
            elif opt in ["-r", "--url"]:
                flags |= IN_URL
                url = unicode(arg, enc, "ignore")
            elif opt in ["-t", "--tag"]:
                newtag = unicode(arg, enc, "ignore")

        # Import flags harness the same functions as their config
        # based counterparts, source_opml and source_url.

        if flags & IN_OPML:
            self.cfg.locals["source_opml"](opml_file, append=True)
            print "OPML imported."

        if flags & IN_URL:
            self.cfg.locals["source_url"](url, append=True, tag=newtag)
            print "URL added."

        # All import options should terminate.

        if flags & (IN_OPML + IN_URL):
            sys.exit(0)

        # If self.cfg had to generate a config, make sure we
        # update first.

        if self.cfg.no_conf:
            self.cfg.log("Conf was auto-generated, adding -u")
            flags |= UPDATE_FIRST

        if flags & UPDATE_FIRST:
            self.cfg.log("Pausing to update...")
            canto_fetch.run(self.cfg, True, True)

        # Detect if there are any new feeds by whether their
        # set path exists. If not, run canto-fetch but don't
        # force it, so canto-fetch intelligently updates.

        for i, f in enumerate(self.cfg.feeds):
            if not os.path.exists(f.path):
                self.cfg.log("Detected unfetched feed: %s." % f.URL)
                canto_fetch.run(self.cfg, True, False)

                # Still no go?
                if not os.path.exists(f.path):
                    self.cfg.log("Failed to fetch %s, removing" % f.URL)
                    self.cfg.feeds[i] = None
                else:
                    self.cfg.log("Fetched.\n")
                break

        # Collapse the feed array, if we had to remove some unfetchables.
        self.cfg.feeds = [f for f in self.cfg.feeds if f is not None]

        self.new = []
        self.old = []
        self.ph = ProcessHandler(self.cfg)

        # Force an update from disk by queueing a work item for each thread.
        # At this point, we just want to do the portion of the update where the
        # disk is read, so PROC_UPDATE is used.

        self.cfg.log("Populating feeds...")
        for f in self.cfg.feeds:
            self.ph.send((PROC_UPDATE, f.URL, []))
        for f in self.cfg.feeds:
            f.merge(self.ph.recv()[1])

        self.ph.send((PROC_GETTAGS,))
        fixedtags = self.ph.recv()

        self.ph.kill_process()

        for i, f in enumerate(self.cfg.feeds):
            self.cfg.feeds[i].tags = fixedtags[i]

        # Now that the tags have all been straightened out, validate the config.
        # Making sure the tags are unique before validation is important because
        # part of validation is the actual creation of Tag() objects.

        try:
            self.cfg.validate()
        except Exception, err:
            print err
            upgrade_help()
            sys.exit(0)
Пример #3
0
def main(enc):
    conf_dir, log_file, conf_file, feed_dir, script_dir, optlist =\
        args.parse_common_args(enc,
            "hvVfdbi:s", ["help","version","verbose","force","daemon",\
                    "background", "interval=", "sysfp"], "canto-fetch")

    try :
        cfg = get_cfg(conf_file, log_file, feed_dir, script_dir)
        cfg.parse()
    except :
        traceback.print_exc()
        sys.exit(-1)

    cfg.log("Canto-fetch v %s (%s)" % \
            ("%d.%d.%d" % VERSION_TUPLE, GIT_SHA), "w")
    cfg.log("Time: %s" % time.asctime())
    cfg.log("Config parsed successfully.")

    def log_func(x):
        if verbose:
            print x
        cfg.log(x)

    #Defaults
    updateInterval = 60
    daemon = False
    background = False
    verbose = False
    force = False

    for opt, arg in optlist :
        if opt in ["-d","--daemon"]:
            daemon = True
        if opt in ["-b","--background"]:
            background = True
            daemon = True
        if opt in ["-i","--interval"]:
            try:
                arg = unicode(arg, enc, "ignore")
                i = int(arg)
                if i < 60:
                    cfg.log("interval must be >= 60 (one minute)")
                else:
                    updateInterval = i
            except:
                cfg.log("%s isn't a valid interval" % arg)
            else:
                cfg.log("interval = %d seconds" % updateInterval)
        if opt in ["-s","--sysfp"]:
            log_func("Using system feedparser")
            global feedparser
            try:
                import feedparser as feedparser_system
                feedparser = feedparser_system
            except:
                log_func("Import failed. Falling back on builtin.")
        if opt in ["-V","--verbose"]:
            verbose = True
        elif opt in ["-f","--force"]:
            force = True

    # Remove any crap out of the directory. This is mostly for
    # cleaning up when the user has removed a feed from the configuration.

    valid_names = [f.URL.replace("/","_") for f in cfg.feeds]
    for file in os.listdir(cfg.feed_dir):
        if not file in valid_names:
            log_func("Deleted extraneous file: %s" % file)
            try:
                os.unlink(cfg.feed_dir + file)
            except:
                pass

    if background:
        # This is a pretty canonical way to do backgrounding.

        pid = os.fork()
        if not pid:
            # New terminal session
            os.setsid()

            os.chdir("/")
            os.umask(0)
            pid = os.fork()
            if pid:
                sys.exit(0)
        else:
            sys.exit(0)

        # Close all possible terminal output
        # file descriptors. 

        os.close(0)
        os.close(1)
        os.close(2)

    if daemon:
        while 1:
            run(cfg, verbose, force)
            time.sleep(updateInterval)
            oldcfg = cfg
            try :
                cfg = get_cfg(conf_file, log_file, feed_dir, script_dir)
                self.cfg.parse()
            except:
                cfg = oldcfg
    else:
        sys.exit(run(cfg, verbose, force))
Пример #4
0
    def __init__(self, stdscr=None):
        signal.signal(signal.SIGUSR2, self.debug_out)

        # Let locale figure itself out
        locale.setlocale(locale.LC_ALL, "")
        enc = locale.getpreferredencoding()

        # If we're canto-fetch, jump to that main function
        if sys.argv[0].endswith("canto-fetch"):
            canto_fetch.main(enc)

        # Parse arguments that canto shares with canto-fetch, return
        # a lot of file locations and an optlist that will contain the
        # parsed, but yet unused canto specific arguments.

        conf_dir, log_file, conf_file, feed_dir, script_dir, optlist =\
                args.parse_common_args(enc,
                    "hvulaor:t:i:n:",
                    ["help","version","update","list","checkall","opml",
                        "import=","url=","checknew=","tag="])

        # Instantiate the config and start the log.
        try:
            self.cfg = get_cfg(conf_file, log_file, feed_dir, script_dir)
            self.cfg.parse()
        except:
            traceback.print_exc()
            upgrade_help()
            sys.exit(-1)

        self.cfg.log("Canto v %s (%s)" % \
                ("%d.%d.%d" % VERSION_TUPLE, GIT_SHA), "w")
        self.cfg.log("Time: %s" % time.asctime())
        self.cfg.log("Config parsed successfully.")

        # If we were passed an existing curses screen (i.e. restart)
        # pass it through to the config.

        self.cfg.stdscr = stdscr
        if self.cfg.stdscr:
            self.restarting = True
        else:
            self.restarting = False
        self.restart = False

        # Default arguments.
        flags = 0
        feed_ct = None
        opml_file = None
        url = None
        newtag = None

        # Note that every single flag that takes an argument has its
        # argument converted to unicode. Saves a lot of bullshit later.

        for opt, arg in optlist:
            if opt in ["-u", "--update"]:
                flags |= UPDATE_FIRST
            elif opt in ["-n", "--checknew"]:
                flags |= CHECK_NEW
                feed_ct = unicode(arg, enc, "ignore")
            elif opt in ["-a", "--checkall"]:
                flags |= CHECK_NEW
            elif opt in ["-l", "--list"]:
                flags |= FEED_LIST
            elif opt in ["-o", "--opml"]:
                flags |= OUT_OPML
            elif opt in ["-i", "--import"]:
                flags |= IN_OPML
                opml_file = unicode(arg, enc, "ignore")
            elif opt in ["-r", "--url"]:
                flags |= IN_URL
                url = unicode(arg, enc, "ignore")
            elif opt in ["-t", "--tag"]:
                newtag = unicode(arg, enc, "ignore")

        # Import flags harness the same functions as their config
        # based counterparts, source_opml and source_url.

        if flags & IN_OPML:
            self.cfg.locals['source_opml'](opml_file, append=True)
            print "OPML imported."

        if flags & IN_URL:
            self.cfg.locals['source_url'](url, append=True, tag=newtag)
            print "URL added."

        # All import options should terminate.

        if flags & (IN_OPML + IN_URL):
            sys.exit(0)

        # If self.cfg had to generate a config, make sure we
        # update first.

        if self.cfg.no_conf:
            self.cfg.log("Conf was auto-generated, adding -u")
            flags |= UPDATE_FIRST

        if flags & UPDATE_FIRST:
            self.cfg.log("Pausing to update...")
            canto_fetch.run(self.cfg, True, True)

        # Detect if there are any new feeds by whether their
        # set path exists. If not, run canto-fetch but don't
        # force it, so canto-fetch intelligently updates.

        for i, f in enumerate(self.cfg.feeds):
            if not os.path.exists(f.path):
                self.cfg.log("Detected unfetched feed: %s." % f.URL)
                canto_fetch.run(self.cfg, True, False)

                #Still no go?
                if not os.path.exists(f.path):
                    self.cfg.log("Failed to fetch %s, removing" % f.URL)
                    self.cfg.feeds[i] = None
                else:
                    self.cfg.log("Fetched.\n")
                break

        # Collapse the feed array, if we had to remove some unfetchables.
        self.cfg.feeds = filter(lambda x: x != None, self.cfg.feeds)

        self.new = []
        self.old = []
        self.ph = ProcessHandler(self.cfg)

        # Force an update from disk by queueing a work item for each thread.
        # At this point, we just want to do the portion of the update where the
        # disk is read, so PROC_UPDATE is used.

        self.cfg.log("Populating feeds...")
        for f in self.cfg.feeds:
            self.ph.send((PROC_UPDATE, f.URL, []))
        for f in self.cfg.feeds:
            f.merge(self.ph.recv()[1])

        self.ph.send((PROC_GETTAGS, ))
        fixedtags = self.ph.recv()

        self.ph.kill_process()

        for i, f in enumerate(self.cfg.feeds):
            self.cfg.feeds[i].tags = fixedtags[i]

        # Now that the tags have all been straightened out, validate the config.
        # Making sure the tags are unique before validation is important because
        # part of validation is the actual creation of Tag() objects.

        try:
            self.cfg.validate()
        except Exception, err:
            print err
            upgrade_help()
            sys.exit(0)