Beispiel #1
0
    def __emit_log(self):
        for source_name, source_data in self.data.iteritems():
            pairs = [("%s= [%s]" % (k, v)) for k, v in source_data.iteritems()]
            logging.info("RPT EMIT(%s): %s" % (source_name, ', '.join(pairs)))

        report_emit_interval_s = Conf.get('report_emit_frequency_s')
        emit_timer = Timer(report_emit_interval_s, self.__emit_log)

        Timers.get_instance().register_timer('emit', emit_timer)
Beispiel #2
0
def _sched_check_changes():
    
    logging.debug("Doing scheduled check for changes.")

    get_change_manager().process_updates()

    # Schedule next invocation.
    t = Timer(Conf.get('change_check_frequency_s'), _sched_check_changes)

    Timers.get_instance().register_timer('change', t)
Beispiel #3
0
def _sched_check_changes():

    logging.debug("Doing scheduled check for changes.")

    get_change_manager().process_updates()

    # Schedule next invocation.
    t = Timer(Conf.get('change_check_frequency_s'), _sched_check_changes)

    Timers.get_instance().register_timer('change', t)
Beispiel #4
0
    def __emit_log(self):
        for source_name, source_data in self.data.iteritems():
            pairs = [ ("%s= [%s]" % (k, v)) 
                        for k, v 
                        in source_data.iteritems() ]
            logging.info("RPT EMIT(%s): %s" % (source_name, ', '.join(pairs)))

        report_emit_interval_s = Conf.get('report_emit_frequency_s')
        emit_timer = Timer(report_emit_interval_s, self.__emit_log)

        Timers.get_instance().register_timer('emit', emit_timer)
Beispiel #5
0
    def __post_status(self):
        """Send the current status to our reporting tool."""

        num_values = self.registry.count(self.resource_name)

        self.report.set_values(self.report_source_name, 'count', 
                               num_values)

        status_post_interval_s = Conf.get('cache_status_post_frequency_s')
        status_timer = Timer(status_post_interval_s, self.__post_status)

        Timers.get_instance().register_timer('status', status_timer)
Beispiel #6
0
def _sched_check_changes():
    logging.debug("Doing scheduled check for changes.")

    try:
        get_change_manager().process_updates()
        logging.debug("Updates have been processed. Rescheduling.")

        # Schedule next invocation.
        t = Timer(Conf.get('change_check_frequency_s'), _sched_check_changes)

        Timers.get_instance().register_timer('change', t)
    except:
        _logger.exception("Exception while managing changes.")
        raise
Beispiel #7
0
    def __cleanup_check(self):
        """Scan the current cache and determine items old-enough to be 
        removed.
        """

        self.__log.debug("Doing clean-up for cache resource with name [%s]." % 
                      (self.resource_name))

        try:
            cache_dict = self.registry.list_raw(self.resource_name)
        except:
            self.__log.exception("Could not do clean-up check with resource-"
                                 "name [%s]." % (self.resource_name))
            raise

        total_keys = [ (key, value_tuple[1]) for key, value_tuple \
                            in cache_dict.iteritems() ]

        cleanup_keys = [ key for key, value_tuple \
                            in cache_dict.iteritems() \
                            if (datetime.now() - value_tuple[1]).seconds > \
                                    self.max_age ]

        self.__log.info("Found (%d) entries to clean-up from entry-cache." % 
                        (len(cleanup_keys)))

        if cleanup_keys:
            for key in cleanup_keys:
                self.__log.debug("Cache entry [%s] under resource-name [%s] "
                                 "will be cleaned-up." % (key, 
                                                          self.resource_name))

                if self.exists(key, no_fault_check=True) == False:
                    self.__log.debug("Entry with ID [%s] has already been "
                                     "cleaned-up." % (key))
                else:
                    try:
                        self.remove(key)
                    except:
                        self.__log.exception("Cache entry [%s] under resource-"
                                             "name [%s] could not be cleaned-"
                                             "up." % (key, self.resource_name))
                        raise

            self.__log.debug("Scheduled clean-up complete.")

        cleanup_interval_s = Conf.get('cache_cleanup_check_frequency_s')
        cleanup_timer = Timer(cleanup_interval_s, self.__cleanup_check)

        Timers.get_instance().register_timer('cleanup', cleanup_timer)
Beispiel #8
0
    def __cleanup_check(self):
        """Scan the current cache and determine items old-enough to be 
        removed.
        """

        self.__log.debug("Doing clean-up for cache resource with name [%s]." %
                         (self.resource_name))

        try:
            cache_dict = self.registry.list_raw(self.resource_name)
        except:
            self.__log.exception("Could not do clean-up check with resource-"
                                 "name [%s]." % (self.resource_name))
            raise

        total_keys = [ (key, value_tuple[1]) for key, value_tuple \
                            in cache_dict.iteritems() ]

        cleanup_keys = [ key for key, value_tuple \
                            in cache_dict.iteritems() \
                            if (datetime.now() - value_tuple[1]).seconds > \
                                    self.max_age ]

        self.__log.info("Found (%d) entries to clean-up from entry-cache." %
                        (len(cleanup_keys)))

        if cleanup_keys:
            for key in cleanup_keys:
                self.__log.debug("Cache entry [%s] under resource-name [%s] "
                                 "will be cleaned-up." %
                                 (key, self.resource_name))

                if self.exists(key, no_fault_check=True) == False:
                    self.__log.debug("Entry with ID [%s] has already been "
                                     "cleaned-up." % (key))
                else:
                    try:
                        self.remove(key)
                    except:
                        self.__log.exception("Cache entry [%s] under resource-"
                                             "name [%s] could not be cleaned-"
                                             "up." % (key, self.resource_name))
                        raise

            self.__log.debug("Scheduled clean-up complete.")

        cleanup_interval_s = Conf.get('cache_cleanup_check_frequency_s')
        cleanup_timer = Timer(cleanup_interval_s, self.__cleanup_check)

        Timers.get_instance().register_timer('cleanup', cleanup_timer)
Beispiel #9
0
def _sched_check_changes():
    logging.debug("Doing scheduled check for changes.")

    try:
        get_change_manager().process_updates()
        logging.debug("Updates have been processed. Rescheduling.")

        # Schedule next invocation.
        t = Timer(Conf.get('change_check_frequency_s'), _sched_check_changes)

        Timers.get_instance().register_timer('change', t)
    except:
        _logger.exception("Exception while managing changes.")
        raise
Beispiel #10
0
    def __post_status(self):
        """Send the current status to our reporting tool."""

        try:
            num_values = self.registry.count(self.resource_name)
        except:
            self.__log.exception("Could not get count of values for resource "
                                 "with name [%s]." % (self.resource_name))
            raise

        try:
            self.report.set_values(self.report_source_name, 'count', 
                                   num_values)
        except:
            self.__log.exception("Cache could not post status for resource "
                                 "with name [%s]." % (self.resource_name))
            raise

        status_post_interval_s = Conf.get('cache_status_post_frequency_s')
        status_timer = Timer(status_post_interval_s, self.__post_status)

        Timers.get_instance().register_timer('status', status_timer)
Beispiel #11
0
    def __post_status(self):
        """Send the current status to our reporting tool."""

        try:
            num_values = self.registry.count(self.resource_name)
        except:
            self.__log.exception("Could not get count of values for resource "
                                 "with name [%s]." % (self.resource_name))
            raise

        try:
            self.report.set_values(self.report_source_name, 'count',
                                   num_values)
        except:
            self.__log.exception("Cache could not post status for resource "
                                 "with name [%s]." % (self.resource_name))
            raise

        status_post_interval_s = Conf.get('cache_status_post_frequency_s')
        status_timer = Timer(status_post_interval_s, self.__post_status)

        Timers.get_instance().register_timer('status', status_timer)
Beispiel #12
0
def mount(auth_storage_filepath, mountpoint, debug=None, nothreads=None, 
          option_string=None):

    fuse_opts = { }
    if option_string:
        for opt_parts in [opt.split('=', 1) \
                          for opt \
                          in option_string.split(',') ]:
            k = opt_parts[0]

            # We need to present a bool type for on/off flags. Since all we
            # have are strings, we'll convert anything with a 'True' or 'False'
            # to a bool, or anything with just a key to True.
            if len(opt_parts) == 2:
                v = opt_parts[1]
                v_lower = v.lower()

                if v_lower == 'true':
                    v = True
                elif v_lower == 'false':
                    v = False
            else:
                v = True

            # We have a list of provided options. See which match against our 
            # application options.

            logging.info("Setting option [%s] to [%s]." % (k, v))

            try:
                Conf.set(k, v)
            except (KeyError) as e:
                logging.debug("Forwarding option [%s] with value [%s] to "
                              "FUSE." % (k, v))

                fuse_opts[k] = v
            except:
                logging.exception("Could not set option [%s]. It is probably "
                                  "invalid." % (k))
                raise

    logging.debug("PERMS: F=%s E=%s NE=%s" % 
                  (Conf.get('default_perm_folder'), 
                   Conf.get('default_perm_file_editable'), 
                   Conf.get('default_perm_file_noneditable')))

    # Assume that any option that wasn't an application option is a FUSE 
    # option. The Python-FUSE interface that we're using is beautiful/elegant,
    # but there's no help support. The user is just going to have to know the
    # options.

    set_auth_cache_filepath(auth_storage_filepath)

    # How we'll appear in diskfree, mtab, etc..
    name = ("gdfs(%s)" % (auth_storage_filepath))

    # Don't start any of the scheduled tasks, such as change checking, cache
    # cleaning, etc. It will minimize outside influence of the logs and state
    # to make it easier to debug.

#    atexit.register(Timers.get_instance().cancel_all)
    if debug:
        Timers.get_instance().set_autostart_default(False)

    fuse = FUSE(GDriveFS(), mountpoint, debug=debug, foreground=debug, 
                nothreads=nothreads, fsname=name, **fuse_opts)
Beispiel #13
0
def mount(auth_storage_filepath,
          mountpoint,
          debug=None,
          nothreads=None,
          option_string=None):

    fuse_opts = {}
    if option_string:
        for opt_parts in [opt.split('=', 1) \
                          for opt \
                          in option_string.split(',') ]:
            k = opt_parts[0]

            # We need to present a bool type for on/off flags. Since all we
            # have are strings, we'll convert anything with a 'True' or 'False'
            # to a bool, or anything with just a key to True.
            if len(opt_parts) == 2:
                v = opt_parts[1]
                v_lower = v.lower()

                if v_lower == 'true':
                    v = True
                elif v_lower == 'false':
                    v = False
            else:
                v = True

            # We have a list of provided options. See which match against our
            # application options.

            logging.info("Setting option [%s] to [%s]." % (k, v))

            try:
                Conf.set(k, v)
            except (KeyError) as e:
                logging.debug("Forwarding option [%s] with value [%s] to "
                              "FUSE." % (k, v))

                fuse_opts[k] = v
            except:
                logging.exception("Could not set option [%s]. It is probably "
                                  "invalid." % (k))
                raise

    logging.debug("PERMS: F=%s E=%s NE=%s" %
                  (Conf.get('default_perm_folder'),
                   Conf.get('default_perm_file_editable'),
                   Conf.get('default_perm_file_noneditable')))

    # Assume that any option that wasn't an application option is a FUSE
    # option. The Python-FUSE interface that we're using is beautiful/elegant,
    # but there's no help support. The user is just going to have to know the
    # options.

    set_auth_cache_filepath(auth_storage_filepath)

    # How we'll appear in diskfree, mtab, etc..
    name = ("gdfs(%s)" % (auth_storage_filepath))

    # Don't start any of the scheduled tasks, such as change checking, cache
    # cleaning, etc. It will minimize outside influence of the logs and state
    # to make it easier to debug.

    #    atexit.register(Timers.get_instance().cancel_all)
    if debug:
        Timers.get_instance().set_autostart_default(False)

    fuse = FUSE(GDriveFS(),
                mountpoint,
                debug=debug,
                foreground=debug,
                nothreads=nothreads,
                fsname=name,
                **fuse_opts)

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('cred_filepath', help='Credentials file')

    subparsers = parser.add_subparsers(help='subcommand help')
    parser_bypath = subparsers.add_parser('bypath', help='Path-based lookups.')
    parser_bypath.add_argument('path', help='Path to identify')
    parser_byid = subparsers.add_parser('byid', help='Path-based lookups.')
    parser_byid.add_argument('id', help='Specific entry to identify')

    args = parser.parse_args()

    set_auth_cache_filepath(args.cred_filepath)
    Timers.get_instance().set_autostart_default(False)

    if 'id' in args:
        entry = get_by_id(args.id)

    if 'path' in args:
        entry = get_by_path(args.path)

    print(entry)
    print

    data = entry.get_data()

    for _type, _dict in data.iteritems():
        print("[%s]\n" % (_type))
    return cache.get(_id)

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('cred_filepath', help='Credentials file')

    subparsers = parser.add_subparsers(help='subcommand help')
    parser_bypath = subparsers.add_parser('bypath', help='Path-based lookups.')
    parser_bypath.add_argument('path', help='Path to identify')
    parser_byid = subparsers.add_parser('byid', help='Path-based lookups.')
    parser_byid.add_argument('id', help='Specific entry to identify')

    args = parser.parse_args()

    set_auth_cache_filepath(args.cred_filepath)
    Timers.get_instance().set_autostart_default(False)

    if 'id' in args:
        entry = get_by_id(args.id)
    
    if 'path' in args:
        entry = get_by_path(args.path)

    print(entry)
    print    

    data = entry.get_data()

    for _type, _dict in data.iteritems():
        print("[%s]\n" % (_type))