def __emit_log(self):
for source_name, source_data in self.data.iteritems():
pairs = [("%s= [%s]" % (k, v)) for k, v in source_data.iteritems()]
logging.info("RPT EMIT(%s): %s" % (source_name, ', '.join(pairs)))
report_emit_interval_s = Conf.get('report_emit_frequency_s')
emit_timer = Timer(report_emit_interval_s, self.__emit_log)
Timers.get_instance().register_timer('emit', emit_timer)
def _sched_check_changes():
logging.debug("Doing scheduled check for changes.")
get_change_manager().process_updates()
# Schedule next invocation.
t = Timer(Conf.get('change_check_frequency_s'), _sched_check_changes)
Timers.get_instance().register_timer('change', t)
def _sched_check_changes():
logging.debug("Doing scheduled check for changes.")
get_change_manager().process_updates()
# Schedule next invocation.
t = Timer(Conf.get('change_check_frequency_s'), _sched_check_changes)
Timers.get_instance().register_timer('change', t)
def __emit_log(self):
for source_name, source_data in self.data.iteritems():
pairs = [ ("%s= [%s]" % (k, v))
for k, v
in source_data.iteritems() ]
logging.info("RPT EMIT(%s): %s" % (source_name, ', '.join(pairs)))
report_emit_interval_s = Conf.get('report_emit_frequency_s')
emit_timer = Timer(report_emit_interval_s, self.__emit_log)
Timers.get_instance().register_timer('emit', emit_timer)
def __post_status(self):
"""Send the current status to our reporting tool."""
num_values = self.registry.count(self.resource_name)
self.report.set_values(self.report_source_name, 'count',
num_values)
status_post_interval_s = Conf.get('cache_status_post_frequency_s')
status_timer = Timer(status_post_interval_s, self.__post_status)
Timers.get_instance().register_timer('status', status_timer)
def _sched_check_changes():
logging.debug("Doing scheduled check for changes.")
try:
get_change_manager().process_updates()
logging.debug("Updates have been processed. Rescheduling.")
# Schedule next invocation.
t = Timer(Conf.get('change_check_frequency_s'), _sched_check_changes)
Timers.get_instance().register_timer('change', t)
except:
_logger.exception("Exception while managing changes.")
raise
def __cleanup_check(self):
"""Scan the current cache and determine items old-enough to be
removed.
"""
self.__log.debug("Doing clean-up for cache resource with name [%s]." %
(self.resource_name))
try:
cache_dict = self.registry.list_raw(self.resource_name)
except:
self.__log.exception("Could not do clean-up check with resource-"
"name [%s]." % (self.resource_name))
raise
total_keys = [ (key, value_tuple[1]) for key, value_tuple \
in cache_dict.iteritems() ]
cleanup_keys = [ key for key, value_tuple \
in cache_dict.iteritems() \
if (datetime.now() - value_tuple[1]).seconds > \
self.max_age ]
self.__log.info("Found (%d) entries to clean-up from entry-cache." %
(len(cleanup_keys)))
if cleanup_keys:
for key in cleanup_keys:
self.__log.debug("Cache entry [%s] under resource-name [%s] "
"will be cleaned-up." % (key,
self.resource_name))
if self.exists(key, no_fault_check=True) == False:
self.__log.debug("Entry with ID [%s] has already been "
"cleaned-up." % (key))
else:
try:
self.remove(key)
except:
self.__log.exception("Cache entry [%s] under resource-"
"name [%s] could not be cleaned-"
"up." % (key, self.resource_name))
raise
self.__log.debug("Scheduled clean-up complete.")
cleanup_interval_s = Conf.get('cache_cleanup_check_frequency_s')
cleanup_timer = Timer(cleanup_interval_s, self.__cleanup_check)
Timers.get_instance().register_timer('cleanup', cleanup_timer)
def __cleanup_check(self):
"""Scan the current cache and determine items old-enough to be
removed.
"""
self.__log.debug("Doing clean-up for cache resource with name [%s]." %
(self.resource_name))
try:
cache_dict = self.registry.list_raw(self.resource_name)
except:
self.__log.exception("Could not do clean-up check with resource-"
"name [%s]." % (self.resource_name))
raise
total_keys = [ (key, value_tuple[1]) for key, value_tuple \
in cache_dict.iteritems() ]
cleanup_keys = [ key for key, value_tuple \
in cache_dict.iteritems() \
if (datetime.now() - value_tuple[1]).seconds > \
self.max_age ]
self.__log.info("Found (%d) entries to clean-up from entry-cache." %
(len(cleanup_keys)))
if cleanup_keys:
for key in cleanup_keys:
self.__log.debug("Cache entry [%s] under resource-name [%s] "
"will be cleaned-up." %
(key, self.resource_name))
if self.exists(key, no_fault_check=True) == False:
self.__log.debug("Entry with ID [%s] has already been "
"cleaned-up." % (key))
else:
try:
self.remove(key)
except:
self.__log.exception("Cache entry [%s] under resource-"
"name [%s] could not be cleaned-"
"up." % (key, self.resource_name))
raise
self.__log.debug("Scheduled clean-up complete.")
cleanup_interval_s = Conf.get('cache_cleanup_check_frequency_s')
cleanup_timer = Timer(cleanup_interval_s, self.__cleanup_check)
Timers.get_instance().register_timer('cleanup', cleanup_timer)
def _sched_check_changes():
logging.debug("Doing scheduled check for changes.")
try:
get_change_manager().process_updates()
logging.debug("Updates have been processed. Rescheduling.")
# Schedule next invocation.
t = Timer(Conf.get('change_check_frequency_s'), _sched_check_changes)
Timers.get_instance().register_timer('change', t)
except:
_logger.exception("Exception while managing changes.")
raise
def __post_status(self):
"""Send the current status to our reporting tool."""
try:
num_values = self.registry.count(self.resource_name)
except:
self.__log.exception("Could not get count of values for resource "
"with name [%s]." % (self.resource_name))
raise
try:
self.report.set_values(self.report_source_name, 'count',
num_values)
except:
self.__log.exception("Cache could not post status for resource "
"with name [%s]." % (self.resource_name))
raise
status_post_interval_s = Conf.get('cache_status_post_frequency_s')
status_timer = Timer(status_post_interval_s, self.__post_status)
Timers.get_instance().register_timer('status', status_timer)
def __post_status(self):
"""Send the current status to our reporting tool."""
try:
num_values = self.registry.count(self.resource_name)
except:
self.__log.exception("Could not get count of values for resource "
"with name [%s]." % (self.resource_name))
raise
try:
self.report.set_values(self.report_source_name, 'count',
num_values)
except:
self.__log.exception("Cache could not post status for resource "
"with name [%s]." % (self.resource_name))
raise
status_post_interval_s = Conf.get('cache_status_post_frequency_s')
status_timer = Timer(status_post_interval_s, self.__post_status)
Timers.get_instance().register_timer('status', status_timer)
def mount(auth_storage_filepath, mountpoint, debug=None, nothreads=None,
option_string=None):
fuse_opts = { }
if option_string:
for opt_parts in [opt.split('=', 1) \
for opt \
in option_string.split(',') ]:
k = opt_parts[0]
# We need to present a bool type for on/off flags. Since all we
# have are strings, we'll convert anything with a 'True' or 'False'
# to a bool, or anything with just a key to True.
if len(opt_parts) == 2:
v = opt_parts[1]
v_lower = v.lower()
if v_lower == 'true':
v = True
elif v_lower == 'false':
v = False
else:
v = True
# We have a list of provided options. See which match against our
# application options.
logging.info("Setting option [%s] to [%s]." % (k, v))
try:
Conf.set(k, v)
except (KeyError) as e:
logging.debug("Forwarding option [%s] with value [%s] to "
"FUSE." % (k, v))
fuse_opts[k] = v
except:
logging.exception("Could not set option [%s]. It is probably "
"invalid." % (k))
raise
logging.debug("PERMS: F=%s E=%s NE=%s" %
(Conf.get('default_perm_folder'),
Conf.get('default_perm_file_editable'),
Conf.get('default_perm_file_noneditable')))
# Assume that any option that wasn't an application option is a FUSE
# option. The Python-FUSE interface that we're using is beautiful/elegant,
# but there's no help support. The user is just going to have to know the
# options.
set_auth_cache_filepath(auth_storage_filepath)
# How we'll appear in diskfree, mtab, etc..
name = ("gdfs(%s)" % (auth_storage_filepath))
# Don't start any of the scheduled tasks, such as change checking, cache
# cleaning, etc. It will minimize outside influence of the logs and state
# to make it easier to debug.
# atexit.register(Timers.get_instance().cancel_all)
if debug:
Timers.get_instance().set_autostart_default(False)
fuse = FUSE(GDriveFS(), mountpoint, debug=debug, foreground=debug,
nothreads=nothreads, fsname=name, **fuse_opts)
def mount(auth_storage_filepath,
mountpoint,
debug=None,
nothreads=None,
option_string=None):
fuse_opts = {}
if option_string:
for opt_parts in [opt.split('=', 1) \
for opt \
in option_string.split(',') ]:
k = opt_parts[0]
# We need to present a bool type for on/off flags. Since all we
# have are strings, we'll convert anything with a 'True' or 'False'
# to a bool, or anything with just a key to True.
if len(opt_parts) == 2:
v = opt_parts[1]
v_lower = v.lower()
if v_lower == 'true':
v = True
elif v_lower == 'false':
v = False
else:
v = True
# We have a list of provided options. See which match against our
# application options.
logging.info("Setting option [%s] to [%s]." % (k, v))
try:
Conf.set(k, v)
except (KeyError) as e:
logging.debug("Forwarding option [%s] with value [%s] to "
"FUSE." % (k, v))
fuse_opts[k] = v
except:
logging.exception("Could not set option [%s]. It is probably "
"invalid." % (k))
raise
logging.debug("PERMS: F=%s E=%s NE=%s" %
(Conf.get('default_perm_folder'),
Conf.get('default_perm_file_editable'),
Conf.get('default_perm_file_noneditable')))
# Assume that any option that wasn't an application option is a FUSE
# option. The Python-FUSE interface that we're using is beautiful/elegant,
# but there's no help support. The user is just going to have to know the
# options.
set_auth_cache_filepath(auth_storage_filepath)
# How we'll appear in diskfree, mtab, etc..
name = ("gdfs(%s)" % (auth_storage_filepath))
# Don't start any of the scheduled tasks, such as change checking, cache
# cleaning, etc. It will minimize outside influence of the logs and state
# to make it easier to debug.
# atexit.register(Timers.get_instance().cancel_all)
if debug:
Timers.get_instance().set_autostart_default(False)
fuse = FUSE(GDriveFS(),
mountpoint,
debug=debug,
foreground=debug,
nothreads=nothreads,
fsname=name,
**fuse_opts)
if __name__ == '__main__':
parser = ArgumentParser()
parser.add_argument('cred_filepath', help='Credentials file')
subparsers = parser.add_subparsers(help='subcommand help')
parser_bypath = subparsers.add_parser('bypath', help='Path-based lookups.')
parser_bypath.add_argument('path', help='Path to identify')
parser_byid = subparsers.add_parser('byid', help='Path-based lookups.')
parser_byid.add_argument('id', help='Specific entry to identify')
args = parser.parse_args()
set_auth_cache_filepath(args.cred_filepath)
Timers.get_instance().set_autostart_default(False)
if 'id' in args:
entry = get_by_id(args.id)
if 'path' in args:
entry = get_by_path(args.path)
print(entry)
print
data = entry.get_data()
for _type, _dict in data.iteritems():
print("[%s]\n" % (_type))
return cache.get(_id)
if __name__ == '__main__':
parser = ArgumentParser()
parser.add_argument('cred_filepath', help='Credentials file')
subparsers = parser.add_subparsers(help='subcommand help')
parser_bypath = subparsers.add_parser('bypath', help='Path-based lookups.')
parser_bypath.add_argument('path', help='Path to identify')
parser_byid = subparsers.add_parser('byid', help='Path-based lookups.')
parser_byid.add_argument('id', help='Specific entry to identify')
args = parser.parse_args()
set_auth_cache_filepath(args.cred_filepath)
Timers.get_instance().set_autostart_default(False)
if 'id' in args:
entry = get_by_id(args.id)
if 'path' in args:
entry = get_by_path(args.path)
print(entry)
print
data = entry.get_data()
for _type, _dict in data.iteritems():
print("[%s]\n" % (_type))