def main():
"""
main entry point
"""
initialize_stderr_logging()
log = logging.getLogger("main")
try:
args = parse_commandline()
except CommandlineError as instance:
log.error("invalid commandline {0}".format(instance))
return 1
initialize_file_logging(args.log_path, args.verbose)
if args.identity_path is not None:
log.info("loading identity from {0}".format(args.identity_path))
nimbusio_identity = \
motoboto.identity.load_identity_from_file(args.identity_path)
if nimbusio_identity is None:
log.error("Unable to load identity from {0}".format(
args.identity_path))
return 1
else:
nimbusio_identity = None
halt_event = Event()
set_signal_handler(halt_event)
file_name_queue = queue.Queue()
try:
notifier = create_notifier(args.watch_path, file_name_queue)
except InotifyError as instance:
log.error("Unable to initialize inotify: {0}".format(instance))
return 1
notifier_thread = create_notifier_thread(halt_event, notifier)
notifier_thread.start()
# we want the notifier running while we do the initial directory scan, so
# we don't miss any files. But this leaves us open to duplicates()
_initial_directory_scan(args.watch_path, file_name_queue)
# so this eliminates duplicates, but leaks a little memory
# if that's a problem, we could clean it out occasionally
file_name_set = set()
log.info("main loop starts")
return_code = 0
while not halt_event.is_set():
try:
file_name = file_name_queue.get(block=True, timeout=1.0)
except queue.Empty:
continue
log.debug("found file_name '{0}'".format(file_name))
if args.pattern is not None:
if not fnmatch.fnmatch(file_name, args.pattern):
log.info("ignoring '{0}': does not match '{1}'".format(
file_name, args.pattern))
continue
if file_name in file_name_set:
log.info("ignoring duplicate file name '{0}'".format(file_name))
continue
file_name_set.add(file_name)
try:
archive_file(args, nimbusio_identity, file_name)
file_path = os.path.join(args.watch_path, file_name)
log.info("deleting {0}".format(file_path))
os.unlink(file_path)
except Exception as instance:
log.exception(instance)
return_code = 1
halt_event.set()
log.info("main loop ends")
notifier_thread.join(timeout=5.0)
# assert not notifier_thread.is_alive
notifier.stop()
log.info("program terminates return_code = {0}".format(return_code))
return return_code
def main():
"""
main entry point for the program
The return value will be the returncode for the program
"""
initialize_stderr_logging()
log = logging.getLogger("main")
try:
args = parse_commandline()
except CommandlineError:
instance = sys.exc_info()[1]
log.error("invalid commandline {0}".format(instance))
return 1
if not "(?P<key>" in args.key_regex:
log.error("invalid key regex {0}".format(args.key_regex))
return 1
try:
key_regex = re.compile(args.key_regex)
except Exception:
instance = sys.exc_info()[1]
log.error("Unable to compile key_regex {0} {1}".format(args.key_regex,
instance))
return 1
initialize_file_logging(args.log_path, args.verbose)
log.info("key regex pattern = '{0}'".format(key_regex.pattern))
halt_event = Event()
set_signal_handler(halt_event)
file_name_queue = queue.Queue()
try:
notifier = create_notifier(args.watch_path, file_name_queue)
except InotifyError as instance:
log.error("Unable to initialize inotify: {0}".format(instance))
return 1
try:
redis = create_redis_connection()
except Exception:
log.exception("Unable to connect to redis")
return 1
# clear REDIS of all keys under our namespace, so that old sets that
# don't have any files anymore don't stay around
existing_keys = redis.keys("_".join([args.redis_prefix, "*"]))
if len(existing_keys) > 0:
log.debug("deleting {0} existing REDIS keys".format(
len(existing_keys), ))
redis.delete(*existing_keys)
# set our up-to-date time as far back as we can: we aren't up to date yet
up_to_date_timestamp_key = "_".join([args.redis_prefix,
_up_to_date_timestamp])
log.debug("setting {0} to {1}".format(up_to_date_timestamp_key, 0))
redis.set(up_to_date_timestamp_key, "0")
notifier_thread = create_notifier_thread(halt_event,
notifier,
file_name_queue)
# we want the notifier running while we do the initial directory scan, so
# we don't miss any files.
notifier_thread.start()
_initial_directory_scan(args.watch_path, file_name_queue)
log.info("main loop starts")
directory_scan_up_to_date = False
up_to_date = False
return_code = 0
while not halt_event.is_set():
try:
file_name, event_name = file_name_queue.get(block=True, timeout=1.0)
except queue.Empty:
continue
except KeyboardInterrupt:
log.warn("KeyboardInterrupt: halting")
halt_event.set()
break
if event_name == directory_scan_finished:
log.debug("setting directory_scan_up_to_date")
directory_scan_up_to_date = True
continue
if event_name == inotify_idle:
if not up_to_date and directory_scan_up_to_date:
current_time = int(time.time())
log.debug("setting {0} to {1}".format(up_to_date_timestamp_key,
current_time))
redis.set(up_to_date_timestamp_key, str(current_time))
up_to_date = True
continue
match_object = key_regex.match(file_name)
if match_object is None:
log.debug("unmatched file name '{0}'".format(file_name))
continue
key = match_object.group("key")
log.debug("found file_name '{0}' key {1} event {2}".format(file_name,
key,
event_name))
redis_key = "_".join([args.redis_prefix, key, ])
try:
_dispatch_table[event_name](redis, redis_key, file_name)
except Exception:
log.exception("{0} {1}".format(file_name, event_name))
return_code = 1
halt_event.set()
log.info("main loop ends")
redis.set(up_to_date_timestamp_key, "0")
notifier_thread.join(timeout=5.0)
# assert not notifier_thread.is_alive
notifier.stop()
log.info("program terminates return_code = {0}".format(return_code))
return return_code