コード例 #1
0
def main():
    """
    main entry point
    """
    initialize_stderr_logging()
    log = logging.getLogger("main")

    try:
        args = parse_commandline()
    except CommandlineError as instance:
        log.error("invalid commandline {0}".format(instance))
        return 1

    initialize_file_logging(args.log_path, args.verbose)

    if args.identity_path is not None:
        log.info("loading identity from {0}".format(args.identity_path))
        nimbusio_identity = \
            motoboto.identity.load_identity_from_file(args.identity_path)
        if nimbusio_identity is None:
            log.error("Unable to load identity from {0}".format(
                      args.identity_path))
            return 1
    else:
        nimbusio_identity = None

    halt_event = Event()
    set_signal_handler(halt_event)

    file_name_queue = queue.Queue()    

    try:
        notifier = create_notifier(args.watch_path, file_name_queue)
    except InotifyError as instance:
        log.error("Unable to initialize inotify: {0}".format(instance))
        return 1

    notifier_thread = create_notifier_thread(halt_event, notifier)
    notifier_thread.start()

    # we want the notifier running while we do the initial directory scan, so
    # we don't miss any files. But this leaves us open to duplicates()
    _initial_directory_scan(args.watch_path, file_name_queue)

    # so this eliminates duplicates, but leaks a little memory
    # if that's a problem, we could clean it out occasionally 
    file_name_set = set()

    log.info("main loop starts")
    return_code = 0
    while not halt_event.is_set():

        try:
            file_name = file_name_queue.get(block=True, timeout=1.0)
        except queue.Empty:
            continue

        log.debug("found file_name '{0}'".format(file_name))

        if args.pattern is not None:
            if not fnmatch.fnmatch(file_name, args.pattern):
                log.info("ignoring '{0}': does not match '{1}'".format(
                         file_name, args.pattern))
                continue

        if file_name in file_name_set:
            log.info("ignoring duplicate file name '{0}'".format(file_name))
            continue
        file_name_set.add(file_name)

        try:
            archive_file(args, nimbusio_identity, file_name)
            file_path = os.path.join(args.watch_path, file_name)
            log.info("deleting {0}".format(file_path))
            os.unlink(file_path)
        except Exception as instance:
            log.exception(instance)
            return_code = 1
            halt_event.set()

    log.info("main loop ends")
    notifier_thread.join(timeout=5.0)
#    assert not notifier_thread.is_alive
    notifier.stop()

    log.info("program terminates return_code = {0}".format(return_code))
    return return_code
コード例 #2
0
def main():
    """
    main entry point for the program
    The return value will be the returncode for the program
    """
    initialize_stderr_logging()
    log = logging.getLogger("main")

    try:
        args = parse_commandline()
    except CommandlineError:
        instance = sys.exc_info()[1]
        log.error("invalid commandline {0}".format(instance))
        return 1

    if not "(?P<key>" in args.key_regex:
        log.error("invalid key regex {0}".format(args.key_regex))
        return 1

    try:
        key_regex = re.compile(args.key_regex)
    except Exception:
        instance = sys.exc_info()[1]
        log.error("Unable to compile key_regex {0} {1}".format(args.key_regex,
                                                               instance))
        return 1        

    initialize_file_logging(args.log_path, args.verbose)

    log.info("key regex pattern = '{0}'".format(key_regex.pattern))

    halt_event = Event()
    set_signal_handler(halt_event)

    file_name_queue = queue.Queue()    

    try:
        notifier = create_notifier(args.watch_path, file_name_queue)
    except InotifyError as instance:
        log.error("Unable to initialize inotify: {0}".format(instance))
        return 1

    try:
        redis = create_redis_connection()
    except Exception:
        log.exception("Unable to connect to redis")
        return 1

    # clear REDIS of all keys under our namespace, so that old sets that 
    # don't have any files anymore don't stay around
    existing_keys = redis.keys("_".join([args.redis_prefix, "*"]))
    if len(existing_keys) > 0: 
        log.debug("deleting {0} existing REDIS keys".format(
                  len(existing_keys), ))
        redis.delete(*existing_keys)

    # set our up-to-date time as far back as we can: we aren't up to date yet
    up_to_date_timestamp_key = "_".join([args.redis_prefix, 
                                         _up_to_date_timestamp])
    log.debug("setting {0} to {1}".format(up_to_date_timestamp_key, 0))
    redis.set(up_to_date_timestamp_key, "0")

    notifier_thread = create_notifier_thread(halt_event, 
                                             notifier, 
                                             file_name_queue)

    # we want the notifier running while we do the initial directory scan, so
    # we don't miss any files. 
    notifier_thread.start()

    _initial_directory_scan(args.watch_path, file_name_queue)

    log.info("main loop starts")
    directory_scan_up_to_date = False
    up_to_date = False
    return_code = 0
    while not halt_event.is_set():

        try:
            file_name, event_name = file_name_queue.get(block=True, timeout=1.0)
        except queue.Empty:
            continue
        except KeyboardInterrupt:
            log.warn("KeyboardInterrupt: halting")
            halt_event.set()
            break

        if event_name == directory_scan_finished:
            log.debug("setting directory_scan_up_to_date")
            directory_scan_up_to_date = True
            continue

        if event_name == inotify_idle:
            if not up_to_date and directory_scan_up_to_date:
                current_time = int(time.time())
                log.debug("setting {0} to {1}".format(up_to_date_timestamp_key, 
                                                      current_time))
                redis.set(up_to_date_timestamp_key, str(current_time))
                up_to_date = True
            continue

        match_object = key_regex.match(file_name)
        if match_object is None:
            log.debug("unmatched file name '{0}'".format(file_name))
            continue
        key = match_object.group("key")

        log.debug("found file_name '{0}' key {1} event {2}".format(file_name, 
                                                                   key,
                                                                   event_name))
        redis_key = "_".join([args.redis_prefix, key, ])
        try:
            _dispatch_table[event_name](redis, redis_key, file_name)
        except Exception:
            log.exception("{0} {1}".format(file_name, event_name))
            return_code = 1
            halt_event.set()

    log.info("main loop ends")
    redis.set(up_to_date_timestamp_key, "0")
    notifier_thread.join(timeout=5.0)
#    assert not notifier_thread.is_alive
    notifier.stop()

    log.info("program terminates return_code = {0}".format(return_code))
    return return_code