示例#1
0
def start_consuming_parsed_articles():
    try:
        print "  Article Analyzer Started..."
        queue.consume_parsed_articles(consume_channel,
                parsed_articles_consumer)
    finally:
        queue.close_connection(connection)
示例#2
0
def start_consuming_analyzed_articles():
    try:
        print "  Metric Writer Started..."
        queue.consume_analyzed_articles(consume_channel,
                analyzed_articles_consumer)
    finally:
        queue.close_connection(connection)
示例#3
0
def start_consuming_scanned_files():
    """ Start consuming messages.  The message body is an article's
        filename.

    """
    try:
        print "  Article Parser Started..."
        queue.consume_scanned_files(consume_channel, scanned_files_consumer)
    finally:
        queue.close_connection(connection)
示例#4
0
def scan_directory(directory):
    """ Recursively scan the given directory file new articles

    """    
    print "  Starting scan..."
    print "  Directory: %s" % directory

    files_queued = 0

    # Initialize a connection and channel to RabbitMQ
    connection, channel = queue.init_connection()
    queue.init_scanned_files(channel, True)
    for root, dirs, files in os.walk(directory):

        # Walk dirs and files in alphabetical order
        dirs.sort()
        files.sort()

        for file in files:
            filename = os.path.join(root, file)

            # Only queue files that match the pattern
            if re.match(scan_match_pattern, filename):
                add_scanned_file_to_queue(channel, filename)
                files_queued += 1

                # Update status...
                if files_queued % updt_freq == 0:
                    print "    * Files Queued: %d..." % files_queued            

        # Ignore any svn dirs
        if '.svn' in dirs:
            dirs.remove('.svn')

    queue.close_connection(connection)

    print ""
    print "  ... scan complete"
    print "    Files Queued: %d" % files_queued
    print ""