def start_consuming_parsed_articles(): try: print " Article Analyzer Started..." queue.consume_parsed_articles(consume_channel, parsed_articles_consumer) finally: queue.close_connection(connection)
def start_consuming_analyzed_articles(): try: print " Metric Writer Started..." queue.consume_analyzed_articles(consume_channel, analyzed_articles_consumer) finally: queue.close_connection(connection)
def start_consuming_scanned_files(): """ Start consuming messages. The message body is an article's filename. """ try: print " Article Parser Started..." queue.consume_scanned_files(consume_channel, scanned_files_consumer) finally: queue.close_connection(connection)
def scan_directory(directory): """ Recursively scan the given directory file new articles """ print " Starting scan..." print " Directory: %s" % directory files_queued = 0 # Initialize a connection and channel to RabbitMQ connection, channel = queue.init_connection() queue.init_scanned_files(channel, True) for root, dirs, files in os.walk(directory): # Walk dirs and files in alphabetical order dirs.sort() files.sort() for file in files: filename = os.path.join(root, file) # Only queue files that match the pattern if re.match(scan_match_pattern, filename): add_scanned_file_to_queue(channel, filename) files_queued += 1 # Update status... if files_queued % updt_freq == 0: print " * Files Queued: %d..." % files_queued # Ignore any svn dirs if '.svn' in dirs: dirs.remove('.svn') queue.close_connection(connection) print "" print " ... scan complete" print " Files Queued: %d" % files_queued print ""