def process_msg(file_path_tuple):
    '''Processes a single email file 
    
    Arguments: 
        file_path_tuple - a tuple of (idx, root, file_name) 
    '''
    
    (idx, root, file_name)  = file_path_tuple
    logging.info('[#%d] file: %s' % (idx, os.path.join(root, file_name)) )
    
    
    _, _, _, _, body_text = parse_plain_text_email(os.path.join(root, file_name))
    tokens = punkt_word_tokenizer(body_text.lower())
    
    return tokens
def index_plain_text_emails(data_folder, 
                            path_index_file, store_dir, 
                            lemmatize = False, stem = False, 
                            nonascii = True):
    '''
    Indexes all the plain text emails in the input directory 
    and stores the index in the store_dir  
    
    Arguments: 
        data_folder - input directory absolute path 
        path_index_file - file paths index file 
        store_dir - index store directory absolute path 
    Returns: 
        None 

    '''
    
    if not os.path.exists(store_dir): 
        os.mkdir(store_dir)
    
    
    if os.path.exists(path_index_file): 
        logging.info('Loading file paths index...')
        file_tuples = load_file_paths_index(path_index_file)
        logging.info('%d files found in the file paths index.' % len(file_tuples))
    else: 
        logging.info('Loading files in the data folder %s...' % data_folder)
        file_tuples = get_file_paths_index(data_folder)
    
        logging.info('%d email documents found.' % len(file_tuples))
    
        store_file_paths_index(path_index_file, file_tuples)
        logging.info('File paths index is stored into %s' % path_index_file)
    
    logging.info('Lucene: Stem = %s, Lemmatize = %s, Number of documents = %d' % (stem, lemmatize, len(file_tuples)))
        
    store = SimpleFSDirectory(File(store_dir))
    writer = IndexWriter(store, STD_ANALYZER, True, IndexWriter.MaxFieldLength.LIMITED)
    
    print 'Lucene:', len(file_tuples), 'files found in %s.' % data_folder
    print 'Lucene: Stem =', stem, 'Lemmatize =', lemmatize, 'Allow non-ASCII =', nonascii  
    
    for ft in file_tuples: 
        idx, root, file_name = ft
        file_path = os.path.join(root, file_name)
        logging.info("[%d] file: %s - adding to Lucene index.", idx, file_name)
        # parses the emails in plain text format 
        receiver, sender, cc, subject, message_text, bcc, date, email_text = parse_plain_text_email(file_path, 
                                                                                                    tokenize = True, 
                                                                                                    lemmatize = lemmatize, 
                                                                                                    stem = stem, 
                                                                                                    nonascii = nonascii)

        doc = Document()
        doc.add(Field(MetadataType.FILE_ID, str(idx), Field.Store.YES, Field.Index.NOT_ANALYZED))
        doc.add(Field(MetadataType.FILE_NAME, file_name, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.YES))
        doc.add(Field(MetadataType.FILE_PATH, file_path, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS))
        doc.add(Field(MetadataType.EMAIL_RECEIVER, receiver, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS))
        doc.add(Field(MetadataType.EMAIL_SENDER, sender, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS))
        doc.add(Field(MetadataType.EMAIL_CC, cc, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS))
        doc.add(Field(MetadataType.EMAIL_SUBJECT, subject, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS))
        #Subodh-Rahul - Added BCC field in indexing.
        doc.add(Field(MetadataType.EMAIL_BCC, bcc, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS))
        #Subodh-Rahul - Added Email-Date field in indexing
        doc.add(Field(MetadataType.EMAIL_DATE, date, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS))
        
        if len(message_text) > 0:
            doc.add(Field(MetadataType.EMAIL_BODY, message_text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES))
        else:
            logging.error("[%d] file: %s - body text is empty.", idx, file_name)
            
        # Adds all documents fields as a separate index so that we can search through them 
        doc.add(Field(MetadataType.ALL, email_text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES))

        writer.addDocument(doc)
        logging.info("[%d] file: %s - added to Lucene index.", idx, file_name)


    writer.commit()
    writer.close()

    logging.info('Lucene: All files are indexed.')
def index_plain_text_emails(data_folder, path_index_file, store_dir, 
                            lemmatize=False, stem=False, nonascii=True):
    '''
    Indexes all the plain text emails in the input directory and stores the 
    index in the store_dir  
    
    Arguments: 
        data_folder - input directory (absolute path)
        path_index_file - file paths index file 
        store_dir - index store directory absolute path 
        lemmatize - lemmatize tokens based on the NLTK WordNet lemmatizer 
        stem - stem tokens 
        nonascii - allow non-ASCII characters  
         
        
    Returns: 
        None 

    '''
    
    if not os.path.exists(store_dir): os.mkdir(store_dir)
    
    if os.path.exists(path_index_file): 
        logging.info('Loading file paths index...')
        file_tuples = load_file_paths_index(path_index_file)
        logging.info('%d files found in the file paths index.' % len(file_tuples))
    else: 
        logging.info('Loading files in the data folder %s...' % data_folder)
        file_tuples = get_file_paths_index(data_folder)
        logging.info('%d email documents found.' % len(file_tuples))    
        store_file_paths_index(path_index_file, file_tuples)
        logging.info('Index file path: %s' % path_index_file)

    schema = Schema(file_id=NUMERIC(int, stored=True), 
                    file_name=ID(stored=True), 
                    file_path=ID(stored=True), 
                    email_reciever=TEXT(stored=True), 
                    email_sender=TEXT(stored=True), 
                    email_cc=TEXT(stored=True), 
                    email_subject=TEXT(stored=True), 
                    email_bcc=TEXT(stored=True),
                    date=ID(stored=True),
                    email_body=TEXT(stored=True),
                    all=TEXT(stored=True))
    ix = create_in(store_dir, schema)
    writer = ix.writer()
    logging.info('Stem = %s, Lemmatize = %s, D = %d, non-ASCII = %s' 
                 % (stem, lemmatize, len(file_tuples), nonascii))
    
    for ft in file_tuples: 
        idx, root, file_name, file_type = ft
        file_path = os.path.join(root, file_name)
        logging.info("[%d] creating index for %s...", idx, file_name)
        
        
        ret = parse_plain_text_email(file_path, lemmatize=lemmatize, stem=stem, 
                                     nonascii=nonascii, file_type=file_type)

        (receiver, sender, cc, subject, body_text, bcc, date, doc_text) = ret
        
        writer.add_document(file_id = idx, 
                            file_name = unicode(file_name), 
                            file_path = unicode(file_path), 
                            email_reciever = unicode(receiver), 
                            email_sender = unicode(sender), 
                            email_cc = unicode(cc),
                            email_subject = unicode(subject), 
                            email_bcc = unicode(bcc), 
                            date = unicode(date), 
                            email_body = unicode(body_text), 
                            all = unicode(doc_text))
 
    writer.commit()
    logging.info('All files are indexed.')