def extract_grams():

    client = MongoClient()
    db = client.news_tfidf

    select_strs = [
        ("se" ,"select ID, story from swift_excel_articles;"),
        ("ne", "select ID, body from nsc_excel_articles;"),
        ("sw", '''select sw.ID, sw.story 
                from swift_articles sw left join swift_excel_articles se on sw.ID = se.swiftID 
                where se.swiftID is null and in_sample=1;'''),
        ("ns", '''select ns.ID, concat(ns.lead_parag, ' ', ns.body) 
                from nsc_articles ns left join nsc_excel_articles ne on ns.ID = ne.nscID 
                where ne.nscID is null and in_sample=1;''')
    ]

    token_ids = get_token_dict(db)
    bi_gram_freqs = defaultdict(int)
    
    t = time.time()
    st = time.time()
    row_count = 0
    
    for source_id, query in select_strs:

        print query

        cnx = get_db_context()
        select_cur = cnx.cursor()
        select_cur.execute(query)
        
        for article_id, article, in select_cur:

            row_count += 1
            if row_count % 10000 == 0:
                print 'processed', row_count, 'rows in', (time.time() - t) / 60, 'minutes'
                t = time.time()

            if row_count % 10000 == 0:
                delete_low_counts(bi_gram_freqs)
                
            if type(article) in (str, unicode) and len(article) > 0:
            
                tokenizer = Tokenizer(article)
                bi_gram_gen = tokenizer.gen_n_grams(n=2)
                counted = set()
                
                for bi_gram in bi_gram_gen:
                    
                    bg_id_tup = (token_ids[bi_gram[0]], token_ids[bi_gram[1]])
                    
                    if bg_id_tup not in counted:
                        bi_gram_freqs[bg_id_tup] += 1
                        counted.add(bg_id_tup)

            if len(bi_gram_freqs) > 1000000:
                pickle_bi_gram_freqs(bi_gram_freqs, source_id)
                bi_gram_freqs = defaultdict(int)                
                        
        pickle_bi_gram_freqs(bi_gram_freqs, source_id)
        bi_gram_freqs = defaultdict(int)
        select_cur.close()
        cnx.close()      

    print 'parsing time: ', (time.time() - st) / 60, 'minutes'
    return bi_gram_freqs
def extract_grams():

    client = MongoClient()
    db = client.news_tfidf

    select_strs = [
        # ("se" ,"select ID, story from swift_excel_articles;"),
        # ("ne", "select ID, body from nsc_excel_articles;"),
        ("sw", "select sw.ID, sw.story from swift_articles sw left join swift_excel_articles se on sw.ID = se.swiftID where se.swiftID is null;"),
        ("ns", "select ns.ID, concat(ns.lead_parag, ' ', ns.body) from nsc_articles ns left join nsc_excel_articles ne on ns.ID = ne.nscID where ne.nscID  is null;")
    ]
    
    id_gen = gen_ids()

    token_ids = get_token_dict(db)

    # token_freqs = dict()
    bi_gram_freqs = dict()
    
    doc_counts = list()
    
    t = time.time()

    row_count = 0
    cnx =  get_db_context()
    
    def incr_token_dict(t_dict, token):
        if token in t_dict:
            t_dict[token]['c'] += 1
        else:
            t_dict[token] = {'i': next(id_gen), 't': token, 'c': 1}

    
    def incr_bi_gram_dict(bg_dict, bg, source_id=None):
        if bg in bg_dict:
            bg_dict[bg]['c'] += 1
        else:
            if source_id:
                bg_dict[bg] = {'g1': bg[0], 'g2': bg[1], 'c': 1, 'source_id': source_id}
            else:
                bg_dict[bg] = {'g1': bg[0], 'g2': bg[1], 'c': 1}

    
    for source_id, query in select_strs:

        print query

        cnx =  get_db_context()
        select_cur = cnx.cursor()
        select_cur.execute(query)
        
        for article_id, article, in select_cur:

            row_count += 1
            if row_count % 5000 == 0:
                print 'processed', row_count, 'rows in', (time.time() - t) / 60, 'minutes'
                t = time.time()

            if type(article) in (str, unicode) and len(article) > 0:
            
                tokenizer = Tokenizer(article)
                # token_gen = tokenizer.gen_tokens()
                bi_gram_gen = tokenizer.gen_n_grams(n=2)
                
                # token_doc_freqs = dict()
                bi_gram_doc_freqs = dict()
                
                # for token in token_gen:
                    
                #     incr_token_dict(token_freqs, token)
                #     incr_token_dict(token_doc_freqs, token)

                for bi_gram in bi_gram_gen:
                    
                    gram1_id = token_ids[bi_gram[0]]
                    gram2_id = token_ids[bi_gram[1]]

                    incr_bi_gram_dict(bi_gram_doc_freqs, (gram1_id, gram2_id))
                    incr_bi_gram_dict(bi_gram_freqs, (gram1_id, gram2_id), source_id)
                    
                doc_counts.append(
                {
                        'sql_id': article_id,
                        'sql_tbl_id': source_id,
                        # 't_counts': token_doc_freqs.values(),
                        'bg_counts': bi_gram_doc_freqs.values()
                    }
                )
                
                if len(doc_counts) > 1000:
                    db_t = time.time()
                    print 'updating db...'
                    db.doc_freq.insert_many(doc_counts)
                    doc_counts = list()
                    
#                         update_doc_bg_counts(db, doc_counts)
#                         doc_counts = list()        

                    insert_bi_gram_freqs(db, source_id, bi_gram_freqs)
                    bi_gram_freqs = dict()

                    print 'db done in', (time.time() - db_t)/60, 'minutes'
                    
        # refresh_token_freqs(db, token_freqs)
        insert_bi_gram_freqs(db, source_id, bi_gram_freqs)
        bi_gram_freqs = dict()

        select_cur.close()
        cnx.close()      

    print 'parsing time: ', time.time() - t
    t = time.time()
    print 'updating...'
    
    db.doc_freq.insert_many(doc_counts)
#         update_doc_bg_counts(db, doc_counts)
    insert_bi_gram_freqs(db, source_id, bi_gram_freqs)