#! /usr/bin/python

# coding: utf-8

"""

Script to process tweets via stdin, expand URLs, add counts of urls/hashtags/mentions, and list of hashtags/mentions.

"""

import sys

from datetime import datetime, timedelta

import time

from email.utils import parsedate_tz

from pymongo import Connection

import simplejson

import re

import hashlib

import string

from collections import defaultdict

import MySQLdb

from connection import dbConnection

import pika

# Connect to mongo

db_names = ['gnip_boston','gnip_processing_errors']

db = dbConnection()

db.create_mongo_connections(mongo_options=[db_name,error_log])

# List of punct to remove from string for track keyword matching

punct = re.escape('!"$%&\'()*+,-./:;<=>?@[\\]^`{|}~')

# keep track of line numbers

#line_num = 0

tweet_total = 0

# Stuff 50k tweets into this sucker

tweets_list = []

connection = pika.BlockingConnection(pika.ConnectionParameters(

host='localhost'))

channel = connection.channel()

channel.queue_declare(queue='mongo_insert',durable=True)

print ' [*] Waiting for messages. To exit press CTRL+C'

def to_datetime(datestring):

time_tuple = parsedate_tz(datestring.strip())

dt = datetime(*time_tuple[:6])

return dt

def batch_insert(ch, method, properties, body):

pass

def single_insert(ch, method, properties, body):

pass

def single_update(ch, method, properties, body):

pass

channel.basic_qos(prefetch_count=1)

channel.basic_consume(single_update,

queue='mongo_insert',

no_ack=True)

channel.start_consuming()