#!/usr/bin/env python

# encoding: utf-8

"""

generic_crawler.py

Created by Brian Eoff on 2011-02-21.

Copyright (c) 2011 __MyCompanyName__. All rights reserved.

"""

import sys

import os

import codecs

import time

import json

import logging

from datetime import datetime

from basestream import Stream

from ConfigParser import SafeConfigParser

from stream_functions import OutputFileHandler

from stream_functions import on_receive, date_to_fname_string

if __name__ == '__main__':

"""

This type of crawler will be a generic. It does not need to interact

with Twitter beyond being connected to the stream.

"""

config_file = sys.argv[1]

crawler_config = sys.argv[2]

parser = SafeConfigParser()

parser.read(config_file)

dataDirectory = parser.get(crawler_config, 'directory')

if not (os.path.exists(dataDirectory)):

os.mkdir(dataDirectory)

time_per_file = parser.getint(crawler_config, 'time_length')

username = parser.get(crawler_config, 'username')

password = parser.get(crawler_config, 'password')

crawler_type = parser.get(crawler_config, 'type')

crawler_id = parser.get(crawler_config, 'id')

logging.basicConfig(filename=crawler_id + '.log', level=logging.ERROR)

stream_url = parser.get(crawler_config, 'stream_url')

trackTerms = []

if parser.has_option(crawler_config, 'terms_file'):

terms_file = open(parser.get(crawler_config, 'terms_file'), 'r')

json_string = terms_file.read()

trackTerms = json.loads(json_string)['terms']

terms_file.close()

while True:

currentDate = datetime.now()

dateStr = date_to_fname_string(currentDate, crawler_config.lower())

output = codecs.open(dataDirectory + '/' + dateStr + '.txt',

encoding='utf-8', mode='w+')

ofh = OutputFileHandler()

ofh.set(output)

try:

stream = Stream(stream_url, username, password, on_receive,

initial_params=trackTerms, filter_type=crawler_type)

print 'Stream Start'

stream.start()

time.sleep(time_per_file)

stream.stop()

print 'Stream Stop'

ofh.close()

except Exception, err:

logging.error(str(datetime.now()) + ':' + str(err))

try:

stream.stop()

ofh.close()

except Exception, err:

logging.error(str(datetime.now()) + ':' + str(err))

logging.warning('Unable to close stream/file.')