#streamG =1

#def setStream(self,streamx):

# global streamG

# streamG = streamx

def __init__(self, output_file):

StreamListener.__init__(self)

self.output_file = os.path.join("/scratch/data/", output_file)

self.tmpData = ""

self.tweet_count = 0

def __exit__(self):

self.f.close()

def on_data(self, data):

try:

if data == "":

return

tmp = json.loads(data)

if tmp == None:

return

if "created_at" not in tmp:

return

coords = [0,0]

created_at = tmp["created_at"]

user = tmp["user"]

screen_name = user["screen_name"]

if "coordinates" in tmp:

if tmp["coordinates"] != None:

if "coordinates" in tmp["coordinates"]:

coords = tmp["coordinates"]["coordinates"]

ids = tmp["id"]

text = tmp["text"]

## Get rid of UTF non-sense....

text = text.encode('ascii', 'ignore').decode('ascii')

text.replace("|","[replaced bar]")

text.replace("\n", " ")

text.replace("\r", " ")

#p = str(idource venv/bin/activate) + "|" + str(created_at) + "|" + str(coords[0]) + "|" + str(coords[1]) + "|" + str(text) + "\n"

p = "<"+str(screen_name)+"> | "+str(created_at)+" | "+str(text) + " <END OF TWEET>\n"

#print str(self.tweet_count)+":::"+p

#if len(p) > 80:

# print p[:79]

#else:

# print p

self.tmpData = self.tmpData + p

#f.write(p)

self.tweet_count += 1

if self.tweet_count % 1000 == 0:

print str(self.tweet_count)

#streamG.dissconnect()

output_file = "%s.%s.txt" % (self.output_file, datetime.date.today().isoformat())

with open(output_file, 'a') as f:

f.write(self.tmpData)

print "writing and flushing data"

self.tmpData = ""

print str(datetime.datetime.now()) + " --- TWITTER_COUNT = " + str(self.tweet_count)

except Exception as e:

exc_type, exc_obj, exc_tb = sys.exc_info()

print "ERROR - " + str(e)

print exc_tb.tb_lineno

def on_error(self, status):

pid = current_process().pid

round = 1

while True:

try:

l = StdOutListener(args['output_file'])

auth = OAuthHandler(args['consumer_key'], args['consumer_secret'])

auth.set_access_token(args['token_key'], args['token_secret'])

print "%i's %i authentication success. (420 means rate limited)" % (pid, round)

stream = Stream(auth, l)

#l.setStream(stream)

#stream.filter(track=['#programming','#google','#android'])

stream.sample()

#locations=[-129.19,23.96,-64.68,50.68]

except KeyboardInterrupt:

print "KeyboardInterrupt sensed, %i exiting..." % pid

stream.disconnect()

return True

except Exception as e:

print e

finally:

stream.disconnect()