def streaming_rss_reader(input_dict, widget, stream=None):
import feedparser
from streams.models import StreamWidgetData
feed = feedparser.parse(input_dict['url'])
output_dict = {}
if stream is None:
output_dict['url'] = feed['items'][0]['link']
else:
try:
swd = StreamWidgetData.objects.get(stream=stream, widget=widget)
data = swd.value
except:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = []
swd.value = data
swd.save()
feed_length = len(feed['items'])
feed['items'].reverse()
for item in feed['items']:
if item['link'] not in data:
data.append(item['link'])
swd.value = data
swd.save()
output_dict['url'] = item['link']
break
else:
from streams.models import HaltStream
raise HaltStream("Halting stream.")
return output_dict
def streaming_rss_reader(input_dict,widget,stream=None):
import feedparser
from streams.models import StreamWidgetData
feed = feedparser.parse(input_dict['url'])
output_dict = {}
if stream is None:
output_dict['url'] = feed['items'][0]['link']
else:
try:
swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
data = swd.value
except:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = []
swd.value = data
swd.save()
feed_length = len(feed['items'])
feed['items'].reverse()
for item in feed['items']:
if item['link'] not in data:
data.append(item['link'])
swd.value = data
swd.save()
output_dict['url'] = item['link']
break
else:
from streams.models import HaltStream
raise HaltStream("Halting stream.")
return output_dict
def streaming_sentiment_graph(input_dict, widget, stream=None):
from streams.models import StreamWidgetData
if stream is None:
return {}
else:
new_tweets = []
for tweet in input_dict['ltw']:
new_tweets.append(
StreamWidgetData(stream=stream, widget=widget, value=tweet))
StreamWidgetData.objects.bulk_create(new_tweets)
return {}
def streaming_triplet_graph(input_dict, widget, stream=None):
from streams.models import StreamWidgetData
if stream is None:
return {}
else:
try:
swd = StreamWidgetData.objects.get(stream=stream, widget=widget)
swd.value = input_dict['triplets']
swd.save()
except:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = input_dict['triplets']
swd.value = data
swd.save()
return {}
def streaming_triplet_graph(input_dict,widget,stream=None):
from streams.models import StreamWidgetData
if stream is None:
return {}
else:
try:
swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
swd.value = input_dict['triplets']
swd.save()
except:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = input_dict['triplets']
swd.value = data
swd.save()
return {}
def streaming_simulate_stream_from_csv(input_dict,widget,stream=None):
from streams.models import StreamWidgetData
import datetime
import csv
csvfile = safeOpen(input_dict['csv'])
csvreader = csv.reader(csvfile,delimiter=";",quotechar='"')
rows = []
ltw = []
i=0
counter = 0
started = False
last_id = "not-started-yet"
if not stream is None:
try:
swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
last_id = swd.value
except:
started = True
else:
started = True
for row in csvreader:
rows.append(row)
if i!=0:
rows[i][1] = datetime.datetime.strptime(rows[i][1],"%m/%d/%Y %I:%M:%S %p")
tweet = {}
tweet['id'] = rows[i][0]
tweet['created_at'] = rows[i][1]
tweet['text'] = rows[i][3].encode('utf-8')
tweet['user'] = rows[i][5].encode('utf-8')
tweet['lang'] = rows[i][11]
if started:
counter = counter + 1
ltw.append(tweet)
if counter == 50 and started:
started = False
if not stream is None:
try:
swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
swd.value = tweet['id']
swd.save()
except:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = tweet['id']
swd.value = data
swd.save()
if tweet['id']==last_id:
started = True
i = i + 1
if counter < 51 and not stream is None and started == True:
try:
swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
swd.value = "done"
swd.save()
except:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = "done"
swd.value = data
swd.save()
output_dict = {}
#print ltw
#print len(ltw)
output_dict['ltw']=ltw
return output_dict
def streaming_sliding_window(input_dict,widget,stream=None):
from streams.models import StreamWidgetData
output_dict = {}
if stream is None:
output_dict['list']=input_dict['list'][:int(input_dict['size'])]
else:
try:
swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
data = swd.value
except:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = []
swd.value = data
swd.save()
size = int(input_dict['size'])
if len(input_dict['list'])>=size:
output_dict['list']=input_dict['list'][:size]
swd.value=output_dict['list']
swd.save()
else:
current_window = input_dict['list'] + swd.value
swd.value = current_window[:size]
output_dict['list']=current_window[:size]
swd.save()
return output_dict
def streaming_twitter(input_dict,widget,stream=None):
import tweepy
from streams.models import StreamWidgetData
from streams.models import HaltStream
if input_dict['cfauth']=="true":
consumer_key="zmK41mqxU3ZNJTFQpYwTdg"
consumer_secret="9StnKNAe20ebDOREQjsVjAjBEiz5R9feZJTGUYWqLo"
access_token="45210078-VydgdJMwhWYjZRvlNbrKj6jfqicUIsdMnRbnaPElL"
access_token_secret="uLvIN3MMxFSxdK4M8P5RYojjUkbc2reqNydYtpT7Ks"
else:
consumer_key = input_dict['ck']
consumer_secret = input_dict['cs']
access_token = input_dict['at']
access_token_secret = input_dict['as']
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
query = input_dict['query']
rate_limit_status = api.rate_limit_status()
if rate_limit_status['resources']['search']['/search/tweets']['remaining']>0:
if stream is None:
try:
ltw = api.new_search(q=input_dict['query'],geocode=input_dict['geocode'],count=100)
except Exception as e:
raise HaltStream("The Twitter API returned an error: "+str(e))
else:
try:
swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
data = swd.value
except Exception as e:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = {}
swd.value = data
swd.save()
if data.has_key(query):
since_id = data[query]
try:
ltw = api.new_search(q=input_dict['query'],geocode=input_dict['geocode'],count=100,since_id=since_id)
except Exception as e:
raise HaltStream("The Twitter API returned an error: "+str(e))
else:
try:
ltw = api.new_search(q=input_dict['query'],geocode=input_dict['geocode'],count=100)
except Exception as e:
raise HaltStream("The Twitter API returned an error: "+str(e))
if len(ltw)>0:
data[query]=ltw[0].id
swd.value = data
swd.save()
else:
import datetime
import time
current_time = int(time.mktime(datetime.datetime.now().timetuple()))
remaining = rate_limit_status['resources']['search']['/search/tweets']['reset']-current_time
if input_dict['cfauth']=="true":
raise HaltStream("The twitter API limit has been reached. Try again in "+str(remaining)+" seconds. Try using your own credentials.")
else:
raise HaltStream("The twitter API limit has been reached. Try again in "+str(remaining)+" seconds.")
output_dict = {}
tweets = []
for tw in ltw:
tweet = {}
tweet['id'] = tw.id
tweet['created_at'] = tw.created_at
tweet['text'] = unicode(tw.text).encode("utf-8")
try:
tweet['user'] = tw.user['screen_name']
except:
tweet['user'] = ""
tweet['lang'] = tw.lang
tweets.append(tweet)
if len(tweets)>0 or stream is None:
output_dict['ltw']=tweets
return output_dict
else:
raise HaltStream("No new results, halting stream.")
def streaming_simulate_stream_from_csv(input_dict, widget, stream=None):
from streams.models import StreamWidgetData
import datetime
import csv
csvfile = safeOpen(input_dict['csv'])
csvreader = csv.reader(csvfile, delimiter=";", quotechar='"')
rows = []
ltw = []
i = 0
counter = 0
started = False
last_id = "not-started-yet"
if not stream is None:
try:
swd = StreamWidgetData.objects.get(stream=stream, widget=widget)
last_id = swd.value
except:
started = True
else:
started = True
for row in csvreader:
rows.append(row)
if i != 0:
rows[i][1] = datetime.datetime.strptime(rows[i][1],
"%m/%d/%Y %I:%M:%S %p")
tweet = {}
tweet['id'] = rows[i][0]
tweet['created_at'] = rows[i][1]
tweet['text'] = rows[i][3].encode('utf-8')
tweet['user'] = rows[i][5].encode('utf-8')
tweet['lang'] = rows[i][11]
if started:
counter = counter + 1
ltw.append(tweet)
if counter == 50 and started:
started = False
if not stream is None:
try:
swd = StreamWidgetData.objects.get(stream=stream,
widget=widget)
swd.value = tweet['id']
swd.save()
except:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = tweet['id']
swd.value = data
swd.save()
if tweet['id'] == last_id:
started = True
i = i + 1
if counter < 51 and not stream is None and started == True:
try:
swd = StreamWidgetData.objects.get(stream=stream, widget=widget)
swd.value = "done"
swd.save()
except:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = "done"
swd.value = data
swd.save()
output_dict = {}
#print ltw
#print len(ltw)
output_dict['ltw'] = ltw
return output_dict
def streaming_sliding_window(input_dict, widget, stream=None):
from streams.models import StreamWidgetData
output_dict = {}
if stream is None:
output_dict['list'] = input_dict['list'][:int(input_dict['size'])]
else:
try:
swd = StreamWidgetData.objects.get(stream=stream, widget=widget)
data = swd.value
except:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = []
swd.value = data
swd.save()
size = int(input_dict['size'])
if len(input_dict['list']) >= size:
output_dict['list'] = input_dict['list'][:size]
swd.value = output_dict['list']
swd.save()
else:
current_window = input_dict['list'] + swd.value
swd.value = current_window[:size]
output_dict['list'] = current_window[:size]
swd.save()
return output_dict
def streaming_twitter(input_dict, widget, stream=None):
import tweepy
from streams.models import StreamWidgetData
from streams.models import HaltStream
if input_dict['cfauth'] == "true":
consumer_key = "zmK41mqxU3ZNJTFQpYwTdg"
consumer_secret = "9StnKNAe20ebDOREQjsVjAjBEiz5R9feZJTGUYWqLo"
access_token = "45210078-VydgdJMwhWYjZRvlNbrKj6jfqicUIsdMnRbnaPElL"
access_token_secret = "uLvIN3MMxFSxdK4M8P5RYojjUkbc2reqNydYtpT7Ks"
else:
consumer_key = input_dict['ck']
consumer_secret = input_dict['cs']
access_token = input_dict['at']
access_token_secret = input_dict['as']
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
query = input_dict['query']
rate_limit_status = api.rate_limit_status()
if rate_limit_status['resources']['search']['/search/tweets'][
'remaining'] > 0:
if stream is None:
try:
ltw = api.new_search(q=input_dict['query'],
geocode=input_dict['geocode'],
count=100)
except Exception as e:
raise HaltStream("The Twitter API returned an error: " +
str(e))
else:
try:
swd = StreamWidgetData.objects.get(stream=stream,
widget=widget)
data = swd.value
except Exception as e:
swd = StreamWidgetData()
swd.stream = stream
swd.widget = widget
data = {}
swd.value = data
swd.save()
if data.has_key(query):
since_id = data[query]
try:
ltw = api.new_search(q=input_dict['query'],
geocode=input_dict['geocode'],
count=100,
since_id=since_id)
except Exception as e:
raise HaltStream("The Twitter API returned an error: " +
str(e))
else:
try:
ltw = api.new_search(q=input_dict['query'],
geocode=input_dict['geocode'],
count=100)
except Exception as e:
raise HaltStream("The Twitter API returned an error: " +
str(e))
if len(ltw) > 0:
data[query] = ltw[0].id
swd.value = data
swd.save()
else:
import datetime
import time
current_time = int(time.mktime(datetime.datetime.now().timetuple()))
remaining = rate_limit_status['resources']['search']['/search/tweets'][
'reset'] - current_time
if input_dict['cfauth'] == "true":
raise HaltStream(
"The twitter API limit has been reached. Try again in " +
str(remaining) + " seconds. Try using your own credentials.")
else:
raise HaltStream(
"The twitter API limit has been reached. Try again in " +
str(remaining) + " seconds.")
output_dict = {}
tweets = []
for tw in ltw:
tweet = {}
tweet['id'] = tw.id
tweet['created_at'] = tw.created_at
tweet['text'] = unicode(tw.text).encode("utf-8")
try:
tweet['user'] = tw.user['screen_name']
except:
tweet['user'] = ""
tweet['lang'] = tw.lang
tweets.append(tweet)
if len(tweets) > 0 or stream is None:
output_dict['ltw'] = tweets
return output_dict
else:
raise HaltStream("No new results, halting stream.")
