def streaming_rss_reader(input_dict, widget, stream=None): import feedparser from streams.models import StreamWidgetData feed = feedparser.parse(input_dict['url']) output_dict = {} if stream is None: output_dict['url'] = feed['items'][0]['link'] else: try: swd = StreamWidgetData.objects.get(stream=stream, widget=widget) data = swd.value except: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = [] swd.value = data swd.save() feed_length = len(feed['items']) feed['items'].reverse() for item in feed['items']: if item['link'] not in data: data.append(item['link']) swd.value = data swd.save() output_dict['url'] = item['link'] break else: from streams.models import HaltStream raise HaltStream("Halting stream.") return output_dict
def streaming_rss_reader(input_dict,widget,stream=None): import feedparser from streams.models import StreamWidgetData feed = feedparser.parse(input_dict['url']) output_dict = {} if stream is None: output_dict['url'] = feed['items'][0]['link'] else: try: swd = StreamWidgetData.objects.get(stream=stream,widget=widget) data = swd.value except: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = [] swd.value = data swd.save() feed_length = len(feed['items']) feed['items'].reverse() for item in feed['items']: if item['link'] not in data: data.append(item['link']) swd.value = data swd.save() output_dict['url'] = item['link'] break else: from streams.models import HaltStream raise HaltStream("Halting stream.") return output_dict
def streaming_sentiment_graph(input_dict, widget, stream=None): from streams.models import StreamWidgetData if stream is None: return {} else: new_tweets = [] for tweet in input_dict['ltw']: new_tweets.append( StreamWidgetData(stream=stream, widget=widget, value=tweet)) StreamWidgetData.objects.bulk_create(new_tweets) return {}
def streaming_triplet_graph(input_dict, widget, stream=None): from streams.models import StreamWidgetData if stream is None: return {} else: try: swd = StreamWidgetData.objects.get(stream=stream, widget=widget) swd.value = input_dict['triplets'] swd.save() except: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = input_dict['triplets'] swd.value = data swd.save() return {}
def streaming_triplet_graph(input_dict,widget,stream=None): from streams.models import StreamWidgetData if stream is None: return {} else: try: swd = StreamWidgetData.objects.get(stream=stream,widget=widget) swd.value = input_dict['triplets'] swd.save() except: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = input_dict['triplets'] swd.value = data swd.save() return {}
def streaming_simulate_stream_from_csv(input_dict,widget,stream=None): from streams.models import StreamWidgetData import datetime import csv csvfile = safeOpen(input_dict['csv']) csvreader = csv.reader(csvfile,delimiter=";",quotechar='"') rows = [] ltw = [] i=0 counter = 0 started = False last_id = "not-started-yet" if not stream is None: try: swd = StreamWidgetData.objects.get(stream=stream,widget=widget) last_id = swd.value except: started = True else: started = True for row in csvreader: rows.append(row) if i!=0: rows[i][1] = datetime.datetime.strptime(rows[i][1],"%m/%d/%Y %I:%M:%S %p") tweet = {} tweet['id'] = rows[i][0] tweet['created_at'] = rows[i][1] tweet['text'] = rows[i][3].encode('utf-8') tweet['user'] = rows[i][5].encode('utf-8') tweet['lang'] = rows[i][11] if started: counter = counter + 1 ltw.append(tweet) if counter == 50 and started: started = False if not stream is None: try: swd = StreamWidgetData.objects.get(stream=stream,widget=widget) swd.value = tweet['id'] swd.save() except: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = tweet['id'] swd.value = data swd.save() if tweet['id']==last_id: started = True i = i + 1 if counter < 51 and not stream is None and started == True: try: swd = StreamWidgetData.objects.get(stream=stream,widget=widget) swd.value = "done" swd.save() except: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = "done" swd.value = data swd.save() output_dict = {} #print ltw #print len(ltw) output_dict['ltw']=ltw return output_dict
def streaming_sliding_window(input_dict,widget,stream=None): from streams.models import StreamWidgetData output_dict = {} if stream is None: output_dict['list']=input_dict['list'][:int(input_dict['size'])] else: try: swd = StreamWidgetData.objects.get(stream=stream,widget=widget) data = swd.value except: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = [] swd.value = data swd.save() size = int(input_dict['size']) if len(input_dict['list'])>=size: output_dict['list']=input_dict['list'][:size] swd.value=output_dict['list'] swd.save() else: current_window = input_dict['list'] + swd.value swd.value = current_window[:size] output_dict['list']=current_window[:size] swd.save() return output_dict
def streaming_twitter(input_dict,widget,stream=None): import tweepy from streams.models import StreamWidgetData from streams.models import HaltStream if input_dict['cfauth']=="true": consumer_key="zmK41mqxU3ZNJTFQpYwTdg" consumer_secret="9StnKNAe20ebDOREQjsVjAjBEiz5R9feZJTGUYWqLo" access_token="45210078-VydgdJMwhWYjZRvlNbrKj6jfqicUIsdMnRbnaPElL" access_token_secret="uLvIN3MMxFSxdK4M8P5RYojjUkbc2reqNydYtpT7Ks" else: consumer_key = input_dict['ck'] consumer_secret = input_dict['cs'] access_token = input_dict['at'] access_token_secret = input_dict['as'] auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) api = tweepy.API(auth) query = input_dict['query'] rate_limit_status = api.rate_limit_status() if rate_limit_status['resources']['search']['/search/tweets']['remaining']>0: if stream is None: try: ltw = api.new_search(q=input_dict['query'],geocode=input_dict['geocode'],count=100) except Exception as e: raise HaltStream("The Twitter API returned an error: "+str(e)) else: try: swd = StreamWidgetData.objects.get(stream=stream,widget=widget) data = swd.value except Exception as e: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = {} swd.value = data swd.save() if data.has_key(query): since_id = data[query] try: ltw = api.new_search(q=input_dict['query'],geocode=input_dict['geocode'],count=100,since_id=since_id) except Exception as e: raise HaltStream("The Twitter API returned an error: "+str(e)) else: try: ltw = api.new_search(q=input_dict['query'],geocode=input_dict['geocode'],count=100) except Exception as e: raise HaltStream("The Twitter API returned an error: "+str(e)) if len(ltw)>0: data[query]=ltw[0].id swd.value = data swd.save() else: import datetime import time current_time = int(time.mktime(datetime.datetime.now().timetuple())) remaining = rate_limit_status['resources']['search']['/search/tweets']['reset']-current_time if input_dict['cfauth']=="true": raise HaltStream("The twitter API limit has been reached. Try again in "+str(remaining)+" seconds. Try using your own credentials.") else: raise HaltStream("The twitter API limit has been reached. Try again in "+str(remaining)+" seconds.") output_dict = {} tweets = [] for tw in ltw: tweet = {} tweet['id'] = tw.id tweet['created_at'] = tw.created_at tweet['text'] = unicode(tw.text).encode("utf-8") try: tweet['user'] = tw.user['screen_name'] except: tweet['user'] = "" tweet['lang'] = tw.lang tweets.append(tweet) if len(tweets)>0 or stream is None: output_dict['ltw']=tweets return output_dict else: raise HaltStream("No new results, halting stream.")
def streaming_simulate_stream_from_csv(input_dict, widget, stream=None): from streams.models import StreamWidgetData import datetime import csv csvfile = safeOpen(input_dict['csv']) csvreader = csv.reader(csvfile, delimiter=";", quotechar='"') rows = [] ltw = [] i = 0 counter = 0 started = False last_id = "not-started-yet" if not stream is None: try: swd = StreamWidgetData.objects.get(stream=stream, widget=widget) last_id = swd.value except: started = True else: started = True for row in csvreader: rows.append(row) if i != 0: rows[i][1] = datetime.datetime.strptime(rows[i][1], "%m/%d/%Y %I:%M:%S %p") tweet = {} tweet['id'] = rows[i][0] tweet['created_at'] = rows[i][1] tweet['text'] = rows[i][3].encode('utf-8') tweet['user'] = rows[i][5].encode('utf-8') tweet['lang'] = rows[i][11] if started: counter = counter + 1 ltw.append(tweet) if counter == 50 and started: started = False if not stream is None: try: swd = StreamWidgetData.objects.get(stream=stream, widget=widget) swd.value = tweet['id'] swd.save() except: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = tweet['id'] swd.value = data swd.save() if tweet['id'] == last_id: started = True i = i + 1 if counter < 51 and not stream is None and started == True: try: swd = StreamWidgetData.objects.get(stream=stream, widget=widget) swd.value = "done" swd.save() except: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = "done" swd.value = data swd.save() output_dict = {} #print ltw #print len(ltw) output_dict['ltw'] = ltw return output_dict
def streaming_sliding_window(input_dict, widget, stream=None): from streams.models import StreamWidgetData output_dict = {} if stream is None: output_dict['list'] = input_dict['list'][:int(input_dict['size'])] else: try: swd = StreamWidgetData.objects.get(stream=stream, widget=widget) data = swd.value except: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = [] swd.value = data swd.save() size = int(input_dict['size']) if len(input_dict['list']) >= size: output_dict['list'] = input_dict['list'][:size] swd.value = output_dict['list'] swd.save() else: current_window = input_dict['list'] + swd.value swd.value = current_window[:size] output_dict['list'] = current_window[:size] swd.save() return output_dict
def streaming_twitter(input_dict, widget, stream=None): import tweepy from streams.models import StreamWidgetData from streams.models import HaltStream if input_dict['cfauth'] == "true": consumer_key = "zmK41mqxU3ZNJTFQpYwTdg" consumer_secret = "9StnKNAe20ebDOREQjsVjAjBEiz5R9feZJTGUYWqLo" access_token = "45210078-VydgdJMwhWYjZRvlNbrKj6jfqicUIsdMnRbnaPElL" access_token_secret = "uLvIN3MMxFSxdK4M8P5RYojjUkbc2reqNydYtpT7Ks" else: consumer_key = input_dict['ck'] consumer_secret = input_dict['cs'] access_token = input_dict['at'] access_token_secret = input_dict['as'] auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) api = tweepy.API(auth) query = input_dict['query'] rate_limit_status = api.rate_limit_status() if rate_limit_status['resources']['search']['/search/tweets'][ 'remaining'] > 0: if stream is None: try: ltw = api.new_search(q=input_dict['query'], geocode=input_dict['geocode'], count=100) except Exception as e: raise HaltStream("The Twitter API returned an error: " + str(e)) else: try: swd = StreamWidgetData.objects.get(stream=stream, widget=widget) data = swd.value except Exception as e: swd = StreamWidgetData() swd.stream = stream swd.widget = widget data = {} swd.value = data swd.save() if data.has_key(query): since_id = data[query] try: ltw = api.new_search(q=input_dict['query'], geocode=input_dict['geocode'], count=100, since_id=since_id) except Exception as e: raise HaltStream("The Twitter API returned an error: " + str(e)) else: try: ltw = api.new_search(q=input_dict['query'], geocode=input_dict['geocode'], count=100) except Exception as e: raise HaltStream("The Twitter API returned an error: " + str(e)) if len(ltw) > 0: data[query] = ltw[0].id swd.value = data swd.save() else: import datetime import time current_time = int(time.mktime(datetime.datetime.now().timetuple())) remaining = rate_limit_status['resources']['search']['/search/tweets'][ 'reset'] - current_time if input_dict['cfauth'] == "true": raise HaltStream( "The twitter API limit has been reached. Try again in " + str(remaining) + " seconds. Try using your own credentials.") else: raise HaltStream( "The twitter API limit has been reached. Try again in " + str(remaining) + " seconds.") output_dict = {} tweets = [] for tw in ltw: tweet = {} tweet['id'] = tw.id tweet['created_at'] = tw.created_at tweet['text'] = unicode(tw.text).encode("utf-8") try: tweet['user'] = tw.user['screen_name'] except: tweet['user'] = "" tweet['lang'] = tw.lang tweets.append(tweet) if len(tweets) > 0 or stream is None: output_dict['ltw'] = tweets return output_dict else: raise HaltStream("No new results, halting stream.")