def get_twitter_messages(request):
twitter_query = TwitterData.objects(
search_term=request.GET.get('search_term', None),
created_at__gte=request.GET.get('from', None),
created_at__lte=request.GET.get('to', None))
twitter_messages = util.get_serialized_data(
twitter_query.order_by('-retweet_count'), 'twitter')
return JsonResponse(twitter_messages, safe=False)
def get_messages_per_day(request):
search_term = request.GET.get('search_term', None)
if search_term is not None:
twitter_query = TwitterData.objects(
search_term=search_term,
created_at__gte=request.GET.get('from', None),
created_at__lte=request.GET.get('to', None))
else:
twitter_query = TwitterData.objects.all()
twitter_messages = util.get_serialized_data(
twitter_query.order_by('created_at'), 'twitter')
result = util.count_daily_messages(twitter_messages)
return JsonResponse(result, safe=False)
def get_pearson_correlation_2(request):
# types: number(total number), neutral, positive, negative
twitter_query = TwitterData.objects(
search_term=request.data['search_term'],
created_at__gte=request.data['from'],
created_at__lte=request.data['to']).order_by('created_at')
stock_query = StockData.objects(
symbol=request.data['symbol'],
date__gte=request.data['from'],
date__lte=request.data['to']).order_by('date')
twitter_messages = util.get_serialized_data(twitter_query, 'twitter')
stock_data = util.get_serialized_data(stock_query, 'stock')
if request.data['type'] == 'number':
summarized_messages = util.count_daily_messages(twitter_messages)
else:
summarized_messages = util.count_different_messages(twitter_messages)
# temp list containing strings with available stock dates
# used for eliminating "non stock days" messages - weekend messages
trimmed_twitter_messages = []
trimmed_stock_data = []
strings_list = []
for stock in stock_data:
strings_list.append(stock['date'])
# eliminating weekend messages
for message in summarized_messages:
for item in strings_list:
if message['date'] in item:
trimmed_twitter_messages.append(message[request.data['type']])
trimmed_stock_data.append(
next((x['close'] for x in stock_data if x['date'] == item),
None))
correlation = numpy.corrcoef(trimmed_twitter_messages,
trimmed_stock_data)[0, 1]
return JsonResponse(correlation, safe=False)
def ner_textblob_process(request):
search_term = request.GET.get('search_term', None)
twitter_data = TwitterData.objects(search_term=search_term)
for tweet in twitter_data:
sentiment = tweet['sentimental']['polarity']
entities = TextBlob(tweet['translated_text'])
for entity in entities.noun_phrases:
temp_entity = entity.lower()
try:
temp = NamedEntityData.objects.get(text=temp_entity)
if sentiment >= 0:
serializer = NamedEntityDataSerializer(
temp, data={"pos_quantity": temp['pos_quantity'] + 1})
else:
serializer = NamedEntityDataSerializer(
temp, data={"neg_quantity": temp['neg_quantity'] + 1})
except NamedEntityData.DoesNotExist:
serializer = NamedEntityDataSerializer(
data={
'text': temp_entity,
'search_term': search_term,
'pos_quantity': 1 if sentiment >= 0 else 0,
'neg_quantity': 1 if sentiment < 0 else 0
})
if serializer.is_valid():
serializer.save()
else:
return JsonResponse(serializer.errors, status=400, safe=False)
return JsonResponse(NamedEntityDataSerializer(NamedEntityData.objects(
search_term=request.GET.get('search_term', None)),
many=True).data,
safe=False)
def get_pearson_correlation(request):
twitter_query = TwitterData.objects(
search_term=request.data['search_term'],
created_at__gte=request.data['from'],
created_at__lte=request.data['to']).order_by('created_at')
stock_query = StockData.objects(
symbol=request.data['symbol'],
date__gte=request.data['from'],
date__lte=request.data['to']).order_by('date')
twitter_messages = util.get_serialized_data(twitter_query, 'twitter')
stock_data = util.get_serialized_data(stock_query, 'stock')
summarized_messages = util.summarize_twitter_data_by_day(twitter_messages)
# temp list containing strings with available stock dates
# used for eliminating "non stock days" messages - weekend messages
trimmed_twitter_messages = []
trimmed_stock_data = []
strings_list = []
for stock in stock_data:
strings_list.append(stock['date'])
# eliminating weekend messages
for message in summarized_messages:
for item in strings_list:
if message['created_at'] in item:
trimmed_twitter_messages.append(message['average_sentimental'])
trimmed_stock_data.append(
next((x['close'] for x in stock_data if x['date'] == item),
None))
correlation = numpy.corrcoef(trimmed_twitter_messages,
trimmed_stock_data)[0, 1]
return JsonResponse(correlation, safe=False)