import json
import pandas as pd
from glob import glob

def convert(x):
    ''' Convert a json string to a flat python dictionary
    which can be passed into Pandas. '''
    ob = json.loads(x)
    for k, v in ob.items():
        if isinstance(v, list):
            ob[k] = ','.join(v)
        elif isinstance(v, dict):
            for kk, vv in v.items():
                ob['%s_%s' % (k, kk)] = vv
            del ob[k]
    return ob

for json_filename in glob('yelp_academic_dataset_review.json'):
     csv_filename = '%s.csv' % json_filename[:-5]
    print 'Converting %s to %s' % (json_filename, csv_filename)
    df = pd.DataFrame([convert(line) for line in file(json_filename)])
    df.to_csv(csv_filename, encoding='utf-8', index=False)

REGISTER 's3://bigdataprojectzongdi/loudacre/piggybank.jar';

DEFINE CSVLoader org.apache.pig.piggybank.storage.CSVLoader();

business = LOAD 's3://bigdataprojectzongdi/loudacre/bid_categ.csv' USING CSVLoader AS (categories:chararray, business_id:chararray);

a = FILTER business BY (categories MATCHES '.*Dentist.*') OR (categories MATCHES '.*Dental.*') OR (categories MATCHES '.*dentist.*') OR (categories MATCHES '.*dental.*');

b = DISTINCT a;

STORE b INTO 's3://bigdataprojectzongdi/output/bid4dentist';

REGISTER 's3://bigdataprojectzongdi/loudacre/piggybank.jar';

DEFINE CSVLoader org.apache.pig.piggybank.storage.CSVLoader();

dentist = LOAD 's3://bigdataprojectzongdi/output/bid4dentist' AS (categories:chararray, business_id:chararray);

reviewall = LOAD 's3://bigdataprojectzongdi/loudacre/sample_no_break.csv' USING CSVLoader AS (
business_id: chararray,
cool: int,
date: chararray,
funny: int,
review_id: chararray,
stars:int,
text: chararray,
useful: int,
user_id: chararray);

dentistreview = JOIN dentist BY business_id, reviewall BY business_id;
temp = LIMIT dentistreview 5;
DUMP temp;
STORE dentistreview INTO 's3://bigdataprojectzongdi/output/mergesample';

dentist = LOAD '/Users/zhou/Desktop/bigdata/dentist_review' AS (categories:chararray,
business_id: chararray,
business_idaa: chararray,
cool: int,
date: chararray,
funny: int,
review_id: chararray,
stars:int,
text: chararray,
useful: int,
user_id: chararray);

/* #####total records################################## */

dent_details = GROUP dentist ALL;
dent_total_num = FOREACH dent_details GENERATE COUNT(dentist.review_id);
DUMP dent_total_num;

/* #####stars########################################## */

dentstars = GROUP dentist BY stars;
starnum = FOREACH dentstars GENERATE COUNT(dentist.review_id);
DUMP starnum;

dent_positive = FILTER dentist BY (stars >= 3);
dent_pos_details = GROUP dent_positive ALL;
dent_pos_num = FOREACH dent_pos_details GENERATE COUNT(dent_positive.review_id);
DUMP dent_pos_num;

dent_negative = FILTER dentist BY (stars < 3);
dent_neg_details = GROUP dent_negative ALL;
dent_neg_num = FOREACH dent_neg_details GENERATE COUNT(dent_negative.review_id);
DUMP dent_neg_num;

#importing Files
posText <- fread("positive-words.csv", header=FALSE, stringsAsFactors=FALSE)
posText <- posText$V1
posText <- unlist(lapply(posText, function(x) { str_split(x, "\n") }))
negText <- read.delim("negative-words.csv", header=FALSE, stringsAsFactors=FALSE)
negText <- negText$V1
negText <- unlist(lapply(negText, function(x) { str_split(x, "\n") }))
pos.words = c(posText, 'upgrade')
neg.words = c(negText, 'with', 'wait', 'waiting','epicfail', 'mechanical')

#define a function
score.sentiment = function(sentences, pos.words, neg.words, .progress='none')
{
  # Parameters
  # sentences: vector of text to score
  # pos.words: vector of words of positive sentiment
  # neg.words: vector of words of negative sentiment
  # .progress: passed to laply() to control of progress bar
  # create a simple array of scores wif laply
  scores = laply(sentences,
                 function(sentence, pos.words, neg.words)
                 {
                   # remove punctuation
                   sentence = gsub("[[:punct:]]", "", sentence)
                   # remove control characters
                   sentence = gsub("[[:cntrl:]]", "", sentence)
                   # remove digits?
                   sentence = gsub('\\d+', '', sentence)
                   # define error handling function when trying tolower
                   tryTolower = function(x)
                   {
                     # create missing value
                     y = NA
                     # tryCatch error
                     try_error = tryCatch(tolower(x), error=function(e) e)
                     # if not an error
                     if (!inherits(try_error, "error"))
                       y = tolower(x)
                     # result
                     return(y)
                   }
                   # use tryTolower wif sapply 
                   sentence = sapply(sentence, tryTolower)
                   # split sentence into words wif str_split (stringr package)
                   word.list = str_split(sentence, "\\s+")
                   words = unlist(word.list)
                   # compare words to teh dictionaries of positive & negative terms
                   pos.matches = match(words, pos.words)
                   neg.matches = match(words, neg.words)
                   # get teh position of teh matched term or NA
                   # we just want a TRUE/FALSE
                   pos.matches = !is.na(pos.matches)
                   neg.matches = !is.na(neg.matches)
                   # final score
                   score = sum(pos.matches) - sum(neg.matches)
                   return(score)
                 }, pos.words, neg.words, .progress=.progress )
  # data frame wif scores for each sentence
  scores.df = data.frame(text=sentences, score=scores)
  return(scores.df)
}
  
scores= score.sentiment((dentist$text), pos.words, neg.words, .progress= 'text')
scores <- as.data.table(scores)
setkey(scores, text)
setkey(dentist, text)
dentist_sentiment <- merge(scores, dentist, by="text")

#seperate the sentiment
dentist_sentiment$polarity <- ifelse(dentist_sentiment$score>0,"positive", 
                                             ifelse(dentist_sentiment$score<0, "negative", ifelse(dentist_sentiment$score==0,
                                                                                                                      "neutral",0)))
#plot the sentiment by polarity
ggplot(data=dentist_sentiment, aes(x=factor(polarity),fill=factor(polarity)))+
  geom_bar() +  geom_text(stat='count', aes(label=..count..), vjust=-1)+ggtitle("Customer Sentiments - Dentist")

## text mining
library(RTextTools)
library(data.table)
library(ggplot2)
library(ROAuth)
library(plyr)
library(stringr)
library(plotly)
library(maxent)

#count the words frequencies
dentist <-fread("dentist.csv", sep = ",", header = TRUE, stringsAsFactors = TRUE)[,-1]
dentist$text <- as.character((dentist$text))
matrix <- create_matrix(dentist$text, language="english", removeSparseTerms = 0.89, removeStopwords = TRUE,
                        removeNumbers = TRUE, stemWords = TRUE, stripWhitespace = TRUE, toLower = TRUE, weighting = weightTf)
mat <- as.matrix(matrix)
mat <- as.data.table(mat)

term.freq <- rowSums(t(as.matrix(mat)))
df <- data.frame(term=names(term.freq), freq=term.freq)
ggplot(df, aes(x=term, y=freq))+geom_bar(stat="identity")+xlab("Terms")+ylab("Count")