def isHateSpeech(self, line): #using open source hate sonar api sonar = Sonar() response = sonar.ping(text=line) if response["top_class"] != "neither": # line is hate speech return 1 else: return 0
def __init__(self, bot, mysql: MySQLWrapper): self.bot = bot self.mysql = mysql self.sonar = Sonar() self.discordForum = Forum("ECC-Discord", "ECC-Discord") self.discordForum.insert(self.mysql) self.discordAuthor = ForumAuthor("Discord", "-1") self.discordAuthor.insert(self.mysql) # self.generalThread = ForumThread("#general-chat", self.discordAuthor, self.discordForum.sqlID, hltvID = "709753463323754539") self.generalThread = ForumThread("#general-chat", self.discordAuthor, self.discordForum.sqlID, hltvID="456834448558653451") self.generalThread.insert(self.mysql) # self.shitpostingThread = ForumThread("#shitposting-and-media", self.discordAuthor, self.discordForum.sqlID, hltvID = "727107131416903750") self.shitpostingThread = ForumThread("#shitposting-and-media", self.discordAuthor, self.discordForum.sqlID, hltvID="468121733929369610") self.shitpostingThread.insert(self.mysql) self.mysql.db.commit()
def hatedetect(hds): ypred = [] sonar = Sonar() for k in hds: check = [] res = sonar.ping(text=k) res_final = json.dumps(res) #person_dict = json.loads(str(res)) res_dict = json.loads(res_final) #print(res_dict) #for x in res_dict["classes"]: if res_dict["classes"][0]["confidence"] > res_dict["classes"][1][ "confidence"]: check = res_dict["classes"][0]["confidence"] val = 0 else: check = res_dict["classes"][1]["confidence"] val = 1 if check > res_dict["classes"][2]["confidence"]: check_final = check else: check_final = res_dict["classes"][2]["confidence"] val = 2 if val == 0 or val == 1: yp = 0 else: yp = 1 ypred.append(yp) return ypred
def refine_jsonl_file(path, votes_threshold=10, hate_limit=0.4, offensive_limit=0.7, general_limit=0.8): sonar = Sonar() name, _ = os.path.splitext(path) refined_name = "refined_{name}.txt".format(name=name) if os.path.exists(refined_name): os.remove(refined_name) refined_file = open(refined_name, mode='w') with open(path, 'rb') as f: # opening file in binary(rb) mode with refined_file as rf: for item in json_lines.reader(f): if int(item['votes']) > 0: text = item['text'] hate_confidence = sonar.ping( text=text)['classes'][0]['confidence'] offensive_confidence = sonar.ping( text=text)['classes'][1]['confidence'] if not ((hate_confidence > hate_limit) or (offensive_confidence > offensive_limit) or (hate_confidence + offensive_confidence > general_limit)): try: print(text, file=rf) except: continue rf.close() f.close()
def classify_texts(texts): """ Classify texts with hate sonar (classifier). """ # initialize parameters and containers texts = texts.split(".")[:-1] rating_map = { "neither": "green", "offensive_language": "orange", "hate_speech": "red" } sonar = Sonar() results = [] # for each text, perform detection and format dictionary for i, sentence in enumerate(texts): sentence_res = sonar.ping(text=sentence) top_class = sentence_res["top_class"] sentence_output = { "index": i, "sentence": sentence, "top_class": top_class, "rating": rating_map[top_class] } results.append(sentence_output) return results
def isAbusiveComment(x): sonar = Sonar() a = sonar.ping(text=x) #print(x) #print(a) hateConfidence = 0.0 offenseConfidence = 0.0 neitherConfidence = 0.0 for result in a['classes'] : #print(result) #print(result['confidence']) #print(result["class_name"]) if result["class_name"] == "hate_speech" : hateConfidence = result['confidence'] if result["class_name"] == "offensive_language" : offenseConfidence = result['confidence'] if result["class_name"] == "neither" : neitherConfidence = result['confidence'] #print("Hate ",hateConfidence, " || offenseConfidence ",offenseConfidence," || neither ",neitherConfidence) rez = False if neitherConfidence > 0.7: rez = False if hateConfidence > 0.6 : rez = True if offenseConfidence >0.55 : rez = True return MessageScreenerResult(rez,hateConfidence,offenseConfidence,"No Tips, Sorry!")
def main(argv): global sonar # setting up the database db = MongoClient().gab # initialising the hateometer sonar = Sonar() # parsing all posts print("hateometing the posts...") # loop over all english posts for post in db.posts.find({'post.language':'en'}, no_cursor_timeout=True): # run hate detection on the post body obj = hateometer(post['post']['body']) postid = post['_id'] # store the results in mongodb db.posts.update_one({'_id':postid},{"$set":{'post.hateometer':obj}}) print("hateometing the comments...") # do the same trick for the comments for comment in db.comments.find({'language':'en'}, no_cursor_timeout=True): obj = hateometer(comment['body']) commentid = comment['_id'] db.comments.update_one({'_id':commentid},{"$set":{'hateometer':obj}}) print("done!")
def receive_tweet(request): tweet = request.GET.get('tweet') print(tweet) tweet = repr(tweet) sonar = Sonar() result = sonar.ping(text=tweet) print(result) json_result = json.dumps(result) return render(request, 'result_api/page2.html', result)
def isHateSpeech(self, line): '''Assign a 'hatespeech score' using sonar api ''' indices = {"hate_speech": 0, "offensive_language": 1, "neither": 2} sonar = Sonar() response = sonar.ping(text=line) indexOfLanguage = indices[response["top_class"]] if response["top_class"] != "neither": return response['classes'][indexOfLanguage]['confidence'] else: return 0
def main(): sonar=Sonar() textArray=readFile() Class=[] hate=[] offensive=[] neither=[] hate_speech_classifier(textArray,Class,hate,offensive,neither,sonar) displayResaults(hate,offensive,neither)
def hateSpeech(comments): sonar = Sonar() print('Inside hatespeech') print('Comments len ='+str(len(comments))) for i in range(len(comments)): x = sonar.ping(text=comments[i].text) if x['top_class'] == "hate_speech": comments[i].hateType = 'hate' elif x['top_class'] == "offensive_language": comments[i].hateType = 'offensive' else: comments[i].hateType = 'neutral' return comments
def do_GET(self): query_components = parse_qs(urlparse(self.path).query) imsi = query_components["imsi"] print(imsi) # Send headers self._set_headers() # Send message back to client sonar = Sonar() message = sonar.ping(text=str(imsi[0])) # Write content as utf-8 data self.wfile.write(bytes(json.dumps(message), "utf-8")) return message
def getOffensiveness(self, string, output_a): sonar = Sonar() data_raw = str(sonar.ping(string)) data_raw_split = data_raw.split(": [{")[1].split("}]}")[0].replace("'class_name': '", "").replace("', 'confidence':", "").split("}, {") output = data_raw_split[0].split(" ")[0] + " " if "e" in data_raw_split[0].split(" ")[1]: output += "0.00" else: output += str('{0:.{1}f}'.format(float(data_raw_split[0].split(" ")[1]) * 100, 2)) + " " output += data_raw_split[1].split(" ")[0] + " " if "e" in data_raw_split[1].split(" ")[1]: output += "0.00" else: output += str('{0:.{1}f}'.format(float(data_raw_split[1].split(" ")[1]) * 100, 2)) + " " output += data_raw_split[2].split(" ")[0] + " " if "e" in data_raw_split[2].split(" ")[1]: output += "0.00" else: output += str('{0:.{1}f}'.format(float(data_raw_split[2].split(" ")[1]) * 100, 2)) output_a.append(output)
def ping_file(dataset_path): """Run ping function on each line in the chat log file :param dataset_path: file path of the dataset (specify directory name if the file is under a folder) """ sonar = Sonar() input_file = open(dataset_path, 'r', encoding="utf-8") # to read and remove "\n" escape characters at the end of each chat message chat_lines = input_file.read().splitlines() # to trim whitespaces before and after each chat message chat_lines = [each_line.strip() for each_line in chat_lines] # to get only the message after the [timestamp] <username> chat_lines = [each_line.partition("> ")[2] for each_line in chat_lines] return [ sonar.ping(each_line) for each_line in tqdm( chat_lines, desc="Processing {} rows".format(len(chat_lines))) ]
def getTweet(api, hashTag, fileName, startDT, numTweet=1): hashTag = "\"#" + hashTag + "\"" startDT = "\"" + startDT + "\"" csvFile = open(fileName, 'w') csvWriter = csv.writer(csvFile) sonar = Sonar() try: for tweet in tweepy.Cursor(api.search, q=hashTag, count=numTweet, lang="en", since=startDT).items(): if tweet.retweeted == False: cleantweet = cleanTweet(tweet) text = "text=\"" + str(cleanTweet) + "\"" csvWriter.writerow([ tweet.created_at, tweet.user.screen_name, str(cleantweet), sonar.ping(text).top_class ]) except: print("Unable to connect Twitter API.")
data.replace('\r', '') input = data.split('\n') splitInput = [ input[x:x + math.ceil(len(input) / 6)] for x in range(0, len(input), math.ceil(len(input) / 6)) ] #print(chunks[3]) #def f(splitInput, splitInputIndex): #print('{}: hello {} from {}'.format( # dt.datetime.now(), name, current_process().name)) #sys.stdout.flush() from hatesonar import Sonar sonar = Sonar() def f(splitInput, splitInputIndex): #offensiveCount = 0 hatefulCount = 0 for i in splitInput[splitInputIndex]: #for i in input: sonarEval = sonar.ping(i) if sonarEval['top_class'] == "neither": continue if sonarEval['top_class'] == "offensive_language": #os.system("cat {} | jq 'select(.body == \"{}\""")' >> offensive{}".format(args['input'], i, args['input'])) #offensiveCount += 1 continue if sonarEval['top_class'] == "hate_speech":
def main(): tweet = sys.argv[1] sonar = Sonar() print(sonar.ping(tweet)['classes'])
def triggers(request): if request.method == 'POST': print(request.POST) data = dict(request.POST) # Driver Code key = 'show_details' one = checkKey(data, key) key = 'check_triggers' two = checkKey(data, key) key = 'show_wordcloud' three = checkKey(data, key) key = 'hate_speech' four = checkKey(data, key) print(one, two, three) #URL Link case if (one == True): url = data['Link'][0] print(url) article = Article(url) article.download() article.parse() authors = article.authors publishdate = article.publish_date #article.text article.nlp() keywords = article.keywords articlesummary = article.summary return render( request, 'consciousApp/triggers.html', { 'authors': authors, 'publishdate': publishdate, 'keywords': keywords, 'articlesummary': articlesummary }) #Show triggers elif (two == True): text = request.POST['input_text'].lower() triggers = [ "9 11", "9-11", "9/11", "ableism", "abusive", "ageism", "alcoholism", "animal abuse", "animal death", "animal violence", "bestiality", "gore", "corpse", "bully", "cannibal", "car accident", "child abuse", "childbirth", "classism", "death", "decapitation", "abuse", "drug", "heroin", "cocaine", "eating disorder", "anorexia", "binge eating", "bulimia", "fatphobia", "forced captivity", "holocaust", "hitler", "homophobia", "hostage", "incest", "kidnap", "murder", "nazi", "overdose", "pedophilia", "prostitution", "PTSD", "racism", "racist", "rape", "raping", "scarification", "self-harm", "self harm", "cutting", "sexism", "slavery", "slurs", "suicide", "suicidal", "swearing", "terminal illness", "terrorism", "torture", "transphobia", "violence", "warfare" ] tw = [] text_file = open( './consciousApp/static/consciousApp/input/triggercheckdata.txt', 'w+') text_file.write(str(text)) text_file.close() for trigger in triggers: if text.find(trigger) > -1: tw.append(trigger) if tw == []: tw.append('No Triggers Found') return render(request, 'consciousApp/triggers.html', { 'text': text, 'triggers': tw, 'data': data }) #Show_cloud elif (three == True): text = request.POST['input_text'].lower() tokens = word_tokenize(text) textdata = nltk.Text(tokens) stopwords = set(STOPWORDS) wordcloud = WordCloud(stopwords=stopwords, max_font_size=50, max_words=100, background_color="white").generate(text) wordcloud.to_file( "./consciousApp/static/consciousApp/output/word-cloud.png") data = "./../../static/consciousApp/output/word-cloud.png" return render(request, 'consciousApp/triggers.html', {'data': data}) elif (four == True): sonar = Sonar() text = request.POST['input_text'].lower() url = data['Link'][0] data = sonar.ping(text=text)["classes"] hate_speech = data[0] hate_speech_confidence = hate_speech["confidence"] * 100 offensive_language = data[1] offensive_language_confidence = offensive_language[ "confidence"] * 100 neither = data[2] neither_confidence = neither["confidence"] * 100 print(type(data)) print(offensive_language_confidence * 100, hate_speech_confidence * 100, neither_confidence * 100) return render( request, 'consciousApp/triggers.html', { 'hate_speech_confidence': hate_speech_confidence, 'offensive_language_confidence': offensive_language_confidence, 'neither_confidence': neither_confidence }) else: return render(request, 'consciousApp/triggers.html')
""" Natural language features defind on a single Comment. """ import logging import nltk from polyglot.detect import Detector from textblob import TextBlob from profanity_check import predict_prob as profanity from hatesonar import Sonar from comment import Comment, CommentFeatures from feature_extraction.nl_sets import * logging.basicConfig(filename='./output.log') hatesonar = Sonar() def compute_nl_features(c: Comment): c.stats = CommentFeatures() stats = c.stats stats['lang'] = comment_languge(c) stats['word_count'] = word_count(c) stats['score'] = c.score stats['controversial'] = c.controversial stats['prp_first'] = percent_first_pronouns(c) stats['prp_second'] = percent_second_pronouns(c) stats['prp_third'] = percent_third_pronouns(c) stats['sent'] = sentiment(c) stats['subj'] = subjectivity(c)
def check(self, txt): sonar = Sonar() res = sonar.ping(text=txt) return json.dumps(res)
def get_report(webpage_html): sonar = Sonar() report = sonar.ping(text=webpage_html) return report
def __init__(self): self.sonar = Sonar()
def sentiment_analysis(): documents = read_documents() sonar = Sonar() print(sonar.ping(text=mtranslate.translate(documents[1]))) analysis_tb = TextBlob(mtranslate.translate(documents[1])) print(analysis_tb.sentiment.polarity)