def __init__(self, datapath, tree): # decision tree get from the learner self.DT = tree if datapath != None: test_file = datapath if len(sys.argv) <= 1: test_file = "../data/origin/test.csv" else: test_file = sys.argv[1] f = open(test_file, "r") lines = f.readlines() f2 = open("btest_output.csv", "w") # get the list of attributes readInput = Datareader(lines) attrNames = readInput.attrNames records = readInput.records resultAttr = readInput.resultAttr typeList = readInput.typeList helper.process(records, attrNames[:len(attrNames)-1], typeList) self.predict(records, tree, resultAttr) for record in records: for attr in attrNames: val = record[attr] if attr == resultAttr: val = int(val) f2.write(str(val) + ',') f2.write('\n')
def __init__(self, datapath, tree): # decision tree get from the learner self.DT = tree if datapath != "None": validate_file = datapath elif len(sys.argv) <= 1: validate_file = "bvalidate.csv" else: validate_file = sys.argv[1] f = open(validate_file, "r") lines = f.readlines() # get the list of attributes readInput = Datareader(lines) attrNames = readInput.attrNames records = readInput.records resultAttr = readInput.resultAttr typeList = readInput.typeList helper.process(records, attrNames, typeList) print print "---> Using the validation set, the prediction accuracy is : " print(self.validate(records, self.DT, resultAttr)) print
def __init__(self, datapath, tree): # decision tree get from the learner self.DT = tree if datapath != "None": validate_file = datapath elif len(sys.argv) <= 1: validate_file = "bvalidate.csv" else: validate_file = sys.argv[1] f = open(validate_file, "r") lines = f.readlines() # get the list of attributes readInput = Datareader(lines) attrNames = readInput.attrNames records = readInput.records resultAttr = readInput.resultAttr typeList = readInput.typeList helper.process(records, attrNames, typeList) print print "---> Using the validation set, the prediction accuracy is : " print(self.validate(records, self.DT, resultAttr)) print
def __init__(self, datapath): if datapath != "None": train_file = datapath elif len(sys.argv) <= 1: train_file = "btrain.csv" #train_file = "../data/test.csv" else: train_file = sys.argv[1] f = open(train_file, "r") lines = f.readlines() lines = lines[:16000] # get the list of attributes readInput = Datareader(lines) attrNames = readInput.attrNames records = readInput.records resultAttr = readInput.resultAttr typeList = readInput.typeList print typeList helper.process(records, attrNames, typeList) print "read " + str(len(attrNames)) + " attributes" print "read " + str(len(records)) + " records" print self.tree = self.DTL(records, attrNames, resultAttr) print "-----> Printing DNF of the decision tree ..." helper.printDNF(self.tree) print
def process( self ): #process have been changed to get the news , process function in helper.py then for url in self.list: #process it into format that bikeso can accept resultPage = fetch(url) result = loads(resultPage)[self.content] yield (result['title'], helper.process(result[self.desc]))
def main(): directory = '/Users/jtim/Dropbox/Academic/research/dissertation/research/output/word-length' if not os.path.exists(directory): os.makedirs(directory) by_author = {} for author, files in corpora.items(): by_author[author] = read_files_into_string(files) # Transform the authors' corpora into lists of word tokens by_author_tokens = {} by_author_length_distributions = {} for author in by_author: tokens = by_author[author].split() # Filter out punctuation by_author_tokens[author] = ([process(token) for token in tokens if any(c.isalpha() for c in token)]) # Get a distribution of token lengths token_lengths = [len(token) for token in by_author_tokens[author]] plt.ion() by_author_length_distributions[author] = nltk.FreqDist(token_lengths) by_author_length_distributions[author].plot(9,title=author,color='grey') plt.savefig('{}/word-length-{}.png'.format(directory, author)) plt.ioff() plt.close("all")
def main(): pure_persian_works = [] arabic_counter = Counter() arabic_vocabulary = set() for name in file_names: if 'ar.txt' in name: with open("{}{}".format(root_dir, name), 'r') as f: #### Error, use Counter instead of set? #### words = set(process(f.read()).split()) arabic_counter.update(words) arabic_vocabulary.update(words) # remove infrequent words for word, count in arabic_counter.items(): if len( word ) < 3: # drop one and two letter words; they have high probablity of being a homograph arabic_vocabulary.remove(word) elif count <= 2: arabic_vocabulary.remove(word) with open("{}pure-persian.txt".format(directory), 'w') as out_file: for name in file_names: if 'fa.txt' in name: with open("{}{}".format(root_dir, name), 'r') as f: #### Error, use Counter instead of set? #### words = set(process(f.read()).split()) if len(words - arabic_vocabulary) > len( words ) * .70: # if x percent of words not in Arabic vocabulary out_file.write(name + "\n") out_file.write("Number of words: {}\n".format( len(words))) out_file.write("Percent Persian words: {}\n".format( len(words - arabic_vocabulary) / len(words))) out_file.write( "Words of possible Arabic origin: {}\n".format( words.intersection(arabic_vocabulary))) out_file.write("------------------------------\n\n") pure_persian_works.append(name) out_file.write("Pure Persian Works:\n") for i in pure_persian_works: out_file.write("\t" + "-" + i + "\n")
def chi_squared(relative_corpus, authors=[]): by_author = {} for key, files in corpora.items(): by_author[key] = read_files_into_string(files) # Transform the authors' corpora into lists of word tokens by_author_tokens = {} by_author_length_distributions = {} for author in by_author: tokens = by_author[author].split() # Filter out punctuation by_author_tokens[author] = ([ process(token) for token in tokens if any(c.isalpha() for c in token) ]) for author in authors: # First, build a joint corpus and identify the 500 most frequent words in it joint_corpus = (by_author_tokens[author] + by_author_tokens[relative_corpus]) joint_freq_dist = nltk.FreqDist(joint_corpus) most_common = list(joint_freq_dist.most_common(500)) # What proportion of the joint corpus is made up of the candidate # author's tokens? author_share = (len(by_author_tokens[author]) / len(joint_corpus)) # Now, let's look at the 500 most common words in the candidate # author's corpus and compare the number of times they can be observed # to what would be expected if the author's writings and the relative # corpus were both random samples from the same distribution. chisquared = 0 for word, joint_count in most_common: # How often do we really see this common word? author_count = by_author_tokens[author].count(word) relative_count = by_author_tokens[relative_corpus].count(word) # How often should we see it? expected_author_count = joint_count * author_share expected_joint_count = joint_count * (1 - author_share) # Add the word's contribution to the chi-squared statistic chisquared += ((author_count - expected_author_count) * (author_count - expected_author_count) / expected_author_count) chisquared += ((relative_count - expected_joint_count) * (relative_count - expected_joint_count) / expected_joint_count) print("The Chi-squared statistic for", author, "compared to", relative_corpus, "is", chisquared)
def main(): persian_words = ['خواهد', 'بايد', 'نزد', 'نماييد', 'اين', 'نموده', 'شما', 'اوست', 'بگو', 'شود', 'خود', 'هست', 'گشت', 'شويد', 'راه', 'بآن', 'امروز', 'نمايد', 'چون', 'شوند', 'دوستان', 'شده', 'بوده', 'آنكه', 'بود', 'آفتاب', 'اند', 'داده', 'فردا', 'شايد', 'چه', 'نيست', 'را' , 'آنچه’, 'شود’, ’آنچه', 'مانده', 'بيني', 'جان', 'باز', 'اگر', 'است', 'آمد', 'كنيد', 'سرا', 'نما', 'ميشود', 'نمود', 'دار', 'نبوده', 'شوي', 'ميفرمايد', 'دوست'] corrupt = [] pure_persian_works = [] arabic_counter = Counter() arabic_vocabulary = set() arabic_works_count = 0 for name in file_names: if 'ar.txt' in name: corrupt_word_count = 0 arabic_works_count += 1 with open(name, 'r') as f: words = set(process(f.read()).split()) arabic_counter.update(words) arabic_vocabulary.update(words) for word in words: if word in persian_words: corrupt_word_count += 1 if corrupt_word_count > 0: corrupt.append(name) print(corrupt) print("{} of {} are corrupted".format(len(corrupt), arabic_works_count))
shared_access_key_value='ak9L18tmI2FssJBIZLz3OCs8U55rcYZaSbwgAR6/B34=') for email in emails: try: # Split and Parse emails # Get Contents # Save to db doc = {} doc['from'] = fx.getContact(email.sender) doc['to'] = fx.getContacts(email.to_recipients) if email.cc_recipients: doc['cc'] = fx.getContacts(email.cc_recipients) doc['subject'] = email.subject.strip() msgs = fx.process(email.text_body) msgs = [[doc['subject']]] + msgs doc['emails'] = msgs doc['intent'] = fx.getIntent(msgs) #doc['intentions'] = fx.getIntentPerLine(msgs) doc['caseid'] = str(uuid.uuid4()) doc['state'] = 'new' doc['handoff'] = False doc['botHasReplied'] = False #print docid #Send Message to Incoming Queue event = Message(json.dumps(doc)) nttBus.send_queue_message('htn.incoming.emails', event) docid = sdmails.insert_one(doc).inserted_id except Exception,e:
# -*- coding: utf-8 -*- """ Created on Thu Apr 30 14:59:44 2020 @author: qinzhen """ import pickle from helper import process #读取单词 en, vocab_en = process("corpus.en", True) de, vocab_de = process("corpus.de") #t存储概率,t1存储e对应的全体f t = dict() t1 = dict() for e in vocab_en: t[e] = dict() #存储键 t1[e] = set() #迭代 for i, sentence in enumerate(en): for e in sentence: t1[e] = t1[e].union(set(de[i])) for e in t: ne = len(t1[e]) for f in t1[e]: t[e][f] = 1.0 / ne
def main(): directory = '/Users/jtim/Dropbox/Academic/research/dissertation/research/output/' if not os.path.exists(directory): os.makedirs(directory) by_author = {} for author, files in corpora.items(): by_author[author] = read_files_into_string(files) authors = ("Bahá'u'llah", "Bahá'u'llah Baghdad", "`Abdu'l-Bahá", "the Báb", "al-Shaykh Murtaḍá al-Ánsárí") # Transform the authors' corpora into lists of word tokens by_author_tokens = {} by_author_length_distributions = {} for author in by_author: tokens = by_author[author].split() # Filter out punctuation by_author_tokens[author] = ([process(token) for token in tokens if any(c.isalpha() for c in token)]) # Get a distribution of token lengths token_lengths = [len(token) for token in by_author_tokens[author]] plt.ion() by_author_length_distributions[author] = nltk.FreqDist(token_lengths) by_author_length_distributions[author].plot(15,title=author,color='grey') plt.savefig('{}figures/word-length-{}.png'.format(directory, author)) plt.ioff() plt.close("all") ### Chi-squared tests ### # Test 1: Compare the distance of Bahá'u'lláh's full corpus to His Baghdad writings # and the distance of Shaykh Murtaḍá's writings to Bahá'u'lláh's Baghdad writings authors_one = ("Bahá'u'llah", "al-Shaykh Murtaḍá al-Ánsárí") for author in authors_one: # First, build a joint corpus and identify the 500 most frequent words in it joint_corpus = (by_author_tokens[author] + by_author_tokens["Bahá'u'llah Baghdad"]) joint_freq_dist = nltk.FreqDist(joint_corpus) most_common = list(joint_freq_dist.most_common(500)) # What proportion of the joint corpus is made up of the candidate # author's (Bahá'u'llah and Shaykh Murtaḍá) tokens? author_share = (len(by_author_tokens[author]) / len(joint_corpus)) # Now, let's look at the 500 most common words in the candidate # author's (Bahá'u'llah and Shaykh Murtaḍá) corpus and compare # the number of times they can be observed to what would be expected # if the author's writings and Bahá'u'llah's Baghdad writings were # both random samples from the same distribution. chisquared = 0 for word, joint_count in most_common: # How often do we really see this common word? author_count = by_author_tokens[author].count(word) baghdad_count = by_author_tokens["Bahá'u'llah Baghdad"].count(word) # How often should we see it? expected_author_count = joint_count * author_share expected_joint_count = joint_count * (1-author_share) # Add the word's contribution to the chi-squared statistic chisquared += ((author_count-expected_author_count) * (author_count-expected_author_count) / expected_author_count) chisquared += ((baghdad_count-expected_joint_count) * (baghdad_count-expected_joint_count) / expected_joint_count) print("The Chi-squared statistic for", author, "compared to Bahá'u'llah's Baghdad writngs is", chisquared) # Test 2: Now consider the relative distance between the writings or `Abdu'l-Bahá` # and the writings of Bahá'u'lláh compared to the writings of Shaykh Murtaḍá. authors_two = ("`Abdu'l-Bahá", "al-Shaykh Murtaḍá al-Ánsárí") for author in authors_two: # First, build a joint corpus and identify the 500 most frequent words in it joint_corpus = (by_author_tokens[author] + by_author_tokens["Bahá'u'llah"]) joint_freq_dist = nltk.FreqDist(joint_corpus) most_common = list(joint_freq_dist.most_common(500)) # What proportion of the joint corpus is made up of the candidate # author's (`Abdu'l-Bahá` and Shaykh Murtaḍá) tokens? author_share = (len(by_author_tokens[author]) / len(joint_corpus)) # Now, let's look at the 500 most common words in the candidate # author's (`Abdu'l-Bahá` and Shaykh Murtaḍá) corpus and compare # the number of times they can be observed to what would be expected # if the author's writings and Bahá'u'llah's writings were both # random samples from the same distribution. chisquared = 0 for word, joint_count in most_common: # How often do we really see this common word? author_count = by_author_tokens[author].count(word) baghdad_count = by_author_tokens["Bahá'u'llah"].count(word) # How often should we see it? expected_author_count = joint_count * author_share expected_joint_count = joint_count * (1-author_share) # Add the word's contribution to the chi-squared statistic chisquared += ((author_count-expected_author_count) * (author_count-expected_author_count) / expected_author_count) chisquared += ((baghdad_count-expected_joint_count) * (baghdad_count-expected_joint_count) / expected_joint_count) print("The Chi-squared statistic for", author, "compared to Bahá'u'llah's writngs is", chisquared)
def read_files_into_string(filenames): strings = [] for filename in filenames: with open(filename) as f: strings.append(process(f.read())) return '\n'.join(strings)
def main(): directory = '/Users/jtim/Dropbox/Academic/sources/corpora/bahai-corpus/output/islamicate-texts/' if not os.path.exists(directory): os.makedirs(directory) # Islamicate variables arabic_islamicate_counter = Counter() persian_islamicate_counter = Counter() arabic_islamicate_vocabulary = set() persian_islamicate_vocabulary = set() arabic_islamicate_word_count = 0 persian_islamicate_word_count = 0 arabic_islamicate_vocabularies = [] persian_islamicate_vocabularies = [] # Baha'i variables arabic_bahai_counter = Counter() persian_bahai_counter = Counter() arabic_bahai_vocabulary = set() persian_bahai_vocabulary = set() arabic_bahai_word_count = 0 persian_bahai_word_count = 0 arabic_bahai_vocabularies = [] persian_bahai_vocabularies = [] # Process Islamicate files for a_file in arabic_islamicate_files: with open(a_file, 'r') as af: words = Counter(process(af.read()).split()) arabic_islamicate_word_count += len(words) arabic_islamicate_vocabulary.update(words) arabic_islamicate_counter.update(words) for p_file in persian_islamicate_files: with open(p_file, 'r') as pf: words = Counter(process(pf.read()).split()) persian_islamicate_word_count += len(words) persian_islamicate_vocabulary.update(words) persian_islamicate_counter.update(words) # Process Baha'i files for a_file in arabic_bahai_files: with open(a_file, 'r') as af: words = Counter(process(af.read()).split()) arabic_bahai_word_count += len(words) arabic_bahai_vocabulary.update(words) arabic_bahai_counter.update(words) for p_file in persian_bahai_files: with open(p_file, 'r') as pf: words = Counter(process(pf.read()).split()) persian_bahai_word_count += len(words) persian_bahai_vocabulary.update(words) persian_bahai_counter.update(words) # Common variables minimum_threshold = 5 for i in range(1, minimum_threshold): # remove minimum_threshold number of words for item in arabic_islamicate_counter.items(): if item[1] <= i: if item[0] in arabic_islamicate_vocabulary: arabic_islamicate_vocabulary.remove(item[0]) for item in persian_islamicate_counter.items(): if item[1] <= i: if item[0] in persian_islamicate_vocabulary: persian_islamicate_vocabulary.remove(item[0]) for item in arabic_bahai_counter.items(): if item[1] <= i: if item[0] in arabic_bahai_vocabulary: arabic_bahai_vocabulary.remove(item[0]) for item in persian_bahai_counter.items(): if item[1] <= i: if item[0] in persian_bahai_vocabulary: persian_bahai_vocabulary.remove(item[0]) combined_bahai_counter = arabic_bahai_counter + persian_bahai_counter bahai_intersection = persian_bahai_vocabulary.intersection( arabic_bahai_vocabulary) islamicate_intersection = persian_islamicate_vocabulary.intersection( arabic_islamicate_vocabulary) # Print Islamicate stats print("Islamicate language statistics:") print("Arabic word count: {}".format(arabic_islamicate_word_count)) print("Persian word count: {}".format(persian_islamicate_word_count)) print("Arabic vocabulary: {}".format(len(arabic_islamicate_vocabulary))) print("Persian vocabulary: {}".format(len(persian_islamicate_vocabulary))) print("intersection: {} ".format(len(islamicate_intersection))) print('\n') with open('{}islamicate_intersection.txt'.format(directory), 'w') as out_file: for w in islamicate_intersection: out_file.write(w + '\n') with open('{}islamicate_intersection_sample.txt'.format(directory), 'w') as out_file: sample = random.sample(list(islamicate_intersection), round((len(islamicate_intersection) / 100)) * 2) for w in sample: out_file.write(w + '\n') # Print Baha'i stats print("Bahá'í language statistics:") print("Arabic word count: {}".format(arabic_bahai_word_count)) print("Persian word count: {}".format(persian_bahai_word_count)) print("Arabic vocabulary: {}".format(len(arabic_bahai_vocabulary))) print("Persian vocabulary: {}".format(len(persian_bahai_vocabulary))) print("intersection: {} ".format(len(bahai_intersection))) print('\n') with open('{}bahai_intersection.txt'.format(directory), 'w') as out_file: for w in bahai_intersection: out_file.write(w + '\n') with open('{}bahai_intersection_sample.txt'.format(directory), 'w') as out_file: sample = random.sample(list(bahai_intersection), round((len(bahai_intersection) / 100)) * 2) for w in sample: out_file.write(w + '\n') # Plot table data = [[4242, 15, 1927, 2], [30000, 2000, 20000, 22000]] columns = ("`Abdu'l-Bahá", "Báb", "Bahá'u'lláh", "Shoghi Effendi") rows = ["Sample", "Known works"] # values = np.arange(0, 2500, 500) # value_increment = 1000 colors = plt.cm.BuPu(np.linspace(0, 0.5, len(rows))) n_rows = len(data) index = np.arange(len(columns)) + 0.3 bar_width = 0.4 for row in range(n_rows): plt.bar(index, data[row], bar_width, bottom=y_offset, color=colors[row]) y_offset = y_offset + data[row] plot = plt.table(cellText=None, cellColours=None, cellLoc='right', colWidths=None, rowLabels=rows, rowColours=None, rowLoc='left', colLabels=columns, colColours=None, colLoc='center', loc='bottom', bbox=None, edges='closed', **kwargs) plot.show()
arabic_counter = Counter() for name in corpus: if 'mmha1' in name: mmha1.append(name) for file in mmha1: with open("{}{}".format(dir, name), 'r') as f, open('/Users/jtim/Desktop/out.txt', 'w') as out: out.write(file) out.write('\n') out.write("--------------------------------------") out.write('\n') out.write(f.read()) for file in mmha1: with open("{}{}".format(dir, name), 'r') as f, open('/Users/jtim/Desktop/out.txt', 'w') as out_c: words = Counter(process(f.read()).split()) arabic_counter.update(words) out_c.write(f.read()) out_c.write('\n') out_c.write("--------------------------------------") out_c.write('\n') out_c.write(str(words)) # print(len(mmha1)) # print(len(arabic_counter)) # print(arabic_counter.most_common(10000))
L = loads(L)[data][list] else: L = loads( L )[data] # f is a function used to generate data,content is extract data from news and desc is to extract news item self.list = map(f, L) #use map to enumerate self.content = content #save state self.desc = desc #save state def process( self ): #process have been changed to get the news , process function in helper.py then for url in self.list: #process it into format that bikeso can accept resultPage = fetch(url) result = loads(resultPage)[self.content] yield (result['title'], helper.process(result[self.desc])) #for test only if __name__ == '__main__': url = 'http://www.imxingzhe.com/api/v4/get_competitions?page=0&limit=500' def f(id): return 'http://www.imxingzhe.com/api/v4/competition_detail?competition_id=' + str( id['id']) tmp = AdvanceWeb(url, 'data', f, 'data', 'description') result = tmp.process() for i in result: print(process(BeautifulSoup(i, 'html.parser')))
data = json.loads(msg,strict=False) print (data) print('Incoming Email Loaded') except Exception as e: print ('Error while loading state....Exit-1 : ' + str(e)) sys.exit(1) #Fix String data Type text_body = data['text_body'].encode('ascii',errors = 'ignore').decode() subject = data['subject'].encode('ascii',errors = 'ignore').decode() frm = data['from']['name'] #Get Email Data structure and intent emails = fx.process(text_body,frm,subject) intent = fx.getIntent(emails) #Prepare the doc doc = {} doc['emails'] = emails doc['intent'] = intent if (intent.get('intent')): doc['request'] = req[intent['intent']] #update the collection result = collection.update_one({'caseid':data['caseid']}, {"$set":doc}, upsert=False) #send Message to reply queue url = "https://sd-ui.azurewebsites.net/task/"+data['caseid']
import helper as app if __name__ == '__main__': app.init() app.process() app.end()
def file_to_string(filenames): strings = [] for filename in filenames: with open(filename) as f: strings.append(process(process_kitab(f.read()))) return '\n'.join(strings)
parser.add_argument('--test_path', default='data/eval.iob') parser.add_argument('--patience', type=int, default=10) parser.add_argument('--number_normalized', type=bool, default=True) parser.add_argument('--use_crf', type=bool, default=True) args = parser.parse_args() use_gpu = torch.cuda.is_available() print('use_crf:', args.use_crf) if not os.path.exists(args.savedir): os.makedirs(args.savedir) test_path = '/content/test.iob' text = input("Enter the sentence :") print(text) process(text, test_path) eval_path = "evaluation" eval_temp = os.path.join(eval_path, "temp") if not os.path.exists(eval_temp): os.makedirs(eval_temp) test_pred_file = eval_temp + '/test_pred.txt' # Loading the vocabulary model_name = args.savedir + '/' + args.feature_extractor + str( args.use_char) + str(args.use_crf) word_vocab = WordVocabulary(args.train_path, args.dev_path, args.test_path, args.number_normalized) label_vocab = LabelVocabulary(args.train_path) alphabet = Alphabet(args.train_path, args.dev_path, args.test_path)