def main_training():
lexicon_loader = LexiconLoader()
scored_lexicon: dict = lexicon_loader.load_all_and_merge()
tr_tweets_loader = LabeledTweetsLoader(TRAINING_INPUT_FILENAME)
tr_labeled_tweets = tr_tweets_loader.parse_tokens_and_labels(
tr_tweets_loader.load_lines())
token_summarizer = TokenSummarizer(scored_lexicon)
feature_extractor = FeatureExtractor(scored_lexicon)
vu = VocabUtil()
nn_input_preparer = NNInputPreparer(vu)
tr_feature_vectors = [] # 2D array of feature vectors
for labeled_tweet in tr_labeled_tweets:
known_token_sequence = token_summarizer.get_known_tokens(
labeled_tweet[0])
feature_vector = feature_extractor.compute_feature_vector(
known_token_sequence)
tr_feature_vectors.append(feature_vector)
tr_network_input = np.array(tr_feature_vectors)
tr_targets = [labeled_tweet[1] for labeled_tweet in tr_labeled_tweets]
tr_targets_one_hot_encoded = nn_input_preparer.rectangular_targets_to_one_hot(
tr_targets)
dev_tweets_loader = LabeledTweetsLoader(DEV_INPUT_FILENAME)
dev_labeled_tweets = dev_tweets_loader.parse_tokens_and_labels(
dev_tweets_loader.load_lines())
dev_feature_vectors = [] # 2D array of feature vectors
for labeled_tweet in dev_labeled_tweets:
known_token_sequence = token_summarizer.get_known_tokens(
labeled_tweet[0])
feature_vector = feature_extractor.compute_feature_vector(
known_token_sequence)
dev_feature_vectors.append(feature_vector)
dev_network_input = np.array(dev_feature_vectors)
dev_targets = [labeled_tweet[1] for labeled_tweet in dev_labeled_tweets]
dev_targets_one_hot_encoded = nn_input_preparer.rectangular_targets_to_one_hot(
dev_targets)
# Every epoch is cheap (< 1ms), so we don't need the ability to continue training from a previous model.
print("Commencing new training run")
model_creator = ModelCreator(vu)
model = model_creator.create_two_dense_model(hidden_layer_size=HIDDEN_SIZE)
cp_filepath = BASE_DIR + 'ep_{epoch}_valacc_{val_accuracy:.5f}.h5'
checkpoint = ModelCheckpoint(cp_filepath,
monitor='val_accuracy',
verbose=1,
save_best_only=False)
model.fit(tr_network_input,
tr_targets_one_hot_encoded,
batch_size=32,
epochs=MAX_EPOCHS,
validation_data=(dev_network_input, dev_targets_one_hot_encoded),
callbacks=[checkpoint])
def model_creator(stream, event, event_close, dataset_name, log_lvl_number):
signal.signal(signal.SIGINT, signal.SIG_IGN)
try:
mc = ModelCreator(stream,
event_close,
sending_events=False,
event_filename=dataset_name + ".csv",
event_types_filename=dataset_name + ".json",
log_lvl=log_lvl_number)
event.set()
mc.start()
except Exception as e:
print("[ERROR] Unhandled exception in Profiler model creator module: {}".format(e))
def validate_all(sizes, droput_rates):
learn_histories_dict = {}
for row_idx, size in enumerate(sizes):
for col_idx, rate in enumerate(droput_rates):
print(f'class_first_size_{size}_dr_rate{rate}')
model_creator = ModelCreator()
model_creator.load_base_pretrained_model()
model_creator.read_prepared_data()
model_creator.train_last_fully_connected_layer(
final_training_mode=True,
first_classif_layer_size=size,
dropout_rate=rate,
logs_file=f'class_first_size_{size}_dr_rate{rate}.csv')
learn_histories_dict[(size, rate)] = model_creator.learn_history
dd.draw_learning_history(
model_creator.learn_history,
title='Uczenie części klasyfikującej',
out_filename=f'class_first_size_{size}_dr_rate{rate}')
dd.draw_learning_history_top_k(
model_creator.learn_history,
title='Uczenie części klasyfikującej',
out_filename=f'class_first_size_{size}_dr_rate{rate}')
dd.draw_two_used_params_values_comparision(
histories_dict=learn_histories_dict,
out_filename='sizes_and_rates_first_classif')
def validate_removal_coef(removal_coefs):
for removal_coef in removal_coefs:
model_creator = ModelCreator()
model_creator.load_base_pretrained_model()
model_creator.read_prepared_data()
model_creator.train_whole_convolutional_network(
removal_coef, final_training_mode=False)
dd.draw_learning_history(
model_creator.learn_history,
f'Wspolczynnik usunietych warstw: {removal_coef}')
model_creator.save_model(
f'models/{model_creator.base_model.name}_{AppParams.layers_trainable_mode.name}_{AppParams.epochs}_{removal_coef}.h5'
)
dd.draw_learning_history(model_creator.learn_history)
del model_creator
def validate_loss_functions(functions):
for func in functions:
model_creator = ModelCreator()
model_creator.load_base_pretrained_model()
model_creator.read_prepared_data()
model_creator.train_last_fully_connected_layer(
final_training_mode=False, loss=func)
dd.draw_learning_history(model_creator.learn_history,
f'Funkcja kary: {func}')
def validate_dropout_rates(dropout_rates):
learn_histories = []
for dropout_rate in dropout_rates:
model_creator = ModelCreator()
model_creator.load_base_pretrained_model()
model_creator.read_prepared_data()
model_creator.train_last_fully_connected_layer(
final_training_mode=False,
dropout_rate=dropout_rate,
logs_file=f'valid_dropout_{dropout_rate}.csv')
learn_histories.append(model_creator.learn_history)
dd.draw_used_params_values_comparision(histories=learn_histories,
values=dropout_rates,
out_filename='dropout_validation')
def validate_first_classif_sizes(sizes):
learn_histories = []
for size in sizes:
model_creator = ModelCreator()
model_creator.load_base_pretrained_model()
model_creator.read_prepared_data()
model_creator.train_last_fully_connected_layer(
final_training_mode=False,
first_classif_layer_size=size,
logs_file=f'valid_size_{size}.csv')
learn_histories.append(model_creator.learn_history)
dd.draw_used_params_values_comparision(histories=learn_histories,
values=sizes,
out_filename='sizes_validation')
def model_creator(event, dataset_name, log_lvl_number):
try:
mc = ModelCreator(MODEL_CREATOR_IN_ADDR_DICT,
event_filename=dataset_name + ".csv",
event_types_filename=dataset_name + ".json",
log_lvl=log_lvl_number)
event.set()
mc.start()
except KeyboardInterrupt:
mc.close()
def valuate_first_classif_sizes_from_last_conv(sizes):
learn_histories = []
for size in sizes:
model_creator = ModelCreator()
model_creator.load_base_pretrained_model()
model_creator.read_prepared_data()
model_creator.train_from_last_convolutional_layer(
final_training_mode=True,
first_classif_layer_size=size,
logs_file=f'valid_size_{size}_last_conv.csv')
learn_histories.append(model_creator.learn_history)
dd.draw_learning_history(model_creator.learn_history,
title='Uczenie od warstwy splotowej',
out_filename=f'last_conv_size_{size}')
dd.draw_learning_history_top_k(model_creator.learn_history,
title='Uczenie części klasyfikującej',
out_filename=f'last_conv_size_{size}')
dd.draw_used_params_values_comparision(
histories=learn_histories,
values=sizes,
out_filename='sizes_validation_last_conv')
from model_creator import ModelCreator
from tf_idf import TfIdf
# tf_idf = TfIdf('train.csv')
# print 'train.csv'
# tf_idf.tfidf('tr-idf-full.csv')
# print 'TfIdf'
# tf_idf.global_stats('global-full.csv')
# print 'global-stats'
mc = ModelCreator("tf-idf-full.csv")
print "model create"
mc.create()
# mt.test()
import datetime
from keras.engine.saving import load_model
import params_validator as pv
from app_params import AppParams
from draw_utils import draw_data as dd
from layers_trainable_modes import LayersTrainableMode
from model_creator import ModelCreator
from svm_model import train_svm
model_creator = ModelCreator()
model_creator.load_base_pretrained_model(
) # załadowanie wytrenowanego już modelu
model_creator.read_prepared_data(
) # odczytanie odpowiednio podzielonych danych
# ZADANIE 1
if AppParams.layers_trainable_mode is LayersTrainableMode.ONLY_CLASSIF:
model_creator.train_last_fully_connected_layer(
logs_file='only_classif.csv')
dd.draw_learning_history(model_creator.learn_history,
title='Uczenie części klasyfikującej',
out_filename='only_classif')
# ZADANIE 2
elif AppParams.layers_trainable_mode is LayersTrainableMode.FROM_LAST_CONV:
model_creator.train_from_last_convolutional_layer(
logs_file='from_last_conv.csv')
dd.draw_learning_history(model_creator.learn_history,
title='Uczenie od ostatniej warstwy splotowej',
def validate_optimizers(optimizers):
for optimizer in optimizers:
model_creator = ModelCreator()
model_creator.load_base_pretrained_model()
model_creator.read_prepared_data()
model_creator.train_last_fully_connected_layer()
def main():
pathname = "/home/kungangli/aes/data/hewlett/"
filename = "training_set_rel3.tsv"
dataloader = HewlettLoader(pathname, filename)
training_data = dataloader.read_training_data()
#scores, text = training_data["2"]['score'], training_data["2"]['essay']
scores, text = [], []
for i in training_data.keys():
scores += training_data[i]['score']
text += training_data[i]['essay']
# Shuffle to mix up the classes, set seed to make it repeatable
#random.seed(1)
#shuffled_scores = []
#shuffled_text = []
#indices = [i for i in xrange(0,len(scores))]
#random.shuffle(indices)
#for i in indices:
# shuffled_scores.append(scores[i])
# shuffled_text.append(text[i])
#text = shuffled_text[:TRAINING_LIMIT]
#scores = shuffled_scores[:TRAINING_LIMIT]
# Model creation and grading
score_subset = scores[:QUICK_TEST_LIMIT]
text_subset = text[:QUICK_TEST_LIMIT]
print "The score subset is: "
print score_subset
prompt1 = "More and more people use computers, but not everyone agrees that this benefits society. Those who support advances in technology believe that computers have a positive effect on people. They teach hand-eye coordination, give people the ability to learn about faraway places and people, and even allow people to talk online with other people. Others have different ideas. Some experts are concerned that people are spending too much time on their computers and less time exercising, enjoying nature, and interacting with family and friends. Write a letter to your local newspaper in which you state your opinion on the effects computers have on people. Persuade the readers to agree with you."
prompt2 = "Censorship in the Libraries. All of us can think of a book that we hope none of our children or any other children have taken off the shelf. But if I have the right to remove that book from the shelf -- that work I abhor --then you also have exactly the same right and so does everyone else. And then we have no books left on the shelf for any of us. -- Katherine Paterson, Author. Write a persuasive essay to a newspaper reflecting your vies on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading"
prompt7 = "Write about patience. Being patient means that you are understanding and tolerant. A patient person experience difficulties without complaining. Do only one of the following: write a story about a time when you were patient OR write a story about a time when someone you know was patient OR write a story in your own way about patience."
prompt8 = "We all understand the benefits of laughter. For example, someone once said, Laughter is the shortest distance between two people. Many other people believe that laughter is an important part of any relationship. Tell a true story in which laughter was one element or part."
prompt = prompt1 + " " + prompt2 + " " + prompt7 + " " + prompt8
model_creator = ModelCreator(score_subset, text_subset, prompt)
results = model_creator.create_model()
print "model creator results: "
print results["cv_kappa"]
print results["cv_mean_absolute_error"]
test_essay_testdataset = "I believe that computers have a positive effect on people. They help you stay in touch with family in a couple different ways they excercise your mind and hands and help you learn and make things easier. Computer's help you keep in touch with people. Say you live in @LOCATION1 and you miss your @CAPS1. You can just send an e-mail and talk all you want. If you don't just want to limit it to words you can add pictures so they can see how much you've grown or if you are well. Even if you're just e-mailing someone down the block it is just as effective as getting up and walking over there. You can also use a computer to make a scrap book card or slide show to show how much you love the person you give them to. Computers @MONTH1 not excercise you whole body but it excersises you mind and hands. You could play solitaire on the computer and come away @PERCENT1 smarter than before. You can play other games of strategy like checkers and chess while still sitting at home being comfortable. Your hands always play a big role while you're on the computer. They need to move the mouse and press the keys on a keyboard. Your hands learn all the keys from memorization. It's like the computer's teaching handi-coordination and studying habit for the future. Computers make human lives easier. Not only do they help kids turn in a nice neatly printed piece or paper for home work but they also help the average person. Teachers use it to keep peoples grades in order and others use it to write reports for various jobs. The @CAPS2 probably uses one to write a speech or to just keep his day in order. Computers make it easier to learn certain topics like the @LOCATION2 history. You can type something into a searcher site and have ton's of websites for one person with, who knows how much imformation. Instead of flipping through all the pages in a dictionary you can look for an online dictionary, type in the word and you have the definition. Computers have positive effects on people because they help you keep close to your family, they challenge your mind to be greater and excercise your hands and they make life easier for kids and the average person. This is why, I think computers have good effects on society"
test_essay_traindataset10 = "Dear @CAPS1, @CAPS2 you imagine a world where technology controls human interaction? Its not such a far fetched idea. Some people @MONTH1 see this as a great advancement. Not me. I think that computers detach people from reality, create false personas, and has no physical benefit. This is what i think the computers effects have on people. First of all, Computers detach people from reality. Its a lot easier for someone to sit on the computer, browsing aimlessly on some social networking website, than go out and actually meet new people. Sure you @CAPS2 send all the friend requests you want, but theres nothing that compares to a first impression that you get from meeting someone in person. The computer can also have an affect on family members and the time spent with them. Secondly, the computer creates false personas. If i had a facebook or @CAPS3 account, it would be very easy to say something false about myself, or someone else. The computer dosn't know better. This allows people to gain, ""friendships"" with others that dont know the real you. I find this as a real problem because eventually lead to shaky or unsteady relationshops. Lastly, I think that the computer offers no physical gain, This nation is in a severe heath crisis. With things easier to get now a days like fast food, and drugs, I don't think that computers help. People need discipline and theres a time and place for everything, including computers. However those that aren't disciplined might use them more than they need to and that isn't good. So @CAPS2 you imagine a world controlled by computers? I think that little by little, thats whats happening, slowly taking away the human element. I think the effects will be severe, detaching reality, creating false personas, and offering no physical gain. I don't think its right, but I hope I've persuaded you to think my way."
test_essay_offtopic = "To Sherlock Holmes she is always THE woman. I have seldom heard him mention her under any other name. In his eyes she eclipses and predominates the whole of her sex. It was not that he felt any emotion akin to love for Irene Adler. All emotions, and that one particularly, were abhorrent to his cold, precise but admirably balanced mind. He was, I take it, the most perfect reasoning and observing machine that the world has seen, but as a lover he would have placed himself in a false position. He never spoke of the softer passions, save with a gibe and a sneer. They were admirable things for the observer--excellent for drawing the veil from men's motives and actions. But for the trained reasoner to admit such intrusions into his own delicate and finely adjusted temperament was to introduce a distracting factor which might throw a doubt upon all his mental results. Grit in a sensitive instrument, or a crack in one of his own high-power lenses, would not be more disturbing than a strong emotion in a nature such as his. And yet there was but one woman to him, and that woman was the late Irene Adler, of dubious and questionable memory"
test_essay_pupil = "The first reason is my family. Over half of my family lives in New Jersey. When I visit, my cousins and I laugh and play all day and night. My uncles and aunts take me to the boardwalk where we ride roller coasters. We devour juicy caramel-covered apples and foot-long hot dogs. My family is fun to be with. The second reason for New Jersey being my favorite place is the weather. Instead of being hot and sweaty, it's always cool and moist. When I think about my visits, I can just feel the crisp fall breeze in my hair. I can just see the white, fluffy winter snow. I can just hear the soft spring trickles of rain splashing on the sidewalks. I can just feel the warm summer sun on my face. The weather is great! The third reason for New Jersey being my favorite place is crabbing. If it's crab season, we crab. We keep the blue crabs and the snow crabs, and we let the others go. Sometimes we catch crabs on hooks, and sometimes we lower crab cages into the bay. Then we pull them out later. One time my brother caught a crab so big that it got stuck in the crab cage! The crab finally got out, but it hurt one of its legs and broke the cage trying. Poor crab!"
test_essay_badessay = "I live in a house that every body in it came from acting. I remember my mom telling me this it you in find your self bad situation, don't forget your smile with 'you'. I think she ment that what ever is the difficulty think always positive. For an example, I grow up in place that full with bad poeple and one time some body try to convinse me to smoke. And smoking it very bad thing. So I started to tell joukes on people that canser and after 2 minutes I change the subject. Or that every time I am getting sick and fill not so good. I am trying to see comedy movies as much as I can. Because I have been told that comedy is the best cure. I think that as an actor on the stage you need to be always ready for something rong, and if you ready and prepard. It will be good and life for your self in you all life and not only there. This experience importent for your benfits, always a positive person and people will love you and get along with you. This mark it the best."
test_essay_randomhigh = "Kentish lyricised constringent gullibly farmerville dynamited lamaist bush. Sour agitate chemotherapist astrateia quorum olecranon filippino episcopize. Painfulness phenolic labyrinthodont assurbanipal denominatively lenses dreg. Innoxiously unsentineled intervaginal thorez separator hilum republication quinquagesima. Jam assafetida nim evincing dilatancy fellowship fasciately messy. Harmonised pout pedicellate froghopper reconform riskier demo bram. Sine racketlike homophonic cravenness halidome excursionary benny transequatorial. Necrotising prereversed waitress paperwork elbowroom ultramodern cauterization unsaturated. Unflaking greely erzerum hypochilia oppugnant recondemnation underspread noncoincident. Unaccepted pyrope laurelled magnific office foehn smoothhound absolutism. Explanator conker surrogated inframarginal hopscotch supremum excel flocci. Saintliness unfabricated remittence chink senecan conjectured beep prototypically."
test_essay_randomlow = "Add you viewing ten equally believe put. Separate families my on drawings do oh offended strictly elegance. Perceive jointure be mistress by jennings properly. An admiration at he discovered difficulty continuing. We in building removing possible suitable friendly on. Nay middleton him admitting consulted and behaviour son household. Recurred advanced he oh together entrance speedily suitable. Ready tried gay state fat could boy its among shall. Continual delighted as elsewhere am convinced unfeeling. Introduced stimulated attachment no by projection. To loud lady whom my mile sold four. Need miss all four case fine age tell. He families my pleasant speaking it bringing it thoughts. View busy dine oh in knew if even. Boy these along far own other equal old fanny charm. Difficulty invitation put introduced see middletons nor preference"
grader = Grader(results)
#results = grader.grade(text[10])
results = grader.grade(prompt)
print "model grader results prompt: "
print results
results = grader.grade(test_essay_offtopic)
print "model grader results test_essay_offtopic: "
print results
results = grader.grade(test_essay_testdataset)
print "model grader results test_essay_testdataset: "
print results
results = grader.grade(test_essay_traindataset10)
print "model grader results test_essay_traindataset10: "
print results
results = grader.grade(test_essay_pupil)
print "model grader results test_essay_pupil:"
print results
results = grader.grade(test_essay_badessay)
print "model grader results test_essay_badessay: "
print results
results = grader.grade(test_essay_randomhigh)
print "model grader results test_essay_randomhigh: "
print results
results = grader.grade(test_essay_randomlow)
print "model grader results test_essay_randomlow: "
print results