def test_train(self): model = NumberRecognizeNN(Resource.INPUT_SIZE, Resource.OUTPUT_SIZE) r = Resource(self.TEST_DIR) trainer = Trainer(model, r) dp = DataProcessor() data, target = r.load_training_data() print("Test Train the model") trainer.train(data, target, epoch=5)
def init_trainer(): class Pack: pass tp = Pack() tp.nb_epoch = 50 tp.input_dimX = 28 tp.input_dimY = 28 tp.nb_classes = 46 + 1 # +1 for "don't know = * " class train_csv = "data/dataset/all_combined_diluted.csv" prep = ImageDataGenerator( featurewise_center=True, # set input mean to 0 over the data samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=True, # divide inputs by std of the data samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range= 30, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range= 0.1, # randomly shift images horizontally (fraction of total width) height_shift_range= 0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=False, # don't horizontally flip images vertical_flip=False) # don't vertically flip images my_trainer = Trainer(train_csv=train_csv, test_csv=None, converters=None, nan_handlers=None, empty_str_handlers=None, training_parameters=tp, preprocessor=prep) return my_trainer
def test_model_api(self): model = NumberRecognizeNN(Resource.INPUT_SIZE, Resource.OUTPUT_SIZE) r = Resource(self.TEST_DIR) trainer = Trainer(model, r) dp = DataProcessor() data, target = r.load_training_data() api_test_size = 200 print("Train the model for API Test.") trainer.train(data[:-api_test_size], target[:-api_test_size], epoch=5) model_api = ModelAPI(r) predicted = model_api.predict(data[-api_test_size:]) teacher = target[-api_test_size:] score = accuracy_score(teacher, predicted) print("Model API score is {}".format(score))
def post(self): resp = {"result": str(-1)} data = self.get_arguments("data[]") r = Resource() if not os.path.isdir(r.model_path): from ml.model import NumberRecognizeNN from ml.trainer import Trainer model = NumberRecognizeNN(r.INPUT_SIZE, r.OUTPUT_SIZE) trainer = Trainer(model, r) x, y = r.load_training_data() trainer.train(x, y) api = ModelAPI(r) if len(data) > 0: _data = [float(d) for d in data] predicted = api.predict(_data) resp["result"] = str(predicted[0]) self.write(resp)
def train(data_file, batch_size, epoch, test_size): r = Resource() dp = DataProcessor() model = NumberRecognizeNN(Resource.INPUT_SIZE, Resource.OUTPUT_SIZE) try: dp.means, dp.stds = r.load_normalization_params() r.load_model(model) print("load the model") except Exception as ex: print("trained model does not exist.") x = None y = None if data_file: x, y = r.load_data(data_file) else: x, y = r.load_training_data() trainer = Trainer(model, r) print("begin training") trainer.train(x, y, batch_size=batch_size, epoch=epoch, test_size=test_size)
def post(self): # レスポンス用マップを作成 resp = {"result": str(-1)} # 送られてきたデータを受け取る data = self.get_arguments("data[]") r = Resource() # path が実在するディレクトリの場合 if not os.path.isdir(r.model_path): # model.pyのインポート from ml.model import NumberRecognizeNN # trainer.pyのインポート from ml.trainer import Trainer # モデル作成クラス (Chainer)をnew model = NumberRecognizeNN(r.INPUT_SIZE, r.OUTPUT_SIZE) # トレーニングクラスをnew trainer = Trainer(model, r) # トレーニングデータを取得 x, y = r.load_training_data() # トレーニング実施 trainer.train(x, y) # モデルをAPIに保存 api = ModelAPI(r) if len(data) > 0: _data = [float(d) for d in data] # 送られてきたデータから予測値を算出 predicted = api.predict(_data) # レスポンスにデータを格納 resp["result"] = str(predicted[0]) # データを送信 self.write(resp)
def run(logdir, model_name, opt_name, loss_name): logger.info(f'model_name: {model_name}') logger.info(f'opt_name: {opt_name}') model = get_model(model_name)( input_shape=(9, 9, 43), classes=MOVE_DIRECTION_LABEL_NUM, activation_name=FLAGS.activation_name) logger.info(model.summary()) if FLAGS.saved_model_path: model = tf.keras.models.load_model(FLAGS.saved_model_path) optimizer = get_optimizer(opt_name) loss = get_loss(loss_name) trainer = Trainer(model, logdir, optimizer, loss, factor=FLAGS.reduce_factor, patience=FLAGS.reduce_patience, min_lr=FLAGS.reduce_factor) with trainer.summary_writer.as_default(): tf.summary.text('parameters', FLAGS.flags_into_string(), step=0) dataset = SFEN(FLAGS.train_csv, FLAGS.valid_csv, FLAGS.epochs, FLAGS.batch_size) # 3143460 if FLAGS.logging_steps: logging_steps = FLAGS.logging_steps else: logging_steps = dataset.train_size // FLAGS.batch_size logger.info(f'logging_steps: {logging_steps}') trainer.train(dataset, logging_steps, FLAGS.learning_rate) trainer.save()
def __init__(self, phone, serverPhone=''): ### redacted self.trainer = Trainer() self.trainer.setup() self.maxList = 5 self.previousMessage = "" self.phone = phone self.serverPhone = serverPhone self.history = [] #List of all previous groups of interactions DELETE self.queueToAsk = [] self.unansweredQuestionTuples = [] self.answeredQuestionTuples = [] self.forward_seen = 0 self.current_forward_questions = [] self.toForwardId = None self.latestPostTime = 0 self.questionToAsk = None self.current = None self.title = "" self.topic = None self.tags = [] self.similarMessage = "" self.similar_questions = [] self.similarCurrent = "" self.acceptedTopics = ["math", "politics", "biology", "cooking", "history", "cs"] self.examples = { 'math': 'Why does the monotone convergence theorem not apply on Riemann integrals?', 'politics': 'What will the consequences of Brexit be on EURONEXT?', 'biology': 'How do homologous pairs find each other?', 'cooking': 'What happens if I use vegetable oil in a cake?', 'history': 'How Did the Mongol Empire Get So Big?', 'cs': 'Proving that the conversion from CNF to DNF is NP-Hard' } self.exampleBodies = { 'math': "I had just learned in measure theory class about the monotone convergence theorem in this version:\nFor every monotonically increasing series of functions fn from measurable space X to [0, infinity],\n\nI tried to find out why this theorem apply only for a Lebesgue integral, but I didn't find a counter example for Riemann integrals, so I would appreciate your help.\n(I guess that f might not be integrable in some cases, but I want a concrete example.)", 'politics': "I've heard that Euronext (which is related to financial products of France, Netherland,...) was physically located near London.\nSuppose that a 'hard' brexit happens. Will be any consequence about the future location of Euronext (even partially).\n\nAnd if it's the case: Will be consequences on the connectivity of the european market place to the other ones?", 'biology': "If I'm not mistaken the only time homologous pairs of chromosomes need to find each other is during gamete formation in preparation for crossover recombination. How do they find each other?", 'cooking': "I want to bake a cake out of baking mix bought at a shop. The mix already contains flour, baking soda, vanillin, other minor components. The manual says I need to add butter.\n\nI'd like to use vegetable oil instead of butter - most likely sunflower seed oil. Is that a good idea? Should I expect any problems? Will the result likely be a decent cake or something to just throw away?", 'history': "Undoubtedly, the largest empire in the existence of human history was the Mongol Empire, once a hodgepodge of warring nomadic tribes from Central Asia before banding together under the banner of Temujin, better known as Genghis Khan.\nBut what was the key to the Mongol Empire's size? Was the horse, the most ideal creature to use for long-distance travel, the exclusive reason? Or were there other factors involved?How Did the Mongol Empire Get So Big?", 'cs': "How can I prove that the conversion from CNF to DNF is NP-Hard?\nI'm not asking for an answer, just some suggestions about how to go about proving it." } self.responses = { 'helloK': '\nWelcome to Qmessage, the messaging service where you can have your questions answered by real people!\n\nReply 1 (ONE) to ask a new question.\nReply 2 (TWO) to forward a previously asked question (and answer) to another contact.\nReply 3 (THREE) for help.', 'askK': "\nYou have selected the topic '{}'.\n\nReply with the title of your question (short and descriptive sentence with appropriate vocabulary).\nReply EXAMPLE to see an example question title.\nReply EXIT to exit.", 'askJ': "\nReply with the topic that corresponds to your question, from the following list: math, politics, biology, cooking, history, or cs (computer science).\nReply EXIT to exit.", 'exampleAsk':"\nA title is a preview or summary of your question, such as '{}'\n", 'askYes':"\nThe title of your question is '{}'.\n\nReply with a descriptive body for your question, so we can post it to the forum and wait for someone to answer it.\nReply EXAMPLE to see an example.\nReply EXIT to exit.", 'askNo': "\nOk. Have a good day! Goodbye for now.", 'askBrowsing':"\nReply 1 (ONE) to rephrase your question under the same topic.\nReply 2 (TWO) to ask a completely new question.\nReply 3 (THREE) in order to post your current question to the forum.\nReply EXIT to exit.", 'forwardK': '\nHere are the titles of your most recently answered questions.\n\nReply with the corresponding question number you wish to forward.\nReply MORE to see older questions.\nReply EXIT to exit.\n\n', 'helpK': '\nQmessage is an SMS-based service that allows you to post questions on a platform where thousands of people can look at and answer them. Any inquiry you have, on topics ranging from biology to cooking, may be answered! If we find similar questions to yours that have already been answered, we forward them immediately; otherwise, we post your question and wait until someone answers it. Keep in mind that this may take a few hours, depending on the relevance of the topic.\nWe also support question sharing, so that you can send your past questions (with answers) to your friends and family.\nTo use the service, simply follow the instructions at each step. Thank you for using Qmessage!', 'errorK': '\nI do not understand your message. Please try another response.\n', 'exampleQ': "\nHere is an example of a complete question:\n\nTitle:\n{}\n\nBody:\n{}\n\n", 'confirmQ': '\nYour question has been sent in! We will notify you when we receive a response. Goodbye for now.', 'forwNum': '\nYou have selected the following question: {}.\n\nReply with the phone number of the person you would like to forward this question and answer to, including the international area code (e.g. +16034445678).\nReply EXIT to exit.', 'similarQ': '\nWe found some questions similar to yours! Below are their titles.\n\nReply with the corresponding question number to look at the answer.\nReply NONE if none of them seem relevant, and we can browse other questions or post yours and wait for an answer.\nReply EXIT to exit.', 'confirmSimilar': "\nBelow is the top-ranked answer to the question\n{}\nReply 1 (ONE) if this was the answer you were looking for.\nReply 2 (TWO) to return to the list of similar questions.\nReply EXIT to exit.\n", 'confirmSimilarExit': "\nGreat! Have a nice day! Goodbye for now.", 'errorQ': '\nThe platform did not accept your question. This is their message:', 'unansweredQ': '\nHi! Someone has answered the following question:\n\n', 'noMoreForwards': '\nAt the moment, you have no more answered questions available to forward.', 'confirmForward': '\nThank you! We have forwarded the information to your contact. Goodbye for now.' }
class User: def __init__(self, phone, serverPhone=''): ### redacted self.trainer = Trainer() self.trainer.setup() self.maxList = 5 self.previousMessage = "" self.phone = phone self.serverPhone = serverPhone self.history = [] #List of all previous groups of interactions DELETE self.queueToAsk = [] self.unansweredQuestionTuples = [] self.answeredQuestionTuples = [] self.forward_seen = 0 self.current_forward_questions = [] self.toForwardId = None self.latestPostTime = 0 self.questionToAsk = None self.current = None self.title = "" self.topic = None self.tags = [] self.similarMessage = "" self.similar_questions = [] self.similarCurrent = "" self.acceptedTopics = ["math", "politics", "biology", "cooking", "history", "cs"] self.examples = { 'math': 'Why does the monotone convergence theorem not apply on Riemann integrals?', 'politics': 'What will the consequences of Brexit be on EURONEXT?', 'biology': 'How do homologous pairs find each other?', 'cooking': 'What happens if I use vegetable oil in a cake?', 'history': 'How Did the Mongol Empire Get So Big?', 'cs': 'Proving that the conversion from CNF to DNF is NP-Hard' } self.exampleBodies = { 'math': "I had just learned in measure theory class about the monotone convergence theorem in this version:\nFor every monotonically increasing series of functions fn from measurable space X to [0, infinity],\n\nI tried to find out why this theorem apply only for a Lebesgue integral, but I didn't find a counter example for Riemann integrals, so I would appreciate your help.\n(I guess that f might not be integrable in some cases, but I want a concrete example.)", 'politics': "I've heard that Euronext (which is related to financial products of France, Netherland,...) was physically located near London.\nSuppose that a 'hard' brexit happens. Will be any consequence about the future location of Euronext (even partially).\n\nAnd if it's the case: Will be consequences on the connectivity of the european market place to the other ones?", 'biology': "If I'm not mistaken the only time homologous pairs of chromosomes need to find each other is during gamete formation in preparation for crossover recombination. How do they find each other?", 'cooking': "I want to bake a cake out of baking mix bought at a shop. The mix already contains flour, baking soda, vanillin, other minor components. The manual says I need to add butter.\n\nI'd like to use vegetable oil instead of butter - most likely sunflower seed oil. Is that a good idea? Should I expect any problems? Will the result likely be a decent cake or something to just throw away?", 'history': "Undoubtedly, the largest empire in the existence of human history was the Mongol Empire, once a hodgepodge of warring nomadic tribes from Central Asia before banding together under the banner of Temujin, better known as Genghis Khan.\nBut what was the key to the Mongol Empire's size? Was the horse, the most ideal creature to use for long-distance travel, the exclusive reason? Or were there other factors involved?How Did the Mongol Empire Get So Big?", 'cs': "How can I prove that the conversion from CNF to DNF is NP-Hard?\nI'm not asking for an answer, just some suggestions about how to go about proving it." } self.responses = { 'helloK': '\nWelcome to Qmessage, the messaging service where you can have your questions answered by real people!\n\nReply 1 (ONE) to ask a new question.\nReply 2 (TWO) to forward a previously asked question (and answer) to another contact.\nReply 3 (THREE) for help.', 'askK': "\nYou have selected the topic '{}'.\n\nReply with the title of your question (short and descriptive sentence with appropriate vocabulary).\nReply EXAMPLE to see an example question title.\nReply EXIT to exit.", 'askJ': "\nReply with the topic that corresponds to your question, from the following list: math, politics, biology, cooking, history, or cs (computer science).\nReply EXIT to exit.", 'exampleAsk':"\nA title is a preview or summary of your question, such as '{}'\n", 'askYes':"\nThe title of your question is '{}'.\n\nReply with a descriptive body for your question, so we can post it to the forum and wait for someone to answer it.\nReply EXAMPLE to see an example.\nReply EXIT to exit.", 'askNo': "\nOk. Have a good day! Goodbye for now.", 'askBrowsing':"\nReply 1 (ONE) to rephrase your question under the same topic.\nReply 2 (TWO) to ask a completely new question.\nReply 3 (THREE) in order to post your current question to the forum.\nReply EXIT to exit.", 'forwardK': '\nHere are the titles of your most recently answered questions.\n\nReply with the corresponding question number you wish to forward.\nReply MORE to see older questions.\nReply EXIT to exit.\n\n', 'helpK': '\nQmessage is an SMS-based service that allows you to post questions on a platform where thousands of people can look at and answer them. Any inquiry you have, on topics ranging from biology to cooking, may be answered! If we find similar questions to yours that have already been answered, we forward them immediately; otherwise, we post your question and wait until someone answers it. Keep in mind that this may take a few hours, depending on the relevance of the topic.\nWe also support question sharing, so that you can send your past questions (with answers) to your friends and family.\nTo use the service, simply follow the instructions at each step. Thank you for using Qmessage!', 'errorK': '\nI do not understand your message. Please try another response.\n', 'exampleQ': "\nHere is an example of a complete question:\n\nTitle:\n{}\n\nBody:\n{}\n\n", 'confirmQ': '\nYour question has been sent in! We will notify you when we receive a response. Goodbye for now.', 'forwNum': '\nYou have selected the following question: {}.\n\nReply with the phone number of the person you would like to forward this question and answer to, including the international area code (e.g. +16034445678).\nReply EXIT to exit.', 'similarQ': '\nWe found some questions similar to yours! Below are their titles.\n\nReply with the corresponding question number to look at the answer.\nReply NONE if none of them seem relevant, and we can browse other questions or post yours and wait for an answer.\nReply EXIT to exit.', 'confirmSimilar': "\nBelow is the top-ranked answer to the question\n{}\nReply 1 (ONE) if this was the answer you were looking for.\nReply 2 (TWO) to return to the list of similar questions.\nReply EXIT to exit.\n", 'confirmSimilarExit': "\nGreat! Have a nice day! Goodbye for now.", 'errorQ': '\nThe platform did not accept your question. This is their message:', 'unansweredQ': '\nHi! Someone has answered the following question:\n\n', 'noMoreForwards': '\nAt the moment, you have no more answered questions available to forward.', 'confirmForward': '\nThank you! We have forwarded the information to your contact. Goodbye for now.' } def next_state(self, message): print('NEW LOOP. State: {}, message: {}\n'.format(self.current, message)) # GENERIC EXIT if message.upper() == "EXIT" or message.upper() == "EXIT ": return ('exit', self.responses['askNo']) if self.current is None or self.current == 'exit': #or self.current is 'error': self.topic = None self.forward_seen = 0 self.current_forward_questions = [] self.toForwardId = None return ('hello', self.responses['helloK']) elif self.current == 'hello': #ASK if '1' == message or 'ONE' in message.upper(): return ('askTopic', self.responses['askJ']) #FORWARD elif '2' == message or 'TWO' in message.upper(): questions = self.generate_forward_options() if len(questions) > 0: self.current_forward_questions = questions message = self.responses['forwardK'] for i, q in enumerate(questions): message += '{}. {}'.format(i+1, htmlToText(q['question_title'])) return ('forward', message) else: self.forward_seen = 0 return ('hello', self.responses['noMoreForwards'] + '\n' + self.responses['helloK']) #HELP elif '3' == message or 'THREE' in message.upper(): return ('hello', self.responses['helpK'] + '\n' + self.responses['helloK']) elif self.current == 'askTopic': if message.lower() in self.acceptedTopics: self.topic = message.lower() return ('ask', self.responses['askK'].format(self.topic)) elif self.current == 'ask': if message.upper() == "EXAMPLE" or message.upper() == "EXAMPLE ": return ('ask', self.responses['exampleAsk'].format(self.examples[self.topic]) + self.responses['askK'].format(self.topic)) elif len(message) < 15: return ('ask', '\nQuestion titles must contain at least 15 characters.\n' + self.responses['askK'].format(self.topic)) else: self.title = message try: # Actually found some similar questions self.tags = self.predict_tags_for_text(self.title) similar = self.similar_qs(self.title, self.tags) #return ('similar', similar) similar = similar[:5] if len(similar) > 0: self.similar = similar message = self.responses['similarQ'] + '\n\n' for i, s in enumerate(similar): message += '{}. {}'.format(i + 1, htmlToText(s['question_title'])) self.similarMessage = message return ('similar', self.similarMessage) #we're only doing one set of questions for this version. Additional similar questions coming in v2. # No similar questions else: return ('askPath', "\nWe found no previously asked questions that resemble yours.\n" + self.responses["askBrowsing"]) except Exception as e: return ('ask', '{}\n{}\n{}\n'.format(self.responses['errorQ'], str(e.args[-1]), self.responses['askK'].format(self.topic))) #ALTERNATIVE elif self.current == 'askPath': # REPHRASE QUESTION if "ONE" in message.upper() or "1" == message: return ('ask', self.responses['askK'].format(self.topic)) # POST QUESTION elif "TWO" in message.upper() or "2" == message: return ('askTopic', self.responses['askJ']) elif "THREE" in message.upper() or "3" == message: return ('askScreen', self.responses['askYes'].format(self.title)) elif self.current == 'askScreen': if "EXAMPLE" == message.upper(): return ('askScreen', self.responses['exampleQ'].format(self.examples[self.topic], self.exampleBodies[self.topic]) + self.responses['askYes'].format(self.title)) elif len(message) < 30: return ('askScreen', '\nQuestion bodies must contain at least 30 characters.\n' + self.responses['askYes'].format(self.title)) else: body = message self.body = body try: self.ask_question(self.topic, self.title, self.body, self.tags) return ("exit", self.responses["confirmQ"]) except Exception as e: return ('askScreen', '{}\n{}\n{}\n'.format(self.responses['errorQ'], str(e.args[-1]), self.responses['askYes'].format(self.title))) elif self.current == 'forward': #FORWARD SELECTED if message.isdigit(): i = int(message) - 1 ##print 'In forwarding, with digit' if i < len(self.current_forward_questions) and i >= 0: self.toForward = self.current_forward_questions[i] self.forward_seen = 0 return ('provideNum', self.responses['forwNum'].format(self.toForward['question_title'])) #MORE FORWARD elif 'MORE' in message.upper(): questions = self.generate_forward_options() if len(questions) > 0: self.current_forward_questions = questions message = self.responses['forwardK'] for i, q in enumerate(questions): message += '{}. {}'.format(i+1, htmlToText(q['question_title'])) return ('forward', message) else: self.forward_seen = 0 return ('hello', self.responses['noMoreForwards'] + '\n' + self.responses['helloK']) elif self.current == 'provideNum': q = self.toForward body = 'Hi, this is Qmessage! User {} has asked to forward you the following question and answer:\n\n'.format(self.phone) body += 'Question Title: {}\n{}'.format(htmlToText(q['question_title']), htmlToText(q['answers'][0])) try: self.send_message(body, message) return ('exit', self.responses['confirmForward']) except Exception as e: return ('provideNum', '\nWe were unable to forward the question. Please make sure that the phone number is valid.\n{}'.format(self.responses['forwNum'].format(self.toForward['question_title']))) self.toForward = None elif self.current == 'similar': if message.isdigit(): i = int(message) - 1 if i < len(self.similar) and i >= 0: s = self.similar[i] self.similarCurrent = '\n{}\n{}'.format(htmlToText(s['question_title']), htmlToText(s['answers'][0])) message = self.responses['confirmSimilar'].format(self.similarCurrent) return ('confirmSim', message) else: return ('similar', "Please enter a valid index\n\n" + self.similarMessage) elif 'NONE' in message.upper(): # ALTERNATIVE return ('askPath', self.responses["askBrowsing"]) elif self.current == 'confirmSim': if message.upper() == "ONE" or message == "1": self.similar = [] self.similarCurrent = "" return ('exit', self.responses['confirmSimilarExit']) elif message.upper() == "TWO" or message == "2": return ('similar', self.similarMessage) print("Previous: {}".format(self.previousMessage)) return (self.current, self.responses['errorK'] + self.previousMessage.replace(self.responses['errorK'], "")) def generate_forward_options(self): print('in generate. answered: {}, seen: {}'.format(len(self.answeredQuestionTuples), self.forward_seen)) if self.forward_seen >= len(self.answeredQuestionTuples): return [] next_batch = self.answeredQuestionTuples[self.forward_seen:][:self.maxList] print(next_batch) forwards = [] for tup in next_batch: forwards += get_questions_and_answers(tup[0], [tup[1]]) self.forward_seen += len(next_batch) return forwards def process_message(self, message): self.history.append(message) curr_state, to_usr = self.next_state(message) self.current = curr_state return to_usr def process_unanswered(self): if len(self.unansweredQuestionTuples) > 0: print('processing unanswered: {}'.format(self.unansweredQuestionTuples)) for topic, qid in self.unansweredQuestionTuples: questions = get_questions_and_answers(topic, [qid]) if len(questions) > 0: q = questions[0] new_tuples = [i for i in self.unansweredQuestionTuples if i[1] != qid] self.unansweredQuestionTuples = new_tuples message = self.responses['unansweredQ'] message += 'Question Title: {}'.format(htmlToText(q['question_title'])) message += 'Answers:\n' for i, a in enumerate(q['answers']): message += '{}. {}\n'.format(i+1, htmlToText(a)) self.answeredQuestionTuples.append((topic, qid)) self.send_message(message) print('after unanswered: {}'.format(self.unansweredQuestionTuples)) def process_queue(self, latestPostTime): now = int(time.time()) print('latestPostTime: {}'.format(latestPostTime)) if (now - latestPostTime) / 60 > 40 and len(self.queueToAsk) > 0: q = self.queueToAsk.pop(0) try: self.ask_question(q[0], q[1], q[2], q[3]) except: pass def send_message(self, body, receiver=None): if receiver is None: receiver = self.phone print('sending {} bytes from {} to {}'.format(len(body), self.serverPhone, receiver)) account_sid = '' ### redacted auth_token = '' ### redacted client = Client(account_sid, auth_token) body = (body[:1550] + '...') if len(body) > 1550 else body body = '.\n{}'.format(body) message = client.messages.create( body=body, from_=self.serverPhone, to=receiver ) def ask_question(self, topic, title, body, tags): try: qid = post_question(topic, title, body, tags) print('Adding question to unanswered questions: {}'.format(qid)) self.unansweredQuestionTuples.append((self.topic, str(qid))) self.latestPostTime = int(time.time()) except Exception as e: print(str(e.args[-1])) if 'You can only post once every 40 minutes.' == str(e.args[-1]): print('Adding question to pending queue!!!') self.queueToAsk.append((topic, title, body, tags)) else: message = str(e.args[-1]).replace("You can only post once every 40 minutes.",""); raise Exception(message) def predict_tags_for_text(self, question): try: tags = self.trainer.predict_tags_for_text(self.topic, question) ###UNCOMMENT FOR FINAL VERSION return tags except Exception as e: print(str(e.args[-1])) return [] def similar_qs(self, title, tags): try: similar = find_similar(self.topic, title, tags) asciiSimilar = [] for s in similar: if htmlToText(s['question_title']) != 'NA' and htmlToText(s['answers'][0]) != 'NA': asciiSimilar.append(s) return asciiSimilar except Exception as e: print(str(e.args[-1])) return []
def train_forever(): global DROPOUT_RATES train_csv = DATASET_PATH nb_epoch = 100 current_i = 0 start_over = False class Pack: pass tp = Pack() tp.nb_epoch = nb_epoch tp.input_dimX = 28 tp.input_dimY = 28 tp.nb_classes = 46 + 1 # +1 for "don't know = * " class cnn_drop = 0.01 dense_drop = 0.1 max_cnn_drop = 0.4 max_dense_drop = 0.5 prep = ImageDataGenerator( featurewise_center=True, # set input mean to 0 over the data samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=True, # divide inputs by std of the data samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range=20, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range=0.2, # randomly shift images horizontally (fraction of total width) height_shift_range=0.2, # randomly shift images vertically (fraction of total height) horizontal_flip=False, # randomly flip images vertical_flip=False) # randomly flip images my_trainer = Trainer(train_csv=train_csv, test_csv=None, converters=None, nan_handlers=None, empty_str_handlers=None, training_parameters=tp, preprocessor=prep) files = glob.glob(PATH) if files is None or len(files) == 0 or start_over: batch_train(my_trainer, None, PATH + FILE_NAME_PREFIX + '%d_epoch.hdf5' % nb_epoch, dropout_rates=DROPOUT_RATES) current_i = nb_epoch old_score = [10000, 0] while True: for i in range(current_i, 1999, nb_epoch): print("epoch:%d" % i) if i == 0 or random.random(5) < 2: model_name_to_load = PATH + FILE_NAME_PREFIX + "_best.hdf5" # + str(i) + '_epoch.hdf5' else: model_name_to_load = PATH + FILE_NAME_PREFIX + str(i) + '_epoch.hdf5' model_name_to_save = PATH + FILE_NAME_PREFIX + str(i + nb_epoch) + '_epoch.hdf5' print("old DROPOUT_RATES: " + ",".join([str(x) for x in DROPOUT_RATES])) score = batch_train(my_trainer, model_name_to_load, model_name_to_save, nb_epoch=nb_epoch, dropout_rates=DROPOUT_RATES) if score[1] - old_score[1] < 0.0: # not enough momentum current_i = i if score[1] - old_score[1] < -0.05: current_i = i - nb_epoch if current_i < 15: current_i = 15 if random.randint(0, 10) < 5: nb_epoch += 1 tp.nb_epoch = nb_epoch my_trainer.training_parameters.nb_epoch = nb_epoch print("number of epochs=%d" % nb_epoch) max_cnn_drop += 0.005 cnn_drop += 0.001 if max_cnn_drop > 0.3: max_cnn_drop = 0.1 if cnn_drop > max_cnn_drop: cnn_drop = max_cnn_drop / 2.0 max_dense_drop += 0.01 dense_drop += 0.0015 if max_dense_drop > 0.75: max_dense_drop = 0.25 if dense_drop > max_dense_drop: dense_drop = max_dense_drop / 2.0 # random regularization param search DROPOUT_RATES = [random.uniform(cnn_drop, max_cnn_drop / 3), random.uniform(cnn_drop, max_cnn_drop / 4), random.uniform(dense_drop, max_dense_drop)] print("new DROPOUT_RATES: " + ",".join([str(x) for x in DROPOUT_RATES])) print(nb_epoch) break old_score = score