def main(train_models: bool, test_models: bool) -> None:
signal.signal(signal.SIGINT, signal_handler)
if train_models:
print('---------------- TRAINING -----------------')
for config in MODELS_CONFIG:
try:
train(config)
except SIGINTError:
print('\n\nTraining stopped. Skipping to the next model.\n\n')
except Exception as e:
print(e)
traceback.print_exc()
print('\n\nSkipping to the next model.\n\n')
if test_models:
test_output_dir = './test_results'
print('---------------- TESTING ----------------')
print('Warning! test_generation_200 disabled!')
test_one_step = TestLotterOneStep(MODELS_CONFIG, test_output_dir)
test_generation_20 = TestGeneration20(MODELS_CONFIG, test_output_dir)
test_generation_200 = TestGeneration200(MODELS_CONFIG, test_output_dir)
print('Test one-step-ahead')
test_one_step.run()
print('Test 20-step-ahead')
test_generation_20.run()
print('Test generation 200')
test_generation_200.run()
def main(args):
size = args.input_size
base_dir = os.path.join('data/', str(size))
if size == 12:
net = 'PNet'
net_factory = net_factorys[0]
end_epoch = FLAGS.end_epoch[0]
elif size == 24:
net = 'RNet'
net_factory = net_factorys[1]
end_epoch = FLAGS.end_epoch[1]
elif size == 48:
net = 'ONet'
net_factory = net_factorys[2]
end_epoch = FLAGS.end_epoch[2]
model_path = os.path.join('model/', net)
if not os.path.exists(model_path):
os.mkdir(model_path)
prefix = os.path.join(model_path, net)
display = FLAGS.display
lr = FLAGS.lr
train(net_factory,
prefix,
end_epoch,
base_dir,
display,
lr,
restore_model=False) # restore_model=False is added by sherk
def main(parser):
opt = parser.parse_args()
with torch.autograd.profiler.profile(use_nvprof=opt['use_nvprof'],
trace_path=opt['trace_path']) as prof:
train(parser)
print(prof.total_average())
sort_cpu = sorted(prof.key_averages(), key=lambda k: k.cpu_time)
sort_cuda = sorted(prof.key_averages(), key=lambda k: k.cuda_time)
def cpu():
for e in sort_cpu:
print(e)
def cuda():
for e in sort_cuda:
print(e)
cpu()
if opt['debug']:
print('`cpu()` prints out cpu-sorted list, '
'`cuda()` prints cuda-sorted list')
pdb.set_trace()
def train():
extract_embeddings.extract_embeddings_func("dataset",
"output/embeddings.pickle",
"face_detection_model",
"openface_nn4.small2.v1.t7")
train_model.train("output/embeddings.pickle", "output/recognizer.pickle",
"output/le.pickle")
return "blah"
def process():
s_print('[process] Starting Process While Loop')
while not exit_flag.wait(timeout=DETAILED_RECORD_PROCESSING_DELAY):
if q.qsize() >= DETAILED_PROCESSING_BATCH_SIZE:
count = DETAILED_PROCESSING_BATCH_SIZE
data = list()
while count > 0:
d = pop_queue()
data.append(d)
q.task_done()
count -= 1
train(data)
def main():
""" main """
parser = ConfigParser()
parser.read('config_spss.ini')
'''
# read train data
[train_data,train_targets,train_clv] = read_data.training_data(parser)
# read val data
[val_data,val_targets,val_clv] = read_data.validation_data(parser)
# read test data
[test_data,test_targets,test_clv] = read_data.test_data(parser)
'''
# read train data
[train_data, train_targets, train_clv] = gen_randdata.gen_randdata(parser)
# print(train_targets)
# read val data
[val_data, val_targets, val_clv] = gen_randdata.gen_randdata(parser)
# read test data
[test_data, test_targets, test_clv] = gen_randdata.gen_randdata(parser)
train_data_list = [train_data, train_targets, train_clv]
val_data_list = [val_data, val_targets, val_clv]
test_data_list = [test_data, test_targets, test_clv]
# Train DNN
# Config Variables
init_meth = parser['strs'].get('init_meth')
olfn = parser['strs'].get('oplayer_fn')
f = ['tanh', 'tanh', olfn]
# Ints
din = parser['ints'].getint('din')
dh1 = parser['ints'].getint('dh1')
dh2 = parser['ints'].getint('dh2')
dh3 = parser['ints'].getint('dh3')
dout = parser['ints'].getint('dout')
# Instatiate the class
l1 = srnlayer.SimpleRecurrentLayer(dh1, din, init_meth, parser)
l2 = srnlayer.SimpleRecurrentLayer(dh2, dh1, init_meth, parser)
l3 = feedforwardlayer.FeedForwardLayer(dout, dh2, init_meth)
l = [l1, l2, l3]
# Call train_mlp function
train_model.train(train_data_list, val_data_list, test_data_list, l, f,
parser)
# Garbage collect
gc.collect()
def Try(self, model_number):
self.train_temp = []
with open("update_train.txt") as tdata:
while True:
line = tdata.readline()
if not line:
break
self.train_temp.append([float(i) for i in line.split()])
self.train_temp = np.array(self.train_temp)
self.result_temp = []
with open("update_result.txt") as rdata:
while True:
line = rdata.readline()
if not line:
break
self.result_temp.append([float(i) for i in line.split()])
self.result_temp = np.array(self.result_temp)
self.y = np.mat(self.result_temp)
self.y = torch.tensor(self.y).float()
run = train_model.train()
run.update(model_number, self.train_temp, self.y)
# go = try_update()
# go.Try(1)
# go.Try(2)
def train_new_model(action_array):
print("_________________ CANNOT FIND A MATCH _______________")
# return 0
# append_new_model(action_array)
# print("+++++++++++++++++++ train(action_array) :",train(action_array))
# return random.uniform(0.8,1)
return train(action_array)
def main(epoch, learn_rate, train_rate, batch_size, l2):
# Download dataset and organize it
pd.prep_data()
# Pipeline
trainset = tf.data.Dataset.list_files('data/train/*.png')
trainset = trainset.shuffle(BUFFER_SIZE)
trainset = trainset.map(ld.load_train)
trainset = trainset.batch(1)
testset = tf.data.Dataset.list_files('data/test/*.png')
testset = testset.map(ld.oad_test)
testset = testset.batch(1)
#
generator = GAN.Generator((256, 256, 3))
discriminator = GAN.Discriminator((256, 256, 3), (256, 256, 3))
gen_optimizer = tf.keras.optimizers.Adam(learning_rate=2e-4,
beta_1=0.5,
beta_2=0.99,
epsilon=epsilon)
dis_optimizer = tf.keras.optimizers.Adam(learning_rate=2e-4,
beta_1=0.5,
beta_2=0.99,
epsilon=epsilon)
# Checkpoint
checkpoint_dir = 'data/training_checkpoints'
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
checkpoint = tf.train.Checkpoint(generator_optimizer=gen_optimizer,
discriminator_optimizer=dis_optimizer,
generator=generator,
discriminator=discriminator)
# Train
tm.train(trainset, EPOCHS)
# Test
# restoring the latest checkpoint in checkpoint_dir
checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))
# Run the trained model on the entire test dataset
for inp, tar in testset.take(5):
generate_images(generator, inp, tar)
def auth_process():
name = get_auth_from_face() #grab name if we can
if name == "n/a": #no face in frame
return "n/a"
elif name == "unknown": #unknown face in frame
to_speak = "want to make a profile?"
mouth.speak_aloud(to_speak)
if ears.listen_for_trigger(["yes", "yeah", "okay", 'sure'], time=3):
to_speak = "What would you like me too call you?"
mouth.speak_aloud(to_speak)
new_name = ears.listen_and_translate(time=5)
mouth.speak_aloud("Look into the Camera")
face_data(new_name, 1)
extract_embeddings()
train() #train the model
name = new_name
else:
to_speak = "Hello " + name + ", awaiting commands"
mouth.speak_aloud(to_speak)
mouth.speak_aloud(vc.decode_and_run(ears.listen_and_translate(5)))
def main():
# Parse input data
args = parsing_inputs()
# Obtain the dataloaders and a dictionary class_to_idx we will use during prediction
trainloader, validloader, testloader, class_to_idx = load_data(args)
# Now we download the model based on the input and select the device we will train it on
possible_inputs = {'vgg16': 25088, 'alexnet': 9216}
model, device = build_model(possible_inputs, args)
# The next step is to define the criterion and train the model
criterion = nn.NLLLoss()
train(model, device, args, trainloader, validloader, criterion)
# We then perform a validation test on new unseen data
with torch.no_grad():
validation_test(model, testloader, device, criterion)
# Finally we save the checkpoint
save_check(args, model, class_to_idx, possible_inputs)
def main(args):
prediction = args.prediction
train = args.train
if prediction:
test_path = args.data_dir
test_file = args.test_file
context = args.Context
res_dir = args.res_dir
test_SNPs = pd.read_csv(test_path + '/' + test_file, sep='\t')
test_SNPs = test_SNPs[['Chromosome','Start','End']]
data = score.get_score(test_SNPs)
'''
kde get transformed score
'''
trans_score = transform.transform(data, context)
'''
load WEVar model, and predict WEVar score
'''
res = model.WEVar(trans_score, context)
test_SNPs['WEVar_{}'.format(context)] = res
test_SNPs.to_csv('{}/{}'.format(res_dir,test_file), index=False, sep='\t')
elif train:
data_path = args.data_dir
train_file = args.train_file
train_SNPs = pd.read_csv(data_path + '/' + train_file, sep='\t')
X = train_SNPs[['Chromosome', 'Start', 'End']]
X = score.get_score(X)
Y = train_SNPs['Labels'].to_numpy()
train_model.train(X,Y, train_file)
train_model.test(X,Y, train_file)
def main(parser):
opt = parser.parse_args()
if opt['torch']:
with torch.autograd.profiler.profile() as prof:
train(parser)
print(prof.total_average())
sort_cpu = sorted(prof.key_averages(), key=lambda k: k.cpu_time)
sort_cuda = sorted(prof.key_averages(), key=lambda k: k.cuda_time)
def cpu():
for e in sort_cpu:
print(e)
def cuda():
for e in sort_cuda:
print(e)
cpu()
if opt['debug']:
print('`cpu()` prints out cpu-sorted list, '
'`cuda()` prints cuda-sorted list')
pdb.set_trace()
else:
pr = cProfile.Profile()
pr.enable()
train(parser)
pr.disable()
s = io.StringIO()
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
print(s.getvalue())
if opt['debug']:
pdb.set_trace()
def executeButtonFunction(self):
if "" == self.data_path.toPlainText():
print("Set Data Path")
return
if config["mode"] == "collect_image":
config["driver_path"] = os.getcwd() + r'\chromedriver_win32\chromedriver.exe'
config["image_save_path"] = self.data_path.toPlainText()
print(config["image_save_path"])
config["keyword"] = self.keyword.text()
download_google_staticimages(config)
elif config["mode"] == "train_model":
config["image_folder"] = self.data_path.toPlainText()
config["epoch"] = int(self.epoch.text())
config["learning_rate"] = float(self.learning_rate.text())
config["weight_decay"] = float(self.weight_decay.text())
train()
elif config["mode"] == "shopping_lense":
config["test_image"] = self.data_path.toPlainText()
test()
self.showResult()
def on_new_client(conn, addr):
data = conn.recv(1024)
data = data.decode("utf-8").strip()
file = data
global data2
data2 = "students/" + data + ".mp4"
print(data)
print(data2)
with open(data2, 'wb') as f:
print('file opened')
while True:
data = conn.recv(1024)
print('data=%s', (data))
if not data:
break
else:
f.write(data)
print('Done sending')
im.makeimage(file)
tr.train(file)
print('Got connection from', addr)
f.close()
def main(args):
size = args.input_size
base_dir = os.path.join('g:/mtcnn-dataset/data/', str(size))
if size == 12:
net = 'PNet'
net_factory = net_factorys[0]
end_epoch = FLAGS.end_epoch[0]
elif size == 24:
net = 'RNet'
net_factory = net_factorys[1]
end_epoch = FLAGS.end_epoch[1]
elif size == 48:
net = 'ONet'
net_factory = net_factorys[2]
end_epoch = FLAGS.end_epoch[2]
model_path = os.path.join('../model/', net)
if not os.path.exists(model_path):
os.mkdir(model_path)
prefix = os.path.join(model_path, net)
display = FLAGS.display
lr = FLAGS.lr
train(net_factory, prefix, end_epoch, base_dir, display, lr)
def start_train(model, train_iter, dev_iter, test_iter):
print("\n cpu_count \n", mu.cpu_count())
torch.set_num_threads(config.num_threads)
if os.path.exists("./Test_Result.txt"):
os.remove("./Test_Result.txt")
model_training, score = train_model.train(train_iter, dev_iter, test_iter,
model, config)
resultlist = []
if os.path.exists("./Test_Result.txt"):
file = open("./Test_Result.txt")
for line in file.readlines():
if line[:10] == "Evaluation":
resultlist.append(float(line[34:41]))
result = sorted(resultlist)
file.close()
file = open("./Test_Result.txt", "a")
file.write("\nThe Best Result is : " + str(result[len(result) - 1]))
file.write("\n")
file.close()
pt.pytesseract.tesseract_cmd = 'C:/Program Files (x86)/Tesseract-OCR/tesseract'
print('started!!!')
##create the game object
game = CHAIRFED()
print("game object created")
epoch = 2 # Number of games played in training,
##switch to toggle between train and test.
train_mode = 0
if train_mode == 1:
# Train the model
hist,loss = train(game, model, epoch, verbose=1)
print(loss)
np.savetxt('loss_history.txt', loss)
print("Training done")
else:
# Test the model
hist = test(game, model, epoch, verbose=1)
print("Testing done")
print('finished!!!')
print(hist)
np.savetxt('win_history.txt', hist)
plt.plot(moving_average_diff(hist))
plt.ylabel('Average number of stableness per quater')
plt.show()
# path for save the data
model_root_dir = "/zfssz6/ST_MCHRI/BIGDATA/P18Z10200N0124/NIPT_CNV/NIPT_CNV/npz_feature_file/winsize_1000"
total_samples_ls_fn = "/zfssz6/ST_MCHRI/BIGDATA/P18Z10200N0124/NIPT_CNV/sample.list"
min_r = 0.1
min_f_deldup = 0.01
min_f_neu = -2
is_norm = False
win_size = 1000
batch_size=256
test_ratio = 0.1
validation_ratio=0.1
example_min_size = 160000
# x_train, y_train = load_train_data(model_root_dir, total_samples_ls_fn,
# win_size, min_r, min_f_deldup, min_f_neu, is_norm, test_ratio, example_min_size)
# X, y=my_generator(model_root_dir, total_samples_ls_fn,
# win_size, min_r, min_f_deldup, min_f_neu, is_norm, test_ratio, example_min_size)
train_generator,validation_generator = my_generator(model_root_dir, total_samples_ls_fn, win_size, min_r, min_f_deldup, min_f_neu, is_norm,
test_ratio=test_ratio, validation_ratio=validation_ratio, batch_size=batch_size, n_classes = 3, shuffle='Local_shuffle')
# print(len(X))
# multi gpu train
train(train_generator,validation_generator, model_root_dir)
# cv_train(x_train, y_train, model_root_dir)
# mnd_train(x_train, y_train, model_root_dir)
background_test_path = args.test_path
# 读取参考训练数据和测试数据
print("[INFO] loading background images...")
x_train_background, y_train_background = load_images(
background_train_path, IMAGE_SIZE)
x_test_background, y_test_background = load_images(
background_train_path, IMAGE_SIZE)
# 初始化模型
print("[INFO] initialize background model...")
background_model = creat_model(FILTERS, KERNEL_SIZE, INPUT_SHAPE,
POOL_SIZE,
len(os.listdir(background_train_path)))
# 训练模型
print("[INFO] compiling background model...")
train(background_model, x_train_background, y_train_background,
x_test_background, y_test_background,
len(y_train_background) // 10, EPOCHS,
'models/backgroud_model.h5')
elif args.function == 2:
# 使用迁移学习训练比赛集
# 比赛训练数据集顶层类型(大类)
evaluation_top_train_path = args.train_path
# 比赛测试数据集
evaluation_top_test_path = args.test_path
# 读取比赛训练数据和测试数据
print("[INFO] loading evaluation images...")
x_train_top_evaluation, y_train_top_evaluation = load_images(
evaluation_top_train_path, IMAGE_SIZE)
x_test_top_evaluation, y_test_top_evaluation = load_images(
evaluation_top_test_path, IMAGE_SIZE)
# 加载并设置模型
print('[INFO] loading model...')
import os, shutil
from keras import backend
import settings
from train_model import train
from evaluate_model import evaluate
for config in settings.TESTS_CONFIG:
print("Training model ", str(config))
train(config)
file = os.path.basename(config.file_name)
os.rename(config.file_name,
os.path.join(settings.completed_tests_folder, file))
shutil.move(config.model_dir,
os.path.join(settings.results_folder, str(config)))
print("Evaluating model ", str(config))
evaluate(config)
backend.clear_session()
def main():
prior = torch.tensor(opt.prior)
input_size = 32
# prepare training data
transform = {
'train':
transforms.Compose([
transforms.Resize(input_size),
transforms.RandomCrop(input_size, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])
]),
'val':
transforms.Compose([
transforms.Resize(input_size),
transforms.CenterCrop(input_size),
transforms.ToTensor(),
transforms.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])
]),
}
label_trainset = datasets.CIFAR10(root=opt.label_dir,
train=True,
download=True,
transform=transform['train'])
label_testset = datasets.CIFAR10(root=opt.label_dir,
train=False,
download=True,
transform=transform['val'])
labelset_index = []
classnum = np.ones(opt.num_classes) * opt.pos_num
i = 0
while (np.sum(classnum) > 0):
image = label_trainset[i][0]
label = label_trainset[i][1]
if (classnum[label] > 0):
labelset_index.append(i)
classnum[label] -= 1
i += 1
label_train_subset = torch.utils.data.Subset(label_trainset,
labelset_index)
target_transform = transforms.Lambda(
lambda target: target + opt.num_classes)
unlabel_set = datasets.ImageFolder(opt.unlabel_dir,
transform['train'],
target_transform=target_transform)
trainloader = torch.utils.data.DataLoader(label_train_subset + unlabel_set,
batch_size=opt.pu_batchsize,
shuffle=True,
num_workers=32)
testloader = torch.utils.data.DataLoader(label_testset,
batch_size=opt.pu_batchsize,
shuffle=False,
num_workers=32)
dataloader = {}
dataloader['train'] = trainloader
dataloader['val'] = testloader
# stage1: get positive data from unlabeled dataset
model_pu = nn.DataParallel(
initialize_model(use_pu=True, num_classes=opt.num_classes)).cuda()
optimizer_pu = optim.SGD(model_pu.parameters(),
lr=opt.pu_lr,
momentum=opt.momentum,
weight_decay=opt.pu_weight_decay)
scheduler_pu = optim.lr_scheduler.MultiStepLR(optimizer_pu,
milestones=[50, 100, 150],
gamma=0.1,
last_epoch=-1)
model_pu = train_pu(model_pu,
dataloader,
optimizer_pu,
scheduler_pu,
prior=prior,
num_classes=opt.num_classes,
num_epochs=opt.pu_num_epochs)
trainloader2 = torch.utils.data.DataLoader(unlabel_set,
batch_size=opt.pu_batchsize,
shuffle=False,
num_workers=32)
positive_index_all = get_positive(model_pu, trainloader2)
print("We have {} positive unlabeled images for all!".format(
len(positive_index_all)))
unlabel_positive_set = torch.utils.data.Subset(unlabel_set,
positive_index_all)
trainloader3 = torch.utils.data.DataLoader(label_train_subset +
unlabel_positive_set,
batch_size=opt.batchsize,
shuffle=True,
num_workers=32)
dataloader['train'] = trainloader3
# stage2: train student model with rkd method
teacher = nn.DataParallel(
initialize_model(use_pu=False, num_classes=opt.num_classes)).cuda()
student = nn.DataParallel(ResNet18(num_classes=opt.num_classes)).cuda()
class_weight = get_class_weight(teacher,
trainloader3,
num_classes=opt.num_classes)
print(class_weight)
class_weights = perturb(class_weight, opt.epsilon, opt.perturb_num)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(student.parameters(),
lr=opt.lr,
momentum=opt.momentum,
weight_decay=opt.weight_decay)
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=[50, 100, 150],
gamma=0.1,
last_epoch=-1)
model_s, hist = train(student,
teacher,
class_weights,
dataloader,
criterion,
optimizer,
scheduler,
num_epochs=opt.num_epochs)
def Predict():
pre = train_model.train()
pre.predict()
def NewModel():
New = train_model.train()
var3 = New.train_model(3)
var1 = New.train_model(1)
var2 = New.train_model(2)
return var1, var2, var3
def main():
#mypath = r'/home/connlab/108IR/will/final/NVSM_pytorch/'
mypath = r'C:/Users/willll/Desktop/WIillll/IRCLass/Final/NVSM_pytorch'
pretrained_model = 'bert-base-uncased'
glove_path = Path(mypath + '/glove')
model_folder = Path(mypath + '/models')
# data_folder = Path(mypath + '/data/processed')
data_folder = Path(mypath + '/Willll/fakedoc')
testing_query_folder = Path(mypath + '/Willll/test/query')
model_path = model_folder / 'nvsm_bert.pt'
batch_size = 140 # for 150, 8053 / 8113MB GPU memory, to tweak
epochs = 1
docs, queries, tokenizer = load_data(data_folder, testing_query_folder,
pretrained_model)
# docs = docs[:20]
doc_names = [doc['name'] for doc in docs]
n_grams, document_ids = create_dataset(
tok_docs=[doc['tokens'] for doc in docs], tokenizer=tokenizer, n=30)
print('N-grams number', len(n_grams))
k_values = [1, 3, 5, 10]
(train_data, eval_data,
eval_train_data) = create_pytorch_datasets(n_grams, document_ids)
print('Train dataset size', len(train_data))
print('Eval dataset size', len(eval_data))
print('Eval (training) dataset size', len(eval_train_data))
train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
eval_loader = DataLoader(eval_data, batch_size=batch_size, shuffle=False)
eval_train_loader = DataLoader(eval_train_data,
batch_size=batch_size,
shuffle=False)
device = torch.device('cuda')
lamb = 1e-3
nvsm = NVSMBERT(
pretrained_model=pretrained_model,
n_doc=len(doc_names),
dim_doc_emb=20,
neg_sampling_rate=10,
).to(device)
# BERT custom optimizer
param_optimizer = list(nvsm.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = BertAdam(params=optimizer_grouped_parameters,
lr=5e-5,
warmup=0.1,
t_total=len(train_loader) * epochs)
train(nvsm=nvsm,
device=device,
optimizer=optimizer,
epochs=epochs,
train_loader=train_loader,
eval_loader=eval_train_loader,
k_values=k_values,
loss_function=loss_function,
lamb=lamb,
print_every=10000)
torch.save(nvsm.state_dict(), model_path)
nvsm.eval()
recall_at_ks = evaluate(
nvsm=nvsm,
device=device,
eval_loader=eval_loader,
recalls=k_values,
loss_function=loss_function,
)
print(generate_eval(k_values, recall_at_ks))
queries_text = [query['tokens'] for query in queries]
queries_name = [query['name'] for query in queries]
evaluation_results = evaluate_queries_bert(nvsm, queries_text, doc_names,
tokenizer, batch_size, device)
print(evaluation_results)
# print(len(ranksResults))
for query_name, query_text, doc_idx in zip(queries_name, queries_text,
evaluation_results):
print(f'{query_name} {query_text:35} -> {doc_names[doc_idx]}')
with open(mypath + './Willll/result.txt', 'w') as f:
f.write('Query,RetrievedDocuments\n')
resuList = ' '
for qIndex, qName in enumerate(queries_name):
f.write(f'{qName},')
f.write(
f'{resuList.join(doc_names[x] for x in ranksResults[qIndex])}\n'
)
btn2 = Button(root,
text="체험자 사진 촬영",
width=20,
height=1,
font=font,
foreground='white',
background=btncolor,
command=lambda: shot(inputText.get()))
btn3 = Button(root,
text="인공지능 학습",
width=20,
height=1,
font=font,
foreground='white',
background=btncolor,
command=lambda: train(inputText.get()))
btn4 = Button(root,
text="스마트 도어 작동",
width=20,
height=1,
font=font,
foreground='white',
background=btncolor,
command=lambda: lock(inputText.get()))
btn5 = Button(root,
text="체험자 사진 삭제",
width=20,
height=1,
font=font,
foreground='white',
background=btncolor,
def on_Button_train_clicked(self):
print("Button train clicked")
attributes_train = self.get_train_attributes()
submit_status = True
k_value = 0
if attributes_train[2] == "KNN":
# pop up screen
try:
import pop_up_entry
except:
print("import error")
pop_up1 = pop_up_entry.pop_up_entry(self.app, attributes_train[2])
k_value, submit_status = pop_up1.get_status()
print(k_value)
print(submit_status)
if submit_status == True:
#make stus bar busy
self.update_status_bar()
self.app.processEvents()
# train model
try:
import train_model
except:
print("import exception")
model1 = train_model.train(attributes_train, k_value, self.app,
self.pca_model)
self.points = model1.get_map_data_points()
self.current_model = model1.get_model()
#update screen
self.accuracy_display.display(
model1.get_model_accuracy()) # set the lcd accuract digit
self.update_screen_widgets(attributes_train)
#make model ready message
self.app.processEvents()
msg = QMessageBox()
msg.setText('Model Ready')
msg.exec_()
try:
import map_load_time
except:
print("import exception")
numsamples = int(attributes_train[0])
loadtime1 = map_load_time.map_load_time()
timetoloadmap = loadtime1.calculate_load_time(numsamples)
# make map ready after a few seconds depending on sample size
self.app.processEvents()
time.sleep(int(timetoloadmap))
self.status_display_map.setText("READY")
self.status_display_map.setStyleSheet(
"QLabel { color: green ; font-weight: bold}")
self.status_display_map.update()
# make map ready message
self.app.processEvents()
msg = QMessageBox()
msg.setText('Geo Map Ready')
msg.exec_()
import train_model
from model_vgg16 import Vggmodel
import tensorflow as tf
import time
vgg = Vggmodel(optimizer=tf.train.GradientDescentOptimizer,
fine_tuning=True,
lr=0.001,
dropout=True,
adaptive_ratio=1.0)
train_model.train(vgg, timestamp=int(time.time()))
if args.cpu:
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
else:
# this will prevent TF from allocating the whole GPU
from keras import backend as K
import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)
import data_generators
import train_model
import validate_model
os.makedirs("data\\training", exist_ok=True)
os.makedirs("data\\validation", exist_ok=True)
os.makedirs("data\\results", exist_ok=True)
if args.action[0] == "data_train":
data_generators.generate_examples(args.original_path[0], "data\\training",
args.count[0], True)
elif args.action[0] == "data_val":
data_generators.generate_examples(args.original_path[0],
"data\\validation", args.count[0], False)
elif args.action[0] == "train":
train_model.train("data\\training")
elif args.action[0] == "eval":
validate_model.evaluate(args.model_name[0])
def classify(classifiers, strategies, report, deploy, results_dir, analysis_dir):
"""Executes the classification process.
The classification process is defined by a sequence of steps:
Start ➜ Gather Data ➜ Prepare Data ➜ Train Models ➜ Deploy Models and Performances ➜ End
Args:
classifiers: A list of classifiers to train.
strategies: A list of strategies to use during training.
report: Boolean value used to define if models peformances will be reported.
deploy: Boolean value used to define if models trained objects will be deployed.
results_dir: A string path to directory where dataframes, models and performances
will be saved.
analysis_dir: A string path to directory where raw spreadsheets are available.
"""
##########################
# START #
##########################
text_column = 'Paragraph'
classes = ['CF – Contribution flow',
'CT – Choose a task',
'FM – Find a mentor',
'TC – Talk to the community',
'BW – Build local workspace',
'DC – Deal with the code',
'SC – Submit the changes']
classifiers_available = {
'rf': RandomForestClassifier(),
'svc': LinearSVC(),
'mnb': MultinomialNB(),
'knn': KNeighborsClassifier(),
'lr': LogisticRegression(),
'mlp': MLPClassifier(),
'sgd': SGDClassifier(),
}
strategies_available = {
'cc': 'classifier-chains',
'lp': 'label-powerset',
'ovr': 'one-vs-the-rest'
}
selected_classifiers = [classifiers_available[classifier]
for classifier in classifiers]
selected_strategies = [strategies_available[strategy]
for strategy in strategies]
##########################
# GATHER DATA #
##########################
print("Gathering dataframe for classification.")
dataframe = check_if_dataframe_copy_exists(results_dir, analysis_dir)
##########################
# PREPARE DATA #
##########################
print("Shuffling and splitting dataframe into training and test sets.")
X_train, X_test, y_train, y_test = shuffle_and_split(
dataframe, [text_column], classes)
print("Converting paragraphs to statistical features.")
X_train, X_test = vectorize_paragraphs(
X_train[text_column].tolist(), X_test[text_column].tolist())
###################################
# TRAIN, REPORT AND DEPLOY MODELS #
###################################
for classifier in selected_classifiers:
classifier_name = type(classifier).__name__
training_args = {
'classifier': classifier,
'X_train': X_train,
'X_test': X_test,
'y_train': y_train,
'y_test': y_test,
}
for strategy in selected_strategies:
print("Executing " + strategy + " classification.")
print("Classifier: " + classifier_name)
deployment_args = {
'strategy': strategy,
'classifier_name': classifier_name,
'results_dir': results_dir,
}
model, performance = train(strategy=strategy, **training_args)
if report:
report_performance(report=performance, **deployment_args)
if deploy:
deploy_model(model=model, **deployment_args)
save_dir = './model_saver/'
isExists = os.path.exists(save_dir)
if not isExists:
os.makedirs(save_dir)
save_path = save_dir + 'model'
method = sys.argv[1]
print("method", method)
if method == "train":
for r in range(10):
print('#############################')
print("Epoch", r, time.asctime(time.localtime(time.time())))
print('#############################')
t1 = time.time()
model_lstm = train(model_lstm, data_path, save_path, r, road)
t2 = time.time()
if r >= 0:
print("testset acc: ", end='')
count_acc(model_lstm, road, test_path)
t3 = time.time()
print("train cost time", t2 - t1, "acc cost time", t3 - t2)
elif method == "test":
epoch = sys.argv[2]
saver = tf.train.Saver()
saver.restore(model_lstm.sess, './model_saver/model-' + epoch)
print("Epoch", epoch, "testset acc: ", end='')
count_acc(model_lstm, road, test_path)
elif method == "beam":
epoch = sys.argv[2]
K = int(sys.argv[3])
