def train(): x_data, y_data = read(os.getcwd() + "\datasets\\1\\train_data.csv") x_train, x_validation, x_test, y_train, y_validation, y_test = get_data( x_data, y_data) path = input("Input output file name for neural network(enter for end): ") while path != "": path = path + ".net" net, res = create_nn(x_train, y_train, os.getcwd() + "\\" + path, 6, 1000, 0.01) test(x_test, y_test, net) path = input( "Input output file name for neural network(enter for end): ")
def test_model( use_cuda, dset_folder, disable_tqdm=False, ): best_model = GAT_MNIST(num_features=util.NUM_FEATURES, num_classes=util.NUM_CLASSES) util.load_model("best", best_model) if use_cuda: best_model = best_model.cuda() test_dset = MNIST(dset_folder, train=False, download=True) test_imgs = test_dset.data.unsqueeze(-1).numpy().astype(np.float64) with multiprocessing.Pool() as p: test_graphs = np.array(p.map(util.get_graph_from_image, test_imgs)) del test_imgs test_labels = test_dset.targets.numpy() test_accs = util.test( best_model, test_graphs, test_labels, list(range(len(test_labels))), use_cuda, desc="Test ", disable_tqdm=disable_tqdm, ) test_acc = 100 * np.mean(test_accs) print("TEST RESULTS: {acc:.2f}%".format(acc=test_acc))
def train(): x_data, y_data = read() x_data = generate_data(x_data) x_train, x_validation, x_test, y_train, y_validation, y_test = get_data_ds3( x_data, y_data) x_train = x_train + x_validation y_train = y_train + y_validation path = input("Input output file name for neural network(enter for end): ") while path != "": path = path + ".net" net, res = create_nn(x_train, y_train, os.getcwd() + "\\" + path, 7, 6000, 0.001) test(x_test, y_test, net) path = input( "Input output file name for neural network(enter for end): ")
def solve(year: int, day: int) -> None: click.echo(f'Year {year}, Day {day}') module = import_module(f'{year}.{day:02d}') data = read_input(year, day) tc1 = read_tc(year, day, 1) if tc1: test(module.solve_1, tc1) part_1_time, part_1_solution = timed(module.solve_1)(data) click.echo(f'Solution 1: {part_1_solution}, Took: {part_1_time}ms') tc2 = read_tc(year, day, 2) if tc2: test(module.solve_2, tc2) part_2_time, part_2_solution = timed(module.solve_2)(data) click.echo(f'Solution 2: {part_2_solution}, Took: {part_2_time}ms')
def testModExp(number1, number2, number3): result = hex(pow(number1, number2, number3) & util.bitMask)[2:] return util.test( result, "modexp {0} {1} {2}".format(util.toBigIntString(number1), util.toBigIntString(number2), util.toBigIntString(number3)))
def test_asyncio(): print('[py/asyncio]') oLooper = util.Looper() oFuture = oLooper.runTask(util.isValid()) if oFuture.result(): for _ in range(10): oLooper.addTask(util.test(2)) oLooper.run()
def post(self): file1 = self.request.files['file1'][0] original_fname = file1['filename'] extension = os.path.splitext(original_fname)[1] fname = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in xrange(6)) final_filename = fname+extension with open(final_filename, 'w') as out_file: out_file.write(file1['body']) font = "dejavusans-alphanumeric" fontimg = mpimg.imread('train/' + font + '.jpg') util.train(fontimg, font) testimg = mpimg.imread(final_filename) self.write(util.test(testimg, font))
def post(self): file1 = self.request.files['file1'][0] original_fname = file1['filename'] extension = os.path.splitext(original_fname)[1] fname = ''.join( random.choice(string.ascii_lowercase + string.digits) for x in xrange(6)) final_filename = fname + extension with open(final_filename, 'w') as out_file: out_file.write(file1['body']) font = "dejavusans-alphanumeric" fontimg = mpimg.imread('train/' + font + '.jpg') util.train(fontimg, font) testimg = mpimg.imread(final_filename) self.write(util.test(testimg, font))
def testSubtract(number1, number2): result = number1 - number2 if result < 0: power = max((len(hex(-result)) // 8 + 1) * 32, 32) result = (1 << power) + result result = hex(result)[2:] # Workaround for bizarre "f" and "0" prefixing. while result[:8] == "f" * 8: result = result[8:] while result[0] == '0': result = result[1:] else: result = hex(result)[2:] return util.test( result, "subtract {0} {1}".format(util.toBigIntString(number1), util.toBigIntString(number2)))
def main(args, test_file, vocab_file, embeddings_file, pretrained_file, max_length=50, gpu_index=0, batch_size=128): device = torch.device("cuda:{}".format(gpu_index) if torch.cuda.is_available() else "cpu") print(20 * "=", " Preparing for testing ", 20 * "=") if platform == "linux" or platform == "linux2": checkpoint = torch.load(pretrained_file) else: checkpoint = torch.load(pretrained_file, map_location=device) # Retrieving model parameters from checkpoint. embeddings = load_embeddings(embeddings_file) print("\t* Loading test data...") test_data = LCQMC_Dataset(test_file, vocab_file, max_length) test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size) print("\t* Building model...") model = RE2(args, embeddings, device=device).to(device) model.load_state_dict(checkpoint["model"]) print(20 * "=", " Testing RE2 model on device: {} ".format(device), 20 * "=") batch_time, total_time, accuracy, auc = test(model, test_loader) print("\n-> Average batch processing time: {:.4f}s, total test time: {:.4f}s, accuracy: {:.4f}%, auc: {:.4f}\n".format(batch_time, total_time, (accuracy*100), auc))
def model_load_test(args, test_df, vocab_file, embeddings_file, pretrained_file, test_prediction_dir, test_prediction_name, mode, max_length=50, gpu_index=0, batch_size=128): device = torch.device( "cuda:{}".format(gpu_index) if torch.cuda.is_available() else "cpu") print(20 * "=", " Preparing for testing ", 20 * "=") if platform == "linux" or platform == "linux2": checkpoint = torch.load(pretrained_file) else: checkpoint = torch.load(pretrained_file, map_location=device) # Retrieving model parameters from checkpoint. embeddings = load_embeddings(embeddings_file) print("\t* Loading test data...") test_data = My_Dataset(test_df, vocab_file, max_length, mode) test_loader = DataLoader(test_data, shuffle=False, batch_size=batch_size) print("\t* Building model...") model = RE2(args, embeddings, device=device).to(device) model.load_state_dict(checkpoint["model"]) print(20 * "=", " Testing RE2 model on device: {} ".format(device), 20 * "=") batch_time, total_time, accuracy, predictions = test(model, test_loader) print( "\n-> Average batch processing time: {:.4f}s, total test time: {:.4f}s, accuracy: {:.4f}%\n" .format(batch_time, total_time, (accuracy * 100))) test_prediction = pd.DataFrame({'prediction': predictions}) if not os.path.exists(test_prediction_dir): os.makedirs(test_prediction_dir) test_prediction.to_csv(os.path.join(test_prediction_dir, test_prediction_name), index=False)
def test_model( use_cuda, dset_folder, disable_tqdm=False, ): best_model = GAT_MNIST(num_features=util.NUM_FEATURES, num_classes=util.NUM_CLASSES) util.load_model("best", best_model) if use_cuda: best_model = best_model.cuda() test_graphs, test_labels = process_dataset(dset_folder, "test") test_accs = util.test( best_model, test_graphs, test_labels, list(range(len(test_labels))), use_cuda, desc="Test ", disable_tqdm=disable_tqdm, ) test_acc = 100 * np.mean(test_accs) print("TEST RESULTS: {acc:.2f}%".format(acc=test_acc))
int(contents[10].split(",")[9]) == 4 and int(contents[12].split(",")[9]) == 5 and int(contents[22].split(",")[9]) == 1 and "ENEMY: Zed" in contents[26] and "10" in contents[27] and "2" in contents[28] and "4" in contents[29] and "3" in contents[30], "bad config file contents") # Clean-up testing resources test_file.close() clean_map(file_name) # End Message util.sprint( "Engine testing Complete! Failed: [" + str(test.failed) + "/" + str(test.run) + "]" ) # Return testing object return test ### SCRIPT #### if __name__ == '__main__': util.test() util_test = test_util() clean_map(file_name) clean_map('__temp2__') maps_test = test_maps() clean_map(file_name) engine_test = test_engine()
def main(args_parser): parser = args_parser args = parser.parse_args() with tf.device('/cpu:0'): #tf.reset_default_graph() DATASET_PATH = args.datasetPath LEARNING_RATE_1 = args.learningRate EPOCHS = args.epochs BATCH_SIZE = args.batchSize NUM_CLASSES = args.numClasses Z_SCORE = args.zScore WEIGHT_DECAY_1 = args.weightDecay #Placeholders learning_rate = tf.placeholder(tf.float32, shape=[], name='learning_rate') weight_decay = tf.placeholder(tf.float32, shape=[], name="weight_decay") #Dataset test_features, test_labels, test_filenames = util.test_input_fn( DATASET_PATH, BATCH_SIZE, EPOCHS) #Model _, _, test_cross_entropy, test_conf_matrix_op, test_accuracy = initiate_vgg_model( test_features, test_labels, test_filenames, NUM_CLASSES, weight_decay, learning_rate, handle="testing") saver = tf.train.Saver() with tf.Session() as sess: with np.printoptions(threshold=np.inf): if not os.path.isdir("./hcc_output/"): raise Exception( "Model file not found. Use Train.py to train a model") else: saver.restore(sess, "./hcc_output/model.ckpt") print("Model restored from Saver files") writer = tf.summary.FileWriter("./short_tensorboard_logs/") writer.add_graph(sess.graph) merged_summary = tf.summary.merge_all() sess.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) for i in range(100): print("Current Testing Iteration : {}/{}".format(i, 100)) summary, _, test_ce, test_acc = util.test( BATCH_SIZE, learning_rate, weight_decay, sess, test_cross_entropy, test_conf_matrix_op, NUM_CLASSES, LEARNING_RATE_1, WEIGHT_DECAY_1, merged_summary, test_accuracy) test_value1, test_value2 = util.confidence_interval( test_acc, Z_SCORE, 32) print("Testing Accuracy : {}".format(test_acc)) print("Testing Loss (Cross Entropy) : {}".format(test_ce)) print("Testing Confidence Interval: [{} , {}]".format( test_value2, test_value1)) writer.add_summary(summary, i)
#!/usr/bin/env python import os, json, socket, sys, util path = os.path.split(os.path.realpath(__file__))[0] os.chdir(path) sys.path.insert(0, path) os.environ['DEVEL'] = 'yes' os.environ['PGHOST'] = os.path.join(path, 'postgres_data/socket') if 'TMUX' in os.environ: # see https://github.com/sagemathinc/smc/issues/563 del os.environ['TMUX'] util.chdir() ports = util.get_ports() hostname = '0.0.0.0' cmd = "service_hub.py --dev --foreground --hostname={hostname} --port={hub_port} --proxy_port=0 --gap=0 {test} start".format( hostname=hostname, hub_port=ports['hub-api'], test=util.test()) util.cmd(cmd)
def testAnd(number1, number2): result = hex((number1 & number2) & util.bitMask)[2:] return util.test( result, "and {0} {1}".format(util.toBigIntString(number1), util.toBigIntString(number2)))
def run(rank, size): # set random seed torch.manual_seed(args.randomSeed+rank) np.random.seed(args.randomSeed) # load data train_loader, test_loader = util.partition_dataset(rank, size, args) num_batches = ceil(len(train_loader.dataset) / float(args.bs)) # load base network topology subGraphs = util.select_graph(args.graphid) # define graph activation scheme if args.matcha: GP = MatchaProcessor(subGraphs, args.budget, rank, size, args.epoch*num_batches, True) else: GP = FixedProcessor(subGraphs, args.budget, rank, size, args.epoch*num_batches, True) # define communicator if args.compress: communicator = ChocoCommunicator(rank, size, GP, 0.9, args.consensus_lr) else: communicator = decenCommunicator(rank, size, GP) # select neural network model model = util.select_model(10, args) model = model.cuda() criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=5e-4, nesterov=args.nesterov) # guarantee all local models start from the same point # can be removed sync_allreduce(model, rank, size) # init recorder comp_time, comm_time = 0, 0 recorder = util.Recorder(args,rank) losses = util.AverageMeter() top1 = util.AverageMeter() tic = time.time() itr = 0 # start training for epoch in range(args.epoch): model.train() # Start training each epoch for batch_idx, (data, target) in enumerate(train_loader): start_time = time.time() # data loading data, target = data.cuda(non_blocking = True), target.cuda(non_blocking = True) # forward pass output = model(data) loss = criterion(output, target) # record training loss and accuracy record_start = time.time() acc1 = util.comp_accuracy(output, target) losses.update(loss.item(), data.size(0)) top1.update(acc1[0], data.size(0)) record_end = time.time() # backward pass loss.backward() update_learning_rate(optimizer, epoch, itr=batch_idx, itr_per_epoch=len(train_loader)) # gradient step optimizer.step() optimizer.zero_grad() end_time = time.time() d_comp_time = (end_time - start_time - (record_end - record_start)) comp_time += d_comp_time # communication happens here d_comm_time = communicator.communicate(model) comm_time += d_comm_time print("batch_idx: %d, rank: %d, comp_time: %.3f, comm_time: %.3f,epoch time: %.3f " % (batch_idx+1,rank,d_comp_time, d_comm_time, comp_time+ comm_time), end='\r') toc = time.time() record_time = toc - tic # time that includes anything epoch_time = comp_time + comm_time # only include important parts # evaluate test accuracy at the end of each epoch test_acc = util.test(model, test_loader) recorder.add_new(record_time,comp_time,comm_time,epoch_time,top1.avg,losses.avg,test_acc) print("rank: %d, epoch: %.3f, loss: %.3f, train_acc: %.3f, test_acc: %.3f epoch time: %.3f" % (rank, epoch, losses.avg, top1.avg, test_acc, epoch_time)) if rank == 0: print("comp_time: %.3f, comm_time: %.3f, comp_time_budget: %.3f, comm_time_budget: %.3f" % (comp_time, comm_time, comp_time/epoch_time, comm_time/epoch_time)) if epoch%10 == 0: recorder.save_to_file() # reset recorders comp_time, comm_time = 0, 0 losses.reset() top1.reset() tic = time.time() recorder.save_to_file()
# These are just test Python sources. print("Hello!") import sys print(sys.path) from time import gmtime, strftime print('Current time is: ' + strftime("%Y-%m-%d %H:%M:%S", gmtime())) from util import test print(test(1)) import this #from turtledemo import nim #nim.main() #nim.turtle.mainloop() #print('loading turtledemo...') #from turtledemo.wikipedia import main #print('Finished loading turtledemo. Starting demo...') #main() #print('Finished demo.') import twistedtest print('Running twistedtest...') twistedtest.run()
"--cmd", type=str, default='train', help="command") flags = parse.parse_args() settings = {} settings['data'] = flags.data settings['net'] = flags.net settings['loss'] = flags.loss settings['step'] = flags.step settings['dump'] = flags.dump + os.sep + flags.net settings['batch_size'] = flags.batch_size settings['test_batch_size'] = flags.test_batch_size settings['width'] = flags.img_width settings['height'] = flags.img_height settings['channel'] = flags.img_channel settings['pts_num'] = flags.points_num settings['grid_num'] = flags.grid_num settings['grid_dim'] = flags.grid_dim settings['epoch_num'] = flags.epoch_num if flags.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = flags.gpu try: if flags.cmd == 'train': util.train(settings) elif flags.cmd == 'test': util.test(settings) except Exception, e: print(e) else: print('Done Runing')
def initUI(self): """ Main Window global parameters """ self.imgOriginal = np.array([]) self.mainWidth = 1280 self.mainHeight = 640 self.main = QLabel() self.imgNpBefore = np.array([]) self.imgNpAfter = np.array([]) self.skin = mpimg.imread('res/skin.jpg') grid = QGridLayout() self.main.setLayout(grid) self.mainBefore = QLabel('Before') self.mainBefore.setAlignment(Qt.AlignHCenter | Qt.AlignVCenter) self.mainBefore.setWordWrap(True) self.mainBefore.setFont(QFont('Monospace', 10)) self.mainAfter = QLabel('After') self.mainAfter.setAlignment(Qt.AlignHCenter | Qt.AlignVCenter) self.mainAfter.setWordWrap(True) self.mainAfter.setFont(QFont('Monospace', 10)) grid.addWidget(self.mainBefore, 0, 0) grid.addWidget(self.mainAfter, 0, 1) """ Menu Bar """ # FILE MENU openFile = QAction('Open', self) openFile.setShortcut('Ctrl+O') openFile.setStatusTip('Open new File') openFile.triggered.connect(self.showDialog) exitAction = QAction('Exit', self) exitAction.setShortcut('Ctrl+Q') exitAction.setStatusTip('Exit application') exitAction.triggered.connect(self.close) moveLeft = QAction('Move Left', self) moveLeft.setShortcut('Ctrl+L') moveLeft.triggered.connect(lambda: self.updateBefore(self.imgNpAfter)) # PROCESS MENU equalizeMenu = QAction('Equalize', self) equalizeMenu.triggered.connect(lambda: self.updateImgAfter(util.equalize(self.imgNpBefore))) histogramMenu = QAction('Histogram', self) #histogramMenu.triggered.connect(lambda: self.updateImgAfter( grayscaleMenu = QAction('Grayscale', self) grayscaleMenu.triggered.connect(lambda: self.updateImgAfter(util.getgrayscale(self.imgNpBefore))) binarizeMenu = QAction('Binarize', self) binarizeMenu.triggered.connect(lambda: self.updateImgAfter(util.otsu(self.imgNpBefore))) gaussianMenu = QAction('Smooth', self) gaussianMenu.triggered.connect(lambda: self.updateImgAfter(util.convolvefft(util.gaussian_filt(), util.getgrayscale(self.imgNpBefore)))) resizeMenu = QAction('Resize', self) resizeMenu.triggered.connect(lambda: self.updateImgAfter(util.downsample(self.imgNpBefore))) segmentMenu = QAction('Segment', self) segmentMenu.triggered.connect(lambda: self.updateImgAfter(util.showobj(util.downsample(self.imgNpBefore, target_height=480), util.segment(util.thin(util.otsu(util.downsample(self.imgNpBefore, target_height=480), bg='light'))), box=False))) # EDGE DETECTION MENU averageMenu = QAction('Average', self) averageMenu.triggered.connect(lambda: self.updateImgAfter(util.degreezero(self.imgNpBefore, type="average"))) differenceMenu = QAction('Difference', self) differenceMenu.triggered.connect(lambda: self.updateImgAfter(util.degreezero(self.imgNpBefore, type="difference"))) homogenMenu = QAction('Homogen', self) homogenMenu.triggered.connect(lambda: self.updateImgAfter(util.degreezero(self.imgNpBefore, type="homogen"))) sobelMenu = QAction('Sobel', self) sobelMenu.triggered.connect(lambda: self.updateImgAfter(util.degreeone(self.imgNpBefore, type="sobel"))) prewittMenu = QAction('Prewitt', self) prewittMenu.triggered.connect(lambda: self.updateImgAfter(util.degreeone(self.imgNpBefore, type="prewitt"))) freichenMenu = QAction('Frei-Chen', self) freichenMenu.triggered.connect(lambda: self.updateImgAfter(util.degreeone(self.imgNpBefore, type="freichen"))) kirschMenu = QAction('Kirsch', self) kirschMenu.triggered.connect(lambda: self.updateImgAfter(util.degreetwo(self.imgNpBefore, type="kirsch"))) # FEATURE MENU chaincodeMenu = QAction('Chain code', self) chaincodeMenu.triggered.connect(lambda: self.updateTxtAfter(str([util.getdirection(chain[n][0], chain[n][1]) for chain in util.segment(util.thin(self.imgNpBefore), cc=True) for n in xrange(len(chain))]))) turncodeMenu = QAction('Turn code', self) turncodeMenu.triggered.connect(lambda: self.updateTxtAfter(str([util.getturncode(cc) for cc in util.segment(util.thin(self.imgNpBefore, bg='light'), cc=False)]))) skeletonMenu = QAction('Zhang-Suen thinning', self) skeletonMenu.triggered.connect(lambda:self.updateImgAfter(util.zhangsuen(util.binarize(self.imgNpBefore, bg='light')))) skinMenu = QAction('Boundary detection', self) skinMenu.triggered.connect(lambda:self.updateImgAfter(util.thin(self.imgNpBefore, bg='light'))) freemanMenu = QAction('Contour profile', self) # RECOGNITION MENU freemantrainfontMenu = QAction('Train Contour Font', self) freemantrainfontMenu.triggered.connect(lambda: util.train(self.imgNpBefore, feats='zs', order='font', setname='font')) freemantrainplatMenu = QAction('Train ZS Plate (GNB)', self) freemantrainplatMenu.triggered.connect(lambda: util.train(self.imgNpBefore, feats='zs', order='plat', setname='plat')) cctctrainfontMenu = QAction('Train CC + TC Font', self) cctctrainplatMenu = QAction('Train CC + TC Plate', self) freemantestfontMenu = QAction('Predict Contour Font', self) freemantestfontMenu.triggered.connect(lambda: self.updateTxtAfter(util.test(self.imgNpBefore, feats='zs', order='font', setname='font'))) freemantestplatMenu = QAction('Predict Contour Plate', self) freemantestplatMenu.triggered.connect(lambda:self.updateTxtAfter(util.test(self.imgNpBefore, feats='zs', order='plat', setname='plat'))) cctctestfontMenu = QAction('Predict CC + TC Font', self) cctctestplatMenu = QAction('Predict CC + TC Plate', self) facesMenu = QAction('Show faces', self) facesMenu.triggered.connect(lambda: self.updateImgAfter(util.getFaces(self.imgNpBefore, self.skin, range=70))) faceMenu = QAction('Show facial features', self) faceMenu.triggered.connect(lambda: self.updateImgAfter(util.showobj(self.imgNpBefore, util.getFaceFeats(self.imgNpBefore, self.skin, range=100), color=False))) # MENU BAR menubar = self.menuBar() fileMenu = menubar.addMenu('&File') fileMenu.addAction(openFile) fileMenu.addAction(exitAction) fileMenu.addAction(moveLeft) processMenu = menubar.addMenu('&Preprocess') #processMenu.addAction(histogramMenu) processMenu.addAction(equalizeMenu) processMenu.addAction(grayscaleMenu) processMenu.addAction(binarizeMenu) processMenu.addAction(gaussianMenu) processMenu.addAction(resizeMenu) processMenu.addAction(segmentMenu) edgeMenu = menubar.addMenu('&Edge Detection') edgeMenu.addAction(averageMenu) edgeMenu.addAction(differenceMenu) edgeMenu.addAction(homogenMenu) edgeMenu.addAction(sobelMenu) edgeMenu.addAction(prewittMenu) edgeMenu.addAction(freichenMenu) edgeMenu.addAction(kirschMenu) featureMenu = menubar.addMenu('&Features') featureMenu.addAction(chaincodeMenu) featureMenu.addAction(turncodeMenu) featureMenu.addAction(skeletonMenu) featureMenu.addAction(skinMenu) featureMenu.addAction(freemanMenu) recogMenu = menubar.addMenu('&Recognition') recogMenu.addAction(freemantrainfontMenu) recogMenu.addAction(freemantrainplatMenu) recogMenu.addAction(cctctrainfontMenu) recogMenu.addAction(cctctrainplatMenu) recogMenu.addAction(freemantestfontMenu) recogMenu.addAction(freemantestplatMenu) recogMenu.addAction(cctctestfontMenu) recogMenu.addAction(cctctestplatMenu) recogMenu.addAction(facesMenu) recogMenu.addAction(faceMenu) #recogMenu.addAction( """ Toolbar, Status Bar, Tooltip """ self.statusBar().showMessage('Ready') QToolTip.setFont(QFont('SansSerif', 10)) #self.setToolTip('This is a <b>QWidget</b> widget') """ Displaying """ self.setGeometry(12, 30, self.mainWidth, self.mainHeight+80) self.setWindowTitle('Pyxel') self.setWindowIcon(QIcon('res/web.png')) self.setCentralWidget(self.main) self.main.setGeometry(QRect(0, 80, self.mainWidth, self.mainHeight)) #self.mainAfter.setGeometry(QRect(self.mainWidth/2, 80, self.mainWidth/2, self.mainHeight)) self.show()
#!/usr/bin/env python import os, json, socket, sys, util path = os.path.split(os.path.realpath(__file__))[0] os.chdir(path) sys.path.insert(0, path) os.environ['DEVEL'] = 'yes' os.environ['PGHOST'] = os.path.join(path, 'postgres_data/socket') if 'TMUX' in os.environ: # see https://github.com/sagemathinc/cocalc/issues/563 del os.environ['TMUX'] util.chdir() ports = util.get_ports() base_url = util.base_url() cmd = "cd ../../ && . smc-env && service_hub.py --dev --foreground --hostname=0.0.0.0 --port={hub_port} --share_port=0 --proxy_port=0 --gap=0 --mentions --base_url={base_url} {test} start".format( base_url=base_url, hub_port=ports['hub'], test=util.test()) print(cmd) util.cmd(cmd)
def cluster_view(request): context = {'Json_Filename':'test.json'} util.test() return render(request, 'cluster_test.html', context)
from util import test test()
if __name__ == '__main__': parser = argparse.ArgumentParser(parents=[tools.argparser]) parser.add_argument('-c', '--config_ft', default='client_data.json', dest='config_ft', help='JSON file containing FT client info.') parser.add_argument('-d', '--duration', default=20, dest='duration', type=int, help='Duration to keep solenoids open (in seconds) [20].') parser.add_argument('--history', default=15, dest='hsize', type=int, help='Number of entries to keep in the soil probe history. [15]') parser.add_argument('-i', '--interval', default=300, dest='interval', type=int, help='Time between sensor readings (in seconds). [300]') parser.add_argument('-n', '--noinit', default=True, dest='init', action='store_false', help='Don\'t initialize devices at startup.') parser.add_argument('-o', '--output', dest='output', help='Directory to use for logging output. [stdout only.]') parser.add_argument('-t', '--test', default=False, dest='test', help='Run a test of the sensors and solenoids. Format: N:M, seconds for each solenoid.') # TODO: Calibrate sensors and set this to a reasonable default. parser.add_argument('-w', '--water-threshold', default=0, dest='water_threshold', type=int, help='AIN value below which we open the solenoid to water.') args = parser.parse_args() if args.init: print 'Initializing...' util.init() if args.test: util.test(args.test) main_loop(args)
use_cuda = not args.no_cuda and torch.cuda.is_available() print(use_cuda) ########################################################################################################################################################## ########################################################################################################################################################## ########################################################################################################################################################## ########################################################################################################################################################## # Define the model model = torch.load("saves/model_after_retraining.ptmodel") print(model) print('accuracy before weight sharing') util.test(model, use_cuda) print("---------------------------------") ########################################################################################################################################################## ########################################################################################################################################################## ########################################################################################################################################################## ########################################################################################################################################################## # Weight sharing apply_weight_sharing(model) print('accuacy after weight sharing') accuracy = util.test(model, use_cuda) # Save the new model os.makedirs('saves', exist_ok=True)
.format(e + 1, val_losses[e], val_err[e], val_size, cur_time - val_time)) print('Global time has passed {:.0f}:{:.0f}:{:.0f}'.format( h, m, s)) print('') # Make a checkpoint. util.save_epoch_result('new_train_result', e, train_losses[e], train_err[e], val_losses[e], val_err[e]) if val_err[e] < best_val: best_val = val_err[e] saver.save(sess, 'new_ckpt/model', global_step=e + 1, write_meta_graph=True) # Make a final checkpoint. saver.save(sess, 'new_ckpt/model-final', write_meta_graph=True) print('Training time: {:.2f}'.format(time.time() - begin)) util.plot_training_result('new_train_result', train_losses, train_err, val_losses, val_err) del (xs_train1, xs_val, xs_train) test_loss, test_err = util.test('new_ckpt/model-final.meta', 'new_ckpt/model-final', xs_test, ys_test, val_batch_size, classes) f = open('new_train_result.txt', 'a') print('Test_loss = {:.3f}, Test_err = {:.2f}'.format(test_loss, test_err), file=f) f.close()
async def test(ctx): await ctx.send(util.test(ctx)) pass
results[rec] = {} for test in testMetricsList: results[rec][test] = -1 for rec in recommenderList: for test in testMetricsList: if not test.__name__ == 'auc': t = 10 else: t = r if type(rec.__self__).__name__ == "userKnn" or type( rec.__self__).__name__ == "itemKnn": eva = matrixEvaluationDict else: eva = evaluationDict results[rec][test] = test(eva, rec, t) s = "recommender " for test in testMetricsList: s += " & " + test.__name__ s += " \\\\\n" for rec in recommenderList: s += type(rec.__self__).__name__ for test in testMetricsList: s += " & " + str(results[rec][test])[0:6] s += " \\\\\n" print s
def main(): parser = get_parser() args = parser.parse_args() setup_seed(args.seed) device = 'cuda:' + str(args.device) train_loader, test_loader = build_dataset(args) train_accuracies = [] test_accuracies = [] class_num = 10 if args.dataset == 'cifar10' else 100 t_net = { 'resnet18': resnet18, 'resnet34': resnet34, 'resnet56': resnet56, 'resnet110': resnet110 }[args.t_model](class_num) t_ckpt_name = 'SGD-CIFAR' + str(class_num) + '-' + args.t_model if args.dataset == 'cifar10': path = '../ckpt/checkpoint/cifar10/' + t_ckpt_name else: path = '../ckpt/checkpoint/cifar100/' + t_ckpt_name ckpt = torch.load(path, map_location=device) t_net.load_state_dict(ckpt['net']) t_net = t_net.to(device) s_ckpt_name = 'SGD-CIFAR' + str( class_num) + '-' + args.s_model + '-student' + '-overhaul2' s_net = { 'resnet18': resnet18, 'resnet20': resnet20, 'resnet34': resnet34, 'resnet56': resnet56, 'resnet110': resnet110 }[args.s_model](class_num) s_net = s_net.to(device) optimizer = optim.SGD(s_net.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) criterion = nn.CrossEntropyLoss() d_net = distillation.Distiller(t_net, s_net) start_epoch = 0 best_acc = 0 start = time.time() for epoch in range(start_epoch, 150): if epoch in [80, 120]: for param_group in optimizer.param_groups: param_group['lr'] *= 0.1 train_acc = train_with_distill(d_net, optimizer, device, train_loader, criterion) test_acc = test(s_net, device, test_loader, criterion) end = time.time() print('epoch %d, train %.3f, test %.3f, time %.3fs' % (epoch, train_acc, test_acc, end - start)) start = time.time() # Save checkpoint. if best_acc < test_acc: best_acc = test_acc if epoch > 80: state = { 'net': s_net.state_dict(), } if not os.path.isdir('../ckpt/checkpoint'): os.mkdir('../ckpt/checkpoint') if args.dataset == 'cifar10': if not os.path.isdir('../ckpt/checkpoint/cifar10'): os.mkdir('../ckpt/checkpoint/cifar10') torch.save( state, os.path.join('../ckpt/checkpoint/cifar10', s_ckpt_name)) elif args.dataset == 'cifar100': if not os.path.isdir('../ckpt/checkpoint/cifar100'): os.mkdir('../ckpt/checkpoint/cifar100') torch.save( state, os.path.join('../ckpt/checkpoint/cifar100', s_ckpt_name)) print('best_acc %.3f' % best_acc) train_accuracies.append(train_acc) test_accuracies.append(test_acc) if not os.path.isdir('../ckpt/curve'): os.mkdir('../ckpt/curve') if args.dataset == 'cifar10': if not os.path.isdir('../ckpt/curve/cifar10'): os.mkdir('../ckpt/curve/cifar10') torch.save( { 'train_acc': train_accuracies, 'test_acc': test_accuracies }, os.path.join('../ckpt/curve/cifar10', s_ckpt_name)) elif args.dataset == 'cifar100': if not os.path.isdir('../ckpt/curve/cifar100'): os.mkdir('../ckpt/curve/cifar100') torch.save( { 'train_acc': train_accuracies, 'test_acc': test_accuracies }, os.path.join('../ckpt/curve/cifar100', s_ckpt_name))
def testDivide(number1, number2): result = hex((number1 // number2) & util.bitMask)[2:] return util.test( result, "divide {0} {1}".format(util.toBigIntString(number1), util.toBigIntString(number2)))
def testXor(number1, number2): result = hex((number1 ^ number2) & util.bitMask)[2:] return util.test( result, "xor {0} {1}".format(util.toBigIntString(number1), util.toBigIntString(number2)))
embedding_size = 10 cin_layer_nums = [40, 30, 20] dnn_hidden = [200, 200] device = torch.device('cuda:0') #device = torch.device('cpu') model = xDeepFM(dataset.feature_size, dataset.field_size, embedding_size, cin_layer_nums, dnn_hidden).to(device) lr = 0.001 weight_l2 = 1e-4 epoch = 10 st = time.time() criterion = torch.nn.BCELoss() optimizer = torch.optim.Adam(params=model.parameters(), lr=lr, weight_decay=weight_l2) for epoch_i in range(epoch): util.train(model, optimizer, train_data_loader, criterion, device, verbose=200) auc = util.test(model, valid_data_loader, device) print('epoch:', epoch_i, 'validation: auc:', auc) print('cost total time: %d' % (time.time() - st)) auc = util.test(model, test_data_loader, device) print('test auc:', auc)
# count acc,loss on trainset _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).sum().item() train_loss += loss.item() counter += 1 if i % 100 == 0: # get acc,loss on trainset acc = correct / total train_loss /= counter # test val_loss, val_acc = test(model, testloader, criterion) print('iteration %d , epoch %d: loss: %.4f val_loss: %.4f acc: %.4f val_acc: %.4f' %(i, epoch, train_loss, val_loss, acc, val_acc)) # save logs and weights with open('log.csv', 'a') as f: writer = csv.writer(f) writer.writerow([i, train_loss, val_loss, acc, val_acc]) if val_acc > max_val_acc: torch.save(model.state_dict(), 'weights.pkl') max_val_acc = val_acc # reset counters correct, total = 0, 0 train_loss, counter = 0, 0
""" import os, json, socket, sys, util path = os.path.split(os.path.realpath(__file__))[0]; os.chdir(path); sys.path.insert(0, path) os.environ['DEVEL']='yes' os.environ['PGHOST']=os.path.join(path, 'postgres_data/socket') util.chdir() ports = util.get_ports() base_url = util.base_url(ports['hub-share-2'], write=False) print('''\n\nBASE URL: {}\n\n'''.format(base_url)) if 'COCALC_PROJECT_PATH' in os.environ: share_path = os.environ['COCALC_PROJECT_PATH'] + '[project_id]' else: share_path= os.path.join(os.environ['SMC_ROOT'], 'data/projects/[project_id]') cmd = "unset NODE_ENV; cd ../../ && . smc-env && service_hub.py --share_path={share_path} --foreground --hostname=0.0.0.0 --port=0 --share_port={share_port} --proxy_port=0 --gap=0 --base_url={base_url} {test} start".format( base_url = base_url, share_port = ports['hub-share-2'], share_path = share_path, test=util.test()) util.cmd(cmd)
def testRShift(number, shiftBy): assert (shiftBy >= 0) result = hex(number >> shiftBy)[2:] nString = util.toBigIntString(number) return util.test(result, "rshift {0} {1}".format(nString, shiftBy))
from sys import argv from util import setup, init, select, test def show_help(): print("""Usage: trainer.py [cmd] cmd: setup or test""") if __name__ == '__main__': init() if len(argv) >= 2: cmd = argv[1] if cmd == 'setup': setup() elif cmd == 'select': select(int(argv[2])) elif cmd == 'test': test() # elif cmd == 'add': # repo = argv[2] else: show_help() else: show_help()
def testLShift(number, shiftBy): assert (shiftBy >= 0) result = hex((number << shiftBy) & util.bitMask)[2:] nString = util.toBigIntString(number) return util.test(result, "lshift {0} {1}".format(nString, shiftBy))
def run_tests(): util.test() sexp.test()
def testComplement(number): result = hex((~number) & util.bitMask)[2:] return util.test(result, "")
def main(cli_args): parser = argparse.ArgumentParser( description="CSCE 496 HW 2, Classify Cifar data") parser.add_argument('--input_dir', type=str, default='/work/cse496dl/shared/homework/02', help='Numpy datafile input') parser.add_argument( '--model_dir', type=str, default='./homework_2/', help='directory where model graph and weights are saved') parser.add_argument('--epoch', type=int, default=100, help="Epoch : number of iterations for the model") parser.add_argument('--batch_size', type=int, default=32, help="Batch Size") parser.add_argument('--model', type=int, help=" '1' for basic model, '2' for best model") parser.add_argument( '--stopCount', type=int, default=100, help="Number of times for dropping accuracy before early stopping") args_input = parser.parse_args(cli_args) if args_input.input_dir: input_dir = args_input.input_dir else: raise ValueError("Provide a valid input data path") if args_input.model_dir: model_dir = args_input.model_dir else: raise ValueError("Provide a valid model data path") if args_input.epoch: epochs = args_input.epoch else: raise ValueError("Epoch value cannot be null and has to be an integer") if args_input.batch_size: batch_size = args_input.batch_size else: raise ValueError( "Batch Size value cannot be null and has to be an integer") if args_input.model: model = args_input.model else: raise ValueError("Model selection must not be empty") if args_input.stopCount: stop_counter = args_input.stopCount else: raise ValueError("StopCount have to be an int") input_dir = '/work/cse496dl/shared/homework/02' #Make output model dir if os.path.exists(model_dir) == False: os.mkdir(model_dir) #Load Data x = tf.placeholder(tf.float32, [None, 32, 32, 3], name='input_placeholder') y = tf.placeholder(tf.float32, [None, 100], name='labels') #Specify Model if (str(model) == '1'): train_images, train_labels, test_images, test_labels, val_images, val_labels = util.load_data( "") _, outputLayer = initiate_basic_model(x) #Run Training with early stopping and save output counter = stop_counter prev_winner = 0 curr_winner = 0 optimizer = tf.train.AdamOptimizer(learning_rate=0.0001) cross_entropy = util.cross_entropy_op(y, outputLayer) global_step_tensor = util.global_step_tensor('global_step_tensor') train_op = util.train_op_basic(cross_entropy, global_step_tensor, optimizer) conf_matrix = util.confusion_matrix_op(y, outputLayer, 100) saver = tf.train.Saver() with tf.Session() as session: session.run(tf.global_variables_initializer()) counter = stop_counter for epoch in range(epochs): if counter > 0: print("Epoch : " + str(epoch)) util.training(batch_size, x, y, train_images, train_labels, session, train_op, conf_matrix, 100) accuracy = util.validation(batch_size, x, y, val_images, val_labels, session, cross_entropy, conf_matrix, 100) if epoch == 0: prev_winner = accuracy print("Saving.......") saver.save(session, os.path.join("./homework_2/", "homework_2")) else: curr_winner = accuracy if (curr_winner > prev_winner) and (counter > 0): prev_winner = curr_winner print("Saving.......") saver.save( session, os.path.join("./homework_2/", "homework_2")) else: counter -= 1 test_accuracy = util.test(batch_size, x, y, test_images, test_labels, session, cross_entropy, conf_matrix, 100) #Calculate the confidence interval value1, value2 = util.confidence_interval( test_accuracy, 1.96, test_images.shape[0]) print("Confidence Interval : " + str(value1) + " , " + str(value2)) else: break elif (str(model) == '2'): sparsity_weight = 5e-3 #Load the data and reshape it train_data = np.load( os.path.join(os.path.join(input_dir, 'imagenet_images.npy'))) train_images, train_labels, test_images, test_labels, val_images, val_labels = util.load_data( "") #train_data = np.reshape(train_data, [-1,32,32,1]) #Add noise to the data noise_level = 0.2 x_noise = x + noise_level * tf.random_normal(tf.shape(x)) code, outputs = initiate_autoencoder(x_noise, 100) #Optimizer for Autoencoder sparsity_loss = tf.norm(code, ord=1, axis=1) reconstruction_loss = tf.reduce_mean(tf.square(outputs - x)) # Mean Square Error total_loss = reconstruction_loss + sparsity_weight * sparsity_loss optimizer = tf.train.AdamOptimizer(learning_rate=0.0001) train_op = optimizer.minimize(total_loss) saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) util.autoencoder_training(x, code, epochs, batch_size, train_data, sess, train_op) saver.save(sess, os.path.join("./homework_2/", "homework_2")) print("Done : " + str(code)) _, outputLayer = initiate_dense_model(code) #Run Training with early stopping and save output counter = stop_counter prev_winner = 0 curr_winner = 0 optimizer = tf.train.AdamOptimizer(learning_rate=0.0001) cross_entropy = util.cross_entropy_op(y, outputLayer) global_step_tensor = util.global_step_tensor('global_step_tensor') #train_op = util.train_op_encoder(cross_entropy, global_step_tensor, optimizer, var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, "code_layer")) train_op = util.train_op_basic(cross_entropy, global_step_tensor, optimizer) conf_matrix = util.confusion_matrix_op(y, outputLayer, 100) with tf.Session() as session: session.run(tf.global_variables_initializer()) if os.path.isfile(os.path.join("./homework_2/", "homework_2")): saver = tf.train.import_meta_graph("homework_2.meta") saver.restore(session, "./homework_2/homework_2") code_encode = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, "code_layer") session.run( tf.variables_initializer(code_encode, name="init_encoded_layer")) tf.stop_gradient( tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, "init_encoded_layer")) counter = stop_counter for epoch in range(epochs): if counter > 0: print("Epoch : " + str(epoch)) util.training(batch_size, x, y, train_images, train_labels, session, train_op, conf_matrix, 100) accuracy = util.validation(batch_size, x, y, val_images, val_labels, session, cross_entropy, conf_matrix, 100) if epoch == 0: prev_winner = accuracy print("Saving.......") saver.save(session, os.path.join("./homework_2/", "homework_2")) else: curr_winner = accuracy if (curr_winner > prev_winner) and (counter > 0): prev_winner = curr_winner print("Saving.......") saver.save( session, os.path.join("./homework_2/", "homework_2")) else: print("Validation Loss : " + str(curr_winner - prev_winner)) counter -= 1 test_accuracy = util.test(batch_size, x, y, test_images, test_labels, session, cross_entropy, conf_matrix, 100) #Calculate the confidence interval value1, value2 = util.confidence_interval( test_accuracy, 1.96, test_images.shape[0]) print("Confidence Interval : " + str(value1) + " , " + str(value2)) else: break
''' モジュールの読み込みと実行Pythonはモジュールを読み込む際に実行をしています。そのため、先程までのような関数のみのモジュールを importしても全く影響はありませんが、特定の処理をするコードを書くとそれが実行されてしまいます。例えば以下の関数を定義しているモジュール util.pyですが、これを importするだけで 4行目が実行されて testと出力されてしまいます。 def test(): print("test") ''' import util util.test() ''' そのため、モジュールとして読み込まれることを想定して開発された Pythonのプログラムファイルは実行されるコードを含まないべきです。 モジュールを読み込んだ際になんらかの初期化処理が必要な場合でも関数などとして提供するほうが行儀はいいです。 なぜなら importをする側は importをするだけで勝手になんらかの処理をすると想定していないからです。 ただ、モジュールがプログラムの起点になることもあれば importされることもあるという場合は「起点となる場合はある処理をする」一方、「モジュールとして呼び出される場合はそれをしない」という実装が必要なことがあります。 これを実現するには特殊な変数 __name__を使います。 これは特殊属性(詳細は本シリーズの下編にて扱います)と呼ばれる高度なトピックなのですが、難しいことは考えずにこれにはモジュール名が入っていると認識して下さい。 たとえば util.pyをモジュールとして読み込めば utilとなりますし、testmodule.pyを読み込めば testmoduleとなります。 ただし、一つ例外がありプログラムの起点となるプログラムはモジュール名がファイル名ではなく __main__となります。 せっかくなので実際に試してみます。以下の3つのファイルで実行してみてください。 hello.py(実行ファイル) nice.py(呼び出しファイル) world.py(呼び出しファイル) '''
bin_search_cases = { 'odds not found -1': ([0, 1, 2, 3, 4], -1), 'odds found 0': ([0, 1, 2, 3, 4], 0), 'odds found 1': ([0, 1, 2, 3, 4], 1), 'odds found 2': ([0, 1, 2, 3, 4], 2), 'odds found 3': ([0, 1, 2, 3, 4], 3), 'odds found 4': ([0, 1, 2, 3, 4], 4), 'evens not found -1': ([0, 1, 2, 3], -1), 'evens found 0': ([0, 1, 2, 3], 0), 'evens found 1': ([0, 1, 2, 3], 1), 'evens found 2': ([0, 1, 2, 3], 2), 'evens found 3': ([0, 1, 2, 3], 3), 'evens not found 4': ([0, 1, 2, 3], 4), } util.test('BINARY SEARCH', algo.bin_search, bin_search_cases) n7 = util.create_node(7) n6 = util.create_node(6) n3 = util.create_node(3, [n6, n7]) n5 = util.create_node(5) n4 = util.create_node(4) n2 = util.create_node(2, [n4, n5]) n1 = util.create_node(1, [n2, n3]) dfs_bfs_cases = { 'map-dfs': (n1, lambda x, r: r + [x.get('v')], []), 'filter-dfs': (n1, lambda x, r: r + [x.get('v')] if x.get('v') > 1 else r, []) }