def main(): if checkpoint.ok: loader = data.Data(args) if args.model.lower() == 'edsr': t_model = EDSR(args, is_teacher=True).to(device) s_model = PAMS_EDSR(args, bias=True).to(device) elif args.model.lower() == 'rdn': t_model = RDN(args, is_teacher=True).to(device) s_model = PAMS_RDN(args).to(device) else: raise ValueError('not expected model = {}'.format(args.model)) if args.pre_train is not None: t_checkpoint = torch.load(args.pre_train) t_model.load_state_dict(t_checkpoint) if args.test_only: if args.refine is None: ckpt = torch.load(f'{args.save}/model/model_best.pth.tar') refine_path = f'{args.save}/model/model_best.pth.tar' else: ckpt = torch.load(f'{args.refine}') refine_path = args.refine s_checkpoint = ckpt['state_dict'] s_model.load_state_dict(s_checkpoint) print(f"Load model from {refine_path}") t = Trainer(args, loader, t_model, s_model, checkpoint) print(f'{args.save} start!') while not t.terminate(): t.train() t.test() checkpoint.done() print(f'{args.save} done!')
def main(argv): parser = argparse.ArgumentParser() parser.add_argument('-o', action='store', dest='output_file', help='The file where to save the solution and, in case, plots') parser.add_argument('-t', action='store', dest='time_limit', type=int, required=True, help='The time limit') parser.add_argument('instance_file', action='store', help='The path to the file of the instance to solve') config = parser.parse_args() print('instance_file = {!r}'.format(config.instance_file)) print('output_file = {!r}'.format(config.output_file)) print('time_limit = {!r}'.format(config.time_limit)) instance = data.Data(config.instance_file) instance.short_info() # if config.output_file is not None: # instance.plot_points(config.output_file+'.png'); # instance.show() sol = solve(instance, config) assert sol.valid_solution() if config.output_file is not None: sol.plot_routes(config.output_file+'_sol'+'.png') sol.write_to_file(config.output_file+'.sol') print("{} routes with total cost {:.1f}" .format(len(sol.routes), sol.cost()))
def listings(self): req = requests.get( self.url, headers={"user-agent": spoof().browser("Chrome", 0)}) soup = bsoup(req.text, "html5lib") titles = soup.findAll("a", {"class": "visual-title dark"}) ttls = np.unique([title.get_text().strip() for title in titles]) dicts = [] patt = re.compile("..\dth\s(Anniversary)") for t in ttls: if t.find("3D") > -1: pass elif t.find("Anniversary") > -1: pass elif t.find("(") > -1: ttl, yr = t.split(" (") dicts.append({ "title": ttl, "year": yr.replace(")", ""), "rank": "", "watched": None }) else: ttl, yr = t, "" dicts.append({ "title": ttl, "year": yr, "rank": "", "watched": None }) m = data.Data(mediatype="movie") m.collect(args=dicts) return m.movies
def test2(): data_obj = dt.Data("african_hiv_econ.csv") headers = data_obj.get_headers() print(data_obj.get_headers()) print( linear_regression(data_obj, [headers[1], headers[6], headers[9], headers[3]], headers[0])[0]) print( linear_regression(data_obj, [headers[1], headers[6], headers[9], headers[3]], headers[0])[1]) print( linear_regression(data_obj, [headers[1], headers[6], headers[9], headers[3]], headers[0])[2]) print( linear_regression(data_obj, [headers[1], headers[6], headers[9], headers[3]], headers[0])[3]) print( linear_regression(data_obj, [headers[1], headers[6], headers[9], headers[3]], headers[0])[4])
def main(argv): if len(argv) < 4: print( "Usage: python %s <data file> <independent header> <dependent header>" ) exit(-1) # read some data data_obj = data.Data(argv[1]) ind_header = argv[2] dep_header = argv[3] # call the analysis function results = analysis.single_linear_regression(data_obj, ind_header, dep_header) # print out the results print("Model: y = %.4fx + %.4f" % (results[0], results[1])) print("R-value: %.3f" % (results[2])) print("P-value: %.3f" % (results[3])) print("Stderr: %.3f" % (results[4])) return
def main(): global model if args.data_test == ['video']: from videotester import VideoTester model = model.Model(args, checkpoint) t = VideoTester(args, model, checkpoint) t.test() else: if checkpoint.ok: args.test_only = True args.resume = -2 args.batch_size = 16 #args.is_fcSim = True loader = data.Data(args) _model = model.Model(args, checkpoint) _loss = loss.Loss(args, checkpoint) if not args.test_only else None t = Trainer(args, loader, _model, _loss, checkpoint) while not t.terminate(): t.train() t.test() checkpoint.done()
def process_gen(self,window,filename1): self.inc = random.randint(0,pow(16,4)-1) self.inc = f"{self.inc :x}" self.inc = f"{self.inc :0>4}" obj = data.Data(self.inc) list1 = obj.generation() if('.txt' in filename1): with open(filename1,'w') as f: for packets in list1: f.write("%s\n"%packets) msg = "One file is generated with 50 packets of data \nand the filename as "+filename1 self.filename = filename1 else: with open(filename1+'.txt','w') as f: for packets in list1: f.write("%s\n"%packets) msg = "One file is generated with 50 packets of data \nand the filename as "+filename1+'.txt' self.filename = filename1+'.txt' process_gen = Tk() process_gen.title("msg box") process_gen.configure(background='grey') label_0 = Label(process_gen, text=msg,font="Times 12",bg='white',fg='black') label_0.grid(row=0,column=0,padx=10,pady=10) process_gen.mainloop()
def __init__(self, task, inference=False): dt_ner = dt.Data(task=task, inference=inference) # Load default model self.nlp = he.load_spacy_model(language=cu.params.get('language'), disable=['ner', 'parser', 'tagger']) # Add flair pipeline #TODO: excluding FALIR for now, to be compared with text analytics # flair_matcher = FlairMatcher(dt_ner.get_path('fn_ner_flair')) # self.nlp.add_pipe(flair_matcher) # Text Analytics ta_matcher = TextAnalyticsMatcher() self.nlp.add_pipe(ta_matcher) # Load phrase matcher self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER") matcher_items = pd.read_csv(dt_ner.get_path('fn_ner_list', dir='asset_dir'), encoding='utf-8', sep='\t') for product in matcher_items['key'].drop_duplicates(): _values = matcher_items[matcher_items['key'] == product] patterns = [self.nlp.make_doc(v) for v in _values.value] self.matcher.add(product, None, *patterns)
def main(argv): if len(argv) < 2: print 'Usage: python %s <all numeric CSV file>' % (argv[0]) exit(-1) try: d = data.Data(argv[1]) except: print 'Unable to open %s' % (argv[1]) exit(-1) print "D :", d an = analysis.Analysis() codebook, codes, errors = an.kmeans_numpy(d, d.get_headers(), 2) an.kmeans_init(d, 2, categories=np.matrix([0, 1, 2, 2123, 141, 2])) print "\nCodebook\n" print codebook print "\nCodes\n" print codes
def build_by_pickle(self, test_data_file, model_dir, data_filename, pickle_file, loadComplEx=False): """ :param pickle_file: pickle embedding :return: """ # load the saved Data() self.this_data = data.Data() data_save_path = join(model_dir, data_filename) self.this_data.load(data_save_path) # load testing data self.load_test_data(test_data_file) self.model_dir = model_dir # used for saving with open(pickle_file, 'rb') as f: ht, r = pickle.load(f) # unpickle self.vec_c = ht self.vec_r = r
def __init__(self): super().__init__() self.DATA_MODULE = dt.Data() self.NUM_SERVER = self.DATA_MODULE.getIpListLength() self.IP_LIST = self.DATA_MODULE.getIpList() self.PORT = self.DATA_MODULE.getPort() self.BUFFER_LENGHT = 128 #Avaliable Threads self.threads = [] #Destroying Windows Flags self.setWindowFlags( QtCore.Qt.Window | QtCore.Qt.CustomizeWindowHint | QtCore.Qt.WindowTitleHint | QtCore.Qt.WindowCloseButtonHint | QtCore.Qt.WindowStaysOnTopHint ) #PyQT sets self.pixmap = QPixmap('images/background.png') #self.setWindowIcon(QtGui.QIcon('images/icon.png')) self.initUI()
def exec(self, sym, data5min, ex1: Btexchange, ord_log=None, strat_log=None): self.symbol = sym self.ex1 = ex1 # print(data5min) self.ts = data5min.iloc[19]['timespan1'] self.lasto = data5min.iloc[19]['open'] self.lasth = data5min.iloc[19]['high'] self.lastl = data5min.iloc[19]['low'] self.lastc = data5min.iloc[19]['close'] # self.candles = "["+str(self.ts)+","+str(self.lasto)+","+str(self.lasth)+","+str(self.lastl)+","+str(self.lastc)+"]" self.candle = [] self.candle.append(self.ts) self.candle.append(self.lasto) self.candle.append(self.lasth) self.candle.append(self.lastl) self.candle.append(self.lastc) self.symbol = sym.upper() datac = data.Data() self.data = data5min # self.datah = datac.geth(self.data) self.cch = 0 self.cc5l = 0 self.cc5n = 0 self.cc5h = 0 # self.ex = exch self.ord_log = ord_log self.strat_log = strat_log self.ta() self.decide()
def __init__(self, task, inference=False, models=['Flair', 'TextAnalytics', 'nerList', 'regEx']): if len(models) < 1: log.warning('Specify at least one model to use for NER.') return None self.models = models dt_ner = dt.Data(task=task, inference=inference) # Load default model self.nlp = he.load_spacy_model(language=cu.params.get('language'), disable=['ner', 'parser', 'tagger']) # Add flairNER to pipeline if 'Flair' in self.models: flair_matcher = FlairMatcher(dt_ner.get_path('fn_ner_flair')) self.nlp.add_pipe(flair_matcher) # Add Azure Text Analytics to pipeline if 'TextAnalytics' in self.models: ta_matcher = TextAnalyticsMatcher() self.nlp.add_pipe(ta_matcher) # Add phrase matcher based on nerList if 'nerList' in self.models: self.matcher = PhraseMatcher(self.nlp.vocab, attr="LOWER") matcher_items = pd.read_csv(dt_ner.get_path('fn_ner_list', dir='asset_dir'), encoding='utf-8', sep='\t') for product in matcher_items['key'].drop_duplicates(): _values = matcher_items[matcher_items['key'] == product] patterns = [self.nlp.make_doc(v) for v in _values.value] self.matcher.add(product, None, *patterns)
def main(): path = sys.argv[1] total_files = len(sys.argv) - 2 test_names = [] for i in range(total_files): test_names.append(sys.argv[i + 2]) datasets = data.Data(path, test_names) model_type = MODEL_TYPES[ "CONVOLUTIONAL"] # HERE GOES THE MODEL WE CONFIGURED IN CORSS VALIDATION epochs = 10 # HERE GOES THE EPOCHS WE CONFIGURED IN CORSS VALIDATION neurons = 64 # HERE GOES THE NEURONS WE CONFIGURED IN CORSS VALIDATION dropout = 0.1 # HERE GOES THE DROPOUT WE CONFIGURED IN CORSS VALIDATION batches_size = 64 # HERE GOES THE BATCH_SIZE WE CONFIGURED IN CORSS VALIDATION m = model.Model(model_type=model_type, train_dataset=datasets.train, neurons=neurons, dropout=dropout, val_dataset=datasets.test, test_files=datasets.test_files, path=path) m.train(epochs, batches_size) m.eval()
def translate(self, headers, magnitudes): '''Translates the variables `headers` in projected dataset in corresponding amounts specified by `magnitudes`. Parameters: ----------- headers: Python list of str. Specifies the variables along which the projected dataset should be translated. magnitudes: Python list of float. Translate corresponding variables in `headers` (in the projected dataset) by these amounts. Returns: ----------- ndarray. shape=(N, num_proj_vars). The translated data (with all variables in the projected). dataset. NOTE: There should be NO homogenous coordinate! ''' newDict = self.data.get_mappings() translation = self.translation_matrix(headers, magnitudes) newData = translation @ self.get_data_homogeneous().transpose() self.data = data.Data(headers = headers, data = newData.transpose()[:,:-1], header2col = dict) return newData.transpose()[:,:-1]
def main(): dclean = data.Data('data-clean.csv') dgood = data.Data('data-good.csv') dnoisy = data.Data('data-noisy.csv') daussie = data.Data('aussiecoast.csv') drb = data.Data('RB 2.csv') dsat = data.Data('SATdata.csv') headersclean=dclean.get_headers() headersgood = dgood.get_headers() headersnoisy = dnoisy.get_headers() headersaussie= daussie.get_headers() headersrb=drb.get_headers() headersat=dsat.get_headers() xheadrb=headersrb[1:3] yheadrb=headersrb[0] xheadaussie=headersaussie[1:3] yheadaussie=headersaussie[0] xheadclean=headersclean[1:3] yheadclean=headersclean[0] xheadgood=headersgood[1:3] yheadgood=headersgood[0] xheadnoisy = headersnoisy[1:3] yheadnoisy = headersnoisy[0] xheadsat=headersat[1:3] yheadsat=headersat[0] print "Multiple Linear Regression Australia Coast Data" print "independent variables", xheadaussie print "dependent variable", yheadaussie print "b, sse, r2, t, p" print linearRegression(daussie,xheadaussie,yheadaussie)
"../../../enron/gender_identified_enron_corpus/graph/partitions/structure_employee_wemail_random_test0.2.csv" ] filename = [ "../../../enron/gender_identified_enron_corpus/graph/partitions/structure_employee_wemail_random_train0.8.csv", "../../../enron/gender_identified_enron_corpus/graph/structure_wemail.csv", "../../../enron/gender_identified_enron_corpus/graph/structure_wemail.csv" ] if len(sys.argv) > 1: this_dim = int(sys.argv[1]) model_path = sys.argv[2] data_path = sys.argv[3] more_filt = [sys.argv[4]] filename = sys.argv[5:] this_data = data.Data() this_data.load_data_enron_csv(filenames=filename) for f in more_filt: this_data.record_more_enron_csv(f) # In[ ]: m_train = Trainer() m_train.build(this_data, dim=this_dim, batch_size=64, num_neg=10, save_path=model_path, data_save_path=data_path, L1=False)
logger = logging.getLogger() if opt.verbose: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) logging.info(opt) if opt.random_seed != 0: random.seed(opt.random_seed) np.random.seed(opt.random_seed) torch.manual_seed(opt.random_seed) torch.cuda.manual_seed_all(opt.random_seed) d = data.Data(opt) logging.info(d.config) makedir_and_clear(opt.output) logging.info("load data ...") train_data = data.loadData(opt.train_file, True, opt.types, opt.type_filter) dev_data = data.loadData(opt.dev_file, True, opt.types, opt.type_filter) if opt.test_file: test_data = data.loadData(opt.test_file, False, opt.types, opt.type_filter) else: test_data = None
def kmeans2(d, headers, K, whiten=True): '''Takes in a Data object, a set of headers, and the number of clusters to create Computes and returns the codebook, codes, and representation error. ''' if isinstance(d, np.ndarray): A = d # assign to A the result getting the data given the headers else: A = d.get_data(headers).T # assign to W the result of calling vq.whiten on A W = vq.whiten(A) # assign to codebook, bookerror the result of calling vq.kmeans2 with W and K codebook, bookerror = vq.kmeans2(W, K) # assign to codes, error the result of calling vq.vq with W and the codebook codes, error = vq.vq(W, codebook) # return codebook, codes, and error return codebook, codes, error if __name__ == '__main__': d = data.Data(sys.argv[1]) headers = d.get_headers() print linear_regression(d, [headers[0], headers[1]], headers[10])
import bilstm_att_model import DGCNN_model import cnn import cnn_att import biattention from sklearn.model_selection import train_test_split import gensim import numpy as np from sklearn.metrics import classification_report import pickle import pandas as pd if __name__ == '__main__': path_train = '../data/train.csv' path_test = '../data/test_stage1.csv' data = data.Data(path_train, path_test) seq_len = data.seq_len[0] seq_len_test = data.seq_len[1] train_id = data.train_id label = data.label test_id = data.test_id word2index = data.word2index index2word = data.index2word x_train, x_dev, y_train, y_dev = train_test_split(train_id, label, random_state=42, test_size=0.2, stratify=label) seqlen_train, seqlen_dev, _, _ = train_test_split(seq_len, label,
# -*- coding: utf-8 -*- """ Created on Tue Nov 19 08:43:41 2019 @author: Ashima """ import data import config import random import numpy as np import neural_network if __name__ == "__main__": #print("Start") data = data.Data() data.read_full(config.TRAIN_INPUT, config.TRAIN_LABELS, True) print("Train data read successfully") X, Y = data.get_data() network = neural_network.Network(config.NUM_TRAIN, config.IMAGE_SIZE * config.IMAGE_SIZE, config.CLASSES) network.train(X, Y) # print("Complete model training") print("Accuracy Plot for Train") network.plot_accuracy('Train') print("Error Plot for Train") network.plot_cost('Train') print("Accuracy Plot for Validation")
def main(argv): #Handle OPTIONS try: opts, args = getopt.getopt(argv[1:], "ht:i:e:") except getopt.GetoptError: print "Invalid option!" usage() #Default args tag = None id = None notificationEmail = None for opt, arg, in opts: if (opt == "-h"): usage() elif (opt == "-t"): tag = arg elif (opt == "-i"): id = arg elif (opt == "-e"): notificationEmail = arg if tag == None: print "A TAG must be specified to run!" usage() if id == None: print "An ID must be specified to run!" usage() #handle VARIABLES variables, job_dir = parseArgs(args) if job_dir == None: print "No JOB_DIR given!" usage() if variables == None: print "Invalid option or variable decleration!" usage() #Finish setting up framework import config, error if not config.init(job_dir): error.warning("No Config file found. Using defaults") error.init() import data, parser, ruleParser, teu #Register our signalHandler now #We will need to start some kinda signal handler at this point so that we can have a graceful exit signal.signal(signal.SIGTERM, signalHandler) #include our command line options if notificationEmail != None: error.addEmailAddr(notificationEmail) variables.append(('TAG', tag)) variables.append(('ID', id)) #Parse the task lists error.message("Parsing the task lists...") taskList = parser.parseTaskList() if error.isError(): error.fatalError("Error parsing the task lists!") error.message("task lists parsed OK!") #Parse Rules error.message("Parsing the rule lists...") ruleList = ruleParser.getRules() if error.isError(): error.fatalError("Error parsing the rules list!") error.message("rule lists parsed OK!") #enforce the rules rulesObj = data.Data() rulesObj.shared.putVar("USER_LIB_PATH", config.getUserLibPath()) rulesObj.shared.putVar("COMMON_LIB_PATH", config.getCommonLibPath()) rulesObj.shared.putVar("TASK_LIST", taskList) error.message("Enforcing the Rules...") teu.runTaskList(ruleList, rulesObj) if error.isError(): error.fatalError("Your job did not pass a FATAL rule") error.message("All FATAL rules passed!") #Start the job (run the tasks) error.message("Running the tasks...") dataObj = data.Data() for var in variables: name = var[0] value = var[1] dataObj.shared.putVar(name, value) teu.runTaskList(taskList, dataObj) if error.isError(): error.message("Some tasks failed during execution!") error.message("Your job may not have completed sucessfuly!") error.message("Trying to exit gracefully...") error.gracefulExit( "Framework Job: " + job_dir + "-" + tag + " completed with errors", "Framework Job: " + job_dir + "-" + tag + " completed, however some tasks failed during execution. Please check logs as your job may not have been successful." ) error.message("All tasks completed sucessfully!") error.gracefulExit( "Job: " + job_dir + "-" + tag + " completed successfully", "The Task Execution Framework has completed Job: " + job_dir + "-" + tag)
def __init__(self, title='nucl.ai Motion Matching'): self.canvas = vispy.scene.SceneCanvas(title=title, size=(1280, 720), bgcolor='black', show=False, keys='interactive') self.params = params.Params() self.widget = self.canvas.central_widget self.view = self.canvas.central_widget.add_view() self.marker = vispy.scene.Markers(pos=numpy.asarray([[0, 0]]), face_color='red', size=0, parent=self.view.scene) self.select = True # identify if the current tick select or advance the path # prepare display self.lines = [] self.colors = [] self.history = [] self.history_pointer = 0 for i in range(self.params.HISTORY_SIZE): line = vispy.scene.Line(parent=self.view.scene, color=COLOR_NEUTRAL, connect='strip', method='agg', width=HISTORY_PATH_WIDTH) line.transform = vispy.visuals.transforms.MatrixTransform() self.history.append(line) for i in range(self.params.TOP_PATHS_NUMBER): path_width = SELECTED_PATH_WIDTH if i == 0 else PATH_WIDTH arrow_size = SELECTED_ARROW_SIZE if i == 0 else ARROW_SIZE color = COLOR_SELECTED if i == 0 else COLOR_NEUTRAL # color = numpy.random.rand(3) # using fixed colors now self.colors.append(color) line = vispy.scene.Line(parent=self.view.scene, color=color, connect='strip', method='agg', width=path_width) line.transform = vispy.visuals.transforms.MatrixTransform() self.lines.append(line) self.timer_toggle = True self.player_position = numpy.asarray([0, 0]) if not os.path.exists('dota2.csv'): print( "ERROR: Please download and extract this file...\nhttps://github.com/aigamedev/nuclai16/releases/download/0.0/dota2.csv.bz2\n" ) sys.exit(-1) self.data = data.Data('dota2.csv', self.params) # init the searched point with some random value - after first mouse move it's a self.data.mouse_xy = ( (numpy.random.rand(2) * 10 - 5) - numpy.asarray(self.canvas.size) / 2) * self.params.SCALE_FACTOR self.grid = vispy.scene.visuals.GridLines(parent=self.view.scene, color=(1, 1, 1, 1)) self.grid.transform = vispy.visuals.transforms.MatrixTransform() self.grid.transform.translate(numpy.asarray(self.canvas.size) / 2) self.canvas.show(visible=True) # HACK: Bug in VisPy 0.5.0-dev requires a click for layout to occur. self.canvas.events.mouse_press() @self.canvas.events.key_press.connect def on_key_press(event): if event.key.name == ' ': if self.timer_toggle: self.timer.stop() else: self.timer.start() self.timer_toggle = not self.timer_toggle @self.canvas.events.resize.connect def on_resize(event): self.grid.transform.reset() self.grid.transform.translate(numpy.asarray(self.canvas.size) / 2) # @TODO: translate paths @self.canvas.events.mouse_move.connect def on_mouse_move(event): self.data.mouse_xy = ( numpy.asarray(self.view.camera.transform.imap(event.pos)) - numpy.asarray(self.canvas.size) / 2) * self.params.SCALE_FACTOR @self.canvas.events.draw.connect def on_draw(event): pass
import tensorflow as tf import model import data # 訓練データ作成担当 g = data.Data() # GPUをすべて使わないオプション config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) tf.keras.backend.set_session(sess) # モデルを作成 model = model.make(tflite=False) # 最適化を定義 optimizer = tf.keras.optimizers.Adam(lr=0.001) model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["categorical_accuracy"]) # コールバック class Callback(tf.keras.callbacks.Callback): def on_epoch_end(self, epoch, logs=None): "各エポック終了時に重みを保存する" model.save("weight.hdf5") cb = Callback() # 途中から学習する場合 initial_epoch = 0 if initial_epoch >= 1:
def main(argv): if len(argv) < 3: print 'usage: python %s <Train CSV file> <Train categories CSV file>' % (argv[0]) exit(-1) # read the features and categories data sets print 'Reading %s and %s' % (argv[1], argv[2]) try: d = data.Data(argv[1]) except: print 'Unable to open %s' % (argv[1]) exit(-1) try: catdata = data.Data(argv[2]) except: print 'Unable to open %s' % (argv[2]) exit(-1) # execute PCA analysis print 'Executing PCA' pcadata = an.pca( d, d.get_headers(), False ) print 'Evaluating eigenvalues' # identify how many dimensions it takes to represent 90% of the variation evals = pcadata.get_eigenvalues() #print "type:",type(evals) evals=np.asmatrix(evals) #print "type2:",type(evals) #print "shape: ",evals.shape esum = np.sum(evals) cum = evals[0,0] cumper = cum / esum i = 1 while cumper < 0.999: cum += evals[0,i] cumper = cum/esum i += 1 print 'Dimensions to reach 99.9% of variation:', i cheaders = pcadata.get_headers()[:i] # cluster the data K = 6 # Use the average of each category as the initial means truecats = catdata.get_data(catdata.get_headers()[0:1]) #tmpcats = truecats - 1 tmpcats = truecats # Don't adjust if we're using corrected labels print 'Clustering to %d clusters' % (K) codebook, codes, errors = an.kmeans(pcadata, cheaders, K, categories = tmpcats) # build a confusion matrix confmtx = [[0,0,0,0,0,0],[0,0,0,0,0,0],[0,0,0,0,0,0],[0,0,0,0,0,0],[0,0,0,0,0,0],[0,0,0,0,0,0]] for i in range(codes.shape[0]): #confmtx[codes[i,0]][int(truecats[i,0])-1] += 1 confmtx[codes[i,0]][int(truecats[i,0])] += 1 # don't adjust print "\nConfusion Matrix:\n" print 'Actual-> Walking Walk-up Walk-dwn Sitting Standing Laying' for i in range(len(confmtx)): s = 'Cluster %d' % (i) for val in confmtx[i]: s += "%10d" % (val) print s print
def setUp(self): self.data = data.Data() self.proximity = s1_proximity.ProximityState()
def main(argv): '''Reads in a training set and a test set and builds two KNN classifiers. One uses all of the data, one uses 10 exemplars. Then it classifies the test data and prints out the results. ''' # usage if len(argv) < 3: print 'Usage: python %s <training data file> <test data file> <optional training category file> <optional test category file>' % ( argv[0]) exit(-1) # read in the training set data_train = data.Data(argv[1]) # read in the test set data_test = data.Data(argv[2]) # compatibility check length or argv if len(argv) > 4: # get the categories of the training data train_cat_data = data.Data(argv[3]) train_cats = train_cat_data.get_data([train_cat_data.get_headers()[0]]) # get the categories of the test data test_cat_data = data.Data(argv[4]) test_cats = test_cat_data.get_data([test_cat_data.get_headers()[0]]) # get the training data A and the test data B A = data_train.get_data(data_train.get_headers()) B = data_test.get_data(data_test.get_headers()) else: # just assume the categories are the last column train_cats = data_train.get_data([data_train.get_headers()[-1]]) test_cats = data_test.get_data([data_test.get_headers()[-1]]) A = data_train.get_data(data_train.get_headers()[:-1]) B = data_test.get_data(data_test.get_headers()[:-1]) #----------------------------------------------------------------------- # create two classifiers knnClass = classifier.KNN() print "Created Classifier, Building Now." # build the classifiers knnClass.build(A, train_cats) print "Built! Now classifying." #----------------------------------------------------------------------- #-Classifies the training set data and prints out a confusion matrix. acats, alabels = knnClass.classify(A) print "Done Classifying." unique, mapping = np.unique(np.array(train_cats.T), return_inverse=True) unique2, mapping2 = np.unique(np.array(alabels.T), return_inverse=True) mtx = knnClass.confusion_matrix( np.matrix(mapping).T, np.matrix(mapping2).T) print "Training Confusion Matrix:" print knnClass.confusion_matrix_str(mtx) #----------------------------------------------------------------------- #----------------------------------------------------------------------- #-Classifies the test set data and prints out a confusion matrix. bcats, blabels = knnClass.classify(B) print "Done Classifying." unique, mapping = np.unique(np.array(test_cats.T), return_inverse=True) unique2, mapping2 = np.unique(np.array(blabels.T), return_inverse=True) mtx1 = knnClass.confusion_matrix( np.matrix(mapping).T, np.matrix(mapping2).T) print "Test Confusion Matrix:" print knnClass.confusion_matrix_str(mtx1) #----------------------------------------------------------------------- #Writes out a new CSV data file with the test set data # and the categories as an extra column data_test.addColumn("KNN Classification", bcats) data_test.toFile(filename="knnClass.csv") return
def __init__(self, width, height): # create a tk object, which is the root window self.root = tk.Tk() # width and height of the window self.initDx = width self.initDy = height # set up the geometry for the window self.root.geometry( "%dx%d+50+30" % (self.initDx, self.initDy) ) # set the title of the window self.root.title("Random Datapoints") # set the maximum size of the window for resizing self.root.maxsize( 1600, 900 ) # setup the menus self.buildMenus() # build the controls self.buildControls() # build the Canvas self.buildCanvas() # bring the window to the front self.root.lift() # - do idle events here to get actual canvas size self.root.update_idletasks() # now we can ask the size of the canvas print self.canvas.winfo_geometry() # set up the key bindings self.setBindings() # set up the application state self.objects = [] # list of data objects that will be drawn in the canvas self.data = None # will hold the raw data someday. self.baseClick = None # used to keep track of mouse movement #create new view object self.v=view.View() #create new data object self.data=data.Data() self.linRegLines = [] self.pcaList = [] self.num_PCA=0 self.num_cluster=0 self.clusterList=[] self.linRegEndpoints = None self.codebook=None self.codes=None #created matrix of endpoints of x,y, and z axes self.axes=np.matrix([[0,0,0,1],[1,0,0,1],[0,0,0,1],[0,1,0,1],[0,0,0,1],[0,0,1,1]]) #list to store the lines for the x, y, and z axes self.lines=[0,0,0] self.colorlist=[] #calls buildAxes function self.buildAxes() #create matrix for data points self.dataPointMatrix=np.zeros((4,4)) #updates the axes self.updateAxes() self.updateFits()
import torch import os import utility import data import model import loss from option import args from torch.nn import DataParallel torch.manual_seed(args.seed) checkpoint = utility.checkpoint(args) print(os.getcwd()) if checkpoint.ok: loader = data.Data(args) model = model.Model(args, checkpoint) model = DataParallel(model) loss = loss.Loss(args, checkpoint) if not args.test_only else None t = Trainer(args, loader, model, loss, checkpoint) while not t.terminate(): t.train() # t.test() checkpoint.done()
def main(argv): '''Reads in a training set and a test set and builds two KNN classifiers. One uses all of the data, one uses 10 exemplars. Then it classifies the test data and prints out the results. ''' # usage if len(argv) < 3: print( 'Usage: python %s <training data file> <test data file> <optional training category file> <optional test category file>' % (argv[0])) exit(-1) # read the training and test sets dtrain = data.Data(argv[1]) dtest = data.Data(argv[2]) # get the categories and the training data A and the test data B if len(argv) > 4: traincatdata = data.Data(argv[3]) testcatdata = data.Data(argv[4]) traincats = traincatdata.get_data([traincatdata.get_headers()[0]]) testcats = testcatdata.get_data([testcatdata.get_headers()[0]]) A = dtrain.get_data(dtrain.get_headers()) B = dtest.get_data(dtest.get_headers()) else: # assume the categories are the last column traincats = dtrain.get_data([dtrain.get_headers()[-1]]) testcats = dtest.get_data([dtest.get_headers()[-1]]) A = dtrain.get_data(dtrain.get_headers()[:-1]) B = dtest.get_data(dtest.get_headers()[:-1]) # create two classifiers, one using 10 exemplars per class knncall = classifiers.KNN() knnc10 = classifiers.KNN() # build the classifiers knncall.build(A, traincats) knnc10.build(A, traincats, 10) # use the classifiers on the test data allcats, alllabels = knncall.classify(B) tencats, tenlabels = knnc10.classify(B) # print the results print('Results using All Exemplars:') print(' True Est') for i in range(allcats.shape[0]): if int(testcats[i, 0]) == int(alllabels[i, 0]): print("%03d: %4d %4d" % (i, int(testcats[i, 0]), int(alllabels[i, 0]))) else: print("%03d: %4d %4d **" % (i, int(testcats[i, 0]), int(alllabels[i, 0]))) print(knnc10) print('Results using 10 Exemplars:') print(' True Est') for i in range(tencats.shape[0]): if int(testcats[i, 0]) == int(tenlabels[i, 0]): print("%03d: %4d %4d" % (i, int(testcats[i, 0]), int(tenlabels[i, 0]))) else: print("%03d: %4d %4d **" % (i, int(testcats[i, 0]), int(tenlabels[i, 0]))) return