def train(self, training_data, validation_data): if not self.initialize_vars: self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) self.initialize_vars = True actions, states, frames = training_data # print(type(frames), frames.shape) for e in range(self.EPOCHS): # do some training for i in range(10): train_loss, _ = self.sess.run( [self.total_loss, self.opt], { self.in_action: actions, self.in_state: states, self.in_frame: frames }) self.train_losses.append(train_loss) print("training loss: ", np.mean(self.train_losses)) # do some validation validation_loss = self.sess.run( [self.total_loss], { self.in_action: validation_data[0], self.in_state: validation_data[1], self.in_frame: validation_data[2] }) self.valid_losses.append(validation_loss) print("validation loss: ", np.mean(self.valid_losses)) util.save(SAVE_GRAPH_NAME, session=self.sess) self.train_losses = [] self.valid_losses = []
def main(): # get argument args = util.get_args() # print argument util.print_args(args) # run te mlus = [] rcs = [] # at every T step for t in range(args.num_test): if t % args.T == 0: repetita_args = util.get_repetita_args(args, t) print('command:', ' '.join(repetita_args)) stdout = util.call(repetita_args) if stdout: print('stdout:', stdout) mlu, rc = util.parse_result(t, stdout, args) if len(mlu) == args.T: mlus.append(mlu) if rc is not None: rcs.append(rc) util.save(mlus, rcs, args)
def create_reference_table(): words = util.load(words_path) times = np.array(util.load(data_path)) pos = util.load(pos_path) table = collections.defaultdict(dict) count = collections.defaultdict(dict) for w, t, p in zip(words, times, pos): if w == "": continue l = len(w) if p not in table or l not in table[p]: table[p][l] = t count[p][l] = 1 else: table[p][l] += t count[p][l] += 1 for p in table: for l in table[p]: table[p][l] = table[p][l] / float(count[p][l]) print(table) util.save(table, 'pos_len_table')
def send_var_fs(self, var_name, val): fname = ut.make_temp_nfs('.pk') ut.save(fname, val) res = self.view().execute('import util as ut; ut.wait_for_file("%s", 30); %s = ut.load("%s")' % (fname, var_name, fname)) res.wait() assert res.successful() os.remove(fname)
def run(self): # Counter for 99999 to save generated password in output file i = 0 # Generate password list by itertools passwords = itertools.product(self.chars, repeat=self.passLen) outputList = [] for password in passwords: # Check counter to save generated passwords into the output file if i == 99999: save(tup=outputList, outFile=self.outputFile) i = 0 # Clear list to avoid of memory leak outputList.clear() # Convert tuple to string it = ''.join(password) print(it) i += 1 # Append the generated password to list outputList.append(it) # Saving the last items that are in memory (RAM) and they are not stored in before save(outputList, self.outputFile)
def plot(results_path, fig_path): data = pd.read_csv( results_path.joinpath("trial_accuracy_means.csv"))\ .groupby('model').get_group('exp')\ .sort('trial') trials = data['trial'] acc = data['median'] * 100 fig, ax = plt.subplots() ax.plot(trials, acc, 'k.') ax.set_xlim(1, 200) ax.set_ylim(70, 100) ax.set_xlabel("Trial", fontsize=14) ax.set_ylabel("Percent correct", fontsize=14) util.clear_right(ax) util.clear_top(ax) util.outward_ticks(ax) fig.set_figheight(3.5) fig.set_figwidth(4.5) plt.draw() plt.tight_layout() pths = [fig_path.joinpath("trial_accuracy.%s" % ext) for ext in ('png', 'pdf')] for pth in pths: util.save(pth, close=False) return pths
def save_model(self, table_name="q_learner_tables.pkl"): """ save trained q learner, aka the q table (and t table and r table if dyna is included) :param table_name:saved table name """ tables = [self.q_table, self.t_table, self.r_table] save(tables, table_name)
def plot(dest, results_path, query): # load human mass responses human_mass = pd\ .read_csv(os.path.join(results_path, "human_mass_responses_by_stimulus.csv"))\ .groupby('version')\ .get_group('H')\ .set_index(['stimulus', 'kappa0'])\ .sortlevel()\ .unstack('kappa0')\ .reorder_levels([1, 0], axis=1) * 100 # load model mass responses model_mass = pd\ .read_csv(os.path.join(results_path, "model_mass_responses_by_stimulus.csv"))\ .set_index(['likelihood', 'counterfactual', 'stimulus', 'kappa0'])\ .sortlevel()\ .unstack('kappa0')\ .reorder_levels([1, 0], axis=1) # load mass correlations mass_corrs = pd\ .read_csv(os.path.join(results_path, "mass_responses_by_stimulus_corrs.csv"))\ .set_index(['version', 'likelihood', 'counterfactual'])\ .sortlevel()\ .ix['H'] # color config plot_config = util.load_config()["plots"] #palette = plot_config["colors"] #colors = [palette[0], palette[2]] colors = ['.4', '.1'] markers = ['o', 's'] lightgrey = plot_config["lightgrey"] sns.set_style("white", {'axes.edgecolor': lightgrey}) # create the figure fig, ax = plt.subplots() plot_kappas(ax, model_mass.ix[('empirical', True)], human_mass, colors, markers) ax.set_xlabel(r"Empirical observer, $p(\kappa=10|F_t,S_t)$") ax.set_ylabel(r"% participants choosing $\kappa=10$") format_mass_plot(ax, lightgrey) add_corr(ax, mass_corrs.ix[('empirical', True)]) # create the legend make_legend(ax, colors, markers) # clear the top and right axis lines sns.despine() # set figure size fig.set_figheight(3) fig.set_figwidth(3.5) plt.draw() plt.tight_layout() # save for pth in dest: util.save(pth, close=False)
def codeProject(args,flag,data): PARAM_KEY = 1 PARAM_PATH = 2 PARAM_FORMATTER = 3 ARGUMENTS = len(args)-1 # JSON mapping files and storage of this if( keyExists("projects",args[1])): if( "stdout" in args[2]): project = json.loads(load("projects/"+args[PARAM_KEY])); # Uses key value storage directory = args[PARAM_PATH] + "/" + args[PARAM_KEY] mkdir(directory) for x in project.keys(): # Reflect that with here _file = json.loads(load("files/"+x)); out = ''; for y in _file: block = str(load("blocks/"+ y)) if(ARGUMENTS == PARAM_FORMATTER): # Alter all the blocks in said fashion block = format.block(block, args[PARAM_FORMATTER]) out += block # Output the file with the correct file name save(directory + "/" + project[x],out) else: error("Error: Project does not exist")
def store(frame_list, tube_bbox, video_id, utterance, person_id, start, end, video_count, chunk_start, args): out, final_bbox = crop_bbox_from_frames(frame_list, tube_bbox, min_frames=args.min_frames, image_shape=args.image_shape, min_size=args.min_size, increase_area=args.increase) if out is None: return [] start += round(chunk_start * REF_FPS) end += round(chunk_start * REF_FPS) name = (person_id + "#" + video_id + "#" + utterance + '#' + str(video_count).zfill(3) + ".mp4") partition = 'test' if person_id in TEST_PERSONS else 'train' save(os.path.join(args.out_folder, partition, name), out, args.format) return [{ 'bbox': '-'.join(map(str, final_bbox)), 'start': start, 'end': end, 'fps': REF_FPS, 'video_id': '#'.join([video_id, person_id]), 'height': frame_list[0].shape[0], 'width': frame_list[0].shape[1], 'partition': partition }]
def vbow(input_file, output_folder): img, img_path = load_images_from_file( input_file) # take all images category by category size = [5, 50, 100, 250, 500, 750, 1000] kp = extract_kp(img) #sift = sift_features(img, kp) gist = gist_features(img, kp) descriptors = [gist] #descriptors = [gist, sift] for d in descriptors: name = d[0] desc_list = d[1] # Takes the descriptor list which is unordered one cat_list = d[ 2] # Takes the sift features that is seperated class by class for train data create_dir(output_folder) create_dir(f"{output_folder}/{name}/") for s in size: create_dir(f"{output_folder}/{name}/{s}/") print(f"--- Create VBoW for {name} features. ---") for s in size: vbow = generate_vbow( s, desc_list, cat_list, 100, 10) # Takes the central points which is visual words save(vbow, img_path, f"{output_folder}/{name}/{s}/")
def accept(self): """ OK """ args = str(self.ui.txtArgs.toPlainText()).replace("\n", " ") self.arglist = args.split(' ') self._working_dir = str(self.ui.txtWorkDir.text()) util.save(args, self._working_dir) QDialog.accept(self)
def closure(): optimizer.zero_grad() vgg_model(input_image) style_score = 0 content_score = 0 for s in style_losses: style_score += s.loss for c in content_losses: content_score += c.loss style_score *= style_weight content_score *= content_weight loss = style_score + content_score loss.backward() # puede darse que la imagen haya acabado con píxeles fuera de [0,1], esto lo arregla input_image.data.clamp_(0, 1) i[0] += 1 if i[0] % 50 == 0: print("iteraciones: {}:".format(i[0])) print( 'Diferencia de estilo : {:4f} Diferencia de contenido: {:4f}' .format(style_score.item(), content_score.item())) if path != "": save(input_image, path.format(i[0])) print() return style_score + content_score
def plot(results_path, fig_path): data = pd.read_csv( results_path.joinpath("trial_time_means.csv"))\ .groupby('model').get_group('exp')\ .sort_values(by='trial') trials = data['trial'] times = data['median'] fig, ax = plt.subplots() ax.plot(trials, times, 'k.') ax.set_xlim(1, 200) ax.set_xlabel("Trial", fontsize=14) ax.set_ylabel("Response time", fontsize=14) util.clear_right(ax) util.clear_top(ax) util.outward_ticks(ax) fig.set_figheight(3.5) fig.set_figwidth(4.5) plt.draw() plt.tight_layout() pths = [fig_path.joinpath("trial_time.%s" % ext) for ext in ('png', 'pdf')] for pth in pths: util.save(pth, close=False) return pths
def codeFile(args,flag,data): PARAM_KEY = 1; PARAM_FILE = 2; # Output file location PARAM_FORMATTER = 3 ARGUMENTS = len(args)-1 # Ability to add a block of code through copy and paste and have it formatted correctly! if( keyExists("files",args[PARAM_KEY])): _file = json.loads(load("files/"+args[PARAM_KEY])); out = '' # loadJSON for x in _file: block = str(load("blocks/"+ x)) if(ARGUMENTS == PARAM_FORMATTER): # Alter all the blocks in said fashion block = format.block(block, args[PARAM_FORMATTER]) out += block out += "\n" # Adds some spacing between blocks # No file specified if(len(args) < 3 ): log(out) else: log("Saving to file "+ args[PARAM_FILE] ) save(args[PARAM_FILE],out) else: error("Error: File does not exist")
def load_dataset(fname, nb_lines): """Load the Amazon dataset if not already present on disc""" import os.path if os.path.isfile('safe/Amazon-'+str(nb_lines)+'.p'): return util.load('safe/Amazon-'+str(nb_lines)+'.p') count = 1 X = [] y = [] with open(fname) as f: for line in f: text, label = read_line(line) #print((label, text)) X.append(text) y.append(label) if count >= nb_lines: break count+=1 #load pretrained dictonary dico = util.load('safe/vocab_gensim.p') preprocessor = text_preprocessing.Preprocessor(dico=dico) X = preprocessor.preprocess(X) #save the loaded dataset in a pickle for speeding up next run util.save((X,y), 'safe/Amazon-'+str(nb_lines)+'.p') return (X, y)
def run_style_transfer(content_image, style_image, input_image, num_steps=300, style_weight=1000000, content_weight=1, path=""): vgg_model, style_losses, content_losses = get_style_model_and_losses( style_image, content_image, style_layers=style_layers, content_layers=content_layers) optimizer = get_input_optimizer(input_image) i = [0] input_image.data.clamp_(0, 1) if path != "": path += "\\{}.jpg" save(input_image, path.format(i[0])) while i[0] <= num_steps: def closure(): optimizer.zero_grad() vgg_model(input_image) style_score = 0 content_score = 0 for s in style_losses: style_score += s.loss for c in content_losses: content_score += c.loss style_score *= style_weight content_score *= content_weight loss = style_score + content_score loss.backward() # puede darse que la imagen haya acabado con píxeles fuera de [0,1], esto lo arregla input_image.data.clamp_(0, 1) i[0] += 1 if i[0] % 50 == 0: print("iteraciones: {}:".format(i[0])) print( 'Diferencia de estilo : {:4f} Diferencia de contenido: {:4f}' .format(style_score.item(), content_score.item())) if path != "": save(input_image, path.format(i[0])) print() return style_score + content_score optimizer.step(closure) return input_image
def plot(dest, results_path, counterfactual, likelihood): if likelihood == 'ipe': likelihood = 'ipe_' + util.get_query() # load in the model data, which includes fitted parameters data = pd\ .read_csv(os.path.join(results_path, 'model_belief_by_trial_fit.csv'))\ .groupby(['fitted', 'counterfactual', 'likelihood'])\ .get_group((True, counterfactual, likelihood))\ .drop_duplicates(['version', 'model', 'pid', 'B'])\ .set_index(['version', 'model', 'pid'])\ .sortlevel() # double check that there is exactly one parameter for each pid assert data.index.is_unique # plotting config stuff plot_config = util.load_config()["plots"] all_colors = plot_config["colors"] colors = {'static': all_colors[1], 'learning': all_colors[0]} # create the figure and plot the histograms fig, axes = plt.subplots(2, 3, sharex=True) for i, version in enumerate(['H', 'G', 'I']): for j, model in enumerate(['static', 'learning']): hist(axes[j, i], data.ix[(version, model)]['B'], plot_config['darkgrey']) # set titles and axis labels for i in range(3): axes[0, i].set_title('Experiment {}'.format(i + 1), y=1.05) for ax in axes[:, 0]: ax.set_ylabel("% participants") for ax in axes[1]: ax.set_xlabel(r"Best fit learning rate ($\beta$)") for i, label in enumerate(['Static', 'Learning']): mid = sum(axes[i, 0].get_ylim()) / 2.0 axes[i, 0].text( -2.6, mid, label, rotation=90, fontsize=12, # same as title font size verticalalignment='center') # clear top and right axis lines sns.despine() # set figure size fig.set_figwidth(6.5) fig.set_figheight(3.25) plt.draw() plt.tight_layout() plt.subplots_adjust(left=0.14) # save for pth in dest: util.save(pth, close=False)
def saveTest(): data = [0,1,2,3,4] name = "test.pkl" util.save(data,name) data2 = util.load(name) print data, data2 return data == data2
def train(args, params): print("Loading training set...") train = load.load_dataset(params['train']) print("Loading dev set...") dev = load.load_dataset(params['dev']) print("Building preprocessor...") preproc = load.Preproc(*train) print("train_set_classes:", preproc.classes) print("Training size: " + str(len(train[0])) + " examples.") print("Dev size: " + str(len(dev[0])) + " examples.") save_dir = make_save_dir(params['save_dir'], args.experiment) util.save(preproc, save_dir) params.update({ "input_shape": [None, 1], "num_categories": len(preproc.classes) }) model = network.build_network(**params) stopping = keras.callbacks.EarlyStopping(patience=30) reduce_lr = keras.callbacks.ReduceLROnPlateau( factor=0.1, patience=2, min_lr=params["learning_rate"] * 0.001) checkpointer = keras.callbacks.ModelCheckpoint( filepath=get_filename_for_saving(save_dir), save_best_only=False) batch_size = params.get("batch_size", 32) # summary = str(model.summary(print_fn=lambda x: fh.write(x + '\n'))) # out = open("/content/ecg/report.txt",'w') # out.write(summary) # out.close if params.get("generator", False): train_gen = load.data_generator(batch_size, preproc, *train) dev_gen = load.data_generator(batch_size, preproc, *dev) model.fit_generator(train_gen, steps_per_epoch=int(len(train[0]) / batch_size), epochs=MAX_EPOCHS, validation_data=dev_gen, validation_steps=int(len(dev[0]) / batch_size), callbacks=[checkpointer, reduce_lr, stopping]) # util.learning_curve(history) else: train_x, train_y = preproc.process(*train) dev_x, dev_y = preproc.process(*dev) model.fit(train_x, train_y, batch_size=batch_size, epochs=MAX_EPOCHS, validation_data=(dev_x, dev_y), callbacks=[checkpointer, reduce_lr, stopping])
def plot(results_path, fig_path): order = ['exp', 'oc', 'th', 'hc', 'bq', 'bqp'] titles = { 'exp': "Human", 'oc': "Oracle", 'th': "Threshold", 'hc': "HC", 'bq': "BQ (equal prior)", 'bqp': "BQ (unequal prior)" } pth = results_path.joinpath("all_response_times.pkl") with open(pth, "r") as fh: times = pickle.load(fh) fig, axes = plt.subplots(1, len(order), sharey=True) for i, key in enumerate(order): ax = axes[i] bins = 200 if key == 'exp': hist, edges = np.histogram( times[key] / 1000., bins=bins, range=(0, 20)) else: hist, edges = np.histogram( times[key], bins=bins, range=(0, bins)) edges = edges[:101] hist = hist[:100] hist = hist * 100 / float(len(times[key])) width = edges[1] - edges[0] ax.bar(edges[:-1], hist, width=width, color='k') ax.set_xlim(0, edges[-1]) ax.set_title(titles[key], fontsize=14) util.clear_right(ax) util.clear_top(ax) util.outward_ticks(ax) if key == 'exp': ax.set_xlabel("RT (seconds)", fontsize=14) else: ax.set_xlabel("Number of actions", fontsize=14) axes[0].set_ylabel("Percent", fontsize=14) fig.set_figheight(2.5) fig.set_figwidth(16) plt.draw() plt.tight_layout() pths = [fig_path.joinpath("response_time_histograms.%s" % ext) for ext in ('png', 'pdf')] for pth in pths: util.save(pth, close=False) return pths
def plot(results_path, fig_path): order = ['exp', 'oc', 'th', 'hc', 'bq', 'bqp'] titles = { 'exp': "Human", 'oc': "Oracle", 'th': "Threshold", 'hc': "HC", 'bq': "BQ (equal prior)", 'bqp': "BQ (unequal prior)" } pth = results_path.joinpath("all_response_times.pkl") with open(pth, "r") as fh: times = pickle.load(fh) fig, axes = plt.subplots(2, 3, sharey=True) for i, key in enumerate(order): ax = axes.flat[i] bins = 200 if key == 'exp': hist, edges = np.histogram( times[key] / 1000., bins=bins, range=(0, 20)) else: hist, edges = np.histogram( times[key], bins=bins, range=(0, bins)) edges = edges[:101] hist = hist[:100] hist = hist * 100 / float(len(times[key])) ax.fill_between(edges[:-1], hist, np.zeros_like(hist), color='#666666') ax.set_xlim(0, edges[-1]) ax.set_title(titles[key]) if key == 'exp': ax.set_xlabel("RT (seconds)") else: ax.set_xlabel("Number of actions") axes[0, 0].set_ylabel("Percent") axes[1, 0].set_ylabel("Percent") sns.despine() fig.set_figheight(4) fig.set_figwidth(6) plt.tight_layout() plt.subplots_adjust(left=0.1) pths = [fig_path.joinpath("response_time_histograms.%s" % ext) for ext in ('png', 'pdf')] for pth in pths: util.save(pth, close=False) return pths
def downloadYear(year): yearString = str(year) examineUrl = '/year/' + yearString limit = -1 navUrls = {} pageUrls = {} navUrls[examineUrl] = False newNavExamined = True count = 0 while (newNavExamined and (limit == -1 or count < limit)): newNavExamined = False for url, examined in navUrls.iteritems(): if not examined: time.sleep(2) try: newPages, newNavs = scanForUrls(url) except: print 'Exception occured ' + sys.exc_info()[0] break for page in newPages: if not page in pageUrls: print 'Adding page ' + page pageUrls[page] = False for nav in newNavs: if not nav in navUrls: print 'Adding nav ' + nav navUrls[nav] = False navUrls[url] = True newNavExamined = True break count += 1 print 'Examined ' + str(count) + ' nav pages' entries = [] for path in pageUrls: time.sleep(2) try: entry = parsePage(path) except: print 'Exception occured ' + sys.exc_info()[0] break entries.append(entry) save(entries, 'data/' + yearString + '.json') print 'Downloaded ' + str(len(entries)) + ' applications'
def run(video_id, args): if not os.path.exists( os.path.join(args.video_folder, video_id.split('#')[0] + '.mp4')): download(video_id.split('#')[0], args) if not os.path.exists( os.path.join(args.video_folder, video_id.split('#')[0] + '.mp4')): print('Can not load video %s, broken link' % video_id.split('#')[0]) return reader = imageio.get_reader( os.path.join(args.video_folder, video_id.split('#')[0] + '.mp4')) fps = reader.get_meta_data()['fps'] df = pd.read_csv(args.metadata) df = df[df['video_id'] == video_id] all_chunks_dict = [{ 'start': df['start'].iloc[j], 'end': df['end'].iloc[j], 'bbox': list(map(int, df['bbox'].iloc[j].split('-'))), 'frames': [] } for j in range(df.shape[0])] ref_fps = df['fps'].iloc[0] ref_height = df['height'].iloc[0] ref_width = df['width'].iloc[0] partition = df['partition'].iloc[0] try: for i, frame in enumerate(reader): for entry in all_chunks_dict: if (i * ref_fps >= entry['start'] * fps) and ( i * ref_fps < entry['end'] * fps): left, top, right, bot = entry['bbox'] left = int(left / (ref_width / frame.shape[1])) top = int(top / (ref_height / frame.shape[0])) right = int(right / (ref_width / frame.shape[1])) bot = int(bot / (ref_height / frame.shape[0])) crop = frame[top:bot, left:right] if args.image_shape is not None: crop = img_as_ubyte( resize(crop, args.image_shape, anti_aliasing=True)) entry['frames'].append(crop) except imageio.core.format.CannotReadFrameError: None for entry in all_chunks_dict: if 'person_id' in df: first_part = df['person_id'].iloc[0] + "#" else: first_part = "" first_part = first_part + '#'.join(video_id.split('#')[::-1]) path = first_part + '#' + str(entry['start']).zfill(6) + '#' + str( entry['end']).zfill(6) + '.mp4' save(os.path.join(args.out_folder, partition, path), entry['frames'], args.format)
def getDataAndLabels(): data = read(DATA_PICKLE) labels = read(LABELS_PICKLE) if data is None or labels is None: data = np.load(DATA_FILE) labels = np.load(LABELS_FILE) save(data, DATA_PICKLE) save(labels, LABELS_PICKLE) labels = transformLabels(labels) return [data, labels]
def train(args, params): print("Loading training set...") train = load.load_dataset(params['train']) print("Loading dev set...") dev = load.load_dataset(params['dev']) print("Building preprocessor...") preproc = load.Preproc(*train) print("Training size: " + str(len(train[0])) + " examples.") print("Dev size: " + str(len(dev[0])) + " examples.") save_dir = make_save_dir(params['save_dir'], args.experiment) util.save(preproc, save_dir) params.update({ "input_shape": [None, 1], "num_categories": len(preproc.classes) }) model = network.build_network(**params) stopping = keras.callbacks.EarlyStopping(patience=8) reduce_lr = keras.callbacks.ReduceLROnPlateau( factor=0.1, patience=2, min_lr=params["learning_rate"] * 0.001) checkpointer = keras.callbacks.ModelCheckpoint( filepath=get_filename_for_saving(save_dir), save_best_only=False) batch_size = params.get("batch_size", 32) if params.get("generator", False): train_gen = load.data_generator(batch_size, preproc, *train) dev_gen = load.data_generator(batch_size, preproc, *dev) model.fit_generator( train_gen, steps_per_epoch=int(len(train[0]) / batch_size), epochs=MAX_EPOCHS, validation_data=dev_gen, validation_steps=int(len(dev[0]) / batch_size), callbacks=[checkpointer, reduce_lr, stopping]) else: train_x, train_y = preproc.process(*train) dev_x, dev_y = preproc.process(*dev) model.fit( train_x, train_y, batch_size=batch_size, epochs=MAX_EPOCHS, validation_data=(dev_x, dev_y), callbacks=[checkpointer, reduce_lr, stopping])
def train(args, params): print("Loading training set...") train = load.load_dataset(params['train']) print("Loading dev set...") dev = load.load_dataset(params['dev']) print("Building preprocessor...") preproc = load.Preproc(*train) print("Training size: " + str(len(train[0])) + " examples.") print("Dev size: " + str(len(dev[0])) + " examples.") save_dir = make_save_dir(params['save_dir'], args.experiment) util.save(preproc, save_dir) params.update({ "input_shape": [None, 1], "num_categories": len(preproc.classes) }) print(params) model = network.build_network(**params) stopping = keras.callbacks.EarlyStopping(patience=8) reduce_lr = keras.callbacks.ReduceLROnPlateau( factor=0.1, patience=2, min_lr=params["learning_rate"] * 0.001) checkpointer = keras.callbacks.ModelCheckpoint( filepath=get_filename_for_saving(save_dir), save_best_only=False) ckpt_best = keras.callbacks.ModelCheckpoint(os.path.join( save_dir, 'best.hdf5'), save_best_only=True) batch_size = params.get("batch_size", 32) if params.get("generator", False): train_gen = load.data_generator(batch_size, preproc, *train) dev_gen = load.data_generator(batch_size, preproc, *dev) model.fit_generator( train_gen, steps_per_epoch=int(len(train[0]) / batch_size), epochs=MAX_EPOCHS, validation_data=dev_gen, validation_steps=int(len(dev[0]) / batch_size), callbacks=[checkpointer, ckpt_best, reduce_lr, stopping]) else: train_x, train_y = preproc.process(*train) dev_x, dev_y = preproc.process(*dev) model.fit(train_x, train_y, batch_size=batch_size, epochs=MAX_EPOCHS, validation_data=(dev_x, dev_y), callbacks=[checkpointer, ckpt_best, reduce_lr, stopping])
def insertBlock(args,flag, data): if(flag == 1): log("Creating Block \"" + get("key") + "\"") save("blocks/"+get("key"),data) set("continued",0) printBlue("Block Creation Complete.") else: set("continued",1) set("prevCmd", "create") set("key",args[1]) log("Enter Block Information and press Ctrl+D when done!")
def save_model(model, model_name, mode, n_hiddens, act_fun, n_comps, batch_norm): assert is_data_loaded(), 'Dataset hasn\'t been loaded' savedir = root_output + data_name + '/' util.make_folder(savedir) filename = create_model_id(model_name, mode, n_hiddens, act_fun, n_comps, batch_norm) util.save(model, savedir + filename + '.pkl')
def self_model(): feature_vocab = _build_feature_vocab(Option.src) from perm import TreeParamPermModel as PM pm = PM(feature_vocab, Option) logger.info('[Train] Loading the source language...:%s' % Option.src) Global.mx_dep = Option.mx_dep_train data_train = Data(Option.src, conll_reader, valid=lambda x: len(x) < Option.mx_len) logger.info('Training...') fit(pm, {'data': data_train, 'mx_sent': Option.mx_sent, 'batch_size': Option.batch_size}, nb_epoch=Option.mx_epoch, callbacks=_callbacks(model='')) save(fn=Option.model, model={'model': pm.state_dict(), 'feature_vocab': feature_vocab})
def getDataAndLabels(): data = read(DATA_PATH) labels = read(LABELS_PATH) if data is None or labels is None: for dir in files: for index in files[dir]: [data, labels] = doStuff(dir, index, data, labels) save(data, DATA_PATH) save(labels, LABELS_PATH) labels = transformLabels(labels) return [data, labels]
def build(root, dest): dest = os.path.join(dest, 'rbnf') shutil.rmtree(dest) makedirs(dest) path = os.path.join(root, 'common/rbnf') names = [os.path.splitext(n)[0] for n in os.listdir(path)] for name in names: tree = readxml(os.path.join(path, '%s.xml' % name)) rbnf = convert(tree) rbnf = to_utf8(rbnf) out = os.path.join(dest, '%s.json' % name) save(out, rbnf)
def handle_update(): statuses = get_statuses() previous_statuses = get_latest_statuses(statuses.keys()) diffs = find_diffs(previous_statuses, statuses) log.debug(f'Changes: {diffs}') if diffs: send_updates(diffs) time = get_time() for location, status in statuses.items(): save(location, time, status)
def run(config): opt_cfg = config["optimizer"] data_cfg = config["data"] # Model model = LinearModel(config=config) model.cuda() if use_cuda else model.cpu() # Loaders batch_size = opt_cfg["batch_size"] preproc = None #loader.Preprocessor(data_cfg["train_set"]) train_ldr = loader.make_loader(data_cfg["train_set"], batch_size, model, config) dev_ldr = loader.make_loader(data_cfg["dev_set"], batch_size, model, config) # Optimizer optimizer = torch.optim.SGD(model.parameters(), lr=opt_cfg["learning_rate"], momentum=opt_cfg["momentum"]) run_state = (0, 0, 0) best_so_far = float("inf") for e in range(opt_cfg["epochs"]): start = time.time() run_state = run_epoch(model, optimizer, train_ldr, *run_state) msg = "Epoch {} completed in {:.2f} (s)." print(msg.format(e, time.time() - start)) dev_loss = eval_loop(model, dev_ldr) print("Dev Loss: {:.5f}".format(dev_loss)) tb.log_value("dev_loss", dev_loss, e) # # Log for tensorboard # tb.log_value("dev_loss", dev_loss, e) # tb.log_value("dev_map", dev_map, e) # # print("Dev Loss: {:.2f}".format(dev_loss)) # print("Dev mAP: {:.2f}".format(dev_map)) # # util.save(model, preproc, config["save_path"]) # # # Save the best model by F1 score on the dev set # if dev_map > best_so_far: # best_so_far = dev_map util.save(model, preproc, config["save_path"], tag="best")
def save(self, file_name): params_dict = {} params_dict['lst_layer_names'] = [ layer.name for layer in self._lst_layers ] params_dict['lst_layer_type'] = self._lst_layer_type params_dict['lst_num_hid'] = self._lst_num_hid params_dict['data_dim'] = self._data_dim for layer in self._lst_layers: layer.add_params_to_dict(params_dict) util.save(file_name, " ".join(params_dict.keys()), params_dict)
def main(): f = dict() f[1] = _make_1d_1d if len(sys.argv) == 1: lv = 1 else: lv = int(sys.argv[1]) eq = f[lv]() save('mathdrill.pdf', eq, '-')
def save_tasks(self, sender): """Save the tasks to the specified file.""" task_file = self.save_dialog['txt_save_file'].text if task_file: if task_file.rfind('.tsk', len(task_file) - 4) == -1: task_file += '.tsk' self.save_dialog.close() if task_file == self.current_task_file: # some bug; even though the file should be closed, I can't overwrite it util.delete(task_file) util.save(self.tasklist.tasks, task_file) else: self.save_dialog['txt_save_file'].text = ''
def gist(input_file, output_folder): img_dict, img_dict_path = load_images_from_file( input_file) # take all images category by category gist = gist_img(img_dict) name = gist[0] desc_list = gist[1] # Takes the descriptor list which is unordered one cat_dict = gist[ 2] # Takes the sift features that is seperated class by class for train data create_dir(output_folder) create_dir(f"{output_folder}/{name}/") save(cat_dict, img_dict_path, f"{output_folder}/{name}/") print(f"\nThe execution of gist is done!\n")
def save_tasks(self, sender): """Save the tasks to the specified file.""" task_file = self.save_dialog["txt_save_file"].text if task_file: if task_file.rfind(".tsk", len(task_file) - 4) == -1: task_file += ".tsk" self.save_dialog.close() if task_file == self.current_task_file: # some bug; even though the file should be closed, I can't overwrite it util.delete(task_file) util.save(self.tasklist.tasks, task_file) else: self.save_dialog["txt_save_file"].text = ""
def save_tasks(self, sender): """Save the tasks to the specified file.""" task_file = self.save_dialog['textfield1'].text if not task_file == '': if task_file.rfind('.tsk', len(task_file) -4) == -1: task_file += '.tsk' self.save_dialog.close() if task_file == self.current_task_file: # some bug; even though the file should be closed, I can't overwrite util.delete(task_file) util.save(self.tasklist.tasks, task_file) else: self.save_dialog['textfield1'].text = ''
def convert_dnss_model(self, new_model): dict = { 'l1_vh': self.weights[1]['w'], 'l1_hb': self.weights[1]['hb'], 'l2_vh': self.weights[2]['w'], 'l2_hb': self.weights[2]['hb'], 'l3_vh': self.weights[3]['w'], 'l3_hb': self.weights[3]['hb'], 'l4_vh': self.weights[4]['w'], 'l4_hb': self.weights[4]['hb'], 'l5_vh': self.weights[5]['w'], 'l5_hb': self.weights[5]['hb'] } varlist = 'l1_vh l1_hb l2_vh l2_hb l3_vh l3_hb l4_vh l4_hb l5_vh l5_hb' util.save(new_model, varlist, dict)
def run(self, phase, state, checkpoint=False, **kwargs): if checkpoint: save(state, self.temp_path) for _ in range(self.max_trials): try: phase(state, self, **kwargs) except GameStateError as e: if checkpoint: state = load(self.temp_path) except Exception as e: print('Error in this phase!') raise e else: break
def insertFile(args,flag,data): # PARAM_CONSTRUCT = 2 PARAM_KEY = 1 PARAM_BLOCKS_BASE = 2 data = [] files = os.listdir("blocks") for x in args[PARAM_BLOCKS_BASE:]: if("unset" not in get("tag")): # Tagging system works! x = get("tag") + "." + x if x in files: data.append(x) else : print "Error: Invalid Block" save("files/" + args[PARAM_KEY], json.dumps(data))
def plot(results_path, fig_path): order = ['hc', 'bq', 'bqp'] titles = { 'exp': "Human", 'hc': "HC", 'bq': "BQ (equal prior)", 'bqp': "BQ (unequal prior)" } results = pd.read_csv( results_path.joinpath("accuracy_means.csv"))\ .set_index(['stimulus', 'theta', 'flipped', 'model'])['median']\ .unstack(['model', 'flipped']) * 100 fig, axes = plt.subplots(1, len(order), sharey=True, sharex=True) for i, key in enumerate(order): ax = axes[i] for flipped in ['same', 'flipped']: ax.plot( results[(key, flipped)], results[('exp', flipped)], '.', alpha=0.8, label=flipped) ax.set_xlabel("Model accuracy", fontsize=14) ax.set_title(titles[key], fontsize=14) util.clear_right(ax) util.clear_top(ax) util.outward_ticks(ax) axes[0].set_ylabel("Human accuracy", fontsize=14) axes[0].set_xlim(-5, 105) axes[0].set_ylim(45, 105) axes[0].legend(loc=0, numpoints=1) fig.set_figheight(3) fig.set_figwidth(8) plt.draw() plt.tight_layout() pths = [fig_path.joinpath("accuracy_scatters.%s" % ext) for ext in ('png', 'pdf')] for pth in pths: util.save(pth, close=False) return pths
def plot_key(key, results_path, fig_path): fig, axes = plt.subplots(4, 5, sharey=True, sharex=True) means = pd.read_csv( results_path.joinpath("theta_time_stimulus.csv"))\ .set_index(['stimulus', 'flipped', 'model'])\ .groupby(level='model').get_group(key) for i, (stim, sdf) in enumerate(means.groupby(level='stimulus')): ax = axes.flat[i] for flipped, stats in sdf.groupby(level='flipped'): lower = stats['median'] - stats['lower'] upper = stats['upper'] - stats['median'] ax.errorbar( stats['modtheta'], stats['median'], yerr=[lower, upper], label=flipped, lw=3) ax.set_xlabel("Rotation") ax.set_xticks([0, 60, 120, 180]) ax.set_xlim(-10, 190) util.clear_right(ax) util.clear_top(ax) util.outward_ticks(ax) ax.set_title("Stim %s" % stim) fig.set_figheight(8) fig.set_figwidth(10) plt.draw() plt.tight_layout() pths = [fig_path.joinpath("response_time_stimuli_%s.%s" % (key, ext)) for ext in ('png', 'pdf')] for pth in pths: util.save(pth, close=False) return pths
def codeBlock(args,flag,data): PARAM_KEY = 1 PARAM_FORMATTER = 2 PARAM_FILE = 3 PARAM_LINE = 4 ARGUMENTS = len(args)-1 # Ability to add a block of code through copy and paste and have it formatted correctly! if( keyExists("blocks",args[PARAM_KEY])): block = load("blocks/"+args[PARAM_KEY]) # Format these blocks if(ARGUMENTS == PARAM_FORMATTER): # Format blocks block = format.block(block, args[PARAM_FORMATTER]) if(ARGUMENTS <= PARAM_FORMATTER): # No file specified log(block) else: if(ARGUMENTS == PARAM_FILE): log("Saving to file "+ args[PARAM_FILE] ) save(args[PARAM_FILE],block) elif(ARGUMENTS >= PARAM_LINE): # Argument for line save(args[PARAM_FILE],block,args[PARAM_LINE]) else: error("Error: Block does not exist")
def insertModule (args,flag,data): # Creates a project based on file jsons on the layout of the code block # PARAM_CONSTRUCT = 2 log("Creating module!") PARAM_KEY = 1 PARAM_NAME_DIR = 2 # key : value PARAM_FILES_BASE = 3 # Use this as key value stores, as we don't have a specific domain and naming data = [] files = os.listdir("files") ARGUMENTS = len(args[PARAM_FILES_BASE:]) - 1 print ARGUMENTS # Another method of constructing a project for x in args[PARAM_FILES_BASE:]: # Split the arguments here s = x.split(":") # Which is a key value store # We assume the first argument is the key value if s[0] in files: data.append(s) else : print "Error: Invalid File" save("projects/" + args[PARAM_KEY], json.dumps(data))
def plot(results_path, fig_path): order = ['exp', 'oc', 'th', 'hc', 'bq', 'bqp'] titles = { 'exp': "Human", 'oc': "Oracle", 'th': "Threshold", 'hc': "HC", 'bq': "BQ (equal prior)", 'bqp': "BQ (unequal prior)" } colors = { 'same': [ 0.16339869, 0.4449827 , 0.69750096], 'flipped': [ 0.72848904, 0.1550173 , 0.19738562] } time_results = pd.read_csv( results_path.joinpath("theta_time.csv")) acc_results = pd.read_csv( results_path.joinpath("theta_accuracy.csv")) fig, axes = plt.subplots(2, len(order), sharex=True) for i, key in enumerate(order): ax = axes[0, i] df = time_results.groupby('model').get_group(key) for flipped, stats in df.groupby('flipped'): if key == 'exp': median = stats['median'] / 1000. lower = (stats['median'] - stats['lower']) / 1000. upper = (stats['upper'] - stats['median']) / 1000. else: median = stats['median'] lower = stats['median'] - stats['lower'] upper = stats['upper'] - stats['median'] ax.errorbar( stats['modtheta'], median, yerr=[lower, upper], lw=3, color=colors[flipped], ecolor=colors[flipped]) ax.set_xticks(np.arange(0, 200, 30)) ax.set_xlim(-10, 190) ax.set_title(titles[key], fontsize=14) if key == 'exp': ax.set_ylabel("Response time", fontsize=14) else: ax.set_ylabel("# actions", fontsize=14) util.sync_ylims(axes[0, order.index('bq')], axes[0, order.index('bqp')]) for i, key in enumerate(order): ax = axes[1, i] df = acc_results.groupby('model').get_group(key) for flipped, stats in df.groupby('flipped'): lower = stats['median'] - stats['lower'] upper = stats['upper'] - stats['median'] ax.errorbar( stats['modtheta'], stats['median'], yerr=[lower, upper], lw=3, color=colors[flipped], ecolor=colors[flipped]) ax.set_xlim(-10, 190) ax.set_ylim(25, 105) ax.set_xticks(np.arange(0, 200, 30)) ax.set_xlabel("Rotation", fontsize=14) ax.set_ylabel("Accuracy", fontsize=14) for ax in axes.flat: util.clear_right(ax) util.clear_top(ax) util.outward_ticks(ax) p0 = plt.Rectangle( (0, 0), 1, 1, fc=colors['same'], ec=colors['same']) p1 = plt.Rectangle( (0, 0), 1, 1, fc=colors['flipped'], ec=colors['flipped']) leg = axes[1, 1].legend( [p0, p1], ["\"same\" pairs", "\"flipped\" pairs"], numpoints=1, fontsize=12, loc='lower center', title='Stimuli') frame = leg.get_frame() frame.set_edgecolor('#FFFFFF') util.sync_ylabel_coords(axes.flat, -0.175) fig.set_figheight(4) fig.set_figwidth(18) plt.draw() plt.tight_layout() plt.subplots_adjust(wspace=0.4) pths = [fig_path.joinpath("response_time_accuracy.%s" % ext) for ext in ('png', 'pdf')] for pth in pths: util.save(pth, close=False) return pths
pl.xlabel('Relative Importance') return fig if __name__ == '__main__': from util import show, save ###ROC### Y_true = [1, 1, 0, 0] pred1 = [0.9, 0.8, 0.6, 0.4] pred2 = [0.8, 0.3, 0.5, 0.4] plot_roc(Y_true, [pred1]) labels = ["Good model", "Bad model"] save(plot_roc(Y_true, [pred1, pred2], labels=labels), "roc.png") ###TRAIN_VS_TEST### from sklearn import linear_model ########################################################################### # Generate sample data n_samples_train, n_samples_test, n_features = 75, 150, 500 #np.random.seed(0) coef = np.random.randn(n_features) coef[50:] = 0.0 # only the top 10 features are impacting the model X = np.random.randn(n_samples_train + n_samples_test, n_features) y = np.dot(X, coef) # Split train and test data X_train, X_test = X[:n_samples_train], X[n_samples_train:]
data = np.empty((lenPatterns, numPatterns)).astype('i4') data[:, :-1] = addresses[:, 1:] data[:, -1] = addresses[:, -1] mem.writeM(addresses, data) # plot what we just wrote for i in xrange(numPatterns): plt.figure(50+curSeq) plt.subplot(1,numPatterns,i+1) plt.imshow((1-addresses[:,i]).reshape((figSize,figSize), order='F'),cmap='gray',interpolation='nearest') plt.xticks([],[]) plt.yticks([],[]) figName="storedSeq"+str(curSeq) save(figName, ext="png", close=True, verbose=True) # now read sequentially with no noise used as input #addresses = inputs.copy() #for curPat in xrange(numPatterns): # a = addresses[:,curPat].copy() # d = mem.read(addresses[:,curPat]).reshape((figSize, figSize), order='F') # plt.figure(numPatterns+curPat) # plot_io(addresses[:,curPat].reshape((figSize, figSize), order='F'), d) # # read until converge, using retrieved data as the next address #
except: print "config_local.py is not present loading configdef.py" from config import * #default configuration if cfg.savedir == "": #cfg.savedir=InriaPosData(basepath=cfg.dbpath).getStorageDir() #where to save cfg.savedir = VOC07Data(basepath=cfg.dbpath).getStorageDir() import sys if len(sys.argv) > 1: cfg.cls = sys.argv[1] cfg.testname = cfg.testpath + cfg.cls + ( "%d" % cfg.numcl) + "_" + cfg.testspec cfg.train = "keep2" util.save(cfg.testname + ".cfg", cfg) cfg.auxdir = cfg.savedir testname = cfg.testname if cfg.multipr == 1: numcore = None else: numcore = cfg.multipr mypool = Pool(numcore) stcfg = stats([{"name": "cfg*"}]) stcfg.report(testname + ".rpt.txt", "w", "Initial Configuration") sts = stats([{
def save_tasks(self, task_file): """Save the task file.""" util.save(self.tasklist.tasks, task_file)
def plot(results_path, fig_path): order = ['exp', 'th', 'bqp'] titles = { 'exp': "Human", 'oc': "Oracle", 'th': "Threshold", 'hc': "HC", 'bq': "BQ (equal prior)", 'bqp': "BQ (unequal prior)" } colors = { 'same': [ 0.16339869, 0.4449827 , 0.69750096], 'flipped': [ 0.72848904, 0.1550173 , 0.19738562] } means = pd.read_csv( results_path.joinpath("theta_time_stimulus.csv"))\ .set_index(['stimulus', 'flipped', 'model'])\ .groupby(level='stimulus').get_group(2) fig, axes = plt.subplots(1, len(order), sharex=True) for i, key in enumerate(order): ax = axes[i] df = means.groupby(level='model').get_group(key) for flipped, stats in df.groupby(level='flipped'): if key == 'exp': median = stats['median'] / 1000. lower = (stats['median'] - stats['lower']) / 1000. upper = (stats['upper'] - stats['median']) / 1000. else: median = stats['median'] lower = stats['median'] - stats['lower'] upper = stats['upper'] - stats['median'] ax.errorbar( stats['modtheta'], median, yerr=[lower, upper], lw=3, color=colors[flipped], ecolor=colors[flipped]) ax.set_xticks(np.arange(0, 200, 60)) ax.set_xlim(-10, 190) ax.set_xlabel("Rotation") ax.set_title(titles[key]) if key == "exp": ax.set_yticks([0.5, 1, 1.5, 2, 2.5, 3]) ax.set_ylabel("Rotation") elif key == "th": ax.set_yticks([0, 10, 20, 30, 40, 50, 60]) ax.set_ylabel("# Steps") elif key == "bqp": ax.set_yticks([10, 15, 20, 25]) ax.set_ylabel("# Steps") p0 = plt.Rectangle( (0, 0), 1, 1, fc=colors['same'], ec=colors['same']) p1 = plt.Rectangle( (0, 0), 1, 1, fc=colors['flipped'], ec=colors['flipped']) leg = axes.flat[0].legend( [p0, p1], ["same", "flipped"], numpoints=1, loc='lower right', bbox_to_anchor=[1.1, 0]) fig.set_size_inches(6, 2) sns.despine() plt.tight_layout() plt.subplots_adjust(left=0.1) pths = [fig_path.joinpath("response_time_stimulus.%s" % ext) for ext in ('png', 'pdf')] for pth in pths: util.save(pth, close=False) return pths
def plot(results_path, fig_path): order = ['exp', 'oc', 'th', 'hc', 'bq', 'bqp'] titles = { 'exp': "Human", 'oc': "Oracle", 'th': "Threshold", 'hc': "HC", 'bq': "BQ (equal prior)", 'bqp': "BQ (unequal prior)" } colors = { 'same': [0.16339869, 0.4449827, 0.69750096], 'flipped': [0.72848904, 0.1550173, 0.19738562] } acc_results = pd.read_csv( results_path.joinpath("theta_accuracy.csv")) fig, axes = plt.subplots(2, 3, sharex=False) for i, key in enumerate(order): ax = axes.flat[i] df = acc_results.groupby('model').get_group(key) for flipped, stats in df.groupby('flipped'): lower = stats['median'] - stats['lower'] upper = stats['upper'] - stats['median'] ax.errorbar( stats['modtheta'], stats['median'], yerr=[lower, upper], lw=3, color=colors[flipped], ecolor=colors[flipped]) ax.set_xlim(-10, 190) ax.set_ylim(-5, 105) ax.set_xticks(np.arange(0, 200, 60)) ax.set_xlabel("Rotation") ax.set_yticks([0, 50, 100]) ax.set_yticklabels([0.0, 0.5, 1.0]) ax.set_title(titles[key]) ax.set_ylabel("Accuracy") p0 = plt.Rectangle( (0, 0), 1, 1, fc=colors['same'], ec=colors['same']) p1 = plt.Rectangle( (0, 0), 1, 1, fc=colors['flipped'], ec=colors['flipped']) leg = axes[0, 0].legend( [p0, p1], ["same", "flipped"], numpoints=1, loc='lower right', bbox_to_anchor=[1.1, 0]) sns.despine() fig.set_size_inches(6, 4) plt.tight_layout() plt.subplots_adjust(left=0.1) pths = [fig_path.joinpath("accuracy.%s" % ext) for ext in ('png', 'pdf')] for pth in pths: util.save(pth, close=False) return pths
# negative phase cm.dot(w_vh, h, target=v) v.add_col_vec(w_v) v.apply_sigmoid() cm.dot(w_vh.T, v, target=h) h.add_col_vec(w_h) h.apply_sigmoid() wu_vh.subtract_dot(v, h.T) wu_v.add_sums(v, axis=1, mult=-1.0) wu_h.add_sums(h, axis=1, mult=-1.0) # update weights w_vh.add_mult(wu_vh, epsilon / batch_size) w_v.add_mult(wu_v, epsilon / batch_size) w_h.add_mult(wu_h, epsilon / batch_size) # calculate reconstruction error v.subtract(v_true) err.append(v.euclid_norm() ** 2 / (num_vis * batch_size)) print "Mean squared error: " + str(np.mean(err)) print "Time: " + str(time.time() - start_time) w_vh.copy_to_host() util.save("weights.dat", "w_vh", {"w_vh": w_vh.numpy_array}) # cm.cublas_shutdown()
# negative phase cm.dot(w_vh, h, target = v) v.add_col_vec(w_v) v.apply_sigmoid() cm.dot(w_vh.T, v, target = h) h.add_col_vec(w_h) h.apply_sigmoid() wu_vh.subtract_dot(v, h.T) wu_v.add_sums(v, axis = 1, mult = -1.) wu_h.add_sums(h, axis = 1, mult = -1.) # update weights w_vh.add_mult(wu_vh, epsilon/batch_size) w_v.add_mult(wu_v, epsilon/batch_size) w_h.add_mult(wu_h, epsilon/batch_size) # calculate reconstruction error v.subtract(v_true) err.append(v.euclid_norm()**2/(num_vis*batch_size)) print "Mean squared error: " + str(np.mean(err)) print "Time: " + str(time.time() - start_time) w_vh.copy_to_host() util.save('weights.dat', 'w_vh', {'w_vh': w_vh.numpy_array}) cm.cublas_shutdown()
def save(self, filename): data = {} for childName, child in self.children.iteritems(): data[childName] = child.doSave() util.save(filename,data)
rr.append(models[idm]["rho"]) w1=numpy.concatenate((w1,waux[-1],-numpy.array([models[idm]["rho"]])/bias)) sizereg[idm]=models[idm]["cost"].size #w2=w #old w w=w1 #add ids clsize and cumsize for each model clsize=[] cumsize=numpy.zeros(cfg.numcl+1,dtype=numpy.int) for l in range(cfg.numcl): models[l]["id"]=l clsize.append(len(waux[l])+1) cumsize[l+1]=cumsize[l]+len(waux[l])+1 clsize=numpy.array(clsize) util.save("%s%d.model"%(testname,0),models) lg.info("Built first model") total=[] posratio=[] cache_full=False #from scipy.ndimage import zoom import detectCRF from multiprocessing import Pool import itertools #just to compute totPosEx when using check points arg=[] for idl,l in enumerate(trPosImages): bb=l["bbox"]
def plot(results_path, fig_path): order = ['exp', 'th', 'bqp'] titles = { 'exp': "Human", 'oc': "Oracle", 'th': "Threshold", 'hc': "HC", 'bq': "BQ (equal prior)", 'bqp': "BQ (unequal prior)" } colors = { 'same': 'r', 'flipped': 'b' } means = pd.read_csv( results_path.joinpath("theta_time_stimulus.csv"))\ .set_index(['stimulus', 'flipped', 'model'])\ .groupby(level='stimulus').get_group(2) fig, axes = plt.subplots(1, len(order), sharex=True) for i, key in enumerate(order): ax = axes[i] df = means.groupby(level='model').get_group(key) for flipped, stats in df.groupby(level='flipped'): if key == 'exp': median = stats['median'] / 1000. lower = (stats['median'] - stats['lower']) / 1000. upper = (stats['upper'] - stats['median']) / 1000. else: median = stats['median'] lower = stats['median'] - stats['lower'] upper = stats['upper'] - stats['median'] ax.errorbar( stats['modtheta'], median, yerr=[lower, upper], lw=3, color=colors[flipped], ecolor=colors[flipped]) ax.set_xticks(np.arange(0, 200, 30)) ax.set_xlim(-10, 190) ax.set_xlabel("Rotation") ax.set_title(titles[key], fontsize=14) if key == 'exp': ax.set_ylabel("Response time", fontsize=14) else: ax.set_ylabel("Number of actions", fontsize=14) # util.sync_ylims(axes[order.index('bq')], axes[order.index('bqp')]) for ax in axes.flat: util.clear_right(ax) util.clear_top(ax) util.outward_ticks(ax) ax.set_axis_bgcolor('0.95') p0 = plt.Rectangle( (0, 0), 1, 1, fc=colors['same'], ec=colors['same']) p1 = plt.Rectangle( (0, 0), 1, 1, fc=colors['flipped'], ec=colors['flipped']) leg = axes[0].legend( [p0, p1], ["same", "flipped"], numpoints=1, fontsize=12, loc='lower right') frame = leg.get_frame() frame.set_edgecolor('#FFFFFF') util.sync_ylabel_coords(axes.flat, -0.175) fig.set_figheight(3) fig.set_figwidth(9) plt.draw() plt.tight_layout() plt.subplots_adjust(wspace=0.4) pths = [fig_path.joinpath("response_time_stimulus.%s" % ext) for ext in ('png', 'pdf')] for pth in pths: util.save(pth, close=False) return pths
def runtest(models,tsImages,cfg,parallel=True,numcore=4,detfun=detectCRF.test,save=False,show=False,pool=None): #parallel=True #cfg.show=not(parallel) #numcore=4 #mycfg= if parallel: if pool!=None: mypool=pool #use already created pool else: mypool = Pool(numcore) arg=[] for idl,l in enumerate(tsImages): #bb=l["bbox"] #for idb,b in enumerate(bb): arg.append({"idim":idl,"file":l["name"],"idbb":0,"bbox":[],"models":models,"cfg":cfg,"flip":False}) print "----------Test-----------" ltdet=[]; if not(parallel): #itr=itertools.imap(detectCRF.test,arg) #itr=itertools.imap(lambda x:detectCRF.test(x,numhyp=1),arg) #this can also be used itr=itertools.imap(detfun,arg) else: #itr=mypool.map(detectCRF.test,arg) itr=mypool.imap(detfun,arg) #for parallle lambda does not work for ii,res in enumerate(itr): if show: im=myimread(arg[ii]["file"]) if tsImages[ii]["bbox"]!=[]: detectCRF.visualize2(res[:3],cfg.N,im,bb=tsImages[ii]["bbox"][0]) else: detectCRF.visualize2(res[:3],cfg.N,im) print [x["scr"] for x in res[:5]] ltdet+=res if parallel: if pool==None: mypool.close() mypool.join() #sort detections ltosort=[-x["scr"] for x in ltdet] lord=numpy.argsort(ltosort) aux=[] for l in lord: aux.append(ltdet[l]) ltdet=aux #save on a file and evaluate with annotations detVOC=[] for l in ltdet: detVOC.append([l["idim"].split("/")[-1].split(".")[0],l["scr"],l["bbox"][1],l["bbox"][0],l["bbox"][3],l["bbox"][2]]) #plot AP tp,fp,scr,tot=VOCpr.VOCprRecord(tsImages,detVOC,show=False,ovr=0.5) pylab.figure(15,figsize=(4,4)) pylab.clf() rc,pr,ap=VOCpr.drawPrfast(tp,fp,tot) pylab.draw() pylab.show() #save in different formats if type(save)==str: testname=save util.savedetVOC(detVOC,testname+".txt") util.save(testname+".det",{"det":ltdet[:500]})#takes a lot of space use only first 500 util.savemat(testname+".mat",{"tp":tp,"fp":fp,"scr":scr,"tot":tot,"rc":rc,"pr":pr,"ap":ap}) pylab.savefig(testname+".png") return ap