def save(self, save_path=None, items=None): path_ = "./config/" if save_path is None else save_path ensure_path(path_) self.model.save(path_, "model") # save model self.trainer.save(path_, "trainer") self.optimizer.save(path_, "optimizer") # save optmizer self.data_loader.save(path_, "data_loader") # save data_loader
def __init__(self, run_label: str, path_to_results: str, pg_model: str = '') -> None: self.path = path_to_results self.run_label: str = run_label utils.ensure_path(f'{path_to_results}/results.csv') utils.ensure_path(f'{path_to_results}/{run_label}/results.csv') self.tensorboard_writer = SummaryWriter( f'{path_to_results}/tboard_logs', comment=f'pg_{pg_model}') self.columns = [ 'env', 'seed', 'run_label', 'run_nr_epochs', 'nr_episodes', 'run_model', 'timestamp', 'epoch', 'step_size', 'epoch_duration', 'nr_steps', 'perf', 'hardware' ] self.results = pd.DataFrame(columns=self.columns) if (not os.path.exists(f'{path_to_results}/results.csv') or os.path.getsize(f'{path_to_results}/results.csv') == 0): with open(f'{self.path}/results.csv', 'w') as f: writer = csv.writer(f) writer.writerow(self.columns)
def save(self, save_path, save_name): ensure_path(save_path) #self.update_before_save() with open(save_path + save_name, 'wb') as f: self.to(torch.device('cpu')) torch.save(self.dict, f) self.to(self.device)
def plot_weight(self, ax=None, save=True, save_path='./', save_name='RNN_Navi_weight_plot.png', cmap='jet'): if ax is None: plt.close('all') row_num, col_num = 2, 2 fig, axes = plt.subplots(nrows=row_num, ncols=col_num, figsize=(5 * col_num, 5 * row_num)) # figsize unit: inches fig.suptitle('Weight Visualization of 1-layer RNN') # plot recurrent weight ax = axes[0, 0] # raises error is row_num==col_num==1 self.plot_recurrent_weight(ax, cmap) # plot input_weight if self.init_method in ['linear']: ax = axes[0, 1] elif self.init_method in ['mlp']: pass else: pass plt.tight_layout() if save: ensure_path(save_path) plt.savefig(save_path + save_name)
def __init__(self, dict_, load=False, options=None): ''' if options is not None: self.receive_options(options) else: raise Exception('Trainer: options is none.') ''' self.dict = dict_ ''' self.epoch_now = get_from_dict(self.dict, 'epoch_now', default=self.epoch_start, write_default=True) self.epoch_start = get_from_dict(self.dict, 'epoch_start', default=1, write_default=True) self.epoch_end = get_from_dict(self.dict, 'epoch_end', default=self.epoch_um, write_default=True) ''' self.epoch_now = 0 #print(self.dict.keys()) self.epoch_num = self.dict['epoch_num'] self.epoch_end = self.epoch_num - 1 # save directory setting self.save_model_path = search_dict(self.dict, ['save_model_path', 'save_dir_model', 'save_path_model'], default='./SavedModels/', write_default=True, write_default_key='save_model_path') #print(self.save_model_path) ensure_path(self.save_model_path) self.save_model = get_from_dict(self.dict, 'save_model', default=True, write_default=True) self.save_after_train = get_from_dict(self.dict, 'save_after_train', default=True, write_default=True) self.save_before_train = get_from_dict(self.dict, 'save_before_train', default=True, write_default=True) if self.save_model: self.save_interval = get_from_dict(self.dict, 'save_model_interval', default=True, write_default=True) self.anal_path = search_dict(self.dict, ['anal_path'], default='./', write_default=True) #print(self.anal_path) ensure_path(self.anal_path)
def main( original_path: str, executable_path: str = EXECUTABLE_PATH, destination_path: str = None, overwrite: bool = False, ): """ Run batch conversion process. Args: original_path: path to the dataset to convert. executable_path: path to the converter executable. destination_path: destination path. overwrite: if True, any previous .edf with the same name will be overwrited. """ print('1 - List the files to convert...') files = [ os.path.abspath(file_) for file_ in list_files(original_path) if os.path.basename(file_).lower().endswith('.eeg') ] n_files = len(files) print('{0} file(s) will be converted.'.format(n_files)) print('2 - Convert files') for index, file_ in enumerate(sorted(files), start=1): # Destination file path if destination_path is None: file_destination_path = file_[:-4] + '.EDF' else: file_destination_path = os.path.join( destination_path, os.path.relpath(file_, original_path)) print( '({0}/{1}) Convert "{2}" to "{3}"'.format( index, n_files, file_, file_destination_path, ), ) ensure_path(path=os.path.dirname(file_destination_path)) if os.path.isfile(file_destination_path) and not overwrite: print('File has already been converted.') else: if os.path.isfile(file_destination_path): print('File has already been converted (will be overwrited).') convert_coh3_to_edf( executable_path=executable_path, eeg_path=file_, edf_path=file_destination_path, ) if n_files: print('3 - Kill the converter process(es).') os.system( 'taskkill /f /im {0}'.format( os.path.basename(executable_path), ), )
def anal(self, title=None, save_path=None, verbose=True): if save_path is None: if title is None: save_path = self.anal_path + 'epoch=%d/' % (self.epoch_index) else: save_path = self.anal_path + 'epoch=%s/' % (title) ensure_path(save_path) self.agent.anal(save_path=save_path, trainer=self)
def __init__(self, dict_, load=False): if options is not None: self.receive_options(options) self.dict = dict_ #set_instance_variable(self, self.dict) self.epoch_num = self.dict['epoch_num'] self.batch_num = self.dict['batch_num'] self.batch_size = self.dict['batch_size'] if not hasattr(self, 'anal_path'): self.anal_path = self.dict.setdefault('anal_path', './anal/') ''' self.epoch_index = get_from_dict(self.dict, 'epoch_index', default=self.epoch_start, write_default=True) self.epoch_start = get_from_dict(self.dict, 'epoch_start', default=1, write_default=True) self.epoch_end = get_from_dict(self.dict, 'epoch_end', default=self.epoch_um, write_default=True) ''' self.epoch_index = 0 self.epoch_end = self.epoch_num - 1 # save directory setting self.save_path = search_dict( self.dict, ['save_path', 'save_model_path', 'save_dir_model'], default='./saved_models/', write_default=True, write_default_key='save_path') ensure_path(self.save_path) self.save = search_dict(self.dict, ['save', 'save_model'], default=True, write_default=True) self.save_after_train = get_from_dict(self.dict, 'save_after_train', default=True, write_default=True) self.save_before_train = get_from_dict(self.dict, 'save_before_train', default=True, write_default=True) self.anal_before_train = get_from_dict(self.dict, 'anal_before_train', default=True, write_default=True) if self.save: self.save_interval = search_dict( self.dict, ['save_interval', 'save_model_interval'], default=int(self.epoch_num / 10), write_default=True) ''' if options is not None: self.options = options self.set_options() ''' self.test_performs = self.dict['test_performs'] = {} self.train_performs = self.dict['train_performs'] = {} self.anal_model = self.dict.setdefault('anal_model', True)
def cross_val(input_path, output_dir, num_folds, dataset_name="data"): kf = KFold(n_splits=num_folds) data = np.array(utils.load_text_as_list(input_path)) fold = 1 for train_index, test_index in kf.split(data): fold_dir = os.path.join(output_dir, "fold_{}".format(fold)) print("Creating fold {} at {}".format(fold, output_dir)) data_train, data_test = data[train_index], data[test_index] utils.save_list_as_text(data_train, utils.ensure_path(os.path.join(fold_dir, "{}.train".format(dataset_name)))) utils.save_list_as_text(data_test, utils.ensure_path(os.path.join(fold_dir, "{}.val".format(dataset_name)))) fold += 1
def ensure_path(params): """ Ensure a certain path """ params = utils.format_params(params) if not 'path' in params: abort('No path set') utils.ensure_path(path=params['path']) print(green("Ensure path `%s`." % (params['path'])))
def fetch_dict(url, dest): assert url.endswith('.tar.bz2'), url filename = os.path.basename(url) utils.fetch(url, os.path.join(DICT_DIR, filename)) utils.run(["tar", "xjvf", filename], cwd=DICT_DIR) name = filename[:-len('.tar.bz2')] path = os.path.join(DICT_DIR, name) utils.run(["./configure", "--vars", "DESTDIR=tmp"], cwd=path) utils.run(["make"], cwd=path) utils.run(["make", "install"], cwd=path) result_dir = os.path.join(path, 'tmp/usr/lib/aspell') utils.ensure_path(dest) for dict_file in os.listdir(result_dir): shutil.copy2(os.path.join(result_dir, dict_file), os.path.join(dest, dict_file))
def fetch_dict(url, dest): assert url.endswith(".tar.bz2"), url filename = os.path.basename(url) utils.fetch(url, os.path.join(DICT_DIR, filename)) utils.run(["tar", "xjvf", filename], cwd=DICT_DIR) name = filename[: -len(".tar.bz2")] path = os.path.join(DICT_DIR, name) utils.run(["./configure", "--vars", "DESTDIR=tmp"], cwd=path) utils.run(["make"], cwd=path) utils.run(["make", "install"], cwd=path) result_dir = os.path.join(path, "tmp/usr/lib/aspell") utils.ensure_path(dest) for dict_file in os.listdir(result_dir): shutil.copy2(os.path.join(result_dir, dict_file), os.path.join(dest, dict_file))
def plot_place_cells_coords(self, ax=None, arena=None, save=True, save_path='./', save_name='place_cells_coords.png'): arena = self.arenas.get_current_arena() if arena is None else arena if ax is None: plt.close('all') fig, ax = plt.subplots() arena.plot_arena(ax, save=False) ax.scatter(self.coords_np[:,0], self.coords_np[:,1], marker='d', color=(0.0, 1.0, 0.0), edgecolors=(0.0,0.0,0.0), label='Start Positions') # marker='d' for diamond ax.set_title('Place Cells Positions') if save: ensure_path(save_path) #cv.imwrite(save_path + save_name, imgs) # so that origin is in left-bottom corner. ensure_path(save_path) plt.savefig(save_path + save_name) plt.close()
def directory(self): """Gets the workflow cache directory. .. note:: The directory is calculated based off of the workflow environment variable. If such variable is not defined, then it defaults to: Alfred 3: ~/Library/Caches/com.runningwithcrayons.Alfred-3/Workflow Data/ Alfred 2: ~/Library/Caches/com.runningwithcrayons.Alfred-2/Workflow Data/' :return: the workflow cache directory. :rtype: ``str``. """ if not self._directory: if self.workflow.environment('workflow_cache'): self._directory = self.workflow.environment('workflow_cache') elif self.workflow.environment('version_build') >= 652: self._directory = os.path.join( os.path.expanduser( '~/Library/Caches/com.runningwithcrayons.Alfred-3/Workflow Data/' ), self.workflow.bundle) elif self.workflow.environment('version_build') < 652: self._directory = os.path.join( os.path.expanduser( '~/Library/Caches/com.runningwithcrayons.Alfred-2/Workflow Data/' ), self.workflow.bundle) return ensure_path(self._directory)
def directory(self): """Gets the workflow cache directory. .. note:: The directory is calculated based off of the workflow environment variable. If such variable is not defined, then it defaults to: Alfred 3: ~/Library/Caches/com.runningwithcrayons.Alfred-3/Workflow Data/ Alfred 2: ~/Library/Caches/com.runningwithcrayons.Alfred-2/Workflow Data/' :return: the workflow cache directory. :rtype: ``str``. """ if not self._directory: if self.workflow.environment('workflow_cache'): self._directory = self.workflow.environment('workflow_cache') elif self.workflow.environment('version_build') >= 652: self._directory = os.path.join( os.path.expanduser('~/Library/Caches/com.runningwithcrayons.Alfred-3/Workflow Data/'), self.workflow.bundle ) elif self.workflow.environment('version_build') < 652: self._directory = os.path.join( os.path.expanduser('~/Library/Caches/com.runningwithcrayons.Alfred-2/Workflow Data/'), self.workflow.bundle ) return ensure_path(self._directory)
def main(args): utils.delete_path(args.log_dir) utils.delete_path(args.save_dir) utils.ensure_path(args.save_dir) utils.ensure_path(args.log_dir) utils.write_dict(vars(args), os.path.join(args.save_dir, 'arguments.csv')) torch.manual_seed(args.seed) cudnn.benchmark = True torch.cuda.manual_seed_all(args.seed) os.environ['CUDA_VISIBLE_DEVICES'] = args.gpus if args.mode == 'train': train(args) elif args.mode == 'test': test(args)
def plot_arena_plt(self, ax=None, save=True, save_path='./', save_name='arena_random_xy.png', line_width=2, color=(0,0,0)): if ax is None: figure, ax = plt.subplots() ax.set_xlim(self.x0, self.x1) ax.set_ylim(self.y0, self.y1) ax.set_xticks(np.linspace(self.x0, self.x1, 5)) ax.set_yticks(np.linspace(self.y0, self.y1, 5)) ax.set_aspect(1) circle = plt.Circle(self.center_coord, self.radius, color=(0.0, 0.0, 0.0), fill=False) ax.add_patch(circle) if save: ensure_path(save_path) plt.savefig(save_path + save_name) return ax
def plot_random_xy_plt(self, ax=None, save=True, save_path='./', save_name='arena_random_xy.png', num=100, color=(0.0,1.0,0.0), plot_arena=True, **kw): if ax is None: figure, ax = plt.subplots() if plot_arena: self.plot_arena_plt(ax, save=False) else: ax.set_xlim(self.x0, self.x1) ax.set_ylim(self.y0, self.y1) ax.set_aspect(1) # so that x and y has same unit length in image. points = self.get_random_xy(num=100) ax.scatter(points[:,0], points[:,1], marker='o', color=color, edgecolors=(0.0,0.0,0.0), label='Start Positions') plt.legend() if save: ensure_path(save_path) plt.savefig(save_path + save_name) return ax
def plot_random_xy_cv(self, img=None, save=True, save_path='./', save_name='arena_random_xy.png', num=100, color=(0,255,0), plot_arena=True, **kw): if img is None: res = search_dict(kw, ['res, resolution'], default=100) res_x, res_y = get_res_xy(res, self.width, self.height) if plot_arena: img = self.plot_arena(save=False, res=res) else: img = np.zeros([res_x, res_y, 3], dtype=np.uint8) img[:,:,:] = (255, 255, 255) else: res_x, res_y = img.shape[0], img.shape[1] points = self.get_random_xy(num=100) points_int = get_int_coords_np(points, self.xy_range, res_x, res_y) for point in points_int: cv.circle(img, (point[0], point[1]), radius=0, color=color, thickness=4) if save: ensure_path(save_path) cv.imwrite(save_path + save_name, img[:, ::-1, :]) return img
def copy_project_files(args): path = args.path if path is None: raise Exception( 'copy_project_files: args.path must not be none. please give path to copy files to' ) ensure_path(args.path) if args.param_path is None: param_path = './params/' print(path) if not path.endswith('/'): path += '/' file_list = [ #'cmd.py', 'Models', 'Optimizers', 'Trainers.py', 'DataLoaders.py', #'Analyzer.py', 'utils.py', 'utils_anal.py', 'utils_model.py', 'config_sys.py', ] copy_files(file_list, path_from='./src/', path_to=path + 'src/') file_list = ['main.py', 'params/__init__.py'] copy_files(file_list, path_from='./', path_to=path) param_files = get_param_files(args) #param_files = list(map(lambda file:param_path + file, param_files)) model_file = param_files['model_file'] optimizer_file = param_files['optimizer_file'] trainer_file = param_files['trainer_file'] data_loader_file = param_files['data_loader_file'] component_files = [ model_file, optimizer_file, trainer_file, data_loader_file ] if param_files.get('config_file') is not None: component_files.append(param_files['config_file']) #print(component_files) copy_files(component_files, path_from=param_path, path_to=path + param_path)
def run(self, inpath, outpath): '''Run the ffmpeg binary with the specified input/outpath paths. Args: inpath: the input path. If inpath is "-", input streaming mode is activated and data can be passed via the stream() method outpath: the output path. Existing files are overwritten, and the directory is created if needed. If outpath is "-", output streaming mode is activated and data can be read via the read() method Raises: ExecutableNotFoundError: if the ffmpeg binary cannot be found ExecutableRuntimeError: if the ffmpeg binary raises an error during execution ''' self.is_input_streaming = (inpath == "-") self.is_output_streaming = (outpath == "-") self._args = ( [self._executable] + self._global_opts + self._in_opts + ["-i", inpath] + self._out_opts + [outpath] ) if not self.is_output_streaming: utils.ensure_path(outpath) try: self._p = Popen(self._args, stdin=PIPE, stdout=PIPE, stderr=PIPE) except OSError as e: if e.errno == errno.ENOENT: raise utils.ExecutableNotFoundError(self._executable) else: raise # Run non-streaming jobs immediately if not (self.is_input_streaming or self.is_output_streaming): err = self._p.communicate()[1] if self._p.returncode != 0: raise utils.ExecutableRuntimeError(self.cmd, err)
def __init__(self, outpath, fps, size): '''Constructs a VideoWriter with OpenCV backend. Args: outpath: the output video path, e.g., "/path/to/video.mp4". Existing files are overwritten, and the directory is created if needed fps: the frame rate size: the (width, height) of each frame Raises: VideoWriterError: if the writer failed to open ''' self.outpath = outpath self.fps = fps self.size = size self._writer = cv2.VideoWriter() utils.ensure_path(self.outpath) self._writer.open(self.outpath, -1, self.fps, self.size, True) if not self._writer.isOpened(): raise VideoWriterError("Unable to open '%s'" % self.outpath)
def plot_perform(self, save_path='./', save_name='perform.png', col_num=3): # plot test_performs epochs = self.test_performs.keys() epochs = np.array(epochs, ) epochs = np.sort(epochs) items = self.test_performs[epochs[0]].keys() item_num = len(items) row_num = item_num // col_num if item_num % col_num > 0: col_num += 1 fig, axes = plt.subplots(nrow=row_num, ncol=col_num) for item, index in enumerate(items): row_index = index // row_num col_index = index % row_num ax = axes[row_index, col_index] ax.set_title plt.suptitle('%s Test Performance'%self.model.dict['name']) ensure_path(save_path) plt.savefig(save_path + save_name)
def clone(params): """ The clone command can be used to clone a new repo. If it's allready an existing path it will prompt for overwrite """ print yellow("Warning git.clone is deprecated from version 1.0") if 'repo_path' in params: abort(red("repo_path is deprecated, use git_repo_path")) if 'repo_url' in params: abort(red("repo_url is deprecated, use git_repo_url")) # Try to get global params params = utils.get_global_params(params, 'git_repo_path', 'git_repo_url') if 'git_repo_path' not in params: abort(red("git_repo_path can't be empty?")) if 'git_repo_url' not in params: abort(red("git_repo_url can't be empty?")) params = utils.format_params(params) if exists(params['git_repo_path']): if confirm("Repo path `%s` found, do you want to reinstall?" % params['git_repo_path']): print(yellow("Repo path `%s` will be deleted" % params['git_repo_path'])) run('rm -Rf %s' % params['git_repo_path']) else: abort("Aborted...") utils.ensure_path(params['git_repo_path']) run('git clone --recursive %s %s' % (params['git_repo_url'], params['git_repo_path'])) print(green("Repo `%s` successfully cloned" % params['git_repo_url']))
def test_ensure_path(self): """ Test the function ensure_path. """ with tempfile.TemporaryDirectory() as tmpdirname: subfolder_path = os.path.join(tmpdirname, 'folder', 'subfolder') folder_path = os.path.join(tmpdirname, 'folder') # No folder and subfolder should exist self.assertEqual(os.path.exists(subfolder_path), False) self.assertEqual(os.path.exists(folder_path), False) # Create folder and subfolder ensure_path(subfolder_path) # Folder and subfolder should exist self.assertEqual(os.path.exists(subfolder_path), True) self.assertEqual(os.path.exists(folder_path), True) # Should not change anything ensure_path(folder_path) # Folder and subfolder should exist self.assertEqual(os.path.exists(subfolder_path), True) self.assertEqual(os.path.exists(folder_path), True)
def backup_db(params): """" This command backups the database based on a backup folder The output dump will be a iso date formatted filename """ print yellow("Warning mysql.backup_db is deprecated from version 1.0") params = utils.format_params(params) command = """ mysqldump -h %(host)s -u %(user)s --password='******' %(database)s > %(backup_file)s """ backup_path = os.path.dirname(params['backup_file']) utils.ensure_path(backup_path) # Make params command_params = {'user': params['user'], 'password': params['password'], 'database': params['database'], 'host': params['host'], 'backup_file':params['backup_file']} with hide('running'): run(command % command_params) with cd(backup_path): filename = os.path.basename(params['backup_file']) clean_filename = os.path.splitext(filename)[0] tarfilename = "%s.tar.gz" % clean_filename run("tar czvf %s %s" % (tarfilename, filename)) run("rm %s" % filename) full_tar_file_path = "%s/%s" % (backup_path ,tarfilename) print(green("Mysql backup `%s` successfully stored." % full_tar_file_path)) if 'download_tar_to_local_file' in params: get(remote_path=full_tar_file_path, local_path=params['download_tar_to_local_file'])
def plot_arena_plt(self, ax=None, save=True, save_path='./', save_name='arena_random_xy.png', line_width=2, color=(0,0,0)): if ax is None: plt.close('all') fig, ax = plt.subplots() ax.set_xlim(self.x0, self.x1) ax.set_ylim(self.y0, self.y1) ax.set_xticks(np.linspace(self.x0, self.x1, 5)) ax.set_yticks(np.linspace(self.y0, self.y1, 5)) ax.set_aspect(1) vertices = self.vertices vertex_num = self.vertices.shape[0] for i in range(vertex_num): vertex_0 = vertices[i] vertex_1 = vertices[(i+1)%vertex_num] ax.add_line(Line2D([vertex_0[0], vertex_1[0]], [vertex_0[1], vertex_1[1]], linewidth=line_width, color=color)) if save: ensure_path(save_path) plt.savefig(save_path + save_name) return ax
def plot_arena_cv(self, img=None, line_color=(0,0,0), line_width=1, line_type=4, save=False, save_path='./', save_name='arena_plot.png', **kw):# line_color: (b, g, r) if img is None: res = search_dict(kw, ['res, resolution'], default=100) res_x, res_y = get_res_xy(res, self.width, self.height) img = np.zeros([res_x, res_y, 3], dtype=np.uint8) img[:,:,:] = (255, 255, 255) vertices, width, height = self.dict['vertices'], self.width, self.height vertex_num = vertices.shape[0] #print('xy_range:%s'%str(xy_range)) for i in range(vertex_num): vertex_0 = get_int_coords(vertices[i, 0], vertices[i, 1], self.xy_range, res_x, res_y) vertex_1 = get_int_coords(vertices[(i+1)%vertex_num, 0], vertices[(i+1)%vertex_num, 1], self.xy_range, res_x, res_y) #print('plot line: (%d, %d) to (%d, %d)'%(vertex_0[0], vertex_0[1], vertex_1[0], vertex_1[1])) cv.line(img, vertex_0, vertex_1, line_color, line_width, line_type) # line_width seems to be in unit of pixel. if save: ensure_path(save_path) #if save_name is None: # save_name = 'arena_plot.png' cv.imwrite(save_path + save_name, img[:, ::-1, :]) # so that origin is in left-bottom corner. return img
def main(): # Basic configuration data_dir = utils.ensure_path('~/data_analysis/fcb/') # group_id = 153748404666241 group_id = 597682743580084 # Load basic arguments log("Parsing basic arguments") missing_arg = utils.check_required_arg(utils.login_opt, utils.password_opt, utils.ch_driver_opt) if missing_arg is not None: utils.exit_program('Missing required argument ' + missing_arg) login = utils.get_arg(utils.login_opt) password = utils.get_arg(utils.password_opt) driver_path = utils.get_arg(utils.ch_driver_opt) log('Starting browser') browser = webdriver.Chrome(executable_path=driver_path) scrapper = FcbBrowserScrapper(browser, login, password) scrapper.log_in() # Scrap users = scrapper.scrap_group_members(group_id) # Temporary code for loading user from file rather from the web # to speed up the development # users_file = utils.ensure_path('~/data_analysis/mlyny_group.txt') # utils.save_data(users, users_file) # users = [] # with open(users_file, encoding='utf-8') as f: # for line in f: # if line == '': # break # s = line.split(',') # users.append((s[0], s[1])) # Process scrapper.process_users(users, data_dir) log('Closing the browser') browser.close() log('The end')
str(args.shot), str(args.query), str(args.way), str(args.validation_way), str(args.step_size), str(args.gamma), str(args.lr), str(args.temperature), str(args.hyperbolic), str(args.dim), str(args.c)[:5], str(args.train_c), str(args.train_x) ]) args.save_path = save_path1 + '_' + save_path2 + '_' + args.exp_addendum ensure_path(args.save_path) else: ensure_path(args.save_path) if args.dataset == 'MiniImageNet': # Handle MiniImageNet from dataloader.mini_imagenet import MiniImageNet as Dataset elif args.dataset == 'CUB': from dataloader.cub import CUB as Dataset else: raise ValueError('Non-supported Dataset.') # train n_batch is 100 by default, val n_batch is 500 by default trainset = Dataset('train', args) train_sampler = CategoriesSampler(trainset.label, 100, args.way, args.shot + args.query)
WINE_RN_WIN_DIR = os.path.join(WINE_RN_DIR, 'win') PYINSTALLER = os.path.join(DRIVE_C, 'PyInstaller-2.1', 'pyinstaller.py') SPEC = os.path.join(BASE_DIR, 'win', 'rednotebook.spec') WINE_SPEC = os.path.join(WINE_RN_WIN_DIR, 'rednotebook.spec') WINE_BUILD = os.path.join(DRIVE_C, 'build') WINE_DIST = os.path.join(DRIVE_C, 'dist') LOCALE_DIR = os.path.join(WINE_DIST, 'share', 'locale') WINE_RN_EXE = os.path.join(WINE_DIST, 'rednotebook.exe') WINE_PYTHON = os.path.join(DRIVE_C, 'Python27', 'python.exe') if os.path.exists(WINE_DIR): answer = raw_input('The build dir exists. Overwrite it? (Y/n): ').strip() if answer and answer.lower() != 'y': sys.exit('Aborting') shutil.rmtree(WINE_DIR) os.environ['WINEPREFIX'] = WINE_DIR os.mkdir(WINE_DIR) run(['tar', '-xzf', WINE_TARBALL, '--directory', WINE_DIR]) archive = '/tmp/rednotebook-archive.tar' run(['git', 'archive', 'HEAD', '-o', archive], cwd=BASE_DIR) utils.ensure_path(WINE_RN_DIR) run(['tar', '-xf', archive], cwd=WINE_RN_DIR) shutil.copy2(SPEC, WINE_SPEC) run(['wine', WINE_PYTHON, PYINSTALLER, '--workpath', WINE_BUILD, '--distpath', DRIVE_C, WINE_SPEC]) # will be built at ...DRIVE_C/dist run(['./build-translations.py', LOCALE_DIR], cwd=DIR) #run(['wine', WINE_RN_EXE])
BASE_DIR = os.path.dirname(DIR) DIST_DIR = os.path.abspath(args.dist_dir) DRIVE_C = os.path.join(DIST_DIR, 'drive_c') BUILD_DIR = os.path.abspath(args.build_dir) assert os.path.exists(BUILD_DIR), BUILD_DIR WINE_RN_DIR = os.path.join(DRIVE_C, 'rednotebook') SPEC = os.path.join(BASE_DIR, 'win', 'rednotebook.spec') WINE_BUILD = os.path.join(DRIVE_C, 'build') WINE_DIST = os.path.join(DRIVE_C, 'dist') LOCALE_DIR = os.path.join(WINE_DIST, 'share', 'locale') WINE_RN_EXE = os.path.join(WINE_DIST, 'rednotebook.exe') WINE_PYTHON = os.path.join(DRIVE_C, 'Python34', 'python.exe') utils.confirm_overwrite(DIST_DIR) os.environ['WINEPREFIX'] = DIST_DIR utils.ensure_path(os.path.dirname(DIST_DIR)) print('Start copying {} to {}'.format(BUILD_DIR, DIST_DIR)) utils.fast_copytree(BUILD_DIR, DIST_DIR) print('Finished copying') archive = '/tmp/rednotebook-archive.tar' stash_name = utils.get_output(['git', 'stash', 'create'], cwd=BASE_DIR) stash_name = stash_name or 'HEAD' run(['git', 'archive', stash_name, '-o', archive], cwd=BASE_DIR) utils.ensure_path(WINE_RN_DIR) run(['tar', '-xf', archive], cwd=WINE_RN_DIR) os.mkdir(os.path.join(DRIVE_C, 'Python34/share')) shutil.copytree(os.path.join(DRIVE_C, 'Python34/Lib/site-packages/gnome/share/gir-1.0/'), os.path.join(DRIVE_C, 'Python34/share/gir-1.0/')) run(['wine', WINE_PYTHON, '-m', 'PyInstaller', '--workpath', WINE_BUILD,
def main(config): svname = args.name if svname is None: svname = 'classifier_{}'.format(config['train_dataset']) svname += '_' + config['model_args']['encoder'] clsfr = config['model_args']['classifier'] if clsfr != 'linear-classifier': svname += '-' + clsfr if args.tag is not None: svname += '_' + args.tag save_path = os.path.join('./save', svname) utils.ensure_path(save_path) utils.set_log_path(save_path) writer = SummaryWriter(os.path.join(save_path, 'tensorboard')) yaml.dump(config, open(os.path.join(save_path, 'config.yaml'), 'w')) #### Dataset #### # train train_dataset = datasets.make(config['train_dataset'], **config['train_dataset_args']) augmentations = [ transforms.Compose([ transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), transforms.Compose([ transforms.RandomResizedCrop(size=(80, 80), scale=(0.08, 1.0), ratio=(0.75, 1.3333)), transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), transforms.Compose([ transforms.RandomRotation(35), transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), transforms.Compose([ transforms.ColorJitter(0.4, 0.4, 0.4, 0.1), transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), transforms.Compose([ transforms.RandomResizedCrop(size=(80, 80), scale=(0.08, 1.0), ratio=(0.75, 1.3333)), transforms.RandomRotation(35), transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), transforms.Compose([ transforms.RandomRotation(35), transforms.ColorJitter(0.4, 0.4, 0.4, 0.1), transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), transforms.Compose([ transforms.RandomResizedCrop(size=(80, 80), scale=(0.08, 1.0), ratio=(0.75, 1.3333)), transforms.ColorJitter(0.4, 0.4, 0.4, 0.1), transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), transforms.Compose([ transforms.RandomRotation(35), transforms.RandomResizedCrop(size=(80, 80), scale=(0.08, 1.0), ratio=(0.75, 1.3333)), transforms.ColorJitter(0.4, 0.4, 0.4, 0.1), transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) ] train_dataset.transform = augmentations[int(config['_a'])] print(train_dataset.transform) print("_a", config['_a']) input("Continue with these augmentations?") train_loader = DataLoader(train_dataset, config['batch_size'], shuffle=True, num_workers=0, pin_memory=True) utils.log('train dataset: {} (x{}), {}'.format(train_dataset[0][0].shape, len(train_dataset), train_dataset.n_classes)) if config.get('visualize_datasets'): utils.visualize_dataset(train_dataset, 'train_dataset', writer) # val if config.get('val_dataset'): eval_val = True val_dataset = datasets.make(config['val_dataset'], **config['val_dataset_args']) val_loader = DataLoader(val_dataset, config['batch_size'], num_workers=0, pin_memory=True) utils.log('val dataset: {} (x{}), {}'.format(val_dataset[0][0].shape, len(val_dataset), val_dataset.n_classes)) if config.get('visualize_datasets'): utils.visualize_dataset(val_dataset, 'val_dataset', writer) else: eval_val = False # few-shot eval if config.get('fs_dataset'): ef_epoch = config.get('eval_fs_epoch') if ef_epoch is None: ef_epoch = 5 eval_fs = True fs_dataset = datasets.make(config['fs_dataset'], **config['fs_dataset_args']) utils.log('fs dataset: {} (x{}), {}'.format(fs_dataset[0][0].shape, len(fs_dataset), fs_dataset.n_classes)) if config.get('visualize_datasets'): utils.visualize_dataset(fs_dataset, 'fs_dataset', writer) n_way = 5 n_query = 15 n_shots = [1, 5] fs_loaders = [] for n_shot in n_shots: fs_sampler = CategoriesSampler(fs_dataset.label, 200, n_way, n_shot + n_query, ep_per_batch=4) fs_loader = DataLoader(fs_dataset, batch_sampler=fs_sampler, num_workers=0, pin_memory=True) fs_loaders.append(fs_loader) else: eval_fs = False ######## #### Model and Optimizer #### if config.get('load'): model_sv = torch.load(config['load']) model = models.load(model_sv) else: model = models.make(config['model'], **config['model_args']) if eval_fs: fs_model = models.make('meta-baseline', encoder=None) fs_model.encoder = model.encoder if config.get('_parallel'): model = nn.DataParallel(model) if eval_fs: fs_model = nn.DataParallel(fs_model) utils.log('num params: {}'.format(utils.compute_n_params(model))) optimizer, lr_scheduler = utils.make_optimizer(model.parameters(), config['optimizer'], **config['optimizer_args']) ######## max_epoch = config['max_epoch'] save_epoch = config.get('save_epoch') max_va = 0. timer_used = utils.Timer() timer_epoch = utils.Timer() for epoch in range(1, max_epoch + 1 + 1): if epoch == max_epoch + 1: if not config.get('epoch_ex'): break train_dataset.transform = train_dataset.default_transform print(train_dataset.transform) train_loader = DataLoader(train_dataset, config['batch_size'], shuffle=True, num_workers=0, pin_memory=True) timer_epoch.s() aves_keys = ['tl', 'ta', 'vl', 'va'] if eval_fs: for n_shot in n_shots: aves_keys += ['fsa-' + str(n_shot)] aves = {k: utils.Averager() for k in aves_keys} # train model.train() writer.add_scalar('lr', optimizer.param_groups[0]['lr'], epoch) for data, label in tqdm(train_loader, desc='train', leave=False): # for data, label in train_loader: data, label = data.cuda(), label.cuda() logits = model(data) loss = F.cross_entropy(logits, label) acc = utils.compute_acc(logits, label) optimizer.zero_grad() loss.backward() optimizer.step() aves['tl'].add(loss.item()) aves['ta'].add(acc) logits = None loss = None # eval if eval_val: model.eval() for data, label in tqdm(val_loader, desc='val', leave=False): data, label = data.cuda(), label.cuda() with torch.no_grad(): logits = model(data) loss = F.cross_entropy(logits, label) acc = utils.compute_acc(logits, label) aves['vl'].add(loss.item()) aves['va'].add(acc) if eval_fs and (epoch % ef_epoch == 0 or epoch == max_epoch + 1): fs_model.eval() for i, n_shot in enumerate(n_shots): np.random.seed(0) for data, _ in tqdm(fs_loaders[i], desc='fs-' + str(n_shot), leave=False): x_shot, x_query = fs.split_shot_query(data.cuda(), n_way, n_shot, n_query, ep_per_batch=4) label = fs.make_nk_label(n_way, n_query, ep_per_batch=4).cuda() with torch.no_grad(): logits = fs_model(x_shot, x_query).view(-1, n_way) acc = utils.compute_acc(logits, label) aves['fsa-' + str(n_shot)].add(acc) # post if lr_scheduler is not None: lr_scheduler.step() for k, v in aves.items(): aves[k] = v.item() t_epoch = utils.time_str(timer_epoch.t()) t_used = utils.time_str(timer_used.t()) t_estimate = utils.time_str(timer_used.t() / epoch * max_epoch) if epoch <= max_epoch: epoch_str = str(epoch) else: epoch_str = 'ex' log_str = 'epoch {}, train {:.4f}|{:.4f}'.format( epoch_str, aves['tl'], aves['ta']) writer.add_scalars('loss', {'train': aves['tl']}, epoch) writer.add_scalars('acc', {'train': aves['ta']}, epoch) if eval_val: log_str += ', val {:.4f}|{:.4f}'.format(aves['vl'], aves['va']) writer.add_scalars('loss', {'val': aves['vl']}, epoch) writer.add_scalars('acc', {'val': aves['va']}, epoch) if eval_fs and (epoch % ef_epoch == 0 or epoch == max_epoch + 1): log_str += ', fs' for n_shot in n_shots: key = 'fsa-' + str(n_shot) log_str += ' {}: {:.4f}'.format(n_shot, aves[key]) writer.add_scalars('acc', {key: aves[key]}, epoch) if epoch <= max_epoch: log_str += ', {} {}/{}'.format(t_epoch, t_used, t_estimate) else: log_str += ', {}'.format(t_epoch) utils.log(log_str) if config.get('_parallel'): model_ = model.module else: model_ = model training = { 'epoch': epoch, 'optimizer': config['optimizer'], 'optimizer_args': config['optimizer_args'], 'optimizer_sd': optimizer.state_dict(), } save_obj = { 'file': __file__, 'config': config, 'model': config['model'], 'model_args': config['model_args'], 'model_sd': model_.state_dict(), 'training': training, } if epoch <= max_epoch: torch.save(save_obj, os.path.join(save_path, 'epoch-last.pth')) if (save_epoch is not None) and epoch % save_epoch == 0: torch.save( save_obj, os.path.join(save_path, 'epoch-{}.pth'.format(epoch))) if aves['va'] > max_va: max_va = aves['va'] torch.save(save_obj, os.path.join(save_path, 'max-va.pth')) else: torch.save(save_obj, os.path.join(save_path, 'epoch-ex.pth')) writer.flush()
def main(config): # Environment setup save_dir = config['save_dir'] utils.ensure_path(save_dir) with open(osp.join(save_dir, 'config.yaml'), 'w') as f: yaml.dump(config, f, sort_keys=False) global log, writer logger = set_logger(osp.join(save_dir, 'log.txt')) log = logger.info writer = SummaryWriter(osp.join(save_dir, 'tensorboard')) os.environ['WANDB_NAME'] = config['exp_name'] os.environ['WANDB_DIR'] = config['save_dir'] if not config.get('wandb_upload', False): os.environ['WANDB_MODE'] = 'dryrun' t = config['wandb'] os.environ['WANDB_API_KEY'] = t['api_key'] wandb.init(project=t['project'], entity=t['entity'], config=config) log('logging init done.') log(f'wandb id: {wandb.run.id}') # Dataset, model and optimizer train_dataset = datasets.make((config['train_dataset'])) test_dataset = datasets.make((config['test_dataset'])) model = models.make(config['model'], args=None).cuda() log(f'model #params: {utils.compute_num_params(model)}') n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) if n_gpus > 1: model = nn.DataParallel(model) optimizer = utils.make_optimizer(model.parameters(), config['optimizer']) train_loader = DataLoader(train_dataset, config['batch_size'], shuffle=True, num_workers=8, pin_memory=True) test_loader = DataLoader(test_dataset, config['batch_size'], num_workers=8, pin_memory=True) # Ready for training max_epoch = config['max_epoch'] n_milestones = config.get('n_milestones', 1) milestone_epoch = max_epoch // n_milestones min_test_loss = 1e18 sample_batch_train = sample_data_batch(train_dataset).cuda() sample_batch_test = sample_data_batch(test_dataset).cuda() epoch_timer = utils.EpochTimer(max_epoch) for epoch in range(1, max_epoch + 1): log_text = f'epoch {epoch}' # Train model.train() adjust_lr(optimizer, epoch, max_epoch, config) log_temp_scalar('lr', optimizer.param_groups[0]['lr'], epoch) ave_scalars = {k: utils.Averager() for k in ['loss']} pbar = tqdm(train_loader, desc='train', leave=False) for data in pbar: data = data.cuda() t = train_step(model, data, data, optimizer) for k, v in t.items(): ave_scalars[k].add(v, len(data)) pbar.set_description(desc=f"train loss:{t['loss']:.4f}") log_text += ', train:' for k, v in ave_scalars.items(): v = v.item() log_text += f' {k}={v:.4f}' log_temp_scalar('train/' + k, v, epoch) # Test model.eval() ave_scalars = {k: utils.Averager() for k in ['loss']} pbar = tqdm(test_loader, desc='test', leave=False) for data in pbar: data = data.cuda() t = eval_step(model, data, data) for k, v in t.items(): ave_scalars[k].add(v, len(data)) pbar.set_description(desc=f"test loss:{t['loss']:.4f}") log_text += ', test:' for k, v in ave_scalars.items(): v = v.item() log_text += f' {k}={v:.4f}' log_temp_scalar('test/' + k, v, epoch) test_loss = ave_scalars['loss'].item() if epoch % milestone_epoch == 0: with torch.no_grad(): pred = model(sample_batch_train).clamp(0, 1) video_batch = torch.cat([sample_batch_train, pred], dim=0) log_temp_videos('train/videos', video_batch, epoch) img_batch = video_batch[:, :, 3, :, :] log_temp_images('train/images', img_batch, epoch) pred = model(sample_batch_test).clamp(0, 1) video_batch = torch.cat([sample_batch_test, pred], dim=0) log_temp_videos('test/videos', video_batch, epoch) img_batch = video_batch[:, :, 3, :, :] log_temp_images('test/images', img_batch, epoch) # Summary and save log_text += ', {} {}/{}'.format(*epoch_timer.step()) log(log_text) model_ = model.module if n_gpus > 1 else model model_spec = config['model'] model_spec['sd'] = model_.state_dict() optimizer_spec = config['optimizer'] optimizer_spec['sd'] = optimizer.state_dict() pth_file = { 'model': model_spec, 'optimizer': optimizer_spec, 'epoch': epoch, } if test_loss < min_test_loss: min_test_loss = test_loss wandb.run.summary['min_test_loss'] = min_test_loss torch.save(pth_file, osp.join(save_dir, 'min-test-loss.pth')) torch.save(pth_file, osp.join(save_dir, 'epoch-last.pth')) writer.flush()
def write_info(self, save_path='./', save_name='Arena Dict.txt'): ensure_path(save_path) utils.write_dict() with open(save_path + save_name, 'w') as f: return # to be implemented
def plot_place_cells(self, act_map=None, arena=None, res=50, plot_num=100, col_num=15, save=True, save_path='./', save_name='place_cells_plot.png', cmap='jet'): arena = self.arena if arena is None else arena if act_map is None: act_map, arena_mask = self.get_act_map(arena=arena, res=res) # [N_num, res_x, res_y] else: res_x, res_y = act_map.shape[1], act_map.shape[2] arena_mask = arena.get_mask(res_x=res_x, res_y=res_y) act_map = act_map[:, ::-1, :] # when plotting image, default origin is on top-left corner. act_max = np.max(act_map) act_min = np.min(act_map) #print('PlaceCells.plot_place_cells: act_min:%.2e act_max:%.2e'%(act_min, act_max)) if plot_num < self.N_num: plot_index = np.sort(random.sample(range(self.N_num), plot_num)) # default order: ascending. else: plot_num = self.N_num plot_index = range(self.N_num) img_num = plot_num + 1 row_num = ( plot_num + 1 ) // col_num if img_num % col_num > 0: row_num += 1 #print('row_num:%d col_num:%d'%(row_num, col_num)) fig, axes = plt.subplots(nrows=row_num, ncols=col_num, figsize=(5*col_num, 5*row_num)) act_map_norm = ( act_map - act_min ) / (act_max - act_min) # normalize to [0, 1] cmap_func = plt.cm.get_cmap(cmap) act_map_mapped = cmap_func(act_map_norm) # [N_num, res_x, res_y, (r,g,b,a)] for i in range(act_map_mapped.shape[0]): act_map_mapped[i,:,:,:] = cv.GaussianBlur(act_map_mapped[i,:,:,:], ksize=(3,3), sigmaX=1, sigmaY=1) arena_mask_white = (~arena_mask).astype(np.int)[:, :, np.newaxis] * np.array([1.0, 1.0, 1.0, 1.0], dtype=np.float) #print(np.sum(arena_mask.astype(np.int))) #print(~arena_mask) #print(arena_mask_white.shape) act_map_mapped = act_map_mapped * arena_mask.astype(np.int)[np.newaxis, :, :, np.newaxis] + arena_mask_white[np.newaxis, :, :, :] #arena_mask = arena_mask[np.newaxis, :, :] #for i in range(act_map_mapped.shape[0]): # act_map_mapped[i, arena_mask] = (1.0, 1.0, 1.0, 1.0) for i in range(plot_num): row_index = (i+1) // col_num col_index = (i+1) % col_num N_index = plot_index[i] ax = axes[row_index, col_index] im = ax.imshow(act_map_mapped[N_index], extent=(arena.x0, arena.x1, arena.y0, arena.y1)) # extent: rescaling axis to arena size. ax.set_xticks(np.linspace(arena.x0, arena.x1, 5)) ax.set_yticks(np.linspace(arena.y0, arena.y1, 5)) ax.set_aspect(1) ax.set_title('Place Cells No.%d @ (%.2f, %.2f)'%(N_index, self.xy[N_index][0], self.xy[N_index][1])) #self.arena.plot_arena(ax, save=False) for i in range(plot_num + 1, row_num * col_num): row_index = i // col_num col_index = i % col_num ax = axes[row_index, col_index] ax.axis('off') ax = axes[0, 0] ax.axis('off') norm = mpl.colors.Normalize(vmin=act_min, vmax=act_max) ax_ = ax.inset_axes([0.0, 0.4, 1.0, 0.2]) # left, bottom, width, height. all are ratios to sub-canvas of ax. cbar = ax.figure.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap), cax=ax_, #pad=.05, # ?Fraction of original axes between colorbar and new image axes fraction=1.0, ticks=np.linspace(act_min, act_max, num=5), aspect = 5, # ratio of colorbar height to width anchor=(0.5, 0.5), # coord of anchor point of colorbar panchor=(0.5, 0.5), # coord of colorbar's anchor point in parent ax. orientation='horizontal') cbar.set_label('Average fire rate', loc='center') ax.axis('off') if save: ensure_path(save_path) #cv.imwrite(save_path + save_name, imgs) # so that origin is in left-bottom corner. plt.savefig(save_path + save_name) plt.close()
def process_users(self, users, directory): utils.ensure_path(directory + '/pics/') c = 0 stale_count = 0 no_such_el_count = 0 unprocessed_ids = [] bad_page_ids = [] html_file = open(directory + '/index.html', 'w', encoding='utf-8') html_file.write('<html><body>\n') for user in users: try: c += 1 log(user) user_id = user[1].strip() log('Processing ' + user_id + " ({}/{})".format(c, len(users))) url = FcbBrowserScrapper.root_url + user_id self.browser.get(url) a_el = self.browser.find_element_by_class_name('profilePicThumb') a_el.click() while True: try: img_el = self.browser.find_element_by_class_name('spotlight') break except NoSuchElementException: try: img_el = self.browser.find_element_by_css_selector('img._4-od') break except NoSuchElementException: pass # The ajax call has not finished yet so lets wait 1 second # and repeat it again time.sleep(1) try: pic_url = img_el.get_attribute('src') except StaleElementReferenceException: stale_count += 1 utils.log('Stale element encountered ({})'.format(stale_count)) unprocessed_ids.append(user_id) continue # Retrieve and... response = requests.get(pic_url) # ...save the photo photo_file = open(directory + '/pics/{}.jpg'.format(user_id), 'wb') photo_file.write(response.content) except NoSuchElementException: utils.log("Failed scrapping page for user {} - no such element".format(user_id)) unprocessed_ids.append(user_id) bad_page_ids.append(user_id) no_such_el_count += 1 # page_row = '<a href="{}"><img src="pics/{}" title="{}"></a>'.format(url, user_id + '.jpg', user[0]) html_file.write(page_row + '\n') utils.log('Total staled: {}'.format(stale_count)) utils.log('Total not found: {}'.format(no_such_el_count)) utils.log('Unprocessed ids: {}'.format(unprocessed_ids)) html_file.write('</body></html>')
def main(config): svname = args.name if svname is None: svname = 'meta_{}-{}shot'.format(config['train_dataset'], config['n_shot']) svname += '_' + config['model'] if config['model_args'].get('encoder'): svname += '-' + config['model_args']['encoder'] if config['model_args'].get('prog_synthesis'): svname += '-' + config['model_args']['prog_synthesis'] svname += '-seed' + str(args.seed) if args.tag is not None: svname += '_' + args.tag save_path = os.path.join(args.save_dir, svname) utils.ensure_path(save_path, remove=False) utils.set_log_path(save_path) writer = SummaryWriter(os.path.join(save_path, 'tensorboard')) yaml.dump(config, open(os.path.join(save_path, 'config.yaml'), 'w')) logger = utils.Logger(file_name=os.path.join(save_path, "log_sdout.txt"), file_mode="a+", should_flush=True) #### Dataset #### n_way, n_shot = config['n_way'], config['n_shot'] n_query = config['n_query'] if config.get('n_train_way') is not None: n_train_way = config['n_train_way'] else: n_train_way = n_way if config.get('n_train_shot') is not None: n_train_shot = config['n_train_shot'] else: n_train_shot = n_shot if config.get('ep_per_batch') is not None: ep_per_batch = config['ep_per_batch'] else: ep_per_batch = 1 random_state = np.random.RandomState(args.seed) print('seed:', args.seed) # train train_dataset = datasets.make(config['train_dataset'], **config['train_dataset_args']) utils.log('train dataset: {} (x{})'.format(train_dataset[0][0].shape, len(train_dataset))) if config.get('visualize_datasets'): utils.visualize_dataset(train_dataset, 'train_dataset', writer) train_sampler = BongardSampler(train_dataset.n_tasks, config['train_batches'], ep_per_batch, random_state.randint(2**31)) train_loader = DataLoader(train_dataset, batch_sampler=train_sampler, num_workers=8, pin_memory=True) # tvals tval_loaders = {} tval_name_ntasks_dict = { 'tval': 2000, 'tval_ff': 600, 'tval_bd': 480, 'tval_hd_comb': 400, 'tval_hd_novel': 320 } # numbers depend on dataset for tval_type in tval_name_ntasks_dict.keys(): if config.get('{}_dataset'.format(tval_type)): tval_dataset = datasets.make( config['{}_dataset'.format(tval_type)], **config['{}_dataset_args'.format(tval_type)]) utils.log('{} dataset: {} (x{})'.format(tval_type, tval_dataset[0][0].shape, len(tval_dataset))) if config.get('visualize_datasets'): utils.visualize_dataset(tval_dataset, 'tval_ff_dataset', writer) tval_sampler = BongardSampler( tval_dataset.n_tasks, n_batch=tval_name_ntasks_dict[tval_type] // ep_per_batch, ep_per_batch=ep_per_batch, seed=random_state.randint(2**31)) tval_loader = DataLoader(tval_dataset, batch_sampler=tval_sampler, num_workers=8, pin_memory=True) tval_loaders.update({tval_type: tval_loader}) else: tval_loaders.update({tval_type: None}) # val val_dataset = datasets.make(config['val_dataset'], **config['val_dataset_args']) utils.log('val dataset: {} (x{})'.format(val_dataset[0][0].shape, len(val_dataset))) if config.get('visualize_datasets'): utils.visualize_dataset(val_dataset, 'val_dataset', writer) val_sampler = BongardSampler(val_dataset.n_tasks, n_batch=900 // ep_per_batch, ep_per_batch=ep_per_batch, seed=random_state.randint(2**31)) val_loader = DataLoader(val_dataset, batch_sampler=val_sampler, num_workers=8, pin_memory=True) ######## #### Model and optimizer #### if config.get('load'): print('loading pretrained model: ', config['load']) model = models.load(torch.load(config['load'])) else: model = models.make(config['model'], **config['model_args']) if config.get('load_encoder'): print('loading pretrained encoder: ', config['load_encoder']) encoder = models.load(torch.load(config['load_encoder'])).encoder model.encoder.load_state_dict(encoder.state_dict()) if config.get('load_prog_synthesis'): print('loading pretrained program synthesis model: ', config['load_prog_synthesis']) prog_synthesis = models.load( torch.load(config['load_prog_synthesis'])) model.prog_synthesis.load_state_dict(prog_synthesis.state_dict()) if config.get('_parallel'): model = nn.DataParallel(model) utils.log('num params: {}'.format(utils.compute_n_params(model))) optimizer, lr_scheduler = utils.make_optimizer(model.parameters(), config['optimizer'], **config['optimizer_args']) ######## max_epoch = config['max_epoch'] save_epoch = config.get('save_epoch') max_va = 0. timer_used = utils.Timer() timer_epoch = utils.Timer() aves_keys = ['tl', 'ta', 'vl', 'va'] tval_tuple_lst = [] for k, v in tval_loaders.items(): if v is not None: loss_key = 'tvl' + k.split('tval')[-1] acc_key = ' tva' + k.split('tval')[-1] aves_keys.append(loss_key) aves_keys.append(acc_key) tval_tuple_lst.append((k, v, loss_key, acc_key)) trlog = dict() for k in aves_keys: trlog[k] = [] for epoch in range(1, max_epoch + 1): timer_epoch.s() aves = {k: utils.Averager() for k in aves_keys} # train model.train() if config.get('freeze_bn'): utils.freeze_bn(model) writer.add_scalar('lr', optimizer.param_groups[0]['lr'], epoch) for data, label in tqdm(train_loader, desc='train', leave=False): x_shot, x_query = fs.split_shot_query(data.cuda(), n_train_way, n_train_shot, n_query, ep_per_batch=ep_per_batch) label_query = fs.make_nk_label(n_train_way, n_query, ep_per_batch=ep_per_batch).cuda() if config['model'] == 'snail': # only use one selected label_query query_dix = random_state.randint(n_train_way * n_query) label_query = label_query.view(ep_per_batch, -1)[:, query_dix] x_query = x_query[:, query_dix:query_dix + 1] if config['model'] == 'maml': # need grad in maml model.zero_grad() logits = model(x_shot, x_query).view(-1, n_train_way) loss = F.cross_entropy(logits, label_query) acc = utils.compute_acc(logits, label_query) optimizer.zero_grad() loss.backward() optimizer.step() aves['tl'].add(loss.item()) aves['ta'].add(acc) logits = None loss = None # eval model.eval() for name, loader, name_l, name_a in [('val', val_loader, 'vl', 'va') ] + tval_tuple_lst: if config.get('{}_dataset'.format(name)) is None: continue np.random.seed(0) for data, _ in tqdm(loader, desc=name, leave=False): x_shot, x_query = fs.split_shot_query( data.cuda(), n_way, n_shot, n_query, ep_per_batch=ep_per_batch) label_query = fs.make_nk_label( n_way, n_query, ep_per_batch=ep_per_batch).cuda() if config[ 'model'] == 'snail': # only use one randomly selected label_query query_dix = random_state.randint(n_train_way) label_query = label_query.view(ep_per_batch, -1)[:, query_dix] x_query = x_query[:, query_dix:query_dix + 1] if config['model'] == 'maml': # need grad in maml model.zero_grad() logits = model(x_shot, x_query, eval=True).view(-1, n_way) loss = F.cross_entropy(logits, label_query) acc = utils.compute_acc(logits, label_query) else: with torch.no_grad(): logits = model(x_shot, x_query, eval=True).view(-1, n_way) loss = F.cross_entropy(logits, label_query) acc = utils.compute_acc(logits, label_query) aves[name_l].add(loss.item()) aves[name_a].add(acc) # post if lr_scheduler is not None: lr_scheduler.step() for k, v in aves.items(): aves[k] = v.item() trlog[k].append(aves[k]) t_epoch = utils.time_str(timer_epoch.t()) t_used = utils.time_str(timer_used.t()) t_estimate = utils.time_str(timer_used.t() / epoch * max_epoch) log_str = 'epoch {}, train {:.4f}|{:.4f}, val {:.4f}|{:.4f}'.format( epoch, aves['tl'], aves['ta'], aves['vl'], aves['va']) for tval_name, _, loss_key, acc_key in tval_tuple_lst: log_str += ', {} {:.4f}|{:.4f}'.format(tval_name, aves[loss_key], aves[acc_key]) writer.add_scalars('loss', {tval_name: aves[loss_key]}, epoch) writer.add_scalars('acc', {tval_name: aves[acc_key]}, epoch) log_str += ', {} {}/{}'.format(t_epoch, t_used, t_estimate) utils.log(log_str) writer.add_scalars('loss', { 'train': aves['tl'], 'val': aves['vl'], }, epoch) writer.add_scalars('acc', { 'train': aves['ta'], 'val': aves['va'], }, epoch) if config.get('_parallel'): model_ = model.module else: model_ = model training = { 'epoch': epoch, 'optimizer': config['optimizer'], 'optimizer_args': config['optimizer_args'], 'optimizer_sd': optimizer.state_dict(), } save_obj = { 'file': __file__, 'config': config, 'model': config['model'], 'model_args': config['model_args'], 'model_sd': model_.state_dict(), 'training': training, } torch.save(save_obj, os.path.join(save_path, 'epoch-last.pth')) torch.save(trlog, os.path.join(save_path, 'trlog.pth')) if (save_epoch is not None) and epoch % save_epoch == 0: torch.save(save_obj, os.path.join(save_path, 'epoch-{}.pth'.format(epoch))) if aves['va'] > max_va: max_va = aves['va'] torch.save(save_obj, os.path.join(save_path, 'max-va.pth')) writer.flush() print('finished training!') logger.close()
from diabeticretinopathy import DiabeticRetinopathy from samplers import CategoriesSampler from convnet import Convnet from utils import pprint, set_gpu, ensure_path, Averager, Timer, count_acc, euclidean_metric if __name__ == '__main__': save_epoch = 10 train_way = 4 test_way = 4 shot = 1 query = 3 gpu = 0 save_path = './save/proto-1' max_epoch = 60 ensure_path(save_path) trainset = DiabeticRetinopathy('train') train_sampler_ = PrototypicalBatchSampler(trainset.label, 3, train_way, shot + query) train_loader = DataLoader(dataset=trainset, batch_sampler=train_sampler_, num_workers=8) valset = DiabeticRetinopathy('val') val_sampler = PrototypicalBatchSampler(valset.label, 4, test_way, shot + query) val_loader = DataLoader(dataset=valset, batch_sampler=val_sampler, num_workers=8, pin_memory=True)
def deploy(params): """ This command will update a repo and copy the contents to another folder. It can be used to create versioned deployments of the codebase When executed it will prompt for the tag to deploy if it's not known """ print yellow("Warning git.deploy is deprecated from version 1.0") # Try to get global params params = utils.get_global_params(params, 'git_repo_path', 'git_repo_url', 'git_branch', 'git_source_path') # Old params if 'tag_path' in params: abort(red("Warning tag_path is deprecated, Use git_source_path !")) if 'target_path' in params: abort(red("Warning target_path is deprecated, Use git_source_path !")) if 'tag' in params: abort(red("Warning tag is deprecated, Use git_branch !")) if 'branch' in params: abort(red("Warning branch is deprecated, Use git_branch !")) if 'repo_path' in params: abort(red("repo_path is deprecated, use git_repo_path !")) # Check required params if 'git_source_path' not in params: abort(red("git_source_path is required !")) if 'git_repo_url' not in params: abort(red("git_repo_url is required !")) if 'git_repo_path' not in params: abort(red("git_repo_path is required !")) if 'git_branch' not in params or len(params['git_branch']) == 0: abort(red("`git_branch` is required !")) params = utils.format_params(params) if not exists(params['git_repo_path']): abort(red("Repo path not existing... is the project installed?")) # If exist remove full source if exists(params['git_source_path']): print(yellow("Deploy target path `%s` allready existed... source data be removed and reset." % params['git_source_path'])) run('rm -Rf %s' % params['git_source_path']) utils.ensure_path(params['git_source_path']) with cd(params['git_repo_path']): run('git fetch') # Update local repo with latest code run('git checkout %s' % (params['git_branch'])) run('git pull origin %s' % (params['git_branch'])) run('git submodule update') # Copy source code to version run('cp -Rf %s/* %s/' % (params['git_repo_path'], params['git_source_path']))