def maybe_download(filename, work_directory, source_url): if gfile.Exists(work_directory): return gfile.MakeDirs(work_directory) filepath = os.path.join(work_directory, filename) if not gfile.Exists(filepath): print('Downloading', filename) temp_file_name, _ = urllib.request.urlretrieve(source_url) gfile.Copy(temp_file_name, filepath) with gfile.GFile(filepath) as f: size = f.size() print('Successfully downloaded', filename, size, 'bytes.') print("Extracting: " + filepath) base_path = os.path.dirname(filepath) with tarfile.open(filepath, "r:gz") as tf: tf.extractall(base_path) os.remove(filepath) return filepath
def maybe_download(filename, work_directory, source_url): """Download the data from source url, unless it's already here. Args: filename: string, name of the file in the directory. work_directory: string, path to working directory. source_url: url to download from if file doesn't exist. Returns: Path to resulting file. """ if not gfile.Exists(work_directory): gfile.MakeDirs(work_directory) filepath = os.path.join(work_directory, filename) if not gfile.Exists(filepath): temp_file_name, _ = urlretrieve_with_retry(source_url) gfile.Copy(temp_file_name, filepath) with gfile.GFile(filepath) as f: size = f.size() print('Successfully downloaded', filename, size, 'bytes.') return filepath
def main(argv): cells = gfile.ListDirectory(FLAGS.src_dir) nsub_list = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 ] cell_id = np.floor(FLAGS.taskid / len(nsub_list)).astype(np.int) cell_string = cells[cell_id] # copy data dst = os.path.join(FLAGS.tmp_dir, cell_string) if not gfile.Exists(dst): print('Started Copy') src = os.path.join(FLAGS.src_dir, cell_string) if not gfile.IsDirectory(FLAGS.tmp_dir): gfile.MkDir(FLAGS.tmp_dir) gfile.Copy(src, dst) print('File copied to destination') else: print('File exists') # Load data data = h5py.File(dst) stimulus = np.array(data.get('stimulus')) stimulus = stimulus[: -1, :] # drop the last frame so that it's the same size as the binned spike train response = np.squeeze(np.array(data.get('response'))) mask_matrix = np.array(data.get('mask')) # Fit with a given number of subunits nsub = nsub_list[FLAGS.taskid % len(nsub_list)] print('Cell: %s, Nsub: %d' % (cell_string, nsub)) get_su_nsub(stimulus, response, mask_matrix, nsub, cell_string)
files[i:i + group_by] for i in range(0, len(files), group_by) ] for fs in groups: imgs = [] for f in fs: imgs.append(sess.run(img, feed_dict={jpeg: f})) tf.logging.info("VGG-16 models for {} images".format(group_by)) tf.logging.info(datetime.now().strftime("%H:%M:%S")) st = int(datetime.now().strftime("%s")) v1, v2 = sess.run( [row, prob], feed_dict={ x: imgs, 'vgg16/dropout_1/random_uniform:0': [[1.0] * 4096], 'vgg16/dropout/random_uniform:0': [[1.0] * 4096] }) for i in range(len(fs)): jobj = { "key": os.path.splitext(os.path.basename(fs[i]))[0], "vector": v1[i].tolist(), "prob": v2[i].tolist() } jfile.write(json.dumps(jobj) + "\n") jfile.flush() ed = int(datetime.now().strftime("%s")) tf.logging.info("duration: {} sec. files: {} - {}".format( ed - st, os.path.basename(fs[0]), os.path.basename(fs[-1]))) gfile.Copy(tmp_json, output_json)
# See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Utilities for manipulating files. """ import os import numpy as np import PIL from tensorflow.python.platform import gfile import cv2 exists = lambda path: gfile.Exists(path) fopen = lambda path, mode: gfile.Open(path, mode) makedirs = lambda path: gfile.MakeDirs(path) listdir = lambda path: gfile.ListDir(path) copyfile = lambda a, b, o: gfile.Copy(a, b, o) def write_image(image_path, rgb): ext = os.path.splitext(image_path)[1] with gfile.GFile(image_path, 'w') as f: img_str = cv2.imencode(ext, rgb[:, :, ::-1])[1].tostring() f.write(img_str) def read_image(image_path, type='rgb'): with fopen(file_name, 'r') as f: I = PIL.Image.open(f) II = np.array(I) if type == 'rgb': II = II[:, :, :3]
def main(argv): # copy data dst = os.path.join(FLAGS.tmp_dir, 'Off_parasol.mat') if not gfile.Exists(dst): print('Started Copy') src = os.path.join(FLAGS.src_dir, 'Off_parasol.mat') if not gfile.IsDirectory(FLAGS.tmp_dir): gfile.MkDir(FLAGS.tmp_dir) gfile.Copy(src, dst) print('File copied to destination') else: print('File exists') # load stimulus file_data = h5py.File(dst, 'r') # Load Masked movie data = file_data.get('maskedMovdd') stimulus = np.array(data) # load cell response cells = file_data.get('cells') cells = np.array(cells) cells = np.squeeze(cells) ttf_log = file_data.get('ttf_log') ttf_avg = file_data.get('ttf_avg') # Load spike Response of cells data = file_data.get('Y') responses = np.array(data) # get mask total_mask_log = np.array(file_data.get('totalMaskAccept_log')) print('Got data') # read line corresponding to task with gfile.Open(FLAGS.task_params_file, 'r') as f: for _ in range(FLAGS.taskid + 1): line = f.readline() line = line[:-1] # Remove \n from end. print(line) # get task parameters by parsing the lines line_split = line.split(';') cell_idx = line_split[0] cell_idx = cell_idx[1:-1].split(',') cell_idx = [int(i) for i in cell_idx] nsub = int(line_split[1]) projection_type = line_split[2] lam_proj = float(line_split[3]) partitions_fit = line_split[4] partitions_fit = partitions_fit[1:-1].split(',') partitions_fit = [int(i) for i in partitions_fit] if len(line_split) == 5: cell_idx_mask = cell_idx else: cell_idx_mask = line_split[5] cell_idx_mask = cell_idx_mask[1:-1].split(',') cell_idx_mask = [int(i) for i in cell_idx] ## print(cell_idx) print(nsub) print(cell_idx_mask) mask = (total_mask_log[cell_idx_mask, :].sum(0) != 0) mask_matrix = np.reshape(mask != 0, [40, 80]) # make mask bigger - add one row one left/right r, c = np.where(mask_matrix) mask_matrix[r.min() - 1:r.max() + 1, c.min() - 1:c.max() + 1] = True neighbor_mat = su_model.get_neighbormat(mask_matrix, nbd=1) mask = np.ndarray.flatten(mask_matrix) stim_use = stimulus[:, mask] resp_use = responses[:, cell_idx] print('Prepared data') # get last 10% as test data np.random.seed(23) frac_test = 0.1 tms_test = np.arange(np.floor(stim_use.shape[0] * (1 - frac_test)), 1 * np.floor(stim_use.shape[0])).astype(np.int) # Random partitions n_partitions = 10 tms_train_validate = np.arange( 0, np.floor(stim_use.shape[0] * (1 - frac_test))).astype(np.int) frac_validate = 0.1 partitions = [] for _ in range(n_partitions): perm = np.random.permutation(tms_train_validate) tms_train = perm[0:np.int(np.floor((1 - frac_validate) * perm.shape[0]))] tms_validate = perm[np. int(np.floor((1 - frac_validate) * perm.shape[0])):np.int(perm.shape[0])] partitions += [{ 'tms_train': tms_train, 'tms_validate': tms_validate, 'tms_test': tms_test }] print('Made partitions') # Do fitting # tms_train = np.arange(0, np.floor(stim_use.shape[0] * 0.8)).astype(np.int) # tms_test = np.arange(np.floor(stim_use.shape[0] * 0.8), # 1 * np.floor(stim_use.shape[0] * 0.9)).astype(np.int) ss = '_'.join([str(cells[ic]) for ic in cell_idx]) for ipartition in partitions_fit: save_filename = os.path.join( FLAGS.save_path, 'Cell_%s_nsub_%d_%s_%.3f_part_%d_jnt.pkl' % (ss, nsub, projection_type, lam_proj, ipartition)) save_filename_partial = os.path.join( FLAGS.save_path_partial, 'Cell_%s_nsub_%d_%s_%.3f_part_%d_jnt.p' 'kl' % (ss, nsub, projection_type, lam_proj, ipartition)) if not gfile.Exists(save_filename): print('Fitting started') op = su_model.Flat_clustering_jnt( stim_use, resp_use, nsub, partitions[ipartition]['tms_train'], partitions[ipartition]['tms_validate'], steps_max=10000, eps=1e-9, projection_type=projection_type, neighbor_mat=neighbor_mat, lam_proj=lam_proj, eps_proj=0.01, save_filename_partial=save_filename_partial) k, b, _, lam_log, lam_log_test, fitting_phase, fit_params = op tms_test = partitions[ipartition]['tms_test'] lam_test_on_test_data = su_model.compute_fr_loss( fit_params[2][0], fit_params[2][1], stim_use[tms_test, :], resp_use[tms_test, :], nl_params=fit_params[2][2]) print('Fitting done') save_dict = { 'K': k, 'b': b, 'lam_log': lam_log, 'lam_log_validate': lam_log_test, 'lam_test': lam_test_on_test_data, 'fitting_phase': fitting_phase, 'fit_params': fit_params } pickle.dump(save_dict, gfile.Open(save_filename, 'w')) print('Saved results')
def main(argv): # copy WN data dst = os.path.join(FLAGS.tmp_dir, 'Off_parasol.mat') if not gfile.Exists(dst): print('Started Copy') src = os.path.join(FLAGS.src_dir, 'Off_parasol.mat') if not gfile.IsDirectory(FLAGS.tmp_dir): gfile.MkDir(FLAGS.tmp_dir) gfile.Copy(src, dst) print('File copied to destination') else: print('File exists') # load stimulus file = h5py.File(dst, 'r') # Load Masked movie data = file.get('maskedMovdd') stimulus = np.array(data) # load cell response cells = file.get('cells') cells = np.array(cells) cells = np.squeeze(cells) ttf_log = file.get('ttf_log') ttf_avg = file.get('ttf_avg') # Load spike Response of cells data = file.get('Y') responses = np.array(data) # get mask total_mask_log = np.array(file.get('totalMaskAccept_log')) print('Got WN data') # Get NULL data dat_null = sio.loadmat( gfile.Open( os.path.join(FLAGS.src_dir, 'OFF_parasol_trial' '_resp_data.mat'), 'r')) # Load Masked movie cids = np.squeeze(np.array(dat_null['cids'])) condition_idx = FLAGS.datarun stimulus_null = dat_null['condMov'][condition_idx, 0] stimulus_null = np.transpose(stimulus_null, [2, 1, 0]) stimulus_null = np.reshape(stimulus_null, [stimulus_null.shape[0], -1]) resp_cell_log = dat_null['resp_cell_log'] print('Got Null data') # read line corresponding to task with gfile.Open(FLAGS.task_params_file, 'r') as f: for itask in range(FLAGS.taskid + 1): line = f.readline() line = line[:-1] # Remove \n from end. print(line) # get task parameters by parsing the lines line_split = line.split(';') cell_idx = line_split[0] cell_idx = cell_idx[1:-1].split(',') cell_idx = [int(i) for i in cell_idx] Nsub = int(line_split[1]) projection_type = line_split[2] lam_proj = float(line_split[3]) #ipartition = int(line_split[4]) #cell_idx_mask = cell_idx partitions_fit = line_split[4] partitions_fit = partitions_fit[1:-1].split(',') partitions_fit = [int(i) for i in partitions_fit] if len(line_split) == 5: cell_idx_mask = cell_idx else: cell_idx_mask = line_split[5] cell_idx_mask = cell_idx_mask[1:-1].split(',') cell_idx_mask = [int(i) for i in cell_idx] ## ## print(cell_idx) print(Nsub) print(cell_idx_mask) mask = (total_mask_log[cell_idx_mask, :].sum(0) != 0) mask_matrix = np.reshape(mask != 0, [40, 80]) # make mask bigger - add one row one left/right r, c = np.where(mask_matrix) mask_matrix[r.min() - 1:r.max() + 1, c.min() - 1:c.max() + 1] = True neighbor_mat = su_model.get_neighbormat(mask_matrix, nbd=1) mask = np.ndarray.flatten(mask_matrix) ## WN preprocess stim_use_wn = stimulus[:, mask] resp_use_wn = responses[:, cell_idx] # get last 10% as test data np.random.seed(23) frac_test = 0.1 tms_test = np.arange(np.floor(stim_use_wn.shape[0] * (1 - frac_test)), 1 * np.floor(stim_use_wn.shape[0])).astype(np.int) # Random partitions n_partitions = 10 tms_train_validate = np.arange( 0, np.floor(stim_use_wn.shape[0] * (1 - frac_test))).astype(np.int) frac_validate = 0.1 partitions_wn = [] for _ in range(n_partitions): perm = np.random.permutation(tms_train_validate) tms_train = perm[0:np.int(np.floor((1 - frac_validate) * perm.shape[0]))] tms_validate = perm[np. int(np.floor((1 - frac_validate) * perm.shape[0])):np.int(perm.shape[0])] partitions_wn += [{ 'tms_train': tms_train, 'tms_validate': tms_validate, 'tms_test': tms_test }] print('Made partitions') print('WN data preprocessed') ## NULL preprocess stimulus_use_null = stimulus_null[:, mask] ttf_use = np.array(ttf_log[cell_idx, :]).astype(np.float32).squeeze() stimulus_use_null = filterMov_time(stimulus_use_null, ttf_use) if len(cell_idx) > 1: print('More than 1 cell not supported!') try: resp_use_null = np.array(resp_cell_log[cell_idx[0], 0][condition_idx, 0]).T.astype(np.float32) except: resp_use_null = np.array(resp_cell_log[cell_idx[0], 0][0, condition_idx].T).astype( np.float32) # Remove first 30 frames due to convolution artifact. stimulus_use_null = stimulus_use_null[30:, :] resp_use_null = resp_use_null[30:, :] n_trials = resp_use_null.shape[1] t_null = resp_use_null.shape[0] tms_train_1tr_null = np.arange(np.floor(t_null / 2)).astype(np.int) tms_test_1tr_null = np.arange(np.ceil(t_null / 2), t_null).astype(np.int) # repeat in time dimension, divide into training and testing. stimulus_use_null = np.tile(stimulus_use_null.T, n_trials).T resp_use_null = np.ndarray.flatten(resp_use_null.T) resp_use_null = np.expand_dims(resp_use_null, 1) tms_train_null = np.array([]) tms_test_null = np.array([]) for itrial in range(n_trials): tms_train_null = np.append(tms_train_null, tms_train_1tr_null + itrial * t_null) tms_test_null = np.append(tms_test_null, tms_test_1tr_null + itrial * t_null) tms_train_null = tms_train_null.astype(np.int) tms_test_null = tms_test_null.astype(np.int) print('NULL data preprocessed') ss = '_'.join([str(cells[ic]) for ic in cell_idx]) for ipartition in partitions_fit: save_filename = os.path.join( FLAGS.save_path, 'Cell_%s_nsub_%d_%s_%.3f_part_%d_jnt.pkl' % (ss, Nsub, projection_type, lam_proj, ipartition)) save_filename_partial = os.path.join( FLAGS.save_path_partial, 'Cell_%s_nsub_%d_%s_%.3f_part_%d_' 'jnt.pkl' % (ss, Nsub, projection_type, lam_proj, ipartition)) ## Do fitting # Fit SU on WN if not gfile.Exists(save_filename): print('Fitting started on WN') op = su_model.Flat_clustering_jnt( stim_use_wn, resp_use_wn, Nsub, partitions_wn[ipartition]['tms_train'], partitions_wn[ipartition]['tms_validate'], steps_max=10000, eps=1e-9, projection_type=projection_type, neighbor_mat=neighbor_mat, lam_proj=lam_proj, eps_proj=0.01, save_filename_partial=save_filename_partial, fitting_phases=[1]) _, _, alpha, lam_log_wn, lam_log_test_wn, fitting_phase, fit_params_wn = op print('Fitting done on WN') # Fit on NULL op = su_model.fit_scales(stimulus_use_null[tms_train_null, :], resp_use_null[tms_train_null, :], stimulus_use_null[tms_test_null, :], resp_use_null[tms_test_null, :], Ns=Nsub, K=fit_params_wn[0][0], b=fit_params_wn[0][1], params=fit_params_wn[0][2], lr=0.01, eps=1e-9) k_null, b_null, nl_params_null, lam_log_null, lam_log_test_null = op # Collect results and save fit_params = fit_params_wn + [[k_null, b_null, nl_params_null]] lam_log = [lam_log_wn, np.array(lam_log_null)] lam_log_test = [lam_log_test_wn, np.array(lam_log_test_null)] save_dict = { 'lam_log': lam_log, 'lam_log_test': lam_log_test, 'fit_params': fit_params } pickle.dump(save_dict, gfile.Open(save_filename, 'w')) print('Saved results')
def main(argv): # Copy data. dst = os.path.join(FLAGS.tmp_dir, 'Off_parasol.mat') if not gfile.Exists(dst): print('Started Copy') src = os.path.join(FLAGS.src_dir, 'Off_parasol.mat') if not gfile.IsDirectory(FLAGS.tmp_dir): gfile.MkDir(FLAGS.tmp_dir) gfile.Copy(src, dst) print('File copied to destination') else: print('File exists') # load stimulus file_data = h5py.File(dst, 'r') # Load Masked movie data = file_data.get('maskedMovdd') stimulus = np.array(data) # load cell response cells = file_data.get('cells') cells = np.array(cells) cells = np.squeeze(cells) ttf_log = file_data.get('ttf_log') ttf_avg = file_data.get('ttf_avg') # Load spike Response of cells data = file_data.get('Y') responses = np.array(data) # get mask total_mask_log = np.array(file_data.get('totalMaskAccept_log')) print('Got data') # read line corresponding to task with gfile.Open(FLAGS.task_params_file, 'r') as f: for _ in range(FLAGS.taskid + 1): line = f.readline() line = line[:-1] # Remove \n from end. print(line) # get task parameters by parsing the lines line_split = line.split(';') cell_idx = line_split[0] cell_idx = cell_idx[1:-1].split(',') cell_idx = [int(i) for i in cell_idx] model = line_split[1] if model == 'su': nsub = int(line_split[2]) projection_type = line_split[3] lam_proj = float(line_split[4]) if model == 'conv': dim_filters = int(line_split[2]) strides = int(line_split[3]) num_filters = int(line_split[4]) frac_train = float(line_split[5]) mask = (total_mask_log[cell_idx, :].sum(0) != 0) mask_matrix = np.reshape(mask != 0, [40, 80]) # make mask bigger - add one row one left/right r, c = np.where(mask_matrix) mask_matrix[r.min() - 1:r.max() + 1, c.min() - 1:c.max() + 1] = True neighbor_mat = su_model.get_neighbormat(mask_matrix, nbd=1) mask = np.ndarray.flatten(mask_matrix) stimulus_2d = np.reshape(stimulus, [-1, 40, 80]) stim_use_2d = stimulus_2d[:, r.min() - 1:r.max() + 1, c.min() - 1:c.max() + 1] stim_use = stimulus[:, mask] resp_use = responses[:, cell_idx] print('Prepared data') # get last 10% as test data np.random.seed(23) frac_test = 0.1 tms_test = np.arange(np.floor(stim_use.shape[0] * (1 - frac_test)), 1 * np.floor(stim_use.shape[0])).astype(np.int) # Random partitions n_partitions = 10 tms_train_validate = np.arange( 0, np.floor(stim_use.shape[0] * (1 - frac_test))).astype(np.int) frac_validate = 0.1 # 'frac_train' needs to be < 0.9 partitions = [] for _ in range(n_partitions): perm = np.random.permutation(tms_train_validate) tms_train = perm[0:np.floor(frac_train * perm.shape[0])] tms_validate = perm[np.floor((1 - frac_validate) * perm.shape[0]):perm.shape[0]] partitions += [{ 'tms_train': tms_train, 'tms_validate': tms_validate, 'tms_test': tms_test }] ipartition = 0 print('Made partitions') # Do fitting # tms_train = np.arange(0, np.floor(stim_use.shape[0] * 0.8)).astype(np.int) # tms_test = np.arange(np.floor(stim_use.shape[0] * 0.8), # 1 * np.floor(stim_use.shape[0] * 0.9)).astype(np.int) ss = '_'.join([str(cells[ic]) for ic in cell_idx]) if model == 'su': save_filename = os.path.join( FLAGS.save_path, 'Cell_%s_su_nsub_%d_%s_%.3f_frac_train' '_%.4f_jnt.pkl' % (ss, nsub, projection_type, lam_proj, frac_train)) save_filename_partial = os.path.join( FLAGS.save_path_partial, 'Cell_%s_su_nsub_%d_%s_%.3f_frac_' 'train_%.4f_jnt.pkl' % (ss, nsub, projection_type, lam_proj, frac_train)) if not gfile.Exists(save_filename): print('Fitting started for SU') op = su_model.Flat_clustering_jnt( stim_use, resp_use, nsub, partitions[ipartition]['tms_train'], partitions[ipartition]['tms_validate'], steps_max=10000, eps=1e-9, projection_type=projection_type, neighbor_mat=neighbor_mat, lam_proj=lam_proj, eps_proj=0.01, save_filename_partial=save_filename_partial) k, b, _, lam_log, lam_log_test, fitting_phase, fit_params = op print('Fitting done') save_dict = { 'K': k, 'b': b, 'lam_log': lam_log, 'lam_log_test': lam_log_test, 'fitting_phase': fitting_phase, 'fit_params': fit_params } pickle.dump(save_dict, gfile.Open(save_filename, 'w')) print('Saved results') if model == 'conv': save_filename = os.path.join( FLAGS.save_path, 'Cell_%s_conv_dim_filter_%d_strides_' '%d_num_filters_%d_frac_train_%.4f_jnt.pkl' % (ss, dim_filters, strides, num_filters, frac_train)) if not gfile.Exists(save_filename): print('Fitting started for CONV') op = conv_model.convolutional_1layer( stim_use_2d, np.squeeze(resp_use), partitions[ipartition]['tms_train'], partitions[ipartition]['tms_validate'], dim_filters=dim_filters, num_filters=num_filters, lr=0.1, num_steps_max=100000, strides=strides, eps=1e-9) loss_train_log, loss_test_log, model_params = op print('Convolutional model fitting done') save_dict = { 'lam_log': loss_train_log, 'lam_log_test': loss_test_log, 'model_params': model_params } pickle.dump(save_dict, gfile.Open(save_filename, 'w')) print('Saved results')
def dump_weights(variables, filepath): with open(".weights.json", "w") as f: data = {key: variables[key].tolist() for key in variables} f.write(json.dumps(data)) gfile.Copy(".weights.json", filepath)
def export_fn(estimator, export_dir_base, checkpoint_path=None, eval_result=None): with ops.Graph().as_default() as g: contrib_variables.create_global_step(g) input_ops = serving_from_csv_input(train_config, args, keep_target) model_fn_ops = estimator._call_model_fn( input_ops.features, None, model_fn_lib.ModeKeys.INFER) output_fetch_tensors = make_output_tensors( train_config=train_config, args=args, input_ops=input_ops, model_fn_ops=model_fn_ops, keep_target=keep_target) signature_def_map = { 'serving_default': signature_def_utils.predict_signature_def( input_ops.default_inputs, output_fetch_tensors) } if not checkpoint_path: # Locate the latest checkpoint checkpoint_path = saver.latest_checkpoint(estimator._model_dir) if not checkpoint_path: raise NotFittedError("Couldn't find trained model at %s." % estimator._model_dir) export_dir = saved_model_export_utils.get_timestamped_export_dir( export_dir_base) with tf_session.Session('') as session: #variables.initialize_local_variables() variables.local_variables_initializer() data_flow_ops.tables_initializer() saver_for_restore = saver.Saver(variables.global_variables(), sharded=True) saver_for_restore.restore(session, checkpoint_path) init_op = control_flow_ops.group( variables.local_variables_initializer(), data_flow_ops.tables_initializer()) # Perform the export builder = saved_model_builder.SavedModelBuilder(export_dir) builder.add_meta_graph_and_variables( session, [tag_constants.SERVING], signature_def_map=signature_def_map, assets_collection=ops.get_collection( ops.GraphKeys.ASSET_FILEPATHS), legacy_init_op=init_op) builder.save(False) # Add the extra assets if assets_extra: assets_extra_path = os.path.join( compat.as_bytes(export_dir), compat.as_bytes('assets.extra')) for dest_relative, source in assets_extra.items(): dest_absolute = os.path.join( compat.as_bytes(assets_extra_path), compat.as_bytes(dest_relative)) dest_path = os.path.dirname(dest_absolute) gfile.MakeDirs(dest_path) gfile.Copy(source, dest_absolute) # only keep the last 3 models saved_model_export_utils.garbage_collect_exports(export_dir_base, exports_to_keep=3) # save the last model to the model folder. # export_dir_base = A/B/intermediate_models/ if keep_target: final_dir = os.path.join(args.job_dir, 'evaluation_model') else: final_dir = os.path.join(args.job_dir, 'model') if file_io.is_directory(final_dir): file_io.delete_recursively(final_dir) file_io.recursive_create_dir(final_dir) _recursive_copy(export_dir, final_dir) return export_dir
def main(argv): cell_idx = FLAGS.taskid file_list = gfile.ListDirectory(FLAGS.src_dir) cell_file = file_list[cell_idx] print('Cell file %s' % cell_file) # copy data dst = os.path.join(FLAGS.tmp_dir, cell_file) if not gfile.Exists(dst): print('Started Copy') src = os.path.join(FLAGS.src_dir, cell_file) if not gfile.IsDirectory(FLAGS.tmp_dir): gfile.MkDir(FLAGS.tmp_dir) gfile.Copy(src, dst) print('File copied to destination') else: print('File exists') # load stimulus, response data data = sio.loadmat(dst) trainMov_filterNSEM = data['trainMov_filterNSEM'] testMov_filterNSEM = data['testMov_filterNSEM'] trainSpksNSEM = data['trainSpksNSEM'] testSpksNSEM = data['testSpksNSEM'] mask = data['mask'] trainMov_filterWN = data['trainMov_filterWN'] testMov_filterWN = data['testMov_filterWN'] trainSpksWN = data['trainSpksWN'] testSpksWN = data['testSpksWN'] # get NSEM stimulus and resposne stimulus_WN = np.array(trainMov_filterWN.transpose(), dtype='float32') response_WN = np.array(np.squeeze(trainSpksWN), dtype='float32') stimulus_NSEM = np.array(trainMov_filterNSEM.transpose(), dtype='float32') response_NSEM = np.array(np.squeeze(trainSpksNSEM), dtype='float32') print('Prepared data') # Do fitting # set random seed. np.random.seed(23) print('Made partitions') # Do fitting # WN data ifrac = 0.8 tms_train_WN = np.arange(0, np.floor(stimulus_WN.shape[0] * ifrac)).astype( np.int) tms_test_WN = np.arange(np.floor(stimulus_WN.shape[0] * ifrac), 1 * np.floor(stimulus_WN.shape[0] * 1)).astype(np.int) # NSEM data ifrac = 0.8 tms_train_NSEM = np.arange(0, np.floor(stimulus_NSEM.shape[0] * ifrac)).astype(np.int) tms_test_NSEM = np.arange(np.floor(stimulus_NSEM.shape[0] * ifrac), 1 * np.floor(stimulus_NSEM.shape[0] * 1)).astype( np.int) ''' eps = 1e-7 for Nsub in [1, 2, 3, 4, 5, 7, 10]: print('Fitting started') # WN fit op = jnt_model.Flat_clustering(stimulus_WN, response_WN, Nsub, tms_train_WN, tms_test_WN, steps_max=10000, eps=eps) K, b, alpha, lam_log, lam_log_test, fitting_phase, fit_params = op WN_fit = {'K': K, 'b': b, 'lam_log': lam_log, 'lam_log_test': lam_log_test} print('WN fit done') # NSEM fit # Just fit the scales # fit NL + b + Kscale K, b, params, loss_log, loss_log_test = jnt_model.fit_scales(stimulus_NSEM[tms_train_NSEM, :], response_NSEM[tms_train_NSEM], stimulus_NSEM[tms_test_NSEM, :], response_NSEM[tms_test_NSEM], Ns=Nsub, K=WN_fit['K'], b=WN_fit['b'], params=[1.0, 0.0], lr=0.001, eps=eps) NSEM_fit_scales = {'K': K, 'b': b, 'nl_params': params, 'lam_log': loss_log, 'lam_log_test': loss_log_test} print('NSEM scales fit') # Fit all params K, b, params, loss_log, loss_log_test = jnt_model.fit_all(stimulus_NSEM[tms_train_NSEM, :], response_NSEM[tms_train_NSEM], stimulus_NSEM[tms_test_NSEM, :], response_NSEM[tms_test_NSEM], Ns=Nsub, K=NSEM_fit_scales['K'], b=NSEM_fit_scales['b'], train_phase=3, params=NSEM_fit_scales['nl_params'], lr=0.001, eps=eps) NSEM_fit_full = {'K': K, 'b': b, 'nl_params': params, 'lam_log': loss_log, 'lam_log_test': loss_log_test} print('NSEM all fit') save_dict = {'WN_fit': WN_fit, 'NSEM_fit_scales': NSEM_fit_scales, 'NSEM_fit_full': NSEM_fit_full} pickle.dump(save_dict, gfile.Open(os.path.join(FLAGS.save_path, 'Cell_%s_nsub_%d_suff_%d_jnt.pkl' % (cell_file, Nsub, 1)), 'w' )) print('Saved results') ''' ''' eps = 1e-7 for Nsub in [1, 2, 3, 4, 5, 7, 10]: print('Fitting started') # Fit all params K = 2*rng.rand(stimulus_NSEM.shape[1], Nsub)-0.5 b = 2*rng.rand(Nsub)-0.5 K, b, params, loss_log, loss_log_test = jnt_model.fit_all(stimulus_NSEM[tms_train_NSEM, :], response_NSEM[tms_train_NSEM], stimulus_NSEM[tms_test_NSEM, :], response_NSEM[tms_test_NSEM], Ns=Nsub, K=K.astype(np.float32), b=b.astype(np.float32), train_phase=3, params=[1.0, 0.0], lr=0.001, eps=eps) NSEM_fit_full = {'K': K, 'b': b, 'nl_params': params, 'lam_log': loss_log, 'lam_log_test': loss_log_test} print('NSEM all (random) fit') save_dict = {'NSEM_fit_full_random': NSEM_fit_full} pickle.dump(save_dict, gfile.Open(os.path.join(FLAGS.save_path, 'Cell_%s_nsub_%d_suff_%d_randomly_init.pkl' % (cell_file, Nsub, 1)), 'w' )) print('Saved results') ''' eps = 1e-7 for Nsub in [1, 2, 3, 4, 5, 7, 10]: print('Fitting started') # NSEM clustering fit op = jnt_model.Flat_clustering(stimulus_NSEM, response_NSEM, Nsub, tms_train_NSEM, tms_test_NSEM, steps_max=10000, eps=eps) K, b, alpha, lam_log, lam_log_test, fitting_phase, fit_params = op NSEM_clustering = { 'K': K, 'b': b, 'lam_log': lam_log, 'lam_log_test': lam_log_test } print('NSEM clustering fit') # NSEM fit # Just fit the scales # fit NL + b + Kscale K, b, params, loss_log, loss_log_test = jnt_model.fit_scales( stimulus_NSEM[tms_train_NSEM, :], response_NSEM[tms_train_NSEM], stimulus_NSEM[tms_test_NSEM, :], response_NSEM[tms_test_NSEM], Ns=Nsub, K=NSEM_clustering['K'], b=NSEM_clustering['b'], params=[1.0, 0.0], lr=0.001, eps=eps) NSEM_fit_scales = { 'K': K, 'b': b, 'nl_params': params, 'lam_log': loss_log, 'lam_log_test': loss_log_test } print('NSEM scales fit') # Fit all params K, b, params, loss_log, loss_log_test = jnt_model.fit_all( stimulus_NSEM[tms_train_NSEM, :], response_NSEM[tms_train_NSEM], stimulus_NSEM[tms_test_NSEM, :], response_NSEM[tms_test_NSEM], Ns=Nsub, K=NSEM_fit_scales['K'], b=NSEM_fit_scales['b'], train_phase=3, params=NSEM_fit_scales['nl_params'], lr=0.001, eps=eps) NSEM_fit_full = { 'K': K, 'b': b, 'nl_params': params, 'lam_log': loss_log, 'lam_log_test': loss_log_test } print('NSEM all fit') save_dict = { 'NSEM_clustering': NSEM_clustering, 'NSEM_fit_scales': NSEM_fit_scales, 'NSEM_fit_full': NSEM_fit_full } pickle.dump( save_dict, gfile.Open( os.path.join( FLAGS.save_path, 'Cell_%s_nsub_%d_suff_%d_NSEM_3_steps.pkl' % (cell_file, Nsub, 1)), 'w')) print('Saved results')
def create_experiment(output_dir): """ Creates a new Experiment instance. Args: output_dir: Output directory for model checkpoints and summaries. """ config = run_config.RunConfig( tf_random_seed=FLAGS.tf_random_seed, save_checkpoints_secs=FLAGS.save_checkpoints_secs, save_checkpoints_steps=FLAGS.save_checkpoints_steps, keep_checkpoint_max=FLAGS.keep_checkpoint_max, keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours) # Load vocabulary info source_vocab_info = vocab.get_vocab_info(FLAGS.vocab_source) target_vocab_info = vocab.get_vocab_info(FLAGS.vocab_target) # Find model class model_class = getattr(models, FLAGS.model) # Parse parameter and merge with defaults hparams = model_class.default_params() if FLAGS.hparams is not None and isinstance(FLAGS.hparams, str): hparams = HParamsParser(hparams).parse(FLAGS.hparams) elif isinstance(FLAGS.hparams, dict): hparams.update(FLAGS.hparams) # Print hparams training_utils.print_hparams(hparams) # One the main worker, save training options and vocabulary if config.is_chief: # Copy vocabulary to output directory gfile.MakeDirs(output_dir) source_vocab_path = os.path.join(output_dir, "vocab_source") gfile.Copy(FLAGS.vocab_source, source_vocab_path, overwrite=True) target_vocab_path = os.path.join(output_dir, "vocab_target") gfile.Copy(FLAGS.vocab_target, target_vocab_path, overwrite=True) # Save train options train_options = training_utils.TrainOptions( hparams=hparams, model_class=FLAGS.model, source_vocab_path=source_vocab_path, target_vocab_path=target_vocab_path) train_options.dump(output_dir) # Create model model = model_class(source_vocab_info=source_vocab_info, target_vocab_info=target_vocab_info, params=hparams) bucket_boundaries = None if FLAGS.buckets: bucket_boundaries = list(map(int, FLAGS.buckets.split(","))) # Create training input function train_input_fn = training_utils.create_input_fn( data_provider_fn=functools.partial( data_utils.make_parallel_data_provider, data_sources_source=FLAGS.train_source, data_sources_target=FLAGS.train_target, shuffle=True, num_epochs=FLAGS.train_epochs, delimiter=FLAGS.delimiter), batch_size=FLAGS.batch_size, bucket_boundaries=bucket_boundaries) # Create eval input function eval_input_fn = training_utils.create_input_fn( data_provider_fn=functools.partial( data_utils.make_parallel_data_provider, data_sources_source=FLAGS.dev_source, data_sources_target=FLAGS.dev_target, shuffle=False, num_epochs=1, delimiter=FLAGS.delimiter), batch_size=FLAGS.batch_size) def model_fn(features, labels, params, mode): """Builds the model graph""" return model(features, labels, params, mode) estimator = tf.contrib.learn.estimator.Estimator(model_fn=model_fn, model_dir=output_dir, config=config) train_hooks = training_utils.create_default_training_hooks( estimator=estimator, sample_frequency=FLAGS.sample_every_n_steps, delimiter=FLAGS.delimiter) eval_metrics = { "log_perplexity": metrics.streaming_log_perplexity(), "bleu": metrics.make_bleu_metric_spec(), } experiment = tf.contrib.learn.experiment.Experiment( estimator=estimator, train_input_fn=train_input_fn, eval_input_fn=eval_input_fn, min_eval_frequency=FLAGS.eval_every_n_steps, train_steps=FLAGS.train_steps, eval_steps=None, eval_metrics=eval_metrics, train_monitors=train_hooks) return experiment
def main(argv): # copy data dst = os.path.join(FLAGS.tmp_dir, 'Off_parasol.mat') if not gfile.Exists(dst): print('Started Copy') src = os.path.join(FLAGS.src_dir, 'Off_parasol.mat') if not gfile.IsDirectory(FLAGS.tmp_dir): gfile.MkDir(FLAGS.tmp_dir) gfile.Copy(src, dst) print('File copied to destination') else: print('File exists') # load stimulus file = h5py.File(dst, 'r') # Load Masked movie data = file.get('maskedMovdd') stimulus = np.array(data) # load cell response cells = file.get('cells') ttf_log = file.get('ttf_log') ttf_avg = file.get('ttf_avg') # Load spike Response of cells data = file.get('Y') responses = np.array(data) # get mask total_mask_log = file.get('totalMaskAccept_log') print('Got data') # get cell and mask nsub_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] if FLAGS.taskid < 107 * len(nsub_list): cell_idx = [np.int(np.floor(FLAGS.taskid / len(nsub_list)))] cellid = cells[np.int(np.floor(FLAGS.taskid / len(nsub_list)))] Nsub = nsub_list[FLAGS.taskid % len(nsub_list)] partition_list = np.arange(10) elif FLAGS.taskid < 107 * len(nsub_list) + 37 * 10: cell_idx = [39, 42, 44, 45] #[np.int(FLAGS.taskid)] cellid = cells[cell_idx] cellid = np.squeeze(cellid) task_id_effective = FLAGS.taskid - 107 * len(nsub_list) partition_list = [task_id_effective % 10] nsub_list_pop = np.arange(4, 41) Nsub = nsub_list_pop[np.int(np.floor(task_id_effective / 10))] elif FLAGS.taskid < 107 * len(nsub_list) + 37 * 10 + 19 * 10: cell_idx = [39, 42] #[np.int(FLAGS.taskid)] cellid = cells[cell_idx] cellid = np.squeeze(cellid) task_id_effective = FLAGS.taskid - 107 * len(nsub_list) - 37 * 10 partition_list = [task_id_effective % 10] nsub_list_pop = np.arange(2, 21) Nsub = nsub_list_pop[np.int(np.floor(task_id_effective / 10))] elif FLAGS.taskid < 107 * len(nsub_list) + 37 * 10 + 19 * 10 + 19 * 10: cell_idx = [44, 45] #[np.int(FLAGS.taskid)] cellid = cells[cell_idx] cellid = np.squeeze(cellid) task_id_effective = FLAGS.taskid - 107 * len( nsub_list) - 37 * 10 - 19 * 10 partition_list = [task_id_effective % 10] nsub_list_pop = np.arange(2, 21) Nsub = nsub_list_pop[np.int(np.floor(task_id_effective / 10))] print(cell_idx) print(Nsub) mask = (total_mask_log[cell_idx, :].sum(0) != 0) mask_matrix = np.reshape(mask != 0, [40, 80]) # make mask bigger - add one row one left/right r, c = np.where(mask_matrix) mask_matrix[r.min() - 1:r.max() + 1, c.min() - 1:c.max() + 1] = True mask = np.ndarray.flatten(mask_matrix) stim_use = stimulus[:, mask] resp_use = responses[:, cell_idx] print('Prepared data') # get last 10% as test data np.random.seed(23) frac_test = 0.1 tms_test = np.arange(np.floor(stim_use.shape[0] * (1 - frac_test)), 1 * np.floor(stim_use.shape[0])).astype(np.int) # Random partitions n_partitions = 10 tms_train_validate = np.arange( 0, np.floor(stim_use.shape[0] * (1 - frac_test))).astype(np.int) frac_validate = 0.1 partitions = [] for ipartition in range(n_partitions): perm = np.random.permutation(tms_train_validate) tms_train = perm[0:np.floor((1 - frac_validate) * perm.shape[0])] tms_validate = perm[np.floor((1 - frac_validate) * perm.shape[0]):perm.shape[0]] partitions += [{ 'tms_train': tms_train, 'tms_validate': tms_validate, 'tms_test': tms_test }] print('Made partitions') # Do fitting # tms_train = np.arange(0, np.floor(stim_use.shape[0] * 0.8)).astype(np.int) # tms_test = np.arange(np.floor(stim_use.shape[0] * 0.8), # 1 * np.floor(stim_use.shape[0] * 0.9)).astype(np.int) for ipartition in partition_list: print(cell_idx, cellid, Nsub) ss = '_'.join([str(ic) for ic in cellid]) save_filename = os.path.join( FLAGS.save_path, 'Cell_%s_nsub_%d_part_%d_jnt.pkl' % (ss, Nsub, ipartition)) if not gfile.Exists(save_filename): print('Fitting started') op = jnt_model.Flat_clustering_jnt( stim_use, resp_use, Nsub, partitions[ipartition]['tms_train'], partitions[ipartition]['tms_validate'], steps_max=10000, eps=1e-9) # op = jnt_model.Flat_clustering_jnt(stim_use, resp_use, Nsub, # tms_train, # tms_test, # steps_max=10000, eps=1e-9) K, b, alpha, lam_log, lam_log_test, fitting_phase, fit_params = op print('Fitting done') save_dict = { 'K': K, 'b': b, 'lam_log': lam_log, 'lam_log_test': lam_log_test, 'fitting_phase': fitting_phase, 'fit_params': fit_params } pickle.dump(save_dict, gfile.Open(save_filename, 'w')) print('Saved results')
""" Download Tiny ImageNet data (http://cs231n.stanford.edu/project.html) if needed """ from __future__ import print_function from six.moves import urllib from tensorflow.python.platform import gfile import tempfile import os import zipfile import src.common.paths as paths if __name__ == "__main__": file_dir = os.path.abspath(paths.DATA_ROOT) with tempfile.NamedTemporaryFile() as tmpfile: temp_file_name = tmpfile.name print("Downloading tiny imagenet dataset (237 MB) to {}".format( temp_file_name)) a = urllib.request.urlretrieve( "http://cs231n.stanford.edu/tiny-imagenet-200.zip", temp_file_name) file_path = os.path.join(file_dir, "tiny_imagenet-200.zip") gfile.Copy(temp_file_name, file_path) print("Extracting zip archive (this may take a while)") with zipfile.ZipFile(file_path, 'r') as f: f.extractall(file_dir)
def main(argv): # parse task params # read line corresponding to task with gfile.Open(FLAGS.task_params_file, 'r') as f: for _ in range(FLAGS.taskid + 1): line = f.readline() print(line) # get task parameters by parsing the line. line_split = line.split(';') cells = gfile.ListDirectory(FLAGS.src_dir) cell_idx = line_split[0] cell_idx = cell_idx[1:-1].split(',') nsub = int(line_split[1]) projection_type = line_split[2] lam_proj = float(line_split[3]) ipartition = int(line_split[4][:-1]) # Copy data for all the data cell_str_final = '' dst_log = [] for icell in cell_idx: icell = int(icell) cell_string = cells[icell] cell_str_final += cell_string # copy data for the corresponding task dst = os.path.join(FLAGS.tmp_dir, cell_string) dst_log += [dst] if not gfile.Exists(dst): print('Started Copy') src = os.path.join(FLAGS.src_dir, cell_string) if not gfile.IsDirectory(FLAGS.tmp_dir): gfile.MkDir(FLAGS.tmp_dir) gfile.Copy(src, dst) print('File %s copied to destination' % cell_string) else: print('File %s exists' % cell_string) # Load data for different cells stim_log = [] resp_log = [] mask_matrix_log = [] for dst in dst_log: print('Loading %s' % dst) data = h5py.File(dst) stimulus = np.array(data.get('stimulus')) stimulus = stimulus[:-1, :] # drop the last frame so that it's # the same size as the binned spike train response = np.squeeze(np.array(data.get('response'))) response = np.expand_dims(response, 1) mask_matrix = np.array(data.get('mask')) stim_log += [stimulus] resp_log += [response] mask_matrix_log += [mask_matrix] # Prepare for fitting across multiple cells # Get total mask mask_matrix_pop = np.array(mask_matrix_log).sum(0) > 0 # Get total response. resp_len = np.min([resp_log[icell].shape[0] for icell in range(4)]) response_pop = np.zeros((resp_len, len(resp_log))) for icell in range(len(resp_log)): response_pop[:, icell] = resp_log[icell][:resp_len, 0] # Get total stimulus. stimulus_pop = np.zeros((resp_len, mask_matrix_pop.sum())) # Find non-zero locations for each mask element nnz_log = [np.where(imask > 0) for imask in mask_matrix_log] nnz_pop = np.where(mask_matrix_pop > 0) for ipix in range(mask_matrix_pop.sum()): print(ipix) r = nnz_pop[0][ipix] c = nnz_pop[1][ipix] stim_pix = np.zeros(resp_len) nc = 0 for icell in range(len(nnz_log)): pix_cell_bool = np.logical_and(nnz_log[icell][0] == r, nnz_log[icell][1] == c) if pix_cell_bool.sum() > 0: pix_cell = np.where(pix_cell_bool > 0)[0][0] stim_pix += stim_log[icell][:resp_len, pix_cell] nc += 1 if nc == 0: print('Error') stim_pix = stim_pix / nc stimulus_pop[:, ipix] = stim_pix # Fit with a given number of subunits print('Starting fitting') get_su_nsub(stimulus_pop, response_pop, mask_matrix_pop, cell_str_final, nsub, projection_type, lam_proj, ipartition)
def plot_trajectory_first_person(dt, orig_maps, out_dir): out_dir = os.path.join(out_dir, FLAGS.config_name+_get_suffix_str(), FLAGS.imset) fu.makedirs(out_dir) # Load the model so that we can render. plt.set_cmap('gray') samples_per_action = 8; wait_at_action = 0; Writer = animation.writers['mencoder'] writer = Writer(fps=3*(samples_per_action+wait_at_action), metadata=dict(artist='anonymous'), bitrate=1800) args = sna.get_args_for_config(FLAGS.config_name + '+bench_'+FLAGS.imset) args.navtask.logdir = None navtask_ = copy.deepcopy(args.navtask) navtask_.camera_param.modalities = ['rgb'] navtask_.task_params.modalities = ['rgb'] sz = 512 navtask_.camera_param.height = sz navtask_.camera_param.width = sz navtask_.task_params.img_height = sz navtask_.task_params.img_width = sz R = lambda: nav_env.get_multiplexer_class(navtask_, 0) R = R() b = R.buildings[0] f = [0 for _ in range(wait_at_action)] + \ [float(_)/samples_per_action for _ in range(samples_per_action)]; # Generate things for it to render. inds_to_do = [] inds_to_do += [1, 4, 10] #1291, 1268, 1273, 1289, 1302, 1426, 1413, 1449, 1399, 1390] for i in inds_to_do: fig = plt.figure(figsize=(10,8)) gs = GridSpec(3,4) gs.update(wspace=0.05, hspace=0.05, left=0.0, top=0.97, right=1.0, bottom=0.) ax = fig.add_subplot(gs[:,:-1]) ax1 = fig.add_subplot(gs[0,-1]) ax2 = fig.add_subplot(gs[1,-1]) ax3 = fig.add_subplot(gs[2,-1]) axes = [ax, ax1, ax2, ax3] # ax = fig.add_subplot(gs[:,:]) # axes = [ax] for ax in axes: ax.set_axis_off() node_ids = dt['all_node_ids'][i, :, 0]*1 # Prune so that last node is not repeated more than 3 times? if np.all(node_ids[-4:] == node_ids[-1]): while node_ids[-4] == node_ids[-1]: node_ids = node_ids[:-1] num_steps = np.minimum(FLAGS.num_steps, len(node_ids)) xyt = b.to_actual_xyt_vec(b.task.nodes[node_ids]) xyt_diff = xyt[1:,:] - xyt[:-1:,:] xyt_diff[:,2] = np.mod(xyt_diff[:,2], 4) ind = np.where(xyt_diff[:,2] == 3)[0] xyt_diff[ind, 2] = -1 xyt_diff = np.expand_dims(xyt_diff, axis=1) to_cat = [xyt_diff*_ for _ in f] perturbs_all = np.concatenate(to_cat, axis=1) perturbs_all = np.concatenate([perturbs_all, np.zeros_like(perturbs_all[:,:,:1])], axis=2) node_ids_all = np.expand_dims(node_ids, axis=1)*1 node_ids_all = np.concatenate([node_ids_all for _ in f], axis=1) node_ids_all = np.reshape(node_ids_all[:-1,:], -1) perturbs_all = np.reshape(perturbs_all, [-1, 4]) imgs = b.render_nodes(b.task.nodes[node_ids_all,:], perturb=perturbs_all) # Get action at each node. actions = [] _, action_to_nodes = b.get_feasible_actions(node_ids) for j in range(num_steps-1): action_to_node = action_to_nodes[j] node_to_action = dict(zip(action_to_node.values(), action_to_node.keys())) actions.append(node_to_action[node_ids[j+1]]) def init_fn(): return fig, gt_dist_to_goal = [] # Render trajectories. def worker(j): # Plot the image. step_number = j/(samples_per_action + wait_at_action) img = imgs[j]; ax = axes[0]; ax.clear(); ax.set_axis_off(); img = img.astype(np.uint8); ax.imshow(img); tt = ax.set_title( "First Person View\n" + "Top corners show diagnostics (distance, agents' action) not input to agent.", fontsize=12) plt.setp(tt, color='white') # Distance to goal. t = 'Dist to Goal:\n{:2d} steps'.format(int(dt['all_d_at_t'][i, step_number])) t = ax.text(0.01, 0.99, t, horizontalalignment='left', verticalalignment='top', fontsize=20, color='red', transform=ax.transAxes, alpha=1.0) t.set_bbox(dict(color='white', alpha=0.85, pad=-0.1)) # Action to take. action_latex = ['$\odot$ ', '$\curvearrowright$ ', '$\curvearrowleft$ ', r'$\Uparrow$ '] t = ax.text(0.99, 0.99, action_latex[actions[step_number]], horizontalalignment='right', verticalalignment='top', fontsize=40, color='green', transform=ax.transAxes, alpha=1.0) t.set_bbox(dict(color='white', alpha=0.85, pad=-0.1)) # Plot the map top view. ax = axes[-1] if j == 0: # Plot the map locs = dt['all_locs'][i,:num_steps,:] goal_loc = dt['all_goal_locs'][i,:,:] xymin = np.minimum(np.min(goal_loc, axis=0), np.min(locs, axis=0)) xymax = np.maximum(np.max(goal_loc, axis=0), np.max(locs, axis=0)) xy1 = (xymax+xymin)/2. - 0.7*np.maximum(np.max(xymax-xymin), 24) xy2 = (xymax+xymin)/2. + 0.7*np.maximum(np.max(xymax-xymin), 24) ax.set_axis_on() ax.patch.set_facecolor((0.333, 0.333, 0.333)) ax.set_xticks([]); ax.set_yticks([]); ax.imshow(orig_maps, origin='lower', vmin=-1.0, vmax=2.0) ax.plot(goal_loc[:,0], goal_loc[:,1], 'g*', markersize=12) locs = dt['all_locs'][i,:1,:] ax.plot(locs[:,0], locs[:,1], 'b.', markersize=12) ax.set_xlim([xy1[0], xy2[0]]) ax.set_ylim([xy1[1], xy2[1]]) locs = dt['all_locs'][i,step_number,:] locs = np.expand_dims(locs, axis=0) ax.plot(locs[:,0], locs[:,1], 'r.', alpha=1.0, linewidth=0, markersize=4) tt = ax.set_title('Trajectory in topview', fontsize=14) plt.setp(tt, color='white') return fig, line_ani = animation.FuncAnimation(fig, worker, (num_steps-1)*(wait_at_action+samples_per_action), interval=500, blit=True, init_func=init_fn) tmp_file_name = 'tmp.mp4' line_ani.save(tmp_file_name, writer=writer, savefig_kwargs={'facecolor':'black'}) out_file_name = os.path.join(out_dir, 'vis_{:04d}.mp4'.format(i)) print(out_file_name) if fu.exists(out_file_name): gfile.Remove(out_file_name) gfile.Copy(tmp_file_name, out_file_name) gfile.Remove(tmp_file_name) plt.close(fig)
def export_savedmodel(self, export_dir_base, serving_input_receiver_fn, assets_extra=None, as_text=False, checkpoint_path=None): """Exports inference graph as a SavedModel into given dir. This method builds a new graph by first calling the serving_input_receiver_fn to obtain feature `Tensor`s, and then calling this `Estimator`'s model_fn to generate the model graph based on those features. It restores the given checkpoint (or, lacking that, the most recent checkpoint) into this graph in a fresh session. Finally it creates a timestamped export directory below the given export_dir_base, and writes a `SavedModel` into it containing a single `MetaGraphDef` saved from this session. The exported `MetaGraphDef` will provide one `SignatureDef` for each element of the export_outputs dict returned from the model_fn, named using the same keys. One of these keys is always signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY, indicating which signature will be served when a serving request does not specify one. For each signature, the outputs are provided by the corresponding `ExportOutput`s, and the inputs are always the input receivers provided by the serving_input_receiver_fn. Extra assets may be written into the SavedModel via the extra_assets argument. This should be a dict, where each key gives a destination path (including the filename) relative to the assets.extra directory. The corresponding value gives the full path of the source file to be copied. For example, the simple case of copying a single file without renaming it is specified as `{'my_asset_file.txt': '/path/to/my_asset_file.txt'}`. Args: export_dir_base: A string containing a directory in which to create timestamped subdirectories containing exported SavedModels. serving_input_receiver_fn: A function that takes no argument and returns a `ServingInputReceiver`. assets_extra: A dict specifying how to populate the assets.extra directory within the exported SavedModel, or `None` if no extra assets are needed. as_text: whether to write the SavedModel proto in text format. checkpoint_path: The checkpoint path to export. If `None` (the default), the most recent checkpoint found within the model directory is chosen. Returns: The string path to the exported directory. Raises: ValueError: if no serving_input_receiver_fn is provided, no export_outputs are provided, or no checkpoint can be found. """ if serving_input_receiver_fn is None: raise ValueError('serving_input_receiver_fn must be defined.') with ops.Graph().as_default() as g: training.create_global_step(g) random_seed.set_random_seed(self._config.tf_random_seed) serving_input_receiver = serving_input_receiver_fn() # Call the model_fn and collect the export_outputs. estimator_spec = self._call_model_fn( features=serving_input_receiver.features, labels=None, mode=model_fn_lib.ModeKeys.PREDICT) # Build the SignatureDefs from receivers and all outputs signature_def_map = build_all_signature_defs( serving_input_receiver.receiver_tensors, estimator_spec.export_outputs) if not checkpoint_path: # Locate the latest checkpoint checkpoint_path = saver.latest_checkpoint(self._model_dir) if not checkpoint_path: raise ValueError("Couldn't find trained model at %s." % self._model_dir) export_dir = get_timestamped_export_dir(export_dir_base) # TODO(soergel): Consider whether MonitoredSession makes sense here with tf_session.Session() as session: saver_for_restore = estimator_spec.scaffold.saver or saver.Saver( sharded=True) saver_for_restore.restore(session, checkpoint_path) # TODO(b/36111876): replace legacy_init_op with main_op mechanism # pylint: disable=protected-access local_init_op = ( estimator_spec.scaffold.local_init_op or monitored_session.Scaffold._default_local_init_op()) # pylint: enable=protected-access # Perform the export builder = saved_model_builder.SavedModelBuilder(export_dir) builder.add_meta_graph_and_variables( session, [tag_constants.SERVING], signature_def_map=signature_def_map, assets_collection=ops.get_collection( ops.GraphKeys.ASSET_FILEPATHS), legacy_init_op=local_init_op) builder.save(as_text) # Add the extra assets if assets_extra: assets_extra_path = os.path.join( compat.as_bytes(export_dir), compat.as_bytes('assets.extra')) for dest_relative, source in assets_extra.items(): dest_absolute = os.path.join( compat.as_bytes(assets_extra_path), compat.as_bytes(dest_relative)) dest_path = os.path.dirname(dest_absolute) gfile.MakeDirs(dest_path) gfile.Copy(source, dest_absolute) return export_dir
def main(argv): # read line corresponding to task with gfile.Open(FLAGS.task_params_file, 'r') as f: for _ in range(FLAGS.taskid + 1): line = f.readline() line = line[:-1] # Remove \n from end. print(line) # get task parameters by parsing the lines line_split = line.split(';') cell_idx = line_split[0] cell_idx = cell_idx[1:-1].split(',') cell_idx = int(cell_idx[0]) file_list = gfile.ListDirectory(FLAGS.src_dir) cell_file = file_list[cell_idx] print('Cell file %s' % cell_file) nsub = int(line_split[1]) projection_type = line_split[2] lam_proj = float(line_split[3]) # copy data dst = os.path.join(FLAGS.tmp_dir, cell_file) if not gfile.Exists(dst): print('Started Copy') src = os.path.join(FLAGS.src_dir, cell_file) if not gfile.IsDirectory(FLAGS.tmp_dir): gfile.MkDir(FLAGS.tmp_dir) gfile.Copy(src, dst) print('File copied to destination') else: print('File exists') # load stimulus, response data try: data = sio.loadmat(dst) trainMov_filterNSEM = data['trainMov_filterNSEM'] testMov_filterNSEM = data['testMov_filterNSEM'] trainSpksNSEM = data['trainSpksNSEM'] testSpksNSEM = data['testSpksNSEM'] mask = data['mask'] neighbor_mat = su_model.get_neighbormat(mask, nbd=1) trainMov_filterWN = data['trainMov_filterWN'] testMov_filterWN = data['testMov_filterWN'] trainSpksWN = data['trainSpksWN'] testSpksWN = data['testSpksWN'] # get NSEM stimulus and resposne stimulus_WN = np.array(trainMov_filterWN.transpose(), dtype='float32') response_WN = np.array(np.squeeze(trainSpksWN), dtype='float32') stimulus_NSEM = np.array(trainMov_filterNSEM.transpose(), dtype='float32') response_NSEM = np.array(np.squeeze(trainSpksNSEM), dtype='float32') print('Prepared data') # Do fitting # set random seed. np.random.seed(23) print('Made partitions') # Do fitting # WN data ifrac = 0.8 tms_train_WN = np.arange(0, np.floor(stimulus_WN.shape[0] * ifrac)).astype(np.int) tms_test_WN = np.arange(np.floor(stimulus_WN.shape[0] * ifrac), 1 * np.floor(stimulus_WN.shape[0] * 1)).astype(np.int) # NSEM data ifrac = 0.8 tms_train_NSEM = np.arange(0, np.floor(stimulus_NSEM.shape[0] * ifrac)).astype(np.int) tms_test_NSEM = np.arange(np.floor(stimulus_NSEM.shape[0] * ifrac), 1 * np.floor(stimulus_NSEM.shape[0] * 1)).astype(np.int) # Give filename ss = str(cell_idx) save_filename = os.path.join(FLAGS.save_path, 'Cell_%s_nsub_%d_%s_%.3f_jnt.pkl' % (ss, nsub, projection_type, lam_proj)) save_filename_partial = os.path.join(FLAGS.save_path_partial, 'Cell_%s_nsub_%d_%s_%.3f_jnt.pkl' % (ss, nsub, projection_type, lam_proj)) ## Do fitting if not gfile.Exists(save_filename): # Fit SU on WN print('Fitting started on WN') op = su_model.Flat_clustering_jnt(stimulus_WN, np.expand_dims(response_WN, 1), nsub, tms_train_WN, tms_test_WN, steps_max=10000, eps=1e-9, projection_type=projection_type, neighbor_mat=neighbor_mat, lam_proj=lam_proj, eps_proj=0.01, save_filename_partial= save_filename_partial, fitting_phases=[1]) _, _, alpha, lam_log_wn, lam_log_test_wn, fitting_phase, fit_params_wn = op print('WN fit done') # Fit on NSEM op = su_model.fit_scales(stimulus_NSEM[tms_train_NSEM, :], np.expand_dims(response_NSEM[tms_train_NSEM], 1), stimulus_NSEM[tms_test_NSEM, :], np.expand_dims(response_NSEM[tms_test_NSEM], 1), Ns=nsub, K=fit_params_wn[0][0], b=fit_params_wn[0][1], params=fit_params_wn[0][2], lr=0.01, eps=1e-9) k_nsem, b_nsem, nl_params_nsem, lam_log_nsem, lam_log_test_nsem = op # Collect results and save fit_params = fit_params_wn + [[k_nsem, b_nsem, nl_params_nsem]] lam_log = [lam_log_wn, np.array(lam_log_nsem)] lam_log_test = [lam_log_test_wn, np.array(lam_log_test_nsem)] save_dict = {'lam_log': lam_log, 'lam_log_test': lam_log_test, 'fit_params': fit_params, 'mask': mask} pickle.dump(save_dict, gfile.Open(save_filename, 'w')) print('Saved results') except: print('Error')