def run(self): if os.path.exists(self.model_dir): latest_step = utils.get_latest_file(self.model_dir) self.agent.load(self.model_dir, latest_step) self.evaluate()
def convert_csv(cursor, mag_id, xml_template_file, output_file): logging.info("Converting CSV") latest_file = get_latest_file(DESTINATION) root, regels = get_xml_file_insert(xml_template_file) df = pd.read_csv(latest_file, sep=';') articles = ', '.join('\'{}\''.format(str(row['Uw artikelnr.']).strip()) for _, row in df.iterrows()) sql_query = 'SELECT ArtCode, ArtIsPartijRegistreren, KingSystem.tabArtikelPartij.ArtPartijNummer as ArtPartijNummer \ from KingSystem.tabArtikel LEFT JOIN KingSystem.tabArtikelPartij \ ON KingSystem.tabArtikel.ArtGid=KingSystem.tabArtikelPartij.ArtPartijArtGid \ WHERE (KingSystem.tabArtikelPartij.ArtPartijIsGeblokkeerdVoorVerkoop = 0 OR KingSystem.tabArtikel.ArtIsPartijRegistreren = 0) AND \ KingSystem.tabArtikel.ArtCode in ({})'.format(articles) # Fetch partij information cursor.execute(sql_query) rows = cursor.fetchall() articles = {} for row in rows: articles[row.ArtCode] = Article(row.ArtCode, row.ArtPartijNummer, row.ArtIsPartijRegistreren) print(len(rows), len(df)) for i, row in df.iterrows(): art_id = row['Uw artikelnr.'].strip() article = articles[art_id] # amount = row['Totaal excl. inslag'] amount = row['Aantal eenheden'] if article.partijregistratie: add_xml(regels, art_id, str(amount).strip(), mag_id, article.partijnummer) else: add_xml(regels, art_id, str(amount).strip(), mag_id) write_xml(output_file, root) os.remove(latest_file)
def run(self): if os.path.exists(self.model_dir): latest_step = utils.get_latest_file(self.model_dir) self.agent.load(self.model_dir, latest_step) else: raise ValueError('Could not reload weights!') self.evaluate()
def visualize_latest_from_visualization_folder(folder='./visualizations/', file=None): if file is None: file = ut.get_latest_file(folder, filter=r'.*\d+\.txt$') ut.print_info('Encoding file: %s' % file.split('/')[-1]) data = np.loadtxt(file) # [0:360] fig = plt.figure() vi.visualize_encodings(data, fast=fast, fig=fig, interactive=True) fig.suptitle(file.split('/')[-1]) fig.tight_layout() plt.show()
def visualize_from_checkpoint(checkpoint, epoch=None): assert os.path.exists(checkpoint) FLAGS.load_from_checkpoint = checkpoint file_filter = r'.*\d+\.txt$' if epoch is None else r'.*e\|%d.*' % epoch latest_file = ut.get_latest_file(folder=checkpoint, filter=file_filter) print(latest_file) ut.print_info('Encoding file: %s' % latest_file.split('/')[-1]) data = np.loadtxt(latest_file) fig = plt.figure() fig.set_size_inches(fig.get_size_inches()[0] * 2, fig.get_size_inches()[1] * 2) entity = EncodingVisualizer(fig, data) # fig.tight_layout() plt.show()
def process_file( output_handler, data_path, target, file_name=None, ndx=None ): """ Function to open and process a file of CSV summary reports """ if not file_name: file_name = get_latest_file(data_path) doc_gen = target_module.DocGenerator(file_name) data_file = join(data_path, file_name) if not ndx: ndx = target.DEFAULT_INDEX with open(data_file, encoding=target.DEFAULT_FILE_ENCODING) as csv_read: csv_reader = csv.DictReader(csv_read) output_handler( ES, doc_gen.generate(csv_reader, ndx) ) return
lock = lock.replace("@", " @LOCATION: ") funcs[func] = PathCall(lock) return funcs if __name__ == '__main__': parser = optparse.OptionParser() parser.add_option("--promo", "-p", help="list of promo log", default=None) (opts, args) = parser.parse_args() # load other targets if opts.promo: BOOTSTRAP.update(load_promo_log(opts.promo).keys()) utils.install_pdb() # pick the latest one from /tmp log_d = utils.get_latest_file("/tmp/fss-*/") if log_d is None: # second choice log_d = os.path.join(ROOT, "data", "sample-fss-output") # perfered one if len(args) >= 1: log_d = args[0] # enable trace output on unittest if not "unit" in log_d: dbg.quiet(["trace"]) analyze_lock_promo(log_d)
def get_model(input_shape, weights_dir, resume, bayesian, vnet, prior_std, kernel_size, activation, padding, alpha, num_gpus, scale_factor=1, weights_path=None): """Loads or creates model. If a weights path is specified, loads from that path. Otherwise, loads the most recently modified model. """ os.makedirs(weights_dir + "/bayesian", exist_ok=True) os.makedirs(weights_dir + "/dropout", exist_ok=True) # Sets variables for bayesian model. if bayesian: checkpoint_path = (weights_dir + "/bayesian/bayesian-{epoch:02d}" "-{val_acc:.3f}-{val_loss:.0f}.h5") if weights_path: latest_weights_path = weights_path else: latest_weights_path = get_latest_file(weights_dir + "/bayesian") net = bayesian_vnet if vnet else bayesian_unet # Sets variables for dropout model. else: checkpoint_path = (weights_dir + "/dropout/dropout-{epoch:02d}" "-{val_acc:.3f}-{val_loss:.2f}.h5") if weights_path: latest_weights_path = weights_path else: latest_weights_path = get_latest_file(weights_dir + "/dropout") net = dropout_vnet if vnet else dropout_unet # Loads or creates model. if latest_weights_path and resume: model = load_model(input_shape, latest_weights_path, net) else: model = net(input_shape, kernel_size=kernel_size, activation=activation, padding=padding, prior_std=prior_std) # Prints model summary. model.summary(line_length=127) # Converts to multi-gpu model if applicable. if num_gpus > 1: model = multi_gpu_model(model, gpus=num_gpus) # Sets loss function. if bayesian: loss = variational_free_energy_loss(model, scale_factor, alpha) else: loss = binary_crossentropy # Compiles model with Adam optimizer. model.compile(loss=loss, optimizer=Adam(), metrics=["accuracy"]) return model, checkpoint_path
lock = lock.strip() lock = lock.replace("@", " @LOCATION: ") funcs[func] = PathCall(lock) return funcs if __name__ == '__main__': parser = optparse.OptionParser() parser.add_option("--promo", "-p", help="list of promo log", default=None) (opts, args) = parser.parse_args() # load other targets if opts.promo: BOOTSTRAP.update(load_promo_log(opts.promo).keys()) utils.install_pdb() # pick the latest one from /tmp log_d = utils.get_latest_file("/tmp/fss-*/") if log_d is None: # second choice log_d = os.path.join(ROOT, "data", "sample-fss-output") # perfered one if len(args) >= 1: log_d = args[0] # enable trace output on unittest if not "unit" in log_d: dbg.quiet(["trace"]) analyze_lock_promo(log_d)
data = np.loadtxt(latest_file) fig = plt.figure() fig.set_size_inches(fig.get_size_inches()[0] * 2, fig.get_size_inches()[1] * 2) entity = EncodingVisualizer(fig, data) # fig.tight_layout() plt.show() fast = True if __name__ == '__main__': cwd = os.getcwd() # cwd = '/mnt/code/vd/TensorFlow_DCIGN/tmp/pred.16c3s2_32c3s2_32c3s2_16c3_f80_f8__i_grid.28c.4' latest = ut.get_latest_file(cwd, filter=r'.*_suf\.encodings\.npy$') print(latest) data = np.load(latest).item() # print(type(data)) # i = data.item() # print(type(i)) # print(i.shape) # print(data['enc']) # print(data) x = data['enc'] # print(x) fig = vis.plot_encoding_crosssection(x, '',
def get_evaluation(path): f = ut.get_latest_file(path, filter=r'.*.npy$') print(f) dict = np.load(f).item() return dict
def run(self): init_env = None keys_to_sample = random.sample(list(self.train_envs), self.num_train_envs) sampled_train_envs = { key: self.train_envs[key] for key in keys_to_sample } # Better way to access first elem of OrderedDict? for env_idx, env in sampled_train_envs.items(): init_env = env break episode, episode_reward, episode_step, done = [0] * self.num_train_envs, [0] * self.num_train_envs, \ [0] * self.num_train_envs, [True] * self.num_train_envs obs, next_obs = [init_env.reset()] * self.num_train_envs, [ init_env.reset() ] * self.num_train_envs start_time = time.time() train_recording = False env_to_rec = 0 if self.reload_weights and os.path.exists(self.model_dir): # Continue training try: latest_step = utils.get_latest_file(self.model_dir) self.agent.load(self.model_dir, latest_step) except: print('Could not reload weights!') while self.step[0] < self.cfg.num_train_steps: if self.resample_envs and self.step[ 0] > 0 and self.step[0] % self.env_resample_rate == 0: keys_to_sample = random.sample(list(self.train_envs), self.num_train_envs) sampled_train_envs = { key: self.train_envs[key] for key in keys_to_sample } for env_idx, (env_tag, env) in enumerate(sampled_train_envs.items()): episode_step[env_idx] = 0 while (episode_step[env_idx] <= env._max_episode_steps - 1): if not train_recording and env_idx == env_to_rec and self.step[ env_idx] % self.train_vid_interval == 0: train_recording = True self.train_video_recorder.init(enabled=True) if done[env_idx] or (episode_step[env_idx] >= env._max_episode_steps - 1): if self.step[env_idx] > 0: self.logger.log('train/duration', time.time() - start_time, self.step[env_idx]) start_time = time.time() if self.step[ env_idx] > 0 and env_idx == env_to_rec and train_recording: file_name = str(self.step[env_idx]) + '_' + env_tag self.train_video_recorder.save(f'{file_name}.mp4') self.train_video_recorder.frames = [] train_recording = False env_to_rec = random.randint( 0, len(sampled_train_envs) - 1) # Evaluate agent periodically if env_idx == 0 and episode[ env_idx] % self.cfg.eval_frequency == 0: # Evaluate an env from training self.logger.log('eval_seen/episode', episode[env_idx], self.step[env_idx]) eval_env = self.train_envs[random.sample( list(self.train_envs), 1)[0]] self.evaluate(phase='seen', eval_env=eval_env) # Evaluate an unseen env self.logger.log('eval_unseen/episode', episode[env_idx], self.step[env_idx]) eval_env = self.eval_envs[random.sample( list(self.eval_envs), 1)[0]] self.evaluate(phase='unseen', eval_env=eval_env) if episode[env_idx] % self.cfg.ckpt_frequency == 0: self.agent.save(self.model_dir, episode[env_idx]) self.logger.log('train/episode_reward', episode_reward[env_idx], self.step[env_idx]) obs[env_idx] = env.reset() done[env_idx] = False episode_reward[env_idx] = 0 episode[env_idx] += 1 self.logger.log('train/episode', episode[env_idx], self.step[env_idx]) self.logger.log('train/env_idx', env_tag, self.step[env_idx]) # sample action for data collection if self.step[env_idx] < self.cfg.num_seed_steps: action = env.action_space.sample() else: with utils.eval_mode(self.agent): action = self.agent.act(obs[env_idx], sample=True) next_obs[env_idx], reward, done[env_idx], _ = env.step( action) if train_recording and env_idx == env_to_rec: self.train_video_recorder.record(env) # allow infinite bootstrap done[env_idx] = float(done[env_idx]) done_no_max = 0 if episode_step[ env_idx] + 1 == env._max_episode_steps - 1 else done[ env_idx] episode_reward[env_idx] += reward self.replay_buffer.add(env_idx, obs[env_idx], action, reward, next_obs[env_idx], done[env_idx], done_no_max) obs[env_idx] = next_obs[env_idx] episode_step[env_idx] += 1 self.step[env_idx] += 1 # Run training update if self.step[env_idx] >= self.cfg.num_seed_steps: #print('Running train update') for _ in range(self.cfg.num_train_iters): self.agent.update(self.replay_buffer, self.num_train_envs, self.logger, self.step[env_idx], env_tag, env_idx) # At the end of each episode, log self.logger.dump( self.step[env_idx], save=(self.step[env_idx] > self.cfg.num_seed_steps), ty='train')