def init(): global api api = config.create_twitter_api() logger.info('Bot Started: %s' % datetime.utcnow()) config.load_cfg() imgpicker.init_img_list() users.init_user_list()
def main(): dirname = sys.argv[1] image_number = int(sys.argv[2]) (model_dir, img_dir, width, height, num_images, ensemble_size, max_levels, \ sampled, assignments, average_img) = config.load_cfg(dirname) init(model_dir, img_dir) print("image_number = %s, num_images = %s"\ % (str(image_number), str(num_images))) test_data, network_data = kmeans2d.assignment_data_to_test_data(\ assignments, image_number, num_images, average_img) input_map = np.zeros((height, width, 3), dtype = np.float64) image_out = np.zeros((height, width, 3), dtype = np.float64) thread_data = [(data, test_data, model_dir) for data in network_data] start_time = time.clock() pool = ThreadPool(8) results = pool.map(predict_thread, thread_data) pool.close() pool.join() for result in results: # predictions (level, network_id, start, count, predictions, test_out) = result kmeans2d.predictions_to_image(image_out, test_out, predictions) # debug data prediction_out = np.zeros((height, width, 3), dtype = np.float64) update_input_map(level, network_id, width, height, test_data,\ [level, network_id, start, count], input_map) kmeans2d.predictions_to_image(prediction_out, test_out, predictions) prediction_file_name = "render_images/prediction_%04d_%04d.png" %\ (level, network_id) misc.imsave(prediction_file_name, prediction_out); image_out = np.divide(image_out, float(ensemble_size)) image_out = np.clip(255.0 * image_out, 0.0, 255.0) end_time = time.clock() image_file_name = "render_images/" + str(sys.argv[2]) + '.png' misc.imsave("render_images/render_input_map.png", input_map) misc.imsave(image_file_name, image_out) print("time = %5.5f" % (end_time - start_time)) print("saved %s" % (image_file_name))
def generate(load_model_path, max_sentences): config.cfg = config.load_cfg(load_model_path) config.cfg["load_model_path"] = load_model_path p_reader = PartialsReader(max_sentences=max_sentences) p_reader.load_dict(config.cfg["dictionary_name"]) p_reader.read_sentences(config.cfg["path"]["continuation"]) import model m = model.Model(cfg=config.cfg) m.build_forward_prop() m.load_model(config.cfg["load_model_path"]) #Revert dictionary for generation reverted_dict = dict([(y, x) for x, y in list(p_reader.vocab_dict.items())]) m.generate(p_reader.id_sequences, reverted_dict)
def test(args): cfg = load_cfg(args.cfg) weight_path = args.wts img_path = args.im_path segnet = SegNet().float().cuda() segnet.load_state_dict(torch.load(weight_path)) segnet.eval() im = cv2.imread(img_path).transpose(2, 0, 1) im = torch.tensor(im[np.newaxis, :], dtype=torch.float).cuda() out = segnet(im) out = out.detach().cpu().numpy().transpose(0, 2, 3, 1) out = np.argmax(out, axis=3).astype(np.uint8)[0] out = out[:, :, np.newaxis] out = out * 20 cv2.imshow('f**k', out) cv2.waitKey(0)
#!/usr/bin/env python3 import config import sys from mbot import bot def usage_hint(): s = "\n" +\ "usage:\n" +\ " $ ./run.py <config>\n" +\ "\n" +\ "example:\n" +\ " $ ./run.py cfg/mb-bot.json" return s if __name__ == "__main__": if len(sys.argv) == 1: print(usage_hint()) sys.exit(0) cfg = config.load_cfg(sys.argv[1]) bot.run(cfg["TOKEN"])
def train(cfg_file): ## config cfg = load_cfg(cfg_file) ## logging FORMAT = '%(levelname)s %(filename)s: %(message)s' logging.basicConfig(level=logging.INFO, format=FORMAT, stream=sys.stdout) logger = logging.getLogger(__name__) ## data loader trainset = get_datasets(cfg.train.datasets, 'train') valset = get_datasets(cfg.val.datasets, 'val') trainloader = DataLoader(trainset, batch_size = cfg.train.batch_size, shuffle = True, num_workers = 4, drop_last = True) valloader = DataLoader(valset, batch_size = 4, shuffle = True, num_workers = 4, drop_last = True) ## network and checkpoint model = get_model(cfg).float().cuda() print(model) if cfg.model.class_weight is not None: weight = torch.Tensor(cfg.model.class_weight) # ignore some labels or set weight Loss = nn.CrossEntropyLoss(weight = weight).cuda() ## optimizer optimizer = Optimizer(model.parameters(), cfg) ## checkpoint save_path = cfg.out_path if not os.path.exists(save_path): os.makedirs(save_path) save_name = os.path.join(save_path, 'model.pytorch') if os.path.exists(save_name): return model_ckpts = os.listdir(save_path) its = [int(os.path.splitext(el)[0].split('_')[2]) for el in model_ckpts if el[:5] == 'model'] ckpt_max_it = 0 if len(its) > 0: ckpt_max_it = max(its) logger.info('resume from checkpoint iter: {}'.format(ckpt_max_it)) model_ckpt = os.path.join(save_path, ''.join(['model_iter_', str(ckpt_max_it), '.pytorch'])) model.load_state_dict(torch.load(model_ckpt)) optimizer.load_checkpoint(save_path, ckpt_max_it) start_it = ckpt_max_it + 1 ## multi-gpu model = nn.DataParallel(model, device_ids = None) ## train result = AttrDict({ 'train_loss': [], 'val_loss': [], 'cfg': cfg }) trainiter = iter(trainloader) for it in range(start_it, cfg.train.max_iter): model.train() optimizer.zero_grad() try: im, label = next(trainiter) if not im.shape[0] == cfg.train.batch_size: continue except StopIteration: trainiter = iter(trainloader) im, label = next(trainiter) im = im.cuda().float() num_class = cfg.model.num_class logits = model(im) label = resize_label(label, logits.shape[2:]) label = label.cuda().long().contiguous().view(-1, ) logits = logits.permute(0, 2, 3, 1).contiguous().view(-1, num_class) loss = Loss(logits, label) loss_value = loss.detach().cpu().numpy() result.train_loss.append(loss_value) loss.backward() optimizer.step() if it == 0: continue if it % 20 == 0: logger.info('iter: {}/{}, loss: {}'.format(it, cfg.train.max_iter, loss_value)) if it % cfg.val.valid_iter == 0: valid_loss, acc_clss, acc_all = val_one_epoch(model, Loss, valloader, cfg) result.val_loss.append(valid_loss) if it % cfg.train.snapshot_iter == 0: save_model_name = os.path.join(save_path, ''.join(['model_iter_', str(it), '.pytorch'])) logger.info('saving snapshot to: {}'.format(save_path)) torch.save(model.module.state_dict(), save_model_name) optimizer.save_checkpoint(save_path, it) logger.info('training done') save_name = os.path.join(save_path, 'model.pytorch') logger.info('saving model to: {}'.format(save_name)) model.cpu() torch.save(model.module.state_dict(), save_name) with open(save_path + '/result.pkl', 'wb') as fw: pickle.dump(result, fw) while True: try: im, label = next(trainiter) except StopIteration: break print('everything done')
help='Specify the path to the config.yml file') args = parser.parse_args() writedb = args.writedb diffs = args.diffs refresh = args.refresh wStatus = args.wStatus updateConfig = args.updateConfig outputErrorDF = args.outputErrorDF debug_flag = "DEBUG" if args.debug_flag else "CRITICAL" cmd_log_level = args.logger_level config_path = Path(args.config_path.replace('"',"").strip()) / "config.yml" import config config.load_cfg(config_path) cfg = config.cfg wStatus_suffix = "_wStatus" if wStatus else "" global export_path global archive_path global error_path global log_path export_path = cfg['informer']['export_path' + wStatus_suffix] archive_path = cfg['ccdw']['archive_path' + wStatus_suffix] error_path = cfg['ccdw']['error_path'] log_path = cfg['ccdw']['log_path'] prefix = cfg['informer']['prefix']
from config import load_cfg from slowfast.models import SlowFast cfg = load_cfg('./test_config.yaml') model = SlowFast(cfg)
global writedb global diffs global refresh global log global cfg run_datetime = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S%f") def timestamp(): return (datetime.datetime.now().isoformat()) import config config.load_cfg() cfg = config.cfg #with open("//helix/divisions/IERG/Data/config.yml","r") as ymlfile: # cfg = yaml.load(ymlfile) parser = argparse.ArgumentParser(description='Import CCDW data') parser.add_argument( '--nodb', dest='writedb', action='store_false', default=True, help='Do not write to database (default: write to database)') parser.add_argument( '--diffs', dest='diffs',
def main(): cfg = config.cfg timestamp = time.strftime('%Y-%m-%d--%H_%M_%S') sys.stdout = Logger(timestamp) # region Parameter arguments parser = argparse.ArgumentParser() parser.add_argument('--mode', choices={'train', 'chat', 'test'}, default='train', help="mode. if not specified, it's in the train mode") parser.add_argument('--cornell', action='store_true', help="use the cornell movie dialogue corpus") parser.add_argument( '--conversations', help="limit the number of conversations used in the dataset") parser.add_argument( '--model', help= 'specify name (timestamp) of a previously used model. If none is provided then the ' 'newest model available will be used.') parser.add_argument( '--load_iter', help= 'if you do not want to use an intermediate version of the trained network, specify the ' 'iteration number here') parser.add_argument('test_file', type=str, nargs='?') parser.add_argument( '--test_conversations', help="limit the number of test conversations used in the dataset") parser.add_argument( '--softmax', action='store_true', help='use standard softmax loss instead of sampled softmax loss') parser.add_argument( '--clear', action='store_true', help="delete all existing models to free up disk space") parser.add_argument( '--keep_prev', action='store_true', help='keep only the most recent version of the trained network') parser.add_argument( '--epochs', help= 'how many times the network should pass over the entire dataset. Note: Due to random ' 'bucketing, this is an approximation.') parser.add_argument('--save_end', action='store_true', help='save the model ONLY at the end of training') parser.add_argument( '--processed_path', help='Specify if you want to use exisiting preprocessed our_data') parser.add_argument( '--no_save_at_exit', action='store_true', help='deactivate automatic model saving at keyboard interrupt') parser.add_argument('--combined_data', action='store_true', help='Both cornell data and our data will be used') args = parser.parse_args() if args.clear: shutil.rmtree(cfg['MODELS_PATH']) if not os.path.exists(cfg['MODELS_PATH']): os.makedirs(cfg['MODELS_PATH']) ########### load old config file if necessary ############ if args.model: # load the corresponding config file cfg = config.load_cfg(args.model) else: if args.mode == 'chat': # default mode: load the config of the last used model saved_models = [ name for name in os.listdir(cfg['MODELS_PATH']) if os.path.isdir(os.path.join(cfg['MODELS_PATH'], name)) ] saved_models = sorted(saved_models) # sorted in ascending order last_model = saved_models[-1] cfg = config.load_cfg(last_model) # pp = pprint.PrettyPrinter(indent=4) # pp.pprint(cfg) else: # create a new model config.adapt_to_dataset(args.cornell, args.combined_data) cfg['MODEL_NAME'] = timestamp directory = os.path.join(cfg['MODELS_PATH'], cfg['MODEL_NAME']) if not os.path.exists(directory): os.makedirs(directory) config.adapt_paths_to_model() ####### process other commmand line arguments ############## if args.conversations: cfg['MAX_TURNS'] = int(args.conversations) if args.test_conversations: cfg['TESTSET_SIZE'] = int(args.test_conversations) if args.softmax: cfg['STANDARD_SOFTMAX'] = args.softmax if args.keep_prev: cfg['KEEP_PREV'] = True if args.load_iter: args.load_iter = int(args.load_iter) if args.epochs: cfg['EPOCHS'] = int(args.epochs) if args.processed_path: cfg['PROCESSED_PATH'] = args.processed_path vocab_sizes_dict = pickle.load( open(os.path.join(cfg['PROCESSED_PATH'], "vocab_sizes"), "rb")) cfg.update(vocab_sizes_dict) if args.no_save_at_exit: cfg['SAVE_AT_EXIT'] = False # endregion ############## read our_data ################# if args.cornell: import cornell_data as data elif args.combined_data: import data_reader as data else: import our_data as data reader = data.Reader(cfg) # reader = data.Reader(cfg) if not os.path.isdir( cfg['PROCESSED_PATH'] ): # Not be done if we're in test mode, chat mode or continue training reader.prepare_raw_data() reader.process_data() print('Data ready!') ########### start using the actual model################## # create checkpoints folder if there isn't one already reader.make_dir(cfg['CPT_PATH']) bot = Chatbot(config=cfg, reader=reader) if args.mode == 'train': bot.train(args.save_end) elif args.mode == 'chat': bot.chat(args.load_iter) elif args.mode == 'test': bot.test_performance(args.test_file, args.load_iter)