def main(argv): parser = argparse.ArgumentParser( description="Update wwe db with rosbag info files", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('action', help='\ action to perform.[all|single]\n\ all - to updated db with all info.json files found recursively in the given path.\n\ single - to update db with a single info json file.\n\ flush - removes all entries from the db.\n\ find - find and print details of the entry in db containing the provided filename.' ) parser.add_argument( '-d', '--dataset_path', help='path to wwe raw dataset dir to search for info.json files') parser.add_argument( '-i', '--info', help='info.json file whose content has to be added to the existing db') parser.add_argument('-f', '--filename', help='filename to search in db') parser.add_argument('-v', '--verbose', action='store_true', help='enable verbose outputs') args = parser.parse_args() # Initialized logger Logger.init(level=Logger.LEVEL_INFO, name="updateDB") if args.verbose: Logger.setLevel(Logger.LEVEL_DEBUG) if args.action == 'all': if args.dataset_path is not None: update_all_info(args.dataset_path) else: Logger.error("Dataset path not provided!") return elif args.action == 'single': if args.info is not None: update_single_info(args.info) else: Logger.error("info.json file not provided to update db!") return elif args.action == "flush": flushdb() elif args.action == "find": if args.filename is not None: findFilename(args.filename) else: Logger.error("filename not provided to search db") return else: Logger.error("unknown action - " + args.action)
def main(argv): parser = argparse.ArgumentParser(description="Calculate distance in km for given gps file") parser.add_argument("filepath", help="path to gps json file") parser.add_argument("-v", "--verbose", action="store_true", help="enable verbose outputs") args = parser.parse_args() Logger.init(level=Logger.LEVEL_INFO, name="GpsDistance") if args.verbose: Logger.setLevel(Logger.LEVEL_DEBUG) if args.filepath is None: print("file not specified!") return if not os.path.exists(args.filepath): print("file doesn't exist - " + args.filepath) return doAction(args.filepath)
def main(opt): exit_code = 0 opt.hostname = os.uname()[1] opt.running = True # cudnn if opt.device > -1: os.environ["CUDA_VISIBLE_DEVICES"] = str(opt.device) device = torch.device('cuda:0') else: device = torch.device('cpu') # seed if opt.manualSeed is None: opt.manualSeed = random.randint(1, 10000) print("Random Seed: ", opt.manualSeed) random.seed(opt.manualSeed) torch.manual_seed(opt.manualSeed) if opt.device > -1: torch.cuda.manual_seed_all(opt.manualSeed) ################################################################################################################## # Data ################################################################################################################## # load config data_opt = load_config(os.path.join('config', opt.corpus, opt.config, 'corpus.yaml')) opt.update(data_opt) # load data corpus = Corpus(opt.dataroot) # split trainset, valset, testset = corpus.split(opt.config, opt.min_freq) # dataloaders # -- train train_loader = Iterator(trainset, opt.batch_size, repeat=False, sort_within_batch=True, device=device) # -- val ts_val = sorted(list(set([ex.timestep for ex in valset]))) val_loaders = [] for t in ts_val: val_t = Dataset(valset.examples, valset.fields, filter_pred=lambda x: x.timestep == t) val_t.sort_key = lambda x: len(x.text) val_t_loader = Iterator(val_t, opt.batch_size, train=False, device=device) val_loaders.append((t, val_t_loader)) val_loaders = OrderedDict(val_loaders) # -- test ts_tests = sorted(list(set([ex.timestep for ex in testset]))) test_loaders = [] if opt.config == 'prediction': for t, loader in val_loaders.items(): test_loaders.append((t, loader)) for t in ts_tests: test_t = Dataset(testset.examples, testset.fields, filter_pred=lambda x: x.timestep == t) test_t.sort_key = lambda x: len(x.text) test_t_loader = Iterator(test_t, opt.batch_size, train=False, device=device) test_loaders.append((t, test_t_loader)) test_loaders = OrderedDict(test_loaders) # opt opt.ntoken = corpus.vocab_size opt.padding_idx = corpus.pad_idx opt.nts = max(ex.timestep for ex in trainset) + 1 opt.nwords = sum(len(ex.text) for ex in trainset) # print info print('Vocab size: {}'.format(opt.ntoken)) print(f'{len(trainset)} training documents with {opt.nwords} tokens on {opt.nts} timesteps') ################################################################################################################## # Model ################################################################################################################## # load config model_opt = load_config(os.path.join('config', opt.corpus, opt.config, '{}.yaml'.format(opt.model))) opt.update(model_opt) # buid model print('Building model...') model = lm_factory(opt).to(device) ################################################################################################################## # Optimizer ################################################################################################################## optimizer = get_lm_optimizer(model, opt) if 'lr_scheduling' in opt: if opt.lr_scheduling == 'linear': opt.min_lr == 0 opt.niter = opt.niter_burnin + opt.niter_scheduling niter = opt.niter_scheduling lr_scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda i: max(0, (niter - i) / niter)) if opt.lr_scheduling == 'reduce_on_plateau': assert opt.min_lr > 0 lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=opt.patience, factor=opt.lr_decay) else: lr_scheduler = None ################################################################################################################## # Log ################################################################################################################## opt.xproot = os.path.join(opt.xproot, opt.corpus, opt.config, opt.model, opt.name) print(f'New experiment logged at {opt.xproot}') logger = Logger(opt.xproot) logger.init(opt) ################################################################################################################## # Trainning ################################################################################################################## print('Training...') pb = trange(opt.niter, ncols=0) ppl_eval = None finished = False itr = -1 try: while not finished: for batch in train_loader: itr += 1 model.train() # io text = batch.text[0][:-1] target = batch.text[0][1:] timestep = batch.timestep # closure log_train = model.closure(text, target, timestep, optimizer, opt) # eval if itr > 0 and itr % opt.niter_checkpoint == 0: model.eval() with torch.no_grad(): score, log_val = evaluate_lm(model, val_loaders, opt) # checkpoint log_train['lr'] = optimizer.param_groups[0]['lr'] logger.log(itr, 'train', log_train) logger.log(itr, 'val', log_val) logger.checkpoint(itr) # reduce_on_plateau lr scheduling if lr_scheduler and itr >= opt.niter_burnin and opt.lr_scheduling == 'reduce_on_plateau': lr_scheduler.step(score) lr = optimizer.param_groups[0]['lr'] if lr < opt.min_lr: finished = True break # progress bar pb.update(opt.niter_checkpoint) pb.set_postfix(chkpt=logger.chkpt, loss=log_train['loss'], score=score, lr=lr) # other lr scheduling if lr_scheduler and itr >= opt.niter_burnin and opt.lr_scheduling != 'reduce_on_plateau': lr_scheduler.step() lr = optimizer.param_groups[0]['lr'] if lr < opt.min_lr: finished = True except KeyboardInterrupt: exit_code = 130 pb.close() print('Evaluating...') model.eval() with torch.no_grad(): _, log_val = evaluate_lm(model, val_loaders, opt) _, results = evaluate_lm(model, test_loaders, opt) log_train['lr'] = optimizer.param_groups[0]['lr'] logger.log(itr, 'train', log_train) logger.log(itr, 'val', log_val) logger.log(itr, 'test', results) logger.checkpoint(itr) logger.terminate(model, optimizer) return exit_code
def main(argv): parser = argparse.ArgumentParser(description="Extracts raw data from rosbag files") parser.add_argument('rosbag', help='Rosbag file to extract data') parser.add_argument('datatype', nargs='+', help='Type of data to be extracted. supported option include [all|info|images|caminfo|gps]') parser.add_argument('--encode', help='[raw|jpeg] when provided with datatype=images, this option extracts images in the corresponding format') parser.add_argument('--hist', action='store_true', help='when provided with datatype=images, this option generates image histograms') parser.add_argument('-o', '--output', help='Dir to dump extracted data') parser.add_argument('-v', '--verbose', action='store_true', help='enable verbose outputs') args = parser.parse_args() # Initialized logger Logger.init(level=Logger.LEVEL_INFO, name="extract_data") if args.verbose: Logger.setLevel(Logger.LEVEL_DEBUG) inputfile = args.rosbag # set ouput dir if provided else extract data in current dir if args.output is not None: basepath = args.output else: basepath = "./" # all sensor data is extracted into <base_path>/extracted dir" outputdir = basepath + "/extracted" Logger.debug('processing ' + inputfile) Logger.debug('extracting to ' + outputdir) # create output dir it not existing for dumping data if not os.path.exists(outputdir): os.makedirs(outputdir) # check if bagfile exists if not os.path.exists(inputfile): Logger.error("File not found: " + inputfile) return bag = None # extract specified datatypes for datatype in args.datatype: if datatype == 'videos': extract_videos(basepath) else: # open bagfile if bag is None: bag = rosbag.Bag(inputfile) if datatype == 'images': hist = False if args.hist: hist = True if args.encode is not None and args.encode == "raw": extract_images(bag, outputdir, img_type="raw", hist=hist) elif args.encode is not None and args.encode == "jpeg": extract_images(bag, outputdir, img_type="jpeg", hist=hist) else: extract_images(bag, outputdir) elif datatype == 'caminfo': extract_cam_info(bag, outputdir) elif datatype == 'imu': extract_imu_info(bag, outputdir) elif datatype == 'gps': extract_gps_info(bag, outputdir) elif datatype == 'info': extract_bag_info(bag, basepath) elif datatype == 'lidar': extract_lidar_points(bag, outputdir) elif datatype == 'all': extract_all(bag, basepath) # close bag and exit if bag is not None: bag.close()