def missing_message_data(path, channels=CHANNELS): for record in list_records(path): glog.info("reading records %s" % record) reader = RecordReader(record) for channel in channels: glog.info("has %d messages" % reader.get_messagenumber(channel)) if reader.get_messagenumber(channel) == 0: return True return False
def list_records(path): glog.info("in list_records:%s" % path) records = [] for (dirpath, _, filenames) in os.walk(path): glog.info('filenames: %s' % filenames) glog.info('dirpath %s' % dirpath) for filename in filenames: end_file = os.path.join(dirpath, filename) glog.info("end_files: %s" % end_file) if record_utils.is_record_file(end_file): records.append(end_file) return records
def sanity_check(input_folder, email_receivers=None): err_msg = None if missing_file(input_folder): err_msg = "One or more files are missing in %s" % input_folder elif parse_error(input_folder): err_msg = "Confige file cannot be parsed in %s" % input_folder elif missing_field(input_folder): err_msg = "One or more field is missing in Confige file %s" % input_folder elif missing_message_data(input_folder): err_msg = "Messages are missing in records of %s" % input_folder else: glog.info("%s Passed sanity check." % input_folder) return True if email_receivers: title = 'Error occurred during data sanity check' email_utils.send_email_error(title, {'Error': err_msg}, email_receivers) glog.error(err_msg) return False
def missing_file(path): vehicles = multi_vehicle_utils.get_vehicle(path) glog.info("vehicles %s" % vehicles) for vehicle in vehicles: # config file conf = os.path.join(path, vehicle, ConfFile) glog.info("vehicles conf %s" % conf) if os.path.exists(conf) is False: glog.error('Missing configuration file in %s' % vehicle) return True # record file glog.info("list of records:" % list_records(os.path.join(path, vehicle))) if len(list_records(os.path.join(path, vehicle))) == 0: glog.error('No record files in %s' % vehicle) return True return False
def missing_field(path): vehicles = multi_vehicle_utils.get_vehicle(path) glog.info("vehicles in missing field: %s" % vehicles) for vehicle in vehicles: conf_file = os.path.join(path, vehicle, ConfFile) glog.info("conf_file: %s" % conf_file) # reset for each vehicle to avoid overwrite pb_value = vehicle_config_pb2.VehicleConfig() conf = proto_utils.get_pb_from_text_file(conf_file, pb_value) glog.info("vehicles conf %s" % conf) if not check_vehicle_id(conf): return True # required field fields = [conf.vehicle_param.brake_deadzone, conf.vehicle_param.throttle_deadzone, conf.vehicle_param.max_acceleration, conf.vehicle_param.max_deceleration] for field in fields: if math.isnan(field): return True return False
if __name__ == '__main__': config = read_config() ham = HAM().to('cuda') flow_dataset = FlowDataset() flow_dataloader = DataLoader( flow_dataset, batch_size=eval(config['train']['BatchSize']), shuffle=eval(config['train']['Shuffle']) ) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(ham.parameters(), lr=1e-3, weight_decay=1e-5) for epoch in range(eval(config['train']['NumEpochs'])): log.info('Running Epoch #{}'.format(epoch + 1)) total_loss = 0 with tqdm(total = len(flow_dataset) // eval(config['train']['BatchSize'])) as pbar: for batch_id, sample in enumerate(flow_dataloader): pbar.update(1) optimizer.zero_grad() X, y = sample prediction = ham(X.type(torch.long)) loss = criterion(prediction, y.to('cuda')) total_loss += loss.item() loss.backward() optimizer.step() log.info('Epoch #{} ==> {:.4f}'.format(epoch+1, total_loss))
#flow_embeddings = np.zeros((10000, 256)) model = HierarchicalAttentionModel().to('cuda') model.load_state_dict( torch.load( wrap_path(config['ModelSavePath']).format(epoch=config['Checkpoint']) ) ) criterion = nn.CrossEntropyLoss() X, y = fetch_dataset() test_dataset = TensorDataset(torch.Tensor(X), torch.Tensor(y)) test_dataloader = DataLoader(test_dataset, batch_size=128) log.info('Running inference...') with torch.no_grad(): num_iteration = len(test_dataloader) // 128 with tqdm(total=num_iteration) as pbar: for batch_id, (test_X, test_y) in enumerate(test_dataloader): preds, flow_embedding = model(test_X.type(torch.long).to('cuda')) loss = criterion(preds, test_y.type(torch.long).to('cuda')) hit += (preds.argmax(dim=1) == test_y.type(torch.long).to('cuda')).sum().item() all_preds += [i.item() for i in preds.argmax(dim=1)] #flow_embeddings[batch_id*128:batch_id*128+flow_embedding.shape[0],] = flow_embedding.data.cpu().numpy() pbar.update(1) accuracy = 100 * (hit / len(test_dataset)) log.info('Accuracy : {:.2f}'.format(accuracy))
return parser.parse_args() def load_model(): ham = HAM().to('cuda') ham.load_state_dict(torch.load('ham.pt')) return ham.eval() if __name__ == '__main__': args = read_args() ham = load_model() with open('label_map.pkl', 'rb') as f: label_map = pickle.load(f) data = torch.Tensor(np.load(args.flow)).type( torch.long).unsqueeze(0).to('cuda') prediction = ham(data) scores = F.softmax(prediction, dim=1).squeeze() item_id = prediction.argmax().item() predicted_item_name = label_map[item_id] confidence = scores[item_id] log.info('Predicted => {} (Confidence {:.2f}%)'.format( predicted_item_name, confidence * 100))
if __name__ == '__main__': config = read_config('train') history = [] model = HierarchicalAttentionModel().to('cuda') criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=eval(config['LearningRate'])) X, y = fetch_dataset() num_split = eval(config['KFoldSplit']) kfold = StratifiedKFold(n_splits=num_split) for epoch in range(eval(config['NumEpoch'])): log.info(f'Running Epoch #{epoch+1}') total_train_loss, total_valid_loss = 0, 0 train_dataset = TensorDataset(torch.Tensor(X), torch.Tensor(y)) train_dataloader = DataLoader(train_dataset, batch_size=eval(config['BatchSize'])) train_loss = 0 num_train_iteration = len(train_dataset) // eval(config['BatchSize']) with tqdm(total=num_train_iteration) as pbar: for batch_id, (train_X, train_y) in enumerate(train_dataloader): pbar.update(1) optimizer.zero_grad() preds = model(train_X.type(torch.long).to('cuda'))