def __init_model(self, entry):
if entry == "train":
self.train_manager = DataManager(batch_size=self.batch_size,
tags=self.tags)
self.total_size = len(self.train_manager.batch_data)
data = {
"batch_size": self.train_manager.batch_size,
"input_size": self.train_manager.input_size,
"vocab": self.train_manager.vocab,
"tag_map": self.train_manager.tag_map,
}
self.save_params(data)
dev_manager = DataManager(batch_size=30, data_type="dev")
self.dev_batch = dev_manager.iteration()
self.model = BiLSTMCRF(
tag_map=self.train_manager.tag_map,
batch_size=self.batch_size,
vocab_size=len(self.train_manager.vocab),
dropout=self.dropout,
embedding_dim=self.embedding_size,
hidden_dim=self.hidden_size,
)
self.restore_model()
elif entry == "predict":
data_map = self.load_params()
input_size = data_map.get("input_size")
self.tag_map = data_map.get("tag_map")
self.vocab = data_map.get("vocab")
self.model = BiLSTMCRF(tag_map=self.tag_map,
vocab_size=input_size,
embedding_dim=self.embedding_size,
hidden_dim=self.hidden_size)
self.restore_model()
def __init__(self,
path_train,
definition,
path_test=None,
c_model_name=Classifier.cnn,
cv_folds=5,
num_epochs=100,
batch_size=8,
learning_rate=0.0003):
"""
Initializer for CNNClassification class.
:param path_train: path to the h5 file for the training dataset
(simulated data)
:param definition: Type of SSW definition to use. example: "CP07",
"U65"
:param path_test: path to the h5 file for the test dataset (real data)
:param c_model_name: CNN classifier name which is going to be used
example: "cnn", "cnn_max_pool"
:param cv_folds: number of folds for the cross-validation which is
used to evaluate the performance on the training dataset.
:param num_epochs: Number of epochs to train the model
:param batch_size: Batch size for Adam optimizer
:param learning_rate: Learning rate for Adam optimizer
"""
# Device configuration
device = torch.device('cuda:' +
os.getenv("CUDA_VISIBLE_DEVICES") if torch.cuda.
is_available() else 'cpu')
SetSeed().set_seed()
self.data_manager_train = DataManager(path_train)
if path_test:
self.data_manager_test = DataManager(path_test)
self.definition = definition
self.cv_folds = cv_folds
self.metric_txt = ["F1", "ROCAUC", "Accuracy"]
self.metrics = [f1_score, roc_auc_score, accuracy_score]
self.model_class_name = c_model_name
# Number of channels in the CNN - number of features to use
num_ts = 3
self.classifier = NeuralNetClassifier(
cnn_model.get_cnn_classes()[c_model_name](num_ts),
criterion=nn.CrossEntropyLoss,
max_epochs=num_epochs,
lr=learning_rate,
batch_size=batch_size,
device=device,
optimizer=torch.optim.Adam,
)
def main_model(self, entry):
"""
Model Initialization
"""
# The Training Process
if entry == "train":
# Training Process: read Training Data from DataManager
self.train_manager = DataManager(batch_size=self.batch_size,
data_type='train',
tags=self.tags)
self.total_size = len(self.train_manager.batch_data)
# Read the corresponding character index (vocab) and other hyper-parameters
data = {
"batch_size": self.train_manager.batch_size,
"input_size": self.train_manager.input_size,
"vocab": self.train_manager.vocab,
"tag_map": self.train_manager.tag_map,
}
save_params(data=data, path=self.model_path)
# Build BiLSTM-CRF Model
self.model = BiLSTMCRF(tag_map=self.train_manager.tag_map,
batch_size=self.batch_size,
vocab_size=len(self.train_manager.vocab),
dropout=self.dropout,
embedding_dim=self.embedding_size,
hidden_dim=self.hidden_size,
max_length=self.max_length)
# Evaluation Process: read Dev Data from DataManager
self.dev_size = DataManager(batch_size=1,
data_type="dev",
tags=self.tags).load_data()
self.dev_manager = DataManager(batch_size=int(self.dev_size),
data_type="dev",
tags=self.tags)
self.dev_batch = self.dev_manager.iteration()
# Restore model if it exists
self.restore_model()
# The Testing & Inference Process
elif entry == "predict":
data_map = load_params(path=self.model_path)
input_size = data_map.get("input_size")
self.tag_map = data_map.get("tag_map")
self.vocab = data_map.get("vocab")
self.model = BiLSTMCRF(tag_map=self.tag_map,
vocab_size=input_size,
dropout=0.0,
embedding_dim=self.embedding_size,
hidden_dim=self.hidden_size,
max_length=self.max_length)
self.restore_model()
def main_model(self, entry):
# The Training Process
if entry == "train":
# Training Process: read Training Data from DataManager
self.train_manager = DataManager(batch_size=self.batch_size, data_type='train', tags=self.tags)
self.total_size = len(self.train_manager.batch_data)
# Read the corresponding character index (vocab) and other hyper-parameters
saved_data = {
"batch_size": self.train_manager.batch_size,
"input_size": self.train_manager.input_size,
"char_vocab": self.train_manager.char_vocab,
"tag_map": self.train_manager.tag_map,
}
save_params(data=saved_data, path=self.model_path)
# Evaluation Process: read Dev Data from DataManager
self.dev_size = DataManager(batch_size=1, data_type="dev", tags=self.tags).load_char_data()
self.dev_manager = DataManager(batch_size=int(self.dev_size), data_type="dev")
self.dev_batch = self.dev_manager.iteration()
# Build BiLSTM-CRF Model
self.model = BiLSTMCRF(
tag_map=self.train_manager.tag_map,
batch_size=self.batch_size,
vocab_size=len(self.train_manager.char_vocab),
dropout=self.dropout,
word_num=self.word_num,
word_dim=self.word_dim,
char_num=self.char_num,
char_dim=self.char_dim,
hidden_dim=self.hidden_size,
)
# Restore model if it exists
self.restore_model()
# The Inference Process
elif entry == "predict":
data = load_params(path=self.model_path)
input_size = data.get("input_size")
self.tag_map = data.get("tag_map")
self.vocab = data.get("char_vocab")
self.model = BiLSTMCRF(
tag_map=self.tag_map,
vocab_size=input_size,
dropout=1.0,
word_num=self.word_num,
word_dim=self.word_dim,
char_num=self.char_num,
char_dim=self.char_dim,
hidden_dim=self.hidden_size,
)
self.restore_model()
def __init__(self):
data_manager = DataManager(inputArguments())
data_manager_2 = DataManager(inputArguments_2())
train_activeset = data_manager.get_datasets(
)['train'] # type: <class 'dataset.TransformableFullMetadataSubset'>, len: 42236
train_new_dataset = data_manager_2.get_datasets()['train']
test_activeset = data_manager.get_datasets()['validation']
test_new_dataset = data_manager_2.get_datasets()['validation']
train_set = []
for indx in range(len(train_activeset)):
train_set.append(train_activeset[indx])
known_indexes = []
with open('kn_indici.txt', 'r') as file:
for line in file:
known_indexes.append(int(line.strip('\n')))
known_set = []
for indx in known_indexes:
known_set.append(train_new_dataset[indx])
test_indexes = []
with open('test_indici_newdataset.txt', 'r') as file:
for line in file:
test_indexes.append(int(line.strip('\n')))
test_set = []
for indx in test_indexes:
test_set.append(test_new_dataset[indx])
grad_test_set = [] # solo per testare gradcam
for indx in range(len(test_activeset)):
test_set.append(test_activeset[indx])
# Solo per testare gradcam
grad_test_set.append(test_activeset[indx])
expl_loader = torch.utils.data.DataLoader(grad_test_set,
batch_size=1,
shuffle=False,
num_workers=2,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=32,
shuffle=False,
num_workers=2,
pin_memory=True)
self.tr = train_set
self.ts = test_loader
self.kn = known_set
self.exp = expl_loader
def init_datasets(self):
self.Positive_data_list,self.Negative_data_list=self.listData1(self.__Param["data_dir"])
if self.__Param["mode"] is "training":
self.DataManager_train_Positive = DataManager(self.Positive_data_list, self.__Param)
self.DataManager_train_Negative = DataManager(self.Negative_data_list, self.__Param)
elif self.__Param["mode"] is "testing":
self.DataManager_test_Positive = DataManager(self.Positive_data_list, self.__Param,shuffle=False)
self.DataManager_test_Negative = DataManager(self.Negative_data_list, self.__Param,shuffle=False)
elif self.__Param["mode"] is "savePb":
pass
else:
raise Exception('got a unexpected mode ')
def update_charts(by):
global chart_natn, chart_club, scatter
global ranking_histo
if not (np.issubdtype(data[by], int) or np.issubdtype(data[by], float)):
return no_update
else:
chart_natn, chart_club, scatter = DataManager().plot_altair(data,
by=by)
ranking_histo = DataManager().plot_histo(data, by=by)
return (chart_natn.to_html(), chart_club.to_html(), scatter.to_html(),
ranking_histo.to_html())
def __init_model(self, entry):
# 模型训练的参数准备
if entry == "train":
#创建训练数据集的管理对象
print(self.tags)
self.train_manager = DataManager(batch_size=self.batch_size, tags=self.tags)
print(self.train_manager.batch_data)
print(len(self.train_manager.batch_data))
self.total_size = len(self.train_manager.batch_data)
# print(self.train_manager.batch_data)
data = {
"batch_size": self.train_manager.batch_size,
"input_size": self.train_manager.input_size,
"vocab": self.train_manager.vocab,
"tag_map": self.train_manager.tag_map,
}
# 保存参数
self.save_params(data)
# 验证数据集的准备
# 创建验证数据集的管理对象
dev_manager = DataManager(batch_size=30, data_type="dev")
# 通过data_manager中的迭代器不断将创建的数据管理器对象赋值到dev_batch中,用于下面计算损失的函数
self.dev_batch = dev_manager.iteration()
# 模型的主体使用的是BiLSTM来进行语义编码，CRF用来约束各个标签
self.model = BiLSTMCRF(
tag_map=self.train_manager.tag_map,
batch_size=self.batch_size,
vocab_size=len(self.train_manager.vocab),
dropout=self.dropout,
embedding_dim=self.embedding_size,
hidden_dim=self.hidden_size,
)
# 加载恢复模型参数
self.restore_model()
# 模型用来预测的参数准备
elif entry == "predict":
data_map = self.load_params()
input_size = data_map.get("input_size")
self.tag_map = data_map.get("tag_map")
self.vocab = data_map.get("vocab")
# 这里创建一个模型对象model
self.model = BiLSTMCRF(
tag_map=self.tag_map,
vocab_size=input_size,
embedding_dim=self.embedding_size,
hidden_dim=self.hidden_size
)
self.restore_model()
def init_datasets(self):
if self.__Param["mode"] != "savePb":
self.image_list_train, self.mask_list_train = self.listData_train(
self.__Param["data_dir"])
self.image_list_valid, self.mask_list_valid = self.listData_val(
self.__Param["data_dir"])
# self.image_list_test, self.mask_list_test = self.listData_test(self.__Param["data_dir"])
self.DataManager_train = DataManager(self.image_list_train,
self.mask_list_train,
self.__Param)
self.DataManager_valid = DataManager(self.image_list_valid,
self.mask_list_valid,
self.__Param,
shuffle=False)
def __init__(self):
self.env_name = 'gaf-environment-v1.0'
self.reward_mult = 100000
self.cash = 100000
self.total_loss = 0.0
self.current_action = None
self.previous_action = None
self.clock = Clock()
self.dm = DataManager(self.clock)
self.pm = PositionManager(self.clock, self.dm, self.cash, 30)
self.benchmark = PositionManager(self.clock, self.dm, self.cash, 1)
self.symbols = self.dm.get_symbols()
self.action_space = gym.spaces.Discrete(3)
self.observation_space = gym.spaces.Box(low=-1,
high=1,
shape=(4, 30, 180))
self.symbols = None
self.final_cash_value = []
#self.print_intro()
self.avg_reward = 0
self.episodes_ran = 0
self.perm_symbols = [
self.dm.current_symbol,
]
self.returns = pd.DataFrame()
def main():
parser = ArgumentParser(
'Preprocess robot data vor DeepVO, This process is destructive.')
parser.add_argument(
'-d',
'--data',
type=str,
required=True,
help='Path to dataset (a folder with "images" and "poses" subfolders.)'
)
parser.add_argument('-f',
'--to-float',
required=False,
default=False,
action='store_true',
help='Convert images array to float')
parser.add_argument('-m',
'--mean-normalize',
required=False,
default=False,
action='store_true',
help='Subtract rgb mean from images')
parser.add_argument('-s',
'--show',
required=False,
default=False,
action='store_true',
help='Show the images')
parser.add_argument('-p',
'--pose',
required=False,
default=False,
action='store_true',
help='Add pi to poses (for range 0-2pi)')
parser.add_argument('-sp',
'--subpi',
required=False,
default=False,
action='store_true',
help='Add pi to poses (for range -pi - +pi)')
args = parser.parse_args()
data_manager = DataManager(args.data,
dtype=np.float32,
batch_size=1,
sequence_length=1)
if args.to_float:
to_float(data_manager)
if args.mean_normalize:
mean_normalize(data_manager)
if args.show:
show_imgs(data_manager)
if args.pose:
add_pi_to_poses(data_manager)
if args.subpi:
sub_pi_from_poses(data_manager)
def main(argv):
manager = DataManager()
manager.load()
sess = tf.Session()
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
model = VAE(beta=flags.beta,
learning_rate=flags.learning_rate,
alpha = flags.alpha,
kappa = flags.kappa,
lagrange_mult_param = flags.lagrange_mult_param,
use_geco = flags.use_geco)
sess.run(tf.global_variables_initializer())
saver = load_checkpoints(sess)
if flags.training:
# Train
train(sess, model, manager, saver)
else:
reconstruct_check_images = manager.get_random_images(10)
# Image reconstruction check
reconstruct_check(sess, model, reconstruct_check_images)
# Disentangle check
disentangle_check(sess, model, manager)
def cross_validation():
for dataset in datasets:
print('INICIANDO VALIDAÇÃO PARA O %s' % dataset.name)
hit_sum_test = []
data_manager = DataManager(dataset)
data_manager.load_data(categorical=False)
for index in range(0, 20):
x_TRAIN, y_TRAIN, x_VALIDATION, y_VALIDATION = data_manager.split_train_test_5fold(
data_manager.X, data_manager.Y)
hit_sum = []
for fold in range(0, 5):
rbf = ELM(number_of_hidden=1000)
rbf.fit(x_TRAIN[fold], y_TRAIN[fold])
hit_sum.append(
rbf.evaluate(x_VALIDATION[fold], y_VALIDATION[fold]))
hit_sum_test.append(np.average(hit_sum))
print('Accuracy: %.2f' % np.average(hit_sum_test))
print('Min: %.2f' % np.min(hit_sum_test))
print('Max: %.2f' % np.max(hit_sum_test))
print('Stand De: %.2f%%' % np.std(hit_sum_test))
def build_data_manager(frame_folder, feature_folder, caption_file, key_frame_info_folder, save_path):
data_manager = DataManager()
data_manager.load_frame_path_info(frame_folder)
data_manager.load_captions(caption_file)
data_manager.load_key_frame_information(key_frame_info_folder)
data_manager.load_features(feature_folder)
data_manager.save(save_path)
def test():
'''
Trains the model and returns its score
'''
matplotlib.rcParams['backend'] = 'Qt5Agg'
matplotlib.get_backend()
D = DataManager(data_name, data_dir)
#Load le model
mdl = model()
Prepro = prepro.Preprocessor()
#D.data['X_train'] = Prepro.removeOutliers(D.data['X_train'])
#D.data['Y_train'] = Prepro.removeOutliers(D.data['Y_train'])
X_train = D.data['X_train']
Y_train = D.data['Y_train'].ravel()
#test de l'entrainement
mdl.fit(X_train, Y_train)
#test de la prediction
Y_hat_train = mdl.predict(D.data['X_train'])
Y_hat_valid = mdl.predict(D.data['X_valid'])
Y_hat_test = mdl.predict(D.data['X_test'])
metric_name, scoring_function = get_metric()
scores = cross_val_score(mdl,
X_train,
Y_train,
cv=5,
scoring=make_scorer(scoring_function))
print('\nCV score (95 perc. CI): %0.2f (+/- %0.2f)' %
(scores.mean(), scores.std() * 2))
def main():
dm = DataManager()
train_x, train_y, val_x, val_y, test_x, test_y = dm.get_inputs(dim=5)
# create model
n_input_dimension = train_x.shape[1]
n_labels = train_y.shape[1]
model = Sequential()
model.add(Dense(50, input_shape=(n_input_dimension, )))
model.add(Dense(50))
model.add(Dense(n_labels, activation='softmax'))
optimizer = Adam(lr=0.0001)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
# callbacks
log_name = "log_" + datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
tb_path = "./logs/tensorboard/{}".format(log_name)
tb = keras.callbacks.TensorBoard(log_dir=tb_path)
es = EarlyStopping(monitor='val_loss', patience=10, verbose=1)
# fit
epochs = 10000
batch_size = 100
model.fit(train_x,
train_y,
epochs=epochs,
batch_size=batch_size,
validation_data=(val_x, val_y),
callbacks=[tb, es])
def main(args, config):
### Set config for either gpu (on cluster) or cpu (on local machine)
if args.gpu:
os.environ["CUDA_VISIBLE_DEVICES"] = os.environ['SGE_GPU'].replace(
"\n", ",")
else:
pass
seed = args.seed
### Set random seed for numpy ops
np.random.seed(seed)
# set random seed for tensorflow graph
# must set random seed before creating session.
# setting seed comes from this GitHubGist https://gist.github.com/tnq177/ce34bcf6b20243b0b5b23c78833e7945
tf.reset_default_graph()
tf.set_random_seed(seed)
print("Creating DataManager and loading training and testing dataset")
manager = DataManager(args)
sess = tf.Session(config=config)
print("Creating model...")
model = VAE(args, manager)
print("Model created!")
sess.run(tf.global_variables_initializer())
# initialize/reset all running variables
sess.run(model.running_vars_initializer)
#create saver for checkpoints saving
saver = utils.create_checkpoint_saver(sess, args, model)
print("Will Start Training Now")
model.train(sess, manager, saver, args)
def main():
if len(sys.argv) < 3:
print('Error: missing path to images and/or confidence threshold\nErro: ausencia do diretorio das imagens e/ou do limiar de confiança')
exit(0)
CONFIDENCE_THRES = float(sys.argv[2])
if CONFIDENCE_THRES > 100 or CONFIDENCE_THRES < 0:
print('Error: Give an Confidence Score Threshold between [0,100]\nErro: Indique um Limiar de Confiança entre [0,100]')
exit(0)
data = DataManager()
net_graph = tf.Graph()
class_net_graph = tf.Graph()
with net_graph.as_default():
net = Network(data.WIDTH, data.HEIGHT, data.CHANNELS)
net.restore_model('model')
with class_net_graph.as_default():
class_net = ClassNetwork(data.WIDTH, data.HEIGHT, data.CHANNELS)
class_net.restore_model('model')
for rel_path, subdirs, files in os.walk(sys.argv[1]):
for name in sorted(files):
if name[-3:] != 'png':
continue
full_path = os.path.join(rel_path, name)
print('processing ' + full_path)
img = cv2.imread(full_path, cv2.IMREAD_COLOR)
img = run(net, class_net, data, img, CONFIDENCE_THRES)
cv2.imwrite(full_path, img)
def load_input_data():
dm = DataManager(INPUT_FILE)
train_feature = dm.get_data_for_variable("wind_60")
train_label = dm.get_data_for_variable("CP07")
# Output should be iterable with features, label for train and test
# shape (1372, 2, 210)
return train_feature, train_label
def validate():
for dataset in datasets:
print('INICIANDO o plot da superfície de decisão PARA O %s' %
dataset.name)
data_manager = DataManager(dataset)
data_manager.load_data(categorical=False)
MSE = []
RMSE = []
for realization in range(0, 10):
x_TRAIN, y_TRAIN, x_VALIDATION, y_VALIDATION = data_manager.split_train_test_5fold(
data_manager.X, data_manager.Y)
for i in range(0, 1):
validation_perceptron = MultiLayerPerceptronRegressor(
activation='tanh', hidden_number=15, learning_rate=0.01)
validation_perceptron.fit(x_TRAIN[i],
y_TRAIN[i],
dataset.value,
epochs=300,
verbose=True)
mse, rmse = validation_perceptron.evaluate(
x_VALIDATION[i], y_VALIDATION[i])
MSE.append(mse)
RMSE.append(rmse)
print('MSE: %.2f' % np.average(MSE))
print('RMSE: %.2f' % np.average(RMSE))
print('Stand De MSE: %.2f%%' % np.std(MSE))
print('Stand De RMSE: %.2f%%' % np.std(RMSE))
def run():
for dataset in datasets:
print('INICIANDO VALIDAÇÃO PARA O %s' % dataset.name)
hit_sum = []
data_manager = DataManager(dataset)
data_manager.load_data(categorical=False)
for i in range(1, 2):
x_TRAIN, y_TRAIN, x_VALIDATION, y_VALIDATION = data_manager.split_train_test_5fold(
data_manager.X, data_manager.Y)
for fold in range(0, 2):
rbf = ELM(number_of_hidden=1000)
rbf.fit(x_TRAIN[fold], y_TRAIN[fold])
hit_sum.append(
rbf.evaluate(x_VALIDATION[fold], y_VALIDATION[fold]))
rmse = np.power(hit_sum, 0.5)
print('MSE: %.25f' % np.average(hit_sum))
print('MSE Std: %.25f' % np.std(hit_sum))
print('RMSE: %.25f' % np.average(rmse))
print('R0.0MSE Std: %.25f' % np.std(rmse))
def __init__(self):
"""
Initialize all data managers.
"""
self.data_manager_list = []
for site_id in range(1, 11):
self.data_manager_list.append(DataManager(site_id))
def main(argv):
manager = DataManager(FLAGS.data)
manager.load()
sess = tf.Session()
normaliser = (FLAGS.latent_size / 10) * ((64 * 64) / manager.input_size)
model = MODEL(latent_size=FLAGS.latent_size,
gamma=normaliser * FLAGS.gamma,
capacity_limit=FLAGS.capacity_limit,
capacity_change_duration=FLAGS.capacity_change_duration,
learning_rate=FLAGS.learning_rate,
n_channels=manager.n_channels)
sess.run(tf.global_variables_initializer())
saver = load_checkpoints(sess)
if FLAGS.training:
# Train
train(sess, model, manager, saver)
else:
reconstruct_check_images = manager.get_random_images(10)
# Image reconstruction check
reconstruct_check(sess, model, reconstruct_check_images)
# Disentangle check
disentangle_check(sess, model, manager)
def main(argv):
np.random.seed(1)
if not os.path.exists(flags.save_dir):
os.mkdir(flags.save_dir)
data_manager = DataManager()
place_cells = PlaceCells()
hd_cells = HDCells()
data_manager.prepare(place_cells, hd_cells)
model = Model(place_cell_size=place_cells.cell_size,
hd_cell_size=hd_cells.cell_size,
sequence_length=flags.sequence_length)
trainer = Trainer(data_manager, model, flags)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# For Tensorboard log
log_dir = flags.save_dir + "/log"
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
# Load checkpoints
saver, start_step = load_checkpoints(sess)
# Train
train(sess, trainer, saver, summary_writer, start_step)
def train(self):
data_manager = DataManager()
passage_vocab_size = len(data_manager.passage_vocab.values())
question_vocab_size = len(data_manager.passage_vocab.values())
data_loader = DataLoader(
data_manager, batch_size=1, shuffle=True, num_workers=4
)
model = AmandaModel(
passage_vocab_size=passage_vocab_size,
question_vocab_size=question_vocab_size
)
optimizer = torch.optim.Adamax(model.parameters())
crt = torch.nn.CrossEntropyLoss()
for epoch in range(self.max_epoch):
print("epcoh {}".format(epoch))
for index, batch in enumerate(data_loader):
model.zero_grad()
passage, question, spans = batch
Pra = model(passage, question)
loss = 0.
for b_index in range(passage.shape[0]):
loss += crt(Pra[b_index], spans[b_index])
print("loss ", loss)
loss.backward()
optimizer.step()
pred = model(passage, question)
def evaluate_map_data(map_resolution, environment):
data_manager = DataManager(DEFAULT_MAP, "perfect")
filter_types = ["normal_filter", "biased_normal_filter", "voxel_filter"]
filter_translation_results = dict()
filter_orientation_results = dict()
filter_iter_results = dict()
point_map = dict()
for filter_name in filter_types:
data_manager.load_data(filter_name)
trans_mean = data_manager.data.post_translation_means()
trans_std = data_manager.data.post_translation_std()
filter_translation_results[filter_name] = (trans_mean, trans_std)
orientation_mean = data_manager.data.post_rotation_means()
orientation_std = data_manager.data.post_rotation_std()
filter_orientation_results[filter_name] = (orientation_mean,
orientation_std)
iter_mean = data_manager.data.iter_means()
iter_std = data_manager.data.iter_std()
filter_iter_results[filter_name] = (iter_mean, iter_std)
point_map[filter_name] = data_manager.data.positions
data_manager.readonly_close()
result_plotter.plot_filter_statistics_translation(
filter_translation_results, map_resolution)
result_plotter.plot_filter_statistics_orientation(
filter_orientation_results, map_resolution)
result_plotter.plot_filter_statistics_iter(filter_iter_results,
map_resolution)
result_plotter.plot_points_on_map(environment, point_map)
def run():
"""
"""
logger = init()
try:
data = DataManager(path_to_raw)
model = ModelManager(data, augment_data)
logger.info(f"Creating new folder in {results_path} to save model object and results...")
folder_name = time.strftime("%Y-%m-%d_%H-%M-%S_RUN")
mkdir(results_path+folder_name)
logger.info(f"{folder_name} created in {results_path}.")
logger.info(f"Saving model object in the created folder...")
with open(results_path + folder_name + "/" + model_name, "wb") as output:
pickle.dump(model, output, pickle.HIGHEST_PROTOCOL)
logger.info("Saving of model object completed.")
if generate_results_report :
report_file = open(results_path + folder_name + "/" + "Run_Report.txt", "w")
report_file.write(write_report_from_run(model, augment_data))
report_file.close()
logger.info("Main program completed successfully.")
return model
except Exception as e:
logger.error("An error occured, program will be interrupted. Details : " + str(e))
raise e
def test():
input_dir = '../../datasets/automl/' # Change this to the directory containing AutoML datasets
if not os.path.isdir(input_dir):
raise ValueError(
"input_dir not found. You can change this value in your ")
small_datasets = ['jasmine', 'dexter', 'adult', 'cadata', 'arturo']
filepath = './sample-haha'
dataset_name = np.random.choice(small_datasets)
set_type = np.random.choice(['train', 'test'])
# set_type = 'test'
D = DataManager(dataset_name,
input_dir,
replace_missing=False,
verbose=verbose)
D_info = D.info
print("dataset_name={}, set_type={}, sparse or dense: {}".format(
dataset_name, set_type, D.info['format']))
metadata, features, labels = _prepare_metadata_features_and_labels(
D, set_type=set_type)
convert_vectors_to_sequence_example(filepath,
metadata,
features,
labels,
D_info,
max_num_examples=None,
num_shards=1)
print_first_sequence_example(filepath)
pprint(D.info)
print("Now you should see 2 or 3 new files in current directory. :)")
def generate_map_data(grid_map, environment, map_name, dir_name, prefix,
sample_count, cloud_size, rangefinder_noise, points):
data_manager = DataManager(map_name, dir_name, prefix)
point_cloud_filters = [
NormalFilter(number_of_bins=20),
BiasedNormalFilter(number_of_bins=20),
AdaptivelyVoxelFilter(2 * grid_map.size),
RandomFilter()
]
#points = [(1.5, 1.5), (3, 1.5), (4.5, 1.5),
# (1.5, 2.5), (3, 3), (5, 2.4),
# (3, 3), (5, 3.5),
# (3.5, 5),
# (1.5, 7), (3.5, 7),
# (2.5, 8), (4, 8), (5, 8.5),
# (6.5, 8.5), (8, 8)]
#points = [(1.5, 1.5), (2, 1.5), (2.5, 1.5), (3, 1.5), (3.5, 1.5),
# (4, 1.5), (4.5, 1.5), (5, 1.5), (5.5, 1.5), (6, 1.5),
# (6.5, 1.5), (7, 1.5), (7.5, 1.5), (8, 1.5), (8.5, 1.5)]
#points = [(5.5, 5.5)]
for point in points:
print("Process point: ", point)
rangefinder = Rangefinder(cloud_size,
range_variance=rangefinder_noise,
angular_variance=rangefinder_noise)
point_cloud = rangefinder.scan(environment, point)
point_cloud.calc_normals()
evaluate_filters(grid_map, point_cloud, point_cloud_filters,
data_manager, sample_count)
def setUp(self):
db_file = "YourAppointment.db"
if os.path.exists(db_file):
os.remove(db_file)
self.dm = DataManager()
with redirect_stdout(StringIO()) as stdout:
self.dm.db.create_table_customers()
