def teestModel(args, folders):
# load the test data
X_test = load_data(os.path.join(folders.data_folder, 'test-images.gz'),
False) / 255.0
y_test = load_data(os.path.join(folders.data_folder, 'test-labels.gz'),
True).reshape(-1)
# predic test data
clf = joblib.load(os.path.join(folders.output_folder, args.modelFileName))
y_hat = clf.predict(X_test)
# examine the confusion matrix
conf_mx = confusion_matrix(y_test, y_hat)
print(conf_mx)
print('Overall accuracy:', np.average(y_hat == y_test))
return conf_mx
def read_data(self, patient_id, week, recording):
recording_session_path = self.get_recording_sessions_path(
patient_id=patient_id, week=week)
path_to_file = glob(os.path.join(recording_session_path +
recording))[-1]
return load_data(path_to_file)
def score(args, folders):
X_test = load_data(os.path.join(folders.data_folder, 'test-images.gz'),
False) / 255.0
y_test = load_data(os.path.join(folders.data_folder, 'test-labels.gz'),
True).reshape(-1)
input_data = "{\"data\": [" + str(list(X_test[args.selected_item])) + "]}"
headers = {'Content-Type': 'application/json'}
# for AKS deployment you'd need to the service key in the header as well
# api_key = service.get_key()
# headers = {'Content-Type':'application/json', 'Authorization':('Bearer '+ api_key)}
resp = requests.post(args.scoring_uri, input_data, headers=headers)
print("POST to url", args.scoring_uri)
#print("input data:", input_data)
print("label:", y_test[args.selected_item])
print("prediction:", resp.text)
# qr = np.vstack(qr)
# x_test_new = np.hstack([x_test_new, qr])
# y_test_new = np.vstack(y_test_new)
def rmse_loss(targets, outputs):
return tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(outputs, targets))))
pc = "../data/pc_with_object_ids13.npy"
hp = "../data/human_pos13.npy"
q = "../data/q13.npy"
pct = "../data/pc_with_object_ids12.npy"
hpt = "../data/human_pos12.npy"
qt = "../data/q12.npy"
x_train, y_train, x_train_ = load_data(pc, hp, q)
# #
# #
# # x_train = np.load("../../data/1filter_points.npy")
# # y_train = np.load("../../data/1op_position.npy")
# #
# # fig = plt.figure()
# # ax = fig.add_subplot(111, projection='3d')
# # ax.scatter(x_train[:,0], x_train[:,1], x_train[:,2], alpha=0.3, c='r') #c=perf_down_sampled.moving
# # ax.scatter(y_train[:,0], y_train[:,1], y_train[:,2]*0.0, alpha=0.3, c='b')
# # plt.show()
# #
# # plt.scatter(x_train[:,0], x_train[:,1])
# # plt.scatter(y_train[:,0], y_train[:,1])
# # plt.show()
# #
def main(args):
# --------------------------------------------------------------------------
# DATA
logger.info('-' * 100)
logger.info('Load data files')
train_exs = utils.load_data(args, args.train_file, args.skip_no_answer )
logger.info('Num train examples = %d' % len(train_exs))
dev_exs = utils.load_data(args, args.dev_file, args.skip_no_answer )
logger.info('Num dev examples = %d' % len(dev_exs))
# Doing Offician evals
# 1) Load the original text to retrieve spans from offsets.
# 2) Load the text answers for each question
dev_texts = utils.load_text(args.dev_json)
dev_offsets = {ex['id']: ex['offsets'] for ex in dev_exs}
dev_answers = utils.load_answers(args.dev_json)
# --------------------------------------------------------------------------
# Model
logger.info('-' * 100)
start_epoch = 0
if args.checkpoint and os.path.isfile(args.model_name + '.checkpoint'):
pass
else:
logger.info('Training model from scratch ...')
model = init_from_scratch(args, train_exs, dev_exs)
model.init_optimizer()
# if args.tune_partial:
# pass
if args.cuda:
model.cuda()
if args.parallel:
model.parallelize()
# --------------------------------------------------------------------------
# DATA ITERATORS
# Two dataset: train and dev. If sort by length it's faster
logger.info('-' * 100)
logger.info('Make data loaders')
train_dataset = data.ReaderDataset(train_exs, model, single_answer=True)
if args.sort_by_len:
train_sampler = data.SortedBatchSampler(train_dataset.lengths(), args.batch_size, shuffle=True)
else:
train_sampler = touch.utils.data.sampler.RandomSampler(train_dataset)
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size = args.batch_size,
sampler = train_sampler,
num_workers = args.data_workers,
collate_fn = vector.batchify,
pin_memory = args.cuda,
)
dev_dataset = data.ReaderDataset(dev_exs, model, single_answer=False)
if args.sort_by_len:
dev_sampler = data.SortedBatchSampler(dev_dataset.lengths(), args.dev_batch_size, shuffle=True)
else:
dev_sampler = touch.utils.data.sampler.RandomSampler(dev_dataset)
dev_loader = torch.utils.data.DataLoader(
dev_dataset,
batch_size = args.dev_batch_size,
sampler = dev_sampler,
num_workers = args.data_workers,
collate_fn = vector.batchify,
pin_memory = args.cuda,
)
# --------------------------------------------------------------------------
# PRINT CONFIG
logger.info('-' * 100)
logger.info('CONFIG:\n%s' % json.dumps(vars(args), indent=4, sort_keys=True))
# --------------------------------------------------------------------------
# TRAIN&VALID LOOP
logger.info('-' * 100)
logger.info('Starting training... ')
stats = {'timer':utils.Timer(), 'epoch':0, 'best_valid':0}
for epoch in range(start_epoch, args.num_epochs):
stats['epoch'] = epoch
#Train
train(args, train_loader, model, stats)
# Validate official (dev)
result = validate_official(args, dev_loader, model, stats, dev_offsets, dev_texts, dev_answers)
if args.lrshrink > 0:
_lr = model.lr_step(result[args.valid_metric])
logger.info('learning rate is %f' % _lr)
# Save best valid
if result[args.valid_metric] > stats['best_valid']:
logger.info('Best valid: %s = %.2f (epoch %d, %d updates)' %
(args.valid_metric, result[args.valid_metric],
stats['epoch'], model.updates))
model_save_name = os.path.join(args.save_dir, args.model_name+str(stats['epoch'])+'.pt')
model.save(model_save_name)
stats['best_valid'] = result[args.valid_metric]
logger.info('Best valid: %s = %.2f (epoch %d, %d updates)' %
(args.valid_metric, stats['best_valid'],
stats['epoch'], model.updates))
from tensorflow.compat.v1 import InteractiveSession
config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config)
pc = "../data/pc_with_object_ids13.npy"
hp = "../data/human_pos13.npy"
q = "../data/q13.npy"
pct = "../../data/pc_with_object_ids14.npy"
hpt = "../../data/human_pos14.npy"
qt = "../../data/q14.npy"
x_train, y_train, x_train_ = load_data(pc, hp, q)
SEQ_LEN = 10
BATCH_SIZE = 256
HIDDEN_UNITS = 256
EPOCHS = 100
VAL_SPLIT = 0.15
# Set random seed for reproducibility
SEED = 2345
random.seed(SEED)
np.random.seed(SEED)
def slice_sequence_examples(sequence, num_steps):
xs = []
for i in range(len(sequence) - num_steps - 1):
utils.reload_scripts()
DATA_DIR = 'data'
RAW_SUB_DIR = '00_raw'
PRO_SUB_DIR = '01_preprocessed'
train_df = None
test_df = None
# PREPROCESSING
RUN_PREPROCESSING = True
if RUN_PREPROCESSING:
train_df = utils.load_data(DATA_DIR, RAW_SUB_DIR, 'train')
test_df = utils.load_data(DATA_DIR, RAW_SUB_DIR, 'test')
preprocessor = preprocessing.DefaultPreprocessor()
processed_train_df = preprocessor.process(train_df)
utils.export_data(processed_train_df, DATA_DIR, PRO_SUB_DIR, 'train')
processed_test_df = preprocessor.process(test_df)
utils.export_data(processed_train_df, DATA_DIR, PRO_SUB_DIR, 'test')
else:
train_df = utils.load_data(DATA_DIR, PRO_SUB_DIR, 'train')
test_df = utils.load_data(DATA_DIR, PRO_SUB_DIR, 'test')
# FEATURE SELECTION
X = train_df['comment_text']
for n in range(n_iterations):
estimate['mu'] = estimate_mu(data, prev_estimate)
estimate['mu_f'] = estimate_mu_f(data, prev_estimate)
estimate['p'] = estimate_p(data, prev_estimate)
prev_estimate.mu[:] = estimate.mu
prev_estimate.mu[:] = estimate.mu_f
prev_estimate.mu[:] = estimate.p
return estimate
if __name__ == '__main__':
from scripts.utils import display_data, display_estimate, load_data
# deriv_data = create_data(load_p_matrix=True, deriv=True, data_filename='probs_deriv')
# data = create_data(load_p_matrix=True, data_filename='probs')
deriv_data = load_data(data_filename='probs_deriv')
data = load_data(data_filename='probs')
diff_data = Data(deriv_data.extent, deriv_data.trans - data.trans, deriv_data.f_1 - data.f_1,
deriv_data.f_2 - data.f_2)
display_data(diff_data)
display_data(data)
estimate = initial_estimate()
display_estimate(estimate)
plt.show()
################################## MAIN #######################################
if __name__ == "__main__":
tic = time() # Start program performance timer
close("all") # Close all previously opened plots
ion() if DISPLAY_PLOTS else ioff()
# Load and preprocess data with all methods that are independent of subset.
data = load_data("trainingData.csv",
"validationData.csv",
N,
DROP_COLUMNS,
dst_null=NO_SIGNAL_VALUE,
drop_val=DROP_VAL)
X, Y = data
# Note that Random Seed is 0. All Validation sets must be created from a
# subset of the train set here.
x_train_o, x_test_o, y_train, y_test = train_test_split(X.values,
Y.values,
test_size=0.2,
random_state=0)
# This filters out samples that do not have enough active WAPs in it
# according to MIN_WAPS. This has to happen after the split because if not,
# the randomness will be affected by missing samples, thus compromising
# test set validity.
student_model.fit(x=X_p,
y=Y_p,
batch_size=8,
validation_data=(Xv_p, Yv_p),
epochs=1,
shuffle=True,
callbacks=callback,
verbose=1)
utils.my_eval(student_model, Xv_p, Yv_p)
return student_model
if __name__ == "__main__":
# loading data and using preprocess for training and validation dataset
(X_data, y_data) = utils.load_data(settings.Xs_dir, settings.ys_dir)[0]
(X_test) = utils.load_test_data(settings.Xt_dir)[0]
utils.mask_unused_gpus()
physical_devices = tf.config.experimental.list_physical_devices('GPU')
if physical_devices:
try:
for gpu in physical_devices:
tf.config.experimental.set_memory_growth(gpu, True)
except RuntimeError as e:
print(e)
assert len(
physical_devices) > 0, "Not enough GPU hardware devices available"
Xt, X, Yt, Y = train_test_split(X_data, y_data, test_size=0.2)