def test_predict(self):
par = PreProcess(r'../data/toy.labeled')
bc = BootCamp(Features())
ds_list = par.parser()
model = DP_Model(boot_camp=bc)
result = model.predict(ds_list)
print(result[0])
def test_fill_tensor(self):
feat = Features()
par = PreProcess(r'../data/toy.labeled')
bc = BootCamp(feat)
soldiers = par.parser()
bc.investigate_soldiers(soldiers)
# bc.truncate_features(10)
bc.train_soldiers(soldiers)
def test_fit(self):
par = PreProcess(r'../data/toy.labeled')
bc = BootCamp(Features())
ds_list = par.parser()
model = DP_Model(boot_camp=bc)
model.fit(ds_list, epochs=50)
results = model.score(ds_list)
# print(model.w)
clean_est = {key: value
for key, value in results[0].items()
if value} # remove empty
print(f"Predicted: {clean_est}")
sorted_ground_truth = dict(sorted(ds_list[0].graph_tag.items()))
print(f"Ground Truth: {sorted_ground_truth}")
def test_main(self):
NUM_EPOCHS = [10]
MODELS = ['base', 'advance']
NUMBER_OF_FEATURES = [500, 5000, 50000, 100_000, 0]
toy_path = r'C:\Users\afinkels\Desktop\private\Technion\Master studies\עיבוד שפה טבעית\HW\hw_repo\nlp_hw\HW2\data\toy.labeled'
toy_10__train_path = r'C:\Users\afinkels\Desktop\private\Technion\Master studies\עיבוד שפה טבעית\HW\hw_repo\nlp_hw\HW2\data\toy_10_train.labeled'
toy_5__train_path = r'C:\Users\afinkels\Desktop\private\Technion\Master studies\עיבוד שפה טבעית\HW\hw_repo\nlp_hw\HW2\data\toy_5_train.labeled'
toy_10_test_path = r'C:\Users\afinkels\Desktop\private\Technion\Master studies\עיבוד שפה טבעית\HW\hw_repo\nlp_hw\HW2\data\toy_10_test.labeled'
train_path = r'C:\Users\afinkels\Desktop\private\Technion\Master studies\עיבוד שפה טבעית\HW\hw_repo\nlp_hw\HW2\data\train.labeled'
test_path = r'C:\Users\afinkels\Desktop\private\Technion\Master studies\עיבוד שפה טבעית\HW\hw_repo\nlp_hw\HW2\data\test.labeled'
DATA_PATH = toy_path
TEST_PATH = toy_path
RESULTS_PATH = r'C:\Users\afinkels\Desktop\private\Technion\Master studies\עיבוד שפה טבעית\HW\hw_repo\nlp_hw\HW2\Test_models'
results_all = []
data = PreProcess(DATA_PATH).parser()
test = PreProcess(TEST_PATH).parser()
# BASE MODEL
bc = BootCamp(Features('bas'))
model = DP_Model(boot_camp=bc)
for n_epochs in NUM_EPOCHS:
start_time = time.time()
model.fit(data,
epochs=n_epochs,
fast=True,
truncate_top=4,
truncate_bottom=1)
train_acc = model.score(data)
test_acc = model.score(test)
results_all.append([
'base',
time.time() - start_time, n_epochs, train_acc, test_acc,
bc.features.num_features
])
print(
f'Finish base model with {n_epochs} epochs at {time.strftime("%X %x")} train_acc{train_acc} and test_acc{test_acc}'
)
df_results = pd.DataFrame(results_all,
columns=[
'Model', 'time', 'epochs', 'train_score',
'val_score', 'n_features'
])
df_results.to_csv(
f'{RESULTS_PATH}\\from_test_re_{time.localtime()}.csv')
def test_fill_tensor(self):
feat = Features()
feat.extract_features(ds)
feat.truncate_features(5)
feat.fill_tensor(ds)
print([mat.toarray() for mat in ds.f]) # for printing
def test_truncate_features(self):
feat = Features()
feat.extract_features(ds)
feat.truncate_features(5)
print("num of keys in updated:")
print(len(list(feat.features.keys())))
def test_extract_features(self):
feat = Features()
feat.extract_features(ds)
print(feat.features.keys())
def test_init(self):
feat = Features()
def test_truncate_features(self):
feat = Features()
par = PreProcess(r'../data/toy.labeled')
bc = BootCamp(feat)
bc.investigate_soldiers(par.parser())
bc.truncate_features(10)
def test_init(self):
feat = Features()
bc =BootCamp(feat)