# option 1 for loading other modules
sys.path.append(
dirname("/Users/dominiquepaul/xCoding/classification_tool/Main/"))
### an alternative menthod for handling file paths ###
# main_path = os.path.dirname(__file__) # this doesnt work when run in a REPL environment
# module_path = os.path.join(main_path,"../" )
# sys.path.append(dirname(module_path))
from preprocessing import join_npy_data
from transfer_learning import Transfer_net
file_paths = [
"./example_output_folder/apparel_image_package_train_val_split_0.npy"
]
x_train, y_train, x_test, y_test, conversion = join_npy_data(
file_paths, training_data_only=False)
# instantiate the empty model
transfer_model = Transfer_net()
# we create our own model architecture by passing configuration values
transfer_model.create_network(layers=2,
neurons=10,
dropout_rate=0.6,
num_output_classes=2)
# convert the data into transfer values
x_train = transfer_model.load_transfer_data(x_train)
# alternatively we could also cache the data
# x_train = transfer_model.load_or_cache_transfer_data(x_train, file_path="./example_output_folder/transfernet_cached_files")
# traing the model
transfer_model.train(x_train,
sys.path.append(dirname(module_path))
### an alternative menthod for handling file paths is by using the absolute path ###
# sys.path.append(dirname("/Users/dominiquepaul/xCoding/classification_tool/Main/modules/"))
from regressionclass import Logistic_regression, Lasso_regression
# folder where different label evaluations are saved
FOLDER_PATH_SAVE = "../Data/wnet_hyperopt_datasets"
# path with the industry dict folders
ind_labels = load_industry_labels(
file_path="./industry_dicts/selection_AutomobileManufacturers.csv")
OBJECT = "car"
automotive_pckgs = ["../Data/np_files/car_image_package_train_test_split0.npy"]
x_train, y_train, x_test, y_test, conversion = join_npy_data(automotive_pckgs)
n_label_list = [3, 5, 8, 10, 15, 20, 25, 50]
# transform or load the data if necessary
for label_amount in tqdm(n_label_list):
x_train_df = create_feature_df(imgs=x_train,
object_name=OBJECT,
ind_labels=ind_labels,
k_labels=label_amount)
x_test_df = create_feature_df(imgs=x_test,
object_name=OBJECT,
ind_labels=ind_labels,
k_labels=label_amount)
x_train_df.to_csv(FOLDER_PATH_SAVE + "/train_{}".format(label_amount))
x_test_df.to_csv(FOLDER_PATH_SAVE + "/test_{}".format(label_amount))
from cnn import cnn_model
from preprocessing import join_npy_data
MAX_EVALS = 20
### loading the data. Two possible methods:
# For running the script on a local machine
# data =["../Data/np_files/car_image_package_train_test_split0.npy"]
# x_train, y_train, x_test, y_test , conversion = join_npy_data(data)
# For running the script on a cloud server machine
data_url = [
'gs://data-imr-unisg/np_array_files/car_image_package_train_val_split_0.npy'
]
x_train, y_train, x_test, y_test, conversion = join_npy_data(
data_url, training_data_only=False)
# File to save first results
out_file = 'out_files/hyperparameter_opt/custom_nn_hyperopt.csv'
with open(out_file, 'w') as csv_file:
writer = csv.writer(csv_file)
# Write the headers to the file
writer.writerow([
'conv_layers', 'conv_filters', 'dense_layers', 'dense_neurons',
'dropout_rate_dense', 'learning_rate', 'run_time', 'val_loss',
'val_accuracy', 'val_f1', 'train_loss', 'train_accuracy', 'train_f1'
])
# the function to be optimised: it takes the parameters and returns the loss (metric to be minimised)
def objective(params):
k_labels=20)
x_test_df_50 = create_feature_df(imgs=x_test,
object_name=OBJECT_NAME,
ind_labels=ind_labels,
k_labels=50)
ALL_PREDICTIONS_DF = pd.DataFrame({"names": names})
# only method that doesnt require a training set
run_wordnet_direct("car", "custom", "Unaugmented")
# run 1/4: own images not augmented
automotive_pckgs = [
os.path.join(DATA_FOLDER_PATH,
"np_files/car_image_package_train_val_split_0.npy")
]
x_train, y_train, _, _, conversion = join_npy_data(automotive_pckgs,
training_data_only=False)
run_custom_network(OBJECT_NAME, "custom", "Unaugmented")
run_transfer_network(OBJECT_NAME, "custom", "Unaugmented")
run_wordnet_indirect_v3(OBJECT_NAME, "custom", "Unaugmented")
run_wordnet_indirect_v4(OBJECT_NAME, "custom", "Unaugmented")
# run 2/4: own images augmented
automotive_pckgs_augmented = [
os.path.join(DATA_FOLDER_PATH,
"np_files/car_image_package_train_val_split_augmented_0.npy"),
os.path.join(DATA_FOLDER_PATH,
"np_files/car_image_package_train_val_split_augmented_1.npy"),
os.path.join(DATA_FOLDER_PATH,
"np_files/car_image_package_train_val_split_augmented_2.npy"),
os.path.join(DATA_FOLDER_PATH,
OBJECT = "car"
ind_labels = load_industry_labels(file_path="./industry_dicts/selection_AutomobileManufacturers.csv")
# non_augmented
#basic_feats = ["max_score", "product_ref_count", "product_ref_sum", "product_reference"]
#wordnet_feats = ["product_count_wordnet", "maxscorevalue_wordnet", "product_sum_wordnet"]
#####################
#### approach 1 #####
#####################
# direct wordnet
automotive_pckgs = ["../Data/np_files4/car_image_package_train_val_split_0.npy"]
x_train, y_train, x_test, y_test, conversion = join_npy_data(automotive_pckgs, training_data_only=False)
words_sought = ["cars","truck"]
predictions = identify_items(x_test[:2], words_sought, k_labels=5, use_synonyms=True)
#####################
#### approach 2 #####
#####################
# regression with basic features
ind_labels = load_industry_labels(file_path="./industry_dicts/selection_AutomobileManufacturers.csv")
automotive_pckgs = ["../Data/np_files/car_image_package_train_test_split0.npy"]
x_train, y_train, x_test, y_test, conversion = join_npy_data(automotive_pckgs)
x_train_df = create_feature_df(imgs=x_test, object_name=OBJECT, ind_labels=ind_labels, k_labels=10, basic_feats=True, wordnet_feats=False)
# train regression
from transfer_learning import Transfer_net
from preprocessing import join_npy_data
EPOCHS = 10000
MAX_EVALS = 20
# File to save first results
out_file = 'out_files/hyperparameter_opt/transfer_learning_hyperopt_out.csv'
with open(out_file, 'w') as csv_file:
writer = csv.writer(csv_file)
# Write the headers to the file
writer.writerow(['neurons','layers','dropout_rate','learning_rate','run_time', 'val_loss', 'val_accuracy', 'train_loss', 'train_accuracy'])
automotive_pckgs = ["../Data/np_files4/car_image_package_train_val_split_0.npy"]
x_data, y_train, _, _, conversion = join_npy_data(automotive_pckgs, training_data_only=False)
# hyperparameter optimization with hyperopt
def objective(params):
t_net = Transfer_net()
t_net.create_network(layers=params["layers"], neurons=params["neurons"], dropout_rate=params["dropout_rate"], num_output_classes=2)
x_train = t_net.load_or_cache_transfer_data(x_data, file_path="../Data/transfernet_files/x_train_T7")
start = timer()
t_net.train(x_train, y_train, learning_rate=params["learning_rate"], epochs=EPOCHS, batch_size=256, verbose=True, tb_logs_dir="./out_files/log_files/transfer_net/")
run_time = timer() - start
val_loss = t_net.hist.history["val_loss"][-1]
val_accuracy = t_net.hist.history["val_acc"][-1]
train_loss = t_net.hist.history["loss"][-1]
train_accuracy = t_net.hist.history["acc"][-1]