def main(): parser = argparse.ArgumentParser( description='Sentiment classifier argument parser.') parser.add_argument( '--build_data', action='store_true', help='build and save data sets (only needed for the first time).') parser.add_argument('--alg', choices=['CNN', 'BiLSTM', 'BiRNN'], required=True, help='algorithm to train the sentiment classifier') parser.add_argument( '--small_subsets', action='store_true', help='train and evaluate on smaller subsets of the data.') parser.add_argument('--outfile', type=str, help='output file name to save trained model.') args = parser.parse_args() # build and save data for the first time if args.build_data: data_loader = DataLoader() data_loader.build_data() # load data from file data_loader = DataLoader() data_loader.load_data() # train model train_classifier(alg=args.alg, data_loader=data_loader, small_subsets=args.small_subsets, outfile=args.outfile)
def main(): s = time.time() dl = DataLoader() print(time.time() - s) for project in dataset_names: result = { project: { i: { algo: {measure: [] for measure in measure_names} for algo in algo_names } for i in range(STEPS) } } counter = 0 while counter != STEPS: try: x_train_scaled, x_test_scaled, y_train, y_test = dl.build_data( project) pos_count = DataLoader.get_positive_count(y_train) all_count = len(y_train) for algo in algo_names: net = Model(FEATURES_NUM, hidden_shape=config.hidden_layer, classes=config.classes, positive_num=pos_count, all_num=all_count) # net.train(x_train_scaled, y_train, config) net.train_batch(x_train_scaled, y_train, config) y_pred, y = net.test(x_test_scaled, y_test, config) result = Evaluator.evaluate(measure_names, y_pred, y, result, project, algo, counter) print('This is the ' + str(counter + 1) + 'times') except BaseException as err: print('The' + str(counter) + 'times loop error') print(err) continue counter += 1 try: # reverse reverse_x_train = x_test_scaled reverse_y_train = y_test reverse_x_test = x_train_scaled reverse_y_test = y_train pos_count = DataLoader.get_positive_count(reverse_y_train) all_count = len(reverse_y_train) net = Model(FEATURES_NUM, hidden_shape=config.hidden_layer, classes=config.classes, positive_num=pos_count, all_num=all_count) net.train_batch(reverse_x_train, reverse_y_train, config) y_pred, y = net.test(reverse_x_test, reverse_y_test, config) result = Evaluator.evaluate(measure_names, y_pred, y, result, project, algo, counter) except BaseException as err: print('The' + str(counter) + 'times loop error') print(err) counter -= 2 finally: counter += 1 continue with open( './results/algo--final--' + str(STEPS) + '--' + project + '.json', 'w') as f: json.dump(result, f)