def load_model(self, model):
assert self.model is None
self.logger.info('load model in %s', model)
self.model = torch.load(open(model, mode='rb'), map_location=self.device)
self.model = self.model.to(self.device)
epoch = int(model.split('_')[-1])
return epoch
def _test(graph):
print("***************** main graph ***************")
print(graph)
subgraphs = model.split(graph)
for i, g in enumerate(subgraphs):
print('------------- subgraph {} --------------'.format(i))
print(g)
def load_model(self, model):
assert self.model is None
self.logger.info('load model in %s', model)
self.model = torch.load(open(model, mode='rb'), map_location=self.device)
self.model = self.model.to(self.device)
epoch = int(model.split('_')[-1])
return epoch
def main():
# instantiate the class
d = DataPrep()
# read the data
# path = '/content/bank-additional-full.csv'
path = sys.argv[1]
data = d.read_data(path)
print('Original shape:', data.shape)
# preprocessing
data = d.treat_null(data)
data = d.outlier_correcter(data)
data = d.generate_features(data)
print('After feature generation:', data.shape)
data = d.scaler(data)
print('After scaling:', data.shape)
data = d.encoder(data)
print('After encoding:', data.shape)
data = d.over_sample(data)
print('After resampling:', data.shape)
data = drop_unwanted(data)
print('After dropping unwanted features:', data.shape)
print(data.head())
# split data
t = Transform()
x, y = t.split(data)
# modeling
m = Model(x, y)
# using mlp (best predictor of the 3)
pred = m.mlp()
pred_df = pd.DataFrame(pred, columns = ['y']) # save the predictions to a df
pred_df.to_csv('pred.csv') # save predictions to csv
# evaluation
x_train, x_test, y_train, y_test = split(x, y)
e = Evaluation()
precision, recall, fscore, support = e.precision_recall_f1_support(y_test, pred)
print('precision:', precision)
print('precision:', precision)
print('precision:', precision)
print('precision:', precision)
def split_graph(graph_in, config):
"""split graph"""
if config == 'auto':
return model.split(graph_in)
subgraphs = []
all_tensors = []
subgraph_idx = 0
config_parts = config.split('|')
for part in config_parts:
tensor_names = part.split(',')
graph_name = "%s_%d" % (graph_in.name, subgraph_idx)
g = graph_in.extract_subgraph(graph_name, tensor_names)
assert len(g.ops) == len(tensor_names)
subgraphs.append(g)
all_tensors += tensor_names
subgraph_idx += 1
if len(all_tensors) < len(graph_in.ops):
graph_name = "%s_%d" % (graph_in.name, subgraph_idx)
g = graph_in.extract_subgraph(graph_name, all_tensors, True)
subgraphs.append(g)
return subgraphs
model.fit(X_train,
y_train,
epochs={{choice([25, 50, 75, 100])}},
batch_size={{choice([16, 32, 64])}},
validation_data=(x_val, y_val),
callbacks=[reduce_lr])
score, acc = model.evaluate(x_val, y_val, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
return model
if __name__ == '__main__':
data = pd.read_csv(os.getcwd() + "/data/DataTurks/dump.csv")
corpus_vocabulary = create_dictionary(data['content'], 10000)
train, test = split(data, 18000)
best_run, best_model = optim.minimize(model=create_model,
data=train,
algo=tpe.suggest,
max_evals=15,
trials=Trials())
print("Evalutation of best performing model:")
print(best_model.evaluate(test['content'], test['label']))
print("Best performing model chosen hyper-parameters:")
print(best_run)
