def test_pre_run_check_bad_credentials(self, mock_boto_client) -> None:
mock_boto_client.side_effect = ClientError(
error_response=mock_boto_client, operation_name=mock_boto_client)
# returns false if we have an upload key in yaml
return_val = pre_run_checks(self.upload_yaml,
check_s3_credentials=True)
self.assertFalse(return_val)
self.assertTrue(mock_boto_client.called)
def test_pre_run_check_bad_credentials_but_no_check(
self, mock_boto_client) -> None:
mock_boto_client.side_effect = ClientError(
error_response=mock_boto_client, operation_name=mock_boto_client)
return_val = pre_run_checks(
YamlHelper('tests/resources/test_no_upload.yaml'),
check_s3_credentials=False)
# returns true if bad creds, but we don't want to check credentials
self.assertTrue(return_val)
def test_pre_run_check_confirm_boto_called(self, mock_boto_client) -> None:
return_val = pre_run_checks(self.upload_yaml,
check_s3_credentials=True)
self.assertTrue(mock_boto_client.called)
def run(config: str) -> None:
"""Run a NEAT pipeline using the given YAML file [neat.yaml]
\f
Args:
config: Specify the YAML file containing instructions of what ML tasks to perform
Returns:
None.
"""
yhelp = YamlHelper(config)
# pre run checks for failing early
if not pre_run_checks(yhelp=yhelp):
raise RuntimeError("Failed pre_run_check")
# generate embeddings if config has 'embeddings' block
if yhelp.do_embeddings() and not os.path.exists(yhelp.embedding_outfile()):
node_embedding_args = yhelp.make_node_embeddings_args()
make_node_embeddings(**node_embedding_args)
if yhelp.do_tsne() and not os.path.exists(yhelp.tsne_outfile()):
graph: Graph = yhelp.load_graph()
tsne_kwargs = yhelp.make_tsne_args(graph)
make_tsne(**tsne_kwargs)
if yhelp.do_classifier():
for classifier in tqdm(yhelp.classifiers()):
model: object = None
if classifier['type'] == 'neural network':
model = MLPModel(classifier, outdir=yhelp.outdir())
elif classifier['type'] in \
['Decision Tree', 'Logistic Regression', 'Random Forest']:
model = SklearnModel(classifier, outdir=yhelp.outdir())
else:
raise NotImplementedError(f"{model} isn't implemented yet")
model.compile()
train_data, validation_data = \
model.make_link_prediction_data(yhelp.embedding_outfile(),
yhelp.main_graph_args(),
yhelp.pos_val_graph_args(),
yhelp.neg_train_graph_args(),
yhelp.neg_val_graph_args(),
yhelp.edge_embedding_method())
history_obj = model.fit(train_data, validation_data)
if yhelp.classifier_history_file_name(classifier):
with open(yhelp.classifier_history_file_name(classifier),
'w') as f: # type: ignore
json.dump(history_obj.history, f)
model.save()
if yhelp.do_upload():
upload_kwargs = yhelp.make_upload_args()
upload_dir_to_s3(**upload_kwargs)
return None