def test_wrong_type_modify(self):
with open("metric.unknown", "w+") as fd:
fd.write("unknown")
fd.flush()
ret = main(["run", "-m", "metric.unknown"])
self.assertEqual(ret, 0)
with MockLoggerHandlers(logger.logger):
reset_logger_error_output()
ret = main(
["metrics", "modify", "metric.unknown", "-t", "unknown"]
)
self.assertEqual(ret, 1)
self.assertIn(
"failed to modify metric file settings - metric type "
"'unknown' is not supported, must be one of "
"[raw, json, csv, tsv, hcsv, htsv]",
logger.logger.handlers[1].stream.getvalue(),
)
ret = main(["metrics", "modify", "metric.unknown", "-t", "CSV"])
self.assertEqual(ret, 0)
reset_logger_standard_output()
ret = main(["metrics", "show", "metric.unknown"])
self.assertEqual(ret, 0)
self.assertIn(
"\tmetric.unknown: unknown",
logger.logger.handlers[0].stream.getvalue(),
)
def test_wrong_type_show(self):
with open("metric.unknown", "w+") as fd:
fd.write("unknown")
fd.flush()
ret = main(["run", "-m", "metric.unknown"])
self.assertEqual(ret, 0)
with MockLoggerHandlers(logger.logger):
reset_logger_standard_output()
ret = main(
[
"metrics",
"show",
"metric.unknown",
"-t",
"unknown",
"-x",
"0,0",
]
)
self.assertEqual(ret, 0)
self.assertIn(
"\tmetric.unknown: unknown",
logger.logger.handlers[0].stream.getvalue(),
)
def test(self):
metrics_file = "metrics_file"
metrics_value = 0.123489015
ret = main([
"run",
"-m",
metrics_file,
"echo {} >> {}".format(metrics_value, metrics_file),
])
self.assertEqual(0, ret)
with MockLoggerHandlers(logger):
reset_logger_standard_output()
ret = main([
"repro",
"--force",
"--metrics",
metrics_file + Stage.STAGE_FILE_SUFFIX,
])
self.assertEqual(0, ret)
expected_metrics_display = "{}: {}".format(metrics_file,
metrics_value)
self.assertIn(expected_metrics_display,
logger.handlers[0].stream.getvalue())
def setUp(self):
logger.logger.handlers = [
logger.logging.StreamHandler(),
logger.logging.StreamHandler(),
]
reset_logger_standard_output()
reset_logger_error_output()
logger.set_default_level()
self.consoleColorRemover = ConsoleFontColorsRemover()
self.consoleColorRemover.__enter__()
def _test(self):
url = get_local_url()
self.main(["remote", "add", "-d", TEST_REMOTE, url])
stage = self.dvc.run(outs=["bar"], cmd="echo bar > bar")
self.main(["push"])
stage_file_path = stage.relpath
content = load_stage_file(stage_file_path)
del content["outs"][0]["md5"]
with open(stage_file_path, "w") as stage_file:
yaml.dump(content, stage_file)
with MockLoggerHandlers(logger):
reset_logger_standard_output()
self.main(["status", "-c"])
self.assertIn(
"Warning: Output 'bar'(Stage: 'bar.dvc') is "
"missing version info. Cache for it will not be "
"collected. Use dvc repro to get your pipeline up to "
"date.",
logger.handlers[0].stream.getvalue(),
)
def test(self):
ret = main(["add", self.FOO, self.BAR])
self.assertEqual(ret, 0)
with MockLoggerHandlers(logger), ConsoleFontColorsRemover():
reset_logger_standard_output()
ret = main(["tag", "list"])
self.assertEqual(ret, 0)
self.assertEqual("", logger.handlers[0].stream.getvalue())
ret = main(["tag", "add", "v1"])
self.assertEqual(ret, 0)
with MockLoggerHandlers(logger), ConsoleFontColorsRemover():
reset_logger_standard_output()
ret = main(["tag", "list"])
self.assertEqual(ret, 0)
self.assertEqual(
logger.handlers[0].stream.getvalue(),
"bar.dvc:\n"
" bar:\n"
" v1:\n"
" md5: 8978c98bb5a48c2fb5f2c4c905768afa\n"
"foo.dvc:\n"
" foo:\n"
" v1:\n"
" md5: acbd18db4cc2f85cedef654fccc4a4d8\n"
"\n",
)
ret = main(["tag", "remove", "v1"])
self.assertEqual(ret, 0)
with MockLoggerHandlers(logger), ConsoleFontColorsRemover():
reset_logger_standard_output()
ret = main(["tag", "list"])
self.assertEqual(ret, 0)
self.assertEqual("", logger.handlers[0].stream.getvalue())
def test_formatted_output(self):
with open("metrics.csv", "w") as fobj:
# Labels are in Spanish to test unicode characters
fobj.write("valor_mse,desviación_mse,data_set\n"
"0.421601,0.173461,entrenamiento\n"
"0.67528,0.289545,pruebas\n"
"0.671502,0.297848,validación\n")
with open("metrics.tsv", "w") as fobj:
# Contains quoted newlines to test output correctness
fobj.write("value_mse\tdeviation_mse\tdata_set\n"
"0.421601\t0.173461\ttrain\n"
'0.67528\t0.289545\t"test\\ning"\n'
"0.671502\t0.297848\tvalidation\n")
with open("metrics.json", "w") as fobj:
fobj.write("{\n"
' "data_set": [\n'
' "train",\n'
' "testing",\n'
' "validation"\n'
" ],\n"
' "deviation_mse": [\n'
' "0.173461",\n'
' "0.289545",\n'
' "0.297848"\n'
" ],\n"
' "value_mse": [\n'
' "0.421601",\n'
' "0.67528",\n'
' "0.671502"\n'
" ]\n"
"}")
with open("metrics.txt", "w") as fobj:
fobj.write("ROC_AUC: 0.64\nKS: 78.9999999996\nF_SCORE: 77\n")
self.dvc.run(
fname="testing_metrics_output.dvc",
metrics_no_cache=[
"metrics.csv",
"metrics.tsv",
"metrics.json",
"metrics.txt",
],
)
self.dvc.metrics.modify("metrics.csv", typ="csv")
self.dvc.metrics.modify("metrics.tsv", typ="tsv")
self.dvc.metrics.modify("metrics.json", typ="json")
with MockLoggerHandlers(logger.logger):
reset_logger_standard_output()
ret = main(["metrics", "show"])
self.assertEqual(ret, 0)
expected_csv = (
u"\tmetrics.csv:\n"
u"\t\tvalor_mse desviación_mse data_set \n"
u"\t\t0.421601 0.173461 entrenamiento \n"
u"\t\t0.67528 0.289545 pruebas \n"
u"\t\t0.671502 0.297848 validación")
expected_tsv = ("\tmetrics.tsv:\n"
"\t\tvalue_mse deviation_mse data_set \n"
"\t\t0.421601 0.173461 train \n"
"\t\t0.67528 0.289545 test\\ning \n"
"\t\t0.671502 0.297848 validation")
expected_txt = ("\tmetrics.txt:\n"
"\t\tROC_AUC: 0.64\n"
"\t\tKS: 78.9999999996\n"
"\t\tF_SCORE: 77\n")
expected_json = ("\tmetrics.json:\n"
"\t\t{\n"
'\t\t "data_set": [\n'
'\t\t "train",\n'
'\t\t "testing",\n'
'\t\t "validation"\n'
"\t\t ],\n"
'\t\t "deviation_mse": [\n'
'\t\t "0.173461",\n'
'\t\t "0.289545",\n'
'\t\t "0.297848"\n'
"\t\t ],\n"
'\t\t "value_mse": [\n'
'\t\t "0.421601",\n'
'\t\t "0.67528",\n'
'\t\t "0.671502"\n'
"\t\t ]\n"
"\t\t}")
stdout = logger.logger.handlers[0].stream.getvalue()
self.assertIn(expected_tsv, stdout)
self.assertIn(expected_csv, stdout)
self.assertIn(expected_txt, stdout)
self.assertIn(expected_json, stdout)
