def test_default(self):
""" Test of a plain standard table. """
with open('input/normal.csv') as f:
to_convert = [row for row in csv.reader(f)]
with open('output/normal.md') as f:
expected = f.read().strip()
self.assertEqual(csvtomd.md_table(to_convert), expected)
def test_jagged(self):
""" Test of a CSV with inconsistent number of fields on each line. """
with open('input/jagged.csv') as f:
to_convert = [row for row in csv.reader(f)]
with open('output/jagged.md') as f:
expected = f.read()
self.assertEqual(csvtomd.md_table(to_convert, padding=2), expected)
def test_default(self):
""" Test of a plain standard table. """
with open('input/normal.csv') as f:
to_convert = [row for row in csv.reader(f)]
with open('output/normal.md') as f:
expected = f.read().strip()
self.assertEqual(csvtomd.md_table(to_convert), expected)
def test_remove_blank_lines(self):
""" Test of a CSV with blank lines, tests both removing the blank
lines and not removing the blank lines.
"""
with open('input/blank_lines.csv') as f:
to_convert = [row for row in csv.reader(f) if len(row) > 0]
with open('output/remove_blank_lines.md') as f:
expected = f.read()
self.assertEqual(csvtomd.md_table(to_convert, padding=2), expected)
def _fn(self, arr):
from csvtomd import md_table
from math import isnan
if len(self.args) > 1:
args = [open(self.args[0], "w")]
else:
args = []
table = []
first = True
for row in map(result_cleaner, arr):
logger.info("Writing row %s", row)
if first:
table.append([str(v) if v else "" for v in row.keys()])
first = False
table.append(
[str(v) if isinstance(v, str) or not isnan(v) else "" for v in row.values()]
)
if len(args):
args[0].write(md_table(table, **self.kwargs) + "\n")
else:
print(md_table(table, **self.kwargs))
def main(argv):
thispath = os.path.dirname(os.path.abspath(__file__))
root_dir = os.path.abspath(thispath + "/../../")
scripts_dir = os.path.abspath(thispath + "/..")
ecs_benchmark_cmd = os.path.abspath(root_dir + "/build/app/ecs_benchmark")
doc_dir = os.path.abspath(root_dir + "/doc")
doc_csv_dir = os.path.abspath(doc_dir + "/csv")
dateinfo = os.popen("date").read()
dateinfo = dateinfo.replace("\n", '')
osinfo = os.popen("uname -o -r -m").read()
osinfo = osinfo.replace("\n", '')
cpuinfo = os.popen("lscpu -y | grep -i 'Model name'").read()
cpuinfo = cpuinfo.replace('Model name:', '').strip()
cpuinfo = cpuinfo.replace("\n", '')
raminfo = os.popen("lsmem | grep -i 'Total online memory'").read()
raminfo = raminfo.replace('Total online memory:', '').strip()
raminfo = raminfo.replace("\n", '')
print(dateinfo)
print("OS: " + osinfo)
print("CPU: " + cpuinfo)
print("RAM: " + raminfo)
print("\n")
## use /usr/bin/time for more measurement
format = 'real %e user %U sys %S %E elapsed CPU: %P Max.Memory: %Mk'
time_cmd = ''
if (os.path.exists('/usr/bin/time')):
time_cmd = '/usr/bin/time -f "' + format + '" '
if config["simplebenchmark"]:
print("Run Simple Benchmarks\n")
updatebenchmarks = config["update"]
if config["simplebenchmark"]:
updatebenchmarks = config["updatelong"]
for fname in updatebenchmarks:
cmd = time_cmd + ' ' + ecs_benchmark_cmd
cmd = cmd + ' --bench ".*\\s+' + fname + '\\s+.*update.*"'
print(cmd + "\n")
os.system(cmd)
print("\n")
print("\n")
cmd = time_cmd + ' ' + ecs_benchmark_cmd
for fname in config["eventbus"]:
cmd = cmd + ' --bench ".*\\s+' + fname + '-eventbus\\s+.*" '
print(cmd + "\n")
os.system(cmd)
print("\n")
print("\n")
cmd = time_cmd + ' ' + ecs_benchmark_cmd
for fname in config["10Mentities"]:
cmd = cmd + ' --bench ".*\\s+' + fname + '\\s+.*10M\\s+entities.*" '
print(cmd + "\n")
os.system(cmd)
print("\n")
print("\n")
csvfiles = {}
csvfiles["update"] = ""
csvfiles["updatelong"] = ""
csvfiles["eventbus"] = ""
csvfiles["10Mentities"] = ""
csvfiles["updatemorecomplex"] = ""
datfiles = {}
datfiles["update"] = ""
datfiles["updatelong"] = ""
datfiles["eventbus"] = ""
datfiles["updatemorecomplex"] = ""
datfiles["update"] = os.path.abspath(doc_dir + "/data-systems-update.dat")
cmd = ecs_benchmark_cmd
for fname in config["update"]:
cmd = cmd + ' --bench ".*\\s+' + fname + '\\s+.*update.*" '
cmd = cmd + ' --colwidth=30 '
if config["gencsvfiles"]:
cmd = cmd + ' --csvoutput=' + doc_csv_dir + ' --csvprefix=update --csvunit seconds '
csvfiles["update"] = os.path.abspath(doc_csv_dir + "/update.csv")
csvfiles["printupdate"] = os.path.abspath(doc_csv_dir +
"/print.update.csv")
if config["plot"]:
cmd = cmd + ' --plotdata > ' + datfiles["update"]
if config["benchmark"] and (
(not 'runbenchmarkupdate' in config or
('runbenchmarkupdate' in config and config["runbenchmarkupdate"])) and
(config["gencsvfiles"] or config["plot"])):
print(cmd + "\n")
os.system(cmd)
print("\n")
print("\n")
datfiles["updatelong"] = os.path.abspath(doc_dir +
"/data-systems-update-long.dat")
cmd = ecs_benchmark_cmd
for fname in config["updatelong"]:
cmd = cmd + ' --bench ".*\\s+' + fname + '\\s+.*update.*" '
cmd = cmd + ' --colwidth=30 '
if config["gencsvfiles"]:
cmd = cmd + ' --csvoutput=' + doc_csv_dir + ' --csvprefix=updatelong --csvunit seconds '
csvfiles["updatelong"] = os.path.abspath(doc_csv_dir + "/updatelong.csv")
csvfiles["printupdatelong"] = os.path.abspath(doc_csv_dir +
"/print.updatelong.csv")
if config["plot"]:
cmd = cmd + ' --plotdata > ' + datfiles["updatelong"]
if config["benchmark"] and (config["runbenchmarkupdatelong"] and
(config["gencsvfiles"] or config["plot"])):
print(cmd + "\n")
os.system(cmd)
print("\n")
print("\n")
datfiles["eventbus"] = os.path.abspath(doc_dir + "/data-eventbus.dat")
cmd = ecs_benchmark_cmd
for fname in config["eventbus"]:
cmd = cmd + ' --bench ".*\\s+' + fname + '-eventbus\\s+.*" '
cmd = cmd + ' --colwidth=30 '
if config["gencsvfiles"]:
cmd = cmd + ' --csvoutput=' + doc_csv_dir + ' --csvprefix=eventbus --csvunit seconds '
csvfiles["eventbus"] = os.path.abspath(doc_csv_dir + "/eventbus.csv")
csvfiles["printeventbus"] = os.path.abspath(doc_csv_dir +
"/print.eventbus.csv")
if config["plot"]:
cmd = cmd + ' --plotdata > ' + datfiles["eventbus"]
if config["benchmark"] and (
(not 'runbenchmarkupdate' in config or
('runbenchmarkeventbus' in config and config["runbenchmarkeventbus"]))
and (config["gencsvfiles"] or config["plot"])):
print(cmd + "\n")
os.system(cmd)
print("\n")
print("\n")
cmd = ecs_benchmark_cmd
for fname in config["10Mentities"]:
cmd = cmd + ' --bench ".*\\s+' + fname + '\\s+.*10M\\s+entities.*" '
cmd = cmd + ' --colwidth=30 '
if config["gencsvfiles"]:
cmd = cmd + ' --csvoutput=' + doc_csv_dir + ' --csvprefix=10Mentities --csvunit seconds '
csvfiles["10Mentities"] = os.path.abspath(doc_csv_dir + "/10Mentities.csv")
csvfiles["print10Mentities"] = os.path.abspath(doc_csv_dir +
"/print.10Mentities.csv")
if config["benchmark"] and ((not 'runbenchmark10Mentities' in config or
('runbenchmark10Mentities' in config
and config["runbenchmark10Mentities"]))
and config["gencsvfiles"]):
print(cmd + "\n")
os.system(cmd)
print("\n")
print("\n")
datfiles["updatemorecomplex"] = os.path.abspath(
doc_dir + "/data-systems-update-morecomplex.dat")
cmd = ecs_benchmark_cmd
for fname in config["updatemorecomplex"]:
cmd = cmd + ' --bench ".*\\s+' + fname + '\\s+.*update.*" '
cmd = cmd + ' --colwidth=30 '
if config["gencsvfiles"]:
cmd = cmd + ' --csvoutput=' + doc_csv_dir + ' --csvprefix=updatemorecomplex --csvunit seconds '
csvfiles["updatemorecomplex"] = os.path.abspath(doc_csv_dir +
"/updatemorecomplex.csv")
csvfiles["printupdatemorecomplex"] = os.path.abspath(
doc_csv_dir + "/print.updatemorecomplex.csv")
if config["plot"]:
cmd = cmd + ' --plotdata > ' + datfiles["updatemorecomplex"]
if config["benchmark"] and (config["runbenchmarkupdatemorecomplex"] and
(config["gencsvfiles"] or config["plot"])):
print(cmd + "\n")
os.system(cmd)
print("\n")
print("\n")
if config["gencsvfiles"]:
if csvfiles["update"]:
updateCSV(csvfiles["update"], "update", replaceColUpdate,
csvfiles["printupdate"])
print("CSV generate: {}".format(csvfiles["printupdate"]))
if csvfiles["updatelong"]:
updateCSV(csvfiles["updatelong"], "update", replaceColUpdateLong,
csvfiles["printupdatelong"])
print("CSV generate: {}".format(csvfiles["printupdatelong"]))
if csvfiles["eventbus"]:
updateCSV(csvfiles["eventbus"], "-eventbus", replaceColEventbus,
csvfiles["printeventbus"])
print("CSV generate: {}".format(csvfiles["printeventbus"]))
if csvfiles["10Mentities"]:
updateCSV(csvfiles["10Mentities"], "10Mentities",
replaceCol10Mentities, csvfiles["print10Mentities"])
print("CSV generate: {}".format(csvfiles["print10Mentities"]))
if csvfiles["updatemorecomplex"]:
updateCSV(csvfiles["updatemorecomplex"], "-morecomplexupdate",
replaceColUpdateMoreComplex,
csvfiles["printupdatemorecomplex"])
print("CSV generate: {}".format(
csvfiles["printupdatemorecomplex"]))
if config["plot"]:
pltfiles = {}
pltfiles['update'] = makePlotScriptFromCSV(
'update', 'ECS Benchmark System Updates', thispath, scripts_dir,
doc_dir, datfiles["update"], csvfiles["printupdate"])
if config["runbenchmarkupdatelong"]:
pltfiles['updatelong'] = makePlotScriptFromCSV(
'updatelong', 'ECS Benchmark System Updates', thispath,
scripts_dir, doc_dir, datfiles["updatelong"],
csvfiles["printupdatelong"])
pltfiles['eventbus'] = makePlotScriptFromCSV(
'eventbus', 'ECS Benchmark Eventbus', thispath, scripts_dir,
doc_dir, datfiles["eventbus"], csvfiles["printeventbus"])
if config["runbenchmarkupdatemorecomplex"]:
pltfiles['updatemorecomplex'] = makePlotScriptFromCSV(
'updatemorecomplex', 'ECS Benchmark System Updates', thispath,
scripts_dir, doc_dir, datfiles["updatemorecomplex"],
csvfiles["printupdatemorecomplex"])
os.chdir(doc_dir)
for frameworkname, pltfile in pltfiles.items():
if frameworkname in datfiles:
print("gnuplot with data {} and plotscript {}".format(
datfiles[frameworkname], pltfile))
os.system('gnuplot ' + pltfile)
os.chdir(root_dir)
if config["genreadme"]:
mdTable10MEntities = ''
if os.path.exists(csvfiles['print10Mentities']):
with open(csvfiles['print10Mentities'], 'r') as f:
table10MEntities = csv_to_table(f, CSV_DELIMITER,
CSV_QUOTECHAR)
mdTable10MEntities = md_table(table10MEntities)
mdTableUpdate = ''
if os.path.exists(csvfiles['printupdate']):
with open(csvfiles['printupdate'], 'r') as f:
tableUpdate = csv_to_table(f, CSV_DELIMITER, CSV_QUOTECHAR)
mdTableUpdate = md_table(tableUpdate)
mdTableUpdateLong = ''
if os.path.exists(csvfiles['printupdatelong']):
with open(csvfiles['printupdatelong'], 'r') as f:
tableUpdateLong = csv_to_table(f, CSV_DELIMITER, CSV_QUOTECHAR)
mdTableUpdateLong = md_table(tableUpdateLong)
mdTableEventbus = ''
if os.path.exists(csvfiles['eventbus']):
with open(csvfiles['printeventbus'], 'r') as f:
tableEventbus = csv_to_table(f, CSV_DELIMITER, CSV_QUOTECHAR)
mdTableEventbus = md_table(tableEventbus)
mdTableUpdateMoreComplex = ''
if os.path.exists(csvfiles['printupdatemorecomplex']):
with open(csvfiles['printupdatemorecomplex'], 'r') as f:
tableUpdateMoreComplex = csv_to_table(f, CSV_DELIMITER,
CSV_QUOTECHAR)
mdTableUpdateMoreComplex = md_table(tableUpdateMoreComplex)
mdTableResult = ''
if config["runbenchmarkupdatelong"]:
mdTableResult = mdTableUpdateLong
else:
mdTableResult = mdTableUpdate
git_doc_dir = 'https://raw.githubusercontent.com/abeimler/ecs_benchmark/develop/doc/'
pngs = {}
pngs['update'] = git_doc_dir + "{}.png".format('update')
if config["runbenchmarkupdatelong"]:
pngs['updatelong'] = git_doc_dir + "{}.png".format('updatelong')
pngs['results'] = pngs['updatelong']
else:
pngs['results'] = pngs['update']
if config["runbenchmarkupdatemorecomplex"]:
pngs['updatemorecomplex'] = git_doc_dir + "{}.png".format(
'updatemorecomplex')
pngs['eventbus'] = git_doc_dir + "{}.png".format('eventbus')
renderinfo = []
for key, value in config['info'].items():
if 'framework' in value and value['framework']:
newvalue = value
newvalue['key'] = key
if 'description' in value:
newdescription = value['description'].split('\n')
newvalue['description'] = newdescription
renderinfo.append(newvalue)
params = {
'dateinfo': "{:%d. %b %Y}".format(datetime.date.today()),
'osinfo': osinfo,
'cpuinfo': cpuinfo,
'raminfo': raminfo,
'table10MEntities': mdTable10MEntities,
'tableUpdate': mdTableUpdate,
'tableUpdateLong': mdTableUpdateLong,
'tableEventbus': mdTableEventbus,
'tableResult': mdTableResult,
'tableUpdateMoreComplex': mdTableUpdateMoreComplex,
'config': config,
'info': renderinfo,
'pngs': pngs
}
readmetmpl = ''
readme_tmpl = os.path.abspath(thispath + "/README.md.mustache")
with open(readme_tmpl, 'r') as readmetmplfile:
readmetmpl = readmetmplfile.read()
readme = pystache.render(readmetmpl, params)
readmefilename = root_dir + "/README.md"
with open(readmefilename, "w") as readmefile:
readmefile.write(readme)
print("generate README: {}".format(readmefilename))
# print(file=open(readmefile))
print("# TSPerf\n", file=open(readmefile, "w"))
print(
"TSPerf is a collection of implementations of time-series forecasting algorithms in Azure cloud and comparison of their performance over benchmark datasets. \
Algorithm implementations are compared by model accuracy, training and scoring time and cost. Each implementation includes all the necessary \
instructions and tools that ensure its reproducibility.",
file=open(readmefile, "a"),
)
print("The following table summarizes benchmarks that are currently included in TSPerf.\n", file=open(readmefile, "a"))
# Read the benchmark table the CSV file and converrt to a table in md format
with open("Benchmarks.csv", "r") as f:
table = csvtomd.csv_to_table(f, ",")
print(csvtomd.md_table(table), file=open(readmefile, "a"))
print("\n\n\n", file=open(readmefile, "a"))
print(
"A complete documentation of TSPerf, along with the instructions for submitting and reviewing implementations, \
can be found [here](./docs/tsperf_rules.md). The tables below show performance of implementations that are developed so far. Source code of \
implementations and instructions for reproducing their performance can be found in submission folders, which are linked in the first column.\n",
file=open(readmefile, "a"),
)
### Write the Energy section
# ============================
print("## Probabilistic energy forecasting performance board\n\n", file=open(readmefile, "a"))
print(
"The following table lists the current submision for the energy forecasting and their respective performances.\n\n",
def test_default():
with open('test/input/normal.csv') as f:
to_convert = [row for row in reader(f)]
with open('test/output/normal.md') as f:
expected = f.read().strip()
md_table(to_convert).should.equal(expected)