host=args.db_server,
port=args.db_port,
user=args.db_user,
passwd=args.db_pass,
db=args.db_name,
)
db_reporter = report.DbReport(
db,
args.db_table,
{
"FilesNumber": "INT UNSIGNED NOT NULL",
"FragmentSize": "BIGINT UNSIGNED NOT NULL",
"BenchName": "VARCHAR(500) NOT NULL",
"BestExecTimeMS": "BIGINT UNSIGNED",
"BestTotalTimeMS": "BIGINT UNSIGNED",
"WorstExecTimeMS": "BIGINT UNSIGNED",
"WorstTotalTimeMS": "BIGINT UNSIGNED",
"AverageExecTimeMS": "BIGINT UNSIGNED",
"AverageTotalTimeMS": "BIGINT UNSIGNED",
},
{
"ScriptName": "mortgage_pandas.py",
"CommitHash": args.commit
},
)
data_directory = args.dp
benchName = "mortgage"
perf_data_path = os.path.join(data_directory, "perf")
perf_format_path = os.path.join(perf_data_path, "Performance_%sQ%s.txt")
schema = ibis.Schema(
names = ["trip_id","vendor_id","pickup_datetime","dropoff_datetime","store_and_fwd_flag","rate_code_id","pickup_longitude","pickup_latitude","dropoff_longitude","dropoff_latitude","passenger_count","trip_distance","fare_amount","extra","mta_tax","tip_amount","tolls_amount","ehail_fee","improvement_surcharge","total_amount","payment_type","trip_type","pickup","dropoff","cab_type","precipitation","snow_depth","snowfall","max_temperature","min_temperature","average_wind_speed","pickup_nyct2010_gid","pickup_ctlabel","pickup_borocode","pickup_boroname","pickup_ct2010","pickup_boroct2010","pickup_cdeligibil","pickup_ntacode","pickup_ntaname","pickup_puma","dropoff_nyct2010_gid","dropoff_ctlabel","dropoff_borocode","dropoff_boroname","dropoff_ct2010","dropoff_boroct2010","dropoff_cdeligibil","dropoff_ntacode","dropoff_ntaname", "dropoff_puma"],
types = ['int32','string','timestamp','timestamp','string','int16','decimal','decimal','decimal','decimal','int16','decimal','decimal','decimal','decimal','decimal','decimal','decimal','decimal','decimal','string','int16','string','string','string','int16','int16','int16','int16','int16','int16','int16','string','int16','string','string','string','string','string','string','string','int16','string','int16','string','string','string','string','string','string','string']
)
db_reporter = None
if args.db_user is not "":
print("Connecting to database")
db = mysql.connector.connect(host=args.db_server, port=args.db_port, user=args.db_user, passwd=args.db_pass, db=args.db_name)
db_reporter = report.DbReport(db, args.db_table, {
'FilesNumber': 'INT UNSIGNED NOT NULL',
'QueryName': 'VARCHAR(500) NOT NULL',
'FirstExecTimeMS': 'BIGINT UNSIGNED',
'WorstExecTimeMS': 'BIGINT UNSIGNED',
'BestExecTimeMS': 'BIGINT UNSIGNED',
'AverageExecTimeMS': 'BIGINT UNSIGNED',
'TotalTimeMS': 'BIGINT UNSIGNED'
}, {
'ScriptName': 'taxibench_ibis.py',
'CommitHash': args.commit
})
# Delete old table
if not args.dnd:
print("Deleting", database_name ,"old database")
try:
conn.drop_database(database_name, force=True)
time.sleep(2)
conn = omnisci_server.connect_to_server()
except Exception as err:
print("Failed to delete", database_name, "old database: ", err)
db_reporter = None
if args.db_user != "":
print("Connecting to database")
db = mysql.connector.connect(
host=args.db_server,
port=args.db_port,
user=args.db_user,
passwd=args.db_pass,
db=args.db_name,
)
db_reporter = report.DbReport(
db,
"taxibench",
{
"FilesNumber": "INT UNSIGNED NOT NULL",
"FragmentSize": "BIGINT UNSIGNED NOT NULL",
"BenchName": "VARCHAR(500) NOT NULL",
"BestExecTimeMS": "BIGINT UNSIGNED",
"BestTotalTimeMS": "BIGINT UNSIGNED",
},
{"ScriptName": "taxibench.py", "CommitHash": args.commit},
)
for fs in args.fs:
print("RUNNING WITH FRAGMENT SIZE", fs)
# Delete old table
if not args.dnd:
print("Deleting taxitestdb old database")
try:
process = subprocess.Popen(
omnisciCmdLine,
stdout=subprocess.PIPE,
db_reporter = None
if args.db_user is not "":
print("Connecting to database")
db = mysql.connector.connect(host=args.db_server,
port=args.db_port,
user=args.db_user,
passwd=args.db_pass,
db=args.db_name)
db_reporter = report.DbReport(
db, args.db_table, {
'FilesNumber': 'INT UNSIGNED NOT NULL',
'FragmentSize': 'BIGINT UNSIGNED NOT NULL',
'BenchName': 'VARCHAR(500) NOT NULL',
'BestExecTimeMS': 'BIGINT UNSIGNED',
'BestTotalTimeMS': 'BIGINT UNSIGNED',
'WorstExecTimeMS': 'BIGINT UNSIGNED',
'WorstTotalTimeMS': 'BIGINT UNSIGNED',
'AverageExecTimeMS': 'BIGINT UNSIGNED',
'AverageTotalTimeMS': 'BIGINT UNSIGNED'
}, {
'ScriptName': 'mortgage_pandas.py',
'CommitHash': args.commit
})
data_directory = args.dp
benchName = "mortgage"
perf_data_path = os.path.join(data_directory, "perf")
perf_format_path = os.path.join(perf_data_path, "Performance_%sQ%s.txt")
bestExecTime = float("inf")
bestTotalTime = float("inf")
omnisciCmdLine = [args.e] + omnisciCmdLine + ["--port", str(args.port)]
db_reporter = None
if args.db_user is not "":
print("Connecting to database")
db = mysql.connector.connect(host=args.db_server,
port=args.db_port,
user=args.db_user,
passwd=args.db_pass,
db=args.db_name)
db_reporter = report.DbReport(
db, "taxibench", {
'FilesNumber': 'INT UNSIGNED NOT NULL',
'FragmentSize': 'BIGINT UNSIGNED NOT NULL',
'BenchName': 'VARCHAR(500) NOT NULL',
'BestExecTimeMS': 'BIGINT UNSIGNED',
'BestTotalTimeMS': 'BIGINT UNSIGNED'
}, {
'ScriptName': 'taxibench.py',
'CommitHash': args.commit
})
for fs in args.fs:
print("RUNNING WITH FRAGMENT SIZE", fs)
# Delete old table
if not args.dnd:
print("Deleting taxitestdb old database")
try:
process = subprocess.Popen(omnisciCmdLine,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,