def main(argv):
# Parses the command-line arguments
parser = argparse.ArgumentParser()
parser.add_argument(
"source_folder",
help="The folder with JSON files to parse to add to the database")
parser.add_argument("clblast_root", help="Root of the CLBlast sources")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="Increase verbosity of the script")
cl_args = parser.parse_args(argv)
# Parses the path arguments
database_filename = os.path.join(cl_args.clblast_root, "scripts",
"database", "database.json")
database_best_filename = os.path.join(cl_args.clblast_root, "scripts",
"database", "database_best.json")
json_files = os.path.join(cl_args.source_folder, "*.json")
cpp_database_path = os.path.join(cl_args.clblast_root, "src", "database",
"kernels")
# Checks whether the command-line arguments are valid
clblast_header = os.path.join(
cl_args.clblast_root, "include",
"clblast.h") # Not used but just for validation
if not os.path.isfile(clblast_header):
raise RuntimeError(
"The path '" + cl_args.clblast_root +
"' does not point to the root of the CLBlast library")
if len(glob.glob(json_files)) < 1:
print("[database] The path '" + cl_args.source_folder +
"' does not contain any JSON files")
# Downloads the database if a local copy is not present
if not os.path.isfile(database_filename):
io.download_database(database_filename, DATABASE_SERVER_URL)
# Loads the database from disk
database = io.load_database(database_filename)
# Loops over all JSON files in the supplied folder
for file_json in glob.glob(json_files):
# Loads the newly imported data
sys.stdout.write("[database] Processing '" + file_json +
"' ") # No newline printed
imported_data = io.load_tuning_results(file_json)
# Fixes the problem that some vendors use multiple different names
for target in VENDOR_TRANSLATION_TABLE:
if imported_data["device_vendor"] == target:
imported_data["device_vendor"] = VENDOR_TRANSLATION_TABLE[
target]
# Adds the new data to the database
old_size = db.length(database)
database = db.add_section(database, imported_data)
new_size = db.length(database)
print("with " + str(new_size - old_size) +
" new items") # Newline printed here
# Stores the modified database back to disk
if len(glob.glob(json_files)) >= 1:
io.save_database(database, database_filename)
# Retrieves the best performing results
print("[database] Calculating the best results per device/kernel...")
database_best_results = bests.get_best_results(database)
# Determines the defaults for other vendors and per vendor
print("[database] Calculating the default values...")
database_defaults = defaults.calculate_defaults(database, cl_args.verbose)
database_best_results["sections"].extend(database_defaults["sections"])
# Optionally outputs the database to disk
if cl_args.verbose:
io.save_database(database_best_results, database_best_filename)
# Outputs the database as a C++ database
print("[database] Producing a C++ database in '" + cpp_database_path +
"'...")
clblast.print_cpp_database(database_best_results, cpp_database_path)
print("[database] All done")
def main(argv):
# Parses the command-line arguments
parser = argparse.ArgumentParser()
parser.add_argument(
"source_folder",
help="The folder with JSON files to parse to add to the database")
parser.add_argument("clblast_root", help="Root of the CLBlast sources")
parser.add_argument("-r",
"--remove_device",
type=str,
default=None,
help="Removes all entries for a specific device")
parser.add_argument("--add_tuning_parameter",
type=str,
default=None,
help="Adds this parameter to existing entries")
parser.add_argument("--add_tuning_parameter_for_kernel",
type=str,
default=None,
help="Adds the above parameter for this kernel")
parser.add_argument(
"--add_tuning_parameter_value",
type=int,
default=0,
help="Set this value as the default for the above parameter")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="Increase verbosity of the script")
cl_args = parser.parse_args(argv)
# Parses the path arguments
database_filename = os.path.join(cl_args.clblast_root, "scripts",
"database", "database.json")
database_best_filename = os.path.join(cl_args.clblast_root, "scripts",
"database", "database_best.json")
json_files = os.path.join(cl_args.source_folder, "*.json")
cpp_database_path = os.path.join(cl_args.clblast_root, "src", "database",
"kernels")
# Checks whether the command-line arguments are valid
clblast_header = os.path.join(
cl_args.clblast_root, "include",
"clblast.h") # Not used but just for validation
if not os.path.isfile(clblast_header):
raise RuntimeError(
"The path '" + cl_args.clblast_root +
"' does not point to the root of the CLBlast library")
if len(glob.glob(json_files)) < 1:
print("[database] The path '" + cl_args.source_folder +
"' does not contain any JSON files")
# Downloads the database if a local copy is not present
if not os.path.isfile(database_filename):
io.download_database(database_filename, DATABASE_SERVER_URL)
# Loads the database from disk
database = io.load_database(database_filename)
# Loops over all JSON files in the supplied folder
for file_json in glob.glob(json_files):
sys.stdout.write("[database] Processing '" + file_json +
"' ") # No newline printed
try:
# Loads the newly imported data
imported_data = io.load_tuning_results(file_json)
# Adds the new data to the database
old_size = db.length(database)
database = db.add_section(database, imported_data)
new_size = db.length(database)
print("with " + str(new_size - old_size) +
" new items") # Newline printed here
except ValueError:
print("--- WARNING: invalid file, skipping")
# Checks for tuning results with mis-matched entries
remove_mismatched_arguments(database)
# Stores the modified database back to disk
if len(glob.glob(json_files)) >= 1:
io.save_database(database, database_filename)
# Removes database entries before continuing
if cl_args.remove_device is not None:
print("[database] Removing all results for device '%s'" %
cl_args.remove_device)
remove_database_entries(database,
{"clblast_device_name": cl_args.remove_device})
#, "kernel_family": "xgemm"})
io.save_database(database, database_filename)
# Adds new tuning parameters to existing database entries
if cl_args.add_tuning_parameter is not None and\
cl_args.add_tuning_parameter_for_kernel is not None:
print(
"[database] Adding tuning parameter: '%s' for kernel '%s' with default %d"
% (cl_args.add_tuning_parameter,
cl_args.add_tuning_parameter_for_kernel,
cl_args.add_tuning_parameter_value))
add_tuning_parameter(database, cl_args.add_tuning_parameter,
cl_args.add_tuning_parameter_for_kernel,
cl_args.add_tuning_parameter_value)
io.save_database(database, database_filename)
# Retrieves the best performing results
print("[database] Calculating the best results per device/kernel...")
database_best_results = bests.get_best_results(database)
# Determines the defaults for other vendors and per vendor
print("[database] Calculating the default values...")
database_defaults = defaults.calculate_defaults(database, cl_args.verbose)
database_best_results["sections"].extend(database_defaults["sections"])
# Optionally outputs the database to disk
if cl_args.verbose:
io.save_database(database_best_results, database_best_filename)
# Outputs the database as a C++ database
print("[database] Producing a C++ database in '" + cpp_database_path +
"'...")
clblast.print_cpp_database(database_best_results, cpp_database_path)
print("[database] All done")
def main(argv):
# Parses the command-line arguments
parser = argparse.ArgumentParser()
parser.add_argument("source_folder", help="The folder with JSON files to parse to add to the database")
parser.add_argument("clblast_root", help="Root of the CLBlast sources")
parser.add_argument("-r", "--remove_device", type=str, default=None, help="Removes all entries for a specific device")
parser.add_argument("--add_tuning_parameter", type=str, default=None, help="Adds this parameter to existing entries")
parser.add_argument("--add_tuning_parameter_for_kernel", type=str, default=None, help="Adds the above parameter for this kernel")
parser.add_argument("--add_tuning_parameter_value", type=int, default=0, help="Set this value as the default for the above parameter")
parser.add_argument("-v", "--verbose", action="store_true", help="Increase verbosity of the script")
cl_args = parser.parse_args(argv)
# Parses the path arguments
database_filename = os.path.join(cl_args.clblast_root, "scripts", "database", "database.json")
database_best_filename = os.path.join(cl_args.clblast_root, "scripts", "database", "database_best.json")
json_files = os.path.join(cl_args.source_folder, "*.json")
cpp_database_path = os.path.join(cl_args.clblast_root, "src", "database", "kernels")
# Checks whether the command-line arguments are valid
clblast_header = os.path.join(cl_args.clblast_root, "include", "clblast.h") # Not used but just for validation
if not os.path.isfile(clblast_header):
raise RuntimeError("The path '" + cl_args.clblast_root +
"' does not point to the root of the CLBlast library")
if len(glob.glob(json_files)) < 1:
print("[database] The path '" + cl_args.source_folder + "' does not contain any JSON files")
# Downloads the database if a local copy is not present
if not os.path.isfile(database_filename):
io.download_database(database_filename, DATABASE_SERVER_URL)
# Loads the database from disk
database = io.load_database(database_filename)
# Loops over all JSON files in the supplied folder
for file_json in glob.glob(json_files):
sys.stdout.write("[database] Processing '" + file_json + "' ") # No newline printed
try:
# Loads the newly imported data
imported_data = io.load_tuning_results(file_json)
# Adds the new data to the database
old_size = db.length(database)
database = db.add_section(database, imported_data)
new_size = db.length(database)
print("with " + str(new_size - old_size) + " new items") # Newline printed here
except ValueError:
print("--- WARNING: invalid file, skipping")
# Checks for tuning results with mis-matched entries
remove_mismatched_arguments(database)
# Stores the modified database back to disk
if len(glob.glob(json_files)) >= 1:
io.save_database(database, database_filename)
# Removes database entries before continuing
if cl_args.remove_device is not None:
print("[database] Removing all results for device '%s'" % cl_args.remove_device)
remove_database_entries(database, {"clblast_device_name": cl_args.remove_device})
#, "kernel_family": "xgemm"})
io.save_database(database, database_filename)
# Adds new tuning parameters to existing database entries
if cl_args.add_tuning_parameter is not None and\
cl_args.add_tuning_parameter_for_kernel is not None:
print("[database] Adding tuning parameter: '%s' for kernel '%s' with default %d" %
(cl_args.add_tuning_parameter, cl_args.add_tuning_parameter_for_kernel,
cl_args.add_tuning_parameter_value))
add_tuning_parameter(database, cl_args.add_tuning_parameter,
cl_args.add_tuning_parameter_for_kernel,
cl_args.add_tuning_parameter_value)
io.save_database(database, database_filename)
# Retrieves the best performing results
print("[database] Calculating the best results per device/kernel...")
database_best_results = bests.get_best_results(database)
# Determines the defaults for other vendors and per vendor
print("[database] Calculating the default values...")
database_defaults = defaults.calculate_defaults(database, cl_args.verbose)
database_best_results["sections"].extend(database_defaults["sections"])
# Optionally outputs the database to disk
if cl_args.verbose:
io.save_database(database_best_results, database_best_filename)
# Outputs the database as a C++ database
print("[database] Producing a C++ database in '" + cpp_database_path + "'...")
clblast.print_cpp_database(database_best_results, cpp_database_path)
print("[database] All done")
def ck_postprocess(i):
ck = i['ck_kernel']
rt = i['run_time']
deps = i['deps']
d = {}
env = i.get('env', {})
# Load both stderr and stdout. Concatenate into one list.
# NB: This assumes that Caffe iterates only once (--iterations=1).
# Otherwise, looping over the log would be required.
# For debugging
# rt['run_cmd_out1']='stdout.log'
# rt['run_cmd_out2']='stderr.log'
# rt['run_output_files']=["clblast_xgemm_1_32.json","clblast_xgemm_2_32.json"]
rf1 = rt['run_cmd_out1']
rf2 = rt['run_cmd_out2']
rf3 = rt['run_output_files'][0]
if len(rt['run_output_files']) > 1:
rf4 = rt['run_output_files'][1]
else:
rf4 = "None"
print("[postprocessing] Loading json output %s " % (rf3))
print("[postprocessing] Loading json output %s " % (rf4))
lst = []
r = {}
if os.path.isfile(rf1):
r = ck.load_text_file({'text_file': rf1, 'split_to_list': 'yes'})
if r['return'] > 0:
r['error'] = "Error loading rf1 file"
return r
lst += r['lst']
if os.path.isfile(rf2):
r = ck.load_text_file({'text_file': rf2, 'split_to_list': 'yes'})
if r['return'] > 0:
r['error'] = "Error loading rf2 file"
return r
lst += r['lst']
c1 = os.path.isfile(rf3)
c2 = os.path.isfile(rf4)
if c1:
rj1 = json.loads(open(rf3).read())
if c2:
rj2 = json.loads(open(rf4).read())
if ((c1 == 0) and (c2 == 0)):
r['return'] = 0
print("[postprocessing] Unable to read json output")
r['return'] = 1
r['error'] = 'Unable to read json output'
return r
if ((c1) == 0):
rj1 = rj2
#### CREATE UNIQUE OUTPUT
print("[postprocessing] Creating dictionary")
d['post_processed'] = 'yes'
# mydict['device_core_clock'] = 'value from script'
# SET ARCH INFORMATION
d['device_vendor'] = rj1['device_vendor']
d['device_type'] = rj1['device_type']
d['device'] = rj1['device']
d['device_compute_units'] = rj1['device_compute_units']
# Notice that often is not really accurated; TBC
d['device_core_clock'] = rj1['device_core_clock']
#Experiment Information
kern_family_len = len(rj1['kernel_family'].split('_'))
kernel_family = rj1['kernel_family'].split("_")[0]
if kern_family_len > 1:
if rj1['kernel_family'].split("_")[1] == "direct":
#concut again special case the kernal family of Xgemm_directNN is the same of other xgemm_direct
kernel_family = kernel_family + "_" + "direct"
d['kernel'] = kernel_family
if 'arg_beta' in d:
d['arg_beta'] = rj1['arg_beta']
if 'arg_m' in d:
d['arg_m'] = rj1['arg_m']
if 'arg_n' in d:
d['arg_n'] = rj1['arg_n']
if 'arg_k' in d:
d['arg_k'] = rj1['arg_k']
if 'arg_alpha' in d:
d['arg_alpha'] = rj1['arg_alpha']
d['precision'] = rj1['precision']
for target in VENDOR_TRANSLATION_TABLE:
if d["device_vendor"] == target:
d["device_vendor"] = VENDOR_TRANSLATION_TABLE[target]
#### Add results per strategy
# print "ADD RESULTs"
l = []
if (c1):
tmp = {'strategy': 'exhaustive', 'result': rj1['results']}
l.append(tmp)
if (c2):
tmp = {'strategy': 'random', 'result': rj2['results']}
l.append(tmp)
d['data'] = l
#GET DEFAULT VALUE from CLBlast
deps_cb = deps['lib-clblast']
b = deps_cb['cus']
pl = b['path_lib']
bench = d['kernel']
bench += ".hpp"
pl = pl.split("install")[0] #### VERIFY WITH INSTALL SCRIPT
pl_suff = "src" + os.sep + "scripts" + os.sep + "database" + os.sep
pl += pl_suff
pl1 = pl + "database" + os.sep
sys.path.append(pl)
sys.path.append(pl1)
####
import database.io as io
import database.db as db
import database.clblast as clblast
import database.bests as bests
import database.defaults as defaults
database_filename = pl + "database.json"
if os.path.isfile(database_filename) and os.path.getsize(
database_filename) == 0:
os.remove(database_filename)
if not os.path.isfile(database_filename):
io.download_database(database_filename, DATABASE_SERVER_URL)
else:
print("[database] DB found")
if os.path.isfile(database_filename):
database = io.load_database(database_filename)
# Retrieves the best performing results
print("[database] Calculating the best results per device/kernel...")
database_best_results = bests.get_best_results(database)
search_vendor = d['device_vendor']
search_device = d['device']
search_device_type = d['device_type']
search_kernel = d['kernel']
search_precision = d['precision']
# Determines the defaults for other vendors and per vendor
print("[database] Calculating the default values...")
database_defaults = defaults.calculate_defaults(
database, 0) #second args denotes VERBOSE or NOT
database_best_results["sections"].extend(database_defaults["sections"])
database_best_filename = 'database_best.json'
# Optionally outputs the database to disk
if VERBOSE:
io.save_database(database_best_results, database_best_filename)
#print("[database] Producing a C++ database in current dir...")
#clblast.print_cpp_database(database_best_results, ".")
#### TEST to get best and default param
best = database_best_results['sections']
ll = []
count = 0
index = -1
for best_entries in best:
#print best_entries['kernel_family'] + " " + search_kernel
if( (best_entries['kernel_family'] == search_kernel) and \
(best_entries['device_vendor'] == search_vendor) and \
(best_entries['precision'] == search_precision) and \
(best_entries['device_type']== search_device_type) and \
((best_entries['device'] == search_device) or (best_entries['device']=='default')) ):
if VERBOSE:
print("[postprocess] Best entry %s found" % (best_entries))
tmp = best_entries
ll.append(tmp)
##### Find Timing in d[data]
stat = []
for s in d['data']:
# print s.get('strategy', {})
result = s.get('result', {})
for rrr in result:
compare = rrr['parameters']
for il in ll:
best = il.get('results', {})[0].get('parameters', {})
dev = il['device']
### comparing value by value
isBest = True
for bb in best.keys():
# print bb, best[bb], compare[bb]
if (best[bb] != compare[bb]):
isBest = False
break
if (isBest):
rrr['device'] = dev
stat.append(rrr)
index = 0
bindex = -1
bres = {}
min_time = sys.float_info.max
for s in d['data']:
# print s.get('strategy', {})
result = s.get('result', {})
for i in result:
index += 1
if (i['time'] < min_time):
min_time = i['time']
bres = i
bindex = index
# print "Best performance: ", min_time,bres
l_m = 0
l_n = 0
l_k = 0
if 'arg_m' in d:
l_m = float(d["arg_m"])
if 'arg_n' in d:
l_n = float(d["arg_n"])
if 'arg_k' in d:
l_k = float(d["arg_k"])
m = l_m
n = l_n
k = l_k
if bres.get('time', '') != '':
gflops = 2.0 * m * n * k
time = float(bres["time"]) / 1000.0
gflops = gflops / time / 1000.0 / 1000.0 / 1000.0
bres['GFLOPS'] = gflops
bestdefstat = {}
defstat = {}
for i in stat:
time = float(i["time"]) / 1000.0
gflops = 2.0 * m * n * k
gflops = gflops / time / 1000.0 / 1000.0 / 1000.0
i['GFLOPS'] = gflops
if i['device'] == 'default':
defstat = i
else:
bestdefstat = i
d["statistics"] = {
'best_configuration': bres,
'default_configuration': defstat,
'default_family': bestdefstat
}
if VERBOSE:
print("[postprocessing] %s" % (d["statistics"]))
if len(ll) > 0:
if VERBOSE:
print(len(ll))
d['db'] = ll
else:
d['db'] = 'na'
rr = {}
rr['return'] = 0
# output_filename='tmp-ck-clblast-tune-'+d['kernel']+'-'+d['arg_m']+'-'+d['arg_n']+'-'+d['arg_k']+'-'+d['precision'] +'.json'
output_filename = 'tmp-ck-clblast-tune.json'
## print output_filename
if d.get('post_processed', '') == 'yes':
rr = ck.save_json_to_file({'json_file': output_filename, 'dict': d})
if rr['return'] > 0: return rr
else:
rr['return'] = 1
rr['error'] = 'FAIL'
print("[postprocessing] Exit code %s" % (rr['return']))
return rr
def ck_preprocess(i):
ck = i['ck_kernel']
del i['ck_kernel']
rt = i['run_time']
deps = i['deps']
env = i.get('env', {})
pass_to_make = i
pli = i['misc']
rr = {}
#print (json.dumps(deps['lib-clblast'], indent=2))
#print deps['lib-clblast']['uoa']
# Load both stderr and stdout. Concatenate into one list.
# NB: This assumes that Caffe iterates only once (--iterations=1).
# Otherwise, looping over the log would be required.
# for target in VENDOR_TRANSLATION_TABLE:
# if d["device_vendor"] == target:
# d["device_vendor"] = VENDOR_TRANSLATION_TABLE[target]
tos = pli['target_os_uoa']
tdid = pli['device_id']
adf = pli['add_to_features']
#compiler=adf['gpgpu'][0]['gpgpu_deps']['compiler']['uoa']
#print (json.dumps(i['env'], indent=2))
docompile = int(env['CK_FORCE_RECOMPILE'])
if docompile == 0:
print("[CK_FORCE_RECOMPILE] Exit %s" % (env['CK_FORCE_RECOMPILE']))
rr["return"] = 0
return rr
#print tos, tdid)
print("[CK_FORCE_RECOMPILE] %s" % (env['CK_FORCE_RECOMPILE']))
#GET DEFAULT VALUE from CLBlast
deps_cb = deps['lib-clblast']
uoa = deps_cb['uoa']
b = deps_cb['cus']
pl = b['path_lib']
#bench="xgemm"
#bench +=".hpp"
pl = pl.split("install")[0] #### VERIFY WITH INSTALL SCRIPT
pl_suff = "src/scripts/database/"
pk = pl
pk_suff = "src/src/database/kernels/"
pl += pl_suff
pl1 = pl + '/database'
pk += pk_suff
# print pl, pk
sys.path.append(pl)
sys.path.append(pl1)
####
import database.io as io
import database.db as db
import database.clblast as clblast
import database.bests as bests
import database.defaults as defaults
best_filename = "database_best.json"
if not os.path.isfile(best_filename):
print("[database] database_best.json not found")
database_filename = pl + "database.json"
if not os.path.isfile(database_filename):
io.download_database(database_filename, DATABASE_SERVER_URL)
else:
print("[database] DB found")
if os.path.isfile(database_filename):
database = io.load_database(database_filename)
# Retrieves the best performing results
print("[database] Calculating the best results per device/kernel...")
database_best_results = bests.get_best_results(database)
# Determines the defaults for other vendors and per vendor
print("[database] Calculating the default values...")
database_defaults = defaults.calculate_defaults(
database, 0) #second args denotes VERBOSE or NOT
database_best_results["sections"].extend(database_defaults["sections"])
database_best_filename = 'database_best.json'
io.save_database(database_best_results, database_best_filename)
# Optionally outputs the database to disk
#### TEST to get best and default param
else:
print("[database] database_best.json found")
print("[database] Loading %s" % (best_filename))
database_best_results = json.loads(open(best_filename).read())
best = database_best_results['sections']
print("[Tuning] Checking new best configuration")
### loadfile To generilize
mybestf = env['CK_CLBLAST_BEST_CONF_FILE']
mybestd = {}
if os.path.isfile(mybestf):
mybestd = json.loads(open(mybestf).read())
del mybestd['data']
#####
MYFIND, k = ck2clblast(best, mybestd)
else:
MYFIND = 0
if MYFIND:
print("[Tuning] Modify databese_best entries")
print("[Tuning] Creating new kernels directory")
cp = os.getcwd()
src_new_kernels = cp + "/kernels_tmp"
if not os.path.exists(src_new_kernels):
os.makedirs(src_new_kernels)
else:
print("[Tuning] %s already exists" % (src_new_kernels))
print("[Tuning] wrinting new kernel: %s " % (src_new_kernels))
clblast.print_cpp_database(database_best_results, src_new_kernels)
rr = make(src_new_kernels, pk, tos, tdid, uoa, k)
else:
print("[Tuning] Nothing to do")
print("[Tuning] Exit")
rr['return'] = 0
#### NO ADD STUFF BELOW
return rr
def ck_preprocess(i):
ck=i['ck_kernel']
del i['ck_kernel']
rt=i['run_time']
deps=i['deps']
env=i.get('env',{})
pass_to_make = i
pli = i['misc']
rr={}
#print (json.dumps(deps['lib-clblast'], indent=2))
#print deps['lib-clblast']['uoa']
# Load both stderr and stdout. Concatenate into one list.
# NB: This assumes that Caffe iterates only once (--iterations=1).
# Otherwise, looping over the log would be required.
# for target in VENDOR_TRANSLATION_TABLE:
# if d["device_vendor"] == target:
# d["device_vendor"] = VENDOR_TRANSLATION_TABLE[target]
tos = pli['target_os_uoa']
tdid = pli['device_id']
adf=pli['add_to_features']
#compiler=adf['gpgpu'][0]['gpgpu_deps']['compiler']['uoa']
#print (json.dumps(i['env'], indent=2))
docompile=int(env['CK_FORCE_RECOMPILE'])
if docompile == 0:
print("[CK_FORCE_RECOMPILE] Exit %s" %(env['CK_FORCE_RECOMPILE']))
rr["return"] = 0
return rr
#print tos, tdid)
print("[CK_FORCE_RECOMPILE] %s" %(env['CK_FORCE_RECOMPILE']))
#GET DEFAULT VALUE from CLBlast
deps_cb= deps['lib-clblast']
uoa= deps_cb['uoa']
b=deps_cb['cus']
pl = b['path_lib']
#bench="xgemm"
#bench +=".hpp"
pl=pl.split("install")[0] #### VERIFY WITH INSTALL SCRIPT
pl_suff= "src/scripts/database/"
pk=pl
pk_suff="src/src/database/kernels/"
pl += pl_suff
pl1 = pl+'/database'
pk += pk_suff
# print pl, pk
sys.path.append(pl)
sys.path.append(pl1)
####
import database.io as io
import database.db as db
import database.clblast as clblast
import database.bests as bests
import database.defaults as defaults
best_filename="database_best.json"
if not os.path.isfile(best_filename):
print("[database] database_best.json not found")
database_filename=pl+"database.json"
if not os.path.isfile(database_filename):
io.download_database(database_filename, DATABASE_SERVER_URL)
else:
print ("[database] DB found")
if os.path.isfile(database_filename):
database = io.load_database(database_filename)
# Retrieves the best performing results
print("[database] Calculating the best results per device/kernel...")
database_best_results = bests.get_best_results(database)
# Determines the defaults for other vendors and per vendor
print("[database] Calculating the default values...")
database_defaults = defaults.calculate_defaults(database, 0) #second args denotes VERBOSE or NOT
database_best_results["sections"].extend(database_defaults["sections"])
database_best_filename='database_best.json'
io.save_database(database_best_results, database_best_filename)
# Optionally outputs the database to disk
#### TEST to get best and default param
else:
print("[database] database_best.json found")
print("[database] Loading %s" % (best_filename))
database_best_results = json.loads(open(best_filename).read())
best = database_best_results['sections']
print("[Tuning] Checking new best configuration")
### loadfile To generilize
mybestf=env['CK_CLBLAST_BEST_CONF_FILE']
mybestd={}
if os.path.isfile(mybestf):
mybestd = json.loads(open(mybestf).read())
del mybestd['data']
#####
MYFIND, k = ck2clblast(best, mybestd)
else:
MYFIND = 0
if MYFIND:
print("[Tuning] Modify databese_best entries")
print("[Tuning] Creating new kernels directory")
cp = os.getcwd()
src_new_kernels = cp+"/kernels_tmp"
if not os.path.exists(src_new_kernels):
os.makedirs(src_new_kernels)
else:
print("[Tuning] %s already exists" % (src_new_kernels))
print("[Tuning] wrinting new kernel: %s " % (src_new_kernels))
clblast.print_cpp_database(database_best_results, src_new_kernels)
rr = make(src_new_kernels, pk, tos , tdid, uoa, k)
else:
print("[Tuning] Nothing to do")
print("[Tuning] Exit")
rr['return']=0
#### NO ADD STUFF BELOW
return rr
def main(argv):
# Parses the command-line arguments
parser = argparse.ArgumentParser()
parser.add_argument("source_folder", help="The folder with JSON files to parse to add to the database")
parser.add_argument("clblast_root", help="Root of the CLBlast sources")
parser.add_argument("-r", "--remove_device", type=str, default=None, help="Removes all entries for a specific device")
parser.add_argument("-v", "--verbose", action="store_true", help="Increase verbosity of the script")
cl_args = parser.parse_args(argv)
# Parses the path arguments
database_filename = os.path.join(cl_args.clblast_root, "scripts", "database", "database.json")
database_best_filename = os.path.join(cl_args.clblast_root, "scripts", "database", "database_best.json")
json_files = os.path.join(cl_args.source_folder, "*.json")
cpp_database_path = os.path.join(cl_args.clblast_root, "src", "database", "kernels")
# Checks whether the command-line arguments are valid
clblast_header = os.path.join(cl_args.clblast_root, "include", "clblast.h") # Not used but just for validation
if not os.path.isfile(clblast_header):
raise RuntimeError("The path '" + cl_args.clblast_root + "' does not point to the root of the CLBlast library")
if len(glob.glob(json_files)) < 1:
print("[database] The path '" + cl_args.source_folder + "' does not contain any JSON files")
# Downloads the database if a local copy is not present
if not os.path.isfile(database_filename):
io.download_database(database_filename, DATABASE_SERVER_URL)
# Loads the database from disk
database = io.load_database(database_filename)
# Loops over all JSON files in the supplied folder
for file_json in glob.glob(json_files):
# Loads the newly imported data
sys.stdout.write("[database] Processing '" + file_json + "' ") # No newline printed
imported_data = io.load_tuning_results(file_json)
# Fixes the problem that some vendors use multiple different names
for target in VENDOR_TRANSLATION_TABLE:
if imported_data["device_vendor"] == target:
imported_data["device_vendor"] = VENDOR_TRANSLATION_TABLE[target]
# Adds the new data to the database
old_size = db.length(database)
database = db.add_section(database, imported_data)
new_size = db.length(database)
print("with " + str(new_size - old_size) + " new items") # Newline printed here
# Checks for tuning results with mis-matched entries
remove_mismatched_arguments(database)
# Stores the modified database back to disk
if len(glob.glob(json_files)) >= 1:
io.save_database(database, database_filename)
# Removes database entries before continuing
if cl_args.remove_device is not None:
print("[database] Removing all results for device '%s'" % cl_args.remove_device)
remove_database_entries(database, {"device": cl_args.remove_device})
io.save_database(database, database_filename)
# Retrieves the best performing results
print("[database] Calculating the best results per device/kernel...")
database_best_results = bests.get_best_results(database)
# Determines the defaults for other vendors and per vendor
print("[database] Calculating the default values...")
database_defaults = defaults.calculate_defaults(database, cl_args.verbose)
database_best_results["sections"].extend(database_defaults["sections"])
# Optionally outputs the database to disk
if cl_args.verbose:
io.save_database(database_best_results, database_best_filename)
# Outputs the database as a C++ database
print("[database] Producing a C++ database in '" + cpp_database_path + "'...")
clblast.print_cpp_database(database_best_results, cpp_database_path)
print("[database] All done")
def ck_postprocess(i):
ck=i['ck_kernel']
rt=i['run_time']
deps=i['deps']
d={}
env=i.get('env',{})
# Load both stderr and stdout. Concatenate into one list.
# NB: This assumes that Caffe iterates only once (--iterations=1).
# Otherwise, looping over the log would be required.
# For debugging
# rt['run_cmd_out1']='stdout.log'
# rt['run_cmd_out2']='stderr.log'
# rt['run_output_files']=["clblast_xgemm_1_32.json","clblast_xgemm_2_32.json"]
rf1=rt['run_cmd_out1']
rf2=rt['run_cmd_out2']
rf3=rt['run_output_files'][0]
rf4=rt['run_output_files'][1]
print("[postprocessing] Loading json output %s " % (rt['run_output_files'][0] ))
print("[postprocessing] Loading json output %s " % (rt['run_output_files'][1] ))
lst=[]
r={}
if os.path.isfile(rf1):
r=ck.load_text_file({'text_file':rf1,'split_to_list':'yes'})
if r['return']>0: return r
lst+=r['lst']
if os.path.isfile(rf2):
r=ck.load_text_file({'text_file':rf2,'split_to_list':'yes'})
if r['return']>0: return r
lst+=r['lst']
c1 = os.path.isfile(rf3)
c2 = os.path.isfile(rf4)
if c1:
rj1 = json.loads(open(rf3).read())
if c2:
rj2= json.loads(open(rf4).read())
if ((c1 == 0) and (c2 == 0)):
r['return'] = 0
print("[postprocessing] Unable to read json output")
r['return'] = 1;
return r;
if ((c1)== 0):
rj1 = rj2
#### CREATE UNIQUE OUTPUT
print("[postprocessing] Creating dictionary")
d['post_processed']='yes'
# mydict['device_core_clock'] = 'value from script'
# SET ARCH INFORMATION
d['device_vendor'] = rj1['device_vendor']
d['device_type'] = rj1['device_type']
d['device'] = rj1['device']
d['device_compute_units'] = rj1['device_compute_units']
# Notice that often is not really accurated; TBC
d['device_core_clock'] = rj1['device_core_clock']
#Experiment Information
kernel_family=rj1['kernel_family'].split("_")[0]
if rj1['kernel_family'].split("_")[1] == "direct":
#concut again special case the kernal family of Xgemm_directNN is the same of other xgemm_direct
kernel_family = kernel_family + "_" + "direct"
d['kernel'] = kernel_family
d['arg_beta'] = rj1['arg_beta']
d['arg_m'] = rj1['arg_m']
d['arg_n'] = rj1['arg_n']
d['arg_k'] = rj1['arg_k']
d['arg_alpha'] = rj1['arg_alpha']
d['precision'] = rj1['precision']
for target in VENDOR_TRANSLATION_TABLE:
if d["device_vendor"] == target:
d["device_vendor"] = VENDOR_TRANSLATION_TABLE[target]
#### Add results per strategy
# print "ADD RESULTs"
l=[]
if ( c1 ):
tmp = {'strategy':'exhaustive', 'result': rj1['results']}
l.append(tmp)
if ( c2 ):
tmp = {'strategy':'random', 'result': rj2['results']}
l.append(tmp)
d['data'] = l
#GET DEFAULT VALUE from CLBlast
deps_cb= deps['lib-clblast']
b=deps_cb['cus']
pl = b['path_lib']
bench=d['kernel']
bench +=".hpp"
pl=pl.split("install")[0] #### VERIFY WITH INSTALL SCRIPT
pl_suff= "src"+os.sep+"scripts"+os.sep+"database"+os.sep
pl += pl_suff
pl1 = pl+"database"+os.sep
sys.path.append(pl)
sys.path.append(pl1)
####
import database.io as io
import database.db as db
import database.clblast as clblast
import database.bests as bests
import database.defaults as defaults
database_filename=pl+"database.json"
if os.path.isfile(database_filename) and os.path.getsize(database_filename)==0:
os.remove(database_filename)
if not os.path.isfile(database_filename):
io.download_database(database_filename, DATABASE_SERVER_URL)
else:
print("[database] DB found")
if os.path.isfile(database_filename):
database = io.load_database(database_filename)
# Retrieves the best performing results
print("[database] Calculating the best results per device/kernel...")
database_best_results = bests.get_best_results(database)
search_vendor=d['device_vendor']
search_device= d['device']
search_device_type=d['device_type']
search_kernel=d['kernel']
search_precision= d['precision']
# Determines the defaults for other vendors and per vendor
print("[database] Calculating the default values...")
database_defaults = defaults.calculate_defaults(database, 0) #second args denotes VERBOSE or NOT
database_best_results["sections"].extend(database_defaults["sections"])
database_best_filename='database_best.json'
# Optionally outputs the database to disk
if VERBOSE:
io.save_database(database_best_results, database_best_filename)
#print("[database] Producing a C++ database in current dir...")
#clblast.print_cpp_database(database_best_results, ".")
#### TEST to get best and default param
best = database_best_results['sections']
ll = []
count = 0
index = -1
for best_entries in best:
#print best_entries['kernel_family'] + " " + search_kernel
if( (best_entries['kernel_family'] == search_kernel) and \
(best_entries['device_vendor'] == search_vendor) and \
(best_entries['precision'] == search_precision) and \
(best_entries['device_type']== search_device_type) and \
((best_entries['device'] == search_device) or (best_entries['device']=='default')) ):
if VERBOSE: print("[postprocess] Best entry %s found" % (best_entries))
tmp=best_entries
ll.append(tmp)
##### Find Timing in d[data]
stat=[]
for s in d['data']:
# print s.get('strategy', {})
result=s.get('result',{})
for rrr in result:
compare = rrr['parameters']
for il in ll:
best = il.get('results',{})[0].get('parameters',{})
dev= il['device']
### comparing value by value
isBest= True
for bb in best.keys():
# print bb, best[bb], compare[bb]
if (best[bb] != compare[bb]):
isBest = False
break
if (isBest):
rrr['device'] = dev
stat.append(rrr)
index=0
bindex=-1
bres={}
min_time=sys.float_info.max
for s in d['data']:
# print s.get('strategy', {})
result=s.get('result',{})
for i in result:
index +=1
if (i['time'] < min_time):
min_time=i['time']
bres=i
bindex=index
# print "Best performance: ", min_time,bres
m = float(d["arg_m"])
n = float(d["arg_n"])
k = float(d["arg_k"])
if bres.get('time','')!='':
gflops = 2.0*m*n*k
time = float(bres["time"])/1000.0
gflops = gflops/time/1000.0/1000.0/1000.0
bres['GFLOPS'] =gflops
bestdefstat={}
defstat = {}
for i in stat:
time=float(i["time"])/1000.0
gflops = 2.0*m*n*k
gflops = gflops/time/1000.0/1000.0/1000.0
i['GFLOPS'] =gflops
if i['device'] == 'default':
defstat=i
else:
bestdefstat=i
d["statistics"] = {'best_configuration': bres, 'default_configuration': defstat, 'default_family':bestdefstat}
if VERBOSE:
print ("[postprocessing] %s" %(d["statistics"]))
if len(ll) > 0:
if VERBOSE:
print (len(ll))
d['db'] = ll
else:
d['db'] = 'na'
rr={}
rr['return']=0
# output_filename='tmp-ck-clblast-tune-'+d['kernel']+'-'+d['arg_m']+'-'+d['arg_n']+'-'+d['arg_k']+'-'+d['precision'] +'.json'
output_filename='tmp-ck-clblast-tune.json'
## print output_filename
if d.get('post_processed','')=='yes':
rr=ck.save_json_to_file({'json_file':output_filename, 'dict':d})
if rr['return']>0: return rr
else:
rr['return']=1
rr['error']='FAIL'
print("[postprocessing] Exit code %s" %(rr['return']))
return rr