def config_packets_predict(algo_payload, matrix_type, matrix_shape,
datagen_dir, train_dir, predict_dir, dense_algos,
config_dir):
"""
This function has two responsibilities. Generate the configuration files for
prediction algorithms and return a dictionary that will be used for execution.
algo_payload : List of tuples
The first tuple index contains algorithm name and the second index contains
family type.
matrix_type: String
Type of matrix to generate e.g dense, sparse, all
matrix_shape: String
Shape of matrix to generate e.g 100k_10
datagen_dir: String
Path of the data generation directory
train_dir: String
Path of the training directory
predict_dir: String
Path of the prediction directory
dense_algos: List
Algorithms that support only dense matrix type
config_dir: String
Location to store to configuration json file
return: Dictionary {string: list}
This dictionary contains algorithms to be executed as keys and the path of configuration
json files to be executed list of values.
"""
config_bundle = {}
for current_algo, current_family in algo_payload:
key_name = current_algo + '.' + current_family
config_bundle[key_name] = []
for current_algo, current_family in algo_payload:
current_matrix_type = mat_type_check(current_family, matrix_type,
dense_algos)
train_folders = relevant_folders(train_dir, current_algo,
current_family, current_matrix_type,
matrix_shape, 'train')
if len(train_folders) == 0:
print('training folders not present for {}'.format(current_algo))
sys.exit()
for current_train_folder in train_folders:
current_data_gen_dir = relevant_folders(datagen_dir, current_algo,
current_family,
current_matrix_type,
matrix_shape, 'data-gen')
if len(current_data_gen_dir) == 0:
print('data-gen folders not present for {}'.format(
current_family))
sys.exit()
save_name = current_train_folder.split('/')[-1]
algo_func = '_'.join([current_algo.lower().replace('-', '_')] +
['predict'])
# current_data_gen_dir has index 0 as we would expect one datagen for each algorithm
conf_path = globals()[algo_func](save_name,
current_data_gen_dir[0],
current_train_folder, predict_dir,
config_dir)
key_name = current_algo + '.' + current_family
config_bundle[key_name].append(conf_path)
return config_bundle
def config_packets_train(algo_payload, matrix_type, matrix_shape, datagen_dir,
train_dir, dense_algos, config_dir):
"""
This function has two responsibilities. Generate the configuration files for
input training algorithms and return a dictionary that will be used for execution.
algo_payload : List of tuples
The first tuple index contains algorithm name and the second index contains
family type.
matrix_type: String
Type of matrix to generate e.g dense, sparse, all
matrix_shape: String
Shape of matrix to generate e.g 100k_10
datagen_dir: String
Path of the data generation directory
train_dir: String
Path of the training directory
dense_algos: List
Algorithms that support only dense matrix type
config_dir: String
Location to store to configuration json file
return: {string: list}
This dictionary contains algorithms to be executed as keys and the path of configuration
json files to be executed list of values.
"""
config_bundle = {}
for current_algo, current_family in algo_payload:
key_name = current_algo + '.' + current_family
config_bundle[key_name] = []
for current_algo, current_family in algo_payload:
current_matrix_type = mat_type_check(current_family, matrix_type,
dense_algos)
data_gen_folders = relevant_folders(datagen_dir, current_algo,
current_family,
current_matrix_type, matrix_shape,
'data-gen')
if len(data_gen_folders) == 0:
print('datagen folders not present for {}'.format(current_family))
sys.exit()
for current_datagen_dir in data_gen_folders:
file_path_last = current_datagen_dir.split('/')[-1]
save_name = '.'.join([current_algo] + [file_path_last])
algo_func = '_'.join([current_family] +
[current_algo.lower().replace('-', '_')] +
['train'])
conf_path = globals()[algo_func](save_name, current_datagen_dir,
train_dir, config_dir)
key_name = current_algo + '.' + current_family
config_bundle[key_name].append(conf_path)
config_packets = {}
# Flatten
for current_algo, current_family in config_bundle.items():
config_packets[current_algo] = reduce(lambda x, y: x + y,
current_family)
return config_packets
def config_packets_predict(algo_payload, matrix_type, matrix_shape, datagen_dir, train_dir, predict_dir, dense_algos, config_dir):
"""
This function has two responsibilities. Generate the configuration files for
prediction algorithms and return a dictionary that will be used for execution.
algo_payload : List of tuples
The first tuple index contains algorithm name and the second index contains
family type.
matrix_type: String
Type of matrix to generate e.g dense, sparse, all
matrix_shape: String
Shape of matrix to generate e.g 100k_10
datagen_dir: String
Path of the data generation directory
train_dir: String
Path of the training directory
predict_dir: String
Path of the prediction directory
dense_algos: List
Algorithms that support only dense matrix type
config_dir: String
Location to store to configuration json file
return: Dictionary {string: list}
This dictionary contains algorithms to be executed as keys and the path of configuration
json files to be executed list of values.
"""
config_bundle = {}
for current_algo, current_family in algo_payload:
key_name = current_algo + '.' + current_family
config_bundle[key_name] = []
for current_algo, current_family in algo_payload:
current_matrix_type = mat_type_check(current_family, matrix_type, dense_algos)
train_folders = relevant_folders(train_dir, current_algo, current_family,
current_matrix_type, matrix_shape, 'train')
if len(train_folders) == 0:
print('training folders not present for {}'.format(current_algo))
sys.exit()
for current_train_folder in train_folders:
current_data_gen_dir = relevant_folders(datagen_dir, current_algo, current_family,
current_matrix_type, matrix_shape, 'data-gen')
if len(current_data_gen_dir) == 0:
print('data-gen folders not present for {}'.format(current_family))
sys.exit()
save_name = current_train_folder.split('/')[-1]
algo_func = '_'.join([current_algo.lower().replace('-', '_')] + ['predict'])
# current_data_gen_dir has index 0 as we would expect one datagen for each algorithm
conf_path = globals()[algo_func](save_name, current_data_gen_dir[0],
current_train_folder, predict_dir, config_dir)
key_name = current_algo + '.' + current_family
config_bundle[key_name].append(conf_path)
return config_bundle
def config_packets_train(algo_payload, matrix_type, matrix_shape, datagen_dir, train_dir, dense_algos, config_dir):
"""
This function has two responsibilities. Generate the configuration files for
input training algorithms and return a dictionary that will be used for execution.
algo_payload : List of tuples
The first tuple index contains algorithm name and the second index contains
family type.
matrix_type: String
Type of matrix to generate e.g dense, sparse, all
matrix_shape: String
Shape of matrix to generate e.g 100k_10
datagen_dir: String
Path of the data generation directory
train_dir: String
Path of the training directory
dense_algos: List
Algorithms that support only dense matrix type
config_dir: String
Location to store to configuration json file
return: {string: list}
This dictionary contains algorithms to be executed as keys and the path of configuration
json files to be executed list of values.
"""
config_bundle = {}
for current_algo, current_family in algo_payload:
key_name = current_algo + '.' + current_family
config_bundle[key_name] = []
for current_algo, current_family in algo_payload:
current_matrix_type = mat_type_check(current_family, matrix_type, dense_algos)
data_gen_folders = relevant_folders(datagen_dir, current_algo, current_family,
current_matrix_type, matrix_shape, 'data-gen')
if len(data_gen_folders) == 0:
print('datagen folders not present for {}'.format(current_family))
sys.exit()
for current_datagen_dir in data_gen_folders:
file_path_last = current_datagen_dir.split('/')[-1]
save_name = '.'.join([current_algo] + [file_path_last])
algo_func = '_'.join([current_family] + [current_algo.lower().replace('-', '_')]
+ ['train'])
conf_path = globals()[algo_func](save_name, current_datagen_dir, train_dir, config_dir)
key_name = current_algo + '.' + current_family
config_bundle[key_name].append(conf_path)
config_packets = {}
# Flatten
for current_algo, current_family in config_bundle.items():
config_packets[current_algo] = reduce(lambda x, y: x + y, current_family)
return config_packets