def existing_data(form, user_configs, username, sess, APP_ROOT):
dataset_name = form['exisiting_files-train_file_exist']
path = os.path.join(APP_ROOT, 'user_data', username, dataset_name)
if form['exisiting_files-configuration'] != 'new_config':
config_name = form['exisiting_files-configuration']
sess.set('config_file', os.path.join(path, config_name, 'config.ini'))
sess.load_config()
return 'parameters'
else:
config_name = config_ops.define_new_config_file(
dataset_name, APP_ROOT, username, sess.get_writer())
sess.set('config_file', os.path.join(path, config_name, 'config.ini'))
if user_configs[dataset_name] and os.path.isfile(
os.path.join(path, user_configs[dataset_name][0],
'config.ini')):
reader = config_reader.read_config(
os.path.join(path, user_configs[dataset_name][0],
'config.ini'))
copyfile(
os.path.join(path, user_configs[dataset_name][0],
'config.ini'),
os.path.join(path, config_name, 'config.ini'))
filename = reader['PATHS']['file']
elif os.path.isfile(os.path.join(path, dataset_name + '.csv')):
filename = os.path.join(path, dataset_name + '.csv')
else:
filename = [
f for f in os.listdir(path)
if os.path.isfile(os.path.join(path, f)) and '.csv' in f
][0]
sess.set('file', os.path.join(path, filename))
sess.get_writer().add_item('PATHS', 'file',
os.path.join(path, filename))
sess.get_writer().write_config(sess.get('config_file'))
return 'slider'
def tensor_board_thread(self, config_file, port):
config_path = config_reader.read_config(config_file).all()['checkpoint_dir']
logging.debug('Starting tensor board')
time.sleep(3)
pro = "tensorboard --host=0.0.0.0 --logdir=" + config_path + " --port=" + port
subprocess.call(pro, shell=True)
logging.debug('Exiting tensor board')
def explain():
if request.method == 'POST':
new_features = feature_util.get_new_features(
request.form, sess.get('defaults'), sess.get('target'),
sess.get('fs').group_by(sess.get('category_list'))['none'])
all_params_config = config_reader.read_config(sess.get('config_file'))
all_params_config.set(
'PATHS', 'checkpoint_dir',
os.path.join(all_params_config.export_dir(),
request.form['radiob']))
labels = feature_util.get_target_labels(
sess.get('target'),
sess.get('data').Category[sess.get('target')], sess.get('fs'))
dtypes = sess.get('fs').group_by(sess.get('category_list'))
result = th.explain_estimator(all_params_config, sess.get('features'),
sess.get('target'), labels,
sess.get('defaults'), dtypes,
new_features, sess.get('df'),
sess.get('data').Category,
int(request.form['num_feat']),
int(request.form['top_labels']))
if result is not None:
fp = BytesIO(str.encode(result.as_html(show_table=True)))
sess.set('explain_fp', fp)
return jsonify(explanation='ok')
return jsonify(explanation=str(result))
else:
return send_file(sess.get('explain_fp'), mimetype='text/html')
def assign_category(self, config_file, df):
feature_dict = self.feature_dict()
unique_values = [
self.unique_value_size_dict.get(key, -1) for key in df.columns
]
category_list = [feature_dict[key] for key in df.columns]
if 'COLUMN_CATEGORIES' in config_reader.read_config(
config_file).keys():
category_list = []
for key in df.columns:
category_list.append(
config_reader.read_config(config_file)['COLUMN_CATEGORIES']
[key])
default_list = self.defaults
frequent_values2frequency = self.frequent_values2frequency
return category_list, unique_values, default_list, frequent_values2frequency
def run():
config = config_reader.read_config()
tables_reader.read(config.connection_string, config.tables)
for destination in config.destination_items:
for table in config.tables:
code = destination.get_code(table)
csharp_project.add_code(destination, table.entity_name, code)
def refresh():
try:
CONFIG_FILE = sess.get('config_file')
export_dir = config_reader.read_config(CONFIG_FILE).export_dir()
checkpoints = run_utils.get_eval_results(export_dir, sess.get_writer(),
CONFIG_FILE)
return jsonify(checkpoints=checkpoints)
except KeyError:
return jsonify(checkpoints='')
def config_ga():
lb = [1] * (station_count + truck_count - 1)
ub = [21] * (station_count + truck_count - 1)
iteration = int(cfg.read_config("iteration_time"))
ga = GA(func=fitness_fun,
size_pop=100,
n_dim=station_count + truck_count - 1,
max_iter=iteration,
prob_mut=0.01,
lb=lb,
ub=ub,
precision=[1] * (station_count + truck_count - 1))
ga.Chrom = generate_new_pop()
if cfg.read_config("crossover_type") is not None:
ga.register(operator_name='crossover', operator=customized_crossover). \
register(operator_name='mutation', operator=customized_mutation)
# TODO: 排序的策略配置（暂时不考虑）
return ga
def test_get_eva_results():
writer = ConfigWriter()
writer.append_config(config_test_file)
results = run_utils.get_eval_results(directory, writer, config_test_file)
assert results == checkpoints
reader = config_reader.read_config(config_test_file)
assert reader['BEST_MODEL']['max_acc'] == '1.0'
assert reader['BEST_MODEL']['max_acc_index'] == '42'
assert reader['BEST_MODEL']['min_loss'] == '0.153'
assert reader['BEST_MODEL']['min_loss_index'] == '42'
def set_form(form, config):
config = read_config(config.get('info', 'config_path'))
if 'copula_factor_algorithm' in config.keys():
form.copula_factor_algorithm.form.gibbs_sampling_n.default = config.get(
'copula_factor_algorithm', 'gibbs_sampling_n')
form.copula_factor_algorithm.form.gibbs_burn_in_n.default = config.get(
'copula_factor_algorithm', 'gibbs_burn_in_n')
form.copula_factor_algorithm.form.gibbs_first_random_seed_n.default = config.get(
'copula_factor_algorithm', 'gibbs_first_random_seed_n')
form.copula_factor_algorithm.form.gibbs_random_seed_update_parameter_n.default = config.get(
'copula_factor_algorithm', 'gibbs_random_seed_update_parameter_n')
form.copula_factor_algorithm.form.process()
if 'edge_weight_algorithm' in config.keys():
form.edge_weight_algorithm.form.bootstrap_n.default = config.get(
'edge_weight_algorithm', 'bootstrap_n')
form.edge_weight_algorithm.form.bootstrap_first_random_seed_n.default = config.get(
'edge_weight_algorithm', 'bootstrap_first_random_seed_n')
form.edge_weight_algorithm.form.bootstrap_random_seed_update_parameter_n.default = config.get(
'edge_weight_algorithm',
'bootstrap_random_seed_update_parameter_n')
form.edge_weight_algorithm.form.process()
if 'pc_algorithm' in config.keys():
form.pc_algorithm.form.causal_discovery_observation_n.default = config.get(
'pc_algorithm', 'causal_discovery_observation_n')
form.pc_algorithm.form.indepTest.default = config.get(
'pc_algorithm', 'indepTest')
form.pc_algorithm.form.alpha.default = config.get(
'pc_algorithm', 'alpha')
form.pc_algorithm.form.numCores.default = config.get(
'pc_algorithm', 'numCores')
form.pc_algorithm.form.verbose.default = config.get(
'pc_algorithm', 'verbose')
form.pc_algorithm.form.NAdelete.default = config.get(
'pc_algorithm', 'NAdelete')
form.pc_algorithm.form.m_max.default = config.get(
'pc_algorithm', 'm.max')
form.pc_algorithm.form.u2pd.default = config.get(
'pc_algorithm', 'u2pd')
form.pc_algorithm.form.skel_method.default = config.get(
'pc_algorithm', 'skel.method')
form.pc_algorithm.form.conservative.default = config.get(
'pc_algorithm', 'conservative')
form.pc_algorithm.form.maj_rule.default = config.get(
'pc_algorithm', 'maj.rule')
form.pc_algorithm.form.solve_confl.default = config.get(
'pc_algorithm', 'solve.confl')
form.pc_algorithm.form.fixedGaps.default = config.get(
'pc_algorithm', 'fixedGaps')
form.pc_algorithm.form.fixedEdges.default = config.get(
'pc_algorithm', 'fixedEdges')
form.pc_algorithm.form.process()
if 'plot_and_display' in config.keys():
form.plot_and_display.form.core_plot_title_str.default = config.get(
'plot_and_display', 'core_plot_title_str')
form.plot_and_display.form.process()
def set_form(form, CONFIG_FILE, hidden_layers):
form.network.form.hidden_layers.default = hidden_layers
form.network.form.process()
# if os.path.isfile(CONFIG_FILE) :
reader = config_reader.read_config(CONFIG_FILE)
if 'EXPERIMENT' in reader.keys():
form.experiment.form.keep_checkpoint_max.default = reader[
'EXPERIMENT']['keep_checkpoint_max']
form.experiment.form.save_checkpoints_steps.default = reader[
'EXPERIMENT']['save_checkpoints_steps']
form.experiment.form.save_summary_steps.default = reader['EXPERIMENT'][
'save_summary_steps']
form.experiment.form.throttle.default = reader['EXPERIMENT'][
'throttle']
form.experiment.form.validation_batch_size.default = reader[
'EXPERIMENT']['validation_batch_size']
if 'NETWORK' in reader.keys():
form.network.form.hidden_layers.default = reader['NETWORK'][
'hidden_layers']
form.network.form.model_name.default = reader['NETWORK']['model_name']
if 'CUSTOM_MODEL' in reader.keys():
form.custom_model.form.custom_model_path.default = reader[
'CUSTOM_MODEL']['custom_model_path']
if 'TRAINING' in config_reader.read_config(CONFIG_FILE).keys():
form.training.form.num_epochs.default = reader['TRAINING'][
'num_epochs']
form.training.form.batch_size.default = reader['TRAINING'][
'batch_size']
form.training.form.optimizer.default = reader['TRAINING']['optimizer']
form.training.form.learning_rate.default = reader['TRAINING'][
'learning_rate']
form.training.form.l1_regularization.default = reader['TRAINING'][
'l1_regularization']
form.training.form.l2_regularization.default = reader['TRAINING'][
'l2_regularization']
form.training.form.dropout.default = reader['TRAINING']['dropout']
form.training.form.activation_fn.default = reader['TRAINING'][
'activation_fn']
form.network.form.process()
form.experiment.form.process()
form.training.form.process()
form.custom_model.form.process()
def test_write_features():
old_categories = ['none-numerical', 'numerical', 'numerical', 'numerical', 'categorical']
writer = ConfigWriter()
writer.append_config(config_test_file)
data.Category = ['none', 'none', 'numerical', 'numerical', 'none']
feature_util.write_features(old_categories, data, writer, config_test_file)
reader = config_reader.read_config(config_test_file)
assert reader['COLUMN_CATEGORIES']['sepal_length'] == 'none-numerical'
assert reader['COLUMN_CATEGORIES']['sepal_width'] == 'none-numerical'
assert reader['COLUMN_CATEGORIES']['petal_length'] == 'numerical'
assert reader['COLUMN_CATEGORIES']['class'] == 'none-categorical'
def run():
all_params_config = config_reader.read_config(sess.get('config_file'))
logfile = os.path.join(all_params_config['PATHS']['log_dir'],
'tensorflow.log')
try:
sess.set('log_fp', open(logfile))
except FileNotFoundError:
open(logfile, 'a').close()
sess.set('log_fp', open(logfile))
try:
sess.get('status')
except KeyError:
sess.set('status', 'paused')
labels = feature_util.get_target_labels(
sess.get('target'),
sess.get('data').Category[sess.get('target')], sess.get('fs'))
export_dir = all_params_config.export_dir()
checkpoints = run_utils.get_eval_results(export_dir, sess.get_writer(),
sess.get('config_file'))
th.run_tensor_board(session['user'], sess.get('config_file'))
if request.method == 'POST':
if request.form['action'] == 'run':
sess.set('status', 'running')
else:
sess.set('status', 'paused')
dtypes = sess.get('fs').group_by(sess.get('category_list'))
th.handle_request(request.form['action'], all_params_config,
sess.get('features'), sess.get('target'), labels,
sess.get('defaults'), dtypes, session['user'],
request.form['resume_from'])
return jsonify(True)
dict_types, categoricals = run_utils.get_dictionaries(
sess.get('defaults'), sess.get('category_list'), sess.get('fs'),
sess.get('target'))
sfeatures = feature_util.remove_target(sess.get('defaults'),
sess.get('target'))
has_hash = 'false'
if any(k in sess.get('category_list') for k in ("hash", "int-hash")):
has_hash = 'true'
return render_template('run.html',
page=5,
features=sfeatures,
target=sess.get('target'),
types=run_utils.get_html_types(dict_types),
categoricals=categoricals,
checkpoints=checkpoints,
port=th.get_port(session['user'],
sess.get('config_file')),
running=sess.get('status'),
has_hash=has_hash)
def config_ga():
constraint_eq = [
lambda x: int(cfg.read_config('bike_total_count')) - sum(x)
]
lb = [0] * station_count
ub = hub.hub
iteration = int(cfg.read_config("iteration_time"))
ga = GA(func=fitness_fun,
size_pop=300,
n_dim=station_count,
max_iter=iteration,
prob_mut=0.01,
lb=lb,
ub=ub,
constraint_eq=constraint_eq,
precision=[1] * 20)
if cfg.read_config("crossover_type") is not None:
ga.register(operator_name='crossover',
operator=crossover.crossover_1point)
# TODO: 变异和排序的策略配置（暂时不考虑）
return ga
def delete():
CONFIG_FILE = sess.get('config_file')
all_params_config = config_reader.read_config(CONFIG_FILE)
export_dir = all_params_config.export_dir()
del_id = request.get_json()['deleteID']
paths = [del_id] if del_id != 'all' else [
d for d in os.listdir(export_dir)
if os.path.isdir(os.path.join(export_dir, d))
]
sys_ops.delete_recursive(paths, export_dir)
checkpoints = run_utils.get_eval_results(export_dir, sess.get_writer(),
CONFIG_FILE)
return jsonify(checkpoints=checkpoints)
def load_config(self):
# read saved config
conf = config_reader.read_config(self.get('config_file'))
# update files and df in config dict
self.set('file', conf['PATHS']['file'])
self.set('train_file', conf['PATHS']['train_file'])
self.set('validation_file', conf['PATHS']['validation_file'])
self.set('df', pd.read_csv(conf['PATHS']['file']))
self.load_features()
# target select
target = conf['TARGET']['target']
self.set_target(target)
self.update_new_features(self.get('category_list'),
list(self.get('defaults').values()))
self.update_writer_conf(conf)
def get_configs_files(app_root, username):
user_configs = {}
existing_datasets = []
configs = {}
user_path = os.path.join(app_root, USER_DATA, username)
user_datasets = [
dataset for dataset in os.listdir(user_path)
if os.path.isdir(os.path.join(user_path, dataset))
]
for user_dataset in user_datasets:
user_configs[user_dataset] = [
config
for config in os.listdir(os.path.join(user_path, user_dataset))
if os.path.isdir(os.path.join(user_path, user_dataset, config))
]
if INPUT_DATA in user_configs[user_dataset]:
user_configs[user_dataset].remove(INPUT_DATA)
if FACTOR_MODEL in user_configs[user_dataset]:
user_configs[user_dataset].remove(FACTOR_MODEL)
for config_file in user_configs[user_dataset]:
dataset_config = user_dataset + '_' + config_file
path_to_config_file = os.path.join(user_path, user_dataset,
config_file, 'config.ini')
config = read_config(path_to_config_file)
configs[dataset_config] = {
'name': config.get_name(),
'path': config.get_path(),
'bootstrap': ''
}
configs[dataset_config]['bootstrap'] = '10' # param by default
configs[dataset_config][
'gibbs_sampling_n'] = '1000' # param by default
configs[dataset_config][
'gibbs_burn_in_n'] = '500' # param by default
if 'edge_weight_algorithm' in config.sections():
configs[dataset_config]['bootstrap'] = config.get_bootstrap_n()
if 'copula_factor_algorithm' in config.sections():
configs[dataset_config][
'gibbs_sampling_n'] = config.get_gibbs_sampling_n()
configs[dataset_config][
'gibbs_burn_in_n'] = config.get_gibbs_burn_in_n()
existing_datasets.append(user_dataset)
return existing_datasets, user_configs, configs
def predict():
new_features = feature_util.get_new_features(
request.form, sess.get('defaults'), sess.get('target'),
sess.get('fs').group_by(sess.get('category_list'))['none'])
all_params_config = config_reader.read_config(sess.get('config_file'))
all_params_config.set(
'PATHS', 'checkpoint_dir',
os.path.join(all_params_config.export_dir(), request.form['radiob']))
labels = feature_util.get_target_labels(
sess.get('target'),
sess.get('data').Category[sess.get('target')], sess.get('fs'))
dtypes = sess.get('fs').group_by(sess.get('category_list'))
final_pred = th.predict_estimator(all_params_config, sess.get('features'),
sess.get('target'), labels,
sess.get('defaults'), dtypes,
new_features, sess.get('df'))
return jsonify(prediction=str(final_pred))
def ga_plot(ga, bike_allocation):
# print(ga.generation_best_Y)
fig, ax = plt.subplots()
generation = []
best = []
valid_gen = 0
for gen in ga.generation_best_Y:
if gen < 0:
best.append(gen)
generation.append(valid_gen)
valid_gen = valid_gen + 1
plt.xlim((0, valid_gen))
plt.ylim((600, 1000))
plt.xlabel('iteration time')
plt.ylabel('fitness value')
my_x_ticks = np.arange(0, valid_gen, 5)
my_y_ticks = np.arange(600, 1000, 50)
plt.xticks(my_x_ticks)
plt.yticks(my_y_ticks)
idx = 0
for item in ga.generation_best_X:
if np.array_equal(item, bike_allocation):
break
idx = idx + 1
f_val = -fitness_fun(bike_allocation)
max_cord = (idx, int(-fitness_fun(bike_allocation)))
max_cord_shift = (idx + 2, int(-fitness_fun(bike_allocation)) + 10)
ax.plot(np.array(generation), -np.array(best), color='red')
plt.vlines(idx, 300, int(f_val), 'black', '--', label='example')
plt.hlines(int(f_val), 0, idx, 'black', '--', label='example')
plt.annotate("(" + str(idx) + ", " + str(-fitness_fun(bike_allocation)) +
")",
xytext=max_cord,
xy=max_cord_shift)
plt.savefig('ga_800_iter' + str(int(cfg.read_config('iteration_time'))) +
"_" + str(time.strftime('%Y%m%d_%H_%M_%S') + ".pdf"),
format='pdf')
plt.show()
def target():
form = Submit()
if form.validate_on_submit():
sess.set_target(json.loads(request.form['selected_rows'])[0])
if 'split_df' in sess.get_config():
train_file, validation_file = preprocessing.split_train_test(
sess.get('split_df'), sess.get('file'), sess.get('target'),
sess.get('df'))
sess.update_split(train_file, validation_file)
sess.get_writer().write_config(sess.get('config_file'))
return redirect(url_for('parameters'))
reader = config_reader.read_config(sess.get('config_file'))
target_selected = reader['TARGET']['target'] if 'TARGET' in reader.keys(
) else 'None'
# filter hash and none columns
data = sess.get('data')[(sess.get('data').Category != 'hash')
& (sess.get('data').Category != 'none')]
return render_template('target_selection.html',
name="Dataset target selection",
form=form,
data=data,
page=3,
target_selected=target_selected)
def load_section(self, section):
reader = config_reader.read_config(self.path)
if section in reader.sections():
for key, val in reader[section].items():
self.add_item(section, key, val)
def test_configreader():
config = config_reader.read_config("../config/default.ini")
assert config['TASK0']['type'] == 'classification'
assert config.get_as_slice("TASK0","ground_truth_column") == -1
print(config['PROCESS']['experiment_ID'])
import time
import matplotlib.pyplot as plt
import numpy as np
from sko.GA import GA
from sko.operators import crossover
import config.config_reader as cfg
from data import hub, user_demand as ud
# station_count
station_count = int(cfg.read_config('station_count'))
def fitness_fun(x):
"""Definition of the fitness function
:param x: the allocation for bicycles at the beginning of each hour
:return:
"""
# user's demand in current hour
user_demand = ud.user_800[0]
# user's borrow count estimated
borrow_amount = np.zeros((station_count, station_count))
# bike available in this hour
available = x
# value of the objective function
fval = 0
demand = np.array(user_demand)
for station_src in range(station_count):
for station_dst in range(station_count):
if station_src == station_dst:
'class': 'Iris-setosa'
}
frequent_values2frequency = {
'sepal_length': (5.0, 10),
'sepal_width': (3.0, 26),
'petal_length': (1.5, 14),
'petal_width': (0.2, 28),
'class': ('Iris-setosa', 50)
}
SAMPLE_DATA_SIZE = 5
data = preprocessing.insert_data(df, categories, unique_values, default_list,
frequent_values2frequency, SAMPLE_DATA_SIZE)
data.Category = categories
labels = feature_util.get_target_labels(target, data.Category[target], fs)
all_params_config = config_reader.read_config(config_file)
export_dir = all_params_config.export_dir()
dtypes = fs.group_by(categories)
category_list = [
'numerical', 'numerical', 'numerical', 'numerical', 'categorical'
]
features = fs.create_tf_features(categories, target)
def test_add_and_get_port():
t.add_port(username, config_file, port)
assert t.get_port(username, config_file) == port
def test_pause_threads():
t.pause_threads(username)
def config():
return config_reader.read_config(sys_ops.abs_path_of(CONFIG_FILE))
def create_config(self, name, dataset_name):
self.add_item('info', 'dataset_name', dataset_name)
self.add_item('info', 'config_name', name)
self.add_item('info', 'config_path', self.path)
if 'info' not in config_reader.read_config(self.path).sections():
self.write_config()
import time
from random import shuffle
import matplotlib.pyplot as plt
import numpy as np
from sko.GA import GA
from sko.operators import crossover, mutation
import config.config_reader as cfg
from data import hub, user_demand as ud, desired_target as tgt
# multi-objective parameter \phi
phi = int(cfg.read_config('phi'))
# station_count
station_count = int(cfg.read_config('station_count'))
# truck count
truck_count = int(cfg.read_config('truck_count'))
# desired schedule target
desired_schedule_target = tgt.target
# initial bike count
initial_bike = ud.initial_bike_count
# truck_capacity
truck_capacity = int(cfg.read_config('truck_capacity'))
def fitness_fun(x):
"""Definition of the fitness function
