def existing_data(form, user_configs, username, sess, APP_ROOT): dataset_name = form['exisiting_files-train_file_exist'] path = os.path.join(APP_ROOT, 'user_data', username, dataset_name) if form['exisiting_files-configuration'] != 'new_config': config_name = form['exisiting_files-configuration'] sess.set('config_file', os.path.join(path, config_name, 'config.ini')) sess.load_config() return 'parameters' else: config_name = config_ops.define_new_config_file( dataset_name, APP_ROOT, username, sess.get_writer()) sess.set('config_file', os.path.join(path, config_name, 'config.ini')) if user_configs[dataset_name] and os.path.isfile( os.path.join(path, user_configs[dataset_name][0], 'config.ini')): reader = config_reader.read_config( os.path.join(path, user_configs[dataset_name][0], 'config.ini')) copyfile( os.path.join(path, user_configs[dataset_name][0], 'config.ini'), os.path.join(path, config_name, 'config.ini')) filename = reader['PATHS']['file'] elif os.path.isfile(os.path.join(path, dataset_name + '.csv')): filename = os.path.join(path, dataset_name + '.csv') else: filename = [ f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) and '.csv' in f ][0] sess.set('file', os.path.join(path, filename)) sess.get_writer().add_item('PATHS', 'file', os.path.join(path, filename)) sess.get_writer().write_config(sess.get('config_file')) return 'slider'
def tensor_board_thread(self, config_file, port): config_path = config_reader.read_config(config_file).all()['checkpoint_dir'] logging.debug('Starting tensor board') time.sleep(3) pro = "tensorboard --host=0.0.0.0 --logdir=" + config_path + " --port=" + port subprocess.call(pro, shell=True) logging.debug('Exiting tensor board')
def explain(): if request.method == 'POST': new_features = feature_util.get_new_features( request.form, sess.get('defaults'), sess.get('target'), sess.get('fs').group_by(sess.get('category_list'))['none']) all_params_config = config_reader.read_config(sess.get('config_file')) all_params_config.set( 'PATHS', 'checkpoint_dir', os.path.join(all_params_config.export_dir(), request.form['radiob'])) labels = feature_util.get_target_labels( sess.get('target'), sess.get('data').Category[sess.get('target')], sess.get('fs')) dtypes = sess.get('fs').group_by(sess.get('category_list')) result = th.explain_estimator(all_params_config, sess.get('features'), sess.get('target'), labels, sess.get('defaults'), dtypes, new_features, sess.get('df'), sess.get('data').Category, int(request.form['num_feat']), int(request.form['top_labels'])) if result is not None: fp = BytesIO(str.encode(result.as_html(show_table=True))) sess.set('explain_fp', fp) return jsonify(explanation='ok') return jsonify(explanation=str(result)) else: return send_file(sess.get('explain_fp'), mimetype='text/html')
def assign_category(self, config_file, df): feature_dict = self.feature_dict() unique_values = [ self.unique_value_size_dict.get(key, -1) for key in df.columns ] category_list = [feature_dict[key] for key in df.columns] if 'COLUMN_CATEGORIES' in config_reader.read_config( config_file).keys(): category_list = [] for key in df.columns: category_list.append( config_reader.read_config(config_file)['COLUMN_CATEGORIES'] [key]) default_list = self.defaults frequent_values2frequency = self.frequent_values2frequency return category_list, unique_values, default_list, frequent_values2frequency
def run(): config = config_reader.read_config() tables_reader.read(config.connection_string, config.tables) for destination in config.destination_items: for table in config.tables: code = destination.get_code(table) csharp_project.add_code(destination, table.entity_name, code)
def refresh(): try: CONFIG_FILE = sess.get('config_file') export_dir = config_reader.read_config(CONFIG_FILE).export_dir() checkpoints = run_utils.get_eval_results(export_dir, sess.get_writer(), CONFIG_FILE) return jsonify(checkpoints=checkpoints) except KeyError: return jsonify(checkpoints='')
def config_ga(): lb = [1] * (station_count + truck_count - 1) ub = [21] * (station_count + truck_count - 1) iteration = int(cfg.read_config("iteration_time")) ga = GA(func=fitness_fun, size_pop=100, n_dim=station_count + truck_count - 1, max_iter=iteration, prob_mut=0.01, lb=lb, ub=ub, precision=[1] * (station_count + truck_count - 1)) ga.Chrom = generate_new_pop() if cfg.read_config("crossover_type") is not None: ga.register(operator_name='crossover', operator=customized_crossover). \ register(operator_name='mutation', operator=customized_mutation) # TODO: 排序的策略配置(暂时不考虑) return ga
def test_get_eva_results(): writer = ConfigWriter() writer.append_config(config_test_file) results = run_utils.get_eval_results(directory, writer, config_test_file) assert results == checkpoints reader = config_reader.read_config(config_test_file) assert reader['BEST_MODEL']['max_acc'] == '1.0' assert reader['BEST_MODEL']['max_acc_index'] == '42' assert reader['BEST_MODEL']['min_loss'] == '0.153' assert reader['BEST_MODEL']['min_loss_index'] == '42'
def set_form(form, config): config = read_config(config.get('info', 'config_path')) if 'copula_factor_algorithm' in config.keys(): form.copula_factor_algorithm.form.gibbs_sampling_n.default = config.get( 'copula_factor_algorithm', 'gibbs_sampling_n') form.copula_factor_algorithm.form.gibbs_burn_in_n.default = config.get( 'copula_factor_algorithm', 'gibbs_burn_in_n') form.copula_factor_algorithm.form.gibbs_first_random_seed_n.default = config.get( 'copula_factor_algorithm', 'gibbs_first_random_seed_n') form.copula_factor_algorithm.form.gibbs_random_seed_update_parameter_n.default = config.get( 'copula_factor_algorithm', 'gibbs_random_seed_update_parameter_n') form.copula_factor_algorithm.form.process() if 'edge_weight_algorithm' in config.keys(): form.edge_weight_algorithm.form.bootstrap_n.default = config.get( 'edge_weight_algorithm', 'bootstrap_n') form.edge_weight_algorithm.form.bootstrap_first_random_seed_n.default = config.get( 'edge_weight_algorithm', 'bootstrap_first_random_seed_n') form.edge_weight_algorithm.form.bootstrap_random_seed_update_parameter_n.default = config.get( 'edge_weight_algorithm', 'bootstrap_random_seed_update_parameter_n') form.edge_weight_algorithm.form.process() if 'pc_algorithm' in config.keys(): form.pc_algorithm.form.causal_discovery_observation_n.default = config.get( 'pc_algorithm', 'causal_discovery_observation_n') form.pc_algorithm.form.indepTest.default = config.get( 'pc_algorithm', 'indepTest') form.pc_algorithm.form.alpha.default = config.get( 'pc_algorithm', 'alpha') form.pc_algorithm.form.numCores.default = config.get( 'pc_algorithm', 'numCores') form.pc_algorithm.form.verbose.default = config.get( 'pc_algorithm', 'verbose') form.pc_algorithm.form.NAdelete.default = config.get( 'pc_algorithm', 'NAdelete') form.pc_algorithm.form.m_max.default = config.get( 'pc_algorithm', 'm.max') form.pc_algorithm.form.u2pd.default = config.get( 'pc_algorithm', 'u2pd') form.pc_algorithm.form.skel_method.default = config.get( 'pc_algorithm', 'skel.method') form.pc_algorithm.form.conservative.default = config.get( 'pc_algorithm', 'conservative') form.pc_algorithm.form.maj_rule.default = config.get( 'pc_algorithm', 'maj.rule') form.pc_algorithm.form.solve_confl.default = config.get( 'pc_algorithm', 'solve.confl') form.pc_algorithm.form.fixedGaps.default = config.get( 'pc_algorithm', 'fixedGaps') form.pc_algorithm.form.fixedEdges.default = config.get( 'pc_algorithm', 'fixedEdges') form.pc_algorithm.form.process() if 'plot_and_display' in config.keys(): form.plot_and_display.form.core_plot_title_str.default = config.get( 'plot_and_display', 'core_plot_title_str') form.plot_and_display.form.process()
def set_form(form, CONFIG_FILE, hidden_layers): form.network.form.hidden_layers.default = hidden_layers form.network.form.process() # if os.path.isfile(CONFIG_FILE) : reader = config_reader.read_config(CONFIG_FILE) if 'EXPERIMENT' in reader.keys(): form.experiment.form.keep_checkpoint_max.default = reader[ 'EXPERIMENT']['keep_checkpoint_max'] form.experiment.form.save_checkpoints_steps.default = reader[ 'EXPERIMENT']['save_checkpoints_steps'] form.experiment.form.save_summary_steps.default = reader['EXPERIMENT'][ 'save_summary_steps'] form.experiment.form.throttle.default = reader['EXPERIMENT'][ 'throttle'] form.experiment.form.validation_batch_size.default = reader[ 'EXPERIMENT']['validation_batch_size'] if 'NETWORK' in reader.keys(): form.network.form.hidden_layers.default = reader['NETWORK'][ 'hidden_layers'] form.network.form.model_name.default = reader['NETWORK']['model_name'] if 'CUSTOM_MODEL' in reader.keys(): form.custom_model.form.custom_model_path.default = reader[ 'CUSTOM_MODEL']['custom_model_path'] if 'TRAINING' in config_reader.read_config(CONFIG_FILE).keys(): form.training.form.num_epochs.default = reader['TRAINING'][ 'num_epochs'] form.training.form.batch_size.default = reader['TRAINING'][ 'batch_size'] form.training.form.optimizer.default = reader['TRAINING']['optimizer'] form.training.form.learning_rate.default = reader['TRAINING'][ 'learning_rate'] form.training.form.l1_regularization.default = reader['TRAINING'][ 'l1_regularization'] form.training.form.l2_regularization.default = reader['TRAINING'][ 'l2_regularization'] form.training.form.dropout.default = reader['TRAINING']['dropout'] form.training.form.activation_fn.default = reader['TRAINING'][ 'activation_fn'] form.network.form.process() form.experiment.form.process() form.training.form.process() form.custom_model.form.process()
def test_write_features(): old_categories = ['none-numerical', 'numerical', 'numerical', 'numerical', 'categorical'] writer = ConfigWriter() writer.append_config(config_test_file) data.Category = ['none', 'none', 'numerical', 'numerical', 'none'] feature_util.write_features(old_categories, data, writer, config_test_file) reader = config_reader.read_config(config_test_file) assert reader['COLUMN_CATEGORIES']['sepal_length'] == 'none-numerical' assert reader['COLUMN_CATEGORIES']['sepal_width'] == 'none-numerical' assert reader['COLUMN_CATEGORIES']['petal_length'] == 'numerical' assert reader['COLUMN_CATEGORIES']['class'] == 'none-categorical'
def run(): all_params_config = config_reader.read_config(sess.get('config_file')) logfile = os.path.join(all_params_config['PATHS']['log_dir'], 'tensorflow.log') try: sess.set('log_fp', open(logfile)) except FileNotFoundError: open(logfile, 'a').close() sess.set('log_fp', open(logfile)) try: sess.get('status') except KeyError: sess.set('status', 'paused') labels = feature_util.get_target_labels( sess.get('target'), sess.get('data').Category[sess.get('target')], sess.get('fs')) export_dir = all_params_config.export_dir() checkpoints = run_utils.get_eval_results(export_dir, sess.get_writer(), sess.get('config_file')) th.run_tensor_board(session['user'], sess.get('config_file')) if request.method == 'POST': if request.form['action'] == 'run': sess.set('status', 'running') else: sess.set('status', 'paused') dtypes = sess.get('fs').group_by(sess.get('category_list')) th.handle_request(request.form['action'], all_params_config, sess.get('features'), sess.get('target'), labels, sess.get('defaults'), dtypes, session['user'], request.form['resume_from']) return jsonify(True) dict_types, categoricals = run_utils.get_dictionaries( sess.get('defaults'), sess.get('category_list'), sess.get('fs'), sess.get('target')) sfeatures = feature_util.remove_target(sess.get('defaults'), sess.get('target')) has_hash = 'false' if any(k in sess.get('category_list') for k in ("hash", "int-hash")): has_hash = 'true' return render_template('run.html', page=5, features=sfeatures, target=sess.get('target'), types=run_utils.get_html_types(dict_types), categoricals=categoricals, checkpoints=checkpoints, port=th.get_port(session['user'], sess.get('config_file')), running=sess.get('status'), has_hash=has_hash)
def config_ga(): constraint_eq = [ lambda x: int(cfg.read_config('bike_total_count')) - sum(x) ] lb = [0] * station_count ub = hub.hub iteration = int(cfg.read_config("iteration_time")) ga = GA(func=fitness_fun, size_pop=300, n_dim=station_count, max_iter=iteration, prob_mut=0.01, lb=lb, ub=ub, constraint_eq=constraint_eq, precision=[1] * 20) if cfg.read_config("crossover_type") is not None: ga.register(operator_name='crossover', operator=crossover.crossover_1point) # TODO: 变异和排序的策略配置(暂时不考虑) return ga
def delete(): CONFIG_FILE = sess.get('config_file') all_params_config = config_reader.read_config(CONFIG_FILE) export_dir = all_params_config.export_dir() del_id = request.get_json()['deleteID'] paths = [del_id] if del_id != 'all' else [ d for d in os.listdir(export_dir) if os.path.isdir(os.path.join(export_dir, d)) ] sys_ops.delete_recursive(paths, export_dir) checkpoints = run_utils.get_eval_results(export_dir, sess.get_writer(), CONFIG_FILE) return jsonify(checkpoints=checkpoints)
def load_config(self): # read saved config conf = config_reader.read_config(self.get('config_file')) # update files and df in config dict self.set('file', conf['PATHS']['file']) self.set('train_file', conf['PATHS']['train_file']) self.set('validation_file', conf['PATHS']['validation_file']) self.set('df', pd.read_csv(conf['PATHS']['file'])) self.load_features() # target select target = conf['TARGET']['target'] self.set_target(target) self.update_new_features(self.get('category_list'), list(self.get('defaults').values())) self.update_writer_conf(conf)
def get_configs_files(app_root, username): user_configs = {} existing_datasets = [] configs = {} user_path = os.path.join(app_root, USER_DATA, username) user_datasets = [ dataset for dataset in os.listdir(user_path) if os.path.isdir(os.path.join(user_path, dataset)) ] for user_dataset in user_datasets: user_configs[user_dataset] = [ config for config in os.listdir(os.path.join(user_path, user_dataset)) if os.path.isdir(os.path.join(user_path, user_dataset, config)) ] if INPUT_DATA in user_configs[user_dataset]: user_configs[user_dataset].remove(INPUT_DATA) if FACTOR_MODEL in user_configs[user_dataset]: user_configs[user_dataset].remove(FACTOR_MODEL) for config_file in user_configs[user_dataset]: dataset_config = user_dataset + '_' + config_file path_to_config_file = os.path.join(user_path, user_dataset, config_file, 'config.ini') config = read_config(path_to_config_file) configs[dataset_config] = { 'name': config.get_name(), 'path': config.get_path(), 'bootstrap': '' } configs[dataset_config]['bootstrap'] = '10' # param by default configs[dataset_config][ 'gibbs_sampling_n'] = '1000' # param by default configs[dataset_config][ 'gibbs_burn_in_n'] = '500' # param by default if 'edge_weight_algorithm' in config.sections(): configs[dataset_config]['bootstrap'] = config.get_bootstrap_n() if 'copula_factor_algorithm' in config.sections(): configs[dataset_config][ 'gibbs_sampling_n'] = config.get_gibbs_sampling_n() configs[dataset_config][ 'gibbs_burn_in_n'] = config.get_gibbs_burn_in_n() existing_datasets.append(user_dataset) return existing_datasets, user_configs, configs
def predict(): new_features = feature_util.get_new_features( request.form, sess.get('defaults'), sess.get('target'), sess.get('fs').group_by(sess.get('category_list'))['none']) all_params_config = config_reader.read_config(sess.get('config_file')) all_params_config.set( 'PATHS', 'checkpoint_dir', os.path.join(all_params_config.export_dir(), request.form['radiob'])) labels = feature_util.get_target_labels( sess.get('target'), sess.get('data').Category[sess.get('target')], sess.get('fs')) dtypes = sess.get('fs').group_by(sess.get('category_list')) final_pred = th.predict_estimator(all_params_config, sess.get('features'), sess.get('target'), labels, sess.get('defaults'), dtypes, new_features, sess.get('df')) return jsonify(prediction=str(final_pred))
def ga_plot(ga, bike_allocation): # print(ga.generation_best_Y) fig, ax = plt.subplots() generation = [] best = [] valid_gen = 0 for gen in ga.generation_best_Y: if gen < 0: best.append(gen) generation.append(valid_gen) valid_gen = valid_gen + 1 plt.xlim((0, valid_gen)) plt.ylim((600, 1000)) plt.xlabel('iteration time') plt.ylabel('fitness value') my_x_ticks = np.arange(0, valid_gen, 5) my_y_ticks = np.arange(600, 1000, 50) plt.xticks(my_x_ticks) plt.yticks(my_y_ticks) idx = 0 for item in ga.generation_best_X: if np.array_equal(item, bike_allocation): break idx = idx + 1 f_val = -fitness_fun(bike_allocation) max_cord = (idx, int(-fitness_fun(bike_allocation))) max_cord_shift = (idx + 2, int(-fitness_fun(bike_allocation)) + 10) ax.plot(np.array(generation), -np.array(best), color='red') plt.vlines(idx, 300, int(f_val), 'black', '--', label='example') plt.hlines(int(f_val), 0, idx, 'black', '--', label='example') plt.annotate("(" + str(idx) + ", " + str(-fitness_fun(bike_allocation)) + ")", xytext=max_cord, xy=max_cord_shift) plt.savefig('ga_800_iter' + str(int(cfg.read_config('iteration_time'))) + "_" + str(time.strftime('%Y%m%d_%H_%M_%S') + ".pdf"), format='pdf') plt.show()
def target(): form = Submit() if form.validate_on_submit(): sess.set_target(json.loads(request.form['selected_rows'])[0]) if 'split_df' in sess.get_config(): train_file, validation_file = preprocessing.split_train_test( sess.get('split_df'), sess.get('file'), sess.get('target'), sess.get('df')) sess.update_split(train_file, validation_file) sess.get_writer().write_config(sess.get('config_file')) return redirect(url_for('parameters')) reader = config_reader.read_config(sess.get('config_file')) target_selected = reader['TARGET']['target'] if 'TARGET' in reader.keys( ) else 'None' # filter hash and none columns data = sess.get('data')[(sess.get('data').Category != 'hash') & (sess.get('data').Category != 'none')] return render_template('target_selection.html', name="Dataset target selection", form=form, data=data, page=3, target_selected=target_selected)
def load_section(self, section): reader = config_reader.read_config(self.path) if section in reader.sections(): for key, val in reader[section].items(): self.add_item(section, key, val)
def test_configreader(): config = config_reader.read_config("../config/default.ini") assert config['TASK0']['type'] == 'classification' assert config.get_as_slice("TASK0","ground_truth_column") == -1 print(config['PROCESS']['experiment_ID'])
import time import matplotlib.pyplot as plt import numpy as np from sko.GA import GA from sko.operators import crossover import config.config_reader as cfg from data import hub, user_demand as ud # station_count station_count = int(cfg.read_config('station_count')) def fitness_fun(x): """Definition of the fitness function :param x: the allocation for bicycles at the beginning of each hour :return: """ # user's demand in current hour user_demand = ud.user_800[0] # user's borrow count estimated borrow_amount = np.zeros((station_count, station_count)) # bike available in this hour available = x # value of the objective function fval = 0 demand = np.array(user_demand) for station_src in range(station_count): for station_dst in range(station_count): if station_src == station_dst:
'class': 'Iris-setosa' } frequent_values2frequency = { 'sepal_length': (5.0, 10), 'sepal_width': (3.0, 26), 'petal_length': (1.5, 14), 'petal_width': (0.2, 28), 'class': ('Iris-setosa', 50) } SAMPLE_DATA_SIZE = 5 data = preprocessing.insert_data(df, categories, unique_values, default_list, frequent_values2frequency, SAMPLE_DATA_SIZE) data.Category = categories labels = feature_util.get_target_labels(target, data.Category[target], fs) all_params_config = config_reader.read_config(config_file) export_dir = all_params_config.export_dir() dtypes = fs.group_by(categories) category_list = [ 'numerical', 'numerical', 'numerical', 'numerical', 'categorical' ] features = fs.create_tf_features(categories, target) def test_add_and_get_port(): t.add_port(username, config_file, port) assert t.get_port(username, config_file) == port def test_pause_threads(): t.pause_threads(username)
def config(): return config_reader.read_config(sys_ops.abs_path_of(CONFIG_FILE))
def create_config(self, name, dataset_name): self.add_item('info', 'dataset_name', dataset_name) self.add_item('info', 'config_name', name) self.add_item('info', 'config_path', self.path) if 'info' not in config_reader.read_config(self.path).sections(): self.write_config()
import time from random import shuffle import matplotlib.pyplot as plt import numpy as np from sko.GA import GA from sko.operators import crossover, mutation import config.config_reader as cfg from data import hub, user_demand as ud, desired_target as tgt # multi-objective parameter \phi phi = int(cfg.read_config('phi')) # station_count station_count = int(cfg.read_config('station_count')) # truck count truck_count = int(cfg.read_config('truck_count')) # desired schedule target desired_schedule_target = tgt.target # initial bike count initial_bike = ud.initial_bike_count # truck_capacity truck_capacity = int(cfg.read_config('truck_capacity')) def fitness_fun(x): """Definition of the fitness function