print(search) for ii in glob(search): print(ii) with open(ii) as infile: for jj in DictReader(infile): seen.add(int(jj['question'])) print(jj) print(seen) sleep(1) return seen if __name__ == "__main__": from util import flags flags.define_string("title_index", None, "Pickle of all titles") flags.define_string("label_path", None, "Where we write page associations") flags.define_string("database", None, "Question database") flags.define_string("performance_output", None, "Where we write user performance") flags.define_string("user", None, "User identifier") flags.InitFlags() seen = already_answered(flags.performance_output, flags.user) al = ActiveLearner(None, flags.label_path) print("Loading question db %s" % flags.database) db = QuestionDatabase(flags.database) pw = PerformanceWriter(flags.performance_output, flags.user) tf = TitleFinder(open(flags.title_index)) questions = db.questions_by_tournament("High School Championship")
search = "%s/%s*.csv" % (path, user) print(search) for ii in glob(search): print(ii) with open(ii) as infile: for jj in DictReader(infile): seen.add(int(jj['question'])) print(jj) print(seen) sleep(1) return seen if __name__ == "__main__": from util import flags flags.define_string("title_index", None, "Pickle of all titles") flags.define_string("label_path", None, "Where we write page associations") flags.define_string("database", None, "Question database") flags.define_string("performance_output", None, "Where we write user performance") flags.define_string("user", None, "User identifier") flags.InitFlags() seen = already_answered(flags.performance_output, flags.user) al = ActiveLearner(None, flags.label_path) print("Loading question db %s" % flags.database) db = QuestionDatabase(flags.database) pw = PerformanceWriter(flags.performance_output, flags.user) tf = TitleFinder(open(flags.title_index)) questions = db.questions_by_tournament("High School Championship")
import os os.environ["CUDA_VISIBLE_DEVICES"] = "" import tensorflow as tf tf.config.experimental_run_functions_eagerly(True) import util.flags as flags from trainer.trainer_base import TrainerBase from input_fn.input_fn_2d.input_fn_generator_polygon2d import InputFnPolygon2D import model_fn.model_fn_2d.model_fn_polygon2d_classifier as models # Model parameter # =============== flags.define_string('model_type', 'ModelPolygonClassifier', 'Model Type to use choose from: ModelTriangle') flags.define_string('loss_mode', "abs_diff", "'abs_diff', 'softmax_crossentropy") flags.define_string('graph', 'GraphConv2MultiFF', 'class name of graph architecture') flags.define_dict( 'graph_params', {"edge_classifier": True}, "key=value pairs defining the configuration of the inference class. see used " "'inference'/'encoder'/'decoder'-class for available options. e.g.[" "mvn (bool), nhidden_lstm_1 (int), nhidden_lstm_2 (int)," "nhidden_lstm_3 (int), dropout_lstm_1 (float), dropout_lstm_2 (float), " "dropout_lstm_3 (float), relu_clip (float)]") flags.define_integer('data_len', 3142, 'F(phi) amount of values saved in one line') flags.define_integer(
import uuid import numpy as np import tensorflow as tf import util.flags as flags import input_fn.input_fn_2d.data_gen_2dt.data_gen_t2d_util.tfr_helper as tfr_helper from util.misc import get_commit_id os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # or any {'0', '1', '2'} set tensorflow logleve 2=warning os.environ["CUDA_VISIBLE_DEVICES"] = "" # hide all gpu's until needed # ======== flags.define_string("data_id", "magic_synthetic_dataset", "select a name unique name for the dataset") flags.define_boolean("to_log_file", False, "if set redirect stdout & stderr to this file in data/syntetic_data/<data_id>/<log-file.log>") flags.define_string("mode", "val", "select 'val' or 'train'") flags.define_list('files_train_val', int, "[int(train_files), int(val_files)]", 'files to generate for train data/val data', default_value=[1000, 10]) flags.define_integer("samples_per_file", 1000, "set number of samples saved in each file") flags.define_integer("jobs", -1, "set number of samples saved in each file") if __name__ == "__main__": main_data_out = "data/synthetic_data/{}".format(flags.FLAGS.data_id) original_out = sys.stdout original_err = sys.stderr if flags.FLAGS.to_log_file: logfile_path = os.path.join(main_data_out, "log_{}_{}.txt".format(flags.FLAGS.data_id, flags.FLAGS.mode)) if not os.path.isdir(os.path.dirname(logfile_path)):
import os os.environ["CUDA_VISIBLE_DEVICES"] = "" import util.flags as flags from trainer.trainer_base import TrainerBase from input_fn.input_fn_2d.input_fn_generator_rp2d import InputFnRegularPolygon2D import model_fn.model_fn_2d.model_fn_rp2d as models # Model parameter # =============== flags.define_string('model_type', 'ModelRegularPolygon', 'Model Type to use choose from: ModelTriangle') flags.define_string('graph', 'GraphConv2MultiFF', 'class name of graph architecture') flags.define_dict('graph_params', {}, "key=value pairs defining the configuration of the inference class. see used " "'inference'/'encoder'/'decoder'-class for available options. e.g.[" "mvn (bool), nhidden_lstm_1 (int), nhidden_lstm_2 (int)," "nhidden_lstm_3 (int), dropout_lstm_1 (float), dropout_lstm_2 (float), " "dropout_lstm_3 (float), relu_clip (float)]") flags.define_integer('data_len', 3142, 'F(phi) amount of values saved in one line') flags.define_integer('max_edges', 8, "Max number of edges must be known (depends on dataset), " "if unknown pick one which is definitv higher than edges in dataset") flags.FLAGS.parse_flags() class TrainerRegularPolygon2D(TrainerBase): def __init__(self): super(TrainerRegularPolygon2D, self).__init__() self._input_fn_generator = InputFnRegularPolygon2D(self._flags) self._model_fn = getattr(models, self._flags.model_type)(self._params)
import os import model_fn.model_fn_2d.model_fn_2dtriangle as model_fn_classes import util.flags as flags from input_fn.input_fn_2d.input_fn_generator_triangle2d import InputFn2DT from trainer.lav_base import LavBase os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # set tf log_level to warning(2), default: info(1) os.environ['TF_CUDNN_USE_AUTOTUNE'] = '0' # no tune necessary, short running time flags.define_string('model_type', 'ModelTriangle', 'Model Type to use choose from: ModelTriangle, ...') flags.define_string('graph', 'GraphBase', 'GraphBase should be enough to load saved model') flags.define_boolean('complex_phi', False, "if set: a=phi.real, b=phi.imag, instead of a=cos(phi) b=sin(phi)-1" "additional flag need for specific input_fn, model, graph") flags.define_boolean('plot', False, "plot results in pdf file, (slow)") flags.FLAGS.parse_flags() class LavTriangle2D(LavBase): def __init__(self): super(LavTriangle2D, self).__init__() self._input_fn_generator = InputFn2DT(self._flags) self._model = getattr(model_fn_classes, self._flags.model_type)(self._params) if __name__ == "__main__": lav = LavTriangle2D() lav.lav()
import util.flags as flags from util.misc import get_commit_id, Tee os.environ["CUDA_VISIBLE_DEVICES"] = "" os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # Training # ======== flags.define_integer( 'epochs', 200, 'Epochs to train. If checkpoint already has these epochs, ' 'a evaluation and export is done') flags.define_integer('samples_per_epoch', 100000, 'Samples shown to the net per epoch.') # flags.define_boolean('calc_ema', False, 'Choose whether you want to use EMA (Exponential Moving Average) ' # 'weights or not,') # flags.define_float('clip_grad', 0.0, 'gradient clipping value: for positive values GLOBAL norm clipping is performed,' # ' for negative values LOCAL norm clipping is performed (default: %(default)s)') flags.define_string('optimizer', 'DecayOptimizer', 'the optimizer used to compute and apply gradients.') flags.define_dict( 'optimizer_params', {}, "key=value pairs defining the configuration of the optimizer.") flags.define_string('learn_rate_schedule', "decay", 'decay, finaldecay, warmupfinaldecay') flags.define_dict( "learn_rate_params", {}, "key=value pairs defining the configuration of the learn_rate_schedule.") # flags.define_string('train_scopes', '', 'Change only variables in this scope during training') flags.define_integer( 'eval_every_n', 1, "Evaluate/Validate every 'n' epochs") # Todo: to be implemented flags.define_string('checkpoint_dir', '', 'Checkpoint to save model information in.') # flags.define_string('warmstart_dir', '', 'load pretrained model (ignored if checkpoint_dir already exists, '
import os import time import tensorflow as tf import util.flags as flags os.environ["CUDA_VISIBLE_DEVICES"] = "" os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' flags.define_string('model_dir', '', 'dir with "export"-folder which was checkpoint_dir before or path to "export"') flags.define_string('val_list', 'lists/dummy_val.lst', '.lst-file specifying the dataset used for validation') flags.define_integer('val_batch_size', 100, 'number of elements in a val_batch') flags.define_list('gpu_devices', int, 'space seperated list of GPU indices to use. ', " ", []) flags.FLAGS.parse_flags() flags.define_float('gpu_memory', 0.0, 'amount of gpu memory in MB if set above 0') flags.define_string("debug_dir", "", "specify dir to save debug outputs, saves are model specific ") flags.define_integer("batch_limiter", -1, "set to positiv value to stop validation after this number of batches") flags.FLAGS.parse_flags() class LavBase(object): def __init__(self): self._flags = flags.FLAGS flags.print_flags() self.set_run_config() self._input_fn_generator = None self._val_dataset = None self._graph_eval = None self._model = None self._model_fn_classes = None self._params = None
import os import logging from trainer.trainer_base import TrainerBase import model_fn.model_fn_2d.model_fn_2dtriangle as models import util.flags as flags from input_fn.input_fn_2d.input_fn_generator_triangle2d import InputFn2DT # Model parameter # =============== flags.define_string('model_type', 'ModelTriangle', 'Model Type to use choose from: ModelTriangle') flags.define_string('graph', 'KerasGraphFF3', 'class name of graph architecture') flags.define_dict( 'graph_params', {}, "key=value pairs defining the configuration of the inference class. see used " "'inference'/'encoder'/'decoder'-class for available options. e.g.[" "mvn (bool), nhidden_lstm_1 (int), nhidden_lstm_2 (int)," "nhidden_lstm_3 (int), dropout_lstm_1 (float), dropout_lstm_2 (float), " "dropout_lstm_3 (float), relu_clip (float)]") flags.define_string('loss_mode', 'point3', 'switch loss calculation, see model_fn_2dtriangle.py') flags.define_integer('data_len', 3142, 'F(phi) amount of values saved in one line') flags.define_boolean( 'complex_phi', False, "if set: a=phi.real, b=phi.imag, instead of a=cos(phi) b=sin(phi)-1") flags.FLAGS.parse_flags() class Trainer2DTriangle(TrainerBase):