print(search)
for ii in glob(search):
print(ii)
with open(ii) as infile:
for jj in DictReader(infile):
seen.add(int(jj['question']))
print(jj)
print(seen)
sleep(1)
return seen
if __name__ == "__main__":
from util import flags
flags.define_string("title_index", None, "Pickle of all titles")
flags.define_string("label_path", None, "Where we write page associations")
flags.define_string("database", None, "Question database")
flags.define_string("performance_output", None,
"Where we write user performance")
flags.define_string("user", None, "User identifier")
flags.InitFlags()
seen = already_answered(flags.performance_output, flags.user)
al = ActiveLearner(None, flags.label_path)
print("Loading question db %s" % flags.database)
db = QuestionDatabase(flags.database)
pw = PerformanceWriter(flags.performance_output, flags.user)
tf = TitleFinder(open(flags.title_index))
questions = db.questions_by_tournament("High School Championship")
search = "%s/%s*.csv" % (path, user)
print(search)
for ii in glob(search):
print(ii)
with open(ii) as infile:
for jj in DictReader(infile):
seen.add(int(jj['question']))
print(jj)
print(seen)
sleep(1)
return seen
if __name__ == "__main__":
from util import flags
flags.define_string("title_index", None, "Pickle of all titles")
flags.define_string("label_path", None, "Where we write page associations")
flags.define_string("database", None, "Question database")
flags.define_string("performance_output", None, "Where we write user performance")
flags.define_string("user", None, "User identifier")
flags.InitFlags()
seen = already_answered(flags.performance_output, flags.user)
al = ActiveLearner(None, flags.label_path)
print("Loading question db %s" % flags.database)
db = QuestionDatabase(flags.database)
pw = PerformanceWriter(flags.performance_output, flags.user)
tf = TitleFinder(open(flags.title_index))
questions = db.questions_by_tournament("High School Championship")
import os
os.environ["CUDA_VISIBLE_DEVICES"] = ""
import tensorflow as tf
tf.config.experimental_run_functions_eagerly(True)
import util.flags as flags
from trainer.trainer_base import TrainerBase
from input_fn.input_fn_2d.input_fn_generator_polygon2d import InputFnPolygon2D
import model_fn.model_fn_2d.model_fn_polygon2d_classifier as models
# Model parameter
# ===============
flags.define_string('model_type', 'ModelPolygonClassifier',
'Model Type to use choose from: ModelTriangle')
flags.define_string('loss_mode', "abs_diff",
"'abs_diff', 'softmax_crossentropy")
flags.define_string('graph', 'GraphConv2MultiFF',
'class name of graph architecture')
flags.define_dict(
'graph_params', {"edge_classifier": True},
"key=value pairs defining the configuration of the inference class. see used "
"'inference'/'encoder'/'decoder'-class for available options. e.g.["
"mvn (bool), nhidden_lstm_1 (int), nhidden_lstm_2 (int),"
"nhidden_lstm_3 (int), dropout_lstm_1 (float), dropout_lstm_2 (float), "
"dropout_lstm_3 (float), relu_clip (float)]")
flags.define_integer('data_len', 3142,
'F(phi) amount of values saved in one line')
flags.define_integer(
import uuid
import numpy as np
import tensorflow as tf
import util.flags as flags
import input_fn.input_fn_2d.data_gen_2dt.data_gen_t2d_util.tfr_helper as tfr_helper
from util.misc import get_commit_id
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # or any {'0', '1', '2'} set tensorflow logleve 2=warning
os.environ["CUDA_VISIBLE_DEVICES"] = "" # hide all gpu's until needed
# ========
flags.define_string("data_id", "magic_synthetic_dataset", "select a name unique name for the dataset")
flags.define_boolean("to_log_file", False,
"if set redirect stdout & stderr to this file in data/syntetic_data/<data_id>/<log-file.log>")
flags.define_string("mode", "val", "select 'val' or 'train'")
flags.define_list('files_train_val', int, "[int(train_files), int(val_files)]",
'files to generate for train data/val data', default_value=[1000, 10])
flags.define_integer("samples_per_file", 1000, "set number of samples saved in each file")
flags.define_integer("jobs", -1, "set number of samples saved in each file")
if __name__ == "__main__":
main_data_out = "data/synthetic_data/{}".format(flags.FLAGS.data_id)
original_out = sys.stdout
original_err = sys.stderr
if flags.FLAGS.to_log_file:
logfile_path = os.path.join(main_data_out, "log_{}_{}.txt".format(flags.FLAGS.data_id, flags.FLAGS.mode))
if not os.path.isdir(os.path.dirname(logfile_path)):
import os
os.environ["CUDA_VISIBLE_DEVICES"] = ""
import util.flags as flags
from trainer.trainer_base import TrainerBase
from input_fn.input_fn_2d.input_fn_generator_rp2d import InputFnRegularPolygon2D
import model_fn.model_fn_2d.model_fn_rp2d as models
# Model parameter
# ===============
flags.define_string('model_type', 'ModelRegularPolygon', 'Model Type to use choose from: ModelTriangle')
flags.define_string('graph', 'GraphConv2MultiFF', 'class name of graph architecture')
flags.define_dict('graph_params', {},
"key=value pairs defining the configuration of the inference class. see used "
"'inference'/'encoder'/'decoder'-class for available options. e.g.["
"mvn (bool), nhidden_lstm_1 (int), nhidden_lstm_2 (int),"
"nhidden_lstm_3 (int), dropout_lstm_1 (float), dropout_lstm_2 (float), "
"dropout_lstm_3 (float), relu_clip (float)]")
flags.define_integer('data_len', 3142, 'F(phi) amount of values saved in one line')
flags.define_integer('max_edges', 8, "Max number of edges must be known (depends on dataset), "
"if unknown pick one which is definitv higher than edges in dataset")
flags.FLAGS.parse_flags()
class TrainerRegularPolygon2D(TrainerBase):
def __init__(self):
super(TrainerRegularPolygon2D, self).__init__()
self._input_fn_generator = InputFnRegularPolygon2D(self._flags)
self._model_fn = getattr(models, self._flags.model_type)(self._params)
import os
import model_fn.model_fn_2d.model_fn_2dtriangle as model_fn_classes
import util.flags as flags
from input_fn.input_fn_2d.input_fn_generator_triangle2d import InputFn2DT
from trainer.lav_base import LavBase
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # set tf log_level to warning(2), default: info(1)
os.environ['TF_CUDNN_USE_AUTOTUNE'] = '0' # no tune necessary, short running time
flags.define_string('model_type', 'ModelTriangle', 'Model Type to use choose from: ModelTriangle, ...')
flags.define_string('graph', 'GraphBase', 'GraphBase should be enough to load saved model')
flags.define_boolean('complex_phi', False, "if set: a=phi.real, b=phi.imag, instead of a=cos(phi) b=sin(phi)-1"
"additional flag need for specific input_fn, model, graph")
flags.define_boolean('plot', False, "plot results in pdf file, (slow)")
flags.FLAGS.parse_flags()
class LavTriangle2D(LavBase):
def __init__(self):
super(LavTriangle2D, self).__init__()
self._input_fn_generator = InputFn2DT(self._flags)
self._model = getattr(model_fn_classes, self._flags.model_type)(self._params)
if __name__ == "__main__":
lav = LavTriangle2D()
lav.lav()
import util.flags as flags
from util.misc import get_commit_id, Tee
os.environ["CUDA_VISIBLE_DEVICES"] = ""
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
# Training
# ========
flags.define_integer(
'epochs', 200, 'Epochs to train. If checkpoint already has these epochs, '
'a evaluation and export is done')
flags.define_integer('samples_per_epoch', 100000,
'Samples shown to the net per epoch.')
# flags.define_boolean('calc_ema', False, 'Choose whether you want to use EMA (Exponential Moving Average) '
# 'weights or not,')
# flags.define_float('clip_grad', 0.0, 'gradient clipping value: for positive values GLOBAL norm clipping is performed,'
# ' for negative values LOCAL norm clipping is performed (default: %(default)s)')
flags.define_string('optimizer', 'DecayOptimizer',
'the optimizer used to compute and apply gradients.')
flags.define_dict(
'optimizer_params', {},
"key=value pairs defining the configuration of the optimizer.")
flags.define_string('learn_rate_schedule', "decay",
'decay, finaldecay, warmupfinaldecay')
flags.define_dict(
"learn_rate_params", {},
"key=value pairs defining the configuration of the learn_rate_schedule.")
# flags.define_string('train_scopes', '', 'Change only variables in this scope during training')
flags.define_integer(
'eval_every_n', 1,
"Evaluate/Validate every 'n' epochs") # Todo: to be implemented
flags.define_string('checkpoint_dir', '',
'Checkpoint to save model information in.')
# flags.define_string('warmstart_dir', '', 'load pretrained model (ignored if checkpoint_dir already exists, '
import os
import time
import tensorflow as tf
import util.flags as flags
os.environ["CUDA_VISIBLE_DEVICES"] = ""
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
flags.define_string('model_dir', '', 'dir with "export"-folder which was checkpoint_dir before or path to "export"')
flags.define_string('val_list', 'lists/dummy_val.lst', '.lst-file specifying the dataset used for validation')
flags.define_integer('val_batch_size', 100, 'number of elements in a val_batch')
flags.define_list('gpu_devices', int, 'space seperated list of GPU indices to use. ', " ", [])
flags.FLAGS.parse_flags()
flags.define_float('gpu_memory', 0.0, 'amount of gpu memory in MB if set above 0')
flags.define_string("debug_dir", "", "specify dir to save debug outputs, saves are model specific ")
flags.define_integer("batch_limiter", -1, "set to positiv value to stop validation after this number of batches")
flags.FLAGS.parse_flags()
class LavBase(object):
def __init__(self):
self._flags = flags.FLAGS
flags.print_flags()
self.set_run_config()
self._input_fn_generator = None
self._val_dataset = None
self._graph_eval = None
self._model = None
self._model_fn_classes = None
self._params = None
import os
import logging
from trainer.trainer_base import TrainerBase
import model_fn.model_fn_2d.model_fn_2dtriangle as models
import util.flags as flags
from input_fn.input_fn_2d.input_fn_generator_triangle2d import InputFn2DT
# Model parameter
# ===============
flags.define_string('model_type', 'ModelTriangle',
'Model Type to use choose from: ModelTriangle')
flags.define_string('graph', 'KerasGraphFF3',
'class name of graph architecture')
flags.define_dict(
'graph_params', {},
"key=value pairs defining the configuration of the inference class. see used "
"'inference'/'encoder'/'decoder'-class for available options. e.g.["
"mvn (bool), nhidden_lstm_1 (int), nhidden_lstm_2 (int),"
"nhidden_lstm_3 (int), dropout_lstm_1 (float), dropout_lstm_2 (float), "
"dropout_lstm_3 (float), relu_clip (float)]")
flags.define_string('loss_mode', 'point3',
'switch loss calculation, see model_fn_2dtriangle.py')
flags.define_integer('data_len', 3142,
'F(phi) amount of values saved in one line')
flags.define_boolean(
'complex_phi', False,
"if set: a=phi.real, b=phi.imag, instead of a=cos(phi) b=sin(phi)-1")
flags.FLAGS.parse_flags()
class Trainer2DTriangle(TrainerBase):
