async def get_file(request: web.Request):
user_id = id_validator(request.match_info['user_id'], 'User')
file_id = id_validator(request.match_info['file_id'], 'File')
user_table = get_model_by_name('user')
user_exists = await request.app['pg'].fetchval(
select([exists().where(user_table.c.user_id == user_id)]))
if not user_exists:
raise web.HTTPNotFound(body=json.dumps({'error': 'User not found'}),
content_type='application/json')
file_table = get_model_by_name('file')
file_exists = await request.app['pg'].fetchval(
select([
exists().where(
and_(file_table.c.id == file_id,
file_table.c.user_id == user_id))
]))
if not file_exists:
raise web.HTTPNotFound(body=json.dumps({'error': 'File not found'}),
content_type='application/json')
file = await request.app['pg'].fetchrow(
file_table.select().where(file_table.c.id == file_id))
file = row_to_dict(file, 'file')
return web.FileResponse(path=file['path'], status=200)
async def post(self):
user_id = id_validator(self.request.match_info['user_id'], 'User')
if self.request.content_type != 'multipart/form-data' or self.request.content_length == 0:
return web.json_response(data=[])
user_table = get_model_by_name('user')
file_table = get_model_by_name('file')
user_exists = await self.request.app['pg'].fetchval(
select([exists().where(user_table.c.user_id == user_id)]))
if not user_exists:
await self.request.app['pg'].fetchrow(
user_table.insert().values(**{'user_id': user_id}))
reader = await self.request.multipart()
upload_folder = self.request.app['config']['UPLOAD_FOLDER']
data = []
while not reader.at_eof():
image = await reader.next()
if not image:
break
file_name, ext = get_ext(image.filename)
generated_file_name = '{}.{}'.format(uuid.uuid4(), ext)
full_path = os.path.abspath(
os.path.join(upload_folder, generated_file_name))
size = 0
with open(full_path, 'wb') as f:
while True:
chunk = await image.read_chunk()
if not chunk:
break
size += len(chunk)
f.write(chunk)
body = {
'user_id': user_id,
'name': image.filename,
'path': full_path,
'size': size
}
file = await self.request.app['pg'].fetchrow(
file_table.insert().values(**body).returning(
literal_column('*')))
file = row_to_dict(file, 'file')
data.append(dict(file))
return web.json_response(data=data)
async def get(self):
user_id = id_validator(self.request.match_info['user_id'], 'User')
user_table = get_model_by_name('user')
user_exists = await self.request.app['pg'].fetchval(
select([exists().where(user_table.c.user_id == user_id)]))
if not user_exists:
raise web.HTTPNotFound(body=json.dumps(
{'error': f'User with id={user_id} not found'}),
content_type='application/json')
file_table = get_model_by_name('file')
files = await self.request.app['pg'].fetch(
file_table.select().where(file_table.c.user_id == user_id))
files = many_row_to_dict(files, 'file')
return web.json_response(data=files)
def __init__(self,
sketches_x,
sketches_y,
class_labels,
model_id,
out_dir,
resume=True,
gpu_id=0):
"""train a new model with the given data. the data
is expected to be in a stroke-3 format.
Args:
sketches_x (array-like): array of N sketches in 3-stroke format
sketches_y (arraly-like): class labels for each of the N sketches
class_labels (array-like): all class labels
model_id (string): unique id for the model, it is used to store, restore checkpoints
out_dir (string): the path where the checkpoints should be stored
resume (bool, optional): If true, it will resume the latest checkpoint. Defaults to True.
gpu_id (int, optional): id of the gpu to run the training on. Defaults to 0.
"""
# create out_dir if not exist
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
# storing the number of classes will be useful for using the embeddings later with different
# number of classes in a testing data.
self.n_classes = len(class_labels)
self.class_labels = class_labels
utils.gpu.setup_gpu(gpu_id)
# prepare the dataset
dataset = self._convert_data(sketches_x, sketches_y, is_training=True)
Model = models.get_model_by_name(self.SKETCHFORMER_MODEL_NAME)
# # update all slow metricss to none
Transformer.slow_metrics = []
self.model = Model(Model.default_hparams(), dataset, out_dir, model_id)
if resume:
print("[run-experiment] resorting checkpoint if exists")
self.model.restore_checkpoint_if_exists("latest")
# continue training
if dataset.n_samples != 0:
self.model.train()
def main(_):
# tf.logging.set_verbosity(tf.logging.ERROR)
np.warnings.filterwarnings('ignore')
util.print('Loading model {} with dataset {}.'.format(
FLAGS.model, FLAGS.dataset))
model_config = models.get_model_by_name(FLAGS.model, dataset=FLAGS.dataset)
if FLAGS.results_dir == '':
results_dir = FLAGS.model + '_' + FLAGS.dataset
else:
results_dir = FLAGS.results_dir
if not tf.io.gfile.exists(results_dir):
tf.io.gfile.makedirs(results_dir)
filename = '{}{}{}{}{}.json'.format(
FLAGS.method, ('_' + FLAGS.learnable_parameterisation_type
if 'VIP' in FLAGS.method else ''),
('_tied' if FLAGS.tied_pparams else ''),
('_reparam_variational' if 'VIP' in FLAGS.method
and FLAGS.reparameterise_variational else ''),
('_discrete_prior'
if 'VIP' in FLAGS.method and FLAGS.discrete_prior else ''))
file_path = os.path.join(results_dir, filename)
if FLAGS.inference == 'VI':
run_vi(model_config, results_dir, file_path)
elif FLAGS.inference == 'HMC':
if FLAGS.method == 'i':
run_interleaved_hmc(model_config, results_dir, file_path)
else:
run_hmc(model_config, results_dir, file_path, tuning=False)
elif FLAGS.inference == 'HMCtuning':
run_hmc(model_config, results_dir, file_path, tuning=True)
def main():
parser = argparse.ArgumentParser(
description='Train modified transformer with sketch data')
parser.add_argument("model_name",
default=None,
help="Model that we are going to train")
parser.add_argument("--id", default="0", help="experiment signature")
parser.add_argument("--data-loader",
default='stroke3-distributed',
help="Data loader that will provide data for model")
parser.add_argument("--dataset", default=None, help="Input data folder")
parser.add_argument("-o",
"--output-dir",
default="",
help="output directory")
parser.add_argument('-p',
"--hparams",
default=None,
help="Parameters to override")
parser.add_argument(
"-g",
"--gpu",
default=0,
type=int,
nargs='+',
help="GPU ID to run on",
)
parser.add_argument('--metrics',
type=str,
nargs='+',
help="selection of metrics you want to calculate")
parser.add_argument("--help-hps",
action="store_true",
help="Prints out the hparams file")
parser.add_argument("-r",
"--resume",
default='latest',
help="One of 'latest' or a checkpoint name")
args = parser.parse_args()
# get our model and data loader classes
Model = models.get_model_by_name(args.model_name)
DataLoader = dataloaders.get_dataloader_by_name(args.data_loader)
# load the config
hps = utils.hparams.combine_hparams_into_one(Model.default_hparams(),
DataLoader.default_hparams())
utils.hparams.load_config(
hps, Model.get_config_filepath(args.output_dir, args.id))
# check for help screams from the void
if args.help_hps:
combined_hps = pprint.pformat(hps.values())
print("\nLoaded parameters from {}: \n{}\n\n".format(
args.model_dir, combined_hps))
return
# optional override of parameters
if args.hparams:
hps.parse(args.hparams)
# build model, load checkpoints
utils.gpu.setup_gpu(args.gpu)
dataset = DataLoader(hps, args.dataset)
model = Model(hps, dataset, args.output_dir, args.id)
model.restore_checkpoint_if_exists(args.resume)
# compute and send metrics
metric_names = args.metrics
metrics_list = {
m: metrics.build_metric_by_name(m, hps.values())
for m in metric_names
}
model.compute_metrics_from(metrics_list)
model.plot_and_send_notification_for(metrics_list)
model.clean_up_tmp_dir()
def main():
# Parsing arguments
parser = argparse.ArgumentParser(description='Training GANs or VAEs')
parser.add_argument('--model', type=str, required=True)
parser.add_argument('--dataset', type=str, required=True)
parser.add_argument('--epoch', type=int, default=200)
parser.add_argument('--batchsize', type=int, default=50)
parser.add_argument('--output', default='output')
parser.add_argument('--z-dims', type=int, default=256)
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--resume', type=str, default=None)
parser.add_argument('--test-mode', action='store_true')
parser.add_argument('--is-conditional', action='store_true')
parser.add_argument('--aux-classifier', action='store_true')
parser.add_argument('--label-smoothing', default=0.0, type=float)
parser.add_argument('--input-noise', default=0.0, type=float)
parser.add_argument('--run-id', '-r', required=True)
parser.add_argument('--checkpoint-every', default='1.', type=str)
parser.add_argument('--notify-every', default='1.', type=str)
parser.add_argument('--lr', default=1e-4, type=float)
parser.add_argument('--dis-loss-control', default=1., type=float)
parser.add_argument('--triplet-weight', default=1., type=float)
parser.add_argument('--embedding-dim', default=256, type=int)
parser.add_argument('--isolate-d-classifier', action='store_true')
parser.add_argument(
'--controlled-losses',
type=str,
nargs='+',
help="strings in format loss_name:weight:control_type:pivot_epoch")
parser.add_argument('--metrics',
type=str,
nargs='+',
help="selection of metrics you want to calculate")
parser.add_argument('--wgan-n-critic', default=5, type=int)
parser.add_argument('--began-gamma', default=0.5, type=float)
args = parser.parse_args()
# select gpu and limit resources if applicable
if 'tensorflow' == K.backend():
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.visible_device_list = str(args.gpu)
set_session(tf.Session(config=config))
# make output directory if not exists
if not os.path.isdir(args.output):
os.mkdir(args.output)
# load datasets
dataset = load_dataset(args.dataset)
model = models.get_model_by_name(args.model)(input_shape=dataset.shape[1:],
**vars(args))
if args.resume:
model.load_model(args.resume)
model.main_loop(dataset, epochs=args.epoch, batchsize=args.batchsize)
def main():
parser = argparse.ArgumentParser(
description='Train modified transformer with sketch data')
parser.add_argument("model_name",
default=None,
help="Model that we are going to train")
parser.add_argument("--id", default="0", help="experiment signature")
parser.add_argument("--dataset", default=None, help="Input data folder")
parser.add_argument("-o",
"--output-dir",
default="",
help="Output directory")
parser.add_argument("-p",
"--hparams",
default=None,
help="Parameters that are specific to one model. They "
"can regard hyperparameters such as number of layers"
"or specifics of training such as an optmiser choice")
parser.add_argument("--base-hparams",
default=None,
help="Model parameters that concern all models. "
"Those are related to logging, checkpointing, "
"notifications and loops")
parser.add_argument("--data-hparams",
default=None,
help="Dataset-related parameters. Regards data"
"formats and preprocessing parameters")
parser.add_argument(
"-g",
"--gpu",
default=0,
type=int,
nargs='+',
help="GPU ID to run on",
)
parser.add_argument("-r",
"--resume",
default=None,
help="One of 'latest' or a checkpoint name")
parser.add_argument("--data-loader",
default='stroke3-distributed',
help="Data loader that will provide data for model")
parser.add_argument("--help-hps",
action="store_true",
help="Prints out each hparams default values")
args = parser.parse_args()
# get our model and data loader classes
Model = models.get_model_by_name(args.model_name)
DataLoader = dataloaders.get_dataloader_by_name(args.data_loader)
# check for desperate help calls from the unending darkness
if args.help_hps:
base_help = pprint.pformat(Model.base_default_hparams().values())
specific_help = pprint.pformat(
Model.specific_default_hparams().values())
data_help = pprint.pformat(DataLoader.default_hparams().values())
print("\nBase model default parameters: \n{}\n\n"
"{} default parameters: \n{}\n\n"
"{} data loader default parameters: \n{}".format(
base_help, args.model_name, specific_help, args.data_loader,
data_help))
return
# parse the parameters, updating defaults
model_hps = Model.parse_hparams(base=args.base_hparams,
specific=args.hparams)
data_hps = DataLoader.parse_hparams(args.data_hparams)
# build model, load checkpoints
utils.gpu.setup_gpu(args.gpu)
dataset = DataLoader(data_hps, args.dataset)
model = Model(model_hps, dataset, args.output_dir, args.id)
if args.resume is not None:
model.restore_checkpoint_if_exists(args.resume)
# combine and save config file
combined_hps = utils.hparams.combine_hparams_into_one(model_hps, data_hps)
utils.hparams.save_config(model.config_filepath, combined_hps)
# train
model.train()
def main():
parser = argparse.ArgumentParser(
description='Train modified transformer with sketch data')
parser.add_argument("experiment_name", default=None,
help="Reference name of experiment that you want to run")
parser.add_argument("--id", default="0", help="Experiment signature")
parser.add_argument("-o", "--output-dir", default="", help="output directory")
parser.add_argument("--exp-hparams", default=None,
help="Parameters to override defaults for experiment")
parser.add_argument("--model-hparams", default=None,
help="Parameters to override defaults for model")
parser.add_argument("-g", "--gpu", default=0, type=int, nargs='+', help="GPU ID to run on", )
parser.add_argument("--model-name", default=None,
help="Model that ou want to experiment on")
parser.add_argument("--model-id", default=None,
help="Model that ou want to experiment on")
parser.add_argument("--data-loader", default='stroke3-distributed',
help="Data loader that will provide data for model, "
"if you want to load a model")
parser.add_argument("--dataset", default=None,
help="Input data folder if you want to load a model")
parser.add_argument("-r", "--resume", default='latest', help="One of 'latest' or a checkpoint name")
parser.add_argument("--help-hps", action="store_true",
help="Prints out the hparams default values")
args = parser.parse_args()
Experiment = experiments.get_experiment_by_name(args.experiment_name)
# check for lost users in the well of despair
if args.help_hps:
hps_description = pprint.pformat(Experiment.default_hparams().values())
print("\nDefault params for experiment {}: \n{}\n\n".format(
args.experiment_name, hps_description))
return
# load model if that is what the experiment requires
utils.gpu.setup_gpu(args.gpu)
if Experiment.requires_model:
Model = models.get_model_by_name(args.model_name)
DataLoader = dataloaders.get_dataloader_by_name(args.data_loader)
# load the modelconfig
model_hps = utils.hparams.combine_hparams_into_one(
Model.default_hparams(), DataLoader.default_hparams())
utils.hparams.load_config(
model_hps, Model.get_config_filepath(args.output_dir, args.model_id))
# optional override of parameters
if args.model_hparams:
model_hps.parse(args.model_hparams)
dataset = DataLoader(model_hps, args.dataset)
model = Model(model_hps, dataset, args.output_dir, args.model_id)
model.restore_checkpoint_if_exists(args.resume)
else:
dataset, model = None, None
experiment_hps = Experiment.parse_hparams(args.exp_hparams)
# finally, run the experiment
experiment = Experiment(experiment_hps, args.id, args.output_dir)
experiment.compute(model)
def compute_score(model_name, image):
"Accepts a model name and a PIL image and returns an integer from 0 to 4"
model_class = get_model_by_name(model_name)
model = model_class()
score = model.predict(image)
return score
def main():
# Parsing arguments
parser = argparse.ArgumentParser(description='Training GANs or VAEs')
parser.add_argument('--model', type=str, required=True)
parser.add_argument('--dataset', type=str, required=True)
parser.add_argument('--epoch', type=int, default=200)
parser.add_argument('--batchsize', type=int, default=50)
parser.add_argument('--output', default='output')
parser.add_argument('--z-dims', type=int, default=256)
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--resume', type=str, default=None)
parser.add_argument('--test-mode', action='store_true')
parser.add_argument('--is-conditional', action='store_true')
parser.add_argument('--aux-classifier', action='store_true')
parser.add_argument('--share-decoder-and-generator', action='store_true')
parser.add_argument('--label-smoothing', default=0.0, type=float)
parser.add_argument('--input-noise', default=0.0, type=float)
parser.add_argument('--run-id', '-r', required=True)
parser.add_argument('--checkpoint-every', default='1.', type=str)
parser.add_argument('--notify-every', default='1.', type=str)
parser.add_argument('--send-every', default=1, type=int)
parser.add_argument('--lr', default=1e-4, type=float)
parser.add_argument('--dis-loss-control', default=1., type=float)
parser.add_argument('--triplet-weight', default=1., type=float)
parser.add_argument('--embedding-dim', default=256, type=int)
parser.add_argument('--isolate-d-classifier', action='store_true')
parser.add_argument(
'--controlled-losses',
type=str,
nargs='+',
help="strings in format loss_name:weight:control_type:pivot_epoch")
parser.add_argument('--metrics',
type=str,
nargs='+',
help="selection of metrics you want to calculate")
parser.add_argument('--wgan-n-critic', default=5, type=int)
parser.add_argument('--began-gamma', default=0.5, type=float)
parser.add_argument('--triplet-margin', default=1., type=float)
parser.add_argument('--n-filters-factor', default=32, type=int)
parser.add_argument('--use-began-equilibrium', action='store_true')
parser.add_argument('--use-alignment-layer', action='store_true')
parser.add_argument('--use-simplified-triplet', action='store_true')
parser.add_argument('--data-folder', default='datasets/files')
parser.add_argument('--use-magan-equilibrium', action='store_true')
parser.add_argument('--topgan-enforce-std-dev', action='store_true')
parser.add_argument('--topgan-use-data-trilet-regularization',
action='store_true')
parser.add_argument('--use-began-loss', action='store_true')
parser.add_argument('--use-gradnorm', action='store_true')
parser.add_argument('--use-sigmoid-triplet', action='store_true')
parser.add_argument('--online-mining', default=None, type=str)
parser.add_argument('--online-mining-ratio', default=4, type=int)
parser.add_argument('--gradnorm-alpha', default=0.5, type=float)
parser.add_argument('--distance-metric', default='l2', type=str)
parser.add_argument('--slack-channel', type=str, default="random")
parser.add_argument('--use-quadruplet', action='store_true')
parser.add_argument('--generator-mining', action='store_true')
args = parser.parse_args()
# select gpu and limit resources if applicable
if 'tensorflow' == K.backend():
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.visible_device_list = str(args.gpu)
set_session(tf.Session(config=config))
# make output directory if not exists
if not os.path.isdir(args.output):
os.mkdir(args.output)
# load datasets
dataset = load_dataset(args.dataset)
model = models.get_model_by_name(args.model)(input_shape=dataset.shape[1:],
**vars(args))
if args.resume:
model.load_model(args.resume)
get_inference_via_optimization(model, dataset)
dataset_name = "mnist"
model_name = AVAILABLE_MODELS[args.model]
loss_fn = AVAILABLE_LOSS_FNS[args.loss]
batch_size = config["batch_size"][dataset_name.lower()]
train_subset_size = config["train_subset_size"][model_name.lower()]
epochs = args.epochs if args.epochs else config["epochs"]
previous_model = None
if args.prev:
if loss_fn not in args.prev:
raise RuntimeError("Use same loss function")
if not os.path.exists(args.prev):
raise FileNotFoundError("Couldn't find {}".format(args.prev))
previous_model = torch.load(args.prev)
num_params = config["num_params"][model_name]
num_models = None
if args.num:
num_models = args.num
model = get_model_by_name(model_name)
dataset = get_dataset(dataset_name, train_subset_size, batch_size)
training = Training(model,
dataset,
num_params,
epochs,
prev_model=previous_model,
num_models=num_models,
loss_fn=loss_fn)
training.start()
training.save()
ITERATIONS = 1
TEMPERATURE = 0
ENCODER_NAME = "oneplane"
AGENT_NAME = "policy_agent"
MODEL_NAME = "policy_gradient_model"
AGENT_FILE_NAME = "q_learning_less_games_1"
LOAD_AGENT = False
SAVE_AGENT = True
CUR_WINS = 0
encoder = encoders.get_encoder_by_name(ENCODER_NAME, ROWS, COLS, MINES)
model = models.get_model_by_name(MODEL_NAME, encoder)
if LOAD_AGENT:
with h5py.File(AGENT_FILE_NAME, 'r') as prev_agent:
agent = agents.load_agent_by_name(AGENT_NAME, prev_agent)
else:
agent = agents.get_agent_by_name(AGENT_NAME, model, encoder)
if TEMPERATURE:
agent.set_temperature(TEMPERATURE)
while True:
print("Creating 100 games with updated agent")
create_experience(agent, model, ROWS, COLS, MINES, 1, 100)
for file_name in os.listdir("experience"):