def __read_configuration(configuration_handler,
filename="tuf.interposition.json",
parent_repository_directory=None,
parent_ssl_certificates_directory=None):
"""
A generic function to read TUF interposition configurations off a file, and
then handle those configurations with a given function. configuration_handler
must be a function which accepts a tuf.interposition.Configuration
instance.
Returns the parsed configurations as a dictionary of configurations indexed
by hostnames."""
INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"
# Configurations indexed by hostnames.
parsed_configurations = {}
try:
with open(filename) as tuf_interposition_json:
tuf_interpositions = json.load(tuf_interposition_json)
configurations = tuf_interpositions.get("configurations", {})
if len(configurations) == 0:
raise InvalidConfiguration(
NO_CONFIGURATIONS.format(filename=filename))
else:
for network_location, configuration in configurations.iteritems(
):
try:
configuration_parser = ConfigurationParser(
network_location,
configuration,
parent_repository_directory=
parent_repository_directory,
parent_ssl_certificates_directory=
parent_ssl_certificates_directory)
configuration = configuration_parser.parse()
configuration_handler(configuration)
parsed_configurations[
configuration.hostname] = configuration
except:
Logger.exception(
INVALID_TUF_CONFIGURATION.format(
network_location=network_location))
raise
except:
Logger.exception(
INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
raise
else:
return parsed_configurations
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(
hyper_params={
"ku_num": 835,
"latent_dim": 600,
"hidden_num": 900,
},
nettype="EmbedDKT",
)
net = get_net(**cfg.hyper_params)
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/train.json",
"$data_dir/test.json",
dataset="assistment_2009_2010",
ctx=mx.gpu(2),
optimizer_params={"learning_rate": 0.001},
hyper_params={
"ku_num": 146,
"hidden_num": 200,
"dropout": 0.5
},
batch_size=16,
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(hyper_params={})
net = get_net(**cfg.hyper_params)
net.initialize()
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/train.csv",
"$data_dir/test.csv",
ctx=mx.cpu(),
optimizer_params={
"learning_rate": 0.01,
"clip_gradient": 1,
},
dataset="a0910",
# init_params_update={"initial_user_item": False},
# workspace="random",
workspace="init",
# params_save=True
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def sym_run(stage: (int, str) = "pesudo"): # pragma: no cover
if isinstance(stage, str):
stage = {
"pseudo": 0,
"real": 1,
"cli": 2,
}[stage]
if stage == 0:
# ############################## Pesudo Test #################################
cfg = Configuration(
hyper_params={
"ku_num": 835,
"hidden_num": 835,
},
ctx="cuda:0,1,2",
)
net = get_net(**cfg.hyper_params)
pseudo_numerical_check(net, cfg)
elif stage == 1:
# ################################# Simple Train ###############################
train(
"$data_dir/train",
"$data_dir/test",
dataset="junyi",
ctx="cuda:0",
optimizer_params={"lr": 0.001},
hyper_params={
"ku_num": 835,
"hidden_num": 835
},
batch_size=16,
)
elif stage == 2:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def __read_configuration(configuration_handler,
filename="tuf.interposition.json",
parent_repository_directory=None,
parent_ssl_certificates_directory=None):
"""
A generic function to read a TUF interposition configuration off the disk,
and handle it. configuration_handler must be a function which accepts a
tuf.interposition.Configuration instance."""
INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"
try:
with open(filename) as tuf_interposition_json:
tuf_interpositions = json.load(tuf_interposition_json)
configurations = tuf_interpositions.get("configurations", {})
if len(configurations) == 0:
raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename))
else:
for network_location, configuration in configurations.iteritems():
try:
configuration_parser = ConfigurationParser(network_location,
configuration, parent_repository_directory=parent_repository_directory,
parent_ssl_certificates_directory=parent_ssl_certificates_directory)
configuration = configuration_parser.parse()
configuration_handler(configuration)
except:
Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location))
raise
except:
Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
raise
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(
hyper_params=dict(
ku_num=835,
key_embedding_dim=50,
value_embedding_dim=200,
hidden_num=900,
key_memory_size=20,
value_memory_size=20,
key_memory_state_dim=50,
value_memory_state_dim=200,
)
)
net = get_net(**cfg.hyper_params)
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/train.json",
"$data_dir/test.json",
dataset="junyi",
ctx=mx.gpu(),
optimizer_params={
"learning_rate": 0.001
},
hyper_params=dict(
ku_num=835,
key_embedding_dim=200,
value_embedding_dim=200,
hidden_num=835,
key_memory_size=40,
dropout=0.5,
),
batch_size=16,
root="../../../",
data_dir="$root_data_dir",
end_epoch=10,
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(
hyper_params={
"model_type": "wclstm",
"class_num": 32,
"embedding_dim": 256,
}
)
embedding_size = {
"w": 100,
"c": 100,
}
net = get_net(embedding_size=embedding_size, **cfg.hyper_params)
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/train.json",
"$data_dir/test.json",
"w:$vec_dir/word.vec.dat,c:$vec_dir/char.vec.dat",
ctx=mx.gpu(),
hyper_params={
"model_type": "wclstm",
"class_num": 32,
"embedding_dim": 256,
},
root="../../../..",
dataset="ctc32",
data_dir="$root_data_dir",
end_epoch=1,
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(
hyper_params=dict(
ku_num=835,
)
)
net = get_net(**cfg.hyper_params)
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/train.json",
"$data_dir/test.json",
ctx=mx.gpu(),
optimizer_params={
"learning_rate": 0.001
},
dump=True,
data_dir="$root_data_dir",
# end_epoch=20,
# root="../../../",
# dataset="assistment_2009_2010",
# hyper_params=dict(
# ku_num=124,
# hidden_num=32,
# latent_dim=32,
# dropout=0.5,
# ),
end_epoch=10,
hyper_params=dict(
ku_num=835,
hidden_num=32,
latent_dim=32,
dropout=0.5,
),
dataset="junyi",
root="/data/tongshiwei/KT",
root_data_dir="$root/$dataset",
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(hyper_params={
"user_num": 1000,
"item_num": 100,
"vec_dim": 100,
"op": "mlp"
},
eval_params={
"unlabeled_value": 0,
"k": [1, 3],
"pointwise": True,
})
net = get_net(**cfg.hyper_params)
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/ml-1m/train.jsonl",
"$data_dir/ml-1m/test.jsonl",
hyper_params={
"num_a": 6040,
"num_b": 3900,
"vec_dim": 128,
"op": "linear"
},
root_data_dir="../../../../",
optimizer_params={"learning_rate": 0.001},
# ctx=[mx.gpu(3)],
ctx=[mx.gpu(5)],
batch_size=16,
# optimizer_params={
# "learning_rate": 0.01
# },
# ctx=[mx.gpu(5), mx.gpu(6), mx.gpu(7), mx.gpu(8)],
# batch_size=256,
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
