def main(cfg: DictConfig, train_id: str) -> None:
cwd = Path.cwd()
myutil.print_config(cfg)
# Setting seed
myutil.set_random_seed(0)
model_file_name = "{}_best.pth".format(cfg.model.name)
# Checking history directory
history_dir = cwd / 'history' / train_id
if (history_dir / model_file_name).exists():
pass
else:
return
# Setting result directory
# All outputs will be written into (p / 'result' / train_id).
if not (cwd / 'result').exists():
(cwd / 'result').mkdir(parents=True)
result_dir = cwd / 'result' / train_id
if result_dir.exists():
# removing files in result_dir?
pass
else:
result_dir.mkdir(parents=True, exist_ok=True)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Testing
testset = get_dataset(cfg)
net = myutil.get_model(cfg)
net.module.load_state_dict(torch.load(str(history_dir / model_file_name)))
predict(result_dir, testset, net, device)
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log('PHASE: build, repository work', code='31')
shutit.log(util.print_config(shutit.cfg))
if cfg['build']['interactive'] >= 3:
print ('\nNow building any modules that need building' +
util.colour('31', '\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
for module_id in module_ids(shutit):
module = shutit.shutit_map[module_id]
shutit.log('considering whether to build: ' + module.module_id,
code='31')
if cfg[module.module_id]['shutit.core.module.build']:
if is_installed(shutit,module):
cfg['build']['report'] = (cfg['build']['report'] +
'\nBuilt already: ' + module.module_id +
' with run order: ' + str(module.run_order))
else:
# We move to the module directory to perform the build, returning immediately afterwards.
revert_dir = os.getcwd()
os.chdir(os.path.dirname(module.__module_file))
shutit.login(prompt_prefix=module_id)
build_module(shutit, module)
shutit.logout()
os.chdir(revert_dir)
if is_to_be_built_or_is_installed(shutit, module):
shutit.log('Starting module')
if not module.start(shutit):
shutit.fail(module.module_id + ' failed on start',
child=shutit.pexpect_children['target'])
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log('PHASE: build, repository work', code='31')
shutit.log(util.print_config(shutit.cfg))
if cfg['build']['interactive'] >= 3:
print ('\nNow building any modules that need building' +
util.colour('31', '\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
for module_id in module_ids(shutit):
module = shutit.shutit_map[module_id]
shutit.log('considering whether to build: ' + module.module_id,
code='31')
if cfg[module.module_id]['shutit.core.module.build']:
if module.is_installed(shutit):
cfg['build']['report'] = (cfg['build']['report'] +
'\nBuilt already: ' + module.module_id +
' with run order: ' +
str(module.run_order))
else:
# We move to the module directory to perform the build, returning immediately afterwards.
revert_dir = os.getcwd()
os.chdir(os.path.dirname(module.__module_file))
shutit.login()
build_module(shutit, module)
shutit.logout()
os.chdir(revert_dir)
if is_built(shutit, module):
shutit.log('Starting module')
if not module.start(shutit):
shutit.fail(module.module_id + ' failed on start',
child=shutit.pexpect_children['container_child'])
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log('PHASE: build, repository work', code='31')
shutit.log(util.print_config(shutit.cfg))
if cfg['build']['interactive'] >= 3:
print ('\nNow building any modules that need building' +
util.colour('31', '\n[Hit return to continue]'))
raw_input('')
for module_id in module_ids(shutit):
module = shutit.shutit_map[module_id]
shutit.log('considering whether to build: ' + module.module_id,
code='31')
if cfg[module.module_id]['build']:
if module.is_installed(shutit):
cfg['build']['report'] = (cfg['build']['report'] +
'\nBuilt already: ' + module.module_id +
' with run order: ' +
str(module.run_order))
else:
revert_dir = os.getcwd()
os.chdir(os.path.dirname(module.__module_file))
build_module(shutit, module)
os.chdir(revert_dir)
if is_built(shutit, module):
shutit.log('Starting module')
if not module.start(shutit):
shutit.fail(module.module_id + ' failed on start',
child=shutit.pexpect_children['container_child'])
def _load_model(self,experiment):
util.set_gpus()
print "Running experiment: {}.".format(experiment)
config = util.get_config("experiments.conf")[experiment]
config["log_dir"] = util.mkdirs(os.path.join(config["log_root"], experiment))
util.print_config(config)
model = cm.CorefModel(config)
saver = tf.train.Saver()
log_dir = config["log_dir"]
with tf.Session() as session:
checkpoint_path = os.path.join(log_dir, "model.max.ckpt")
saver.restore(session, checkpoint_path)
self.model=model
self.session=session
def write_single(self):
if len(sys.argv) > 1:
name = sys.argv[1]
print(
"Running experiment: {} (from command-line argument).".format(
name))
else:
name = os.environ["EXP"]
print("Running experiment: {} (from environment variable).".format(
name))
config = util.get_config("experiments.conf")[name]
config["log_dir"] = util.mkdirs(os.path.join(config["log_root"], name))
config["batch_size"] = -1
config["max_tokens_per_batch"] = -1
config["lm_path"] = self.input_hdf5
config["eval_path"] = self.input_json
config["output_path"] = self.output_path
util.print_config(config)
data = LSGNData(config)
model = SRLModel(data, config)
evaluator = LSGNEvaluator(config)
variables_to_restore = []
for var in tf.global_variables():
print(var.name)
if "module/" not in var.name:
variables_to_restore.append(var)
saver = tf.train.Saver(variables_to_restore)
log_dir = config["log_dir"]
with tf.Session() as session:
checkpoint_path = os.path.join(log_dir, "model.max.ckpt")
print("Evaluating {}".format(checkpoint_path))
tf.global_variables_initializer().run()
saver.restore(session, checkpoint_path)
evaluator.evaluate(session, data, model.predictions, model.loss)
session.close()
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log('PHASE: build, repository work', code='31')
shutit.log(util.print_config(shutit.cfg))
if cfg['build']['interactive'] >= 3:
print ('\nNow building any modules that need building' +
util.colour('31', '\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
module_id_list = module_ids(shutit)
if cfg['build']['deps_only']:
module_id_list_build_only = filter(lambda x: cfg[x]['shutit.core.module.build'], module_id_list)
for module_id in module_id_list:
module = shutit.shutit_map[module_id]
shutit.log('considering whether to build: ' + module.module_id,
code='31')
if cfg[module.module_id]['shutit.core.module.build']:
if is_installed(shutit,module):
cfg['build']['report'] = (cfg['build']['report'] +
'\nBuilt already: ' + module.module_id +
' with run order: ' + str(module.run_order))
else:
# We move to the module directory to perform the build, returning immediately afterwards.
if cfg['build']['deps_only'] and module_id == module_id_list_build_only[-1]:
# If this is the last module, and we are only building deps, stop here.
cfg['build']['report'] = (cfg['build']['report'] + '\nSkipping: ' +
module.module_id + ' with run order: ' + str(module.run_order) +
'\n\tas this is the final module and we are building dependencies only')
else:
revert_dir = os.getcwd()
cfg['target']['module_root_dir'] = os.path.dirname(module.__module_file)
shutit.chdir(cfg['target']['module_root_dir'])
shutit.login(prompt_prefix=module_id)
build_module(shutit, module)
shutit.logout()
shutit.chdir(revert_dir)
if is_installed(shutit, module):
shutit.log('Starting module')
if not module.start(shutit):
shutit.fail(module.module_id + ' failed on start',
child=shutit.pexpect_children['target_child'])
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-o",
"--output_path",
type=str,
help="path of the output file",
default=None)
parser.add_argument("-e",
"--eps",
type=float,
help="size of the perturbation",
default=0)
parser.add_argument("-m",
"--method",
type=str,
choices=["fgsm", "rand"],
help="perturbation method",
default="fgsm")
parser.add_argument("name",
type=str,
help="name of the experiment",
default=None)
args = parser.parse_args()
if args.name is not None:
name = args.name
print(
"Running experiment: {} (from environment variable).".format(name))
else:
name = os.environ["EXP"]
print("Running experiment: {} (from command-line argument).".format(
name))
config = util.get_config("experiments.conf")[name]
config["log_dir"] = util.mkdirs(os.path.join(config["log_root"], name))
config["batch_size"] = -1
config["max_tokens_per_batch"] = -1
# Use dev lm, if provided.
if config["lm_path"] and "lm_path_dev" in config and config["lm_path_dev"]:
config["lm_path"] = config["lm_path_dev"]
config["adv"] = True
config["adv_eps"] = args.eps
config["adv_method"] = args.method # or "rand" for random perturbation
config["output_path"] = args.output_path
print("---" * 60)
util.print_config(config)
print("---" * 60)
# exit(0)
data = LSGNData(config)
model = SRLModel(data, config)
evaluator = LSGNEvaluator(config)
variables_to_restore = []
for var in tf.global_variables():
print(var.name)
if "module/" not in var.name:
variables_to_restore.append(var)
saver = tf.train.Saver(variables_to_restore)
log_dir = config["log_dir"]
with tf.Session() as session:
checkpoint_path = os.path.join(log_dir, "model.max.ckpt")
print("Evaluating {}".format(checkpoint_path))
tf.global_variables_initializer().run()
saver.restore(session, checkpoint_path)
evaluator.evaluate(session, data, model.predictions, model.loss)
def shutit_main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- sc - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
shutit = shutit_global.shutit
cfg = shutit.cfg
util.parse_args(cfg)
if cfg['action']['skeleton']:
util.create_skeleton(shutit)
return
if cfg['action']['serve']:
import shutit_srv
shutit_srv.start()
return
util.load_configs(shutit)
shutit_module_init(shutit)
conn_container(shutit)
errs = []
errs.extend(check_deps(shutit))
# Show dependency graph
if cfg['action']['show_depgraph']:
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
if module_id in shutit.cfg and shutit.cfg[module_id]['build']
])
digraph = digraph + '\n}'
shutit.log(digraph, force_stdout=True)
return
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['show_config']:
shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']),
force_stdout=True)
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
errs.extend(check_ready(shutit))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_container(shutit)
shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False,
force_stdout=True, code='31')
if shutit.cfg['build']['interactive'] >= 3:
shutit.log('\n' +
'The build is complete. You should now have a container ' +
'called ' + shutit.cfg['container']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/default.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' +
'default test runner in test.sh\n\n' +
'You can inspect the details of the build in the container\'s ' +
'/root/shutit_build directory.', force_stdout=True, code='31')
def shutit_main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- sc - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if sys.stdout.isatty():
if spawn.find_executable('shutit') is None:
# try the current directory, the .. directory, or the ../shutit directory, the ~/shutit
pwd = os.getcwd()
path_to_shutit = ''
for d in ('.','..','~','~/shutit'):
if os.path.isfile(os.path.expanduser(d) + '/shutit'):
path_to_shutit = d + '/shutit'
res = util.util_raw_input(prompt='shutit appears not to be on your path - would you like me to add it to your ~/.bashrc (Y/n)? ')
if res not in ('n','N'):
bashrc = os.path.expanduser('~/') + '.bashrc'
if os.path.isfile(bashrc):
with open(bashrc, "a") as myfile:
#http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch
myfile.write('export PATH="$PATH:' + path_to_shutit + '"\n')
break
if path_to_shutit == '':
while True:
res = util.util_raw_input(prompt='shutit appears not to be on your path - please input the path to your shutit dir\n')
if os.path.isfile(os.path.expanduser(res) + '/shutit'):
path_to_shutit = res + '/shutit'
bashrc = os.path.expanduser('~/') + '.bashrc'
if os.path.isfile(bashrc):
with open(bashrc, "a") as myfile:
myfile.write('\nexport PATH="$PATH:' + path_to_shutit + '"\n')
break
if path_to_shutit != '':
util.util_raw_input(prompt='\nPath set up - please open new terminal and re-run command\n')
sys.exit()
shutit = shutit_global.shutit
cfg = shutit.cfg
util.parse_args(cfg)
if cfg['action']['skeleton']:
util.create_skeleton(shutit)
return
if cfg['action']['serve']:
import shutit_srv
shutit_srv.start()
return
util.load_configs(shutit)
shutit_module_init(shutit)
conn_container(shutit)
errs = []
errs.extend(check_deps(shutit))
# Show dependency graph
if cfg['action']['show_depgraph']:
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build']
])
digraph = digraph + '\n}'
shutit.log(digraph, force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
return
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['show_config']:
shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']),
force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
errs.extend(check_ready(shutit))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_container(shutit)
shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False,
force_stdout=True, code='31')
if shutit.cfg['build']['interactive'] >= 3:
shutit.log('\n' +
'The build is complete. You should now have a container ' +
'called ' + shutit.cfg['container']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/default.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' +
'default test runner in test.sh\n\n' +
'You can inspect the details of the build in the container\'s ' +
'/root/shutit_build directory.', force_stdout=True, code='31')
# Mark the build as completed
shutit.cfg['build']['completed'] = True
def predict():
# util.set_gpus()
path = "./tempDir/"
path2 = "./tempDir2/"
os.mkdir("./tempDir2/")
data_list = os.listdir(path)
for data_file in data_list[1:]:
name = "final"
input_filename = path + data_file
output_filename = path2 + data_file
print("Running experiment: {}.".format(name))
config = util.get_config("experiments.conf")[name]
config["log_dir"] = util.mkdirs(os.path.join(config["log_root"], name))
print("Loading data from: {}.".format(input_filename))
config["eval_path"] = input_filename
config["batch_size"] = -1
config["max_tokens_per_batch"] = -1
# Use dev lm, if provided.
if config["lm_path"] and "lm_path_dev" in config and config[
"lm_path_dev"]:
config["lm_path"] = config["lm_path_dev"]
util.print_config(config)
data = LSGNData(config)
model = SRLModel(data, config)
evaluator = LSGNEvaluator(config)
# Load data and model.
eval_data, eval_tensors, doc_level_eval_data = data.load_eval_data()
variables_to_restore = []
for var in tf.global_variables():
#print var.name
if "module/" not in var.name:
variables_to_restore.append(var)
saver = tf.train.Saver(variables_to_restore)
log_dir = config["log_dir"]
with tf.Session() as session:
checkpoint_path = os.path.join(log_dir, "model.max.ckpt")
tf.global_variables_initializer().run()
saver.restore(session, checkpoint_path)
with open(output_filename, "w") as f:
#for example_num, (tensorized_example, example) in enumerate(model.eval_data):
for i, doc_tensors in enumerate(eval_tensors):
feed_dict = dict(
zip(data.input_tensors, [
input_utils.pad_batch_tensors(doc_tensors, tn)
for tn in data.input_names + data.label_names
]))
predict_names = []
for tn in data.predict_names:
if tn in model.predictions:
predict_names.append(tn)
predict_tensors = [
model.predictions[tn] for tn in predict_names
] + [model.loss]
predict_tensors = session.run(predict_tensors,
feed_dict=feed_dict)
predict_dict = dict(
zip(predict_names + ["loss"], predict_tensors))
doc_example = doc_level_eval_data[i]
sentences = doc_example["sentences"]
predictions = inference_utils.srl_decode(
sentences, predict_dict, data.srl_labels_inv, config)
doc_example["predicted_srl"] = []
word_offset = 0
for j, sentence in enumerate(sentences):
for pred, args in predictions["srl"][j].items():
doc_example["predicted_srl"].extend([[
int(pred + word_offset),
int(a[0] + word_offset),
int(a[1] + word_offset), a[2]
] for a in args])
word_offset += len(sentence)
f.write(json.dumps(doc_example))
f.write("\n")
if (i + 1) % 10 == 0:
print("Decoded {} documents.".format(i + 1))
def main(cfg: DictConfig) -> None:
cwd = Path(hydra.utils.get_original_cwd())
myutil.print_config(cfg)
# Setting history directory
# All outputs will be written into (p / "history" / train_id).
train_id = tid.generate_train_id(cfg)
history_dir = cwd / "history" / train_id
if not history_dir.exists():
history_dir.mkdir(parents=True, exist_ok=True)
cfg_path = history_dir / "config.yaml"
if cfg_path.exists():
existing_cfg = OmegaConf.load(str(history_dir / "config.yaml"))
if not myutil.is_same_config(cfg, existing_cfg):
raise ValueError("Train ID {} already exists, but config is different".format(train_id))
# Saving cfg
OmegaConf.save(cfg, str(history_dir / "config.yaml"))
# Setting seed
if cfg.seed is not None:
myutil.set_random_seed(cfg.seed)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Training
trainloaders, valloaders = get_data_loaders(cfg)
# Confirming dataset
dataiter = iter(trainloaders)
inputs, labels = next(dataiter)
hdrpy.io.write(
history_dir / 'input_sample.jpg',
inputs[0].clone().detach().numpy().transpose((1, 2, 0)))
hdrpy.io.write(
history_dir / 'label_sample.jpg',
labels[0].clone().detach().numpy().transpose((1, 2, 0)))
net = myutil.get_model(cfg)
# Checking initial DNN
# outputs = net(inputs.to(device)).to('cpu').clone().detach()
# hdrpy.io.write(
# history_dir / 'initial_output_sample.jpg',
# outputs[0].clone().detach().numpy().transpose((1, 2, 0)))
criterion = nn.L1Loss()
optimizer = get_optimizer(net.parameters(), cfg)
scheduler = get_scheduler(optimizer, cfg)
extensions = [ModelSaver(directory=history_dir,
name=lambda x: cfg.model.name+"_best.pth",
trigger=MinValueTrigger(mode="validation", key="loss")),
HistorySaver(directory=history_dir,
name=lambda x: cfg.model.name+"_history.pth",
trigger=IntervalTrigger(period=1))]
trainer = RegressorTrainer(net, optimizer, criterion, trainloaders,
scheduler=scheduler, extensions=extensions,
init_epoch=0,
device=device)
trainer.train(cfg.epoch, valloaders)
# Checking trained DNN
# outputs = net(inputs.to(device)).to('cpu').clone().detach()
# hdrpy.io.write(
# history_dir / 'output_sample.jpg',
# outputs[0].clone().detach().numpy().transpose((1, 2, 0)))
# print(outputs[0])
save_model(net, str(history_dir / "{}.pth".format(cfg.model.name)))
def record_config(self): """ Put the config in a file in the container. """ self.send_file(self.cfg['build']['build_db_dir'] + '/' + self.cfg['build']['build_id'] + '/' + self.cfg['build']['build_id'] + '.cfg',util.print_config(self.cfg))
def shutit_main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- sc - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if sys.stdout.isatty():
if spawn.find_executable('shutit') is None:
# try the current directory, the .. directory, or the ../shutit directory, the ~/shutit
pwd = os.getcwd()
path_to_shutit = ''
for d in ('.','..','~','~/shutit'):
if os.path.isfile(os.path.expanduser(d) + '/shutit'):
path_to_shutit = d + '/shutit'
res = util.util_raw_input(prompt='shutit appears not to be on your path - would you like me to add it to your ~/.bashrc (Y/n)? ')
if res not in ('n','N'):
bashrc = os.path.expanduser('~/') + '.bashrc'
if os.path.isfile(bashrc):
with open(bashrc, "a") as myfile:
#http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch
myfile.write('export PATH="$PATH:' + path_to_shutit + '"\n')
break
if path_to_shutit == '':
while True:
res = util.util_raw_input(prompt='shutit appears not to be on your path - please input the path to your shutit dir\n')
if os.path.isfile(os.path.expanduser(res) + '/shutit'):
path_to_shutit = res + '/shutit'
bashrc = os.path.expanduser('~/') + '.bashrc'
if os.path.isfile(bashrc):
with open(bashrc, "a") as myfile:
myfile.write('\nexport PATH="$PATH:' + path_to_shutit + '"\n')
break
if path_to_shutit != '':
util.util_raw_input(prompt='\nPath set up - please open new terminal and re-run command\n')
sys.exit()
shutit = shutit_global.shutit
cfg = shutit.cfg
util.parse_args(cfg)
if cfg['action']['skeleton']:
util.create_skeleton(shutit)
return
if cfg['action']['serve']:
import shutit_srv
shutit_srv.start()
return
util.load_configs(shutit)
shutit_module_init(shutit)
conn_container(shutit)
errs = []
errs.extend(check_deps(shutit))
# Show dependency graph
if cfg['action']['show_depgraph']:
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build']
])
digraph = digraph + '\n}'
shutit.log(digraph, force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
return
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['show_config']:
shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']),
force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
errs.extend(check_ready(shutit))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_container(shutit)
shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False,
force_stdout=True, code='31')
if shutit.cfg['build']['interactive'] >= 3:
shutit.log('\n' +
'The build is complete. You should now have a container ' +
'called ' + shutit.cfg['container']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/default.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' +
'default test runner in test.sh\n\n' +
'You can inspect the details of the build in the container\'s ' +
'/root/shutit_build directory.', force_stdout=True, code='31')
# Mark the build as completed
shutit.cfg['build']['completed'] = True
def shutit_main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- list_configs - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
shutit = shutit_global.shutit
cfg = shutit.cfg
util.parse_args(shutit)
if cfg['action']['skeleton']:
util.create_skeleton(shutit)
cfg['build']['completed'] = True
return
if cfg['action']['serve']:
import shutit_srv
cfg['build']['interactive'] = 0
revert_dir = os.getcwd()
os.chdir(sys.path[0])
shutit_srv.start()
os.chdir(revert_dir)
return
util.load_configs(shutit)
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if util.determine_interactive() and spawn.find_executable('shutit') is None:
setup_shutit_path(cfg)
util.load_mod_from_file(shutit, os.path.join(shutit.shutit_main_dir, 'setup.py'))
util.load_shutit_modules(shutit)
if cfg['action']['list_modules']:
util.list_modules(shutit)
sys.exit(0)
init_shutit_map(shutit)
config_collection(shutit)
conn_target(shutit)
errs = []
errs.extend(check_deps(shutit))
if cfg['action']['list_deps']:
# Show dependency graph
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build']
])
digraph = digraph + '\n}'
f = file(cfg['build']['log_config_path'] + '/digraph.txt','w')
f.write(digraph)
f.close()
digraph_all = 'digraph depgraph {\n'
digraph_all = digraph_all + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
])
digraph_all = digraph_all + '\n}'
f = file(cfg['build']['log_config_path'] + '/digraph_all.txt','w')
f.write(digraph_all)
f.close()
shutit.log('\n================================================================================\n' + digraph_all, force_stdout=True)
shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png\n', force_stdout=True)
shutit.log('\n================================================================================\n', force_stdout=True)
shutit.log('\n\n' + digraph, force_stdout=True)
shutit.log('\n================================================================================\n' + digraph, force_stdout=True)
shutit.log('\nAbove is the digraph for all modules configured to be built in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png\n', force_stdout=True)
shutit.log('\n================================================================================\n', force_stdout=True)
# Exit now
sys.exit(0)
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['list_configs'] or cfg['build']['debug']:
shutit.log(util.print_config(cfg, history=cfg['list_configs']['cfghistory']),
force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
f = file(cfg['build']['log_config_path'] + '/cfg.txt','w')
f.write(util.print_config(cfg, history=cfg['list_configs']['cfghistory']))
f.close()
shutit.log('================================================================================', force_stdout=True)
shutit.log('Config details placed in: ' + cfg['build']['log_config_path'], force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
shutit.log('To render the digraph of this build into an image run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph.gv ' + cfg['build']['log_config_path'] + '/digraph.txt && dot -Tpdf -o digraph.pdf ' + cfg['build']['log_config_path'] + '/digraph.gv\n\n', force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
shutit.log('To render the digraph of all visible modules into an image, run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph_all.gv ' + cfg['build']['log_config_path'] + '/digraph_all.txt && dot -Tpdf -o digraph_all.pdf ' + cfg['build']['log_config_path'] + '/digraph_all.gv\n\n', force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
shutit.log('\nConfiguration details have been written to the folder: ' + cfg['build']['log_config_path'] + '\n', force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
if cfg['action']['list_configs']:
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
# Cache the results of check_ready at the start.
errs.extend(check_ready(shutit, throw_error=False))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_target(shutit)
shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False,
force_stdout=True, code='31')
if cfg['build']['build_log']:
shutit.cfg['build']['report_final_messages'] += "Build log file: " + cfg['host']['logfile']
# Show final report messages (ie messages to show after standard report).
if shutit.cfg['build']['report_final_messages'] != '':
shutit.log(shutit.cfg['build']['report_final_messages'], prefix=False,
force_stdout=True, code='31')
if shutit.cfg['build']['interactive'] >= 3:
shutit.log('\n' +
'The build is complete. You should now have a target ' +
'called ' + shutit.cfg['target']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/build.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh. These may need ' +
'tweaking for your particular environment, eg sudo\n\n' +
'You can inspect the details of the build in the target image\'s ' +
'/root/shutit_build directory.', force_stdout=True, code='31')
# Mark the build as completed
shutit.cfg['build']['completed'] = True
def main(cfg: DictConfig) -> None:
cwd = Path(hydra.utils.get_original_cwd())
myutil.print_config(cfg)
# Setting history directory
# All outputs will be written into (p / "history" / train_id).
train_id = tid.generate_train_id(cfg)
history_dir = cwd / "history" / train_id
if not history_dir.exists():
history_dir.mkdir(parents=True, exist_ok=True)
cfg_path = history_dir / "config.yaml"
if cfg_path.exists():
existing_cfg = OmegaConf.load(str(history_dir / "config.yaml"))
if not myutil.is_same_config(cfg, existing_cfg):
raise ValueError(
"Train ID {} already exists, but config is different".format(
train_id))
# Saving cfg
OmegaConf.save(cfg, str(history_dir / "config.yaml"))
# Setting seed
if cfg.seed is not None:
myutil.set_random_seed(cfg.seed)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Training
trainloaders, valloaders, classes = get_data_loaders(cfg)
net = myutil.get_model(classes, cfg)
if device.type == "cuda":
net = torch.nn.DataParallel(net)
net = net.to(device)
summary(net, input_size=(1, 32, 32))
criterion = nn.CrossEntropyLoss()
optimizer = get_optimizer(net.parameters(), cfg)
scheduler = get_scheduler(optimizer, cfg)
extensions = [
ModelSaver(directory=history_dir,
name=lambda x: cfg.model.name + "_best.pth",
trigger=MaxValueTrigger(mode="validation",
key="total acc")),
HistorySaver(directory=history_dir,
name=lambda x: cfg.model.name + "_history.pth",
trigger=IntervalTrigger(period=1))
]
trainer = ClassifierTrainer(net,
optimizer,
criterion,
trainloaders,
scheduler=scheduler,
extensions=extensions,
init_epoch=0,
device=device)
trainer.train(cfg.epoch, valloaders, classes)
save_model(net, str(history_dir / "{}.pth".format(cfg.model.name)))
import coref_model as cm
import util
if __name__ == "__main__":
if "GPU" in os.environ:
util.set_gpus(int(os.environ["GPU"]))
else:
util.set_gpus()
if len(sys.argv) > 1:
name = sys.argv[1]
print("Running experiment: {} (from command-line argument).".format(name))
else:
name = os.environ["EXP"]
print("Running experiment: {} (from environment variable).".format(name))
config = util.get_config("experiments.conf")[name]
config["log_dir"] = util.mkdirs(os.path.join(config["log_root"], name))
util.print_config(config)
model = cm.CorefModel(config)
saver = tf.train.Saver()
log_dir = config["log_dir"]
with tf.Session() as session:
checkpoint_path = os.path.join(log_dir, "model.max.ckpt")
print("Evaluating {}".format(checkpoint_path))
saver.restore(session, checkpoint_path)
model.evaluate(session, official_stdout=True)
else:
name = os.environ["EXP"]
print("Running experiment: {} (from environment variable).".format(name))
config = util.get_config("experiments.conf")[name]
config["log_dir"] = util.mkdirs(os.path.join(config["log_root"], name))
# Dynamic batch size.
config["batch_size"] = -1
config["max_tokens_per_batch"] = -1
# Use dev lm, if provided.
if config["lm_path"] and "lm_path_dev" in config and config["lm_path_dev"]:
config["lm_path"] = config["lm_path_dev"]
util.print_config(config)
data = LSGNData(config)
model = SRLModel(data, config)
evaluator = LSGNEvaluator(config)
variables_to_restore = []
for var in tf.global_variables():
if "module/" not in var.name:
variables_to_restore.append(var)
else:
print("Not restoring from checkpoint:", var.name)
saver = tf.train.Saver(variables_to_restore)
log_dir = config["log_dir"]
assert not ("final" in name) # Make sure we don't override a finalized checkpoint.
def shutit_main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- sc - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if sys.stdout.isatty() and spawn.find_executable('shutit') is None:
setup_shutit_path()
shutit = shutit_global.shutit
cfg = shutit.cfg
util.parse_args(shutit)
if cfg['action']['skeleton']:
util.create_skeleton(shutit)
cfg['build']['completed'] = True
return
if cfg['action']['serve']:
import shutit_srv
cfg['build']['interactive'] = 0
revert_dir = os.getcwd()
os.chdir(sys.path[0])
shutit_srv.start()
os.chdir(revert_dir)
return
util.load_configs(shutit)
shutit_module_init(shutit)
conn_target(shutit)
errs = []
errs.extend(check_deps(shutit))
if cfg['action']['show_config']:
# Show dependency graph
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build']
])
digraph = digraph + '\n}'
shutit.cfg['build']['depgraph'] = digraph
digraph_all = 'digraph depgraph {\n'
digraph_all = digraph_all + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
])
digraph_all = digraph_all + '\n}'
shutit.cfg['build']['depgraph_all'] = digraph_all
shutit.log('\n================================================================================\n' + digraph_all, force_stdout=True)
shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png', force_stdout=True)
shutit.log('\n================================================================================\n', force_stdout=True)
shutit.log('\n\n' + digraph, force_stdout=True)
shutit.log('\n================================================================================\n' + digraph, force_stdout=True)
shutit.log('\nAbove is the digraph for this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png', force_stdout=True)
shutit.log('\n================================================================================\n', force_stdout=True)
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['show_config']:
shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']),
force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
errs.extend(check_ready(shutit))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_target(shutit)
shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False,
force_stdout=True, code='31')
if shutit.cfg['build']['interactive'] >= 3:
shutit.log('\n' +
'The build is complete. You should now have a target ' +
'called ' + shutit.cfg['target']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/default.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' +
'default test runner in test.sh\n\n' +
'You can inspect the details of the build in the target\'s ' +
'/root/shutit_build directory.', force_stdout=True, code='31')
# Mark the build as completed
shutit.cfg['build']['completed'] = True
