def run_local(self):
if self.meta_args["rerun"]:
self.update_analytics()
logging.info("Running %d tasks locally" % len(self.configurations))
for conf in self.configurations:
train = glob.glob(os.path.join(conf["fold_dir"], "*train.txt*"))[0]
test = glob.glob(os.path.join(conf["fold_dir"], "*test.txt*"))[0]
conf["test_queries"] = test
conf["training_queries"] = train
if not os.path.exists(conf["output_dir"]):
try:
os.makedirs(conf["output_dir"])
except:
pass
config_bk = os.path.join(conf["output_dir"], "config_bk.yml")
config_bk_file = open(config_bk, "w")
yaml.dump(conf, config_bk_file, default_flow_style=False, Dumper=Dumper)
config_bk_file.close()
e = GenericExperiment("-f " + config_bk)
r = e.run_experiment()
log_file = self.store(conf, r)
self.update_analytics_file(log_file)
logging.info("Done with %s, run %d" % (conf["output_dir"], conf["run_id"]))
logging.info("Done")
def run_local(self):
if self.meta_args["rerun"]:
self.update_analytics()
logging.info("Running %d tasks locally" % len(self.configurations))
for conf in self.configurations:
train = glob.glob(os.path.join(conf["fold_dir"], "*train.txt*"))[0]
test = glob.glob(os.path.join(conf["fold_dir"], "*test.txt*"))[0]
conf["test_queries"] = test
conf["training_queries"] = train
if not os.path.exists(conf["output_dir"]):
try:
os.makedirs(conf["output_dir"])
except:
pass
config_bk = os.path.join(conf["output_dir"], "config_bk.yml")
config_bk_file = open(config_bk, "w")
yaml.dump(conf,
config_bk_file,
default_flow_style=False,
Dumper=Dumper)
config_bk_file.close()
e = GenericExperiment("-f " + config_bk)
r = e.run_experiment()
log_file = self.store(conf, r)
self.update_analytics_file(log_file)
logging.info("Done with %s, run %d" %
(conf["output_dir"], conf["run_id"]))
logging.info("Done")
def run_task(conf):
if not os.path.exists(conf["output_dir"]):
try:
os.makedirs(conf["output_dir"])
except:
pass
logging.basicConfig(format='%(asctime)s %(module)s: %(message)s',
level=logging.INFO,
filename=os.path.join(conf["output_dir"],
"output.%d.log" %
conf["run_id"]))
logging.info("Starting %s, run %d" % (conf["output_dir"], conf["run_id"]))
if os.environ.get('TMPDIR'):
fold = "/".join(conf["fold_dir"].strip("/").split("/")[-2:])
train = glob.glob("/scratch/*/" + fold + "/*train.txt*")[0]
test = glob.glob("/scratch/*/" + fold + "/*test.txt*")[0]
else:
train = glob.glob(os.path.join(conf["fold_dir"], "*train.txt*"))[0]
test = glob.glob(os.path.join(conf["fold_dir"], "*test.txt*"))[0]
conf["test_queries"] = test
conf["training_queries"] = train
if not os.path.exists(conf["output_dir"]):
try:
os.makedirs(conf["output_dir"])
except:
pass
config_bk = os.path.join(conf["output_dir"], "config_bk.%d.yml" %
conf["run_id"])
config_bk_file = open(config_bk, "w")
yaml.dump(conf,
config_bk_file,
default_flow_style=False,
Dumper=Dumper)
config_bk_file.close()
e = GenericExperiment("-f " + config_bk)
return e.run_experiment()
def run_task(conf):
if not os.path.exists(conf["output_dir"]):
try:
os.makedirs(conf["output_dir"])
except:
pass
logging.basicConfig(format='%(asctime)s %(module)s: %(message)s',
level=logging.INFO,
filename=os.path.join(conf["output_dir"],
"output.%d.log" %
conf["run_id"]))
logging.info("Starting %s, run %d" % (conf["output_dir"], conf["run_id"]))
if os.environ.get('TMPDIR'):
fold = "/".join(conf["fold_dir"].strip("/").split("/")[-2:])
train = glob.glob("/scratch/*/" + fold + "/*train.txt*")[0]
test = glob.glob("/scratch/*/" + fold + "/*test.txt*")[0]
else:
train = glob.glob(os.path.join(conf["fold_dir"], "*train.txt*"))[0]
test = glob.glob(os.path.join(conf["fold_dir"], "*test.txt*"))[0]
conf["test_queries"] = test
conf["training_queries"] = train
if not os.path.exists(conf["output_dir"]):
try:
os.makedirs(conf["output_dir"])
except:
pass
config_bk = os.path.join(conf["output_dir"], "config_bk.%d.yml" %
conf["run_id"])
config_bk_file = open(config_bk, "w")
yaml.dump(conf,
config_bk_file,
default_flow_style=False,
Dumper=Dumper)
config_bk_file.close()
e = GenericExperiment("-f " + config_bk)
return e.run_experiment()
