def __init__(self, exp_script, max_time="2:00:00"):
self._session = SSHSession("avakas.mcia.univ-bordeaux.fr",
"thimunzer",
key_file=open(
"/home/tmunzer/.ssh/no_passphrase",
"r"))
self._session.command("module load torque")
self.exp_script = exp_script
self.advise_sleep = 60
if isinstance(max_time, str):
self.max_time = max_time
else:
assert isinstance(max_time, int)
self.max_time = str(max_time) + ":00:00"
def __init__(self, exp_script, root, max_time="2:00:00"):
with open(os.path.dirname(__file__) + "/config.json") as config_file:
config = json.load(config_file)
self._session = SSHSession("avakas.mcia.univ-bordeaux.fr",
config["user"],
key_file=open(config["key_path"], "r"))
self._session.command("module load torque")
self.exp_script = exp_script
self.root = root
self.advise_sleep = 60
if isinstance(max_time, str):
self.max_time = max_time
else:
assert isinstance(max_time, int)
self.max_time = str(max_time) + ":00:00"
def __init__(self, exp_script, max_time="2:00:00"):
self._session = SSHSession("avakas.mcia.univ-bordeaux.fr", "thimunzer", key_file=open("/home/tmunzer/.ssh/no_passphrase", "r"))
self._session.command("module load torque")
self.exp_script = exp_script
self.advise_sleep = 60
if isinstance(max_time, str):
self.max_time = max_time
else:
assert isinstance(max_time, int)
self.max_time = str(max_time) + ":00:00"
def __init__(self, exp_script, root, max_time="2:00:00"):
with open(os.path.dirname(__file__) + "/config.json") as config_file:
config = json.load(config_file)
self._session = SSHSession("avakas.mcia.univ-bordeaux.fr", config["user"],
key_file=open(config["key_path"], "r"))
self._session.command("module load torque")
self.exp_script = exp_script
self.root = root
self.advise_sleep = 60
if isinstance(max_time, str):
self.max_time = max_time
else:
assert isinstance(max_time, int)
self.max_time = str(max_time) + ":00:00"
class AvakasJobQueue(JobQueue):
def __init__(self, exp_script, root, max_time="2:00:00"):
with open(os.path.dirname(__file__) + "/config.json") as config_file:
config = json.load(config_file)
self._session = SSHSession("avakas.mcia.univ-bordeaux.fr",
config["user"],
key_file=open(config["key_path"], "r"))
self._session.command("module load torque")
self.exp_script = exp_script
self.root = root
self.advise_sleep = 60
if isinstance(max_time, str):
self.max_time = max_time
else:
assert isinstance(max_time, int)
self.max_time = str(max_time) + ":00:00"
def add_job(self, directory):
# print
# print "add_job : ", directory
with open(directory + "/job.pbs", "w") as job_file:
job_file.write("\
#PBS -o Repos/{4}/{0}/out\n\
#PBS -e Repos/{4}/{0}/err\n\
#PBS -l walltime={1}\n\
#PBS -l nodes=1:ppn=2\n\
#PBS -N {2}\n\
\n\
printf \"/tmp/$PBS_JOBID\" > Repos/Inria/thibaut/{0}/tmp_dir_name\n\
source virtualenvs/py2.7/bin/activate;\n\
cd Repos/{4}/;\n\
python {3} {0};\n\
sleep 1;\n\
touch {0}/finished\n".format(directory, self.max_time, directory[:-10],
self.exp_script, self.root))
self._session.create_path("{}/{}".format(self.root, directory))
self._session.put("{}/params.json".format(directory),
"{}/{}/params.json".format(self.root, directory))
self._session.put("{}/job.pbs".format(directory),
"{}/{}/job.pbs".format(self.root, directory))
self._session.command("qsub {}/{}/job.pbs".format(
self.root, directory))
def is_launched(self, directory):
return self._session.path_exists("{}/{}".format(self.root, directory))
def is_done(self, directory):
# print
# print "is_done : ", directory
if os.path.isfile(directory + "/results.json"):
return True
# elif not self._session.is_dir("Repos/Inria/thibaut/{}".format(directory)):
# return False
elif not self._session.path_exists("{}/{}/finished".format(
self.root, directory)):
# print "not done :", directory
# self._session.command("qsub Repos/Inria/thibaut/{}/job.pbs".format(directory))
return False
else:
if self._session.path_exists("{}/{}/results.json".format(
self.root, directory)):
self._session.get(
"{}/{}/results.json".format(self.root, directory),
"{}/results.json".format(directory))
return True
else:
print "relaunch : ", directory
# assert False
# self._session.command("rm Repos/Inria/thibaut/{}/finished".format(directory))
# self._session.command("qsub Repos/Inria/thibaut/{}/job.pbs".format(directory))
return False
def get_results(self, directory):
# print
# print "get_results : ", directory
with open(directory + "/results.json", "r") as results_file:
results = json.load(results_file)
return results
class AvakasJobQueue(JobQueue):
def __init__(self, exp_script, root, max_time="2:00:00"):
with open(os.path.dirname(__file__) + "/config.json") as config_file:
config = json.load(config_file)
self._session = SSHSession("avakas.mcia.univ-bordeaux.fr", config["user"],
key_file=open(config["key_path"], "r"))
self._session.command("module load torque")
self.exp_script = exp_script
self.root = root
self.advise_sleep = 60
if isinstance(max_time, str):
self.max_time = max_time
else:
assert isinstance(max_time, int)
self.max_time = str(max_time) + ":00:00"
def add_job(self, directory):
# print
# print "add_job : ", directory
with open(directory + "/job.pbs", "w") as job_file:
job_file.write("\
#PBS -o Repos/{4}/{0}/out\n\
#PBS -e Repos/{4}/{0}/err\n\
#PBS -l walltime={1}\n\
#PBS -l nodes=1:ppn=2\n\
#PBS -N {2}\n\
\n\
printf \"/tmp/$PBS_JOBID\" > Repos/Inria/thibaut/{0}/tmp_dir_name\n\
source virtualenvs/py2.7/bin/activate;\n\
cd Repos/{4}/;\n\
python {3} {0};\n\
sleep 1;\n\
touch {0}/finished\n".format(directory, self.max_time, directory[:-10], self.exp_script, self.root))
self._session.create_path("{}/{}".format(self.root, directory))
self._session.put("{}/params.json".format(directory),
"{}/{}/params.json".format(self.root, directory))
self._session.put("{}/job.pbs".format(directory),
"{}/{}/job.pbs".format(self.root, directory))
self._session.command("qsub {}/{}/job.pbs".format(self.root, directory))
def is_launched(self, directory):
return self._session.path_exists("{}/{}".format(self.root, directory))
def is_done(self, directory):
# print
# print "is_done : ", directory
if os.path.isfile(directory + "/results.json"):
return True
# elif not self._session.is_dir("Repos/Inria/thibaut/{}".format(directory)):
# return False
elif not self._session.path_exists("{}/{}/finished".format(self.root, directory)):
# print "not done :", directory
# self._session.command("qsub Repos/Inria/thibaut/{}/job.pbs".format(directory))
return False
else:
if self._session.path_exists("{}/{}/results.json".format(self.root, directory)):
self._session.get("{}/{}/results.json".format(self.root, directory),
"{}/results.json".format(directory))
return True
else:
print "relaunch : ", directory
# assert False
# self._session.command("rm Repos/Inria/thibaut/{}/finished".format(directory))
# self._session.command("qsub Repos/Inria/thibaut/{}/job.pbs".format(directory))
return False
def get_results(self, directory):
# print
# print "get_results : ", directory
with open(directory + "/results.json", "r") as results_file:
results = json.load(results_file)
return results