def submit_job(self, verbose=False):
#store copy of calculation
self.calcs.append(copy.deepcopy(self.calc))
#copy the input file to the equivalent location on the server
self.send_to_home(self.calc.label + '.com')
#run submission script and collect job info
ssh = remote.connect_server(ssh=True)
i, o, e = ssh.exec_command(
'submit_calc {fld} {inp} {p} {m} {t} {q} {v}'.format(
host=config.get('gaussian', 'gauss_host'),
fld=self.host_dir,
inp=self.calc.label + '.com',
p=self.calc.job_params['nodes'],
m=self.calc.job_params['memory'],
t=int(self.calc.job_params['time']),
q=self.calc.job_params['queue'],
v=self.calc.job_paras['version']))
qsub_output = o.readlines() + e.readlines()
ssh.close()
#sanity check
if len(qsub_output) == 0:
raise remote.PBSUtilQSubError(
"Failed to launch Gaussian %s, qsub gave no stdoutput" %
self.calc.label)
if verbose:
print '\n%s\n' % qsub_output
#extract and return job_id
pbs_id = remote.parse_qsub_output(qsub_output[0])[0]
return pbs_id
def server_files_equal_v2(serv_files, local_files):
"""checks whether a list of files on the server are the same as a list of local by comparing line 133 and the last 10 lines of the files,
returns a list of booleans indicating whether each file is equivalent"""
if len(serv_files) != len(local_files):
raise AttributeError('Cannot compare an unequal list of files')
#{fl} enclosed in quotes to allow for directories with spaces in them
serv_commands = [
"sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;"
.format(fl=serv_file) for serv_file in serv_files
]
serv_command = "".join(serv_commands)
ssh = remote.connect_server(ssh=True)
stdin, stdout, stderr = ssh.exec_command(serv_command)
#last element is empty space because of the way split works
serv_lines = stdout.read().split(
'server_files_equal_v2_chunk_done\n')[0:-1]
ssh.close()
local_commands = [
"sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;"
.format(fl=os.path.realpath(local_file)) for local_file in local_files
]
local_command = "".join(local_commands)
p = Popen(local_command, stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = p.communicate()
home_lines = stdout.split('server_files_equal_v2_chunk_done\n')[0:-1]
return [home_lines[i] == serv_lines[i] for i in range(len(home_lines))]
def server_data_unequal(list_mols):
from ase_extensions.ase_utils import get_active_dirs
"""checks whether calculation data from log files on the server is the same as local calculation data by comparing fingerprints of the files"""
fingerprints = [mol.calc.fingerprint for mol in list_mols]
home_dir, scratch_dir = get_active_dirs()
home_files = [mol.calc.label + '.log' for mol in list_mols]
serv_files = [scratch_dir + '/' + fn for fn in home_files]
serv_commands = [
"sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;"
.format(fl=serv_file) for serv_file in serv_files
]
serv_command = "".join(serv_commands)
ssh = remote.connect_server(ssh=True)
stdin, stdout, stderr = ssh.exec_command(serv_command)
#last element is empty space because of the way split works
serv_fingerprints = stdout.read().split(
'server_files_equal_v2_chunk_done\n')[0:-1]
ssh.close()
return [
fingerprints[i] != serv_fingerprints[i]
for i in range(len(fingerprints))
]
def server_files_equal(serv_file, local_file):
"""checks whether a local file is the same as the file on a server by comparing last 10 lines of the file"""
try:
#{fl} enclosed in quotes to allow for directories with spaces in them
serv_command = "head -n100 '{fl}'; tail -n10 '{fl}'".format(
fl=serv_file)
local_command = "head -n100 '{fl}'; tail -n10 '{fl}'".format(
fl=os.path.realpath(local_file))
ssh = remote.connect_server(ssh=True)
stdin, stdout, stderr = ssh.exec_command(serv_command)
serv_last_lines = stdout.read()
ssh.close()
except OSError:
return False
try:
p = Popen(local_command, stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = p.communicate()
home_last_lines = stdout
except OSError:
return False
if serv_last_lines == home_last_lines:
return True
else:
return False
def server_file_exists(serv_file, sftp=None):
"""checks a given file exists on the server, if an sftp connection is provided that is used (and subsequently left open)"""
sftp_gen = False
if not sftp:
sftp_gen = True
ssh, sftp = remote.connect_server(ssh=True, sftp=True)
try:
sftp.stat(serv_file)
return True
except IOError:
return False
finally:
if sftp_gen:
ssh.close()
sftp.close()
def server_file_exists(serv_file, sftp=None):
"""checks a given file exists on the server, if an sftp connection is provided that is used (and subsequently left open)"""
sftp_gen = False
if not sftp:
sftp_gen = True
ssh, sftp = remote.connect_server(ssh=True, sftp=True)
try:
sftp.stat(serv_file)
return True
except IOError:
return False
finally:
if sftp_gen:
ssh.close()
sftp.close()
def latest_restarts(list_mols):
home_dir, scratch_dir = get_active_dirs()
home_files = [mol.calc.label + '.log' for mol in list_mols]
serv_files = [scratch_dir + '/' + fn for fn in home_files]
ssh = connect_server(ssh=True)
serv_files = [get_latest_restart_name(file_n, ssh) for file_n in serv_files]
home_files = [sfn.replace(scratch_dir + '/', '') for sfn in serv_files]
ssh.close()
new_mols = copy.deepcopy(list_mols)
for i, mol in enumerate(new_mols):
mol.calc.label = home_files[i].replace('.log','')
return new_mols
def latest_restarts(list_mols):
home_dir, scratch_dir = get_active_dirs()
home_files = [mol.calc.label + '.log' for mol in list_mols]
serv_files = [scratch_dir + '/' + fn for fn in home_files]
ssh = connect_server(ssh=True)
serv_files = [
get_latest_restart_name(file_n, ssh) for file_n in serv_files
]
home_files = [sfn.replace(scratch_dir + '/', '') for sfn in serv_files]
ssh.close()
new_mols = copy.deepcopy(list_mols)
for i, mol in enumerate(new_mols):
mol.calc.label = home_files[i].replace('.log', '')
return new_mols
def get_latest_restart_name(file_n, ssh=None):
"""get's the file name of the final restarted calculation from the server"""
ssh_created = False
if not ssh:
ssh = connect_server(ssh=True)
ssh_created = True
i,o,e = ssh.exec_command('ls {fn}_restart_{{?,??}}.log'.format(fn=file_n.replace('.log', '')))
ls_output = o.readlines()
clean_ls = [fn.strip() for fn in ls_output]
clean_ls.sort(key=lambda fn: int(re.search('[0-9]+.log', fn).group().replace('.log' , '')))
if ssh_created:
ssh.close()
try:
return clean_ls[-1]
except IndexError:
return file_n
def get_latest_restart_name(file_n, ssh=None):
"""get's the file name of the final restarted calculation from the server"""
ssh_created = False
if not ssh:
ssh = connect_server(ssh=True)
ssh_created = True
i, o, e = ssh.exec_command(
'ls {fn}_restart_{{?,??}}.log'.format(fn=file_n.replace('.log', '')))
ls_output = o.readlines()
clean_ls = [fn.strip() for fn in ls_output]
clean_ls.sort(key=lambda fn: int(
re.search('[0-9]+.log', fn).group().replace('.log', '')))
if ssh_created:
ssh.close()
try:
return clean_ls[-1]
except IndexError:
return file_n
def server_data_unequal(list_mols):
from ase_extensions.ase_utils import get_active_dirs
"""checks whether calculation data from log files on the server is the same as local calculation data by comparing fingerprints of the files"""
fingerprints = [mol.calc.fingerprint for mol in list_mols]
home_dir,scratch_dir = get_active_dirs()
home_files = [mol.calc.label + '.log' for mol in list_mols]
serv_files = [scratch_dir + '/' + fn for fn in home_files]
serv_commands = ["sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;".format(fl=serv_file) for serv_file in serv_files]
serv_command = "".join(serv_commands)
ssh = remote.connect_server(ssh=True)
stdin, stdout, stderr = ssh.exec_command(serv_command)
#last element is empty space because of the way split works
serv_fingerprints = stdout.read().split('server_files_equal_v2_chunk_done\n')[0:-1]
ssh.close()
return [fingerprints[i] != serv_fingerprints[i] for i in range(len(fingerprints))]
def submit_job(self,verbose=False):
#store copy of calculation
self.calcs.append(copy.deepcopy(self.calc))
#copy the input file to the equivalent location on the server
self.send_to_home(self.calc.label + '.com')
#run submission script and collect job info
ssh = remote.connect_server(ssh=True)
i,o,e = ssh.exec_command('submit_calc {fld} {inp} {p} {m} {t} {q} {v}'.format(host= config.get('gaussian', 'gauss_host'), fld=self.host_dir, inp=self.calc.label + '.com',
p=self.calc.job_params['nodes'], m=self.calc.job_params['memory'], t=int(self.calc.job_params['time']),
q=self.calc.job_params['queue'], v=self.calc.job_paras['version']))
qsub_output = o.readlines() + e.readlines()
ssh.close()
#sanity check
if len(qsub_output) == 0:
raise remote.PBSUtilQSubError("Failed to launch Gaussian %s, qsub gave no stdoutput" % self.calc.label)
if verbose:
print '\n%s\n' % qsub_output
#extract and return job_id
pbs_id = remote.parse_qsub_output(qsub_output[0])[0]
return pbs_id
def server_files_equal_v2(serv_files, local_files):
"""checks whether a list of files on the server are the same as a list of local by comparing line 133 and the last 10 lines of the files,
returns a list of booleans indicating whether each file is equivalent"""
if len(serv_files) != len(local_files):
raise AttributeError('Cannot compare an unequal list of files')
#{fl} enclosed in quotes to allow for directories with spaces in them
serv_commands = ["sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;".format(fl=serv_file) for serv_file in serv_files]
serv_command = "".join(serv_commands)
ssh = remote.connect_server(ssh=True)
stdin, stdout, stderr = ssh.exec_command(serv_command)
#last element is empty space because of the way split works
serv_lines = stdout.read().split('server_files_equal_v2_chunk_done\n')[0:-1]
ssh.close()
local_commands = ["sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;".format(fl=os.path.realpath(local_file)) for local_file in local_files]
local_command = "".join(local_commands)
p = Popen(local_command, stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = p.communicate()
home_lines = stdout.split('server_files_equal_v2_chunk_done\n')[0:-1]
return [home_lines[i] == serv_lines[i] for i in range(len(home_lines))]
def server_files_equal(serv_file, local_file):
"""checks whether a local file is the same as the file on a server by comparing last 10 lines of the file"""
try:
#{fl} enclosed in quotes to allow for directories with spaces in them
serv_command = "head -n100 '{fl}'; tail -n10 '{fl}'".format(fl=serv_file)
local_command = "head -n100 '{fl}'; tail -n10 '{fl}'".format(fl=os.path.realpath(local_file))
ssh = remote.connect_server(ssh=True)
stdin, stdout, stderr = ssh.exec_command(serv_command)
serv_last_lines = stdout.read()
ssh.close()
except OSError:
return False
try:
p = Popen(local_command, stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = p.communicate()
home_last_lines = stdout
except OSError:
return False
if serv_last_lines == home_last_lines:
return True
else:
return False