def submit_job(self, verbose=False): #store copy of calculation self.calcs.append(copy.deepcopy(self.calc)) #copy the input file to the equivalent location on the server self.send_to_home(self.calc.label + '.com') #run submission script and collect job info ssh = remote.connect_server(ssh=True) i, o, e = ssh.exec_command( 'submit_calc {fld} {inp} {p} {m} {t} {q} {v}'.format( host=config.get('gaussian', 'gauss_host'), fld=self.host_dir, inp=self.calc.label + '.com', p=self.calc.job_params['nodes'], m=self.calc.job_params['memory'], t=int(self.calc.job_params['time']), q=self.calc.job_params['queue'], v=self.calc.job_paras['version'])) qsub_output = o.readlines() + e.readlines() ssh.close() #sanity check if len(qsub_output) == 0: raise remote.PBSUtilQSubError( "Failed to launch Gaussian %s, qsub gave no stdoutput" % self.calc.label) if verbose: print '\n%s\n' % qsub_output #extract and return job_id pbs_id = remote.parse_qsub_output(qsub_output[0])[0] return pbs_id
def server_files_equal_v2(serv_files, local_files): """checks whether a list of files on the server are the same as a list of local by comparing line 133 and the last 10 lines of the files, returns a list of booleans indicating whether each file is equivalent""" if len(serv_files) != len(local_files): raise AttributeError('Cannot compare an unequal list of files') #{fl} enclosed in quotes to allow for directories with spaces in them serv_commands = [ "sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;" .format(fl=serv_file) for serv_file in serv_files ] serv_command = "".join(serv_commands) ssh = remote.connect_server(ssh=True) stdin, stdout, stderr = ssh.exec_command(serv_command) #last element is empty space because of the way split works serv_lines = stdout.read().split( 'server_files_equal_v2_chunk_done\n')[0:-1] ssh.close() local_commands = [ "sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;" .format(fl=os.path.realpath(local_file)) for local_file in local_files ] local_command = "".join(local_commands) p = Popen(local_command, stdout=PIPE, stderr=PIPE, shell=True) stdout, stderr = p.communicate() home_lines = stdout.split('server_files_equal_v2_chunk_done\n')[0:-1] return [home_lines[i] == serv_lines[i] for i in range(len(home_lines))]
def server_data_unequal(list_mols): from ase_extensions.ase_utils import get_active_dirs """checks whether calculation data from log files on the server is the same as local calculation data by comparing fingerprints of the files""" fingerprints = [mol.calc.fingerprint for mol in list_mols] home_dir, scratch_dir = get_active_dirs() home_files = [mol.calc.label + '.log' for mol in list_mols] serv_files = [scratch_dir + '/' + fn for fn in home_files] serv_commands = [ "sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;" .format(fl=serv_file) for serv_file in serv_files ] serv_command = "".join(serv_commands) ssh = remote.connect_server(ssh=True) stdin, stdout, stderr = ssh.exec_command(serv_command) #last element is empty space because of the way split works serv_fingerprints = stdout.read().split( 'server_files_equal_v2_chunk_done\n')[0:-1] ssh.close() return [ fingerprints[i] != serv_fingerprints[i] for i in range(len(fingerprints)) ]
def server_files_equal(serv_file, local_file): """checks whether a local file is the same as the file on a server by comparing last 10 lines of the file""" try: #{fl} enclosed in quotes to allow for directories with spaces in them serv_command = "head -n100 '{fl}'; tail -n10 '{fl}'".format( fl=serv_file) local_command = "head -n100 '{fl}'; tail -n10 '{fl}'".format( fl=os.path.realpath(local_file)) ssh = remote.connect_server(ssh=True) stdin, stdout, stderr = ssh.exec_command(serv_command) serv_last_lines = stdout.read() ssh.close() except OSError: return False try: p = Popen(local_command, stdout=PIPE, stderr=PIPE, shell=True) stdout, stderr = p.communicate() home_last_lines = stdout except OSError: return False if serv_last_lines == home_last_lines: return True else: return False
def server_file_exists(serv_file, sftp=None): """checks a given file exists on the server, if an sftp connection is provided that is used (and subsequently left open)""" sftp_gen = False if not sftp: sftp_gen = True ssh, sftp = remote.connect_server(ssh=True, sftp=True) try: sftp.stat(serv_file) return True except IOError: return False finally: if sftp_gen: ssh.close() sftp.close()
def latest_restarts(list_mols): home_dir, scratch_dir = get_active_dirs() home_files = [mol.calc.label + '.log' for mol in list_mols] serv_files = [scratch_dir + '/' + fn for fn in home_files] ssh = connect_server(ssh=True) serv_files = [get_latest_restart_name(file_n, ssh) for file_n in serv_files] home_files = [sfn.replace(scratch_dir + '/', '') for sfn in serv_files] ssh.close() new_mols = copy.deepcopy(list_mols) for i, mol in enumerate(new_mols): mol.calc.label = home_files[i].replace('.log','') return new_mols
def latest_restarts(list_mols): home_dir, scratch_dir = get_active_dirs() home_files = [mol.calc.label + '.log' for mol in list_mols] serv_files = [scratch_dir + '/' + fn for fn in home_files] ssh = connect_server(ssh=True) serv_files = [ get_latest_restart_name(file_n, ssh) for file_n in serv_files ] home_files = [sfn.replace(scratch_dir + '/', '') for sfn in serv_files] ssh.close() new_mols = copy.deepcopy(list_mols) for i, mol in enumerate(new_mols): mol.calc.label = home_files[i].replace('.log', '') return new_mols
def get_latest_restart_name(file_n, ssh=None): """get's the file name of the final restarted calculation from the server""" ssh_created = False if not ssh: ssh = connect_server(ssh=True) ssh_created = True i,o,e = ssh.exec_command('ls {fn}_restart_{{?,??}}.log'.format(fn=file_n.replace('.log', ''))) ls_output = o.readlines() clean_ls = [fn.strip() for fn in ls_output] clean_ls.sort(key=lambda fn: int(re.search('[0-9]+.log', fn).group().replace('.log' , ''))) if ssh_created: ssh.close() try: return clean_ls[-1] except IndexError: return file_n
def get_latest_restart_name(file_n, ssh=None): """get's the file name of the final restarted calculation from the server""" ssh_created = False if not ssh: ssh = connect_server(ssh=True) ssh_created = True i, o, e = ssh.exec_command( 'ls {fn}_restart_{{?,??}}.log'.format(fn=file_n.replace('.log', ''))) ls_output = o.readlines() clean_ls = [fn.strip() for fn in ls_output] clean_ls.sort(key=lambda fn: int( re.search('[0-9]+.log', fn).group().replace('.log', ''))) if ssh_created: ssh.close() try: return clean_ls[-1] except IndexError: return file_n
def server_data_unequal(list_mols): from ase_extensions.ase_utils import get_active_dirs """checks whether calculation data from log files on the server is the same as local calculation data by comparing fingerprints of the files""" fingerprints = [mol.calc.fingerprint for mol in list_mols] home_dir,scratch_dir = get_active_dirs() home_files = [mol.calc.label + '.log' for mol in list_mols] serv_files = [scratch_dir + '/' + fn for fn in home_files] serv_commands = ["sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;".format(fl=serv_file) for serv_file in serv_files] serv_command = "".join(serv_commands) ssh = remote.connect_server(ssh=True) stdin, stdout, stderr = ssh.exec_command(serv_command) #last element is empty space because of the way split works serv_fingerprints = stdout.read().split('server_files_equal_v2_chunk_done\n')[0:-1] ssh.close() return [fingerprints[i] != serv_fingerprints[i] for i in range(len(fingerprints))]
def submit_job(self,verbose=False): #store copy of calculation self.calcs.append(copy.deepcopy(self.calc)) #copy the input file to the equivalent location on the server self.send_to_home(self.calc.label + '.com') #run submission script and collect job info ssh = remote.connect_server(ssh=True) i,o,e = ssh.exec_command('submit_calc {fld} {inp} {p} {m} {t} {q} {v}'.format(host= config.get('gaussian', 'gauss_host'), fld=self.host_dir, inp=self.calc.label + '.com', p=self.calc.job_params['nodes'], m=self.calc.job_params['memory'], t=int(self.calc.job_params['time']), q=self.calc.job_params['queue'], v=self.calc.job_paras['version'])) qsub_output = o.readlines() + e.readlines() ssh.close() #sanity check if len(qsub_output) == 0: raise remote.PBSUtilQSubError("Failed to launch Gaussian %s, qsub gave no stdoutput" % self.calc.label) if verbose: print '\n%s\n' % qsub_output #extract and return job_id pbs_id = remote.parse_qsub_output(qsub_output[0])[0] return pbs_id
def server_files_equal_v2(serv_files, local_files): """checks whether a list of files on the server are the same as a list of local by comparing line 133 and the last 10 lines of the files, returns a list of booleans indicating whether each file is equivalent""" if len(serv_files) != len(local_files): raise AttributeError('Cannot compare an unequal list of files') #{fl} enclosed in quotes to allow for directories with spaces in them serv_commands = ["sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;".format(fl=serv_file) for serv_file in serv_files] serv_command = "".join(serv_commands) ssh = remote.connect_server(ssh=True) stdin, stdout, stderr = ssh.exec_command(serv_command) #last element is empty space because of the way split works serv_lines = stdout.read().split('server_files_equal_v2_chunk_done\n')[0:-1] ssh.close() local_commands = ["sed -n 133p '{fl}'; tail -n10 '{fl}'; echo server_files_equal_v2_chunk_done;".format(fl=os.path.realpath(local_file)) for local_file in local_files] local_command = "".join(local_commands) p = Popen(local_command, stdout=PIPE, stderr=PIPE, shell=True) stdout, stderr = p.communicate() home_lines = stdout.split('server_files_equal_v2_chunk_done\n')[0:-1] return [home_lines[i] == serv_lines[i] for i in range(len(home_lines))]
def server_files_equal(serv_file, local_file): """checks whether a local file is the same as the file on a server by comparing last 10 lines of the file""" try: #{fl} enclosed in quotes to allow for directories with spaces in them serv_command = "head -n100 '{fl}'; tail -n10 '{fl}'".format(fl=serv_file) local_command = "head -n100 '{fl}'; tail -n10 '{fl}'".format(fl=os.path.realpath(local_file)) ssh = remote.connect_server(ssh=True) stdin, stdout, stderr = ssh.exec_command(serv_command) serv_last_lines = stdout.read() ssh.close() except OSError: return False try: p = Popen(local_command, stdout=PIPE, stderr=PIPE, shell=True) stdout, stderr = p.communicate() home_last_lines = stdout except OSError: return False if serv_last_lines == home_last_lines: return True else: return False