def b_remote_exists(remote_user_host, remote_dir, file_pattern): # check if file exist command_str = 'ssh %s "ls %s/%s"' % (remote_user_host, remote_dir, file_pattern) status, result = basic.getstatusoutput(command_str) if status != 0 and 'No such file or directory' in result: return False return True
def get_remote_folder(remote_dir, folder_pattern): command_str = 'ssh $tesia_host "ls -d %s/%s"' % (remote_dir, folder_pattern) status, result = basic.getstatusoutput(command_str) # r_scan =$(ssh hlc @ 10.0.0.203 ls -d / home / hlc / Data / rock / Synchrotron / * _0${scan}_ *) # print(status) # print(result) dir_list = result.split() # print(dir_list) return dir_list
def copy_file_folder_from_remote_machine(remote_user_host, remote_file_folder, local_path): basic.outputlogMessage('copy remote file or folder %s ' % remote_file_folder) command_str = 'scp -r %s:%s %s' % (remote_user_host, remote_file_folder, local_path) print(command_str) status, result = basic.getstatusoutput(command_str) if status != 0: print(result) if 'No such file or directory' in result: return False sys.exit(1) return True
def main(): basic.setlogfile('scp_log.txt') while True: # get remote dir basic.outputlogMessage('get remote folders') remote_folders = get_remote_folder(remote_dir, folder_pattern) basic.outputlogMessage("%d remote folders" % len(remote_folders)) folder_list = get_local_folder(local_dir, folder_pattern) basic.outputlogMessage("%d local folders" % len(folder_list)) folder_name_list = [os.path.basename(item) for item in folder_list] for idx, r_folders in enumerate(remote_folders): folder_name = os.path.basename(r_folders) if folder_name in folder_name_list: continue basic.outputlogMessage('copy trained folder in %s' % folder_name) command_str = 'scp -r ${tesia_host}:%s %s/%s' % ( r_folders, local_dir, folder_name) print(command_str) status, result = basic.getstatusoutput(command_str) if status != 0: sys.exit(1) folder_list = get_local_folder(local_dir, folder_pattern) # update local folder # reomve incomplete folders for folder in folder_list: res_json = os.path.join(folder, 'result.json') if os.path.isfile(res_json) and os.path.getsize(res_json) > 0: continue else: basic.outputlogMessage('remove incomplete folder: %s' % os.path.basename(folder)) io_function.delete_file_or_dir(folder) basic.outputlogMessage('wait five hours') time.sleep(3600 * 5) # wait five hours pass
def get_remote_folder_list(remote_user_host, remote_dir, folder_pattern): ''' get folder list in remote machine of specific directory Args: remote_host: user and hostname like: [email protected] or $tesia remote_dir: remote directory folder_pattern: pattern Returns: the folder list ''' command_str = 'ssh %s "ls -d %s/%s"' % (remote_user_host, remote_dir, folder_pattern) print(command_str) status, result = basic.getstatusoutput(command_str) if status != 0: print(result) if 'No such file or directory' in result: return [] sys.exit(1) dir_list = result.split() return dir_list