def rsync_command(files, read=True, path=None, delete=False, precheck=0): """ run rsync, syncing the files (or folders) listed in files, assumed to be paths or partial paths from the RSYNC_LOCAL_ROOT """ if cf.PIPELINE_LOCAL: return network.ensure_ssh_connection(network.RSYNC_ADDRESS) if path: full_path = RSYNC_PATH + path + "/./" else: full_path = RSYNC_PATH with tempfile.NamedTemporaryFile(delete=True) as tmp: tmp.file.write("\n".join(files)) tmp.file.close() flags = RSYNC_FLAGS if delete: flags = "--delete "+flags try: logger.info("running rsync for files: %r",files) if read: if precheck == 0: # Reading from remote repository into local repository cmd = " ".join(["rsync", flags, full_path, RSYNC_LOG_FILE_FLAG, "--files-from={}".format(tmp.name), cf.RSYNC_LOCAL_ROOT, RSYNC_LOG_PIPE_FLAG]) elif precheck == 1: # Reading from remote repository into local 'precheck' repository cmd = " ".join(["rsync", flags, full_path, RSYNC_LOG_FILE_FLAG, "--files-from={}".format(tmp.name), os.path.join(cf.RSYNC_LOCAL_ROOT, 'precheck'), RSYNC_LOG_PIPE_FLAG]) elif precheck == 2: # Reading from remote 'precheck' repository into local 'precheck' repository cmd = " ".join(["rsync", flags, os.path.join(full_path, 'precheck'), RSYNC_LOG_FILE_FLAG, "--files-from={}".format(tmp.name), os.path.join(cf.RSYNC_LOCAL_ROOT, 'precheck'), RSYNC_LOG_PIPE_FLAG]) else: logger.exception("rsync_command - received invalid precheck option %r", precheck) raise cf.RsyncRuntimeError("rsync_command - failure for command: `%s`" % cmd) else: cmd = " ".join(["rsync", flags, RSYNC_LOG_FILE_FLAG, "--files-from={}".format(tmp.name), cf.RSYNC_LOCAL_ROOT, full_path, RSYNC_LOG_PIPE_FLAG]) logger.debug("rsync command = %r",cmd) out = subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError as e: logger.exception("rsync_command - failure for command: %s" % cmd) raise cf.RsyncRuntimeError("rsync_command - failure for command: `%s`" % cmd)
def rsync_command_read_wildcard(files, path=None, precheck=0): """ run rsync, syncing the files (or folders) listed in files, assumed to be paths or partial paths from the RSYNC_LOCAL_ROOT """ if cf.PIPELINE_LOCAL: return network.ensure_ssh_connection(network.RSYNC_ADDRESS) for filename in files: if path: full_path = RSYNC_PATH + path + "/./" else: full_path = RSYNC_PATH full_path += filename flags = RSYNC_FLAGS try: logger.info("running rsync for files: %r",filename) if precheck == 0: # Reading from remote repository into local repository cmd = " ".join(["rsync", flags, full_path, RSYNC_LOG_FILE_FLAG, cf.RSYNC_LOCAL_ROOT, RSYNC_LOG_PIPE_FLAG]) elif precheck == 1: # Reading from remote repository into local 'precheck' repository cmd = " ".join(["rsync", flags, full_path, RSYNC_LOG_FILE_FLAG, os.path.join(cf.RSYNC_LOCAL_ROOT,'precheck'), RSYNC_LOG_PIPE_FLAG]) elif precheck == 2: # Reading from remote 'precheck' repository into local 'precheck' repository cmd = " ".join(["rsync", flags, os.path.join(full_path, 'precheck'), RSYNC_LOG_FILE_FLAG, os.path.join(cf.RSYNC_LOCAL_ROOT,'precheck'), RSYNC_LOG_PIPE_FLAG]) else: logger.exception("rsync_command - received invalid precheck option %r", precheck) raise cf.RsyncRuntimeError("rsync_command - failure for command: `%s`" % cmd) logger.debug("rsync command = %r",cmd) out = subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError as e: logger.exception("rsync_command_read_wildcard - failure for command: %s" % cmd) raise cf.RsyncRuntimeError("rsync_command_read_wildcard - failure for command: `%s`" % cmd)