def rsync_command(files, read=True, path=None, delete=False, precheck=0):
""" run rsync, syncing the files (or folders) listed in files, assumed to be paths or partial
paths from the RSYNC_LOCAL_ROOT
"""
if cf.PIPELINE_LOCAL:
return
network.ensure_ssh_connection(network.RSYNC_ADDRESS)
if path:
full_path = RSYNC_PATH + path + "/./"
else:
full_path = RSYNC_PATH
with tempfile.NamedTemporaryFile(delete=True) as tmp:
tmp.file.write("\n".join(files))
tmp.file.close()
flags = RSYNC_FLAGS
if delete:
flags = "--delete "+flags
try:
logger.info("running rsync for files: %r",files)
if read:
if precheck == 0:
# Reading from remote repository into local repository
cmd = " ".join(["rsync", flags, full_path, RSYNC_LOG_FILE_FLAG,
"--files-from={}".format(tmp.name), cf.RSYNC_LOCAL_ROOT, RSYNC_LOG_PIPE_FLAG])
elif precheck == 1:
# Reading from remote repository into local 'precheck' repository
cmd = " ".join(["rsync", flags, full_path, RSYNC_LOG_FILE_FLAG,
"--files-from={}".format(tmp.name), os.path.join(cf.RSYNC_LOCAL_ROOT, 'precheck'),
RSYNC_LOG_PIPE_FLAG])
elif precheck == 2:
# Reading from remote 'precheck' repository into local 'precheck' repository
cmd = " ".join(["rsync", flags, os.path.join(full_path, 'precheck'), RSYNC_LOG_FILE_FLAG,
"--files-from={}".format(tmp.name), os.path.join(cf.RSYNC_LOCAL_ROOT, 'precheck'),
RSYNC_LOG_PIPE_FLAG])
else:
logger.exception("rsync_command - received invalid precheck option %r", precheck)
raise cf.RsyncRuntimeError("rsync_command - failure for command: `%s`" % cmd)
else:
cmd = " ".join(["rsync", flags, RSYNC_LOG_FILE_FLAG,
"--files-from={}".format(tmp.name), cf.RSYNC_LOCAL_ROOT, full_path, RSYNC_LOG_PIPE_FLAG])
logger.debug("rsync command = %r",cmd)
out = subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError as e:
logger.exception("rsync_command - failure for command: %s" % cmd)
raise cf.RsyncRuntimeError("rsync_command - failure for command: `%s`" % cmd)
def rsync_command_read_wildcard(files, path=None, precheck=0):
""" run rsync, syncing the files (or folders) listed in files, assumed to be paths or partial
paths from the RSYNC_LOCAL_ROOT
"""
if cf.PIPELINE_LOCAL:
return
network.ensure_ssh_connection(network.RSYNC_ADDRESS)
for filename in files:
if path:
full_path = RSYNC_PATH + path + "/./"
else:
full_path = RSYNC_PATH
full_path += filename
flags = RSYNC_FLAGS
try:
logger.info("running rsync for files: %r",filename)
if precheck == 0:
# Reading from remote repository into local repository
cmd = " ".join(["rsync", flags, full_path, RSYNC_LOG_FILE_FLAG,
cf.RSYNC_LOCAL_ROOT, RSYNC_LOG_PIPE_FLAG])
elif precheck == 1:
# Reading from remote repository into local 'precheck' repository
cmd = " ".join(["rsync", flags, full_path, RSYNC_LOG_FILE_FLAG,
os.path.join(cf.RSYNC_LOCAL_ROOT,'precheck'), RSYNC_LOG_PIPE_FLAG])
elif precheck == 2:
# Reading from remote 'precheck' repository into local 'precheck' repository
cmd = " ".join(["rsync", flags, os.path.join(full_path, 'precheck'), RSYNC_LOG_FILE_FLAG,
os.path.join(cf.RSYNC_LOCAL_ROOT,'precheck'), RSYNC_LOG_PIPE_FLAG])
else:
logger.exception("rsync_command - received invalid precheck option %r", precheck)
raise cf.RsyncRuntimeError("rsync_command - failure for command: `%s`" % cmd)
logger.debug("rsync command = %r",cmd)
out = subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError as e:
logger.exception("rsync_command_read_wildcard - failure for command: %s" % cmd)
raise cf.RsyncRuntimeError("rsync_command_read_wildcard - failure for command: `%s`" % cmd)
