def run_rsync(cfgs, **kwargs):
opts = ["rsync"]
content = kwargs.get('content')
log_fp = kwargs.get('log', sys.stdout)
dry_run = kwargs.get('dry_run', True)
run_verbose = kwargs.get('run_verbose', False)
if dry_run:
opts.append("-n")
for c in cfgs:
src = c.get('src')
dst = c.get('dst')
if not (src and dst):
content.append(">> rsync w/o src or dst defined: `%s'" % (c,))
continue
rsync_cmd = opts[:]
if 'opts' in c:
rsync_cmd.extend(c.get('opts'))
if run_verbose:
rsync_cmd.append("--progress")
rsync_cmd.extend((src, dst))
log_fp.write(">>> begin @ %s, `%s'\n" %
(datetime.now(), " ".join(rsync_cmd)))
log_fp.flush()
timeout = c.get('timeout', 3600)
if not dry_run:
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(timeout)
if osp.exists(dst):
sandbox.enable(dst)
p = subprocess.Popen(rsync_cmd, stdout=log_fp, stderr=log_fp)
try:
ret = p.wait()
if ret != 0:
content.append(">> `%s' failed w/ %d\n" %
(" ".join(rsync_cmd), ret))
except KeyboardInterrupt:
p.terminate()
except TimeoutException, e:
p.terminate()
content.append(">> `%s' timeout in %ds\n" %
(" ".join(rsync_cmd), timeout))
signal.alarm(0)
sandbox.disable()
log_fp.write(">>> end @ %s, `%s'\n\n" %
(datetime.now(), " ".join(rsync_cmd)))
log_fp.flush()
def run_sftp(cfgs, **kwargs):
content = kwargs.get('content')
log_fp = kwargs.get('log', sys.stdout)
dry_run = kwargs.get('dry_run', True)
for c in cfgs:
src = c.get('src')
dst = c.get('dst')
host = c.get('host')
port = c.get('port', 22)
name = c.get('name')
if not (src and dst and host and name):
content.append(">> sftp w/o src/dst/host/name defined: `%s'" % (c,))
continue
sftp_cmd = "sftp://%s/%s => %s/%s" % (host, src, dst, name)
log_fp.write(">>> begin @ %s, `%s'\n" %
(datetime.now(), sftp_cmd))
log_fp.flush()
timeout = c.get('timeout', 3600)
if not dry_run:
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(timeout)
sandbox.enable(dst)
sftp = None
try:
dstfile = osp.join(dst, name)
if not dry_run:
user = c.get('user')
passwd = c.get('passwd')
transport = paramiko.Transport((host, port))
transport.connect(username = user, password = passwd)
sftp = paramiko.SFTPClient.from_transport(transport)
tmpfile = ".".join((dstfile, "tmp"))
sftp.get(src, tmpfile)
os.rename(tmpfile, dstfile)
sftp.close()
transport.close()
sftp = None
transport = None
else:
log_fp.write(">>> will retr `%s' to `%s'\n" % (src, dstfile))
except paramiko.ssh_exception.SSHException, e:
content.append(">> `%s' error %s\n" % (sftp_cmd, e))
except KeyboardInterrupt:
pass
def run_ftp(cfgs, **kwargs):
content = kwargs.get('content')
log_fp = kwargs.get('log', sys.stdout)
dry_run = kwargs.get('dry_run', True)
for c in cfgs:
src = c.get('src')
dst = c.get('dst')
host = c.get('host')
name = c.get('name')
if not (src and dst and host and name):
content.append(">> ftp w/o src/dst/host/name defined: `%s'" % (c,))
continue
ftp_cmd = "ftp://%s/%s => %s/%s" % (host, src, dst, name)
log_fp.write(">>> begin @ %s, `%s'\n" %
(datetime.now(), ftp_cmd))
log_fp.flush()
timeout = c.get('timeout', 3600)
if not dry_run:
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(timeout)
sandbox.enable(dst)
ftp = None
try:
dstfile = osp.join(dst, name)
if not dry_run:
ftp = ftplib.FTP(host)
user = c.get('user')
passwd = c.get('passwd')
if user and passwd:
ftp.login(user, passwd)
if c.get('pasv'):
ftp.set_pasv(c.get('pasv'))
tmpfile = ".".join((dstfile, "tmp"))
ftp.retrbinary(" ".join(("RETR", src)),
open(tmpfile, 'wb').write)
os.rename(tmpfile, dstfile)
ftp.quit()
ftp = None
else:
log_fp.write(">>> will retr `%s' to `%s'\n" % (src, dstfile))
except ftplib.all_errors, e:
content.append(">> `%s' error %s\n" % (ftp_cmd, e))
except KeyboardInterrupt:
pass
def run_command(cfgs, **kwargs):
content = kwargs.get('content')
log_fp = kwargs.get('log', sys.stdout)
dry_run = kwargs.get('dry_run', True)
for c in cfgs:
sandbox_dir = c.get('dir')
command = c.get('cmd')
run_cmd = []
if not command or not sandbox_dir:
content.append(">> command and dir must both be defined: `%s'" %
(c,))
continue
now = datetime.now().strftime("%Y%m%d-%H%M")
for c in command:
c = c % locals()
run_cmd.append(c)
log_fp.write(">>> begin @ %s, `%s'\n" %
(datetime.now(), " ".join(command)))
if dry_run:
log_fp.write(">>> will run command `%s'\n" % (" ".join(run_cmd)))
else:
sandbox.enable(sandbox_dir)
p = subprocess.Popen(run_cmd, stdout=log_fp, stderr=log_fp)
try:
ret = p.wait()
if ret != 0:
content.append(">> `%s' failed w/ %d\n" %
(" ".join(run_cmd), ret))
else:
log_fp.write(">>> command `%s` success\n" %
" ".join(run_cmd))
except KeyboardInterrupt:
p.terminate()
sandbox.disable()
log_fp.write(">>> end @ %s, `%s'\n" %
(datetime.now(), " ".join(command)))
log_fp.flush()
def run_mysql(cfgs, **kwargs):
opts = ["mysqldump"]
content = kwargs.get('content')
log_fp = kwargs.get('log', sys.stdout)
dry_run = kwargs.get('dry_run', True)
for c in cfgs:
name = c.get('name')
db = c.get('db', '--all-databases')
dst = c.get('dst')
if not name or not dst:
content.append(">> mysqldump w/o dst defined: `%s'" % (c,))
continue
mysql_cmd = opts[:]
if 'opts' in c:
mysql_cmd.extend(c.get('opts'))
mysql_cmd.extend((db,))
log_fp.write(">>> begin @ %s, `%s'\n" %
(datetime.now(), " ".join(mysql_cmd)))
log_fp.flush()
filename = osp.join(dst, "%s-%s.mysql.gz" %
(datetime.now().strftime("%Y%m%d-%H%M"), name))
saved_tmp = os.environ.get('TMPDIR')
if osp.isdir(dst):
sandbox.enable(dst)
else:
content.append(">> `%s' not exists or is not directory" % (dst,))
continue
if dry_run:
log_fp.write(">>> will save file to %s\n" %
(osp.join(dst, filename)))
else:
if c.get('tmp'):
os.environ['TMPDIR'] = c.get('tmp')
dump_fp = tempfile.TemporaryFile()
p = subprocess.Popen(mysql_cmd, stdout=dump_fp, stderr=log_fp)
try:
ret = p.wait()
if ret != 0:
content.append(">> `%s' failed w/ %d\n" %
(" ".join(mysql_cmd), ret))
else:
gzfile = gzip.open(filename, "wb")
dump_fp.seek(0, os.SEEK_END)
s0 = dump_fp.tell()
dump_fp.seek(0, os.SEEK_SET)
gzfile.writelines(dump_fp)
gzfile.close()
dump_fp.close()
s1 = os.stat(filename).st_size
log_fp.write(">>> save file to %s, size %d (%.0f%%)\n" %
(osp.join(dst, filename), s1, s1*100.0/s0))
except KeyboardInterrupt:
p.terminate()
if os.environ.get('TMPDIR'):
if saved_tmp:
os.environ['TMPDIR'] = saved_tmp
else:
os.environ.pop('TMPDIR')
sandbox.disable()
log_fp.write(">>> end @ %s, `%s'\n\n" %
(datetime.now(), " ".join(mysql_cmd)))
log_fp.flush()