Example #1
0
def run_rsync(cfgs, **kwargs):
    opts = ["rsync"]
    content = kwargs.get('content')
    log_fp = kwargs.get('log', sys.stdout)
    dry_run = kwargs.get('dry_run', True)
    run_verbose = kwargs.get('run_verbose', False)

    if dry_run:
        opts.append("-n")

    for c in cfgs:
        src = c.get('src')
        dst = c.get('dst')
        if not (src and dst):
            content.append(">> rsync w/o src or dst defined: `%s'" % (c,))
            continue
        rsync_cmd = opts[:]
        if 'opts' in c:
            rsync_cmd.extend(c.get('opts'))
        if run_verbose:
            rsync_cmd.append("--progress")
        rsync_cmd.extend((src, dst))

        log_fp.write(">>> begin @ %s, `%s'\n" %
                     (datetime.now(), " ".join(rsync_cmd)))
        log_fp.flush()

        timeout = c.get('timeout', 3600)

        if not dry_run:
            signal.signal(signal.SIGALRM, alarm_handler)
            signal.alarm(timeout)

        if osp.exists(dst):
            sandbox.enable(dst)

        p = subprocess.Popen(rsync_cmd, stdout=log_fp, stderr=log_fp)
        try:
            ret = p.wait()
            if ret != 0:
                content.append(">> `%s' failed w/ %d\n" %
                               (" ".join(rsync_cmd), ret))
        except KeyboardInterrupt:
            p.terminate()
        except TimeoutException, e:
            p.terminate()
            content.append(">> `%s' timeout in %ds\n" %
                           (" ".join(rsync_cmd), timeout))

        signal.alarm(0)

        sandbox.disable()

        log_fp.write(">>> end @ %s, `%s'\n\n" %
                     (datetime.now(), " ".join(rsync_cmd)))
        log_fp.flush()
Example #2
0
def run_sftp(cfgs, **kwargs):
    content = kwargs.get('content')
    log_fp = kwargs.get('log', sys.stdout)
    dry_run = kwargs.get('dry_run', True)

    for c in cfgs:
        src = c.get('src')
        dst = c.get('dst')
        host = c.get('host')
        port = c.get('port', 22)
        name = c.get('name')
        if not (src and dst and host and name):
            content.append(">> sftp w/o src/dst/host/name defined: `%s'" % (c,))
            continue
        sftp_cmd = "sftp://%s/%s => %s/%s" % (host, src, dst, name)

        log_fp.write(">>> begin @ %s, `%s'\n" %
                     (datetime.now(), sftp_cmd))
        log_fp.flush()

        timeout = c.get('timeout', 3600)

        if not dry_run:
            signal.signal(signal.SIGALRM, alarm_handler)
            signal.alarm(timeout)

        sandbox.enable(dst)

        sftp = None
        try:
            dstfile = osp.join(dst, name)
            if not dry_run:
                user = c.get('user')
                passwd = c.get('passwd')
                transport = paramiko.Transport((host, port))
                transport.connect(username = user, password = passwd)
                sftp = paramiko.SFTPClient.from_transport(transport)
                tmpfile = ".".join((dstfile, "tmp"))
                sftp.get(src, tmpfile)
                os.rename(tmpfile, dstfile)
                sftp.close()
                transport.close()
                sftp = None
                transport = None
            else:
                log_fp.write(">>> will retr `%s' to `%s'\n" % (src, dstfile))
        except paramiko.ssh_exception.SSHException, e:
            content.append(">> `%s' error %s\n" % (sftp_cmd, e))
        except KeyboardInterrupt:
            pass
Example #3
0
def run_ftp(cfgs, **kwargs):
    content = kwargs.get('content')
    log_fp = kwargs.get('log', sys.stdout)
    dry_run = kwargs.get('dry_run', True)

    for c in cfgs:
        src = c.get('src')
        dst = c.get('dst')
        host = c.get('host')
        name = c.get('name')
        if not (src and dst and host and name):
            content.append(">> ftp w/o src/dst/host/name defined: `%s'" % (c,))
            continue
        ftp_cmd = "ftp://%s/%s => %s/%s" % (host, src, dst, name)

        log_fp.write(">>> begin @ %s, `%s'\n" %
                     (datetime.now(), ftp_cmd))
        log_fp.flush()

        timeout = c.get('timeout', 3600)

        if not dry_run:
            signal.signal(signal.SIGALRM, alarm_handler)
            signal.alarm(timeout)

        sandbox.enable(dst)

        ftp = None
        try:
            dstfile = osp.join(dst, name)
            if not dry_run:
                ftp = ftplib.FTP(host)
                user = c.get('user')
                passwd = c.get('passwd')
                if user and passwd:
                    ftp.login(user, passwd)
                if c.get('pasv'):
                    ftp.set_pasv(c.get('pasv'))
                tmpfile = ".".join((dstfile, "tmp"))
                ftp.retrbinary(" ".join(("RETR", src)),
                               open(tmpfile, 'wb').write)
                os.rename(tmpfile, dstfile)
                ftp.quit()
                ftp = None
            else:
                log_fp.write(">>> will retr `%s' to `%s'\n" % (src, dstfile))
        except ftplib.all_errors, e:
            content.append(">> `%s' error %s\n" % (ftp_cmd, e))
        except KeyboardInterrupt:
            pass
Example #4
0
def run_command(cfgs, **kwargs):
    content = kwargs.get('content')
    log_fp = kwargs.get('log', sys.stdout)
    dry_run = kwargs.get('dry_run', True)

    for c in cfgs:
        sandbox_dir = c.get('dir')
        command = c.get('cmd')
        run_cmd = []

        if not command or not sandbox_dir:
            content.append(">> command and dir must both be defined: `%s'" %
                           (c,))
            continue

        now = datetime.now().strftime("%Y%m%d-%H%M")

        for c in command:
            c = c % locals()
            run_cmd.append(c)

        log_fp.write(">>> begin @ %s, `%s'\n" %
                     (datetime.now(), " ".join(command)))

        if dry_run:
            log_fp.write(">>> will run command `%s'\n" % (" ".join(run_cmd)))
        else:
            sandbox.enable(sandbox_dir)
            p = subprocess.Popen(run_cmd, stdout=log_fp, stderr=log_fp)
            try:
                ret = p.wait()
                if ret != 0:
                    content.append(">> `%s' failed w/ %d\n" %
                                   (" ".join(run_cmd), ret))
                else:
                    log_fp.write(">>> command `%s` success\n" %
                                 " ".join(run_cmd))
            except KeyboardInterrupt:
                p.terminate()

        sandbox.disable()

        log_fp.write(">>> end @ %s, `%s'\n" %
                     (datetime.now(), " ".join(command)))
        log_fp.flush()
Example #5
0
def run_mysql(cfgs, **kwargs):
    opts = ["mysqldump"]
    content = kwargs.get('content')
    log_fp = kwargs.get('log', sys.stdout)
    dry_run = kwargs.get('dry_run', True)

    for c in cfgs:
        name = c.get('name')
        db = c.get('db', '--all-databases')
        dst = c.get('dst')
        if not name or not dst:
            content.append(">> mysqldump w/o dst defined: `%s'" % (c,))
            continue

        mysql_cmd = opts[:]
        if 'opts' in c:
            mysql_cmd.extend(c.get('opts'))
        mysql_cmd.extend((db,))

        log_fp.write(">>> begin @ %s, `%s'\n" %
                     (datetime.now(), " ".join(mysql_cmd)))
        log_fp.flush()

        filename = osp.join(dst, "%s-%s.mysql.gz" %
                            (datetime.now().strftime("%Y%m%d-%H%M"), name))

        saved_tmp = os.environ.get('TMPDIR')

        if osp.isdir(dst):
            sandbox.enable(dst)
        else:
            content.append(">> `%s' not exists or is not directory" % (dst,))
            continue

        if dry_run:
            log_fp.write(">>> will save file to %s\n" %
                         (osp.join(dst, filename)))
        else:
            if c.get('tmp'):
                os.environ['TMPDIR'] = c.get('tmp')
            dump_fp = tempfile.TemporaryFile()
            p = subprocess.Popen(mysql_cmd, stdout=dump_fp, stderr=log_fp)
            try:
                ret = p.wait()
                if ret != 0:
                    content.append(">> `%s' failed w/ %d\n" %
                                   (" ".join(mysql_cmd), ret))
                else:
                    gzfile = gzip.open(filename, "wb")
                    dump_fp.seek(0, os.SEEK_END)
                    s0 = dump_fp.tell()
                    dump_fp.seek(0, os.SEEK_SET)
                    gzfile.writelines(dump_fp)
                    gzfile.close()
                    dump_fp.close()
                    s1 = os.stat(filename).st_size
                    log_fp.write(">>> save file to %s, size %d (%.0f%%)\n" %
                                 (osp.join(dst, filename), s1, s1*100.0/s0))
            except KeyboardInterrupt:
                p.terminate()

        if os.environ.get('TMPDIR'):
            if saved_tmp:
                os.environ['TMPDIR'] = saved_tmp
            else:
                os.environ.pop('TMPDIR')

        sandbox.disable()

        log_fp.write(">>> end @ %s, `%s'\n\n" %
                     (datetime.now(), " ".join(mysql_cmd)))
        log_fp.flush()