Esempio n. 1
0
    class put:
        def setup(self):
            self.c = Connection('localhost')
            self.remote = self._tmp('file.txt')

        def base_case(self):
            # Copy file from 'local' (support dir) to 'remote' (tempdir)
            # TODO: consider path.py for contextmanager
            cwd = os.getcwd()
            os.chdir(self._support())
            try:
                # TODO: wrap chdir at the Connection level
                self.c.sftp().chdir(self._tmp())
                result = self.c.put('file.txt')
            finally:
                os.chdir(cwd)

            # Make sure it arrived
            ok_(os.path.exists(self.remote))
            eq_(open(self.remote).read(), "yup\n")
            # Sanity check result object
            eq_(result.remote, self.remote)
            eq_(result.orig_remote, None)
            eq_(result.local, self._support('file.txt'))
            eq_(result.orig_local, 'file.txt')

        def file_like_objects(self):
            fd = BytesIO()
            fd.write(b"yup\n")
            result = self.c.put(local=fd, remote=self.remote)
            eq_(open(self.remote).read(), "yup\n")
            eq_(result.remote, self.remote)
            ok_(result.local is fd)

        def mode_preservation(self):
            # Use a dummy file which is given an unusual, highly unlikely to be
            # default umask, set of permissions (oct 641, aka -rw-r----x)
            local = self._tmp('funky-local.txt')
            with open(local, 'w') as fd:
                fd.write('whatever')
            os.chmod(local, 0o641)
            remote = self._tmp('funky-remote.txt')
            self.c.put(remote=remote, local=local)
            eq_(stat.S_IMODE(os.stat(remote).st_mode), 0o641)
Esempio n. 2
0
    class put:
        def setup(self):
            self.c = Connection('localhost')
            self.remote = self._tmp('file.txt')

        def base_case(self):
            # Copy file from 'local' (support dir) to 'remote' (tempdir)
            # TODO: consider path.py for contextmanager
            cwd = os.getcwd()
            os.chdir(self._support())
            try:
                # TODO: wrap chdir at the Connection level
                self.c.sftp().chdir(self._tmp())
                result = self.c.put('file.txt')
            finally:
                os.chdir(cwd)

            # Make sure it arrived
            ok_(os.path.exists(self.remote))
            eq_(open(self.remote).read(), "yup\n")
            # Sanity check result object
            eq_(result.remote, self.remote)
            eq_(result.orig_remote, None)
            eq_(result.local, self._support('file.txt'))
            eq_(result.orig_local, 'file.txt')

        def file_like_objects(self):
            fd = BytesIO()
            fd.write(b"yup\n")
            result = self.c.put(local=fd, remote=self.remote)
            eq_(open(self.remote).read(), "yup\n")
            eq_(result.remote, self.remote)
            ok_(result.local is fd)

        def mode_preservation(self):
            # Use a dummy file which is given an unusual, highly unlikely to be
            # default umask, set of permissions (oct 641, aka -rw-r----x)
            local = self._tmp('funky-local.txt')
            with open(local, 'w') as fd:
                fd.write('whatever')
            os.chmod(local, 0o641)
            remote = self._tmp('funky-remote.txt')
            self.c.put(remote=remote, local=local)
            eq_(stat.S_IMODE(os.stat(remote).st_mode), 0o641)
Esempio n. 3
0
    class put:

        def setup(self):
            self.c = Connection("localhost")
            self.remote = path.local.mkdtemp().join("file.txt").realpath()

        def base_case(self):
            # Copy file from 'local' (support dir) to 'remote' (tempdir)
            local_dir = _support()
            with path.local(local_dir).as_cwd():
                tmpdir = self.remote.dirpath()
                # TODO: wrap chdir at the Connection level
                self.c.sftp().chdir(str(tmpdir))
                result = self.c.put("file.txt")
            # Make sure it arrived
            assert self.remote.check()
            assert self.remote.read() == "yup\n"
            # Sanity check result object
            assert result.remote == self.remote
            assert result.orig_remote is None
            assert result.local == _support("file.txt")
            assert result.orig_local == "file.txt"

        def file_like_objects(self):
            fd = BytesIO()
            fd.write(b"yup\n")
            remote_str = str(self.remote)
            result = self.c.put(local=fd, remote=remote_str)
            assert self.remote.read() == "yup\n"
            assert result.remote == remote_str
            assert result.local is fd

        def mode_preservation(self, tmpdir):
            # Use a dummy file which is given an unusual, highly unlikely to be
            # default umask, set of permissions (oct 641, aka -rw-r----x)
            local = tmpdir.join("funky-local.txt")
            local.write("whatever")
            local.chmod(0o641)
            remote = tmpdir.join("funky-remote.txt")
            self.c.put(remote=str(remote), local=str(local))
            assert stat.S_IMODE(remote.stat().mode) == 0o641
Esempio n. 4
0
    class put:
        def setup(self):
            self.c = Connection("localhost")
            self.remote = path.local.mkdtemp().join("file.txt").realpath()

        def base_case(self):
            # Copy file from 'local' (support dir) to 'remote' (tempdir)
            local_dir = _support()
            with path.local(local_dir).as_cwd():
                tmpdir = self.remote.dirpath()
                # TODO: wrap chdir at the Connection level
                self.c.sftp().chdir(str(tmpdir))
                result = self.c.put("file.txt")
            # Make sure it arrived
            assert self.remote.check()
            assert self.remote.read() == "yup\n"
            # Sanity check result object
            assert result.remote == self.remote
            assert result.orig_remote is None
            assert result.local == _support("file.txt")
            assert result.orig_local == "file.txt"

        def file_like_objects(self):
            fd = BytesIO()
            fd.write(b"yup\n")
            remote_str = str(self.remote)
            result = self.c.put(local=fd, remote=remote_str)
            assert self.remote.read() == "yup\n"
            assert result.remote == remote_str
            assert result.local is fd

        def mode_preservation(self, tmpdir):
            # Use a dummy file which is given an unusual, highly unlikely to be
            # default umask, set of permissions (oct 641, aka -rw-r----x)
            local = tmpdir.join("funky-local.txt")
            local.write("whatever")
            local.chmod(0o641)
            remote = tmpdir.join("funky-remote.txt")
            self.c.put(remote=str(remote), local=str(local))
            assert stat.S_IMODE(remote.stat().mode) == 0o641
Esempio n. 5
0
def main():
    args = docopt(__doc__, version='Colab Tool 1.0')
    SP.enabled = args['--verbose']

    root_path = args.get('--root-path')
    if not root_path:
        root_path = environ['MUSICGEN_ROOT_PATH']
    root_path = Path(root_path)

    auth = args.get('--authority')
    if not auth:
        auth = environ['MUSICGEN_AUTHORITY']
    userinfo, netloc = auth.split('@')
    _, password = userinfo.split(':')
    host, port = netloc.split(':')
    port = int(port)

    connect_kwargs = {'password': password}
    SP.print('Connecting to %s' % host)
    conn = Connection(host, 'root', port, connect_kwargs=connect_kwargs)
    sftp = conn.sftp()
    SP.print('Changing to dir "%s".' % root_path)
    remote_mkdir_safe(sftp, root_path)
    sftp.chdir(str(root_path))
    if args['get-data']:
        get_data(conn, sftp)
    elif args['upload-code']:
        upload_code(conn, sftp)
    elif args['upload-caches']:
        corpus_path = Path(args['<corpus-path>'])
        upload_caches(conn, corpus_path)
    elif args['upload-file']:
        local_path = Path(args['<local-file>'])
        upload_file(conn, local_path)
    elif args['upload-and-run-file']:
        src = Path(args['<file>'])
        dst = src.parent
        if args['--drop-path']:
            dst = Path('.')
        upload_files(conn, [(src, dst)])
        if args['--drop-path']:
            src = src.name

        run_python_file(conn, root_path, str(src), args['<args>'])
    elif args['run-file']:
        run_python_file(conn, root_path, args['<file>'], args['<args>'])
    else:
        assert False
Esempio n. 6
0
def deploy_production(c, file_type, package_name):
    #current_day = time.strftime("%y%m%d")
    # print(current_day)
    local_update_files_folder = 'e:/release_files/'
    remote_update_files_folder = '/cygdrive/c/update_files/'
    remote_backup_folder = '/cygdrive/c/update_files/production_web_backup/'
    remote_web_site_folder = '/cygdrive/c/progra~2/THC/'
    conn = Connection(host='11.11.11.10', user='******',
                      connect_kwargs={'password': "******"})
    with conn.cd(remote_update_files_folder):
        logger.info('begin to upload file...')
        begin_time = time.time()
        cbk, pbar = show_progress_bar(ascii=True, unit='b', unit_scale=True)
        result = conn.sftp().put("{0}{1}.zip".format(local_update_files_folder, package_name), "%s%s.zip" % (
            remote_update_files_folder, package_name), callback=cbk)
        pbar.close()
        # conn.sftp().put("{0}{1}.zip".format( local_update_files_folder, package_name), remote_update_files_folder + package_name + ".zip", callback=progress_bar)
        end_time = time.time()
        logger.info('file upload finished...')
        logger.info("Uploaded {0}{1}.zip -> {2}{1}.zip".format(
            local_update_files_folder, package_name, remote_update_files_folder))
        logger.info("used {:.5}s".format(end_time - begin_time))
        time.sleep(1)
        logger.info('unzip the uploaded files...')
        conn.run("unzip -o {0}.zip -d .".format(package_name))
    with conn.cd(remote_backup_folder):
        conn.run('rm -rf ./thc/*')
        logger.info('copy files to the backup folder...')
        conn.run("cp -a -f -v ../{0}/* ./thc/".format(package_name))
        logger.info('backup the files ...')
        conn.run("./backup_thc.bat")
        if file_type == 'dll':
            logger.info('stopping service...')
            conn.run('{0}stop.bat'.format(remote_web_site_folder))
            logger.info('update release files ...')
            #conn.run('cp -a -f -v ./thc/* {0}'.format(remote_web_site_folder))
            conn.run("./copy_to_thc.bat")
            logger.info('starting service ...')
            conn.run('{0}start.bat'.format(remote_web_site_folder))
        else:
            logger.info('update release files ...')
            #conn.run('cp -a -f -v ./thc/* {0}'.format(remote_web_site_folder))
            conn.run("./copy_to_thc.bat")
        logger.info('deploy finished.')
Esempio n. 7
0
class SshTool():
    def __init__(self, dev_info, logger: Logger = logger):

        # self.dev_info = dev_info

        self.connect_info = dict(
            host=dev_info['hostname'],
            user=dev_info['username'],
            connect_kwargs={'password': dev_info['password']})

        self.logger = logger

    def __enter__(self):

        self.client = Connection(**self.connect_info)
        self.sftp = self.client.sftp()

        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def close(self):
        self.client.close()

    def cd(self, path):
        return self.client.cd(path)

    def run(self, command, **kwargs):
        return self.client.run(command, **kwargs)

    def local(self, *args, **kwargs):
        return self.client(*args, **kwargs)

    def exists(self, path):
        """judge whether the path existes
        """
        try:
            self.sftp.stat(path)
        except IOError as e:
            if 'No such file' in str(e):
                return False
            raise
        else:
            return True

    def isdir(self, path):
        try:
            return stat.S_ISDIR(self.sftp.stat(path).st_mode)
        except IOError:
            return False

    def mkdir(self, path, ignore_existed=True):
        """make a directory, if ignore_existed == True, it will happen error if the dir existes
        """
        try:
            if self.exists(path):
                return False
            self.sftp.mkdir(path)
            return True
        except IOError as e:
            if ignore_existed:
                pass
            else:
                return False
        return True

    def rm(self, path):
        if self.exists(path) and not self.isdir(path):
            self.sftp.remove(path)

    def rm_dir(self, path):

        if self.exists(path):

            files = self.sftp.listdir(path=path)

            for f in files:
                filepath = '{}/{}'.format(path, f)
                if self.isdir(filepath):
                    self.rm_dir(filepath)
                else:
                    self.sftp.remove(filepath)
            self.sftp.rmdir(path)

    def put(self, *args, **kwargs):
        try:
            self.client.put(*args, **kwargs)
            return True
        except IOError as e:
            self.logger.error('transfer failed! err: {}'.format(e))
            return False

    def put_dir(self, source_dir, dest_dir):

        target_dir = Path(dest_dir, Path(source_dir).name)
        if self.exists(target_dir.__str__()):
            self.rm_dir(target_dir.__str__())

        local_tar_file = Path(tempfile.gettempdir(),
                              os.path.basename(source_dir) + '.tar.gz')
        if local_tar_file.exists():
            os.remove(local_tar_file)

        self.__make_targz(source_dir, local_tar_file)

        remote_tar_file = Path(dest_dir, local_tar_file.name).__str__()

        self.put(local_tar_file, dest_dir)
        self.run('python3 -m tarfile -e {} {}'.format(remote_tar_file,
                                                      dest_dir))
        self.rm(remote_tar_file)

        os.remove(local_tar_file)

        return True

    def __make_targz(self, source_dir, output_filename):
        with tarfile.open(output_filename, 'w:gz') as tar:
            tar.add(source_dir)