class put:
def setup(self):
self.c = Connection('localhost')
self.remote = self._tmp('file.txt')
def base_case(self):
# Copy file from 'local' (support dir) to 'remote' (tempdir)
# TODO: consider path.py for contextmanager
cwd = os.getcwd()
os.chdir(self._support())
try:
# TODO: wrap chdir at the Connection level
self.c.sftp().chdir(self._tmp())
result = self.c.put('file.txt')
finally:
os.chdir(cwd)
# Make sure it arrived
ok_(os.path.exists(self.remote))
eq_(open(self.remote).read(), "yup\n")
# Sanity check result object
eq_(result.remote, self.remote)
eq_(result.orig_remote, None)
eq_(result.local, self._support('file.txt'))
eq_(result.orig_local, 'file.txt')
def file_like_objects(self):
fd = BytesIO()
fd.write(b"yup\n")
result = self.c.put(local=fd, remote=self.remote)
eq_(open(self.remote).read(), "yup\n")
eq_(result.remote, self.remote)
ok_(result.local is fd)
def mode_preservation(self):
# Use a dummy file which is given an unusual, highly unlikely to be
# default umask, set of permissions (oct 641, aka -rw-r----x)
local = self._tmp('funky-local.txt')
with open(local, 'w') as fd:
fd.write('whatever')
os.chmod(local, 0o641)
remote = self._tmp('funky-remote.txt')
self.c.put(remote=remote, local=local)
eq_(stat.S_IMODE(os.stat(remote).st_mode), 0o641)
class put:
def setup(self):
self.c = Connection('localhost')
self.remote = self._tmp('file.txt')
def base_case(self):
# Copy file from 'local' (support dir) to 'remote' (tempdir)
# TODO: consider path.py for contextmanager
cwd = os.getcwd()
os.chdir(self._support())
try:
# TODO: wrap chdir at the Connection level
self.c.sftp().chdir(self._tmp())
result = self.c.put('file.txt')
finally:
os.chdir(cwd)
# Make sure it arrived
ok_(os.path.exists(self.remote))
eq_(open(self.remote).read(), "yup\n")
# Sanity check result object
eq_(result.remote, self.remote)
eq_(result.orig_remote, None)
eq_(result.local, self._support('file.txt'))
eq_(result.orig_local, 'file.txt')
def file_like_objects(self):
fd = BytesIO()
fd.write(b"yup\n")
result = self.c.put(local=fd, remote=self.remote)
eq_(open(self.remote).read(), "yup\n")
eq_(result.remote, self.remote)
ok_(result.local is fd)
def mode_preservation(self):
# Use a dummy file which is given an unusual, highly unlikely to be
# default umask, set of permissions (oct 641, aka -rw-r----x)
local = self._tmp('funky-local.txt')
with open(local, 'w') as fd:
fd.write('whatever')
os.chmod(local, 0o641)
remote = self._tmp('funky-remote.txt')
self.c.put(remote=remote, local=local)
eq_(stat.S_IMODE(os.stat(remote).st_mode), 0o641)
class put:
def setup(self):
self.c = Connection("localhost")
self.remote = path.local.mkdtemp().join("file.txt").realpath()
def base_case(self):
# Copy file from 'local' (support dir) to 'remote' (tempdir)
local_dir = _support()
with path.local(local_dir).as_cwd():
tmpdir = self.remote.dirpath()
# TODO: wrap chdir at the Connection level
self.c.sftp().chdir(str(tmpdir))
result = self.c.put("file.txt")
# Make sure it arrived
assert self.remote.check()
assert self.remote.read() == "yup\n"
# Sanity check result object
assert result.remote == self.remote
assert result.orig_remote is None
assert result.local == _support("file.txt")
assert result.orig_local == "file.txt"
def file_like_objects(self):
fd = BytesIO()
fd.write(b"yup\n")
remote_str = str(self.remote)
result = self.c.put(local=fd, remote=remote_str)
assert self.remote.read() == "yup\n"
assert result.remote == remote_str
assert result.local is fd
def mode_preservation(self, tmpdir):
# Use a dummy file which is given an unusual, highly unlikely to be
# default umask, set of permissions (oct 641, aka -rw-r----x)
local = tmpdir.join("funky-local.txt")
local.write("whatever")
local.chmod(0o641)
remote = tmpdir.join("funky-remote.txt")
self.c.put(remote=str(remote), local=str(local))
assert stat.S_IMODE(remote.stat().mode) == 0o641
class put:
def setup(self):
self.c = Connection("localhost")
self.remote = path.local.mkdtemp().join("file.txt").realpath()
def base_case(self):
# Copy file from 'local' (support dir) to 'remote' (tempdir)
local_dir = _support()
with path.local(local_dir).as_cwd():
tmpdir = self.remote.dirpath()
# TODO: wrap chdir at the Connection level
self.c.sftp().chdir(str(tmpdir))
result = self.c.put("file.txt")
# Make sure it arrived
assert self.remote.check()
assert self.remote.read() == "yup\n"
# Sanity check result object
assert result.remote == self.remote
assert result.orig_remote is None
assert result.local == _support("file.txt")
assert result.orig_local == "file.txt"
def file_like_objects(self):
fd = BytesIO()
fd.write(b"yup\n")
remote_str = str(self.remote)
result = self.c.put(local=fd, remote=remote_str)
assert self.remote.read() == "yup\n"
assert result.remote == remote_str
assert result.local is fd
def mode_preservation(self, tmpdir):
# Use a dummy file which is given an unusual, highly unlikely to be
# default umask, set of permissions (oct 641, aka -rw-r----x)
local = tmpdir.join("funky-local.txt")
local.write("whatever")
local.chmod(0o641)
remote = tmpdir.join("funky-remote.txt")
self.c.put(remote=str(remote), local=str(local))
assert stat.S_IMODE(remote.stat().mode) == 0o641
def main():
args = docopt(__doc__, version='Colab Tool 1.0')
SP.enabled = args['--verbose']
root_path = args.get('--root-path')
if not root_path:
root_path = environ['MUSICGEN_ROOT_PATH']
root_path = Path(root_path)
auth = args.get('--authority')
if not auth:
auth = environ['MUSICGEN_AUTHORITY']
userinfo, netloc = auth.split('@')
_, password = userinfo.split(':')
host, port = netloc.split(':')
port = int(port)
connect_kwargs = {'password': password}
SP.print('Connecting to %s' % host)
conn = Connection(host, 'root', port, connect_kwargs=connect_kwargs)
sftp = conn.sftp()
SP.print('Changing to dir "%s".' % root_path)
remote_mkdir_safe(sftp, root_path)
sftp.chdir(str(root_path))
if args['get-data']:
get_data(conn, sftp)
elif args['upload-code']:
upload_code(conn, sftp)
elif args['upload-caches']:
corpus_path = Path(args['<corpus-path>'])
upload_caches(conn, corpus_path)
elif args['upload-file']:
local_path = Path(args['<local-file>'])
upload_file(conn, local_path)
elif args['upload-and-run-file']:
src = Path(args['<file>'])
dst = src.parent
if args['--drop-path']:
dst = Path('.')
upload_files(conn, [(src, dst)])
if args['--drop-path']:
src = src.name
run_python_file(conn, root_path, str(src), args['<args>'])
elif args['run-file']:
run_python_file(conn, root_path, args['<file>'], args['<args>'])
else:
assert False
def deploy_production(c, file_type, package_name):
#current_day = time.strftime("%y%m%d")
# print(current_day)
local_update_files_folder = 'e:/release_files/'
remote_update_files_folder = '/cygdrive/c/update_files/'
remote_backup_folder = '/cygdrive/c/update_files/production_web_backup/'
remote_web_site_folder = '/cygdrive/c/progra~2/THC/'
conn = Connection(host='11.11.11.10', user='******',
connect_kwargs={'password': "******"})
with conn.cd(remote_update_files_folder):
logger.info('begin to upload file...')
begin_time = time.time()
cbk, pbar = show_progress_bar(ascii=True, unit='b', unit_scale=True)
result = conn.sftp().put("{0}{1}.zip".format(local_update_files_folder, package_name), "%s%s.zip" % (
remote_update_files_folder, package_name), callback=cbk)
pbar.close()
# conn.sftp().put("{0}{1}.zip".format( local_update_files_folder, package_name), remote_update_files_folder + package_name + ".zip", callback=progress_bar)
end_time = time.time()
logger.info('file upload finished...')
logger.info("Uploaded {0}{1}.zip -> {2}{1}.zip".format(
local_update_files_folder, package_name, remote_update_files_folder))
logger.info("used {:.5}s".format(end_time - begin_time))
time.sleep(1)
logger.info('unzip the uploaded files...')
conn.run("unzip -o {0}.zip -d .".format(package_name))
with conn.cd(remote_backup_folder):
conn.run('rm -rf ./thc/*')
logger.info('copy files to the backup folder...')
conn.run("cp -a -f -v ../{0}/* ./thc/".format(package_name))
logger.info('backup the files ...')
conn.run("./backup_thc.bat")
if file_type == 'dll':
logger.info('stopping service...')
conn.run('{0}stop.bat'.format(remote_web_site_folder))
logger.info('update release files ...')
#conn.run('cp -a -f -v ./thc/* {0}'.format(remote_web_site_folder))
conn.run("./copy_to_thc.bat")
logger.info('starting service ...')
conn.run('{0}start.bat'.format(remote_web_site_folder))
else:
logger.info('update release files ...')
#conn.run('cp -a -f -v ./thc/* {0}'.format(remote_web_site_folder))
conn.run("./copy_to_thc.bat")
logger.info('deploy finished.')
class SshTool():
def __init__(self, dev_info, logger: Logger = logger):
# self.dev_info = dev_info
self.connect_info = dict(
host=dev_info['hostname'],
user=dev_info['username'],
connect_kwargs={'password': dev_info['password']})
self.logger = logger
def __enter__(self):
self.client = Connection(**self.connect_info)
self.sftp = self.client.sftp()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self):
self.client.close()
def cd(self, path):
return self.client.cd(path)
def run(self, command, **kwargs):
return self.client.run(command, **kwargs)
def local(self, *args, **kwargs):
return self.client(*args, **kwargs)
def exists(self, path):
"""judge whether the path existes
"""
try:
self.sftp.stat(path)
except IOError as e:
if 'No such file' in str(e):
return False
raise
else:
return True
def isdir(self, path):
try:
return stat.S_ISDIR(self.sftp.stat(path).st_mode)
except IOError:
return False
def mkdir(self, path, ignore_existed=True):
"""make a directory, if ignore_existed == True, it will happen error if the dir existes
"""
try:
if self.exists(path):
return False
self.sftp.mkdir(path)
return True
except IOError as e:
if ignore_existed:
pass
else:
return False
return True
def rm(self, path):
if self.exists(path) and not self.isdir(path):
self.sftp.remove(path)
def rm_dir(self, path):
if self.exists(path):
files = self.sftp.listdir(path=path)
for f in files:
filepath = '{}/{}'.format(path, f)
if self.isdir(filepath):
self.rm_dir(filepath)
else:
self.sftp.remove(filepath)
self.sftp.rmdir(path)
def put(self, *args, **kwargs):
try:
self.client.put(*args, **kwargs)
return True
except IOError as e:
self.logger.error('transfer failed! err: {}'.format(e))
return False
def put_dir(self, source_dir, dest_dir):
target_dir = Path(dest_dir, Path(source_dir).name)
if self.exists(target_dir.__str__()):
self.rm_dir(target_dir.__str__())
local_tar_file = Path(tempfile.gettempdir(),
os.path.basename(source_dir) + '.tar.gz')
if local_tar_file.exists():
os.remove(local_tar_file)
self.__make_targz(source_dir, local_tar_file)
remote_tar_file = Path(dest_dir, local_tar_file.name).__str__()
self.put(local_tar_file, dest_dir)
self.run('python3 -m tarfile -e {} {}'.format(remote_tar_file,
dest_dir))
self.rm(remote_tar_file)
os.remove(local_tar_file)
return True
def __make_targz(self, source_dir, output_filename):
with tarfile.open(output_filename, 'w:gz') as tar:
tar.add(source_dir)
