def distribute_code(mode: CodeDistributionMode, connection: Connection, dst: str): if mode == CodeDistributionMode.SYNC: print("Syncing the source folder ..") src = pathlib.Path().absolute() / '*' rsync(connection, str(src), dst, exclude=['.git', '__pycache__', 'outputs']) else: raise NotImplementedError("Not implemented yet")
def test_rsync_file(): # setup d = 'test_rsync' remote_f = 'remote\ file.txt' local_f = 'local_file.txt' directory(d) with cd(d): rm(remote_f) append(remote_f, 'Hello world') # download file src = '%s/%s' % (d, remote_f) rsync( remote_dir=src, local_dir=local_f, upload=False ) assert contains(local_f, 'Hello world', runner=local) is True # append to and upload file append(local_f, 'Hello from local', runner=local) rsync( remote_dir=src, local_dir=local_f, ) # check uploaded file with cd(d): assert contains(remote_f, 'Hello from local') is True # cleanup rm(local_f, runner=local) rmdir(d)
def release(c): server = c.server.connection wheel_path = glob.glob("dist/*.whl")[0] wheel_name = os.path.basename(wheel_path) remote_dir = c.server.remote_dir secret_key = c.server.secret_key with server.cd(remote_dir): server.put(wheel_path, remote=os.path.join(remote_dir, "backend", wheel_name)) with server.cd("backend"): server.run("python3 -m virtualenv --clear .venv") server.run( f'echo "export CORONASTATS_SECRET_KEY={secret_key}" >> .venv/bin/activate' ) server.run( 'echo "export FLASK_APP=coronastats:create_app" >> .venv/bin/activate' ) with server.prefix("source .venv/bin/activate"): server.run(f"pip3 install {wheel_name}") server.run(f"rm {wheel_name}") server.run("rm -rf frontend/*") server.run("ln app.db frontend/database.sqlite") rsync(server, "../frontend/public/*", os.path.join(remote_dir, "frontend")) server.put( "serverconfig/supervisor.conf", remote=os.path.join(remote_dir, "supervisor.conf"), ) server.put("serverconfig/nginx.conf", remote=os.path.join(remote_dir, "nginx.conf"))
def update_files(c): print(c) c.run('mkdir -p ' + WORKING_DIR) real_path = os.path.realpath(__file__) dir_path = os.path.dirname(real_path) rsync( c, source=dir_path + "/.", target=WORKING_DIR, rsync_opts='--progress', exclude=[ #'services/env_vars-staging', '.vscode', 'jenkins_ci/jenkins_home/*', 'jenkins_ci/ssh_config' 'jenkins_ci/jenkins.war' '.vscode/*', 'fab*', '.git', '.gitignore', '*.pyc', '.idea', #'infra/keycloak/keycloak_secrets', #'services/platform-config' '__pycache__', ])
def copy_src(c): source_path = f'./src/{DATASET}' dest_path = str(SRC_DIR) c.run(f'mkdir -p {dest_path}') rsync(c, source_path, dest_path, delete=True, ssh_opts="-i ~/.ssh/pi_cluster -o StrictHostKeyChecking=no")
def synchronize_code(context: Context): import os import sys # Path /cygdrive/d/cygwin64/bin/rsync rsync_path = os.environ.get("RSYNC_PATH", None) or "rsync" ssh_path = os.environ.get( "SSH_PATH", None ) or "/cygdrive/d/cygwin64/bin/ssh" if sys.platform == "win32" else "ssh" remote_base_path = "/home/pcdinh/code" ssh_identity_file_path = convert_to_posix_path( os.path.normpath("".join(context.ssh_config["identityfile"]))) if sys.platform == "win32": rsync_cmd = r"{} -pthrvz --exclude='.git/' --exclude='.idea/' --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r --rsh='{} -i {} -p 22 ' {} {}@{}:{}" context.local( rsync_cmd.format( rsync_path, ssh_path, # context.ssh_config => dict(hostname, port, user, identityfile) ssh_identity_file_path, convert_to_posix_path(get_base_path()), context.ssh_config["user"], context.ssh_config["hostname"], remote_base_path)) else: rsync(context, convert_to_posix_path(get_base_path()), "/home/pcdinh/code", rsync_opts='--chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r --verbose')
def sync(c): rsync(c, '.', f'/root/{project}', exclude=[ '.venv', '.git', '/static', '.DS_Store', '.env', '__pycache__', '*.pyc', '*.log', '*.pid' ])
def deployrsync(c): """ 部署最新内容到远程服务器 """ if not isinstance(c, Connection): raise Exit('Use -H to provide a host!') pdir = get_static() transfers.rsync(c, pdir, DEPLOY_DIR, exclude=[]) logger.warn('RSYNC [%s] to [%s]', pdir, DEPLOY_DIR)
def upload(): print("upload") print("Moving to folder... \n\n") if ("." not in filename): rsync(c, lpath, (rpath)) else: c.put(lpath, rpath) c.run('ls') return
def deploytesting(connection): rsync(connection, ".", TESTING_SSH_FOLDER_PATH, **SERVER_RSYNC_OPTS) compose = "docker-compose -f docker-compose-testing.yml" with connection.cd(TESTING_SSH_FOLDER_PATH): connection.run(f"{compose} build") connection.run(f"{compose} stop") connection.run(f"{compose} up -d")
def copy_terraform_to_remote(self): logging.info('transfer terraform dir to remote') tf_dir = os.path.abspath( os.path.join(os.getcwd(), os.path.pardir, os.path.pardir)) source = os.path.join(tf_dir, 'ssn-helm-charts') remote_dir = '/home/{}/terraform/'.format(self.user_name) with Console.ssh(self.ip, self.user_name, self.pkey_path) as conn: conn.run('mkdir -p {}'.format(remote_dir)) rsync(conn, source, remote_dir)
def deploy_client(c): exclude = ['node_modules/*', 'about.html', 'd3.html'] srcdir = path(os.getcwd(), 'client/') destdir = path(SRV_PATH, 'client') rsync(linode, srcdir, destdir, exclude=exclude) with linode.cd(destdir): linode.run('npm install --silent')
def push_profile(cx): """Push my configuration from local user. Assumes bootstrap install is done.""" homedir = osp.expandvars('$HOME') rsync(cx, f"hotexamples_com/.{{cookiecutter.owner_nickname}}.d", f"hotexamples_com/", rsync_opts="-ahi --stats", )
def deploy(ctx): with ctx.prefix("source ~/.virtualenvs/inertia/bin/activate"): with ctx.cd("apps/django-inertia-demo"): ctx.run("git pull") ctx.run("pip install -r requirements.txt") ctx.run("python manage.py migrate") ctx.run("python populate.py") ctx.run("python manage.py collectstatic --noinput") rsync(ctx, "static/", "apps/django-inertia-demo/") ctx.run("sudo supervisorctl restart inertia")
def upload(c, target=PROJECT_DIR_NAME, clean_build=False): local_dir_path = os.path.dirname(__file__) if clean_build: c.run(f'rm -rf {target}') rsync(c, f'{local_dir_path}/', f'{target}/', exclude=['.git'], rsync_opts='-C --filter=":- .gitignore"')
def push_profile(cx): """Push my configuration from local user. Assumes bootstrap install is done.""" homedir = osp.expandvars('$HOME') rsync( cx, f"hotexamples_com/.salotz.d", f"hotexamples_com/", rsync_opts="-ahi --stats", )
def deploy(ctx): directory_app = "apps/newapp" app_name = "newapp" local("python manage.py collectstatic --noinput", echo=True) local("find . -name '__pycache__' |xargs rm -rf ", echo=True) rsync(ctx, ".", directory_app, exclude=exclude_dirs) with ctx.cd(directory_app): with ctx.prefix(f"source ~/{directory_app}/.env/bin/activate"): ctx.run("pip install -r requirements.txt") ctx.run("python manage.py migrate") ctx.run(f"sudo supervisorctl restart {app_name}")
def pull_project(cx): # get the directory to push to target_dir = Path(os.getcwd()).parent rsync( cx, os.getcwd(), target_dir, rsync_opts="-ahi --stats --filter=':- .gitignore' --update", )
def sync(c): rsync( c, './', "/srv/django-projects/" + env['project_remote'] + "/", exclude=("fabfile.py", "*.pyc", ".git*", "*.db", "*.log", "venv", "uploads", 'media', '*.tar.gz'), delete=False, rsync_opts="", ) # TODO: consider/test instead passing rsync_opts="--no-perms" -- see https://unix.stackexchange.com/questions/65621/rsync-is-changing-my-directory-permissions c.sudo('chmod -R g+w /srv/django-projects/' + env['project_remote'])
def deploy(ctx): run("npm install", echo=True) run("rm -rf market/static/dist/", echo=True) run("npm run build", echo=True) run("python manage.py collectstatic --noinput", echo=True) run("find . -name '__pycache__' |xargs rm -rf ", echo=True) rsync(ctx, ".", "apps/market", exclude=exclude_dirs) with ctx.cd("apps/market"): with ctx.prefix("source ~/.virtualenvs/market/bin/activate"): ctx.run("pip install -r requirements.txt") ctx.run("python manage.py migrate") ctx.run("sudo supervisorctl restart market")
def deploy(ctx): local("yarn build", echo=True) local("python manage.py collectstatic --noinput", echo=True) rsync(ctx, "static/", "apps/django-sockpuppet-expo/static/", exclude=exclude_dirs) with ctx.cd("apps/django-sockpuppet-expo"): ctx.run('git pull') with ctx.prefix("source .env/bin/activate"): ctx.run('pip3.8 install -r requirements.in') ctx.run('python3.8 manage.py migrate') ctx.run("sudo supervisorctl restart expo:*")
def push_project(cx): # get the directory to push to target_dir = Path(os.getcwd()).parent cx.run(f"mkdir -p {target_dir}") rsync( cx, os.getcwd(), target_dir, rsync_opts="-ahi --stats --filter=':- .gitignore'", )
def install_wheels(c): # keep trailing slash to copy contents of source_path into dest_path source_path = './wheels/' dest_path = '/tmp/wheels' c.run(f'mkdir -p {dest_path}') rsync(c, source_path, dest_path, delete=True, ssh_opts="-i ~/.ssh/pi_cluster -o StrictHostKeyChecking=no") pip_bin = BIN_DIR / 'pip' c.run(f'{pip_bin} install {dest_path}/*.whl')
def rsync(self, exclude=[], is_windows=False): """ 部署最新程序到远程服务器 """ if is_windows: # 因为 windows 下面的 rsync 不支持 windows 风格的绝对路径,转换成相对路径 pdir = str(self.basedir.relative_to('.').resolve()) else: pdir = str(self.basedir.resolve()) if not pdir.endswith('/'): pdir += '/' deploy_dir = self.get_remote_path() self.init_remote_dir(deploy_dir) transfers.rsync(self.conn, pdir, deploy_dir, exclude=exclude) logger.warn('RSYNC [%s] to [%s]', pdir, deploy_dir)
def pull_project(cx): """Pull the project files to the LXD container. Ignores according to the gitignore file """ # get the directory to push to target_dir = Path(os.getcwd()).parent rsync(cx, os.getcwd(), target_dir, rsync_opts="-ahi --stats --filter=':- .gitignore' --update", )
def run(con: Connection, command: str, params: List[str], asynchronous: bool = False) -> Union['Result', 'Promise']: cfg = RunConfig() src = pathlib.Path().absolute() / '*' dst = 'experiment' rsync(con, str(src), dst, exclude=['.git', '__pycache__', 'outputs']) command = command + ' ' + ' '.join(params) command = f"source ~/.bash_profile; cd {dst}; {cfg.python} {command}" result = con.run(command, asynchronous=asynchronous) return result
def push_project(cx): """Push the project files to the LXD container. Ignores according to the gitignore file """ # get the directory to push to target_dir = Path(os.getcwd()).parent cx.run(f"mkdir -p {target_dir}") rsync(cx, os.getcwd(), target_dir, rsync_opts="-ahi --stats --filter=':- .gitignore'", )
def deploy_server(c): # update folder except client/ dir # symlink should have worked # npm update # restart nginx/supervisor whatever include = ['server.js', 'package.json', 'package-lock.json'] tmpdir = path(os.getcwd(), 'tmp/') # mv to tmp os.makedirs(tmpdir, exist_ok=True) for file in include: shutil.copy(file, tmpdir) rsync(linode, tmpdir, SRV_PATH) # run npm install with linode.cd(SRV_PATH): linode.run('npm install --silent') shutil.rmtree(tmpdir)
def rsync(source, target, exclude=(), delete=False, strict_host_keys=True, rsync_opts="--progress -pthrvz", ssh_opts=''): # https://fabric-patchwork.readthedocs.io/en/latest/api/transfers.html#patchwork.transfers.rsync return transfers.rsync( c=global_context.conn, source=source, target=target, exclude=exclude, delete=delete, strict_host_keys=strict_host_keys, rsync_opts=rsync_opts, ssh_opts=ssh_opts, )
def deploy( c, withclean=False, withbackup=False, ): if withclean: clean(c) if withbackup: backup(c) os.system("cd {}client && make build-prod".format(LOCAL_PATH_SITE)) rsync(c, LOCAL_PATH_SITE, REMOTE_PATH_SITE, exclude=[ '*.pyc', '__pycache__', '.DS_Store', '.env', 'env', '.vscode', '.git', 'node_modules', 'server_media', 'server_static', 'htmlcov', '.coverage', '.expo', 'web-build', 'web-report', ]) with c.cd(REMOTE_PATH_SITE): c.run('docker-compose build') c.run('docker-compose stop') c.run('docker-compose up -d')
def rsync_upload(c): """ Uploads the project with rsync excluding some files and folders. """ # excludes = ["*.pyc", "*.pyo", "*.db", ".DS_Store", ".coverage", # "local_settings.py", "/static", "/.git", "/.hg"] excludes = ["*.pyc", "*.pyo", "*.db", ".DS_Store", ".coverage", "local_settings.py", "/static", "/.git", "/.hg"] local_dir = os.getcwd() + os.sep print(c) print(local_dir) print(env.proj_path) cmd = f"rsync -a --exclude \"{' '.join(excludes)}\" {local_dir} {env.user}@{env.hosts[0]}:{env.proj_path}" print(cmd) os.system(cmd) return return rsync(c, source=local_dir, target=env.proj_path, exclude=excludes)
def upload(context): # print("upload: context=%s", context) seperatorLine() fabFilePath = fabLoadedPath() # print("fabFilePath=%s" % fabFilePath) localProjectRootPath = os.path.join(fabFilePath, "..") # print("localProjectRootPath=%s" % localProjectRootPath) # seperatorLine() # print("Local environment:") # context.run("uname -a") # context.run("pwd") # context.run("ls -lha") # seperatorLine() remoteConn = Connection(host=RemoteHost, user=RemoteUser) # print(remoteConn) # seperatorLine() # print("Remote Server:") # remoteConn.run('uname -a') # remoteConn.run('pwd') # print("remote path: %s" % RemotePathRoot) # remoteConn.run('ls -lha %s' % (RemotePathRoot)) # remoteConn.run('cd %s && pwd && ls -lha' % RemotePathRoot) # putFileResult = remoteConn.put('fabfile.py', remote=RemotePathRoot) # print("Uploaded {0.local} to {0.remote}".format(putFileResult)) syncSource = localProjectRootPath syncTarget = RemotePathRoot # syncExclude = [".DS_Store", "data/", "processData/", "*.log", "*.pyc", "__pycache__"] syncExclude = [ ".DS_Store", ".git/", ".idea/", "*.pyc", "__pycache__", "dump.rdb", "debug/", "logs/", "runtime/", "tmp/"] syncResp = rsync( remoteConn, source=syncSource, target=syncTarget, # Note: be careful to add `delete`, to void to delete unexpected files # delete=True, exclude=syncExclude) seperatorLine() # print("Sync lcoal %s to remote %s while exclude %s -> return %s" % # (syncSource, syncTarget, syncExclude, syncResp)) print("Sync lcoal:\n%s\nto remote:\n%s\nwhile exclude:\n%s\n-> return:\n%s" % (syncSource, syncTarget, syncExclude, syncResp))
def rsync_src(ctx): rsync(ctx, source='.', target=PROJECT_PATH, exclude=('core/.env', '.vagrant', '.idea', '.git'), delete=True, strict_host_keys=True, rsync_opts='-rz', ssh_opts = '')
def publish(c): local('pelican -s publishconf.py') rsync(c, deploy_path + '/', dest_path, exclude=".DS_Store", delete=True)
def _expect(self, cxn, expected, kwargs=None): if kwargs is None: kwargs = {} rsync(cxn, local, remote, **kwargs) command = cxn.local.call_args[0][0] assert expected == command