def distribute_code(mode: CodeDistributionMode, connection: Connection, dst: str):
if mode == CodeDistributionMode.SYNC:
print("Syncing the source folder ..")
src = pathlib.Path().absolute() / '*'
rsync(connection, str(src), dst, exclude=['.git', '__pycache__', 'outputs'])
else:
raise NotImplementedError("Not implemented yet")
def test_rsync_file():
# setup
d = 'test_rsync'
remote_f = 'remote\ file.txt'
local_f = 'local_file.txt'
directory(d)
with cd(d):
rm(remote_f)
append(remote_f, 'Hello world')
# download file
src = '%s/%s' % (d, remote_f)
rsync(
remote_dir=src,
local_dir=local_f,
upload=False
)
assert contains(local_f, 'Hello world', runner=local) is True
# append to and upload file
append(local_f, 'Hello from local', runner=local)
rsync(
remote_dir=src,
local_dir=local_f,
)
# check uploaded file
with cd(d):
assert contains(remote_f, 'Hello from local') is True
# cleanup
rm(local_f, runner=local)
rmdir(d)
def release(c):
server = c.server.connection
wheel_path = glob.glob("dist/*.whl")[0]
wheel_name = os.path.basename(wheel_path)
remote_dir = c.server.remote_dir
secret_key = c.server.secret_key
with server.cd(remote_dir):
server.put(wheel_path,
remote=os.path.join(remote_dir, "backend", wheel_name))
with server.cd("backend"):
server.run("python3 -m virtualenv --clear .venv")
server.run(
f'echo "export CORONASTATS_SECRET_KEY={secret_key}" >> .venv/bin/activate'
)
server.run(
'echo "export FLASK_APP=coronastats:create_app" >> .venv/bin/activate'
)
with server.prefix("source .venv/bin/activate"):
server.run(f"pip3 install {wheel_name}")
server.run(f"rm {wheel_name}")
server.run("rm -rf frontend/*")
server.run("ln app.db frontend/database.sqlite")
rsync(server, "../frontend/public/*", os.path.join(remote_dir, "frontend"))
server.put(
"serverconfig/supervisor.conf",
remote=os.path.join(remote_dir, "supervisor.conf"),
)
server.put("serverconfig/nginx.conf",
remote=os.path.join(remote_dir, "nginx.conf"))
def update_files(c):
print(c)
c.run('mkdir -p ' + WORKING_DIR)
real_path = os.path.realpath(__file__)
dir_path = os.path.dirname(real_path)
rsync(
c,
source=dir_path + "/.",
target=WORKING_DIR,
rsync_opts='--progress',
exclude=[
#'services/env_vars-staging',
'.vscode',
'jenkins_ci/jenkins_home/*',
'jenkins_ci/ssh_config'
'jenkins_ci/jenkins.war'
'.vscode/*',
'fab*',
'.git',
'.gitignore',
'*.pyc',
'.idea',
#'infra/keycloak/keycloak_secrets',
#'services/platform-config'
'__pycache__',
])
def copy_src(c):
source_path = f'./src/{DATASET}'
dest_path = str(SRC_DIR)
c.run(f'mkdir -p {dest_path}')
rsync(c, source_path, dest_path, delete=True,
ssh_opts="-i ~/.ssh/pi_cluster -o StrictHostKeyChecking=no")
def synchronize_code(context: Context):
import os
import sys
# Path /cygdrive/d/cygwin64/bin/rsync
rsync_path = os.environ.get("RSYNC_PATH", None) or "rsync"
ssh_path = os.environ.get(
"SSH_PATH", None
) or "/cygdrive/d/cygwin64/bin/ssh" if sys.platform == "win32" else "ssh"
remote_base_path = "/home/pcdinh/code"
ssh_identity_file_path = convert_to_posix_path(
os.path.normpath("".join(context.ssh_config["identityfile"])))
if sys.platform == "win32":
rsync_cmd = r"{} -pthrvz --exclude='.git/' --exclude='.idea/' --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r --rsh='{} -i {} -p 22 ' {} {}@{}:{}"
context.local(
rsync_cmd.format(
rsync_path,
ssh_path,
# context.ssh_config => dict(hostname, port, user, identityfile)
ssh_identity_file_path,
convert_to_posix_path(get_base_path()),
context.ssh_config["user"],
context.ssh_config["hostname"],
remote_base_path))
else:
rsync(context,
convert_to_posix_path(get_base_path()),
"/home/pcdinh/code",
rsync_opts='--chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r --verbose')
def sync(c):
rsync(c,
'.',
f'/root/{project}',
exclude=[
'.venv', '.git', '/static', '.DS_Store', '.env', '__pycache__',
'*.pyc', '*.log', '*.pid'
])
def deployrsync(c):
""" 部署最新内容到远程服务器
"""
if not isinstance(c, Connection):
raise Exit('Use -H to provide a host!')
pdir = get_static()
transfers.rsync(c, pdir, DEPLOY_DIR, exclude=[])
logger.warn('RSYNC [%s] to [%s]', pdir, DEPLOY_DIR)
def upload():
print("upload")
print("Moving to folder... \n\n")
if ("." not in filename):
rsync(c, lpath, (rpath))
else:
c.put(lpath, rpath)
c.run('ls')
return
def deploytesting(connection):
rsync(connection, ".", TESTING_SSH_FOLDER_PATH, **SERVER_RSYNC_OPTS)
compose = "docker-compose -f docker-compose-testing.yml"
with connection.cd(TESTING_SSH_FOLDER_PATH):
connection.run(f"{compose} build")
connection.run(f"{compose} stop")
connection.run(f"{compose} up -d")
def copy_terraform_to_remote(self):
logging.info('transfer terraform dir to remote')
tf_dir = os.path.abspath(
os.path.join(os.getcwd(), os.path.pardir, os.path.pardir))
source = os.path.join(tf_dir, 'ssn-helm-charts')
remote_dir = '/home/{}/terraform/'.format(self.user_name)
with Console.ssh(self.ip, self.user_name, self.pkey_path) as conn:
conn.run('mkdir -p {}'.format(remote_dir))
rsync(conn, source, remote_dir)
def deploy_client(c):
exclude = ['node_modules/*', 'about.html', 'd3.html']
srcdir = path(os.getcwd(), 'client/')
destdir = path(SRV_PATH, 'client')
rsync(linode, srcdir, destdir, exclude=exclude)
with linode.cd(destdir):
linode.run('npm install --silent')
def push_profile(cx):
"""Push my configuration from local user. Assumes bootstrap install is
done."""
homedir = osp.expandvars('$HOME')
rsync(cx,
f"hotexamples_com/.{{cookiecutter.owner_nickname}}.d",
f"hotexamples_com/",
rsync_opts="-ahi --stats",
)
def deploy(ctx):
with ctx.prefix("source ~/.virtualenvs/inertia/bin/activate"):
with ctx.cd("apps/django-inertia-demo"):
ctx.run("git pull")
ctx.run("pip install -r requirements.txt")
ctx.run("python manage.py migrate")
ctx.run("python populate.py")
ctx.run("python manage.py collectstatic --noinput")
rsync(ctx, "static/", "apps/django-inertia-demo/")
ctx.run("sudo supervisorctl restart inertia")
def upload(c, target=PROJECT_DIR_NAME, clean_build=False):
local_dir_path = os.path.dirname(__file__)
if clean_build:
c.run(f'rm -rf {target}')
rsync(c,
f'{local_dir_path}/',
f'{target}/',
exclude=['.git'],
rsync_opts='-C --filter=":- .gitignore"')
def push_profile(cx):
"""Push my configuration from local user. Assumes bootstrap install is
done."""
homedir = osp.expandvars('$HOME')
rsync(
cx,
f"hotexamples_com/.salotz.d",
f"hotexamples_com/",
rsync_opts="-ahi --stats",
)
def deploy(ctx):
directory_app = "apps/newapp"
app_name = "newapp"
local("python manage.py collectstatic --noinput", echo=True)
local("find . -name '__pycache__' |xargs rm -rf ", echo=True)
rsync(ctx, ".", directory_app, exclude=exclude_dirs)
with ctx.cd(directory_app):
with ctx.prefix(f"source ~/{directory_app}/.env/bin/activate"):
ctx.run("pip install -r requirements.txt")
ctx.run("python manage.py migrate")
ctx.run(f"sudo supervisorctl restart {app_name}")
def pull_project(cx):
# get the directory to push to
target_dir = Path(os.getcwd()).parent
rsync(
cx,
os.getcwd(),
target_dir,
rsync_opts="-ahi --stats --filter=':- .gitignore' --update",
)
def sync(c):
rsync(
c,
'./',
"/srv/django-projects/" + env['project_remote'] + "/",
exclude=("fabfile.py", "*.pyc", ".git*", "*.db", "*.log", "venv",
"uploads", 'media', '*.tar.gz'),
delete=False,
rsync_opts="",
)
# TODO: consider/test instead passing rsync_opts="--no-perms" -- see https://unix.stackexchange.com/questions/65621/rsync-is-changing-my-directory-permissions
c.sudo('chmod -R g+w /srv/django-projects/' + env['project_remote'])
def deploy(ctx):
run("npm install", echo=True)
run("rm -rf market/static/dist/", echo=True)
run("npm run build", echo=True)
run("python manage.py collectstatic --noinput", echo=True)
run("find . -name '__pycache__' |xargs rm -rf ", echo=True)
rsync(ctx, ".", "apps/market", exclude=exclude_dirs)
with ctx.cd("apps/market"):
with ctx.prefix("source ~/.virtualenvs/market/bin/activate"):
ctx.run("pip install -r requirements.txt")
ctx.run("python manage.py migrate")
ctx.run("sudo supervisorctl restart market")
def deploy(ctx):
local("yarn build", echo=True)
local("python manage.py collectstatic --noinput", echo=True)
rsync(ctx,
"static/",
"apps/django-sockpuppet-expo/static/",
exclude=exclude_dirs)
with ctx.cd("apps/django-sockpuppet-expo"):
ctx.run('git pull')
with ctx.prefix("source .env/bin/activate"):
ctx.run('pip3.8 install -r requirements.in')
ctx.run('python3.8 manage.py migrate')
ctx.run("sudo supervisorctl restart expo:*")
def push_project(cx):
# get the directory to push to
target_dir = Path(os.getcwd()).parent
cx.run(f"mkdir -p {target_dir}")
rsync(
cx,
os.getcwd(),
target_dir,
rsync_opts="-ahi --stats --filter=':- .gitignore'",
)
def install_wheels(c):
# keep trailing slash to copy contents of source_path into dest_path
source_path = './wheels/'
dest_path = '/tmp/wheels'
c.run(f'mkdir -p {dest_path}')
rsync(c,
source_path,
dest_path,
delete=True,
ssh_opts="-i ~/.ssh/pi_cluster -o StrictHostKeyChecking=no")
pip_bin = BIN_DIR / 'pip'
c.run(f'{pip_bin} install {dest_path}/*.whl')
def rsync(self, exclude=[], is_windows=False):
""" 部署最新程序到远程服务器
"""
if is_windows:
# 因为 windows 下面的 rsync 不支持 windows 风格的绝对路径，转换成相对路径
pdir = str(self.basedir.relative_to('.').resolve())
else:
pdir = str(self.basedir.resolve())
if not pdir.endswith('/'):
pdir += '/'
deploy_dir = self.get_remote_path()
self.init_remote_dir(deploy_dir)
transfers.rsync(self.conn, pdir, deploy_dir, exclude=exclude)
logger.warn('RSYNC [%s] to [%s]', pdir, deploy_dir)
def pull_project(cx):
"""Pull the project files to the LXD container.
Ignores according to the gitignore file
"""
# get the directory to push to
target_dir = Path(os.getcwd()).parent
rsync(cx,
os.getcwd(),
target_dir,
rsync_opts="-ahi --stats --filter=':- .gitignore' --update",
)
def run(con: Connection,
command: str,
params: List[str],
asynchronous: bool = False) -> Union['Result', 'Promise']:
cfg = RunConfig()
src = pathlib.Path().absolute() / '*'
dst = 'experiment'
rsync(con, str(src), dst, exclude=['.git', '__pycache__', 'outputs'])
command = command + ' ' + ' '.join(params)
command = f"source ~/.bash_profile; cd {dst}; {cfg.python} {command}"
result = con.run(command, asynchronous=asynchronous)
return result
def push_project(cx):
"""Push the project files to the LXD container.
Ignores according to the gitignore file
"""
# get the directory to push to
target_dir = Path(os.getcwd()).parent
cx.run(f"mkdir -p {target_dir}")
rsync(cx,
os.getcwd(),
target_dir,
rsync_opts="-ahi --stats --filter=':- .gitignore'",
)
def deploy_server(c):
# update folder except client/ dir
# symlink should have worked
# npm update
# restart nginx/supervisor whatever
include = ['server.js', 'package.json', 'package-lock.json']
tmpdir = path(os.getcwd(), 'tmp/')
# mv to tmp
os.makedirs(tmpdir, exist_ok=True)
for file in include:
shutil.copy(file, tmpdir)
rsync(linode, tmpdir, SRV_PATH)
# run npm install
with linode.cd(SRV_PATH):
linode.run('npm install --silent')
shutil.rmtree(tmpdir)
def rsync(source, target, exclude=(), delete=False, strict_host_keys=True, rsync_opts="--progress -pthrvz", ssh_opts=''):
# https://fabric-patchwork.readthedocs.io/en/latest/api/transfers.html#patchwork.transfers.rsync
return transfers.rsync(
c=global_context.conn,
source=source,
target=target,
exclude=exclude,
delete=delete,
strict_host_keys=strict_host_keys,
rsync_opts=rsync_opts,
ssh_opts=ssh_opts,
)
def deploy(
c,
withclean=False,
withbackup=False,
):
if withclean:
clean(c)
if withbackup:
backup(c)
os.system("cd {}client && make build-prod".format(LOCAL_PATH_SITE))
rsync(c,
LOCAL_PATH_SITE,
REMOTE_PATH_SITE,
exclude=[
'*.pyc',
'__pycache__',
'.DS_Store',
'.env',
'env',
'.vscode',
'.git',
'node_modules',
'server_media',
'server_static',
'htmlcov',
'.coverage',
'.expo',
'web-build',
'web-report',
])
with c.cd(REMOTE_PATH_SITE):
c.run('docker-compose build')
c.run('docker-compose stop')
c.run('docker-compose up -d')
def rsync_upload(c):
"""
Uploads the project with rsync excluding some files and folders.
"""
# excludes = ["*.pyc", "*.pyo", "*.db", ".DS_Store", ".coverage",
# "local_settings.py", "/static", "/.git", "/.hg"]
excludes = ["*.pyc", "*.pyo", "*.db", ".DS_Store", ".coverage",
"local_settings.py", "/static", "/.git", "/.hg"]
local_dir = os.getcwd() + os.sep
print(c)
print(local_dir)
print(env.proj_path)
cmd = f"rsync -a --exclude \"{' '.join(excludes)}\" {local_dir} {env.user}@{env.hosts[0]}:{env.proj_path}"
print(cmd)
os.system(cmd)
return
return rsync(c, source=local_dir, target=env.proj_path,
exclude=excludes)
def upload(context):
# print("upload: context=%s", context)
seperatorLine()
fabFilePath = fabLoadedPath()
# print("fabFilePath=%s" % fabFilePath)
localProjectRootPath = os.path.join(fabFilePath, "..")
# print("localProjectRootPath=%s" % localProjectRootPath)
# seperatorLine()
# print("Local environment:")
# context.run("uname -a")
# context.run("pwd")
# context.run("ls -lha")
# seperatorLine()
remoteConn = Connection(host=RemoteHost, user=RemoteUser)
# print(remoteConn)
# seperatorLine()
# print("Remote Server:")
# remoteConn.run('uname -a')
# remoteConn.run('pwd')
# print("remote path: %s" % RemotePathRoot)
# remoteConn.run('ls -lha %s' % (RemotePathRoot))
# remoteConn.run('cd %s && pwd && ls -lha' % RemotePathRoot)
# putFileResult = remoteConn.put('fabfile.py', remote=RemotePathRoot)
# print("Uploaded {0.local} to {0.remote}".format(putFileResult))
syncSource = localProjectRootPath
syncTarget = RemotePathRoot
# syncExclude = [".DS_Store", "data/", "processData/", "*.log", "*.pyc", "__pycache__"]
syncExclude = [
".DS_Store", ".git/", ".idea/", "*.pyc", "__pycache__",
"dump.rdb", "debug/", "logs/", "runtime/", "tmp/"]
syncResp = rsync(
remoteConn,
source=syncSource,
target=syncTarget,
# Note: be careful to add `delete`, to void to delete unexpected files
# delete=True,
exclude=syncExclude)
seperatorLine()
# print("Sync lcoal %s to remote %s while exclude %s -> return %s" %
# (syncSource, syncTarget, syncExclude, syncResp))
print("Sync lcoal:\n%s\nto remote:\n%s\nwhile exclude:\n%s\n-> return:\n%s" % (syncSource, syncTarget, syncExclude, syncResp))
def rsync_src(ctx):
rsync(ctx, source='.', target=PROJECT_PATH, exclude=('core/.env', '.vagrant', '.idea', '.git'),
delete=True, strict_host_keys=True, rsync_opts='-rz', ssh_opts = '')
def publish(c):
local('pelican -s publishconf.py')
rsync(c, deploy_path + '/', dest_path, exclude=".DS_Store", delete=True)
def _expect(self, cxn, expected, kwargs=None):
if kwargs is None:
kwargs = {}
rsync(cxn, local, remote, **kwargs)
command = cxn.local.call_args[0][0]
assert expected == command
