def deployment(remote_machine, code = SERVER_SCRIPT):
with remote_machine.tempdir() as tmp:
copy(RPYC_ROOT, tmp)
delete(tmp // ".pyc", tmp // "*/.pyc")
with remote_machine.cwd(tmp):
with remote_machine.env(PYTHONPATH = remote_machine.env.get("PYTHONPATH", "") + ":%s" % (tmp,)):
proc = (remote_machine.python << code).popen()
line = ""
try:
line = proc.stdout.readline()
remote_port = int(line.strip())
except Exception:
try:
proc.kill()
except Exception:
pass
stdout, stderr = proc.communicate()
raise ProcessExecutionError(proc.argv, proc.returncode, line + stdout, stderr)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("localhost", 0))
local_port = s.getsockname()[1]
s.close()
with remote_machine.tunnel(local_port, remote_port) as tun:
try:
yield DeployedServer(local_port)
finally:
try:
proc.kill()
except Exception:
pass
def main(self):
delete(local.cwd // "*.egg-info", "build", "dist")
if self.upload:
local.python("setup.py", "register")
local.python("setup.py", "sdist", "--formats=zip,gztar", "bdist_wininst", "--plat-name=win32",
"upload" if self.upload else None)
delete(local.cwd // "*.egg-info", "build")
def main(self, src, dst):
if local.path(dst).exists():
if not self.overwrite:
logger.debug("Oh no! That's terrible")
raise ValueError("Destination already exists")
else:
delete(dst)
logger.debug("I'm going to copy %s to %s", src, dst)
copy(src, dst)
logger.debug("Great success")
def main(self, src, dst):
if local.path(dst).exists():
if not self.overwrite:
logger.debug("Oh no! That's terrible")
raise ValueError("Destination already exists")
else:
delete(dst)
logger.debug("I'm going to copy %s to %s", src, dst)
copy(src, dst)
logger.debug("Great success")
def deployment(remote_machine):
"""Sets up a temporary, short-lived RPyC deployment on the given remote machine.
A deployment:
1. Creates a temporary directory on the remote machine, and copies RPyC's code
from the local machine to the remote temporary directory.
2. Starts an RPyC server on the remote machine, binding to an arbitrary TCP port,
allowing only in-bound connections (connections from the same machine).
The server reports the chosen port over ``stdout``.
3. An SSH tunnel is created from a local port on the local host, to the remote
machine's chosen TCP port. This tunnel is authenticated and encrypted.
4. The deployment returns a :class:`DeployedServer <rpyc.utils.zerodeploy.DeployedServer>`
object, which can be used to connect to the newly-spawned server.
5. When the deployment context is exited, the SSH tunnel is torn down, the remote
server is terminated and the temporary directory deleted.
:param remote_machine: a ``plumbum.SshMachine`` instance, representing an SSH
connection to the desired remote machine
"""
RPYC_ROOT = local.path(rpyc.__file__).dirname
with remote_machine.tempdir() as tmp:
copy(RPYC_ROOT, tmp)
delete(tmp // ".pyc", tmp // "*/.pyc")
with remote_machine.cwd(tmp):
with remote_machine.env(PYTHONPATH = remote_machine.env.get("PYTHONPATH", "") + ":%s" % (tmp,)):
proc = (remote_machine.python << SERVER_SCRIPT).popen()
line = ""
try:
line = proc.stdout.readline()
remote_port = int(line.strip())
except Exception:
try:
proc.kill()
except Exception:
pass
stdout, stderr = proc.communicate()
raise ProcessExecutionError(proc.argv, proc.returncode, line + stdout, stderr)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("localhost", 0))
local_port = s.getsockname()[1]
s.close()
with remote_machine.tunnel(local_port, remote_port) as tun:
try:
yield DeployedServer(local_port)
finally:
try:
proc.kill()
except Exception:
pass
def main(self):
delete(local.cwd // "*.egg-info", "build", "dist")
if self.upload:
local.python("setup.py", "register")
local.python("setup.py", "sdist", "--formats=zip,gztar",
"upload" if self.upload else "")
local.python("setup.py", "bdist_wininst", "--plat-name=win32",
"upload" if self.upload else "")
delete(local.cwd // "*.egg-info", "build")
def test_copy_move_delete(self):
from plumbum.cmd import touch
with local.tempdir() as dir:
(dir / "orog").mkdir()
(dir / "orog" / "rec").mkdir()
for i in range(20):
touch(dir / "orog" / ("f%d.txt" % (i, )))
for i in range(20, 40):
touch(dir / "orog" / "rec" / ("f%d.txt" % (i, )))
move(dir / "orog", dir / "orig")
s1 = sorted(f.basename for f in (dir / "orig").walk())
copy(dir / "orig", dir / "dup")
s2 = sorted(f.basename for f in (dir / "dup").walk())
self.assertEqual(s1, s2)
with SshMachine("localhost") as rem:
with rem.tempdir() as dir2:
copy(dir / "orig", dir2)
s3 = sorted(f.basename for f in (dir2 / "orig").walk())
self.assertEqual(s1, s3)
copy(dir2 / "orig", dir2 / "dup")
s4 = sorted(f.basename for f in (dir2 / "dup").walk())
self.assertEqual(s1, s4)
copy(dir2 / "dup", dir / "dup2")
s5 = sorted(f.basename for f in (dir / "dup2").walk())
self.assertEqual(s1, s5)
with SshMachine("localhost") as rem2:
with rem2.tempdir() as dir3:
copy(dir2 / "dup", dir3)
s6 = sorted(f.basename
for f in (dir3 / "dup").walk())
self.assertEqual(s1, s6)
move(dir3 / "dup", dir / "superdup")
self.assertFalse((dir3 / "dup").exists())
s7 = sorted(f.basename
for f in (dir / "superdup").walk())
self.assertEqual(s1, s7)
# test rm
delete(dir)
def test_copy_move_delete(self):
from plumbum.cmd import touch
with local.tempdir() as dir:
(dir / "orog").mkdir()
(dir / "orog" / "rec").mkdir()
for i in range(20):
touch(dir / "orog" / ("f%d.txt" % (i,)))
for i in range(20,40):
touch(dir / "orog" / "rec" / ("f%d.txt" % (i,)))
move(dir / "orog", dir / "orig")
s1 = sorted(f.basename for f in (dir / "orig").walk())
copy(dir / "orig", dir / "dup")
s2 = sorted(f.basename for f in (dir / "dup").walk())
self.assertEqual(s1, s2)
with SshMachine("localhost") as rem:
with rem.tempdir() as dir2:
copy(dir / "orig", dir2)
s3 = sorted(f.basename for f in (dir2 / "orig").walk())
self.assertEqual(s1, s3)
copy(dir2 / "orig", dir2 / "dup")
s4 = sorted(f.basename for f in (dir2 / "dup").walk())
self.assertEqual(s1, s4)
copy(dir2 / "dup", dir / "dup2")
s5 = sorted(f.basename for f in (dir / "dup2").walk())
self.assertEqual(s1, s5)
with SshMachine("localhost") as rem2:
with rem2.tempdir() as dir3:
copy(dir2 / "dup", dir3)
s6 = sorted(f.basename for f in (dir3 / "dup").walk())
self.assertEqual(s1, s6)
move(dir3 / "dup", dir / "superdup")
self.assertFalse((dir3 / "dup").exists())
s7 = sorted(f.basename for f in (dir / "superdup").walk())
self.assertEqual(s1, s7)
# test rm
delete(dir)
def main(self):
delete("build", "dist", "MANIFEST", local.cwd // "*.egg-info")
# generate zip, tar.gz, and win32 installer
if self.publish:
print("registering...")
local.python("setup.py", "register")
print("uploading zip and tar.gz")
local.python("setup.py", "sdist", "--formats=zip,gztar", "upload")
print("uploading win installer")
local.python("setup.py", "bdist_wininst", "--plat-name=win32", "upload")
# upload to sourceforge
print("uploading to sourceforge")
dst = "gangesmaster,[email protected]:/home/frs/project/r/rp/rpyc/main/%s/" % (version_string,)
local["rsync"]("-rv", "dist/", dst)
else:
local.python("setup.py", "sdist", "--formats=zip,gztar")
local.python("setup.py", "bdist_wininst", "--plat-name=win32")
delete("build", local.cwd // "*.egg-info")
print("Built", [f.basename for f in local.cwd / "dist"])
def main(self):
delete("build", "dist", "MANIFEST", local.cwd // "*.egg-info")
# generate zip, tar.gz, and win32 installer
if self.publish:
print("registering...")
local.python("setup.py", "register")
print("uploading zip and tar.gz")
local.python("setup.py", "sdist", "--formats=zip,gztar", "upload")
print("uploading win installer")
local.python("setup.py", "bdist_wininst", "--plat-name=win32",
"upload")
# upload to sourceforge
print("uploading to sourceforge")
dst = "gangesmaster,[email protected]:/home/frs/project/r/rp/rpyc/main/%s/" % (
version_string, )
local["rsync"]("-rv", "dist/", dst)
else:
local.python("setup.py", "sdist", "--formats=zip,gztar")
local.python("setup.py", "bdist_wininst", "--plat-name=win32")
delete("build", local.cwd // "*.egg-info")
print("Built", [f.basename for f in local.cwd / "dist"])
def deployment(remote_machine, code=SERVER_SCRIPT):
with remote_machine.tempdir() as tmp:
copy(RPYC_ROOT, tmp)
delete(tmp // ".pyc", tmp // "*/.pyc")
with remote_machine.cwd(tmp):
with remote_machine.env(
PYTHONPATH=remote_machine.env.get("PYTHONPATH", "") +
":%s" % (tmp, )):
proc = (remote_machine.python << code).popen()
line = ""
try:
line = proc.stdout.readline()
remote_port = int(line.strip())
except Exception:
try:
proc.kill()
except Exception:
pass
stdout, stderr = proc.communicate()
raise ProcessExecutionError(proc.argv, proc.returncode,
line + stdout, stderr)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("localhost", 0))
local_port = s.getsockname()[1]
s.close()
with remote_machine.tunnel(local_port, remote_port) as tun:
try:
yield DeployedServer(local_port)
finally:
try:
proc.kill()
except Exception:
pass
def create_cache(output_dir, pkg_repos, bundle_repos, packages, debug, args_as_specfile, force_download_core, offline, **localvars):
core_file = download_araqne_core(force_download_core, offline)
if sys.platform == "cygwin" or sys.platform == "win32":
java = local['java.exe']
nul = 'NUL'
else:
java = local['java']
nul = '/dev/null'
java_f = None
timestamps = None
buildinfos = None
try:
java_f = (java['-Daraqne.core.allow_default_password=true', '-Daraqne.telnet.port=7204', '-jar', core_file] > nul).popen()
if java_f == None:
sys.exit(1)
try:
time.sleep(2)
tn = Telnet('localhost', 7204)
login(tn)
pkg_repo_cnt = 0
for repo in pkg_repos:
repo_name = 'pkg_repo_' + str(pkg_repo_cnt)
pkg_repo_cnt += 1
execute(tn, 'pkg.addRepository {0} {1}'.format(repo_name, repo))
bundle_repo_cnt = 0
for i, repo in enumerate(bundle_repos):
repo_name = 'bundle_repo_' + str(bundle_repo_cnt)
bundle_repo_cnt += 1
execute(tn, 'bundle.addRepository {0} {1} {2}'.format(repo_name, repo, 1000 + len(bundle_repos) - i))
for pkg_spec in [_.split(':') for _ in packages]:
result = execute(tn, 'pkg.install {0} {1}'.format(pkg_spec[0], pkg_spec[1]))
print result[2]
sys.stdout.flush()
time.sleep(1)
tn.close()
print 'done.'
except:
traceback.print_exc()
finally:
if java_f is not None:
if debug:
print 'DEBUG MODE: process is continuously running on PID', java_f.proc.pid, ', port 7204'
print 'You should manually shutdown the instance.'
else:
java_f.kill()
java_f.wait()
trash = glob.glob('cache/*/version0.0/bundle.jar-*')
for p in trash:
delete(local.path(p))
jars = glob.glob('cache/*/version0.0/bundle.jar')
jars = [(int(re.search(r'bundle([0-9]*)', x).group(1)), x) for x in jars if re.search(r'bundle([0-9]*)', x)]
jars = sorted(jars, key=lambda x: x[0])
bundle_infos = []
for i, (s, v, t, c, ct, r, desc, br) in ((i, read_jar(f)) for (i, f) in jars):
bi = {}
bi['bundle_id'] = i
bi['commit_id'] = c if c else None
bi['commit_time'] = strftime('%Y-%m-%d %H:%M:%S', strptime(ct[:-4], '%d.%m.%Y @ %H\\:%M\\:%S')) if ct else None
bi['remote'] = r.replace('\\', '') if r else None
bi['symbolic_name'] = s
bi['version'] = v
bi['branch'] = br
bi['desc'] = desc
bi['timestamp'] = strftime('%Y-%m-%d %H:%M:%S', time.localtime(float(t)/1000)) if t else None
bundle_infos.append(bi)
e = json.JSONEncoder()
open('bundle_infos.json', 'w').writelines(
[e.encode(entry) + '\n' for entry in bundle_infos])
except:
traceback.print_exc()
from plumbum import local from plumbum.cmd import wc, ls, echo, grep from plumbum.utils import copy, delete delete(local.cwd // "*/*.pyc")