def __init__(self, remote_machine, server_class = "ThreadedServer"):
self.proc = None
self.tun = None
self._tmpdir_ctx = None
rpyc_root = local.path(rpyc.__file__).dirname
self._tmpdir_ctx = remote_machine.tempdir()
tmp = self._tmpdir_ctx.__enter__()
copy(rpyc_root, tmp)
script = (tmp / "deployed-rpyc.py")
script.write(SERVER_SCRIPT.replace("$SERVER$", server_class))
self.proc = remote_machine.python.popen(script, new_session = True)
line = ""
try:
line = self.proc.stdout.readline()
remote_port = int(line.strip())
except Exception:
try:
self.proc.terminate()
except Exception:
pass
stdout, stderr = self.proc.communicate()
raise ProcessExecutionError(self.proc.argv, self.proc.returncode, line + stdout, stderr)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("localhost", 0))
self.local_port = s.getsockname()[1]
s.close()
self.tun = remote_machine.tunnel(self.local_port, remote_port)
def makeHTMLdir(fileName,
stacheDir = 'VPsupportScripts',
GLOWPATH = '/Users/jonschull-MBPR/glowscript',
scriptNames ="""/lib/jquery/IDE/jquery.min.js
/lib/jquery/IDE/jquery-ui.custom.min.js
/package/glow.2.7.min.js
/package/RSrun.2.7.min.js
/css/redmond/jquery-ui.custom.css
/css/ide.css""".split() ):
"""create a stacheDir if necessary.
create HTMLdir
create supportScripts folder
fill it with scriptNames from stacheDir
pytest
"""
verbose=False
HTMLdir = fileName.replace('.py','')
if not os.path.exists(stacheDir):
makeSupportScriptStache(stacheDir, GLOWPATH, scriptNames)
prepareHTMLdir(HTMLdir)
#print(f'copy({fileName}, {HTMLdir})')
copy(fileName, HTMLdir)
destDir = HTMLdir + '/' + 'supportScripts'
delete(destDir)
if verbose: print('stacheDir', stacheDir)
#print('destDir', destDir)
copy(stacheDir,destDir)
if verbose: print('makeHTMLdir\t', destDir, 'created and filled')
return {'HTMLdir': HTMLdir, 'destDir': destDir}
def main(self):
represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
yaml.add_representer(OrderedDict, represent_dict_order)
make_extra()
with open_db() as (url, machine,
dbpath), local.tempdir() as tmpdir:
# expand caselist as it's usually a text file
yml = read_project_yml()
pipelines = yml['pipelines']
for pipeline in yml['pipelines']:
pipeline['paths']['caselist'] = readCaselist(pipeline['paths'][
'caselist'])
yml['pipelines'] = pipelines
tmpyml = tmpdir / 'pnldash.yml'
with open(tmpyml, 'w') as f:
yaml.dump(yml, f, default_flow_style=False)
if not (tmpyml).exists():
raise Exception('Error: could not write out temporary yaml file: {}'.format(tmpyml))
destdir = dbpath / (local.cwd.replace('/', '---')[3:])
log.info('Copy files to central database...')
copy(PNLDASH_FILES, destdir)
copy(tmpyml, destdir)
print("Copied")
print('\n'.join(PNLDASH_FILES))
print(tmpyml)
print('to {}'.format(url))
def _copy_master_image(self, node, current_path):
if not node.master_image_filepath: return
local_img = os.path.join(current_path,'master_image.' + node.master_image_filepath.split(os.sep)[-1])
remote_img = os.path.join(settings.remote_files, node.master_image_filepath[6:])
if os.path.exists(local_img): return
try:
copy(remote.path(remote_img), local.path(local_img))
except Exception, e:
logging.warning(e)
def _copy_photo(self, node, current_path):
if not node.img_path: return
local_img = os.path.join(current_path, node.img_path)
remote_img = os.path.join(settings.remote_files, "photo", node.img_path)
if os.path.exists(local_img): return
try:
copy(remote.path(remote_img), local.path(local_img))
except Exception, e:
logging.warning(e)
def main(self):
rmd = local.path(__file__).dirname / "../pnldashboard.Rmd"
with open_db() as (url, machine, dbpath):
log.info("Make directory at '{}'".format(url))
dbpath.mkdir()
log.info("Copy '{}'".format(rmd))
copy(rmd, dbpath)
log.info("Made database at '{}'".format(url))
def copy_owner(self, src, dst, mode=None, user=None, group=None):
""" copy with sudo """
tmpname = "/tmp/{0}".format(next(tempfile._get_candidate_names()))
copy(src, self.env.host.path(tmpname))
with self.env.sudo():
self.env.command("mv", tmpname, dst)
# if sudo to a specific user, fix ownership
owner = self.env._sudouser or user
self._fixattrs(dst, mode, owner, group)
def main(self):
Rcmd = "library('rmarkdown'); render('pnldashboard.Rmd')"
with open_db() as (url, machine, dbpath):
log.info("Make dashboard for database at '{}'".format(url))
with machine.cwd(dbpath):
_ = machine['R']('-e', Rcmd)
copy(dbpath / 'pnldashboard.html', local.cwd)
log.info("Made 'pnldashboard.html'")
def main(self, src, dst):
if local.path(dst).exists():
if not self.overwrite:
logger.debug("Oh no! That's terrible")
raise ValueError("Destination already exists")
else:
delete(dst)
logger.debug("I'm going to copy %s to %s", src, dst)
copy(src, dst)
logger.debug("Great success")
def test_copy_move_delete(self):
from plumbum.cmd import touch
with local.tempdir() as dir:
(dir / "orog").mkdir()
(dir / "orog" / "rec").mkdir()
for i in range(20):
touch(dir / "orog" / ("f%d.txt" % (i, )))
for i in range(20, 40):
touch(dir / "orog" / "rec" / ("f%d.txt" % (i, )))
move(dir / "orog", dir / "orig")
s1 = sorted(f.basename for f in (dir / "orig").walk())
copy(dir / "orig", dir / "dup")
s2 = sorted(f.basename for f in (dir / "dup").walk())
self.assertEqual(s1, s2)
with SshMachine("localhost") as rem:
with rem.tempdir() as dir2:
copy(dir / "orig", dir2)
s3 = sorted(f.basename for f in (dir2 / "orig").walk())
self.assertEqual(s1, s3)
copy(dir2 / "orig", dir2 / "dup")
s4 = sorted(f.basename for f in (dir2 / "dup").walk())
self.assertEqual(s1, s4)
copy(dir2 / "dup", dir / "dup2")
s5 = sorted(f.basename for f in (dir / "dup2").walk())
self.assertEqual(s1, s5)
with SshMachine("localhost") as rem2:
with rem2.tempdir() as dir3:
copy(dir2 / "dup", dir3)
s6 = sorted(f.basename
for f in (dir3 / "dup").walk())
self.assertEqual(s1, s6)
move(dir3 / "dup", dir / "superdup")
self.assertFalse((dir3 / "dup").exists())
s7 = sorted(f.basename
for f in (dir / "superdup").walk())
self.assertEqual(s1, s7)
# test rm
delete(dir)
def test_copy_move_delete(self):
from plumbum.cmd import touch
with local.tempdir() as dir:
(dir / "orog").mkdir()
(dir / "orog" / "rec").mkdir()
for i in range(20):
touch(dir / "orog" / ("f%d.txt" % (i, )))
for i in range(20, 40):
touch(dir / "orog" / "rec" / ("f%d.txt" % (i, )))
move(dir / "orog", dir / "orig")
s1 = sorted(f.name for f in (dir / "orig").walk())
copy(dir / "orig", dir / "dup")
s2 = sorted(f.name for f in (dir / "dup").walk())
assert s1 == s2
with SshMachine("localhost") as rem:
with rem.tempdir() as dir2:
copy(dir / "orig", dir2)
s3 = sorted(f.name for f in (dir2 / "orig").walk())
assert s1 == s3
copy(dir2 / "orig", dir2 / "dup")
s4 = sorted(f.name for f in (dir2 / "dup").walk())
assert s1 == s4
copy(dir2 / "dup", dir / "dup2")
s5 = sorted(f.name for f in (dir / "dup2").walk())
assert s1 == s5
with SshMachine("localhost") as rem2:
with rem2.tempdir() as dir3:
copy(dir2 / "dup", dir3)
s6 = sorted(f.name for f in (dir3 / "dup").walk())
assert s1 == s6
move(dir3 / "dup", dir / "superdup")
assert not (dir3 / "dup").exists()
s7 = sorted(f.name
for f in (dir / "superdup").walk())
assert s1 == s7
# test rm
delete(dir)
def test_copy_move_delete(self):
from plumbum.cmd import touch
with local.tempdir() as dir:
(dir / "orog").mkdir()
(dir / "orog" / "rec").mkdir()
for i in range(20):
touch(dir / "orog" / ("f%d.txt" % (i,)))
for i in range(20,40):
touch(dir / "orog" / "rec" / ("f%d.txt" % (i,)))
move(dir / "orog", dir / "orig")
s1 = sorted(f.basename for f in (dir / "orig").walk())
copy(dir / "orig", dir / "dup")
s2 = sorted(f.basename for f in (dir / "dup").walk())
self.assertEqual(s1, s2)
with SshMachine("localhost") as rem:
with rem.tempdir() as dir2:
copy(dir / "orig", dir2)
s3 = sorted(f.basename for f in (dir2 / "orig").walk())
self.assertEqual(s1, s3)
copy(dir2 / "orig", dir2 / "dup")
s4 = sorted(f.basename for f in (dir2 / "dup").walk())
self.assertEqual(s1, s4)
copy(dir2 / "dup", dir / "dup2")
s5 = sorted(f.basename for f in (dir / "dup2").walk())
self.assertEqual(s1, s5)
with SshMachine("localhost") as rem2:
with rem2.tempdir() as dir3:
copy(dir2 / "dup", dir3)
s6 = sorted(f.basename for f in (dir3 / "dup").walk())
self.assertEqual(s1, s6)
move(dir3 / "dup", dir / "superdup")
self.assertFalse((dir3 / "dup").exists())
s7 = sorted(f.basename for f in (dir / "superdup").walk())
self.assertEqual(s1, s7)
# test rm
delete(dir)
def test_copy_move_delete(self):
from plumbum.cmd import touch
with local.tempdir() as dir:
(dir / "orog").mkdir()
(dir / "orog" / "rec").mkdir()
for i in range(20):
touch(dir / "orog" / ("f%d.txt" % (i,)))
for i in range(20,40):
touch(dir / "orog" / "rec" / ("f%d.txt" % (i,)))
move(dir / "orog", dir / "orig")
s1 = sorted(f.name for f in (dir / "orig").walk())
copy(dir / "orig", dir / "dup")
s2 = sorted(f.name for f in (dir / "dup").walk())
assert s1 == s2
with SshMachine("localhost") as rem:
with rem.tempdir() as dir2:
copy(dir / "orig", dir2)
s3 = sorted(f.name for f in (dir2 / "orig").walk())
assert s1 == s3
copy(dir2 / "orig", dir2 / "dup")
s4 = sorted(f.name for f in (dir2 / "dup").walk())
assert s1 == s4
copy(dir2 / "dup", dir / "dup2")
s5 = sorted(f.name for f in (dir / "dup2").walk())
assert s1 == s5
with SshMachine("localhost") as rem2:
with rem2.tempdir() as dir3:
copy(dir2 / "dup", dir3)
s6 = sorted(f.name for f in (dir3 / "dup").walk())
assert s1 == s6
move(dir3 / "dup", dir / "superdup")
assert not (dir3 / "dup").exists()
s7 = sorted(f.name for f in (dir / "superdup").walk())
assert s1 == s7
# test rm
delete(dir)
def testBuild(target='lyte.py', fileNames=['lyte.py', 'pyonly.py']):
local.cwd.chdir(initialDir)
global testDirName, fileName
testDirName = target.replace('.py', '') + timestamp
print(f'\n\nTESTBUILD: making {testDirName}')
mkdir(testDirName)
for fileName in fileNames:
print(f'TESTBUILD: copying {fileName} -> {testDirName}/{fileName}')
copy(fileName, f'{testDirName}/{fileName}')
local.cwd.chdir(testDirName)
print(pwd())
print(f'TESTBUILD: running ipython3 {testDirName}/{target}\n')
ipython3 = local['ipython3']
ipython3(target)
def copy(self, src, dst, mode=None, user=None, group=None):
dst = self.env.host.path(dst)
copy(src, dst)
self._fixattrs(dst, mode, user, group)
return self
def _prepare_post(self):
plumbum_utils.copy(self.postfilePath, self.remote.cwd)
self._battleCry.clarify(['-T', self.mimeType, '-p',
self.postfilePath.basename])
def copy(self, src, dst, mode=None, user=None, group=None):
dst = self.env.host.path(dst)
copy(src, dst)
self._fixattrs(dst, mode, user, group)
return self
def create_project(self):
plumbum_utils.copy(self.SKELETON_PROJECT, self.inputContainer)
log.info("created project at %s" % (self.inputContainer))
