def initial_server_setup(ip, user):
# As soon as the server is created, we can log in as root
c = Connection(ip, user='******')
# Lets create a user and add it to the sudoers group
# c.run('adduser {}'.format(user))
# c.run('usermod -aG sudo {}'.format(user))
# c.run('su - {}'.format(user))
# c.run('mkdir ~/.ssh')
# c.run('chmod 700 ~/.ssh')
# Lets create an ssh-key pair if it does not exist:
ssh_dir = os.path.join(os.environ.get('HOME'), '.ssh')
ssh_filename = os.path.join(ssh_dir, 'id_rsa')
if not os.path.exists(ssh_dir):
os.makedirs(ssh_dir)
make_ssh_key(c, ssh_filename)
else:
if not os.path.isfile(ssh_filename):
make_ssh_key(c, ssh_filename)
# Upload the file to the server
ssh_filename_pub = ssh_filename + '.pub'
c.put(ssh_filename_pub, remote='/home/{}/'.format(user))
c.run('cd /home/{}'.format(user))
c.run('cat {} >> .ssh/authorized_keys'.format('id_rsa.pub'))
def write_configuration(conn: Connection,
training_config: CS1TrainingRunConfig):
"""
Sets up the directories and params.yaml file for the current experiment on Medulla
Args:
conn: fabric.Connection object
training_config: CS1TrainingConfig object from parsed YAML
theta_experiment_dir: ThetaGPU experiment directory
"""
top_dir = training_config.medulla_experiment_path
if not top_dir.is_absolute():
raise ValueError("medulla_experiment_path must be absolute")
logger.info(f"Creating {top_dir} on {conn.host}")
conn.run(f"mkdir -p {top_dir}")
conn.run(f"mkdir -p {training_config.sim_data_dir}")
conn.run(f"mkdir -p {training_config.data_dir}")
conn.run(f"mkdir -p {training_config.eval_data_dir}")
conn.run(f"mkdir -p {training_config.model_dir}")
conn.run(f"touch {training_config.global_path}")
h5_dir = training_config.initial_h5_transfer_dir
if h5_dir and h5_dir.is_dir():
for f in h5_dir.glob("*.h5"):
conn.put(f.as_posix(), training_config.sim_data_dir.as_posix())
with NamedTemporaryFile(mode="w", delete=False) as fp:
yaml.dump(json.loads(training_config.json()), fp)
fp.flush()
conn.put(fp.name, top_dir.joinpath("params.yaml").as_posix())
class FabricApi:
def __init__(self, ip, user, password):
self.conn = Connection(ip, user, connect_kwargs={'password': password})
def update_symbolic_link(self, src, dest):
try:
self.conn.run("[ -e {} ]".format(src))
self.conn.run("rm -rf {}".format(src))
except:
pass
self.conn.run("ln -s {} {}".format(dest, src))
def send_file(self, src, dest):
try:
self.conn.run("[ -e {} ]".format(dest))
return True
except:
pass
self.conn.run("mkdir -p {}".format(os.path.dirname(dest)))
self.conn.put(src, "{}.tar.gz".format(dest))
self.conn.run("tar -xf {}.tar.gz -C {}".format(dest,
os.path.dirname(dest)))
self.conn.run("rm -rf {}.tar.gz".format(dest))
def add_new_node_ssh():
nodes_new = {'172.17.0.12': '111111'}
node_num = 3
with open('/etc/hosts', 'a') as f:
for ip, pwd in node_new.items():
f.write('%s node%s' % (ip, node_num))
node_num += 1
c = Connection(ip,
port=22,
user='******',
connect_kwargs={'password': pwd})
c.get('/root/.ssh/id_rsa.pub', '/root/.ssh/id_rsa.pub.bak')
c.local(
'cat /root/.ssh/id_rsa.pub.bak >> /root/.ssh/authorized_keys')
c.local('rm -f /root/.ssh/id_rsa.pub.bak')
nodes.update(nodes_new)
for ip, pwd in nodes.items():
c = Connection(ip,
port=22,
user='******',
connect_kwargs={'password': pwd})
c.run('rm -f /etc/hosts')
c.put('/etc/hosts', '/etc/hosts')
a = c.local('find /root/.ssh/ -name authorized_keys')
if a.stdout.find('authorized_keys') != -1:
c.run('rm -f /root/.ssh/authorized_keys')
c.put('/root/.ssh/authorized_keys', '/root/.ssh/authorized_keys')
print('over')
def setup_squid(self, server, startup_script_file_location,
squid_service_location):
service_name = squid_service_location.replace('.service', '')
sudo_config = Config(overrides={'sudo': {'password': server.password}})
server_connection = Connection(
host=server.ip_address,
user=server.username,
connect_kwargs={"password": server.password},
config=sudo_config)
try:
server_connection.sudo('reboot')
except UnexpectedExit as e:
print("Caught error while rebooting machine")
self.wait_for_server(server)
server_connection.sudo('apt-get update')
server_connection.sudo('apt-get install dos2unix')
file_transfer_result = server_connection.put(
startup_script_file_location, remote='/usr/bin/')
server_connection.sudo('chmod +x /usr/bin/' +
startup_script_file_location)
server_connection.sudo('dos2unix /usr/bin/' +
startup_script_file_location)
file_transfer_result = server_connection.put(
squid_service_location, remote='/etc/systemd/system/')
server_connection.sudo(f'systemctl enable {service_name}')
try:
server_connection.sudo('reboot')
except UnexpectedExit as e:
print("Caught error while rebooting machine")
def upload_archive():
print('uploading...')
# establishing connection to the server
c = Connection(
host="178.128.193.27",
user="******",
connect_kwargs={
"key_filename": "D:\projects\doggo\.ssh\openSSH",
},
)
# uploading the zip to myproject directory
c.put(zipName, 'myproject/')
print('exctracting...')
# unziping and overwriting the files
c.run('cd myproject && unzip -o %s' % zipName)
# removing the archive
c.run('cd myproject && rm %s' % zipName)
# does server restart correctly?
# sudo -S -p '[sudo] password: '******'\[sudo\] password:'******'1992\n',
)
# does not work : c.config.sudo.password('1992')
print('restarting...')
c.sudo('systemctl restart myproject', pty=True, watchers=[sudopass])
def put_dir(con: Connection, from_dir: str, to_dir: str):
""" put a directory to server.
"""
con.local("tar cfz {0}.tar.gz {0}".format(from_dir))
con.put("{}.tar.gz".format(from_dir), "{}".format(to_dir))
con.run("tar zxf {1}/{0}.tar.gz -C {1}".format(from_dir, to_dir))
con.local("rm {}.tar.gz".format(from_dir))
def deploy(c):
user = input('Input login user name: ')
host = input('Input login host: ')
root = input('Input project root path：')
user_pass = getpass.getpass('Input login user pass：')
result = c.run('python setup.py --fullname', hide=True)
dist = result.stdout.strip()
filename = '%s.tar.gz' % dist
result = c.run('python setup.py --name', hide=True)
name = result.stdout.strip()
remote = Connection('%s@%s' % (user, host),
connect_kwargs={"password": user_pass})
remote.run('cd %s && ls -al' % root)
return False
remote.put('./dist/%s' %
filename, remote='%s' % root)
remote.run('cd %s &&\
source .env/bin/activate &&\
ls -al && type python &&\
pip install %s &&\
supervisorctl restart %s' % (root, filename, name))
def deploy_ccp_docs(
ccp_doc_root="public_html/py/ciscoconfparse",
ccp_bundle_name="ccp.tar.gz",
doc_host="",
password="",
):
# Run 'make html' in directory: sphinx-doc/
run("cd sphinx-doc && make clean && make html") # local command
run("cd sphinx-doc/_build/html && tar cvfz {0} *".format(
os.path.expanduser("~/" + ccp_bundle_name)))
# Run 'make clean' in directory: sphinx-doc/
run("cd sphinx-doc && make clean") # local command
# ssh with a password...
conn = Connection("mpenning@{}".format(doc_host),
connect_kwargs={"password": password})
conn.put(
local=os.path.expanduser("~/{0}".format(ccp_bundle_name)),
remote=ccp_bundle_name,
)
# Delete all the old files
conn.run("rm -rf {0}/*".format(ccp_doc_root))
# Move the new files to ccp_doc_root
conn.run("mv {0} {1}".format(ccp_bundle_name, ccp_doc_root))
with conn.cd(ccp_doc_root):
conn.run("tar xvfz {0}".format(ccp_bundle_name))
conn.run("rm {0}".format(ccp_bundle_name))
def install_hashistack(nodes):
# TODO: make this concurrent in threads
for node in nodes:
hostname = node["hostname"]
conn = Connection(hostname)
# Create all the script directories that will be needed
conn.run('mkdir -p /home/ubuntu/scripts/consul')
conn.run('mkdir -p /home/ubuntu/scripts/vault')
conn.run('mkdir -p /home/ubuntu/scripts/nomad')
conn.run('mkdir -p /home/ubuntu/scripts/terraform')
conn.run('mkdir -p /home/ubuntu/scripts/packer')
conn.run('mkdir -p /home/ubuntu/scripts/waypoint')
conn.run('mkdir -p /home/ubuntu/scripts/boundary')
conn.put("../../bash/rasppi/install.sh", "/home/ubuntu/install.sh")
conn.put("../../bash/rasppi/consul/setup_basics.sh",
"/home/ubuntu/scripts/consul/setup_basics.sh")
# conn.put("../../bash/rasppi/consul/setup_primary.sh", "/home/ubuntu/scripts/consul/setup_primary.sh")
# Set the permissions on all the install scripts
conn.run("chmod 755 *.sh")
conn.run("chmod 755 scripts/**/*.sh")
# Then install all the binaries
install_result = conn.run("./install.sh")
print(install_result)
basics_result = conn.run("./scripts/consul/setup_basics.sh")
print(basics_result)
def start_vid(request):
global filename_global
filename_global = 'user_{0}_{1}'.format(request.user.id,
request.user.username)
with open('./runcode/motion.conf') as fin, open(
'./runcode/motion_new.conf', 'w') as fout:
for i, item in enumerate(fin, 1):
if i == 450: # 450 - dir for saving stuff
item = 'target_dir "' + '/home/pi/runcode/data/videos/' + filename_global + '"\n'
print(item)
if i == 473: # 473 - filename pattern
global f
f = request.user.username + '-' + timezone.now().strftime(
'%d-%m-%y_%H-%M-%S')
# f_global = f
item = 'movie_filename ' + f + '\n'
print(item)
fout.write(item)
c = Connection(host=pi_ip,
user='******',
connect_kwargs={'password': pi_pwd},
connect_timeout=10)
c.put('./runcode/motion_new.conf', 'runcode/motion_new.conf')
cmd = " echo " + pi_pwd + " | sudo -S motion -b -c /runcode/motion_new.conf"
c.run(cmd)
global a
a = 1
sleep(1.5
) # don't have any other option as of now to wait for iframe loading
return HttpResponseRedirect('/runcode/')
def deploy(c):
remote_user = '******'
remote_password = '******'
remote_host = '165.22.105.187'
config = Config(overrides={'sudo': {'password': remote_password}})
connect_kwarg = {'password': remote_password, 'allow_agent': False}
conn = Connection(host = remote_host, user=remote_user, config=config, connect_kwargs=connect_kwarg)
print('Connected with remote machine')
print('Copy sources')
conn.put('app.py')
conn.put("config.json")
print("install requirements")
conn.sudo("pip3 install Flask Flask-CORS")
print("Shutdown previous server")
conn.sudo('pkill -F server.pid', warn=True)
print('Start server')
conn.sudo("nohup python3 app.py &> logs.txt & echo $! > server.pid")
print("Sucsess!")
conn.close()
def _update_settings(c: Connection, source_folder, sitename):
settings_path = source_folder + '/superlists/settings.py'
secret_key_path = source_folder + '/superlists/secret_key.py'
loc_tmp_dir = 'tmp_remote_settings'
loc_new_settings_path = loc_tmp_dir + '/settings.py'
loc_old_settings_path = loc_tmp_dir + '/settings_old.py'
loc_secret_key_path = loc_tmp_dir + '/secret_key.py'
os.mkdir(loc_tmp_dir)
try:
c.get(settings_path,
local=os.getcwd() + '/' + loc_tmp_dir + '/settings.py')
os.rename(loc_tmp_dir + '/settings.py', loc_old_settings_path)
with open(loc_old_settings_path, 'r') as f:
content = f.read()
new_content = re.sub("DEBUG = True", "DEBUG = False", content)
new_content = re.sub(r'ALLOWED_HOSTS = \[.*\]',
'ALLOWED_HOSTS = ["{}"]'.format(sitename),
new_content)
new_content = re.sub(r"SECRET_KEY = '.*'",
'from .secret_key import SECRET_KEY',
new_content)
with open(loc_new_settings_path, 'w') as nf:
nf.write(new_content)
if not _exists(c, secret_key_path):
chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
key = ''.join(random.SystemRandom().choice(chars)
for _ in range(50))
with open(loc_secret_key_path, 'w') as nkey:
nkey.write('SECRET_KEY = "{}"'.format(key))
c.put(os.getcwd() + '/' + loc_secret_key_path,
remote=source_folder + '/superlists/')
c.put(os.getcwd() + '/' + loc_new_settings_path,
remote=source_folder + '/superlists/')
finally:
shutil.rmtree(loc_tmp_dir)
def job(self, run: "Run", payload: dict, device: Device) -> dict:
username, password = run.get_credentials(device)
fabric_connection = Connection(
host=device.ip_address,
port=device.port,
user=username,
connect_kwargs={"password": password},
)
source_code = run.sub(run.source_code, locals())
match = run.sub(run.content_match, locals())
run.log("info", f"Running Unix Shell Script {self.name} on {device.name}")
script_file_name = "unix_shell_script_service.sh"
with StringIO(run.source_code) as script_file:
fabric_connection.put(script_file, script_file_name)
if run.privileged_mode:
if not device.enable_password:
raise Exception(
f"Service {self.name} requested privileged mode on device "
f"with no configured enable_password: {device.name}"
)
result = fabric_connection.sudo(
f"bash {script_file_name}", password=device.enable_password
)
else:
result = fabric_connection.run(f"bash {script_file_name}")
fabric_connection.run(f"rm {script_file_name}")
return {
"match": match,
"negative_logic": run.negative_logic,
"result": f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}",
"success": result.ok and run.match_content(result, match),
}
def submit_job(connection: Connection, job_script: str) -> str:
job_script = textwrap.dedent(job_script.lstrip())
connection.put(io.StringIO(job_script), "test.slm")
connection.sudo("mkdir -p --mode=777 /mnt/shared/test", in_stream=False)
res = connection.run("sbatch --chdir=/mnt/shared/test --wait test.slm", timeout=timedelta(minutes=10).seconds, in_stream=False)
job_id = res.stdout.split()[-1]
return job_id
def deploy(c):
# 手动输入服务器地址、用户名及密码
host = input('host:')
user = input('user:'******'password:'******'22', user=user, connect_kwargs={'password': password})
# 删除服务器上老版本的tar文件
conn.run('rm -f %s' % _REMOTE_TMP_TAR)
# 上传新版本的tar文件
conn.put(os.path.join(_LOCAL_PKG_PATH, _TAR_FILE), _REMOTE_TMP_TAR)
# 在服务器上创建新目录
newdir = 'www-%s' % datetime.now().strftime('%y-%m-%d_%H.%M.%S')
with conn.cd(_REMOTE_BASE_DIR):
conn.run('mkdir %s' % newdir)
# 解压tar文件至新目录
with conn.cd('%s/%s' % (_REMOTE_BASE_DIR, newdir)):
conn.run('tar -xzvf %s' % _REMOTE_TMP_TAR)
# 重置软链接
with conn.cd(_REMOTE_BASE_DIR):
conn.run('rm -f www')
conn.run('ln -s %s www' % newdir)
# 将app.py设置为可执行
conn.run('chmod a+x %s/www/app.py' % _REMOTE_BASE_DIR)
# 重启Python服务和Nginx服务
conn.run('supervisorctl stop pythonweb')
conn.run('supervisorctl start pythonweb')
conn.run('/etc/init.d/nginx reload')
def deploy(c):
remote_user = '******'
remote_password = '******'
remote_host = '178.62.2.141'
config = Config(overrides={'sudo': {'password': remote_password}})
connect_kwarg = {'password': remote_password, 'allow_agent': False}
conn = Connection(host=remote_host,
user=remote_user,
config=config,
connect_kwargs=connect_kwarg)
print("Connected with remote machine")
print("Copy sources")
conn.put("app.py")
conn.put("config.json")
print("Install requirements")
conn.sudo("pip3 install Flask Flask-CORS")
print("Shutdown previous server")
conn.sudo("pkill -F server.pid", warn=True)
print("Start server")
conn.sudo("nohup python3 app.py &> logs.txt & echo $! > server.pid")
print("Success!")
conn.close()
class Fabapi(object):
def __init__(self, hostip):
connect_kwargs = {
'pkey': settings.DEPLOYKEY,
# 'password':''
}
self.c = Connection(host=hostip,
user=settings.DEPLOYUSER,
port=22,
**connect_kwargs)
def locald(self, ld):
return self.c.local(ld)
def remoted(self, rd, sudoif=0):
# if sudoif == 1:
# return sudo(rd, pty=False)
# else:
# return run(rd)
return self.c.run(rd, pty=False, shell=False)
def getfile(self, local_dest, remote_dest):
self.c.get(remote_dest, local_dest)
def putfile(self, local_dest, remote_dest):
self.c.put(local_dest, remote_dest, mode=0o664)
def isexists(self, rootdir):
res = int(
self.c.run(" [ -e {0} ] && echo 1 || echo 0".format(rootdir),
shell=False))
return res
def setup_node(self, ip):
ssh_conn = SSH_Connection(host=ip,
user=self.config.get('remote', 'user'),
connect_kwargs=self.ssh_custom_key)
ssh_conn.run('apt-get -y update && apt-get -y upgrade', hide=True)
ssh_conn.run('apt-get -y install openmpi-bin', hide=True)
ssh_conn.run('mkdir -p /root/bin', hide=True)
if self.config.get('local', 'deployable_code').startswith('http'):
# downloading binary from a trusted non-public address
ssh_conn.run('cd /root/bin && wget %s' %
self.config.get('local', 'deployable_code'),
hide=True)
else:
# uploading binary from local; requires broadband connection
ssh_conn.put(self.config.get('local', 'deployable_code'),
'/root/bin/Pcrystal')
if self.config.get('local', 'deployable_code').endswith('.gz'):
# binary may be gzipped, without subfolders, with an arbitrary archive name,
# but the name of the binary must remain Pcrystal
ssh_conn.run(
'cd /root/bin && tar xvf %s' %
self.config.get('local', 'deployable_code').split('/')[-1],
hide=True)
ssh_conn.run('ln -sf /root/bin/Pcrystal /usr/bin/Pcrystal', hide=True)
# print and ensure versions
result = ssh_conn.run('/usr/bin/mpirun --allow-run-as-root -V',
hide=True)
logging.info(result.stdout)
result = ssh_conn.run('cat /etc/issue', hide=True)
logging.info(result.stdout)
result = ssh_conn.run('grep -c ^processor /proc/cpuinfo', hide=True)
logging.info(result.stdout)
def deploy(c):
conn = Connection("trotsky")
with conn.cd("~/src/untitled-incremental"):
conn.run("git fetch")
conn.run("git stash")
conn.run("git pull")
conn.run("git stash pop")
conn.put("main.wasm")
def restore(self, host, src):
c = Connection(host)
c.put(src, f"{self.directory}/backup.tgz")
c.run(f"cd {self.directory} && docker-compose stop")
c.run(f"cd {self.directory} && rm -rf db images", warn=True)
c.run(f"cd {self.directory} && tar -xvzf backup.tgz")
c.run(f"rm {self.directory}/backup.tgz")
c.run(f"cd {self.directory} && docker-compose start")
def _check_or_copy_wandb_key(hostname: str, ssh_session: fabric.Connection):
try:
ssh_session.run("test -f $HOME/.netrc")
except UnexpectedExit:
print(
f"Wandb api key not found in {hostname}. Copying it from {DEFAULT_WANDB_KEY}"
)
ssh_session.put(DEFAULT_WANDB_KEY, ".netrc")
def deploy(c):
r = Connection("zerojoy")
r.put("dist/usb_gadget.tar", "zerojoy.tar")
f = c.run("poetry version -s")
wheel = f"zerojoy-{f.stdout.strip()}-py3-none-any.whl"
r.put(f"dist/{wheel}")
with r.prefix(". .venv/bin/activate"):
r.run(f"pip install --no-index --force-reinstall {wheel}")
restart(r)
def passwordlessSSH(user,publicIPs,keyFile,cwd):
#Change the permissions of authorized_keys to 600
subprocess.check_output(['bash','-c', "chmod 600 keys/authorized_keys_updated"])
print("\nCopy the Authorized Keys file to all instances")
for IPAddress in publicIPs:
print IPAddress
connection = Connection(host=IPAddress, user=user, connect_kwargs = {'key_filename': ['' + keyFile + ''] })
connection.put(''+cwd +'/keys/authorized_keys_updated',remote='/home/'+user+'/.ssh/authorized_keys',preserve_mode=True)
def start(self):
""" start node
:return: ip
"""
# launch server. ubuntu.
name = names.sample(1).item().lower()
size = [s for s in self.lc.list_sizes() if s.name == "t3.nano"][0]
image = self.lc.list_images(ex_image_ids=["ami-03d8261f577d71b6a"])[0]
node = self.lc.create_node(
name,
size,
image,
ex_keyname="key",
ex_spot=True,
ex_security_groups=["proxy"],
ex_metadata=dict(app="proxy", ready=False),
)
log.info(f"waiting for {name} to start")
node = self.lc.wait_until_running([node])[0][0]
self.node = node
ip = node.public_ips[0]
self.session = self.get_session(ip)
# configure using fabric
con = Connection(
ip,
user="******",
connect_kwargs=dict(key_filename=f"{HOME}/.aws/key"),
)
self.con = con
# retry until ssh available
Retry(tries=3, delay=2, warn=1)(con.open)()
con.put(f"{HERE}/tinyproxy.conf")
con.run(
"sudo apt-get -qq update && "
"sudo apt-get -y -qq install dos2unix tinyproxy && "
"dos2unix -q tinyproxy.conf && "
"sudo cp tinyproxy.conf /etc/tinyproxy/tinyproxy.conf && "
"sudo service tinyproxy restart",
hide="both",
)
# wait for proxy to be working
try:
self.check_proxy()
except:
log.error(
f"Failed to start proxy for {node.extra.instance_id} at {ip}")
raise
# make available
self.lc.ex_create_tags(node, dict(ready="True"))
log.info(f" {ip} started")
return ip
def upload_runbook(con: Connection, dst: str):
print("Uploading runbook to the master node results dir ...")
try:
create_runbook(RUNBOOK_PATH, setup_file=README_PATH)
con.run(f"mkdir -p {dst}")
con.put(README_PATH, dst)
con.put(RUNBOOK_PATH, dst)
finally:
if os.path.exists(RUNBOOK_PATH):
os.remove(RUNBOOK_PATH)
def test_3_fabric_upload():
remove(testfilename) if isfile(testfilename) else None
localfile = open(testfilename, 'w')
localfile.write(test_text)
localfile.close()
c = Connection(host=hostname, user=username, connect_kwargs={'password': password})
c.put(testfilename, remotepath)
output = c.run(f'cat {remotepath}').stdout
assert output == test_text
remove(testfilename)
def copyFile(user, fileName, destinationPath, publicIPs, keyFile):
for IPAddress in publicIPs:
print IPAddress
connection = Connection(
host=IPAddress,
user=user,
connect_kwargs={'key_filename': ['' + keyFile + '']})
connection.run('ls -ltr', pty=True)
connection.put(fileName,
remote=destinationPath + fileName,
preserve_mode=True)
def create_or_update_dot_env(c: fabric.Connection, env_args, env_file):
"""
Create or update the remote .env file by merging its contents with the
variables passed in as CLI args and the contents of the passed in local .env
file. Generate SECRET_KEY if not provided and prompt for DOMAIN/EMAIL if not
provided.
"""
info("Creating/updating the remote .env")
local_env = {}
if env_file:
sub_info(f"Reading variables from {env_file}")
file_env = dotenv.dotenv_values(env_file)
local_env.update(file_env)
if env_args:
sub_info("Compiling variables passed through CLI")
cli_env_content = "\n".join(env_args)
cli_env = dotenv.dotenv_values(stream=io.StringIO(cli_env_content))
local_env.update(cli_env)
if files.exists(c, ".env"):
sub_info("Reading variables from remote .env")
remote_env_content = c.run("cat .env", hide=True).stdout
remote_env = dotenv.dotenv_values(
stream=io.StringIO(remote_env_content))
else:
remote_env = {}
for key, value in local_env.items():
if key in remote_env:
prompt_text = f"{key} is already set to {remote_env[key]}, are you sure that you want to set it to {value}?"
if not confirm(prompt_text):
continue
sub_info(f"Setting {key} to {value}")
remote_env[key] = value
if not remote_env.get("SECRET_KEY"):
sub_info("Generating value for SECRET_KEY")
remote_env["SECRET_KEY"] = "".join(
random.choices(string.ascii_letters + string.digits, k=50))
for key in ["DOMAIN", "EMAIL"]:
while not remote_env.get(key):
value = prompt(f"Enter value for {key}")
remote_env[key] = value
sub_info("Writing variables to remote .env")
remote_env_content = "\n".join(f"{key}={value or ''}"
for key, value in remote_env.items())
remote_env_content_bytes = io.BytesIO(remote_env_content.encode("utf-8"))
c.put(remote_env_content_bytes, f"{c.cwd}/.env")
def run_py_code(self, code=None):
filename = "./runcode/running/a.py"
if not code:
code = self.code
with open(filename, "w") as f:
f.write(code)
c = Connection(host=pi_ip,
user='******',
connect_kwargs={'password': '******'})
c.put("./runcode/running/a.py", './runcode/running/a.py')
c.close()
self.test_py_code(filename)
return self.stderr, self.stdout
class put:
def setup(self):
self.c = Connection('localhost')
self.remote = self._tmp('file.txt')
def base_case(self):
# Copy file from 'local' (support dir) to 'remote' (tempdir)
# TODO: consider path.py for contextmanager
cwd = os.getcwd()
os.chdir(self._support())
try:
# TODO: wrap chdir at the Connection level
self.c.sftp().chdir(self._tmp())
result = self.c.put('file.txt')
finally:
os.chdir(cwd)
# Make sure it arrived
ok_(os.path.exists(self.remote))
eq_(open(self.remote).read(), "yup\n")
# Sanity check result object
eq_(result.remote, self.remote)
eq_(result.orig_remote, None)
eq_(result.local, self._support('file.txt'))
eq_(result.orig_local, 'file.txt')
def file_like_objects(self):
fd = BytesIO()
fd.write(b"yup\n")
result = self.c.put(local=fd, remote=self.remote)
eq_(open(self.remote).read(), "yup\n")
eq_(result.remote, self.remote)
ok_(result.local is fd)
def mode_preservation(self):
# Use a dummy file which is given an unusual, highly unlikely to be
# default umask, set of permissions (oct 641, aka -rw-r----x)
local = self._tmp('funky-local.txt')
with open(local, 'w') as fd:
fd.write('whatever')
os.chmod(local, 0o641)
remote = self._tmp('funky-remote.txt')
self.c.put(remote=remote, local=local)
eq_(stat.S_IMODE(os.stat(remote).st_mode), 0o641)
class put:
def setup(self):
self.c = Connection("localhost")
self.remote = path.local.mkdtemp().join("file.txt").realpath()
def base_case(self):
# Copy file from 'local' (support dir) to 'remote' (tempdir)
local_dir = _support()
with path.local(local_dir).as_cwd():
tmpdir = self.remote.dirpath()
# TODO: wrap chdir at the Connection level
self.c.sftp().chdir(str(tmpdir))
result = self.c.put("file.txt")
# Make sure it arrived
assert self.remote.check()
assert self.remote.read() == "yup\n"
# Sanity check result object
assert result.remote == self.remote
assert result.orig_remote is None
assert result.local == _support("file.txt")
assert result.orig_local == "file.txt"
def file_like_objects(self):
fd = BytesIO()
fd.write(b"yup\n")
remote_str = str(self.remote)
result = self.c.put(local=fd, remote=remote_str)
assert self.remote.read() == "yup\n"
assert result.remote == remote_str
assert result.local is fd
def mode_preservation(self, tmpdir):
# Use a dummy file which is given an unusual, highly unlikely to be
# default umask, set of permissions (oct 641, aka -rw-r----x)
local = tmpdir.join("funky-local.txt")
local.write("whatever")
local.chmod(0o641)
remote = tmpdir.join("funky-remote.txt")
self.c.put(remote=str(remote), local=str(local))
assert stat.S_IMODE(remote.stat().mode) == 0o641
class RemoteCommandExecutor:
"""Execute remote commands on the cluster master node."""
USERNAMES = {
"alinux": "ec2-user",
"centos6": "centos",
"centos7": "centos",
"ubuntu1404": "ubuntu",
"ubuntu1604": "ubuntu",
}
def __init__(self, cluster):
self.__connection = Connection(
host=cluster.master_ip,
user=self.USERNAMES[cluster.os],
forward_agent=True,
connect_kwargs={"key_filename": cluster.ssh_key},
)
self.__user_at_hostname = "{0}@{1}".format(self.USERNAMES[cluster.os], cluster.master_ip)
def __del__(self):
try:
self.__connection.close()
except Exception as e:
# Catch all exceptions if we fail to close the clients
logging.warning("Exception raised when closing remote ssh client: {0}".format(e))
def run_remote_command(self, command, log_error=True, additional_files=None, raise_on_error=True, login_shell=True):
"""
Execute remote command on the cluster master node.
:param command: command to execute.
:param log_error: log errors.
:param additional_files: additional files to copy before executing script.
:param raise_on_error: if True raises a RemoteCommandExecutionError on failures
:param login_shell: if True prepends /bin/bash --login -c to the given command
:return: result of the execution.
"""
if isinstance(command, list):
command = " ".join(command)
self._copy_additional_files(additional_files)
logging.info("Executing remote command command on {0}: {1}".format(self.__user_at_hostname, command))
if login_shell:
command = "/bin/bash --login -c {0}".format(shlex.quote(command))
result = self.__connection.run(command, warn=True, pty=True, hide=False)
result.stdout = "\n".join(result.stdout.splitlines())
result.stderr = "\n".join(result.stderr.splitlines())
if result.failed and raise_on_error:
if log_error:
logging.error(
"Command {0} failed with error:\n{1}\nand output:\n{2}".format(
command, result.stderr, result.stdout
)
)
raise RemoteCommandExecutionError(result)
return result
def run_remote_script(self, script_file, args=None, log_error=True, additional_files=None):
"""
Execute a script remotely on the cluster master node.
Script is copied to the master home dir before being executed.
:param script_file: local path to the script to execute remotely.
:param args: args to pass to the script when invoked.
:param log_error: log errors.
:param additional_files: additional files to copy before executing script.
:return: result of the execution.
"""
script_name = os.path.basename(script_file)
self.__connection.put(script_file, script_name)
if not args:
args = []
return self.run_remote_command(
["/bin/bash", "--login", script_name] + args, log_error=log_error, additional_files=additional_files
)
def _copy_additional_files(self, files):
for file in files or []:
self.__connection.put(file, os.path.basename(file))
"key_filename": "c:\Users\Guodong\.ssh\exportedkey201310171355",
}
# Superuser privileges via auto-response
sudo_pass_auto_respond = Responder(
pattern=r'\[sudo\] password:'******'mypassword\n',
)
# create connection
cxn = Connection('192.168.88.19', config=fabric_config)
# do tasks on host
print cxn.run("uname -a", hide=True).stdout
print cxn.sudo("whoami", hide=True).stdout
cxn.run('sudo whoami', pty=True, watchers=[sudo_pass_auto_respond])
cxn.put(__file__, "/tmp/this.py")
cxn.run("sudo rm -f /tmp/this.py")
# cxn.get("/tmp/this.py", "this.py")
print disk_free(cxn)
# config multiple servers with methods 1
for host in ('192.168.88.19', '192.168.88.20', '192.168.88.21'):
result = Connection(host, config=fabric_config).run('uname -s', hide=True)
print("{}: {}".format(host, result.stdout.strip()))
# config multiple servers, M2
results = Group('192.168.88.19', '192.168.88.20', '192.168.88.21', config=fabric_config).run('uname -s', hide=True)
for connection, result in results.items():
print("{0.host}: {1.stdout}".format(connection, result))
