def backup(c):
c = Connection('aw')
""" backup data from aw mongo """
with c.cd('/var/www/anwen/db'):
c.run('. ~/.zshrc && python3 db_in_out.py -o')
c.run('tar czf aw_yaml.tar.gz data')
with c.cd('/var/www/anwen/docs/shares'):
c.run('tar czf aw_md.tar.gz *.md')
with c.cd('/var/www/anwen/static/upload/'):
c.run('tar czf upload.tar.gz img')
print('download yaml:')
with CD(os.path.join(os.getcwd(), 'db/')):
c.get('/var/www/anwen/db/aw_yaml.tar.gz', 'aw_yaml.tar.gz')
c.local('tar zxf aw_yaml.tar.gz')
c.local('rm aw_yaml.tar.gz')
print('download md:')
with CD(os.path.join(os.getcwd(), 'docs/shares/')):
c.get('/var/www/anwen/docs/shares/aw_md.tar.gz', 'aw_md.tar.gz')
c.local('tar zxf aw_md.tar.gz')
c.local('rm aw_md.tar.gz')
print('download img:')
return
with CD(os.path.join(os.getcwd(), 'static/upload/')):
c.get('/var/www/anwen/static/upload/upload.tar.gz', 'upload.tar.gz')
c.local('tar zxf upload.tar.gz img')
c.local('rm upload.tar.gz')
class sudo:
def setup(self):
# NOTE: assumes a user configured for passworded (NOT
# passwordless)_sudo, whose password is 'mypass', is executing the
# test suite. I.e. our travis-ci setup.
config = Config(
{"sudo": {"password": "******"}, "run": {"hide": True}}
)
self.cxn = Connection("localhost", config=config)
def sudo_command(self):
"""
Run command via sudo on host localhost
"""
skip_outside_travis()
assert self.cxn.sudo("whoami").stdout.strip() == "root"
def mixed_sudo_and_normal_commands(self):
"""
Run command via sudo, and not via sudo, on localhost
"""
skip_outside_travis()
logname = os.environ["LOGNAME"]
assert self.cxn.run("whoami").stdout.strip() == logname
assert self.cxn.sudo("whoami").stdout.strip() == "root"
def backol(c):
c = Connection('aw')
""" backup data from aw mongo """
t = time.time()
with c.cd('/var/www/anwen/db'):
c.run('. ~/.zshrc && python3 db_in_out.py -o')
c.run('tar czf aw_yaml_{}.tar.gz data'.format(t))
with c.cd('/var/www/anwen/docs/shares'):
c.run('tar czf aw_md_{}.tar.gz *.md'.format(t))
with c.cd('/var/www/anwen/static/upload/'):
c.run('tar czf upload_{}.tar.gz img'.format(t))
def mixed_use_of_local_and_run(self):
"""
Run command truly locally, and over SSH via localhost
"""
cxn = Connection("localhost")
result = cxn.local("echo foo", hide=True)
assert result.stdout == "foo\n"
assert not cxn.is_connected # meh way of proving it didn't use SSH yet
result = cxn.run("echo foo", hide=True)
assert cxn.is_connected # NOW it's using SSH
assert result.stdout == "foo\n"
def large_remote_commands_finish_cleanly(self):
# Guards against e.g. cleanup finishing before actually reading all
# data from the remote end. Which is largely an issue in Invoke-level
# code but one that only really manifests when doing stuff over the
# network. Yay computers!
path = "/usr/share/dict/words"
cxn = Connection("localhost")
with open(path) as fd:
words = [x.strip() for x in fd.readlines()]
stdout = cxn.run("cat {}".format(path), hide=True).stdout
lines = [x.strip() for x in stdout.splitlines()]
# When bug present, # lines received is significantly fewer than the
# true count in the file (by thousands).
assert len(lines) == len(words)
def copyDirectory(user, directoryName, destinationPath, publicIPs, keyFile):
for IPAddress in publicIPs:
config = Config({'identity': 'us-1east.pem'})
connection = Connection(
host=IPAddress,
user=user,
config=config,
connect_kwargs={'key_filename': ['' + keyFile + '']})
filePaths = []
connection.run('mkdir ' + directoryName + '')
for root, directories, filenames in os.walk(directoryName):
#Directories
for directory in directories:
directoryPath = os.path.join(root, directory)
index = directoryPath.find('/')
print directoryPath[index + 1:]
connection.run('mkdir ' + directoryPath + '')
for filename in filenames:
filePaths.append(os.path.join(root, filename))
for filePath in filePaths:
connection.put(filePath, remote=filePath, preserve_mode=True)
def main(args):
with open(os.path.expanduser(args.config_file), 'r') as fh:
config = yaml.load(fh)
# Create an archive
dirname = get_dirname(args.repo)
logger.info("Using repo {}".format(dirname))
repo = Repo(args.repo)
assert not repo.is_dirty()
archive_name = dirname
git_tag = next((tag for tag in repo.tags if tag.commit == repo.head.commit), None)
if git_tag:
archive_name += '_' + git_tag
else:
archive_name += '_' + repo.head.object.hexsha
if args.extra_tag:
archive_name += '_' + args.extra_tag
logger.info("Creating repo archive {}".format(archive_name))
archive = "{0}.tar.gz".format(archive_name)
archive_path = ospj(args.repo, archive)
run_dir = os.getcwd()
os.chdir(args.repo)
os.system("git-archive-all {}".format(archive))
os.chdir(run_dir)
logger.info("Archive created.")
# Transfer archive to remote
remote_dir = config['hosts']['irma']['archive_dir']
Connection('irma').put(archive_path, remote=remote_dir)
logger.info("Archive successfully transferred to irma")
# Extract remote archive
c = Connection('irma')
remote_archive_path = ospj(remote_dir, archive)
remote_extracted_path = remote_archive_path.replace('.tar.gz', '')
c.run('rm -r {} || true'.format(remote_extracted_path))
c.run('cd {}; tar -xvzf {}'.format(remote_dir, remote_archive_path))
# Create a link from dev or latest to the new archive
if args.mode == 'dev':
link_name = "{}_dev".format(dirname)
else:
link_name = "{}_latest".format(dirname)
c.run('cd {}; ln -sfn {} {}'.format(remote_dir, remote_extracted_path, link_name))
logger.info("Linking: {} {}".format(remote_extracted_path, link_name))
logger.info("{} successfully linked as the new {}".format(dirname, link_name))
def batch_run(source_root: str,
python_file: str,
key_file: str,
job_assignments: Dict[str, List[str]],
respawner: bool = False) -> None:
if respawner:
respawner_file = source_root + "/generative_playground/molecules/train/respawner.py"
respawner_prefix = 'python ' + respawner_file + ' '
else:
respawner_prefix = ''
for ip, jobs in job_assignments.items():
c = Connection('ubuntu@{}'.format(ip),
connect_kwargs={'key_filename': key_file})
for job in jobs:
for _ in range(5):
try:
# sometimes this fails for no discernible reason, and succeeds on second try
screen_name = str(uuid.uuid4())
c.run((
"screen -dmS {screen_name}; "
"screen -S {screen_name} -X stuff 'source activate pytorch_p36'$(echo -ne '\015'); "
"screen -S {screen_name} -X stuff 'export PYTHONPATH=$PYTHONPATH:{src_root}/'$(echo -ne '\015'); "
"screen -S {screen_name} -X stuff '{respawner_prefix}python {python_file} {job_id}'$(echo -ne '\015');"
).format(src_root=source_root,
python_file=python_file,
job_id=job,
screen_name=screen_name,
respawner_prefix=respawner_prefix))
print('Started job {} on screen {}'.format(
job, screen_name))
break
except:
pass
print('All jobs running')
def main():
print("{}@{}".format(mach['u'], mach['h']))
c = Connection("{}@{}".format(mach['u'], mach['h']),
port=11022,
connect_kwargs={"password": mach['p']})
with c.cd(mach['ws']):
c.run('git reset --hard')
c.run('git pull')
c.run('python3 deploy.py')
def run(con: Connection,
command: str,
params: List[str],
asynchronous: bool = False) -> Union['Result', 'Promise']:
cfg = RunConfig()
src = pathlib.Path().absolute() / '*'
dst = 'experiment'
rsync(con, str(src), dst, exclude=['.git', '__pycache__', 'outputs'])
command = command + ' ' + ' '.join(params)
command = f"source ~/.bash_profile; cd {dst}; {cfg.python} {command}"
result = con.run(command, asynchronous=asynchronous)
return result
def run_script(request):
site_pk = request.POST.get("site_id", 0)
action = request.POST.get("action", "")
# 什么也不做
if not action:
return jsonFailed(5, "没有要执行的文件")
site = Site.objects.filter(pk=site_pk).first()
if not site:
return jsonFailed(1, msg="没有找到该站点！该站点可能已经被删除")
action_dict = {
'restart': site.restart,
'deploy': site.deploy,
'update_cert': site.update_cert
}
file_name = action_dict.get(action, "")
if not file_name:
return jsonFailed(1, msg="该站点未设置脚本文件")
# 连接服务器
user = '******'
host = settings.SERVER_HOST
key_filename = settings.KEY_FILENAME_PATH
result = Connection(host,
user=user,
connect_kwargs={"key_filename": key_filename})
# script_file = os.path.join(settings.SCRIPT_PATH, file_name)
try:
# 要执行的命令
with result.cd(script_file):
result = result.run("python3 %s" % file_name)
msg = result.stdout
return jsonSuccess(msg="返回结果：" + msg)
# print('return_code', pwd.return_code)
# print('stdout', pwd.stdout)
except UnexpectedExit as e:
return jsonFailed(2, msg="执行命令错误")
class Remote:
def __init__(self, ip, user, password, port=22):
self.ip = ip
self.port = port
self.user = user
self.password = password
self.path = ""
self.git = Git(self)
def connect(self):
self.con = Connection(self.ip,
port=self.port,
user=self.user,
connect_kwargs={"password": self.password})
if (self.con):
print(
colored("Remote VPS Conntectd",
'yellow',
attrs=['reverse', 'bold']))
else:
print(
colored("Couldn't connect to VPS",
'red',
attrs=['reverse', 'bold']))
def setPath(self, path):
Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True)
self.path = path
def _exe(self, cmd):
try:
out = self.con.run(cmd)
return out.stdout.strip()
except Exception as e:
print("Error")
print(e)
def exe(self, cmd, path=None):
if (path):
self.path = path
print("")
print(
colored(self.path + "$", 'cyan', attrs=['reverse', 'bold']) +
colored(" " + cmd, 'green', attrs=['reverse', 'bold']))
with self.con.cd(self.path):
return self._exe(cmd)
def kernels(ctx: Connection):
"""returns sorted list of installed kernels
"""
result = ctx.run("ls /boot/vmlinuz*", hide=True)
klist = []
for release in result.stdout.split("\n"):
release = release.replace("/boot/vmlinuz-", "")
version = release.split("-")[0]
if version:
klist.append(Kernel(version, release))
def version_key(x: Kernel):
v = x.version.split(".")
return int(v[0]), int(v[1]), int(v[2])
return sorted(klist, key=version_key)
def run(con: Connection,
mode: CodeDistributionMode,
command: str,
params: List[str],
run_config: str,
asynchronous: bool = False) -> Union['Result', 'Promise']:
cfg = RunConfig(config=run_config)
dst = 'experiment'
distribute_code(mode, con, dst)
command = command + ' ' + ' '.join(params)
command = f"source ~/.bash_profile; cd {dst}; {cfg.python} {command}"
result = con.run(command, asynchronous=asynchronous)
return result
def update_pcnest(c):
remote_pcnest_dir = '/cygdrive/d/THC/IRRSvc'
for host in ['11.11.11.18', '11.11.11.28', '11.11.11.38']:
conn = Connection(host, user='******',
connect_kwargs={'password': "******"})
logger.info(host)
conn.run('uname -s')
with conn.cd(remote_pcnest_dir):
conn.run('./kill.bat')
conn.run('../update_irrsvc.bat')
logger.info('{0} pcnest update completed.'.format(host))
logger.info('all the pcnest update completed.')
def pull_latest_source(c: fabric.Connection, repo, checkout):
"""Pull the latest source and checkout the desired branch/commit."""
info(f"Pulling the latest source and checking out {checkout}")
if files.exists(c, ".git"):
sub_info("Resetting the working tree")
c.run("git reset --hard")
sub_info("Fetching the latest source")
c.run(f"git fetch --tag")
else:
if repo is None:
repo = prompt("Enter repository to pull source from")
sub_info(f"Cloning {repo}")
c.run(f"git clone {repo} .")
sub_info(f"Checking out {checkout}")
c.run(f"git checkout {checkout}")
def iperf3(c, dest_host, ca_host=None, parallel=None, stdout=False):
server_pid = None
try:
# 1. get conn to admin
dest_conn = Connection(dest_host, connect_timeout=5)
# 2. run iperf3 in daemon mode & save pid
dest_conn.sudo("iperf3 -s -D")
server_pid = dest_conn.run("pidof -s iperf3", hide=True).stdout
ca_host_group = get_ca_host_group(ca_host)
if parallel is not None:
parallel = "-P {}".format(int(parallel))
cmd = "iperf3 -J -c {} {}".format(dest_host, parallel)
for conn in ca_host_group:
print("Working on {}".format(conn.host))
try:
result = conn.run(cmd, pty=True, hide=True, warn=True).stdout
except socket.timeout:
print("Connection timed out after 5s to {}".format(conn.host))
continue
except socket.gaierror as e:
print(str(e))
continue
tr = TestResult(result)
ca = conn.host.split('.')[0]
cw_cmd = ("aws --profile test cloudwatch put-metric-data --region us-east-1 "
"--namespace 'Capture Agents' --dimensions CaptureAgent={} "
"--metric-name Sent_Mbps --value {} --unit 'Megabits/Second'") \
.format(ca, tr.sent_Mbps)
conn.local(cw_cmd)
finally:
if server_pid is not None:
print("Stopping iperf3 server")
dest_conn.sudo("kill {}".format(server_pid))
def pull_ec2():
print('Connecting to EC2 instance...')
instance = Connection(EC2_INSTANCE, 'ec2-user', connect_kwargs={'key_filename':'{}/.ssh/id_rsa'.format(os.path.expanduser('~'))})
print('Uploading compose files...')
instance.run('mkdir -p app')
for filename in COMPOSE_FILES:
instance.put(filename, remote='app/')
print('Logging into ECR from EC2 instance...')
instance.run('$(aws ecr get-login --region us-west-1 --no-include-email)')
print('Running compose files...')
instance.run('cd app && docker-compose -f docker-compose.yml -f production.yml pull && docker-compose -f docker-compose.yml -f production.yml up -d cms')
def deploy(c, server='test'):
if server not in HOST_CONF.keys():
return
password = getpass.getpass("What's your ssh password?")
config = Config(overrides={'connect_kwargs': {'password': password}, 'run': {'echo': True}})
host_conf = HOST_CONF[server]
remote_path = '/home/{user}/'.format(user=host_conf[1])
my_c = Connection(host=host_conf[0], user=host_conf[1], port=host_conf[2], config=config)
# 部署前端代码
front_code_dir = remote_path + 'workspace/craftsman_front/'
back_code_dir = remote_path + 'workspace/craftsman_back/'
from_templates_path = front_code_dir + 'templates/'
dest_templates_path = back_code_dir + 'templates/'
from_static = front_code_dir + 'static/'
dest_static = back_code_dir + 'static/'
with my_c.cd(front_code_dir):
my_c.run("git reset --hard")
my_c.run("git pull origin test" if server == 'test' else "git pull")
my_c.run("rsync -r {from_temp} {dest_temp}".format(from_temp=from_templates_path,
dest_temp=dest_templates_path))
my_c.run("rsync -r {from_static} {dest_static}".format(from_static=from_static,
dest_static=dest_static))
def update_report(self, force=False):
report_path = os.path.join(dir_path, self.report_dir, self.host)
if not os.path.isfile(report_path) or time.time() - os.path.getmtime(
report_path) > self.refresh_rate:
c = Connection(self.host, user=USER)
env = {"LDAP_URL": self.ldap_url, "CCGROUP": self.cc_group}
p = "&&".join(['{}={}'.format(k, v) for k, v in env.items()])
with c.prefix(p):
outputs = c.run(open(report_sh).read(), replace_env=True)
self._stdout = outputs.stdout
with open(report_path, 'w') as fp:
fp.write(self._stdout)
else:
self._stdout = open(report_path).read()
self.last_report_date = os.path.getmtime(report_path)
def send_cmd(args, node, node_ip):
if args.pswd:
credentials = {'password': cluster.passwords[node]}
else:
credentials = {
'password': cluster.passwords[node],
'key_filename': cluster.sshkeys[node]
}
try:
c = Connection(node_ip,
user=cluster.users[node],
connect_kwargs=credentials)
result = c.run(args.cmd, hide=args.quiet)
except:
print('Problem with node {}\n'.format(node))
def getSignalStrength():
pub = rospy.Publisher('/radio_signal', Int16, queue_size=10)
pub2 = rospy.Publisher('/radio_tx_rate', Int16, queue_size=10)
rospy.init_node('radio_interface', anonymous=True)
rate = rospy.Rate(2)
ssh_connection = Connection(host="[email protected]",
connect_kwargs={"password": "******"})
while not rospy.is_shutdown():
raw_output = ssh_connection.run(
'mca-status | grep -Ei "signal|TxRate"', hide=True)
msg = "{0.stdout}"
signal_string = msg.format(raw_output)
initial_split = signal_string.split('=')
signal_split = initial_split[1].split('\r')[0]
tx_split = initial_split[2].split('\r')[0]
pub.publish(int(signal_split))
pub2.publish(int(float(tx_split)))
rate.sleep()
def deploy(host, user, password, image, name):
""" Deploy Django app from docker container on specified host """
tar_dir(name, name + '.tar.gz')
connection = Connection(host=host,
port=22,
user=user,
connect_kwargs={'password': password})
connection.run('rm -rf {0} && rm -f {1}'.format(name, name + '.tar.gz'))
connection.put(name + '.tar.gz', remote=name + '.tar.gz')
connection.run('tar -xvzf ' + name + '.tar.gz')
connection.run('docker pull {0}'.format(image))
connection.run('docker stop $(docker ps -a -q)')
connection.run('''
docker run -d -p 8000:8000 -v ~/{1}:/dist-truth-checker/{1} {0} /bin/bash -c "
python manage.py collectstatic --settings=config.settings.production;
uwsgi uwsgi.ini"
'''.format(image, name))
connection.close()
class Server:
def __init__(self, config):
'''
初始化服务器链接
'''
self.conn = Connection(host=config.host,
user=config.user,
connect_kwargs={"password": config.password})
def exec(self, command):
'''
执行远程命令
'''
return self.conn.run(command, hide=True)
def close(self):
'''
关闭远程链接
'''
self.conn.close()
class RemoteShell(object):
"""RemoteShell class is used run remote shell commands against nodes using fabric.
"""
def __init__(self, options):
assert options["ssh_user"] is not None, 'ssh_user is required option'
assert options["ssh_host"] is not None, 'ssh_host is required option'
assert options["ssh_port"] is not None, 'ssh_port is required option'
assert options[
"private_key_file"] is not None, 'private_key_file is required option'
self.ssh_conn = Connection(
host=options.get("ssh_host"),
user=options.get("ssh_user"),
port=options.get("ssh_port"),
connect_kwargs={'key_filename': [options.get("private_key_file")]})
def run_command_raw(self, command):
return retry_network_errors(
lambda: self.ssh_conn.run(command, hide=True, warn=True))
def run_command(self, command):
result = self.run_command_raw(command)
if result.exited:
raise YBOpsRuntimeError("Remote shell command '{}' failed with "
"return code '{}' and error '{}'".format(
command.encode('utf-8'),
result.stderr.encode('utf-8'),
result.exited))
return result
def put_file(self, local_path, remote_path):
return retry_network_errors(
lambda: self.ssh_conn.put(local_path, remote_path))
# Checks if the file exists on the remote, and if not, it puts it there.
def put_file_if_not_exists(self, local_path, remote_path, file_name):
result = self.run_command('ls ' + remote_path)
if file_name not in result.stdout:
self.put_file(local_path, os.path.join(remote_path, file_name))
def start(self):
try:
conn = Connection(self.ip,user=self.user,
connect_kwargs={
"key_filename":self.key_filename,
},
)
except ValueError as e:
print("Value error while connecting to ",self.ip);
exit(-1);
current_time = datetime.datetime.utcnow().strftime("%d-%m-%Y_%H-%M-%S");
output_file = self.ip + "_" + self.server_ip + "_iperf3_test_client_" + current_time + ".out";
self.command = "nohup /usr/bin/python3 " + self.command + " " + self.config_file + self.args[0] + output_file\
+ self.args[1];
result = conn.run(self.command,hide=True);
print(self.command,' returned ',result.exited);
conn.close();
if (result.exited != 0):
print('Failed to run command: ',self.command, ' Exiting..');
exit(-1);
def get_rdms_from_neci(iter, job_folder):
from fabric import Connection
remote_ip = os.getenv('REMOTE_MACHINE_IP')
user = os.getenv('USER')
c = Connection(remote_ip, user=user)
molcas_WorkDir = os.getenv('MOLCAS_WorkDir')
remote_WorkDir = os.getenv('REMOTE_NECI_WorkDir')
neci_WorkDir = remote_WorkDir + job_folder + '/'
print('Copying RDMs and NECI output from')
print(neci_WorkDir)
print(' to ')
print(molcas_WorkDir)
c.get(neci_WorkDir + 'TwoRDM_aaaa.1',
local=molcas_WorkDir + 'TwoRDM_aaaa.1') # ,local=molcas_WorkDir)
c.get(neci_WorkDir + 'TwoRDM_abab.1',
local=molcas_WorkDir + 'TwoRDM_abab.1')
c.get(neci_WorkDir + 'TwoRDM_abba.1',
local=molcas_WorkDir + 'TwoRDM_abba.1')
c.get(neci_WorkDir + 'TwoRDM_bbbb.1',
local=molcas_WorkDir + 'TwoRDM_bbbb.1')
c.get(neci_WorkDir + 'TwoRDM_baba.1',
local=molcas_WorkDir + 'TwoRDM_baba.1')
c.get(neci_WorkDir + 'TwoRDM_baab.1',
local=molcas_WorkDir + 'TwoRDM_baab.1')
c.get(neci_WorkDir + 'out', local=molcas_WorkDir + 'neci.out')
# iter=0
with c.cd(neci_WorkDir):
iter_folder = 'Iter_' + str(iter)
c.run('mkdir {0}'.format(iter_folder))
c.run('mv TwoRDM* {0}'.format(iter_folder))
c.run('mv out {0}/neci.out'.format(iter_folder))
c.run('cp FCIMCStats {0}/.'.format(iter_folder))
c.run('tar -cf {0}.tar.gz {0}'.format(iter_folder, iter_folder))
c.get(neci_WorkDir + iter_folder + '.tar.gz',
local=molcas_WorkDir + iter_folder + '.tar.gz')
# c.run('rm -r {0}'.format(neci_WorkDir))
c.close()
def backup():
'''
备份数据库
'''
# 配置sudo命令的密码
config = Config(overrides={'sudo': {'password': remote_su_pass}})
# 以明文方式配置用户登录密码
conn = Connection(ip,
user=remote_user,
config=config,
connect_kwargs={
"allow_agent": False,
"password": remote_pass
})
f = 'backup-%s.sql' % datetime.now().strftime('%y-%m-%d_%H.%M.%S')
with conn.cd('/tmp'):
conn.run(
'mysqldump --user={} --password=\'{}\' --single-transaction --routines --triggers --events --skip-extended-insert {}>{}'
.format(remote_sql_user, remote_sql_pass, remote_sql_db, f))
conn.run('tar -czvf %s.tar.gz %s' % (f, f))
conn.get('/tmp/%s.tar.gz' % f, 'backup/%s.tar.gz' % f)
conn.run('rm -f %s' % f)
conn.run('rm -f %s.tar.gz' % f)
def exec_listdir(self):
try:
conn = Connection(self.ip,user="******",
connect_kwargs={
"key_filename":self.key_filename,
},
)
print("exec_listdir: conn: ",conn);
except ValueError as e:
print("Value error while connecting to ",self.ip);
exit(-1);
current_time = datetime.datetime.utcnow().strftime("%d-%m-%Y_%H-%M-%S");
output_file = self.ip + "_listdir_" + current_time + ".out";
self.command = "nohup " + self.command + " " + self.args[0] + output_file + self.args[1];
result = conn.run(self.command,hide=True);
print(self.command,' returned ',result.exited);
conn.close();
if (result.exited != 0):
print('Failed to run command: ',self.command, ' Exiting..');
exit(-1);
def setup_master(master: str, passphrase: str, user: str) -> None:
connect_kwargs = {'passphrase': passphrase}
node = Connection(master, user=user, connect_kwargs=connect_kwargs)
home = get_home_path(node)[0]
# install(node) # Not calling because of Fabric bug
# redis conf
logfile = f'logfile "{home}/redis.log"'
node.put('redis_configs/redis.conf', 'redis-stable/redis.conf')
node.run(f'echo {logfile} >> redis-stable/redis.conf')
# sentinal conf
logfile = f'logfile "{home}/sentinel.log"'
sentinel_monitor = f'sentinel monitor mymaster {master} 6379 2'
node.put('redis_configs/sentinel.conf', 'redis-stable/sentinel.conf')
node.run(f'echo {logfile} >> redis-stable/sentinel.conf')
node.run(f"sed -i 's/placeholder-line/{sentinel_monitor}/g' redis-stable/sentinel.conf")
class SshSession:
def __init__(self, host, user, key=None):
self.host = host
self.port = 22
self.user = user
self.key = key
self.pkey = paramiko.RSAKey.from_private_key(io.StringIO(self.key))
self.connection = Connection(self.host,
user=self.user,
port=22,
connect_kwargs={'pkey': self.pkey})
@retry(tries=5, delay=10)
def connect(self):
self.connection.open()
def run(self, command, **kwargs):
# http://docs.pyinvoke.org/en/latest/api/runners.html#invoke.runners.Runner.run
if not self.connection.is_connected:
self.connect()
self.connection.run(command, hide=True, **kwargs)
def mkdir(self, path, **kwargs):
self.connection.run('mkdir {}'.format(path), **kwargs)
def python(self, code, **kwargs):
self.connection.run('python {}'.format(code), **kwargs)
@contextmanager
def cd(self, *path):
cd_path = '/'.join(list(path))
self.connection.command_cwds.append(cd_path)
try:
yield
finally:
self.connection.command_cwds.pop()
@contextmanager
def activate(self, env):
self.connection.command_prefixes.append(
'conda activate {}'.format(env))
try:
yield
finally:
self.connection.command_prefixes.pop()
def getipexternal():
if server.startswith("http"):
# Get ip from external page
dbg("getipexternal: getting ip from external page=" + server)
from urllib import request
result = request.urlopen(server)
code = result.getcode()
html = str(result.read())
regex = matchip.replace("\\\\", "\\")
match = re.search(regex, html)
if match == None:
ip = ""
else:
ip = match.group()
dbg("getipexternal: got result=" + str(code) + " and ip=" + ip)
return ip
else:
# Get ip from router
dbg("getipexgternal: getting ip from router=" + server)
from fabric import Connection
ssh = Connection(
host=server,
port=str(port),
user=user,
connect_kwargs={"password": password}
)
result = ssh.run(matchip, hide=True)
code = result.exited
ip = result.stdout.strip()
dbg("getipexternal: got result=" + str(code) + " and ip=" + ip)
return ip
class SshWorker:
"""Обертка для использования библиотеки Fabric."""
def __init__(self, ssh_info: SshInfo):
self.ssh_info = ssh_info
self.fab: Union[Connection, None] = None
def connect(self):
self.fab = Connection(
host=self.ssh_info.host,
port=self.ssh_info.port,
user=self.ssh_info.user,
connect_kwargs={'password': self.ssh_info.password})
def ssh_cmd(self, cmd: str, warn=False):
print(
f'[{datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] >> {cmd}'
)
result = self.fab.run(command=cmd, warn=warn)
return result
def put(self, local_src, remote_dst):
result = self.fab.put(local_src, remote_dst)
print(f'Uploaded "{result.local}" to "{result.remote}"')
def login_and_execute(host, usr, pwd, cmd="hostname"):
lpwd = str(pwd).lower()
print("Trying username ", end='')
print(bcolors.BOLD + usr + bcolors.ENDC, end='')
print(" and password ", end='')
print(bcolors.BOLD + lpwd + bcolors.ENDC, end='')
print("...", end='')
try:
c = Connection(host=host, user=usr, connect_kwargs={"password": lpwd})
# hide stderr and stdout
result = c.run(cmd, hide=True)
print(bcolors.OKGREEN + "success." + bcolors.ENDC)
c.close()
except paramiko.ssh_exception.AuthenticationException as e:
print(bcolors.FAIL + str(e).lower() + bcolors.ENDC)
return None
except UnexpectedExit as e:
print(bcolors.FAIL + str(e).lower() + bcolors.ENDC)
return None
lines = result.stdout.splitlines()
if len(lines) > 1:
# the second line contains our cmd output
out = lines[1]
if out is not None and out != "":
return out
else:
print(bcolors.WARNING + "Command produced no output:" +
bcolors.ENDC)
print(bcolors.HEADER + cmd + bcolors.ENDC)
return None
def do_check_disk(threadName, q):
# 从队列里获取目标
i = q.get(timeout=2)
c = Connection(
host=i[2],
user='******',
connect_timeout=5,
connect_kwargs={"key_filename": ['/home/devops/.ssh/id_30_rsa']})
try:
disk_result = c.run(
"df -hP / | grep / | awk '{print $(NF -2) $(NF -1)}'",
pty=True,
hide=True,
)
disk_free, disk_use_percent = disk_result.stdout.split("G")
disk_free = int(disk_free)
disk_use_percent = int(disk_use_percent.replace('%', ''))
if disk_free < 30 and disk_use_percent > 95:
# print(i[0],"\t",i[2],"\t",disk_free)
logger.error(" {} {} disk usage alert!".format(i[0], i[2]))
except Exception as e:
logger.error(" {} {} host disk check fail,since:{}".format(
i[0], i[2], e))
def check(hosts: List[Host]) -> List[Notification]:
# check all hosts and disks
logger = logging.getLogger("check")
notifications = []
for host in hosts:
try:
con = Connection(host=host.host, user=host.user)
result = con.run("df -h", hide=True)
except Exception as e:
print("Unable to run commands on {}: {}".format(host.name, str(e)))
continue
else:
if result.ok:
# successful df command
disks = get_disks(host, result.stdout)
if len(disks) > 0:
notifications.append(Notification.from_disk(host, disks))
else:
logger.info("No notification is needed for '{}'".format(
host.name))
else:
# failed to get usage
continue
return notifications
class RemoteCommandExecutor:
"""Execute remote commands on the cluster master node."""
USERNAMES = {
"alinux": "ec2-user",
"centos6": "centos",
"centos7": "centos",
"ubuntu1404": "ubuntu",
"ubuntu1604": "ubuntu",
}
def __init__(self, cluster):
self.__connection = Connection(
host=cluster.master_ip,
user=self.USERNAMES[cluster.os],
forward_agent=True,
connect_kwargs={"key_filename": cluster.ssh_key},
)
self.__user_at_hostname = "{0}@{1}".format(self.USERNAMES[cluster.os], cluster.master_ip)
def __del__(self):
try:
self.__connection.close()
except Exception as e:
# Catch all exceptions if we fail to close the clients
logging.warning("Exception raised when closing remote ssh client: {0}".format(e))
def run_remote_command(self, command, log_error=True, additional_files=None, raise_on_error=True, login_shell=True):
"""
Execute remote command on the cluster master node.
:param command: command to execute.
:param log_error: log errors.
:param additional_files: additional files to copy before executing script.
:param raise_on_error: if True raises a RemoteCommandExecutionError on failures
:param login_shell: if True prepends /bin/bash --login -c to the given command
:return: result of the execution.
"""
if isinstance(command, list):
command = " ".join(command)
self._copy_additional_files(additional_files)
logging.info("Executing remote command command on {0}: {1}".format(self.__user_at_hostname, command))
if login_shell:
command = "/bin/bash --login -c {0}".format(shlex.quote(command))
result = self.__connection.run(command, warn=True, pty=True, hide=False)
result.stdout = "\n".join(result.stdout.splitlines())
result.stderr = "\n".join(result.stderr.splitlines())
if result.failed and raise_on_error:
if log_error:
logging.error(
"Command {0} failed with error:\n{1}\nand output:\n{2}".format(
command, result.stderr, result.stdout
)
)
raise RemoteCommandExecutionError(result)
return result
def run_remote_script(self, script_file, args=None, log_error=True, additional_files=None):
"""
Execute a script remotely on the cluster master node.
Script is copied to the master home dir before being executed.
:param script_file: local path to the script to execute remotely.
:param args: args to pass to the script when invoked.
:param log_error: log errors.
:param additional_files: additional files to copy before executing script.
:return: result of the execution.
"""
script_name = os.path.basename(script_file)
self.__connection.put(script_file, script_name)
if not args:
args = []
return self.run_remote_command(
["/bin/bash", "--login", script_name] + args, log_error=log_error, additional_files=additional_files
)
def _copy_additional_files(self, files):
for file in files or []:
self.__connection.put(file, os.path.basename(file))
def test(c):
c = Connection('aw')
result = c.run('uname -s', hide=True)
msg = "Ran {0.command!r} on {0.connection.host}, got stdout:\n{0.stdout}"
print(msg.format(result))
print(c.run('hostname'))
fabric_config['user'] = '******'
fabric_config['connect_kwargs'] = {
"key_filename": "c:\Users\Guodong\.ssh\exportedkey201310171355",
}
# Superuser privileges via auto-response
sudo_pass_auto_respond = Responder(
pattern=r'\[sudo\] password:'******'mypassword\n',
)
# create connection
cxn = Connection('192.168.88.19', config=fabric_config)
# do tasks on host
print cxn.run("uname -a", hide=True).stdout
print cxn.sudo("whoami", hide=True).stdout
cxn.run('sudo whoami', pty=True, watchers=[sudo_pass_auto_respond])
cxn.put(__file__, "/tmp/this.py")
cxn.run("sudo rm -f /tmp/this.py")
# cxn.get("/tmp/this.py", "this.py")
print disk_free(cxn)
# config multiple servers with methods 1
for host in ('192.168.88.19', '192.168.88.20', '192.168.88.21'):
result = Connection(host, config=fabric_config).run('uname -s', hide=True)
print("{}: {}".format(host, result.stdout.strip()))
# config multiple servers, M2
results = Group('192.168.88.19', '192.168.88.20', '192.168.88.21', config=fabric_config).run('uname -s', hide=True)
for connection, result in results.items():
