class sudo:
def setup(self):
# NOTE: assumes a user configured for passworded (NOT
# passwordless)_sudo, whose password is 'mypass', is executing the
# test suite. I.e. our travis-ci setup.
config = Config({
"sudo": {
"password": "******"
},
"run": {
"hide": True
}
})
self.cxn = Connection("localhost", config=config)
def sudo_command(self):
"""
Run command via sudo on host localhost
"""
skip_outside_travis()
assert self.cxn.sudo("whoami").stdout.strip() == "root"
def mixed_sudo_and_normal_commands(self):
"""
Run command via sudo, and not via sudo, on localhost
"""
skip_outside_travis()
logname = os.environ["LOGNAME"]
assert self.cxn.run("whoami").stdout.strip() == logname
assert self.cxn.sudo("whoami").stdout.strip() == "root"
def setup_squid(self, servers, startup_script_file_location,
squid_service_location):
service_name = squid_service_location.replace('.service', '')
for x in servers:
sudo_config = Config(overrides={'sudo': {'password': x.password}})
server_connection = Connection(
host=x.ip_address,
user=x.username,
connect_kwargs={"password": x.password},
config=sudo_config)
server_connection.sudo('apt-get install dos2unix')
time.sleep(1)
file_transfer_result = server_connection.put(
startup_script_file_location, remote='/usr/bin/')
server_connection.sudo('chmod +x /usr/bin/' +
startup_script_file_location)
server_connection.sudo('dos2unix /usr/bin/' +
startup_script_file_location)
file_transfer_result = server_connection.put(
squid_service_location, remote='/etc/systemd/system/')
server_connection.sudo(f'systemctl enable {service_name}')
try:
server_connection.sudo('reboot')
except UnexpectedExit as e:
print("Caught error while rebooting machine")
class sudo:
def setup(self):
# NOTE: assumes a user configured for passworded (NOT
# passwordless)_sudo, whose password is 'mypass', is executing the
# test suite. I.e. our travis-ci setup.
config = Config({
'sudo': {
'password': '******',
},
'run': {
'hide': True,
},
})
self.cxn = Connection('localhost', config=config)
def sudo_command(self):
"""
Run command via sudo on host localhost
"""
skip_outside_travis()
assert self.cxn.sudo('whoami').stdout.strip() == 'root'
def mixed_sudo_and_normal_commands(self):
"""
Run command via sudo, and not via sudo, on localhost
"""
skip_outside_travis()
logname = os.environ['LOGNAME']
assert self.cxn.run('whoami').stdout.strip() == logname
assert self.cxn.sudo('whoami').stdout.strip() == 'root'
class sudo:
def setup(self):
# NOTE: assumes a user configured for passworded (NOT
# passwordless)_sudo, whose password is 'mypass', is executing the
# test suite. I.e. our travis-ci setup.
config = Config(
{"sudo": {"password": "******"}, "run": {"hide": True}}
)
self.cxn = Connection("localhost", config=config)
def sudo_command(self):
"""
Run command via sudo on host localhost
"""
skip_outside_travis()
assert self.cxn.sudo("whoami").stdout.strip() == "root"
def mixed_sudo_and_normal_commands(self):
"""
Run command via sudo, and not via sudo, on localhost
"""
skip_outside_travis()
logname = os.environ["LOGNAME"]
assert self.cxn.run("whoami").stdout.strip() == logname
assert self.cxn.sudo("whoami").stdout.strip() == "root"
def upload_archive():
print('uploading...')
# establishing connection to the server
c = Connection(
host="178.128.193.27",
user="******",
connect_kwargs={
"key_filename": "D:\projects\doggo\.ssh\openSSH",
},
)
# uploading the zip to myproject directory
c.put(zipName, 'myproject/')
print('exctracting...')
# unziping and overwriting the files
c.run('cd myproject && unzip -o %s' % zipName)
# removing the archive
c.run('cd myproject && rm %s' % zipName)
# does server restart correctly?
# sudo -S -p '[sudo] password: '******'\[sudo\] password:'******'1992\n',
)
# does not work : c.config.sudo.password('1992')
print('restarting...')
c.sudo('systemctl restart myproject', pty=True, watchers=[sudopass])
def submit_job(connection: Connection, job_script: str) -> str:
job_script = textwrap.dedent(job_script.lstrip())
connection.put(io.StringIO(job_script), "test.slm")
connection.sudo("mkdir -p --mode=777 /mnt/shared/test", in_stream=False)
res = connection.run("sbatch --chdir=/mnt/shared/test --wait test.slm", timeout=timedelta(minutes=10).seconds, in_stream=False)
job_id = res.stdout.split()[-1]
return job_id
class OperationsUtil():
def __init__(self):
self.rootUser="******"
self.rootPass="******"
self.conn=Connection(host='[email protected]',connect_kwargs={"password": self.rootPass})
def createUser(self, username, password, dirname):
if not dirname:
dirname = username
command = "useradd -p "+password+" -m -d /home/"+dirname+"/ -g users -s /bin/bash "+username
try:
val = self.conn.sudo(command,password=self.rootPass,hide=True).stdout.strip()
return ('User Created: '+username,0)
except Exception as e:
return ('Cannot create user',1)
def viewUser(self):
command = "awk -F: '{ print $1}' /etc/passwd"
return self.conn.run(command,hide=True).stdout.strip()
def deluser(self,username):
command = "userdel -f "+username
try:
val = self.conn.sudo(command,password=self.rootPass,hide=True).stdout.strip()
return ('User Deleted: '+username,0)
except Exception as e:
return ('Cannot delete user',1)
def updatePriv(self,username):
command = "usermod -aG sudo "+username
try:
val = self.conn.sudo(command,password=self.rootPass).stdout.strip()
return ('User Privilege Granted',0)
except Exception as e:
return ('Cannot Grant user Privileges',1)
def deploy(local):
sudo_pass = getpass.getpass("[sudo] password: "******"git push")
remote = Connection("abig", config=config)
remote.run("cd /home/rc/webapps/topicos4 && git pull")
remote.run("cd /home/rc/webapps/topicos4 && source /home/rc/.virtualenvs/topicos4/bin/activate && pip install -r requirements.txt")
remote.sudo("sudo supervisorctl restart topicos4")
def update_repo(conn: Connection, branch: str = "master"):
"""Update remote Git repo to use the latest code"""
logger.info("Updating git repository to run the latest code.")
conn.sudo(f"chown -R ubuntu:ubuntu {CODE_PATH}", echo=True)
with conn.cd(CODE_PATH):
conn.run("git fetch --quiet", echo=True)
conn.run(f"git checkout --quiet {branch}", echo=True)
conn.run("git pull --quiet", echo=True)
logger.info("Done updating repo.")
def deploy_and_reload(c):
"Runs 'git pull' and reloads Nginx"
for host in HOSTS:
print('\n\n*** CONNECTING TO: %s' % (host))
config = Config(overrides={'sudo': {'password': sudo_pass}})
c = Connection(host, config=config)
c.run('cd %s && git pull [email protected]:openstate/%s.git' %
(DIR, GIT_REPO))
c.sudo('bash -c "cd %s && ./reload.sh"' % (DIR))
def restart_apache(context, region=DEFAULT_REGION, ip=None):
"""
Restart the application on the servers
"""
for ip_address in _get_ips(region_name=region):
if ip and ip_address != ip:
continue
cnx = Connection(ip_address, user=DEFAULT_USER)
print(f"Restart on {ip_address}")
cnx.sudo(f"systemctl restart flask.service")
def set_run_id(conn: Connection, run_id: str):
"""Set git to use the commit for a given run ID"""
logger.info("Setting up repo using a run id %s", run_id)
conn.sudo(f"chown -R ubuntu:ubuntu {CODE_PATH}", echo=True)
_, _, _, commit = read_run_id(run_id)
with conn.cd(CODE_PATH):
conn.run("git fetch --quiet", echo=True)
conn.run(f"git checkout --quiet {commit}", echo=True)
logger.info("Done updating repo.")
def deploy(ctx):
timestr = time.strftime("%Y%m%d-%H%M%S")
# print("Hello world!")
sudo_pass = getpass.getpass("What's your sudo password?")
config = Config(overrides={"sudo": {"password": sudo_pass}})
c = Connection("*****@*****.**", config=config)
# c.run(
# "mysqldump -u root -p'"
# + sudo_pass
# + "' meal_project > ~/mysql_backups/meal_project_"
# + timestr
# + ".sql"
# )
c.sudo("docker exec mysql mysqldump -u root -p'" + sudo_pass +
"' meal_project > ~/mysql_backups/meal_project_" + timestr + ".sql")
## from mysql docker site:
##$ docker exec some-mysql sh -c 'exec mysqldump --all-databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > /some/path/on/your/host/all-databases.sql
site_folder = "/home/odroid/meal_project"
c.run(f"mkdir -p {site_folder}")
# current_commit = c.local("HOME=~ git log -n 1 --format=%H")
with c.cd(site_folder):
if c.run("test -d .git", warn=True).failed:
c.run(f"git clone {REPO_URL} .")
else:
# c.run("git pull origin master --force")
c.run("git fetch")
c.run("git reset --hard origin/master")
# PIP load from requirements.txt
venv_command = "source venv/bin/activate"
pip_command = "venv/bin/pip3 install -r requirements.txt"
c.run(venv_command + " && " + pip_command)
# Reload static files
static_cmd = "python manage.py collectstatic --noinput"
c.run(venv_command + " && " + static_cmd)
# Database Make Migration
makem_cmd = "python manage.py makemigrations --settings=meal_project.settings.production"
c.run(venv_command + " && " + makem_cmd)
# Database Migrate
migrate_cmd = (
"python manage.py migrate --settings=meal_project.settings.production"
)
c.run(venv_command + " && " + migrate_cmd)
# Restart Apache
c.sudo("service apache2 restart")
def check_status(c: Connection) -> None:
# 检查硬盘使用状态
try:
result = c.sudo("df -h", hide=True)
except invoke.exceptions.UnexpectedExit as uee:
print(colored(uee, 'white', 'on_red'))
raise uee
except invoke.exceptions.Failure as fe:
print(colored(fe, 'white', 'on_cyan'))
raise fe
except invoke.exceptions.ThreadException as te:
print(colored(te, 'white', 'on_cyan'))
raise te
except Exception as e:
print(colored(str(e), 'white', 'on_yellow'))
raise e
else:
msg = f"Ran {result.command!r} on {result.connection.host}, got stdout:\n{result.stdout}"
print(msg)
# 检查内存使用状态
try:
result = c.sudo("free -m", hide=True)
except invoke.exceptions.UnexpectedExit as uee:
print(colored(uee, 'white', 'on_red'))
raise uee
except invoke.exceptions.Failure as fe:
print(colored(fe, 'white', 'on_cyan'))
raise fe
except invoke.exceptions.ThreadException as te:
print(colored(te, 'white', 'on_cyan'))
raise te
except Exception as e:
print(colored(str(e), 'white', 'on_yellow'))
raise e
else:
msg = f"Ran {result.command!r} on {result.connection.host}, got stdout:\n{result.stdout}"
print(msg)
# 检查io和cpu使用情况
try:
result = c.sudo("iostat 1 1", hide=True)
except invoke.exceptions.UnexpectedExit as uee:
print(colored(uee, 'white', 'on_red'))
raise uee
except invoke.exceptions.Failure as fe:
print(colored(fe, 'white', 'on_cyan'))
raise fe
except invoke.exceptions.ThreadException as te:
print(colored(te, 'white', 'on_cyan'))
raise te
except Exception as e:
print(colored(str(e), 'white', 'on_yellow'))
raise e
else:
msg = f"Ran {result.command!r} on {result.connection.host}, got stdout:\n{result.stdout}"
print(msg)
def restart_server(c):
password = getpass.getpass('Your sudo password: '******'sudo': {'password': password}})
conn = Connection(host, config=config)
commands = [
'supervisorctl stop {}'.format(service_name),
'supervisorctl start {}'.format(service_name),
'/etc/init.d/nginx reload',
]
for cmd in commands:
conn.sudo(cmd, hide='stderr')
def init(ctx, cfg=None):
if not cfg:
raise ValueError("No config set, provide a config file.")
host = cfg.get("host")
user = cfg.get("user")
echo(f"Connecting to {user} @ {host}")
conn = Connection(host, user=user)
conn.sudo("apt-get -y update")
conn.sudo("apt-get -y upgrade")
def load_modules(root):
"""Insert necessary kernel modules"""
print(" - Loading modules.")
v = vagrant.Vagrant(root=root)
cmd = "modprobe iscsi_tcp"
con = Connection(v.user_hostname_port(),
connect_kwargs={
"key_filename": v.keyfile(),
})
con.sudo(cmd)
def register_container_in_vm(root, version):
"""Export fresh container to kubernetes cluster"""
print(" - Adding container to the registry.")
v = vagrant.Vagrant(root=root)
cmd = "docker load < ./build/src/_output/joviandss-csi:" + version
con = Connection(v.user_hostname_port(),
connect_kwargs={
"key_filename": v.keyfile(),
})
con.sudo(cmd)
def deploy_and_up(c):
("Runs 'git pull', 'docker-compose up -d' (required if you update "
"Nginx version) and reloads Nginx")
for host in HOSTS:
print('\n\n*** CONNECTING TO: %s' % (host))
config = Config(overrides={'sudo': {'password': sudo_pass}})
c = Connection(host, config=config)
c.run('cd %s && git pull [email protected]:openstate/%s.git' %
(DIR, GIT_REPO))
c.sudo('bash -c "cd %s && docker-compose up -d"' % (DIR))
c.sudo('bash -c "cd %s && ./reload.sh"' % (DIR))
def upload_and_install_rpm(cnx: fabric.Connection):
s = Settings()
try:
cnx.put(s.qualys_rpm, s.qualys_rpm.name)
except OSError as e:
log.error(f'* os error: {e}')
return DeploymentResult.UPLOAD_FAILED
cnx.sudo(f'rpm --install {s.qualys_rpm.name}', hide=True)
cnx.sudo(
f'/usr/local/qualys/cloud-agent/bin/qualys-cloud-agent.sh ActivationId={s.qualys_activation_id} '
f'CustomerId={s.qualys_customer_id}',
hide=True)
return DeploymentResult.INSTALL_SUCCEEDED
def deploy(c):
sudo_pass = os.environ.get('SUDO_PW')
if not sudo_pass:
sudo_pass = getpass.getpass("sudo password? (or set $SUDO_PW): ")
config = Config(overrides={'sudo': {'password': sudo_pass}})
c = Connection(host='165.227.33.196',
user=os.environ['POSTGRES_US'],
config=config)
with c.cd('hellowebbooks/hellowebbooks'):
c.run('git pull origin master')
c.sudo('systemctl restart gunicorn')
c.sudo('journalctl -u gunicorn --since "1 min ago"')
def publish_container(root, argsi, version):
"""Publish tested container to dockerhub"""
print(" - Publish container to dockerhub.")
v = vagrant.Vagrant(root=root)
con = Connection(v.user_hostname_port(),
connect_kwargs={
"key_filename": v.keyfile(),
})
login_to_docker = ("docker login -u opene -p " + args.password)
con.sudo(login_to_docker)
if args.dpl == True:
print(" - Publishing with tag latest.")
set_tag_latest = ("docker tag opene/joviandss-csi:" + version +
" opene/joviandss-csi:latest")
con.sudo(set_tag_latest)
upload_latest = "docker push opene/joviandss-csi:latest"
con.sudo(upload_latest)
if args.dpv == True:
print(" - Publishing with tag " + version)
upload_latest = "docker push opene/joviandss-csi:" + version
con.sudo(upload_latest)
return
def gitsetup(ctx, url, repo, virtualenvdir):
# url example: 'dev.tetraoutcomes.com'
# repo example: 'tetra.git'
# virtualenv example: 'dev')
conn = Connection(ctx.host, ctx.user, connect_kwargs=ctx.connect_kwargs)
print('***************************set up sites directory ****************************')
conn.sudo('mkdir -p /home/django/sites/{}/logs/'.format(url))
conn.sudo('chown -R django:django /home/django/sites')
print('*** git clone ***')
with conn.cd('/home/django/sites/{}'.format(url)):
#conn.run('git clone [email protected]:tkwon/{} src'.format(repo))
conn.run('virtualenv {} --python=python3'.format(virtualenvdir))
conn.run('{}/bin/pip3 install pip-tools'.format(virtualenvdir))
def deploy_and_compile(c):
sudo_pass = getpass.getpass("Enter your sudo password on %s: " % SERVER)
config = Config(overrides={'sudo': {'password': sudo_pass}})
c = Connection(SERVER, config=config)
c.sudo('bash -c "cd %s && git pull"' % (DIR))
output = c.sudo('docker inspect --format="{{.State.Status}}" %s' %
(NODE_CONTAINER))
if output.stdout.strip() != 'running':
raise Exit(
'\n*** ERROR: The %s container, used to compile the assets, is '
'not running. Please build/run/start the container.' %
(NODE_CONTAINER))
c.sudo('docker exec %s yarn build:production' % (NODE_CONTAINER))
def deploy(c):
remote_user = "******"
remote_password = "******"
remote_host = "159.65.6.97"
config = Config(overrides={'sudo': {'password': remote_password}})
connect_kwarg = {'password': remote_password, 'allow_agent': False}
conn = Connection(host=remote_host,
user=remote_user,
config=config,
connect_kwargs=connect_kwarg)
print("Success")
conn.put("app.py")
conn.put("config.json")
conn.put("meal.json")
conn.put("orders.json")
conn.put("promo.json")
conn.put("users.json")
conn.put("promotions.json")
print("Success!")
print("Install requirements:")
conn.sudo("pip3 install Flask Flask-CORS")
conn.sudo("pip3 install twilio")
print("Killdown")
conn.sudo("pkill -F server.pid", warn=True)
print("Start server")
conn.sudo("nohup python3 app.py &> logs.txt & echo $! > server.pid")
conn.close()
def start_luigi_scheduler(conn: Connection, instance):
"""Start the Luigi scheduling server"""
ip = instance["ip"]
url = f"http://{ip}:8082/static/visualiser/index.html"
logger.info("Starting Luigi scheduling server")
log_dir = "/home/ubuntu/code/data/outputs/luigid"
conn.run(f"mkdir -p {log_dir}")
cmd_str = ("/home/ubuntu/code/env/bin/luigid"
" --background"
f" --logdir {log_dir}"
" --address 0.0.0.0"
" --port 8082")
conn.sudo(cmd_str, echo=True)
logger.info("Started Luigi scheduling server")
logger.info("Luigi server available at %s", url)
def set_ntp(c):
ntp_server = config.server.ntp_server
ntp_conn = Connection(ntp_server, config=ssh_config)
ntp_conn.sudo('service ntpd restart', pty=True)
sudo_conn.sudo('ntpdate {}'.format(ntp_server), pty=True, warn=True)
sudo_conn.sudo('''sh -c "echo '#!/bin/bash' > /etc/cron.daily/myntp"''',
pty=True)
sudo_conn.sudo(
'''sh -c "echo 'ntpdate {}' >> /etc/cron.daily/myntp"'''.format(
ntp_server),
pty=True)
sudo_conn.sudo('chmod 755 /etc/cron.daily/myntp', pty=True)
ntp_conn.sudo('rm -f /etc/cron.daily/myntp', pty=True, warn=True)
def manager_fix():
"""
Uses fabric to securely use a sudo password, ssh to each server and restart mysql_manager.
:return: None, print info to screen
"""
sudo_pass = getpass("Env Password: "******"/home/rob/Desktop/list.txt", "r") as f:
servers = f.readlines()
servers = [i.strip("\n") for i in servers]
for i in servers:
conn = Connection(i, config=config)
conn.sudo("service mysql_manager restart")
return None
def build_code(root, version):
v = vagrant.Vagrant(root=root)
cd_src = "cd ~/go/src/github.com/open-e/JovianDSS-KubernetesCSI; "
# Get dependency
cmd = cd_src + "go get ./... ;"
con = Connection(v.user_hostname_port(),
connect_kwargs={
"key_filename": v.keyfile(),
})
out = con.run(cmd)
# Start plugin
cmd = cd_src + "make joviandss-container;"
con = Connection(v.user_hostname_port(),
connect_kwargs={
"key_filename": v.keyfile(),
})
out = con.run(cmd)
cmd = ("sudo docker save -o ~/go/src/github.com/open-e/" +
"JovianDSS-KubernetesCSI/_output/joviandss-csi:" + version +
" opene/joviandss-csi:" + version)
con = Connection(v.user_hostname_port(),
connect_kwargs={
"key_filename": v.keyfile(),
})
out = con.sudo(cmd)
def virt_list(host_start, host_end, site):
"""
First collect a sudo password. Next we dig with the +short flag and if we get an IP address back we continue with
ping. If no IP back we skip to the next host. If we get an IP we ping the host to make sure it actually is alive.
Not alive? Iterate to the next host. Alive? SSH to it and and print out the result of virsh list --all.
Since this requires ping I've added a progress bar as it takes a bit to work through all of the hosts and otherwise
looks stalled.
:param host_start: server number to start at
:param host_end: server number to end at
:param site: physical site of servers, e.g. tym
:return: None, print info to screen
"""
sudo_pass = getpass("Env Password: "******"Progress"):
dig = str(
run(f"dig +short vm{i}.product.{site}.domain.com",
shell=True,
stdout=PIPE))
dig = ''.join(dig)
if "stdout=b''" in dig:
continue
else:
ping = str(
run(f"ping -c 4 vm{i}.{site}.domain.com",
shell=True,
stdout=PIPE))
ping = ''.join(ping)
if "redirect.com" in ping or "100% packet loss" in ping:
continue
else:
conn = Connection(f"vm{i}.{site}.domain.com", config=config)
virsh_all = str(conn.sudo("virsh list --all", hide="stdout"))
print(f"vm{i}.{site}.domain.com\n{virsh_all}")
return None
def job(self, run: "Run", payload: dict, device: Device) -> dict:
username, password = run.get_credentials(device)
fabric_connection = Connection(
host=device.ip_address,
port=device.port,
user=username,
connect_kwargs={"password": password},
)
source_code = run.sub(run.source_code, locals())
match = run.sub(run.content_match, locals())
run.log("info", f"Running Unix Shell Script {self.name} on {device.name}")
script_file_name = "unix_shell_script_service.sh"
with StringIO(run.source_code) as script_file:
fabric_connection.put(script_file, script_file_name)
if run.privileged_mode:
if not device.enable_password:
raise Exception(
f"Service {self.name} requested privileged mode on device "
f"with no configured enable_password: {device.name}"
)
result = fabric_connection.sudo(
f"bash {script_file_name}", password=device.enable_password
)
else:
result = fabric_connection.run(f"bash {script_file_name}")
fabric_connection.run(f"rm {script_file_name}")
return {
"match": match,
"negative_logic": run.negative_logic,
"result": f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}",
"success": result.ok and run.match_content(result, match),
}
fabric_config['connect_kwargs'] = {
"key_filename": "c:\Users\Guodong\.ssh\exportedkey201310171355",
}
# Superuser privileges via auto-response
sudo_pass_auto_respond = Responder(
pattern=r'\[sudo\] password:'******'mypassword\n',
)
# create connection
cxn = Connection('192.168.88.19', config=fabric_config)
# do tasks on host
print cxn.run("uname -a", hide=True).stdout
print cxn.sudo("whoami", hide=True).stdout
cxn.run('sudo whoami', pty=True, watchers=[sudo_pass_auto_respond])
cxn.put(__file__, "/tmp/this.py")
cxn.run("sudo rm -f /tmp/this.py")
# cxn.get("/tmp/this.py", "this.py")
print disk_free(cxn)
# config multiple servers with methods 1
for host in ('192.168.88.19', '192.168.88.20', '192.168.88.21'):
result = Connection(host, config=fabric_config).run('uname -s', hide=True)
print("{}: {}".format(host, result.stdout.strip()))
# config multiple servers, M2
results = Group('192.168.88.19', '192.168.88.20', '192.168.88.21', config=fabric_config).run('uname -s', hide=True)
for connection, result in results.items():
print("{0.host}: {1.stdout}".format(connection, result))
