def subservient_to_runtime_overrides(self):
env = self.env
env.sudo_password = "******"
config = Config.from_v1(
env, overrides={"sudo": {"password": "******"}}
)
assert config.sudo.password == "runtime"
def connect_kwargs_also_merged_with_imported_values(self):
self.env["key_filename"] = "whatever"
conf = Config.from_v1(
self.env, overrides={"connect_kwargs": {"meh": "effort"}}
)
assert conf.connect_kwargs["key_filename"] == "whatever"
assert conf.connect_kwargs["meh"] == "effort"
def does_not_affect_explicit_object(self, method):
sc = SSHConfig()
c = Config(ssh_config=sc, overrides={"load_ssh_configs": False})
# Implicit loading still doesn't happen...sanity check
assert not method.called
# Real test: the obj we passed in is present as usual
assert c.base_ssh_config is sc
def defaults_to_merger_of_global_defaults(self):
# I.e. our global_defaults + Invoke's global_defaults
c = Config()
# From invoke's global_defaults
assert c.run.warn is False
# From ours
assert c.port == 22
def setup_squid(self, servers, startup_script_file_location,
squid_service_location):
service_name = squid_service_location.replace('.service', '')
for x in servers:
sudo_config = Config(overrides={'sudo': {'password': x.password}})
server_connection = Connection(
host=x.ip_address,
user=x.username,
connect_kwargs={"password": x.password},
config=sudo_config)
server_connection.sudo('apt-get install dos2unix')
time.sleep(1)
file_transfer_result = server_connection.put(
startup_script_file_location, remote='/usr/bin/')
server_connection.sudo('chmod +x /usr/bin/' +
startup_script_file_location)
server_connection.sudo('dos2unix /usr/bin/' +
startup_script_file_location)
file_transfer_result = server_connection.put(
squid_service_location, remote='/etc/systemd/system/')
server_connection.sudo(f'systemctl enable {service_name}')
try:
server_connection.sudo('reboot')
except UnexpectedExit as e:
print("Caught error while rebooting machine")
def rollback():
'''回滚版本'''
# 配置sudo命令的密码
config = Config(overrides={'sudo': {'password': remote_su_pass}})
# 以明文方式配置用户登录密码
conn = Connection(ip,
user=remote_user,
config=config,
connect_kwargs={
"allow_agent": False,
"password": remote_pass
})
with conn.cd(remote_wwwroot):
r = conn.run("ls -p -1", hide=True) # hide禁用服务器将输出复制到中断的行为
files = [
s[:-1] for s in RE_FILES.split(r.stdout)
if s.startswith('www-') and s.endswith('/')
]
files.sort(reverse=True)
r = conn.run('ls -l www', hide=True)
ss = r.stdout.split(' -> ')
if len(ss) != 2:
print('ERROR: \'www\' is not a symbol link.')
return
current = ss[1].split('/')[-1].strip()
print('Found current symbol link points to: %s\n' % current)
try:
print(files, "*", current, "*")
index = files.index(current)
except ValueError as e:
print('ERROR: symbol link is invalid.')
return
if len(files) == index + 1:
print('ERROR: already the oldest version.')
return
old = files[index + 1]
print('==================================================')
for f in files:
if f == current:
print(' Current ---> %s' % current)
elif f == old:
print(' Rollback to ---> %s' % old)
else:
print(' %s' % f)
print('==================================================')
print('')
yn = input('continue? y/N ')
if yn != 'y' and yn != 'Y':
print('Rollback cancelled.')
return
print('Start rollback...')
conn.sudo('rm -f {}/www'.format(remote_wwwroot))
conn.sudo('ln -s {}/{} {}/www'.format(remote_wwwroot, old, remote_wwwroot))
conn.sudo('chown -h www-data:www-data {}/www'.format(remote_wwwroot))
conn.sudo('chown -H -R www-data:www-data {}/www'.format(remote_wwwroot))
conn.sudo('supervisorctl stop awesome')
conn.sudo('supervisorctl start awesome')
conn.sudo('/etc/init.d/nginx reload')
print('ROLLBACKED OK.')
def both_paths_loaded_if_both_exist_with_user_winning(self):
c = Config(user_ssh_path=self._user_path,
system_ssh_path=self._system_path)
names = c.base_ssh_config.get_hostnames()
expected = {"user", "system", "shared", "*"}
assert names == expected
# Expect the user value (321), not the system one (123)
assert c.base_ssh_config.lookup("shared")["port"] == "321"
def subservient_to_runtime_overrides(self):
env = self.env
env.sudo_password = "******"
config = Config.from_v1(
env, overrides={"sudo": {
"password": "******"
}})
assert config.sudo.password == "runtime"
def connect_kwargs_also_merged_with_imported_values(self):
self.env["key_filename"] = "whatever"
conf = Config.from_v1(
self.env, overrides={"connect_kwargs": {
"meh": "effort"
}})
assert conf.connect_kwargs["key_filename"] == "whatever"
assert conf.connect_kwargs["meh"] == "effort"
def has_various_Fabric_specific_default_keys(self):
c = Config()
assert c.port == 22
assert c.user == get_local_user()
assert c.forward_agent is False
assert c.connect_kwargs == {}
assert c.timeouts.connect is None
assert c.ssh_config_path is None
def defaults_to_empty_sshconfig_obj_if_no_files_found(self):
c = Config(**self._empty_kwargs)
# TODO: Currently no great public API that lets us figure out if
# one of these is 'empty' or not. So for now, expect an empty inner
# SSHConfig._config from an un-.parse()d such object. (AFAIK, such
# objects work fine re: .lookup, .get_hostnames etc.)
assert type(c.base_ssh_config) is SSHConfig
assert c.base_ssh_config._config == []
def runtime_path_does_not_die_silently(self):
try:
Config(runtime_ssh_path="sure/thing/boss/whatever/you/say")
except IOError as e:
assert "No such file or directory" in str(e)
assert e.errno == errno.ENOENT
else:
assert False, "Bad runtime path didn't raise IOError!"
def does_not_skip_loading_runtime_path(self, method):
Config(
runtime_ssh_path=self._runtime_path,
overrides={"load_ssh_configs": False},
)
# Expect that loader method did still run (and, as usual, that
# it did not load any other files)
method.assert_called_once_with(self._runtime_path)
def setup(self):
# NOTE: assumes a user configured for passworded (NOT
# passwordless)_sudo, whose password is 'mypass', is executing the
# test suite. I.e. our travis-ci setup.
config = Config(
{"sudo": {"password": "******"}, "run": {"hide": True}}
)
self.cxn = Connection("localhost", config=config)
def deploy(local):
sudo_pass = getpass.getpass("[sudo] password: "******"git push")
remote = Connection("abig", config=config)
remote.run("cd /home/rc/webapps/topicos4 && git pull")
remote.run("cd /home/rc/webapps/topicos4 && source /home/rc/.virtualenvs/topicos4/bin/activate && pip install -r requirements.txt")
remote.sudo("sudo supervisorctl restart topicos4")
def __init__(self, cluster):
self.cluster = cluster
# Autentication
self.sudo_pass = self.cluster.get(
'password') #getpass.getpass("What's your sudo password?")
self.config = Config(overrides={'sudo': {'password': self.sudo_pass}})
self.cnx = None
def global_defaults():
defaults = FabConfig.global_defaults()
ours = {
'platform': 'Win64',
'configuration': 'DevelopmentEditor',
'type': 'Editor',
}
merge_dicts(defaults, ours)
return defaults
def deploy(host, user, password, version, resetdb, resetconf, component):
''' deploy services to specified machine via fabric
example:
python3 -m tao.tools deploy_docker -h 10.1.2.13 -p dx@666 -c console-ui:master -c console:master
python3 -m tao.tools deploy_docker -h 10.1.2.13 -p dx@666 -c harbor.dx-corp.top/ctu-group/ctu-console-ui:master
'''
config = Config(overrides={'sudo': {'password': password}})
sql_component = [c for c in component if c.name == _SQL_COMPONENT]
if sql_component:
sql_version = sql_component[0].version
else:
sql_version = version or 'master'
_echo("sql_version: %s" % sql_version)
component = [c for c in component if c.name != _SQL_COMPONENT]
if not component:
_echo("no components to deploy")
raise RuntimeError
with Connection(host,
user,
config=config,
connect_kwargs={'password': password}) as conn:
conn.version = version
conn.reset_db = resetdb
conn.reset_conf = resetconf
conn.run('test -e docker || mkdir docker') # create docker directory
all_containers = conn.run('docker ps -a')
container_names = [
e.split()[-1] for e in all_containers.stdout.splitlines()[1:]
]
_echo('login to harbor')
_docker_login(conn)
if conn.reset_db:
_echo('init mysql database')
_init_mysql(conn, sql_version)
_echo('init license file')
_init_license(conn)
_echo('init zookeeper')
if 'zookeeper' in container_names:
_echo('"zookeeper" is already running!')
else:
conn.run(
'docker run --name zookeeper --restart always --network host --log-opt max-size=256m -d harbor.dx-corp.top/basic/zookeeper:3.4.10'
)
time.sleep(5)
component.sort(key=_sort_component)
for app_item in component:
_echo('start to deploy %s:%s' % (app_item.image, app_item.version))
_do_deploy(conn, app_item, container_names)
if 'jmxmon' not in container_names:
_echo('jmxmon not running, start it!')
conn.run(
'docker run --rm -d --network host --name jmxmon harbor.dx-corp.top/aladdin/jmxmon:0.0.2'
)
_echo('sleep 5s and show running containers')
time.sleep(5)
conn.run('docker ps')
def create_session_on_server(ssh_config_path, email):
config = Config(runtime_ssh_path=ssh_config_path)
host = _get_hostname(ssh_config_path)
with Connection(host=host, config=config) as c:
manage_dot_py = _get_manage_dot_py(c.host)
env_vars = _get_server_env_vars(c, c.host)
c.config.run.env = env_vars
session_key = c.run(f'{manage_dot_py} create_session {email}',
hide=True).stdout.strip()
return session_key
def forwards_arbitrary_kwargs_to_init(self):
config = Config.from_v1(
self.env,
# Vanilla Invoke
overrides={"some": "value"},
# Fabric
system_ssh_path="/what/ever",
)
assert config.some == "value"
assert config._system_ssh_path == "/what/ever"
def both_paths_loaded_if_both_exist_with_user_winning(self):
c = Config(
user_ssh_path=self._user_path,
system_ssh_path=self._system_path,
)
names = c.base_ssh_config.get_hostnames()
expected = {'user', 'system', 'shared', '*'}
assert names == expected
# Expect the user value (321), not the system one (123)
assert c.base_ssh_config.lookup('shared')['port'] == '321'
def global_defaults():
defaults = FabConfig.global_defaults()
ours = {
'tasks': {
'search_root': 'fub',
'collection_name': 'tasks',
},
}
merge_dicts(defaults, ours)
return defaults
def func():
"""
Description
"""
sudo_pass = getpass("Env Password: "******"vm{i}.{site}.domain.com", config=config)
virsh_all = str(conn.sudo("virsh list --all", hide="stdout"))
return None
def forwards_arbitrary_kwargs_to_init(self):
config = Config.from_v1(
self.env,
# Vanilla Invoke
overrides={"some": "value"},
# Fabric
system_ssh_path="/what/ever",
)
assert config.some == "value"
assert config._system_ssh_path == "/what/ever"
def get_sudo():
'''
Create Config object with sudo password
:return: Fabric/Config object
'''
return Config(overrides={
'sudo': {
'password': getpass("What's your sudo password? ")
}
})
def deploy_and_reload(c):
"Runs 'git pull' and reloads Nginx"
for host in HOSTS:
print('\n\n*** CONNECTING TO: %s' % (host))
config = Config(overrides={'sudo': {'password': sudo_pass}})
c = Connection(host, config=config)
c.run('cd %s && git pull [email protected]:openstate/%s.git' %
(DIR, GIT_REPO))
c.sudo('bash -c "cd %s && ./reload.sh"' % (DIR))
def deploy(ctx):
timestr = time.strftime("%Y%m%d-%H%M%S")
# print("Hello world!")
sudo_pass = getpass.getpass("What's your sudo password?")
config = Config(overrides={"sudo": {"password": sudo_pass}})
c = Connection("*****@*****.**", config=config)
# c.run(
# "mysqldump -u root -p'"
# + sudo_pass
# + "' meal_project > ~/mysql_backups/meal_project_"
# + timestr
# + ".sql"
# )
c.sudo("docker exec mysql mysqldump -u root -p'" + sudo_pass +
"' meal_project > ~/mysql_backups/meal_project_" + timestr + ".sql")
## from mysql docker site:
##$ docker exec some-mysql sh -c 'exec mysqldump --all-databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > /some/path/on/your/host/all-databases.sql
site_folder = "/home/odroid/meal_project"
c.run(f"mkdir -p {site_folder}")
# current_commit = c.local("HOME=~ git log -n 1 --format=%H")
with c.cd(site_folder):
if c.run("test -d .git", warn=True).failed:
c.run(f"git clone {REPO_URL} .")
else:
# c.run("git pull origin master --force")
c.run("git fetch")
c.run("git reset --hard origin/master")
# PIP load from requirements.txt
venv_command = "source venv/bin/activate"
pip_command = "venv/bin/pip3 install -r requirements.txt"
c.run(venv_command + " && " + pip_command)
# Reload static files
static_cmd = "python manage.py collectstatic --noinput"
c.run(venv_command + " && " + static_cmd)
# Database Make Migration
makem_cmd = "python manage.py makemigrations --settings=meal_project.settings.production"
c.run(venv_command + " && " + makem_cmd)
# Database Migrate
migrate_cmd = (
"python manage.py migrate --settings=meal_project.settings.production"
)
c.run(venv_command + " && " + migrate_cmd)
# Restart Apache
c.sudo("service apache2 restart")
def restart_server(c):
password = getpass.getpass('Your sudo password: '******'sudo': {'password': password}})
conn = Connection(host, config=config)
commands = [
'supervisorctl stop {}'.format(service_name),
'supervisorctl start {}'.format(service_name),
'/etc/init.d/nginx reload',
]
for cmd in commands:
conn.sudo(cmd, hide='stderr')
def connect(host):
remote_config = Config()
settings = {"hide": True, "warn": True}
remote_config["sudo"].update(settings)
remote_config["run"].update(settings)
if False: # ignore sudo for now, will be required when working
sudo_pass = getpass.getpass("sudo password: "******"sudo"].update({"password": sudo_pass})
return Connection(host, config=remote_config)
def deploy(c):
sudo_pass = getpass.getpass("Enter your sudo password on %s: " % SERVER)
config = Config(overrides={'sudo': {'password': sudo_pass}})
c = Connection(SERVER, config=config)
# Pull from GitHub
c.run('bash -c "cd %s && git pull [email protected]:openstate/%s.git"' %
(DIR, GIT_REPO))
# Reload app
c.run('bash -c "cd %s && touch uwsgi-touch-reload"' % (DIR))
def conn(c=None, one=False, invoke=False):
if globing.invoke or invoke:
from invoke import Context
c = Context(Config())
c.host = 'local'
return c
""" 确保传入的是connect，不是local的context
"""
if not hasattr(c, 'host'):
c = hosts.one() if one else hosts.conn(0)
print("connection [{}]".format(c.host))
return c
def _establish_fabric_connection(self) -> None:
if not self._connection:
config = Config(overrides={'sudo': {'password': self._password}})
self._connection = Connection(self._ipaddress,
config=config,
user=self._username,
connect_timeout=CONNECTION_TIMEOUT,
connect_kwargs={
'password': self._password,
'allow_agent': False,
'look_for_keys': False
})
def _prepare(self):
self.ssh_is_up = threading.Event()
self.ssh_up_thread_termination = threading.Event()
self.ssh_config = Config(
overrides={
'load_ssh_config': False,
'UserKnownHostsFile': self.known_hosts_file,
'ServerAliveInterval': 300,
'StrictHostKeyChecking': 'no'
})
self.start_ssh_up_thread()
super()._prepare()
def deploy_and_up(c):
("Runs 'git pull', 'docker-compose up -d' (required if you update "
"Nginx version) and reloads Nginx")
for host in HOSTS:
print('\n\n*** CONNECTING TO: %s' % (host))
config = Config(overrides={'sudo': {'password': sudo_pass}})
c = Connection(host, config=config)
c.run('cd %s && git pull [email protected]:openstate/%s.git' %
(DIR, GIT_REPO))
c.sudo('bash -c "cd %s && docker-compose up -d"' % (DIR))
c.sudo('bash -c "cd %s && ./reload.sh"' % (DIR))
def may_use_lazy_plus_explicit_methods_to_control_flow(self, method):
c = Config(lazy=True)
assert not method.called
c.set_runtime_ssh_path(self._runtime_path)
c.load_ssh_config()
method.assert_called_once_with(self._runtime_path)
def ssh_config_path(self):
self.env.ssh_config_path = "/where/ever"
config = Config.from_v1(self.env, lazy=True)
assert config.ssh_config_path == "/where/ever"
def _conf(self, **kwargs):
self.env.update(kwargs)
return Config.from_v1(self.env)
def must_be_given_explicit_env_arg(self):
config = Config.from_v1(
env=Lexicon(self.env, sudo_password="******")
)
assert config.sudo.password == "sikrit"
