def deploy(): publish_conf() SerialGroup(*HOST_ADDR).run('cd %s && python ctrl.py pull %s' % (REMOTE_CONF_DIR, IMAGE_URL)) debug('pull image done.') SerialGroup(*HOST_ADDR).run('cd %s && python ctrl.py run %s' % (REMOTE_CONF_DIR, IMAGE_URL)) debug('run container done.')
def retrieve_smis(server_list, out_filename=".smi_saves"): out_file = open(out_filename, "w", encoding="utf-8") server_group = SerialGroup(*server_list) try: return server_group.run('echo "new connection:" && hostname && \ nvidia-smi', out_stream=out_file, hide=True) except GroupException as e: problem_list = [] for key, val in e.result.items(): if type(val) is not Result or val.failed: problem_list.append(key.original_host) return 'Err', problem_list
def load_es_package(): ctx = SerialGroup(*deploy_hosts) ctx.run('mkdir -p %s' % file_load_path) ctx.run( 'cd %s && wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.2.tar.gz' ' -e use_proxy=yes -e http_proxy=rec-httpproxy01:3128' % file_load_path) ctx.run('cd %s && tar -zxvf elasticsearch-6.2.2.tar.gz' % file_load_path)
def execute_on_hosts(cls, args, command): coordinator = Connection(args.config["coordinator"][0]) cls.service(coordinator, command, True) workers = SerialGroup(*(args.config["workers"])) for host in workers: cls.service(host, command, False)
def errors_in_execution_capture_and_continue_til_end(self): cxns = [Mock(name=x) for x in ("host1", "host2", "host3")] class OhNoz(Exception): pass onoz = OhNoz() cxns[1].run.side_effect = onoz g = SerialGroup.from_connections(cxns) try: g.run("whatever", hide=True) except GroupException as e: result = e.result else: assert False, "Did not raise GroupException!" succeeded = { cxns[0]: cxns[0].run.return_value, cxns[2]: cxns[2].run.return_value, } failed = {cxns[1]: onoz} expected = succeeded.copy() expected.update(failed) assert result == expected assert result.succeeded == succeeded assert result.failed == failed
def errors_in_execution_capture_and_continue_til_end(self): cxns = [Mock(name=x) for x in ("host1", "host2", "host3")] class OhNoz(Exception): pass onoz = OhNoz() cxns[1].run.side_effect = onoz g = SerialGroup.from_connections(cxns) try: g.run("whatever", hide=True) except GroupException as e: result = e.result else: assert False, "Did not raise GroupException!" succeeded = { cxns[0]: cxns[0].run.return_value, cxns[2]: cxns[2].run.return_value, } failed = {cxns[1]: onoz} expected = succeeded.copy() expected.update(failed) assert result == expected assert result.succeeded == succeeded assert result.failed == failed
def install_cmd(cls, args): logging.info("Installing packages %s" % args.rpm) coordinator = Connection(args.config["coordinator"][0]) cls.install(coordinator, args, True) workers = SerialGroup(*(args.config["workers"])) for host in workers: cls.install(host, args, False)
def returns_results_mapping(self): cxns = [Mock(name=x) for x in ("host1", "host2", "host3")] g = SerialGroup.from_connections(cxns) result = g.run("whatever", hide=True) assert isinstance(result, GroupResult) expected = {x: x.run.return_value for x in cxns} assert result == expected assert result.succeeded == expected assert result.failed == {}
def returns_results_mapping(self): cxns = [Mock(name=x) for x in ("host1", "host2", "host3")] g = SerialGroup.from_connections(cxns) result = g.run("whatever", hide=True) assert isinstance(result, GroupResult) expected = {x: x.run.return_value for x in cxns} assert result == expected assert result.succeeded == expected assert result.failed == {}
def executes_arguments_on_contents_run_serially(self): "executes arguments on contents' run() serially" cxns = [Connection(x) for x in ("host1", "host2", "host3")] args = ("command",) kwargs = {"hide": True, "warn": True} for index, cxn in enumerate(cxns): side_effect = _make_serial_tester(cxns, index, args, kwargs) cxn.run = Mock(side_effect=side_effect) g = SerialGroup.from_connections(cxns) g.run(*args, **kwargs) # Sanity check, e.g. in case none of them were actually run for cxn in cxns: cxn.run.assert_called_with(*args, **kwargs)
def executes_arguments_on_contents_run_serially(self): "executes arguments on contents' run() serially" cxns = [Connection(x) for x in ("host1", "host2", "host3")] args = ("command",) kwargs = {"hide": True, "warn": True} for index, cxn in enumerate(cxns): side_effect = _make_serial_tester(cxns, index, args, kwargs) cxn.run = Mock(side_effect=side_effect) g = SerialGroup.from_connections(cxns) g.run(*args, **kwargs) # Sanity check, e.g. in case none of them were actually run for cxn in cxns: cxn.run.assert_called_with(*args, **kwargs)
def executes_arguments_on_contents_run_serially(self, method): "executes arguments on contents' run() serially" cxns = [Connection(x) for x in ("host1", "host2", "host3")] args = ARGS_BY_METHOD[method] kwargs = KWARGS_BY_METHOD[method] for index, cxn in enumerate(cxns): side_effect = _make_serial_tester(method, cxns, index, args, kwargs) setattr(cxn, method, Mock(side_effect=side_effect)) g = SerialGroup.from_connections(cxns) getattr(g, method)(*args, **kwargs) # Sanity check, e.g. in case none of them were actually run for cxn in cxns: getattr(cxn, method).assert_called_with(*args, **kwargs)
def __init__(self): host_list = open('InstancesConfigurations/public_ips', 'r').read().splitlines() self.connections = [] self.pool = Serial() party_id = 0 for host in host_list: c = Connection(host, user='******', connect_kwargs={ 'key_filename': ['%s/Keys/matrix.pem' % Path.home()] }) c.party_id = party_id party_id += 1 self.connections.append(c) self.pool.append(c)
def create_connections_(cfg, sudo_passwd, pty): """ Create connections from the config and stage. """ roles = cfg.settings['roles'] stage = roles[cfg.stage] target_hosts = stage['target-hosts'] cf_settings = { 'run': { 'echo': True, 'env': dict(os.environ), 'pty': pty, }, 'sudo': { 'password': sudo_passwd }, } cf = ConnectionConfig(cf_settings) group = SerialGroup(*target_hosts, config=cf) cfg.conn_pool = group cf = invoke.config.Config(cf_settings) ctx = invoke.context.Context(cf) cfg.invoker = ctx
def config_env(): ctx = SerialGroup(*deploy_hosts) ctx.run("whoami") ctx.run("hostname") ctx.run( "echo 'set nu\nset incsearch\nset hlsearch\nset background=dark\n' >> ~/.vimrc " ) ctx.run("echo 'export ES_JAVA_OPTS=\"-Xms28g -Xmx28g\"\n" "export ES_HEAP_SIZE=28G' >> /home/app/.bash_profile ") ctx.run('sudo sysctl -w vm.max_map_count=262144')
# load config config = configparser.ConfigParser() config.read('config.ini') # set coordinator coordinator_connections = [] coordinator_user = config['COORDINATOR'].pop('user') coordinator_password = config['COORDINATOR'].pop('password') coordinator_catalog_path = config['COORDINATOR'].pop('catalog_path') for host_key in config['COORDINATOR'].keys(): host = config['COORDINATOR'][host_key] conn = Connection(host=host, user=coordinator_user, connect_kwargs={'password': coordinator_password}) coordinator_connections.append(conn) coordinator_group = SerialGroup.from_connections(coordinator_connections) # set worker worker_connections = [] worker_user = config['WORKER'].pop('user') worker_password = config['WORKER'].pop('password') worker_catalog_path = config['WORKER'].pop('catalog_path') for host_key in config['WORKER'].keys(): host = config['WORKER'][host_key] conn = Connection(host=host, user=worker_user, connect_kwargs={'password': worker_password}) worker_connections.append(conn) worker_group = SerialGroup.from_connections(worker_connections)
from fabric import Connection, Config from fabric import SerialGroup as Group # create group of remote servers group = Group('localhost', 'localhost', user='******', port=2222) # apply commands to multiple servers group.run('whoami') group.run('uname -a') # return GroupResult obj results = group.run('whoami') # show specific log for connection, result in results.items(): print("{0.host}: {1.stdout}".format(connection, result))
def main(cmd, path=LOG_PATH): for cxn in SerialGroup(*HOSTS): print_result(cxn.host, cd_and_run(cxn, cmd, path))
"key_filename": "c:\Users\Guodong\.ssh\exportedkey201310171355", } # Superuser privileges via auto-response sudo_pass_auto_respond = Responder( pattern=r'\[sudo\] password:'******'mypassword\n', ) # create connection cxn = Connection('192.168.88.19', config=fabric_config) # do tasks on host print cxn.run("uname -a", hide=True).stdout print cxn.sudo("whoami", hide=True).stdout cxn.run('sudo whoami', pty=True, watchers=[sudo_pass_auto_respond]) cxn.put(__file__, "/tmp/this.py") cxn.run("sudo rm -f /tmp/this.py") # cxn.get("/tmp/this.py", "this.py") print disk_free(cxn) # config multiple servers with methods 1 for host in ('192.168.88.19', '192.168.88.20', '192.168.88.21'): result = Connection(host, config=fabric_config).run('uname -s', hide=True) print("{}: {}".format(host, result.stdout.strip())) # config multiple servers, M2 results = Group('192.168.88.19', '192.168.88.20', '192.168.88.21', config=fabric_config).run('uname -s', hide=True) for connection, result in results.items(): print("{0.host}: {1.stdout}".format(connection, result))
def restart(): SerialGroup(*HOST_ADDR).run('cd %s && python ctrl.py restart' % REMOTE_CONF_DIR)
def clean(): SerialGroup(*HOST_ADDR).run('cd %s && python ctrl.py clean' % REMOTE_CONF_DIR)
"""tests for the 'fabric' package (v2.x) Most of these examples are taken from the fabric documentation: http://docs.fabfile.org/en/2.5/getting-started.html See fabric-LICENSE for its' license. """ from fabric import Connection c = Connection('web1') result = c.run('uname -s') c.run(command='echo run with kwargs') from fabric import SerialGroup as Group results = Group('web1', 'web2', 'mac1').run('uname -s') from fabric import SerialGroup as Group pool = Group('web1', 'web2', 'web3') pool.run('ls') # using the 'fab' command-line tool from fabric import task @task def upload_and_unpack(c): if c.run('test -f /opt/mydata/myfile', warn=True).failed: c.put('myfiles.tgz', '/opt/mydata') c.run('tar -C /opt/mydata -xzvf /opt/mydata/myfiles.tgz')
def stop(): SerialGroup(*HOST_ADDR).run('cd %s && python ctrl.py stop' % REMOTE_CONF_DIR)
def disk_free(c): uname = c.run('uname -s', hide=True) if 'Linux' in uname.stdout: command = "df -h / | tail -n1 | awk '{print $5}'" return c.run(command, hide=True).stdout.strip() err = "No idea how to get disk space on {}!".format(uname) raise Exception(err) # hosts = ["{}@{}".format(row[2], row[0]) for row in get_sheet(file_name='hosts.xlsx') if len([col for col in row if col]) > 1] hosts = [] for row in get_sheet(file_name='hosts.xlsx'): row = [col for col in row if col] if len(row) == 1: continue hosts.append("{}@{}".format(row[2], row[0])) pp(hosts) print() print(SerialGroup(*hosts)) print() for cxn in SerialGroup(*hosts): print("{}: {}".format(cxn, disk_free(cxn)))
c.local("cmd1; cmd2") # $getCommand="cmd1; cmd2" c.sudo("cmd1; cmd2") # $getCommand="cmd1; cmd2" c.local(command="cmd1; cmd2") # $getCommand="cmd1; cmd2" c.run(command="cmd1; cmd2") # $getCommand="cmd1; cmd2" c.sudo(command="cmd1; cmd2") # $getCommand="cmd1; cmd2" # fully qualified usage c2 = connection.Connection("web2") c2.run("cmd1; cmd2") # $getCommand="cmd1; cmd2" ################################################################################ # SerialGroup ################################################################################ results = SerialGroup("web1", "web2", "mac1").run("cmd1; cmd2") # $getCommand="cmd1; cmd2" pool = SerialGroup("web1", "web2", "web3") pool.run("cmd1; cmd2") # $getCommand="cmd1; cmd2" # fully qualified usage group.SerialGroup("web1", "web2", "mac1").run("cmd1; cmd2") # $getCommand="cmd1; cmd2" ################################################################################ # ThreadingGroup ################################################################################ results = ThreadingGroup("web1", "web2", "mac1").run("cmd1; cmd2") # $getCommand="cmd1; cmd2" pool = ThreadingGroup("web1", "web2", "web3") pool.run("cmd1; cmd2") # $getCommand="cmd1; cmd2"
sudo_pass_auto_respond = Responder( pattern=r'\[sudo\] password:'******'mypassword\n', ) # create connection cxn = Connection('192.168.88.19', config=fabric_config) # do tasks on host print cxn.run("uname -a", hide=True).stdout print cxn.sudo("whoami", hide=True).stdout cxn.run('sudo whoami', pty=True, watchers=[sudo_pass_auto_respond]) cxn.put(__file__, "/tmp/this.py") cxn.run("sudo rm -f /tmp/this.py") # cxn.get("/tmp/this.py", "this.py") print disk_free(cxn) # config multiple servers with methods 1 for host in ('192.168.88.19', '192.168.88.20', '192.168.88.21'): result = Connection(host, config=fabric_config).run('uname -s', hide=True) print("{}: {}".format(host, result.stdout.strip())) # config multiple servers, M2 results = Group('192.168.88.19', '192.168.88.20', '192.168.88.21', config=fabric_config).run('uname -s', hide=True) for connection, result in results.items(): print("{0.host}: {1.stdout}".format(connection, result))
from fabric import SerialGroup result = SerialGroup('host1','host2','host3','host4','host5').run('hostname')
def date(): SerialGroup(*HOST_ADDR).run('cd %s && python ctrl.py date' % REMOTE_CONF_DIR)
from fabric import SerialGroup result = SerialGroup('demo.itversity.com', 'cdhhost.itversity.com').run('hostname') result
def exec_cmd(cmd): SerialGroup(*HOST_ADDR).run('cd %s && python ctrl.py exec_cmd "%s"' % (REMOTE_CONF_DIR, cmd))
class FabricTasks: def __init__(self): host_list = open('InstancesConfigurations/public_ips', 'r').read().splitlines() self.connections = [] self.pool = Serial() party_id = 0 for host in host_list: c = Connection(host, user='******', connect_kwargs={ 'key_filename': ['%s/Keys/matrix.pem' % Path.home()] }) c.party_id = party_id party_id += 1 self.connections.append(c) self.pool.append(c) def pre_process(self, task_idx): """ Execute pre process tasks on the remote hosts. :param task_idx: int :return: 0 on success, 1 on failure """ for conn in self.pool: conn.put('Execution/pre_process.py', Path.home()) try: self.pool.run('python3 pre_process.py %s' % task_idx) except BaseException as e: print(e.message) return 1 return 0 def install_git_project(self, username, password, git_branch, working_directory, git_address, external): """ :param username: string :param password: string :param git_branch: string :param working_directory: list[string] :param git_address: list[string] :param external: string :return: """ # result = self.pool.run('ls' % working_directory, warn=True).failed # if result: self.pool.run('rm -rf %s' % working_directory) r = self.pool.run( 'git clone %s %s' % (git_address.format(username, password), working_directory)) self.pool.run('cd %s && git pull' % working_directory) self.pool.run('cd %s && git checkout %s ' % (working_directory, git_branch)) if external: self.pool.run('cd %s/MATRIX && ./build.sh') else: if self.pool.run('cd %s && ls CMakeLists.txt' % working_directory, warn=True).succeeded: self.pool.run( 'cd %s && rm -rf CMakeFiles CMakeCache.txt Makefile' % working_directory) self.pool.run('cd %s && cmake .' % working_directory) self.pool.run('cd %s && make' % working_directory) self.pool.run('cd %s && 7za -y x \"*.7z\"' % working_directory, warn=True) def run_protocol(self, cxn, config_file, args, executable_name, working_directory, party_id): with open(config_file, 'r') as data_file: data = json.load(data_file, object_pairs_hook=OrderedDict) external_protocol = json.loads(data['isExternal'].lower()) protocol_name = data['protocol'] if 'aws' in data['CloudProviders']: regions = data['CloudProviders']['aws']['regions'] elif 'azure' in data['CloudProviders']: regions = data['CloudProviders']['azure']['regions'] elif len(data['CloudProviders']) > 1: regions = data['CloudProviders']['aws']['regions'] + data[ 'CloudProviders']['scaleway']['regions'] else: regions = [] args = ' '.join(args) # local execution if len(regions) == 0: number_of_parties = len(self.pool) for idx in range(number_of_parties): if external_protocol: cxn.run('cd %s && ./%s %s %s &' % (working_directory, executable_name, idx, args)) else: cxn.run('cd %s && ./%s partyID %s %s &' % (working_directory, executable_name, idx, args)) # remote execution else: # copy parties file to hosts if len(regions) > 1: cxn.put('InstancesConfigurations/parties%s.conf' % party_id, working_directory) cxn.run('mv parties%s.conf parties.conf' % party_id) else: cxn.put( '/home/liork/MATRIX/InstancesConfigurations/parties.conf') cxn.run('mv parties.conf HyperMPC') cxn.run("kill -9 `ps aux | grep %s | awk '{print $2}'`" % executable_name, warn=True) # libscapi protocol if not external_protocol: cxn.run('cd %s && chmod +x %s' % (working_directory, executable_name)) cxn.run('cd %s && ./%s partyID %s %s &' % (working_directory, executable_name, party_id, args)) # external protocols else: # with coordinator if 'coordinatorConfig' in data: cxn.put('InstancesConfigurations/parties.conf', '%s/MATRIX' % working_directory) if protocol_name == 'SCALE-MAMBA': cxn.put('InstancesConfigurations/public_ips', '%s/MATRIX' % working_directory) cxn.put(cxn.connect_kwags.key_filename, '%s/MATRIX' % working_directory) if party_id == 0: coordinator_executable = data['coordinatorExecutable'] coordinator_args = ' '.join(data['coordinatorConfig']) cxn.run('./%s %s' % (coordinator_executable, coordinator_args)) else: # run the parties that depends in the coordinator self.pool.run('./%s %s %s' % (executable_name, party_id - 1, args)) with open('Execution/execution_log.log', 'a+') as log_file: log_file.write('%s\n' % args) else: # run external protocols with no coordinator self.pool.run('. ./%s %s %s' % (executable_name, party_id, args)) with open('Execution/execution_log.log', 'a+') as log_file: log_file.write('%s\n' % args) def run_protocol_profiler(self): pass def run_protocol_latency(self): pass def update_libscapi(self): pass def get_logs(self): pass def test_task(self, c, party_id): print(c, party_id)
from fabric import SerialGroup, Connection import os # List of master's HAProxy _haproxies_hosts = [ '10.148.0.37', ] _ssh_user = '******' _ssh_private_key = 'haproxy' conns = [Connection(host, user=_ssh_user, connect_kwargs={ "key_filename": _ssh_private_key, }) for host in _haproxies_hosts] group = SerialGroup.from_connections(conns) for conn in conns: conn.put('./haproxy.cfg', '/etc/haproxy/haproxy.cfg') result = group.run('systemctl reload haproxy')
#!/usr/bin/env python3 # -*- coding: utf-8 -*- import getpass from fabric import SerialGroup sudo_pass = getpass.getpass("input your password: "******"password": sudo_pass}) #pool = SerialGroup('arch-od-tracker02.beta1.fn', 'arch-od-tracker03.beta1.fn', user='******', connect_kwargs={"password": '******'} ) pool.run('hostname')
def scan(): hosts = ["giga1"] grp = SerialGroup(*hosts) for sta in grp: pprint(wifi.scan(sta, phy="02:00"))