def __init__(self, host_id, session_id, session_config): host = Host.objects.select_related('group').get(pk=host_id) hostname = host.hostname or host.name config = MINKE_FABRIC_CONFIG.clone() self.host = host # At first try to load the hostgroup- and host-config... for obj in (host.group, host): if not obj or not obj.config: continue if obj.config in MINKE_HOST_CONFIG: config.load_snakeconfig(MINKE_HOST_CONFIG[obj.config]) else: msg = 'Invalid MINKE_HOST_CONFIG for {}'.format(obj) logger.warning(msg) # At least load the session-config... config.load_snakeconfig(session_config or dict()) # Initialize the connection... self.con = Connection(hostname, host.username, host.port, config=config) # Initialize the session... self.minke_session = MinkeSession.objects.get(pk=session_id) REGISTRY.reload(self.minke_session.session_name) session_cls = REGISTRY[self.minke_session.session_name] self.session = session_cls(self.con, self.minke_session)
def setConnection(self): self.connection = Connection(host=self.properties['ip'], user=self.properties['username'], connect_kwargs={ "key_filename": self.properties['keyFile'], })
def __init__(self, host, user, password=None, ssh_private_key=None): """ Args: host (str): server host user (str): user name password (str): password, default None; If password is None, use ssh_key. ssh_private_key (Path, str): ssh public key path, default None; If ssh_key is None, use password. """ if password is None and ssh_private_key is None: logger.warning( "ssh_private_key and password are both none, ssh_private_key will use `~/.ssh/id_rsa`!" ) ssh_private_key = str(Path("~/.ssh/id_rsa").expanduser().resolve()) if ssh_private_key is not None: ssh_key = str(Path(ssh_private_key).expanduser().resolve()) connect_kwargs = {"key_filename": ssh_key} self.conn = Connection(host=host, user=user, connect_kwargs=connect_kwargs) else: config = Config(overrides={"sudo": {"password": password}}) connect_kwargs = {"password": password} self.conn = Connection(host=host, user=user, connect_kwargs=connect_kwargs, config=config) self._home_dir = None
def ssh_run(servername, cmd, sudo): try: user = os.environ['SSH_USER'] passw = os.environ['SSH_PASS'] sudotf = sudo command = cmd sname = servername print('user: %s passwd: %s servername: %s' % (user, passw, sname)) config = Config(overrides={'sudo': {'password': passw}}) c = Connection(host=sname, user=user, port=22, config=config, connect_kwargs={"password": passw}) if sudotf: result = c.sudo(command, pty=True, hide='stderr') if result.exited == 0: print("In SUDO") print(result.ok) print(result.stdout.strip()) return result.exited else: result = c.run(SSH_COMMAND, pty=True, hide='stderr') if result.exited == 0: print("In NOSUDO") print(result.ok) print(result.stdout.strip()) return result.exited except Exception as e: print("ssh_run Exception Caught: %s" % e) return "Exception Caught Clearing Certificate"
def rsyn_file(): passwd_file = "/home/tomcat/shiwm-service/py-scripts/get_sftp_list/scripts/rsync.pwd" rsync_bin = "/usr/bin/rsync" source_dir = "/home/tomcat/shiwm-service/sftp-duizhang" c = Connection("127.0.0.1") res = c.run( " %s -azvP %s --password-file=%s [email protected]::sftp-duizhang" % (rsync_bin, source_dir, passwd_file)) logging.info("Rsync message: %s" % res.stdout)
def set_fabric_connection (self): """ SET CONNECTION PREFS """ self.fabric_connection = Connection(host = self.ssh_target_ip, port = self.ssh_target_port, user = self.ssh_target_user) self.fabric_connection.connect_kwargs.password = self.ssh_target_pswd return
def deploy(c): c = Connection(os.environ.get("SSH_ADDRESS"), connect_kwargs={"passphrase": os.environ.get("PASSPHRASE")}) c.run(f"mkdir -p {SITE_FOLDER}") with c.cd(f"{SITE_FOLDER}"): _get_latest_sources(c) _update_pipenv(c) _update_staticfiles(c) _update_migrations(c)
def deploy(): print("========================================") print("deploying to server") # + os.environ.get("ftp_host")) print("========================================") c = Connection(host='www.jonaso.de', port=21) result = c.run('uname -s') print(result.stdout.strip()) print(result.exited) print(result.ok) print(result.command) print(result.connection) print(result.connection.host) sys.exit() try: # cleanup # local_cleanup() # compress the folder # local("tar -zcvf %s %s" % (tarFilename, localdir)) pass # upload the tar file to the remote host # put(tarFilename, join(remotedir, tarFilename), use_sudo=True, mirror_local_mode=True) # with cd(remotedir): # untar the folder # sudo("tar -xvf " + tarFilename) # modify perms # TODO: check if this is necessary # sudo("chmod 755 " + remotedir) # drop the database # sudo("mysqladmin -f -u%s -p\"%s\" drop %s" % (dbUsername, dbPassword, dbName)) # sudo("cp -r wordpress/dist ./"); # sudo("rm -rf ./wordpress/dist"); # sudo("cp -r wordpress/static ./"); # sudo("rm -rf ./wordpress/static"); # sudo("cp -r wordpress/favicon.* ./"); # # sudo("rm -f ./favicon.*"); # sudo("cp -r wordpress/.htaccess ./.htaccess"); # sudo("rm -f ./wordpress/.htaccess"); finally: # cleanup # local_cleanup() # remote cleanup # remove the tar file and sql file # sudo("rm -f " + join(remotedir, localdir)) pass
class FabricWrapper: def __init__( self, host, user=None, password=None, key_filename=None, key_passphrase=None, port=22, sudo=False ): self.sudo = sudo if not user and not password and not key_filename: # Get details from ~/.ssh/config self.connection = Connection(host) elif key_filename and not password: self.connection = Connection( host=host, user=user, port=port, connect_kwargs={ 'key_filename': key_filename, 'passphrase': key_passphrase } ) elif not key_filename and password: self.connection = Connection( host=host, user=user, port=port, connect_kwargs={ 'password': password } ) elif key_filename and password: self.connection = Connection( host=host, user=user, port=port, connect_kwargs={ 'key_filename': key_filename, 'passphrase': key_passphrase if key_passphrase else password } ) else: logging.error( crayons.red(f'You need to provide either a private key_filename or password to connect to {host} with user: {user}') ) self.connection = None def execute(self, command, **kwargs): if not self.connection: logging.error(crayons.red('No connection object instantiated.')) return None return self.connection.sudo(command, **kwargs) if self.sudo else self.connection.run(command, **kwargs)
def bootstrap(ctx): with Connection(HOST, user='******') as connection: upgrade_server(connection) time.sleep(REBOOT_DELAY) with Connection(HOST, user='******') as connection: install_packages(connection) create_user(connection) with Connection(HOST, user='******') as connection: setup_user(connection)
def del_file(file_path): c = Connection('localhost') with c.cd("/home/tomcat/shiwm-service/"): res = c.run('[ -d %s ] && echo 00 || echo 11' % file_path, hide=True).stdout logging.info("The res is %s" % res) if res.startswith("00"): c.run("rm -rf ./sftp-duizhang/") logging.info("The die is exits, delete it now.") c.run("mkdir -p sftp-duizhang") logging.info("The die isn't exits, mkdir it now.") else: c.run("mkdir -p sftp-duizhang") logging.info("The die isn't exits, mkdir it now.")
def remote_notebook_start(self, instance): """ Do notebook start command on the remote server.""" self.log_user("remote_notebook_start()") # Setup environments env = self.get_env() lenv='' for key in env: lenv = lenv + key + "=" + env[key] + " " # End setup environment worker_ip_address_string = instance.private_ip_address start_notebook_cmd = self.cmd + self.get_args() start_notebook_cmd = " ".join(start_notebook_cmd) self.log_user("remote_notebook_start private ip: %s" % worker_ip_address_string) with Connection(user=self.user.name, host=worker_ip_address_string, connect_kwargs=FABRIC_CONNECT_KWARGS) as c: yield sudo(c, "%s %s --user=%s --notebook-dir=/home/%s/ --allow-root > /tmp/jupyter.log 2>&1 &" % (lenv, start_notebook_cmd,self.user.name,self.user.name), pty=False) self.log_user("remote_notebook_start private ip: %s, waiting." % worker_ip_address_string) notebook_running = yield self.is_notebook_running(worker_ip_address_string, attempts=10) self.log_user("remote_notebook_start private ip: %s, running: %s" % (worker_ip_address_string, notebook_running)) num_remote_notebook_start_retries = 0 while not notebook_running and num_remote_notebook_start_retries < REMOTE_NOTEBOOK_START_RETRY_MAX: yield sudo(c, "%s %s --user=%s --notebook-dir=/home/%s/ --allow-root > /tmp/jupyter.log 2>&1 &" % (lenv, start_notebook_cmd,self.user.name,self.user.name), pty=False) self.log_user("remote_notebook_start private ip: %s, retry attempt %s/%s. waiting..." % (worker_ip_address_string, num_remote_notebook_start_retries + 1, REMOTE_NOTEBOOK_START_RETRY_MAX)) yield gen.sleep(3) # Wait for 3 seconds before checking whether the notebook server started notebook_running = yield self.is_notebook_running(worker_ip_address_string, attempts=10) self.log_user("remote_notebook_start private ip: %s, running: %s" % (worker_ip_address_string, notebook_running)) if notebook_running: break # break loop num_remote_notebook_start_retries += 1
def main(unused_argv=0): producer = Producer() # load host names from file filename = os.getcwd() + "/" + HOSTS_FILE hosts = [line.rstrip('\n') for line in open(filename)] for host in hosts: print("host: %s" % host) if not host or not bool(host.strip()): continue host = host.strip() #TODO add try catch #try: c = Connection(host=host, user=USER, connect_kwargs={ "key_filename": os.getcwd() + "/hostsecrets/" + KEY_FILE, }) with c: res = remote_info(c, host) print('res:%s' % res) if len(res) > 0: # send message if it's not empty producer.produce(TOPIC, res) #except e: producer.flush() print("done")
def get_connection(host_name): config = HOSTS[host_name] return Connection(config["host"], connect_kwargs={ "key_filename": config["key_filename"], "password": config["password"] })
def ec2_connection(request, ec2_instance, ec2_instance_type, region): """ Fixture to establish connection with EC2 instance if necessary :param request: pytest test request :param ec2_instance: ec2_instance pytest fixture :param ec2_instance_type: ec2_instance_type pytest fixture :param region: Region where ec2 instance is launched :return: Fabric connection object """ instance_id, instance_pem_file = ec2_instance ip_address = ec2_utils.get_public_ip(instance_id, region=region) LOGGER.info(f"Instance ip_address: {ip_address}") user = ec2_utils.get_instance_user(instance_id, region=region) LOGGER.info(f"Connecting to {user}@{ip_address}") conn = Connection(user=user, host=ip_address, connect_kwargs={"key_filename": [instance_pem_file]}) random.seed(f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}") unique_id = random.randint(1, 100000) artifact_folder = f"ts-benchmark-{unique_id}-folder" ArtifactsHandler.upload_torchserve_folder_to_instance( conn, artifact_folder) def delete_s3_artifact_copy(): ArtifactsHandler.cleanup_temp_s3_folder(artifact_folder) request.addfinalizer(delete_s3_artifact_copy) return conn
def dist_model(src_model, node_info, remote_path="xt_archive"): """ Distribute model to remote node. :param src_model: :param node_info: :param remote_path: :return: """ # optimize local node check if node_info[0] in ("127.0.0.1", ): return None server_ip = get_host_ip() if node_info[0] == server_ip: return None _basename = os.path.basename(src_model) if _basename in ("none", "None", None): # init model with (none, none) return None target_file = glob.glob("{}*".format(src_model)) _ip, _user, _password = node_info destination_model = os.path.join("/home", _user, remote_path + "/") with Connection(_ip, user=_user, connect_kwargs={"password": _password}) as connect: # fixme: multi-case running on the same node _workspace = os.path.join("/home", _user, remote_path) for _item in target_file: logging.debug("dist model: {}--> {}".format( _item, destination_model)) connect.put(_item, destination_model) return [destination_model + _basename]
def make_worker_ami(config, ec2, security_group_list): """ Sets up worker components, runs before jupyterhub setup, after common setup. """ instance = launch_server(config, ec2, security_group_list, size=int(config.worker_ebs_size)) instance.wait_until_exists() instance.wait_until_running() with Connection(host=instance.public_ip_address, user=config.server_username, connect_kwargs={"key_filename": KEY_PATH}) as connection: # Wait for server to finish booting (keep trying until you can successfully run a command on the server via ssh) retry(connection.run, "# waiting for ssh to be connectable...", max_retries=100) connection.sudo("apt-get -qq -y update") connection.sudo( "apt-get -qq -y install -q python python-dev python-pip") connection.sudo("pip install --upgrade pip") connection.sudo("apt-get -qq -y remove -q python-pip") connection.sudo("hash -r") connection.sudo("apt-get -qq -y install -q python3-pip sqlite") connection.sudo("pip3 install --upgrade pip") connection.sudo("apt-get -qq -y remove -q python3-pip") connection.sudo("hash -r") connection.put("jupyterhub_files/requirements_jupyterhub.txt", remote_path="/var/tmp/") connection.sudo( "pip3 install --quiet -r /var/tmp/requirements_jupyterhub.txt") connection.sudo("pip3 -q install ipython jupyter ipykernel nbgrader") connection.sudo("pip2 -q install ipykernel --upgrade") # register Python 3 and 2 kernel connection.sudo("python3 -m ipykernel install") connection.sudo("python2 -m ipykernel install") connection.sudo("chmod 755 /mnt") connection.sudo("chown ubuntu /mnt") # Create AMI for workers logger.info("Creating worker AMI") ami_name = "jupyter-hub-%s-worker-image" % config.cluster_name worker_ami = instance.create_image(Name=ami_name) # Wait until AMI is ready or 300 seconds have elapsed to allow for server restart for i in range(100): if get_resource(config.region).Image( worker_ami.id).state == "available": break logger.info("AMI not ready yet (running for ~%s seconds)" % (i * 3)) sleep(3) instance.terminate() return worker_ami.id
def new_user(username, ssh_key, host): with Connection( host=host, user='******', port=22, connect_kwargs={"key_filename": "/Users/panyan/.ssh/id_rsa"}, connect_timeout=5) as c: create_user(c, username, ssh_key)
def get_connection(ctx): if isinstance(ctx, Connection): conn = ctx else: conn = Connection(ctx.host, ctx.user, connect_kwargs=ctx.connect_kwargs) if conn is None: sys.exit("Failed to get connection") return conn
def main(): fab = Connection(host='www.xachf.com', user='******', port=22, connect_timeout=20, connect_kwargs={"password": "******"}) # print(fab) res = fab.run('uname -s', hide=True) # 'command', 'connection', 'encoding', 'env', 'exited', 'failed', 'hide', 'ok', 'pty', 'return_code', 'shell', 'stderr', 'stdout', 'tail' # print(dir(res)) # print(res.stdout.strip()) # print(res.command, res.connection.host,res.connection.user) if res.ok: print('{}>>用户{} 执行[{}]命令成功'.format(res.connection.host, res.connection.user, res.command)) else: print('{}>>用户{} 执行[{}]命令失败'.format(res.connection.host, res.connection.user, res.command))
def setUp(self): self.host = Host.objects.get(name='localhost') self.server = Server.objects.get(host=self.host) self.user = User.objects.get(username='******') config = MINKE_FABRIC_CONFIG.clone() self.con = Connection(self.host.hostname, self.host.username, config=config) self._REGISTRY = sessions.REGISTRY.copy()
def wait_until_SSHable(self, ip_address_string, max_retries=1): """ Run a meaningless bash command (a comment) inside a retry statement. """ self.log_user("wait_until_SSHable()") with Connection(user=WORKER_USERNAME, host=ip_address_string, connect_kwargs=FABRIC_CONNECT_KWARGS) as c: self.log_user("wait_until_SSHable max_retries:%s" % max_retries, level=logging.DEBUG) ret = yield run(c, "# waiting for ssh to be connectable for user %s..." % self.user.name, max_retries=max_retries) self.log_user("wait_until_SSHable completed return: %s" % ret, level=logging.DEBUG) if ret == "RETRY_FAILED": ret = "SSH_CONNECTION_FAILED" return (ret)
def ConnectNode(u, p, h): print("----> Connecting to ", h) global connect try: connect = Connection(host=h, user=u, connect_kwargs={"password": p}) except: print( "Wrong ssh connexion parameters. Please check also that you are using fabric v2.0" ) # trouver un moyen de faire printer cela return (connect)
def connect_to_host(ctx): if isinstance(ctx, Connection): return ctx else: password = ENV_VALUES['password'] config = Config(overrides={'sudo': {'password': password}}) conn = Connection(ctx.host, ctx.user, connect_kwargs=ctx.connect_kwargs, config=config) return conn
def __init__(self, task_queue, result_queue, lggr, appconfig): super(M3TaskPerformer, self).__init__() if appconfig['ENVNAME'] == 'local': self.host = 'mugopala@localhost' else: self.host = None self.logger = lggr self.conn = Connection(self.host) # TODO implement any other environment initialization self.task_queue = task_queue self.result_queue = result_queue
def remote_run(server_ip, host, passwd, cmd, remote_env): """Run command in remote node.""" print("remote_env:", remote_env) _env_export = "export PATH={}/bin:$PATH".format(remote_env["conda"]) if "env" in remote_env.keys(): for _key, _val in remote_env["env"].items(): _env_export += "&& export {}={}".format(_key, _val) with Connection(server_ip, user=host, connect_kwargs={"password": passwd}) as connect: with connect.prefix("{}".format(_env_export)): connect.run(cmd, pty=False, echo=True)
def main(): hosts = ['192.168.19.128', '192.168.19.129'] Config.ssh_config_path = 'ssh_config' # SSH Config File try: for host in hosts: con = Connection(host) # All Rask Run print() print('==============') print(ahc(con, 'date', hide=True, hostNameShow=False)) print(hostname(con, hide=True)) print(ip(con, hide=True)) print(disk_free(con, hide=True)) print(chpasswd(con, 'python', hide=True)) con.close() except RuntimeError as r: sys.stderr.write('=== Runtime Error! ===') sys.stderr.write(str(r.args)) print('ALL Done!')
def distribute_xt_if_need(config, remote_env, remote_path="xt_archive"): """ Distribute Xingtian sourcecode among use's node configure. :param config: config instance from config.yaml :param remote_env: remote conda environment path :param remote_path: path to store the wheel file. 'xt_archive' default. :return: """ local_ip = get_host_ip() # check could if distribute or not remote_ip_list = list() for _key in ( "node_config", "test_node_config", ): if _key not in config.keys(): continue for _ip, _user, _password in config.get(_key): # local need not distribute if _ip in (local_ip, "127.0.0.1"): continue remote_ip_list.append((_ip, _user, _password)) if not remote_ip_list: logging.debug("Don't distribute xingtian without remote ip set.") return True dist_path = _package_xt() if not remote_env: logging.fatal("must assign remote env in yaml.") for _ip, _user, _password in remote_ip_list: with Connection(_ip, user=_user, connect_kwargs={"password": _password}) as connect: _workspace = os.path.join("/tmp") target_whl = glob.glob("{}/xingtian*.whl".format(dist_path)) logging.info("found dist: {}".format(target_whl)) for _whl in target_whl: _name = os.path.basename(_whl) _remote_cmd = "pip install {}/{} --upgrade --force-reinstall --no-deps".format( _workspace, _name) logging.info( "re-install xingtian in remote-{} conda env {} >>> \n" "{}".format(_ip, remote_env["conda"], _remote_cmd)) connect.put(os.path.join(dist_path, _name), remote=_workspace) with connect.prefix("export PATH={}/bin:$PATH".format( remote_env["conda"])): connect.run(_remote_cmd, pty=False)
def generateSSHKey(self): display('Generating ssh key...') c = Connection('localhost') c.local('rm -f "{keyFile}.*"'.format(**self.properties)) c.local("echo 'yes' | ssh-keygen -t rsa -f {keyFile} -C {username} -N '' ".format(**self.properties),hide='out') c.close() #p = Popen("echo 'yes' | ssh-keygen -t rsa -f {keyFile} -C {username} -N '' ".format(**self.properties), # stdout=PIPE, # shell=True, # stderr=PIPE # ) #print(p.communicate()) with open (self.properties['pubKeyFile'],'r') as f: display('Opening {}'.format(self.properties['pubKeyFile'])) self.pub = f.read().strip()
def _inner(): logger.warning( f'将本地文件夹代码 {python_proj_dir} 上传到远程 {host} 的 {remote_dir} 文件夹。') t_start = time.perf_counter() uploader = ParamikoFolderUploader( host, port, user, password, python_proj_dir, remote_dir, path_pattern_exluded_tuple, file_suffix_tuple_exluded, only_upload_within_the_last_modify_time, file_volume_limit, sftp_log_level) uploader.upload() logger.info( f'上传 本地文件夹代码 {python_proj_dir} 上传到远程 {host} 的 {remote_dir} 文件夹耗时 {round(time.perf_counter() - t_start, 3)} 秒' ) # conn.run(f'''export PYTHONPATH={remote_dir}:$PYTHONPATH''') func_name = task_fun.__name__ queue_name = task_fun.consumer.queue_name process_mark = f'fsdf_fabric_mark__{queue_name}__{func_name}' conn = Connection( host, port=port, user=user, connect_kwargs={"password": password}, ) kill_shell = f'''ps -aux|grep {process_mark}|grep -v grep|awk '{{print $2}}' |xargs kill -9''' logger.warning(f'{kill_shell} 命令杀死 {process_mark} 标识的进程') uploader.ssh.exec_command(kill_shell) # conn.run(kill_shell, encoding='utf-8',warn=True) # 不想提示,免得烦扰用户以为有什么异常了。所以用上面的paramiko包的ssh.exec_command python_exec_str = f'''export is_fsdf_remote_run=1;export PYTHONPATH={remote_dir}:$PYTHONPATH ;python3 -c "from {relative_module} import {func_name};{func_name}.multi_process_consume({process_num})" -fsdfmark {process_mark} ''' shell_str = f'''cd {remote_dir}; {python_exec_str}''' extra_shell_str2 = extra_shell_str # 内部函数对外部变量不能直接改。 if not extra_shell_str2.endswith(';') and extra_shell_str != '': extra_shell_str2 += ';' shell_str = extra_shell_str2 + shell_str logger.warning(f'使用语句 {shell_str} 在远程机器 {host} 上启动任务消费') conn.run(shell_str, encoding='utf-8', **invoke_runner_kwargs)
def sanity_test_from_v1(c): """ Run some very quick in-process sanity tests on a dual fabric1-v-2 env. Assumes Fabric 2+ is already installed as 'fabric2'. """ # This cannot, by definition, work under Python 3 as Fabric 1 is not Python # 3 compatible. PYTHON = environ.get("TRAVIS_PYTHON_VERSION", "") if PYTHON.startswith("3") or PYTHON == "pypy3": return c.run("pip install 'fabric<2'") # Make darn sure the two copies of fabric are coming from install root, not # local directory - which would result in 'fabric' always being v2! for serious in (getcwd(), ""): if serious in sys.path: # because why would .remove be idempotent?! sys.path.remove(serious) from fabric.api import env from fabric2 import Connection env.gateway = "some-gateway" env.no_agent = True env.password = "******" env.user = "******" env.host_string = "localghost" env.port = "2222" cxn = Connection.from_v1(env) config = cxn.config assert config.run.pty is True assert config.gateway == "some-gateway" assert config.connect_kwargs.password == "sikrit" assert config.sudo.password == "sikrit" assert cxn.host == "localghost" assert cxn.user == "admin" assert cxn.port == 2222
def deploy(c, restart=False, sync=False): "push code to a server (configure your own here)" c.run("git push --all") conn = Connection("trotsky") with conn.cd("src/pyborg-1up"): conn.run("git fetch") conn.run("git stash") conn.run("git pull") conn.run("git stash pop") if sync: conn.run("~/.local/bin/pipenv sync") # they all use the same pipenv managed virtualenv if restart: for unit in ["pyborg_discord", "pyborg_http", "pyborg_twitter", "pyborg_mastodon"]: conn.sudo("systemctl restart {}".format(unit), pty=True) print("Deploy Completed.")