Ejemplo n.º 1
0
    def __init__(self, host_id, session_id, session_config):
        host = Host.objects.select_related('group').get(pk=host_id)
        hostname = host.hostname or host.name
        config = MINKE_FABRIC_CONFIG.clone()
        self.host = host

        # At first try to load the hostgroup- and host-config...
        for obj in (host.group, host):
            if not obj or not obj.config: continue
            if obj.config in MINKE_HOST_CONFIG:
                config.load_snakeconfig(MINKE_HOST_CONFIG[obj.config])
            else:
                msg = 'Invalid MINKE_HOST_CONFIG for {}'.format(obj)
                logger.warning(msg)

        # At least load the session-config...
        config.load_snakeconfig(session_config or dict())

        # Initialize the connection...
        self.con = Connection(hostname,
                              host.username,
                              host.port,
                              config=config)

        # Initialize the session...
        self.minke_session = MinkeSession.objects.get(pk=session_id)
        REGISTRY.reload(self.minke_session.session_name)
        session_cls = REGISTRY[self.minke_session.session_name]
        self.session = session_cls(self.con, self.minke_session)
Ejemplo n.º 2
0
 def setConnection(self):
     self.connection = Connection(host=self.properties['ip'],
                                  user=self.properties['username'],
                                  connect_kwargs={
                                      "key_filename":
                                      self.properties['keyFile'],
                                  })
Ejemplo n.º 3
0
    def __init__(self, host, user, password=None, ssh_private_key=None):
        """
        Args:
            host (str): server host
            user (str): user name
            password (str): password, default None; If password is None, use ssh_key.
            ssh_private_key (Path, str): ssh public key path, default None; If ssh_key is None, use password.
        """
        if password is None and ssh_private_key is None:
            logger.warning(
                "ssh_private_key and password are both none, ssh_private_key will use `~/.ssh/id_rsa`!"
            )
            ssh_private_key = str(Path("~/.ssh/id_rsa").expanduser().resolve())

        if ssh_private_key is not None:
            ssh_key = str(Path(ssh_private_key).expanduser().resolve())
            connect_kwargs = {"key_filename": ssh_key}
            self.conn = Connection(host=host,
                                   user=user,
                                   connect_kwargs=connect_kwargs)
        else:
            config = Config(overrides={"sudo": {"password": password}})
            connect_kwargs = {"password": password}
            self.conn = Connection(host=host,
                                   user=user,
                                   connect_kwargs=connect_kwargs,
                                   config=config)

        self._home_dir = None
Ejemplo n.º 4
0
def ssh_run(servername, cmd, sudo):
    try:
        user = os.environ['SSH_USER']
        passw = os.environ['SSH_PASS']
        sudotf = sudo
        command = cmd
        sname = servername
        print('user: %s passwd: %s servername: %s' % (user, passw, sname))
        config = Config(overrides={'sudo': {'password': passw}})
        c = Connection(host=sname,
                       user=user,
                       port=22,
                       config=config,
                       connect_kwargs={"password": passw})
        if sudotf:
            result = c.sudo(command, pty=True, hide='stderr')
            if result.exited == 0:
                print("In SUDO")
                print(result.ok)
                print(result.stdout.strip())
                return result.exited
        else:
            result = c.run(SSH_COMMAND, pty=True, hide='stderr')
            if result.exited == 0:
                print("In NOSUDO")
                print(result.ok)
                print(result.stdout.strip())
                return result.exited
    except Exception as e:
        print("ssh_run Exception Caught: %s" % e)
        return "Exception Caught Clearing Certificate"
Ejemplo n.º 5
0
def rsyn_file():
    passwd_file = "/home/tomcat/shiwm-service/py-scripts/get_sftp_list/scripts/rsync.pwd"
    rsync_bin = "/usr/bin/rsync"
    source_dir = "/home/tomcat/shiwm-service/sftp-duizhang"
    c = Connection("127.0.0.1")
    res = c.run(
        " %s -azvP %s --password-file=%s [email protected]::sftp-duizhang" %
        (rsync_bin, source_dir, passwd_file))
    logging.info("Rsync message: %s" % res.stdout)
Ejemplo n.º 6
0
 def set_fabric_connection (self):
     """
     SET CONNECTION PREFS
     """
     self.fabric_connection = Connection(host = self.ssh_target_ip,
                                         port = self.ssh_target_port,
                                         user = self.ssh_target_user)
     self.fabric_connection.connect_kwargs.password = self.ssh_target_pswd
     return
Ejemplo n.º 7
0
def deploy(c):
    c = Connection(os.environ.get("SSH_ADDRESS"),
                   connect_kwargs={"passphrase": os.environ.get("PASSPHRASE")})
    c.run(f"mkdir -p {SITE_FOLDER}")
    with c.cd(f"{SITE_FOLDER}"):
        _get_latest_sources(c)
        _update_pipenv(c)
        _update_staticfiles(c)
        _update_migrations(c)
Ejemplo n.º 8
0
def deploy():
    print("========================================")
    print("deploying to server")  # + os.environ.get("ftp_host"))
    print("========================================")

    c = Connection(host='www.jonaso.de', port=21)
    result = c.run('uname -s')
    print(result.stdout.strip())
    print(result.exited)
    print(result.ok)
    print(result.command)
    print(result.connection)
    print(result.connection.host)
    sys.exit()

    try:
        # cleanup
        # local_cleanup()

        # compress the folder
        # local("tar -zcvf %s %s" % (tarFilename, localdir))

        pass

        # upload the tar file to the remote host
        # put(tarFilename, join(remotedir, tarFilename), use_sudo=True, mirror_local_mode=True)

        # with cd(remotedir):

        # untar the folder
        # sudo("tar -xvf " + tarFilename)

        # modify perms # TODO: check if this is necessary
        # sudo("chmod 755 " + remotedir)
        # drop the database
        # sudo("mysqladmin -f -u%s -p\"%s\" drop %s" % (dbUsername, dbPassword, dbName))

        # sudo("cp -r wordpress/dist ./");
        # sudo("rm -rf ./wordpress/dist");
        # sudo("cp -r wordpress/static ./");
        # sudo("rm -rf ./wordpress/static");
        # sudo("cp -r wordpress/favicon.* ./");
        # # sudo("rm -f ./favicon.*");
        # sudo("cp -r wordpress/.htaccess ./.htaccess");
        # sudo("rm -f ./wordpress/.htaccess");

    finally:

        # cleanup
        # local_cleanup()

        # remote cleanup
        # remove the tar file and sql file
        # sudo("rm -f " + join(remotedir, localdir))

        pass
Ejemplo n.º 9
0
class FabricWrapper:
    def __init__(
            self,
            host,
            user=None,
            password=None,
            key_filename=None,
            key_passphrase=None,
            port=22,
            sudo=False
    ):
        self.sudo = sudo
        if not user and not password and not key_filename:
            # Get details from ~/.ssh/config
            self.connection = Connection(host)
        elif key_filename and not password:
            self.connection = Connection(
                host=host,
                user=user,
                port=port,
                connect_kwargs={
                    'key_filename': key_filename,
                    'passphrase': key_passphrase
                }
            )
        elif not key_filename and password:
            self.connection = Connection(
                host=host,
                user=user,
                port=port,
                connect_kwargs={
                    'password': password
                }
            )
        elif key_filename and password:
            self.connection = Connection(
                host=host,
                user=user,
                port=port,
                connect_kwargs={
                    'key_filename': key_filename,
                    'passphrase': key_passphrase if key_passphrase else password
                }
            )
        else:
            logging.error(
                crayons.red(f'You need to provide either a private key_filename or password to connect to {host} with user: {user}')
            )
            self.connection = None

    def execute(self, command, **kwargs):
        if not self.connection:
            logging.error(crayons.red('No connection object instantiated.'))
            return None
        return self.connection.sudo(command, **kwargs) if self.sudo else self.connection.run(command, **kwargs)
Ejemplo n.º 10
0
def bootstrap(ctx):

    with Connection(HOST, user='******') as connection:
        upgrade_server(connection)

    time.sleep(REBOOT_DELAY)

    with Connection(HOST, user='******') as connection:
        install_packages(connection)
        create_user(connection)

    with Connection(HOST, user='******') as connection:
        setup_user(connection)
Ejemplo n.º 11
0
def del_file(file_path):
    c = Connection('localhost')
    with c.cd("/home/tomcat/shiwm-service/"):
        res = c.run('[ -d %s ] && echo 00 || echo 11' % file_path,
                    hide=True).stdout
        logging.info("The res is %s" % res)
        if res.startswith("00"):
            c.run("rm -rf ./sftp-duizhang/")
            logging.info("The die is exits, delete it now.")
            c.run("mkdir -p sftp-duizhang")
            logging.info("The die isn't exits, mkdir it now.")
        else:
            c.run("mkdir -p sftp-duizhang")
            logging.info("The die isn't exits, mkdir it now.")
Ejemplo n.º 12
0
    def remote_notebook_start(self, instance):
        """ Do notebook start command on the remote server."""
        self.log_user("remote_notebook_start()")

        # Setup environments
        env = self.get_env()
        lenv=''
        for key in env:
            lenv = lenv + key + "=" + env[key] + " "
        # End setup environment
        worker_ip_address_string = instance.private_ip_address
        start_notebook_cmd = self.cmd + self.get_args()
        start_notebook_cmd = " ".join(start_notebook_cmd)
        self.log_user("remote_notebook_start private ip: %s" % worker_ip_address_string)
        with Connection(user=self.user.name, host=worker_ip_address_string, connect_kwargs=FABRIC_CONNECT_KWARGS) as c:
            yield sudo(c, "%s %s --user=%s --notebook-dir=/home/%s/ --allow-root > /tmp/jupyter.log 2>&1 &" % (lenv, start_notebook_cmd,self.user.name,self.user.name),  pty=False)
            self.log_user("remote_notebook_start private ip: %s, waiting." % worker_ip_address_string)
            notebook_running = yield self.is_notebook_running(worker_ip_address_string, attempts=10)
            self.log_user("remote_notebook_start private ip: %s, running: %s" % (worker_ip_address_string, notebook_running))
            num_remote_notebook_start_retries = 0
            while not notebook_running and num_remote_notebook_start_retries < REMOTE_NOTEBOOK_START_RETRY_MAX:
                yield sudo(c, "%s %s --user=%s --notebook-dir=/home/%s/ --allow-root > /tmp/jupyter.log 2>&1 &" % (lenv, start_notebook_cmd,self.user.name,self.user.name),  pty=False)
                self.log_user("remote_notebook_start private ip: %s, retry attempt %s/%s. waiting..." % (worker_ip_address_string, num_remote_notebook_start_retries + 1, REMOTE_NOTEBOOK_START_RETRY_MAX))
                yield gen.sleep(3) # Wait for 3 seconds before checking whether the notebook server started
                notebook_running = yield self.is_notebook_running(worker_ip_address_string, attempts=10)
                self.log_user("remote_notebook_start private ip: %s, running: %s" % (worker_ip_address_string, notebook_running))
                if notebook_running:
                    break # break loop
                num_remote_notebook_start_retries += 1
Ejemplo n.º 13
0
def main(unused_argv=0):

    producer = Producer()
    # load host names from file
    filename = os.getcwd() + "/" + HOSTS_FILE
    hosts = [line.rstrip('\n') for line in open(filename)]

    for host in hosts:
        print("host: %s" % host)
        if not host or not bool(host.strip()):
            continue
        host = host.strip()
        #TODO add try catch
        #try:
        c = Connection(host=host,
                       user=USER,
                       connect_kwargs={
                           "key_filename":
                           os.getcwd() + "/hostsecrets/" + KEY_FILE,
                       })
        with c:
            res = remote_info(c, host)
            print('res:%s' % res)
            if len(res) > 0:
                # send message if it's not empty
                producer.produce(TOPIC, res)
        #except e:
    producer.flush()
    print("done")
Ejemplo n.º 14
0
def get_connection(host_name):
    config = HOSTS[host_name]
    return Connection(config["host"],
                      connect_kwargs={
                          "key_filename": config["key_filename"],
                          "password": config["password"]
                      })
Ejemplo n.º 15
0
def ec2_connection(request, ec2_instance, ec2_instance_type, region):
    """
    Fixture to establish connection with EC2 instance if necessary
    :param request: pytest test request
    :param ec2_instance: ec2_instance pytest fixture
    :param ec2_instance_type: ec2_instance_type pytest fixture
    :param region: Region where ec2 instance is launched
    :return: Fabric connection object
    """
    instance_id, instance_pem_file = ec2_instance
    ip_address = ec2_utils.get_public_ip(instance_id, region=region)
    LOGGER.info(f"Instance ip_address: {ip_address}")
    user = ec2_utils.get_instance_user(instance_id, region=region)
    LOGGER.info(f"Connecting to {user}@{ip_address}")
    conn = Connection(user=user,
                      host=ip_address,
                      connect_kwargs={"key_filename": [instance_pem_file]})

    random.seed(f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}")
    unique_id = random.randint(1, 100000)

    artifact_folder = f"ts-benchmark-{unique_id}-folder"

    ArtifactsHandler.upload_torchserve_folder_to_instance(
        conn, artifact_folder)

    def delete_s3_artifact_copy():
        ArtifactsHandler.cleanup_temp_s3_folder(artifact_folder)

    request.addfinalizer(delete_s3_artifact_copy)

    return conn
Ejemplo n.º 16
0
def dist_model(src_model, node_info, remote_path="xt_archive"):
    """
    Distribute model to remote node.

    :param src_model:
    :param node_info:
    :param remote_path:
    :return:
    """
    # optimize local node check
    if node_info[0] in ("127.0.0.1", ):
        return None
    server_ip = get_host_ip()
    if node_info[0] == server_ip:
        return None

    _basename = os.path.basename(src_model)
    if _basename in ("none", "None", None):  # init model with (none, none)
        return None

    target_file = glob.glob("{}*".format(src_model))

    _ip, _user, _password = node_info
    destination_model = os.path.join("/home", _user, remote_path + "/")
    with Connection(_ip, user=_user, connect_kwargs={"password":
                                                     _password}) as connect:
        # fixme: multi-case running on the same node
        _workspace = os.path.join("/home", _user, remote_path)
        for _item in target_file:
            logging.debug("dist model: {}--> {}".format(
                _item, destination_model))
            connect.put(_item, destination_model)
    return [destination_model + _basename]
Ejemplo n.º 17
0
def make_worker_ami(config, ec2, security_group_list):
    """ Sets up worker components, runs before jupyterhub setup, after common setup. """
    instance = launch_server(config,
                             ec2,
                             security_group_list,
                             size=int(config.worker_ebs_size))
    instance.wait_until_exists()
    instance.wait_until_running()

    with Connection(host=instance.public_ip_address,
                    user=config.server_username,
                    connect_kwargs={"key_filename": KEY_PATH}) as connection:
        # Wait for server to finish booting (keep trying until you can successfully run a command on the server via ssh)
        retry(connection.run,
              "# waiting for ssh to be connectable...",
              max_retries=100)

        connection.sudo("apt-get -qq -y update")

        connection.sudo(
            "apt-get -qq -y install -q python python-dev python-pip")
        connection.sudo("pip install --upgrade pip")
        connection.sudo("apt-get -qq -y remove -q python-pip")
        connection.sudo("hash -r")

        connection.sudo("apt-get -qq -y install -q python3-pip sqlite")
        connection.sudo("pip3 install --upgrade pip")
        connection.sudo("apt-get -qq -y remove -q python3-pip")
        connection.sudo("hash -r")

        connection.put("jupyterhub_files/requirements_jupyterhub.txt",
                       remote_path="/var/tmp/")
        connection.sudo(
            "pip3 install --quiet -r /var/tmp/requirements_jupyterhub.txt")

        connection.sudo("pip3 -q install ipython jupyter ipykernel nbgrader")
        connection.sudo("pip2 -q install ipykernel --upgrade")

        # register Python 3 and 2 kernel
        connection.sudo("python3 -m ipykernel install")
        connection.sudo("python2 -m ipykernel install")
        connection.sudo("chmod 755 /mnt")
        connection.sudo("chown ubuntu /mnt")

    # Create AMI for workers
    logger.info("Creating worker AMI")
    ami_name = "jupyter-hub-%s-worker-image" % config.cluster_name
    worker_ami = instance.create_image(Name=ami_name)

    # Wait until AMI is ready or 300 seconds have elapsed to allow for server restart
    for i in range(100):
        if get_resource(config.region).Image(
                worker_ami.id).state == "available":
            break
        logger.info("AMI not ready yet (running for ~%s seconds)" % (i * 3))
        sleep(3)

    instance.terminate()
    return worker_ami.id
Ejemplo n.º 18
0
def new_user(username, ssh_key, host):
    with Connection(
            host=host,
            user='******',
            port=22,
            connect_kwargs={"key_filename": "/Users/panyan/.ssh/id_rsa"},
            connect_timeout=5) as c:
        create_user(c, username, ssh_key)
Ejemplo n.º 19
0
def get_connection(ctx):
    if isinstance(ctx, Connection):
        conn = ctx
    else:
        conn = Connection(ctx.host, ctx.user,
                          connect_kwargs=ctx.connect_kwargs)
    if conn is None:
        sys.exit("Failed to get connection")
    return conn
Ejemplo n.º 20
0
def main():
    fab = Connection(host='www.xachf.com',
                     user='******',
                     port=22,
                     connect_timeout=20,
                     connect_kwargs={"password": "******"})
    # print(fab)
    res = fab.run('uname -s', hide=True)
    # 'command', 'connection', 'encoding', 'env', 'exited', 'failed', 'hide', 'ok', 'pty', 'return_code', 'shell', 'stderr', 'stdout', 'tail'
    # print(dir(res))
    # print(res.stdout.strip())
    # print(res.command, res.connection.host,res.connection.user)
    if res.ok:
        print('{}>>用户{} 执行[{}]命令成功'.format(res.connection.host,
                                           res.connection.user, res.command))
    else:
        print('{}>>用户{} 执行[{}]命令失败'.format(res.connection.host,
                                           res.connection.user, res.command))
Ejemplo n.º 21
0
 def setUp(self):
     self.host = Host.objects.get(name='localhost')
     self.server = Server.objects.get(host=self.host)
     self.user = User.objects.get(username='******')
     config = MINKE_FABRIC_CONFIG.clone()
     self.con = Connection(self.host.hostname,
                           self.host.username,
                           config=config)
     self._REGISTRY = sessions.REGISTRY.copy()
Ejemplo n.º 22
0
 def wait_until_SSHable(self, ip_address_string, max_retries=1):
     """ Run a meaningless bash command (a comment) inside a retry statement. """
     self.log_user("wait_until_SSHable()")
     with Connection(user=WORKER_USERNAME, host=ip_address_string, connect_kwargs=FABRIC_CONNECT_KWARGS) as c:
         self.log_user("wait_until_SSHable max_retries:%s" % max_retries, level=logging.DEBUG)
         ret = yield run(c, "# waiting for ssh to be connectable for user %s..." % self.user.name, max_retries=max_retries)
     self.log_user("wait_until_SSHable completed return: %s" % ret, level=logging.DEBUG)
     if ret == "RETRY_FAILED":
        ret = "SSH_CONNECTION_FAILED"
     return (ret)
Ejemplo n.º 23
0
def ConnectNode(u, p, h):
    print("----> Connecting to ", h)
    global connect

    try:
        connect = Connection(host=h, user=u, connect_kwargs={"password": p})
    except:
        print(
            "Wrong ssh connexion parameters. Please check also that you are using fabric v2.0"
        )  # trouver un moyen de faire printer cela
    return (connect)
Ejemplo n.º 24
0
def connect_to_host(ctx):
    if isinstance(ctx, Connection):
        return ctx
    else:
        password = ENV_VALUES['password']
        config = Config(overrides={'sudo': {'password': password}})
        conn = Connection(ctx.host,
                          ctx.user,
                          connect_kwargs=ctx.connect_kwargs,
                          config=config)
        return conn
Ejemplo n.º 25
0
 def __init__(self, task_queue, result_queue, lggr, appconfig):
     super(M3TaskPerformer, self).__init__()
     if appconfig['ENVNAME'] == 'local':
         self.host = 'mugopala@localhost'
     else:
         self.host = None
     self.logger = lggr
     self.conn = Connection(self.host)
     # TODO implement any other environment initialization
     self.task_queue = task_queue
     self.result_queue = result_queue
Ejemplo n.º 26
0
def remote_run(server_ip, host, passwd, cmd, remote_env):
    """Run command in remote node."""
    print("remote_env:", remote_env)
    _env_export = "export PATH={}/bin:$PATH".format(remote_env["conda"])
    if "env" in remote_env.keys():
        for _key, _val in remote_env["env"].items():
            _env_export += "&& export {}={}".format(_key, _val)

    with Connection(server_ip, user=host, connect_kwargs={"password":
                                                          passwd}) as connect:
        with connect.prefix("{}".format(_env_export)):
            connect.run(cmd, pty=False, echo=True)
Ejemplo n.º 27
0
def main():
    hosts = ['192.168.19.128', '192.168.19.129']
    Config.ssh_config_path = 'ssh_config'  # SSH Config File

    try:
        for host in hosts:
            con = Connection(host)
            # All Rask Run
            print()
            print('==============')
            print(ahc(con, 'date', hide=True, hostNameShow=False))
            print(hostname(con, hide=True))
            print(ip(con, hide=True))
            print(disk_free(con, hide=True))
            print(chpasswd(con, 'python', hide=True))
            con.close()

    except RuntimeError as r:
        sys.stderr.write('=== Runtime Error! ===')
        sys.stderr.write(str(r.args))
    print('ALL Done!')
Ejemplo n.º 28
0
def distribute_xt_if_need(config, remote_env, remote_path="xt_archive"):
    """
    Distribute Xingtian sourcecode among use's node configure.

    :param config: config instance from config.yaml
    :param remote_env: remote conda environment path
    :param remote_path: path to store the wheel file. 'xt_archive' default.
    :return:
    """
    local_ip = get_host_ip()

    # check could if distribute or not
    remote_ip_list = list()
    for _key in (
            "node_config",
            "test_node_config",
    ):
        if _key not in config.keys():
            continue
        for _ip, _user, _password in config.get(_key):
            # local need not distribute
            if _ip in (local_ip, "127.0.0.1"):
                continue
            remote_ip_list.append((_ip, _user, _password))

    if not remote_ip_list:
        logging.debug("Don't distribute xingtian without remote ip set.")
        return True

    dist_path = _package_xt()
    if not remote_env:
        logging.fatal("must assign remote env in yaml.")

    for _ip, _user, _password in remote_ip_list:
        with Connection(_ip,
                        user=_user,
                        connect_kwargs={"password": _password}) as connect:
            _workspace = os.path.join("/tmp")
            target_whl = glob.glob("{}/xingtian*.whl".format(dist_path))
            logging.info("found dist: {}".format(target_whl))
            for _whl in target_whl:
                _name = os.path.basename(_whl)
                _remote_cmd = "pip install {}/{} --upgrade --force-reinstall --no-deps".format(
                    _workspace, _name)
                logging.info(
                    "re-install xingtian in remote-{} conda env {} >>> \n"
                    "{}".format(_ip, remote_env["conda"], _remote_cmd))

                connect.put(os.path.join(dist_path, _name), remote=_workspace)
                with connect.prefix("export PATH={}/bin:$PATH".format(
                        remote_env["conda"])):
                    connect.run(_remote_cmd, pty=False)
Ejemplo n.º 29
0
 def generateSSHKey(self):
     display('Generating ssh key...')
     c = Connection('localhost')
     c.local('rm -f "{keyFile}.*"'.format(**self.properties))
     c.local("echo 'yes' | ssh-keygen -t rsa -f {keyFile} -C {username}  -N '' ".format(**self.properties),hide='out')
     c.close()
     #p = Popen("echo 'yes' | ssh-keygen -t rsa -f {keyFile} -C {username} -N '' ".format(**self.properties),
     #              stdout=PIPE,
     #              shell=True,
     #              stderr=PIPE
     #               )
     #print(p.communicate())
     with open (self.properties['pubKeyFile'],'r') as f:
         display('Opening {}'.format(self.properties['pubKeyFile']))
         self.pub = f.read().strip()
Ejemplo n.º 30
0
    def _inner():
        logger.warning(
            f'将本地文件夹代码 {python_proj_dir}  上传到远程 {host} 的 {remote_dir} 文件夹。')
        t_start = time.perf_counter()
        uploader = ParamikoFolderUploader(
            host, port, user, password, python_proj_dir, remote_dir,
            path_pattern_exluded_tuple, file_suffix_tuple_exluded,
            only_upload_within_the_last_modify_time, file_volume_limit,
            sftp_log_level)
        uploader.upload()
        logger.info(
            f'上传 本地文件夹代码 {python_proj_dir}  上传到远程 {host} 的 {remote_dir} 文件夹耗时 {round(time.perf_counter() - t_start, 3)} 秒'
        )
        # conn.run(f'''export PYTHONPATH={remote_dir}:$PYTHONPATH''')

        func_name = task_fun.__name__
        queue_name = task_fun.consumer.queue_name

        process_mark = f'fsdf_fabric_mark__{queue_name}__{func_name}'
        conn = Connection(
            host,
            port=port,
            user=user,
            connect_kwargs={"password": password},
        )
        kill_shell = f'''ps -aux|grep {process_mark}|grep -v grep|awk '{{print $2}}' |xargs kill -9'''
        logger.warning(f'{kill_shell} 命令杀死 {process_mark} 标识的进程')
        uploader.ssh.exec_command(kill_shell)
        # conn.run(kill_shell, encoding='utf-8',warn=True)  # 不想提示,免得烦扰用户以为有什么异常了。所以用上面的paramiko包的ssh.exec_command

        python_exec_str = f'''export is_fsdf_remote_run=1;export PYTHONPATH={remote_dir}:$PYTHONPATH ;python3 -c "from {relative_module} import {func_name};{func_name}.multi_process_consume({process_num})"  -fsdfmark {process_mark} '''
        shell_str = f'''cd {remote_dir}; {python_exec_str}'''
        extra_shell_str2 = extra_shell_str  # 内部函数对外部变量不能直接改。
        if not extra_shell_str2.endswith(';') and extra_shell_str != '':
            extra_shell_str2 += ';'
        shell_str = extra_shell_str2 + shell_str
        logger.warning(f'使用语句 {shell_str} 在远程机器 {host} 上启动任务消费')
        conn.run(shell_str, encoding='utf-8', **invoke_runner_kwargs)
Ejemplo n.º 31
0
def sanity_test_from_v1(c):
    """
    Run some very quick in-process sanity tests on a dual fabric1-v-2 env.

    Assumes Fabric 2+ is already installed as 'fabric2'.
    """
    # This cannot, by definition, work under Python 3 as Fabric 1 is not Python
    # 3 compatible.
    PYTHON = environ.get("TRAVIS_PYTHON_VERSION", "")
    if PYTHON.startswith("3") or PYTHON == "pypy3":
        return
    c.run("pip install 'fabric<2'")
    # Make darn sure the two copies of fabric are coming from install root, not
    # local directory - which would result in 'fabric' always being v2!
    for serious in (getcwd(), ""):
        if serious in sys.path:  # because why would .remove be idempotent?!
            sys.path.remove(serious)

    from fabric.api import env
    from fabric2 import Connection

    env.gateway = "some-gateway"
    env.no_agent = True
    env.password = "******"
    env.user = "******"
    env.host_string = "localghost"
    env.port = "2222"
    cxn = Connection.from_v1(env)
    config = cxn.config
    assert config.run.pty is True
    assert config.gateway == "some-gateway"
    assert config.connect_kwargs.password == "sikrit"
    assert config.sudo.password == "sikrit"
    assert cxn.host == "localghost"
    assert cxn.user == "admin"
    assert cxn.port == 2222
Ejemplo n.º 32
0
def deploy(c, restart=False, sync=False):
    "push code to a server (configure your own here)"

    c.run("git push --all")
    conn = Connection("trotsky")
    with conn.cd("src/pyborg-1up"):
        conn.run("git fetch")
        conn.run("git stash")
        conn.run("git pull")
        conn.run("git stash pop")
        if sync:
            conn.run("~/.local/bin/pipenv sync")  # they all use the same pipenv managed virtualenv
        if restart:
            for unit in ["pyborg_discord", "pyborg_http", "pyborg_twitter", "pyborg_mastodon"]:
                conn.sudo("systemctl restart {}".format(unit), pty=True)
        print("Deploy Completed.")