예제 #1
0
class VIRLSim(object):
    "Defines the simulation element and holds configuration information of a VIRL simulation."

    # will sleep for 'timeout / INTERVAL' when waiting for sim to start
    INTERVAL = 30

    def __init__(self,
                 host,
                 user,
                 password,
                 filename,
                 logger=None,
                 timeout=300,
                 port=19399):
        super(VIRLSim, self).__init__()
        self._host = host
        self._port = port
        self._filename = filename
        self._logger = logger
        self._timeout = timeout
        self._session = requests.Session()
        self._username = user
        self._password = password
        self._session.auth = (user, password)
        self._sim_id = None
        self._lxc_port = None
        self._lxc_host = host
        self._no_start = False
        self._semaphore = Semaphore()
        self._ssh_client = None
        self._ssh_interact = None

    def _url(self, method='', roster=False):
        """Return the proper URL given the set vars and the
        method parameter."""
        api = 'simengine'
        if roster:
            api = 'roster'
        return "http://{}:{}/{}/rest/{}".format(self._host, self._port, api,
                                                method)

    def _request(self, verb, method, *args, **kwargs):
        url = self._url(method, roster=kwargs.pop('roster', False))
        r = self._session.request(verb, url, *args, **kwargs)
        if not r.ok:
            self.log(ERROR, 'VIRL API [%s]: %s', r.status_code,
                     r.json().get('cause'))
        return r

    def _post(self, method, *args, **kwargs):
        return self._request('POST', method, *args, **kwargs)

    def _get(self, method, *args, **kwargs):
        return self._request('GET', method, *args, **kwargs)

    def _delete(self, method, *args, **kwargs):
        return self._request('DELETE', method, *args, **kwargs)

    def log(self, level, *args, **kwargs):
        "Send the message in args to the logger with the given level."
        sim = self._sim_id if self._sim_id is not None else '<unknown>'
        newargs = list(args)
        newargs[0] = ': '.join((sim, args[0]))
        if self._logger is not None:
            self._logger.log(level, *newargs, **kwargs)

    def isLogDebug(self):
        "Is logging enabled?"
        return self._logger.getEffectiveLevel() == DEBUG

    @property
    def simId(self):
        "The simulation ID on the VIRL host."
        return self._sim_id

    @simId.setter
    def simId(self, value):
        self._sim_id = value

    @property
    def simHost(self):
        "Returns the name of the simulation host."
        return self._host

    @property
    def simUser(self):
        "Returns the name of the user running the sim."
        return self._username

    @property
    def simPass(self):
        "Returns the password used to run the simulation."
        return self._password

    @property
    def simTimeout(self):
        "Returns the timeout set for the simulation."
        return self._timeout

    @property
    def sshInteract(self):
        "Returns a SSH object to interact with the simulation via LXC."
        if self._ssh_interact is not None:
            return self._ssh_interact
        return self.sshOpen()

    @property
    def simPollInterval(self):
        "Returns the poll interval (how often to check state) for the sim."
        interval = self._timeout // self.INTERVAL
        if interval == 0:
            interval = self._timeout
        return interval

    def lock(self):
        "Lock the simulation, admit only one at a time."
        self._semaphore.acquire()

    def unlock(self):
        "Unlocks the simulation."
        self._semaphore.release()

    def startSim(self):
        "This function will start a simulation using the provided .virl file."
        sim_name = os.path.basename(os.path.splitext(self._filename)[0])
        self.log(WARN, 'Starting [%s]' % sim_name)

        # for debugging purposes. uses existing sim, does not start nor stop
        if self._no_start and self._sim_id:
            return True

        # Open .virl file and assign it to the variable
        ok = False
        try:
            with open(os.path.expanduser(self._filename), 'rb') as virl_file:
                # Parameter which will be passed to the server with the API call
                params = dict(file=sim_name)

                # Make an API call and assign the response information to the
                # variable
                r = self._post('launch', params=params, data=virl_file)

                # Check if call was successful, if true log it and return the value
                if r.status_code == 200:
                    self._sim_id = r.text
                    self.log(WARN, 'Simulation started.')
                ok = r.ok
        except IOError as e:
            self.log(CRITICAL, 'open file: %s', e)
        return ok

    def waitForSimStart(self):
        """Returns True if the sim is started and all nodes are active/reachable
        waits for self._timeout (default 5min)."""

        active = False
        self.log(WARN, 'Waiting %ds to become active...', self._timeout)

        # for testing purposes
        if self._no_start and self._sim_id:
            return True

        endtime = datetime.utcnow() + timedelta(seconds=self._timeout)
        while not active and endtime > datetime.utcnow():

            # Make an API call and assign the response information to the
            # variable
            r = self._get('nodes/%s' % self._sim_id)
            if not r.ok:
                return False

            # check if all nodes are active AND reachable
            nodes = r.json()[self._sim_id]
            active = True
            for node in nodes.values():
                if node['state'] == 'SHUTOFF':
                    continue
                if not (node['state'] == 'ACTIVE' and node['reachable']):
                    active = False
                    break

            # wait if not
            if not active:
                sleep(self.simPollInterval)

        # for testing purposes
        #active = False
        #nodes['csr1000v-1']['reachable'] = False

        if active:
            self.log(WARN, "Simulation is active.")
        else:
            self.log(ERROR, "Timeout... aborting!")

            # write status log file
            with open("status-%s.log" % self._sim_id, "w") as fh:
                fh.write(dumps(self.getStatus(), indent=2))

            for name, node in nodes.items():
                state = node['state']
                reachable = node['reachable']
                if state == 'ACTIVE' and not reachable:
                    self.log(ERROR, "%s: %s, %s", name, state, reachable)

                    subtype, serial_port = self.getNodeDetail(name)
                    if serial_port is not None:
                        postMortem(self, name, subtype, self._host,
                                   serial_port)
                        self.log(ERROR, "error log written!")

        return active

    def stopSim(self, wait=False):
        "This function will stop the simulation."
        self.log(WARN, 'Simulation stop...')

        # for debugging purposes
        if self._no_start and self._sim_id:
            return

        # ensure SSH sessions are closed, if open
        self.sshClose()

        # Make an API call and assign the response information to the variable
        r = self._get('stop/%s' % self._sim_id)

        # Check if call was successful
        if r.status_code == 200:
            self.log(INFO, 'Simulation stop initiated.')

            # should we wait until all nodes are stopped?
            if wait:
                status = self.getStatus()
                waited = 0
                while not status['state'] == "DONE":

                    # print(dumps(status, indent=2))
                    seconds = self.simPollInterval
                    self.log(INFO, 'sleeping %ds' % seconds)
                    sleep(seconds)

                    # only wait for so long before giving up
                    waited += seconds
                    if waited > self._timeout / 2:
                        self.log(CRITICAL, 'Simulation did NOT stop.')
                        break
                    status = self.getStatus()

                if status['state'] == "DONE":
                    self.log(INFO, 'Simulation finally stopped.')
            # we might rely on the _sim_id after stop
            # for logging purposes.
            # self._sim_id = None

    def getNodeDetail(self, node):
        """Get the node subtype and console port of the given node
        guest|csr1kv-single-test-9DYnbf|virl|csr1000v-1
        """
        self.log(INFO, "Getting console port for [%s]...", node)
        r = self._get('', roster=True)
        if r.ok:
            for k, v in r.json().items():
                f = k.split('|')
                if len(f) > 1 and f[1] == self._sim_id and f[3] == node:
                    return (v.get('NodeSubtype'), v.get('PortConsole'))
        return ('unknown', 0)

    def getEvents(self):
        "Get the events associated with the sim."
        self.log(INFO, "Getting events...")
        r = self._get('events/%s' % self._sim_id)
        if r.ok:
            return r.json()
        return '{}'

    def getStatus(self):
        "Get the status messages associated with the sim."
        self.log(INFO, "Getting status messages...")
        r = self._get('status/%s' % self._sim_id)
        if r.ok:
            return r.json()
        return '{}'

    def getInterfaces(self, node):
        """Return the list of interfaces for the given node or
        None if not found."""
        self.log(INFO, "Getting interfaces for [%s]...", node)
        params = dict(nodes=node)
        r = self._get('interfaces/%s' % self._sim_id, params=params)
        if r.ok:
            interfaces = r.json().get(self._sim_id).get(node)
            return interfaces
        self.log(ERROR, 'node not found: %s', node)
        return None

    def getInterfaceId(self, node, interface):
        "Get the interface index for the given interface name."
        self.log(INFO, "Getting ID from name [%s]...", interface)
        interfaces = self.getInterfaces(node)
        for key, intfc in interfaces.items():
            if intfc.get('name') == interface:
                self.log(INFO, "Found id: %s", key)
                return key
        self.log(ERROR, "Can't find specified interface %s", interface)
        return None

    def createCapture(self, node, interface, pcap_filter, count):
        """Create a packet capture for the simulation using the given
        parameters in cfg."""
        self.log(WARN, "Starting packet capture...")

        # get interface based on name
        interface = self.getInterfaceId(node, interface)
        if interface is None:
            self.log(ERROR, "interface not found", interface)
            return None

        params = dict(node=node, interface=interface, count=count)
        params['pcap-filter'] = pcap_filter
        r = self._post('capture/%s' % self._sim_id, params=params)
        # did it work?
        cap_id = ""
        if r.ok:
            cap_id = list(r.json().keys())[0]
            self.log(INFO, "Created packet capture (%s)", cap_id)
        return cap_id

    def deleteCapture(self, cap_id):
        "Delete the given packet capture with cap_id for the simulation."
        self.log(INFO, "Deleting packet capture...")

        params = dict(capture=cap_id)
        r = self._delete('capture/%s' % self._sim_id, params=params)
        return r.ok

    def waitForCapture(self, cap_id, wait=None):
        """Wait until the packet capture is done. check for the 'running'
        state according to the set wait time divided by INTERVAL
        divisor."""

        if wait is None:
            wait = self._timeout

        done = False
        self.log(INFO, 'Waiting %ds for capture [%s]', wait, cap_id)

        endtime = datetime.utcnow() + timedelta(seconds=wait)
        while not done and endtime > datetime.utcnow():

            # Make an API call and assign the response information to the
            # variable
            r = self._get('capture/%s' % self._sim_id)
            if not r.ok:
                return False

            # check if all nodes are active AND reachable
            captures = r.json()
            for cid, cval in captures.items():
                if cid == cap_id and not cval.get('running'):
                    done = True
                    break

            # wait if not
            if not done:
                sleep(self.simPollInterval)

        if done:
            self.log(WARN, "Capture has finished.")
        else:
            self.log(ERROR, "Timeout... aborting!")

        return done

    def downloadCapture(self, pcap_id):
        "Download the finished capture and write it into a file."

        content = 'application/vnd.tcpdump.pcap'
        params = dict(capture=pcap_id)
        headers = dict(accept=content)
        self.log(WARN, 'Downloading capture file...')

        # Make an API call and assign the response information to the
        # variable
        r = self._get('capture/%s' % self._sim_id,
                      params=params,
                      headers=headers)
        if r.status_code == 200 and r.headers.get('content-type') == content:
            # "ContentDisposition":
            # "attachment; filename=V1_Ethernet1_1_2016-10-15-17-18-18.pcap
            filename = r.headers.get('Content-Disposition').split('=')[1]
            with open(filename, "wb") as fh:
                fh.write(r.content)
        else:
            self.log(ERROR, "problem... %s", r.headers)
            return False

        self.log(WARN, "Download finished.")
        return True

    def getMgmtIP(self, node):
        "Return the management IP of the given Node name."

        interfaces = self.getInterfaces(node)
        if interfaces is None:
            return None

        for key, intfc in interfaces.items():
            if key == 'management' and intfc.get('ip-address') is not None:
                address = intfc.get('ip-address').split('/')[0]
                self.log(INFO, "Found mgmt ip for %s: %s", node, address)
                return address

        self.log(ERROR, "Can't find Mgmt IP")
        return None

    def getLXCPort(self):
        """Return the TCP port of the LXC host for the simulation
        the LXC can then be reached via the sim host on this port using
        SSH as the protocol."""

        if self._lxc_port is not None:
            return self._lxc_port

        self._semaphore.acquire()
        interfaces = self.getInterfaces('~mgmt-lxc')
        if interfaces is not None:
            for key, intfc in interfaces.items():
                if key != 'management' and \
                   intfc.get('external-ip-address') is not None:
                    self._lxc_port = int(intfc.get('external-port'))
                    self.log(INFO, "Found LXC port: %s", self._lxc_port)
                    break

        # crude hack to make it work with ngrok
        tmp_lxc = os.environ.get('VIRL_LXC_PORT', None)
        if tmp_lxc is not None and tmp_lxc:
            self._lxc_port = int(tmp_lxc)
        tmp_host = os.environ.get('VIRL_LXC_HOST', None)
        if tmp_host is not None and tmp_host:
            self._lxc_host = tmp_host

        if self._lxc_port is None:
            self.log(ERROR, "Can't find LXC port")
        self._semaphore.release()
        return self._lxc_port

    def sshOpen(self, timeout=5):
        "Opens a SSH connection to the mgmt LXC."
        if self._ssh_interact is not None:
            return self._ssh_interact

        self.log(WARN, 'Acquiring LXC SSH session')
        self._ssh_client = paramiko.SSHClient()
        paramiko.hostkeys.HostKeys(filename=os.devnull)
        # client.load_system_host_keys()
        self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        # the below might update self._lxc_host if the env var is set during
        # the execution of getLXCPort().
        port = self.getLXCPort()
        try:
            self._ssh_client.connect(hostname=self._lxc_host,
                                     username=self._username,
                                     pkey=None,
                                     look_for_keys=False,
                                     allow_agent=False,
                                     password=self._password,
                                     port=port)
        except (paramiko.AuthenticationException, paramiko.SSHException) as e:
            self.log(CRITICAL, 'SSH connect failed: %s' % e)
            return None

        self._ssh_interact = SSHClientInteraction(self._ssh_client,
                                                  timeout=timeout,
                                                  display=self.isLogDebug())
        return self._ssh_interact

    def sshClose(self):
        "Closes the connection to the mgmt LXC, if it exists."
        if self._ssh_interact is None:
            return
        self.log(WARN, 'Closing LXC SSH session')
        self._ssh_interact.close()
        self._ssh_client.close()
        self._ssh_interact = self._ssh_client = None
예제 #2
0
class NetDevices(object):

    #currentDateTime = datetime.datime.now()
    number_of_devices = 0
    failed_login_attempts = 0
    max_failed_login = 3
    successList = []
    failedList = []
    startTime = time.time()

    debugLock = threading.Lock()
    errorLock = threading.Lock()
    successLock = threading.Lock()
    failedLock = threading.Lock()
    resultsLock = threading.Lock()
    outputLock = threading.Lock()
    completeLock = threading.Lock()

    maxNumberOfThreads = 6
    threadLimiter = threading.BoundedSemaphore(maxNumberOfThreads)

    threadCounter = 0

    def locking(cls):
        lock = thread.Lock()
        lock.acquire()
        lock.release()

    @classmethod
    def set_configs(cls, configs):
        pass

    @staticmethod
    def model_check(show_verison):
        pass

    @classmethod
    def promptUserCred(cls):
        from getpass import getpass
        username = get_input("Please enter your username: "******"./logs/paramiko.log")
        try:
            for message in messages:

                cls.formatter = logging.Formatter(
                    '%(asctime)s:%(levelname)s:%(name)s:%(message)s')
                cls.debug_logger = logging.getLogger(__name__)
                cls.debug_logger.setLevel(logging.DEBUG)
                cls.debug_file_handler = logging.FileHandler(
                    './logs/debug.log')
                cls.debug_file_handler.setFormatter(cls.formatter)
                cls.debug_logger.addHandler(cls.debug_file_handler)

                cls.debug_logger.debug(message)

        finally:
            cls.debugLock.release()

    @classmethod
    def displayOutput(cls, self):

        cls.outputLock.acquire()

        outputBorder = (
            '\n\n\n' + '-' * 79 + '\n' + '            Hostname: ' +
            self.hostname + '            ' +
            datetime.datetime.now().strftime("%b %d %Y %H:%M:%S") +
            '          \n' +
            '------------------------------<---Output Screen--->----------------------------\n'
            + '-' * 79 + '\n')

        print(colored(outputBorder, 'cyan'))
        '''Displaying device output'''
        for output in self.stdout:
            print(output)

        cls.outputLock.release()

    @classmethod
    def displayResults(cls):

        outputBorder = (
            '-' * 79 + '\n---------------------------------' +
            datetime.datetime.now().strftime("%b %d %Y %H:%M:%S") +
            '--------------------------\n------------------------------<---Results Screen--->---------------------------\n'
            + '-' * 79 + '\n')

        try:
            if not os.path.exists('./logs/'):
                os.makedirs('./logs/')

                NetDevices.logger_error('Path created:  ./logs/')
                NetDevices.logger_debug('Path created:  ./logs/')

            with open(os.path.join('./logs/results.log'), 'a') as resultFile:

                print(colored(outputBorder, 'cyan'))

                resultFile.write(outputBorder)

                for result in cls.successList:
                    print(colored(result, 'green'))
                    resultFile.write(result + '\n')

                resultFile.write(
                    '\n------------------------------<---Failures--->---------------------------------\n'
                )
                print(
                    colored(
                        '\n------------------------------<---Failures--->---------------------------------\n',
                        'red'))

                for result in cls.failedList:
                    print(colored(result, 'red'))
                    resultFile.write(result + '\n')

        except IOError:
            NetDevices.logger_error('results.log file created.')
            NetDevices.logger_debug('results.log file created.')

            with open(os.path.join('results.log'), 'w') as resultFile:

                print(colored(outputBorder, 'cyan'))

                resultFile.write(outputBorder)

                for result in cls.successList:
                    print(result)
                    resultFile.write(result + '\n')

                resultFile.write(
                    '------------------------------<---Failures--->---------------------------------\n'
                )
                print(
                    '------------------------------<---Failures--->---------------------------------\n'
                )
                for result in cls.failedList:
                    print(result)
                    resultFile.write(result + '\n')

        print(
            colored(
                "\nScript took --- %s seconds ---" %
                (time.time() - cls.startTime) + '\n', 'cyan'))

    def threadMethod(function):
        '''
        @threadMethod
        def func_to_be_threaded(self):
            #main body

            see ---> https://stackoverflow.com/questions/19846332/python-threading-inside-a-class
        '''
        def wrapper(*args, **kwargs):

            thread = threading.Thread(target=function,
                                      args=args,
                                      kwargs=kwargs)
            thread.daemon = True
            thread.start()
            return thread

        return wrapper

    def __init__(self,
                 hostname='generic',
                 model='generic',
                 configs="generic",
                 user='******',
                 password='',
                 timeout=10,
                 display=True):
        self.hostname = hostname
        self.model = model
        self.configs = configs
        self.user = user
        self.password = password
        self.timeout = timeout
        self.display = display
        self.stdout = []
        self.commandList = None
        self.debugList = []
        self.errorList = []

        ###expect prediction
        #self.expect_predict = expect_predict

        ###methods to connect, SSH, telnet, console
        #self.connect_method = connect_method

        NetDevices.number_of_devices += 1

    @threadMethod
    def connect(self, method='SSH'):

        NetDevices.threadLimiter.acquire()
        NetDevices.threadCounter += 1
        NetDevices.logger_debug("\nNumber of threads active: " +
                                str(NetDevices.threadCounter))

        try:
            if method == 'SSH':
                self.remote_conn_pre = paramiko.SSHClient()
                self.remote_conn_pre.set_missing_host_key_policy(
                    paramiko.AutoAddPolicy())
                try:
                    self.remote_conn_pre.connect(hostname=self.hostname,
                                                 port=22,
                                                 username=self.user,
                                                 password=self.password,
                                                 timeout=self.timeout,
                                                 look_for_keys=False)

                except paramiko.ssh_exception.socket.gaierror:
                    print(
                        colored(
                            'Connection Timeout or unknown host on: [{}]'.
                            format(self.hostname), 'red'))
                    NetDevices.failedLock.acquire()
                    NetDevices.failedList.append(
                        'Connection Timeout or unknown host on: [{}]'.format(
                            self.hostname))
                    NetDevices.failedLock.release()
                    NetDevices.logger_error(
                        'Connection Timeout or unknown host on: [{}]'.format(
                            self.hostname))
                    NetDevices.logger_debug(
                        'Connection Timeout or unknown host on: [{}]'.format(
                            self.hostname))
                    return

                except paramiko.ssh_exception.socket.timeout:
                    print(
                        colored(
                            'Connection Timeout or unknown host on:  [{}]'.
                            format(self.hostname), 'red'))
                    NetDevices.failedLock.acquire()
                    NetDevices.failedList.append(
                        'Connection Timeout or unknown host on: [{}]'.format(
                            self.hostname))
                    NetDevices.failedLock.release()
                    NetDevices.logger_error(
                        'Connection Timeout or unknown host on: [{}]'.format(
                            self.hostname))
                    NetDevices.logger_debug(
                        'Connection Timeout or unknown host on: [{}]'.format(
                            self.hostname))
                    return

                ###-----catches authentication failures, limit 2 failures and script will stop

                except paramiko.ssh_exception.AuthenticationException:
                    print(
                        colored('Authentication failed on: ' + self.hostname,
                                'red'))
                    NetDevices.failedLock.acquire()
                    NetDevices.failedList.append(
                        'Authentication failed on: [{}]'.format(self.hostname))
                    NetDevices.failedLock.release()
                    NetDevices.logger_error(
                        'Authentication failed on: [{}]'.format(self.hostname))
                    NetDevices.logger_debug(
                        'Authentication failed on: [{}]'.format(self.hostname))

                    NetDevices.failed_login_attempts += 1

                    print(
                        colored(('Number of failed attempts: ' +
                                 NetDevices.failed_login_attempts), 'red'))
                    if (NetDevices.failed_login_attempts >
                            NetDevices.max_failed_login):
                        print(
                            colored(
                                'Warning!!! Too many authentication failures, danger of your account being locked out!!!',
                                'red'))
                        NetDevices.logger_error(
                            'Too many failed authentication' +
                            NetDevices.failed_login_attempts +
                            'number of failed attempts, script cancelled.')
                        NetDevices.logger_debug(
                            'Too many failed authentication' +
                            NetDevices.failed_login_attempts +
                            'number of failed attempts, script cancelled.')

                        sys.exit('!!!!!!Script will stop!!!!!!')

                except Exception as e:
                    print(
                        colored(
                            ("Connection error: [{}]\n Traceback error: {}".
                             format(self.hostname, str(e))), 'red'))
                    NetDevices.logger_error(
                        "Connection error: [{}]\n Traceback error: {}".format(
                            self.hostname, str(e)))
                    NetDevices.logger_debug()
                    NetDevices.failedLock.acquire()
                    NetDevices.failedList.append(
                        "Connection error: [{}]\n Traceback error: {}".format(
                            self.hostname, str(e)))
                    NetDevices.failedLock.release()

                    return

                ###------interact expect starts and enable mode
                try:
                    self.interact = SSHClientInteraction(self.remote_conn_pre,
                                                         timeout=self.timeout,
                                                         display=False)
                    print(
                        colored(('Connection established on: : [{}]'.format(
                            self.hostname)), 'green'))
                    self.stdout.append(self.interact.current_output)

                    NetDevices.logger_debug(
                        'Connection established on: [{}]'.format(
                            self.hostname))

                    self.interact.expect(['.*\#', '.*\>'],
                                         timeout=self.timeout)
                    if self.interact.last_match == '.*\>':
                        self.interact.send('enable')
                        self.stdout.append(self.interact.current_output)

                        self.interact.expect('.*assword:*.',
                                             timeout=self.timeout)
                        self.interact.send(self.password)
                        self.stdout.append(self.interact.current_output)

                    self.interact.expect(['.*\#', '.*\>'])
                    self.interact.send('terminal length 0')
                    self.stdout.append(self.interact.current_output)

                    if self.interact.last_match == '.*\>':
                        time.sleep(1)
                        self.interact.send('enable')
                        self.interact.expect('.*assword:*.')
                        time.sleep(1)
                        self.stdout.append(interact.current_output)
                        self.interact.send(self.password)
                        self.interact.expect(['.*\#', '.*\>'])
                        self.interact.send('terminal length 0')
                        self.stdout.append(interact.current_output)
                        if self.interact.last_match == '.*\>':
                            self.stdout.append(self.interact.current_output)
                            self.interact.close()
                            print(
                                colored(
                                    ('Failed to enter Enable mode on: [{}]'.
                                     format(self.hostname)), 'red'))
                            NetDevices.failedList.append(
                                'Failed to enter Enable mode on: [{}]'.format(
                                    self.hostname))
                            NetDevices.logger_error(
                                'Failed to enter Enable mode on: [{}]'.format(
                                    self.hostname))
                            NetDevices.logger_debug(
                                'Failed to enter Enable mode on: [{}]'.format(
                                    self.hostname))
                            self.remote_conn_pre.close()

                except paramiko.ssh_exception.socket.timeout:
                    self.interact.close()
                    self.remote_conn_pre.close()
                    print(
                        colored((
                            'Connection lost...Exception Socket Timeout on: [{}]'
                            .format(self.hostname)), 'red'))
                    NetDevices.failedLock.acquire()
                    NetDevices.failedList.append(
                        'Connection lost...Exception Socket Timeout on: [{}]'.
                        format(self.hostname))
                    NetDevices.failedLock.release()
                    NetDevices.logger_error(
                        'Connection lost...Exception Socket Timeout on: [{}]'.
                        format(self.hostname))
                    NetDevices.logger_debug(
                        'Connection lost...Exception Socket Timeout on: [{}]'.
                        format(self.hostname))
                    return

                ###-------------------------------------sending commands------------------------------------------------------------

                if self.model == 'linux':
                    pass
                '''
                can pass commands single or pass *commands list
                '''
                if self.remote_conn_pre.get_transport() == None:
                    return
                try:

                    for command in self.commandList:
                        self.interact.send(command)
                        #time.sleep(1)
                        #self.stdout.append(self.interact.current_output)
                        #print(self.interact.current_output)
                        self.interact.expect(['.*\#', '.*\(y/n\) '])
                        #print(self.interact.current_output)
                        self.stdout.append(self.interact.current_output)
                        if self.interact.last_match == '.*\(y/n\) ':
                            self.interact.send('y')
                            self.stdout.append(self.interact.current_output)
                            time.sleep(2)
                            self.interact.expect('.*\#', timeout=180)
                            self.stdout.append(self.interact.current_output)

                    NetDevices.successLock.acquire()
                    try:
                        NetDevices.successList.append(
                            'Commands successful on: [{}]'.format(
                                self.hostname))
                    finally:
                        NetDevices.successLock.release()

                except Exception as e:
                    NetDevices.logger_error(
                        "Exception: on sending command on: [{}] Traceback error: {}"
                        .format(self.hostname, str(e)))
                    NetDevices.logger_debug()
                    self.stdout.append(self.interact.current_output)
                    NetDevices.failedLock.acquire()
                    NetDevices.failedList.append(
                        'Exception: on sending command on: [{}]'.format(
                            self.hostname))
                    NetDevices.failedLock.release()

                    #traceback.print_exc()

                self.log_output()

                outputBorder = (
                    '\n' + '-' * 79 + '\n' + '-' * 79 + '\n' +
                    '                   Hostname: ' + self.hostname.strip() +
                    '             ' +
                    datetime.datetime.now().strftime("%b %d %Y %H:%M:%S") +
                    '\n' + '-' * 79 + '\n' +
                    '----------------------------<---Commands completed--->-------------------------\n'
                )

                NetDevices.completeLock.acquire()
                print(colored(outputBorder, 'cyan'))
                NetDevices.completeLock.release()

                if self.display == True:
                    NetDevices.displayOutput(self)

            NetDevices.logger_error(*self.errorList)
            NetDevices.logger_debug(*self.debugList)
        finally:
            NetDevices.threadLimiter.release()
            NetDevices.threadCounter -= 1

            NetDevices.logger_debug("\nNumber of threads active: " +
                                    str(NetDevices.threadCounter))

    def setCommands(self, *commands):
        self.commandList = [command for command in commands]

    def send(self, *commands):
        pass

    def log_output(self):
        import datetime
        currentDateTime = datetime.datetime.now().strftime("%b %d %Y %H:%M")

        try:
            if not os.path.exists('./device-logs/' + currentDateTime):
                os.makedirs('./device-logs/' + currentDateTime)

                NetDevices.logger_error('Path created:  ./device-logs/')
                NetDevices.logger_debug('Path created:  ./device-logs/')
                NetDevices.logger_error('Path created:  ./device-logs/' +
                                        currentDateTime)
                NetDevices.logger_debug('Path created:  ./device-logs/' +
                                        currentDateTime)

            with open(
                    os.path.join('./device-logs/' + currentDateTime,
                                 self.hostname + '.log'), 'a') as outputFile:
                outputFile.write('-' * 79 + '\n')
                outputFile.write('Hostname: ' + self.hostname)
                outputFile.write(
                    '------------' +
                    datetime.datetime.now().strftime("%b %d %Y %H:%M:%S") +
                    '----------------------\n')
                outputFile.write(
                    '------------------------------<---Output Screen--->----------------------------\n'
                )
                outputFile.write('-' * 79 + '\n')

                for outline in self.stdout:
                    outputFile.write(outline + '\n')
            outputFile.close()

        except IOError:
            NetDevices.logger_error(self.hostname + '.log file created.')
            NetDevices.logger_debug(self.hostname + '.log file created.')

            with open(
                    os.path.join('./device-logs/' + currentDateTime,
                                 self.hostname + '.log'), 'w') as outputFile:
                outputFile.write('-' * 79 + '\n')
                outputFile.write('Hostname: ' + self.hostname)
                outputFile.write(
                    '------------' +
                    datetime.datetime.now().strftime("%b %d %Y %H:%M:%S") +
                    '----------------------\n')
                outputFile.write(
                    '------------------------------<---Output Screen--->----------------------------\n'
                )
                outputFile.write('-' * 79 + '\n')

                for outline in self.stdout:
                    outputFile.write(outline + '\n')
            outputFile.close()

    def make_excel(self, command, search_string):
        '''
        function to send a command and capture specific data and enter into excel file.
        '''
        pass

    def variable_command(self, currentDevice, *commands):
        '''
        function to use variable to loop through different devices.  Example: tftp different configs to specific matching device.
        '''
        pass

    def reboot_wait(numberOfDevices, waitTime):
        '''
        function to set a wait time for stagger reboots on hosts to prevent network being overburdened.
        '''
        pass

    def prompt_yes_no(answer='y'):
        '''
        function to pass in y or n when prompt with entering yes or no example, erasing configs.
        '''
        pass

    def tftpServer():

        pass

    def upgradeFirmware():
        if model == x:
            pass

    def file_transfer(method='tftp'):
        pass

    def read_output(self):
        pass

    def print_output(self):
        pass

        ##something

    def log_file(self):
        pass

    def login(self):
        pass

    def read_file(self, filename):
        pass

    def write_file(self, filename):
        pass