Exemplo n.º 1
0
    def __init__(self, host, username=None, password=None, port=22,
                 private_key_file=None, private_key_password=None,
                 compression=True):

        self._con = None

        # connection options
        cnopts = CnOpts()
        cnopts.compression = compression
        # set hostkeys to None, if not provided
        if private_key_file is None:
            cnopts.hostkeys = None

        if password is None:
            logging.debug('No password provided, using key auth.')
            # NOTE:
            #   Ducking exceptions, so that they can be handled
            #   by the main module however it wants.
            self._con = Connection(host=host, username=username,
                                   port=port,
                                   private_key_file=private_key_file,
                                   # Ignore LineLengthBear, PycodeStyleBear
                                   private_key_password=private_key_password,
                                   cnopts=cnopts)

        if self._con is None:
            self._con = Connection(host, username=username, port=port,
                                   password=password, cnopts=cnopts)
Exemplo n.º 2
0
 def connect(self):
     if not self.__connection:
         self.__connection = Connection(self.server,
                                        self.username,
                                        self.private_key,
                                        cnopts=self.connection_opts)
     return self
Exemplo n.º 3
0
 def __init__(self,
              hostname: str = "test.rebex.net",
              username: str = "demo",
              password: str = "password"):
     self.srv = Connection(host=hostname,
                           username=username,
                           password=password)
     self.state = self.srv.pwd
class SFTPConnector:
    def __init__(self, download_folder, **kwargs):
        cnopts = CnOpts()
        cnopts.hostkeys = None
        self._sftp = Connection(host=kwargs["host"],
                                username=kwargs["username"],
                                password=kwargs["password"],
                                cnopts=cnopts)
        print("File server connection established...")
        self._folder = kwargs["folder"]
        self._download_folder = download_folder

    def get_latest_file(self, folder=None):
        if folder is not None:
            self._sftp.chdir(folder)

        latest = 0
        latest_file = None

        is_dir = False

        for fileattr in self._sftp.listdir_attr():
            if fileattr.st_mtime > latest:
                is_dir = stat.S_ISDIR(fileattr.st_mode)
                latest = fileattr.st_mtime
                latest_file = fileattr.filename

        print(latest_file, "will be downloaded. It is the latest file")
        latest_downloaded = self.download_latest_file(folder, latest_file,
                                                      is_dir)

        return latest_downloaded

    def download_latest_file(self, root_dir, latest, is_dir):
        if latest is not None:
            if is_dir:
                print(latest +
                      " is the most recent folder. Changing directory...")
                return self.get_latest_file(latest)
            print(latest, "will be downloaded...")
            self._sftp.get(self._folder + latest,
                           self._download_folder + latest)

        return self._download_folder + latest

    def get_all_files(self, folder=None):
        if folder is not None:
            self._sftp.chdir(folder)

        all_files = list(
            map(lambda x: (x.filename, stat.S_ISDIR(x.st_mode)),
                self._sftp.listdir_attr()))
        for e in all_files:
            yield e

    def close(self):
        self._sftp.close()
 def __init__(self, download_folder, **kwargs):
     cnopts = CnOpts()
     cnopts.hostkeys = None
     self._sftp = Connection(host=kwargs["host"],
                             username=kwargs["username"],
                             password=kwargs["password"],
                             cnopts=cnopts)
     print("File server connection established...")
     self._folder = kwargs["folder"]
     self._download_folder = download_folder
Exemplo n.º 6
0
def upload_folder(sftp: pysftp.Connection,
                  from_computer: str,
                  to_server: str,
                  is_last: bool = False,
                  step: int = 1,
                  blanks: List[bool] = None) -> None:
    """
    Copy a file or a folder
    :param sftp: The sftp connection
    :param from_computer: The file or folder to copy
    :param to_server: The folder to copy the file/folder inside
    :param is_last: The boolean that say if it is the last file/folder
    :param step: The number of space to space printing
    :param blanks: List of booleans that say if the repository is empty
    """

    if blanks is None:
        blanks = [False]
    if len(blanks) <= step // 4:
        blanks.append(False)

    # Print space and tree before folder

    for i in range(step // 4):
        if blanks[i]:
            print(' ' * 4, end="")
        else:
            print('|   ', end="")
    if is_last:
        print('└───', end="")
        blanks[step // 4] = True
    else:
        print('├───', end="")

    # Change current name
    to_server += '/' + from_computer.split('/')[-1]

    # Check the type of the path
    if os.path.isfile(from_computer):
        # We are in case of a file, so we can upload it
        sftp.put(from_computer, to_server)
        print(f" File '{from_computer.split('/')[-1]}' uploaded")

    else:
        # We are in case of a folder
        sftp.execute(f"mkdir {to_server}")
        print(f" Folder '{from_computer.split('/')[-1]}' created")

        index = 1
        for item in os.listdir(from_computer):
            upload_folder(sftp, f"{from_computer}/{item}", to_server,
                          len(os.listdir(from_computer)) == index, step + 4,
                          blanks)
            index += 1
        blanks[step // 4] = False
def remove_dir_remote(sftp: pysftp.Connection, remotepath: str):
    try:
        sftp.rmdir(remotepath)
        return True

    except OSError as e:
        if e.strerror is None:
            print(ERROR_PREFACE, "could not remove directory " + remotepath)
        else:
            print(ERROR_PREFACE, e.strerror)
        return False
Exemplo n.º 8
0
def download_file(conn: pysftp.Connection, file) -> None:
    target = os.path.join(config.local_directory, file.group('title'))
    if not os.path.exists(target):
        os.mkdir(target)
    print(f'Downloading "{file.string}" into "{target}"')
    start = time.time()
    conn.get(file.string, os.path.join(target, file.string))
    filesize = os.path.getsize(os.path.join(target, file.string))
    time_elapsed = time.time() - start
    print(colored('[SUCCESS]', 'green'), end=' ')
    print(
        f'Finished downloading "{file.string}" after {round(time_elapsed)} seconds. ({round(filesize/(time_elapsed*10e5), 2)} MB/s)'
    )
Exemplo n.º 9
0
def uploadFilesToPath(fls: list, targetPath: str,
                      sftpConnection: sftp.Connection):
    print('Uploading files: ' + str(fls))
    for f in fls:
        if (f in ignore):
            print('File ignored: ' + f)
            continue
        destF = f.split('\\')[-1]
        try:
            sftpConnection.put(localpath=f,
                               remotepath=targetPath + destF,
                               confirm=True)
        except Exception as e:
            print('Unable to upload file: ' + str(f) + '\nError: ' + str(e))
Exemplo n.º 10
0
class ConnectionManager:
    state: str = ""
    svr: str = ""

    def __init__(self,
                 hostname: str = "test.rebex.net",
                 username: str = "demo",
                 password: str = "password"):
        self.srv = Connection(host=hostname,
                              username=username,
                              password=password)
        self.state = self.srv.pwd

    def __del__(self):
        self.srv.close()
Exemplo n.º 11
0
 def connect(self):
     try:
         if self.credentials.cred_type == SSHCredentials.CredentialsType.KEY:
             self.client = Connection(self.domain, username=self.credentials.login,
                                      private_key=self.credentials.key)
         else:
             self.client = Connection(self.domain, username=self.credentials.login,
                                      password=self.credentials.password)
     except:
         FailingConnection("SFTP connection failing")
     else:
         try:
             self.client.chdir(self.remote_location)
         except FileNotFoundError:
             raise UnknownRemoteLocation("Remote folder can't be found")
         self.status = self.Status.CONNECTED
Exemplo n.º 12
0
def download_new_files(last_date: typing.Optional[datetime.date]):
    new_dates = []
    new_filenames = []
    opts = CnOpts()
    opts.hostkeys = None
    with Connection(settings.SFTP_HOST, username=settings.SFTP_USER,
                    private_key=settings.SFTP_PRIVATE_KEY, cnopts=opts) as conn:
        with conn.cd(settings.SFTP_DIR):
            dir_listing = conn.listdir()
            for filename in dir_listing:
                date = parse_filename(filename, settings.ACCOUNT_CODE)

                if date:
                    stat = conn.stat(filename)
                    if stat.st_size > SIZE_LIMIT_BYTES:
                        logger.error('%s is too large (%s), download skipped.'
                                     % (filename, stat.st_size))
                        continue

                    if last_date is None or date > last_date:
                        local_path = os.path.join(settings.DS_NEW_FILES_DIR,
                                                  filename)
                        new_filenames.append(local_path)
                        new_dates.append(date)
                        conn.get(filename, localpath=local_path)

    if new_dates and new_filenames:
        sorted_dates, sorted_files = zip(*sorted(zip(new_dates, new_filenames)))
        return NewFiles(list(sorted_dates), list(sorted_files))
    else:
        return NewFiles([], [])
Exemplo n.º 13
0
def put(sftp: pysftp.Connection, filename: str):
    try:
        sftp.put(filename, os.path.basename(filename), preserve_mtime=False)
        return True
    except FileNotFoundError:
        print("Error:", FILE_NOT_FOUND_ERROR_MESSAGE)
        return False
    except OSError:
        if sftp.isdir(filename):
            # The file  we want to upload is a directory on remote.
            # pysftp doesn't handle this case well, so we'll print a helpful
            # error message of our own.
            print("Error:", FOLDER_CONFLICT_ERROR_MESSAGE)
            return False
        else:
            # It's some other error; we'll let the main handler deal with it.
            raise
Exemplo n.º 14
0
def list_dir(sftp: pysftp.Connection):
    """sftp.listdir() returns a list containing the names of files in the current (".") directory.
    Note: It does not print the list, so we must return the list to main for that to be done.
    """
    for entry in sftp.listdir("."):
        if not entry.startswith('.'):
            print(entry, end='    ')
    print()
Exemplo n.º 15
0
def test_upload_raise(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip):
    host = '127.0.0.1'
    port = ftp_server3.port
    cnopts = CnOpts()
    cnopts.hostkeys = None
    ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
    with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
        with pytest.raises(loader.LoaderException):
            ft.upload(conn, skip_existing=skip)
Exemplo n.º 16
0
def download_data(url, user, passwd, file_trans, skip_existing, **kwargs):
    downloaded = []
    with Connection(url, user, password=passwd, **kwargs) as conn:
        for ft in file_trans:
            try:
                ft.download(conn, skip_existing)
                downloaded.append(ft)
            except loader.LoaderException as e:
                print(e.message)
    return downloaded
Exemplo n.º 17
0
def upload_data(url, user, passwd, file_trans, skip_existing):
    with Connection(url, user, password=passwd) as conn:
        count = 0
        for ft in file_trans:
            try:
                ft.upload(conn, skip_existing)
                count += 1
            except loader.LoaderException as e:
                print(e.message)
    return count
Exemplo n.º 18
0
def updateFiles():
    global connection, needed_files
    try:
        os.mkdir(localLogsDir)
    except FileExistsError:
        pass
    try:
        connection = Connection(host, user, password=paswd)
    except ConnectionException:
        print('Connection failed')
        return False

    else:
        connection.chdir(remoteLogsDir)
        needed_files = compareFileList(remoteLogsDir, localLogsDir)
        downloadFiles(needed_files, localLogsDir)
    logging.info('Update compited')
    print('Update compited')
    return True
Exemplo n.º 19
0
def rename_remote_file(sftp: pysftp.Connection, before: str, after: str):
    '''
	Method to rename a file on remote server.
	'''

    #Path is set to current working directory
    path = '.'

    try:
        # Rename file name from rename.txt to renamed.txt on remote server
        # If rename is unsuccessful, then None is returned
        sftp.rename(before, after)
        return True
    except OSError as e:
        if e.strerror is None:
            print(ERROR_PREFACE, "could not rename file or directory.")
        else:
            print(ERROR_PREFACE, e.strerror)
        return False
Exemplo n.º 20
0
 def get_client(self):
     cnopts = CnOpts()
     cnopts.hostkeys = None
     sftp = Connection(self.conf['domain'],
                       username=self.conf['user'],
                       password=self.conf['password'],
                       cnopts=cnopts,
                       default_path=self.conf['remote_path'])
     LOGGER.info("Successfully connected to '%s' SFTP server",
                 self.conf['domain'])
     return sftp
Exemplo n.º 21
0
class SFTPService:  # pylint: disable=too-few-public-methods
    """SFTP  Service class."""

    DEFAUILT_CONNECT_SERVER = 'CAS'

    @staticmethod
    def get_connection(
            server_name: str = DEFAUILT_CONNECT_SERVER) -> Connection:
        """Return a SFTP connection."""
        # pylint: disable=protected-access
        return SFTPService._connect(server_name)

    @staticmethod
    def _connect(server_name: str) -> Connection:

        sftp_configs = current_app.config.get('SFTP_CONFIGS')
        # if not passed , connect to CAS server always. to make the existing code work
        if not server_name or server_name not in sftp_configs.keys():
            server_name = SFTPService.DEFAUILT_CONNECT_SERVER

        connect_configs = sftp_configs.get(server_name)

        sftp_host: str = connect_configs.get('SFTP_HOST')
        cnopts = CnOpts()
        # only for local development set this to false .
        if connect_configs.get('SFTP_VERIFY_HOST').lower() == 'false':
            cnopts.hostkeys = None
        else:
            ftp_host_key_data = connect_configs.get('SFTP_HOST_KEY').encode()
            key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data))
            cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key)

        sftp_port: int = connect_configs.get('SFTP_PORT')
        sft_credentials = {
            'username':
            connect_configs.get('SFTP_USERNAME'),
            # private_key should be the absolute path to where private key file lies since sftp
            'private_key':
            connect_configs.get('FTP_PRIVATE_KEY_LOCATION'),
            'private_key_pass':
            connect_configs.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE')
        }

        # to support local testing. SFTP CAS server should run in private key mode
        if password := connect_configs.get('SFTP_PASSWORD'):
            sft_credentials['password'] = password

        sftp_connection = Connection(host=sftp_host,
                                     **sft_credentials,
                                     cnopts=cnopts,
                                     port=sftp_port)
        current_app.logger.debug('sftp_connection successful')
        return sftp_connection
Exemplo n.º 22
0
def check_if_sftp_remotepath_exists(sftp: Connection, remotepath: str):

    """
    This error checking function requires a running sftp connection.
    """

    if sftp.exists(remotepath=remotepath):
        return True
    else:
        raise IOError(
            f"{remotepath} seems not to exist. Did you check spelling errors?"
        )
    return
Exemplo n.º 23
0
def test_download(ftp_server1, file_tree1, tmp_path, local, remote, name, arch, skip):
    host = '127.0.0.1'
    port = ftp_server1.port
    cnopts = CnOpts()
    cnopts.hostkeys = None
    ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
    with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
        ft.download(conn, skip_existing=skip)
        filename = ft._arch_name
        mode = 'rb' if arch else 'r'
        with open(tmp_path / local / filename, mode) as f:
            data = f.read()
            assert data == ftp_server1.content_provider.get(remote + '/' + filename)
Exemplo n.º 24
0
def test_upload2(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip, content):
    host = '127.0.0.1'
    port = ftp_server3.port
    cnopts = CnOpts()
    cnopts.hostkeys = None
    ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
    with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
        ft.upload(conn, skip_existing=skip)
        filename = ft._arch_name
        data = ftp_server3.content_provider.get(remote + '/' + filename)
        if arch:
            data = loader.get_archivator(arch).decompress(data)
        assert data.decode() == content
Exemplo n.º 25
0
 def __init__(self, serverDNS):
     """
     The init script for the class
     """
     self.serverDNS = str(serverDNS)
     BASE_DIR = dirname(dirname(__file__))
     config = ConfigReader()
     self.userName = config.get('saturnnode', 'user')
     self.keyFile = join(BASE_DIR, config.get('saturnring',
                                              'privatekeyfile'))
     self.rembashpath = config.get('saturnnode', 'bashpath')
     self.rempypath = config.get('saturnnode', 'pythonpath')
     self.iscsiconfdir = join(BASE_DIR,
                              config.get('saturnring', 'iscsiconfigdir'))
     self.remoteinstallLoc = config.get('saturnnode', 'install_location')
     self.localbashscripts = join(BASE_DIR,
                                  config.get('saturnring', 'bashscripts'))
     try:
         self.srv = Connection(self.serverDNS, self.userName, self.keyFile)
     except:
         logger.critical(
             "Failed SSH-exec connection on Saturn server %s; possible cause: %s"
             % (self.serverDNS, format_exc()))
Exemplo n.º 26
0
class Storage(BaseStorage):
    """ SFTP Storage """
    name = 'SFTP'
    SFTP_HOST = getattr(settings, 'DBBACKUP_SFTP_HOST', None)
    SFTP_USER = getattr(settings, 'DBBACKUP_SFTP_USER', None)
    SFTP_PASSWORD = getattr(settings, 'DBBACKUP_SFTP_PASSWORD', None)
    SFTP_PATH = getattr(settings, 'DBBACKUP_SFTP_PATH', ".")
    SFTP_PATH = '/%s/' % SFTP_PATH.strip('/')
    SFTP_PASSIVE_MODE = getattr(settings, 'DBBACKUP_SFTP_PASSIVE_MODE', False)

    def __init__(self, server_name=None):
        self._check_settings()
        self.sftp = Connection(
            host = self.SFTP_HOST,
            username = self.SFTP_USER,
            password = self.SFTP_PASSWORD)

    def _check_settings(self):
        """ Check we have all the required settings defined. """
        if not self.SFTP_HOST:
            raise StorageError('%s storage requires DBBACKUP_SFTP_HOST to be defined in settings.' % self.name)

    ###################################
    #  DBBackup Storage Methods
    ###################################

    @property
    def backup_dir(self):
        return self.SFTP_PATH

    def delete_file(self, filepath):
        """ Delete the specified filepath. """
        self.sftp.remove(filepath)

    def list_directory(self, raw=False):
        """ List all stored backups for the specified. """
        return sorted(self.sftp.listdir(self.SFTP_PATH))

    def write_file(self, filehandle, filename):
        """ Write the specified file. """
        filehandle.seek(0)
        backuppath = os.path.join(self.SFTP_PATH, filename)
        self.sftp.putfo(filehandle, backuppath)

    def read_file(self, filepath):
        """ Read the specified file and return it's handle. """
        outputfile = tempfile.SpooledTemporaryFile(
            max_size=10 * 1024 * 1024,
            dir=dbbackup_settings.TMP_DIR)
        self.sftp.getfo(filepath, outputfile)
        return outputfile
Exemplo n.º 27
0
class Storage(BaseStorage):
    """ SFTP Storage """
    name = 'SFTP'
    SFTP_HOST = getattr(settings, 'DBBACKUP_SFTP_HOST', None)
    SFTP_USER = getattr(settings, 'DBBACKUP_SFTP_USER', None)
    SFTP_PASSWORD = getattr(settings, 'DBBACKUP_SFTP_PASSWORD', None)
    SFTP_PATH = getattr(settings, 'DBBACKUP_SFTP_PATH', ".")
    SFTP_PATH = '/%s/' % SFTP_PATH.strip('/')
    SFTP_PASSIVE_MODE = getattr(settings, 'DBBACKUP_SFTP_PASSIVE_MODE', False)

    def __init__(self, server_name=None):
        self._check_settings()
        self.sftp = Connection(host=self.SFTP_HOST,
                               username=self.SFTP_USER,
                               password=self.SFTP_PASSWORD)

    def _check_settings(self):
        """ Check we have all the required settings defined. """
        if not self.SFTP_HOST:
            raise StorageError(
                '%s storage requires DBBACKUP_SFTP_HOST to be defined in settings.'
                % self.name)

    ###################################
    #  DBBackup Storage Methods
    ###################################

    @property
    def backup_dir(self):
        return self.SFTP_PATH

    def delete_file(self, filepath):
        """ Delete the specified filepath. """
        self.sftp.remove(filepath)

    def list_directory(self, raw=False):
        """ List all stored backups for the specified. """
        return sorted(self.sftp.listdir(self.SFTP_PATH))

    def write_file(self, filehandle, filename):
        """ Write the specified file. """
        filehandle.seek(0)
        backuppath = os.path.join(self.SFTP_PATH, filename)
        self.sftp.putfo(filehandle, backuppath)

    def read_file(self, filepath):
        """ Read the specified file and return it's handle. """
        outputfile = tempfile.SpooledTemporaryFile(
            max_size=10 * 1024 * 1024, dir=dbbackup_settings.TMP_DIR)
        self.sftp.getfo(filepath, outputfile)
        return outputfile
Exemplo n.º 28
0
    def _download_them(self, _sftp: pysftp.Connection):
        """
        This method does the hard work downloading files. It also preserves the modified time
        after downloaded from the SFTP host.
        The call back lambda function prints the % byes downloaded,
        which is based on the size of the file.

        Parameters
        ----------
        _sftp : pysftp.Connection
                a sftp connection object

        """
        entries = _sftp.listdir(self._srcDir)

        for entry in entries:
            if _sftp.isfile(entry) and self._is_file_asked(entry):

                _sftp.get(os.path.join(self._srcDir, entry),
                          os.path.join(self._destDir, entry),
                          callback=lambda transfered, size: audit_params(operation=Sc.OPERATION_DOWNLOAD,
                                                                         status=Sc.STATUS_COMPLETE,
                                                                         comments="{} {} ({})%".format('>>-->', entry, str("%.2f" % (100*(int(transfered)/int(size)))))),
                          preserve_mtime=True)
Exemplo n.º 29
0
 def __init__(self,serverDNS):
     """
     The init script for the class
     """
     self.serverDNS = str(serverDNS)
     BASE_DIR = dirname(dirname(__file__))
     config = ConfigReader()
     self.userName = config.get('saturnnode','user')
     self.keyFile = join(BASE_DIR,config.get('saturnring','privatekeyfile'))
     self.rembashpath = config.get('saturnnode','bashpath')
     self.rempypath = config.get('saturnnode','pythonpath')
     self.iscsiconfdir = join(BASE_DIR,config.get('saturnring','iscsiconfigdir'))
     self.remoteinstallLoc = config.get('saturnnode','install_location')
     self.localbashscripts = join(BASE_DIR,config.get('saturnring','bashscripts'))
     try:
         self.srv = Connection(self.serverDNS,self.userName,self.keyFile)
     except:
         logger.critical("Failed SSH-exec connection on Saturn server %s; possible cause: %s" % (self.serverDNS,format_exc()) )
Exemplo n.º 30
0
 def sendPhotos(self, host, name, pskw, floor=0):
     try:
         with Connection(host, username=name, password=pskw) as sftp:
             if (sftp.isdir('/home/pi/Documents/Master/data')):
                 sftp.chdir('/home/pi/Documents/Master/data')
                 if not sftp.isdir("Grower{}".format(floor)):
                     sftp.makedirs("Grower{}".format(floor))
                 sftp.chdir("Grower{}".format(floor))
                 sftp.makedirs(self.photoPath(False))
                 sftp.put_r(
                     self.photoPath(True),
                     '/home/pi/Documents/Master/data/Grower{}/{}'.format(
                         floor, self.photoPath(False)),
                     preserve_mtime=False)
                 return True
             else:
                 return False
     except:
         return False
def ftp_upload(dir_filename_list):
    "Uploads a list of (dir, filename)"
    # Required to fix pysftp bug
    cnopts = CnOpts()
    cnopts.hostkeys = None

    with Connection(
            host=b64decode(config["ftp"]["host"]).decode("utf-8"),
            username=b64decode(config["ftp"]["user"]).decode("utf-8"),
            password=b64decode(config["ftp"]["password"]).decode("utf-8"),
            port=int(b64decode(config["ftp"]["port"]).decode("utf-8")),
            cnopts=cnopts,
    ) as sftp:
        logger.info("SFTP connection OK")
        for dir_filename in dir_filename_list:
            file_path = join(dir_filename[0], dir_filename[1])
            logger.info("Uploading %s..." % (file_path))
            sftp.put(file_path, f"{config['dir']['remote']}/{dir_filename[1]}")
            logger.info("FTP uploaded successfully: %s" % (file_path))
Exemplo n.º 32
0
class SFTPService:  # pylint: disable=too-few-public-methods
    """SFTP  Service class."""

    __instance: Connection = None

    @staticmethod
    def get_connection() -> Connection:
        """Return a SFTP connection."""
        # pylint: disable=protected-access
        if not SFTPService.__instance or not SFTPService.__instance._sftp_live:
            SFTPService.__instance = SFTPService._connect()
        return SFTPService.__instance

    @staticmethod
    def _connect() -> Connection:

        sftp_host: str = current_app.config.get('CAS_SFTP_HOST')
        cnopts = CnOpts()
        # only for local development set this to false .
        if current_app.config.get('SFTP_VERIFY_HOST').lower() == 'false':
            cnopts.hostkeys = None
        else:
            host_key = current_app.config.get('CAS_SFTP_HOST_KEY')
            ftp_host_key_data = current_app.config.get('CAS_SFTP_HOST_KEY').encode()
            key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data))
            cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key)

        sftp_port: int = current_app.config.get('CAS_SFTP_PORT')
        sft_credentials = {
            'username': current_app.config.get('CAS_SFTP_USER_NAME'),
            # private_key should be the absolute path to where private key file lies since sftp
            'private_key': current_app.config.get('BCREG_FTP_PRIVATE_KEY_LOCATION'),
            'private_key_pass': current_app.config.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE')
        }

        # to support local testing. SFTP CAS server should run in private key mode
        if password := current_app.config.get('CAS_SFTP_PASSWORD'):
            sft_credentials['password'] = password

        sftp_connection = Connection(host=sftp_host, **sft_credentials, cnopts=cnopts, port=sftp_port)
        current_app.logger.debug('sftp_connection successful')
        return sftp_connection
Exemplo n.º 33
0
class SFTP:
    def __init__(self, sftp_server_config):
        self.server = sftp_server_config['host_address']
        self.username = sftp_server_config['user_name']
        self.private_key = sftp_server_config['key_path']
        self.sftp_folder = sftp_server_config['sftp_folder']
        self.connection_opts = CnOpts()
        self.connection_opts.hostkeys = None
        self.__connection: Connection = None

    def connect(self):
        if not self.__connection:
            self.__connection = Connection(self.server,
                                           self.username,
                                           self.private_key,
                                           cnopts=self.connection_opts)
        return self

    def put(self, file, remote_path):
        return self.__connection.put(file, remote_path)

    def remove(self, path):
        return self.__connection.remove(path)

    def rmdir(self, path):
        return self.__connection.rmdir(path)

    def listdir(self, path):
        return self.__connection.listdir(path)

    def is_dir(self, path):
        return self.__connection.isdir(path)

    def is_file(self, path):
        return self.__connection.isfile(path)

    def close(self):
        self.__connection.close()
        self.__connection = None
Exemplo n.º 34
0
def ftp_upload(sourcefile: str) -> str:


    def prepare_remote_folder(conn) -> None:
        "Create the necessary folder(s) on the remote server and change the directory accordingly"
        if config.preserve_folders_on_remote:
            full_remote_dir = os.path.join(config.remote_directory, get_date_folder())
        else:
            full_remote_dir = config.remote_directory
        if not conn.exists(full_remote_dir):
            conn.makedirs(full_remote_dir)
        conn.chdir(full_remote_dir)


    extension = get_extension(sourcefile)
    with Connection(config.sftp_address, username=config.username, password=config.password, port=config.sftp_port,
                    private_key=config.private_key, private_key_pass=config.private_key_pass) as conn:
        prepare_remote_folder(conn)
        extension = get_extension(sourcefile)
        dest_name = generate_filename(config.length, extension)
        while conn.exists(dest_name):
            dest_name = generate_filename(config.length, extension)
        conn.put(sourcefile, dest_name)
        return dest_name
Exemplo n.º 35
0
class PollServer():
    """
    This is the controller that calls/runs scripts on a Saturn server
    as required by saturnring

    """
    def __init__(self,serverDNS):
        """
        The init script for the class
        """
        self.serverDNS = str(serverDNS)
        BASE_DIR = dirname(dirname(__file__))
        config = ConfigReader()
        self.userName = config.get('saturnnode','user')
        self.keyFile = join(BASE_DIR,config.get('saturnring','privatekeyfile'))
        self.rembashpath = config.get('saturnnode','bashpath')
        self.rempypath = config.get('saturnnode','pythonpath')
        self.iscsiconfdir = join(BASE_DIR,config.get('saturnring','iscsiconfigdir'))
        self.remoteinstallLoc = config.get('saturnnode','install_location')
        self.localbashscripts = join(BASE_DIR,config.get('saturnring','bashscripts'))
        try:
            self.srv = Connection(self.serverDNS,self.userName,self.keyFile)
        except:
            logger.critical("Failed SSH-exec connection on Saturn server %s; possible cause: %s" % (self.serverDNS,format_exc()) )

    def InstallScripts(self):
        """
        Copy bash scripts from the saturnringserver into the saturn server via sftp
        """
        #srv = Connection(self.serverDNS,self.userName,self.keyFile)
        self.srv.execute ('mkdir -p '+self.remoteinstallLoc+'saturn-bashscripts/')
        self.srv.chdir(self.remoteinstallLoc+'saturn-bashscripts/')
        locallist=listdir(self.localbashscripts)
        for localfile in locallist:
            self.srv.put(self.localbashscripts+localfile)
            self.srv.execute("chmod 777 "+self.remoteinstallLoc+'saturn-bashscripts/'+localfile)
        #srv.close()
        logger.info("Installed scripts")

    def Exec(self,command):
        """
        Helper function for executing a remote command over an SSH tunnel
        """
        rtncmd = -1
        try:
            #srv = Connection(self.serverDNS,self.userName,self.keyFile)
            rtncmd=self.srv.execute(command)
            #srv.close()
        except:
            logger.error("Failed SSH-exec command: %s on Saturn server %s" % (command, self.serverDNS))
            logger.error(format_exc())
        return rtncmd

    def ParseLVM(self,strList,delimitStr,paraList):
        """
        Parse lvdisplay and vgdisplay strings and populate 
        dictionaries with relevant information
        """
        rtnDict ={}
        valueDict={}
        for aLine in strList:
            if (delimitStr in aLine):
                if len(valueDict) == len(paraList):
                    rtnDict[valueDict[paraList[0]]]=valueDict
                    valueDict = {}
                continue
            else:
                for anItem in paraList:
                    if anItem in aLine:
                        valueDict[anItem] = aLine.split(anItem)[1].strip()
                        if '%' in valueDict[anItem]:
                            valueDict[anItem] = float(valueDict[anItem][:-2])
                            continue
                        if '/' in valueDict[anItem]:
                            valueDict[anItem] = valueDict[anItem].split('/')[0]
                        if 'GiB' in valueDict[anItem]:
                            valueDict[anItem] = float(valueDict[anItem].split('GiB')[0])*1
                            continue
                        if 'MiB' in valueDict[anItem]:
                            valueDict[anItem] = float(valueDict[anItem].split('MiB')[0])*0.001
                            continue
                        continue
        if len(valueDict) == len(paraList):
             rtnDict[valueDict[paraList[0]]] = valueDict

        logger.info(rtnDict)
        return rtnDict


    def UpdateLVs(self,vgObject):
        """
        Update LV information, called to monitor and update capacity.
        """
        lvdict = self.GetLVs(vgObject.vguuid)
        if "No LVs " in lvdict:
            logger.info("There are no LVs in %s to run UpdateLVs on in Saturn host %s" %(vgObject.vguuid, self.serverDNS))
            return 0
        if lvdict == -1:
            logger.error ("Could not run GetLVs (perhaps there are no LVs in this VG yet?)")
            return -1
        lvs = LV.objects.filter(vg=vgObject)
        for lvName,lvinfo in lvdict.iteritems():
            if len(lvs.filter(lvname=lvName)):
                preexistLV=lvs.filter(lvname=lvName)[0]
            	preexistLV.lvsize=lvinfo['LV Size']
            	preexistLV.save(update_fields=['lvsize'])
            else:
                logger.warn("Found orphan LV %s in VG %s on host %s" %(lvName,vgObject.vguuid,self.serverDNS))


    def GetLVs(self,vguuid):
        """
        Wrapper for parselvm (for LVs), actually populating the DB is done by the UpdateLV function  
        """
        execCmd = " ".join(['sudo','vgdisplay', '-c','|','grep',vguuid,'|','cut -d: -f1'])
        vgname = self.Exec(execCmd)[0].strip()
        if vgname == -1:
            logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS))
            return -1
        execCmd=" ".join(['sudo','lvdisplay','--units g',vgname])
        lvStrList = self.Exec(execCmd)
        if lvStrList ==[""]:
            return "No LVs in %s" %(vguuid,)

        if lvStrList == -1:
            logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS))
            return -1
        delimitStr = '--- Logical volume ---'
        paraList=['LV Name','LV UUID','LV Size']
        lvs = self.ParseLVM(lvStrList,delimitStr,paraList)
        return lvs

    def GetVG(self): #Unit test this again
        """
        Wrapper for parseLVM (for VGs)+populating the DB
        """
        delimitStr = '--- Volume group ---'
        paraList = ['VG Name','VG Size','PE Size','Total PE', 'Free  PE / Size', 'VG UUID']
        execCmd = " ".join(['sudo','vgdisplay','--units g'])
        vgStrList = self.Exec(execCmd)
        if vgStrList == -1:
            return -1
        vgs = self.ParseLVM(vgStrList,delimitStr,paraList)
        #logger.info("VGStating on %s returns %s " % (self.serverDNS, str(vgs)) )
        rtnvguuidList = ""
        for vgname in vgs:
            try:
                execCmd = " ".join(['sudo',self.remoteinstallLoc+'saturn-bashscripts/vgstats.sh',vgname])
                cmdStr = self.Exec(execCmd)
                logger.info(self.serverDNS+": "+" ".join(['sudo',self.rembashpath,self.remoteinstallLoc+'saturn-bashscripts/vgstats.sh',vgname])+': returned '+str(cmdStr))
                maxavl = float(cmdStr[0].rstrip())
                totalGB = float(cmdStr[1].rstrip())
                isThin = bool(int(cmdStr[2].rstrip()))
            except:
                logger.warn("Unable to run VGscan, disabling VG on "+self.serverDNS)
                logger.warn(format_exc())
                try:
                    vg = VG.objects.get(vguuid=vgs[vgname]['VG UUID'])
                    vg.in_error = True
                    vg.save(update_fields=['in_error'])
                except:
                    logger.error("VG not found in DB: %s" % ( vgs[vgname]['VG UUID'],))
                return 3

            existingvgs = VG.objects.filter(vguuid=vgs[vgname]['VG UUID'])
            if len(existingvgs)==1:
                existingvg = existingvgs[0]
                existingvg.in_error=False
                existingvg.CurrentAllocGB = totalGB-maxavl#Target.objects.filter(targethost=existingvg.vghost).aggregate(Sum('sizeinGB'))['sizeinGB__sum']
                existingvg.totalGB=totalGB
                existingvg.maxavlGB=maxavl
                existingvg.is_thin=isThin
                existingvg.vgsize = vgs[vgname]['VG Size']
                existingvg.save(update_fields=['totalGB','maxavlGB','vgsize','CurrentAllocGB','in_error','is_thin'])
                logger.info( "Ran in existingVG loop")
            else:
                logger.info("Found new VG, adding\n" + str(vgs[vgname]))
                myvg = VG(vghost=StorageHost.objects.get(dnsname=self.serverDNS),vgsize=vgs[vgname]['VG Size'],
                        vguuid=vgs[vgname]['VG UUID'],vgpesize=vgs[vgname]['PE Size'],
                        vgtotalpe=vgs[vgname]['Total PE'],
                        vgfreepe=vgs[vgname]['Free  PE / Size'],
                        totalGB=totalGB,maxavlGB=maxavl, is_thin=isThin)
                myvg.save()#force_update=True)
            rtnvguuidList = rtnvguuidList+ ','+ vgs[vgname]['VG UUID']
        return rtnvguuidList[1:]

    
    def GitSave(self,vguuid,commentStr):
        """
        Check in changes to config files into git repository
        """
        try:
            #srv = Connection(self.serverDNS,self.userName,self.keyFile)
            self.srv.get('/temp/scst.conf',self.iscsiconfdir+self.serverDNS+'.scst.conf')
            self.srv.get('/temp/'+vguuid,self.iscsiconfdir+self.serverDNS+'.'+vguuid+'.lvm')
            try:
                repo = Repo(self.iscsiconfdir)
                filelist = [ f for f in listdir(self.iscsiconfdir) if isfile(join(self.iscsiconfdir,f)) ]
                repo.stage(filelist)
                repo.do_commit(commentStr)
            except:
                var = format_exc()
                logger.error("During GitSave: %s: Git save error: %s" % (commentStr, var))
        except:
            var = format_exc()
            logger.error("During GitSave: %s: PYSFTP download error: %s" % (commentStr, var))

    
    def CreateTarget(self,iqnTarget,iqnInit,sizeinGB,storageip1,storageip2,vguuid):
        """
        Create iSCSI target by running the createtarget script; 
        and save latest scst.conf from the remote server (overwrite)
        """
        #self.srv = Connection(self.serverDNS,self.userName,self.keyFile)
        cmdStr = " ".join(['sudo',self.rembashpath,self.remoteinstallLoc+'saturn-bashscripts/createtarget.sh',str(sizeinGB),iqnTarget,storageip1,storageip2,iqnInit,vguuid])
        #srv.close()
        logger.info ("Launching createtarget with \n%s" %(cmdStr,))
        exStr=self.Exec(cmdStr)
        if exStr == -1:
            return -1

        commentStr = "Trying to create target %s " %( iqnTarget, )

        self.GitSave(vguuid,commentStr)
        logger.info("Execution report for %s:  %s" %(cmdStr,"\t".join(exStr)))
        if "SUCCESS" in str(exStr):
            logger.info("Returning successful createtarget run")
            return 1
        else:
            logger.error("Returning failed createtarget run")
            return 0

    def GetTargetsState(self):
        """
        Read targets to determine their latest state via the parsetarget script
        """
        cmdStr = " ".join(["sudo",self.rempypath,self.remoteinstallLoc+'saturn-bashscripts/parsetarget.py'])
        exStr = self.Exec(cmdStr)
        if exStr == -1:
            return -1
        for eachLine in exStr:
            iqntar = eachLine.split()[0]
            tar = Target.objects.filter(iqntar=iqntar)
            if len(tar)==1:
                tar = tar[0]
                if "no session" in eachLine:
                    tar.sessionup=False
                    tar.rkbpm = 0
                    tar.wkbpm = 0
                else:
                    tar.sessionup=True
                    rkb = long(eachLine.split()[1])
                    tar.rkbpm = long(rkb-tar.rkb)
                    tar.rkb=rkb
                    wkb = long(eachLine.split()[2])
                    wpm = long(wkb-tar.wkb)
                    tar.wkbpm = wpm
                    tar.wkb=wkb
                tar.save()
            else:
                logger.warn("Found target %s on %s that does not exist in the DB" % (iqntar,self.serverDNS) )

    def DeleteTarget(self,iqntar,vguuid):
        """
        Delete target
        """
        logger.info("Trying to delete target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS))
        if self.GetTargetsState() == -1:
            logger.error("Could not GetTargetsState while deleting %s" %(iqntar,))
            return -1
        try:
            tar = Target.objects.get(iqntar=iqntar)
        except:
            logger.warn("Could not find deletion target in DB, exiting. "+iqntar)
            return -1
        if not tar.sessionup:
            cmdStr = " ".join(["sudo",self.rembashpath,self.remoteinstallLoc+'saturn-bashscripts/removetarget.sh',iqntar,vguuid])
            exStr = self.Exec(cmdStr)
            if exStr == -1:
                return -1
            self.GitSave(vguuid,"Trying to delete  target %s " %( iqntar,))
            success1 = False
            success2 = False
            for eachLine in exStr:
                logger.info(eachLine)
                if "Removing virtual target '"+iqntar+"' from driver 'iscsi': done" in eachLine:
                    success1=True
                if "successfully removed" in eachLine:
                    success2=True
            if success1==True and success2==True:
                logger.info("Successful deletion of target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS))

                return 1
            else:
                logger.error("Error deleting target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS))
                return -1
        return -1

    def GetInterfaces(self):
        """
        Scan and get network interfaces into saturnring DB
        """
        #cmdStr = 'ifconfig | grep -oE "inet addr:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d: -f2'
        cmdStr = 'ip addr | grep -oE "inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d" "  -f2'
        ipadds=self.Exec(cmdStr)
        if ipadds == -1:
            return -1

        sh = StorageHost.objects.get(dnsname=self.serverDNS)
        superuser=User.objects.filter(is_superuser=True).order_by('username')[0]
        for addr in ipadds:
            try:
                addr = addr.rstrip()
                if "127.0.0.1" in addr: #Ignore loopback addresses
                    continue
                socket.inet_aton(addr)
                interfaces = Interface.objects.filter(ip=addr)
                if len(interfaces) != 1: #If 0, then new interface
                    Interface.objects.filter(ip=addr).delete()
                    logger.info("Adding newly discovered interface %s to storage host %s " % (addr, self.serverDNS))
                    try:
                        newInterface = Interface(storagehost=sh,ip=addr,owner=superuser)
                        newInterface.save()
                        for eachIPRange in IPRange.objects.all():
                            if ipaddress.ip_address(unicode(addr)) in ipaddress.ip_network(unicode(eachIPRange.iprange)):
                                eachIPRange.hosts.add(sh)
                                eachIPRange.save()
                                newInterface.iprange.add(eachIPRange)
                                newInterface.owner=eachIPRange.owner
                                newInterface.save()
                    except:
                        logger.warn("Error saving newly discovered Interface %s  of host %s" % (addr, self.serverDNS))
                        var = format_exc()
                        logger.warn(var)
                else:
                    if interfaces[0].storagehost.dnsname != self.serverDNS:
                        Interface.objects.filter(ip=addr).delete()
                        logger.warn("IP address %s was reassigned to another host" % (addr,))
            except socket.error:
                logger.warn("Invalid IP address %s retuned in GetInterfaces call on Saturn server %s " % (addr, self.serverDNS ))
                var = format_exc()
                logger.warn(var)
Exemplo n.º 36
0
 def __init__(self, server_name=None):
     self._check_settings()
     self.sftp = Connection(
         host = self.SFTP_HOST,
         username = self.SFTP_USER,
         password = self.SFTP_PASSWORD)
Exemplo n.º 37
0
class PollServer():
    """
    This is the controller that calls/runs scripts on a Saturn server
    as required by saturnring

    """
    def __init__(self,serverDNS):
        """
        The init script for the class
        """
        try:
            self.serverDNS = str(serverDNS)
            self.hostobject = StorageHost.objects.get(dnsname=self.serverDNS)
            BASE_DIR = dirname(dirname(__file__))
            config = ConfigReader()
            self.userName = config.get('saturnnode','user')
            self.keyFile = join(BASE_DIR,config.get('saturnring','privatekeyfile'))
            self.rembashpath = config.get('saturnnode','bashpath')
            self.rempypath = config.get('saturnnode','pythonpath')
            self.iscsiconfdir = join(BASE_DIR,config.get('saturnring','iscsiconfigdir'))
            self.remoteinstallLoc = config.get('saturnnode','install_location')
            self.localbashscripts = join(BASE_DIR,config.get('saturnring','bashscripts'))
        except:
            logger.critical("Error setting up configuration for server "+self.serverDNS)
            logger.critical(format_exc())
        try:
            self.srv = Connection(self.serverDNS,self.userName,self.keyFile)
        except:
            logger.critical("Failed SSH-exec connection on Saturn server %s; possible cause: %s" % (self.serverDNS,format_exc()) )
            self.srv="inError"
    
    def CheckServer(self):
        if self.srv == 'inError':
            return -1
        remotePath = join(self.remoteinstallLoc,'saturn-bashscripts')
        cmdStr = " ".join([join(remotePath,'checkserver.sh'), '2> checkservererror.log'])
        #logger.info("Executing %s on %s" %(cmdStr,self.serverDNS))
        rtnStrList = self.Exec(cmdStr)
        if (rtnStrList == -1):
            return -2
        else:
            for aLine in rtnStrList:
                if "FAILURE" in aLine:
                    logger.error(self.serverDNS + ": "+ str(rtnStrList))
                    return -3
        return 0

    def InstallScripts(self):
        """
        Copy bash scripts from the saturnringserver into the saturn server via sftp
        """
        rtnVal = -1
        try:
            if self.srv == "inError":
                raise Exception('Server SSH connection object inError')
            remotePath = join(self.remoteinstallLoc,'saturn-bashscripts')
            self.srv.execute (" ".join(['mkdir', '-p', remotePath]))
            self.srv.chdir(remotePath)
            locallist=listdir(self.localbashscripts)
            for localfile in locallist:
                self.srv.put(join(self.localbashscripts,localfile))
                self.srv.execute(" ".join(["chmod", "777",join(remotePath,localfile)]))
            #Update rc.local for luks reboot functionality
            luksopenscriptpath = join(remotePath,'luksonreboot.sh');
            self.srv.execute("sudo sed -i '/luksonreboot.sh/d' /etc/rc.local") #delete pre-existing line if any
            self.srv.execute("sudo sed -i '/^exit 0/i " + '/bin/bash ' + luksopenscriptpath +"' /etc/rc.local")
            logger.info("Installed scripts on "+ self.serverDNS)
            rtnVal = 1
        except:
            logger.error('Could not install scripts on '+self.serverDNS)
            logger.error(format_exc())
        finally:
            return rtnVal



    def Exec(self,command):
        """
        Helper function for executing a remote command over an SSH tunnel
        """
        rtncmd = -1
        if self.srv=="inError":
            logger.error("There is no ssh connection object for server: %s" %(self.serverDNS,))
            return -1
        try:
            #srv = Connection(self.serverDNS,self.userName,self.keyFile)
            rtncmd=self.srv.execute(command)
            #srv.close()
        except:
            logger.error("Failed SSH-exec command: %s on Saturn server %s" % (command, self.serverDNS))
            logger.error(format_exc())
        return rtncmd


    def GetFile(self,remotePath,localPath):
        """
        Get a file from the remote server.
        return 1 on success, -1 on error
        """
        try:
            self.srv.get(remotePath,localPath)
            #logger.info("Copying file %s from remote server %s to local path %s succeeded" %(remotePath,self.serverDNS,localPath))
            return 1
        except:
            logger.error("Error copying file %s from remote server %s to local path %s" %(remotePath,self.serverDNS,localPath))
            logger.error(format_exc())
            return -1

    def PutKeyFile(self,keyfileName):
        """
        Copy over the keyfile to be used for creating the LUKs encrypted DM volumes
        """
        remoteKeyfileDir = join(self.remoteinstallLoc,'keys')
        try:
            self.Exec (" ".join(['mkdir','-p',remoteKeyfileDir]))
            self.srv.chdir(remoteKeyfileDir)
            self.srv.put(join(self.iscsiconfdir,keyfileName))
            self.remoteKeyfilePath = join(remoteKeyfileDir,keyfileName)
            rtnString = self.Exec ('test -f ' + self.remoteKeyfilePath + '&&  echo "OK Putkeyfile" ')
            logger.info(rtnString)
            if "OK Putkeyfile" not in str(rtnString):
                raise ValueError("Putkey didnt install file")
        except ValueError:
            logger.error("Failed to put keyfile on Saturn server %s at location %s" %(self.serverDNS,join(remoteKeyfileDir,keyfileName)))
            logger.error(format_exc())
            return -1

        return self.remoteKeyfilePath

    def DelKeyFile(self,keyfileName):
        """
        Delete key file from saturn server
        """
        remoteKeyfileDir = join(self.remoteinstallLoc,'keys')
        self.srv.execute('rm '+ join(remoteKeyfileDir,keyfileName))
        rtnString = self.Exec ('test ! -f ' + join(self.iscsiconfdir,keyfileName)+ ' &&  echo "OK Deleted keyfile"')
        return rtnString


    def ParseLVM(self,strList,delimitStr,paraList):
        """
        Parse lvdisplay and vgdisplay strings and populate 
        dictionaries with relevant information
        """
        rtnDict ={}
        valueDict={}
        for aLine in strList:
            if (delimitStr in aLine):
                if len(valueDict) == len(paraList):
                    rtnDict[valueDict[paraList[0]]]=valueDict
                    valueDict = {}
                continue
            else:
                for anItem in paraList:
                    if anItem in aLine:
                        valueDict[anItem] = aLine.split(anItem)[1].strip()
                        if '%' in valueDict[anItem]:
                            valueDict[anItem] = float(valueDict[anItem][:-2])
                            continue
                        if '/' in valueDict[anItem]:
                            valueDict[anItem] = valueDict[anItem].split('/')[0]
                        if (('GiB' in valueDict[anItem]) and ('Size' in aLine)):
                            valueDict[anItem] = float(valueDict[anItem].split('GiB')[0])*1
                            continue
                        if (('MiB' in valueDict[anItem]) and ('Size' in aLine)):
                            valueDict[anItem] = float(valueDict[anItem].split('MiB')[0])*0.001
                            continue
                        continue
        if len(valueDict) == len(paraList):
             rtnDict[valueDict[paraList[0]]] = valueDict

        #logger.info(rtnDict)
        return rtnDict


    def UpdateLVs(self,vgObject):
        """
        Update LV information, called to monitor and update capacity.
        """
        lvdict = self.GetLVs(vgObject.vguuid)
        if "No LVs " in lvdict:
            logger.info("There are no LVs in %s to run UpdateLVs on in Saturn host %s" %(vgObject.vguuid, self.serverDNS))
            return 0
        if lvdict == -1:
            logger.error ("Could not run GetLVs (perhaps there are no LVs in this VG yet?)")
            return -1
        lvs = set(LV.objects.filter(vg=vgObject))
        lvDict = {}
        for eachlv in lvs:
            lvDict[eachlv.lvname] = eachlv

        for lvName,lvinfo in lvdict.iteritems():
            if lvName in lvDict:
                preexistLV=lvDict[lvName]
            	preexistLV.lvsize=lvinfo['LV Size']
            	preexistLV.save(update_fields=['lvsize'])
            else:
                logger.warn("Found orphan LV %s in VG %s on host %s" %(lvName,vgObject.vguuid,self.serverDNS))


    def GetLVs(self,vguuid):
        """
        Wrapper for parselvm (for LVs), actually populating the DB is done by the UpdateLV function  
        """
        execCmd = " ".join(['sudo','vgdisplay', '-c','|','grep',vguuid,'|','cut -d: -f1'])
        vgname = self.Exec(execCmd)[0].strip()
        if vgname == -1:
            logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS))
            return -1
        execCmd=" ".join(['sudo','lvdisplay','--units g',vgname])
        lvStrList = self.Exec(execCmd)
        if lvStrList ==[""]:
            return "No LVs in %s" %(vguuid,)

        if lvStrList == -1:
            logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS))
            return -1
        delimitStr = '--- Logical volume ---'
        paraList=['LV Name','LV UUID','LV Size']
        lvs = self.ParseLVM(lvStrList,delimitStr,paraList)
        return lvs

    def GetVG(self): #Unit test this again
        """
        Wrapper for parseLVM (for VGs)+populating the DB
        """
        delimitStr = '--- Volume group ---'
        paraList = ['VG Name','VG Size','PE Size','Total PE', 'Free  PE / Size', 'VG UUID']
        execCmd = " ".join(['sudo','vgdisplay','--units g'])
        vgStrList = self.Exec(execCmd)
        if vgStrList == -1:
            logger.error("Error in GetVG while executing %s on server %s " %(execCmd,self.serverDNS))
            return -1
        vgs = self.ParseLVM(vgStrList,delimitStr,paraList)
        #logger.info("VGStating on %s returns %s " % (self.serverDNS, str(vgs)) )
        rtnvguuidList = ""
        for vgname in vgs:
            try:
                execCmd = " ".join(['sudo',join(self.remoteinstallLoc,'saturn-bashscripts/vgstats.sh'),vgname,' 2> error.log'])
                cmdStr = self.Exec(execCmd)
                maxavl = float(cmdStr[0].rstrip())
                totalGB = float(cmdStr[1].rstrip())
                isThin = bool(int(cmdStr[2].rstrip()))
            except:
                logger.warn("Unable to run VGscan, disabling VG on "+self.serverDNS)
                logger.warn(format_exc())
                try:
                    vg = VG.objects.get(vguuid=vgs[vgname]['VG UUID'])
                    vg.in_error = True
                    vg.save(update_fields=['in_error'])
                except:
                    logger.error("VG not found in DB: %s" % ( vgs[vgname]['VG UUID'],))
                return 3

            existingvgs = VG.objects.filter(vguuid=vgs[vgname]['VG UUID'])
            if len(existingvgs)==1:
                existingvg = existingvgs[0]
                existingvg.in_error=False
                existingvg.CurrentAllocGB = totalGB-maxavl#Target.objects.filter(targethost=existingvg.vghost).aggregate(Sum('sizeinGB'))['sizeinGB__sum']
                existingvg.totalGB=totalGB
                existingvg.maxavlGB=maxavl
                existingvg.is_thin=isThin
                existingvg.vgsize = vgs[vgname]['VG Size']
                existingvg.save(update_fields=['totalGB','maxavlGB','vgsize','CurrentAllocGB','in_error','is_thin'])
                #logger.info( "Ran in existingVG loop")
            else:
                logger.info("Found new VG, adding\n" + str(vgs[vgname]))
                myvg = VG(vghost=StorageHost.objects.get(dnsname=self.serverDNS),vgsize=vgs[vgname]['VG Size'],
                        vguuid=vgs[vgname]['VG UUID'],vgpesize=vgs[vgname]['PE Size'],
                        vgtotalpe=vgs[vgname]['Total PE'],
                        vgfreepe=vgs[vgname]['Free  PE / Size'],
                        totalGB=totalGB,maxavlGB=maxavl, is_thin=isThin)
                myvg.save()#force_update=True)
            rtnvguuidList = rtnvguuidList+ ','+ vgs[vgname]['VG UUID']
        return rtnvguuidList[1:]

    def GitSave(self,commentStr):
        """
        Check in changes to config files into git repository
        """
        try:
            repo = Repo(self.iscsiconfdir)
            filelist = [ f for f in listdir(self.iscsiconfdir) if isfile(join(self.iscsiconfdir,f)) ]
            repo.stage(filelist)
            repo.do_commit(commentStr)
            return 1
        except:
            var = format_exc()
            logger.error("During GitSave %s: Git save error: %s" % (commentStr,var))
            return -1

    def CreateTarget(self,iqnTarget,iqnInit,sizeinGB,storageip1,storageip2,vguuid,isencrypted):
        """
        Create iSCSI target by running the createtarget script; 
        and save latest scst.conf from the remote server (overwrite)
        """
        #self.srv = Connection(self.serverDNS,self.userName,self.keyFile)
        if str(isencrypted) != '1':
            cmdStr = " ".join(['sudo',self.rembashpath,join(self.remoteinstallLoc,'saturn-bashscripts','createtarget.sh'),
            str(sizeinGB),iqnTarget,storageip1,storageip2,iqnInit,vguuid, '2> createtarget.sh-error.log'])
        else:
            try:
                self.remotekeyfilelocation = self.PutKeyFile("cryptokey")
                cmdStr = " ".join(['sudo',self.rembashpath,join(self.remoteinstallLoc,'saturn-bashscripts','createencryptedtarget.sh'),
                str(sizeinGB),iqnTarget,storageip1,storageip2,iqnInit,vguuid,self.remotekeyfilelocation,'2> createencryptedtarget.sh-error.log'])
                if self.remotekeyfilelocation == -1:
                    raise ValueError("Putkey failed")

            except:
                logger.error("Error setting up encrypted target: %s " %(iqnTarget,))
                logger.error(format_exc())
                return -1

        #srv.close()
        logger.info ("Launching createtarget with \n%s" %(cmdStr,))
        exStr=self.Exec(cmdStr)
        if exStr == -1:
            return -1

        commentStr = "Trying to create target %s " %( iqnTarget, )
        try:
            if self.GetFile('/temp/scst.conf',self.iscsiconfdir+self.serverDNS+'.scst.conf')==-1:
                raise Exception('Error getting scst.conf')
            if self.GetFile(join('/temp',vguuid),join(self.iscsiconfdir,self.serverDNS+'.'+vguuid+'.lvm'))==-1:
                raise Exception('Error getting LVM configuration file %s' %(vguuid+'.lvm',))
            if self.GitSave(commentStr) == -1:
                raise Exception('Error in GitSave')
        except:
            logger.warning('Unable to save updated config files on ring server')
            logger.warning(format_exc())

        logger.info("Execution report for %s:  %s" %(cmdStr,"\t".join(exStr)))
        if "SUCCESS" in str(exStr):
            logger.info("Returning successful createtarget run")
            return 1
        else:
            logger.error("Returning failed createtarget run:" + str(exStr))
            return 0

    def GetTargetsState(self):
        """
        Read targets to determine their latest state via the parsetarget script
        """
        cmdStr = " ".join(["sudo",self.rempypath, join(self.remoteinstallLoc,'saturn-bashscripts','parsetarget.py'), '2> parsetargeterror.txt'])
        exStr = self.Exec(cmdStr)
        #logger.info("Parse target returns " +str(exStr))
        if exStr == -1:
            return -1
        try:
            targetDic = {}
            hostTargets = set(Target.objects.filter(targethost=self.hostobject))
            for eachTarget in hostTargets:
                targetDic[eachTarget.iqntar] = eachTarget
            for eachLine in exStr:
                iqntar = eachLine.split()[0]
                if iqntar in targetDic:
                    tar = targetDic[iqntar]
                    if "no session" in eachLine:
                        tar.sessionup=False
                        tar.rkbpm = 0
                        tar.wkbpm = 0
                    else:
                        tar.sessionup=True
                        rkb = long(eachLine.split()[1])
                        tar.rkbpm = long(rkb-tar.rkb)
                        tar.rkb=rkb
                        wkb = long(eachLine.split()[2])
                        wpm = long(wkb-tar.wkb)
                        tar.wkbpm = wpm
                        tar.wkb=wkb
                    tar.save()
                else:
                    logger.warn("Found target %s on %s that does not exist in the DB" % (iqntar,self.serverDNS) )
        except:
            logger.error("Error reading iSCSI target state for %s on server %s" %(iqntar,self.serverDNS))
            logger.error(format_exc())
            return -1
        return 0


    def DeleteTarget(self,iqntar,vguuid,lvolname):
        """
        Delete target
        """
        logger.info("Trying to delete target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS))
        if self.GetTargetsState() == -1:
            logger.error("Could not GetTargetsState while deleting %s" %(iqntar,))
            return -1
        try:
            tar = Target.objects.get(iqntar=iqntar)
        except:
            logger.warn("Could not find deletion target in DB, exiting. "+iqntar)
            return -1
        if not tar.sessionup:
            cmdStr = " ".join(["sudo",self.rembashpath,join(self.remoteinstallLoc,'saturn-bashscripts','removetarget.sh'),iqntar,vguuid,lvolname])
            exStr = self.Exec(cmdStr)
            if exStr == -1:
                return -1
            try:
                if self.GetFile('/temp/scst.conf',self.iscsiconfdir+self.serverDNS+'.scst.conf') == -1:
                    raise Exception('Error getting scst configuration file to store locally')
                if self.GetFile('/temp/'+vguuid,self.iscsiconfdir+self.serverDNS+'.'+vguuid+'.lvm') == -1:
                    raise Exception('Error getting LVM configuration to store locally')
                if self.GitSave("Trying to delete target "+iqntar) == -1:
                    raise Exception('Error with gitsave in delete target')
            except:
                logger.error("Error getting configuration files after deletion of target")
                logger.error(format_exc())
            success1 = False
            success2 = False
            logger.info(exStr)
            for eachLine in exStr:
                if "Removing virtual target '"+iqntar+"' from driver 'iscsi': done" in eachLine:
                    success1=True
                if "successfully removed" in eachLine:
                    success2=True
            if success1==True and success2==True:
                logger.info("Successful deletion of target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS))
                return 1
            else:
                logger.error("Error deleting target %s from VG %s on host %s; command execution returned %s" %(iqntar,vguuid,self.serverDNS,str(exStr)))
                return -1
        else:
            logger.error("Target state of %s is set to session up, will not try to delete it." %(iqntar,))

        return -1

    def GetInterfaces(self):
        """
        Scan and get network interfaces into saturnring DB
        """
        #cmdStr = 'ifconfig | grep -oE "inet addr:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d: -f2'
        cmdStr = 'ip addr | grep -oE "inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d" "  -f2 2> ipaddrerror.txt'
        ipadds=self.Exec(cmdStr)
        if ipadds == -1:
            return -1
        rtnVal = 0
        sh = self.hostobject
        superuser=User.objects.filter(is_superuser=True).order_by('username')[0]
        for addr in ipadds:
            try:
                addr = addr.rstrip()
                if "127.0.0.1" in addr: #Ignore loopback addresses
                    continue
                socket.inet_aton(addr)
                interfaces = Interface.objects.filter(ip=addr)
                if len(interfaces) != 1: #If 0, then new interface
                    Interface.objects.filter(ip=addr).delete()
                    logger.info("Adding newly discovered interface %s to storage host %s " % (addr, self.serverDNS))
                    try:
                        newInterface = Interface(storagehost=sh,ip=addr,owner=superuser)
                        newInterface.save()
                        for eachIPRange in IPRange.objects.all():
                            if ipaddress.ip_address(unicode(addr)) in ipaddress.ip_network(unicode(eachIPRange.iprange)):
                                eachIPRange.hosts.add(sh)
                                eachIPRange.save()
                                newInterface.iprange.add(eachIPRange)
                                newInterface.owner=eachIPRange.owner
                                newInterface.save()
                    except:
                        logger.warn("Error saving newly discovered Interface %s  of host %s" % (addr, self.serverDNS))
                        var = format_exc()
                        logger.warn(var)
                        rtnVal = -1
                else:
                    if interfaces[0].storagehost.dnsname != self.serverDNS:
                        Interface.objects.filter(ip=addr).delete()
                        logger.warn("IP address %s was reassigned to another host" % (addr,))
            except socket.error:
                logger.warn("Invalid IP address %s retuned in GetInterfaces call on Saturn server %s " % (addr, self.serverDNS ))
                var = format_exc()
                logger.warn(var)
                rtnVal = -1
        return rtnVal

    def InsertCrypttab(self,base_LV,enc_LV,keyfilePath):
        """
        Insert entry into /etc/crypttab
        """
        #Needed for the crypttab 
        baselvpath = self.Exec(" ".join(["sudo sh -c 'lvs -o lv_path | grep ",base_LV," | tr -d \" \"'"]))[0].strip()
        if baselvpath == -1:
            return -1
        logger.info("BaseLVPATH = " + baselvpath)
        cmdStr = " ".join(["sudo sh -c 'echo \"" + enc_LV, baselvpath, keyfilePath, "luks\" >> /etc/crypttab; mkdir -p /temp; cp /etc/crypttab /temp/crypttab; chmod 666 /temp/crypttab'"])
        logger.info("InsertCrypttab: "+cmdStr)
        rtnVal = self.Exec(cmdStr)
        if rtnVal == -1:
           logger.error("Error in InsertCrypttab while executing %s" %(cmdStr,))
        try:
            if self.GetFile('/temp/crypttab',self.iscsiconfdir+self.serverDNS+'.crypttab') == -1:
                raise Exception('Could not get crypttab file')
            if self.GitSave("Trying the insert crypttab entry " + cmdStr) == -1:
                raise Exception('Could not get gitsave to work for crypttab')
        except:
            logger.error('Error with getfile/gitsave during crypttab insert operations on %s' %(self.serverDNS,))
            logger.error(format_exc())

    def DeleteCrypttab(self,lvStr):
        """
        Delete entry from /etc/crypttab
        LVStr can be either the encrypted LV name or the base LV name
        """
        cmdStr = " ".join(['sudo sed -i','/'+lvStr+'/d','/etc/crypttab']) + "; sudo mkdir -p /temp; sudo cp /etc/crypttab /temp/crypttab; sudo chmod 666 /temp/crypttab"
        logger.info("DeleteCrypttab: "+cmdStr)
        rtnVal = self.Exec(cmdStr)
        if rtnVal == -1:
            return rtnVal
            logger.error("Error in DeleteCrypttab while executing %s" %(cmdStr,))
        try:
            if self.GetFile('/temp/crypttab',self.iscsiconfdir+self.serverDNS+'.crypttab') == -1:
                raise Exception('Could not get crypttab file')
            if self.GitSave("Trying the insert crypttab entry " + cmdStr) == -1:
                raise Exception('Could not get gitsave to work for crypttab')
        except:
            logger.error('Error with getfile/gitsave during crypttab delete operations on %s' %(self.serverDNS,))
            logger.error(format_exc())