def __init__(self, host, username=None, password=None, port=22,
private_key_file=None, private_key_password=None,
compression=True):
self._con = None
# connection options
cnopts = CnOpts()
cnopts.compression = compression
# set hostkeys to None, if not provided
if private_key_file is None:
cnopts.hostkeys = None
if password is None:
logging.debug('No password provided, using key auth.')
# NOTE:
# Ducking exceptions, so that they can be handled
# by the main module however it wants.
self._con = Connection(host=host, username=username,
port=port,
private_key_file=private_key_file,
# Ignore LineLengthBear, PycodeStyleBear
private_key_password=private_key_password,
cnopts=cnopts)
if self._con is None:
self._con = Connection(host, username=username, port=port,
password=password, cnopts=cnopts)
def connect(self):
if not self.__connection:
self.__connection = Connection(self.server,
self.username,
self.private_key,
cnopts=self.connection_opts)
return self
def __init__(self,
hostname: str = "test.rebex.net",
username: str = "demo",
password: str = "password"):
self.srv = Connection(host=hostname,
username=username,
password=password)
self.state = self.srv.pwd
class SFTPConnector:
def __init__(self, download_folder, **kwargs):
cnopts = CnOpts()
cnopts.hostkeys = None
self._sftp = Connection(host=kwargs["host"],
username=kwargs["username"],
password=kwargs["password"],
cnopts=cnopts)
print("File server connection established...")
self._folder = kwargs["folder"]
self._download_folder = download_folder
def get_latest_file(self, folder=None):
if folder is not None:
self._sftp.chdir(folder)
latest = 0
latest_file = None
is_dir = False
for fileattr in self._sftp.listdir_attr():
if fileattr.st_mtime > latest:
is_dir = stat.S_ISDIR(fileattr.st_mode)
latest = fileattr.st_mtime
latest_file = fileattr.filename
print(latest_file, "will be downloaded. It is the latest file")
latest_downloaded = self.download_latest_file(folder, latest_file,
is_dir)
return latest_downloaded
def download_latest_file(self, root_dir, latest, is_dir):
if latest is not None:
if is_dir:
print(latest +
" is the most recent folder. Changing directory...")
return self.get_latest_file(latest)
print(latest, "will be downloaded...")
self._sftp.get(self._folder + latest,
self._download_folder + latest)
return self._download_folder + latest
def get_all_files(self, folder=None):
if folder is not None:
self._sftp.chdir(folder)
all_files = list(
map(lambda x: (x.filename, stat.S_ISDIR(x.st_mode)),
self._sftp.listdir_attr()))
for e in all_files:
yield e
def close(self):
self._sftp.close()
def __init__(self, download_folder, **kwargs):
cnopts = CnOpts()
cnopts.hostkeys = None
self._sftp = Connection(host=kwargs["host"],
username=kwargs["username"],
password=kwargs["password"],
cnopts=cnopts)
print("File server connection established...")
self._folder = kwargs["folder"]
self._download_folder = download_folder
def upload_folder(sftp: pysftp.Connection,
from_computer: str,
to_server: str,
is_last: bool = False,
step: int = 1,
blanks: List[bool] = None) -> None:
"""
Copy a file or a folder
:param sftp: The sftp connection
:param from_computer: The file or folder to copy
:param to_server: The folder to copy the file/folder inside
:param is_last: The boolean that say if it is the last file/folder
:param step: The number of space to space printing
:param blanks: List of booleans that say if the repository is empty
"""
if blanks is None:
blanks = [False]
if len(blanks) <= step // 4:
blanks.append(False)
# Print space and tree before folder
for i in range(step // 4):
if blanks[i]:
print(' ' * 4, end="")
else:
print('| ', end="")
if is_last:
print('└───', end="")
blanks[step // 4] = True
else:
print('├───', end="")
# Change current name
to_server += '/' + from_computer.split('/')[-1]
# Check the type of the path
if os.path.isfile(from_computer):
# We are in case of a file, so we can upload it
sftp.put(from_computer, to_server)
print(f" File '{from_computer.split('/')[-1]}' uploaded")
else:
# We are in case of a folder
sftp.execute(f"mkdir {to_server}")
print(f" Folder '{from_computer.split('/')[-1]}' created")
index = 1
for item in os.listdir(from_computer):
upload_folder(sftp, f"{from_computer}/{item}", to_server,
len(os.listdir(from_computer)) == index, step + 4,
blanks)
index += 1
blanks[step // 4] = False
def remove_dir_remote(sftp: pysftp.Connection, remotepath: str):
try:
sftp.rmdir(remotepath)
return True
except OSError as e:
if e.strerror is None:
print(ERROR_PREFACE, "could not remove directory " + remotepath)
else:
print(ERROR_PREFACE, e.strerror)
return False
def download_file(conn: pysftp.Connection, file) -> None:
target = os.path.join(config.local_directory, file.group('title'))
if not os.path.exists(target):
os.mkdir(target)
print(f'Downloading "{file.string}" into "{target}"')
start = time.time()
conn.get(file.string, os.path.join(target, file.string))
filesize = os.path.getsize(os.path.join(target, file.string))
time_elapsed = time.time() - start
print(colored('[SUCCESS]', 'green'), end=' ')
print(
f'Finished downloading "{file.string}" after {round(time_elapsed)} seconds. ({round(filesize/(time_elapsed*10e5), 2)} MB/s)'
)
def uploadFilesToPath(fls: list, targetPath: str,
sftpConnection: sftp.Connection):
print('Uploading files: ' + str(fls))
for f in fls:
if (f in ignore):
print('File ignored: ' + f)
continue
destF = f.split('\\')[-1]
try:
sftpConnection.put(localpath=f,
remotepath=targetPath + destF,
confirm=True)
except Exception as e:
print('Unable to upload file: ' + str(f) + '\nError: ' + str(e))
class ConnectionManager:
state: str = ""
svr: str = ""
def __init__(self,
hostname: str = "test.rebex.net",
username: str = "demo",
password: str = "password"):
self.srv = Connection(host=hostname,
username=username,
password=password)
self.state = self.srv.pwd
def __del__(self):
self.srv.close()
def connect(self):
try:
if self.credentials.cred_type == SSHCredentials.CredentialsType.KEY:
self.client = Connection(self.domain, username=self.credentials.login,
private_key=self.credentials.key)
else:
self.client = Connection(self.domain, username=self.credentials.login,
password=self.credentials.password)
except:
FailingConnection("SFTP connection failing")
else:
try:
self.client.chdir(self.remote_location)
except FileNotFoundError:
raise UnknownRemoteLocation("Remote folder can't be found")
self.status = self.Status.CONNECTED
def download_new_files(last_date: typing.Optional[datetime.date]):
new_dates = []
new_filenames = []
opts = CnOpts()
opts.hostkeys = None
with Connection(settings.SFTP_HOST, username=settings.SFTP_USER,
private_key=settings.SFTP_PRIVATE_KEY, cnopts=opts) as conn:
with conn.cd(settings.SFTP_DIR):
dir_listing = conn.listdir()
for filename in dir_listing:
date = parse_filename(filename, settings.ACCOUNT_CODE)
if date:
stat = conn.stat(filename)
if stat.st_size > SIZE_LIMIT_BYTES:
logger.error('%s is too large (%s), download skipped.'
% (filename, stat.st_size))
continue
if last_date is None or date > last_date:
local_path = os.path.join(settings.DS_NEW_FILES_DIR,
filename)
new_filenames.append(local_path)
new_dates.append(date)
conn.get(filename, localpath=local_path)
if new_dates and new_filenames:
sorted_dates, sorted_files = zip(*sorted(zip(new_dates, new_filenames)))
return NewFiles(list(sorted_dates), list(sorted_files))
else:
return NewFiles([], [])
def put(sftp: pysftp.Connection, filename: str):
try:
sftp.put(filename, os.path.basename(filename), preserve_mtime=False)
return True
except FileNotFoundError:
print("Error:", FILE_NOT_FOUND_ERROR_MESSAGE)
return False
except OSError:
if sftp.isdir(filename):
# The file we want to upload is a directory on remote.
# pysftp doesn't handle this case well, so we'll print a helpful
# error message of our own.
print("Error:", FOLDER_CONFLICT_ERROR_MESSAGE)
return False
else:
# It's some other error; we'll let the main handler deal with it.
raise
def list_dir(sftp: pysftp.Connection):
"""sftp.listdir() returns a list containing the names of files in the current (".") directory.
Note: It does not print the list, so we must return the list to main for that to be done.
"""
for entry in sftp.listdir("."):
if not entry.startswith('.'):
print(entry, end=' ')
print()
def test_upload_raise(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip):
host = '127.0.0.1'
port = ftp_server3.port
cnopts = CnOpts()
cnopts.hostkeys = None
ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
with pytest.raises(loader.LoaderException):
ft.upload(conn, skip_existing=skip)
def download_data(url, user, passwd, file_trans, skip_existing, **kwargs):
downloaded = []
with Connection(url, user, password=passwd, **kwargs) as conn:
for ft in file_trans:
try:
ft.download(conn, skip_existing)
downloaded.append(ft)
except loader.LoaderException as e:
print(e.message)
return downloaded
def upload_data(url, user, passwd, file_trans, skip_existing):
with Connection(url, user, password=passwd) as conn:
count = 0
for ft in file_trans:
try:
ft.upload(conn, skip_existing)
count += 1
except loader.LoaderException as e:
print(e.message)
return count
def updateFiles():
global connection, needed_files
try:
os.mkdir(localLogsDir)
except FileExistsError:
pass
try:
connection = Connection(host, user, password=paswd)
except ConnectionException:
print('Connection failed')
return False
else:
connection.chdir(remoteLogsDir)
needed_files = compareFileList(remoteLogsDir, localLogsDir)
downloadFiles(needed_files, localLogsDir)
logging.info('Update compited')
print('Update compited')
return True
def rename_remote_file(sftp: pysftp.Connection, before: str, after: str):
'''
Method to rename a file on remote server.
'''
#Path is set to current working directory
path = '.'
try:
# Rename file name from rename.txt to renamed.txt on remote server
# If rename is unsuccessful, then None is returned
sftp.rename(before, after)
return True
except OSError as e:
if e.strerror is None:
print(ERROR_PREFACE, "could not rename file or directory.")
else:
print(ERROR_PREFACE, e.strerror)
return False
def get_client(self):
cnopts = CnOpts()
cnopts.hostkeys = None
sftp = Connection(self.conf['domain'],
username=self.conf['user'],
password=self.conf['password'],
cnopts=cnopts,
default_path=self.conf['remote_path'])
LOGGER.info("Successfully connected to '%s' SFTP server",
self.conf['domain'])
return sftp
class SFTPService: # pylint: disable=too-few-public-methods
"""SFTP Service class."""
DEFAUILT_CONNECT_SERVER = 'CAS'
@staticmethod
def get_connection(
server_name: str = DEFAUILT_CONNECT_SERVER) -> Connection:
"""Return a SFTP connection."""
# pylint: disable=protected-access
return SFTPService._connect(server_name)
@staticmethod
def _connect(server_name: str) -> Connection:
sftp_configs = current_app.config.get('SFTP_CONFIGS')
# if not passed , connect to CAS server always. to make the existing code work
if not server_name or server_name not in sftp_configs.keys():
server_name = SFTPService.DEFAUILT_CONNECT_SERVER
connect_configs = sftp_configs.get(server_name)
sftp_host: str = connect_configs.get('SFTP_HOST')
cnopts = CnOpts()
# only for local development set this to false .
if connect_configs.get('SFTP_VERIFY_HOST').lower() == 'false':
cnopts.hostkeys = None
else:
ftp_host_key_data = connect_configs.get('SFTP_HOST_KEY').encode()
key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data))
cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key)
sftp_port: int = connect_configs.get('SFTP_PORT')
sft_credentials = {
'username':
connect_configs.get('SFTP_USERNAME'),
# private_key should be the absolute path to where private key file lies since sftp
'private_key':
connect_configs.get('FTP_PRIVATE_KEY_LOCATION'),
'private_key_pass':
connect_configs.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE')
}
# to support local testing. SFTP CAS server should run in private key mode
if password := connect_configs.get('SFTP_PASSWORD'):
sft_credentials['password'] = password
sftp_connection = Connection(host=sftp_host,
**sft_credentials,
cnopts=cnopts,
port=sftp_port)
current_app.logger.debug('sftp_connection successful')
return sftp_connection
def check_if_sftp_remotepath_exists(sftp: Connection, remotepath: str):
"""
This error checking function requires a running sftp connection.
"""
if sftp.exists(remotepath=remotepath):
return True
else:
raise IOError(
f"{remotepath} seems not to exist. Did you check spelling errors?"
)
return
def test_download(ftp_server1, file_tree1, tmp_path, local, remote, name, arch, skip):
host = '127.0.0.1'
port = ftp_server1.port
cnopts = CnOpts()
cnopts.hostkeys = None
ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
ft.download(conn, skip_existing=skip)
filename = ft._arch_name
mode = 'rb' if arch else 'r'
with open(tmp_path / local / filename, mode) as f:
data = f.read()
assert data == ftp_server1.content_provider.get(remote + '/' + filename)
def test_upload2(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip, content):
host = '127.0.0.1'
port = ftp_server3.port
cnopts = CnOpts()
cnopts.hostkeys = None
ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
ft.upload(conn, skip_existing=skip)
filename = ft._arch_name
data = ftp_server3.content_provider.get(remote + '/' + filename)
if arch:
data = loader.get_archivator(arch).decompress(data)
assert data.decode() == content
def __init__(self, serverDNS):
"""
The init script for the class
"""
self.serverDNS = str(serverDNS)
BASE_DIR = dirname(dirname(__file__))
config = ConfigReader()
self.userName = config.get('saturnnode', 'user')
self.keyFile = join(BASE_DIR, config.get('saturnring',
'privatekeyfile'))
self.rembashpath = config.get('saturnnode', 'bashpath')
self.rempypath = config.get('saturnnode', 'pythonpath')
self.iscsiconfdir = join(BASE_DIR,
config.get('saturnring', 'iscsiconfigdir'))
self.remoteinstallLoc = config.get('saturnnode', 'install_location')
self.localbashscripts = join(BASE_DIR,
config.get('saturnring', 'bashscripts'))
try:
self.srv = Connection(self.serverDNS, self.userName, self.keyFile)
except:
logger.critical(
"Failed SSH-exec connection on Saturn server %s; possible cause: %s"
% (self.serverDNS, format_exc()))
class Storage(BaseStorage):
""" SFTP Storage """
name = 'SFTP'
SFTP_HOST = getattr(settings, 'DBBACKUP_SFTP_HOST', None)
SFTP_USER = getattr(settings, 'DBBACKUP_SFTP_USER', None)
SFTP_PASSWORD = getattr(settings, 'DBBACKUP_SFTP_PASSWORD', None)
SFTP_PATH = getattr(settings, 'DBBACKUP_SFTP_PATH', ".")
SFTP_PATH = '/%s/' % SFTP_PATH.strip('/')
SFTP_PASSIVE_MODE = getattr(settings, 'DBBACKUP_SFTP_PASSIVE_MODE', False)
def __init__(self, server_name=None):
self._check_settings()
self.sftp = Connection(
host = self.SFTP_HOST,
username = self.SFTP_USER,
password = self.SFTP_PASSWORD)
def _check_settings(self):
""" Check we have all the required settings defined. """
if not self.SFTP_HOST:
raise StorageError('%s storage requires DBBACKUP_SFTP_HOST to be defined in settings.' % self.name)
###################################
# DBBackup Storage Methods
###################################
@property
def backup_dir(self):
return self.SFTP_PATH
def delete_file(self, filepath):
""" Delete the specified filepath. """
self.sftp.remove(filepath)
def list_directory(self, raw=False):
""" List all stored backups for the specified. """
return sorted(self.sftp.listdir(self.SFTP_PATH))
def write_file(self, filehandle, filename):
""" Write the specified file. """
filehandle.seek(0)
backuppath = os.path.join(self.SFTP_PATH, filename)
self.sftp.putfo(filehandle, backuppath)
def read_file(self, filepath):
""" Read the specified file and return it's handle. """
outputfile = tempfile.SpooledTemporaryFile(
max_size=10 * 1024 * 1024,
dir=dbbackup_settings.TMP_DIR)
self.sftp.getfo(filepath, outputfile)
return outputfile
class Storage(BaseStorage):
""" SFTP Storage """
name = 'SFTP'
SFTP_HOST = getattr(settings, 'DBBACKUP_SFTP_HOST', None)
SFTP_USER = getattr(settings, 'DBBACKUP_SFTP_USER', None)
SFTP_PASSWORD = getattr(settings, 'DBBACKUP_SFTP_PASSWORD', None)
SFTP_PATH = getattr(settings, 'DBBACKUP_SFTP_PATH', ".")
SFTP_PATH = '/%s/' % SFTP_PATH.strip('/')
SFTP_PASSIVE_MODE = getattr(settings, 'DBBACKUP_SFTP_PASSIVE_MODE', False)
def __init__(self, server_name=None):
self._check_settings()
self.sftp = Connection(host=self.SFTP_HOST,
username=self.SFTP_USER,
password=self.SFTP_PASSWORD)
def _check_settings(self):
""" Check we have all the required settings defined. """
if not self.SFTP_HOST:
raise StorageError(
'%s storage requires DBBACKUP_SFTP_HOST to be defined in settings.'
% self.name)
###################################
# DBBackup Storage Methods
###################################
@property
def backup_dir(self):
return self.SFTP_PATH
def delete_file(self, filepath):
""" Delete the specified filepath. """
self.sftp.remove(filepath)
def list_directory(self, raw=False):
""" List all stored backups for the specified. """
return sorted(self.sftp.listdir(self.SFTP_PATH))
def write_file(self, filehandle, filename):
""" Write the specified file. """
filehandle.seek(0)
backuppath = os.path.join(self.SFTP_PATH, filename)
self.sftp.putfo(filehandle, backuppath)
def read_file(self, filepath):
""" Read the specified file and return it's handle. """
outputfile = tempfile.SpooledTemporaryFile(
max_size=10 * 1024 * 1024, dir=dbbackup_settings.TMP_DIR)
self.sftp.getfo(filepath, outputfile)
return outputfile
def _download_them(self, _sftp: pysftp.Connection):
"""
This method does the hard work downloading files. It also preserves the modified time
after downloaded from the SFTP host.
The call back lambda function prints the % byes downloaded,
which is based on the size of the file.
Parameters
----------
_sftp : pysftp.Connection
a sftp connection object
"""
entries = _sftp.listdir(self._srcDir)
for entry in entries:
if _sftp.isfile(entry) and self._is_file_asked(entry):
_sftp.get(os.path.join(self._srcDir, entry),
os.path.join(self._destDir, entry),
callback=lambda transfered, size: audit_params(operation=Sc.OPERATION_DOWNLOAD,
status=Sc.STATUS_COMPLETE,
comments="{} {} ({})%".format('>>-->', entry, str("%.2f" % (100*(int(transfered)/int(size)))))),
preserve_mtime=True)
def __init__(self,serverDNS):
"""
The init script for the class
"""
self.serverDNS = str(serverDNS)
BASE_DIR = dirname(dirname(__file__))
config = ConfigReader()
self.userName = config.get('saturnnode','user')
self.keyFile = join(BASE_DIR,config.get('saturnring','privatekeyfile'))
self.rembashpath = config.get('saturnnode','bashpath')
self.rempypath = config.get('saturnnode','pythonpath')
self.iscsiconfdir = join(BASE_DIR,config.get('saturnring','iscsiconfigdir'))
self.remoteinstallLoc = config.get('saturnnode','install_location')
self.localbashscripts = join(BASE_DIR,config.get('saturnring','bashscripts'))
try:
self.srv = Connection(self.serverDNS,self.userName,self.keyFile)
except:
logger.critical("Failed SSH-exec connection on Saturn server %s; possible cause: %s" % (self.serverDNS,format_exc()) )
def sendPhotos(self, host, name, pskw, floor=0):
try:
with Connection(host, username=name, password=pskw) as sftp:
if (sftp.isdir('/home/pi/Documents/Master/data')):
sftp.chdir('/home/pi/Documents/Master/data')
if not sftp.isdir("Grower{}".format(floor)):
sftp.makedirs("Grower{}".format(floor))
sftp.chdir("Grower{}".format(floor))
sftp.makedirs(self.photoPath(False))
sftp.put_r(
self.photoPath(True),
'/home/pi/Documents/Master/data/Grower{}/{}'.format(
floor, self.photoPath(False)),
preserve_mtime=False)
return True
else:
return False
except:
return False
def ftp_upload(dir_filename_list):
"Uploads a list of (dir, filename)"
# Required to fix pysftp bug
cnopts = CnOpts()
cnopts.hostkeys = None
with Connection(
host=b64decode(config["ftp"]["host"]).decode("utf-8"),
username=b64decode(config["ftp"]["user"]).decode("utf-8"),
password=b64decode(config["ftp"]["password"]).decode("utf-8"),
port=int(b64decode(config["ftp"]["port"]).decode("utf-8")),
cnopts=cnopts,
) as sftp:
logger.info("SFTP connection OK")
for dir_filename in dir_filename_list:
file_path = join(dir_filename[0], dir_filename[1])
logger.info("Uploading %s..." % (file_path))
sftp.put(file_path, f"{config['dir']['remote']}/{dir_filename[1]}")
logger.info("FTP uploaded successfully: %s" % (file_path))
class SFTPService: # pylint: disable=too-few-public-methods
"""SFTP Service class."""
__instance: Connection = None
@staticmethod
def get_connection() -> Connection:
"""Return a SFTP connection."""
# pylint: disable=protected-access
if not SFTPService.__instance or not SFTPService.__instance._sftp_live:
SFTPService.__instance = SFTPService._connect()
return SFTPService.__instance
@staticmethod
def _connect() -> Connection:
sftp_host: str = current_app.config.get('CAS_SFTP_HOST')
cnopts = CnOpts()
# only for local development set this to false .
if current_app.config.get('SFTP_VERIFY_HOST').lower() == 'false':
cnopts.hostkeys = None
else:
host_key = current_app.config.get('CAS_SFTP_HOST_KEY')
ftp_host_key_data = current_app.config.get('CAS_SFTP_HOST_KEY').encode()
key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data))
cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key)
sftp_port: int = current_app.config.get('CAS_SFTP_PORT')
sft_credentials = {
'username': current_app.config.get('CAS_SFTP_USER_NAME'),
# private_key should be the absolute path to where private key file lies since sftp
'private_key': current_app.config.get('BCREG_FTP_PRIVATE_KEY_LOCATION'),
'private_key_pass': current_app.config.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE')
}
# to support local testing. SFTP CAS server should run in private key mode
if password := current_app.config.get('CAS_SFTP_PASSWORD'):
sft_credentials['password'] = password
sftp_connection = Connection(host=sftp_host, **sft_credentials, cnopts=cnopts, port=sftp_port)
current_app.logger.debug('sftp_connection successful')
return sftp_connection
class SFTP:
def __init__(self, sftp_server_config):
self.server = sftp_server_config['host_address']
self.username = sftp_server_config['user_name']
self.private_key = sftp_server_config['key_path']
self.sftp_folder = sftp_server_config['sftp_folder']
self.connection_opts = CnOpts()
self.connection_opts.hostkeys = None
self.__connection: Connection = None
def connect(self):
if not self.__connection:
self.__connection = Connection(self.server,
self.username,
self.private_key,
cnopts=self.connection_opts)
return self
def put(self, file, remote_path):
return self.__connection.put(file, remote_path)
def remove(self, path):
return self.__connection.remove(path)
def rmdir(self, path):
return self.__connection.rmdir(path)
def listdir(self, path):
return self.__connection.listdir(path)
def is_dir(self, path):
return self.__connection.isdir(path)
def is_file(self, path):
return self.__connection.isfile(path)
def close(self):
self.__connection.close()
self.__connection = None
def ftp_upload(sourcefile: str) -> str:
def prepare_remote_folder(conn) -> None:
"Create the necessary folder(s) on the remote server and change the directory accordingly"
if config.preserve_folders_on_remote:
full_remote_dir = os.path.join(config.remote_directory, get_date_folder())
else:
full_remote_dir = config.remote_directory
if not conn.exists(full_remote_dir):
conn.makedirs(full_remote_dir)
conn.chdir(full_remote_dir)
extension = get_extension(sourcefile)
with Connection(config.sftp_address, username=config.username, password=config.password, port=config.sftp_port,
private_key=config.private_key, private_key_pass=config.private_key_pass) as conn:
prepare_remote_folder(conn)
extension = get_extension(sourcefile)
dest_name = generate_filename(config.length, extension)
while conn.exists(dest_name):
dest_name = generate_filename(config.length, extension)
conn.put(sourcefile, dest_name)
return dest_name
class PollServer():
"""
This is the controller that calls/runs scripts on a Saturn server
as required by saturnring
"""
def __init__(self,serverDNS):
"""
The init script for the class
"""
self.serverDNS = str(serverDNS)
BASE_DIR = dirname(dirname(__file__))
config = ConfigReader()
self.userName = config.get('saturnnode','user')
self.keyFile = join(BASE_DIR,config.get('saturnring','privatekeyfile'))
self.rembashpath = config.get('saturnnode','bashpath')
self.rempypath = config.get('saturnnode','pythonpath')
self.iscsiconfdir = join(BASE_DIR,config.get('saturnring','iscsiconfigdir'))
self.remoteinstallLoc = config.get('saturnnode','install_location')
self.localbashscripts = join(BASE_DIR,config.get('saturnring','bashscripts'))
try:
self.srv = Connection(self.serverDNS,self.userName,self.keyFile)
except:
logger.critical("Failed SSH-exec connection on Saturn server %s; possible cause: %s" % (self.serverDNS,format_exc()) )
def InstallScripts(self):
"""
Copy bash scripts from the saturnringserver into the saturn server via sftp
"""
#srv = Connection(self.serverDNS,self.userName,self.keyFile)
self.srv.execute ('mkdir -p '+self.remoteinstallLoc+'saturn-bashscripts/')
self.srv.chdir(self.remoteinstallLoc+'saturn-bashscripts/')
locallist=listdir(self.localbashscripts)
for localfile in locallist:
self.srv.put(self.localbashscripts+localfile)
self.srv.execute("chmod 777 "+self.remoteinstallLoc+'saturn-bashscripts/'+localfile)
#srv.close()
logger.info("Installed scripts")
def Exec(self,command):
"""
Helper function for executing a remote command over an SSH tunnel
"""
rtncmd = -1
try:
#srv = Connection(self.serverDNS,self.userName,self.keyFile)
rtncmd=self.srv.execute(command)
#srv.close()
except:
logger.error("Failed SSH-exec command: %s on Saturn server %s" % (command, self.serverDNS))
logger.error(format_exc())
return rtncmd
def ParseLVM(self,strList,delimitStr,paraList):
"""
Parse lvdisplay and vgdisplay strings and populate
dictionaries with relevant information
"""
rtnDict ={}
valueDict={}
for aLine in strList:
if (delimitStr in aLine):
if len(valueDict) == len(paraList):
rtnDict[valueDict[paraList[0]]]=valueDict
valueDict = {}
continue
else:
for anItem in paraList:
if anItem in aLine:
valueDict[anItem] = aLine.split(anItem)[1].strip()
if '%' in valueDict[anItem]:
valueDict[anItem] = float(valueDict[anItem][:-2])
continue
if '/' in valueDict[anItem]:
valueDict[anItem] = valueDict[anItem].split('/')[0]
if 'GiB' in valueDict[anItem]:
valueDict[anItem] = float(valueDict[anItem].split('GiB')[0])*1
continue
if 'MiB' in valueDict[anItem]:
valueDict[anItem] = float(valueDict[anItem].split('MiB')[0])*0.001
continue
continue
if len(valueDict) == len(paraList):
rtnDict[valueDict[paraList[0]]] = valueDict
logger.info(rtnDict)
return rtnDict
def UpdateLVs(self,vgObject):
"""
Update LV information, called to monitor and update capacity.
"""
lvdict = self.GetLVs(vgObject.vguuid)
if "No LVs " in lvdict:
logger.info("There are no LVs in %s to run UpdateLVs on in Saturn host %s" %(vgObject.vguuid, self.serverDNS))
return 0
if lvdict == -1:
logger.error ("Could not run GetLVs (perhaps there are no LVs in this VG yet?)")
return -1
lvs = LV.objects.filter(vg=vgObject)
for lvName,lvinfo in lvdict.iteritems():
if len(lvs.filter(lvname=lvName)):
preexistLV=lvs.filter(lvname=lvName)[0]
preexistLV.lvsize=lvinfo['LV Size']
preexistLV.save(update_fields=['lvsize'])
else:
logger.warn("Found orphan LV %s in VG %s on host %s" %(lvName,vgObject.vguuid,self.serverDNS))
def GetLVs(self,vguuid):
"""
Wrapper for parselvm (for LVs), actually populating the DB is done by the UpdateLV function
"""
execCmd = " ".join(['sudo','vgdisplay', '-c','|','grep',vguuid,'|','cut -d: -f1'])
vgname = self.Exec(execCmd)[0].strip()
if vgname == -1:
logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS))
return -1
execCmd=" ".join(['sudo','lvdisplay','--units g',vgname])
lvStrList = self.Exec(execCmd)
if lvStrList ==[""]:
return "No LVs in %s" %(vguuid,)
if lvStrList == -1:
logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS))
return -1
delimitStr = '--- Logical volume ---'
paraList=['LV Name','LV UUID','LV Size']
lvs = self.ParseLVM(lvStrList,delimitStr,paraList)
return lvs
def GetVG(self): #Unit test this again
"""
Wrapper for parseLVM (for VGs)+populating the DB
"""
delimitStr = '--- Volume group ---'
paraList = ['VG Name','VG Size','PE Size','Total PE', 'Free PE / Size', 'VG UUID']
execCmd = " ".join(['sudo','vgdisplay','--units g'])
vgStrList = self.Exec(execCmd)
if vgStrList == -1:
return -1
vgs = self.ParseLVM(vgStrList,delimitStr,paraList)
#logger.info("VGStating on %s returns %s " % (self.serverDNS, str(vgs)) )
rtnvguuidList = ""
for vgname in vgs:
try:
execCmd = " ".join(['sudo',self.remoteinstallLoc+'saturn-bashscripts/vgstats.sh',vgname])
cmdStr = self.Exec(execCmd)
logger.info(self.serverDNS+": "+" ".join(['sudo',self.rembashpath,self.remoteinstallLoc+'saturn-bashscripts/vgstats.sh',vgname])+': returned '+str(cmdStr))
maxavl = float(cmdStr[0].rstrip())
totalGB = float(cmdStr[1].rstrip())
isThin = bool(int(cmdStr[2].rstrip()))
except:
logger.warn("Unable to run VGscan, disabling VG on "+self.serverDNS)
logger.warn(format_exc())
try:
vg = VG.objects.get(vguuid=vgs[vgname]['VG UUID'])
vg.in_error = True
vg.save(update_fields=['in_error'])
except:
logger.error("VG not found in DB: %s" % ( vgs[vgname]['VG UUID'],))
return 3
existingvgs = VG.objects.filter(vguuid=vgs[vgname]['VG UUID'])
if len(existingvgs)==1:
existingvg = existingvgs[0]
existingvg.in_error=False
existingvg.CurrentAllocGB = totalGB-maxavl#Target.objects.filter(targethost=existingvg.vghost).aggregate(Sum('sizeinGB'))['sizeinGB__sum']
existingvg.totalGB=totalGB
existingvg.maxavlGB=maxavl
existingvg.is_thin=isThin
existingvg.vgsize = vgs[vgname]['VG Size']
existingvg.save(update_fields=['totalGB','maxavlGB','vgsize','CurrentAllocGB','in_error','is_thin'])
logger.info( "Ran in existingVG loop")
else:
logger.info("Found new VG, adding\n" + str(vgs[vgname]))
myvg = VG(vghost=StorageHost.objects.get(dnsname=self.serverDNS),vgsize=vgs[vgname]['VG Size'],
vguuid=vgs[vgname]['VG UUID'],vgpesize=vgs[vgname]['PE Size'],
vgtotalpe=vgs[vgname]['Total PE'],
vgfreepe=vgs[vgname]['Free PE / Size'],
totalGB=totalGB,maxavlGB=maxavl, is_thin=isThin)
myvg.save()#force_update=True)
rtnvguuidList = rtnvguuidList+ ','+ vgs[vgname]['VG UUID']
return rtnvguuidList[1:]
def GitSave(self,vguuid,commentStr):
"""
Check in changes to config files into git repository
"""
try:
#srv = Connection(self.serverDNS,self.userName,self.keyFile)
self.srv.get('/temp/scst.conf',self.iscsiconfdir+self.serverDNS+'.scst.conf')
self.srv.get('/temp/'+vguuid,self.iscsiconfdir+self.serverDNS+'.'+vguuid+'.lvm')
try:
repo = Repo(self.iscsiconfdir)
filelist = [ f for f in listdir(self.iscsiconfdir) if isfile(join(self.iscsiconfdir,f)) ]
repo.stage(filelist)
repo.do_commit(commentStr)
except:
var = format_exc()
logger.error("During GitSave: %s: Git save error: %s" % (commentStr, var))
except:
var = format_exc()
logger.error("During GitSave: %s: PYSFTP download error: %s" % (commentStr, var))
def CreateTarget(self,iqnTarget,iqnInit,sizeinGB,storageip1,storageip2,vguuid):
"""
Create iSCSI target by running the createtarget script;
and save latest scst.conf from the remote server (overwrite)
"""
#self.srv = Connection(self.serverDNS,self.userName,self.keyFile)
cmdStr = " ".join(['sudo',self.rembashpath,self.remoteinstallLoc+'saturn-bashscripts/createtarget.sh',str(sizeinGB),iqnTarget,storageip1,storageip2,iqnInit,vguuid])
#srv.close()
logger.info ("Launching createtarget with \n%s" %(cmdStr,))
exStr=self.Exec(cmdStr)
if exStr == -1:
return -1
commentStr = "Trying to create target %s " %( iqnTarget, )
self.GitSave(vguuid,commentStr)
logger.info("Execution report for %s: %s" %(cmdStr,"\t".join(exStr)))
if "SUCCESS" in str(exStr):
logger.info("Returning successful createtarget run")
return 1
else:
logger.error("Returning failed createtarget run")
return 0
def GetTargetsState(self):
"""
Read targets to determine their latest state via the parsetarget script
"""
cmdStr = " ".join(["sudo",self.rempypath,self.remoteinstallLoc+'saturn-bashscripts/parsetarget.py'])
exStr = self.Exec(cmdStr)
if exStr == -1:
return -1
for eachLine in exStr:
iqntar = eachLine.split()[0]
tar = Target.objects.filter(iqntar=iqntar)
if len(tar)==1:
tar = tar[0]
if "no session" in eachLine:
tar.sessionup=False
tar.rkbpm = 0
tar.wkbpm = 0
else:
tar.sessionup=True
rkb = long(eachLine.split()[1])
tar.rkbpm = long(rkb-tar.rkb)
tar.rkb=rkb
wkb = long(eachLine.split()[2])
wpm = long(wkb-tar.wkb)
tar.wkbpm = wpm
tar.wkb=wkb
tar.save()
else:
logger.warn("Found target %s on %s that does not exist in the DB" % (iqntar,self.serverDNS) )
def DeleteTarget(self,iqntar,vguuid):
"""
Delete target
"""
logger.info("Trying to delete target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS))
if self.GetTargetsState() == -1:
logger.error("Could not GetTargetsState while deleting %s" %(iqntar,))
return -1
try:
tar = Target.objects.get(iqntar=iqntar)
except:
logger.warn("Could not find deletion target in DB, exiting. "+iqntar)
return -1
if not tar.sessionup:
cmdStr = " ".join(["sudo",self.rembashpath,self.remoteinstallLoc+'saturn-bashscripts/removetarget.sh',iqntar,vguuid])
exStr = self.Exec(cmdStr)
if exStr == -1:
return -1
self.GitSave(vguuid,"Trying to delete target %s " %( iqntar,))
success1 = False
success2 = False
for eachLine in exStr:
logger.info(eachLine)
if "Removing virtual target '"+iqntar+"' from driver 'iscsi': done" in eachLine:
success1=True
if "successfully removed" in eachLine:
success2=True
if success1==True and success2==True:
logger.info("Successful deletion of target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS))
return 1
else:
logger.error("Error deleting target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS))
return -1
return -1
def GetInterfaces(self):
"""
Scan and get network interfaces into saturnring DB
"""
#cmdStr = 'ifconfig | grep -oE "inet addr:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d: -f2'
cmdStr = 'ip addr | grep -oE "inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d" " -f2'
ipadds=self.Exec(cmdStr)
if ipadds == -1:
return -1
sh = StorageHost.objects.get(dnsname=self.serverDNS)
superuser=User.objects.filter(is_superuser=True).order_by('username')[0]
for addr in ipadds:
try:
addr = addr.rstrip()
if "127.0.0.1" in addr: #Ignore loopback addresses
continue
socket.inet_aton(addr)
interfaces = Interface.objects.filter(ip=addr)
if len(interfaces) != 1: #If 0, then new interface
Interface.objects.filter(ip=addr).delete()
logger.info("Adding newly discovered interface %s to storage host %s " % (addr, self.serverDNS))
try:
newInterface = Interface(storagehost=sh,ip=addr,owner=superuser)
newInterface.save()
for eachIPRange in IPRange.objects.all():
if ipaddress.ip_address(unicode(addr)) in ipaddress.ip_network(unicode(eachIPRange.iprange)):
eachIPRange.hosts.add(sh)
eachIPRange.save()
newInterface.iprange.add(eachIPRange)
newInterface.owner=eachIPRange.owner
newInterface.save()
except:
logger.warn("Error saving newly discovered Interface %s of host %s" % (addr, self.serverDNS))
var = format_exc()
logger.warn(var)
else:
if interfaces[0].storagehost.dnsname != self.serverDNS:
Interface.objects.filter(ip=addr).delete()
logger.warn("IP address %s was reassigned to another host" % (addr,))
except socket.error:
logger.warn("Invalid IP address %s retuned in GetInterfaces call on Saturn server %s " % (addr, self.serverDNS ))
var = format_exc()
logger.warn(var)
def __init__(self, server_name=None):
self._check_settings()
self.sftp = Connection(
host = self.SFTP_HOST,
username = self.SFTP_USER,
password = self.SFTP_PASSWORD)
class PollServer():
"""
This is the controller that calls/runs scripts on a Saturn server
as required by saturnring
"""
def __init__(self,serverDNS):
"""
The init script for the class
"""
try:
self.serverDNS = str(serverDNS)
self.hostobject = StorageHost.objects.get(dnsname=self.serverDNS)
BASE_DIR = dirname(dirname(__file__))
config = ConfigReader()
self.userName = config.get('saturnnode','user')
self.keyFile = join(BASE_DIR,config.get('saturnring','privatekeyfile'))
self.rembashpath = config.get('saturnnode','bashpath')
self.rempypath = config.get('saturnnode','pythonpath')
self.iscsiconfdir = join(BASE_DIR,config.get('saturnring','iscsiconfigdir'))
self.remoteinstallLoc = config.get('saturnnode','install_location')
self.localbashscripts = join(BASE_DIR,config.get('saturnring','bashscripts'))
except:
logger.critical("Error setting up configuration for server "+self.serverDNS)
logger.critical(format_exc())
try:
self.srv = Connection(self.serverDNS,self.userName,self.keyFile)
except:
logger.critical("Failed SSH-exec connection on Saturn server %s; possible cause: %s" % (self.serverDNS,format_exc()) )
self.srv="inError"
def CheckServer(self):
if self.srv == 'inError':
return -1
remotePath = join(self.remoteinstallLoc,'saturn-bashscripts')
cmdStr = " ".join([join(remotePath,'checkserver.sh'), '2> checkservererror.log'])
#logger.info("Executing %s on %s" %(cmdStr,self.serverDNS))
rtnStrList = self.Exec(cmdStr)
if (rtnStrList == -1):
return -2
else:
for aLine in rtnStrList:
if "FAILURE" in aLine:
logger.error(self.serverDNS + ": "+ str(rtnStrList))
return -3
return 0
def InstallScripts(self):
"""
Copy bash scripts from the saturnringserver into the saturn server via sftp
"""
rtnVal = -1
try:
if self.srv == "inError":
raise Exception('Server SSH connection object inError')
remotePath = join(self.remoteinstallLoc,'saturn-bashscripts')
self.srv.execute (" ".join(['mkdir', '-p', remotePath]))
self.srv.chdir(remotePath)
locallist=listdir(self.localbashscripts)
for localfile in locallist:
self.srv.put(join(self.localbashscripts,localfile))
self.srv.execute(" ".join(["chmod", "777",join(remotePath,localfile)]))
#Update rc.local for luks reboot functionality
luksopenscriptpath = join(remotePath,'luksonreboot.sh');
self.srv.execute("sudo sed -i '/luksonreboot.sh/d' /etc/rc.local") #delete pre-existing line if any
self.srv.execute("sudo sed -i '/^exit 0/i " + '/bin/bash ' + luksopenscriptpath +"' /etc/rc.local")
logger.info("Installed scripts on "+ self.serverDNS)
rtnVal = 1
except:
logger.error('Could not install scripts on '+self.serverDNS)
logger.error(format_exc())
finally:
return rtnVal
def Exec(self,command):
"""
Helper function for executing a remote command over an SSH tunnel
"""
rtncmd = -1
if self.srv=="inError":
logger.error("There is no ssh connection object for server: %s" %(self.serverDNS,))
return -1
try:
#srv = Connection(self.serverDNS,self.userName,self.keyFile)
rtncmd=self.srv.execute(command)
#srv.close()
except:
logger.error("Failed SSH-exec command: %s on Saturn server %s" % (command, self.serverDNS))
logger.error(format_exc())
return rtncmd
def GetFile(self,remotePath,localPath):
"""
Get a file from the remote server.
return 1 on success, -1 on error
"""
try:
self.srv.get(remotePath,localPath)
#logger.info("Copying file %s from remote server %s to local path %s succeeded" %(remotePath,self.serverDNS,localPath))
return 1
except:
logger.error("Error copying file %s from remote server %s to local path %s" %(remotePath,self.serverDNS,localPath))
logger.error(format_exc())
return -1
def PutKeyFile(self,keyfileName):
"""
Copy over the keyfile to be used for creating the LUKs encrypted DM volumes
"""
remoteKeyfileDir = join(self.remoteinstallLoc,'keys')
try:
self.Exec (" ".join(['mkdir','-p',remoteKeyfileDir]))
self.srv.chdir(remoteKeyfileDir)
self.srv.put(join(self.iscsiconfdir,keyfileName))
self.remoteKeyfilePath = join(remoteKeyfileDir,keyfileName)
rtnString = self.Exec ('test -f ' + self.remoteKeyfilePath + '&& echo "OK Putkeyfile" ')
logger.info(rtnString)
if "OK Putkeyfile" not in str(rtnString):
raise ValueError("Putkey didnt install file")
except ValueError:
logger.error("Failed to put keyfile on Saturn server %s at location %s" %(self.serverDNS,join(remoteKeyfileDir,keyfileName)))
logger.error(format_exc())
return -1
return self.remoteKeyfilePath
def DelKeyFile(self,keyfileName):
"""
Delete key file from saturn server
"""
remoteKeyfileDir = join(self.remoteinstallLoc,'keys')
self.srv.execute('rm '+ join(remoteKeyfileDir,keyfileName))
rtnString = self.Exec ('test ! -f ' + join(self.iscsiconfdir,keyfileName)+ ' && echo "OK Deleted keyfile"')
return rtnString
def ParseLVM(self,strList,delimitStr,paraList):
"""
Parse lvdisplay and vgdisplay strings and populate
dictionaries with relevant information
"""
rtnDict ={}
valueDict={}
for aLine in strList:
if (delimitStr in aLine):
if len(valueDict) == len(paraList):
rtnDict[valueDict[paraList[0]]]=valueDict
valueDict = {}
continue
else:
for anItem in paraList:
if anItem in aLine:
valueDict[anItem] = aLine.split(anItem)[1].strip()
if '%' in valueDict[anItem]:
valueDict[anItem] = float(valueDict[anItem][:-2])
continue
if '/' in valueDict[anItem]:
valueDict[anItem] = valueDict[anItem].split('/')[0]
if (('GiB' in valueDict[anItem]) and ('Size' in aLine)):
valueDict[anItem] = float(valueDict[anItem].split('GiB')[0])*1
continue
if (('MiB' in valueDict[anItem]) and ('Size' in aLine)):
valueDict[anItem] = float(valueDict[anItem].split('MiB')[0])*0.001
continue
continue
if len(valueDict) == len(paraList):
rtnDict[valueDict[paraList[0]]] = valueDict
#logger.info(rtnDict)
return rtnDict
def UpdateLVs(self,vgObject):
"""
Update LV information, called to monitor and update capacity.
"""
lvdict = self.GetLVs(vgObject.vguuid)
if "No LVs " in lvdict:
logger.info("There are no LVs in %s to run UpdateLVs on in Saturn host %s" %(vgObject.vguuid, self.serverDNS))
return 0
if lvdict == -1:
logger.error ("Could not run GetLVs (perhaps there are no LVs in this VG yet?)")
return -1
lvs = set(LV.objects.filter(vg=vgObject))
lvDict = {}
for eachlv in lvs:
lvDict[eachlv.lvname] = eachlv
for lvName,lvinfo in lvdict.iteritems():
if lvName in lvDict:
preexistLV=lvDict[lvName]
preexistLV.lvsize=lvinfo['LV Size']
preexistLV.save(update_fields=['lvsize'])
else:
logger.warn("Found orphan LV %s in VG %s on host %s" %(lvName,vgObject.vguuid,self.serverDNS))
def GetLVs(self,vguuid):
"""
Wrapper for parselvm (for LVs), actually populating the DB is done by the UpdateLV function
"""
execCmd = " ".join(['sudo','vgdisplay', '-c','|','grep',vguuid,'|','cut -d: -f1'])
vgname = self.Exec(execCmd)[0].strip()
if vgname == -1:
logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS))
return -1
execCmd=" ".join(['sudo','lvdisplay','--units g',vgname])
lvStrList = self.Exec(execCmd)
if lvStrList ==[""]:
return "No LVs in %s" %(vguuid,)
if lvStrList == -1:
logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS))
return -1
delimitStr = '--- Logical volume ---'
paraList=['LV Name','LV UUID','LV Size']
lvs = self.ParseLVM(lvStrList,delimitStr,paraList)
return lvs
def GetVG(self): #Unit test this again
"""
Wrapper for parseLVM (for VGs)+populating the DB
"""
delimitStr = '--- Volume group ---'
paraList = ['VG Name','VG Size','PE Size','Total PE', 'Free PE / Size', 'VG UUID']
execCmd = " ".join(['sudo','vgdisplay','--units g'])
vgStrList = self.Exec(execCmd)
if vgStrList == -1:
logger.error("Error in GetVG while executing %s on server %s " %(execCmd,self.serverDNS))
return -1
vgs = self.ParseLVM(vgStrList,delimitStr,paraList)
#logger.info("VGStating on %s returns %s " % (self.serverDNS, str(vgs)) )
rtnvguuidList = ""
for vgname in vgs:
try:
execCmd = " ".join(['sudo',join(self.remoteinstallLoc,'saturn-bashscripts/vgstats.sh'),vgname,' 2> error.log'])
cmdStr = self.Exec(execCmd)
maxavl = float(cmdStr[0].rstrip())
totalGB = float(cmdStr[1].rstrip())
isThin = bool(int(cmdStr[2].rstrip()))
except:
logger.warn("Unable to run VGscan, disabling VG on "+self.serverDNS)
logger.warn(format_exc())
try:
vg = VG.objects.get(vguuid=vgs[vgname]['VG UUID'])
vg.in_error = True
vg.save(update_fields=['in_error'])
except:
logger.error("VG not found in DB: %s" % ( vgs[vgname]['VG UUID'],))
return 3
existingvgs = VG.objects.filter(vguuid=vgs[vgname]['VG UUID'])
if len(existingvgs)==1:
existingvg = existingvgs[0]
existingvg.in_error=False
existingvg.CurrentAllocGB = totalGB-maxavl#Target.objects.filter(targethost=existingvg.vghost).aggregate(Sum('sizeinGB'))['sizeinGB__sum']
existingvg.totalGB=totalGB
existingvg.maxavlGB=maxavl
existingvg.is_thin=isThin
existingvg.vgsize = vgs[vgname]['VG Size']
existingvg.save(update_fields=['totalGB','maxavlGB','vgsize','CurrentAllocGB','in_error','is_thin'])
#logger.info( "Ran in existingVG loop")
else:
logger.info("Found new VG, adding\n" + str(vgs[vgname]))
myvg = VG(vghost=StorageHost.objects.get(dnsname=self.serverDNS),vgsize=vgs[vgname]['VG Size'],
vguuid=vgs[vgname]['VG UUID'],vgpesize=vgs[vgname]['PE Size'],
vgtotalpe=vgs[vgname]['Total PE'],
vgfreepe=vgs[vgname]['Free PE / Size'],
totalGB=totalGB,maxavlGB=maxavl, is_thin=isThin)
myvg.save()#force_update=True)
rtnvguuidList = rtnvguuidList+ ','+ vgs[vgname]['VG UUID']
return rtnvguuidList[1:]
def GitSave(self,commentStr):
"""
Check in changes to config files into git repository
"""
try:
repo = Repo(self.iscsiconfdir)
filelist = [ f for f in listdir(self.iscsiconfdir) if isfile(join(self.iscsiconfdir,f)) ]
repo.stage(filelist)
repo.do_commit(commentStr)
return 1
except:
var = format_exc()
logger.error("During GitSave %s: Git save error: %s" % (commentStr,var))
return -1
def CreateTarget(self,iqnTarget,iqnInit,sizeinGB,storageip1,storageip2,vguuid,isencrypted):
"""
Create iSCSI target by running the createtarget script;
and save latest scst.conf from the remote server (overwrite)
"""
#self.srv = Connection(self.serverDNS,self.userName,self.keyFile)
if str(isencrypted) != '1':
cmdStr = " ".join(['sudo',self.rembashpath,join(self.remoteinstallLoc,'saturn-bashscripts','createtarget.sh'),
str(sizeinGB),iqnTarget,storageip1,storageip2,iqnInit,vguuid, '2> createtarget.sh-error.log'])
else:
try:
self.remotekeyfilelocation = self.PutKeyFile("cryptokey")
cmdStr = " ".join(['sudo',self.rembashpath,join(self.remoteinstallLoc,'saturn-bashscripts','createencryptedtarget.sh'),
str(sizeinGB),iqnTarget,storageip1,storageip2,iqnInit,vguuid,self.remotekeyfilelocation,'2> createencryptedtarget.sh-error.log'])
if self.remotekeyfilelocation == -1:
raise ValueError("Putkey failed")
except:
logger.error("Error setting up encrypted target: %s " %(iqnTarget,))
logger.error(format_exc())
return -1
#srv.close()
logger.info ("Launching createtarget with \n%s" %(cmdStr,))
exStr=self.Exec(cmdStr)
if exStr == -1:
return -1
commentStr = "Trying to create target %s " %( iqnTarget, )
try:
if self.GetFile('/temp/scst.conf',self.iscsiconfdir+self.serverDNS+'.scst.conf')==-1:
raise Exception('Error getting scst.conf')
if self.GetFile(join('/temp',vguuid),join(self.iscsiconfdir,self.serverDNS+'.'+vguuid+'.lvm'))==-1:
raise Exception('Error getting LVM configuration file %s' %(vguuid+'.lvm',))
if self.GitSave(commentStr) == -1:
raise Exception('Error in GitSave')
except:
logger.warning('Unable to save updated config files on ring server')
logger.warning(format_exc())
logger.info("Execution report for %s: %s" %(cmdStr,"\t".join(exStr)))
if "SUCCESS" in str(exStr):
logger.info("Returning successful createtarget run")
return 1
else:
logger.error("Returning failed createtarget run:" + str(exStr))
return 0
def GetTargetsState(self):
"""
Read targets to determine their latest state via the parsetarget script
"""
cmdStr = " ".join(["sudo",self.rempypath, join(self.remoteinstallLoc,'saturn-bashscripts','parsetarget.py'), '2> parsetargeterror.txt'])
exStr = self.Exec(cmdStr)
#logger.info("Parse target returns " +str(exStr))
if exStr == -1:
return -1
try:
targetDic = {}
hostTargets = set(Target.objects.filter(targethost=self.hostobject))
for eachTarget in hostTargets:
targetDic[eachTarget.iqntar] = eachTarget
for eachLine in exStr:
iqntar = eachLine.split()[0]
if iqntar in targetDic:
tar = targetDic[iqntar]
if "no session" in eachLine:
tar.sessionup=False
tar.rkbpm = 0
tar.wkbpm = 0
else:
tar.sessionup=True
rkb = long(eachLine.split()[1])
tar.rkbpm = long(rkb-tar.rkb)
tar.rkb=rkb
wkb = long(eachLine.split()[2])
wpm = long(wkb-tar.wkb)
tar.wkbpm = wpm
tar.wkb=wkb
tar.save()
else:
logger.warn("Found target %s on %s that does not exist in the DB" % (iqntar,self.serverDNS) )
except:
logger.error("Error reading iSCSI target state for %s on server %s" %(iqntar,self.serverDNS))
logger.error(format_exc())
return -1
return 0
def DeleteTarget(self,iqntar,vguuid,lvolname):
"""
Delete target
"""
logger.info("Trying to delete target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS))
if self.GetTargetsState() == -1:
logger.error("Could not GetTargetsState while deleting %s" %(iqntar,))
return -1
try:
tar = Target.objects.get(iqntar=iqntar)
except:
logger.warn("Could not find deletion target in DB, exiting. "+iqntar)
return -1
if not tar.sessionup:
cmdStr = " ".join(["sudo",self.rembashpath,join(self.remoteinstallLoc,'saturn-bashscripts','removetarget.sh'),iqntar,vguuid,lvolname])
exStr = self.Exec(cmdStr)
if exStr == -1:
return -1
try:
if self.GetFile('/temp/scst.conf',self.iscsiconfdir+self.serverDNS+'.scst.conf') == -1:
raise Exception('Error getting scst configuration file to store locally')
if self.GetFile('/temp/'+vguuid,self.iscsiconfdir+self.serverDNS+'.'+vguuid+'.lvm') == -1:
raise Exception('Error getting LVM configuration to store locally')
if self.GitSave("Trying to delete target "+iqntar) == -1:
raise Exception('Error with gitsave in delete target')
except:
logger.error("Error getting configuration files after deletion of target")
logger.error(format_exc())
success1 = False
success2 = False
logger.info(exStr)
for eachLine in exStr:
if "Removing virtual target '"+iqntar+"' from driver 'iscsi': done" in eachLine:
success1=True
if "successfully removed" in eachLine:
success2=True
if success1==True and success2==True:
logger.info("Successful deletion of target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS))
return 1
else:
logger.error("Error deleting target %s from VG %s on host %s; command execution returned %s" %(iqntar,vguuid,self.serverDNS,str(exStr)))
return -1
else:
logger.error("Target state of %s is set to session up, will not try to delete it." %(iqntar,))
return -1
def GetInterfaces(self):
"""
Scan and get network interfaces into saturnring DB
"""
#cmdStr = 'ifconfig | grep -oE "inet addr:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d: -f2'
cmdStr = 'ip addr | grep -oE "inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d" " -f2 2> ipaddrerror.txt'
ipadds=self.Exec(cmdStr)
if ipadds == -1:
return -1
rtnVal = 0
sh = self.hostobject
superuser=User.objects.filter(is_superuser=True).order_by('username')[0]
for addr in ipadds:
try:
addr = addr.rstrip()
if "127.0.0.1" in addr: #Ignore loopback addresses
continue
socket.inet_aton(addr)
interfaces = Interface.objects.filter(ip=addr)
if len(interfaces) != 1: #If 0, then new interface
Interface.objects.filter(ip=addr).delete()
logger.info("Adding newly discovered interface %s to storage host %s " % (addr, self.serverDNS))
try:
newInterface = Interface(storagehost=sh,ip=addr,owner=superuser)
newInterface.save()
for eachIPRange in IPRange.objects.all():
if ipaddress.ip_address(unicode(addr)) in ipaddress.ip_network(unicode(eachIPRange.iprange)):
eachIPRange.hosts.add(sh)
eachIPRange.save()
newInterface.iprange.add(eachIPRange)
newInterface.owner=eachIPRange.owner
newInterface.save()
except:
logger.warn("Error saving newly discovered Interface %s of host %s" % (addr, self.serverDNS))
var = format_exc()
logger.warn(var)
rtnVal = -1
else:
if interfaces[0].storagehost.dnsname != self.serverDNS:
Interface.objects.filter(ip=addr).delete()
logger.warn("IP address %s was reassigned to another host" % (addr,))
except socket.error:
logger.warn("Invalid IP address %s retuned in GetInterfaces call on Saturn server %s " % (addr, self.serverDNS ))
var = format_exc()
logger.warn(var)
rtnVal = -1
return rtnVal
def InsertCrypttab(self,base_LV,enc_LV,keyfilePath):
"""
Insert entry into /etc/crypttab
"""
#Needed for the crypttab
baselvpath = self.Exec(" ".join(["sudo sh -c 'lvs -o lv_path | grep ",base_LV," | tr -d \" \"'"]))[0].strip()
if baselvpath == -1:
return -1
logger.info("BaseLVPATH = " + baselvpath)
cmdStr = " ".join(["sudo sh -c 'echo \"" + enc_LV, baselvpath, keyfilePath, "luks\" >> /etc/crypttab; mkdir -p /temp; cp /etc/crypttab /temp/crypttab; chmod 666 /temp/crypttab'"])
logger.info("InsertCrypttab: "+cmdStr)
rtnVal = self.Exec(cmdStr)
if rtnVal == -1:
logger.error("Error in InsertCrypttab while executing %s" %(cmdStr,))
try:
if self.GetFile('/temp/crypttab',self.iscsiconfdir+self.serverDNS+'.crypttab') == -1:
raise Exception('Could not get crypttab file')
if self.GitSave("Trying the insert crypttab entry " + cmdStr) == -1:
raise Exception('Could not get gitsave to work for crypttab')
except:
logger.error('Error with getfile/gitsave during crypttab insert operations on %s' %(self.serverDNS,))
logger.error(format_exc())
def DeleteCrypttab(self,lvStr):
"""
Delete entry from /etc/crypttab
LVStr can be either the encrypted LV name or the base LV name
"""
cmdStr = " ".join(['sudo sed -i','/'+lvStr+'/d','/etc/crypttab']) + "; sudo mkdir -p /temp; sudo cp /etc/crypttab /temp/crypttab; sudo chmod 666 /temp/crypttab"
logger.info("DeleteCrypttab: "+cmdStr)
rtnVal = self.Exec(cmdStr)
if rtnVal == -1:
return rtnVal
logger.error("Error in DeleteCrypttab while executing %s" %(cmdStr,))
try:
if self.GetFile('/temp/crypttab',self.iscsiconfdir+self.serverDNS+'.crypttab') == -1:
raise Exception('Could not get crypttab file')
if self.GitSave("Trying the insert crypttab entry " + cmdStr) == -1:
raise Exception('Could not get gitsave to work for crypttab')
except:
logger.error('Error with getfile/gitsave during crypttab delete operations on %s' %(self.serverDNS,))
logger.error(format_exc())