class Storage(BaseStorage):
""" SFTP Storage """
name = 'SFTP'
SFTP_HOST = getattr(settings, 'DBBACKUP_SFTP_HOST', None)
SFTP_USER = getattr(settings, 'DBBACKUP_SFTP_USER', None)
SFTP_PASSWORD = getattr(settings, 'DBBACKUP_SFTP_PASSWORD', None)
SFTP_PATH = getattr(settings, 'DBBACKUP_SFTP_PATH', ".")
SFTP_PATH = '/%s/' % SFTP_PATH.strip('/')
SFTP_PASSIVE_MODE = getattr(settings, 'DBBACKUP_SFTP_PASSIVE_MODE', False)
def __init__(self, server_name=None):
self._check_settings()
self.sftp = Connection(
host = self.SFTP_HOST,
username = self.SFTP_USER,
password = self.SFTP_PASSWORD)
def _check_settings(self):
""" Check we have all the required settings defined. """
if not self.SFTP_HOST:
raise StorageError('%s storage requires DBBACKUP_SFTP_HOST to be defined in settings.' % self.name)
###################################
# DBBackup Storage Methods
###################################
@property
def backup_dir(self):
return self.SFTP_PATH
def delete_file(self, filepath):
""" Delete the specified filepath. """
self.sftp.remove(filepath)
def list_directory(self, raw=False):
""" List all stored backups for the specified. """
return sorted(self.sftp.listdir(self.SFTP_PATH))
def write_file(self, filehandle, filename):
""" Write the specified file. """
filehandle.seek(0)
backuppath = os.path.join(self.SFTP_PATH, filename)
self.sftp.putfo(filehandle, backuppath)
def read_file(self, filepath):
""" Read the specified file and return it's handle. """
outputfile = tempfile.SpooledTemporaryFile(
max_size=10 * 1024 * 1024,
dir=dbbackup_settings.TMP_DIR)
self.sftp.getfo(filepath, outputfile)
return outputfile
class Storage(BaseStorage):
""" SFTP Storage """
name = 'SFTP'
SFTP_HOST = getattr(settings, 'DBBACKUP_SFTP_HOST', None)
SFTP_USER = getattr(settings, 'DBBACKUP_SFTP_USER', None)
SFTP_PASSWORD = getattr(settings, 'DBBACKUP_SFTP_PASSWORD', None)
SFTP_PATH = getattr(settings, 'DBBACKUP_SFTP_PATH', ".")
SFTP_PATH = '/%s/' % SFTP_PATH.strip('/')
SFTP_PASSIVE_MODE = getattr(settings, 'DBBACKUP_SFTP_PASSIVE_MODE', False)
def __init__(self, server_name=None):
self._check_settings()
self.sftp = Connection(host=self.SFTP_HOST,
username=self.SFTP_USER,
password=self.SFTP_PASSWORD)
def _check_settings(self):
""" Check we have all the required settings defined. """
if not self.SFTP_HOST:
raise StorageError(
'%s storage requires DBBACKUP_SFTP_HOST to be defined in settings.'
% self.name)
###################################
# DBBackup Storage Methods
###################################
@property
def backup_dir(self):
return self.SFTP_PATH
def delete_file(self, filepath):
""" Delete the specified filepath. """
self.sftp.remove(filepath)
def list_directory(self, raw=False):
""" List all stored backups for the specified. """
return sorted(self.sftp.listdir(self.SFTP_PATH))
def write_file(self, filehandle, filename):
""" Write the specified file. """
filehandle.seek(0)
backuppath = os.path.join(self.SFTP_PATH, filename)
self.sftp.putfo(filehandle, backuppath)
def read_file(self, filepath):
""" Read the specified file and return it's handle. """
outputfile = tempfile.SpooledTemporaryFile(
max_size=10 * 1024 * 1024, dir=dbbackup_settings.TMP_DIR)
self.sftp.getfo(filepath, outputfile)
return outputfile
class SFTP:
def __init__(self, sftp_server_config):
self.server = sftp_server_config['host_address']
self.username = sftp_server_config['user_name']
self.private_key = sftp_server_config['key_path']
self.sftp_folder = sftp_server_config['sftp_folder']
self.connection_opts = CnOpts()
self.connection_opts.hostkeys = None
self.__connection: Connection = None
def connect(self):
if not self.__connection:
self.__connection = Connection(self.server,
self.username,
self.private_key,
cnopts=self.connection_opts)
return self
def put(self, file, remote_path):
return self.__connection.put(file, remote_path)
def remove(self, path):
return self.__connection.remove(path)
def rmdir(self, path):
return self.__connection.rmdir(path)
def listdir(self, path):
return self.__connection.listdir(path)
def is_dir(self, path):
return self.__connection.isdir(path)
def is_file(self, path):
return self.__connection.isfile(path)
def close(self):
self.__connection.close()
self.__connection = None
def _process_file(self, sftp_conn: pysftp.Connection, f: dict,
field_dict: dict, record_info: object,
record_creator: object):
"""Process single remote file.
:param sftp_conn: [description]
:type sftp_conn: pysftp.Connection
:param f: [description]
:type f: dict
:param field_dict: [description]
:type field_dict: dict
:param record_creator: [description]
:type record_creator: object
"""
# These fields are shared by all tool modes (directories only for LIST_FILES)
record_info[field_dict['Filename']].set_from_string(
record_creator, f['filename'])
record_info[field_dict['Size']].set_from_int32(record_creator,
f['size'])
record_info[field_dict['TimeAdded']].set_from_string(
record_creator, f['atime'])
record_info[field_dict['TimeModified']].set_from_string(
record_creator, f['mtime'])
if self.tool_mode == self.ToolMode.LIST_FILES:
# Fields only present for LIST_FILES mode
record_info[field_dict['UID']].set_from_string(
record_creator, f['uid'])
record_info[field_dict['GID']].set_from_string(
record_creator, f['gid'])
record_info[field_dict['Mode']].set_from_string(
record_creator, f['mode'])
record_info[field_dict['IsDirectory']].set_from_bool(
record_creator, f['is_dir'])
record_info[field_dict['IsFile']].set_from_bool(
record_creator, f['is_file'])
elif self.tool_mode != self.ToolMode.LIST_FILES:
# Download files if not in LIST_FILES mode
if self.tool_mode == self.ToolMode.DOWNLOAD_TO_BLOB:
# Generate temporary filename
out_fname = self.alteryx_engine.create_temp_file_name('tmp')
elif self.tool_mode == self.ToolMode.DOWNLOAD_TO_PATH:
# Build local path for download
out_fname = os.path.join(self.output_settings['local_path'],
f['filename'])
# Download file
try:
# Download file to temporary folder
sftp_conn.get(f['filename'], localpath=out_fname)
except IOError as e:
self.output_message('Error transferring file "{}": {}'.format(
f['filename'], e))
# Generate BLOB for Alteryx
if self.tool_mode == self.ToolMode.DOWNLOAD_TO_BLOB:
# Read file as binary for blob
with open(out_fname, 'rb') as temp_f:
blob_content = temp_f.read()
if self.tool_mode == self.ToolMode.DOWNLOAD_TO_BLOB:
# Add file as blob
record_info[field_dict[
self.output_settings['blobfield']]].set_from_blob(
record_creator, blob_content)
elif self.tool_mode == self.ToolMode.DOWNLOAD_TO_PATH:
# Add file path
record_info[field_dict['FilePath']].set_from_string(
record_creator, out_fname)
# Finalize record for this file and push
out_record = record_creator.finalize_record()
self.output_anchor.push_record(out_record, False)
# Reset for next file
record_creator.reset()
# Handle file after it has been downloaded
if self.file_handling == self.FileHandling.MOVE_FILES:
# Check if target folder exists
if not sftp_conn.exists(self.sftp_settings['move_path']):
self.output_message(
"The target folder {} does not exist.".format(
self.sftp_settings['move_path']),
messageType=Sdk.EngineMessageType.warning)
return
# Check if it is actually a folder
if not sftp_conn.isdir(self.sftp_settings['move_path']):
self.output_message(
"The target folder {} is not a directory.".format(
self.sftp_settings['move_path']),
messageType=Sdk.EngineMessageType.warning)
return
# Try to move file
try:
sftp_conn.rename(
sftp_conn.pwd + "/" + f['filename'],
sftp_conn.normalize(self.sftp_settings['move_path']) +
"/" + f['filename'])
except IOError as e:
self.output_message('Error moving file "{}": {}'.format(
f['filename'], e))
else:
self.output_message('File {} moved to {}.'.format(
f['filename'],
sftp_conn.normalize(self.sftp_settings['move_path'])),
messageType=Sdk.EngineMessageType.info)
elif self.file_handling == self.FileHandling.DELETE_FILES:
# Simply delete file
try:
sftp_conn.remove(sftp_conn.pwd + "/" + f['filename'])
except IOError as e:
self.output_message('Error deleting file "{}": {}'.format(
f['filename'], e))
else:
self.output_message('File {} deleted from server.'.format(
f['filename']),
messageType=Sdk.EngineMessageType.info)
