def test_download2(ftp_server1, config1, file_tree1, tmp_path, download_cnt, skip):
url, file_trans = read_config(tmp_path / 'ftp-config.toml')
port = ftp_server1.port
cnopts = CnOpts()
cnopts.hostkeys = None
downloads = download_data(url, 'user1', '1234', file_trans, skip, port=port, cnopts=cnopts)
assert len(downloads) == download_cnt
def __init__(self, host, username=None, password=None, port=22,
private_key_file=None, private_key_password=None,
compression=True):
self._con = None
# connection options
cnopts = CnOpts()
cnopts.compression = compression
# set hostkeys to None, if not provided
if private_key_file is None:
cnopts.hostkeys = None
if password is None:
logging.debug('No password provided, using key auth.')
# NOTE:
# Ducking exceptions, so that they can be handled
# by the main module however it wants.
self._con = Connection(host=host, username=username,
port=port,
private_key_file=private_key_file,
# Ignore LineLengthBear, PycodeStyleBear
private_key_password=private_key_password,
cnopts=cnopts)
if self._con is None:
self._con = Connection(host, username=username, port=port,
password=password, cnopts=cnopts)
def download_new_files(last_date: typing.Optional[datetime.date]):
new_dates = []
new_filenames = []
opts = CnOpts()
opts.hostkeys = None
with Connection(settings.SFTP_HOST, username=settings.SFTP_USER,
private_key=settings.SFTP_PRIVATE_KEY, cnopts=opts) as conn:
with conn.cd(settings.SFTP_DIR):
dir_listing = conn.listdir()
for filename in dir_listing:
date = parse_filename(filename, settings.ACCOUNT_CODE)
if date:
stat = conn.stat(filename)
if stat.st_size > SIZE_LIMIT_BYTES:
logger.error('%s is too large (%s), download skipped.'
% (filename, stat.st_size))
continue
if last_date is None or date > last_date:
local_path = os.path.join(settings.DS_NEW_FILES_DIR,
filename)
new_filenames.append(local_path)
new_dates.append(date)
conn.get(filename, localpath=local_path)
if new_dates and new_filenames:
sorted_dates, sorted_files = zip(*sorted(zip(new_dates, new_filenames)))
return NewFiles(list(sorted_dates), list(sorted_files))
else:
return NewFiles([], [])
def _connect(server_name: str) -> Connection:
sftp_configs = current_app.config.get('SFTP_CONFIGS')
# if not passed , connect to CAS server always. to make the existing code work
if not server_name or server_name not in sftp_configs.keys():
server_name = SFTPService.DEFAUILT_CONNECT_SERVER
connect_configs = sftp_configs.get(server_name)
sftp_host: str = connect_configs.get('SFTP_HOST')
cnopts = CnOpts()
# only for local development set this to false .
if connect_configs.get('SFTP_VERIFY_HOST').lower() == 'false':
cnopts.hostkeys = None
else:
ftp_host_key_data = connect_configs.get('SFTP_HOST_KEY').encode()
key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data))
cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key)
sftp_port: int = connect_configs.get('SFTP_PORT')
sft_credentials = {
'username':
connect_configs.get('SFTP_USERNAME'),
# private_key should be the absolute path to where private key file lies since sftp
'private_key':
connect_configs.get('FTP_PRIVATE_KEY_LOCATION'),
'private_key_pass':
connect_configs.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE')
}
# to support local testing. SFTP CAS server should run in private key mode
if password := connect_configs.get('SFTP_PASSWORD'):
sft_credentials['password'] = password
def test_upload_raise(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip):
host = '127.0.0.1'
port = ftp_server3.port
cnopts = CnOpts()
cnopts.hostkeys = None
ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
with pytest.raises(loader.LoaderException):
ft.upload(conn, skip_existing=skip)
def __init__(self, download_folder, **kwargs):
cnopts = CnOpts()
cnopts.hostkeys = None
self._sftp = Connection(host=kwargs["host"],
username=kwargs["username"],
password=kwargs["password"],
cnopts=cnopts)
print("File server connection established...")
self._folder = kwargs["folder"]
self._download_folder = download_folder
def get_client(self):
cnopts = CnOpts()
cnopts.hostkeys = None
sftp = Connection(self.conf['domain'],
username=self.conf['user'],
password=self.conf['password'],
cnopts=cnopts,
default_path=self.conf['remote_path'])
LOGGER.info("Successfully connected to '%s' SFTP server",
self.conf['domain'])
return sftp
def connect(self):
opts = CnOpts()
opts.hostkeys = None
try:
conn = sftp.Connection(self.hostname,
username=self.username,
password=self.password,
cnopts=opts)
except ConnectionError:
raise SystemExit("Connection error")
self.backup_files_network(self.src_dir, conn)
def test_download(ftp_server1, file_tree1, tmp_path, local, remote, name, arch, skip):
host = '127.0.0.1'
port = ftp_server1.port
cnopts = CnOpts()
cnopts.hostkeys = None
ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
ft.download(conn, skip_existing=skip)
filename = ft._arch_name
mode = 'rb' if arch else 'r'
with open(tmp_path / local / filename, mode) as f:
data = f.read()
assert data == ftp_server1.content_provider.get(remote + '/' + filename)
def test_upload2(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip, content):
host = '127.0.0.1'
port = ftp_server3.port
cnopts = CnOpts()
cnopts.hostkeys = None
ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
ft.upload(conn, skip_existing=skip)
filename = ft._arch_name
data = ftp_server3.content_provider.get(remote + '/' + filename)
if arch:
data = loader.get_archivator(arch).decompress(data)
assert data.decode() == content
def ftp_upload(dir_filename_list):
"Uploads a list of (dir, filename)"
# Required to fix pysftp bug
cnopts = CnOpts()
cnopts.hostkeys = None
with Connection(
host=b64decode(config["ftp"]["host"]).decode("utf-8"),
username=b64decode(config["ftp"]["user"]).decode("utf-8"),
password=b64decode(config["ftp"]["password"]).decode("utf-8"),
port=int(b64decode(config["ftp"]["port"]).decode("utf-8")),
cnopts=cnopts,
) as sftp:
logger.info("SFTP connection OK")
for dir_filename in dir_filename_list:
file_path = join(dir_filename[0], dir_filename[1])
logger.info("Uploading %s..." % (file_path))
sftp.put(file_path, f"{config['dir']['remote']}/{dir_filename[1]}")
logger.info("FTP uploaded successfully: %s" % (file_path))
def _connect() -> Connection:
sftp_host: str = current_app.config.get('CAS_SFTP_HOST')
cnopts = CnOpts()
# only for local development set this to false .
if current_app.config.get('SFTP_VERIFY_HOST').lower() == 'false':
cnopts.hostkeys = None
else:
host_key = current_app.config.get('CAS_SFTP_HOST_KEY')
ftp_host_key_data = current_app.config.get('CAS_SFTP_HOST_KEY').encode()
key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data))
cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key)
sftp_port: int = current_app.config.get('CAS_SFTP_PORT')
sft_credentials = {
'username': current_app.config.get('CAS_SFTP_USER_NAME'),
# private_key should be the absolute path to where private key file lies since sftp
'private_key': current_app.config.get('BCREG_FTP_PRIVATE_KEY_LOCATION'),
'private_key_pass': current_app.config.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE')
}
# to support local testing. SFTP CAS server should run in private key mode
if password := current_app.config.get('CAS_SFTP_PASSWORD'):
sft_credentials['password'] = password
from django.shortcuts import render, redirect
from django.http import HttpResponse, HttpResponseRedirect
from django.template import loader
from django.urls import reverse
from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import UserCreationForm
from .forms import ConnectionForm
from io import BytesIO
from .models import Connection
from pysftp import Connection as SftpConnection
from pysftp import CnOpts, SSHException
CNOPTS = CnOpts()
CNOPTS.hostkeys = None
def index(request):
"""
Used as main entry point. Propose of registration.
Redirect if you already logged in.
:param request: user enter on site.
:return: 'all_connections' if user authenticated else 'index' page.
"""
if request.user.is_authenticated:
return redirect('all_connections')
return render(request, 'file_manager/index.html')
def user_login(request):
