def test_download2(ftp_server1, config1, file_tree1, tmp_path, download_cnt, skip): url, file_trans = read_config(tmp_path / 'ftp-config.toml') port = ftp_server1.port cnopts = CnOpts() cnopts.hostkeys = None downloads = download_data(url, 'user1', '1234', file_trans, skip, port=port, cnopts=cnopts) assert len(downloads) == download_cnt
def __init__(self, host, username=None, password=None, port=22, private_key_file=None, private_key_password=None, compression=True): self._con = None # connection options cnopts = CnOpts() cnopts.compression = compression # set hostkeys to None, if not provided if private_key_file is None: cnopts.hostkeys = None if password is None: logging.debug('No password provided, using key auth.') # NOTE: # Ducking exceptions, so that they can be handled # by the main module however it wants. self._con = Connection(host=host, username=username, port=port, private_key_file=private_key_file, # Ignore LineLengthBear, PycodeStyleBear private_key_password=private_key_password, cnopts=cnopts) if self._con is None: self._con = Connection(host, username=username, port=port, password=password, cnopts=cnopts)
def download_new_files(last_date: typing.Optional[datetime.date]): new_dates = [] new_filenames = [] opts = CnOpts() opts.hostkeys = None with Connection(settings.SFTP_HOST, username=settings.SFTP_USER, private_key=settings.SFTP_PRIVATE_KEY, cnopts=opts) as conn: with conn.cd(settings.SFTP_DIR): dir_listing = conn.listdir() for filename in dir_listing: date = parse_filename(filename, settings.ACCOUNT_CODE) if date: stat = conn.stat(filename) if stat.st_size > SIZE_LIMIT_BYTES: logger.error('%s is too large (%s), download skipped.' % (filename, stat.st_size)) continue if last_date is None or date > last_date: local_path = os.path.join(settings.DS_NEW_FILES_DIR, filename) new_filenames.append(local_path) new_dates.append(date) conn.get(filename, localpath=local_path) if new_dates and new_filenames: sorted_dates, sorted_files = zip(*sorted(zip(new_dates, new_filenames))) return NewFiles(list(sorted_dates), list(sorted_files)) else: return NewFiles([], [])
def _connect(server_name: str) -> Connection: sftp_configs = current_app.config.get('SFTP_CONFIGS') # if not passed , connect to CAS server always. to make the existing code work if not server_name or server_name not in sftp_configs.keys(): server_name = SFTPService.DEFAUILT_CONNECT_SERVER connect_configs = sftp_configs.get(server_name) sftp_host: str = connect_configs.get('SFTP_HOST') cnopts = CnOpts() # only for local development set this to false . if connect_configs.get('SFTP_VERIFY_HOST').lower() == 'false': cnopts.hostkeys = None else: ftp_host_key_data = connect_configs.get('SFTP_HOST_KEY').encode() key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data)) cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key) sftp_port: int = connect_configs.get('SFTP_PORT') sft_credentials = { 'username': connect_configs.get('SFTP_USERNAME'), # private_key should be the absolute path to where private key file lies since sftp 'private_key': connect_configs.get('FTP_PRIVATE_KEY_LOCATION'), 'private_key_pass': connect_configs.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE') } # to support local testing. SFTP CAS server should run in private key mode if password := connect_configs.get('SFTP_PASSWORD'): sft_credentials['password'] = password
def test_upload_raise(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip): host = '127.0.0.1' port = ftp_server3.port cnopts = CnOpts() cnopts.hostkeys = None ft = loader.FileTransfer(name, tmp_path / local, remote, arch) with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn: with pytest.raises(loader.LoaderException): ft.upload(conn, skip_existing=skip)
def __init__(self, download_folder, **kwargs): cnopts = CnOpts() cnopts.hostkeys = None self._sftp = Connection(host=kwargs["host"], username=kwargs["username"], password=kwargs["password"], cnopts=cnopts) print("File server connection established...") self._folder = kwargs["folder"] self._download_folder = download_folder
def get_client(self): cnopts = CnOpts() cnopts.hostkeys = None sftp = Connection(self.conf['domain'], username=self.conf['user'], password=self.conf['password'], cnopts=cnopts, default_path=self.conf['remote_path']) LOGGER.info("Successfully connected to '%s' SFTP server", self.conf['domain']) return sftp
def connect(self): opts = CnOpts() opts.hostkeys = None try: conn = sftp.Connection(self.hostname, username=self.username, password=self.password, cnopts=opts) except ConnectionError: raise SystemExit("Connection error") self.backup_files_network(self.src_dir, conn)
def test_download(ftp_server1, file_tree1, tmp_path, local, remote, name, arch, skip): host = '127.0.0.1' port = ftp_server1.port cnopts = CnOpts() cnopts.hostkeys = None ft = loader.FileTransfer(name, tmp_path / local, remote, arch) with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn: ft.download(conn, skip_existing=skip) filename = ft._arch_name mode = 'rb' if arch else 'r' with open(tmp_path / local / filename, mode) as f: data = f.read() assert data == ftp_server1.content_provider.get(remote + '/' + filename)
def test_upload2(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip, content): host = '127.0.0.1' port = ftp_server3.port cnopts = CnOpts() cnopts.hostkeys = None ft = loader.FileTransfer(name, tmp_path / local, remote, arch) with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn: ft.upload(conn, skip_existing=skip) filename = ft._arch_name data = ftp_server3.content_provider.get(remote + '/' + filename) if arch: data = loader.get_archivator(arch).decompress(data) assert data.decode() == content
def ftp_upload(dir_filename_list): "Uploads a list of (dir, filename)" # Required to fix pysftp bug cnopts = CnOpts() cnopts.hostkeys = None with Connection( host=b64decode(config["ftp"]["host"]).decode("utf-8"), username=b64decode(config["ftp"]["user"]).decode("utf-8"), password=b64decode(config["ftp"]["password"]).decode("utf-8"), port=int(b64decode(config["ftp"]["port"]).decode("utf-8")), cnopts=cnopts, ) as sftp: logger.info("SFTP connection OK") for dir_filename in dir_filename_list: file_path = join(dir_filename[0], dir_filename[1]) logger.info("Uploading %s..." % (file_path)) sftp.put(file_path, f"{config['dir']['remote']}/{dir_filename[1]}") logger.info("FTP uploaded successfully: %s" % (file_path))
def _connect() -> Connection: sftp_host: str = current_app.config.get('CAS_SFTP_HOST') cnopts = CnOpts() # only for local development set this to false . if current_app.config.get('SFTP_VERIFY_HOST').lower() == 'false': cnopts.hostkeys = None else: host_key = current_app.config.get('CAS_SFTP_HOST_KEY') ftp_host_key_data = current_app.config.get('CAS_SFTP_HOST_KEY').encode() key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data)) cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key) sftp_port: int = current_app.config.get('CAS_SFTP_PORT') sft_credentials = { 'username': current_app.config.get('CAS_SFTP_USER_NAME'), # private_key should be the absolute path to where private key file lies since sftp 'private_key': current_app.config.get('BCREG_FTP_PRIVATE_KEY_LOCATION'), 'private_key_pass': current_app.config.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE') } # to support local testing. SFTP CAS server should run in private key mode if password := current_app.config.get('CAS_SFTP_PASSWORD'): sft_credentials['password'] = password
from django.shortcuts import render, redirect from django.http import HttpResponse, HttpResponseRedirect from django.template import loader from django.urls import reverse from django.contrib.auth import authenticate, login, logout from django.contrib.auth.decorators import login_required from django.contrib.auth.forms import UserCreationForm from .forms import ConnectionForm from io import BytesIO from .models import Connection from pysftp import Connection as SftpConnection from pysftp import CnOpts, SSHException CNOPTS = CnOpts() CNOPTS.hostkeys = None def index(request): """ Used as main entry point. Propose of registration. Redirect if you already logged in. :param request: user enter on site. :return: 'all_connections' if user authenticated else 'index' page. """ if request.user.is_authenticated: return redirect('all_connections') return render(request, 'file_manager/index.html') def user_login(request):