def _connect(server_name: str) -> Connection: sftp_configs = current_app.config.get('SFTP_CONFIGS') # if not passed , connect to CAS server always. to make the existing code work if not server_name or server_name not in sftp_configs.keys(): server_name = SFTPService.DEFAUILT_CONNECT_SERVER connect_configs = sftp_configs.get(server_name) sftp_host: str = connect_configs.get('SFTP_HOST') cnopts = CnOpts() # only for local development set this to false . if connect_configs.get('SFTP_VERIFY_HOST').lower() == 'false': cnopts.hostkeys = None else: ftp_host_key_data = connect_configs.get('SFTP_HOST_KEY').encode() key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data)) cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key) sftp_port: int = connect_configs.get('SFTP_PORT') sft_credentials = { 'username': connect_configs.get('SFTP_USERNAME'), # private_key should be the absolute path to where private key file lies since sftp 'private_key': connect_configs.get('FTP_PRIVATE_KEY_LOCATION'), 'private_key_pass': connect_configs.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE') } # to support local testing. SFTP CAS server should run in private key mode if password := connect_configs.get('SFTP_PASSWORD'): sft_credentials['password'] = password
def download_new_files(last_date: typing.Optional[datetime.date]): new_dates = [] new_filenames = [] opts = CnOpts() opts.hostkeys = None with Connection(settings.SFTP_HOST, username=settings.SFTP_USER, private_key=settings.SFTP_PRIVATE_KEY, cnopts=opts) as conn: with conn.cd(settings.SFTP_DIR): dir_listing = conn.listdir() for filename in dir_listing: date = parse_filename(filename, settings.ACCOUNT_CODE) if date: stat = conn.stat(filename) if stat.st_size > SIZE_LIMIT_BYTES: logger.error('%s is too large (%s), download skipped.' % (filename, stat.st_size)) continue if last_date is None or date > last_date: local_path = os.path.join(settings.DS_NEW_FILES_DIR, filename) new_filenames.append(local_path) new_dates.append(date) conn.get(filename, localpath=local_path) if new_dates and new_filenames: sorted_dates, sorted_files = zip(*sorted(zip(new_dates, new_filenames))) return NewFiles(list(sorted_dates), list(sorted_files)) else: return NewFiles([], [])
def test_download2(ftp_server1, config1, file_tree1, tmp_path, download_cnt, skip): url, file_trans = read_config(tmp_path / 'ftp-config.toml') port = ftp_server1.port cnopts = CnOpts() cnopts.hostkeys = None downloads = download_data(url, 'user1', '1234', file_trans, skip, port=port, cnopts=cnopts) assert len(downloads) == download_cnt
def __init__(self, host, username=None, password=None, port=22, private_key_file=None, private_key_password=None, compression=True): self._con = None # connection options cnopts = CnOpts() cnopts.compression = compression # set hostkeys to None, if not provided if private_key_file is None: cnopts.hostkeys = None if password is None: logging.debug('No password provided, using key auth.') # NOTE: # Ducking exceptions, so that they can be handled # by the main module however it wants. self._con = Connection(host=host, username=username, port=port, private_key_file=private_key_file, # Ignore LineLengthBear, PycodeStyleBear private_key_password=private_key_password, cnopts=cnopts) if self._con is None: self._con = Connection(host, username=username, port=port, password=password, cnopts=cnopts)
def test_upload_raise(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip): host = '127.0.0.1' port = ftp_server3.port cnopts = CnOpts() cnopts.hostkeys = None ft = loader.FileTransfer(name, tmp_path / local, remote, arch) with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn: with pytest.raises(loader.LoaderException): ft.upload(conn, skip_existing=skip)
def __init__(self, download_folder, **kwargs): cnopts = CnOpts() cnopts.hostkeys = None self._sftp = Connection(host=kwargs["host"], username=kwargs["username"], password=kwargs["password"], cnopts=cnopts) print("File server connection established...") self._folder = kwargs["folder"] self._download_folder = download_folder
def connect(self): opts = CnOpts() opts.hostkeys = None try: conn = sftp.Connection(self.hostname, username=self.username, password=self.password, cnopts=opts) except ConnectionError: raise SystemExit("Connection error") self.backup_files_network(self.src_dir, conn)
def get_client(self): cnopts = CnOpts() cnopts.hostkeys = None sftp = Connection(self.conf['domain'], username=self.conf['user'], password=self.conf['password'], cnopts=cnopts, default_path=self.conf['remote_path']) LOGGER.info("Successfully connected to '%s' SFTP server", self.conf['domain']) return sftp
def test_download(ftp_server1, file_tree1, tmp_path, local, remote, name, arch, skip): host = '127.0.0.1' port = ftp_server1.port cnopts = CnOpts() cnopts.hostkeys = None ft = loader.FileTransfer(name, tmp_path / local, remote, arch) with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn: ft.download(conn, skip_existing=skip) filename = ft._arch_name mode = 'rb' if arch else 'r' with open(tmp_path / local / filename, mode) as f: data = f.read() assert data == ftp_server1.content_provider.get(remote + '/' + filename)
def test_upload2(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip, content): host = '127.0.0.1' port = ftp_server3.port cnopts = CnOpts() cnopts.hostkeys = None ft = loader.FileTransfer(name, tmp_path / local, remote, arch) with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn: ft.upload(conn, skip_existing=skip) filename = ft._arch_name data = ftp_server3.content_provider.get(remote + '/' + filename) if arch: data = loader.get_archivator(arch).decompress(data) assert data.decode() == content
def test_upload_from_sftp_handles_partial_uploads( self, _mock_upload_controller: mock.MagicMock, _mock_download_controller: mock.MagicMock, mock_download_delegate_factory: mock.MagicMock, mock_sftp_auth: mock.MagicMock, mock_environment: mock.MagicMock, ) -> None: region_code = "us_xx" mock_environment.return_value = "staging" request_args = {"region": region_code, "date": "2021-01-01"} headers = {"X-Appengine-Cron": "test-cron"} mock_download_delegate_factory.return_value = Mock( spec=BaseSftpDownloadDelegate, root_directory=lambda _, candidate_paths: ".", filter_paths=lambda _, candidate_paths: candidate_paths, post_process_downloads=lambda _, download_directory_path: None, ) mock_sftp_auth.return_value = SftpAuth("host", "username", "password", CnOpts()) response = self.client.post("/upload_from_sftp", query_string=request_args, headers=headers) self.assertEqual(HTTPStatus.MULTI_STATUS, response.status_code)
def __init__(self, sftp_server_config): self.server = sftp_server_config['host_address'] self.username = sftp_server_config['user_name'] self.private_key = sftp_server_config['key_path'] self.sftp_folder = sftp_server_config['sftp_folder'] self.connection_opts = CnOpts() self.connection_opts.hostkeys = None self.__connection: Connection = None
def __init__(self, server, username, prefix, password=None, rsa_path=None): CloudBackend.__init__(self, prefix) self.server = server self.username = username self.password = password self.prefix = prefix self.rsa_path = rsa_path self.cnopts = CnOpts() self.cnopts.hostkeys = None
def ftp_upload(dir_filename_list): "Uploads a list of (dir, filename)" # Required to fix pysftp bug cnopts = CnOpts() cnopts.hostkeys = None with Connection( host=b64decode(config["ftp"]["host"]).decode("utf-8"), username=b64decode(config["ftp"]["user"]).decode("utf-8"), password=b64decode(config["ftp"]["password"]).decode("utf-8"), port=int(b64decode(config["ftp"]["port"]).decode("utf-8")), cnopts=cnopts, ) as sftp: logger.info("SFTP connection OK") for dir_filename in dir_filename_list: file_path = join(dir_filename[0], dir_filename[1]) logger.info("Uploading %s..." % (file_path)) sftp.put(file_path, f"{config['dir']['remote']}/{dir_filename[1]}") logger.info("FTP uploaded successfully: %s" % (file_path))
def set_up_connection_options(prefix: str, host: str) -> CnOpts: connection_options = CnOpts() try: connection_options.get_hostkey(host) except SSHException as s: hostkey = secrets.get_secret(f"{prefix}_hostkey") if hostkey is None: raise ValueError( f"Unable to find hostkey for secret key {prefix}_hostkey" ) from s hostkeyEntry = HostKeyEntry.from_line(hostkey) if hostkeyEntry: key = hostkeyEntry.key name, keytype, _ = hostkey.split(" ") connection_options.hostkeys.add(name, keytype, key) else: raise ValueError( f"Unable to add hostkey to connection_options for secret key {prefix}_hostkey" ) from s return connection_options
def __init__( self, hostname: str, username: Optional[str], password: Optional[str], connection_options: Optional[CnOpts], ): self.hostname = hostname self.username = username self.password = password # connection_options hold advanced options that help when creating a pysftp Connection # such as overriding where to check for hosts or enabling compression. self.connection_options = connection_options if connection_options else CnOpts()
def _connect() -> Connection: sftp_host: str = current_app.config.get('CAS_SFTP_HOST') cnopts = CnOpts() # only for local development set this to false . if current_app.config.get('SFTP_VERIFY_HOST').lower() == 'false': cnopts.hostkeys = None else: host_key = current_app.config.get('CAS_SFTP_HOST_KEY') ftp_host_key_data = current_app.config.get('CAS_SFTP_HOST_KEY').encode() key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data)) cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key) sftp_port: int = current_app.config.get('CAS_SFTP_PORT') sft_credentials = { 'username': current_app.config.get('CAS_SFTP_USER_NAME'), # private_key should be the absolute path to where private key file lies since sftp 'private_key': current_app.config.get('BCREG_FTP_PRIVATE_KEY_LOCATION'), 'private_key_pass': current_app.config.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE') } # to support local testing. SFTP CAS server should run in private key mode if password := current_app.config.get('CAS_SFTP_PASSWORD'): sft_credentials['password'] = password
from django.shortcuts import render, redirect from django.http import HttpResponse, HttpResponseRedirect from django.template import loader from django.urls import reverse from django.contrib.auth import authenticate, login, logout from django.contrib.auth.decorators import login_required from django.contrib.auth.forms import UserCreationForm from .forms import ConnectionForm from io import BytesIO from .models import Connection from pysftp import Connection as SftpConnection from pysftp import CnOpts, SSHException CNOPTS = CnOpts() CNOPTS.hostkeys = None def index(request): """ Used as main entry point. Propose of registration. Redirect if you already logged in. :param request: user enter on site. :return: 'all_connections' if user authenticated else 'index' page. """ if request.user.is_authenticated: return redirect('all_connections') return render(request, 'file_manager/index.html') def user_login(request):
def conn(sftpsrv): """return a dictionary holding argument info for the pysftp client""" cnopts = CnOpts() cnopts.hostkeys.load('sftpserver.pub') return {'host': sftpsrv.host, 'port': sftpsrv.port, 'username': '******', 'password': '******', 'default_path': '/home/test', 'cnopts': cnopts}
class EEXSpecs: """ Defines relevant information related to the downloading process EEX' sFTP server. Does not include Note ---- Example Abbreviations: Power Future History Austria Derivatives Contained Data: ATBM EEX Austrian Power Base Month Future ATBQ EEX Austrian Power Base Quarter Future ATBY EEX Austrian Power Base Year Future ATPM EEX Austrian Power Peak Month Future ATPQ EEX Austrian Power Peak Quarter Future ATPY EEX Austrian Power Peak Year Future See --- https://www.eex.com/fileadmin/EEX/Downloads/Market_Data/EEX_Group_DataSource/sFTP_Server/EEX_Group_DataSource_SFTP_XLSX_Interface_Specification_v008_09_09_2020.pdf, Page 19 """ base_path_power_futures: Path = "/market_data/power/at/derivatives/csv" # TODO: Get ssh-pk from config via paramiko connection_options: CnOpts = CnOpts( knownhosts=Path(__file__).resolve().parent / "sshpk.txt" ) countries: list = field(default_factory=lambda: ["AT", "DE", "FR", "NL", "BE"]) contained_data_contracts: list = field( default_factory=lambda: ["BM", "PM", "BQ", "PQ", "BY", "PY"] ) contract_country_codes: dict = field( default_factory=lambda: { "FR": "F7", "AT": "AT", "DE": "DE", "BE": "Q1", "NL": "Q0", } ) def create_filename(date: datetime.date, country: str): return "PowerFutureResults_{}_{}.csv".format(country, date) def get_contained_data_codes_per_country(self, country: str): """ Returns the cartesian product of a single a country and all considered EEX contract types Return ------ TODO: Connect tuple items specs = EEXSpecs() specs.get_encoded_contracts_per_country("BE")) >> [('Q1', 'BM'), ('Q1', 'PM'), ('Q1', 'BQ'), ('Q1', 'PQ'), ('Q1', 'BY'), ('Q1', 'PY')] """ return list(product([self.contract_country_codes[country]], self.contracts))