def test_upload_from_sftp_handles_partial_uploads(
self,
_mock_upload_controller: mock.MagicMock,
_mock_download_controller: mock.MagicMock,
mock_download_delegate_factory: mock.MagicMock,
mock_sftp_auth: mock.MagicMock,
mock_environment: mock.MagicMock,
) -> None:
region_code = "us_xx"
mock_environment.return_value = "staging"
request_args = {"region": region_code, "date": "2021-01-01"}
headers = {"X-Appengine-Cron": "test-cron"}
mock_download_delegate_factory.return_value = Mock(
spec=BaseSftpDownloadDelegate,
root_directory=lambda _, candidate_paths: ".",
filter_paths=lambda _, candidate_paths: candidate_paths,
post_process_downloads=lambda _, download_directory_path: None,
)
mock_sftp_auth.return_value = SftpAuth("host", "username", "password",
CnOpts())
response = self.client.post("/upload_from_sftp",
query_string=request_args,
headers=headers)
self.assertEqual(HTTPStatus.MULTI_STATUS, response.status_code)
def download_new_files(last_date: typing.Optional[datetime.date]):
new_dates = []
new_filenames = []
opts = CnOpts()
opts.hostkeys = None
with Connection(settings.SFTP_HOST, username=settings.SFTP_USER,
private_key=settings.SFTP_PRIVATE_KEY, cnopts=opts) as conn:
with conn.cd(settings.SFTP_DIR):
dir_listing = conn.listdir()
for filename in dir_listing:
date = parse_filename(filename, settings.ACCOUNT_CODE)
if date:
stat = conn.stat(filename)
if stat.st_size > SIZE_LIMIT_BYTES:
logger.error('%s is too large (%s), download skipped.'
% (filename, stat.st_size))
continue
if last_date is None or date > last_date:
local_path = os.path.join(settings.DS_NEW_FILES_DIR,
filename)
new_filenames.append(local_path)
new_dates.append(date)
conn.get(filename, localpath=local_path)
if new_dates and new_filenames:
sorted_dates, sorted_files = zip(*sorted(zip(new_dates, new_filenames)))
return NewFiles(list(sorted_dates), list(sorted_files))
else:
return NewFiles([], [])
def _connect(server_name: str) -> Connection:
sftp_configs = current_app.config.get('SFTP_CONFIGS')
# if not passed , connect to CAS server always. to make the existing code work
if not server_name or server_name not in sftp_configs.keys():
server_name = SFTPService.DEFAUILT_CONNECT_SERVER
connect_configs = sftp_configs.get(server_name)
sftp_host: str = connect_configs.get('SFTP_HOST')
cnopts = CnOpts()
# only for local development set this to false .
if connect_configs.get('SFTP_VERIFY_HOST').lower() == 'false':
cnopts.hostkeys = None
else:
ftp_host_key_data = connect_configs.get('SFTP_HOST_KEY').encode()
key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data))
cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key)
sftp_port: int = connect_configs.get('SFTP_PORT')
sft_credentials = {
'username':
connect_configs.get('SFTP_USERNAME'),
# private_key should be the absolute path to where private key file lies since sftp
'private_key':
connect_configs.get('FTP_PRIVATE_KEY_LOCATION'),
'private_key_pass':
connect_configs.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE')
}
# to support local testing. SFTP CAS server should run in private key mode
if password := connect_configs.get('SFTP_PASSWORD'):
sft_credentials['password'] = password
def test_download2(ftp_server1, config1, file_tree1, tmp_path, download_cnt, skip):
url, file_trans = read_config(tmp_path / 'ftp-config.toml')
port = ftp_server1.port
cnopts = CnOpts()
cnopts.hostkeys = None
downloads = download_data(url, 'user1', '1234', file_trans, skip, port=port, cnopts=cnopts)
assert len(downloads) == download_cnt
def __init__(self, host, username=None, password=None, port=22,
private_key_file=None, private_key_password=None,
compression=True):
self._con = None
# connection options
cnopts = CnOpts()
cnopts.compression = compression
# set hostkeys to None, if not provided
if private_key_file is None:
cnopts.hostkeys = None
if password is None:
logging.debug('No password provided, using key auth.')
# NOTE:
# Ducking exceptions, so that they can be handled
# by the main module however it wants.
self._con = Connection(host=host, username=username,
port=port,
private_key_file=private_key_file,
# Ignore LineLengthBear, PycodeStyleBear
private_key_password=private_key_password,
cnopts=cnopts)
if self._con is None:
self._con = Connection(host, username=username, port=port,
password=password, cnopts=cnopts)
def __init__(self, sftp_server_config):
self.server = sftp_server_config['host_address']
self.username = sftp_server_config['user_name']
self.private_key = sftp_server_config['key_path']
self.sftp_folder = sftp_server_config['sftp_folder']
self.connection_opts = CnOpts()
self.connection_opts.hostkeys = None
self.__connection: Connection = None
def test_upload_raise(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip):
host = '127.0.0.1'
port = ftp_server3.port
cnopts = CnOpts()
cnopts.hostkeys = None
ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
with pytest.raises(loader.LoaderException):
ft.upload(conn, skip_existing=skip)
def __init__(self, server, username, prefix, password=None, rsa_path=None):
CloudBackend.__init__(self, prefix)
self.server = server
self.username = username
self.password = password
self.prefix = prefix
self.rsa_path = rsa_path
self.cnopts = CnOpts()
self.cnopts.hostkeys = None
def __init__(self, download_folder, **kwargs):
cnopts = CnOpts()
cnopts.hostkeys = None
self._sftp = Connection(host=kwargs["host"],
username=kwargs["username"],
password=kwargs["password"],
cnopts=cnopts)
print("File server connection established...")
self._folder = kwargs["folder"]
self._download_folder = download_folder
def connect(self):
opts = CnOpts()
opts.hostkeys = None
try:
conn = sftp.Connection(self.hostname,
username=self.username,
password=self.password,
cnopts=opts)
except ConnectionError:
raise SystemExit("Connection error")
self.backup_files_network(self.src_dir, conn)
def get_client(self):
cnopts = CnOpts()
cnopts.hostkeys = None
sftp = Connection(self.conf['domain'],
username=self.conf['user'],
password=self.conf['password'],
cnopts=cnopts,
default_path=self.conf['remote_path'])
LOGGER.info("Successfully connected to '%s' SFTP server",
self.conf['domain'])
return sftp
def test_download(ftp_server1, file_tree1, tmp_path, local, remote, name, arch, skip):
host = '127.0.0.1'
port = ftp_server1.port
cnopts = CnOpts()
cnopts.hostkeys = None
ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
ft.download(conn, skip_existing=skip)
filename = ft._arch_name
mode = 'rb' if arch else 'r'
with open(tmp_path / local / filename, mode) as f:
data = f.read()
assert data == ftp_server1.content_provider.get(remote + '/' + filename)
def test_upload2(ftp_server3, file_tree2, tmp_path, local, remote, name, arch, skip, content):
host = '127.0.0.1'
port = ftp_server3.port
cnopts = CnOpts()
cnopts.hostkeys = None
ft = loader.FileTransfer(name, tmp_path / local, remote, arch)
with Connection(host, port=port, username='******', password='******', cnopts=cnopts) as conn:
ft.upload(conn, skip_existing=skip)
filename = ft._arch_name
data = ftp_server3.content_provider.get(remote + '/' + filename)
if arch:
data = loader.get_archivator(arch).decompress(data)
assert data.decode() == content
def __init__(
self,
hostname: str,
username: Optional[str],
password: Optional[str],
connection_options: Optional[CnOpts],
):
self.hostname = hostname
self.username = username
self.password = password
# connection_options hold advanced options that help when creating a pysftp Connection
# such as overriding where to check for hosts or enabling compression.
self.connection_options = connection_options if connection_options else CnOpts()
def ftp_upload(dir_filename_list):
"Uploads a list of (dir, filename)"
# Required to fix pysftp bug
cnopts = CnOpts()
cnopts.hostkeys = None
with Connection(
host=b64decode(config["ftp"]["host"]).decode("utf-8"),
username=b64decode(config["ftp"]["user"]).decode("utf-8"),
password=b64decode(config["ftp"]["password"]).decode("utf-8"),
port=int(b64decode(config["ftp"]["port"]).decode("utf-8")),
cnopts=cnopts,
) as sftp:
logger.info("SFTP connection OK")
for dir_filename in dir_filename_list:
file_path = join(dir_filename[0], dir_filename[1])
logger.info("Uploading %s..." % (file_path))
sftp.put(file_path, f"{config['dir']['remote']}/{dir_filename[1]}")
logger.info("FTP uploaded successfully: %s" % (file_path))
def set_up_connection_options(prefix: str, host: str) -> CnOpts:
connection_options = CnOpts()
try:
connection_options.get_hostkey(host)
except SSHException as s:
hostkey = secrets.get_secret(f"{prefix}_hostkey")
if hostkey is None:
raise ValueError(
f"Unable to find hostkey for secret key {prefix}_hostkey"
) from s
hostkeyEntry = HostKeyEntry.from_line(hostkey)
if hostkeyEntry:
key = hostkeyEntry.key
name, keytype, _ = hostkey.split(" ")
connection_options.hostkeys.add(name, keytype, key)
else:
raise ValueError(
f"Unable to add hostkey to connection_options for secret key {prefix}_hostkey"
) from s
return connection_options
def _connect() -> Connection:
sftp_host: str = current_app.config.get('CAS_SFTP_HOST')
cnopts = CnOpts()
# only for local development set this to false .
if current_app.config.get('SFTP_VERIFY_HOST').lower() == 'false':
cnopts.hostkeys = None
else:
host_key = current_app.config.get('CAS_SFTP_HOST_KEY')
ftp_host_key_data = current_app.config.get('CAS_SFTP_HOST_KEY').encode()
key = paramiko.RSAKey(data=decodebytes(ftp_host_key_data))
cnopts.hostkeys.add(sftp_host, 'ssh-rsa', key)
sftp_port: int = current_app.config.get('CAS_SFTP_PORT')
sft_credentials = {
'username': current_app.config.get('CAS_SFTP_USER_NAME'),
# private_key should be the absolute path to where private key file lies since sftp
'private_key': current_app.config.get('BCREG_FTP_PRIVATE_KEY_LOCATION'),
'private_key_pass': current_app.config.get('BCREG_FTP_PRIVATE_KEY_PASSPHRASE')
}
# to support local testing. SFTP CAS server should run in private key mode
if password := current_app.config.get('CAS_SFTP_PASSWORD'):
sft_credentials['password'] = password
from django.shortcuts import render, redirect
from django.http import HttpResponse, HttpResponseRedirect
from django.template import loader
from django.urls import reverse
from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import UserCreationForm
from .forms import ConnectionForm
from io import BytesIO
from .models import Connection
from pysftp import Connection as SftpConnection
from pysftp import CnOpts, SSHException
CNOPTS = CnOpts()
CNOPTS.hostkeys = None
def index(request):
"""
Used as main entry point. Propose of registration.
Redirect if you already logged in.
:param request: user enter on site.
:return: 'all_connections' if user authenticated else 'index' page.
"""
if request.user.is_authenticated:
return redirect('all_connections')
return render(request, 'file_manager/index.html')
def user_login(request):
def conn(sftpsrv):
"""return a dictionary holding argument info for the pysftp client"""
cnopts = CnOpts()
cnopts.hostkeys.load('sftpserver.pub')
return {'host': sftpsrv.host, 'port': sftpsrv.port, 'username': '******',
'password': '******', 'default_path': '/home/test', 'cnopts': cnopts}
class EEXSpecs:
"""
Defines relevant information related to the downloading process EEX' sFTP server. Does not include
Note
----
Example Abbreviations: Power Future History Austria
Derivatives Contained Data:
ATBM EEX Austrian Power Base Month Future
ATBQ EEX Austrian Power Base Quarter Future
ATBY EEX Austrian Power Base Year Future
ATPM EEX Austrian Power Peak Month Future
ATPQ EEX Austrian Power Peak Quarter Future
ATPY EEX Austrian Power Peak Year Future
See
---
https://www.eex.com/fileadmin/EEX/Downloads/Market_Data/EEX_Group_DataSource/sFTP_Server/EEX_Group_DataSource_SFTP_XLSX_Interface_Specification_v008_09_09_2020.pdf, Page 19
"""
base_path_power_futures: Path = "/market_data/power/at/derivatives/csv"
# TODO: Get ssh-pk from config via paramiko
connection_options: CnOpts = CnOpts(
knownhosts=Path(__file__).resolve().parent / "sshpk.txt"
)
countries: list = field(default_factory=lambda: ["AT", "DE", "FR", "NL", "BE"])
contained_data_contracts: list = field(
default_factory=lambda: ["BM", "PM", "BQ", "PQ", "BY", "PY"]
)
contract_country_codes: dict = field(
default_factory=lambda: {
"FR": "F7",
"AT": "AT",
"DE": "DE",
"BE": "Q1",
"NL": "Q0",
}
)
def create_filename(date: datetime.date, country: str):
return "PowerFutureResults_{}_{}.csv".format(country, date)
def get_contained_data_codes_per_country(self, country: str):
"""
Returns the cartesian product of a single a country and all considered EEX contract types
Return
------
TODO: Connect tuple items
specs = EEXSpecs()
specs.get_encoded_contracts_per_country("BE"))
>> [('Q1', 'BM'), ('Q1', 'PM'), ('Q1', 'BQ'), ('Q1', 'PQ'), ('Q1', 'BY'), ('Q1', 'PY')]
"""
return list(product([self.contract_country_codes[country]], self.contracts))