def tearDown(self):
cleared = Tools.move_dir_to_trash_can(self.tmpdir)
if not cleared:
rmtree(self.tmpdir)
os.environ.pop(ENV.settings)
os.environ.pop("TMPDIR")
tempfile.tempdir = None
Database.reset()
SettingsManager.reset()
DebugSettingsManager.reset()
def tearDown(self):
#clearing repertories
cleared = Tools.move_dir_to_trash_can(self.tmpdir)
if not cleared:
rmtree(self.tmpdir)
SFTPserver.stop_server()
os.environ.pop(ENV.settings)
os.environ.pop("TMPDIR")
tempfile.tempdir = None
Database.reset()
SettingsManager.reset()
DebugSettingsManager.reset()
def tearDown(self):
cleared = Tools.move_dir_to_trash_can(self.tmpdir)
if not cleared:
rmtree(self.tmpdir)
# Stop sftp server
SFTPserver.stop_server()
os.environ.pop(ENV.settings)
os.environ.pop("TMPDIR")
tempfile.tempdir = None
Database.reset()
SettingsManager.reset()
HarnessTree.reset()
DebugSettingsManager.reset()
def sftp_dir(self, dir_path, destination_dir):
"""
Connect to a remote directory by sftp and list the files to download.
Then the size of the files are used to compute a global timeout and all
files are then downloaded.
Dissemination is a failure if any file is over the size limit or if the
staging post doesn't exist.
Otheriwse, if the timeout is hit, the function returns a failure to
download but there can be another attempt when the instruction file
gets reprocessed.
"""
files_to_sftp = []
try:
transport = paramiko.Transport((self.hostname, self.port))
transport.connect(username=self.user, password=self.password)
sftp = paramiko.SFTPClient.from_transport(transport)
sftp.chdir(dir_path)
required_bandwith = 0
# for each file we update the files_to_sftp list.
# This list contains a tuple of 4 elements
# (staging post path, file_to_download_path, file_downloaded_path, skip)
# with skip = False if file has not been downloaded and True otherwise
# The reason for skip is that we still want to keep track of files
# that should have been retried for an instruction file, even if there was no need
# to download it a second time. Keeping track of those files allows
# to link them to their request id in the database
for item in sftp.listdir(dir_path):
file_path = os.path.join(dir_path, item)
destination_path = os.path.join(destination_dir, item)
# if the file has already been fetched by a previous instruction file,
# we don't do it again
if os.path.isfile(destination_path):
files_to_sftp.append((dir_path, file_path, destination_path, True))
LOGGER.debug("File %s already downloaded, moving on", file_path)
continue
mode = sftp.stat(file_path).st_mode
# ignore directories
if S_ISDIR(mode):
continue
# check for file_size
size = sftp.stat(file_path).st_size
max_size = SettingsManager.get("processFilesize")
# if size > max_size, diffusion failed.
# conversion in Mbytes with shift_expr << operator
if max_size is not None and size > max_size*(1 << 20):
raise FileOverSizeLimit
required_bandwith += size
# if a file, get it
LOGGER.debug('file %s found on openwis staging post',
file_path
)
destination_path = self.check_zip(item, destination_dir)
files_to_sftp.append((dir_path, file_path, destination_path, False))
sftp.close()
transport.close()
# initialize the multiprocessing manager
# this creates a pool of workers that automatically dispatch the
# downloads between them
nb_workers = SettingsManager.get("getSFTPlimitConn")
if DEBUG:
pool = DebugSettingsManager.sftp_pool(processes=nb_workers)
else:
pool = multiprocessing.Pool(processes=nb_workers)
# launch the pool of jobs
results = pool.starmap_async(self._sftp_file, files_to_sftp)
pool.close()
timeout = self.compute_timeout(required_bandwith)
nb_downloads = sum([not i[3] for i in files_to_sftp])
if nb_downloads == 0:
LOGGER.debug("No files to download or required files already downloaded once.")
sftp_success = True
else:
try:
LOGGER.debug("Attempting download of %i files, for a total size of "
" %f. Timeout is fixed at %s s.", nb_downloads,
required_bandwith, timeout)
start = time()
# activate the timeout. If the jobs are not finished by timeout,
# results.get will trigger a multiprocessing.TimeoutError
results.get(timeout=timeout)
delta_t = time() - start
LOGGER.debug("Files downloaded in %f seconds" % delta_t)
sftp_success = True
except multiprocessing.TimeoutError:
LOGGER.error(
"Timeout exceeded for fetching files on staging post.")
sftp_success = False
# check download success and update database
sftp_success = self.check_download_success(files_to_sftp, sftp_success)
sftp.close()
# case where the sftp interface (paramiko) returns an exception
except (paramiko.SSHException,
paramiko.ssh_exception.NoValidConnectionsError):
LOGGER.exception("Couldn't connect to %s", self.hostname)
sftp_success = False
# case where a file is too big
except FileOverSizeLimit:
msg = ('file %s found on openwis staging post'
'is over the size limit %f. Dissemination '
'failed' % (file_path, max_size))
LOGGER.exception(msg)
Database.update_field_by_query("requestStatus", REQ_STATUS.failed,
**dict(fullrequestId=self.req_id))
Database.update_field_by_query("message", msg,
**dict(fullrequestId=self.req_id))
sftp_success = False
# case where the staging post path is incorrect
except FileNotFoundError:
msg = ('Incorrect path %s for openwis staging post'
'Dissemination failed' % dir_path)
LOGGER.exception(msg)
Database.update_field_by_query("requestStatus", REQ_STATUS.failed,
**dict(fullrequestId=self.req_id))
Database.update_field_by_query("message", msg,
**dict(fullrequestId=self.req_id))
sftp_success = False
# update database
# The database is updated with as many new entries as there were files
# downloaded
files_downloaded = self.update(sftp_success, files_to_sftp)
return sftp_success, files_downloaded
from functools import partial
from lxml import etree
import paramiko
from setproctitle import setproctitle
from settings.settings_manager import SettingsManager, DebugSettingsManager
from utils.log_setup import setup_logging
from utils.setup_tree import HarnessTree
from utils.database import Database, Diffusion
from utils.const import (REQ_STATUS, TIMEOUT_BUFFER, DEBUG_TIMEOUT, TIMEOUT,
PRIORITIES, MAX_REGEX, ENV)
from utils.tools import Tools, Incrementator
from webservice.server.application import APP
try:
DEBUG = strtobool(os.environ.get(ENV.debug) or DebugSettingsManager.get("debug"))
except ValueError:
DEBUG = False
try:
TEST_SFTP = strtobool(os.environ.get("MFSERV_HARNESS_TEST_SFTP") or DebugSettingsManager.get("test_sftp"))
except ValueError:
TEST_SFTP = False
# initialize LOGGER
LOGGER = logging.getLogger(__name__)
LOGGER.debug("Logging configuration set up for %s", __name__)
class FileOverSizeLimit(Exception):
pass
class FileManager:
from time import sleep, time
from re import match
from setproctitle import setproctitle
from settings.settings_manager import DebugSettingsManager, SettingsManager
from utils.const import DEBUG_TIMEOUT, ENV, TIMEOUT, TIMEOUT_BUFFER
from utils.log_setup import setup_logging
from utils.setup_tree import HarnessTree
from utils.tools import Tools
# initialize LOGGER
LOGGER = logging.getLogger(__name__)
LOGGER.debug("Logging configuration set up for %s", __name__)
try:
DEBUG = strtobool(
os.environ.get(ENV.debug) or DebugSettingsManager.get("debug"))
except ValueError:
DEBUG = False
class DifmetSender:
_running = False
dir_c = None
dir_d = None
nb_workers = None
@classmethod
def process(cls, max_loops=0):
cls.nb_workers = SettingsManager.get("sendFTPlimitConn")
# in debug mode, it is possible to set