def __init__(self, req_id, uri, diss_info, client_ip):
self.req_id = req_id
self.uri = uri
self.diss_info = diss_info
self.date_reception = gmtime()
self.request_file = ""
self._diff_externalid = None
self.hostname = self.get_hostname(client_ip)
#load settings, if it has not been done
if not SettingsManager.is_loaded():
SettingsManager.load_settings()
# initialize LOGGER
setup_logging()
# setup repertory structure
HarnessTree.setup_tree()
# Setting up database if necessary
if Database.get_database() is None:
from webservice.server.application import APP
Database.initialize_database(APP)
LOGGER.debug("Database setup")
LOGGER.debug("Created a Notification object with id %s", req_id)
def setup_tree(cls, update=False):
if update:
LOGGER.info("Settings modified, performing a check on the "
"repertories tree structure, updating it if necessary")
if not SettingsManager.is_loaded():
error_msg = ("Attempting to setup repertories "
"tree structure before the SettingsManager "
"has been loaded")
LOGGER.error(error_msg)
raise RuntimeError(error_msg)
def setup_repertory(dir_path):
if not os.path.isdir(dir_path):
try:
os.makedirs(dir_path, exist_ok=True)
LOGGER.debug(
"Repertory %s created recursively.", dir_path)
except Exception as msg_error:
LOGGER.exception(
"Couldn't create repertory %s.",dir_path)
return dir_path
if DEBUG:
tempfile.gettempdir()
tempfile.tempdir = None
harnais_dir = os.path.join(tempfile.gettempdir(), "harnais")
else:
harnais_dir = SettingsManager.get("harnaisDir")
# repertory of temporary dissRequest JSON files
cls._repertories["temp_dissRequest_A"] = setup_repertory(
join(harnais_dir,
REPERTORY_TREE.temp_dissrequest_a)
)
cls._repertories["temp_dissRequest_B"] = setup_repertory(
join(harnais_dir,
REPERTORY_TREE.temp_dissrequest_b)
)
cls._repertories["temp_dissRequest_C"] = setup_repertory(
join(harnais_dir,
REPERTORY_TREE.temp_dissrequest_c)
)
cls._repertories["temp_dissRequest_D"] = setup_repertory(
join(harnais_dir,
REPERTORY_TREE.temp_dissrequest_d)
)
cls._repertories["dir_ack"] = dir_ack = SettingsManager.get("harnaisAckDir")
if not os.path.isdir(cls._repertories["dir_ack"]):
LOGGER.error("Ack repertory %s does "
"not exist", dir_ack)
# storing the settings file signature
cls._checksum = SettingsManager.get_checksum()
def rename(self):
"""
Rename files using user specified regex
"""
# Extract all the regex specified by users.
# Each regex should be a dictionnary with two keys,
# pattern_in and pattern_out for the substitution.
# The program can extract up to MAX_REGEX from the settings
# like this in the settings.yaml :
# fileRegex1:
# pattern_in:
# pattern_out:
# fileRegex2:
# pattern_in:
# pattern_out:
# ...
# fileRegex<MAX_REGEX>:
# pattern_in:
# pattern_out:
# Each set of regex is applied one after the other.
# Except for tmpregex that has only one value corresponding to
# pattern_out
regex_settings = [SettingsManager.get(
"fileRegex%i" % i, {}) for i in range(1, MAX_REGEX+1)]
new_filename = self.original_filename
zip_detector = r"^tmp\.zip\..*"
# we check if the file is a zipped one
# by looking if it matches r"^tmp\.zip\..*"
if re.match(zip_detector, new_filename) is not None:
zip_regex = SettingsManager.get("tmpregex")
if zip_regex is None:
LOGGER.error("No regex defined in tmpregex settings !")
else:
new_filename = self._rename_by_regex(new_filename, zip_detector, zip_regex)
else:
for idx, regex_instruction in enumerate(regex_settings):
reg = regex_instruction.get("pattern_in", None)
repl = regex_instruction.get("pattern_out", None)
if None not in (reg, repl):
new_filename = self._rename_by_regex(new_filename, reg, repl)
elif idx == 0:
LOGGER.error("No regex defined in fileregex1 settings !")
# we check that the file renamed fits difmet standards
filename_ok = self.check_filename(new_filename)
if filename_ok:
# update record with new filename
Database.update_field_by_query("final_file", new_filename,
**dict(original_file=self.original_filename))
self.new_filename = new_filename
# we return if the process returned a fit new filename or not
return filename_ok
def tearDown(self):
cleared = Tools.move_dir_to_trash_can(self.tmpdir)
if not cleared:
rmtree(self.tmpdir)
os.environ.pop(ENV.settings)
os.environ.pop("TMPDIR")
tempfile.tempdir = None
Database.reset()
SettingsManager.reset()
def __init__(self, req_id, hostname):
# path to file on staging post
self.staging_path = SettingsManager.get("openwisStagingPath") or ""
self.hostname = SettingsManager.get("openwisHost") or hostname
self.user = SettingsManager.get("openwisSftpUser") or None
self.password = SettingsManager.get("openwisSftpPassword") or None
self.port = SettingsManager.get("openwisSftpPort") or None
self.req_id = req_id + hostname
def get_logger():
"""
Fetch logger
"""
SettingsManager.load_settings()
# initialize LOGGER
setup_logging()
logger = logging.getLogger(__name__)
logger.debug("Logging configuration set up for %s", __name__)
return logger
def tearDown(self):
#clearing repertories
cleared = Tools.move_dir_to_trash_can(self.tmpdir)
if not cleared:
rmtree(self.tmpdir)
SFTPserver.stop_server()
os.environ.pop(ENV.settings)
os.environ.pop("TMPDIR")
tempfile.tempdir = None
Database.reset()
SettingsManager.reset()
DebugSettingsManager.reset()
def tearDown(self):
cleared = Tools.move_dir_to_trash_can(self.tmpdir)
if not cleared:
rmtree(self.tmpdir)
# Stop sftp server
SFTPserver.stop_server()
os.environ.pop(ENV.settings)
os.environ.pop("TMPDIR")
tempfile.tempdir = None
Database.reset()
SettingsManager.reset()
HarnessTree.reset()
DebugSettingsManager.reset()
def tearDown(self):
#stopping FTP server
FTPserver.stop_server()
#clearing repertories
cleared = Tools.move_dir_to_trash_can(self.tmpdir)
if not cleared:
rmtree(self.tmpdir)
# cleaning up environment
os.environ.pop(ENV.settings)
os.environ.pop("TMPDIR")
tempfile.tempdir = None
SettingsManager.reset()
HarnessTree.reset()
def get(cls, key):
if not SettingsManager.is_loaded():
SettingsManager.load_settings()
LOGGER.info("Settings loaded")
#check if settings have been modified
if SettingsManager.get_checksum() != cls._checksum:
LOGGER.info("Settings have been modified")
cls.setup_tree(update=True)
elif not os.path.isdir(cls._repertories[key]):
cls.setup_tree(update=False)
return cls._repertories[key]
def __change_settings(self):
"""Changes the settings of the application by saving new values in the xml file."""
settings_form = SettingsForm(self.__main_form, self.__conditions)
conditions = settings_form.get_settings()
if conditions != None:
self.__conditions = conditions
# Writing the new settings on xml file
try:
SettingsManager.write_settings(conditions)
except IOError as err:
messagebox.showerror(title="Error",
message=err,
parent=self.__main_form)
def process(cls, max_loops=0):
cls.nb_workers = SettingsManager.get("sendFTPlimitConn")
# in debug mode, it is possible to set
pool_method = cls.get_pool_method()
cls.pool = pool_method(processes=cls.nb_workers)
counter = 0
cls.setup_process()
while cls._running:
counter += 1
cls.signal_loop(counter)
cls.load_settings()
cls.update_workers()
# idle time
idle_time = SettingsManager.get("sendFTPIdle")
sleep(idle_time)
# get settings
cls.dir_c = dir_c = HarnessTree.get("temp_dissRequest_C")
cls.dir_d = dir_d = HarnessTree.get("temp_dissRequest_D")
# move back any remaining file from D to C
cls.move_back_files()
# get files in C
max_files = cls.nb_workers
list_files_c = cls.get_file_list(dir_c, max_files)
files_to_ftp = cls.move_files(list_files_c, dir_d)
for file_ in files_to_ftp:
file_expired = cls.check_file_age(file_)
if file_expired:
# TODO we need to find a way to update the info to the database
# would require looking at the file compressed though
Tools.remove_file(file_, "difmet archive", LOGGER)
continue
size = os.stat(file_).st_size
timeout = cls.compute_timeout(size, file_)
# start download
# renaming file to prevent any operation on it.
cls.lock_file(file_)
res = cls.pool.apply_async(
cls.abortable_ftp, (cls.upload_file, file_, dir_c, dir_d),
dict(timeout=timeout))
# for testing and debugging purpose only
cls.check_end_loop(counter, max_loops)
def _get_priority(self):
donot_compute_priority = SettingsManager.get("sla")
default_priority = SettingsManager.get("defaultPriority", PRIORITIES.default)
if donot_compute_priority:
highest_priority = default_priority
else:
priority_list = []
for req_id in self.instructions.keys():
priority = self._get_instr(req_id, "diffpriority")
priority_list.append(priority)
highest_priority = min(priority_list + [default_priority])
return highest_priority
def check_file_age(filename):
time_limit = SettingsManager.get("keepFileTimeSender") or None
if time_limit is not None:
check = (time() - os.stat(filename).st_mtime) > time_limit
else:
check = False
return check
def setup_logging(default_path=DEFAULT_PATH):
"""Setup logging configuration
"""
path = os.path.abspath(default_path)
if os.path.exists(path):
with open(path, 'rt') as file_:
config = yaml.safe_load(file_.read())
# use harnaislogdir settings if it has been set.
log_dir, dir_error_msg = _get_logdir()
if log_dir is not None:
if os.path.isdir(log_dir):
change_log_dir(config, log_dir)
else:
raise NotADirectoryError(dir_error_msg)
elif SettingsManager.is_loaded():
logging.warning(
"Logdir filed in settings_harnais.yaml has not been set.\n"
"Log files repertories will be those defined in the %s file",
default_path)
else:
error_msg = "Attempting to setup logs before the SettingsManager has been loaded"
logging.error(error_msg)
raise RuntimeError(error_msg)
logging.config.dictConfig(config)
else:
raise FileNotFoundError("Couldn't resolve logging configuration "
"file path {f}".format(f=path))
def setUp(self):
self.tmpdir = mkdtemp(prefix='harnais_')
harnais_dir = join(self.tmpdir, "harnais")
os.mkdir(harnais_dir)
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
self.difmet_deposit = join(self.tmpdir, "difmet_deposit")
os.mkdir(self.difmet_deposit)
self.ack_dir = join(self.tmpdir, "ack_dir")
os.mkdir(self.ack_dir)
self.hostname = hostname = socket.gethostname()
port = os.environ.get(ENV.port) or PORT
os.environ[ENV.soap_url] = ('http://{hostname}:{port}/harnais-diss-v2/'
'webservice/Dissemination?wsdl'.format(hostname=hostname,
port=port))
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=harnais_dir,
harnaisDir=harnais_dir,
harnaisAckDir=self.ack_dir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort = 3373,
processFileIdle = 10,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
def main():
SettingsManager.load_settings()
# initialize LOGGER
setup_logging()
# setup repertory structure
HarnessTree.setup_tree()
# Setting up database
Database.initialize_database(APP)
LOGGER.info("setup complete")
hostname = socket.gethostname()
port = os.environ.get(ENV.port) or PORT
LOGGER.warning("Starting application through Flask development server."
" This is NOT a production environment.")
LOGGER.info(
"Launching Flask development server "
"on hostname %s on port %s", hostname, port)
APP.run(host=hostname, port=port)
def update_workers(cls):
# update the number of workers if necessary
nbw = SettingsManager.get("sendFTPlimitConn")
if nbw != cls.nb_workers:
#wait for every upload to be finished
cls.pool.close()
cls.pool.join()
#update workers
cls.nb_workers = nbw
cls.pool = pool_method(processes=nbw)
def check_file_age(filename):
"""
Discard files that are too old
"""
time_limit = SettingsManager.get("keepFileTime") or None
if time_limit is not None:
check = (time() - os.stat(filename).st_mtime) > time_limit
else:
check = False
return check
def check_transfer(cls, filename, ftp):
# check if file exists on remote server
ftpdir = SettingsManager.get("dissFtpDir")
ftp.cwd(ftpdir)
if filename in [name for name, data in list(ftp.mlsd())]:
upload_ok = True
else:
upload_ok = False
ftp.quit()
return upload_ok
def compute_priority(priority, sla):
# priority is scaled from 1 (highest) to 4 (lowest)
# sla is 0 (BRONZE), 1 (SILVER), 2 (GOLD)
priority_activated = SettingsManager.get("sla")
if type(priority_activated) == str:
priority_activated = strtobool(priority_activated)
if priority_activated == False:
result = SettingsManager.get("defaultPriority") or PRIORITIES.default
elif priority ==1:
result = PRIORITIES.maximum
elif priority >=2:
default_priority = SettingsManager.get("defaultPriority") or PRIORITIES.default
result = default_priority + priority - 2*sla
result = max(PRIORITIES.maximum, result)
result = min(PRIORITIES.minimum, result)
return result
def _get_end_date():
limit = SettingsManager.get("fileEndLive") or 0
if limit ==0:
end_date = None
else:
# convert limit in minutes in seconds
end_date = datetime.utcnow() + timedelta(seconds=limit*60)
end_date = end_date.strftime("%Y-%m-%dT%H:%M:%SZ")
return end_date
def setUp(self):
# create repertories and configure harnesss
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir
), testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(dict(SettingsManager._parameters), file_)
SettingsManager.reset()
# configure SOAP server
self.hostname = hostname = socket.gethostname()
self.port = port = os.environ.get(ENV.port) or PORT
self.soap_url = os.environ[ENV.soap_url]= ('http://{hostname}:{port}/harnais-diss-v2/'
'webservice/Dissemination?wsdl'.format(hostname=hostname,
port=port))
SoapServer.create_server()
# connect to soap WSDL
self.client = Client(self.soap_url)
self.factory = self.client.type_factory('http://dissemination.harness.openwis.org/')
def setUp(self):
self.tmpdir = mkdtemp(prefix='harnais_')
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
# # prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(
harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
bandwidth=5,
),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
def upload_file(cls, file_, ftp):
upload_ok = False
start = time()
file_locked = file_ + ".lock"
with open(file_locked, 'rb') as file_transfered:
ftpdir = SettingsManager.get("dissFtpDir")
ftp.cwd(ftpdir)
# renaming in tmp
file_renamed = basename(file_) + ".tmp"
ftp.storbinary('STOR ' + file_renamed, file_transfered)
ftp.rename(file_renamed, basename(file_))
upload_ok = True
return upload_ok, time() - start
def get_file_list(cls, dirname, maxfiles):
overflow = SettingsManager.get("ManagerOverflow")
list_entries = os.listdir(dirname)
list_entries = [os.path.join(dirname, entry) for entry in list_entries]
list_files = [
entry for entry in list_entries if not os.path.isdir(entry)]
list_files.sort(key=lambda x: os.stat(x).st_mtime)
if overflow is not None and len(list_files) > overflow:
LOGGER.warning("%s repertory is overflowing. "
"Number of files %i over the limit %i",
cls.dir_a, len(list_files), overflow)
list_files = list_files[-maxfiles:]
return list_files
def setUp(self):
# configuring settings and repertories
DebugSettingsManager.debug = "False"
DebugSettingsManager.test_sftp = "True"
self.tmpdir = mkdtemp(prefix='harnais_')
harnais_dir = join(self.tmpdir, "harnais")
os.mkdir(harnais_dir)
os.environ["TMPDIR"] = self.tmpdir
self.staging_post = join(self.tmpdir, "staging_post")
os.mkdir(self.staging_post)
self.difmet_deposit = join(self.tmpdir, "difmet_deposit")
os.mkdir(self.difmet_deposit)
self.ack_dir = join(self.tmpdir, "ack_dir")
os.mkdir(self.ack_dir)
# create files on staging post
for i in range(2):
with open(join(self.staging_post,"A_SNFR30LFPW270700_C_LFPW_20180927070000_%i.txt" % i),"w") as file_out:
file_out.write("Dummy staging post test file")
# configure soap server
self.hostname = hostname = socket.gethostname()
self.port = port = os.environ.get(ENV.port) or PORT
self.soap_url = os.environ[ENV.soap_url]= ('http://{hostname}:{port}/harnais-diss-v2/'
'webservice/Dissemination?wsdl'.format(hostname=hostname,
port=port))
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=harnais_dir,
harnaisDir=harnais_dir,
harnaisAckDir=self.ack_dir,
openwisStagingPath=gettempdir(),
openwisHost="localhost",
openwisSftpUser="******",
openwisSftpPassword="******",
openwisSftpPort = 3373,
processFileIdle = 10,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(dict(SettingsManager._parameters), file_)
SettingsManager.reset()
def compute_timeout(required_bandwith):
# compute timeout by doing timeout = size/bandwidth + buffer
bandwidth = SettingsManager.get("bandwidth")
if bandwidth in [None, 0]:
LOGGER.warning("Incorrect value for harness settings bandwidth. "
"Sftp timeout set to default TIMEOUT %i s.",
TIMEOUT)
timeout = TIMEOUT
elif DEBUG:
timeout = DEBUG_TIMEOUT
LOGGER.debug("Sftp debug timeout set to %s s", timeout)
else:
# conversion in Mbits/s with shift_expr << operator
timeout = (required_bandwith/(1 << 17))/bandwidth + TIMEOUT_BUFFER
LOGGER.debug("Sftp timeout computed to %s s", timeout)
# start download
return timeout
def test_sending(self):
"""
Testing FTP transfer of 5 small files
"""
# prepare settings
SettingsManager.load_settings()
SettingsManager.update(dict(harnaisLogdir=self.tmpdir,
harnaisDir=self.tmpdir,
harnaisAckDir=self.tmpdir,
dissHost="0.0." + "0.0",
dissFtpUser="******",
dissFtpPasswd="12345",
bandwidth=10,
dissFtpDir=self.difmet_deposit,
dissFtpMode=None,
dissFtpPort=2121,
sendFTPlimitConn=5,
sendFTPIdle=10),
testing=True)
os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml")
with open(os.environ[ENV.settings], "w") as file_:
yaml.dump(SettingsManager._parameters, file_)
setup_logging()
SettingsManager.load_settings()
dir_C = HarnessTree.get("temp_dissRequest_C")
dir_D = HarnessTree.get("temp_dissRequest_D")
#create dummy files of size 1000 bits to send
for i in range(5):
filename = "package_file_%i.tar.gz" % i
with open(join(dir_C, filename), "wb") as file_out:
size = 1000
file_out.seek(size - 1)
file_out.write(b"\0")
DifmetSender.process(max_loops=3)
list_dwld = [
'package_file_0.tar.gz', 'package_file_2.tar.gz',
'package_file_4.tar.gz', 'package_file_1.tar.gz',
'package_file_3.tar.gz'
]
expected_result = True
for file_ in os.listdir(self.difmet_deposit):
expected_result = expected_result and (file_ in list_dwld)
self.assertTrue(expected_result)
def __read_settings(self):
# Default algorithm to calculate pi
pi_algorithm = PiAlgorithmType.BBP
alpha_approximation_algorithm = AlphaAlgorithmType.Newton
precision = 11
conditions = None
try:
conditions = SettingsManager.read_settings()
except IOError as err:
conditions = CalculationConditions(pi_algorithm,
alpha_approximation_algorithm,
precision)
messagebox.showwarning(title="Warning",
message=err,
parent=self.__main_form)
return conditions