def __init__(self, req_id, uri, diss_info, client_ip): self.req_id = req_id self.uri = uri self.diss_info = diss_info self.date_reception = gmtime() self.request_file = "" self._diff_externalid = None self.hostname = self.get_hostname(client_ip) #load settings, if it has not been done if not SettingsManager.is_loaded(): SettingsManager.load_settings() # initialize LOGGER setup_logging() # setup repertory structure HarnessTree.setup_tree() # Setting up database if necessary if Database.get_database() is None: from webservice.server.application import APP Database.initialize_database(APP) LOGGER.debug("Database setup") LOGGER.debug("Created a Notification object with id %s", req_id)
def setup_tree(cls, update=False): if update: LOGGER.info("Settings modified, performing a check on the " "repertories tree structure, updating it if necessary") if not SettingsManager.is_loaded(): error_msg = ("Attempting to setup repertories " "tree structure before the SettingsManager " "has been loaded") LOGGER.error(error_msg) raise RuntimeError(error_msg) def setup_repertory(dir_path): if not os.path.isdir(dir_path): try: os.makedirs(dir_path, exist_ok=True) LOGGER.debug( "Repertory %s created recursively.", dir_path) except Exception as msg_error: LOGGER.exception( "Couldn't create repertory %s.",dir_path) return dir_path if DEBUG: tempfile.gettempdir() tempfile.tempdir = None harnais_dir = os.path.join(tempfile.gettempdir(), "harnais") else: harnais_dir = SettingsManager.get("harnaisDir") # repertory of temporary dissRequest JSON files cls._repertories["temp_dissRequest_A"] = setup_repertory( join(harnais_dir, REPERTORY_TREE.temp_dissrequest_a) ) cls._repertories["temp_dissRequest_B"] = setup_repertory( join(harnais_dir, REPERTORY_TREE.temp_dissrequest_b) ) cls._repertories["temp_dissRequest_C"] = setup_repertory( join(harnais_dir, REPERTORY_TREE.temp_dissrequest_c) ) cls._repertories["temp_dissRequest_D"] = setup_repertory( join(harnais_dir, REPERTORY_TREE.temp_dissrequest_d) ) cls._repertories["dir_ack"] = dir_ack = SettingsManager.get("harnaisAckDir") if not os.path.isdir(cls._repertories["dir_ack"]): LOGGER.error("Ack repertory %s does " "not exist", dir_ack) # storing the settings file signature cls._checksum = SettingsManager.get_checksum()
def rename(self): """ Rename files using user specified regex """ # Extract all the regex specified by users. # Each regex should be a dictionnary with two keys, # pattern_in and pattern_out for the substitution. # The program can extract up to MAX_REGEX from the settings # like this in the settings.yaml : # fileRegex1: # pattern_in: # pattern_out: # fileRegex2: # pattern_in: # pattern_out: # ... # fileRegex<MAX_REGEX>: # pattern_in: # pattern_out: # Each set of regex is applied one after the other. # Except for tmpregex that has only one value corresponding to # pattern_out regex_settings = [SettingsManager.get( "fileRegex%i" % i, {}) for i in range(1, MAX_REGEX+1)] new_filename = self.original_filename zip_detector = r"^tmp\.zip\..*" # we check if the file is a zipped one # by looking if it matches r"^tmp\.zip\..*" if re.match(zip_detector, new_filename) is not None: zip_regex = SettingsManager.get("tmpregex") if zip_regex is None: LOGGER.error("No regex defined in tmpregex settings !") else: new_filename = self._rename_by_regex(new_filename, zip_detector, zip_regex) else: for idx, regex_instruction in enumerate(regex_settings): reg = regex_instruction.get("pattern_in", None) repl = regex_instruction.get("pattern_out", None) if None not in (reg, repl): new_filename = self._rename_by_regex(new_filename, reg, repl) elif idx == 0: LOGGER.error("No regex defined in fileregex1 settings !") # we check that the file renamed fits difmet standards filename_ok = self.check_filename(new_filename) if filename_ok: # update record with new filename Database.update_field_by_query("final_file", new_filename, **dict(original_file=self.original_filename)) self.new_filename = new_filename # we return if the process returned a fit new filename or not return filename_ok
def tearDown(self): cleared = Tools.move_dir_to_trash_can(self.tmpdir) if not cleared: rmtree(self.tmpdir) os.environ.pop(ENV.settings) os.environ.pop("TMPDIR") tempfile.tempdir = None Database.reset() SettingsManager.reset()
def __init__(self, req_id, hostname): # path to file on staging post self.staging_path = SettingsManager.get("openwisStagingPath") or "" self.hostname = SettingsManager.get("openwisHost") or hostname self.user = SettingsManager.get("openwisSftpUser") or None self.password = SettingsManager.get("openwisSftpPassword") or None self.port = SettingsManager.get("openwisSftpPort") or None self.req_id = req_id + hostname
def get_logger(): """ Fetch logger """ SettingsManager.load_settings() # initialize LOGGER setup_logging() logger = logging.getLogger(__name__) logger.debug("Logging configuration set up for %s", __name__) return logger
def tearDown(self): #clearing repertories cleared = Tools.move_dir_to_trash_can(self.tmpdir) if not cleared: rmtree(self.tmpdir) SFTPserver.stop_server() os.environ.pop(ENV.settings) os.environ.pop("TMPDIR") tempfile.tempdir = None Database.reset() SettingsManager.reset() DebugSettingsManager.reset()
def tearDown(self): cleared = Tools.move_dir_to_trash_can(self.tmpdir) if not cleared: rmtree(self.tmpdir) # Stop sftp server SFTPserver.stop_server() os.environ.pop(ENV.settings) os.environ.pop("TMPDIR") tempfile.tempdir = None Database.reset() SettingsManager.reset() HarnessTree.reset() DebugSettingsManager.reset()
def tearDown(self): #stopping FTP server FTPserver.stop_server() #clearing repertories cleared = Tools.move_dir_to_trash_can(self.tmpdir) if not cleared: rmtree(self.tmpdir) # cleaning up environment os.environ.pop(ENV.settings) os.environ.pop("TMPDIR") tempfile.tempdir = None SettingsManager.reset() HarnessTree.reset()
def get(cls, key): if not SettingsManager.is_loaded(): SettingsManager.load_settings() LOGGER.info("Settings loaded") #check if settings have been modified if SettingsManager.get_checksum() != cls._checksum: LOGGER.info("Settings have been modified") cls.setup_tree(update=True) elif not os.path.isdir(cls._repertories[key]): cls.setup_tree(update=False) return cls._repertories[key]
def __change_settings(self): """Changes the settings of the application by saving new values in the xml file.""" settings_form = SettingsForm(self.__main_form, self.__conditions) conditions = settings_form.get_settings() if conditions != None: self.__conditions = conditions # Writing the new settings on xml file try: SettingsManager.write_settings(conditions) except IOError as err: messagebox.showerror(title="Error", message=err, parent=self.__main_form)
def process(cls, max_loops=0): cls.nb_workers = SettingsManager.get("sendFTPlimitConn") # in debug mode, it is possible to set pool_method = cls.get_pool_method() cls.pool = pool_method(processes=cls.nb_workers) counter = 0 cls.setup_process() while cls._running: counter += 1 cls.signal_loop(counter) cls.load_settings() cls.update_workers() # idle time idle_time = SettingsManager.get("sendFTPIdle") sleep(idle_time) # get settings cls.dir_c = dir_c = HarnessTree.get("temp_dissRequest_C") cls.dir_d = dir_d = HarnessTree.get("temp_dissRequest_D") # move back any remaining file from D to C cls.move_back_files() # get files in C max_files = cls.nb_workers list_files_c = cls.get_file_list(dir_c, max_files) files_to_ftp = cls.move_files(list_files_c, dir_d) for file_ in files_to_ftp: file_expired = cls.check_file_age(file_) if file_expired: # TODO we need to find a way to update the info to the database # would require looking at the file compressed though Tools.remove_file(file_, "difmet archive", LOGGER) continue size = os.stat(file_).st_size timeout = cls.compute_timeout(size, file_) # start download # renaming file to prevent any operation on it. cls.lock_file(file_) res = cls.pool.apply_async( cls.abortable_ftp, (cls.upload_file, file_, dir_c, dir_d), dict(timeout=timeout)) # for testing and debugging purpose only cls.check_end_loop(counter, max_loops)
def _get_priority(self): donot_compute_priority = SettingsManager.get("sla") default_priority = SettingsManager.get("defaultPriority", PRIORITIES.default) if donot_compute_priority: highest_priority = default_priority else: priority_list = [] for req_id in self.instructions.keys(): priority = self._get_instr(req_id, "diffpriority") priority_list.append(priority) highest_priority = min(priority_list + [default_priority]) return highest_priority
def check_file_age(filename): time_limit = SettingsManager.get("keepFileTimeSender") or None if time_limit is not None: check = (time() - os.stat(filename).st_mtime) > time_limit else: check = False return check
def setup_logging(default_path=DEFAULT_PATH): """Setup logging configuration """ path = os.path.abspath(default_path) if os.path.exists(path): with open(path, 'rt') as file_: config = yaml.safe_load(file_.read()) # use harnaislogdir settings if it has been set. log_dir, dir_error_msg = _get_logdir() if log_dir is not None: if os.path.isdir(log_dir): change_log_dir(config, log_dir) else: raise NotADirectoryError(dir_error_msg) elif SettingsManager.is_loaded(): logging.warning( "Logdir filed in settings_harnais.yaml has not been set.\n" "Log files repertories will be those defined in the %s file", default_path) else: error_msg = "Attempting to setup logs before the SettingsManager has been loaded" logging.error(error_msg) raise RuntimeError(error_msg) logging.config.dictConfig(config) else: raise FileNotFoundError("Couldn't resolve logging configuration " "file path {f}".format(f=path))
def setUp(self): self.tmpdir = mkdtemp(prefix='harnais_') harnais_dir = join(self.tmpdir, "harnais") os.mkdir(harnais_dir) os.environ["TMPDIR"] = self.tmpdir self.staging_post = join(self.tmpdir, "staging_post") os.mkdir(self.staging_post) self.difmet_deposit = join(self.tmpdir, "difmet_deposit") os.mkdir(self.difmet_deposit) self.ack_dir = join(self.tmpdir, "ack_dir") os.mkdir(self.ack_dir) self.hostname = hostname = socket.gethostname() port = os.environ.get(ENV.port) or PORT os.environ[ENV.soap_url] = ('http://{hostname}:{port}/harnais-diss-v2/' 'webservice/Dissemination?wsdl'.format(hostname=hostname, port=port)) SettingsManager.load_settings() SettingsManager.update(dict(harnaisLogdir=harnais_dir, harnaisDir=harnais_dir, harnaisAckDir=self.ack_dir, openwisStagingPath=gettempdir(), openwisHost="localhost", openwisSftpUser="******", openwisSftpPassword="******", openwisSftpPort = 3373, processFileIdle = 10, dissHost="0.0." + "0.0", dissFtpUser="******", dissFtpPasswd="12345", dissFtpDir=self.difmet_deposit, dissFtpMode=None, dissFtpPort=2121, sendFTPlimitConn=5), testing=True) os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml") with open(os.environ[ENV.settings], "w") as file_: yaml.dump(SettingsManager._parameters, file_) setup_logging()
def main(): SettingsManager.load_settings() # initialize LOGGER setup_logging() # setup repertory structure HarnessTree.setup_tree() # Setting up database Database.initialize_database(APP) LOGGER.info("setup complete") hostname = socket.gethostname() port = os.environ.get(ENV.port) or PORT LOGGER.warning("Starting application through Flask development server." " This is NOT a production environment.") LOGGER.info( "Launching Flask development server " "on hostname %s on port %s", hostname, port) APP.run(host=hostname, port=port)
def update_workers(cls): # update the number of workers if necessary nbw = SettingsManager.get("sendFTPlimitConn") if nbw != cls.nb_workers: #wait for every upload to be finished cls.pool.close() cls.pool.join() #update workers cls.nb_workers = nbw cls.pool = pool_method(processes=nbw)
def check_file_age(filename): """ Discard files that are too old """ time_limit = SettingsManager.get("keepFileTime") or None if time_limit is not None: check = (time() - os.stat(filename).st_mtime) > time_limit else: check = False return check
def check_transfer(cls, filename, ftp): # check if file exists on remote server ftpdir = SettingsManager.get("dissFtpDir") ftp.cwd(ftpdir) if filename in [name for name, data in list(ftp.mlsd())]: upload_ok = True else: upload_ok = False ftp.quit() return upload_ok
def compute_priority(priority, sla): # priority is scaled from 1 (highest) to 4 (lowest) # sla is 0 (BRONZE), 1 (SILVER), 2 (GOLD) priority_activated = SettingsManager.get("sla") if type(priority_activated) == str: priority_activated = strtobool(priority_activated) if priority_activated == False: result = SettingsManager.get("defaultPriority") or PRIORITIES.default elif priority ==1: result = PRIORITIES.maximum elif priority >=2: default_priority = SettingsManager.get("defaultPriority") or PRIORITIES.default result = default_priority + priority - 2*sla result = max(PRIORITIES.maximum, result) result = min(PRIORITIES.minimum, result) return result
def _get_end_date(): limit = SettingsManager.get("fileEndLive") or 0 if limit ==0: end_date = None else: # convert limit in minutes in seconds end_date = datetime.utcnow() + timedelta(seconds=limit*60) end_date = end_date.strftime("%Y-%m-%dT%H:%M:%SZ") return end_date
def setUp(self): # create repertories and configure harnesss self.tmpdir = mkdtemp(prefix='harnais_') os.environ["TMPDIR"] = self.tmpdir self.staging_post = join(self.tmpdir, "staging_post") os.mkdir(self.staging_post) # prepare settings SettingsManager.load_settings() SettingsManager.update(dict(harnaisLogdir=self.tmpdir, harnaisDir=self.tmpdir, harnaisAckDir=self.tmpdir ), testing=True) os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml") with open(os.environ[ENV.settings], "w") as file_: yaml.dump(dict(SettingsManager._parameters), file_) SettingsManager.reset() # configure SOAP server self.hostname = hostname = socket.gethostname() self.port = port = os.environ.get(ENV.port) or PORT self.soap_url = os.environ[ENV.soap_url]= ('http://{hostname}:{port}/harnais-diss-v2/' 'webservice/Dissemination?wsdl'.format(hostname=hostname, port=port)) SoapServer.create_server() # connect to soap WSDL self.client = Client(self.soap_url) self.factory = self.client.type_factory('http://dissemination.harness.openwis.org/')
def setUp(self): self.tmpdir = mkdtemp(prefix='harnais_') os.environ["TMPDIR"] = self.tmpdir self.staging_post = join(self.tmpdir, "staging_post") os.mkdir(self.staging_post) # # prepare settings SettingsManager.load_settings() SettingsManager.update(dict( harnaisLogdir=self.tmpdir, harnaisDir=self.tmpdir, harnaisAckDir=self.tmpdir, bandwidth=5, ), testing=True) os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml") with open(os.environ[ENV.settings], "w") as file_: yaml.dump(SettingsManager._parameters, file_) setup_logging()
def upload_file(cls, file_, ftp): upload_ok = False start = time() file_locked = file_ + ".lock" with open(file_locked, 'rb') as file_transfered: ftpdir = SettingsManager.get("dissFtpDir") ftp.cwd(ftpdir) # renaming in tmp file_renamed = basename(file_) + ".tmp" ftp.storbinary('STOR ' + file_renamed, file_transfered) ftp.rename(file_renamed, basename(file_)) upload_ok = True return upload_ok, time() - start
def get_file_list(cls, dirname, maxfiles): overflow = SettingsManager.get("ManagerOverflow") list_entries = os.listdir(dirname) list_entries = [os.path.join(dirname, entry) for entry in list_entries] list_files = [ entry for entry in list_entries if not os.path.isdir(entry)] list_files.sort(key=lambda x: os.stat(x).st_mtime) if overflow is not None and len(list_files) > overflow: LOGGER.warning("%s repertory is overflowing. " "Number of files %i over the limit %i", cls.dir_a, len(list_files), overflow) list_files = list_files[-maxfiles:] return list_files
def setUp(self): # configuring settings and repertories DebugSettingsManager.debug = "False" DebugSettingsManager.test_sftp = "True" self.tmpdir = mkdtemp(prefix='harnais_') harnais_dir = join(self.tmpdir, "harnais") os.mkdir(harnais_dir) os.environ["TMPDIR"] = self.tmpdir self.staging_post = join(self.tmpdir, "staging_post") os.mkdir(self.staging_post) self.difmet_deposit = join(self.tmpdir, "difmet_deposit") os.mkdir(self.difmet_deposit) self.ack_dir = join(self.tmpdir, "ack_dir") os.mkdir(self.ack_dir) # create files on staging post for i in range(2): with open(join(self.staging_post,"A_SNFR30LFPW270700_C_LFPW_20180927070000_%i.txt" % i),"w") as file_out: file_out.write("Dummy staging post test file") # configure soap server self.hostname = hostname = socket.gethostname() self.port = port = os.environ.get(ENV.port) or PORT self.soap_url = os.environ[ENV.soap_url]= ('http://{hostname}:{port}/harnais-diss-v2/' 'webservice/Dissemination?wsdl'.format(hostname=hostname, port=port)) SettingsManager.load_settings() SettingsManager.update(dict(harnaisLogdir=harnais_dir, harnaisDir=harnais_dir, harnaisAckDir=self.ack_dir, openwisStagingPath=gettempdir(), openwisHost="localhost", openwisSftpUser="******", openwisSftpPassword="******", openwisSftpPort = 3373, processFileIdle = 10, dissHost="0.0." + "0.0", dissFtpUser="******", dissFtpPasswd="12345", dissFtpDir=self.difmet_deposit, dissFtpMode=None, dissFtpPort=2121, sendFTPlimitConn=5), testing=True) os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml") with open(os.environ[ENV.settings], "w") as file_: yaml.dump(dict(SettingsManager._parameters), file_) SettingsManager.reset()
def compute_timeout(required_bandwith): # compute timeout by doing timeout = size/bandwidth + buffer bandwidth = SettingsManager.get("bandwidth") if bandwidth in [None, 0]: LOGGER.warning("Incorrect value for harness settings bandwidth. " "Sftp timeout set to default TIMEOUT %i s.", TIMEOUT) timeout = TIMEOUT elif DEBUG: timeout = DEBUG_TIMEOUT LOGGER.debug("Sftp debug timeout set to %s s", timeout) else: # conversion in Mbits/s with shift_expr << operator timeout = (required_bandwith/(1 << 17))/bandwidth + TIMEOUT_BUFFER LOGGER.debug("Sftp timeout computed to %s s", timeout) # start download return timeout
def test_sending(self): """ Testing FTP transfer of 5 small files """ # prepare settings SettingsManager.load_settings() SettingsManager.update(dict(harnaisLogdir=self.tmpdir, harnaisDir=self.tmpdir, harnaisAckDir=self.tmpdir, dissHost="0.0." + "0.0", dissFtpUser="******", dissFtpPasswd="12345", bandwidth=10, dissFtpDir=self.difmet_deposit, dissFtpMode=None, dissFtpPort=2121, sendFTPlimitConn=5, sendFTPIdle=10), testing=True) os.environ[ENV.settings] = join(self.tmpdir, "settings_testing.yaml") with open(os.environ[ENV.settings], "w") as file_: yaml.dump(SettingsManager._parameters, file_) setup_logging() SettingsManager.load_settings() dir_C = HarnessTree.get("temp_dissRequest_C") dir_D = HarnessTree.get("temp_dissRequest_D") #create dummy files of size 1000 bits to send for i in range(5): filename = "package_file_%i.tar.gz" % i with open(join(dir_C, filename), "wb") as file_out: size = 1000 file_out.seek(size - 1) file_out.write(b"\0") DifmetSender.process(max_loops=3) list_dwld = [ 'package_file_0.tar.gz', 'package_file_2.tar.gz', 'package_file_4.tar.gz', 'package_file_1.tar.gz', 'package_file_3.tar.gz' ] expected_result = True for file_ in os.listdir(self.difmet_deposit): expected_result = expected_result and (file_ in list_dwld) self.assertTrue(expected_result)
def __read_settings(self): # Default algorithm to calculate pi pi_algorithm = PiAlgorithmType.BBP alpha_approximation_algorithm = AlphaAlgorithmType.Newton precision = 11 conditions = None try: conditions = SettingsManager.read_settings() except IOError as err: conditions = CalculationConditions(pi_algorithm, alpha_approximation_algorithm, precision) messagebox.showwarning(title="Warning", message=err, parent=self.__main_form) return conditions