def __delitem__(self, index): if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": actual_value = self[index] if actual_value: del self.database[self.filename][index] PyFunceble.LOGGER.info( "Cleaned the data related to " f"{repr(index)} and {repr(self.filename)} " f"from the database.") elif PyFunceble.CONFIGURATION.db_type in [ "mariadb", "mysql", ]: # pragma: no cover with session.Session() as db_session: # pylint: disable=no-member to_delete = (db_session.query(Mined).join(Status).filter( Status.tested == index).all()) for row in to_delete: # pylint: disable=no-member with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison delete_query = Mined.__table__.delete().where( Mined.id == row.id) db_session.execute(delete_query) db_session.commit() PyFunceble.LOGGER.info( "Cleaned the data related to " f"{repr(index)} and {repr(self.filename)} " f"from the database.")
def remove(self, subject, history_member): """ Removes the given subject from the database assigned to the currently tested file. :param str subject: The subject we are working with. :param str history_member: The history member to delete. """ if self.authorized: while True: actual_value = self[subject] if (isinstance(actual_value, (list, set)) and history_member in actual_value): if PyFunceble.CONFIGURATION.db_type == "json": try: actual_value.remove(history_member) PyFunceble.LOGGER.info( f"Removed {repr(history_member)} (mined) " f"From the subset of {repr(subject)}.") except ValueError: # pragma: no cover pass elif PyFunceble.CONFIGURATION.db_type in [ "mariadb", "mysql" ]: # We construct the query string. with session.Session() as db_session: # pylint: disable=no-member to_delete = ( db_session.query(Mined).join(Status).join(File) .filter(File.id == Mined.file_id).filter( Status.tested == subject).filter( Mined.mined == history_member).all()) for row in to_delete: with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison delete_query = Mined.__table__.delete().where( Mined.id == row.id) db_session.execute(delete_query) db_session.commit() PyFunceble.LOGGER.info( "Cleaned the data related to " f"{repr(subject)}, {repr(history_member)} (mined) and " f"{repr(self.filename)} and from " f"the database.") else: # pragma: no cover break if not self[subject]: # pragma: no cover del self[subject] self.save()
def __setitem__(self, index, value): # pylint: disable=too-many-branches if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": actual_value = self[index] if actual_value: if isinstance(actual_value, dict): if isinstance(value, dict): # pragma: no cover self.database[self.filename][index].update(value) else: # pragma: no cover self.database[self.filename][index] = value elif isinstance(actual_value, list): if isinstance(value, list): self.database[self.filename][index].extend(value) else: # pragma: no cover self.database[self.filename][index].append(value) self.database[ self.filename][index] = PyFunceble.helpers.List( self.database[self.filename][index]).format() else: # pragma: no cover self.database[self.filename][index] = value else: if self.filename not in self.database: # pragma: no cover self.database[self.filename] = {} self.database[self.filename][index] = value PyFunceble.LOGGER.info( f"Inserted {repr(value)} into the subset of {repr(index)}") elif PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: try: # pylint: disable=no-member status = (db_session.query(Status).filter( Status.tested == index).one()) except NoResultFound: pass with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison in_db = (db_session.query(Mined).filter( Mined.mined == value).filter( Mined.subject_id == status.id).all()) if not in_db: mined = Mined( subject_id=status.id, mined=value, file_id=status.file_id, ) with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison db_session.add(mined) db_session.commit() db_session.refresh(mined)
def generate_files_of_status( self, status, include_entries_without_changes=False ): # pragma: no cover """ Generates the status file of all subjects of the given status. :param str status: A status to filter. :param bool include_entries_without_changes: Descriptive enough. """ if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: fetched = ( # pylint: disable=no-member, singleton-comparison db_session.query(Status) .join(File) .filter(File.path == self.file) .filter(Status.status == status) .filter(Status.test_completed == True) .all() ) if fetched: for data in fetched: with session.Session() as db_session: try: # pylint: disable=no-member whois_record = ( db_session.query(WhoisRecord) .filter(WhoisRecord.subject == data.tested) .one() ) whois_server = whois_record.server except NoResultFound: whois_server = None generate = PyFunceble.output.Generate( data.tested, f"file_{self.file_type}", data.status, source=data.status_source, expiration_date=data.expiration_date, http_status_code=data.http_status_code, whois_server=whois_server, filename=self.file, end=True, ) if include_entries_without_changes: generate.status_file(exclude_file_generation=False) else: generate.status_file( exclude_file_generation=self.inactive_db.authorized and data.status not in self.list_of_up_statuses and data.tested in self.inactive_db.to_retest )
def clean(self): """ Cleans the database. """ if self.authorized: # We are authorized to operate. if PyFunceble.CONFIGURATION.db_type == "json": # We empty the database. self.database[self.filename] = {} # And we save the current database state. PyFunceble.helpers.Dict(self.database).to_json_file( self.database_file) PyFunceble.LOGGER.info( "Cleaned the data related to " f"{repr(self.filename)} from {repr(self.database_file)}.") elif PyFunceble.CONFIGURATION.db_type in [ "mysql", "mariadb", ]: # pragma: no cover with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison try: file_object = (db_session.query(File).filter( File.path == self.filename).one()) file_object.test_completed = True db_session.add(file_object) db_session.commit() except NoResultFound: pass with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison # Now we only replace the test_completed # flag for the inactive/invalid one. to_update = (db_session.query(Status).join(File).filter( File.path == self.filename).filter( Status.status.notin_( PyFunceble.core.CLI.get_up_statuses())).filter( Status.test_completed == True).all()) for status_object in to_update: status_object.test_completed = False with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison db_session.add(status_object) db_session.commit()
def save_into_database(cls, output, filename): # pragma: no cover """ Saves the current status inside the database. """ if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: output = output.copy() if (isinstance(output["http_status_code"], str) and not output["http_status_code"].isdigit()): output["http_status_code"] = None output["tested_at"] = datetime.utcnow() if not filename: filename = "simple" status_input = output.copy() undesirable_status = ["dns_lookup", "whois_record", "whois_server"] for index in undesirable_status: del status_input[index] with session.Session() as db_session: # pylint: disable=no-member try: file = db_session.query(File).filter( File.path == filename).one() except NoResultFound: file = File(path=filename) db_session.add(file) db_session.commit() db_session.refresh(file) with session.Session() as db_session: # pylint: disable=no-member try: status = (db_session.query(Status).filter( Status.file_id == file.id).filter( Status.tested == status_input["tested"]).one()) except NoResultFound: status = Status(file_id=file.id) for index, value in status_input.items(): setattr(status, index, value) status.test_completed = True with session.Session() as db_session: # pylint: disable=no-member db_session.add(status) db_session.commit() PyFunceble.LOGGER.debug(f"Saved into database:\n{output}")
def __get_or_generate_complements_mysql(self): # pragma: no cover """ Gets or generates the complements while working with as MySQL/MariaDB formatted database. """ result = [] with session.Session() as db_session: # pylint: disable=singleton-comparison,no-member result = (db_session.query(Status).join(File).filter( File.path == self.filename).filter( Status.is_complement == True).all()) if result: result = [x.tested for x in result] else: result = self.__generate_complements() for subject in result: with session.Session() as db_session: try: file = (db_session.query(File).filter( File.path == self.filename).one()) except NoResultFound: file = File(path=self.filename) db_session.add(file) db_session.commit() db_session.refresh(file) with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison try: status = (db_session.query(Status).filter( Status.tested == subject).one()) except NoResultFound: status = Status( file_id=file.id, is_complement=True, tested=subject, tested_at=datetime.fromtimestamp(10), test_completed=False, ) db_session.add(status) db_session.commit() return result
def get_already_tested(self): """ Returns the list of subjects which were already tested as a set. """ PyFunceble.LOGGER.info( "Getting the list of already tested (DATASET WONT BE LOGGED)") # raise Exception("AUTHORIZED", self.authorized) if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": if (PyFunceble.CONFIGURATION.multiprocess and get_start_method() == "spawn"): # pragma: no cover self.load() try: return { y for state, x in self.database[self.filename].items() for y in x if state not in ["complements"] } except KeyError: # pragma: no cover pass elif PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison result = (db_session.query(Status).join(File).filter( File.path == self.filename).filter( Status.test_completed == True).all()) if result: return {x.tested for x in result} return set() # pragma: no cover
def __contains__(self, subject): if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": if (PyFunceble.CONFIGURATION.multiprocess and get_start_method() == "spawn"): # pragma: no cover self.load() if subject not in self.is_present_cache: self.is_present_cache[subject] = False if self[subject]: self.is_present_cache[subject] = True return self.is_present_cache[subject] if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: # pylint: disable=no-member result = (db_session.query(Status).join(File).filter( File.path == self.filename ).filter(Status.tested == subject).filter( Status.status.notin_( PyFunceble.core.CLI.get_up_statuses())).count() > 0) if result: PyFunceble.LOGGER.info( f"{subject} is present into the database.") else: PyFunceble.LOGGER.info( f"{subject} is not present into the database.") return result return False # pragma: no cover
def __getitem__(self, index): if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": if index in self.database: return self.database[index] return None if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: fetched = None with session.Session() as db_session: try: # pylint: disable=no-member fetched = (db_session.query(WhoisRecord).filter( WhoisRecord.subject == index)).one() return { "epoch": fetched.epoch, "expiration_date": fetched.expiration_date, "state": fetched.state, "record": fetched.record, } except NoResultFound: pass return None # pragma: no cover
def __contains__(self, subject): if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": if subject not in self.is_present_cache: self.is_present_cache[subject] = False if self[subject]: self.is_present_cache[subject] = True return self.is_present_cache[subject] if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: try: # pylint: disable=no-member data = (db_session.query(Status).join(File).filter( File.path == self.filename ).filter(Status.tested == subject).filter( Status.status.notin_( PyFunceble.core.CLI.get_up_statuses())).one()) except NoResultFound: data = [] if data: return True return False # pragma: no cover
def __contains__(self, index): if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": if index in self.database: PyFunceble.LOGGER.info( f"{index} is present into the database.") return True PyFunceble.LOGGER.info( f"{index} is not present into the database.") return False if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: try: # pylint: disable=no-member _ = (db_session.query(WhoisRecord).filter( WhoisRecord.subject == index)).one() PyFunceble.LOGGER.info( f"{index} is present into the database.") return True except NoResultFound: PyFunceble.LOGGER.info( f"{index} is not present into the database.") return False # pragma: no cover
def __delitem__(self, index): if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": actual_value = self[index] if actual_value: del self.database[self.filename][index] PyFunceble.LOGGER.info( "Cleaned the data related to " f"{repr(index)} and {repr(self.filename)} " f"from the database.") elif PyFunceble.CONFIGURATION.db_type in [ "mariadb", "mysql", ]: # pragma: no cover with session.Session() as db_session: # pylint: disable=no-member status = (db_session.query(Status).filter( Status.tested == index).one()) for mined in status.mined: db_session.delete(mined) db_session.commit() PyFunceble.LOGGER.info( "Cleaned the data related to " f"{repr(index)} and {repr(self.filename)} " f"from the database.")
def update_counters(self): # pragma: no cover """ Updates the counters. """ if self.authorized and self.parent: # We are authorized to operate. # We create a list of all status we are working with. statuses = PyFunceble.STATUS.official.keys() # We preset the number of tested. tested = 0 for status in statuses: # We loop through the list of status. if PyFunceble.CONFIGURATION.db_type == "json": try: # We get the number of tested of the currently read # status. tested_for_status = len(self.database[self.filename][ PyFunceble.STATUS.official[status]]) # We then update/transfert it to its global place. PyFunceble.INTERN["counter"]["number"][ status] = tested_for_status PyFunceble.LOGGER.debug( f"Counter of {repr(status)} set to {tested_for_status}." ) # We finally increate the number of tested. tested += tested_for_status except KeyError: PyFunceble.INTERN["counter"]["number"][status] = 0 continue elif PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison result = (db_session.query(Status).join(File).filter( File.path == self.filename).filter( Status.status == PyFunceble.STATUS.official[status]).filter( Status.test_completed == True).all()) fetched = len(result) PyFunceble.INTERN["counter"]["number"][ status] = fetched PyFunceble.LOGGER.debug( f"Counter of {repr(status)} set to {fetched}.") # We then update/transfer it to its global place. tested += fetched # We update/transfert the number of tested globally. PyFunceble.INTERN["counter"]["number"]["tested"] = tested PyFunceble.LOGGER.debug(f"Totally tested set to {repr(tested)}.")
def list_of_mined(self): """ Provides the list of mined domains so that they can be tested. :return: The list of mined domains. The returned format is the following: :: [ (index_to_delete_after_test, mined), (index_to_delete_after_test, mined), (index_to_delete_after_test, mined) ] :rtype: list """ PyFunceble.LOGGER.info( "Getting the list of previously mined data. (DATASET WONT BE LOGGED)" ) # We initiate a variable which will return the result. result = [] if self.authorized: # We are authorized to operate. if PyFunceble.CONFIGURATION.db_type == "json": for subject in self.database[self.filename].keys(): # We loop through the available list of status # from the database. for element in self[subject]: # We then loop through the data associatied to # the currently read status. result.append((subject, element)) elif PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: # pylint: disable=no-member fetched = ( db_session.query(Mined) .join(File) .join(Status) .filter(File.id == Mined.file_id) .filter(Status.id == Mined.subject_id) .all() ) if fetched: result = [(x.subject.tested, x.mined) for x in fetched] # We return the result. return result
def __setitem_mysql(cls, index, value): with session.Session() as db_session: try: # pylint: disable=no-member record = (db_session.query(WhoisRecord).filter( WhoisRecord.subject == index).one()) except NoResultFound: record = WhoisRecord(subject=index, ) for db_key, db_value in value.items(): if not PyFunceble.CONFIGURATION.store_whois_record and db_key == "record": continue setattr(record, db_key, db_value) with session.Session() as db_session: # pylint: disable=no-member db_session.add(record) db_session.commit() PyFunceble.LOGGER.info(f"Inserted into the database: \n {value}")
def __getitem__(self, index): if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": if index in self.database[self.filename]: return self.database[self.filename][index] if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: # pylint: disable=no-member fetched = (db_session.query(Mined).join(Status).filter( Status.id == Mined.subject_id).all()) if fetched: return {x.mined for x in fetched} return None
def get_to_retest(self): """ Returns a set of subject to restest. """ PyFunceble.LOGGER.info( "Getting the list of subjects to retest (DATASET WONT BE LOGGED)") if (self.authorized and PyFunceble.CONFIGURATION.days_between_db_retest >= 0 and self.filename in self.database): if PyFunceble.CONFIGURATION.db_type == "json": result = set() if (PyFunceble.CONFIGURATION.multiprocess and get_start_method() == "spawn"): # pragma: no cover self.load() for subject, info in self.database[self.filename].items(): if ("last_retested_at_epoch" in info and info["last_retested_at_epoch"]): if (datetime.utcnow() > datetime.fromtimestamp( info["last_retested_at_epoch"]) + self.days): result.add(subject) else: result.add(subject) return result if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: try: # pylint: disable=no-member result = (db_session.query(Status).join(File).filter( File.path == self.filename).filter( Status.status.notin_( PyFunceble.core.CLI.get_up_statuses())). filter(datetime.utcnow() > Status.tested_at + self.days).all()) except NoResultFound: result = [] if result: return {x.tested for x in result} return set()
def __contains__(self, index): # pragma: no cover if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": if (PyFunceble.CONFIGURATION.multiprocess and get_start_method() == "spawn"): # pragma: no cover self.load() if self.filename in self.database: for status, status_data in self.database[ self.filename].items(): if status == "complements": continue if index in status_data: PyFunceble.LOGGER.info( f"{index} is present into the database.") return True PyFunceble.LOGGER.info( f"{index} is not present into the database.") return False if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison result = (db_session.query(Status).join(File).filter( Status.tested == index).filter( File.path == self.filename).count() > 0) if result: PyFunceble.LOGGER.info( f"{index} is present into the database.") else: PyFunceble.LOGGER.info( f"{index} is not present into the database.") return result PyFunceble.LOGGER.info( f"Could not check if {index} is present into the database. " "Unauthorized action.") return False
def is_empty(self): """ Checks if the database related to the currently tested file is empty. """ if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": if (self.filename not in self.database or not self.database[self.filename]): PyFunceble.LOGGER.info( "File to test was not previously indexed.") return True PyFunceble.LOGGER.info("File to test was previously indexed.") return False if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: # Here we don't really check if it is empty. # What we do, is that we check that everything was # tested. with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison try: result = (db_session.query(File).filter( File.path == self.filename).one()) PyFunceble.LOGGER.info( f"File was completely tested: {not result.test_completed}" ) return result.test_completed except NoResultFound: pass PyFunceble.LOGGER.info( # pragma: no cover "Could not check if the file to test " "was previously indexed. Unauthorized action.") return False # pragma: no cover
def __contains__(self, index): # pragma: no cover if self.authorized: if PyFunceble.CONFIGURATION.db_type == "json": if self.filename in self.database: for status, status_data in self.database[ self.filename].items(): if status == "complements": continue if index in status_data: PyFunceble.LOGGER.info( f"{index} is present into the database.") return True PyFunceble.LOGGER.info( f"{index} is not present into the database.") return False if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]: with session.Session() as db_session: try: # pylint: disable=no-member, singleton-comparison _ = (db_session.query(Status).join(File).filter( Status.tested == index).filter( File.path == self.filename).one()) except NoResultFound: PyFunceble.LOGGER.info( f"{index} is not present into the database.") return False PyFunceble.LOGGER.info( f"{index} is present into the database.") return True PyFunceble.LOGGER.info( f"Could not check if {index} is present into the database. " "Unauthorized action.") return False
def almost_everything(self, clean_all=False, file_path=False): """ Delete almost all discovered files. :param bool clean_all: Tell the subsystem if we have to clean everything instesd of almost everything. """ if ( "do_not_clean" not in PyFunceble.INTERN or not PyFunceble.INTERN["do_not_clean"] ): # We get the list of file to delete. to_delete = self.file_to_delete(clean_all) if ( not PyFunceble.abstracts.Version.is_local_cloned() and clean_all ): # pragma: no cover to_delete.extend(self.databases_to_delete()) for file in to_delete: # We loop through the list of file to delete. # And we delete the currently read file. PyFunceble.helpers.File(file).delete() PyFunceble.LOGGER.info(f"Deleted: {file}") if PyFunceble.CONFIGURATION.db_type in [ "mariadb", "mysql", ]: # pragma: no cover if file_path: with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison to_delete = ( db_session.query(Status) .join(File) .filter(File.path == file_path) .filter(File.test_completed == True) .filter( Status.status.in_(PyFunceble.core.CLI.get_up_statuses()) ) .all() ) for row in to_delete: with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison delete_query = Status.__table__.delete().where( Status.id == row.id ) db_session.execute(delete_query) db_session.commit() with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison file_object = ( db_session.query(File).filter(File.path == file_path).one() ) file_object.test_completed = False with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison db_session.add(file_object) db_session.commit() else: with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison to_delete = db_session.query( # pylint: disable=no-member File ).all() for row in to_delete: # pylint: disable=no-member with session.Session() as db_session: # pylint: disable=no-member, singleton-comparison delete_query = File.__table__.delete().where( File.id == row.id ) db_session.execute(delete_query) db_session.commit() if ( not PyFunceble.abstracts.Version.is_local_cloned() and clean_all ): # pragma: no cover PyFunceble.load_config() PyFunceble.LOGGER.info("Reloaded configuration.")