def __delitem__(self, index):
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
actual_value = self[index]
if actual_value:
del self.database[self.filename][index]
PyFunceble.LOGGER.info(
"Cleaned the data related to "
f"{repr(index)} and {repr(self.filename)} "
f"from the database.")
elif PyFunceble.CONFIGURATION.db_type in [
"mariadb",
"mysql",
]: # pragma: no cover
with session.Session() as db_session:
# pylint: disable=no-member
to_delete = (db_session.query(Mined).join(Status).filter(
Status.tested == index).all())
for row in to_delete:
# pylint: disable=no-member
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
delete_query = Mined.__table__.delete().where(
Mined.id == row.id)
db_session.execute(delete_query)
db_session.commit()
PyFunceble.LOGGER.info(
"Cleaned the data related to "
f"{repr(index)} and {repr(self.filename)} "
f"from the database.")
def remove(self, subject, history_member):
"""
Removes the given subject from the database assigned to the
currently tested file.
:param str subject: The subject we are working with.
:param str history_member: The history member to delete.
"""
if self.authorized:
while True:
actual_value = self[subject]
if (isinstance(actual_value, (list, set))
and history_member in actual_value):
if PyFunceble.CONFIGURATION.db_type == "json":
try:
actual_value.remove(history_member)
PyFunceble.LOGGER.info(
f"Removed {repr(history_member)} (mined) "
f"From the subset of {repr(subject)}.")
except ValueError: # pragma: no cover
pass
elif PyFunceble.CONFIGURATION.db_type in [
"mariadb", "mysql"
]:
# We construct the query string.
with session.Session() as db_session:
# pylint: disable=no-member
to_delete = (
db_session.query(Mined).join(Status).join(File)
.filter(File.id == Mined.file_id).filter(
Status.tested == subject).filter(
Mined.mined == history_member).all())
for row in to_delete:
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
delete_query = Mined.__table__.delete().where(
Mined.id == row.id)
db_session.execute(delete_query)
db_session.commit()
PyFunceble.LOGGER.info(
"Cleaned the data related to "
f"{repr(subject)}, {repr(history_member)} (mined) and "
f"{repr(self.filename)} and from "
f"the database.")
else: # pragma: no cover
break
if not self[subject]: # pragma: no cover
del self[subject]
self.save()
def __setitem__(self, index, value): # pylint: disable=too-many-branches
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
actual_value = self[index]
if actual_value:
if isinstance(actual_value, dict):
if isinstance(value, dict): # pragma: no cover
self.database[self.filename][index].update(value)
else: # pragma: no cover
self.database[self.filename][index] = value
elif isinstance(actual_value, list):
if isinstance(value, list):
self.database[self.filename][index].extend(value)
else: # pragma: no cover
self.database[self.filename][index].append(value)
self.database[
self.filename][index] = PyFunceble.helpers.List(
self.database[self.filename][index]).format()
else: # pragma: no cover
self.database[self.filename][index] = value
else:
if self.filename not in self.database: # pragma: no cover
self.database[self.filename] = {}
self.database[self.filename][index] = value
PyFunceble.LOGGER.info(
f"Inserted {repr(value)} into the subset of {repr(index)}")
elif PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
try:
# pylint: disable=no-member
status = (db_session.query(Status).filter(
Status.tested == index).one())
except NoResultFound:
pass
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
in_db = (db_session.query(Mined).filter(
Mined.mined == value).filter(
Mined.subject_id == status.id).all())
if not in_db:
mined = Mined(
subject_id=status.id,
mined=value,
file_id=status.file_id,
)
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
db_session.add(mined)
db_session.commit()
db_session.refresh(mined)
def generate_files_of_status(
self, status, include_entries_without_changes=False
): # pragma: no cover
"""
Generates the status file of all subjects of the given status.
:param str status: A status to filter.
:param bool include_entries_without_changes: Descriptive enough.
"""
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
fetched = (
# pylint: disable=no-member, singleton-comparison
db_session.query(Status)
.join(File)
.filter(File.path == self.file)
.filter(Status.status == status)
.filter(Status.test_completed == True)
.all()
)
if fetched:
for data in fetched:
with session.Session() as db_session:
try:
# pylint: disable=no-member
whois_record = (
db_session.query(WhoisRecord)
.filter(WhoisRecord.subject == data.tested)
.one()
)
whois_server = whois_record.server
except NoResultFound:
whois_server = None
generate = PyFunceble.output.Generate(
data.tested,
f"file_{self.file_type}",
data.status,
source=data.status_source,
expiration_date=data.expiration_date,
http_status_code=data.http_status_code,
whois_server=whois_server,
filename=self.file,
end=True,
)
if include_entries_without_changes:
generate.status_file(exclude_file_generation=False)
else:
generate.status_file(
exclude_file_generation=self.inactive_db.authorized
and data.status not in self.list_of_up_statuses
and data.tested in self.inactive_db.to_retest
)
def clean(self):
"""
Cleans the database.
"""
if self.authorized:
# We are authorized to operate.
if PyFunceble.CONFIGURATION.db_type == "json":
# We empty the database.
self.database[self.filename] = {}
# And we save the current database state.
PyFunceble.helpers.Dict(self.database).to_json_file(
self.database_file)
PyFunceble.LOGGER.info(
"Cleaned the data related to "
f"{repr(self.filename)} from {repr(self.database_file)}.")
elif PyFunceble.CONFIGURATION.db_type in [
"mysql",
"mariadb",
]: # pragma: no cover
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
try:
file_object = (db_session.query(File).filter(
File.path == self.filename).one())
file_object.test_completed = True
db_session.add(file_object)
db_session.commit()
except NoResultFound:
pass
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
# Now we only replace the test_completed
# flag for the inactive/invalid one.
to_update = (db_session.query(Status).join(File).filter(
File.path == self.filename).filter(
Status.status.notin_(
PyFunceble.core.CLI.get_up_statuses())).filter(
Status.test_completed == True).all())
for status_object in to_update:
status_object.test_completed = False
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
db_session.add(status_object)
db_session.commit()
def save_into_database(cls, output, filename): # pragma: no cover
"""
Saves the current status inside the database.
"""
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
output = output.copy()
if (isinstance(output["http_status_code"], str)
and not output["http_status_code"].isdigit()):
output["http_status_code"] = None
output["tested_at"] = datetime.utcnow()
if not filename:
filename = "simple"
status_input = output.copy()
undesirable_status = ["dns_lookup", "whois_record", "whois_server"]
for index in undesirable_status:
del status_input[index]
with session.Session() as db_session:
# pylint: disable=no-member
try:
file = db_session.query(File).filter(
File.path == filename).one()
except NoResultFound:
file = File(path=filename)
db_session.add(file)
db_session.commit()
db_session.refresh(file)
with session.Session() as db_session:
# pylint: disable=no-member
try:
status = (db_session.query(Status).filter(
Status.file_id == file.id).filter(
Status.tested == status_input["tested"]).one())
except NoResultFound:
status = Status(file_id=file.id)
for index, value in status_input.items():
setattr(status, index, value)
status.test_completed = True
with session.Session() as db_session:
# pylint: disable=no-member
db_session.add(status)
db_session.commit()
PyFunceble.LOGGER.debug(f"Saved into database:\n{output}")
def __get_or_generate_complements_mysql(self): # pragma: no cover
"""
Gets or generates the complements while working with
as MySQL/MariaDB formatted database.
"""
result = []
with session.Session() as db_session:
# pylint: disable=singleton-comparison,no-member
result = (db_session.query(Status).join(File).filter(
File.path == self.filename).filter(
Status.is_complement == True).all())
if result:
result = [x.tested for x in result]
else:
result = self.__generate_complements()
for subject in result:
with session.Session() as db_session:
try:
file = (db_session.query(File).filter(
File.path == self.filename).one())
except NoResultFound:
file = File(path=self.filename)
db_session.add(file)
db_session.commit()
db_session.refresh(file)
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
try:
status = (db_session.query(Status).filter(
Status.tested == subject).one())
except NoResultFound:
status = Status(
file_id=file.id,
is_complement=True,
tested=subject,
tested_at=datetime.fromtimestamp(10),
test_completed=False,
)
db_session.add(status)
db_session.commit()
return result
def get_already_tested(self):
"""
Returns the list of subjects which were already tested as a set.
"""
PyFunceble.LOGGER.info(
"Getting the list of already tested (DATASET WONT BE LOGGED)")
# raise Exception("AUTHORIZED", self.authorized)
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
if (PyFunceble.CONFIGURATION.multiprocess
and get_start_method() == "spawn"): # pragma: no cover
self.load()
try:
return {
y
for state, x in self.database[self.filename].items()
for y in x if state not in ["complements"]
}
except KeyError: # pragma: no cover
pass
elif PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
result = (db_session.query(Status).join(File).filter(
File.path == self.filename).filter(
Status.test_completed == True).all())
if result:
return {x.tested for x in result}
return set() # pragma: no cover
def __contains__(self, subject):
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
if (PyFunceble.CONFIGURATION.multiprocess
and get_start_method() == "spawn"): # pragma: no cover
self.load()
if subject not in self.is_present_cache:
self.is_present_cache[subject] = False
if self[subject]:
self.is_present_cache[subject] = True
return self.is_present_cache[subject]
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
# pylint: disable=no-member
result = (db_session.query(Status).join(File).filter(
File.path == self.filename
).filter(Status.tested == subject).filter(
Status.status.notin_(
PyFunceble.core.CLI.get_up_statuses())).count() >
0)
if result:
PyFunceble.LOGGER.info(
f"{subject} is present into the database.")
else:
PyFunceble.LOGGER.info(
f"{subject} is not present into the database.")
return result
return False # pragma: no cover
def __getitem__(self, index):
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
if index in self.database:
return self.database[index]
return None
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
fetched = None
with session.Session() as db_session:
try:
# pylint: disable=no-member
fetched = (db_session.query(WhoisRecord).filter(
WhoisRecord.subject == index)).one()
return {
"epoch": fetched.epoch,
"expiration_date": fetched.expiration_date,
"state": fetched.state,
"record": fetched.record,
}
except NoResultFound:
pass
return None # pragma: no cover
def __contains__(self, subject):
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
if subject not in self.is_present_cache:
self.is_present_cache[subject] = False
if self[subject]:
self.is_present_cache[subject] = True
return self.is_present_cache[subject]
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
try:
# pylint: disable=no-member
data = (db_session.query(Status).join(File).filter(
File.path == self.filename
).filter(Status.tested == subject).filter(
Status.status.notin_(
PyFunceble.core.CLI.get_up_statuses())).one())
except NoResultFound:
data = []
if data:
return True
return False # pragma: no cover
def __contains__(self, index):
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
if index in self.database:
PyFunceble.LOGGER.info(
f"{index} is present into the database.")
return True
PyFunceble.LOGGER.info(
f"{index} is not present into the database.")
return False
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
try:
# pylint: disable=no-member
_ = (db_session.query(WhoisRecord).filter(
WhoisRecord.subject == index)).one()
PyFunceble.LOGGER.info(
f"{index} is present into the database.")
return True
except NoResultFound:
PyFunceble.LOGGER.info(
f"{index} is not present into the database.")
return False # pragma: no cover
def __delitem__(self, index):
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
actual_value = self[index]
if actual_value:
del self.database[self.filename][index]
PyFunceble.LOGGER.info(
"Cleaned the data related to "
f"{repr(index)} and {repr(self.filename)} "
f"from the database.")
elif PyFunceble.CONFIGURATION.db_type in [
"mariadb",
"mysql",
]: # pragma: no cover
with session.Session() as db_session:
# pylint: disable=no-member
status = (db_session.query(Status).filter(
Status.tested == index).one())
for mined in status.mined:
db_session.delete(mined)
db_session.commit()
PyFunceble.LOGGER.info(
"Cleaned the data related to "
f"{repr(index)} and {repr(self.filename)} "
f"from the database.")
def update_counters(self): # pragma: no cover
"""
Updates the counters.
"""
if self.authorized and self.parent:
# We are authorized to operate.
# We create a list of all status we are working with.
statuses = PyFunceble.STATUS.official.keys()
# We preset the number of tested.
tested = 0
for status in statuses:
# We loop through the list of status.
if PyFunceble.CONFIGURATION.db_type == "json":
try:
# We get the number of tested of the currently read
# status.
tested_for_status = len(self.database[self.filename][
PyFunceble.STATUS.official[status]])
# We then update/transfert it to its global place.
PyFunceble.INTERN["counter"]["number"][
status] = tested_for_status
PyFunceble.LOGGER.debug(
f"Counter of {repr(status)} set to {tested_for_status}."
)
# We finally increate the number of tested.
tested += tested_for_status
except KeyError:
PyFunceble.INTERN["counter"]["number"][status] = 0
continue
elif PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
result = (db_session.query(Status).join(File).filter(
File.path == self.filename).filter(
Status.status ==
PyFunceble.STATUS.official[status]).filter(
Status.test_completed == True).all())
fetched = len(result)
PyFunceble.INTERN["counter"]["number"][
status] = fetched
PyFunceble.LOGGER.debug(
f"Counter of {repr(status)} set to {fetched}.")
# We then update/transfer it to its global place.
tested += fetched
# We update/transfert the number of tested globally.
PyFunceble.INTERN["counter"]["number"]["tested"] = tested
PyFunceble.LOGGER.debug(f"Totally tested set to {repr(tested)}.")
def list_of_mined(self):
"""
Provides the list of mined domains so that they can
be tested.
:return:
The list of mined domains.
The returned format is the following:
::
[
(index_to_delete_after_test, mined),
(index_to_delete_after_test, mined),
(index_to_delete_after_test, mined)
]
:rtype: list
"""
PyFunceble.LOGGER.info(
"Getting the list of previously mined data. (DATASET WONT BE LOGGED)"
)
# We initiate a variable which will return the result.
result = []
if self.authorized:
# We are authorized to operate.
if PyFunceble.CONFIGURATION.db_type == "json":
for subject in self.database[self.filename].keys():
# We loop through the available list of status
# from the database.
for element in self[subject]:
# We then loop through the data associatied to
# the currently read status.
result.append((subject, element))
elif PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
# pylint: disable=no-member
fetched = (
db_session.query(Mined)
.join(File)
.join(Status)
.filter(File.id == Mined.file_id)
.filter(Status.id == Mined.subject_id)
.all()
)
if fetched:
result = [(x.subject.tested, x.mined) for x in fetched]
# We return the result.
return result
def __setitem_mysql(cls, index, value):
with session.Session() as db_session:
try:
# pylint: disable=no-member
record = (db_session.query(WhoisRecord).filter(
WhoisRecord.subject == index).one())
except NoResultFound:
record = WhoisRecord(subject=index, )
for db_key, db_value in value.items():
if not PyFunceble.CONFIGURATION.store_whois_record and db_key == "record":
continue
setattr(record, db_key, db_value)
with session.Session() as db_session:
# pylint: disable=no-member
db_session.add(record)
db_session.commit()
PyFunceble.LOGGER.info(f"Inserted into the database: \n {value}")
def __getitem__(self, index):
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
if index in self.database[self.filename]:
return self.database[self.filename][index]
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
# pylint: disable=no-member
fetched = (db_session.query(Mined).join(Status).filter(
Status.id == Mined.subject_id).all())
if fetched:
return {x.mined for x in fetched}
return None
def get_to_retest(self):
"""
Returns a set of subject to restest.
"""
PyFunceble.LOGGER.info(
"Getting the list of subjects to retest (DATASET WONT BE LOGGED)")
if (self.authorized
and PyFunceble.CONFIGURATION.days_between_db_retest >= 0
and self.filename in self.database):
if PyFunceble.CONFIGURATION.db_type == "json":
result = set()
if (PyFunceble.CONFIGURATION.multiprocess
and get_start_method() == "spawn"): # pragma: no cover
self.load()
for subject, info in self.database[self.filename].items():
if ("last_retested_at_epoch" in info
and info["last_retested_at_epoch"]):
if (datetime.utcnow() > datetime.fromtimestamp(
info["last_retested_at_epoch"]) + self.days):
result.add(subject)
else:
result.add(subject)
return result
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
try:
# pylint: disable=no-member
result = (db_session.query(Status).join(File).filter(
File.path == self.filename).filter(
Status.status.notin_(
PyFunceble.core.CLI.get_up_statuses())).
filter(datetime.utcnow() > Status.tested_at +
self.days).all())
except NoResultFound:
result = []
if result:
return {x.tested for x in result}
return set()
def __contains__(self, index): # pragma: no cover
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
if (PyFunceble.CONFIGURATION.multiprocess
and get_start_method() == "spawn"): # pragma: no cover
self.load()
if self.filename in self.database:
for status, status_data in self.database[
self.filename].items():
if status == "complements":
continue
if index in status_data:
PyFunceble.LOGGER.info(
f"{index} is present into the database.")
return True
PyFunceble.LOGGER.info(
f"{index} is not present into the database.")
return False
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
result = (db_session.query(Status).join(File).filter(
Status.tested == index).filter(
File.path == self.filename).count() > 0)
if result:
PyFunceble.LOGGER.info(
f"{index} is present into the database.")
else:
PyFunceble.LOGGER.info(
f"{index} is not present into the database.")
return result
PyFunceble.LOGGER.info(
f"Could not check if {index} is present into the database. "
"Unauthorized action.")
return False
def is_empty(self):
"""
Checks if the database related to the currently tested
file is empty.
"""
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
if (self.filename not in self.database
or not self.database[self.filename]):
PyFunceble.LOGGER.info(
"File to test was not previously indexed.")
return True
PyFunceble.LOGGER.info("File to test was previously indexed.")
return False
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
# Here we don't really check if it is empty.
# What we do, is that we check that everything was
# tested.
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
try:
result = (db_session.query(File).filter(
File.path == self.filename).one())
PyFunceble.LOGGER.info(
f"File was completely tested: {not result.test_completed}"
)
return result.test_completed
except NoResultFound:
pass
PyFunceble.LOGGER.info( # pragma: no cover
"Could not check if the file to test "
"was previously indexed. Unauthorized action.")
return False # pragma: no cover
def __contains__(self, index): # pragma: no cover
if self.authorized:
if PyFunceble.CONFIGURATION.db_type == "json":
if self.filename in self.database:
for status, status_data in self.database[
self.filename].items():
if status == "complements":
continue
if index in status_data:
PyFunceble.LOGGER.info(
f"{index} is present into the database.")
return True
PyFunceble.LOGGER.info(
f"{index} is not present into the database.")
return False
if PyFunceble.CONFIGURATION.db_type in ["mariadb", "mysql"]:
with session.Session() as db_session:
try:
# pylint: disable=no-member, singleton-comparison
_ = (db_session.query(Status).join(File).filter(
Status.tested == index).filter(
File.path == self.filename).one())
except NoResultFound:
PyFunceble.LOGGER.info(
f"{index} is not present into the database.")
return False
PyFunceble.LOGGER.info(
f"{index} is present into the database.")
return True
PyFunceble.LOGGER.info(
f"Could not check if {index} is present into the database. "
"Unauthorized action.")
return False
def almost_everything(self, clean_all=False, file_path=False):
"""
Delete almost all discovered files.
:param bool clean_all:
Tell the subsystem if we have to clean everything instesd
of almost everything.
"""
if (
"do_not_clean" not in PyFunceble.INTERN
or not PyFunceble.INTERN["do_not_clean"]
):
# We get the list of file to delete.
to_delete = self.file_to_delete(clean_all)
if (
not PyFunceble.abstracts.Version.is_local_cloned() and clean_all
): # pragma: no cover
to_delete.extend(self.databases_to_delete())
for file in to_delete:
# We loop through the list of file to delete.
# And we delete the currently read file.
PyFunceble.helpers.File(file).delete()
PyFunceble.LOGGER.info(f"Deleted: {file}")
if PyFunceble.CONFIGURATION.db_type in [
"mariadb",
"mysql",
]: # pragma: no cover
if file_path:
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
to_delete = (
db_session.query(Status)
.join(File)
.filter(File.path == file_path)
.filter(File.test_completed == True)
.filter(
Status.status.in_(PyFunceble.core.CLI.get_up_statuses())
)
.all()
)
for row in to_delete:
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
delete_query = Status.__table__.delete().where(
Status.id == row.id
)
db_session.execute(delete_query)
db_session.commit()
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
file_object = (
db_session.query(File).filter(File.path == file_path).one()
)
file_object.test_completed = False
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
db_session.add(file_object)
db_session.commit()
else:
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
to_delete = db_session.query( # pylint: disable=no-member
File
).all()
for row in to_delete:
# pylint: disable=no-member
with session.Session() as db_session:
# pylint: disable=no-member, singleton-comparison
delete_query = File.__table__.delete().where(
File.id == row.id
)
db_session.execute(delete_query)
db_session.commit()
if (
not PyFunceble.abstracts.Version.is_local_cloned() and clean_all
): # pragma: no cover
PyFunceble.load_config()
PyFunceble.LOGGER.info("Reloaded configuration.")
