def adminInfo(output=None):
return {
"stats": getDBStats(True),
"plugins": plugManager.getPlugins(),
"updateOutput": filterUpdateField(output),
"signalPhoneRegistered":
getInfo('subscriptions')['registeredPhone'] != '',
"watchlistInfos": getInfo('watchlist'),
"subscriptionsInfos": getInfo('subscriptions')
}
def get_via4_indexes(self):
via4 = getInfo("via4")
result = []
if via4:
for index in via4.get("searchables", []):
result.append(("via4", index))
return result
def download_site(self, url):
self.logger.debug("Downloading from url: {}".format(url))
session = self.get_session()
try:
with session.get(url) as response:
try:
self.last_modified = parse_datetime(
response.headers["last-modified"], ignoretz=True)
except KeyError:
self.logger.error(
"Did not receive last-modified header in the response; setting to default "
"(01-01-1970) and force update! Headers received: {}".
format(response.headers))
# setting to last_modified to default value
self.last_modified = parse_datetime("01-01-1970")
self.logger.debug(
"Last {} modified value: {} for URL: {}".format(
self.feed_type, self.last_modified, url))
i = getInfo(self.feed_type.lower())
if i is not None:
if self.last_modified == i["last-modified"]:
self.logger.info(
"{}'s are not modified since the last update".
format(self.feed_type))
self.file_queue.get_full_list()
self.do_process = False
if self.do_process:
content_type = response.headers["content-type"]
self.logger.debug(
"URL: {} fetched Content-Type: {}".format(
url, content_type))
wd, filename = self.store_file(
response_content=response.content,
content_type=content_type,
url=url,
)
if filename is not None:
self.file_queue.put((wd, filename))
else:
self.logger.error(
"Unable to retrieve a filename; something went wrong during the storing of the file"
)
sys.exit(1)
except Exception as err:
self.logger.info(
"Exception encountered during download from: {}. Please check the logs for more information!"
.format(url))
self.logger.error(
"Exception encountered during the download from: {}. Error encountered: {}"
.format(url, err))
sys.exit(1)
def create_indexes(self):
for index in self.iter_indexes():
self.setIndex(index[0], index[1])
via4 = getInfo("via4")
if via4:
for index in via4.get("searchables", []):
self.setIndex("via4", index, False)
def sendNotification(cve, new=False, signal_groups=[]):
registeredPhone = getInfo('subscriptions')['registeredPhone']
message = _makeMessage(cve, new)
if (message is not None and message is not "Error"):
with open(runPath + '/../log/notification_logs', 'a+') as f:
f.write(str(datetime.datetime.now()) + ' ' + message + '\n')
if signal_groups is None:
return
for group in signal_groups:
subprocess.run([
"signal-cli", "-u", registeredPhone, "send", "-m", message,
"-g", group
])
def download_site(self, url):
self.logger.debug("Downloading from url: {}".format(url))
session = self.get_session()
with session.get(url) as response:
self.last_modified = parse_datetime(
response.headers["last-modified"], ignoretz=True
)
self.logger.debug(
"Last {} modified value: {} for URL: {}".format(self.feed_type, self.last_modified, url)
)
i = getInfo(self.feed_type.lower())
if i is not None:
if self.last_modified == i["last-modified"]:
self.logger.info(
"{}'s are not modified since the last update".format(
self.feed_type
)
)
self.file_queue.get_full_list()
self.do_process = False
if self.do_process:
content_type = response.headers["content-type"]
self.logger.debug("URL: {} fetched Content-Type: {}".format(url, content_type))
wd, filename = self.store_file(
response_content=response.content, content_type=content_type, url=url
)
if filename is not None:
self.file_queue.put((wd, filename))
else:
self.logger.error(
"Unable to retrieve a filename; something went wrong during the storing of the file"
)
sys.exit(1)
def _update_watchlist(self):
if isBlacklisted(self.doc):
return
infos = getInfo('watchlist')
cve = dict(self.doc)
done = False
for entity in getAllEntities():
found = False
for cpe in entity['cpes']:
if found:
break
for vc in cve['vulnerable_configuration']:
if cpe in vc:
#cve is impacting an entity cpe
done = True
found = True
wcve = getWatchlistCVE(cve['id'], entity['name'], cpe)
if wcve and wcve['cvss3'] == cve['cvss3']:
updateWatchlistCVELastmodified(
{
'id': wcve['id'],
'entity': entity['name'],
'product': cpe
}, cve['last-modified'])
break
cve['entity'] = entity['name']
cve['product'] = cpe
cve['treated'] = False
if infos is None or cve['cvss3'] is None or (
cve['cvss3'] and cve['cvss3'] >= float(
infos['w_threshold_entity'])):
updateWatchlistCVE(cve)
if infos is None or cve['cvss3'] is None or (
cve['cvss3'] and cve['cvss3'] >= float(
infos['threshold_entity'])):
sendNotification(
cve,
new=True if wcve is None else False,
signal_groups=self.signal_groups)
if done:
return
#cve is not impacting an entity
wcve = getWatchlistCVEbyID(cve['id'])
if wcve and wcve['cvss3'] == cve['cvss3']:
updateWatchlistCVELastmodified({'id': wcve['id']},
cve['last-modified'])
return
cve['entity'] = 'all'
cve['product'] = cve['vulnerable_configuration'][0] if cve[
'vulnerable_configuration'] else 'unknown'
cve['treated'] = False
if infos is None or cve['cvss3'] is None or (
cve['cvss3']
and cve['cvss3'] >= float(infos['w_threshold_critical'])):
updateWatchlistCVE(cve)
if infos is None or (cve['cvss3'] and cve['cvss3'] >= float(
infos['threshold_critical'])):
sendNotification(cve,
new=True if wcve is None else False,
signal_groups=self.signal_groups)
def admin_home():
if getInfo('subscriptions') is None or 'registeredPhone' not in getInfo('subscriptions'):
setColInfo('subscriptions', 'registeredPhone', '')
return render_template("admin.html", status="default", **adminInfo())
def generate_minimal_query(f):
query = []
# retrieving lists
if f["rejectedSelect"] == "hide":
query.append({
"summary":
re.compile(
r"^(?!\*\* REJECT \*\*\s+DO NOT USE THIS CANDIDATE NUMBER.*)")
})
# cvss / cvss3 logic
if f["cvssVersion"] == "V2":
cvss_filter_field = "cvss"
else:
cvss_filter_field = "cvss3"
if f["cvssSelect"] == "above":
query.append({cvss_filter_field: {"$gt": float(f["cvss"])}})
elif f["cvssSelect"] == "equals":
query.append({cvss_filter_field: float(f["cvss"])})
elif f["cvssSelect"] == "below":
query.append({cvss_filter_field: {"$lt": float(f["cvss"])}})
# date logic
if f["timeSelect"] != "all":
if f["startDate"]:
startDate = parse_datetime(f["startDate"],
ignoretz=True,
dayfirst=True)
if f["endDate"]:
endDate = parse_datetime(f["endDate"],
ignoretz=True,
dayfirst=True)
if f["timeSelect"] == "from":
query.append({f["timeTypeSelect"]: {"$gt": startDate}})
elif f["timeSelect"] == "until":
query.append({f["timeTypeSelect"]: {"$lt": endDate}})
elif f["timeSelect"] == "between":
query.append(
{f["timeTypeSelect"]: {
"$gt": startDate,
"$lt": endDate
}})
elif f["timeSelect"] == "outside":
query.append({
"$or": [
{
f["timeTypeSelect"]: {
"$lt": startDate
}
},
{
f["timeTypeSelect"]: {
"$gt": endDate
}
},
]
})
# status logic
if f['statusSelect'] != 'all':
if f['statusSelect'] == 'treated_only':
query.append({'treated': True})
if f['statusSelect'] == 'not_treated_only':
query.append({'treated': {'$ne': True}})
# type logic
threshold = 0
infos = getInfo('watchlist')
if infos:
threshold = float(infos['w_threshold_critical'])
if f['typeSelect'] != 'all':
if f['typeSelect'] == 'important_only':
query.append({
'$or': [{
'entity': {
'$ne': 'all'
}
}, {
'cvss3': {
'$gte': threshold
}
}]
})
# if f['typeSelect'] == 'not_important_only':
# query.append({'$and':{'entity': 'all', 'cvss3':{'$lt':threshold}}})
return query
cpd = CPEDownloads()
cpd.populate()
elif args.f and source["name"] == "cve":
cvd = CVEDownloads()
cvd.populate()
else:
if isinstance(source["updater"], str):
subprocess.Popen((shlex.split(source["updater"]))).wait()
else:
if source['name'] == 'cve':
if getInfo(
'subscriptions'
) is None or 'registeredPhone' not in getInfo(
'subscriptions'):
groups = None
else:
registeredPhone = getInfo(
'subscriptions')['registeredPhone']
ret = subprocess.run([
"signal-cli", "-u", registeredPhone,
"listGroups"
],
capture_output=True)
tmp = (ret.stdout.decode()).split('\n')
groups = []
for string in tmp:
if string: