def test_isotime_local():
local_date = now_as_local()
local_format = re.compile(r'[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}.*')
assert isinstance(local_date, str)
assert local_format.match(local_date)
assert epoch_to_local(local_to_epoch(local_date)) == local_date
assert local_date == epoch_to_local(iso_to_epoch(epoch_to_iso(local_to_epoch(local_date))))
def _get_version_map(self):
self.engine_map = {}
engine_list = []
newest_dat = 0
oldest_dat = now()
url = self.cfg.get('BASE_URL') + "stat/engines"
try:
r = self.session.get(url=url, timeout=self.timeout)
except requests.exceptions.Timeout:
raise Exception("Metadefender service timeout.")
engines = r.json()
for engine in engines:
if self.cfg.get("MD_VERSION") == 4:
name = self._format_engine_name(engine["eng_name"])
version = engine['eng_ver']
def_time = engine['def_time']
etype = engine['engine_type']
elif self.cfg.get("MD_VERSION") == 3:
name = self._format_engine_name(engine["eng_name"]).replace(
"scanengine", "")
version = engine['eng_ver']
def_time = engine['def_time'].replace(" AM", "").replace(
" PM", "").replace("/", "-").replace(" ", "T")
def_time = def_time[6:10] + "-" + def_time[:5] + def_time[
10:] + "Z"
etype = engine['eng_type']
else:
raise Exception("Unknown metadefender version")
# Compute newest DAT
dat_epoch = iso_to_epoch(def_time)
if dat_epoch > newest_dat:
newest_dat = dat_epoch
if dat_epoch < oldest_dat and dat_epoch != 0 and etype in [
"av", "Bundled engine"
]:
oldest_dat = dat_epoch
self.engine_map[name] = {
'version': version,
'def_time': iso_to_local(def_time)[:19]
}
engine_list.append(name)
engine_list.append(version)
engine_list.append(def_time)
self.newest_dat = epoch_to_local(newest_dat)[:19]
self.oldest_dat = epoch_to_local(oldest_dat)[:19]
self.dat_hash = hashlib.md5("".join(engine_list)).hexdigest()
def test_isotime_iso():
iso_date = now_as_iso()
iso_format = re.compile(r'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z')
assert isinstance(iso_date, str)
assert iso_format.match(iso_date)
assert epoch_to_iso(iso_to_epoch(iso_date)) == iso_date
assert iso_date == epoch_to_iso(local_to_epoch(epoch_to_local(iso_to_epoch(iso_date))))
def __init__(self, cfg=None):
super(MetaDefender, self).__init__(cfg)
self.dat_hash = "0"
self.engine_map = {}
self.engine_list = []
self.newest_dat = epoch_to_local(0)
self.oldest_dat = now_as_local()
self.session = None
self._updater_id = "ENABLE_SERVICE_BLK_MSG"
self.timeout = cfg.get('MD_TIMEOUT', (self.SERVICE_TIMEOUT * 2) / 3)
self.init_vmap = False
def test_isotime_rounding_error():
for t in ["2020-01-29 18:41:25.758416", "2020-01-29 18:41:25.127600"]:
epoch = local_to_epoch(t)
local = epoch_to_local(epoch)
assert local == t
def test_isotime_epoch():
epoch_date = now(200)
assert epoch_date == local_to_epoch(epoch_to_local(epoch_date))
assert epoch_date == iso_to_epoch(epoch_to_iso(epoch_date))
assert isinstance(epoch_date, float)
def test_get_version_map(metadefender_class_instance):
from requests import Session, exceptions
from assemblyline.common.isotime import epoch_to_local
metadefender_class_instance.session = Session()
node = "http://blah:1/"
metadefender_class_instance.nodes[node] = {}
with requests_mock.Mocker() as m:
m.get(f"{node}stat/engines", status_code=200, json=[])
metadefender_class_instance._get_version_map(node)
metadefender_class_instance.nodes[node].pop("oldest_dat")
assert metadefender_class_instance.nodes[node] == {
"engine_count": 0,
"newest_dat": epoch_to_local(0)[:19],
"engine_list": ""
}
with pytest.raises(Exception):
m.get(f"{node}stat/engines", exc=exceptions.Timeout)
metadefender_class_instance._get_version_map(node)
with pytest.raises(Exception):
m.get(f"{node}stat/engines", exc=exceptions.ConnectionError)
metadefender_class_instance._get_version_map(node)
# MD Version 4
metadefender_class_instance.nodes[node] = {"engine_map": {}}
m.get(f"{node}stat/engines",
status_code=200,
json=[{
"active": False,
"eng_name": "blah",
"eng_ver": "blah",
"def_time": "1999-09-09T12:12:12",
"engine_type": "blah"
}])
metadefender_class_instance._get_version_map(node)
metadefender_class_instance.nodes[node].pop("oldest_dat")
assert metadefender_class_instance.nodes[node] == {
"engine_count": 0,
"newest_dat": '1999-09-09 12:12:14',
"engine_list": 'blahblah1999-09-09T12:12:12',
"engine_map": {
'blah': {
'def_time': '1999-09-09 12:12:14',
'version': 'blah'
}
}
}
m.get(f"{node}stat/engines",
status_code=200,
json=[{
"active": True,
"state": "blah",
"eng_name": "blah",
"eng_ver": "blah",
"def_time": "1999-09-09T12:12:12",
"engine_type": "blah"
}])
metadefender_class_instance._get_version_map(node)
metadefender_class_instance.nodes[node].pop("oldest_dat")
assert metadefender_class_instance.nodes[node] == {
"engine_count": 1,
"newest_dat": '1999-09-09 12:12:14',
"engine_list": 'blahblah1999-09-09T12:12:12',
"engine_map": {
'blah': {
'def_time': '1999-09-09 12:12:14',
'version': 'blah'
}
}
}
# MD Version 3
metadefender_class_instance.config["md_version"] = 3
m.get(f"{node}stat/engines",
status_code=200,
json=[{
"active": False,
"eng_name": "blah",
"eng_ver": "blah",
"def_time": "09-09-1999T12:12:12",
"engine_type": "blah",
"eng_type": "blah"
}])
metadefender_class_instance._get_version_map(node)
metadefender_class_instance.nodes[node].pop("oldest_dat")
assert metadefender_class_instance.nodes[node] == {
"engine_count": 0,
"newest_dat": '1999-09-09 12:12:12',
"engine_list": 'blahblah1999-09-09T12:12:12Z',
"engine_map": {
'blah': {
'def_time': '1999-09-09 12:12:12',
'version': 'blah'
}
}
}
# etype
for etype in ["av", "Bundled engine"]:
m.get(f"{node}stat/engines",
status_code=200,
json=[{
"active": False,
"eng_name": "blah",
"eng_ver": "blah",
"def_time": "09-09-1999T12:12:12",
"engine_type": "blah",
"eng_type": etype
}])
metadefender_class_instance._get_version_map(node)
assert metadefender_class_instance.nodes[node] == {
"engine_count": 0,
"newest_dat": '1999-09-09 12:12:12',
"engine_list": 'blahblah1999-09-09T12:12:12Z',
"engine_map": {
'blah': {
'def_time': '1999-09-09 12:12:12',
'version': 'blah'
}
},
"oldest_dat": '1999-09-09 12:12:12',
}
# Invalid MD Version
metadefender_class_instance.config["md_version"] = 2
with pytest.raises(Exception):
m.get(f"{node}stat/engines",
status_code=200,
json=[{
"active": False,
"eng_name": "blah",
"eng_ver": "blah",
"def_time": "09-09-1999T12:12:12",
"engine_type": "blah",
"eng_type": "blah"
}])
metadefender_class_instance._get_version_map(node)
# Failed states
metadefender_class_instance.config["md_version"] = 4
for state in ["removed", "temporary failed", "permanently failed"]:
m.get(f"{node}stat/engines",
status_code=200,
json=[{
"active": False,
"state": state,
"eng_name": "blah",
"eng_ver": "blah",
"def_time": "1999-09-09T12:12:12",
"engine_type": "blah"
}])
metadefender_class_instance._get_version_map(node)
metadefender_class_instance.nodes[node].pop("oldest_dat")
assert metadefender_class_instance.nodes[node] == {
"engine_count": 0,
"newest_dat": '1999-09-09 12:12:14',
"engine_list": 'blahblah1999-09-09T12:12:12',
"engine_map": {
'blah': {
'def_time': '1999-09-09 12:12:14',
'version': 'blah'
}
}
}
def start(self) -> None:
self.log.debug("MetaDefender service started")
base_urls: List[str] = []
if type(self.config.get("base_url")) == str:
base_urls = [self.config.get("base_url")]
elif type(self.config.get("base_url")) == list:
for base_url in self.config.get("base_url"):
prepared_base_url = base_url.replace(" ", "")
base_urls.append(prepared_base_url)
else:
raise Exception(
"Invalid format for BASE_URL service variable (must be str or list)"
)
av_config: Dict[str, Any] = self.config.get("av_config", {})
self.blocklist: List[str] = av_config.get("blocklist", [])
self.kw_score_revision_map: Dict[str, int] = av_config.get(
"kw_score_revision_map", {})
self.sig_score_revision_map: Dict[str, int] = av_config.get(
"sig_score_revision_map", {})
# Initialize a list of all nodes with default data
for index, url in enumerate(base_urls):
self.nodes[url] = {
'engine_map': {},
'engine_count': 0,
'engine_list': "default",
'newest_dat': epoch_to_local(0),
'oldest_dat': now_as_local(),
'file_count': 0,
'queue_times': [],
'average_queue_time': 0
}
# Get version map for all of the nodes
self.session = Session()
engine_count = 0
for node in list(self.nodes.keys()):
try:
self._get_version_map(node)
except Exception as e:
self.log.error(
f"Unable to contact {node} due to {e}. Removing from node list."
)
del self.nodes[node]
continue
engine_count += self.nodes[node]['engine_count']
if len(list(self.nodes.keys())) == 0:
raise Exception(
"All MetaDefender Core nodes are down. Please ensure that the URLs are correct in the "
"service config and that the MetaDefender Core REST API is running at each one."
)
if engine_count == 0:
raise Exception(
f"MetaDefender Core nodes {list(self.nodes.keys())} have an active engine_count of 0"
)
# On first launch, choose random node to start with
if not self.current_node:
while True:
self.current_node = random.choice(list(self.nodes.keys()))
# Check to see if the chosen node has a version map, else try to get version map again
if self.nodes[self.current_node]['engine_count'] >= 1:
self.log.info(
f"Node ({self.current_node}) chosen at launch")
break
else:
self._get_version_map(self.current_node)
# Start the global timer
if not self.start_time:
self.start_time = time.time()
def _get_version_map(self, node: str) -> None:
"""
Get the versions of all engines running on a given node
:param node: The IP of the MetaDefender node
:return: None
"""
newest_dat = 0
oldest_dat = now()
engine_list = []
active_engines = 0
failed_states = ["removed", "temporary failed", "permanently failed"]
url = urljoin(node, 'stat/engines')
try:
self.log.debug(f"_get_version_map: GET {url}")
r = self.session.get(url=url, timeout=self.timeout)
engines = r.json()
for engine in engines:
if engine['active'] and engine["state"] not in failed_states:
active_engines += 1
if self.config.get("md_version") == 4:
name = self._format_engine_name(engine["eng_name"])
version = engine['eng_ver']
def_time = engine['def_time']
etype = engine['engine_type']
elif self.config.get("md_version") == 3:
name = self._format_engine_name(
engine["eng_name"]).replace("scanengine", "")
version = engine['eng_ver']
def_time = engine['def_time'].replace(" AM", "").replace(
" PM", "").replace("/", "-").replace(" ", "T")
def_time = def_time[6:10] + "-" + def_time[:5] + def_time[
10:] + "Z"
etype = engine['eng_type']
else:
raise Exception("Unknown version of MetaDefender")
# Compute newest DAT
dat_epoch = iso_to_epoch(def_time)
if dat_epoch > newest_dat:
newest_dat = dat_epoch
if dat_epoch < oldest_dat and dat_epoch != 0 and etype in [
"av", "Bundled engine"
]:
oldest_dat = dat_epoch
self.nodes[node]['engine_map'][name] = {
'version': version,
'def_time': iso_to_local(def_time)[:19]
}
engine_list.append(name)
engine_list.append(version)
engine_list.append(def_time)
self.nodes[node]['engine_count'] = active_engines
self.nodes[node]['newest_dat'] = epoch_to_local(newest_dat)[:19]
self.nodes[node]['oldest_dat'] = epoch_to_local(oldest_dat)[:19]
self.nodes[node]['engine_list'] = "".join(engine_list)
except exceptions.Timeout:
raise Exception(
f"Node ({node}) timed out after {self.timeout}s while trying to get engine version map"
)
except ConnectionError:
raise Exception(
f"Unable to connect to node ({node}) while trying to get engine version map"
)
