def test_unit_upload_annotate_normalize(service_unit, file_hydra_1_binary):
file = werkzeug.datastructures.FileStorage(stream=BytesIO(file_hydra_1_binary), content_type="application/vnd.tcpdump.pcap", filename="file.pcap")
unit1, _ = service_unit.unit_upload(file)
unit2 = service_unit.unit_annotate(unit1.id_unit, "Unit #1", "Desc unit #1", ["L1", "L2"])
assert unit1.id_unit == unit2.id_unit
annotated_unit = service_unit.unit_normalize(
id_unit=unit1.id_unit,
mac_mapping=Mapping.create_from_dict([
{
"original": "00:A0:C9:14:C8:29",
"replacement": ["1.2.3.4"]
}
]),
ip_details=IPDetails(
target_nodes=[
"1.2.3.4"
],
intermediate_nodes=[],
source_nodes=[]
),
tcp_timestamp_mapping=Mapping()
)
assert annotated_unit.id_annotated_unit
assert annotated_unit.name == "Unit #1"
assert annotated_unit.description == "Desc unit #1"
def test_find(service_unit):
file = werkzeug.datastructures.FileStorage(content_type="application/vnd.tcpdump.pcap", filename="dump.pcap")
unit, _ = service_unit.unit_upload(file)
units = service_unit.get_units()
assert len(units) == 1
assert units[0].stage == "upload"
_ = service_unit.unit_annotate(unit.id_unit, "Unit #1", "Desc unit #1", ["L1", "L2"])
units = service_unit.get_units()
assert len(units) == 1
assert units[0].stage == "annotate"
service_unit.unit_normalize(
id_unit=unit.id_unit,
ip_mapping=Mapping(),
mac_mapping=Mapping(),
ip_details=IPDetails([], [], []),
timestamp=123456.12
)
units = service_unit.get_units()
assert len(units) == 0
def create_ann_unit(service_unit, name, labels=None):
if not labels:
labels = ["IMPORTANT", "SECOND_LABEL"]
file = werkzeug.datastructures.FileStorage(
stream=BytesIO(get_hydra_file()),
content_type="application/vnd.tcpdump.pcap",
filename="file.pcap")
unit1, _ = service_unit.unit_upload(file)
service_unit.unit_annotate(unit1.id_unit, name, "Description %s" % name,
labels)
annotated_unit = service_unit.unit_normalize(
id_unit=unit1.id_unit,
mac_mapping=Mapping.create_from_dict([{
"mac": "00:A0:C9:14:C8:29",
"ips": ["1.2.3.4"]
}],
keys=("mac", "ips")),
ip_details=IPDetails(target_nodes=["1.2.3.4"],
intermediate_nodes=[],
source_nodes=[]),
tcp_timestamp_mapping=Mapping.create_from_dict([], keys=("ip", "min")))
return annotated_unit
def test_invalid_unit_normalize(service_unit):
with pytest.raises(UnitDoesntExistsException):
service_unit.unit_normalize(
id_unit=123456789,
mac_mapping=Mapping(),
ip_details=IPDetails([], [], []),
tcp_timestamp_mapping=Mapping()
)
def start_mix_generation(self, id_mix):
"""
Start async mix file generation
:param id_mix: id of existing mix
:return: id_mix_generation
"""
mix = self.get_mix(id_mix)
if not mix:
raise MixDoesntExistsException(id_mix)
annotated_units_data = []
for origin in mix.origins:
annotated_units_data.append(
dict(
id_annotated_unit=origin.id_annotated_unit,
ip_mapping=Mapping.create_from_dict(
json.loads(origin.ip_mapping)),
mac_mapping=Mapping.create_from_dict(
json.loads(origin.mac_mapping)),
port_mapping=json.loads(origin.port_mapping),
at_timestamp=origin.timestamp,
))
if not (all([
self._exits_ann_unit(ann_unit["id_annotated_unit"])
for ann_unit in annotated_units_data
])):
raise AnnotatedUnitDoesntExistsException()
mix_generation = ModelMixFileGeneration(
id_mix=mix.id_mix,
creation_time=datetime.now(),
expired=False,
progress=0,
)
self._session.add(mix_generation)
self._session.commit()
id_mix_generation = mix_generation.id_mix_generation
self._session_maker.remove()
self._engine.dispose()
p = multiprocessing.Process(target=self.generate_mix,
args=(id_mix_generation,
annotated_units_data))
p.daemon = True
p.start()
p.join()
return id_mix_generation
def create_ann_unit(service_unit, name, labels=None):
if not labels:
labels = ["IMPORTANT", "SECOND_LABEL"]
file = werkzeug.datastructures.FileStorage(
stream=BytesIO(get_hydra_file()),
content_type="application/vnd.tcpdump.pcap",
filename="file.pcap"
)
unit1, _ = service_unit.unit_upload(file)
service_unit.unit_annotate(
unit1.id_unit, name, "Description %s" % name, labels
)
annotated_unit = service_unit.unit_normalize(
id_unit=unit1.id_unit,
ip_mapping=Mapping.create_from_dict([
{
"original": "1.2.3.4",
"replacement": "172.16.0.0"
}
]),
mac_mapping=Mapping.create_from_dict([
{
"original": "00:A0:C9:14:C8:29",
"replacement": "00:A0:C9:14:C8:29"
}
]),
ip_details=IPDetails(
target_nodes=[
"172.16.0.0"
],
intermediate_nodes=[
"172.16.0.0"
],
source_nodes=[
"172.16.0.0"
]
),
timestamp=1541346574.1234
)
return annotated_unit
def test_unit_normalize_invalid_ip_details(service_unit, file_hydra_1_binary):
file = werkzeug.datastructures.FileStorage(stream=BytesIO(file_hydra_1_binary), content_type="application/vnd.tcpdump.pcap", filename="file.pcap")
unit1, _ = service_unit.unit_upload(file)
unit2 = service_unit.unit_annotate(unit1.id_unit, "Unit #1", "Desc unit #1", ["L1", "L2"])
assert unit1.id_unit == unit2.id_unit
with pytest.raises(IPDetailsUnknownIPException):
service_unit.unit_normalize(
id_unit=unit1.id_unit,
ip_mapping=Mapping.create_from_dict([
{
"original": "1.2.3.4",
"replacement": "172.16.0.0"
}
]),
mac_mapping=Mapping.create_from_dict([]),
ip_details=IPDetails(
target_nodes=["172.16.0.1"],
intermediate_nodes=["172.16.0.0"],
source_nodes=["172.16.0.0"]
),
timestamp=1541346574.1234
)
with pytest.raises(IPDetailsUnknownIPException):
service_unit.unit_normalize(
id_unit=unit1.id_unit,
ip_mapping=Mapping.create_from_dict([
{
"original": "1.2.3.4",
"replacement": "172.16.0.0"
}
]),
mac_mapping=Mapping.create_from_dict([]),
ip_details=IPDetails(
target_nodes=["172.16.0.0"],
intermediate_nodes=["172.16.0.1"],
source_nodes=["172.16.0.0"]
),
timestamp=1541346574.1234
)
with pytest.raises(IPDetailsUnknownIPException):
service_unit.unit_normalize(
id_unit=unit1.id_unit,
ip_mapping=Mapping.create_from_dict([
{
"original": "1.2.3.4",
"replacement": "172.16.0.0"
}
]),
mac_mapping=Mapping.create_from_dict([]),
ip_details=IPDetails(
target_nodes=["172.16.0.0"],
intermediate_nodes=["172.16.0.0"],
source_nodes=["172.16.0.1"]
),
timestamp=1541346574.1234
)
annotated_unit = service_unit.unit_normalize(
id_unit=unit1.id_unit,
ip_mapping=Mapping.create_from_dict([
{
"original": "1.2.3.4",
"replacement": "172.16.0.0"
}
]),
mac_mapping=Mapping.create_from_dict([]),
ip_details=IPDetails(
target_nodes=["172.16.0.0"],
intermediate_nodes=["172.16.0.0"],
source_nodes=["172.16.0.0"]
),
timestamp=1541346574.1234
)
assert annotated_unit.id_annotated_unit
def test_unit_normalize_invalid_ip_details(service_unit, file_hydra_1_binary):
file = werkzeug.datastructures.FileStorage(stream=BytesIO(file_hydra_1_binary), content_type="application/vnd.tcpdump.pcap", filename="file.pcap")
unit1, _ = service_unit.unit_upload(file)
unit2 = service_unit.unit_annotate(unit1.id_unit, "Unit #1", "Desc unit #1", ["L1", "L2"])
assert unit1.id_unit == unit2.id_unit
# Outdated test, io_mapping no longer
# with pytest.raises(IPDetailsUnknownIPException):
# service_unit.unit_normalize(
# id_unit=unit1.id_unit,
# ip_mapping=Mapping.create_from_dict([
# {
# "original": "1.2.3.4",
# "replacement": "172.16.0.0"
# }
# ]),
# mac_mapping=Mapping.create_from_dict([]),
# ip_details=IPDetails(
# target_nodes=["172.16.0.1"],
# intermediate_nodes=["172.16.0.0"],
# source_nodes=["172.16.0.0"]
# ),
# timestamp=Mapping()
# )
# with pytest.raises(IPDetailsUnknownIPException):
# service_unit.unit_normalize(
# id_unit=unit1.id_unit,
# ip_mapping=Mapping.create_from_dict([
# {
# "original": "1.2.3.4",
# "replacement": "172.16.0.0"
# }
# ]),
# mac_mapping=Mapping.create_from_dict([]),
# ip_details=IPDetails(
# target_nodes=["172.16.0.0"],
# intermediate_nodes=["172.16.0.1"],
# source_nodes=["172.16.0.0"]
# ),
# timestamp=1541346574.1234
# )
# with pytest.raises(TraceNormalizerError):
# service_unit.unit_normalize(
# id_unit=unit1.id_unit,
# mac_mapping=Mapping.create_from_dict([]),
# ip_details=IPDetails(
# target_nodes=["1.2.3.4"],
# intermediate_nodes=[],
# source_nodes=[]
# ),
# tcp_timestamp_mapping=Mapping()
# )
annotated_unit = service_unit.unit_normalize(
id_unit=unit1.id_unit,
mac_mapping=Mapping.create_from_dict([]),
ip_details=IPDetails(
target_nodes=["1.2.3.4"],
intermediate_nodes=[],
source_nodes=[]
),
tcp_timestamp_mapping=Mapping()
)
assert annotated_unit.id_annotated_unit