def test_basic_data(self):
data = [
{
'one': {
'device': {
'key1': 'value1',
'key2': 'value2',
},
},
'two': {
'device': {
'key1': 'value1',
'key2': 'value2',
},
},
'database': {
'device': {
'key2': 'value2',
},
},
},
{
'three': {
'device': {
'key1': 'value2',
'key3': 'value3',
},
},
},
]
merged = merge_data(*data)
self.assertEqual(
merged, {
'key1': {
('one', 'two'): 'value1',
('three', ): 'value2',
},
'key2': {
('database', 'one', 'two'): 'value2',
},
'key3': {
('three', ): 'value3',
},
})
merged = merge_data(*data, only_multiple=True)
self.assertEqual(
merged, {
'key1': {
('one', 'two'): 'value1',
('three', ): 'value2',
},
'key3': {
('three', ): 'value3',
},
})
def test_basic_data(self):
data = [
{
'one': {
'device': {
'key1': 'value1',
'key2': 'value2',
},
},
'two': {
'device': {
'key1': 'value1',
'key2': 'value2',
},
},
},
{
'three': {
'device': {
'key1': 'value2',
'key3': 'value3',
},
},
},
]
merged = merge_data(*data)
self.assertEqual(merged, {
'key1': {
('one', 'two'): 'value1',
('three',): 'value2',
},
'key2': {
('one', 'two'): 'value2',
},
'key3': {
('three',): 'value3',
},
})
merged = merge_data(*data, only_multiple=True)
self.assertEqual(merged, {
'key1': {
('one', 'two'): 'value1',
('three',): 'value2',
},
})
def test_basic_data(self):
data = [
{
"one": {"device": {"key1": "value1", "key2": "value2"}},
"two": {"device": {"key1": "value1", "key2": "value2"}},
"database": {"device": {"key2": "value2"}},
},
{"three": {"device": {"key1": "value2", "key3": "value3"}}},
]
merged = merge_data(*data)
self.assertEqual(
merged,
{
"key1": {("one", "two"): "value1", ("three",): "value2"},
"key2": {("database", "one", "two"): "value2"},
"key3": {("three",): "value3"},
},
)
merged = merge_data(*data, only_multiple=True)
self.assertEqual(
merged, {"key1": {("one", "two"): "value1", ("three",): "value2"}, "key3": {("three",): "value3"}}
)
def _append_shares_to_device(ip, device, data, external_priorities={}):
device_data = get_device_data(device)
full_data = merge_data(
data,
{
'database': {'device': device_data},
},
only_multiple=True,
)
append_merged_proposition(full_data, device, external_priorities)
selected_data = select_data(full_data, external_priorities)
for share_mount in selected_data.get('disk_shares', []):
_create_or_update_share_mount(ip, device, share_mount)
def _append_shares_to_device(ip, device, data, external_priorities={}):
device_data = get_device_data(device)
full_data = merge_data(
data,
{
'database': {'device': device_data},
},
only_multiple=True,
)
append_merged_proposition(full_data, device, external_priorities)
selected_data = select_data(full_data, external_priorities)
parsed_mounts = set()
for share_mount in selected_data.get('disk_shares', []):
status, mount = _create_or_update_share_mount(ip, device, share_mount)
if mount:
parsed_mounts.add(mount.pk)
device.disksharemount_set.exclude(pk__in=parsed_mounts).delete()
def _append_shares_to_device(ip, device, data, external_priorities={}):
device_data = get_device_data(device)
full_data = merge_data(
data,
{
'database': {
'device': device_data
},
},
only_multiple=True,
)
append_merged_proposition(full_data, device, external_priorities)
selected_data = select_data(full_data, external_priorities)
parsed_mounts = set()
for share_mount in selected_data.get('disk_shares', []):
status, mount = _create_or_update_share_mount(ip, device, share_mount)
if mount:
parsed_mounts.add(mount.pk)
device.disksharemount_set.exclude(pk__in=parsed_mounts).delete()
def _append_connections_to_device(device, data, external_priorities):
device_data = get_device_data(device)
full_data = merge_data(
data,
{
'database': {'device': device_data},
},
only_multiple=True,
)
append_merged_proposition(full_data, device, external_priorities)
selected_data = select_data(full_data, external_priorities)
parsed_connections = set()
for conn_data in selected_data.get('connections', []):
conn = _create_or_update_connection(device, conn_data)
if conn:
parsed_connections.add(conn.pk)
Connection.objects.filter(
Q(outbound=device),
Q(connection_type=ConnectionType.network),
~Q(pk__in=parsed_connections),
).delete()
def _append_connections_to_device(device, data, external_priorities):
device_data = get_device_data(device)
full_data = merge_data(
data,
{
'database': {'device': device_data},
},
only_multiple=True,
)
append_merged_proposition(full_data, device, external_priorities)
selected_data = select_data(full_data, external_priorities)
parsed_connections = set()
for conn_data in selected_data.get('connections', []):
conn = _create_or_update_connection(device, conn_data)
if conn:
parsed_connections.add(conn.pk)
Connection.objects.filter(
Q(outbound=device),
Q(connection_type=ConnectionType.network),
~Q(pk__in=parsed_connections),
).delete()
def _save_job_results(job_id, start_ts):
if (int(time.time()) - start_ts) > 86100: # 24h - 5min
return
try:
job = rq.job.Job.fetch(job_id, django_rq.get_connection())
except rq.exceptions.NoSuchJobError:
return # job with this id does not exist...
if job.result is None and not job.is_failed:
# we must wait...
_enqueue_save_job_results(job_id)
return
elif job.is_failed:
# nothing to do...
return
external_priorities = get_external_results_priorities(job.result)
# management?
is_management = False
if job.args:
try:
is_management = IPAddress.objects.filter(
address=job.args[0]).values_list('is_management', flat=True)[0]
except IndexError:
pass
# first... update device
devices, ids_lookup, sn_lookup, macs_lookup = _find_devices(job.result)
if len(devices) > 1:
raise AutomergerError(
'Many devices found for: ids=%s, sn=%s, macs=%s' %
(ids_lookup, sn_lookup, macs_lookup))
used_serial_numbers = set()
used_mac_addresses = set()
for device in devices:
device_data = get_device_data(device)
if 'serial_number' in device_data:
used_serial_numbers |= set([device_data['serial_number']])
if 'mac_addresses' in device_data:
used_mac_addresses |= set(device_data['mac_addresses'])
data = merge_data(
job.result,
{
'database': {
'device': device_data
},
},
only_multiple=True,
)
append_merged_proposition(data, device, external_priorities)
selected_data = _select_data(data, external_priorities, is_management)
set_device_data(device, selected_data, save_priority=SAVE_PRIORITY)
device.save(priority=SAVE_PRIORITY)
# now... we create new devices from `garbage`
if not devices:
garbage = {}
for plugin_name, plugin_result in job.result.items():
if 'device' not in plugin_result:
continue
if 'serial_number' in plugin_result['device']:
if plugin_result['device'][
'serial_number'] in used_serial_numbers:
continue
if 'mac_addresses' in plugin_result['device']:
if set(plugin_result['device']['mac_addresses'], ) != set(
plugin_result['device']['mac_addresses'],
) - used_mac_addresses:
continue
if any((
plugin_result['device'].get('serial_number'),
plugin_result['device'].get('mac_addresses'),
)):
garbage[plugin_name] = plugin_result
if garbage:
data = merge_data(garbage)
selected_data = _select_data(data, external_priorities)
if all((
any((
selected_data.get('serial_number'),
selected_data.get('mac_addresses', []),
)),
any((
selected_data.get('model_name'),
selected_data.get('type'),
)),
)):
device_from_data(selected_data, save_priority=SAVE_PRIORITY)
# mark this scan results
update_scan_summary(job)
def _save_job_results(job_id, start_ts):
if (int(time.time()) - start_ts) > 86100: # 24h - 5min
return
try:
job = rq.job.Job.fetch(job_id, django_rq.get_connection())
except rq.exceptions.NoSuchJobError:
return # job with this id does not exist...
if job.result is None and not job.is_failed:
# we must wait...
_enqueue_save_job_results(job_id)
return
elif job.is_failed:
# nothing to do...
return
external_priorities = get_external_results_priorities(job.result)
# management?
is_management = False
if job.args:
try:
is_management = IPAddress.objects.filter(
address=job.args[0]
).values_list(
'is_management', flat=True
)[0]
except IndexError:
pass
# first... update device
devices, ids_lookup, sn_lookup, macs_lookup = _find_devices(job.result)
if len(devices) > 1:
raise AutomergerError(
'Many devices found for: ids=%s, sn=%s, macs=%s' % (
ids_lookup, sn_lookup, macs_lookup
)
)
used_serial_numbers = set()
used_mac_addresses = set()
for device in devices:
device_data = get_device_data(device)
if 'serial_number' in device_data:
used_serial_numbers |= set([device_data['serial_number']])
if 'mac_addresses' in device_data:
used_mac_addresses |= set(device_data['mac_addresses'])
data = merge_data(
job.result,
{
'database': {'device': device_data},
},
only_multiple=True,
)
append_merged_proposition(data, device, external_priorities)
selected_data = select_data(data, external_priorities, is_management)
set_device_data(device, selected_data, save_priority=SAVE_PRIORITY)
device.save(priority=SAVE_PRIORITY)
# now... we create new devices from `garbage`
if not devices:
garbage = {}
for plugin_name, plugin_result in job.result.items():
if 'device' not in plugin_result:
continue
if 'serial_number' in plugin_result['device']:
if plugin_result['device'][
'serial_number'
] in used_serial_numbers:
continue
if 'mac_addresses' in plugin_result['device']:
if set(
plugin_result['device']['mac_addresses'],
) != set(
plugin_result['device']['mac_addresses'],
) - used_mac_addresses:
continue
if any((
plugin_result['device'].get('serial_number'),
plugin_result['device'].get('mac_addresses'),
)):
garbage[plugin_name] = plugin_result
if garbage:
data = merge_data(garbage)
selected_data = select_data(data, external_priorities)
if all((
any(
(
selected_data.get('serial_number'),
selected_data.get('mac_addresses', []),
)
),
any(
(
selected_data.get('model_name'),
selected_data.get('type'),
)
),
)):
device_from_data(selected_data, save_priority=SAVE_PRIORITY)
# mark this scan results
update_scan_summary(job)
