def setUp(self):
device = Device.create(
ethernets=[('', 'deadbeefcafe', 0)],
model_name='HAL 9000',
model_type=DeviceType.unknown,
remarks="I'm sorry, Dave.",
)
self.deployment = Deployment(
hostname='discovery.one',
ip='127.0.0.1',
mac='deadbeefcafe',
device=device,
preboot=None,
venture=None,
venture_role=None,
)
self.deployment.save()
self.ip = IPAddress(address='127.0.0.1', device=device)
self.ip.save()
IPAddress(address='127.0.0.2', device=device).save()
share_model = ComponentModel(type=ComponentType.share, name="share")
share_model.save()
share = DiskShare(wwn='x' * 33, device=device, model=share_model)
share.save()
DiskShareMount(share=share, device=device).save()
OperatingSystem.create(os_name='GladOS',
dev=device,
family='',
priority=0)
Software.create(dev=device,
model_name='soft',
path='/',
family='',
priority=0)
def test_disk_shares_and_exports(self):
model = ComponentModel(
type=ComponentType.share,
name="3par share",
)
model.save()
share = DiskShare(
device=self.device,
model=model,
label="pr0n",
size="2048",
wwn="deadbeefcafe1234",
)
share.save()
address = IPAddress(address='127.0.0.1')
address.save()
DiskShareMount(
device=self.device,
share=share,
address=address,
).save()
data = get_device_data(Device.objects.get(sn='123456789'))
exports = data['disk_exports']
mounts = data['disk_shares']
self.assertEqual(len(exports), 1)
self.assertEqual(len(mounts), 1)
self.assertEqual(mounts[0]['serial_number'], "deadbeefcafe1234")
def test_disk_shares_and_exports(self):
model = ComponentModel(
type=ComponentType.share,
name="3par share",
)
model.save()
share = DiskShare(
device=self.device,
model=model,
label="pr0n",
size="2048",
wwn="deadbeefcafe1234",
)
share.save()
address = IPAddress(address='127.0.0.1')
address.save()
DiskShareMount(
device=self.device,
share=share,
address=address,
).save()
data = get_device_data(Device.objects.get(sn='123456789'))
exports = data['disk_exports']
mounts = data['disk_shares']
self.assertEqual(len(exports), 1)
self.assertEqual(len(mounts), 1)
self.assertEqual(mounts[0]['serial_number'], "deadbeefcafe1234")
def test_name(self):
ip = IPAddress(address='127.0.0.1')
ip.save()
with mock.patch('ralph.discovery.plugins.snmp.snmp_command') as snmp_command:
snmp_command.return_value = [[None, b'Testing name']]
is_up, message = snmp._snmp('127.0.0.1', 'public', (1,3,6,1,2,1,1,1,0))
self.assertEqual(message, 'Testing name')
self.assertEqual(is_up, True)
def test_ip_is_public_or_no(self):
ip_list = [
('92.143.123.123', True),
('10.168.123.123', False),
]
for ip, is_public in ip_list:
new_ip_address = IPAddress(address=ip)
new_ip_address.save()
self.assertEquals(new_ip_address.is_public, is_public)
class ApiPricingTest(TestCase):
def setUp(self):
self.venture, created = Venture.objects.get_or_create(
name="Sample venture",
)
self.venture.save()
self.device, created = Device.objects.get_or_create(
name="Device1",
venture=self.venture,
)
self.device_without_venture, created = Device.objects.get_or_create(
name="Device2",
)
self.ip1 = IPAddress(address='1.1.1.1', device=self.device)
self.ip2 = IPAddress(address='2.2.2.2', venture=self.venture)
self.ip1.save()
self.ip2.save()
self.device.save()
self.device_without_venture.save()
def test_get_device_by_name_with_venture(self):
# device with venture
device1 = api_pricing.get_device_by_name("Device1")
self.assertEqual(device1['device_id'], self.device.id)
self.assertEqual(device1['venture_id'], self.venture.id)
def test_get_device_by_name_without_venture(self):
# device without venture
device2 = api_pricing.get_device_by_name("Device2")
self.assertEqual(device2['device_id'], self.device_without_venture.id)
self.assertIsNone(device2['venture_id'])
def test_get_device_by_name_wrong_name(self):
# device does not exist
device3 = api_pricing.get_device_by_name("Device3")
self.assertEqual(device3, {})
def test_get_ip_info(self):
result = api_pricing.get_ip_info(ipaddress='1.1.1.1')
self.assertEquals(result, {
'device_id': self.device.id,
'venture_id': self.venture.id,
})
def test_get_ip_info_without_device(self):
result = api_pricing.get_ip_info(ipaddress='2.2.2.2')
self.assertEquals(result, {
'venture_id': self.venture.id,
})
def test_get_ip_info_empty(self):
result = api_pricing.get_ip_info(ipaddress='3.3.3.3')
self.assertEquals(result, {})
def test_ip_is_public_or_no(self):
ip_list = [
('92.143.123.123', True),
('10.168.123.123', False),
]
for ip, is_public in ip_list:
new_ip_address = IPAddress(address=ip)
new_ip_address.save()
self.assertEquals(new_ip_address.is_public, is_public)
def test_name(self):
ip = IPAddress(address='127.0.0.1')
ip.save()
with mock.patch(
'ralph.discovery.plugins.snmp.snmp_command') as snmp_command:
snmp_command.return_value = [[None, b'Testing name']]
is_up, message = snmp._snmp('127.0.0.1', 'public',
(1, 3, 6, 1, 2, 1, 1, 1, 0))
self.assertEqual(message, 'Testing name')
self.assertEqual(is_up, True)
def _ip_create_record(form, request, device):
hostname = form.cleaned_data.get('ip_new_hostname')
address = form.cleaned_data.get('ip_new_address')
if hostname and address:
if IPAddress.objects.filter(address=address).exists():
messages.error(
request,
"An IP address entry for %s already exists." % address)
return
record = IPAddress(address=address, hostname=hostname, device=device)
record.save()
messages.success(request,
"An IP address entry for %s created." % address)
def setUp(self):
self.venture, created = Venture.objects.get_or_create(name="Sample venture")
self.venture.save()
self.device, created = Device.objects.get_or_create(name="Device1", venture=self.venture)
self.device_without_venture, created = Device.objects.get_or_create(name="Device2")
self.ip1 = IPAddress(address="1.1.1.1", device=self.device)
self.ip2 = IPAddress(address="2.2.2.2", venture=self.venture)
self.ip1.save()
self.ip2.save()
self.device.save()
self.device_without_venture.save()
class CleanPluginTest(TestCase):
def setUp(self):
device = Device.create(
ethernets=[('', 'deadbeefcafe', 0)],
model_name='HAL 9000',
model_type=DeviceType.unknown,
remarks="I'm sorry, Dave.",
)
self.deployment = Deployment(
hostname='discovery.one',
ip='127.0.0.1',
mac='deadbeefcafe',
device=device,
preboot=None,
venture=None,
venture_role=None,
)
self.deployment.save()
self.ip = IPAddress(address='127.0.0.1', device=device)
self.ip.save()
IPAddress(address='127.0.0.2', device=device).save()
share_model = ComponentModel(type=ComponentType.share, name="share")
share_model.save()
share = DiskShare(wwn='x' * 33, device=device, model=share_model)
share.save()
DiskShareMount(share=share, device=device).save()
OperatingSystem.create(os_name='GladOS',
dev=device,
family='',
priority=0)
Software.create(dev=device,
model_name='soft',
path='/',
family='',
priority=0)
def test_clean_plugin(self):
clean(self.deployment.id)
device = Device.objects.get(id=self.deployment.device.id)
self.assertEqual(
device.remarks,
"-- Remarks below are for old role -/- from %s --\n"
"I'm sorry, Dave." % datetime.date.today().strftime('%Y-%m-%d'),
)
self.assertEquals(device.ipaddress_set.count(), 1)
self.assertEquals(device.ipaddress_set.all()[0].address, '127.0.0.1')
self.assertFalse(device.diskshare_set.exists())
self.assertFalse(device.disksharemount_set.exists())
self.assertFalse(device.software_set.exists())
self.assertFalse(device.operatingsystem_set.exists())
def run_http(ip):
headers, document = get_http_info(ip)
family = guess_family(headers, document)
ip_address, created = IPAddress.concurrent_get_or_create(address=ip)
ip_address.http_family = family
ip_address.save(update_last_seen=True)
return family
def setUp(self):
device = Device.create(
ethernets=[('', 'deadbeefcafe', 0)],
model_name='HAL 9000',
model_type=DeviceType.unknown,
remarks="I'm sorry, Dave.",
)
self.deployment = Deployment(
hostname='discovery.one',
ip='127.0.0.1',
mac='deadbeefcafe',
device=device,
preboot=None,
venture=None,
venture_role=None,
)
self.deployment.save()
self.ip = IPAddress(address='127.0.0.1', device=device)
self.ip.save()
IPAddress(address='127.0.0.2', device=device).save()
share_model = ComponentModel(type=ComponentType.share, name="share")
share_model.save()
share = DiskShare(wwn='x'*33, device=device, model=share_model)
share.save()
DiskShareMount(share=share, device=device).save()
OperatingSystem.create(os_name='GladOS', dev=device, family='',
priority=0)
Software.create(dev=device, model_name='soft', path='/', family='',
priority=0)
def make_device(ilo, ip):
if ilo.model.startswith('HP ProLiant BL'):
t = DeviceType.blade_server
else:
t = DeviceType.rack_server
ethernets = [Eth(label, mac, speed=None) for label, mac in ilo.ethernets]
dev = Device.create(
ethernets=ethernets,
model_name=ilo.model,
model_type=t,
sn=ilo.sn,
name=ilo.name,
mgmt_firmware=ilo.firmware,
)
dev.save(update_last_seen=True, priority=SAVE_PRIORITY)
ipaddr, created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.device = dev
ipaddr.is_management = True
ipaddr.save()
if dev.parent and dev.parent.management:
dev.management = dev.parent.management
else:
dev.management = ipaddr
dev.save(priority=SAVE_PRIORITY)
return dev
def run_idrac(ip):
idrac = IDRAC(ip)
base_info = idrac.get_base_info()
model_name = "{} {}".format(
base_info['manufacturer'].replace(" Inc.", ""),
base_info['model']
)
ethernets = _save_ethernets(idrac.get_ethernets())
ip_address, _ = IPAddress.concurrent_get_or_create(address=ip)
ip_address.is_management = True
ip_address.save()
dev = Device.create(
ethernets=ethernets,
model_name=model_name,
sn=base_info['sn'],
model_type=DeviceType.rack_server,
)
dev.management = ip_address
dev.save(priority=SAVE_PRIORITY)
ip_address.device = dev
ip_address.save()
_save_cpu(dev, idrac.get_cpu())
_save_memory(dev, idrac.get_memory())
_save_storage(dev, idrac.get_storage())
_save_fc_cards(dev, idrac.get_fc_cards())
return model_name
def _cisco_snmp_model(model_oid, sn_oid, **kwargs):
ip = str(kwargs['ip'])
version = kwargs.get('snmp_version')
if version == '3':
community = SNMP_V3_AUTH
else:
community = str(kwargs['community'])
model = snmp_command(
ip,
community,
model_oid,
attempts=2,
timeout=3,
snmp_version=version,
)
sn = snmp_command(
ip,
community,
sn_oid,
attempts=2,
timeout=3,
snmp_version=version,
)
if not (model and sn):
return False, "silent.", kwargs
sn = unicode(sn[0][1])
model = 'Cisco %s' % unicode(model[0][1])
dev = Device.create(sn=sn, model_name=model, model_type=DeviceType.switch)
ip_address, created = IPAddress.concurrent_get_or_create(address=str(ip))
ip_address.device = dev
ip_address.is_management = True
ip_address.save()
return True, sn, kwargs
def _run_ssh_catalyst(ip):
ssh = _connect_ssh(ip)
try:
mac = '\n'.join(ssh.cisco_command(
"show version | include Base ethernet MAC Address"
))
raw = '\n'.join(ssh.cisco_command("show inventory"))
finally:
ssh.close()
mac = mac.strip()
if mac.startswith("Base ethernet MAC Address") and ':' in mac:
ethernets = [
Eth(
"Base ethernet MAC Address",
mac.split(':', 1)[1].strip(),
None,
),
]
else:
ethernets = None
inventory = list(cisco_inventory(raw))
serials = [inv['sn'] for inv in inventory]
dev_inv = inventory[0]
try:
dev = Device.objects.get(sn__in=serials)
except MultipleObjectsReturned:
raise Error(
"Stacked devices with serials %r should be merged.",
serials,
)
except Device.DoesNotExist:
sn = dev_inv['sn']
model_name='Cisco %s' % dev_inv['pid']
else:
# This is a stacked device, use the base device for it
sn = dev.sn
model_name = dev.model.name
dev = Device.create(
ethernets=ethernets,
sn=sn,
model_name=model_name,
model_type=DeviceType.switch,
name=dev_inv['descr'][:255],
)
dev.save(update_last_seen=True)
for inv in inventory:
cisco_component(dev, inv)
ip_address, created = IPAddress.concurrent_get_or_create(address=str(ip))
if created:
ip_address.hostname = network.hostname(ip_address.address)
ip_address.device = dev
ip_address.is_management = True
ip_address.save(update_last_seen=True)
return dev.name
def perform_move(self, address, new_ip, new_hostname):
old_ipaddress = IPAddress.objects.get(address=address)
device = old_ipaddress.device
mac = None
for r in Record.objects.filter(
db.Q(name=old_ipaddress.hostname) |
db.Q(content=old_ipaddress.hostname) |
db.Q(content=old_ipaddress.address)
):
r.delete()
for e in DHCPEntry.objects.filter(ip=old_ipaddress.address):
mac = e.mac
e.delete()
old_ipaddress.device = None
old_ipaddress.save()
reset_dns(new_hostname, new_ip)
new_ipaddress, c = IPAddress.concurrent_get_or_create(
address=new_ip,
)
new_ipaddress.device = device
new_ipaddress.hostname = new_hostname
new_ipaddress.save()
if mac:
entry = DHCPEntry(ip=new_ip, mac=mac)
entry.save()
pricing.device_update_cached(device)
def attach_ip(dev, ip):
"""Attach the IP address"""
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.device = dev
ipaddr.is_management = False
ipaddr.save()
def scan_address_job(
ip_address=None,
plugins=None,
results=None,
automerge=AUTOMERGE_MODE,
**kwargs
):
"""
The function that is actually running on the worker.
"""
job = rq.get_current_job()
available_plugins = getattr(settings, 'SCAN_PLUGINS', {}).keys()
if not plugins:
plugins = available_plugins
run_postprocessing = not (set(available_plugins) - set(plugins))
if ip_address and plugins:
if not kwargs:
ip, created = IPAddress.concurrent_get_or_create(
address=ip_address,
)
kwargs = {
'snmp_community': ip.snmp_community,
'snmp_version': ip.snmp_version,
'http_family': ip.http_family,
'snmp_name': ip.snmp_name,
}
results = _run_plugins(ip_address, plugins, job, **kwargs)
if run_postprocessing:
_scan_postprocessing(results, job, ip_address)
# Run only when automerge mode is enabled and some change was detected.
# When `change` state is not available just run it...
if automerge and job.meta.get('changed', True):
save_job_results(job.id)
return results
def _save_shares(dev, luns, mounts):
wwns = []
for lun, volume in luns.iteritems():
rest, wwn_end = lun.rsplit('_', 1)
try:
share = DiskShare.objects.get(wwn__endswith=wwn_end)
except DiskShare.DoesNotExist:
continue
wwns.append(share.wwn)
clients = mounts.get(volume, [])
for client in clients:
ipaddr, ip_created = IPAddress.concurrent_get_or_create(
address=client)
mount, created = DiskShareMount.concurrent_get_or_create(
address=ipaddr, device=ipaddr.device, share=share, server=dev)
mount.volume = volume
mount.save(update_last_seen=True)
if not clients:
mount, created = DiskShareMount.concurrent_get_or_create(
address=None, device=None, share=share, server=dev)
mount.volume = volume
mount.save(update_last_seen=True)
for mount in DiskShareMount.objects.filter(server=dev).exclude(
share__wwn__in=wwns):
mount.delete()
def _run_ssh_catalyst(ip):
ssh = _connect_ssh(ip)
try:
raw = '\n'.join(ssh.cisco_command("show inventory"))
finally:
ssh.close()
inventory = list(cisco_inventory(raw))
serials = [inv['sn'] for inv in inventory]
try:
dev = Device.objects.get(sn__in=serials)
except Device.DoesNotExist:
dev_inv = inventory[0]
dev = Device.create(sn=dev_inv['sn'],
model_name='Cisco %s' % dev_inv['pid'],
model_type=DeviceType.switch,
name=dev_inv['descr'][:255])
dev.save(update_last_seen=True)
for inv in inventory:
cisco_component(dev, inv)
ip_address, created = IPAddress.concurrent_get_or_create(address=str(ip))
if created:
ip_address.hostname = network.hostname(ip_address.address)
ip_address.device = dev
ip_address.is_management = True
ip_address.save(update_last_seen=True)
return dev.name
def attach_ip(dev, ip):
"""Attach the IP address"""
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.device = dev
ipaddr.is_management = False
ipaddr.save()
def _save_shares(dev, luns, mounts):
wwns = []
for lun, volume in luns.iteritems():
rest, wwn_end = lun.rsplit('_', 1)
try:
share = DiskShare.objects.get(wwn__endswith=wwn_end)
except DiskShare.DoesNotExist:
continue
wwns.append(share.wwn)
clients = mounts.get(volume, [])
for client in clients:
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=client)
mount, created = DiskShareMount.concurrent_get_or_create(
address=ipaddr, device=ipaddr.device, share=share, server=dev)
mount.volume = volume
mount.save(update_last_seen=True)
if not clients:
mount, created = DiskShareMount.concurrent_get_or_create(
address=None, device=None, share=share, server=dev)
mount.volume = volume
mount.save(update_last_seen=True)
for mount in DiskShareMount.objects.filter(
server=dev
).exclude(
share__wwn__in=wwns
):
mount.delete()
def _run_ssh_catalyst(ip):
ssh = _connect_ssh(ip)
try:
raw = "\n".join(ssh.cisco_command("show inventory"))
finally:
ssh.close()
inventory = list(cisco_inventory(raw))
serials = [inv["sn"] for inv in inventory]
try:
dev = Device.objects.get(sn__in=serials)
except Device.DoesNotExist:
dev_inv = inventory[0]
dev = Device.create(
sn=dev_inv["sn"],
model_name="Cisco %s" % dev_inv["pid"],
model_type=DeviceType.switch,
name=dev_inv["descr"][:255],
)
dev.save(update_last_seen=True)
for inv in inventory:
cisco_component(dev, inv)
ip_address, created = IPAddress.concurrent_get_or_create(address=str(ip))
if created:
ip_address.hostname = network.hostname(ip_address.address)
ip_address.device = dev
ip_address.is_management = True
ip_address.save(update_last_seen=True)
return dev.name
def _get_master(ssh, data=None):
if data is None:
stdin, stdout, stderr = ssh.exec_command("cat /etc/pve/cluster.cfg")
data = stdout.read()
nodes = {}
current_node = None
for line in data.splitlines():
line = line.strip()
if line.endswith("{"):
current_node = line.replace("{", "").strip()
nodes[current_node] = {}
elif line.endswith("}"):
current_node = None
elif ":" in line and current_node:
key, value = (v.strip() for v in line.split(":", 1))
nodes[current_node][key] = value
for node, pairs in nodes.iteritems():
is_master = node.startswith("master")
try:
ip = pairs["IP"]
except KeyError:
continue
if is_master:
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.save()
return ipaddr
def scan_address_job(
ip_address=None,
plugins=None,
results=None,
automerge=AUTOMERGE_MODE,
called_from_ui=False,
**kwargs
):
"""The function that is actually running on the worker."""
job = rq.get_current_job()
available_plugins = getattr(settings, 'SCAN_PLUGINS', {}).keys()
if not plugins:
plugins = available_plugins
run_postprocessing = not (set(available_plugins) - set(plugins))
if ip_address and plugins:
if not kwargs:
ip, created = IPAddress.concurrent_get_or_create(
address=ip_address,
)
if not (ip.snmp_name and ip.snmp_community):
message = "SNMP name/community is missing. Forcing autoscan."
job.meta['messages'] = [
(ip_address, 'ralph.scan', 'info', message)
]
job.save()
autoscan_address(ip_address)
kwargs = {
'snmp_community': ip.snmp_community,
'snmp_version': ip.snmp_version,
'http_family': ip.http_family,
'snmp_name': ip.snmp_name,
}
results = _run_plugins(ip_address, plugins, job, **kwargs)
if run_postprocessing:
_scan_postprocessing(results, job, ip_address)
if automerge and job.meta.get('changed', True):
# Run only when automerge mode is enabled and some change was
# detected. When `change` state is not available just run it...
save_job_results(job.id)
elif not called_from_ui and job.args and job.meta.get('changed', True):
# Run only when some change was detected. When `change` state is
# not available just run it...
try:
ip_obj = IPAddress.objects.select_related().get(
address=job.args[0] # job.args[0] == ip_address
)
except IPAddress.DoesNotExist:
pass
else:
for plugin_name in getattr(
settings, 'SCAN_POSTPROCESS_ENABLED_JOBS', []
):
try:
module = import_module(plugin_name)
except ImportError as e:
logger.error(unicode(e))
else:
module.run_job(ip_obj)
return results
def _cisco_snmp_model(model_oid, sn_oid, **kwargs):
ip = str(kwargs['ip'])
version = kwargs.get('snmp_version')
if version == '3':
community = SNMP_V3_AUTH
else:
community = str(kwargs['community'])
model = snmp_command(
ip,
community,
model_oid,
attempts=2,
timeout=3,
snmp_version=version,
)
sn = snmp_command(
ip,
community,
sn_oid,
attempts=2,
timeout=3,
snmp_version=version,
)
if not (model and sn):
return False, "silent.", kwargs
sn = unicode(sn[0][1])
model = 'Cisco %s' % unicode(model[0][1])
dev = Device.create(sn=sn, model_name=model, model_type=DeviceType.switch)
ip_address, created = IPAddress.concurrent_get_or_create(address=str(ip))
ip_address.device = dev
ip_address.is_management = True
ip_address.save()
return True, sn, kwargs
def run_http_ibm_system_x(ip):
session_id = get_session_id(ip)
management_url = "http://%s/wsman" % ip
model_name = get_model_name(management_url, session_id)
sn = get_sn(management_url, session_id)
macs = get_mac_addresses(management_url, session_id)
ethernets = [Eth(label=label, mac=mac, speed=0)
for (label, mac) in macs]
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.is_management = True
ipaddr.save()
dev = Device.create(
ethernets=ethernets,
model_name=model_name,
sn=sn,
model_type=DeviceType.rack_server,
)
dev.management = ipaddr
dev.save(priority=SAVE_PRIORITY)
ipaddr.device = dev
ipaddr.save()
detected_memory = get_memory(management_url, session_id)
detected_memory_indexes = [x.get('index') for x in detected_memory]
for m in dev.memory_set.exclude(index__in=detected_memory_indexes):
m.delete()
for m in detected_memory:
index = m['index']
mem, _ = Memory.concurrent_get_or_create(index=index, device=dev)
mem.label = m['label']
mem.size = m['size']
mem.save(priority=SAVE_PRIORITY)
mem.model, c = ComponentModel.concurrent_get_or_create(
name='RAM %s %dMiB' % (mem.label, mem.size), size=mem.size,
type=ComponentType.memory.id,
family=mem.label, cores=0
)
mem.save(priority=SAVE_PRIORITY)
detected_processors = get_processors(management_url, session_id)
detected_processors_keys = [x.get('index') for x in detected_processors]
for cpu in dev.processor_set.exclude(index__in=detected_processors_keys):
cpu.delete()
# add new
for p in detected_processors:
processor_model, _ = ComponentModel.concurrent_get_or_create(
name=p.get('label'),
speed=p.get('speed'),
type=ComponentType.processor.id,
family=p.get('family'),
cores=p.get('cores')
)
processor, _ = Processor.concurrent_get_or_create(
device=dev,
index=p.get('index'),
)
processor.label = p.get('label')
processor.model = processor_model
processor.speed = p.get('speed')
processor.save()
return model_name
def run_http_ibm_system_x(ip):
session_id = get_session_id(ip)
management_url = "http://%s/wsman" % ip
model_name = get_model_name(management_url, session_id)
sn = get_sn(management_url, session_id)
macs = get_mac_addresses(management_url, session_id)
ethernets = [Eth(label=label, mac=mac, speed=0)
for (label, mac) in macs]
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.is_management = True
ipaddr.save()
dev = Device.create(
ethernets=ethernets,
model_name=model_name,
sn=sn,
model_type=DeviceType.rack_server,
)
dev.management = ipaddr
dev.save(priority=SAVE_PRIORITY)
ipaddr.device = dev
ipaddr.save()
detected_memory = get_memory(management_url, session_id)
detected_memory_indexes = [x.get('index') for x in detected_memory]
for m in dev.memory_set.exclude(index__in=detected_memory_indexes):
m.delete()
for m in detected_memory:
index = m['index']
mem, _ = Memory.concurrent_get_or_create(index=index, device=dev)
mem.label = m['label']
mem.size = m['size']
mem.model, c = ComponentModel.create(
ComponentType.memory,
size=mem.size,
priority=SAVE_PRIORITY,
)
mem.save(priority=SAVE_PRIORITY)
detected_processors = get_processors(management_url, session_id)
detected_processors_keys = [x.get('index') for x in detected_processors]
for cpu in dev.processor_set.exclude(index__in=detected_processors_keys):
cpu.delete()
# add new
for p in detected_processors:
processor_model, _ = ComponentModel.create(
ComponentType.processor,
speed=p.get('speed'),
family=p.get('family'),
cores=p.get('cores'),
name=p.get('label'),
priority=SAVE_PRIORITY,
)
processor, _ = Processor.concurrent_get_or_create(
device=dev,
index=p.get('index'),
)
processor.label = p.get('label')
processor.model = processor_model
processor.speed = p.get('speed')
processor.save(priority=SAVE_PRIORITY)
return model_name
class CleanPluginTest(TestCase):
def setUp(self):
device = Device.create(
ethernets=[('', 'deadbeefcafe', 0)],
model_name='HAL 9000',
model_type=DeviceType.unknown,
remarks="I'm sorry, Dave.",
)
self.deployment = Deployment(
hostname='discovery.one',
ip='127.0.0.1',
mac='deadbeefcafe',
device=device,
preboot=None,
venture=None,
venture_role=None,
)
self.deployment.save()
self.ip = IPAddress(address='127.0.0.1', device=device)
self.ip.save()
IPAddress(address='127.0.0.2', device=device).save()
share_model = ComponentModel(type=ComponentType.share, name="share")
share_model.save()
share = DiskShare(wwn='x' * 33, device=device, model=share_model)
share.save()
DiskShareMount(share=share, device=device).save()
OperatingSystem.create(os_name='GladOS', dev=device, family='',
priority=0)
Software.create(dev=device, model_name='soft', path='/', family='',
priority=0)
def test_clean_plugin(self):
clean(self.deployment.id)
device = Device.objects.get(id=self.deployment.device.id)
self.assertEqual(
device.remarks,
"-- Remarks below are for old role -/- from %s --\n"
"I'm sorry, Dave." % datetime.date.today().strftime('%Y-%m-%d'),
)
self.assertEquals(device.ipaddress_set.count(), 1)
self.assertEquals(device.ipaddress_set.all()[0].address, '127.0.0.1')
self.assertFalse(device.diskshare_set.exists())
self.assertFalse(device.disksharemount_set.exists())
self.assertFalse(device.software_set.exists())
self.assertFalse(device.operatingsystem_set.exists())
def _ip_create_record(form, request, device):
hostname = form.cleaned_data.get('ip_new_hostname')
address = form.cleaned_data.get('ip_new_address')
if hostname and address:
if IPAddress.objects.filter(address=address).exists():
messages.error(
request,
"An IP address entry for %s already exists."
% address
)
return
record = IPAddress(address=address, hostname=hostname,
device=device)
record.save()
messages.success(request,
"An IP address entry for %s created." %
address)
def save_device_data(data, remote_ip):
device = data['device']
ethernets = [
Eth(e.get('label'), MACAddressField.normalize(e.get('mac')),
str_to_ethspeed(e.get('speed'))) for e in data['ethernets']
if MACAddressField.normalize(e.get('mac')) not in MAC_PREFIX_BLACKLIST
]
sn = device.get('sn')
vendor = device.get('vendor', '')
if not ethernets and not sn:
raise NoRequiredDataError('No MAC addresses and no device SN.')
ip_addresses = [
e['ipaddress'] for e in data['ethernets'] if e.get('ipaddress')
]
if not ip_addresses:
raise NoRequiredIPAddressError(
"Couldn't find any IP address for this device.")
try:
dev = Device.create(sn=sn,
ethernets=ethernets,
model_name='%s %s %s' % (
device.get('caption'),
vendor,
device.get('version'),
),
model_type=DeviceType.unknown,
priority=SAVE_PRIORITY)
except ValueError as e:
DiscoveryWarning(
message="Failed to create device: " + str(e),
plugin=__name__,
ip=str(remote_ip),
).save()
return None
dev.save(priority=SAVE_PRIORITY)
os = data['operating_system']
o = OperatingSystem.create(dev,
os_name=os.get('label'),
family='Windows',
priority=SAVE_PRIORITY)
o.memory = int(os['memory'])
o.storage = int(os['storage'])
o.cores_count = int(os['corescount'])
o.save(priority=SAVE_PRIORITY)
for ip in ip_addresses:
ip_address, _ = IPAddress.concurrent_get_or_create(address=str(ip))
ip_address.device = dev
ip_address.is_management = False
ip_address.save()
vendor = vendor.lower()
is_virtual = any(virtual in vendor for virtual in CPU_VIRTUAL_LIST)
save_processors(data['processors'], dev, is_virtual)
save_memory(data['memory'], dev)
save_storage(data['storage'], dev)
save_shares(data['shares'], dev, ip_address)
save_fibre_channel(data['fcs'], dev)
save_software(data.get('software', []), dev)
return dev
def save_device_data(data, remote_ip):
device = data['device']
ethernets = [
Eth(e.get('label'), MACAddressField.normalize(e.get('mac')),
str_to_ethspeed(e.get('speed')))
for e in data['ethernets']
if MACAddressField.normalize(e.get('mac')) not in MAC_PREFIX_BLACKLIST]
sn = device.get('sn')
vendor = device.get('vendor', '')
if not ethernets and not sn:
raise NoRequiredDataError('No MAC addresses and no device SN.')
ip_addresses = [
e['ipaddress'] for e in data['ethernets'] if e.get('ipaddress')
]
if not ip_addresses:
raise NoRequiredIPAddressError(
"Couldn't find any IP address for this device."
)
try:
dev = Device.create(
sn=sn,
ethernets=ethernets,
model_name='%s %s %s' % (
device.get('caption'),
vendor,
device.get('version'),
),
model_type=DeviceType.unknown, priority=SAVE_PRIORITY
)
except ValueError as e:
DiscoveryWarning(
message="Failed to create device: " + str(e),
plugin=__name__,
ip=str(remote_ip),
).save()
return None
dev.save(priority=SAVE_PRIORITY)
os = data['operating_system']
o = OperatingSystem.create(dev, os_name=os.get('label'),
family='Windows', priority=SAVE_PRIORITY)
o.memory = int(os['memory'])
o.storage = int(os['storage'])
o.cores_count = int(os['corescount'])
o.save(priority=SAVE_PRIORITY)
for ip in ip_addresses:
ip_address, _ = IPAddress.concurrent_get_or_create(address=str(ip))
ip_address.device = dev
ip_address.is_management = False
ip_address.save()
vendor = vendor.lower()
is_virtual = any(virtual in vendor for virtual in CPU_VIRTUAL_LIST)
save_processors(data['processors'], dev, is_virtual)
save_memory(data['memory'], dev)
save_storage(data['storage'], dev)
save_shares(data['shares'], dev, ip_address)
save_fibre_channel(data['fcs'], dev)
save_software(data.get('software', []), dev)
return dev
def scan_address_job(ip_address=None,
plugins=None,
results=None,
automerge=AUTOMERGE_MODE,
called_from_ui=False,
**kwargs):
"""The function that is actually running on the worker."""
job = rq.get_current_job()
available_plugins = getattr(settings, 'SCAN_PLUGINS', {}).keys()
if not plugins:
plugins = available_plugins
run_postprocessing = not (set(available_plugins) - set(plugins))
if ip_address and plugins:
if not kwargs:
ip, created = IPAddress.concurrent_get_or_create(
address=ip_address, )
if not (ip.snmp_name and ip.snmp_community):
message = ("SNMP name and community is missing. Forcing "
" autoscan.")
job.meta['messages'] = [(ip_address, 'ralph.scan', 'info',
message)]
job.save()
autoscan_address(ip_address)
# since autoscan_address can update some fields on IPAddress,
# we need to refresh it here
ip = IPAddress.objects.get(address=ip_address)
kwargs = {
'snmp_community': ip.snmp_community,
'snmp_version': ip.snmp_version,
'http_family': ip.http_family,
'snmp_name': ip.snmp_name,
}
results = _run_plugins(ip_address, plugins, job, **kwargs)
if run_postprocessing:
_scan_postprocessing(results, job, ip_address)
if automerge and job.meta.get('changed', True):
# Run only when automerge mode is enabled and some change was
# detected. When `change` state is not available just run it...
save_job_results(job.id)
elif not called_from_ui and job.args:
try:
ip_obj = IPAddress.objects.select_related().get(
address=job.args[0] # job.args[0] == ip_address
)
except IPAddress.DoesNotExist:
pass
else:
for plugin_name in getattr(settings,
'SCAN_POSTPROCESS_ENABLED_JOBS',
[]):
try:
module = import_module(plugin_name)
except ImportError as e:
logger.error(unicode(e))
else:
module.run_job(ip_obj, plugins_results=results)
return results
def _add_dev_system(ip, pairs, parent, raw, counts, dev_id):
dev = _dev(DeviceType.blade_system, pairs, parent, raw)
ip_address, created = IPAddress.concurrent_get_or_create(address=str(ip))
if created:
ip_address.hostname = network.hostname(ip_address.address)
ip_address.device = dev
ip_address.is_management = True # FIXME: how do we know for sure?
ip_address.save(update_last_seen=True) # no priorities for IP addresses
return dev
def _add_dev_system(ip, pairs, parent, raw, counts, dev_id):
dev = _dev(DeviceType.blade_system, pairs, parent, raw)
ip_address, created = IPAddress.concurrent_get_or_create(address=str(ip))
if created:
ip_address.hostname = network.hostname(ip_address.address)
ip_address.device = dev
ip_address.is_management = True # FIXME: how do we know for sure?
ip_address.save(update_last_seen=True) # no priorities for IP addresses
return dev
def setUp(self):
self.venture, created = Venture.objects.get_or_create(
name="Sample venture", )
self.venture.save()
self.device, created = Device.objects.get_or_create(
name="Device1",
venture=self.venture,
)
self.device_without_venture, created = Device.objects.get_or_create(
name="Device2", )
self.ip1 = IPAddress(address='1.1.1.1', device=self.device)
self.ip2 = IPAddress(address='2.2.2.2', venture=self.venture)
self.ip1.save()
self.ip2.save()
self.device.save()
self.device_without_venture.save()
def _get_master(ssh, ip, data=None):
if data is None:
stdin, stdout, stderr = ssh.exec_command("cat /etc/pve/cluster.cfg")
data = stdout.read()
if not data:
stdin, stdout, stderr = ssh.exec_command("pvesh get /nodes")
data = stdout.read()
if data:
nodes = json.loads(data)
for node in nodes:
node_name = node['node']
stdin, stdout, stderr = ssh.exec_command(
'pvesh get "/nodes/%s/dns"' % node_name)
dns = json.loads(stdout.read())
ip = dns['dns1']
break
if not data:
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.save()
return ipaddr
nodes = {}
current_node = None
for line in data.splitlines():
line = line.strip()
if line.endswith('{'):
current_node = line.replace('{', '').strip()
nodes[current_node] = {}
elif line.endswith('}'):
current_node = None
elif ':' in line and current_node:
key, value = (v.strip() for v in line.split(':', 1))
nodes[current_node][key] = value
for node, pairs in nodes.iteritems():
is_master = node.startswith('master')
try:
ip = pairs['IP']
except KeyError:
continue
if is_master:
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.save()
return ipaddr
def _get_master(ssh, ip, data=None):
if data is None:
stdin, stdout, stderr = ssh.exec_command("cat /etc/pve/cluster.cfg")
data = stdout.read()
if not data:
stdin, stdout, stderr = ssh.exec_command("pvesh get /nodes")
data = stdout.read()
if data:
nodes = json.loads(data)
for node in nodes:
node_name = node['node']
stdin, stdout, stderr = ssh.exec_command(
'pvesh get "/nodes/%s/dns"' % node_name)
dns = json.loads(stdout.read())
ip = dns['dns1']
break
if not data:
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.save()
return ipaddr
nodes = {}
current_node = None
for line in data.splitlines():
line = line.strip()
if line.endswith('{'):
current_node = line.replace('{', '').strip()
nodes[current_node] = {}
elif line.endswith('}'):
current_node = None
elif ':' in line and current_node:
key, value = (v.strip() for v in line.split(':', 1))
nodes[current_node][key] = value
for node, pairs in nodes.iteritems():
is_master = node.startswith('master')
try:
ip = pairs['IP']
except KeyError:
continue
if is_master:
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.save()
return ipaddr
def setUp(self):
self.dc = DataCenter(name="test_dc")
self.dc.save()
self.net1 = Network(
name="test1",
data_center=self.dc,
address="192.168.0.1/16",
)
self.net1.save()
self.net2 = Network(
name="test2",
data_center=self.dc,
address="192.168.0.1/17",
)
self.net2.save()
self.net3 = Network(
name="test3",
data_center=self.dc,
address="192.168.128.1/17",
reserved=5,
reserved_top_margin=5,
)
self.net3.save()
self.net4 = Network(
name="test4",
data_center=self.dc,
address="192.168.133.1/24",
)
self.net4.save()
self.net5 = Network(
name="test5",
data_center=self.dc,
address="192.169.133.1/24",
)
self.net5.save()
self.ip1 = IPAddress(address="192.168.128.10")
self.ip1.save()
self.ip2 = IPAddress(address="192.168.133.10")
self.ip2.save()
self.ip3 = IPAddress(address="192.168.128.11")
self.ip3.save()
def clean(deployment_id):
"""Prepare an existing device for deployment by cleaning old information."""
deployment = Deployment.objects.get(id=deployment_id)
if deployment.status != DeploymentStatus.open:
return True
do_clean(deployment.device, deployment.user)
ip, created = IPAddress.concurrent_get_or_create(address=deployment.ip)
ip.device = deployment.device
ip.hostname = deployment.hostname
ip.save()
return True
def clean(deployment_id):
"""Prepare an existing device for deployment by cleaning old information."""
deployment = Deployment.objects.get(id=deployment_id)
if deployment.status != DeploymentStatus.open:
return True
do_clean(deployment.device, deployment.user)
ip, created = IPAddress.concurrent_get_or_create(address=deployment.ip)
ip.device=deployment.device
ip.hostname=deployment.hostname
ip.save()
return True
def _save_device(ip, name, model_name, sn, macs):
ethernets = [Eth(mac=mac, label='MAC', speed=0) for mac in macs]
dev = Device.create(sn=sn, model_name=model_name, ethernets=ethernets,
model_type=DeviceType.storage, name=name)
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.device = dev
ipaddr.is_management = True
ipaddr.save()
dev.management = ipaddr
dev.save()
return dev
def run_ssh_asa(ip):
ssh = _connect_ssh(ip)
try:
lines = ssh.asa_command(
"show version | grep (^Hardware|Boot microcode|^Serial|address is)"
)
raw_inventory = '\n'.join(ssh.asa_command("show inventory"))
finally:
ssh.close()
pairs = parse.pairs(lines=[line.strip() for line in lines])
sn = pairs.get('Serial Number', None)
model, ram, cpu = pairs['Hardware'].split(',')
boot_firmware = pairs['Boot microcode']
ethernets = []
for i in xrange(99):
try:
junk, label, mac = pairs['%d' % i].split(':')
except KeyError:
break
mac = mac.split(',', 1)[0]
mac = mac.replace('address is', '')
mac = mac.replace('.', '').upper().strip()
label = label.strip()
ethernets.append(Eth(label, mac, speed=None))
dev = Device.create(ethernets=ethernets,
sn=sn,
model_name=model,
model_type=DeviceType.firewall,
boot_firmware=boot_firmware)
dev.save(update_last_seen=True)
inventory = list(cisco_inventory(raw_inventory))
for inv in inventory:
cisco_component(dev, inv)
ipaddr, created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.device = dev
ipaddr.is_management = True
ipaddr.save()
for label, mac, speed in ethernets:
eth, created = Ethernet.concurrent_get_or_create(
mac=mac,
defaults={'device': dev},
)
eth.label = label
eth.device = dev
eth.save()
return model
def _save_device(ip, name, model_name, sn, mac):
model, model_created = DeviceModel.concurrent_get_or_create(
name = 'Onstor %s' % model_name, type=DeviceType.storage.id)
dev, dev_created = Device.concurrent_get_or_create(sn=sn, model=model)
dev.save()
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.device = dev
ipaddr.is_management = True
ipaddr.save(update_last_seen=True)
dev.management = ipaddr
dev.save(update_last_seen=True)
return dev
def assign_ips(dev, ip_addresses):
ip_addresses = {str(ip) for ip in ip_addresses}
for addr in IPAddress.objects.filter(device=dev, is_management=False):
if addr.address in ip_addresses:
continue
addr.device = None
addr.save()
for ip in ip_addresses:
addr, created = IPAddress.concurrent_get_or_create(address=ip)
addr.device = dev
addr.last_puppet = datetime.datetime.now()
addr.save()
def _save_device(ip, name, model_name, sn, mac):
model, model_created = DeviceModel.concurrent_get_or_create(
name='Onstor %s' % model_name, type=DeviceType.storage.id)
dev, dev_created = Device.concurrent_get_or_create(sn=sn, model=model)
dev.save()
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.device = dev
ipaddr.is_management = True
ipaddr.save(update_last_seen=True)
dev.management = ipaddr
dev.save(update_last_seen=True)
return dev
def assign_ips(dev, ip_addresses):
ip_addresses = {str(ip) for ip in ip_addresses}
for addr in IPAddress.objects.filter(device=dev, is_management=False):
if addr.address in ip_addresses:
continue
addr.device = None
addr.save()
for ip in ip_addresses:
addr, created = IPAddress.concurrent_get_or_create(address=ip)
addr.device = dev
addr.last_puppet = datetime.datetime.now()
addr.save()
def _save_device(ip, name, model_name, sn):
model, model_created = DeviceModel.concurrent_get_or_create(
name='3PAR %s' % model_name,
defaults={'type': DeviceType.storage.id},
)
dev = Device.create(sn=sn, model=model)
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.device = dev
ipaddr.is_management = True
ipaddr.save()
dev.management = ipaddr
dev.save()
return dev
def _save_device(ip, name, model_name, sn):
model, model_created = DeviceModel.concurrent_get_or_create(
name='3PAR %s' % model_name,
defaults={'type': DeviceType.storage.id},
)
dev = Device.create(sn=sn, model=model)
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.device = dev
ipaddr.is_management = True
ipaddr.save()
dev.management = ipaddr
dev.save()
return dev
def _run_ipmi(ip):
try:
ipmi = IPMI(ip)
fru = ipmi.get_fru()
except AuthError:
try:
ipmi = IPMI(ip, 'ADMIN')
fru = ipmi.get_fru()
except AuthError:
ipmi = IPMI(ip, 'ADMIN', 'ADMIN')
fru = ipmi.get_fru()
mc = ipmi.get_mc()
top = fru['/SYS']
if not top:
top = fru['Builtin FRU Device']
if not top:
raise AnswerError('Incompatible answer.')
name, name_clean = _clean(top['Product Name'])
sn, sn_clean = _clean(top['Product Serial'])
if sn in SERIAL_BLACKLIST:
sn = None
model_type = DeviceType.rack_server
if name.lower().startswith('ipmi'):
model_type = DeviceType.unknown
mac = ipmi.get_mac()
if mac:
ethernets = [Eth(label='IPMI MAC', mac=mac, speed=0)]
else:
ethernets = []
ethernets.extend(_get_ipmi_ethernets(fru))
dev = Device.create(ethernets=ethernets,
priority=SAVE_PRIORITY,
sn=sn,
model_name=name.title(),
model_type=model_type)
firmware = mc.get('Firmware Revision')
if firmware:
dev.mgmt_firmware = 'rev %s' % firmware
_add_ipmi_lan(dev, mac)
_add_ipmi_components(dev, fru)
dev.save(update_last_seen=True, priority=SAVE_PRIORITY)
ip_address, created = IPAddress.concurrent_get_or_create(address=str(ip))
ip_address.device = dev
ip_address.is_management = True
if created:
ip_address.hostname = network.hostname(ip_address.address)
ip_address.snmp_name = name
ip_address.save(update_last_seen=True) # no priorities for IP addresses
return name
def test_validate_ip_owner(self):
_validate_ip_owner('127.0.0.1', 'deadbeefcafe', 0)
another_device = Device.create(
ethernets=[('', 'deadbeefcafd', 0)],
model_name='splat',
model_type=DeviceType.unknown,
)
ip = IPAddress(address='127.0.0.1')
ip.save()
with self.assertRaises(forms.ValidationError):
_validate_ip_owner('127.0.0.1', 'deadbeefcafe', 0)
ip.device = Device.create(
ethernets=[('', 'deadbeefcafe', 0)],
model_name='splat',
model_type=DeviceType.unknown,
)
ip.device.save()
ip.save()
_validate_ip_owner('127.0.0.1', 'deadbeefcafe', 0)
ip.device = another_device
ip.save()
with self.assertRaises(forms.ValidationError):
_validate_ip_owner('127.0.0.1', 'deadbeefcafe', 0)
def _add_cluster_member(ssh, ip):
stdin, stdout, stderr = ssh.exec_command("ifconfig eth0 | head -n 1")
mac = stdout.readline().split()[-1]
dev = Device.create(ethernets=[Eth(label='eth0', mac=mac, speed=0)],
model_name='Proxmox',
model_type=DeviceType.unknown)
Software.create(dev, 'proxmox', 'Proxmox', family='Virtualization').save()
ipaddr, ip_created = IPAddress.concurrent_get_or_create(address=ip)
ipaddr.is_management = False
ipaddr.device = dev
ipaddr.save()
return dev
