def test_extract_item():
with open("./tests/test.html", "rb") as f:
response = httpx.Response(
200, request=httpx.Request("Get", "https://test.com"), content=f.read()
)
class Item:
pass
# extract item with xpath and regex
item_extractor = Extractor(Item)
item_extractor.add_extractor(
"paragraph",
lambda x: html.fromstring(x.text).xpath("/html/body/div/p/text()")[0],
)
item_extractor.add_extractor(
"title", lambda x: re.findall(r"<title>([A-Z a-z]+)</title>", x.text)[0]
)
item = item_extractor.extract(response)
assert item.paragraph == "test"
assert item.title == "Test html"
# extract with jpath
response = httpx.Response(
200,
request=httpx.Request("Get", "https://test.com"),
content=b'{"a": {"b": {"c": 1}}, "d": null}',
)
item_extractor = Extractor(Item)
item_extractor.add_extractor(
"author", lambda x: jpath.get_all("a.b.c", x.json())[0]
)
item_extractor.add_extractor("freedom", lambda x: jpath.get_all("d", x.json())[0])
item = item_extractor.extract(response)
assert item.author == 1
assert item.freedom is None
# ItemNestExtractor tests
with open("./tests/test.html", "rb") as f:
response = httpx.Response(
200, request=httpx.Request("Get", "https://test.com"), content=f.read()
)
item_nest_extractor = NestExtractor(
Item, lambda x: html.fromstring(x.text).xpath('//div[@id="nest"]/div')
)
item_nest_extractor.add_extractor("xpath_key", lambda x: x.xpath("./p/text()")[0])
item_nest_extractor.add_extractor(
"regex_key",
lambda x: re.findall(r"regex(\d+)</", html.tostring(x, encoding="unicode"))[0],
)
temp = 1
for item in item_nest_extractor.extract_items(response):
assert item.xpath_key == str(temp)
assert item.regex_key == str(temp)
temp += 1
def handle_lshw_fibre_cards(dev, lshw, is_virtual=False, priority=0):
buses = []
for bus in jpath.get_all('..bus', lshw):
if not bus:
continue
if isinstance(bus, list):
buses.extend(bus)
else:
buses.append(bus)
buses = filter(lambda item: item['id'].startswith('fiber'), buses)
buses.sort(key=lambda item: item['handle'])
handled_buses = set()
detected_fc_cards = set()
for bus in buses:
handle = unicode(bus['handle'])
m = re.search(r"([1-9][0-9]*)", handle)
if not m:
continue
physid = m.group(1)
if physid in handled_buses:
continue
fib, created = FibreChannel.concurrent_get_or_create(
device=dev, physical_id=physid)
fib.label = "{} {}".format(bus['vendor'], bus['product'])
fib.model, c = ComponentModel.create(
ComponentType.fibre,
family=bus['vendor'],
name=bus['product'],
priority=priority,
)
fib.save(priority=priority)
handled_buses.add(physid)
detected_fc_cards.add(fib.pk)
dev.fibrechannel_set.exclude(pk__in=detected_fc_cards).delete()
def handle_lshw_fibre_cards(dev, lshw, is_virtual=False, priority=0):
buses = []
for bus in jpath.get_all('..bus', lshw):
if not bus:
continue
if isinstance(bus, list):
buses.extend(bus)
else:
buses.append(bus)
buses = filter(lambda item: item['id'].startswith('fiber'), buses)
buses.sort(key=lambda item: item['handle'])
handled_buses = set()
detected_fc_cards = set()
for bus in buses:
handle = unicode(bus['handle'])
m = re.search(r"([1-9][0-9]*)", handle)
if not m:
continue
physid = m.group(1)
if physid in handled_buses:
continue
fib, created = FibreChannel.concurrent_get_or_create(device=dev,
physical_id=physid)
fib.label = "{} {}".format(bus['vendor'], bus['product'])
extra = fib.label
fib.model, c = ComponentModel.concurrent_get_or_create(
type=ComponentType.fibre.id, family=bus['vendor'],
extra_hash=hashlib.md5(extra).hexdigest())
fib.model.extra = extra
fib.model.name = bus['product']
fib.model.save(priority=priority)
fib.save(priority=priority)
handled_buses.add(physid)
detected_fc_cards.add(fib.pk)
dev.fibrechannel_set.exclude(pk__in=detected_fc_cards).delete()
def handle_lshw_fibre_cards(dev, lshw, is_virtual=False, priority=0):
buses = []
for bus in jpath.get_all('..bus', lshw):
if not bus:
continue
if isinstance(bus, list):
buses.extend(bus)
else:
buses.append(bus)
buses = filter(lambda item: item['id'].startswith('fiber'), buses)
buses.sort(key=lambda item: item['physid'])
handled_buses = set()
for bus in buses:
physid = unicode(bus['physid'])
for handled in handled_buses:
if physid.startswith(handled):
break
else:
fib, created = FibreChannel.concurrent_get_or_create(device=dev,
physical_id=physid)
fib.label = "{} {}".format(bus['vendor'], bus['product'])
extra = fib.label
fib.model, c = ComponentModel.concurrent_get_or_create(
type=ComponentType.fibre.id, family=bus['vendor'],
extra_hash=hashlib.md5(extra).hexdigest(),
extra=extra)
fib.model.name = bus['product']
fib.model.save(priority=priority)
fib.save(priority=priority)
handled_buses.add(physid)
def handle_lshw_fibrechannel_cards(lshw):
buses = []
for bus in jpath.get_all('..bus', lshw):
if not bus:
continue
if isinstance(bus, list):
buses.extend(bus)
else:
buses.append(bus)
buses = filter(lambda item: item['id'].startswith('fiber'), buses)
buses.sort(key=lambda item: item['handle'])
handled_buses = set()
fc_cards = []
for bus in buses:
handle = unicode(bus['handle'])
m = FC_CARD_PHYSICAL_ID_EXPRESSION.search(handle)
if not m:
continue
physid = m.group(1)
if physid in handled_buses:
continue
handled_buses.add(physid)
fc_cards.append({
'physical_id': physid,
'label': "{} {}".format(bus['vendor'], bus['product']),
'model_name': bus['product'],
})
return fc_cards
def handle_lshw_fibre_cards(dev, lshw, is_virtual=False, priority=0):
buses = []
for bus in jpath.get_all("..bus", lshw):
if not bus:
continue
if isinstance(bus, list):
buses.extend(bus)
else:
buses.append(bus)
buses = filter(lambda item: item["id"].startswith("fiber"), buses)
buses.sort(key=lambda item: item["handle"])
handled_buses = set()
detected_fc_cards = set()
for bus in buses:
handle = unicode(bus["handle"])
m = re.search(r"([1-9][0-9]*)", handle)
if not m:
continue
physid = m.group(1)
if physid in handled_buses:
continue
fib, created = FibreChannel.concurrent_get_or_create(device=dev, physical_id=physid)
fib.label = "{} {}".format(bus["vendor"], bus["product"])
fib.model, c = ComponentModel.create(
ComponentType.fibre, family=bus["vendor"], name=bus["product"], priority=priority
)
fib.save(priority=priority)
handled_buses.add(physid)
detected_fc_cards.add(fib.pk)
dev.fibrechannel_set.exclude(pk__in=detected_fc_cards).delete()
def handle_lshw_fibrechannel_cards(lshw):
buses = []
for bus in jpath.get_all('..bus', lshw):
if not bus:
continue
if isinstance(bus, list):
buses.extend(bus)
else:
buses.append(bus)
buses = filter(lambda item: item['id'].startswith('fiber'), buses)
buses.sort(key=lambda item: item['handle'])
handled_buses = set()
fc_cards = []
for bus in buses:
handle = unicode(bus['handle'])
m = FC_CARD_PHYSICAL_ID_EXPRESSION.search(handle)
if not m:
continue
physid = m.group(1)
if physid in handled_buses:
continue
handled_buses.add(physid)
fc_cards.append({
'physical_id': physid,
'label': "{} {}".format(bus['vendor'], bus['product']),
'model_name': bus['product'],
})
return fc_cards
def search(pattern: str, data: Response) -> typing.List[typing.Any]:
# convert data to json object
if isinstance(data, Response):
data = data.simple_json
elif isinstance(data, str):
data = ujson.loads(data)
return jpath.get_all(pattern, data)
def get_storage_from_lshw(lshw, no_ignore=False):
storages = []
for storage in jpath.get_all('..disk', lshw):
if not storage:
continue
if isinstance(storage, list):
storages.extend(storage)
else:
storages.append(storage)
mount_points = set()
for storage in storages:
if 'logicalname' not in storage:
continue
ln = storage['logicalname']
if isinstance(ln, list):
mount_points.update(ln)
else:
mount_points.add(ln)
parsed_storages = []
for storage in storages:
if 'size' in storage:
size = storage['size']
elif 'capacity' in storage:
size = storage['capacity']
else:
# empty slot
continue
sn = unicode(storage.get('serial') or '') or None
if sn and sn.startswith('OCZ-'):
sn = sn.replace('OCZ-', '')
if (not sn or (sn.startswith('QM000') and not no_ignore) or
(storage.get('vendor', '').strip().lower() in
DISK_VENDOR_BLACKLIST) or
(storage.get('product', '').strip().lower() in
DISK_PRODUCT_BLACKLIST)):
continue
mount_point = storage.get('logicalname', None)
storage_size = int(size['value'])
storage_size /= units.size_divisor[size['units']]
storage_size = int(storage_size)
storage_speed = 0
label = ''
if storage.get('vendor', '').strip():
label = storage['vendor'].strip() + ' '
if storage.get('product', '').strip():
label += storage['product'].strip()
elif storage.get('description', '').strip():
label += storage['description'].strip()
else:
label += 'Generic disk'
parsed_storages.append({
'mount_point': mount_point,
'sn': sn,
'size': storage_size,
'speed': storage_speed,
'label': label,
})
return mount_points, parsed_storages
def get_storage_from_lshw(lshw, no_ignore=False):
storages = []
for storage in jpath.get_all('..disk', lshw):
if not storage:
continue
if isinstance(storage, list):
storages.extend(storage)
else:
storages.append(storage)
mount_points = set()
for storage in storages:
if 'logicalname' not in storage:
continue
ln = storage['logicalname']
if isinstance(ln, list):
mount_points.update(ln)
else:
mount_points.add(ln)
parsed_storages = []
for storage in storages:
if 'size' in storage:
size = storage['size']
elif 'capacity' in storage:
size = storage['capacity']
else:
# empty slot
continue
sn = unicode(storage.get('serial') or '') or None
if sn and sn.startswith('OCZ-'):
sn = sn.replace('OCZ-', '')
if (not sn or (sn.startswith('QM000') and not no_ignore) or
(storage.get('vendor', '').strip().lower() in
DISK_VENDOR_BLACKLIST) or
(storage.get('product', '').strip().lower() in
DISK_PRODUCT_BLACKLIST)):
continue
mount_point = storage.get('logicalname', None)
storage_size = int(size['value'])
storage_size /= units.size_divisor[size['units']]
storage_size = int(storage_size)
storage_speed = 0
label = ''
if storage.get('vendor', '').strip():
label = storage['vendor'].strip() + ' '
if storage.get('product', '').strip():
label += storage['product'].strip()
elif storage.get('description', '').strip():
label += storage['description'].strip()
else:
label += 'Generic disk'
parsed_storages.append({
'mount_point': mount_point,
'sn': sn,
'size': storage_size,
'speed': storage_speed,
'label': label,
})
return mount_points, parsed_storages
def search_pattern(context, json_path, exp_value):
data = context.resp.json()
logger.debug(data)
match_all = jpath.get_all(json_path, data)
for match in match_all:
logger.debug("Actual value from response: {}".format(match))
assert exp_value in match, "Pattern not found in response. Pattern :{}, Actual value: {}".format(
exp_value, match)
logger.info("Pattern \"{}\" found in response".format(exp_value))
def handle_lshw_storage(lshw):
storages = []
for storage in jpath.get_all('..disk', lshw):
if not storage:
continue
if isinstance(storage, list):
storages.extend(storage)
else:
storages.append(storage)
detected_storages = []
for storage in storages:
if 'size' in storage:
size = storage['size']
elif 'capacity' in storage:
size = storage['capacity']
else:
# empty slot
continue
sn = unicode(storage.get('serial') or '') or None
if sn and sn.startswith('OCZ-'):
sn = sn.replace('OCZ-', '')
if (
not sn or
sn.startswith('QM000') or
storage.get(
'vendor', '',
).strip().lower() in DISK_VENDOR_BLACKLIST or
storage.get(
'product', '',
).strip().lower() in DISK_PRODUCT_BLACKLIST
):
continue
mount_point = storage.get('logicalname', None)
storage_size = int(size['value'])
storage_size /= units.size_divisor[size['units']]
storage_size = int(storage_size)
label = ''
if storage.get('vendor', '').strip():
label = storage['vendor'].strip() + ' '
if storage.get('product', '').strip():
label += storage['product'].strip()
elif storage.get('description', '').strip():
label += storage['description'].strip()
else:
label += 'Generic disk'
family = storage['vendor'].strip() or 'Generic disk'
detected_storages.append({
'mount_point': mount_point,
'serial_number': sn,
'size': storage_size,
'label': label,
'family': family,
})
return detected_storages
def handle_lshw_storage(lshw):
storages = []
for storage in jpath.get_all('..disk', lshw):
if not storage:
continue
if isinstance(storage, list):
storages.extend(storage)
else:
storages.append(storage)
detected_storages = []
for storage in storages:
if 'size' in storage:
size = storage['size']
elif 'capacity' in storage:
size = storage['capacity']
else:
# empty slot
continue
sn = unicode(storage.get('serial') or '') or None
if sn and sn.startswith('OCZ-'):
sn = sn.replace('OCZ-', '')
if (
not sn or
sn.startswith('QM000') or
storage.get(
'vendor', '',
).strip().lower() in DISK_VENDOR_BLACKLIST or
storage.get(
'product', '',
).strip().lower() in DISK_PRODUCT_BLACKLIST
):
continue
mount_point = storage.get('logicalname', None)
storage_size = int(size['value'])
storage_size /= units.size_divisor[size['units']]
storage_size = int(storage_size)
label = ''
if storage.get('vendor', '').strip():
label = storage['vendor'].strip() + ' '
if storage.get('product', '').strip():
label += storage['product'].strip()
elif storage.get('description', '').strip():
label += storage['description'].strip()
else:
label += 'Generic disk'
family = storage['vendor'].strip() or 'Generic disk'
detected_storages.append({
'mount_point': mount_point,
'serial_number': sn,
'size': storage_size,
'label': label,
'family': family,
})
return detected_storages
def get_storage_from_lshw(lshw, no_ignore=False):
storages = []
for storage in jpath.get_all("..disk", lshw):
if not storage:
continue
if isinstance(storage, list):
storages.extend(storage)
else:
storages.append(storage)
mount_points = set()
for storage in storages:
if "logicalname" not in storage:
continue
ln = storage["logicalname"]
if isinstance(ln, list):
mount_points.update(ln)
else:
mount_points.add(ln)
parsed_storages = []
for storage in storages:
if "size" in storage:
size = storage["size"]
elif "capacity" in storage:
size = storage["capacity"]
else:
# empty slot
continue
sn = unicode(storage.get("serial") or "") or None
if (
not sn
or (sn.startswith("QM000") and not no_ignore)
or (storage.get("vendor", "").strip().lower() in DISK_VENDOR_BLACKLIST)
or (storage.get("product", "").strip().lower() in DISK_PRODUCT_BLACKLIST)
):
continue
mount_point = storage.get("logicalname", None)
storage_size = int(size["value"])
storage_size /= units.size_divisor[size["units"]]
storage_size = int(storage_size)
storage_speed = 0
label = ""
if storage.get("vendor", "").strip():
label = storage["vendor"].strip() + " "
if storage.get("product", "").strip():
label += storage["product"].strip()
elif storage.get("description", "").strip():
label += storage["description"].strip()
else:
label += "Generic disk"
parsed_storages.append(
{"mount_point": mount_point, "sn": sn, "size": storage_size, "speed": storage_speed, "label": label}
)
return mount_points, parsed_storages
def handle_lshw_mac_addresses(lshw):
mac_addresses = set()
ethernets = sorted(
(e for e in jpath.get_all('..network', lshw) if e),
key=_get_logical_name,
)
for i, ethernet in enumerate(untangle(ethernets)):
try:
mac = MACAddressField.normalize(ethernet['serial'])
except (ValueError, KeyError):
continue
if not mac:
continue
mac_addresses.add(mac)
return list(mac_addresses)
def handle_lshw_mac_addresses(lshw):
mac_addresses = set()
ethernets = sorted(
(e for e in jpath.get_all('..network', lshw) if e),
key=_get_logical_name,
)
for i, ethernet in enumerate(untangle(ethernets)):
try:
mac = MACAddressField.normalize(ethernet['serial'])
except (ValueError, KeyError):
continue
if not mac:
continue
mac_addresses.add(mac)
return list(mac_addresses)
def handle_lshw_ethernets(lshw):
ethernets = sorted((e for e in jpath.get_all("..network", lshw) if e), key=get_logical_name)
for i, ethernet in enumerate(untangle(ethernets)):
try:
mac = MACAddressField.normalize(ethernet["serial"])
except (ValueError, KeyError):
continue
if not mac:
continue
full_name = ethernet["product"]
if ethernet["vendor"] not in full_name:
full_name = "{} {}".format(ethernet["vendor"], full_name)
label = "{}: {}".format(get_logical_name(ethernet), full_name)
caps = set(ethernet["capabilities"].keys())
if "1000bt-fd" in caps or "1000bt" in caps:
speed = EthernetSpeed.s1gbit.id
elif "100bt-fd" in caps or "100bt" in caps:
speed = EthernetSpeed.s100mbit.id
else:
speed = None
yield Eth(label, mac, speed)
def handle_lshw_ethernets(lshw):
ethernets = sorted((e for e in jpath.get_all('..network', lshw) if e),
key=get_logical_name)
for i, ethernet in enumerate(untangle(ethernets)):
try:
mac = MACAddressField.normalize(ethernet['serial'])
except (ValueError, KeyError):
continue
if not mac:
continue
full_name = ethernet['product']
if ethernet['vendor'] not in full_name:
full_name = "{} {}".format(ethernet['vendor'], full_name)
label = "{}: {}".format(get_logical_name(ethernet), full_name)
caps = set(ethernet['capabilities'].keys())
if '1000bt-fd' in caps or '1000bt' in caps:
speed = EthernetSpeed.s1gbit.id
elif '100bt-fd' in caps or '100bt' in caps:
speed = EthernetSpeed.s100mbit.id
else:
speed = None
yield Eth(label, mac, speed)
def handle_lshw_storage(dev, lshw, is_virtual=False, priority=0):
storages = []
for storage in jpath.get_all('..disk', lshw):
if not storage:
continue
if isinstance(storage, list):
storages.extend(storage)
else:
storages.append(storage)
storages.sort(key=get_logical_name)
mount_points = set()
for stor in storages:
if 'logicalname' not in stor:
continue
ln = stor['logicalname']
if isinstance(ln, list):
mount_points.update(ln)
else:
mount_points.add(ln)
dev.storage_set.filter(mount_point__in=mount_points).delete()
for storage in storages:
if 'size' in storage:
size = storage['size']
elif 'capacity' in storage:
size = storage['capacity']
else:
# empty slot
continue
sn = unicode(storage.get('serial') or '') or None
if not sn or sn.startswith('QM000') or \
storage.get('vendor', '').strip().lower() in DISK_VENDOR_BLACKLIST or \
storage.get('product', '').strip().lower() in DISK_PRODUCT_BLACKLIST:
continue
if sn:
stor, created = Storage.concurrent_get_or_create(sn=sn, device=dev)
stor.mount_point = storage.get('logicalname', None)
else:
stor, created = Storage.concurrent_get_or_create(
sn=None,
device=dev,
mount_point=storage.get('logicalname', None))
stor.size = int(size['value'])
stor.size /= units.size_divisor[size['units']]
stor.size = int(stor.size)
stor.speed = 0
if storage.get('vendor', '').strip():
stor.label = storage['vendor'].strip() + ' '
else:
stor.label = ''
if storage.get('product', '').strip():
stor.label += storage['product'].strip()
elif storage.get('description', '').strip():
stor.label += storage['description'].strip()
else:
stor.label += 'Generic disk'
caps = storage['capabilities']
extra = "\n".join([
": ".join((unicode(key), unicode(caps[key]) or ''))
for key in sorted(caps.keys())
])
stor.model, c = ComponentModel.concurrent_get_or_create(
size=stor.size,
speed=stor.speed,
type=ComponentType.disk.id,
family='',
extra_hash=hashlib.md5(extra).hexdigest(),
extra=extra)
stor.model.name = '{} {}MiB'.format(stor.label, stor.size)
stor.model.save(priority=priority)
stor.save(priority=priority)
def handle_lshw_storage(dev, lshw, is_virtual=False, priority=0):
storages = []
for storage in jpath.get_all("..disk", lshw):
if not storage:
continue
if isinstance(storage, list):
storages.extend(storage)
else:
storages.append(storage)
storages.sort(key=get_logical_name)
mount_points = set()
for stor in storages:
if "logicalname" not in stor:
continue
ln = stor["logicalname"]
if isinstance(ln, list):
mount_points.update(ln)
else:
mount_points.add(ln)
dev.storage_set.filter(mount_point__in=mount_points).delete()
for storage in storages:
if "size" in storage:
size = storage["size"]
elif "capacity" in storage:
size = storage["capacity"]
else:
# empty slot
continue
sn = unicode(storage.get("serial") or "") or None
if (
not sn
or sn.startswith("QM000")
or storage.get("vendor", "").strip().lower() in DISK_VENDOR_BLACKLIST
or storage.get("product", "").strip().lower() in DISK_PRODUCT_BLACKLIST
):
continue
if sn:
stor, created = Storage.concurrent_get_or_create(sn=sn, device=dev)
stor.mount_point = storage.get("logicalname", None)
else:
stor, created = Storage.concurrent_get_or_create(
sn=None, device=dev, mount_point=storage.get("logicalname", None)
)
stor.size = int(size["value"])
stor.size /= units.size_divisor[size["units"]]
stor.size = int(stor.size)
stor.speed = 0
if storage.get("vendor", "").strip():
stor.label = storage["vendor"].strip() + " "
else:
stor.label = ""
if storage.get("product", "").strip():
stor.label += storage["product"].strip()
elif storage.get("description", "").strip():
stor.label += storage["description"].strip()
else:
stor.label += "Generic disk"
caps = storage["capabilities"]
extra = "\n".join([": ".join((unicode(key), unicode(caps[key]) or "")) for key in sorted(caps.keys())])
stor.model, c = ComponentModel.concurrent_get_or_create(
size=stor.size,
speed=stor.speed,
type=ComponentType.disk.id,
family="",
extra_hash=hashlib.md5(extra).hexdigest(),
extra=extra,
)
stor.model.name = "{} {}MiB".format(stor.label, stor.size)
stor.model.save(priority=priority)
stor.save(priority=priority)
exit()
logw('Сreating an additional field in the host-matrix based on data from Vulners')
# формируем доп-поля в матрице на основе данных от вулнерса
current_host = 0
logw('Processed hosts')
for h in h_matrix:
current_host += 1
try:
# список словарей из bulletitID + SCORE, для этого хоста
h_bulletins = list()
# список словарей из PKG + его SCORE + его bulletitID, для этого хоста
h_packages_tmp = list()
# из форматированного через jpath-джейсона получаем пакет, его бюллетень и балл
for row in jpath.get_all(jpath=jpath_mask, data=h['vuln_data']):
pkg = row['package']
bull = row['bulletinID']
score = row['cvss']['score']
# добавляем double-словарь (bull,score) во временный список (для будущего шаманства с бюллетенями)
h_bulletins.append({'name': bull, 'score': score})
# добавляем triple-словарь (pkg,score,bull) во второй временный список (для будущего шаманства с пакетами)
h_packages_tmp.append({'name': pkg, 'score': score, 'bull': bull})
# убираем дубли одинаковые пакеты, но разные баллы; оставляя только самый высокий бал
h_packages = list()
# для каждого пакета в списке "пакет-балл"
for r in h_packages_tmp:
pkg = r['name']
# оставляем во временном списке словарей (куцом) только сторки с пакетами = пакету текущей строки
h_pkg_tmp = [i for i in h_packages_tmp if i['name'] == pkg]
def handle_lshw_storage(dev, lshw, is_virtual=False, priority=0):
storages = []
for storage in jpath.get_all('..disk', lshw):
if not storage:
continue
if isinstance(storage, list):
storages.extend(storage)
else:
storages.append(storage)
storages.sort(key=get_logical_name)
mount_points = set()
for stor in storages:
if 'logicalname' not in stor:
continue
ln = stor['logicalname']
if isinstance(ln, list):
mount_points.update(ln)
else:
mount_points.add(ln)
dev.storage_set.filter(mount_point__in=mount_points).delete()
for storage in storages:
if 'size' in storage:
size = storage['size']
elif 'capacity' in storage:
size = storage['capacity']
else:
# empty slot
continue
sn = unicode(storage.get('serial') or '') or None
if not sn or sn.startswith('QM000') or \
storage.get('vendor', '').strip().lower() in DISK_VENDOR_BLACKLIST or \
storage.get('product', '').strip().lower() in DISK_PRODUCT_BLACKLIST:
continue
if sn:
stor, created = Storage.concurrent_get_or_create(sn=sn, device=dev)
stor.mount_point = storage.get('logicalname', None)
else:
stor, created = Storage.concurrent_get_or_create(sn=None, device=dev,
mount_point=storage.get('logicalname', None))
stor.size = int(size['value'])
stor.size /= units.size_divisor[size['units']]
stor.size = int(stor.size)
stor.speed = 0
if storage.get('vendor', '').strip():
stor.label = storage['vendor'].strip() + ' '
else:
stor.label = ''
if storage.get('product', '').strip():
stor.label += storage['product'].strip()
elif storage.get('description', '').strip():
stor.label += storage['description'].strip()
else:
stor.label += 'Generic disk'
caps = storage['capabilities']
extra = "\n".join([": ".join((unicode(key), unicode(caps[key]) or '')) for key in
sorted(caps.keys())])
stor.model, c = ComponentModel.concurrent_get_or_create(
size=stor.size, speed=stor.speed, type=ComponentType.disk.id,
family='', extra_hash=hashlib.md5(extra).hexdigest(), extra=extra)
stor.model.name = '{} {}MiB'.format(stor.label, stor.size)
stor.model.save(priority=priority)
stor.save(priority=priority)