Пример #1
0
def test_extract_item():
    with open("./tests/test.html", "rb") as f:
        response = httpx.Response(
            200, request=httpx.Request("Get", "https://test.com"), content=f.read()
        )

    class Item:
        pass

    # extract item with xpath and regex
    item_extractor = Extractor(Item)
    item_extractor.add_extractor(
        "paragraph",
        lambda x: html.fromstring(x.text).xpath("/html/body/div/p/text()")[0],
    )
    item_extractor.add_extractor(
        "title", lambda x: re.findall(r"<title>([A-Z a-z]+)</title>", x.text)[0]
    )
    item = item_extractor.extract(response)
    assert item.paragraph == "test"
    assert item.title == "Test html"
    # extract with jpath
    response = httpx.Response(
        200,
        request=httpx.Request("Get", "https://test.com"),
        content=b'{"a": {"b": {"c": 1}}, "d": null}',
    )
    item_extractor = Extractor(Item)
    item_extractor.add_extractor(
        "author", lambda x: jpath.get_all("a.b.c", x.json())[0]
    )
    item_extractor.add_extractor("freedom", lambda x: jpath.get_all("d", x.json())[0])
    item = item_extractor.extract(response)
    assert item.author == 1
    assert item.freedom is None
    # ItemNestExtractor tests
    with open("./tests/test.html", "rb") as f:
        response = httpx.Response(
            200, request=httpx.Request("Get", "https://test.com"), content=f.read()
        )
    item_nest_extractor = NestExtractor(
        Item, lambda x: html.fromstring(x.text).xpath('//div[@id="nest"]/div')
    )
    item_nest_extractor.add_extractor("xpath_key", lambda x: x.xpath("./p/text()")[0])
    item_nest_extractor.add_extractor(
        "regex_key",
        lambda x: re.findall(r"regex(\d+)</", html.tostring(x, encoding="unicode"))[0],
    )
    temp = 1
    for item in item_nest_extractor.extract_items(response):
        assert item.xpath_key == str(temp)
        assert item.regex_key == str(temp)
        temp += 1
Пример #2
0
def handle_lshw_fibre_cards(dev, lshw, is_virtual=False, priority=0):
    buses = []
    for bus in jpath.get_all('..bus', lshw):
        if not bus:
            continue
        if isinstance(bus, list):
            buses.extend(bus)
        else:
            buses.append(bus)
    buses = filter(lambda item: item['id'].startswith('fiber'), buses)
    buses.sort(key=lambda item: item['handle'])
    handled_buses = set()
    detected_fc_cards = set()
    for bus in buses:
        handle = unicode(bus['handle'])
        m = re.search(r"([1-9][0-9]*)", handle)
        if not m:
            continue
        physid = m.group(1)
        if physid in handled_buses:
            continue
        fib, created = FibreChannel.concurrent_get_or_create(
            device=dev, physical_id=physid)
        fib.label = "{} {}".format(bus['vendor'], bus['product'])
        fib.model, c = ComponentModel.create(
            ComponentType.fibre,
            family=bus['vendor'],
            name=bus['product'],
            priority=priority,
        )
        fib.save(priority=priority)
        handled_buses.add(physid)
        detected_fc_cards.add(fib.pk)
    dev.fibrechannel_set.exclude(pk__in=detected_fc_cards).delete()
Пример #3
0
def handle_lshw_fibre_cards(dev, lshw, is_virtual=False, priority=0):
    buses = []
    for bus in jpath.get_all('..bus', lshw):
        if not bus:
            continue
        if isinstance(bus, list):
            buses.extend(bus)
        else:
            buses.append(bus)
    buses = filter(lambda item: item['id'].startswith('fiber'), buses)
    buses.sort(key=lambda item: item['handle'])
    handled_buses = set()
    detected_fc_cards = set()
    for bus in buses:
        handle = unicode(bus['handle'])
        m = re.search(r"([1-9][0-9]*)", handle)
        if not m:
            continue
        physid = m.group(1)
        if physid in handled_buses:
            continue
        fib, created = FibreChannel.concurrent_get_or_create(device=dev,
            physical_id=physid)
        fib.label = "{} {}".format(bus['vendor'], bus['product'])
        extra = fib.label
        fib.model, c = ComponentModel.concurrent_get_or_create(
            type=ComponentType.fibre.id, family=bus['vendor'],
            extra_hash=hashlib.md5(extra).hexdigest())
        fib.model.extra = extra
        fib.model.name = bus['product']
        fib.model.save(priority=priority)
        fib.save(priority=priority)
        handled_buses.add(physid)
        detected_fc_cards.add(fib.pk)
    dev.fibrechannel_set.exclude(pk__in=detected_fc_cards).delete()
Пример #4
0
def handle_lshw_fibre_cards(dev, lshw, is_virtual=False, priority=0):
    buses = []
    for bus in jpath.get_all('..bus', lshw):
        if not bus:
            continue
        if isinstance(bus, list):
            buses.extend(bus)
        else:
            buses.append(bus)
    buses = filter(lambda item: item['id'].startswith('fiber'), buses)
    buses.sort(key=lambda item: item['physid'])
    handled_buses = set()
    for bus in buses:
        physid = unicode(bus['physid'])
        for handled in handled_buses:
            if physid.startswith(handled):
                break
        else:
            fib, created = FibreChannel.concurrent_get_or_create(device=dev,
                physical_id=physid)
            fib.label = "{} {}".format(bus['vendor'], bus['product'])
            extra = fib.label
            fib.model, c = ComponentModel.concurrent_get_or_create(
                type=ComponentType.fibre.id, family=bus['vendor'],
                            extra_hash=hashlib.md5(extra).hexdigest(),
                            extra=extra)
            fib.model.name = bus['product']
            fib.model.save(priority=priority)
            fib.save(priority=priority)
        handled_buses.add(physid)
Пример #5
0
def handle_lshw_fibrechannel_cards(lshw):
    buses = []
    for bus in jpath.get_all('..bus', lshw):
        if not bus:
            continue
        if isinstance(bus, list):
            buses.extend(bus)
        else:
            buses.append(bus)
    buses = filter(lambda item: item['id'].startswith('fiber'), buses)
    buses.sort(key=lambda item: item['handle'])
    handled_buses = set()
    fc_cards = []
    for bus in buses:
        handle = unicode(bus['handle'])
        m = FC_CARD_PHYSICAL_ID_EXPRESSION.search(handle)
        if not m:
            continue
        physid = m.group(1)
        if physid in handled_buses:
            continue
        handled_buses.add(physid)
        fc_cards.append({
            'physical_id': physid,
            'label': "{} {}".format(bus['vendor'], bus['product']),
            'model_name': bus['product'],
        })
    return fc_cards
Пример #6
0
def handle_lshw_fibre_cards(dev, lshw, is_virtual=False, priority=0):
    buses = []
    for bus in jpath.get_all("..bus", lshw):
        if not bus:
            continue
        if isinstance(bus, list):
            buses.extend(bus)
        else:
            buses.append(bus)
    buses = filter(lambda item: item["id"].startswith("fiber"), buses)
    buses.sort(key=lambda item: item["handle"])
    handled_buses = set()
    detected_fc_cards = set()
    for bus in buses:
        handle = unicode(bus["handle"])
        m = re.search(r"([1-9][0-9]*)", handle)
        if not m:
            continue
        physid = m.group(1)
        if physid in handled_buses:
            continue
        fib, created = FibreChannel.concurrent_get_or_create(device=dev, physical_id=physid)
        fib.label = "{} {}".format(bus["vendor"], bus["product"])
        fib.model, c = ComponentModel.create(
            ComponentType.fibre, family=bus["vendor"], name=bus["product"], priority=priority
        )
        fib.save(priority=priority)
        handled_buses.add(physid)
        detected_fc_cards.add(fib.pk)
    dev.fibrechannel_set.exclude(pk__in=detected_fc_cards).delete()
Пример #7
0
def handle_lshw_fibrechannel_cards(lshw):
    buses = []
    for bus in jpath.get_all('..bus', lshw):
        if not bus:
            continue
        if isinstance(bus, list):
            buses.extend(bus)
        else:
            buses.append(bus)
    buses = filter(lambda item: item['id'].startswith('fiber'), buses)
    buses.sort(key=lambda item: item['handle'])
    handled_buses = set()
    fc_cards = []
    for bus in buses:
        handle = unicode(bus['handle'])
        m = FC_CARD_PHYSICAL_ID_EXPRESSION.search(handle)
        if not m:
            continue
        physid = m.group(1)
        if physid in handled_buses:
            continue
        handled_buses.add(physid)
        fc_cards.append({
            'physical_id': physid,
            'label': "{} {}".format(bus['vendor'], bus['product']),
            'model_name': bus['product'],
        })
    return fc_cards
Пример #8
0
 def search(pattern: str, data: Response) -> typing.List[typing.Any]:
     # convert data to json object
     if isinstance(data, Response):
         data = data.simple_json
     elif isinstance(data, str):
         data = ujson.loads(data)
     return jpath.get_all(pattern, data)
Пример #9
0
def get_storage_from_lshw(lshw, no_ignore=False):
    storages = []
    for storage in jpath.get_all('..disk', lshw):
        if not storage:
            continue
        if isinstance(storage, list):
            storages.extend(storage)
        else:
            storages.append(storage)
    mount_points = set()
    for storage in storages:
        if 'logicalname' not in storage:
            continue
        ln = storage['logicalname']
        if isinstance(ln, list):
            mount_points.update(ln)
        else:
            mount_points.add(ln)
    parsed_storages = []
    for storage in storages:
        if 'size' in storage:
            size = storage['size']
        elif 'capacity' in storage:
            size = storage['capacity']
        else:
            # empty slot
            continue
        sn = unicode(storage.get('serial') or '') or None
        if sn and sn.startswith('OCZ-'):
            sn = sn.replace('OCZ-', '')
        if (not sn or (sn.startswith('QM000') and not no_ignore) or
            (storage.get('vendor', '').strip().lower() in
                DISK_VENDOR_BLACKLIST) or
            (storage.get('product', '').strip().lower() in
                DISK_PRODUCT_BLACKLIST)):
            continue
        mount_point = storage.get('logicalname', None)
        storage_size = int(size['value'])
        storage_size /= units.size_divisor[size['units']]
        storage_size = int(storage_size)
        storage_speed = 0
        label = ''
        if storage.get('vendor', '').strip():
            label = storage['vendor'].strip() + ' '
        if storage.get('product', '').strip():
            label += storage['product'].strip()
        elif storage.get('description', '').strip():
            label += storage['description'].strip()
        else:
            label += 'Generic disk'
        parsed_storages.append({
            'mount_point': mount_point,
            'sn': sn,
            'size': storage_size,
            'speed': storage_speed,
            'label': label,
        })
    return mount_points, parsed_storages
Пример #10
0
def get_storage_from_lshw(lshw, no_ignore=False):
    storages = []
    for storage in jpath.get_all('..disk', lshw):
        if not storage:
            continue
        if isinstance(storage, list):
            storages.extend(storage)
        else:
            storages.append(storage)
    mount_points = set()
    for storage in storages:
        if 'logicalname' not in storage:
            continue
        ln = storage['logicalname']
        if isinstance(ln, list):
            mount_points.update(ln)
        else:
            mount_points.add(ln)
    parsed_storages = []
    for storage in storages:
        if 'size' in storage:
            size = storage['size']
        elif 'capacity' in storage:
            size = storage['capacity']
        else:
            # empty slot
            continue
        sn = unicode(storage.get('serial') or '') or None
        if sn and sn.startswith('OCZ-'):
            sn = sn.replace('OCZ-', '')
        if (not sn or (sn.startswith('QM000') and not no_ignore) or
            (storage.get('vendor', '').strip().lower() in
                DISK_VENDOR_BLACKLIST) or
            (storage.get('product', '').strip().lower() in
                DISK_PRODUCT_BLACKLIST)):
            continue
        mount_point = storage.get('logicalname', None)
        storage_size = int(size['value'])
        storage_size /= units.size_divisor[size['units']]
        storage_size = int(storage_size)
        storage_speed = 0
        label = ''
        if storage.get('vendor', '').strip():
            label = storage['vendor'].strip() + ' '
        if storage.get('product', '').strip():
            label += storage['product'].strip()
        elif storage.get('description', '').strip():
            label += storage['description'].strip()
        else:
            label += 'Generic disk'
        parsed_storages.append({
            'mount_point': mount_point,
            'sn': sn,
            'size': storage_size,
            'speed': storage_speed,
            'label': label,
        })
    return mount_points, parsed_storages
Пример #11
0
def search_pattern(context, json_path, exp_value):
    data = context.resp.json()
    logger.debug(data)
    match_all = jpath.get_all(json_path, data)
    for match in match_all:
        logger.debug("Actual value from response: {}".format(match))
        assert exp_value in match, "Pattern not found in response. Pattern :{}, Actual value: {}".format(
            exp_value, match)
    logger.info("Pattern \"{}\" found in response".format(exp_value))
Пример #12
0
def handle_lshw_storage(lshw):
    storages = []
    for storage in jpath.get_all('..disk', lshw):
        if not storage:
            continue
        if isinstance(storage, list):
            storages.extend(storage)
        else:
            storages.append(storage)
    detected_storages = []
    for storage in storages:
        if 'size' in storage:
            size = storage['size']
        elif 'capacity' in storage:
            size = storage['capacity']
        else:
            # empty slot
            continue
        sn = unicode(storage.get('serial') or '') or None
        if sn and sn.startswith('OCZ-'):
            sn = sn.replace('OCZ-', '')
        if (
            not sn or
            sn.startswith('QM000') or
            storage.get(
                'vendor', '',
            ).strip().lower() in DISK_VENDOR_BLACKLIST or
            storage.get(
                'product', '',
            ).strip().lower() in DISK_PRODUCT_BLACKLIST
        ):
            continue
        mount_point = storage.get('logicalname', None)
        storage_size = int(size['value'])
        storage_size /= units.size_divisor[size['units']]
        storage_size = int(storage_size)
        label = ''
        if storage.get('vendor', '').strip():
            label = storage['vendor'].strip() + ' '
        if storage.get('product', '').strip():
            label += storage['product'].strip()
        elif storage.get('description', '').strip():
            label += storage['description'].strip()
        else:
            label += 'Generic disk'
        family = storage['vendor'].strip() or 'Generic disk'
        detected_storages.append({
            'mount_point': mount_point,
            'serial_number': sn,
            'size': storage_size,
            'label': label,
            'family': family,
        })
    return detected_storages
Пример #13
0
def handle_lshw_storage(lshw):
    storages = []
    for storage in jpath.get_all('..disk', lshw):
        if not storage:
            continue
        if isinstance(storage, list):
            storages.extend(storage)
        else:
            storages.append(storage)
    detected_storages = []
    for storage in storages:
        if 'size' in storage:
            size = storage['size']
        elif 'capacity' in storage:
            size = storage['capacity']
        else:
            # empty slot
            continue
        sn = unicode(storage.get('serial') or '') or None
        if sn and sn.startswith('OCZ-'):
            sn = sn.replace('OCZ-', '')
        if (
            not sn or
            sn.startswith('QM000') or
            storage.get(
                'vendor', '',
            ).strip().lower() in DISK_VENDOR_BLACKLIST or
            storage.get(
                'product', '',
            ).strip().lower() in DISK_PRODUCT_BLACKLIST
        ):
            continue
        mount_point = storage.get('logicalname', None)
        storage_size = int(size['value'])
        storage_size /= units.size_divisor[size['units']]
        storage_size = int(storage_size)
        label = ''
        if storage.get('vendor', '').strip():
            label = storage['vendor'].strip() + ' '
        if storage.get('product', '').strip():
            label += storage['product'].strip()
        elif storage.get('description', '').strip():
            label += storage['description'].strip()
        else:
            label += 'Generic disk'
        family = storage['vendor'].strip() or 'Generic disk'
        detected_storages.append({
            'mount_point': mount_point,
            'serial_number': sn,
            'size': storage_size,
            'label': label,
            'family': family,
        })
    return detected_storages
Пример #14
0
def get_storage_from_lshw(lshw, no_ignore=False):
    storages = []
    for storage in jpath.get_all("..disk", lshw):
        if not storage:
            continue
        if isinstance(storage, list):
            storages.extend(storage)
        else:
            storages.append(storage)
    mount_points = set()
    for storage in storages:
        if "logicalname" not in storage:
            continue
        ln = storage["logicalname"]
        if isinstance(ln, list):
            mount_points.update(ln)
        else:
            mount_points.add(ln)
    parsed_storages = []
    for storage in storages:
        if "size" in storage:
            size = storage["size"]
        elif "capacity" in storage:
            size = storage["capacity"]
        else:
            # empty slot
            continue
        sn = unicode(storage.get("serial") or "") or None
        if (
            not sn
            or (sn.startswith("QM000") and not no_ignore)
            or (storage.get("vendor", "").strip().lower() in DISK_VENDOR_BLACKLIST)
            or (storage.get("product", "").strip().lower() in DISK_PRODUCT_BLACKLIST)
        ):
            continue
        mount_point = storage.get("logicalname", None)
        storage_size = int(size["value"])
        storage_size /= units.size_divisor[size["units"]]
        storage_size = int(storage_size)
        storage_speed = 0
        label = ""
        if storage.get("vendor", "").strip():
            label = storage["vendor"].strip() + " "
        if storage.get("product", "").strip():
            label += storage["product"].strip()
        elif storage.get("description", "").strip():
            label += storage["description"].strip()
        else:
            label += "Generic disk"
        parsed_storages.append(
            {"mount_point": mount_point, "sn": sn, "size": storage_size, "speed": storage_speed, "label": label}
        )
    return mount_points, parsed_storages
Пример #15
0
def handle_lshw_mac_addresses(lshw):
    mac_addresses = set()
    ethernets = sorted(
        (e for e in jpath.get_all('..network', lshw) if e),
        key=_get_logical_name,
    )
    for i, ethernet in enumerate(untangle(ethernets)):
        try:
            mac = MACAddressField.normalize(ethernet['serial'])
        except (ValueError, KeyError):
            continue
        if not mac:
            continue
        mac_addresses.add(mac)
    return list(mac_addresses)
Пример #16
0
def handle_lshw_mac_addresses(lshw):
    mac_addresses = set()
    ethernets = sorted(
        (e for e in jpath.get_all('..network', lshw) if e),
        key=_get_logical_name,
    )
    for i, ethernet in enumerate(untangle(ethernets)):
        try:
            mac = MACAddressField.normalize(ethernet['serial'])
        except (ValueError, KeyError):
            continue
        if not mac:
            continue
        mac_addresses.add(mac)
    return list(mac_addresses)
Пример #17
0
def handle_lshw_ethernets(lshw):
    ethernets = sorted((e for e in jpath.get_all("..network", lshw) if e), key=get_logical_name)
    for i, ethernet in enumerate(untangle(ethernets)):
        try:
            mac = MACAddressField.normalize(ethernet["serial"])
        except (ValueError, KeyError):
            continue
        if not mac:
            continue
        full_name = ethernet["product"]
        if ethernet["vendor"] not in full_name:
            full_name = "{} {}".format(ethernet["vendor"], full_name)
        label = "{}: {}".format(get_logical_name(ethernet), full_name)
        caps = set(ethernet["capabilities"].keys())
        if "1000bt-fd" in caps or "1000bt" in caps:
            speed = EthernetSpeed.s1gbit.id
        elif "100bt-fd" in caps or "100bt" in caps:
            speed = EthernetSpeed.s100mbit.id
        else:
            speed = None
        yield Eth(label, mac, speed)
Пример #18
0
def handle_lshw_ethernets(lshw):
    ethernets = sorted((e for e in jpath.get_all('..network', lshw) if e),
                       key=get_logical_name)
    for i, ethernet in enumerate(untangle(ethernets)):
        try:
            mac = MACAddressField.normalize(ethernet['serial'])
        except (ValueError, KeyError):
            continue
        if not mac:
            continue
        full_name = ethernet['product']
        if ethernet['vendor'] not in full_name:
            full_name = "{} {}".format(ethernet['vendor'], full_name)
        label = "{}: {}".format(get_logical_name(ethernet), full_name)
        caps = set(ethernet['capabilities'].keys())
        if '1000bt-fd' in caps or '1000bt' in caps:
            speed = EthernetSpeed.s1gbit.id
        elif '100bt-fd' in caps or '100bt' in caps:
            speed = EthernetSpeed.s100mbit.id
        else:
            speed = None
        yield Eth(label, mac, speed)
Пример #19
0
def handle_lshw_storage(dev, lshw, is_virtual=False, priority=0):
    storages = []
    for storage in jpath.get_all('..disk', lshw):
        if not storage:
            continue
        if isinstance(storage, list):
            storages.extend(storage)
        else:
            storages.append(storage)
    storages.sort(key=get_logical_name)
    mount_points = set()
    for stor in storages:
        if 'logicalname' not in stor:
            continue
        ln = stor['logicalname']
        if isinstance(ln, list):
            mount_points.update(ln)
        else:
            mount_points.add(ln)
    dev.storage_set.filter(mount_point__in=mount_points).delete()
    for storage in storages:
        if 'size' in storage:
            size = storage['size']
        elif 'capacity' in storage:
            size = storage['capacity']
        else:
            # empty slot
            continue
        sn = unicode(storage.get('serial') or '') or None
        if not sn or sn.startswith('QM000') or \
                storage.get('vendor', '').strip().lower() in DISK_VENDOR_BLACKLIST or \
                storage.get('product', '').strip().lower() in DISK_PRODUCT_BLACKLIST:
            continue
        if sn:
            stor, created = Storage.concurrent_get_or_create(sn=sn, device=dev)
            stor.mount_point = storage.get('logicalname', None)
        else:
            stor, created = Storage.concurrent_get_or_create(
                sn=None,
                device=dev,
                mount_point=storage.get('logicalname', None))
        stor.size = int(size['value'])
        stor.size /= units.size_divisor[size['units']]
        stor.size = int(stor.size)
        stor.speed = 0
        if storage.get('vendor', '').strip():
            stor.label = storage['vendor'].strip() + ' '
        else:
            stor.label = ''
        if storage.get('product', '').strip():
            stor.label += storage['product'].strip()
        elif storage.get('description', '').strip():
            stor.label += storage['description'].strip()
        else:
            stor.label += 'Generic disk'
        caps = storage['capabilities']
        extra = "\n".join([
            ": ".join((unicode(key), unicode(caps[key]) or ''))
            for key in sorted(caps.keys())
        ])
        stor.model, c = ComponentModel.concurrent_get_or_create(
            size=stor.size,
            speed=stor.speed,
            type=ComponentType.disk.id,
            family='',
            extra_hash=hashlib.md5(extra).hexdigest(),
            extra=extra)
        stor.model.name = '{} {}MiB'.format(stor.label, stor.size)
        stor.model.save(priority=priority)
        stor.save(priority=priority)
Пример #20
0
def handle_lshw_storage(dev, lshw, is_virtual=False, priority=0):
    storages = []
    for storage in jpath.get_all("..disk", lshw):
        if not storage:
            continue
        if isinstance(storage, list):
            storages.extend(storage)
        else:
            storages.append(storage)
    storages.sort(key=get_logical_name)
    mount_points = set()
    for stor in storages:
        if "logicalname" not in stor:
            continue
        ln = stor["logicalname"]
        if isinstance(ln, list):
            mount_points.update(ln)
        else:
            mount_points.add(ln)
    dev.storage_set.filter(mount_point__in=mount_points).delete()
    for storage in storages:
        if "size" in storage:
            size = storage["size"]
        elif "capacity" in storage:
            size = storage["capacity"]
        else:
            # empty slot
            continue
        sn = unicode(storage.get("serial") or "") or None
        if (
            not sn
            or sn.startswith("QM000")
            or storage.get("vendor", "").strip().lower() in DISK_VENDOR_BLACKLIST
            or storage.get("product", "").strip().lower() in DISK_PRODUCT_BLACKLIST
        ):
            continue
        if sn:
            stor, created = Storage.concurrent_get_or_create(sn=sn, device=dev)
            stor.mount_point = storage.get("logicalname", None)
        else:
            stor, created = Storage.concurrent_get_or_create(
                sn=None, device=dev, mount_point=storage.get("logicalname", None)
            )
        stor.size = int(size["value"])
        stor.size /= units.size_divisor[size["units"]]
        stor.size = int(stor.size)
        stor.speed = 0
        if storage.get("vendor", "").strip():
            stor.label = storage["vendor"].strip() + " "
        else:
            stor.label = ""
        if storage.get("product", "").strip():
            stor.label += storage["product"].strip()
        elif storage.get("description", "").strip():
            stor.label += storage["description"].strip()
        else:
            stor.label += "Generic disk"
        caps = storage["capabilities"]
        extra = "\n".join([": ".join((unicode(key), unicode(caps[key]) or "")) for key in sorted(caps.keys())])
        stor.model, c = ComponentModel.concurrent_get_or_create(
            size=stor.size,
            speed=stor.speed,
            type=ComponentType.disk.id,
            family="",
            extra_hash=hashlib.md5(extra).hexdigest(),
            extra=extra,
        )
        stor.model.name = "{} {}MiB".format(stor.label, stor.size)
        stor.model.save(priority=priority)
        stor.save(priority=priority)
Пример #21
0
    exit()

logw('Сreating an additional field in the host-matrix based on data from Vulners')
# формируем доп-поля в матрице на основе данных от вулнерса
current_host = 0
logw('Processed hosts')
for h in h_matrix:
    current_host += 1
    try:
        # список словарей из bulletitID + SCORE, для этого хоста
        h_bulletins = list()
        # список словарей из PKG + его SCORE + его bulletitID, для этого хоста
        h_packages_tmp = list()

        # из форматированного через jpath-джейсона получаем пакет, его бюллетень и балл
        for row in jpath.get_all(jpath=jpath_mask, data=h['vuln_data']):
            pkg = row['package']
            bull = row['bulletinID']
            score = row['cvss']['score']
            # добавляем double-словарь (bull,score) во временный список (для будущего шаманства с бюллетенями)
            h_bulletins.append({'name': bull, 'score': score})
            # добавляем triple-словарь (pkg,score,bull) во второй временный список (для будущего шаманства с пакетами)
            h_packages_tmp.append({'name': pkg, 'score': score, 'bull': bull})

        # убираем дубли одинаковые пакеты, но разные баллы; оставляя только самый высокий бал
        h_packages = list()
        # для каждого пакета в списке "пакет-балл"
        for r in h_packages_tmp:
            pkg = r['name']
            # оставляем во временном списке словарей (куцом) только сторки с пакетами = пакету текущей строки
            h_pkg_tmp = [i for i in h_packages_tmp if i['name'] == pkg]
Пример #22
0
def handle_lshw_storage(dev, lshw, is_virtual=False, priority=0):
    storages = []
    for storage in jpath.get_all('..disk', lshw):
        if not storage:
            continue
        if isinstance(storage, list):
            storages.extend(storage)
        else:
            storages.append(storage)
    storages.sort(key=get_logical_name)
    mount_points = set()
    for stor in storages:
        if 'logicalname' not in stor:
            continue
        ln = stor['logicalname']
        if isinstance(ln, list):
            mount_points.update(ln)
        else:
            mount_points.add(ln)
    dev.storage_set.filter(mount_point__in=mount_points).delete()
    for storage in storages:
        if 'size' in storage:
            size = storage['size']
        elif 'capacity' in storage:
            size = storage['capacity']
        else:
            # empty slot
            continue
        sn = unicode(storage.get('serial') or '') or None
        if not sn or sn.startswith('QM000') or \
                storage.get('vendor', '').strip().lower() in DISK_VENDOR_BLACKLIST or \
                storage.get('product', '').strip().lower() in DISK_PRODUCT_BLACKLIST:
            continue
        if sn:
            stor, created = Storage.concurrent_get_or_create(sn=sn, device=dev)
            stor.mount_point = storage.get('logicalname', None)
        else:
            stor, created = Storage.concurrent_get_or_create(sn=None, device=dev,
                mount_point=storage.get('logicalname', None))
        stor.size = int(size['value'])
        stor.size /= units.size_divisor[size['units']]
        stor.size = int(stor.size)
        stor.speed = 0
        if storage.get('vendor', '').strip():
            stor.label = storage['vendor'].strip() + ' '
        else:
            stor.label = ''
        if storage.get('product', '').strip():
            stor.label += storage['product'].strip()
        elif storage.get('description', '').strip():
            stor.label += storage['description'].strip()
        else:
            stor.label += 'Generic disk'
        caps = storage['capabilities']
        extra = "\n".join([": ".join((unicode(key), unicode(caps[key]) or '')) for key in
            sorted(caps.keys())])
        stor.model, c = ComponentModel.concurrent_get_or_create(
            size=stor.size, speed=stor.speed, type=ComponentType.disk.id,
            family='', extra_hash=hashlib.md5(extra).hexdigest(), extra=extra)
        stor.model.name =  '{} {}MiB'.format(stor.label, stor.size)
        stor.model.save(priority=priority)
        stor.save(priority=priority)