Beispiel #1
0
    def put(self, tablename, key, value, secondary_indexes=None):
        """
        Insert a value in a table.
        :param tablename: a table name
        :param key: a string key
        :param value: a value
        :param secondary_indexes: (facultative) a list of secondary index
        :return: a dict that represents the value that has been inserted
        """
        if secondary_indexes is None:
            secondary_indexes = []

        # Increase version numbers of the table and the object
        self._incr_version_number(tablename)
        self._incr_object_version_number(tablename, key)
        # Add the version number
        object_version_number = self.get_object_version_number(tablename, key)
        value["___version_number"] = object_version_number
        # Dump python object to JSON field.
        json_value = ujson_dumps(value)
        fetched = self.redis_client.hset(tablename,
                                         "%s:id:%s" % (tablename, key),
                                         json_value)
        for secondary_index in secondary_indexes:
            secondary_value = value[secondary_index]
            sec_index_key = "sec_index:%s:%s:%s" % (tablename, secondary_index, secondary_value)
            sec_index_value = "%s:id:%s" % (tablename, key)
            fetched = self.redis_client.sadd(sec_index_key, sec_index_value)
        result = value if fetched else None
        result = convert_unicode_dict_to_utf8(result)
        return result
async def work_with_queue_result(queue_out: asyncio.Queue, filename,
                                 mode_write) -> None:
    """

    :param queue_out:
    :param filename:
    :param mode_write:
    :return:
    """
    if mode_write == 'a':
        method_write_result = write_to_file
    else:
        method_write_result = write_to_stdout
    async with aiofiles_open(filename, mode=mode_write) as file_with_results:
        while True:
            line = await queue_out.get()
            if line == b"check for end":
                break
            if line:
                await method_write_result(file_with_results, line)
    await asyncio.sleep(0.5)
    # region dev
    if args.statistics:
        stop_time = datetime.datetime.now()
        _delta_time = stop_time - start_time
        duration_time_sec = _delta_time.total_seconds()
        statistics = {
            'duration': duration_time_sec,
            'valid targets': count_input,
            'success': count_good,
            'fails': count_error
        }
        async with aiofiles_open('/dev/stdout', mode='wb') as stats:
            await stats.write(ujson_dumps(statistics).encode('utf-8') + b'\n')
Beispiel #3
0
def test_loads_citm_catalog_ujson(benchmark):
    benchmark.group = "citm_catalog.json deserialization"
    benchmark.extra_info["lib"] = "ujson"
    data = read_fixture_str("citm_catalog.json.xz")
    benchmark.extra_info["correct"] = json_loads(ujson_dumps(
        ujson_loads(data))) == json_loads(data)
    benchmark(ujson_loads, data)
Beispiel #4
0
def test_ujson(everything: Everything):
    converter = ujson_make_converter()
    raw = ujson_dumps(converter.unstructure(everything))
    assert (converter.structure(
        ujson_loads(raw),
        Everything,
    ) == everything)
Beispiel #5
0
def test_ujson(everything: Everything):
    from ujson import dumps as ujson_dumps
    from ujson import loads as ujson_loads

    converter = ujson_make_converter()
    raw = ujson_dumps(converter.unstructure(everything))
    assert converter.structure(ujson_loads(raw), Everything) == everything
Beispiel #6
0
def test_loads_citm_catalog_ujson(benchmark):
    benchmark.group = 'citm_catalog.json deserialization'
    benchmark.extra_info['lib'] = 'ujson'
    data = read_fixture_str("citm_catalog.json.xz")
    benchmark.extra_info['correct'] = json_loads(ujson_dumps(
        ujson_loads(data))) == json_loads(data)
    benchmark(ujson_loads, data)
Beispiel #7
0
    async def run(self):
        while True:
            line = await self.in_queue.get()
            if line == STOP_SIGNAL:
                break
            if line:
                await self.async_writer(self.io, line)

        await asyncio.sleep(0.5)
        if self.stats:
            statistics = self.stats.dict()
            if self.output_file == '/dev/stdout':
                await self.io.write(
                    ujson_dumps(statistics).encode('utf-8') + b'\n')
            else:
                async with aiofiles_open('/dev/stdout', mode='wb') as stats:
                    await stats.write(
                        ujson_dumps(statistics).encode('utf-8') + b'\n')
Beispiel #8
0
    async def send_result(self, result: Optional[Dict]):
        if result:
            try:
                success = access_dot_path(result, 'data.tcp.status')
            except:
                success = 'unknown-error'
            try:
                content_length = int(
                    access_dot_path(result,
                                    'data.tcp.result.response.content_length'))
            except:
                content_length = 0

            if self.stats:
                if success == 'success':
                    self.stats.count_good += 1
                else:
                    self.stats.count_error += 1

            line = None
            line_out = None
            try:
                if self.success_only:
                    if success == 'success':
                        line = result
                else:
                    line = result
            except Exception:
                pass

            if line:
                if self.body_not_empty:
                    if content_length > 0:
                        line_out = ujson_dumps(line)
                else:
                    line_out = ujson_dumps(line)

            if line_out:
                await self.output_queue.put(line_out)
Beispiel #9
0
 def put(self, tablename, key, value, secondary_indexes=None):
     """
     Insert a value in a table.
     :param tablename: a table name
     :param key: a string key
     :param value: a value
     :param secondary_indexes: (facultative) a list of secondary index
     :return: a dict that represents the value that has been inserted
     """
     if secondary_indexes is None:
         secondary_indexes = []
     json_value = ujson_dumps(value)
     etcd_key = "%s/%s" % (tablename, key)
     etcd_sec_idx_key = "%s_%s" % (tablename, key)
     fetched = self.etcd_client.write("/%s" % (etcd_key), json_value)
     for secondary_index in secondary_indexes:
         secondary_value = value[secondary_index]
         fetched = self.etcd_client.write("sec_index/%s/%s/%s/%s" % (
             tablename, secondary_index, secondary_value, etcd_sec_idx_key
         ), etcd_sec_idx_key)
     result = value if fetched else None
     result = convert_unicode_dict_to_utf8(result)
     return result
Beispiel #10
0
import os
import sys

# Use ujson if available
try:
    from ujson import dumps as ujson_dumps

    try:
        assert (ujson_dumps(
            'http://example.com/',
            escape_forward_slashes=False) == '"http://example.com/"')
    except Exception as e:  # pragma: no cover
        sys.stderr.write('ujson w/o forward-slash escaping not available,\
defaulting to regular json\n')
        raise

    def json_encode(obj):
        return ujson_dumps(obj, escape_forward_slashes=False)

except:  # pragma: no cover
    from json import dumps as json_encode

try:  # pragma: no cover
    from collections import OrderedDict
except ImportError:  # pragma: no cover
    from ordereddict import OrderedDict

from argparse import ArgumentParser, RawTextHelpFormatter
from bisect import insort

from six import StringIO
Beispiel #11
0
 def json_encode(obj):
     return ujson_dumps(obj, escape_forward_slashes=False)
Beispiel #12
0
 def json_encode(obj):
     return ujson_dumps(obj, escape_forward_slashes=False)
Beispiel #13
0
 def render(self, content):
     return ujson_dumps(content).encode('utf-8')
async def worker_single_run(target: NamedTuple, semaphore: asyncio.Semaphore,
                            queue_out: asyncio.Queue) -> None:
    """
    сопрограмма, осуществляет подключение к Target,
    отправку и прием данных, формирует результата в виде dict
    :param target:
    :param semaphore:
    :return:
    """
    global count_good
    global count_error
    async with semaphore:
        result = None
        status_data = False
        key = None
        try:
            future_connection = asyncssh.connect(host=target.ip,
                                                 port=target.port,
                                                 username=target.username,
                                                 password=target.password,
                                                 known_hosts=None)
            conn = await asyncio.wait_for(future_connection,
                                          timeout=target.timeout_connection)
            async with conn:
                try:
                    _key = conn.get_server_host_key()
                    try:
                        server_version = conn.get_extra_info('server_version')
                    except:
                        server_version = ''

                    h_alg = _key.algorithm
                    h_alg = h_alg.decode('utf-8')
                    if h_alg == "ssh-rsa":
                        _key_hex = md5(_key.public_data).hexdigest()
                        key = {h_alg: {'md5': _key_hex}}
                    else:
                        _key_hex = sha256(_key.public_data).hexdigest()
                        key = {h_alg: {'sha256': _key_hex}}
                except:
                    pass
                try:
                    _result = await conn.run(target.command,
                                             check=True,
                                             timeout=target.timeout_read)
                    result_data_str = _result.stdout
                    status_data = True  # trivial check that's all Ok? need rethink
                except Exception as e:
                    result = create_template_error(target, str(e))
                    await asyncio.sleep(0.005)
        except Exception as e:
            result = create_template_error(target, str(e))
            await asyncio.sleep(0.005)
        if status_data:
            try:
                result = result_data_str.encode('utf-8')
            except:
                result = b'all good, but not command'
            add_info = None
            if key:
                add_info = {'fingerprint': []}
                add_info['fingerprint'].append(key)
                if server_version:
                    add_info['version'] = server_version
            result = make_document_from_response(result, target, add_info)
        if result:
            success = return_value_from_dict(result, "data.ssh.status")
            if success == "success":
                count_good += 1
            else:
                count_error += 1
            line = None
            try:
                if args.show_only_success:
                    if success == "success":
                        line = ujson_dumps(result)
                else:
                    line = ujson_dumps(result)
            except Exception as e:
                pass
            if line:
                await queue_out.put(line)
Beispiel #15
0
import os
import sys

# Use ujson if available
try:
    from ujson import dumps as ujson_dumps

    try:
        assert (ujson_dumps('http://example.com/',
                            escape_forward_slashes=False) ==
                '"http://example.com/"')
    except Exception as e:  # pragma: no cover
        sys.stderr.write('ujson w/o forward-slash escaping not available,\
defaulting to regular json\n')
        raise

    def json_encode(obj):
        return ujson_dumps(obj, escape_forward_slashes=False)

except:  # pragma: no cover
    from json import dumps as json_encode

try:  # pragma: no cover
    from collections import OrderedDict
except ImportError:  # pragma: no cover
    from ordereddict import OrderedDict


from argparse import ArgumentParser, RawTextHelpFormatter
from bisect import insort
Beispiel #16
0
def dumps(obj):
    """ Dumps json and formats datetime to isotime """
    return ujson_dumps(cvt_nested_datetime_isoformat(obj))
async def worker_single_fingerprint(target: NamedTuple,
                                    semaphore: asyncio.Semaphore,
                                    queue_out: asyncio.Queue) -> None:
    global count_good
    global count_error
    async with semaphore:
        result = None
        _results = []
        for algorithm in target.algorithms:
            status = False
            key = None
            try:
                if algorithm != 'host':
                    future_connection = _get_server_host_key(
                        host=target.ip,
                        port=target.port,
                        server_host_key_algs=algorithm,
                        client_version='AsyncSSHChecker')
                    status = True

                elif algorithm == 'host':
                    future_connection = _get_server_host_key(
                        host=target.ip,
                        port=target.port,
                        client_version='AsyncSSHChecker')
                    status = True

                if status:
                    server_version, key = await asyncio.wait_for(
                        future_connection, timeout=target.timeout_connection)

                if key:
                    function_hash = default_host_key_algorithms[algorithm]
                    function_hash_name = default_host_key_algorithms[
                        algorithm].__name__
                    function_hash_name = function_hash_name.replace(
                        'openssl_', '')
                    _key_hex = function_hash(key.public_data).hexdigest()

                    # key_md5 = ':'.join(_key_md5[i:i + 2] for i in range(0, len(_key_md5), 2))
                    if algorithm == 'host':
                        _current_algorithm = key.algorithm
                        current_algorithm = _current_algorithm.decode('utf-8')
                    else:
                        current_algorithm = algorithm

                    _results.append(
                        {current_algorithm: {
                            function_hash_name: _key_hex
                        }})
            except:
                pass
        if _results:
            result = b''
            add_info = {'fingerprint': _results}
            if server_version:
                add_info['version'] = server_version
            result = make_document_from_response(result, target, add_info)
        else:
            result = create_template_error(target, 'no results')
            await asyncio.sleep(0.005)
        if result:
            success = return_value_from_dict(result, "data.ssh.status")
            if success == "success":
                count_good += 1
            else:
                count_error += 1
            line = None
            try:
                if args.show_only_success:
                    if success == "success":
                        line = ujson_dumps(result)
                else:
                    line = ujson_dumps(result)
            except Exception as e:
                pass
            if line:
                await queue_out.put(line)
Beispiel #18
0
    async def do(self, target: Target):
        async with self.semaphore:
            result = None
            future_connection = asyncio.open_connection(target.ip, target.port)
            try:
                reader, writer = await asyncio.wait_for(
                    future_connection, timeout=target.conn_timeout)
            except Exception as e:
                await asyncio.sleep(0.005)
                try:
                    future_connection.close()
                    del future_connection
                except Exception as e:
                    pass
                result = create_error_template(target, str(e))
            else:
                try:
                    for i in range(15, 43):
                        TLS_ClientHello[i] = (TLS_ClientHello[i]
                                              ^ randint(0, 4294967294)) & 0xFF
                    writer.write(bytes(TLS_ClientHello))
                    await writer.drain()
                    await asyncio.sleep(0.05)
                    desc_text = '1 step: read 65535 bytes'
                    status_data, answer = await single_read(
                        reader,
                        target,
                        custom_max_size=65535,
                        operation_description=desc_text)
                    if status_data:
                        if answer[0] != 0x16:
                            message_error_text = "Not a C2. TLS ContentType state"
                            raise AssertionError(message_error_text)
                        if answer[5] != 0x02:
                            message_error_text = f"Not a C2. Expected ServerHello (02) got ({answer[5]:02X})"
                            raise AssertionError(message_error_text)

                        for i in range(15, 267):
                            TLS_HelloRequest[i] = (
                                TLS_HelloRequest[i]
                                ^ randint(0, 4294967294)) & 0xFF

                        packed_size = pack(">H", FULL_SIZE)
                        for i, j in enumerate(range(276, 278)):
                            TLS_HelloRequest[j] = packed_size[i]

                        for i, j in enumerate(range(278, 278 + len(SECRET))):
                            TLS_HelloRequest[j] = SECRET[i]

                        for i in range(278 + len(SECRET), 278 + FULL_SIZE):
                            TLS_HelloRequest[i] = (
                                TLS_HelloRequest[i]
                                ^ randint(0, 4294967294)) & 0xFF

                        writer.write(bytes(TLS_HelloRequest))
                        await writer.drain()
                        await asyncio.sleep(0.05)
                        desc_text = f'2 step: read 5 bytes, timeout={target.read_timeout}'
                        status_data, answer = await single_read(
                            reader,
                            target,
                            custom_max_size=5,
                            operation_description=desc_text)
                        if status_data:
                            if answer[0] != 0x14:
                                message_error_text = f"Not a C2. Expected ChangeCipherSpec (0x14) got ({answer[0]:02X})"
                                raise AssertionError(message_error_text)
                            length = unpack(">H", bytes(answer[3:]))[0]
                            if length > 0x3ff9:
                                message_error_text = f"Not a C2. ChangeCipherSpec too big"
                                raise AssertionError(message_error_text)
                            await asyncio.sleep(0.05)
                            desc_text = f'3 step: read {length}+5 bytes, timeout={target.read_timeout}'
                            status_data, _answer = await single_read(
                                reader,
                                target,
                                custom_max_size=length + 5,
                                operation_description=desc_text)
                            await asyncio.sleep(0.05)
                            if status_data:
                                answer = _answer[-5:]
                                if answer[0] != 0x16:
                                    message_error_text = f"Not a C2. Expected Handshake (0x16) got ({answer[0]:02X})"
                                    raise AssertionError(message_error_text)

                                length = unpack(">H", bytes(answer[3:]))[0]
                                if length > 0x3ff9:
                                    message_error_text = f"Not a C2. Handshake too big"
                                    raise AssertionError(message_error_text)
                                desc_text = f'4 step: read {length} bytes, timeout={target.read_timeout}'
                                status_data, answer = await single_read(
                                    reader,
                                    target,
                                    custom_max_size=length,
                                    operation_description=desc_text)
                                if status_data:
                                    server_secret = answer[:len(SECRET)]
                                    key_stream = derive_key(server_secret)
                                    need_payload = pack_payload(key_stream,
                                                                  pack("<H", 0x200) + \
                                                                  pack("<H", 0x03) + \
                                                                  b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")
                                    writer.write(need_payload)
                                    await writer.drain()
                                    await asyncio.sleep(0.05)
                                    desc_text = f'5 step: read 5 bytes, timeout={target.read_timeout}'
                                    status_data, answer = await single_read(
                                        reader,
                                        target,
                                        custom_max_size=5,
                                        operation_description=desc_text)
                                    length = unpack(">H", bytes(answer[3:]))[0]
                                    desc_text = f'6 step: read {length} bytes, timeout={target.read_timeout}'
                                    status_data, answer = await single_read(
                                        reader,
                                        target,
                                        custom_max_size=length,
                                        operation_description=desc_text)
                                    c2_answer = key_stream.decrypt(answer)
                                    command_id = unpack("<H", c2_answer[:2])[0]
                                    module_id = unpack("<H", c2_answer[2:4])[0]
                                    if command_id not in COMMANDS:
                                        message_error_text = f"Not a C2. Invalid response command id"
                                        raise AssertionError(
                                            message_error_text)
                                    if module_id != 3:
                                        message_error_text = f"Not a C2. Invalid response module id"
                                        raise AssertionError(
                                            message_error_text)
                                    result = make_document_from_response(
                                        b'C2 found', target)
                                    await asyncio.sleep(0.005)
                                else:
                                    result = answer
                            else:
                                result = _answer
                        else:
                            result = answer
                    else:
                        result = answer  # get errors
                    try:
                        writer.close()
                    except BaseException:
                        pass
                except AssertionError as e:
                    result = create_error_template(target, str(e))
                    try:
                        future_connection.close()
                    except Exception:
                        pass
                    await asyncio.sleep(0.005)
                    try:
                        writer.close()
                    except Exception:
                        pass
                except Exception as e:
                    result = create_error_template(target, str(e))
                    try:
                        future_connection.close()
                    except Exception:
                        pass
                    await asyncio.sleep(0.005)
                    try:
                        writer.close()
                    except Exception:
                        pass

            if result:
                try:
                    success = access_dot_path(result, 'data.tcp.status')
                except:
                    success = 'unknown-error'
                try:
                    content_length = int(
                        access_dot_path(
                            result, 'data.tcp.result.response.content_length'))
                except:
                    content_length = 0

                if self.stats:
                    if success == 'success':
                        self.stats.count_good += 1
                    else:
                        self.stats.count_error += 1

                line = None
                line_out = None
                try:
                    if self.success_only:
                        if success == 'success':
                            line = result
                    else:
                        line = result
                except Exception:
                    pass

                if line:
                    if self.body_not_empty:
                        if content_length > 0:
                            line_out = ujson_dumps(line)
                    else:
                        line_out = ujson_dumps(line)

                if line_out:
                    await self.output_queue.put(line_out)
Beispiel #19
0
    async def do(self, target: Target):
        """
        сопрограмма, осуществляет подключение к Target, отправку и прием данных, формирует результата в виде dict
        """
        def return_ip_from_deep(sess, response) -> str:
            try:
                ip_port = response.connection.transport.get_extra_info(
                    'peername')
                if is_ip(ip_port[0]):
                    return ip_port[0]
            except BaseException:
                pass
            try:
                _tmp_conn_key = sess.connector._conns.items()
                for k, v in _tmp_conn_key:
                    _h = v[0][0]
                    ip_port = _h.transport.get_extra_info('peername')
                    if is_ip(ip_port[0]):
                        return ip_port[0]
            except BaseException:
                pass
            return ''

        def update_line(json_record, target):
            json_record['ip'] = target.ip
            # json_record['ip_v4_int'] = int(ip_address(target.ip))
            # json_record['datetime'] = datetime.datetime.utcnow()
            # json_record['port'] = int(target.port)
            return json_record

        async with self.semaphore:
            result = None
            timeout = ClientTimeout(total=target.total_timeout)

            # region tmp disable
            trace_config = TraceConfig()
            trace_config.on_request_start.append(on_request_start)
            trace_config.on_request_end.append(on_request_end)
            # endregion
            resolver = AsyncResolver(nameservers=['8.8.8.8', '8.8.4.4'])
            # resolver = None
            # https://github.com/aio-libs/aiohttp/issues/2228  - closed
            if target.ssl_check:

                conn = TCPConnector(
                    ssl=False,
                    family=2,  # need set current family (only IPv4)
                    limit_per_host=0,
                    resolver=resolver)
                session = ClientSession(timeout=timeout,
                                        connector=conn,
                                        response_class=WrappedResponseClass,
                                        trace_configs=[trace_config])
                simple_zero_sleep = 0.250
            else:
                simple_zero_sleep = 0.001
                session = ClientSession(
                    connector=TCPConnector(
                        limit_per_host=0,
                        family=2,  # need set current family (only IPv4)
                        resolver=resolver),
                    timeout=timeout,
                    trace_configs=[trace_config])
            selected_proxy_connection = None
            try:
                selected_proxy_connection = next(
                    self.app_config.proxy_connections)
            except:
                pass
            try:
                async with session.request(
                        target.method,
                        target.url,
                        timeout=timeout,
                        headers=target.headers,
                        cookies=target.cookies,
                        allow_redirects=target.allow_redirects,
                        data=target.payload,
                        proxy=selected_proxy_connection,
                        trace_request_ctx=self.trace_request_ctx) as response:
                    _default_record = create_template_struct(target)
                    if target.ssl_check:
                        cert = convert_bytes_to_cert(response.peer_cert)
                        if not self.app_config.without_certraw:
                            _default_record['data']['http']['result'][
                                'response']['request']['tls_log'][
                                    'handshake_log']['server_certificates'][
                                        'certificate']['raw'] = b64encode(
                                            response.peer_cert).decode('utf-8')
                        if cert:
                            _default_record['data']['http']['result'][
                                'response']['request']['tls_log'][
                                    'handshake_log']['server_certificates'][
                                        'certificate']['parsed'] = cert
                    _default_record['data']['http']['status'] = "success"
                    _default_record['data']['http']['result']['response'][
                        'status_code'] = response.status
                    # region
                    _header = {}
                    for key in response.headers:
                        _header[key.lower().replace(
                            '-', '_')] = response.headers.getall(key)
                    _default_record['data']['http']['result']['response'][
                        'headers'] = _header
                    # endregion
                    if target.method in [
                            'GET', 'POST', 'PUT', 'DELETE', 'UPDATE'
                    ]:
                        buffer = b""
                        try:
                            read_c = asyncio.wait_for(
                                read_http_content(response, n=target.max_size),
                                timeout=target.total_timeout)
                            buffer = await read_c
                        except Exception as e:
                            pass
                        else:
                            if filter_bytes(buffer, target):
                                _default_record['data']['http']['result'][
                                    'response']['content_length'] = len(buffer)
                                _default_record['data']['http']['result'][
                                    'response']['body'] = ''
                                try:
                                    _default_record['data']['http']['result'][
                                        'response']['body'] = buffer.decode()
                                except Exception as e:
                                    pass
                                if not self.app_config.without_base64:
                                    try:
                                        _base64_data = b64encode(
                                            buffer).decode('utf-8')
                                        _default_record['data']['http'][
                                            'result']['response'][
                                                'body_raw'] = _base64_data
                                    except Exception as e:
                                        pass
                                if not self.app_config.without_hashs:
                                    try:
                                        hashs = {
                                            'sha256': sha256,
                                            'sha1': sha1,
                                            'md5': md5
                                        }
                                        for namehash, func in hashs.items():
                                            hm = func()
                                            hm.update(buffer)
                                            _default_record['data']['http'][
                                                'result']['response'][
                                                    f'body_{namehash}'] = hm.hexdigest(
                                                    )
                                    except Exception as e:
                                        pass
                                result = update_line(_default_record, target)
                            else:
                                # TODO: добавить статус success-not-contain для обозначения того,
                                #  что сервис найден, но не попал под фильтр?
                                result = create_error_template(
                                    target,
                                    error_str='',
                                    status_string='success-not-contain')
                    if result:
                        if not result['ip']:
                            result['ip'] = return_ip_from_deep(
                                session, response)
            except Exception as exp:
                error_str = ''
                try:
                    error_str = exp.strerror
                except:
                    pass
                result = create_error_template(target, error_str,
                                               type(exp).__name__)
                await asyncio.sleep(simple_zero_sleep)
                try:
                    await session.close()
                except:
                    pass
                try:
                    await conn.close()
                except:
                    pass
            if result:
                if 'duration' in self.trace_request_ctx:
                    request_duration = self.trace_request_ctx['duration']
                    result['data']['http']['duration'] = request_duration
                success = access_dot_path(result, "data.http.status")
                if self.stats:
                    if success == "success":
                        self.stats.count_good += 1
                    else:
                        self.stats.count_error += 1
                if not (self.app_config.status_code == CONST_ANY_STATUS):
                    response_status = access_dot_path(
                        result, 'data.http.result.response.status_code')
                    if response_status:
                        if self.app_config.status_code != response_status:
                            error_str = f'status code: {response_status} is not equal to filter: {self.app_config.status_code}'
                            result = create_error_template(
                                target,
                                error_str=error_str,
                                status_string='success-not-need-status')
                            self.stats.count_good -= 1
                            self.stats.count_error += 1
                line = None
                try:
                    if self.success_only:
                        if success == "success":
                            line = ujson_dumps(result)
                    else:
                        line = ujson_dumps(result)
                except Exception:
                    pass
                if line:
                    await self.output_queue.put(line)

            await asyncio.sleep(simple_zero_sleep)
            try:
                await session.close()
            except:
                pass
            try:
                await conn.close()
            except:
                pass
Beispiel #20
0
    async def do(self, target: Target):
        """
        сопрограмма, осуществляет подключение к Target, отправку и прием данных, формирует результата в виде dict
        """
        async with self.semaphore:
            result = None
            certificate_dict = None
            cert_bytes_base64 = None

            if target.ssl_check:  # если при запуске в настройках указано --use-ssl - то контекст ssl
                ssl_context = ssl_create_unverified_context()
                future_connection = asyncio.open_connection(
                    target.ip,
                    target.port,
                    ssl=ssl_context,
                    ssl_handshake_timeout=target.ssl_timeout)
            else:
                future_connection = asyncio.open_connection(
                    target.ip, target.port)
            try:
                reader, writer = await asyncio.wait_for(
                    future_connection, timeout=target.conn_timeout)
                if target.ssl_check:
                    try:
                        # noinspection PyProtectedMember
                        _sub_ssl = writer._transport.get_extra_info(
                            'ssl_object')
                        cert_bytes = _sub_ssl.getpeercert(binary_form=True)
                        cert_bytes_base64 = b64encode(cert_bytes).decode(
                            'utf-8')
                        certificate_dict = convert_bytes_to_cert(cert_bytes)
                    except BaseException:
                        pass
            except Exception as e:
                await asyncio.sleep(0.005)
                try:
                    future_connection.close()
                    del future_connection
                except Exception as e:
                    pass
                result = create_error_template(target, str(e))
            else:
                try:
                    status_data = False
                    if target.payload:  # если указан payload - то он и отправляется в первую очередь
                        writer.write(target.payload)
                        await writer.drain()
                    if target.mode == 'single':
                        status_data, data_or_error_result = await single_read(
                            reader, target)
                    elif target.mode == 'multi':
                        status_data, data_or_error_result = await asyncio.wait_for(
                            multi_read(reader, target),
                            timeout=target.read_timeout)
                    if status_data:
                        check_filter = filter_bytes(data_or_error_result,
                                                    target)
                        if check_filter:
                            result = make_document_from_response(
                                data_or_error_result, target)
                            if target.ssl_check:
                                if cert_bytes_base64:
                                    result['data']['tcp']['result']['response'][
                                        'request']['tls_log']['handshake_log'][
                                            'server_certificates'][
                                                'certificate'][
                                                    'raw'] = cert_bytes_base64
                                if certificate_dict:
                                    result['data']['tcp']['result']['response'][
                                        'request']['tls_log']['handshake_log'][
                                            'server_certificates'][
                                                'certificate'][
                                                    'parsed'] = certificate_dict

                        else:
                            # TODO: добавить статус success-not-contain для обозначения того,
                            #  что сервис найден, но не попал под фильтр
                            pass
                        await asyncio.sleep(0.005)
                    else:
                        result = data_or_error_result  # get errors
                    try:
                        writer.close()
                    except BaseException:
                        pass
                except Exception as e:
                    result = create_error_template(target, str(e))
                    try:
                        future_connection.close()
                    except Exception:
                        pass
                    await asyncio.sleep(0.005)
                    try:
                        writer.close()
                    except Exception:
                        pass
            if result:
                success = access_dot_path(result, "data.tcp.status")
                if self.stats:
                    if success == "success":
                        self.stats.count_good += 1
                    else:
                        self.stats.count_error += 1
                line = None
                try:
                    if self.success_only:
                        if success == "success":
                            line = ujson_dumps(result)
                    else:
                        line = ujson_dumps(result)
                except Exception:
                    pass
                if line:
                    await self.output_queue.put(line)