def put(self, tablename, key, value, secondary_indexes=None): """ Insert a value in a table. :param tablename: a table name :param key: a string key :param value: a value :param secondary_indexes: (facultative) a list of secondary index :return: a dict that represents the value that has been inserted """ if secondary_indexes is None: secondary_indexes = [] # Increase version numbers of the table and the object self._incr_version_number(tablename) self._incr_object_version_number(tablename, key) # Add the version number object_version_number = self.get_object_version_number(tablename, key) value["___version_number"] = object_version_number # Dump python object to JSON field. json_value = ujson_dumps(value) fetched = self.redis_client.hset(tablename, "%s:id:%s" % (tablename, key), json_value) for secondary_index in secondary_indexes: secondary_value = value[secondary_index] sec_index_key = "sec_index:%s:%s:%s" % (tablename, secondary_index, secondary_value) sec_index_value = "%s:id:%s" % (tablename, key) fetched = self.redis_client.sadd(sec_index_key, sec_index_value) result = value if fetched else None result = convert_unicode_dict_to_utf8(result) return result
async def work_with_queue_result(queue_out: asyncio.Queue, filename, mode_write) -> None: """ :param queue_out: :param filename: :param mode_write: :return: """ if mode_write == 'a': method_write_result = write_to_file else: method_write_result = write_to_stdout async with aiofiles_open(filename, mode=mode_write) as file_with_results: while True: line = await queue_out.get() if line == b"check for end": break if line: await method_write_result(file_with_results, line) await asyncio.sleep(0.5) # region dev if args.statistics: stop_time = datetime.datetime.now() _delta_time = stop_time - start_time duration_time_sec = _delta_time.total_seconds() statistics = { 'duration': duration_time_sec, 'valid targets': count_input, 'success': count_good, 'fails': count_error } async with aiofiles_open('/dev/stdout', mode='wb') as stats: await stats.write(ujson_dumps(statistics).encode('utf-8') + b'\n')
def test_loads_citm_catalog_ujson(benchmark): benchmark.group = "citm_catalog.json deserialization" benchmark.extra_info["lib"] = "ujson" data = read_fixture_str("citm_catalog.json.xz") benchmark.extra_info["correct"] = json_loads(ujson_dumps( ujson_loads(data))) == json_loads(data) benchmark(ujson_loads, data)
def test_ujson(everything: Everything): converter = ujson_make_converter() raw = ujson_dumps(converter.unstructure(everything)) assert (converter.structure( ujson_loads(raw), Everything, ) == everything)
def test_ujson(everything: Everything): from ujson import dumps as ujson_dumps from ujson import loads as ujson_loads converter = ujson_make_converter() raw = ujson_dumps(converter.unstructure(everything)) assert converter.structure(ujson_loads(raw), Everything) == everything
def test_loads_citm_catalog_ujson(benchmark): benchmark.group = 'citm_catalog.json deserialization' benchmark.extra_info['lib'] = 'ujson' data = read_fixture_str("citm_catalog.json.xz") benchmark.extra_info['correct'] = json_loads(ujson_dumps( ujson_loads(data))) == json_loads(data) benchmark(ujson_loads, data)
async def run(self): while True: line = await self.in_queue.get() if line == STOP_SIGNAL: break if line: await self.async_writer(self.io, line) await asyncio.sleep(0.5) if self.stats: statistics = self.stats.dict() if self.output_file == '/dev/stdout': await self.io.write( ujson_dumps(statistics).encode('utf-8') + b'\n') else: async with aiofiles_open('/dev/stdout', mode='wb') as stats: await stats.write( ujson_dumps(statistics).encode('utf-8') + b'\n')
async def send_result(self, result: Optional[Dict]): if result: try: success = access_dot_path(result, 'data.tcp.status') except: success = 'unknown-error' try: content_length = int( access_dot_path(result, 'data.tcp.result.response.content_length')) except: content_length = 0 if self.stats: if success == 'success': self.stats.count_good += 1 else: self.stats.count_error += 1 line = None line_out = None try: if self.success_only: if success == 'success': line = result else: line = result except Exception: pass if line: if self.body_not_empty: if content_length > 0: line_out = ujson_dumps(line) else: line_out = ujson_dumps(line) if line_out: await self.output_queue.put(line_out)
def put(self, tablename, key, value, secondary_indexes=None): """ Insert a value in a table. :param tablename: a table name :param key: a string key :param value: a value :param secondary_indexes: (facultative) a list of secondary index :return: a dict that represents the value that has been inserted """ if secondary_indexes is None: secondary_indexes = [] json_value = ujson_dumps(value) etcd_key = "%s/%s" % (tablename, key) etcd_sec_idx_key = "%s_%s" % (tablename, key) fetched = self.etcd_client.write("/%s" % (etcd_key), json_value) for secondary_index in secondary_indexes: secondary_value = value[secondary_index] fetched = self.etcd_client.write("sec_index/%s/%s/%s/%s" % ( tablename, secondary_index, secondary_value, etcd_sec_idx_key ), etcd_sec_idx_key) result = value if fetched else None result = convert_unicode_dict_to_utf8(result) return result
import os import sys # Use ujson if available try: from ujson import dumps as ujson_dumps try: assert (ujson_dumps( 'http://example.com/', escape_forward_slashes=False) == '"http://example.com/"') except Exception as e: # pragma: no cover sys.stderr.write('ujson w/o forward-slash escaping not available,\ defaulting to regular json\n') raise def json_encode(obj): return ujson_dumps(obj, escape_forward_slashes=False) except: # pragma: no cover from json import dumps as json_encode try: # pragma: no cover from collections import OrderedDict except ImportError: # pragma: no cover from ordereddict import OrderedDict from argparse import ArgumentParser, RawTextHelpFormatter from bisect import insort from six import StringIO
def json_encode(obj): return ujson_dumps(obj, escape_forward_slashes=False)
def render(self, content): return ujson_dumps(content).encode('utf-8')
async def worker_single_run(target: NamedTuple, semaphore: asyncio.Semaphore, queue_out: asyncio.Queue) -> None: """ сопрограмма, осуществляет подключение к Target, отправку и прием данных, формирует результата в виде dict :param target: :param semaphore: :return: """ global count_good global count_error async with semaphore: result = None status_data = False key = None try: future_connection = asyncssh.connect(host=target.ip, port=target.port, username=target.username, password=target.password, known_hosts=None) conn = await asyncio.wait_for(future_connection, timeout=target.timeout_connection) async with conn: try: _key = conn.get_server_host_key() try: server_version = conn.get_extra_info('server_version') except: server_version = '' h_alg = _key.algorithm h_alg = h_alg.decode('utf-8') if h_alg == "ssh-rsa": _key_hex = md5(_key.public_data).hexdigest() key = {h_alg: {'md5': _key_hex}} else: _key_hex = sha256(_key.public_data).hexdigest() key = {h_alg: {'sha256': _key_hex}} except: pass try: _result = await conn.run(target.command, check=True, timeout=target.timeout_read) result_data_str = _result.stdout status_data = True # trivial check that's all Ok? need rethink except Exception as e: result = create_template_error(target, str(e)) await asyncio.sleep(0.005) except Exception as e: result = create_template_error(target, str(e)) await asyncio.sleep(0.005) if status_data: try: result = result_data_str.encode('utf-8') except: result = b'all good, but not command' add_info = None if key: add_info = {'fingerprint': []} add_info['fingerprint'].append(key) if server_version: add_info['version'] = server_version result = make_document_from_response(result, target, add_info) if result: success = return_value_from_dict(result, "data.ssh.status") if success == "success": count_good += 1 else: count_error += 1 line = None try: if args.show_only_success: if success == "success": line = ujson_dumps(result) else: line = ujson_dumps(result) except Exception as e: pass if line: await queue_out.put(line)
import os import sys # Use ujson if available try: from ujson import dumps as ujson_dumps try: assert (ujson_dumps('http://example.com/', escape_forward_slashes=False) == '"http://example.com/"') except Exception as e: # pragma: no cover sys.stderr.write('ujson w/o forward-slash escaping not available,\ defaulting to regular json\n') raise def json_encode(obj): return ujson_dumps(obj, escape_forward_slashes=False) except: # pragma: no cover from json import dumps as json_encode try: # pragma: no cover from collections import OrderedDict except ImportError: # pragma: no cover from ordereddict import OrderedDict from argparse import ArgumentParser, RawTextHelpFormatter from bisect import insort
def dumps(obj): """ Dumps json and formats datetime to isotime """ return ujson_dumps(cvt_nested_datetime_isoformat(obj))
async def worker_single_fingerprint(target: NamedTuple, semaphore: asyncio.Semaphore, queue_out: asyncio.Queue) -> None: global count_good global count_error async with semaphore: result = None _results = [] for algorithm in target.algorithms: status = False key = None try: if algorithm != 'host': future_connection = _get_server_host_key( host=target.ip, port=target.port, server_host_key_algs=algorithm, client_version='AsyncSSHChecker') status = True elif algorithm == 'host': future_connection = _get_server_host_key( host=target.ip, port=target.port, client_version='AsyncSSHChecker') status = True if status: server_version, key = await asyncio.wait_for( future_connection, timeout=target.timeout_connection) if key: function_hash = default_host_key_algorithms[algorithm] function_hash_name = default_host_key_algorithms[ algorithm].__name__ function_hash_name = function_hash_name.replace( 'openssl_', '') _key_hex = function_hash(key.public_data).hexdigest() # key_md5 = ':'.join(_key_md5[i:i + 2] for i in range(0, len(_key_md5), 2)) if algorithm == 'host': _current_algorithm = key.algorithm current_algorithm = _current_algorithm.decode('utf-8') else: current_algorithm = algorithm _results.append( {current_algorithm: { function_hash_name: _key_hex }}) except: pass if _results: result = b'' add_info = {'fingerprint': _results} if server_version: add_info['version'] = server_version result = make_document_from_response(result, target, add_info) else: result = create_template_error(target, 'no results') await asyncio.sleep(0.005) if result: success = return_value_from_dict(result, "data.ssh.status") if success == "success": count_good += 1 else: count_error += 1 line = None try: if args.show_only_success: if success == "success": line = ujson_dumps(result) else: line = ujson_dumps(result) except Exception as e: pass if line: await queue_out.put(line)
async def do(self, target: Target): async with self.semaphore: result = None future_connection = asyncio.open_connection(target.ip, target.port) try: reader, writer = await asyncio.wait_for( future_connection, timeout=target.conn_timeout) except Exception as e: await asyncio.sleep(0.005) try: future_connection.close() del future_connection except Exception as e: pass result = create_error_template(target, str(e)) else: try: for i in range(15, 43): TLS_ClientHello[i] = (TLS_ClientHello[i] ^ randint(0, 4294967294)) & 0xFF writer.write(bytes(TLS_ClientHello)) await writer.drain() await asyncio.sleep(0.05) desc_text = '1 step: read 65535 bytes' status_data, answer = await single_read( reader, target, custom_max_size=65535, operation_description=desc_text) if status_data: if answer[0] != 0x16: message_error_text = "Not a C2. TLS ContentType state" raise AssertionError(message_error_text) if answer[5] != 0x02: message_error_text = f"Not a C2. Expected ServerHello (02) got ({answer[5]:02X})" raise AssertionError(message_error_text) for i in range(15, 267): TLS_HelloRequest[i] = ( TLS_HelloRequest[i] ^ randint(0, 4294967294)) & 0xFF packed_size = pack(">H", FULL_SIZE) for i, j in enumerate(range(276, 278)): TLS_HelloRequest[j] = packed_size[i] for i, j in enumerate(range(278, 278 + len(SECRET))): TLS_HelloRequest[j] = SECRET[i] for i in range(278 + len(SECRET), 278 + FULL_SIZE): TLS_HelloRequest[i] = ( TLS_HelloRequest[i] ^ randint(0, 4294967294)) & 0xFF writer.write(bytes(TLS_HelloRequest)) await writer.drain() await asyncio.sleep(0.05) desc_text = f'2 step: read 5 bytes, timeout={target.read_timeout}' status_data, answer = await single_read( reader, target, custom_max_size=5, operation_description=desc_text) if status_data: if answer[0] != 0x14: message_error_text = f"Not a C2. Expected ChangeCipherSpec (0x14) got ({answer[0]:02X})" raise AssertionError(message_error_text) length = unpack(">H", bytes(answer[3:]))[0] if length > 0x3ff9: message_error_text = f"Not a C2. ChangeCipherSpec too big" raise AssertionError(message_error_text) await asyncio.sleep(0.05) desc_text = f'3 step: read {length}+5 bytes, timeout={target.read_timeout}' status_data, _answer = await single_read( reader, target, custom_max_size=length + 5, operation_description=desc_text) await asyncio.sleep(0.05) if status_data: answer = _answer[-5:] if answer[0] != 0x16: message_error_text = f"Not a C2. Expected Handshake (0x16) got ({answer[0]:02X})" raise AssertionError(message_error_text) length = unpack(">H", bytes(answer[3:]))[0] if length > 0x3ff9: message_error_text = f"Not a C2. Handshake too big" raise AssertionError(message_error_text) desc_text = f'4 step: read {length} bytes, timeout={target.read_timeout}' status_data, answer = await single_read( reader, target, custom_max_size=length, operation_description=desc_text) if status_data: server_secret = answer[:len(SECRET)] key_stream = derive_key(server_secret) need_payload = pack_payload(key_stream, pack("<H", 0x200) + \ pack("<H", 0x03) + \ b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00") writer.write(need_payload) await writer.drain() await asyncio.sleep(0.05) desc_text = f'5 step: read 5 bytes, timeout={target.read_timeout}' status_data, answer = await single_read( reader, target, custom_max_size=5, operation_description=desc_text) length = unpack(">H", bytes(answer[3:]))[0] desc_text = f'6 step: read {length} bytes, timeout={target.read_timeout}' status_data, answer = await single_read( reader, target, custom_max_size=length, operation_description=desc_text) c2_answer = key_stream.decrypt(answer) command_id = unpack("<H", c2_answer[:2])[0] module_id = unpack("<H", c2_answer[2:4])[0] if command_id not in COMMANDS: message_error_text = f"Not a C2. Invalid response command id" raise AssertionError( message_error_text) if module_id != 3: message_error_text = f"Not a C2. Invalid response module id" raise AssertionError( message_error_text) result = make_document_from_response( b'C2 found', target) await asyncio.sleep(0.005) else: result = answer else: result = _answer else: result = answer else: result = answer # get errors try: writer.close() except BaseException: pass except AssertionError as e: result = create_error_template(target, str(e)) try: future_connection.close() except Exception: pass await asyncio.sleep(0.005) try: writer.close() except Exception: pass except Exception as e: result = create_error_template(target, str(e)) try: future_connection.close() except Exception: pass await asyncio.sleep(0.005) try: writer.close() except Exception: pass if result: try: success = access_dot_path(result, 'data.tcp.status') except: success = 'unknown-error' try: content_length = int( access_dot_path( result, 'data.tcp.result.response.content_length')) except: content_length = 0 if self.stats: if success == 'success': self.stats.count_good += 1 else: self.stats.count_error += 1 line = None line_out = None try: if self.success_only: if success == 'success': line = result else: line = result except Exception: pass if line: if self.body_not_empty: if content_length > 0: line_out = ujson_dumps(line) else: line_out = ujson_dumps(line) if line_out: await self.output_queue.put(line_out)
async def do(self, target: Target): """ сопрограмма, осуществляет подключение к Target, отправку и прием данных, формирует результата в виде dict """ def return_ip_from_deep(sess, response) -> str: try: ip_port = response.connection.transport.get_extra_info( 'peername') if is_ip(ip_port[0]): return ip_port[0] except BaseException: pass try: _tmp_conn_key = sess.connector._conns.items() for k, v in _tmp_conn_key: _h = v[0][0] ip_port = _h.transport.get_extra_info('peername') if is_ip(ip_port[0]): return ip_port[0] except BaseException: pass return '' def update_line(json_record, target): json_record['ip'] = target.ip # json_record['ip_v4_int'] = int(ip_address(target.ip)) # json_record['datetime'] = datetime.datetime.utcnow() # json_record['port'] = int(target.port) return json_record async with self.semaphore: result = None timeout = ClientTimeout(total=target.total_timeout) # region tmp disable trace_config = TraceConfig() trace_config.on_request_start.append(on_request_start) trace_config.on_request_end.append(on_request_end) # endregion resolver = AsyncResolver(nameservers=['8.8.8.8', '8.8.4.4']) # resolver = None # https://github.com/aio-libs/aiohttp/issues/2228 - closed if target.ssl_check: conn = TCPConnector( ssl=False, family=2, # need set current family (only IPv4) limit_per_host=0, resolver=resolver) session = ClientSession(timeout=timeout, connector=conn, response_class=WrappedResponseClass, trace_configs=[trace_config]) simple_zero_sleep = 0.250 else: simple_zero_sleep = 0.001 session = ClientSession( connector=TCPConnector( limit_per_host=0, family=2, # need set current family (only IPv4) resolver=resolver), timeout=timeout, trace_configs=[trace_config]) selected_proxy_connection = None try: selected_proxy_connection = next( self.app_config.proxy_connections) except: pass try: async with session.request( target.method, target.url, timeout=timeout, headers=target.headers, cookies=target.cookies, allow_redirects=target.allow_redirects, data=target.payload, proxy=selected_proxy_connection, trace_request_ctx=self.trace_request_ctx) as response: _default_record = create_template_struct(target) if target.ssl_check: cert = convert_bytes_to_cert(response.peer_cert) if not self.app_config.without_certraw: _default_record['data']['http']['result'][ 'response']['request']['tls_log'][ 'handshake_log']['server_certificates'][ 'certificate']['raw'] = b64encode( response.peer_cert).decode('utf-8') if cert: _default_record['data']['http']['result'][ 'response']['request']['tls_log'][ 'handshake_log']['server_certificates'][ 'certificate']['parsed'] = cert _default_record['data']['http']['status'] = "success" _default_record['data']['http']['result']['response'][ 'status_code'] = response.status # region _header = {} for key in response.headers: _header[key.lower().replace( '-', '_')] = response.headers.getall(key) _default_record['data']['http']['result']['response'][ 'headers'] = _header # endregion if target.method in [ 'GET', 'POST', 'PUT', 'DELETE', 'UPDATE' ]: buffer = b"" try: read_c = asyncio.wait_for( read_http_content(response, n=target.max_size), timeout=target.total_timeout) buffer = await read_c except Exception as e: pass else: if filter_bytes(buffer, target): _default_record['data']['http']['result'][ 'response']['content_length'] = len(buffer) _default_record['data']['http']['result'][ 'response']['body'] = '' try: _default_record['data']['http']['result'][ 'response']['body'] = buffer.decode() except Exception as e: pass if not self.app_config.without_base64: try: _base64_data = b64encode( buffer).decode('utf-8') _default_record['data']['http'][ 'result']['response'][ 'body_raw'] = _base64_data except Exception as e: pass if not self.app_config.without_hashs: try: hashs = { 'sha256': sha256, 'sha1': sha1, 'md5': md5 } for namehash, func in hashs.items(): hm = func() hm.update(buffer) _default_record['data']['http'][ 'result']['response'][ f'body_{namehash}'] = hm.hexdigest( ) except Exception as e: pass result = update_line(_default_record, target) else: # TODO: добавить статус success-not-contain для обозначения того, # что сервис найден, но не попал под фильтр? result = create_error_template( target, error_str='', status_string='success-not-contain') if result: if not result['ip']: result['ip'] = return_ip_from_deep( session, response) except Exception as exp: error_str = '' try: error_str = exp.strerror except: pass result = create_error_template(target, error_str, type(exp).__name__) await asyncio.sleep(simple_zero_sleep) try: await session.close() except: pass try: await conn.close() except: pass if result: if 'duration' in self.trace_request_ctx: request_duration = self.trace_request_ctx['duration'] result['data']['http']['duration'] = request_duration success = access_dot_path(result, "data.http.status") if self.stats: if success == "success": self.stats.count_good += 1 else: self.stats.count_error += 1 if not (self.app_config.status_code == CONST_ANY_STATUS): response_status = access_dot_path( result, 'data.http.result.response.status_code') if response_status: if self.app_config.status_code != response_status: error_str = f'status code: {response_status} is not equal to filter: {self.app_config.status_code}' result = create_error_template( target, error_str=error_str, status_string='success-not-need-status') self.stats.count_good -= 1 self.stats.count_error += 1 line = None try: if self.success_only: if success == "success": line = ujson_dumps(result) else: line = ujson_dumps(result) except Exception: pass if line: await self.output_queue.put(line) await asyncio.sleep(simple_zero_sleep) try: await session.close() except: pass try: await conn.close() except: pass
async def do(self, target: Target): """ сопрограмма, осуществляет подключение к Target, отправку и прием данных, формирует результата в виде dict """ async with self.semaphore: result = None certificate_dict = None cert_bytes_base64 = None if target.ssl_check: # если при запуске в настройках указано --use-ssl - то контекст ssl ssl_context = ssl_create_unverified_context() future_connection = asyncio.open_connection( target.ip, target.port, ssl=ssl_context, ssl_handshake_timeout=target.ssl_timeout) else: future_connection = asyncio.open_connection( target.ip, target.port) try: reader, writer = await asyncio.wait_for( future_connection, timeout=target.conn_timeout) if target.ssl_check: try: # noinspection PyProtectedMember _sub_ssl = writer._transport.get_extra_info( 'ssl_object') cert_bytes = _sub_ssl.getpeercert(binary_form=True) cert_bytes_base64 = b64encode(cert_bytes).decode( 'utf-8') certificate_dict = convert_bytes_to_cert(cert_bytes) except BaseException: pass except Exception as e: await asyncio.sleep(0.005) try: future_connection.close() del future_connection except Exception as e: pass result = create_error_template(target, str(e)) else: try: status_data = False if target.payload: # если указан payload - то он и отправляется в первую очередь writer.write(target.payload) await writer.drain() if target.mode == 'single': status_data, data_or_error_result = await single_read( reader, target) elif target.mode == 'multi': status_data, data_or_error_result = await asyncio.wait_for( multi_read(reader, target), timeout=target.read_timeout) if status_data: check_filter = filter_bytes(data_or_error_result, target) if check_filter: result = make_document_from_response( data_or_error_result, target) if target.ssl_check: if cert_bytes_base64: result['data']['tcp']['result']['response'][ 'request']['tls_log']['handshake_log'][ 'server_certificates'][ 'certificate'][ 'raw'] = cert_bytes_base64 if certificate_dict: result['data']['tcp']['result']['response'][ 'request']['tls_log']['handshake_log'][ 'server_certificates'][ 'certificate'][ 'parsed'] = certificate_dict else: # TODO: добавить статус success-not-contain для обозначения того, # что сервис найден, но не попал под фильтр pass await asyncio.sleep(0.005) else: result = data_or_error_result # get errors try: writer.close() except BaseException: pass except Exception as e: result = create_error_template(target, str(e)) try: future_connection.close() except Exception: pass await asyncio.sleep(0.005) try: writer.close() except Exception: pass if result: success = access_dot_path(result, "data.tcp.status") if self.stats: if success == "success": self.stats.count_good += 1 else: self.stats.count_error += 1 line = None try: if self.success_only: if success == "success": line = ujson_dumps(result) else: line = ujson_dumps(result) except Exception: pass if line: await self.output_queue.put(line)