def test_cache_memoize_typed(cache: Cache):
"""Test that cache.memoize() can factor in argument types as part of the cache key."""
@cache.memoize()
def untyped(a):
return a
@cache.memoize(typed=True)
def typed(a):
return a
assert untyped(1) is untyped(1.0)
assert typed(1) is not typed(1.0)
assert len(cache) == 3
untyped_keys = [
key for key in cache.keys()
if key.startswith(f"{untyped.__module__}.{untyped.__name__}")
]
typed_keys = [
key for key in cache.keys()
if key.startswith(f"{typed.__module__}.{typed.__name__}")
]
assert len(untyped_keys) == 1
assert len(typed_keys) == 2
def test_cache_full(cache: Cache):
"""Test that cache.full() returns whether the cache is full or not."""
for n in range(cache.maxsize):
assert not cache.full()
cache.set(n, n)
assert cache.full()
def test_cache_delete_expired(cache: Cache, timer: Timer):
"""Test that cache.delete_expired() removes all expired keys."""
ttl = 1
ttl_keys = list(range(5))
non_ttl_keys = list(range(5, 10))
all_keys = ttl_keys + non_ttl_keys
for key in ttl_keys:
cache.set(key, key, ttl=ttl)
for key in non_ttl_keys:
cache.set(key, key)
assert len(cache) == len(all_keys)
cache.delete_expired()
assert len(cache) == len(all_keys)
timer.time = ttl
cache.delete_expired()
assert len(cache) == len(non_ttl_keys)
for key in ttl_keys:
assert not cache.has(key)
for key in non_ttl_keys:
assert cache.has(key)
def test_cache_size(cache: Cache):
"""Test that cache.size() returns the number of cache keys."""
assert cache.size() == len(cache) == 0
for n in range(1, 50):
cache.set(n, n)
assert cache.size() == len(cache) == n
def test_cache_set_many(cache: Cache):
"""Test that cache.set_many() sets multiple cache key/values."""
items = {"a": 1, "b": 2, "c": 3}
cache.set_many(items)
for key, value in items.items():
assert cache.get(key) == value
def test_cache_memoize_ttl(cache: Cache, timer: Timer):
"""Test that cache.memoize() can set a TTL."""
ttl1 = 5
ttl2 = ttl1 + 1
@cache.memoize(ttl=ttl1)
def func1(a):
return a
@cache.memoize(ttl=ttl2)
def func2(a):
return a
func1(1)
func2(1)
assert len(cache) == 2
key1, key2 = tuple(cache.keys())
timer.time = ttl1 - 1
assert cache.has(key1)
assert cache.has(key2)
timer.time = ttl1
assert not cache.has(key1)
assert cache.has(key2)
timer.time = ttl2
assert not cache.has(key2)
class ScanQRCode(threading.Thread):
def __init__(self, q):
threading.Thread.__init__(self)
self.running = True
self.queue = q
self.vs = None
self.cache = Cache(ttl=5)
def stopThread(self):
self.running = False
if not self.vs:
self.vs.stop()
def run(self):
self.vs = VideoStream(usePiCamera=True).start()
time.sleep(2)
logging.info("scan qrcode thread is running...")
while self.running:
frame = self.vs.read()
frame = imutils.resize(frame, width=400)
barcodes = pyzbar.decode(frame)
for barcode in barcodes:
data = barcode.data
if self.cache.has(data):
continue
self.cache.add(data, None)
msg = Msg(msgType=MsgType.SCAN_QR_CODE, load=barcode.data)
logging.info("scan qrcode thread put msg: " + str(msg.load))
self.queue.put(msg)
def test_cache_copy(cache: Cache):
"""Test that cache.copy() returns a copy of the cache."""
items = {"a": 1, "b": 2, "c": 3}
cache.set_many(items)
copied = cache.copy()
assert copied == items
assert copied is not cache._cache
def test_cache_clear(cache: Cache):
"""Test that cache.clear() deletes all cache keys."""
items = {"a": 1, "b": 2, "c": 3}
cache.set_many(items)
assert len(cache) == len(items)
cache.clear()
assert len(cache) == 0
def test_cache_default():
"""Test that Cache can set the default for Cache.get()."""
cache = Cache(default=True)
assert cache.get(1) is True
assert 1 not in cache
assert cache.get(2, default=False) is False
assert 2 not in cache
def test_cache_delete_many(cache: Cache, items: dict,
iteratee: t.Union[list, str, t.Pattern,
t.Callable], expected: dict):
"""Test that cache.delete_many() deletes multiple cache key/values filtered by an iteratee."""
cache.set_many(items)
cache.delete_many(iteratee)
assert dict(cache.items()) == expected
def test_cache_repr(cache: Cache):
"""Test that repr(cache) returns a representation of the cache object."""
assert repr(cache) == "Cache([])"
cache.set("a", 1)
cache.set("b", 2)
cache.set("c", 3)
assert repr(cache) == "Cache([('a', 1), ('b', 2), ('c', 3)])"
class Predictor_Ocr:
def __init__(self, ip, port, password):
self.ip = ip
self.port = port
self.password = password
self.redis_pool = redis.ConnectionPool(host=ip,
port=port,
db=0,
password=password,
encoding='utf-8')
self.cache = Cache()
self.cur = os.getcwd()
def get_model(self, model_key):
if self.cache.has(model_key) is True:
return self.cache.get(model_key)
else:
path = "%s/%s/data/%s.h5" % (self.cur, model_key, model_key)
model = load_model(path)
self.cache.set(model_key, model)
return model
def work(self):
while True:
width = 64
height = 64
red = redis.Redis(connection_pool=self.redis_pool)
task_str = red.lpop("aiocr")
if task_str is not None:
task = json.loads(task_str)
create_user_id = task["create_user_id"]
image_id = task["image_id"]
polygon_id = task["polygon_id"]
image = task["image"]
algorithm_set = task["algorithm"]
image_list = [image]
image_batch = np.array(image_list, dtype=np.float32).reshape(
-1, width, height, 1)
image_batch = image_batch / 255.0
out_list = []
top_k = 5
thr = 0.1
for algorithm in algorithm_set:
path = "%s/%s/data/elem_list.json" % (self.cur, algorithm)
with open(path, 'r') as f:
elem_list = json.load(f)
model = self.get_model(algorithm)
out = model.predict(image_batch)[0]
top_candidate = out.argsort()[::-1][0:top_k]
for item in top_candidate:
if out[item] > thr and elem_list[item] > -1:
out_list.append(elem_list[item])
key = "%s_%s_%s_%s" % ("rs_aiocr", create_user_id, image_id,
polygon_id)
red.set(key, json.dumps(out_list))
time.sleep(0.005)
def __init__(self, data_config_file="dataBase.yaml"):
"""
初始化
"""
self.mysql_conf = setup.MySql(data_config_file)
self.engine = create_engine(self.mysql_conf.PyMySql_STR)
self.db_session_factory = sessionmaker(bind=self.engine)
self.app = setup.App()
self.cache = Cache()
def __init__(self, pcap_collection_data, max_queue_size, work_queue,
interface, custom_tag, return_deep_info, http_filter_json,
cache_size, session_size, bpf_filter, timeout, debug):
"""
构造函数
:param max_queue_size: 资产队列最大长度
:param work_queue: 捕获资产数据消息发送队列
:param interface: 捕获流量的网卡名
:param custom_tag: 数据标签，用于区分不同的采集引擎
:param return_deep_info: 是否处理更多信息，包括原始请求、响应头和正文
:param http_filter_json: HTTP过滤器配置，支持按状态和内容类型过滤
:param cache_size: 缓存的已处理数据条数，120秒内重复的数据将不会重复采集
:param session_size: 缓存的HTTP/TCP会话数量，30秒未使用的会话将被自动清除
:param bpf_filter: 数据包底层过滤器
:param timeout: 采集程序的运行超时时间，默认为启动后1小时自动退出
:param debug: 调试开关
"""
self.pcap_collection_data = pcap_collection_data
self.total_msg_num = 0
self.max_queue_size = max_queue_size
self.work_queue = work_queue
self.debug = debug
self.timeout = timeout
self.bpf_filter = bpf_filter
self.cache_size = cache_size
self.session_size = session_size
self.http_filter_json = http_filter_json
self.return_deep_info = return_deep_info
self.custom_tag = custom_tag
self.interface = interface
self.sniffer = pcap.pcap(self.interface,
snaplen=65535,
promisc=True,
timeout_ms=self.timeout,
immediate=False)
self.sniffer.setfilter(self.bpf_filter)
self.tcp_stream_cache = Cache(maxsize=self.session_size,
ttl=30,
timer=time.time,
default=None)
if self.cache_size:
self.tcp_cache = LRUCache(maxsize=self.cache_size,
ttl=120,
timer=time.time,
default=None)
self.http_cache = LRUCache(maxsize=self.cache_size,
ttl=120,
timer=time.time,
default=None)
# http数据分析正则
self.decode_request_regex = re.compile(
r'^([A-Z]+) +([^ \r\n]+) +HTTP/\d+(?:\.\d+)?[^\r\n]*(.*?)$', re.S)
self.decode_response_regex = re.compile(
r'^HTTP/(\d+(?:\.\d+)?) (\d+)[^\r\n]*(.*?)$', re.S)
self.decode_body_regex = re.compile(
rb'<meta[^>]+?charset=[\'"]?([a-z\d\-]+)[\'"]?', re.I)
def test_cache_get(cache: Cache):
"""Test that cache.get() returns a cache key value or a default value if missing."""
key, value = ("key", "value")
assert cache.get(key) is None
assert cache.get(key, default=1) == 1
assert key not in cache
cache.set(key, value)
assert cache.get(key) == value
def test_cache_get_default_callable(cache: Cache):
"""Test that cache.get() uses a default function when value is not found to set cache keys."""
def default(key):
return key
key = "key"
assert cache.get(key) is None
assert cache.get(key, default=default) == key
assert key in cache
def __init__(self, ip, port, password):
self.ip = ip
self.port = port
self.password = password
self.redis_pool = redis.ConnectionPool(host=ip,
port=port,
db=0,
password=password,
encoding='utf-8')
self.cache = Cache()
self.cur = os.getcwd()
def test_cache_iter(cache: Cache):
"""Test that iterating over cache yields each cache key."""
items: dict = {"a": 1, "b": 2, "c": 3}
cache.set_many(items)
keys = []
for key in cache:
assert cache.get(key) == items[key]
keys.append(key)
assert set(keys) == set(items)
def test_cache_has(cache: Cache):
"""Test that cache.has() returns whether a key exists or not."""
key, value = ("key", "value")
assert not cache.has(key)
assert key not in cache
cache.set(key, value)
assert cache.has(key)
assert key in cache
def __init__(self, work_queue, interface, custom_tag, return_deep_info,
http_filter_json, cache_size, session_size, bpf_filter,
timeout, debug):
"""
构造函数
:param work_queue: 捕获资产数据消息发送队列
:param interface: 捕获流量的网卡名
:param custom_tag: 数据标签，用于区分不同的采集引擎
:param return_deep_info: 是否处理更多信息，包括原始请求、响应头和正文
:param http_filter_json: HTTP过滤器配置，支持按状态和内容类型过滤
:param cache_size: 缓存的已处理数据条数，120秒内重复的数据将不会发送Syslog
:param session_size: 缓存的HTTP/TCP会话数量，16秒未使用的会话将被自动清除
:param bpf_filter: 数据包底层过滤器
:param timeout: 采集程序的运行超时时间，默认为启动后1小时自动退出
:param debug: 调试开关
"""
self.work_queue = work_queue
self.debug = debug
self.timeout = timeout
self.bpf_filter = bpf_filter
self.cache_size = cache_size
self.session_size = session_size
self.http_filter_json = http_filter_json
self.return_deep_info = return_deep_info
self.custom_tag = custom_tag
self.interface = interface
self.pktcap = pyshark.LiveCapture(interface=self.interface,
bpf_filter=self.bpf_filter,
use_json=False,
debug=self.debug)
if self.session_size:
self.http_stream_cache = Cache(maxsize=self.session_size,
ttl=16,
timer=time.time,
default=None)
self.tcp_stream_cache = Cache(maxsize=self.session_size,
ttl=16,
timer=time.time,
default=None)
if self.cache_size:
self.http_cache = LRUCache(maxsize=self.cache_size,
ttl=120,
timer=time.time,
default=None)
self.tcp_cache = LRUCache(maxsize=self.cache_size,
ttl=120,
timer=time.time,
default=None)
# 检测页面编码的正则表达式
self.encode_regex = re.compile(
rb'<meta [^>]*?charset=["\']?([a-z\-\d]+)["\'>]?', re.I)
def test_cache_default_callable():
"""Test that Cache can set a default function for Cache.get()."""
def default(key):
return False
def default_override(key):
return key
cache = Cache(default=default)
assert cache.get("key1") is False
assert cache.get("key1", default=default_override) is False
assert cache.get("key2", default=default_override) == "key2"
assert cache.get("key3", default=3) == 3
def test_cache_delete(cache: Cache):
"""Test that cache.delete() removes a cache key."""
key, value = ("key", "value")
cache.set(key, value)
assert cache.has(key)
cache.delete(key)
assert not cache.has(key)
cache.delete(key)
assert not cache.has(key)
def test_cache_popitem(cache: Cache):
"""Test that cache.popitem() removes the oldest entries first."""
keys = list(range(cache.maxsize))
for key in keys:
cache.set(key, str(key))
for i, key in enumerate(keys):
k, v = cache.popitem()
assert k == key
assert v == str(key)
assert len(cache) == (cache.maxsize - (i + 1))
with pytest.raises(KeyError):
cache.popitem()
def test_cache_add(cache: Cache):
"""Test that cache.add() sets a cache key but only if it doesn't exist."""
key, value = ("key", "value")
ttl = 2
cache.add(key, value, ttl)
assert cache.get(key) == value
assert cache.expire_times()[key] == ttl
cache.add(key, value, ttl + 1)
assert cache.expire_times()[key] == ttl
cache.set(key, value, ttl + 1)
assert cache.expire_times()[key] == ttl + 1
def test_cache_memoize_async(cache: Cache):
"""Test that cache.memoize() can decorate async functions."""
loop = asyncio.get_event_loop()
marker = 1
@cache.memoize()
async def func(a):
return a, marker
assert asyncio.iscoroutinefunction(func)
assert len(cache) == 0
result = loop.run_until_complete(func("a"))
assert result == ("a", 1)
assert len(cache) == 1
assert list(cache.values())[0] == ("a", 1)
marker += 1
result = loop.run_until_complete(func("a"))
assert result == ("a", 1)
assert len(cache) == 1
result = loop.run_until_complete(func("b"))
assert result == ("b", 2)
assert len(cache) == 2
def test_cache_memoize_func_attrs(cache: Cache):
"""Test that cache.memoize() adds attributes to decorated function."""
marker = 1
value: dict = {}
def original(a):
return a, marker
memoized = cache.memoize()(original)
assert memoized.cache is cache
assert memoized.uncached is original
assert hasattr(memoized, "cache_key") and callable(memoized.cache_key)
_, mark_x = memoized(value)
assert mark_x == marker
marker += 1
_, mark_y = memoized(value)
assert mark_y != marker
assert mark_y == mark_x
_, mark_z = memoized.uncached(value)
assert mark_z == marker
class XMLParser():
def __init__(self, path):
self.cache = Cache()
self.root = etree.parse(r'{0}'.format(path))
def storeElementWithKey(self, element):
udid = str(uuid.uuid4())
self.cache.set(udid, element)
return udid
def getStorElementWithKey(self, udid):
print(udid)
return self.cache.get(udid)
def getElementByXpath(self, xpath):
var = self.root.xpath(xpath)
return self.storeElementWithKey(var)
def test_cache_set_ttl_override(cache: Cache, timer: Timer):
"""Test that cache.set() can override the default TTL."""
default_ttl = 1
cache.configure(ttl=default_ttl)
cache.set("key1", "value1")
cache.set("key2", "value2", ttl=default_ttl + 1)
timer.time = default_ttl
assert not cache.has("key1")
assert cache.has("key2")
timer.time = default_ttl + 1
assert not cache.has("key2")
def __init__(self,interface,display_filter,syslog_ip,syslog_port,custom_tag,return_deep_info,filter_rules,cache_size,bpf_filter,timeout,debug): self.debug = debug self.timeout = timeout self.bpf_filter = bpf_filter self.cache_size = cache_size self.filter_rules = filter_rules self.return_deep_info = return_deep_info self.custom_tag = custom_tag self.syslog_ip = syslog_ip self.syslog_port = syslog_port self.log_obj = _logging(self.syslog_ip,self.syslog_port) self.interface = interface self.display_filter = display_filter self.pktcap = pyshark.LiveCapture(interface=self.interface, bpf_filter=self.bpf_filter, use_json=True, display_filter=self.display_filter, debug=self.debug) self.http_cache = Cache(maxsize=self.cache_size, ttl=120, timer=time.time, default=None) self.tcp_cache = Cache(maxsize=self.cache_size, ttl=120, timer=time.time, default=None) # 检测页面编码的正则表达式 self.encode_regex = re.compile(b'<meta [^>]*?charset=["\']?([^"\'\s]+)["\']?', re.I)