def test_inject_data_env(monkeypatch, monkeyed_requests_get, cleanuptmp,
fake_msg, fake_msg_with_arrays, fake_msg_as_jsonwb,
fake_msg_with_arrays_jsonwb, test_proto_url,
test_proto_mid, test_proto_msg,
test_proto_with_arrays_url,
test_proto_with_arrays_mid,
test_proto_with_arrays_msg):
monkeypatch.setattr('requests.get', monkeyed_requests_get)
assert (test_proto_mid not in data.list_known_protobufs())
assert (test_proto_with_arrays_mid not in data.list_known_protobufs())
register_proto_from_url(test_proto_url)
register_proto_from_url(test_proto_with_arrays_url)
assert (test_proto_mid in data.list_known_protobufs())
assert (test_proto_with_arrays_mid in data.list_known_protobufs())
# patching
data.myredis = fakeredis.FakeStrictRedis()
monkeypatch.setattr('acumos_proto_viewer.data.get_raw_data_source_count',
lambda x, y: 0)
monkeypatch.setattr('acumos_proto_viewer.data._get_bucket',
lambda: 'asdf0')
monkeypatch.setattr('time.time', lambda: 55555555555)
msgb = fake_msg()
data.inject_data(msgb, test_proto_url, test_proto_msg)
msgbwa = fake_msg_with_arrays()
data.inject_data(msgbwa, test_proto_with_arrays_url,
test_proto_with_arrays_msg)
_verify_inject_test(fake_msg_as_jsonwb, fake_msg_with_arrays_jsonwb,
test_proto_mid, test_proto_msg,
test_proto_with_arrays_mid, test_proto_with_arrays_msg)
cleanuptmp()
def inject_data(binarydata, proto_url, message_name):
"""
Injects data into the appropriate queue.
Raises SchemaNotReachable if the proto_url is invalid.
Answers true if expected keys are found, otherwise false.
In the future if the data moves to a database this would go away
"""
# register the proto file. Will return immediately if already exists
model_id = register_proto_from_url(proto_url)
count = get_raw_data_source_count(model_id, message_name)
index = _get_raw_data_source_index(model_id, message_name)
_logger.debug("inject_data: message_name %s sequence %d", message_name, count + 1)
j = _msg_to_json_preserve_bytes(binarydata, model_id, message_name, count + 1)
# safeguard against malformed data
act_keys = sorted(j.keys())
exp_keys = sorted(proto_data_structure[model_id]["messages"][message_name]["properties"].keys())
if act_keys == exp_keys:
# this auto creates the key if it does not exist yet #https://myredis.io/commands/lpush
try:
pickled_json_message = pickle.dumps(j)
myredis.rpush(index, pickled_json_message)
except Exception as exc:
_logger.error("inject_data: failed to pickle data or upload it to redis")
_logger.exception(exc)
if count == 0:
_logger.debug("inject_data: created new data source with TTL of one day")
myredis.expire(index, 60 * 60 * 24)
else:
_logger.warning("inject_data: dropped message {0} due to unexpected keys: received {1} expected {2}".format(message_name, act_keys, exp_keys))
return False
return True
def test_register_load(monkeypatch, monkeyed_requests_get, cleanuptmp,
test_proto_url, test_proto_mid,
test_proto_with_arrays_url, test_proto_with_arrays_mid):
monkeypatch.setattr('requests.get', monkeyed_requests_get)
register_proto_from_url(test_proto_url)
register_proto_from_url(test_proto_with_arrays_url)
assert (test_proto_mid in data.list_known_protobufs())
assert (test_proto_with_arrays_mid in data.list_known_protobufs())
test_pb2 = load_proto(test_proto_mid)
test_pb2 = load_proto(test_proto_mid) # cache hit
assert isinstance(test_pb2, ModuleType)
test2_pb2 = load_proto(test_proto_with_arrays_mid)
test2_pb2 = load_proto(test_proto_with_arrays_mid) # cache hit
assert isinstance(test2_pb2, ModuleType)
cleanuptmp()
def test_msg_to_json_preserve_bytes(
monkeypatch, monkeyed_requests_get, cleanuptmp, fake_msg_as_jsonwb,
fake_msg, fake_msg_with_arrays_jsonwb, fake_msg_with_arrays,
test_proto_url, test_proto_mid, test_proto_msg,
test_proto_with_arrays_url, test_proto_with_arrays_mid,
test_proto_with_arrays_msg):
monkeypatch.setattr('requests.get', monkeyed_requests_get)
register_proto_from_url(test_proto_url)
register_proto_from_url(test_proto_with_arrays_url)
msgb = fake_msg()
# the method under test here generates a timestamp so we have to fake that
monkeypatch.setattr('time.time', lambda: 55555555555)
jpb = data._msg_to_json_preserve_bytes(msgb, test_proto_mid,
test_proto_msg, 1)
assert (jpb == fake_msg_as_jsonwb())
msgba = fake_msg_with_arrays()
jpb2 = data._msg_to_json_preserve_bytes(msgba, test_proto_with_arrays_mid,
test_proto_with_arrays_msg, 1)
assert (jpb2 == fake_msg_with_arrays_jsonwb())
cleanuptmp()
def test_protobuf_to_js(monkeypatch, monkeyed_requests_get, cleanuptmp,
test_proto_url, test_proto_mid,
test_proto_with_arrays_url,
test_proto_with_arrays_mid):
monkeypatch.setattr('requests.get', monkeyed_requests_get)
register_proto_from_url(test_proto_url)
register_proto_from_url(test_proto_with_arrays_url)
assert (test_proto_mid in data.list_known_protobufs())
assert (test_proto_with_arrays_mid in data.list_known_protobufs())
js = _protobuf_to_js(test_proto_mid)
assert js == {
"definitions": {
"mygreatpackage.Data1": {
"title": "Data1",
"type": "object",
"properties": {
"a": {
"type": "number"
},
"b": {
"type": "number"
},
"c": {
"type": "integer",
"minimum": -2147483648,
"maximum": 2147483647
},
"d": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"e": {
"type": "boolean"
},
"f": {
"type": "string"
},
"g": {
"type": "string"
}
}
},
"mygreatpackage.Data2": {
"title": "Data2",
"type": "object",
"properties": {
"a": {
"$ref": "#/definitions/mygreatpackage.Data1"
},
"b": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/mygreatpackage.Data1"
}
},
"c": {
"type": "object",
"additionalProperties": {
"type": "integer",
"minimum": -2147483648,
"maximum": 2147483647
}
},
"d": {
"type": "array",
"items": {
"$ref": "#/definitions/mygreatpackage.Data1"
}
},
"e": {
"type": "array",
"items": {
"type": "integer",
"minimum": -2147483648,
"maximum": 2147483647
}
}
}
}
}
}
js = _protobuf_to_js(test_proto_with_arrays_mid)
print(js)
assert (js == {
"definitions": {
"YKhGXjKWHYsPwKJFfEPnmoHOkDkPKBxX.ImageTagSet": {
"title": "ImageTagSet",
"type": "object",
"properties": {
"image": {
"type": "array",
"items": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
}
},
"tag": {
"type": "array",
"items": {
"type": "string"
}
},
"score": {
"type": "array",
"items": {
"type": "number"
}
}
}
}
}
})
cleanuptmp()