async def on_message(msg):
if msg[0] == ord(self.__class__._msgType['notify']):
wrapper = self._addrBook.Wrapper()
wrapper.ParseFromString(msg[1:])
notification = reflection.MakeClass(
self._addrBook.DESCRIPTOR.message_types_by_name[
wrapper.name[4:]])()
notification.ParseFromString(wrapper.data)
if wrapper.name[4:] in self._msgPool.keys():
for index in self._msgPool[wrapper.name[4:]]:
if index != -1:
cb = self._msgPool[wrapper.name[4:]][index]
if asyncio.iscoroutinefunction(cb):
await cb(protobuf_to_dict(notification))
else:
cb(protobuf_to_dict(notification))
elif msg[0] == ord(self.__class__._msgType['res']):
# get index
index = msg[1] + msg[2] * 256
if index in self._msgPool.keys():
# callback
self._wrapper.ParseFromString(msg[3:])
res = self._msgPool[index][0]()
res.ParseFromString(self._wrapper.data)
if self._msgPool[index][1] is not None:
if asyncio.iscoroutinefunction(
self._msgPool[index][1]):
await self._msgPool[index][1](protobuf_to_dict(res)
)
else:
self._msgPool[index][1](protobuf_to_dict(res))
self._msgPool.pop(index)
def run_evaluation_data(self, data_path: str, result_path: str):
request = rekcurd_pb2.EvaluationResultRequest(data_path=data_path,
result_path=result_path)
for raw_response in self.stub.EvaluationResult(
request, metadata=self.__metadata):
details = []
for detail in raw_response.detail:
details.append(
dict(protobuf_to_dict(detail,
including_default_value_fields=True),
input=self.__get_value_from_io(detail.input),
label=self.__get_value_from_io(detail.label),
output=self.__get_value_from_io(detail.output),
score=detail.score[0]
if len(detail.score) == 1 else list(detail.score)))
metrics = raw_response.metrics
metrics_response = dict(
protobuf_to_dict(metrics, including_default_value_fields=True),
label=[self.__get_value_from_io(l) for l in metrics.label])
response = protobuf_to_dict(raw_response,
including_default_value_fields=True)
response['detail'] = details
response['status'] = True
response['metrics'] = metrics_response
yield response
def conf_reply(self, response):
"""Set RRC measurements configuration reply."""
self._conf_reply = protobuf_to_dict(response)
reply = protobuf_to_dict(response)
ue = RUNTIME.tenants[self.tenant_id].ues[self.ue]
event_type = response.WhichOneof("event_types")
conf = reply[event_type]["mUE_rrc_meas_conf"]["repl"]
self._conf_reply = \
self._conf_reply[event_type]["mUE_rrc_meas_conf"]["repl"]
if conf["status"] != configs_pb2.CREQS_SUCCESS:
return
del conf["rnti"]
del conf["status"]
if "ue_rrc_state" in conf:
ue.rrc_state = conf["ue_rrc_state"]
del conf["ue_rrc_state"]
if "capabilities" in conf:
ue.capabilities = conf["capabilities"]
del conf["capabilities"]
ue.rrc_meas_config = conf
def _make_json_entry(header, item):
"""
Combine a trace entry header and item into a single JSON-serializable
entry. Return this entry as a ``dict``.
Some things to note:
* The header's ``size`` field is removed - it is not required in the
JSON
* Enums are replaced by their numerical value (so that they can be
written to JSON)
"""
# If the entry is a fork, then we have to make the child traces
# JSON-serializable as well
if header.type == TraceEntries_pb2.TRACE_FORK:
children = {
state_id: _make_json_trace(trace)
for state_id, trace in item.children.items()
}
item = TraceEntryFork(children)
header_dict = protobuf_to_dict(header, use_enum_labels=True)
entry = header_dict.copy()
if isinstance(item, TraceEntryFork):
entry.update({'children': item.children})
else:
entry.update(protobuf_to_dict(item, use_enum_labels=True))
return entry
def test_message_with_proto3_enum_protobuf_to_dict(self):
m = SomeMessage()
m.enum_field = 0
d = protobuf_to_dict(m)
self.assertEqual(d['enum_field'], 0)
m = SomeMessage()
m.enum_field = 1
d = protobuf_to_dict(m)
self.assertEqual(d['enum_field'], 1)
def test_message_with_proto3_bool_protobuf_to_dict(self):
m = SomeMessage()
m.bool_field = False
d = protobuf_to_dict(m)
self.assertEqual(d['bool_field'], False)
m = SomeMessage()
m.bool_field = True
d = protobuf_to_dict(m)
self.assertEqual(d['bool_field'], True)
def test_including_default_value_fields(self):
m = MessageOfTypes()
d = protobuf_to_dict(m)
assert d == {}
d = protobuf_to_dict(m, including_default_value_fields=True)
for field in m.DESCRIPTOR.fields:
if field.name != 'nested':
assert field.name in d, field.name
m2 = dict_to_protobuf(MessageOfTypes, d)
assert m == m2
def test_including_default_value_fields(self):
m = MessageOfTypes()
d = protobuf_to_dict(m)
assert d == {}
d = protobuf_to_dict(m, including_default_value_fields=True)
for field in m.DESCRIPTOR.fields:
if field.name != 'nested':
assert field.name in d, field.name
m2 = dict_to_protobuf(MessageOfTypes, d)
assert m == m2
def _make_json_entry(header, item, state):
"""
Combine a trace entry header and item into a single JSON-serializable
entry. Return this entry as a ``dict``.
Some things to note:
* The header's ``size`` field is removed - it is not required in the
JSON
* Enums are replaced by their numerical value (so that they can be
written to JSON)
"""
# If the entry is a fork, then we have to make the child traces
# JSON-serializable as well
if header.type == TraceEntries_pb2.TRACE_FORK:
children = {}
for state_id, trace in item.children.items():
new_state = state.clone()
children[state_id] = _make_json_trace(trace, new_state)
item = TraceEntryFork(children)
elif header.type == TraceEntries_pb2.TRACE_OSINFO:
state.modules.kernel_start = item.kernel_start
elif header.type == TraceEntries_pb2.TRACE_MOD_LOAD:
state.modules.add(Module(item))
elif header.type == TraceEntries_pb2.TRACE_MOD_UNLOAD:
try:
state.modules.remove(Module(item))
except Exception:
pass
header_dict = protobuf_to_dict(header, use_enum_labels=True)
entry = header_dict.copy()
if isinstance(item, TraceEntryFork):
entry.update({'children': item.children})
else:
entry.update(protobuf_to_dict(item, use_enum_labels=True))
try:
mod = state.modules.get(header.pid, header.pc)
rel_pc = mod.to_native(header.pc)
if rel_pc:
entry.update({'module': {'name': mod.path, 'pc': rel_pc}})
except Exception as e:
logger.debug('Error while computing module: %s', e)
return entry
def test_use_enum_labels(self, add_class_metadata,
overwrite_package_with_name):
m = self.populate_MessageOfTypes()
d = protobuf_to_dict(
m,
use_enum_labels=True,
add_class_metadata=add_class_metadata,
overwrite_package_with_name=overwrite_package_with_name)
self.compare(m,
d, ['enm', 'enmRepeated', 'nestedRepeated'],
add_class_metadata=add_class_metadata,
overwrite_package_with_name=overwrite_package_with_name)
assert d['enm'] == 'C'
assert d['enmRepeated'] == ['A', 'C']
m2 = dict_to_protobuf(MessageOfTypes, d)
assert m == m2
d['enm'] = 'MEOW'
with nose.tools.assert_raises(KeyError):
dict_to_protobuf(MessageOfTypes, d)
d['enm'] = 'A'
d['enmRepeated'] = ['B']
dict_to_protobuf(MessageOfTypes, d)
d['enmRepeated'] = ['CAT']
with nose.tools.assert_raises(KeyError):
dict_to_protobuf(MessageOfTypes, d)
def parse_main_request(response_content, response_status, subrequests):
log.debug('Parsing main RPC response...')
if response_status != 200:
log.warning('Unexpected HTTP server response - needs 200 got %s', response_status)
log.debug('HTTP output: \n%s', response_content)
return False
if response_content is None:
log.warning('Empty server response!')
return False
response_proto = RpcEnvelope.Response()
try:
response_proto.ParseFromString(response_content)
except:
log.exception('Could not parse response: ')
return False
log.debug('Protobuf structure of rpc response:\n\r%s', response_proto)
response_proto_dict = protobuf_to_dict(response_proto)
response_proto_dict = parse_sub_responses(response_proto, subrequests, response_proto_dict)
return response_proto_dict
def prepare_response(self, response):
resp_dict = protobuf_to_dict(response)
# Remove the recipe_ids field
resp_dict.pop('recipe_ids', None)
resp_dict.pop('test', None)
resp_dict.pop('duration', None)
if not resp_dict.has_key('recipes'):
return {'data': [], 'debug': {'duration': response.duration}}
for i, recipe in enumerate(resp_dict['recipes']):
resp_dict['recipes'][i]['cook_time'] = recipe['time']['cook']
resp_dict['recipes'][i]['prep_time'] = recipe['time']['prep']
resp_dict['recipes'][i]['ready_time'] = recipe['time']['ready']
for j, ingredient in enumerate(recipe['ingredients']):
resp_dict['recipes'][i]['ingredients'][j]['units'] = 'ounce'
resp_dict['recipes'][i]['ingredients'][j]['picture'] = 'not-implemented-yet.png'
# Replace the ingrids repeated field with a single ID.
if ingredient.has_key('ingrids'):
resp_dict['recipes'][i]['ingredients'][j]['id'] = ingredient['ingrids'][0]
resp_dict.pop('ingrids', None)
else:
resp_dict['recipes'][i]['ingredients'][j]['id'] = -1
return {'data': resp_dict['recipes'],
'debug': {'duration': response.duration}}
def _parse_main_request(self, response_raw, subrequests):
self.log.debug('Parsing main RPC response...')
if response_raw.status_code != 200:
self.log.warning(
'Unexpected HTTP server response - needs 200 got %s',
response_raw.status_code)
self.log.debug('HTTP output: \n%s', response_raw.content)
return False
if response_raw.content is None:
self.log.warning('Empty server response!')
return False
response_proto = RpcEnvelope.Response()
try:
response_proto.ParseFromString(response_raw.content)
except google.protobuf.message.DecodeError as e:
self.log.warning('Could not parse response: %s', str(e))
return False
self.log.debug('Protobuf structure of rpc response:\n\r%s',
response_proto)
response_proto_dict = protobuf_to_dict(response_proto)
response_proto_dict = self._parse_sub_responses(
response_proto, subrequests, response_proto_dict)
return response_proto_dict
def get(self):
fingerprint = self.get_argument('fingerprint')
CHECK_EQ(len(fingerprint), 32, 'invalid fingerprint: %s' % fingerprint)
provenance_list = provenance.BuildResourceProvenanceList(fingerprint)
provenance_list.sort(key = lambda p : p.start_time_sec)
total_job_run_time_sec = 0
prev_end_time_sec = 0
items = []
for p in provenance_list:
CHECK_GE(p.start_time_sec, prev_end_time_sec) # our method of adding run times assumes no two jobs overlap... need more complex method otherwise
prev_end_time_sec = p.end_time_sec
CHECK_GE(p.end_time_sec, p.start_time_sec) # check invariant
job_run_time_sec = p.end_time_sec - p.start_time_sec
print total_job_run_time_sec
item_data = protobuf_to_dict(p)
item_data['squeeze_time_start_sec'] = total_job_run_time_sec
total_job_run_time_sec += job_run_time_sec
item_data['squeeze_time_end_sec'] = total_job_run_time_sec
items.append(item_data)
provenance_json = json.dumps(items)
print provenance_json
params = {'provenance_json' : provenance_json
}
self.render('resource.html', **params)
return
def get(self):
fingerprint = self.get_argument('fingerprint')
CHECK_EQ(len(fingerprint), 32, 'invalid fingerprint: %s' % fingerprint)
provenance_list = provenance.BuildResourceProvenanceList(fingerprint)
provenance_list.sort(key=lambda p: p.start_time_sec)
total_job_run_time_sec = 0
prev_end_time_sec = 0
items = []
for p in provenance_list:
CHECK_GE(
p.start_time_sec, prev_end_time_sec
) # our method of adding run times assumes no two jobs overlap... need more complex method otherwise
prev_end_time_sec = p.end_time_sec
CHECK_GE(p.end_time_sec, p.start_time_sec) # check invariant
job_run_time_sec = p.end_time_sec - p.start_time_sec
print total_job_run_time_sec
item_data = protobuf_to_dict(p)
item_data['squeeze_time_start_sec'] = total_job_run_time_sec
total_job_run_time_sec += job_run_time_sec
item_data['squeeze_time_end_sec'] = total_job_run_time_sec
items.append(item_data)
provenance_json = json.dumps(items)
print provenance_json
params = {'provenance_json': provenance_json}
self.render('resource.html', **params)
return
def update(self):
"""
gets raw data from MTA GTFS-rt API
data is updated ~once/min, so no need to call more often than that. must be called before querying arrival times.
"""
if len(self.api_key) < 40 or len(self.api_key) > 40:
print("invalid API key")
return
feed = gtfs_realtime_pb2.FeedMessage()
# gets proper URL based on chosen train route
for route_url in urldict:
if self.train in route_url:
mta_url = urldict[route_url]
response = requests.get(mta_url, headers={'x-api-key' : (self.api_key)})
feed.ParseFromString(response.content)
mta_raw_output = protobuf_to_dict(feed)
print(mta_raw_output)
for key in mta_raw_output:
if key is 'vehicle':
self.vehicle_updates = {val for key, val in mta_raw_output.items() if 'vehicle' in key}
self.trip_updates = {val for key, val in mta_raw_output.items() if 'trip_update' in key}
print(self.trip_updates)
print(self.vehicle_updates)
def trainTimeLookUp(
station
): #This function fetches the data from the MTA and only keeps the data for the stations of interest
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(
'http://datamine.mta.info/mta_esi.php?key={}&feed_id={}'.format(
api_key, station[1]))
feed.ParseFromString(response.content)
subway_feed = protobuf_to_dict(feed)
realtime_data = subway_feed['entity']
collected_times = []
for trains in realtime_data:
if trains.get('trip_update', False) != False:
unique_train_schedule = trains['trip_update']
if unique_train_schedule.get('stop_time_update', False) != False:
ExpTime = unique_train_schedule.get('stop_time_update')
trainInfo = unique_train_schedule['trip']
trainLine = trainInfo['route_id']
for stop in ExpTime:
if stop.get('stop_id') == station[0] + 'N':
output = stop.get('departure')
collected_times.append(
[trainLine, 'N',
output.get('time')])
elif stop.get('stop_id') == station[0] + 'S':
output = stop.get('departure')
collected_times.append(
[trainLine, 'S',
output.get('time')])
return np.asarray(collected_times)
def receive_packet(self):
wrapper_packet = SSL_WrapperPacket()
try:
data = self.connection.recv(2048)
except timeout:
self.logger.debug('No Vision Frame received.')
return
try:
wrapper_packet.ParseFromString(data)
except DecodeError:
self.logger.error('VisionReceiver had trouble decoding a packet!')
wrapper_packet = protobuf_to_dict(wrapper_packet)
detection_packet = wrapper_packet.get('detection', None)
if detection_packet:
current_time_offset = time.time() - detection_packet['t_capture']
if VisionReceiver.TIME_OFFSET is None:
self.logger.debug('Offset time between system is {:.0f} sec.'.format(current_time_offset))
VisionReceiver.TIME_OFFSET = current_time_offset
if abs(VisionReceiver.TIME_OFFSET - current_time_offset) > VisionReceiver.MAX_TIME_OFFSET_DIFFERENCE:
self.logger.debug('Offset time between system was reset to {:.0f} sec (was {:.0f} sec)'.format(current_time_offset,
VisionReceiver.TIME_OFFSET))
self.logger.warning('You might be receiving vision frame from more then one source.')
VisionReceiver.TIME_OFFSET = current_time_offset
detection_packet['t_capture'] = VisionReceiver.TIME_OFFSET + detection_packet['t_capture']
self._link[detection_packet['camera_id']] = detection_packet
def test_extensions(self, add_class_metadata, overwrite_package_with_name):
m = MessageOfTypes()
primitives = {
extDouble: 123.4,
extString: "string",
NestedExtension.extInt: 4
}
for key, value in primitives.items():
m.Extensions[key] = value
m.Extensions[NestedExtension.extNested].req = "nested"
# Confirm compatibility with JSON serialization
res = json.loads(
json.dumps(
protobuf_to_dict(
m,
add_class_metadata=add_class_metadata,
overwrite_package_with_name=overwrite_package_with_name)))
assert '___X' in res
exts = res['___X']
assert set(exts.keys()) == set(
[str(f.number) for f, _ in m.ListFields() if f.is_extension])
for key, value in primitives.items():
assert exts[str(key.number)] == value
assert exts[str(NestedExtension.extNested.number)]['req'] == 'nested'
deser = dict_to_protobuf(MessageOfTypes, res)
assert deser
for key, value in primitives.items():
assert deser.Extensions[key] == m.Extensions[key]
assert deser.Extensions[NestedExtension.extNested].req == m.Extensions[
NestedExtension.extNested].req
def friends(request, list_type='friends'):
logging.info("djrs.view.friends")
harness = getWebHarness()
if harness.is_connected() is False:
return HttpResponseRedirect(reverse('djrs_login'))
template_vars = {}
acceptable_types = ['all', 'friends', 'connected', 'self']
if list_type not in acceptable_types:
template_vars['error_message'] = 'Invalid Arguments Friends Listing'
return render_to_response('djrs_error.dtml',
template_vars,
context_instance=RequestContext(request))
try:
(req_id, msg_id) = harness.request_peer_list(list_type)
resp = harness.specific_response(req_id)
if resp:
(resp_id, resp_msg) = resp
dict_msg = protobuf_to_dict(resp_msg)
template_vars['friend_list'] = core_personlist(dict_msg['peers'])
except Exception, e:
logging.info("Unexpected Exception: %s" % (e))
def _parse_main_response(self, response_raw, subrequests):
self.log.debug('Parsing main RPC response...')
response_proto = ResponseEnvelope()
try:
response_proto.ParseFromString(response_raw)
except message.DecodeError as e:
raise MalformedNianticResponseException(
'Could not parse response.') from e
self.log.debug('Protobuf structure of rpc response:\n\r%s',
response_proto)
response_proto_dict = protobuf_to_dict(response_proto)
response_proto_dict = self._parse_sub_responses(
subrequests, response_proto_dict)
if not self.state.message8 and 'platform_returns' in response_proto_dict:
for plat_response in response_proto_dict['platform_returns']:
if plat_response['type'] == 8:
try:
resp = PlatEightResponse()
resp.ParseFromString(plat_response['response'])
self.state.message8 = resp.message
except KeyError:
pass
break
if not response_proto_dict:
raise MalformedNianticResponseException(
'Could not convert protobuf to dict.')
return response_proto_dict
def retFeed(feedId):
url = 'http://datamine.mta.info/mta_esi.php?key={}&feed_id={}'.format(
mtaKey, str(feedId))
r = requests.get(url)
feed.ParseFromString(r.content)
f = protobuf_to_dict(feed)
return f
def classify(uri):
client = vision.ImageAnnotatorClient.from_service_account_json(
'./google_credentials.json')
response = client.annotate_image({
'image': {
'source': {
'image_uri': uri
}
},
'features': [{
'type':
vision.enums.Feature.Type.SAFE_SEARCH_DETECTION
}, {
'type': vision.enums.Feature.Type.LABEL_DETECTION
}]
})
if response.error.message:
raise Exception(response.error.message)
raw = protobuf_to_dict(response)
return {
'result': {
'flags': {
**raw['safe_search_annotation'], 'spam':
vision.enums.Likelihood.UNKNOWN
}
},
'raw': raw
}
def get_hyperparameter(path):
"""
Reads the tf.Event files generated by `hp.hparams` in order to retrieve model hyperparameters
Args:
path (str): Path to the `events.out.tfevents.*.v2` file
Returns:
Dict: A dict. with keys given by the names of the hyperparameters and their values
"""
si = summary_iterator(path)
for event in si:
for value in event.summary.value:
proto_bytes = value.metadata.plugin_data.content
plugin_data = plugin_data_pb2.HParamsPluginData.FromString(
proto_bytes)
if plugin_data.HasField("session_start_info"):
hp = plugin_data.session_start_info.hparams
# convert protocol buffer to dict.
hp = {k: list(protobuf_to_dict(hp[k]).values())[0] for k in hp}
return hp
return False
def process_transit(self):
from google.transit import gtfs_realtime_pb2
self.feed = gtfs_realtime_pb2.FeedMessage()
self.feed.ParseFromString(self.transit)
from protobuf_to_dict import protobuf_to_dict
self.transit_dict = protobuf_to_dict(self.feed)
pass
def query_cassandra(start, end, session, device):
upmu_data = []
start_day = int(start / (1000 * 24 * 60 * 60))
end_day = int(end / (1000 * 24 * 60 * 60)) + 1
days = ','.join(str(i) for i in range(start_day, end_day + 1))
query = "SELECT * FROM upmu_data WHERE DEVICE = '{0}' AND DAY IN ({1}) AND timestamp_msec >= {2} AND timestamp_msec < {3}".format(
device, days, start, end)
statement = SimpleStatement(query, fetch_size=100)
for user_row in session.execute(statement):
try:
data = upmuDataProtobuf_pb2.upmuData()
data.ParseFromString(user_row.data)
assert isinstance(data, upmuDataProtobuf_pb2.upmuData)
pyobj = protobuf_to_dict(data)
secs = float(pyobj['timeStamp'])
interval_msec = pyobj['sampleIntervalMsec'] / 1000
for datum in pyobj['sample']:
upmu_data.append(
(secs, datum['C1mag'], datum['C1angle'], datum['C2mag'],
datum['C2angle'], datum['C3mag'], datum['C3angle'],
datum['L1mag'], datum['L1angle'], datum['L2mag'],
datum['L2angle'], datum['L3mag'], datum['L3angle']))
secs += interval_msec
except AttributeError:
pass
return upmu_data
def loads(data, wrapper=dict):
"""
Loads Manifest content into a Python object.
:param data: A byte-like object with the contents of an Appinfo file.
:param wrapper: A wrapping object for key-value pairs.
:return: A dictionary with Manifest data.
"""
if not isinstance(data, (bytes, bytearray)):
raise TypeError(
'can only load a bytes-like object as a Manifest but got ' +
type(data).__name__)
offset = 0
parsed = wrapper()
int32 = struct.Struct('<I')
while True:
msg_id, = int32.unpack_from(data, offset)
offset += int32.size
if msg_id == MSG_EOF:
break
msg_size, = int32.unpack_from(data, offset)
offset += int32.size
msg_data = data[offset:offset + msg_size]
offset += msg_size
message = MessageClass[msg_id]()
message.ParseFromString(msg_data)
parsed[MSG_NAMES[msg_id]] = wrapper(protobuf_to_dict(message))
return parsed
def encode_part(self, obj):
d = {}
d['Content-Type'] = '%s; proto="%s"; serial="%s"' % (self.MIME_TYPE,
obj.DESCRIPTOR.full_name,
self.SERIAL_FORMAT)
d[TRANSFER_ENCODING] = 'inline'
return d, protobuf_to_dict(obj)
def handle_response(self, measurements_response):
""" Handle an Measurements response message.
Args:
measurements_response, a RRC Measurements report message
Returns:
None
"""
measurements_response_dict = protobuf_to_dict(measurements_response)[PRT_UE_RRC_MEASUREMENTS_RESPONSE]
measurements = measurements_response_dict["measurements"]
try:
rnti = measurements_response_dict["rnti"]
ue = self.vbsp.ues[rnti]
except KeyError:
LOG.error(" Unknown UE to VBSP (%s)", (self.vbsp_id))
return
ue.PCell_rsrp = measurements["PCell_rsrp"]
ue.PCell_rsrq = measurements["PCell_rsrq"]
if "meas_result_neigh_cells" in measurements:
meas_result_neigh_cells = measurements["meas_result_neigh_cells"]
for measurement_RAT in meas_result_neigh_cells:
for measurement in meas_result_neigh_cells[measurement_RAT]:
ue.rrc_measurements[measurement["phys_cell_id"]] = {"measId": measurements["measId"],
"RAT_type": measurement_RAT,
"rsrp": measurement["meas_result"]["rsrp"],
"rsrq": measurement["meas_result"]["rsrq"]
}
self.handle_callback(self)
def test_value_in_dict_is_none(self):
m = self.populate_MessageOfTypes()
res = protobuf_to_dict(m)
res['optional_string'] = None
res['nested'] = None
d = dict_to_protobuf(res, MessageOfTypes)
self.assertEqual(d.optional_string, '')
def pb2json_from_file(a_file):
# --------------------------------------------
# Read in pbuf file
# --------------------------------------------
with open(a_file) as l_data_file:
l_pbuf_serialized = l_data_file.read()
l_pbuf = waflz_pb2.config_t()
try:
l_pbuf.ParseFromString(l_pbuf_serialized)
except Exception as e:
g_logger.error(
'Error: performing pbuf parse. type: %s error: %s, doc: %s, message: %s'
% (type(e), e, e.__doc__, e.message))
raise
# By default protobuf-to-dict converts byte arrays into base64 encoded strings,
# but we want them to be actual byte arrays which we will parse and format
# ourselves.
l_type_callable_map = copy(TYPE_CALLABLE_MAP)
l_type_callable_map[FieldDescriptor.TYPE_BYTES] = str
# use_enum_labels causes protobuf_to_dict to use the enum names as strings rather
# than the integer representation of that value
l_dict = protobuf_to_dict(l_pbuf,
type_callable_map=l_type_callable_map,
use_enum_labels=True)
# TODO try/except???
print(json.dumps(l_dict))
def main():
db = resultdb.ResultDatabase()
results = []
for i, result in enumerate(db.GetResults()):
result.eval1.object_precision.Clear()
result.eval1.object_recall.Clear()
result.eval1.confusion_matrix.Clear()
result.eval1.confusion_matrix_item_freq.Clear()
result.eval2.Clear()
results.append(protobuf_to_dict(result))
f = open('lpbench_results.json', 'w')
f.write(json.dumps(results))
f.close()
# configs = [r.config for r in db.GetResults()]
# property_sets = AggregateProtoProperties(configs)
# property_names = []
# color_maps = {}
# for property, values in property_sets.iteritems():
# property_names.append(property)
# #color_map = GenerateColorMap(values)
# #color_maps[property] = color_map
# print property
return 0
def test_basics(self):
m = self.populate_MessageOfTypes()
d = protobuf_to_dict(m)
self.compare(m, d, ['nestedRepeated'])
m2 = dict_to_protobuf(d, MessageOfTypes)
assert m == m2
def get_form_kwargs(self):
kwargs = {"initial": self.get_initial(), "prefix": self.get_prefix()}
submit = ExtensionSubmit()
submit.ParseFromString(self.request.body)
type_callable_map = copy(TYPE_CALLABLE_MAP)
type_callable_map[FieldDescriptor.TYPE_BYTES] = lambda x: "[binary content]"
pb_dict = protobuf_to_dict(submit, type_callable_map=type_callable_map, use_enum_labels=True)
data = dict(
description=submit.common_data.description,
email=submit.common_data.user_email,
page_url=submit.web_data.url,
feedback_data=pb_dict,
ip=get_client_ip(self.request),
)
files = dict()
if submit.screenshot.binary_content:
files["screenshot"] = SimpleUploadedFile("screenshot.png", submit.screenshot.binary_content)
if submit.blackbox.data:
files["blackbox"] = SimpleUploadedFile("blackbox.tar", submit.blackbox.data)
for attach in submit.product_specific_binary_data:
key = "attached_file"
logs_key = "system_logs"
if attach.name == u"system_logs.zip" and logs_key not in files:
key = logs_key
files[key] = SimpleUploadedFile(attach.name, attach.data)
kwargs.update(dict(data=data, files=files))
return kwargs
def _parse_main_request(self, response_raw, subrequests):
self.log.debug('Parsing main RPC response...')
if response_raw.status_code != 200:
self.log.warning('Unexpected HTTP server response - needs 200 got %s', response_raw.status_code)
self.log.debug('HTTP output: \n%s', response_raw.content)
return False
if response_raw.content is None:
self.log.warning('Empty server response!')
return False
response_proto = RpcEnvelope.Response()
try:
response_proto.ParseFromString(response_raw.content)
except DecodeError as e:
self.log.warning('Could not parse response: %s', str(e))
return False
self.log.debug('Protobuf structure of rpc response:\n\r%s', response_proto)
response_proto_dict = protobuf_to_dict(response_proto)
response_proto_dict = self._parse_sub_responses(response_proto, subrequests, response_proto_dict)
return response_proto_dict
def main():
db = resultdb.ResultDatabase()
results = []
for i, result in enumerate(db.GetResults()):
result.eval1.object_precision.Clear()
result.eval1.object_recall.Clear()
result.eval1.confusion_matrix.Clear()
result.eval1.confusion_matrix_item_freq.Clear()
result.eval2.Clear()
results.append(protobuf_to_dict(result))
f = open('lpbench_results.json', 'w')
f.write(json.dumps(results))
f.close()
# configs = [r.config for r in db.GetResults()]
# property_sets = AggregateProtoProperties(configs)
# property_names = []
# color_maps = {}
# for property, values in property_sets.iteritems():
# property_names.append(property)
# #color_map = GenerateColorMap(values)
# #color_maps[property] = color_map
# print property
return 0
def _parse_main_request(self, response_raw, subrequests):
self.log.debug('Parsing main RPC response...')
if response_raw.status_code != 200:
self.log.warning('Unexpected HTTP server response - needs 200 got %s', response_raw.status_code)
self.log.debug('HTTP output: \n%s', response_raw.content)
return False
if response_raw.content is None:
self.log.warning('Empty server response!')
return False
response_proto = RpcEnvelope.Response()
try:
response_proto.ParseFromString(response_raw.content)
except google.protobuf.message.DecodeError as e:
self.log.warning('Could not parse response: %s', str(e))
return False
self.log.debug('Protobuf structure of rpc response:\n\r%s', response_proto)
try:
self.log.debug('Decode raw over protoc (protoc has to be in your PATH):\n\r%s', self.decode_raw(response_raw.content))
except:
print("You don't have protoc installed")
pass
response_proto_dict = protobuf_to_dict(response_proto)
response_proto_dict = self._parse_sub_responses(response_proto, subrequests, response_proto_dict)
return response_proto_dict
def test_basics(self):
m = self.populate_MessageOfTypes()
d = protobuf_to_dict(m)
self.compare(m, d, ['nestedRepeated', 'nestedMap'])
m2 = dict_to_protobuf(MessageOfTypes, d)
assert m == m2
def loads(data, wrapper=dict):
"""
Loads Manifest content into a Python object.
:param data: A byte-like object with the contents of an Appinfo file.
:param wrapper: A wrapping object for key-value pairs.
:return: A dictionary with Manifest data.
"""
if not isinstance(data, (bytes, bytearray)):
raise TypeError('can only load a bytes-like object as a Manifest but got ' + type(data).__name__)
offset = 0
parsed = wrapper()
int32 = struct.Struct('<I')
while True:
msg_id, = int32.unpack_from(data, offset)
offset += int32.size
if msg_id == MSG_EOF:
break
msg_size, = int32.unpack_from(data, offset)
offset += int32.size
msg_data = data[offset:offset + msg_size]
offset += msg_size
message = MessageClass[msg_id]()
message.ParseFromString(msg_data)
parsed[MSG_NAMES[msg_id]] = wrapper(protobuf_to_dict(message))
return parsed
def test_pass_instance(self):
m = self.populate_MessageOfTypes()
d = protobuf_to_dict(m)
d['dubl'] = 1
m2 = dict_to_protobuf(m, d)
assert m is m2
assert m.dubl == 1
def run_service_info(self):
request = rekcurd_pb2.ServiceInfoRequest()
response_protobuf = self.stub.ServiceInfo(request,
metadata=self.__metadata)
response = protobuf_to_dict(response_protobuf,
including_default_value_fields=True)
return response
def _unpack_user_list(self, action):
userList = []
for user in action.contacts.user:
user_data = protobuf_to_dict(user)
if user_data != {} and user_data != self.local_user:
userList.append(user_data)
return userList
def get_form_kwargs(self):
kwargs = {
'initial': self.get_initial(),
'prefix': self.get_prefix(),
}
submit = ExtensionSubmit()
submit.ParseFromString(self.request.body)
type_callable_map = copy(TYPE_CALLABLE_MAP)
type_callable_map[FieldDescriptor.TYPE_BYTES] = lambda x: '[binary content]'
pb_dict = protobuf_to_dict(submit, type_callable_map=type_callable_map, use_enum_labels=True)
data = dict(
description=submit.common_data.description,
email=submit.common_data.user_email,
page_url=submit.web_data.url,
feedback_data=pb_dict,
)
files = dict()
if submit.screenshot.binary_content:
files['screenshot'] = SimpleUploadedFile('screenshot.png', submit.screenshot.binary_content)
if submit.blackbox.data:
files['blackbox'] = SimpleUploadedFile('blackbox.tar', submit.blackbox.data)
for attach in submit.product_specific_binary_data:
key = 'attached_file'
logs_key = 'system_logs'
if attach.name == u'system_logs.zip' and logs_key not in files:
key = logs_key
files[key] = SimpleUploadedFile(attach.name, attach.data)
kwargs.update(dict(data=data, files=files))
return kwargs
def staion_time_lookup(station):
api_key = 'bf83cb9a7f437c52c034339b889bf6e1'
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(
'http://datamine.mta.info/mta_esi.php?key={}&feed_id=26'.format(
api_key))
feed.ParseFromString(response.content)
f.write("Ping API: " + str(time.time()) + "\n")
subway_feed = protobuf_to_dict(feed)
train_data = subway_feed['entity']
collected_times = []
for trains in train_data:
if trains.get('trip_update', False) != False:
unique_train_schedule = trains['trip_update']
unique_arrival_times = unique_train_schedule['stop_time_update']
for scheduled_arrivals in unique_arrival_times:
if scheduled_arrivals.get('stop_id', False) == station:
time_data = scheduled_arrivals['arrival']
unique_time = time_data['time']
if unique_time != None:
collected_times.append(unique_time)
collected_times.sort()
return collected_times
def _handle_ue_rrc_measurements_reply(self, measurements_response):
measurements_response_dict = protobuf_to_dict(
measurements_response)[PRT_UE_RRC_MEASUREMENTS_RESPONSE]
measurements = measurements_response_dict["measurements"]
try:
rnti = measurements_response_dict["rnti"]
ue = self.vbsp.ues[rnti]
except KeyError:
LOG.error(" Unknown UE to VBSP (%s)", (self.vbsp_id))
return
ue.PCell_rsrp = measurements["PCell_rsrp"]
ue.PCell_rsrq = measurements["PCell_rsrq"]
if "meas_result_neigh_cells" in measurements:
meas_result_neigh_cells = measurements["meas_result_neigh_cells"]
for measurement_RAT in meas_result_neigh_cells:
for measurement in meas_result_neigh_cells[measurement_RAT]:
ue.rrc_measurements[measurement["phys_cell_id"]] = {
"measId": measurements["measId"],
"RAT_type": measurement_RAT,
"rsrp": measurement["meas_result"]["rsrp"],
"rsrq": measurement["meas_result"]["rsrq"]
}
def _parse_main_response(self, response_raw, subrequests, use_dict=True):
self.log.debug('Parsing main RPC response...')
if response_raw.status_code == 400:
raise BadRequestException("400: Bad Request")
if response_raw.status_code == 403:
raise NianticIPBannedException(
"Seems your IP Address is banned or something else went badly wrong..."
)
elif response_raw.status_code in (502, 503, 504):
raise NianticOfflineException('{} Server Error'.format(
response_raw.status_code))
elif response_raw.status_code != 200:
error = 'Unexpected HTTP server response - needs 200 got {}'.format(
response_raw.status_code)
self.log.warning(error)
self.log.debug('HTTP output: \n%s',
response_raw.content.decode('utf-8'))
raise UnexpectedResponseException(error)
if not response_raw.content:
self.log.warning('Empty server response!')
raise MalformedNianticResponseException('Empty server response!')
response_proto = ResponseEnvelope()
try:
response_proto.ParseFromString(response_raw.content)
except message.DecodeError as e:
self.log.error('Could not parse response: %s', e)
raise MalformedNianticResponseException(
'Could not decode response.')
self.log.debug('Protobuf structure of rpc response:\n\r%s',
response_proto)
try:
self.log.debug(
'Decode raw over protoc (protoc has to be in your PATH):\n\r%s',
self.decode_raw(response_raw.content).decode('utf-8'))
except Exception:
self.log.debug('Error during protoc parsing - ignored.')
if use_dict:
response_proto_dict = protobuf_to_dict(response_proto)
if 'returns' in response_proto_dict:
del response_proto_dict['returns']
else:
response_proto_dict = {'envelope': response_proto}
if not response_proto_dict:
raise MalformedNianticResponseException(
'Could not convert protobuf to dict.')
response_proto_dict = self._parse_sub_responses(
response_proto, subrequests, response_proto_dict, use_dict)
#It can't be done before
if not use_dict:
del response_proto_dict['envelope'].returns[:]
return response_proto_dict
def test_pass_instance(self):
m = self.populate_MessageOfTypes()
d = protobuf_to_dict(m)
d['dubl'] = 1
m2 = dict_to_protobuf(d, m)
assert m is m2
assert m.dubl == 1
def test_incomplete(self):
m = self.populate_MessageOfTypes()
d = protobuf_to_dict(m)
d.pop('dubl')
m2 = dict_to_protobuf(d, MessageOfTypes)
assert m2.dubl == 0
assert m != m2
def test_incomplete(self):
m = self.populate_MessageOfTypes()
d = protobuf_to_dict(m)
d.pop('dubl')
m2 = dict_to_protobuf(MessageOfTypes, d)
assert m2.dubl == 0
assert m != m2
def _handle_rrc_meas_conf_repl(self, main_msg):
"""Handle an incoming UE's RRC Measurements configuration reply.
Args:
message, a message containing RRC Measurements configuration in UE
Returns:
None
"""
event_type = main_msg.WhichOneof("event_types")
msg = protobuf_to_dict(main_msg)
rrc_m_conf_repl = msg[event_type]["mUE_rrc_meas_conf"]["repl"]
rnti = rrc_m_conf_repl["rnti"]
if rnti not in RUNTIME.ues:
return
ue = self.RUNTIME[rnti]
if rrc_m_conf_repl["status"] != configs_pb2.CREQS_SUCCESS:
return
del rrc_m_conf_repl["rnti"]
del rrc_m_conf_repl["status"]
if "ue_rrc_state" in rrc_m_conf_repl:
ue.rrc_state = rrc_m_conf_repl["ue_rrc_state"]
del rrc_m_conf_repl["ue_rrc_state"]
if "capabilities" in rrc_m_conf_repl:
ue.capabilities = rrc_m_conf_repl["capabilities"]
del rrc_m_conf_repl["capabilities"]
ue.rrc_meas_config = rrc_m_conf_repl
def meas_reply(self, response):
"""Set RRC measurements reply."""
tenant = RUNTIME.tenants[self.tenant_id]
ue_addr = (self.vbs, self.ue)
ue = tenant.ues[ue_addr]
self._meas_reply = protobuf_to_dict(response)
event_type = response.WhichOneof("event_types")
meas = self._meas_reply[event_type]["mRRC_meas"]["repl"]
if "PCell_rsrp" in meas:
ue.pcell_rsrp = meas["PCell_rsrp"]
if "PCell_rsrq" in meas:
ue.pcell_rsrq = meas["PCell_rsrq"]
if "neigh_meas" in meas:
for k in meas["neigh_meas"].keys():
if k == "EUTRA_meas":
# EUTRA measurement result
for m in meas["neigh_meas"][k]:
if m["phys_cell_id"] not in ue.rrc_meas:
self._meas[m["phys_cell_id"]] = {}
ue.rrc_meas[m["phys_cell_id"]] = {}
ue.rrc_meas[m["phys_cell_id"]]["RAT_type"] = "EUTRA"
if "meas_result" in m:
if "rsrp" in m["meas_result"]:
self._meas[m["phys_cell_id"]]["rsrp"] = \
m["meas_result"]["rsrp"]
ue.rrc_meas[m["phys_cell_id"]]["rsrp"] = \
m["meas_result"]["rsrp"]
else:
self._meas[m["phys_cell_id"]]["rsrp"] = -139
ue.rrc_meas[m["phys_cell_id"]]["rsrp"] = -139
if "rsrq" in m["meas_result"]:
self._meas[m["phys_cell_id"]]["rsrq"] = \
m["meas_result"]["rsrq"]
ue.rrc_meas[m["phys_cell_id"]]["rsrq"] = \
m["meas_result"]["rsrq"]
else:
self._meas[m["phys_cell_id"]]["rsrq"] = -19
ue.rrc_meas[m["phys_cell_id"]]["rsrq"] = -19
else:
self._meas[m["phys_cell_id"]]["rsrp"] = -139
self._meas[m["phys_cell_id"]]["rsrq"] = -19
ue.rrc_meas[m["phys_cell_id"]]["rsrp"] = -139
ue.rrc_meas[m["phys_cell_id"]]["rsrq"] = -19
self._meas_reply = meas
def AggregateProtoProperties(protos):
""" Converts a list of protobuffers into a flattened list of property path and unique value sets """
property_sets = {}
for proto in protos:
print str(protobuf_to_dict(proto))
for path_tuple, val in objwalk(protobuf_to_dict(proto)):
path = ''
for element in path_tuple:
if path:
path += '.'
path += str(element)
if path not in property_sets:
property_sets[path] = set()
property_sets[path].add(val)
return property_sets
def test_strict(self):
m = self.populate_MessageOfTypes()
d = protobuf_to_dict(m)
d['meow'] = 1
with nose.tools.assert_raises(KeyError):
m2 = dict_to_protobuf(d, MessageOfTypes)
m2 = dict_to_protobuf(d, MessageOfTypes, strict=False)
assert m == m2
def test_message_with_proto3_map_protobuf_to_dict(self):
m = SomeMessage()
m.some_map['key1'] = 'value1'
m.some_map['key2'] = 'value2'
d = protobuf_to_dict(m)
some_map = d['some_map']
self.assertEqual(some_map['key1'], m.some_map['key1'])
self.assertEqual(some_map['key2'], m.some_map['key2'])
def lambda_handler(event, ctx):
url = event.get('url')
if url:
feed = fetch_feed(url)
return json.dumps(protobuf_to_dict(feed))
else:
return {'error_message': 'Requires the `url` param'}
def test_nested_ignore_none(self):
m = MessageOfTypes()
m.nestedMap['123'].req = '42'
d = protobuf_to_dict(m)
d['nestedMap']['123']['req'] = None
m2 = dict_to_protobuf(MessageOfTypes, d, ignore_none=True)
assert m2.nestedMap['123'].req == ''
def _save_as_json(self, key, raw_log_path, json_usrs_log_path):
"""
save data to file out path as json format
Arguments:
- `key`: boto key object
- `raw_log_path`: raw log path
- `json_usrs_log_path`: json usrs log path
"""
self.logger.info("save_as_json starts")
src_filename = key.name.split("/")[-1]
for file_suffix_excluder in self.file_suffix_excluders:
if src_filename.endswith(file_suffix_excluder):
return ""
pass
elms = src_filename.split("_")
guid = elms[self.user_idx] # guid generated by UUID.random() or index of imei
src_file_path = "%s/%s/%s" % (raw_log_path, guid, src_filename)
# ungzip
fpin = gzip.open(src_file_path, "rb")
content = fpin.read()
fpin.close()
#content = content.decode('utf-8') # TODO: required to supress error?
# deseriazlie
log_datas = LogData_pb2.LogDatas()
log_datas.ParseFromString(content)
# to dict
log_datas_dict = protobuf_to_dict(log_datas)
# to json
log_datas_json = json.dumps(log_datas_dict)
# save
dst_filename = "%s.json" % src_filename.split(".%s" % self.OUT_GZ_EXTENSION)[0]
guid_path = "%s/%s" % (json_usrs_log_path, guid)
dst_file_path = "%s/%s/%s" % (json_usrs_log_path, guid, dst_filename)
if self.user_idx == 0: # imei
imei = log_datas_dict["baseInfo"][1]["value"]
guid_path = "%s/%s" % (json_usrs_log_path, imei)
dst_file_path = "%s/%s/%s" % (json_usrs_log_path, imei, dst_filename)
pass
if not os.path.exists(guid_path):
os.mkdir(guid_path)
self._save(log_datas_json, dst_file_path)
else:
self._save(log_datas_json, dst_file_path)
pass
self.logger.info("save_as_json finished")
return dst_file_path
def pb2dict(pb):
"""
Convert protobuf to a dict of values good for instantiating
loxi objects (or any other objects). We specialize the protobuf_to_dict
library call with our modified decoders.
:param pb: protobuf as loaded into Python
:return: dict of values
"""
return protobuf_to_dict(pb, type_callable_map)
def login_proxy_body_unserialize2(serialize_data):
protobuf_login_proxy = testim_pb_pb2.login_proxy()
#反序列化
protobuf_login_proxy.ParseFromString(serialize_data)
res = protobuf_to_dict(protobuf_login_proxy)
print type(res)
print json.dumps(res).decode('unicode_escape')
def test_nested_repeated(self):
m = self.populate_MessageOfTypes()
m.nestedRepeated.extend([MessageOfTypes.NestedType(req=str(i)) for i in range(10)])
d = protobuf_to_dict(m)
self.compare(m, d, exclude=['nestedRepeated'])
assert d['nestedRepeated'] == [{'req': str(i)} for i in range(10)]
m2 = dict_to_protobuf(d, MessageOfTypes)
assert m == m2
