async def on_message(msg): if msg[0] == ord(self.__class__._msgType['notify']): wrapper = self._addrBook.Wrapper() wrapper.ParseFromString(msg[1:]) notification = reflection.MakeClass( self._addrBook.DESCRIPTOR.message_types_by_name[ wrapper.name[4:]])() notification.ParseFromString(wrapper.data) if wrapper.name[4:] in self._msgPool.keys(): for index in self._msgPool[wrapper.name[4:]]: if index != -1: cb = self._msgPool[wrapper.name[4:]][index] if asyncio.iscoroutinefunction(cb): await cb(protobuf_to_dict(notification)) else: cb(protobuf_to_dict(notification)) elif msg[0] == ord(self.__class__._msgType['res']): # get index index = msg[1] + msg[2] * 256 if index in self._msgPool.keys(): # callback self._wrapper.ParseFromString(msg[3:]) res = self._msgPool[index][0]() res.ParseFromString(self._wrapper.data) if self._msgPool[index][1] is not None: if asyncio.iscoroutinefunction( self._msgPool[index][1]): await self._msgPool[index][1](protobuf_to_dict(res) ) else: self._msgPool[index][1](protobuf_to_dict(res)) self._msgPool.pop(index)
def run_evaluation_data(self, data_path: str, result_path: str): request = rekcurd_pb2.EvaluationResultRequest(data_path=data_path, result_path=result_path) for raw_response in self.stub.EvaluationResult( request, metadata=self.__metadata): details = [] for detail in raw_response.detail: details.append( dict(protobuf_to_dict(detail, including_default_value_fields=True), input=self.__get_value_from_io(detail.input), label=self.__get_value_from_io(detail.label), output=self.__get_value_from_io(detail.output), score=detail.score[0] if len(detail.score) == 1 else list(detail.score))) metrics = raw_response.metrics metrics_response = dict( protobuf_to_dict(metrics, including_default_value_fields=True), label=[self.__get_value_from_io(l) for l in metrics.label]) response = protobuf_to_dict(raw_response, including_default_value_fields=True) response['detail'] = details response['status'] = True response['metrics'] = metrics_response yield response
def conf_reply(self, response): """Set RRC measurements configuration reply.""" self._conf_reply = protobuf_to_dict(response) reply = protobuf_to_dict(response) ue = RUNTIME.tenants[self.tenant_id].ues[self.ue] event_type = response.WhichOneof("event_types") conf = reply[event_type]["mUE_rrc_meas_conf"]["repl"] self._conf_reply = \ self._conf_reply[event_type]["mUE_rrc_meas_conf"]["repl"] if conf["status"] != configs_pb2.CREQS_SUCCESS: return del conf["rnti"] del conf["status"] if "ue_rrc_state" in conf: ue.rrc_state = conf["ue_rrc_state"] del conf["ue_rrc_state"] if "capabilities" in conf: ue.capabilities = conf["capabilities"] del conf["capabilities"] ue.rrc_meas_config = conf
def _make_json_entry(header, item): """ Combine a trace entry header and item into a single JSON-serializable entry. Return this entry as a ``dict``. Some things to note: * The header's ``size`` field is removed - it is not required in the JSON * Enums are replaced by their numerical value (so that they can be written to JSON) """ # If the entry is a fork, then we have to make the child traces # JSON-serializable as well if header.type == TraceEntries_pb2.TRACE_FORK: children = { state_id: _make_json_trace(trace) for state_id, trace in item.children.items() } item = TraceEntryFork(children) header_dict = protobuf_to_dict(header, use_enum_labels=True) entry = header_dict.copy() if isinstance(item, TraceEntryFork): entry.update({'children': item.children}) else: entry.update(protobuf_to_dict(item, use_enum_labels=True)) return entry
def test_message_with_proto3_enum_protobuf_to_dict(self): m = SomeMessage() m.enum_field = 0 d = protobuf_to_dict(m) self.assertEqual(d['enum_field'], 0) m = SomeMessage() m.enum_field = 1 d = protobuf_to_dict(m) self.assertEqual(d['enum_field'], 1)
def test_message_with_proto3_bool_protobuf_to_dict(self): m = SomeMessage() m.bool_field = False d = protobuf_to_dict(m) self.assertEqual(d['bool_field'], False) m = SomeMessage() m.bool_field = True d = protobuf_to_dict(m) self.assertEqual(d['bool_field'], True)
def test_including_default_value_fields(self): m = MessageOfTypes() d = protobuf_to_dict(m) assert d == {} d = protobuf_to_dict(m, including_default_value_fields=True) for field in m.DESCRIPTOR.fields: if field.name != 'nested': assert field.name in d, field.name m2 = dict_to_protobuf(MessageOfTypes, d) assert m == m2
def _make_json_entry(header, item, state): """ Combine a trace entry header and item into a single JSON-serializable entry. Return this entry as a ``dict``. Some things to note: * The header's ``size`` field is removed - it is not required in the JSON * Enums are replaced by their numerical value (so that they can be written to JSON) """ # If the entry is a fork, then we have to make the child traces # JSON-serializable as well if header.type == TraceEntries_pb2.TRACE_FORK: children = {} for state_id, trace in item.children.items(): new_state = state.clone() children[state_id] = _make_json_trace(trace, new_state) item = TraceEntryFork(children) elif header.type == TraceEntries_pb2.TRACE_OSINFO: state.modules.kernel_start = item.kernel_start elif header.type == TraceEntries_pb2.TRACE_MOD_LOAD: state.modules.add(Module(item)) elif header.type == TraceEntries_pb2.TRACE_MOD_UNLOAD: try: state.modules.remove(Module(item)) except Exception: pass header_dict = protobuf_to_dict(header, use_enum_labels=True) entry = header_dict.copy() if isinstance(item, TraceEntryFork): entry.update({'children': item.children}) else: entry.update(protobuf_to_dict(item, use_enum_labels=True)) try: mod = state.modules.get(header.pid, header.pc) rel_pc = mod.to_native(header.pc) if rel_pc: entry.update({'module': {'name': mod.path, 'pc': rel_pc}}) except Exception as e: logger.debug('Error while computing module: %s', e) return entry
def test_use_enum_labels(self, add_class_metadata, overwrite_package_with_name): m = self.populate_MessageOfTypes() d = protobuf_to_dict( m, use_enum_labels=True, add_class_metadata=add_class_metadata, overwrite_package_with_name=overwrite_package_with_name) self.compare(m, d, ['enm', 'enmRepeated', 'nestedRepeated'], add_class_metadata=add_class_metadata, overwrite_package_with_name=overwrite_package_with_name) assert d['enm'] == 'C' assert d['enmRepeated'] == ['A', 'C'] m2 = dict_to_protobuf(MessageOfTypes, d) assert m == m2 d['enm'] = 'MEOW' with nose.tools.assert_raises(KeyError): dict_to_protobuf(MessageOfTypes, d) d['enm'] = 'A' d['enmRepeated'] = ['B'] dict_to_protobuf(MessageOfTypes, d) d['enmRepeated'] = ['CAT'] with nose.tools.assert_raises(KeyError): dict_to_protobuf(MessageOfTypes, d)
def parse_main_request(response_content, response_status, subrequests): log.debug('Parsing main RPC response...') if response_status != 200: log.warning('Unexpected HTTP server response - needs 200 got %s', response_status) log.debug('HTTP output: \n%s', response_content) return False if response_content is None: log.warning('Empty server response!') return False response_proto = RpcEnvelope.Response() try: response_proto.ParseFromString(response_content) except: log.exception('Could not parse response: ') return False log.debug('Protobuf structure of rpc response:\n\r%s', response_proto) response_proto_dict = protobuf_to_dict(response_proto) response_proto_dict = parse_sub_responses(response_proto, subrequests, response_proto_dict) return response_proto_dict
def prepare_response(self, response): resp_dict = protobuf_to_dict(response) # Remove the recipe_ids field resp_dict.pop('recipe_ids', None) resp_dict.pop('test', None) resp_dict.pop('duration', None) if not resp_dict.has_key('recipes'): return {'data': [], 'debug': {'duration': response.duration}} for i, recipe in enumerate(resp_dict['recipes']): resp_dict['recipes'][i]['cook_time'] = recipe['time']['cook'] resp_dict['recipes'][i]['prep_time'] = recipe['time']['prep'] resp_dict['recipes'][i]['ready_time'] = recipe['time']['ready'] for j, ingredient in enumerate(recipe['ingredients']): resp_dict['recipes'][i]['ingredients'][j]['units'] = 'ounce' resp_dict['recipes'][i]['ingredients'][j]['picture'] = 'not-implemented-yet.png' # Replace the ingrids repeated field with a single ID. if ingredient.has_key('ingrids'): resp_dict['recipes'][i]['ingredients'][j]['id'] = ingredient['ingrids'][0] resp_dict.pop('ingrids', None) else: resp_dict['recipes'][i]['ingredients'][j]['id'] = -1 return {'data': resp_dict['recipes'], 'debug': {'duration': response.duration}}
def _parse_main_request(self, response_raw, subrequests): self.log.debug('Parsing main RPC response...') if response_raw.status_code != 200: self.log.warning( 'Unexpected HTTP server response - needs 200 got %s', response_raw.status_code) self.log.debug('HTTP output: \n%s', response_raw.content) return False if response_raw.content is None: self.log.warning('Empty server response!') return False response_proto = RpcEnvelope.Response() try: response_proto.ParseFromString(response_raw.content) except google.protobuf.message.DecodeError as e: self.log.warning('Could not parse response: %s', str(e)) return False self.log.debug('Protobuf structure of rpc response:\n\r%s', response_proto) response_proto_dict = protobuf_to_dict(response_proto) response_proto_dict = self._parse_sub_responses( response_proto, subrequests, response_proto_dict) return response_proto_dict
def get(self): fingerprint = self.get_argument('fingerprint') CHECK_EQ(len(fingerprint), 32, 'invalid fingerprint: %s' % fingerprint) provenance_list = provenance.BuildResourceProvenanceList(fingerprint) provenance_list.sort(key = lambda p : p.start_time_sec) total_job_run_time_sec = 0 prev_end_time_sec = 0 items = [] for p in provenance_list: CHECK_GE(p.start_time_sec, prev_end_time_sec) # our method of adding run times assumes no two jobs overlap... need more complex method otherwise prev_end_time_sec = p.end_time_sec CHECK_GE(p.end_time_sec, p.start_time_sec) # check invariant job_run_time_sec = p.end_time_sec - p.start_time_sec print total_job_run_time_sec item_data = protobuf_to_dict(p) item_data['squeeze_time_start_sec'] = total_job_run_time_sec total_job_run_time_sec += job_run_time_sec item_data['squeeze_time_end_sec'] = total_job_run_time_sec items.append(item_data) provenance_json = json.dumps(items) print provenance_json params = {'provenance_json' : provenance_json } self.render('resource.html', **params) return
def get(self): fingerprint = self.get_argument('fingerprint') CHECK_EQ(len(fingerprint), 32, 'invalid fingerprint: %s' % fingerprint) provenance_list = provenance.BuildResourceProvenanceList(fingerprint) provenance_list.sort(key=lambda p: p.start_time_sec) total_job_run_time_sec = 0 prev_end_time_sec = 0 items = [] for p in provenance_list: CHECK_GE( p.start_time_sec, prev_end_time_sec ) # our method of adding run times assumes no two jobs overlap... need more complex method otherwise prev_end_time_sec = p.end_time_sec CHECK_GE(p.end_time_sec, p.start_time_sec) # check invariant job_run_time_sec = p.end_time_sec - p.start_time_sec print total_job_run_time_sec item_data = protobuf_to_dict(p) item_data['squeeze_time_start_sec'] = total_job_run_time_sec total_job_run_time_sec += job_run_time_sec item_data['squeeze_time_end_sec'] = total_job_run_time_sec items.append(item_data) provenance_json = json.dumps(items) print provenance_json params = {'provenance_json': provenance_json} self.render('resource.html', **params) return
def update(self): """ gets raw data from MTA GTFS-rt API data is updated ~once/min, so no need to call more often than that. must be called before querying arrival times. """ if len(self.api_key) < 40 or len(self.api_key) > 40: print("invalid API key") return feed = gtfs_realtime_pb2.FeedMessage() # gets proper URL based on chosen train route for route_url in urldict: if self.train in route_url: mta_url = urldict[route_url] response = requests.get(mta_url, headers={'x-api-key' : (self.api_key)}) feed.ParseFromString(response.content) mta_raw_output = protobuf_to_dict(feed) print(mta_raw_output) for key in mta_raw_output: if key is 'vehicle': self.vehicle_updates = {val for key, val in mta_raw_output.items() if 'vehicle' in key} self.trip_updates = {val for key, val in mta_raw_output.items() if 'trip_update' in key} print(self.trip_updates) print(self.vehicle_updates)
def trainTimeLookUp( station ): #This function fetches the data from the MTA and only keeps the data for the stations of interest feed = gtfs_realtime_pb2.FeedMessage() response = requests.get( 'http://datamine.mta.info/mta_esi.php?key={}&feed_id={}'.format( api_key, station[1])) feed.ParseFromString(response.content) subway_feed = protobuf_to_dict(feed) realtime_data = subway_feed['entity'] collected_times = [] for trains in realtime_data: if trains.get('trip_update', False) != False: unique_train_schedule = trains['trip_update'] if unique_train_schedule.get('stop_time_update', False) != False: ExpTime = unique_train_schedule.get('stop_time_update') trainInfo = unique_train_schedule['trip'] trainLine = trainInfo['route_id'] for stop in ExpTime: if stop.get('stop_id') == station[0] + 'N': output = stop.get('departure') collected_times.append( [trainLine, 'N', output.get('time')]) elif stop.get('stop_id') == station[0] + 'S': output = stop.get('departure') collected_times.append( [trainLine, 'S', output.get('time')]) return np.asarray(collected_times)
def receive_packet(self): wrapper_packet = SSL_WrapperPacket() try: data = self.connection.recv(2048) except timeout: self.logger.debug('No Vision Frame received.') return try: wrapper_packet.ParseFromString(data) except DecodeError: self.logger.error('VisionReceiver had trouble decoding a packet!') wrapper_packet = protobuf_to_dict(wrapper_packet) detection_packet = wrapper_packet.get('detection', None) if detection_packet: current_time_offset = time.time() - detection_packet['t_capture'] if VisionReceiver.TIME_OFFSET is None: self.logger.debug('Offset time between system is {:.0f} sec.'.format(current_time_offset)) VisionReceiver.TIME_OFFSET = current_time_offset if abs(VisionReceiver.TIME_OFFSET - current_time_offset) > VisionReceiver.MAX_TIME_OFFSET_DIFFERENCE: self.logger.debug('Offset time between system was reset to {:.0f} sec (was {:.0f} sec)'.format(current_time_offset, VisionReceiver.TIME_OFFSET)) self.logger.warning('You might be receiving vision frame from more then one source.') VisionReceiver.TIME_OFFSET = current_time_offset detection_packet['t_capture'] = VisionReceiver.TIME_OFFSET + detection_packet['t_capture'] self._link[detection_packet['camera_id']] = detection_packet
def test_extensions(self, add_class_metadata, overwrite_package_with_name): m = MessageOfTypes() primitives = { extDouble: 123.4, extString: "string", NestedExtension.extInt: 4 } for key, value in primitives.items(): m.Extensions[key] = value m.Extensions[NestedExtension.extNested].req = "nested" # Confirm compatibility with JSON serialization res = json.loads( json.dumps( protobuf_to_dict( m, add_class_metadata=add_class_metadata, overwrite_package_with_name=overwrite_package_with_name))) assert '___X' in res exts = res['___X'] assert set(exts.keys()) == set( [str(f.number) for f, _ in m.ListFields() if f.is_extension]) for key, value in primitives.items(): assert exts[str(key.number)] == value assert exts[str(NestedExtension.extNested.number)]['req'] == 'nested' deser = dict_to_protobuf(MessageOfTypes, res) assert deser for key, value in primitives.items(): assert deser.Extensions[key] == m.Extensions[key] assert deser.Extensions[NestedExtension.extNested].req == m.Extensions[ NestedExtension.extNested].req
def friends(request, list_type='friends'): logging.info("djrs.view.friends") harness = getWebHarness() if harness.is_connected() is False: return HttpResponseRedirect(reverse('djrs_login')) template_vars = {} acceptable_types = ['all', 'friends', 'connected', 'self'] if list_type not in acceptable_types: template_vars['error_message'] = 'Invalid Arguments Friends Listing' return render_to_response('djrs_error.dtml', template_vars, context_instance=RequestContext(request)) try: (req_id, msg_id) = harness.request_peer_list(list_type) resp = harness.specific_response(req_id) if resp: (resp_id, resp_msg) = resp dict_msg = protobuf_to_dict(resp_msg) template_vars['friend_list'] = core_personlist(dict_msg['peers']) except Exception, e: logging.info("Unexpected Exception: %s" % (e))
def _parse_main_response(self, response_raw, subrequests): self.log.debug('Parsing main RPC response...') response_proto = ResponseEnvelope() try: response_proto.ParseFromString(response_raw) except message.DecodeError as e: raise MalformedNianticResponseException( 'Could not parse response.') from e self.log.debug('Protobuf structure of rpc response:\n\r%s', response_proto) response_proto_dict = protobuf_to_dict(response_proto) response_proto_dict = self._parse_sub_responses( subrequests, response_proto_dict) if not self.state.message8 and 'platform_returns' in response_proto_dict: for plat_response in response_proto_dict['platform_returns']: if plat_response['type'] == 8: try: resp = PlatEightResponse() resp.ParseFromString(plat_response['response']) self.state.message8 = resp.message except KeyError: pass break if not response_proto_dict: raise MalformedNianticResponseException( 'Could not convert protobuf to dict.') return response_proto_dict
def retFeed(feedId): url = 'http://datamine.mta.info/mta_esi.php?key={}&feed_id={}'.format( mtaKey, str(feedId)) r = requests.get(url) feed.ParseFromString(r.content) f = protobuf_to_dict(feed) return f
def classify(uri): client = vision.ImageAnnotatorClient.from_service_account_json( './google_credentials.json') response = client.annotate_image({ 'image': { 'source': { 'image_uri': uri } }, 'features': [{ 'type': vision.enums.Feature.Type.SAFE_SEARCH_DETECTION }, { 'type': vision.enums.Feature.Type.LABEL_DETECTION }] }) if response.error.message: raise Exception(response.error.message) raw = protobuf_to_dict(response) return { 'result': { 'flags': { **raw['safe_search_annotation'], 'spam': vision.enums.Likelihood.UNKNOWN } }, 'raw': raw }
def get_hyperparameter(path): """ Reads the tf.Event files generated by `hp.hparams` in order to retrieve model hyperparameters Args: path (str): Path to the `events.out.tfevents.*.v2` file Returns: Dict: A dict. with keys given by the names of the hyperparameters and their values """ si = summary_iterator(path) for event in si: for value in event.summary.value: proto_bytes = value.metadata.plugin_data.content plugin_data = plugin_data_pb2.HParamsPluginData.FromString( proto_bytes) if plugin_data.HasField("session_start_info"): hp = plugin_data.session_start_info.hparams # convert protocol buffer to dict. hp = {k: list(protobuf_to_dict(hp[k]).values())[0] for k in hp} return hp return False
def process_transit(self): from google.transit import gtfs_realtime_pb2 self.feed = gtfs_realtime_pb2.FeedMessage() self.feed.ParseFromString(self.transit) from protobuf_to_dict import protobuf_to_dict self.transit_dict = protobuf_to_dict(self.feed) pass
def query_cassandra(start, end, session, device): upmu_data = [] start_day = int(start / (1000 * 24 * 60 * 60)) end_day = int(end / (1000 * 24 * 60 * 60)) + 1 days = ','.join(str(i) for i in range(start_day, end_day + 1)) query = "SELECT * FROM upmu_data WHERE DEVICE = '{0}' AND DAY IN ({1}) AND timestamp_msec >= {2} AND timestamp_msec < {3}".format( device, days, start, end) statement = SimpleStatement(query, fetch_size=100) for user_row in session.execute(statement): try: data = upmuDataProtobuf_pb2.upmuData() data.ParseFromString(user_row.data) assert isinstance(data, upmuDataProtobuf_pb2.upmuData) pyobj = protobuf_to_dict(data) secs = float(pyobj['timeStamp']) interval_msec = pyobj['sampleIntervalMsec'] / 1000 for datum in pyobj['sample']: upmu_data.append( (secs, datum['C1mag'], datum['C1angle'], datum['C2mag'], datum['C2angle'], datum['C3mag'], datum['C3angle'], datum['L1mag'], datum['L1angle'], datum['L2mag'], datum['L2angle'], datum['L3mag'], datum['L3angle'])) secs += interval_msec except AttributeError: pass return upmu_data
def loads(data, wrapper=dict): """ Loads Manifest content into a Python object. :param data: A byte-like object with the contents of an Appinfo file. :param wrapper: A wrapping object for key-value pairs. :return: A dictionary with Manifest data. """ if not isinstance(data, (bytes, bytearray)): raise TypeError( 'can only load a bytes-like object as a Manifest but got ' + type(data).__name__) offset = 0 parsed = wrapper() int32 = struct.Struct('<I') while True: msg_id, = int32.unpack_from(data, offset) offset += int32.size if msg_id == MSG_EOF: break msg_size, = int32.unpack_from(data, offset) offset += int32.size msg_data = data[offset:offset + msg_size] offset += msg_size message = MessageClass[msg_id]() message.ParseFromString(msg_data) parsed[MSG_NAMES[msg_id]] = wrapper(protobuf_to_dict(message)) return parsed
def encode_part(self, obj): d = {} d['Content-Type'] = '%s; proto="%s"; serial="%s"' % (self.MIME_TYPE, obj.DESCRIPTOR.full_name, self.SERIAL_FORMAT) d[TRANSFER_ENCODING] = 'inline' return d, protobuf_to_dict(obj)
def handle_response(self, measurements_response): """ Handle an Measurements response message. Args: measurements_response, a RRC Measurements report message Returns: None """ measurements_response_dict = protobuf_to_dict(measurements_response)[PRT_UE_RRC_MEASUREMENTS_RESPONSE] measurements = measurements_response_dict["measurements"] try: rnti = measurements_response_dict["rnti"] ue = self.vbsp.ues[rnti] except KeyError: LOG.error(" Unknown UE to VBSP (%s)", (self.vbsp_id)) return ue.PCell_rsrp = measurements["PCell_rsrp"] ue.PCell_rsrq = measurements["PCell_rsrq"] if "meas_result_neigh_cells" in measurements: meas_result_neigh_cells = measurements["meas_result_neigh_cells"] for measurement_RAT in meas_result_neigh_cells: for measurement in meas_result_neigh_cells[measurement_RAT]: ue.rrc_measurements[measurement["phys_cell_id"]] = {"measId": measurements["measId"], "RAT_type": measurement_RAT, "rsrp": measurement["meas_result"]["rsrp"], "rsrq": measurement["meas_result"]["rsrq"] } self.handle_callback(self)
def test_value_in_dict_is_none(self): m = self.populate_MessageOfTypes() res = protobuf_to_dict(m) res['optional_string'] = None res['nested'] = None d = dict_to_protobuf(res, MessageOfTypes) self.assertEqual(d.optional_string, '')
def pb2json_from_file(a_file): # -------------------------------------------- # Read in pbuf file # -------------------------------------------- with open(a_file) as l_data_file: l_pbuf_serialized = l_data_file.read() l_pbuf = waflz_pb2.config_t() try: l_pbuf.ParseFromString(l_pbuf_serialized) except Exception as e: g_logger.error( 'Error: performing pbuf parse. type: %s error: %s, doc: %s, message: %s' % (type(e), e, e.__doc__, e.message)) raise # By default protobuf-to-dict converts byte arrays into base64 encoded strings, # but we want them to be actual byte arrays which we will parse and format # ourselves. l_type_callable_map = copy(TYPE_CALLABLE_MAP) l_type_callable_map[FieldDescriptor.TYPE_BYTES] = str # use_enum_labels causes protobuf_to_dict to use the enum names as strings rather # than the integer representation of that value l_dict = protobuf_to_dict(l_pbuf, type_callable_map=l_type_callable_map, use_enum_labels=True) # TODO try/except??? print(json.dumps(l_dict))
def main(): db = resultdb.ResultDatabase() results = [] for i, result in enumerate(db.GetResults()): result.eval1.object_precision.Clear() result.eval1.object_recall.Clear() result.eval1.confusion_matrix.Clear() result.eval1.confusion_matrix_item_freq.Clear() result.eval2.Clear() results.append(protobuf_to_dict(result)) f = open('lpbench_results.json', 'w') f.write(json.dumps(results)) f.close() # configs = [r.config for r in db.GetResults()] # property_sets = AggregateProtoProperties(configs) # property_names = [] # color_maps = {} # for property, values in property_sets.iteritems(): # property_names.append(property) # #color_map = GenerateColorMap(values) # #color_maps[property] = color_map # print property return 0
def test_basics(self): m = self.populate_MessageOfTypes() d = protobuf_to_dict(m) self.compare(m, d, ['nestedRepeated']) m2 = dict_to_protobuf(d, MessageOfTypes) assert m == m2
def get_form_kwargs(self): kwargs = {"initial": self.get_initial(), "prefix": self.get_prefix()} submit = ExtensionSubmit() submit.ParseFromString(self.request.body) type_callable_map = copy(TYPE_CALLABLE_MAP) type_callable_map[FieldDescriptor.TYPE_BYTES] = lambda x: "[binary content]" pb_dict = protobuf_to_dict(submit, type_callable_map=type_callable_map, use_enum_labels=True) data = dict( description=submit.common_data.description, email=submit.common_data.user_email, page_url=submit.web_data.url, feedback_data=pb_dict, ip=get_client_ip(self.request), ) files = dict() if submit.screenshot.binary_content: files["screenshot"] = SimpleUploadedFile("screenshot.png", submit.screenshot.binary_content) if submit.blackbox.data: files["blackbox"] = SimpleUploadedFile("blackbox.tar", submit.blackbox.data) for attach in submit.product_specific_binary_data: key = "attached_file" logs_key = "system_logs" if attach.name == u"system_logs.zip" and logs_key not in files: key = logs_key files[key] = SimpleUploadedFile(attach.name, attach.data) kwargs.update(dict(data=data, files=files)) return kwargs
def _parse_main_request(self, response_raw, subrequests): self.log.debug('Parsing main RPC response...') if response_raw.status_code != 200: self.log.warning('Unexpected HTTP server response - needs 200 got %s', response_raw.status_code) self.log.debug('HTTP output: \n%s', response_raw.content) return False if response_raw.content is None: self.log.warning('Empty server response!') return False response_proto = RpcEnvelope.Response() try: response_proto.ParseFromString(response_raw.content) except DecodeError as e: self.log.warning('Could not parse response: %s', str(e)) return False self.log.debug('Protobuf structure of rpc response:\n\r%s', response_proto) response_proto_dict = protobuf_to_dict(response_proto) response_proto_dict = self._parse_sub_responses(response_proto, subrequests, response_proto_dict) return response_proto_dict
def _parse_main_request(self, response_raw, subrequests): self.log.debug('Parsing main RPC response...') if response_raw.status_code != 200: self.log.warning('Unexpected HTTP server response - needs 200 got %s', response_raw.status_code) self.log.debug('HTTP output: \n%s', response_raw.content) return False if response_raw.content is None: self.log.warning('Empty server response!') return False response_proto = RpcEnvelope.Response() try: response_proto.ParseFromString(response_raw.content) except google.protobuf.message.DecodeError as e: self.log.warning('Could not parse response: %s', str(e)) return False self.log.debug('Protobuf structure of rpc response:\n\r%s', response_proto) try: self.log.debug('Decode raw over protoc (protoc has to be in your PATH):\n\r%s', self.decode_raw(response_raw.content)) except: print("You don't have protoc installed") pass response_proto_dict = protobuf_to_dict(response_proto) response_proto_dict = self._parse_sub_responses(response_proto, subrequests, response_proto_dict) return response_proto_dict
def test_basics(self): m = self.populate_MessageOfTypes() d = protobuf_to_dict(m) self.compare(m, d, ['nestedRepeated', 'nestedMap']) m2 = dict_to_protobuf(MessageOfTypes, d) assert m == m2
def loads(data, wrapper=dict): """ Loads Manifest content into a Python object. :param data: A byte-like object with the contents of an Appinfo file. :param wrapper: A wrapping object for key-value pairs. :return: A dictionary with Manifest data. """ if not isinstance(data, (bytes, bytearray)): raise TypeError('can only load a bytes-like object as a Manifest but got ' + type(data).__name__) offset = 0 parsed = wrapper() int32 = struct.Struct('<I') while True: msg_id, = int32.unpack_from(data, offset) offset += int32.size if msg_id == MSG_EOF: break msg_size, = int32.unpack_from(data, offset) offset += int32.size msg_data = data[offset:offset + msg_size] offset += msg_size message = MessageClass[msg_id]() message.ParseFromString(msg_data) parsed[MSG_NAMES[msg_id]] = wrapper(protobuf_to_dict(message)) return parsed
def test_pass_instance(self): m = self.populate_MessageOfTypes() d = protobuf_to_dict(m) d['dubl'] = 1 m2 = dict_to_protobuf(m, d) assert m is m2 assert m.dubl == 1
def run_service_info(self): request = rekcurd_pb2.ServiceInfoRequest() response_protobuf = self.stub.ServiceInfo(request, metadata=self.__metadata) response = protobuf_to_dict(response_protobuf, including_default_value_fields=True) return response
def _unpack_user_list(self, action): userList = [] for user in action.contacts.user: user_data = protobuf_to_dict(user) if user_data != {} and user_data != self.local_user: userList.append(user_data) return userList
def get_form_kwargs(self): kwargs = { 'initial': self.get_initial(), 'prefix': self.get_prefix(), } submit = ExtensionSubmit() submit.ParseFromString(self.request.body) type_callable_map = copy(TYPE_CALLABLE_MAP) type_callable_map[FieldDescriptor.TYPE_BYTES] = lambda x: '[binary content]' pb_dict = protobuf_to_dict(submit, type_callable_map=type_callable_map, use_enum_labels=True) data = dict( description=submit.common_data.description, email=submit.common_data.user_email, page_url=submit.web_data.url, feedback_data=pb_dict, ) files = dict() if submit.screenshot.binary_content: files['screenshot'] = SimpleUploadedFile('screenshot.png', submit.screenshot.binary_content) if submit.blackbox.data: files['blackbox'] = SimpleUploadedFile('blackbox.tar', submit.blackbox.data) for attach in submit.product_specific_binary_data: key = 'attached_file' logs_key = 'system_logs' if attach.name == u'system_logs.zip' and logs_key not in files: key = logs_key files[key] = SimpleUploadedFile(attach.name, attach.data) kwargs.update(dict(data=data, files=files)) return kwargs
def staion_time_lookup(station): api_key = 'bf83cb9a7f437c52c034339b889bf6e1' feed = gtfs_realtime_pb2.FeedMessage() response = requests.get( 'http://datamine.mta.info/mta_esi.php?key={}&feed_id=26'.format( api_key)) feed.ParseFromString(response.content) f.write("Ping API: " + str(time.time()) + "\n") subway_feed = protobuf_to_dict(feed) train_data = subway_feed['entity'] collected_times = [] for trains in train_data: if trains.get('trip_update', False) != False: unique_train_schedule = trains['trip_update'] unique_arrival_times = unique_train_schedule['stop_time_update'] for scheduled_arrivals in unique_arrival_times: if scheduled_arrivals.get('stop_id', False) == station: time_data = scheduled_arrivals['arrival'] unique_time = time_data['time'] if unique_time != None: collected_times.append(unique_time) collected_times.sort() return collected_times
def _handle_ue_rrc_measurements_reply(self, measurements_response): measurements_response_dict = protobuf_to_dict( measurements_response)[PRT_UE_RRC_MEASUREMENTS_RESPONSE] measurements = measurements_response_dict["measurements"] try: rnti = measurements_response_dict["rnti"] ue = self.vbsp.ues[rnti] except KeyError: LOG.error(" Unknown UE to VBSP (%s)", (self.vbsp_id)) return ue.PCell_rsrp = measurements["PCell_rsrp"] ue.PCell_rsrq = measurements["PCell_rsrq"] if "meas_result_neigh_cells" in measurements: meas_result_neigh_cells = measurements["meas_result_neigh_cells"] for measurement_RAT in meas_result_neigh_cells: for measurement in meas_result_neigh_cells[measurement_RAT]: ue.rrc_measurements[measurement["phys_cell_id"]] = { "measId": measurements["measId"], "RAT_type": measurement_RAT, "rsrp": measurement["meas_result"]["rsrp"], "rsrq": measurement["meas_result"]["rsrq"] }
def _parse_main_response(self, response_raw, subrequests, use_dict=True): self.log.debug('Parsing main RPC response...') if response_raw.status_code == 400: raise BadRequestException("400: Bad Request") if response_raw.status_code == 403: raise NianticIPBannedException( "Seems your IP Address is banned or something else went badly wrong..." ) elif response_raw.status_code in (502, 503, 504): raise NianticOfflineException('{} Server Error'.format( response_raw.status_code)) elif response_raw.status_code != 200: error = 'Unexpected HTTP server response - needs 200 got {}'.format( response_raw.status_code) self.log.warning(error) self.log.debug('HTTP output: \n%s', response_raw.content.decode('utf-8')) raise UnexpectedResponseException(error) if not response_raw.content: self.log.warning('Empty server response!') raise MalformedNianticResponseException('Empty server response!') response_proto = ResponseEnvelope() try: response_proto.ParseFromString(response_raw.content) except message.DecodeError as e: self.log.error('Could not parse response: %s', e) raise MalformedNianticResponseException( 'Could not decode response.') self.log.debug('Protobuf structure of rpc response:\n\r%s', response_proto) try: self.log.debug( 'Decode raw over protoc (protoc has to be in your PATH):\n\r%s', self.decode_raw(response_raw.content).decode('utf-8')) except Exception: self.log.debug('Error during protoc parsing - ignored.') if use_dict: response_proto_dict = protobuf_to_dict(response_proto) if 'returns' in response_proto_dict: del response_proto_dict['returns'] else: response_proto_dict = {'envelope': response_proto} if not response_proto_dict: raise MalformedNianticResponseException( 'Could not convert protobuf to dict.') response_proto_dict = self._parse_sub_responses( response_proto, subrequests, response_proto_dict, use_dict) #It can't be done before if not use_dict: del response_proto_dict['envelope'].returns[:] return response_proto_dict
def test_pass_instance(self): m = self.populate_MessageOfTypes() d = protobuf_to_dict(m) d['dubl'] = 1 m2 = dict_to_protobuf(d, m) assert m is m2 assert m.dubl == 1
def test_incomplete(self): m = self.populate_MessageOfTypes() d = protobuf_to_dict(m) d.pop('dubl') m2 = dict_to_protobuf(d, MessageOfTypes) assert m2.dubl == 0 assert m != m2
def test_incomplete(self): m = self.populate_MessageOfTypes() d = protobuf_to_dict(m) d.pop('dubl') m2 = dict_to_protobuf(MessageOfTypes, d) assert m2.dubl == 0 assert m != m2
def _handle_rrc_meas_conf_repl(self, main_msg): """Handle an incoming UE's RRC Measurements configuration reply. Args: message, a message containing RRC Measurements configuration in UE Returns: None """ event_type = main_msg.WhichOneof("event_types") msg = protobuf_to_dict(main_msg) rrc_m_conf_repl = msg[event_type]["mUE_rrc_meas_conf"]["repl"] rnti = rrc_m_conf_repl["rnti"] if rnti not in RUNTIME.ues: return ue = self.RUNTIME[rnti] if rrc_m_conf_repl["status"] != configs_pb2.CREQS_SUCCESS: return del rrc_m_conf_repl["rnti"] del rrc_m_conf_repl["status"] if "ue_rrc_state" in rrc_m_conf_repl: ue.rrc_state = rrc_m_conf_repl["ue_rrc_state"] del rrc_m_conf_repl["ue_rrc_state"] if "capabilities" in rrc_m_conf_repl: ue.capabilities = rrc_m_conf_repl["capabilities"] del rrc_m_conf_repl["capabilities"] ue.rrc_meas_config = rrc_m_conf_repl
def meas_reply(self, response): """Set RRC measurements reply.""" tenant = RUNTIME.tenants[self.tenant_id] ue_addr = (self.vbs, self.ue) ue = tenant.ues[ue_addr] self._meas_reply = protobuf_to_dict(response) event_type = response.WhichOneof("event_types") meas = self._meas_reply[event_type]["mRRC_meas"]["repl"] if "PCell_rsrp" in meas: ue.pcell_rsrp = meas["PCell_rsrp"] if "PCell_rsrq" in meas: ue.pcell_rsrq = meas["PCell_rsrq"] if "neigh_meas" in meas: for k in meas["neigh_meas"].keys(): if k == "EUTRA_meas": # EUTRA measurement result for m in meas["neigh_meas"][k]: if m["phys_cell_id"] not in ue.rrc_meas: self._meas[m["phys_cell_id"]] = {} ue.rrc_meas[m["phys_cell_id"]] = {} ue.rrc_meas[m["phys_cell_id"]]["RAT_type"] = "EUTRA" if "meas_result" in m: if "rsrp" in m["meas_result"]: self._meas[m["phys_cell_id"]]["rsrp"] = \ m["meas_result"]["rsrp"] ue.rrc_meas[m["phys_cell_id"]]["rsrp"] = \ m["meas_result"]["rsrp"] else: self._meas[m["phys_cell_id"]]["rsrp"] = -139 ue.rrc_meas[m["phys_cell_id"]]["rsrp"] = -139 if "rsrq" in m["meas_result"]: self._meas[m["phys_cell_id"]]["rsrq"] = \ m["meas_result"]["rsrq"] ue.rrc_meas[m["phys_cell_id"]]["rsrq"] = \ m["meas_result"]["rsrq"] else: self._meas[m["phys_cell_id"]]["rsrq"] = -19 ue.rrc_meas[m["phys_cell_id"]]["rsrq"] = -19 else: self._meas[m["phys_cell_id"]]["rsrp"] = -139 self._meas[m["phys_cell_id"]]["rsrq"] = -19 ue.rrc_meas[m["phys_cell_id"]]["rsrp"] = -139 ue.rrc_meas[m["phys_cell_id"]]["rsrq"] = -19 self._meas_reply = meas
def AggregateProtoProperties(protos): """ Converts a list of protobuffers into a flattened list of property path and unique value sets """ property_sets = {} for proto in protos: print str(protobuf_to_dict(proto)) for path_tuple, val in objwalk(protobuf_to_dict(proto)): path = '' for element in path_tuple: if path: path += '.' path += str(element) if path not in property_sets: property_sets[path] = set() property_sets[path].add(val) return property_sets
def test_strict(self): m = self.populate_MessageOfTypes() d = protobuf_to_dict(m) d['meow'] = 1 with nose.tools.assert_raises(KeyError): m2 = dict_to_protobuf(d, MessageOfTypes) m2 = dict_to_protobuf(d, MessageOfTypes, strict=False) assert m == m2
def test_message_with_proto3_map_protobuf_to_dict(self): m = SomeMessage() m.some_map['key1'] = 'value1' m.some_map['key2'] = 'value2' d = protobuf_to_dict(m) some_map = d['some_map'] self.assertEqual(some_map['key1'], m.some_map['key1']) self.assertEqual(some_map['key2'], m.some_map['key2'])
def lambda_handler(event, ctx): url = event.get('url') if url: feed = fetch_feed(url) return json.dumps(protobuf_to_dict(feed)) else: return {'error_message': 'Requires the `url` param'}
def test_nested_ignore_none(self): m = MessageOfTypes() m.nestedMap['123'].req = '42' d = protobuf_to_dict(m) d['nestedMap']['123']['req'] = None m2 = dict_to_protobuf(MessageOfTypes, d, ignore_none=True) assert m2.nestedMap['123'].req == ''
def _save_as_json(self, key, raw_log_path, json_usrs_log_path): """ save data to file out path as json format Arguments: - `key`: boto key object - `raw_log_path`: raw log path - `json_usrs_log_path`: json usrs log path """ self.logger.info("save_as_json starts") src_filename = key.name.split("/")[-1] for file_suffix_excluder in self.file_suffix_excluders: if src_filename.endswith(file_suffix_excluder): return "" pass elms = src_filename.split("_") guid = elms[self.user_idx] # guid generated by UUID.random() or index of imei src_file_path = "%s/%s/%s" % (raw_log_path, guid, src_filename) # ungzip fpin = gzip.open(src_file_path, "rb") content = fpin.read() fpin.close() #content = content.decode('utf-8') # TODO: required to supress error? # deseriazlie log_datas = LogData_pb2.LogDatas() log_datas.ParseFromString(content) # to dict log_datas_dict = protobuf_to_dict(log_datas) # to json log_datas_json = json.dumps(log_datas_dict) # save dst_filename = "%s.json" % src_filename.split(".%s" % self.OUT_GZ_EXTENSION)[0] guid_path = "%s/%s" % (json_usrs_log_path, guid) dst_file_path = "%s/%s/%s" % (json_usrs_log_path, guid, dst_filename) if self.user_idx == 0: # imei imei = log_datas_dict["baseInfo"][1]["value"] guid_path = "%s/%s" % (json_usrs_log_path, imei) dst_file_path = "%s/%s/%s" % (json_usrs_log_path, imei, dst_filename) pass if not os.path.exists(guid_path): os.mkdir(guid_path) self._save(log_datas_json, dst_file_path) else: self._save(log_datas_json, dst_file_path) pass self.logger.info("save_as_json finished") return dst_file_path
def pb2dict(pb): """ Convert protobuf to a dict of values good for instantiating loxi objects (or any other objects). We specialize the protobuf_to_dict library call with our modified decoders. :param pb: protobuf as loaded into Python :return: dict of values """ return protobuf_to_dict(pb, type_callable_map)
def login_proxy_body_unserialize2(serialize_data): protobuf_login_proxy = testim_pb_pb2.login_proxy() #反序列化 protobuf_login_proxy.ParseFromString(serialize_data) res = protobuf_to_dict(protobuf_login_proxy) print type(res) print json.dumps(res).decode('unicode_escape')
def test_nested_repeated(self): m = self.populate_MessageOfTypes() m.nestedRepeated.extend([MessageOfTypes.NestedType(req=str(i)) for i in range(10)]) d = protobuf_to_dict(m) self.compare(m, d, exclude=['nestedRepeated']) assert d['nestedRepeated'] == [{'req': str(i)} for i in range(10)] m2 = dict_to_protobuf(d, MessageOfTypes) assert m == m2