def __mermaid_str__(self): results = [] from google.protobuf.json_format import MessageToDict content = MessageToDict(self._pb_body, preserving_proto_field_name=True) _id = f'{self._mermaid_id[:3]}~Document~' for idx, c in enumerate(self.chunks): results.append( f'{_id} --> "{idx + 1}/{len(self.chunks)}" {c._mermaid_id[:3]}~Document~: chunks' ) results.append(c.__mermaid_str__()) for idx, c in enumerate(self.matches): results.append( f'{_id} ..> "{idx + 1}/{len(self.matches)}" {c._mermaid_id[:3]}~Document~: matches' ) results.append(c.__mermaid_str__()) if 'chunks' in content: content.pop('chunks') if 'matches' in content: content.pop('matches') if content: results.append(f'class {_id}{{') for k, v in content.items(): if isinstance(v, (str, int, float, bytes)): results.append(f'+{k} {str(v)[:10]}') else: results.append(f'+{k}({type(getattr(self, k, v))})') results.append('}') return '\n'.join(results)
def diarized_transcribe(gcred, gcs_uri, speakercount): os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = gcred client = speech_v1p1beta1.SpeechClient() audio = beta_types.RecognitionAudio(uri=gcs_uri) config = beta_types.RecognitionConfig( encoding=beta_enums.RecognitionConfig.AudioEncoding.FLAC, language_code='en-US', enable_speaker_diarization=True, diarization_speaker_count=speakercount, enable_word_time_offsets=True, model='video', enable_automatic_punctuation=True) operation = client.long_running_recognize(config, audio) response = operation.result(timeout=3600) transcript = MessageToDict(response) transcript = transcript.get('results') transcript = transcript.pop() transcript = transcript.get('alternatives') transcript = transcript.pop() transcript = transcript.get('words') return transcript
def _save_loss(self, request, data_type): task_id = ObjectId(request.task_id) if int(os.getenv( 'SERVER', 1)) else request.task_id result = MessageToDict(request) result.pop('secret', None) loss_method = getattr(self.utils, f"add_{data_type}_result_to_experiment") loss_method(db=self.fl_db, task_id=task_id, loss=request.loss, experiment_id=ObjectId(request.experiment_id), client=request.client) return task_id, result
def parser_mind(path: bytes, content_type, parser_url, sender): parse = find_parse_method(parser_url) data = helper_read_path(path) url = urlparse(parser_url) parser_tag = url.scheme if url.scheme else DEFAULT_PARSER_TAG user_id, = struct.unpack('I', data[:4]) datetime, = struct.unpack('I', data[4:12]) snapshot = Snapshot() snapshot.ParseFromString(data[12:]) try: result = getattr(snapshot, parser_tag) except AttributeError: return 2 dict_result_fields = MessageToDict(result, preserving_proto_field_name=True) if parser_tag == 'color_image': dict_result_fields['data'] = base64.b64decode( dict_result_fields.pop('data')) dict_result = { parser_tag: dict_result_fields, 'user_id': user_id, 'datetime': snapshot.datetime } res = json.dumps(parse(dict_result)) if sender is None: return res # the caller did not give any where to send, so he will get back the result return sender(res, 'mind')
def to_dict(self): claim = MessageToDict(self.claim.message, preserving_proto_field_name=True) claim.update(claim.pop(self.claim_type)) if 'languages' in claim: claim['languages'] = self.langtags return claim
def _save_model_updates(self, request, weights_stream): task_id = ObjectId(request.task_id) if int(os.getenv( 'SERVER', 1)) else request.task_id local_model_path = self.utils.get_clients_response_path( experiment_id=ObjectId(request.experiment_id), task_id=task_id, client=request.client) result = MessageToDict(request) result.pop('secret', None) result['model_update'] = local_model_path logging.info(f'Parsing Model from {request.client}...') first_row = next(weights_stream) result['model_update'] = json.loads(first_row.model_update) json.dump(result, open(local_model_path, 'w')) result.pop('model_update', None) return task_id, result
async def parse_time_series(time_series: TimeSeries): d = MessageToDict(time_series._pb) points = d.pop('points', []) for p in points: p = parse_point(p) event = {} event.update(d) event.update(p) yield event
def train_model_response(self, request, context): """ This is simply lets the global_server know that the worker finished training. Marks task as done """ if not self.utils.client_is_valid(request.client, request.secret): return globalserver_pb2.DefaultResponse( message='Client not known....', ok=False) logging.info( f"received train_model_response from {request.client}/{request.experiment_id}" ) task_id = ObjectId(request.task_id) if int(os.getenv( 'SERVER', 1)) else request.task_id result = MessageToDict(request) result.pop('secret', None) logging.info(f'{request.client} finished training') return self._finish_up_task_response(request, task_id, result)
def _get_model_parameters(self, request): if request.model_id != '': model_parameters_path = self.utils.get_model_parameter_path( request.model_id) with open(model_parameters_path, 'r') as reader: # todo load config from db? model_parameters = reader.read().encode('utf-8') model_parameters = json.loads(model_parameters) experiment_id, task_id, result = ObjectId(request.model_id), '', '' elif request.experiment_id != '': experiment_id = ObjectId(request.experiment_id) task_id = '' if request.task_id == '' else ObjectId( request.task_id) result = MessageToDict(request) result.pop('secret', None) model_parameters = self.utils.load_global_model( db=self.fl_db, experiment_id=experiment_id) return model_parameters, experiment_id, task_id, result
def handler_format_mind(result_name, path): """ The data is a path to the file that the contains the snapshot (in bytes in the format: user_id + datetime of the snapshot + the snapshot itself). Parses it and extracts from it the dict of the current parser (result_name). """ with gzip.open(path, 'rb') as reader: user_id, = struct.unpack('I', reader.read(4)) datetime, = struct.unpack('L', reader.read(8)) snapshot = Snapshot() snapshot.ParseFromString(reader.read()) result = getattr(snapshot, result_name) dict_result_fields = MessageToDict(result, preserving_proto_field_name=True) if result_name == 'color_image': dict_result_fields['data'] = base64.b64decode(dict_result_fields.pop('data')) dict_result = {result_name: dict_result_fields, 'user_id': user_id, 'datetime': datetime} return dict_result
def reader_mind(driver): """ A reader that handles data in mind format (that means, as the specifications in mind_pb2.py, and the format: user size - user - snapshot size - snapshot - snapshot size - snapshot and so on. :param driver: the driver that gets the data """ logger.debug('in reader_mind start') try: logger.debug('in reader_mind going to read size') raw_user_size = driver.read(4) if raw_user_size == b'': print("File is empty, Job is done") yield None user_size, = struct.unpack('I', raw_user_size) user = User() user.ParseFromString(driver.read(user_size)) dict_user = MessageToDict(user, preserving_proto_field_name=True, use_integers_for_enums=True) dict_user['user_id'] = int(dict_user.pop('user_id')) logger.debug('yielding user') yield {'data': json.dumps(dict_user), 'content_type': 'mind/user'} logger.debug('start reading snapshots') raw_user_id = struct.pack('I', user.user_id) while True: logger.debug('in reader_mind_mind going to read snapshot') raw_snap_size = driver.read(4) if raw_snap_size == b'': print("Done reading snapshots") yield None snap_size, = struct.unpack('I', raw_snap_size) raw_snapshot = driver.read(snap_size) snap = Snapshot() snap.ParseFromString(raw_snapshot) logger.debug(f'{snap.datetime=}') raw_datetime = struct.pack('L', snap.datetime) yield { 'data': raw_user_id + raw_datetime + raw_snapshot, 'content_type': 'mind/snapshot' } logger.debug('going to yield None') yield None except TypeError: raise TypeError("The given file has not the right form")
def list_pending_channels(self) -> List[PendingChannels]: request = ln.PendingChannelsRequest() response = self.lnd_client.PendingChannels(request, timeout=1) pending_channels = [] pending_types = [ 'pending_open_channels', 'pending_closing_channels', 'pending_force_closing_channels', 'waiting_close_channels' ] for pending_type in pending_types: for pending_channel in getattr(response, pending_type): channel_dict = MessageToDict(pending_channel) nested_data = channel_dict.pop('channel') # noinspection PyDictCreation flat_dict = {**channel_dict, **nested_data} flat_dict['pending_type'] = pending_type pending_channel_model = PendingChannels(**flat_dict) pending_channels.append(pending_channel_model) return pending_channels
def read_pb(config_file): """ Read from protocol buffer file. """ # pylint: disable=import-error try: from google.protobuf import text_format from google.protobuf.json_format import MessageToDict except ImportError: raise ImportError('Protocol Buffer library is not installed') # Read PB file from . import config_pb2 config_pb = config_pb2.Experiment() _file = open(config_file, "rb") text_format.Merge(_file.read(), config_pb) _file.close() # Load parameters and return config = MessageToDict(config_pb) config['fidel_space'] = config.pop('fidelSpace', {}) return load_parameters(config)
def get_list(self, page, sort_field, sort_desc, search, filters, page_size=None): response = self.ln.list_pending_channels() pending_channels = [] pending_types = [ 'pending_open_channels', 'pending_closing_channels', 'pending_force_closing_channels', 'waiting_close_channels' ] for pending_type in pending_types: for pending_channel in getattr(response, pending_type): channel_dict = MessageToDict(pending_channel) nested_data = channel_dict.pop('channel') flat_dict = {**channel_dict, **nested_data} flat_dict['pending_type'] = pending_type pending_channel_model = PendingChannel(**flat_dict) pending_channels.append(pending_channel_model) return len(pending_channels), pending_channels
async def play(self, ws, observation): success = True request_data = api.Request( data=api.RequestData(ability_id=True, unit_type_id=True, upgrade_id=True) ) await asyncio.wait_for(ws.send(request_data.SerializeToString()), 5) try: result = await asyncio.wait_for(ws.recv(), 5) data_response = api.Response.FromString(result) game_data = data_response.data request_game_info = api.Request(game_info=api.RequestGameInfo()) await asyncio.wait_for(ws.send(request_game_info.SerializeToString()), 5) result = await asyncio.wait_for(ws.recv(), 5) game_info_response = api.Response.FromString(result) # If game is still on if game_data.units: obj = decode_observation(observation.observation.observation, game_data, game_info_response) obs = MessageToDict(observation) obs = str(obs) obs = obs.replace("\'", "\"") obs = obs.replace("False", "false") obs = obs.replace("True", "true") obs = json.loads(obs,encoding="UTF-8") game_meta = api.Request( game_info=api.RequestGameInfo() ) await ws.send(game_meta.SerializeToString()) result = await ws.recv() game_meta = api.Response.FromString(result) game_meta = MessageToDict(game_meta) game_meta = str(game_meta) game_meta = game_meta.replace("\'", "\"") game_meta = game_meta.replace("False", "false") game_meta = game_meta.replace("True", "true") game_meta = json.loads(game_meta,encoding="UTF-8") game_meta = game_meta.get("gameInfo", None) game_meta.pop("modNames") game_meta.pop("options") game_meta.pop("mapName") if("localMapPath" in game_meta.keys()): game_meta.pop("localMapPath") game_meta.pop("playerInfo") game_meta.update(game_meta["startRaw"]) game_meta.pop("startRaw") game_meta.pop("mapSize") await self.process_step(ws, obj, raw=(obs, game_meta, game_data)) # function = self.decision_function # alvailable_actions = self.query_alvailable_actions() # to_do_action = function(observation, alvailable_actions) # while(to_do_action and alvailable_actions): # self.send_order(self, to_do_action) # to_do_action = self.query_alvailable_actions() except asyncio.TimeoutError: return False return True
class ResourceDefinition: def __init__(self, resource, resource_type, resource_id): self.resource = MessageToDict(resource) self.transforms = transforms.Transforms.from_rd(self.resource) self.rd = self.resource[resource_type] self.resource_name = coalesce(self.resource.get('name'), self.rd.get('name')) self.resource_desc = coalesce(self.resource.get('description'), self.rd.get('description')) self.resource_id = coalesce(resource_id, self.resource.get('id'), self.rd.get('id')) if self.resource_id is None: raise ValueError(f"Illegal resource, missing id: {self.resource}") self.exportable = True @property def path(self) -> Tuple[Optional[str], Optional[str], Optional[str]]: raise NotImplementedError(self) def dir_path(self, origin_path: 'ResourcePath', target: str): ''' when a target path is provided, it should override the root-level resource id, based on the origin resource path. ''' return resolve_path(target, origin_path, self.path) @property def name(self): return self.rd.get('name') or self.resource.get('name') @property def description(self): return self.resource.get('description') or self.rd.get( 'description', '') def dependencies(self): raise NotImplementedError(self) def delete_deps(self): return [] def dependees(self): return [] def apply_inputs(self, rs: 'ResourceSession'): pass @staticmethod def from_resource_proto( m: resource_pb2.Resource, override_path=(None, None, None) ) -> 'ResourceDefinition': if m.version > global_values.API_VERSION: raise ValueError(f'Version out of date (found {m.version}, ' f'running {global_values.API_VERSION})') sh.debug(m) t = m.WhichOneof('type') typed_resource = getattr(m, t) if isinstance(typed_resource, resource_pb2.DataService): resource = DataServiceDef(m, override_path) elif isinstance(typed_resource, resource_pb2.Dataflow): resource = DataflowDef(m, override_path) elif isinstance(typed_resource, resource_pb2.Component): resource = ComponentDef(m, override_path) elif isinstance(typed_resource, resource_pb2.Group): resource = GroupDef(m, override_path) elif isinstance(typed_resource, resource_pb2.DataFeed): resource = DataFeedDef(m, override_path) else: raise KeyError(f"Unrecognizable resource {m}") return resource # need to hit the api to generate the resource(s) for a definition # garuanteed to have dependencies resolve inside rs maps def to_resource(self, rs: 'ResourceSession') -> Resource: json = {} json['id'] = self.resource_id json['name'] = self.name json['description'] = self.description return self.make_resource(json, rs) @abc.abstractmethod def make_resource(self, json: dict, rs: 'ResourceSession') -> Resource: pass def list(self): pass def apply(self, rs: 'ResourceSession') -> Resource: res = self.to_resource(rs) res.apply() return res def delete(self): pass def all_transforms(self): return self.transforms + flatten(child.transforms for child in self.contained()) def dump(self, origin, options): if not self.exportable: return target = self.get_dump_target(origin, options) self.do_dump(target, options.directory) def do_dump(self, target: Optional[str], directory): cleanup = [] self.rd.pop('id', None) self.resource.pop('id', None) parent_dir = None if target is not None: parent_dir, _ = os.path.split(target) fns = filter_none([ t.from_api(directory=directory, parent_directory=parent_dir, res_def=self) for t in self.transforms ]) if len(fns) > 0: prefixes, post_fns = zip(*fns) else: prefixes, post_fns = [], [] try: if target is not None: f = open(target, mode='w') cleanup.append(f.close) else: f = sys.stdout f.write(f'# id: {self.resource_id}\n') f.writelines(sorted(prefixes)) yaml_data = yaml.dump(self.resource, default_flow_style=False) for post_fn in post_fns: if post_fn is not None: yaml_data = post_fn(yaml_data) f.write(yaml_data) finally: for c in cleanup: c() def contained(self) -> List['ResourceDefinition']: return [] def get_dump_target(self, origin_path: 'ResourcePath', options) -> Optional[str]: if options.directory is None: return options.output base_path = options.output dirs, t = os.path.split(base_path) if t == "" and origin_path.rd_path[0] is not None: t = origin_path.resource.resource_id suff = self.dir_path(origin_path.rd_path, t) + '.yaml' target = os.path.join(dirs, suff) sh.debug(f'target for {self.path}: {target}') os.makedirs(os.path.split(target)[0], exist_ok=True) return target def __repr__(self): return f'<{type(self)}>: {self.resource_id}'
async def game_log_as_json(lobby, uuid): logging.info("Loading game log") jsonOutput = {} req = pb.ReqGameRecord() req.game_uuid = uuid req.client_version_string = os.environ.get('MJS_CLIENT_VERSION_STRING') res = await lobby.fetch_game_record(req) #head = pb.ResGameRecord() #head.ParseFromString(res) record_wrapper = pb.Wrapper() if res.data_url != "": async with aiohttp.ClientSession() as session: headers = {'content-type': 'text/html; charset=UTF-8'} async with session.get(res.data_url, headers=headers) as response: bData = await response.read() record_wrapper.ParseFromString(bData) else: record_wrapper.ParseFromString(res.data) game_details = pb.GameDetailRecords() game_details.ParseFromString(record_wrapper.data) if len(game_details.records) != 0: records = game_details.records elif len(game_details.actions) != 0: records = [ action.result for action in game_details.actions if len(action.result) > 0 ] game_records_count = len(records) round_record_wrapper = pb.Wrapper() jsonOutput["Game"] = MessageToDict(res)["head"] jsonOutput["Game"]["Data_Url"] = res.data_url jsonOutput["Game"]["Rounds"] = [] round = -1 for i in range(0, game_records_count): round_record_wrapper.ParseFromString(records[i]) if round_record_wrapper.name == ".lq.RecordNewRound": round += 1 round_data = pb.RecordNewRound() round_data.ParseFromString(round_record_wrapper.data) #jsonStr += json.dumps(MessageToDict(round_data)) #jsonOutput[f"Round{i}"] = MessageToDict(round_data) jsonOutput["Game"]["Rounds"].append(MessageToDict(round_data)) jsonOutput["Game"]["Rounds"][round]["Tile"] = [] # jsonOutput.update(MessageToDict(round_data)) #jsonParts.append(list(MessageToDict(round_data).items())) elif round_record_wrapper.name == ".lq.RecordDiscardTile": discard_tile = pb.RecordDiscardTile() discard_tile.ParseFromString(round_record_wrapper.data) newTile = MessageToDict(discard_tile) newTile["TileType"] = "Discard" #jsonOutput[f"Discard{i}"] = MessageToDict(discard_tile) jsonOutput["Game"]["Rounds"][round]["Tile"].append(newTile) elif round_record_wrapper.name == ".lq.RecordDealTile": deal_tile = pb.RecordDealTile() deal_tile.ParseFromString(round_record_wrapper.data) newTile = MessageToDict(deal_tile) newTile["TileType"] = "Draw" #jsonOutput[f"Deal{i}"] = MessageToDict(deal_tile) jsonOutput["Game"]["Rounds"][round]["Tile"].append(newTile) elif round_record_wrapper.name == ".lq.RecordChiPengGang": call_tile = pb.RecordChiPengGang() call_tile.ParseFromString(round_record_wrapper.data) newTile = MessageToDict(call_tile) # if newTile["type"] == 2: # newTile["TileType"] = "OpenKan" # else: newTile["TileType"] = "Call" jsonOutput["Game"]["Rounds"][round]["Tile"].append(newTile) elif round_record_wrapper.name == ".lq.RecordBaBei": call_pei = pb.RecordBaBei() call_pei.ParseFromString(round_record_wrapper.data) newTile = MessageToDict(call_pei) newTile["TileType"] = "Pei" jsonOutput["Game"]["Rounds"][round]["Tile"].append(newTile) elif round_record_wrapper.name == ".lq.RecordAnGangAddGang": add_to_kan = pb.RecordAnGangAddGang() add_to_kan.ParseFromString(round_record_wrapper.data) newTile = MessageToDict(add_to_kan) if newTile["type"] == 2: newTile["TileType"] = "AddKan" elif newTile["type"] == 3: newTile["TileType"] = "AnKan" newTile["tile"] = newTile["tiles"] newTile.pop('tiles', None) newTile.pop('type', None) jsonOutput["Game"]["Rounds"][round]["Tile"].append(newTile) elif round_record_wrapper.name == ".lq.RecordHule": # data = bytearray(round_record_wrapper.data, "utf-8") round_result = pb.RecordHuleInfo() # round_result.ParseFromString(round_record_wrapper.data) # round_result.ParseFromString((round_record_wrapper.data).encode('utf-8').strip()) # jsonOutput[f"RoundResult{i}"] = MessageToDict(round_result) #else: # jsonOutput["Game"]["Rounds"][round]["Tile"].append(round_record_wrapper.name) return jsonOutput
async def load_replay(self, replay_file, id=0): print(replay_file) async with websockets.connect("ws://{0}:{1}/sc2api".format( self.host.address, self.host.port)) as ws: replay_meta = api.Request(replay_info=api.RequestReplayInfo( replay_path=replay_file)) await ws.send(replay_meta.SerializeToString()) result = await ws.recv() metadata = api.Response.FromString(result) self.replay_info = { "map": metadata.replay_info.map_name, "races": [ metadata.replay_info.player_info[0].player_info. race_requested, metadata.replay_info.player_info[1].player_info. race_requested, ], "results": [ metadata.replay_info.player_info[0].player_result.result, metadata.replay_info.player_info[1].player_result.result, ], } print(self.replay_info) msg = api.Request(start_replay=api.RequestStartReplay( replay_path=replay_file, observed_player_id=id, options=api.InterfaceOptions(raw=True, score=False), )) await ws.send(msg.SerializeToString()) time.sleep(1) result = await ws.recv() response = api.Response.FromString(result) print(response) game_meta = api.Request(game_info=api.RequestGameInfo()) await ws.send(game_meta.SerializeToString()) result = await ws.recv() game_meta = api.Response.FromString(result) game_meta = MessageToDict(game_meta) game_meta = str(game_meta) game_meta = game_meta.replace("\'", "\"") game_meta = game_meta.replace("False", "false") game_meta = game_meta.replace("True", "true") game_meta = json.loads(game_meta, encoding="UTF-8") if "gameInfo" in game_meta.keys(): game_meta = game_meta.get("gameInfo", None) game_meta.pop("modNames") game_meta.pop("options") game_meta.pop("mapName") if ("localMapPath" in game_meta.keys()): game_meta.pop("localMapPath") game_meta.pop("playerInfo") game_meta.update(game_meta["startRaw"]) game_meta.pop("startRaw") game_meta.pop("mapSize") self.game_info = game_meta self.status = "started" return True else: return False