def __mermaid_str__(self):
results = []
from google.protobuf.json_format import MessageToDict
content = MessageToDict(self._pb_body,
preserving_proto_field_name=True)
_id = f'{self._mermaid_id[:3]}~Document~'
for idx, c in enumerate(self.chunks):
results.append(
f'{_id} --> "{idx + 1}/{len(self.chunks)}" {c._mermaid_id[:3]}~Document~: chunks'
)
results.append(c.__mermaid_str__())
for idx, c in enumerate(self.matches):
results.append(
f'{_id} ..> "{idx + 1}/{len(self.matches)}" {c._mermaid_id[:3]}~Document~: matches'
)
results.append(c.__mermaid_str__())
if 'chunks' in content:
content.pop('chunks')
if 'matches' in content:
content.pop('matches')
if content:
results.append(f'class {_id}{{')
for k, v in content.items():
if isinstance(v, (str, int, float, bytes)):
results.append(f'+{k} {str(v)[:10]}')
else:
results.append(f'+{k}({type(getattr(self, k, v))})')
results.append('}')
return '\n'.join(results)
def diarized_transcribe(gcred, gcs_uri, speakercount):
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = gcred
client = speech_v1p1beta1.SpeechClient()
audio = beta_types.RecognitionAudio(uri=gcs_uri)
config = beta_types.RecognitionConfig(
encoding=beta_enums.RecognitionConfig.AudioEncoding.FLAC,
language_code='en-US',
enable_speaker_diarization=True,
diarization_speaker_count=speakercount,
enable_word_time_offsets=True,
model='video',
enable_automatic_punctuation=True)
operation = client.long_running_recognize(config, audio)
response = operation.result(timeout=3600)
transcript = MessageToDict(response)
transcript = transcript.get('results')
transcript = transcript.pop()
transcript = transcript.get('alternatives')
transcript = transcript.pop()
transcript = transcript.get('words')
return transcript
def _save_loss(self, request, data_type):
task_id = ObjectId(request.task_id) if int(os.getenv(
'SERVER', 1)) else request.task_id
result = MessageToDict(request)
result.pop('secret', None)
loss_method = getattr(self.utils,
f"add_{data_type}_result_to_experiment")
loss_method(db=self.fl_db,
task_id=task_id,
loss=request.loss,
experiment_id=ObjectId(request.experiment_id),
client=request.client)
return task_id, result
def parser_mind(path: bytes, content_type, parser_url, sender):
parse = find_parse_method(parser_url)
data = helper_read_path(path)
url = urlparse(parser_url)
parser_tag = url.scheme if url.scheme else DEFAULT_PARSER_TAG
user_id, = struct.unpack('I', data[:4])
datetime, = struct.unpack('I', data[4:12])
snapshot = Snapshot()
snapshot.ParseFromString(data[12:])
try:
result = getattr(snapshot, parser_tag)
except AttributeError:
return 2
dict_result_fields = MessageToDict(result,
preserving_proto_field_name=True)
if parser_tag == 'color_image':
dict_result_fields['data'] = base64.b64decode(
dict_result_fields.pop('data'))
dict_result = {
parser_tag: dict_result_fields,
'user_id': user_id,
'datetime': snapshot.datetime
}
res = json.dumps(parse(dict_result))
if sender is None:
return res # the caller did not give any where to send, so he will get back the result
return sender(res, 'mind')
def to_dict(self):
claim = MessageToDict(self.claim.message,
preserving_proto_field_name=True)
claim.update(claim.pop(self.claim_type))
if 'languages' in claim:
claim['languages'] = self.langtags
return claim
def _save_model_updates(self, request, weights_stream):
task_id = ObjectId(request.task_id) if int(os.getenv(
'SERVER', 1)) else request.task_id
local_model_path = self.utils.get_clients_response_path(
experiment_id=ObjectId(request.experiment_id),
task_id=task_id,
client=request.client)
result = MessageToDict(request)
result.pop('secret', None)
result['model_update'] = local_model_path
logging.info(f'Parsing Model from {request.client}...')
first_row = next(weights_stream)
result['model_update'] = json.loads(first_row.model_update)
json.dump(result, open(local_model_path, 'w'))
result.pop('model_update', None)
return task_id, result
async def parse_time_series(time_series: TimeSeries):
d = MessageToDict(time_series._pb)
points = d.pop('points', [])
for p in points:
p = parse_point(p)
event = {}
event.update(d)
event.update(p)
yield event
def train_model_response(self, request, context):
"""
This is simply lets the global_server know that the worker finished training.
Marks task as done
"""
if not self.utils.client_is_valid(request.client, request.secret):
return globalserver_pb2.DefaultResponse(
message='Client not known....', ok=False)
logging.info(
f"received train_model_response from {request.client}/{request.experiment_id}"
)
task_id = ObjectId(request.task_id) if int(os.getenv(
'SERVER', 1)) else request.task_id
result = MessageToDict(request)
result.pop('secret', None)
logging.info(f'{request.client} finished training')
return self._finish_up_task_response(request, task_id, result)
def _get_model_parameters(self, request):
if request.model_id != '':
model_parameters_path = self.utils.get_model_parameter_path(
request.model_id)
with open(model_parameters_path,
'r') as reader: # todo load config from db?
model_parameters = reader.read().encode('utf-8')
model_parameters = json.loads(model_parameters)
experiment_id, task_id, result = ObjectId(request.model_id), '', ''
elif request.experiment_id != '':
experiment_id = ObjectId(request.experiment_id)
task_id = '' if request.task_id == '' else ObjectId(
request.task_id)
result = MessageToDict(request)
result.pop('secret', None)
model_parameters = self.utils.load_global_model(
db=self.fl_db, experiment_id=experiment_id)
return model_parameters, experiment_id, task_id, result
def handler_format_mind(result_name, path):
"""
The data is a path to the file that the contains the snapshot (in bytes in the format: user_id + datetime
of the snapshot + the snapshot itself).
Parses it and extracts from it the dict of the current parser (result_name).
"""
with gzip.open(path, 'rb') as reader:
user_id, = struct.unpack('I', reader.read(4))
datetime, = struct.unpack('L', reader.read(8))
snapshot = Snapshot()
snapshot.ParseFromString(reader.read())
result = getattr(snapshot, result_name)
dict_result_fields = MessageToDict(result, preserving_proto_field_name=True)
if result_name == 'color_image':
dict_result_fields['data'] = base64.b64decode(dict_result_fields.pop('data'))
dict_result = {result_name: dict_result_fields, 'user_id': user_id, 'datetime': datetime}
return dict_result
def reader_mind(driver):
"""
A reader that handles data in mind format (that means, as the specifications in mind_pb2.py,
and the format: user size - user - snapshot size - snapshot - snapshot size - snapshot and so on.
:param driver: the driver that gets the data
"""
logger.debug('in reader_mind start')
try:
logger.debug('in reader_mind going to read size')
raw_user_size = driver.read(4)
if raw_user_size == b'':
print("File is empty, Job is done")
yield None
user_size, = struct.unpack('I', raw_user_size)
user = User()
user.ParseFromString(driver.read(user_size))
dict_user = MessageToDict(user,
preserving_proto_field_name=True,
use_integers_for_enums=True)
dict_user['user_id'] = int(dict_user.pop('user_id'))
logger.debug('yielding user')
yield {'data': json.dumps(dict_user), 'content_type': 'mind/user'}
logger.debug('start reading snapshots')
raw_user_id = struct.pack('I', user.user_id)
while True:
logger.debug('in reader_mind_mind going to read snapshot')
raw_snap_size = driver.read(4)
if raw_snap_size == b'':
print("Done reading snapshots")
yield None
snap_size, = struct.unpack('I', raw_snap_size)
raw_snapshot = driver.read(snap_size)
snap = Snapshot()
snap.ParseFromString(raw_snapshot)
logger.debug(f'{snap.datetime=}')
raw_datetime = struct.pack('L', snap.datetime)
yield {
'data': raw_user_id + raw_datetime + raw_snapshot,
'content_type': 'mind/snapshot'
}
logger.debug('going to yield None')
yield None
except TypeError:
raise TypeError("The given file has not the right form")
def list_pending_channels(self) -> List[PendingChannels]:
request = ln.PendingChannelsRequest()
response = self.lnd_client.PendingChannels(request, timeout=1)
pending_channels = []
pending_types = [
'pending_open_channels', 'pending_closing_channels',
'pending_force_closing_channels', 'waiting_close_channels'
]
for pending_type in pending_types:
for pending_channel in getattr(response, pending_type):
channel_dict = MessageToDict(pending_channel)
nested_data = channel_dict.pop('channel')
# noinspection PyDictCreation
flat_dict = {**channel_dict, **nested_data}
flat_dict['pending_type'] = pending_type
pending_channel_model = PendingChannels(**flat_dict)
pending_channels.append(pending_channel_model)
return pending_channels
def read_pb(config_file):
""" Read from protocol buffer file. """
# pylint: disable=import-error
try:
from google.protobuf import text_format
from google.protobuf.json_format import MessageToDict
except ImportError:
raise ImportError('Protocol Buffer library is not installed')
# Read PB file
from . import config_pb2
config_pb = config_pb2.Experiment()
_file = open(config_file, "rb")
text_format.Merge(_file.read(), config_pb)
_file.close()
# Load parameters and return
config = MessageToDict(config_pb)
config['fidel_space'] = config.pop('fidelSpace', {})
return load_parameters(config)
def get_list(self,
page,
sort_field,
sort_desc,
search,
filters,
page_size=None):
response = self.ln.list_pending_channels()
pending_channels = []
pending_types = [
'pending_open_channels', 'pending_closing_channels',
'pending_force_closing_channels', 'waiting_close_channels'
]
for pending_type in pending_types:
for pending_channel in getattr(response, pending_type):
channel_dict = MessageToDict(pending_channel)
nested_data = channel_dict.pop('channel')
flat_dict = {**channel_dict, **nested_data}
flat_dict['pending_type'] = pending_type
pending_channel_model = PendingChannel(**flat_dict)
pending_channels.append(pending_channel_model)
return len(pending_channels), pending_channels
async def play(self, ws, observation):
success = True
request_data = api.Request(
data=api.RequestData(ability_id=True, unit_type_id=True, upgrade_id=True)
)
await asyncio.wait_for(ws.send(request_data.SerializeToString()), 5)
try:
result = await asyncio.wait_for(ws.recv(), 5)
data_response = api.Response.FromString(result)
game_data = data_response.data
request_game_info = api.Request(game_info=api.RequestGameInfo())
await asyncio.wait_for(ws.send(request_game_info.SerializeToString()), 5)
result = await asyncio.wait_for(ws.recv(), 5)
game_info_response = api.Response.FromString(result)
# If game is still on
if game_data.units:
obj = decode_observation(observation.observation.observation, game_data, game_info_response)
obs = MessageToDict(observation)
obs = str(obs)
obs = obs.replace("\'", "\"")
obs = obs.replace("False", "false")
obs = obs.replace("True", "true")
obs = json.loads(obs,encoding="UTF-8")
game_meta = api.Request(
game_info=api.RequestGameInfo()
)
await ws.send(game_meta.SerializeToString())
result = await ws.recv()
game_meta = api.Response.FromString(result)
game_meta = MessageToDict(game_meta)
game_meta = str(game_meta)
game_meta = game_meta.replace("\'", "\"")
game_meta = game_meta.replace("False", "false")
game_meta = game_meta.replace("True", "true")
game_meta = json.loads(game_meta,encoding="UTF-8")
game_meta = game_meta.get("gameInfo", None)
game_meta.pop("modNames")
game_meta.pop("options")
game_meta.pop("mapName")
if("localMapPath" in game_meta.keys()):
game_meta.pop("localMapPath")
game_meta.pop("playerInfo")
game_meta.update(game_meta["startRaw"])
game_meta.pop("startRaw")
game_meta.pop("mapSize")
await self.process_step(ws, obj, raw=(obs, game_meta, game_data))
# function = self.decision_function
# alvailable_actions = self.query_alvailable_actions()
# to_do_action = function(observation, alvailable_actions)
# while(to_do_action and alvailable_actions):
# self.send_order(self, to_do_action)
# to_do_action = self.query_alvailable_actions()
except asyncio.TimeoutError:
return False
return True
class ResourceDefinition:
def __init__(self, resource, resource_type, resource_id):
self.resource = MessageToDict(resource)
self.transforms = transforms.Transforms.from_rd(self.resource)
self.rd = self.resource[resource_type]
self.resource_name = coalesce(self.resource.get('name'),
self.rd.get('name'))
self.resource_desc = coalesce(self.resource.get('description'),
self.rd.get('description'))
self.resource_id = coalesce(resource_id, self.resource.get('id'),
self.rd.get('id'))
if self.resource_id is None:
raise ValueError(f"Illegal resource, missing id: {self.resource}")
self.exportable = True
@property
def path(self) -> Tuple[Optional[str], Optional[str], Optional[str]]:
raise NotImplementedError(self)
def dir_path(self, origin_path: 'ResourcePath', target: str):
'''
when a target path is provided, it should override the root-level
resource id, based on the origin resource path.
'''
return resolve_path(target, origin_path, self.path)
@property
def name(self):
return self.rd.get('name') or self.resource.get('name')
@property
def description(self):
return self.resource.get('description') or self.rd.get(
'description', '')
def dependencies(self):
raise NotImplementedError(self)
def delete_deps(self):
return []
def dependees(self):
return []
def apply_inputs(self, rs: 'ResourceSession'):
pass
@staticmethod
def from_resource_proto(
m: resource_pb2.Resource, override_path=(None, None, None)
) -> 'ResourceDefinition':
if m.version > global_values.API_VERSION:
raise ValueError(f'Version out of date (found {m.version}, '
f'running {global_values.API_VERSION})')
sh.debug(m)
t = m.WhichOneof('type')
typed_resource = getattr(m, t)
if isinstance(typed_resource, resource_pb2.DataService):
resource = DataServiceDef(m, override_path)
elif isinstance(typed_resource, resource_pb2.Dataflow):
resource = DataflowDef(m, override_path)
elif isinstance(typed_resource, resource_pb2.Component):
resource = ComponentDef(m, override_path)
elif isinstance(typed_resource, resource_pb2.Group):
resource = GroupDef(m, override_path)
elif isinstance(typed_resource, resource_pb2.DataFeed):
resource = DataFeedDef(m, override_path)
else:
raise KeyError(f"Unrecognizable resource {m}")
return resource
# need to hit the api to generate the resource(s) for a definition
# garuanteed to have dependencies resolve inside rs maps
def to_resource(self, rs: 'ResourceSession') -> Resource:
json = {}
json['id'] = self.resource_id
json['name'] = self.name
json['description'] = self.description
return self.make_resource(json, rs)
@abc.abstractmethod
def make_resource(self, json: dict, rs: 'ResourceSession') -> Resource:
pass
def list(self):
pass
def apply(self, rs: 'ResourceSession') -> Resource:
res = self.to_resource(rs)
res.apply()
return res
def delete(self):
pass
def all_transforms(self):
return self.transforms + flatten(child.transforms
for child in self.contained())
def dump(self, origin, options):
if not self.exportable:
return
target = self.get_dump_target(origin, options)
self.do_dump(target, options.directory)
def do_dump(self, target: Optional[str], directory):
cleanup = []
self.rd.pop('id', None)
self.resource.pop('id', None)
parent_dir = None
if target is not None:
parent_dir, _ = os.path.split(target)
fns = filter_none([
t.from_api(directory=directory,
parent_directory=parent_dir,
res_def=self) for t in self.transforms
])
if len(fns) > 0:
prefixes, post_fns = zip(*fns)
else:
prefixes, post_fns = [], []
try:
if target is not None:
f = open(target, mode='w')
cleanup.append(f.close)
else:
f = sys.stdout
f.write(f'# id: {self.resource_id}\n')
f.writelines(sorted(prefixes))
yaml_data = yaml.dump(self.resource, default_flow_style=False)
for post_fn in post_fns:
if post_fn is not None:
yaml_data = post_fn(yaml_data)
f.write(yaml_data)
finally:
for c in cleanup:
c()
def contained(self) -> List['ResourceDefinition']:
return []
def get_dump_target(self, origin_path: 'ResourcePath',
options) -> Optional[str]:
if options.directory is None:
return options.output
base_path = options.output
dirs, t = os.path.split(base_path)
if t == "" and origin_path.rd_path[0] is not None:
t = origin_path.resource.resource_id
suff = self.dir_path(origin_path.rd_path, t) + '.yaml'
target = os.path.join(dirs, suff)
sh.debug(f'target for {self.path}: {target}')
os.makedirs(os.path.split(target)[0], exist_ok=True)
return target
def __repr__(self):
return f'<{type(self)}>: {self.resource_id}'
async def game_log_as_json(lobby, uuid):
logging.info("Loading game log")
jsonOutput = {}
req = pb.ReqGameRecord()
req.game_uuid = uuid
req.client_version_string = os.environ.get('MJS_CLIENT_VERSION_STRING')
res = await lobby.fetch_game_record(req)
#head = pb.ResGameRecord()
#head.ParseFromString(res)
record_wrapper = pb.Wrapper()
if res.data_url != "":
async with aiohttp.ClientSession() as session:
headers = {'content-type': 'text/html; charset=UTF-8'}
async with session.get(res.data_url, headers=headers) as response:
bData = await response.read()
record_wrapper.ParseFromString(bData)
else:
record_wrapper.ParseFromString(res.data)
game_details = pb.GameDetailRecords()
game_details.ParseFromString(record_wrapper.data)
if len(game_details.records) != 0:
records = game_details.records
elif len(game_details.actions) != 0:
records = [
action.result for action in game_details.actions
if len(action.result) > 0
]
game_records_count = len(records)
round_record_wrapper = pb.Wrapper()
jsonOutput["Game"] = MessageToDict(res)["head"]
jsonOutput["Game"]["Data_Url"] = res.data_url
jsonOutput["Game"]["Rounds"] = []
round = -1
for i in range(0, game_records_count):
round_record_wrapper.ParseFromString(records[i])
if round_record_wrapper.name == ".lq.RecordNewRound":
round += 1
round_data = pb.RecordNewRound()
round_data.ParseFromString(round_record_wrapper.data)
#jsonStr += json.dumps(MessageToDict(round_data))
#jsonOutput[f"Round{i}"] = MessageToDict(round_data)
jsonOutput["Game"]["Rounds"].append(MessageToDict(round_data))
jsonOutput["Game"]["Rounds"][round]["Tile"] = []
# jsonOutput.update(MessageToDict(round_data))
#jsonParts.append(list(MessageToDict(round_data).items()))
elif round_record_wrapper.name == ".lq.RecordDiscardTile":
discard_tile = pb.RecordDiscardTile()
discard_tile.ParseFromString(round_record_wrapper.data)
newTile = MessageToDict(discard_tile)
newTile["TileType"] = "Discard"
#jsonOutput[f"Discard{i}"] = MessageToDict(discard_tile)
jsonOutput["Game"]["Rounds"][round]["Tile"].append(newTile)
elif round_record_wrapper.name == ".lq.RecordDealTile":
deal_tile = pb.RecordDealTile()
deal_tile.ParseFromString(round_record_wrapper.data)
newTile = MessageToDict(deal_tile)
newTile["TileType"] = "Draw"
#jsonOutput[f"Deal{i}"] = MessageToDict(deal_tile)
jsonOutput["Game"]["Rounds"][round]["Tile"].append(newTile)
elif round_record_wrapper.name == ".lq.RecordChiPengGang":
call_tile = pb.RecordChiPengGang()
call_tile.ParseFromString(round_record_wrapper.data)
newTile = MessageToDict(call_tile)
# if newTile["type"] == 2:
# newTile["TileType"] = "OpenKan"
# else:
newTile["TileType"] = "Call"
jsonOutput["Game"]["Rounds"][round]["Tile"].append(newTile)
elif round_record_wrapper.name == ".lq.RecordBaBei":
call_pei = pb.RecordBaBei()
call_pei.ParseFromString(round_record_wrapper.data)
newTile = MessageToDict(call_pei)
newTile["TileType"] = "Pei"
jsonOutput["Game"]["Rounds"][round]["Tile"].append(newTile)
elif round_record_wrapper.name == ".lq.RecordAnGangAddGang":
add_to_kan = pb.RecordAnGangAddGang()
add_to_kan.ParseFromString(round_record_wrapper.data)
newTile = MessageToDict(add_to_kan)
if newTile["type"] == 2:
newTile["TileType"] = "AddKan"
elif newTile["type"] == 3:
newTile["TileType"] = "AnKan"
newTile["tile"] = newTile["tiles"]
newTile.pop('tiles', None)
newTile.pop('type', None)
jsonOutput["Game"]["Rounds"][round]["Tile"].append(newTile)
elif round_record_wrapper.name == ".lq.RecordHule":
# data = bytearray(round_record_wrapper.data, "utf-8")
round_result = pb.RecordHuleInfo()
# round_result.ParseFromString(round_record_wrapper.data)
# round_result.ParseFromString((round_record_wrapper.data).encode('utf-8').strip())
# jsonOutput[f"RoundResult{i}"] = MessageToDict(round_result)
#else:
# jsonOutput["Game"]["Rounds"][round]["Tile"].append(round_record_wrapper.name)
return jsonOutput
async def load_replay(self, replay_file, id=0):
print(replay_file)
async with websockets.connect("ws://{0}:{1}/sc2api".format(
self.host.address, self.host.port)) as ws:
replay_meta = api.Request(replay_info=api.RequestReplayInfo(
replay_path=replay_file))
await ws.send(replay_meta.SerializeToString())
result = await ws.recv()
metadata = api.Response.FromString(result)
self.replay_info = {
"map":
metadata.replay_info.map_name,
"races": [
metadata.replay_info.player_info[0].player_info.
race_requested,
metadata.replay_info.player_info[1].player_info.
race_requested,
],
"results": [
metadata.replay_info.player_info[0].player_result.result,
metadata.replay_info.player_info[1].player_result.result,
],
}
print(self.replay_info)
msg = api.Request(start_replay=api.RequestStartReplay(
replay_path=replay_file,
observed_player_id=id,
options=api.InterfaceOptions(raw=True, score=False),
))
await ws.send(msg.SerializeToString())
time.sleep(1)
result = await ws.recv()
response = api.Response.FromString(result)
print(response)
game_meta = api.Request(game_info=api.RequestGameInfo())
await ws.send(game_meta.SerializeToString())
result = await ws.recv()
game_meta = api.Response.FromString(result)
game_meta = MessageToDict(game_meta)
game_meta = str(game_meta)
game_meta = game_meta.replace("\'", "\"")
game_meta = game_meta.replace("False", "false")
game_meta = game_meta.replace("True", "true")
game_meta = json.loads(game_meta, encoding="UTF-8")
if "gameInfo" in game_meta.keys():
game_meta = game_meta.get("gameInfo", None)
game_meta.pop("modNames")
game_meta.pop("options")
game_meta.pop("mapName")
if ("localMapPath" in game_meta.keys()):
game_meta.pop("localMapPath")
game_meta.pop("playerInfo")
game_meta.update(game_meta["startRaw"])
game_meta.pop("startRaw")
game_meta.pop("mapSize")
self.game_info = game_meta
self.status = "started"
return True
else:
return False
