def _handle_registration_acknowledgement(self, message):
"""
The server sends back an acknowledgement. Process that acknowledgement
and send back a ready message to the server.
:param message: An acknowledge register message.
:type message: ServerToSimulator protobuf class
:return: A ready message
:rtype: SimulatorToServer protobuf message
"""
if message.message_type != ServerToSimulator.ACKNOWLEDGE_REGISTER:
error = 'Expected ACKNOWLEDGE_REGISTER but got {}'.format(
MessageToString(message))
raise RuntimeError(error)
if not message.acknowledge_register_data:
error = 'Missing data in ACKNOWLEDGE_REGISTER message {}'.format(
MessageToString(message))
raise RuntimeError(error)
self._state = DriverState.ACTIVE
self._base_protocol.handle_register_acknowledgement(
message.acknowledge_register_data)
# Difference between training and predicting is here... instead of
# sending a READY, send an initial STATE.
reply = SimulatorToServer()
self._simulator_protocol.generate_state_message(reply)
return reply
def command_loop():
periodic.read()
if count[0] % (2 * command_rate_hz) == 0:
logger.info(
'right: %s\nleft: %s',
MessageToString(right_motor.get_state(), as_one_line=True),
MessageToString(left_motor.get_state(), as_one_line=True),
)
right_motor.send_velocity_command(0.0)
right_motor_aft.send_velocity_command(0.0)
left_motor.send_velocity_command(0.0)
left_motor_aft.send_velocity_command(0.0)
count[0] += 1
def ReLU_OPT_Create_Prototxt(original_prototxt_path, original_model_path,
optimized_prototxt_path):
net_param = caffe_pb2.NetParameter()
new_net_param = caffe_pb2.NetParameter()
with open(original_prototxt_path, 'rt') as f:
Parse(f.read(), net_param)
for layer_idx in range(0, len(net_param.layer)):
layer = net_param.layer[layer_idx]
pre_layer = net_param.layer[layer_idx - 1]
if layer.type == 'ReLU' and \
(pre_layer.type == 'Convolution' or pre_layer.type == 'ConvolutionDepthwise' or pre_layer.type == 'DepthwiseConvolution' or \
pre_layer.type == 'InnerProduct' or pre_layer.type == 'Eltwise'):
if pre_layer.type == 'Convolution' or pre_layer.type == 'ConvolutionDepthwise' or pre_layer.type == 'DepthwiseConvolution':
new_net_param.layer[-1].type = 'ConvolutionReLU'
elif pre_layer.type == 'Eltwise':
new_net_param.layer[-1].type = 'EltwiseReLU'
else:
new_net_param.layer[-1].type = 'InnerProductReLU'
if layer.top[0] == layer.bottom[0]:
continue
else:
new_net_param.layer[-1].top[0] = layer.top[0]
else:
new_net_param.layer.extend([layer])
new_net_param.name = net_param.name
with open(optimized_prototxt_path, 'wt') as f:
f.write(MessageToString(new_net_param))
print "ReLU OPT : Create Optimized Prototxt Done."
print bcolors.OKGREEN + "ReLU OPT : Model at " + original_model_path + "." + bcolors.ENDC
print bcolors.OKGREEN + "ReLU OPT : Prototxt at " + optimized_prototxt_path + "." + bcolors.ENDC
def _record(self, send_or_recv, message):
self.recording_queue.put(send_or_recv)
if message:
body = MessageToString(message, as_one_line=True)
else:
body = 'None'
self.recording_queue.put(body)
def _print_single_message(data, unknown_fields):
from pyatv.mrp.protobuf import ProtocolMessage
parsed = ProtocolMessage()
parsed.ParseFromString(data)
output = MessageToString(parsed, print_unknown_fields=unknown_fields)
print(output)
def test_grpc_call(self):
with grpc.insecure_channel('localhost:' + self.port) as channel:
stub = LanguageDetectStub(channel)
request = Input()
request.input = self.sentences
response = stub.infer(request)
print(MessageToString(response))
# print(MessageToDict(response))
print(response.language[0].sentence.encode('utf-8'))
# self.assertEqual(response, '')
request = Input()
request.input = self.sentences_1
response = stub.infer(request)
# print(MessageToDict(response))
print(response)
# self.assertEqual(response, '')
request = Input()
request.input = self.sentences_2
response = stub.infer(request)
# print(MessageToDict(response))
print(response)
# self.assertEqual(response, '')
request = Input()
request.input = self.sentences_3
response = stub.infer(request)
# print(MessageToDict(response))
print(response)
# self.assertEqual(response, '')
request = Input()
request.input = self.sentences_4
response = stub.infer(request)
# print(MessageToDict(response))
print(response)
def AFFine_OPT_Create_Prototxt(original_prototxt_path, optimized_prototxt_path):
net_param = caffe_pb2.NetParameter()
new_net_param = caffe_pb2.NetParameter()
with open(original_prototxt_path, 'rt') as f:
Parse(f.read(), net_param)
layer_num = len(net_param.layer)
parameter_layer_type = ['Convolution', 'InnerProduct']
merge_layer_type = ['Scale', 'BatchNorm']
for layer_idx in range(0, layer_num):
layer = net_param.layer[layer_idx]
if layer.type not in merge_layer_type:
new_net_param.layer.extend([layer])
else:
if layer.type == 'Scale' and len(layer.bottom) != 1:
# In case, scale layer has two bottom blob, then scale layer can't be merged into CONV/IP.
new_net_param.layer.extend([layer])
else:
continue
if layer.type in parameter_layer_type:
if layer_idx+1 < layer_num:
if net_param.layer[layer_idx+1].type in merge_layer_type and len(net_param.layer[layer_idx+1].bottom)==1:
# In case, scale layer has two bottom blob, then scale layer can't be merged into CONV/IP.
if layer.type == 'Convolution':
new_net_param.layer[-1].convolution_param.bias_term = True
else:
new_net_param.layer[-1].inner_product_param.bias_term = True
new_net_param.name = net_param.name
with open(optimized_prototxt_path, 'wt') as f:
f.write(MessageToString(new_net_param))
print "BN SCALE OPT : Create Optimized Prototxt Done."
print bcolors.OKGREEN + "BN SCALE OPT : Prototxt at " + optimized_prototxt_path + "." + bcolors.ENDC
def pbin_dump(path):
name = os.path.basename(path)
TBTypeName = name.split('.')[0]
tb = getattr(res, TBTypeName)()
buff = open(path, "r").read()
tb.ParseFromString(buff)
print(MessageToString(tb))
def _replace_graph_node_names(graph, mapping):
# regex, match all mapped name
all_nodes_regex = re.compile(
_node_name_regex_tpl.format('|'.join(mapping.keys())))
# old graph text
graph_text = MessageToString(graph)
# replace all node name
obfuscated_graph_text = io.StringIO()
last_match_end = 0
while True:
match = all_nodes_regex.search(graph_text, last_match_end)
if match is None:
break
# prefix
match_beg, match_end = match.span('name')
obfuscated_graph_text.write(graph_text[last_match_end:match_beg])
last_match_end = match_end
# node name
node_name = graph_text[match_beg:match_end]
obfuscated_graph_text.write(mapping.get(node_name, node_name))
obfuscated_graph_text.write(graph_text[last_match_end:])
obfuscated_graph = GraphDef()
Parse(obfuscated_graph_text.getvalue(), obfuscated_graph)
obfuscated_graph_text.close()
return obfuscated_graph
def generate_state_message(self, message):
message.message_type = SimulatorToServer.STATE
message.sim_id = self._simulator_id
state = self._simulator.get_state()
if self._current_reward_name:
reward = getattr(self._simulator, self._current_reward_name)()
else:
reward = 0.0
log.debug('generate_state_message => state = %s', pformat(state))
terminal = state.is_terminal
state_message = self._output_schema()
convert_state_to_proto(state_message, state.state)
current_state_data = message.state_data.add()
current_state_data.state = state_message.SerializeToString()
current_state_data.reward = reward
current_state_data.terminal = terminal
# add action taken
last_action = self._simulator.get_last_action()
if last_action is not None:
actions_msg = self._prediction_schema()
convert_state_to_proto(actions_msg, last_action)
current_state_data.action_taken = actions_msg.SerializeToString()
if self._log_state_messages:
log.debug('Generated simulator state %s', MessageToString(message))
def __init__(self, expected, message):
super(UnexpectedMessageError, self).__init__(
'Expected {} but got {}'.format(
expected,
MessageToString(message, as_one_line=True)
)
)
def _record(self, send_or_recv, message):
yield self.recording_queue.put(send_or_recv)
if message:
yield self.recording_queue.put(
MessageToString(message, as_one_line=True))
else:
yield self.recording_queue.put('None')
def Inpt_OPT_New_Weight(original_prototxt_path, original_model_path,
optimized_prototxt_path, new_model_path, scale):
net_param = caffe_pb2.NetParameter()
with open(original_prototxt_path, 'rt') as f:
Parse(f.read(), net_param)
layer_num = len(net_param.layer)
input_layer_type = ['Data', 'Input', 'AnnotatedData']
for layer_idx in range(0, layer_num):
layer = net_param.layer[layer_idx]
if layer.type not in input_layer_type:
assert (layer.type == 'Convolution' or layer.type == 'InnerProduct'
), "## ERROR : First Layer MUST BE CONV or IP. ##"
target_layer_name = layer.name
break
else:
try:
net_param.layer[layer_idx].transform_param.scale = 1.0
except:
print bcolors.WARNING + "INPUT PREPROCESS (SCALE) OPT : ** WARNING ** NO SCALE found in DATA layer." + bcolors.ENDC
new_net = caffe.Net(original_prototxt_path, str(original_model_path),
caffe.TEST)
new_net.params[target_layer_name][0].data[
...] = new_net.params[target_layer_name][0].data[...] * scale
new_net.save(new_model_path)
with open(optimized_prototxt_path, 'wt') as f:
f.write(MessageToString(net_param))
print "INPUT PREPROCESS (SCALE) OPT : Merge Input Scale Done."
print bcolors.OKGREEN + "INPUT PREPROCESS (SCALE) OPT : Model at " + new_model_path + "." + bcolors.ENDC
print bcolors.OKGREEN + "INPUT PREPROCESS (SCALE) OPT : Prototxt at " + optimized_prototxt_path + "." + bcolors.ENDC
def work(self, thread_queue, handle):
logger = get_logger("Worker")
while True:
if self._should_stop:
logger.info("receive stop signal, stopping worker...")
break
try:
ch, method, props, message = thread_queue.get_nowait()
except QueueEmpty:
logger.debug("worker got no work to do")
time.sleep(0.5)
continue
try:
handle(message)
except Exception as e:
# 如果处理消息出错就不会 ACK
if self._message_type is not None:
parsed_message = self._message_type()
parsed_message.ParseFromString(message)
logger.exception(
"handle message %s error %s",
MessageToString(parsed_message, as_one_line=True),
e,
)
else:
logger.exception("handle message %s error %s", message, e)
def _announce_recv(self):
data, address = self._mc_recv_sock.recvfrom(_g_datagram_size)
# Ignore self-announcements
if address[1] == self._mc_send_sock.getsockname()[1]:
return
# Ignore non-local announcements
if not host_is_local(address[0], address[1]):
logger.warning('ignoring non-local announcement: %s:%s',
address[0], address[1])
return
announce = Announce()
announce.ParseFromString(data)
# Ignore faulty announcements
if announce.host != '127.0.0.1' or announce.port != address[1]:
logger.warning(
'announcement does not match sender... rejecting %s',
MessageToString(announce, as_one_line=True))
return
# Store the announcement
announce.recv_stamp.GetCurrentTime()
self._services['%s:%d' % (announce.host, announce.port)] = announce
for q in self._announce_subscribers:
q.put_nowait(announce)
def export_sparse_graph_v4(self, output_v4_dir, emb_dim, space_list, order=1, comm_hint_type='common'):
from xdl.python.training.v4 import dense_input_conf_pb2, embed_dimension_xdl_code_pb2, f2id_pb2
from google.protobuf.text_format import MessageToString
embed_dim_list = embed_dimension_xdl_code_pb2.EmbedDimList()
embed_dim_list.model_signature = str(long(time.mktime(datetime.datetime.utcnow().timetuple()) * 1000000))
if order == 1:
comm_block = embed_dim_list.dense_input.input_blocks.add()
ncomm_block = embed_dim_list.dense_input.input_blocks.add()
else:
ncomm_block = embed_dim_list.dense_input.input_blocks.add()
comm_block = embed_dim_list.dense_input.input_blocks.add()
comm_block.name = 'comm'
ncomm_block.name = 'ncomm'
comm_block.hint_type = dense_input_conf_pb2.UNKNOWN if comm_hint_type == 'unknown' else dense_input_conf_pb2.COMMON
ncomm_block.hint_type = dense_input_conf_pb2.UNCOMMON
comm_index = 0
ncomm_index = 0
for i in xrange(len(graph_tag().sparse_list)):
name = graph_tag().sparse_list[i]
table = graph_tag().fea_dict[name]['table']
embed_dim = embed_dim_list.embed_dim_list.add()
embed_dim.embed_dim = emb_dim
embed_dim.fea_groupid = name
embed_dim.fea_group_global_offset = i + 1
embed_dim.fea_type = "ncommon" if table == 0 else "common"
embed_field = ncomm_block.embed_fields.add() if table == 0 else comm_block.embed_fields.add()
if table == 0:
embed_field.index = ncomm_index
ncomm_index += 1
else:
embed_field.index = comm_index
comm_index += 1
embed_field.dim = emb_dim
embed_field.fea_group_id = name
embed_field.op = dense_input_conf_pb2.KSUM
with open('embed-dim-xdl-code-conf', 'wb') as f: # dense_input_conf.pb
f.write(MessageToString(embed_dim_list))
output(output_v4_dir, 'embed-dim-xdl-code-conf')
f2id_list = f2id_pb2.F2IdList()
for gid in xrange(len(space_list)):
for feature_groupid in space_list[gid]:
f2_id = f2id_list.item.add()
f2_id.feature_groupid = feature_groupid
f2_id.fid = gid
with open('embed.best.meta', 'wb') as f: # f2id.pb
f.write(MessageToString(f2id_list))
output(output_v4_dir, 'embed.best.meta')
def __init__(self, missing_field, message):
super(MalformedMessageError, self).__init__(
'Could not locate {} in message {} - got {}'.format(
missing_field,
type(message).__name__,
MessageToString(message, as_one_line=True)
)
)
async def ipc_recv():
uri = 'ws://localhost:8989'
async with websockets.connect(uri) as websocket:
while True:
msg = await websocket.recv()
event = Event()
event.ParseFromString(msg)
print(MessageToString(event, as_one_line=True))
def Memo_OPT_Inplace_Memory(original_prototxt_path, original_model_path,
optimized_prototxt_path):
inplace_operation_type = [
'Scale', 'BatchNorm', 'ReLU', 'PReLU', 'Softmax', 'TanH', 'ELU',
'Dropout'
]
net_param = caffe_pb2.NetParameter()
with open(original_prototxt_path, 'rt') as f:
Parse(f.read(), net_param)
layer_num = len(net_param.layer)
parameter_blob_name = []
blob_pair = {}
for layer_idx in range(0, layer_num):
layer = net_param.layer[layer_idx]
if layer.type in inplace_operation_type and len(layer.bottom) == 1:
if layer.bottom[0] in parameter_blob_name:
if layer.bottom[0] != layer.top[0]:
# inplace opt
blob_pair[layer.top[0]] = layer.bottom[0]
print "MEMORY In-PLACE OPT : " + layer.name + " : Top Blob [" + layer.top[
0] + "] => [" + layer.bottom[0] + "]"
net_param.layer[layer_idx].top[0] = layer.bottom[0]
else:
# optimized
continue
else:
if blob_pair.has_key(layer.bottom[0]):
# change bottom blob name
blob_pair[layer.top[0]] = blob_pair[layer.bottom[0]]
print "MEMORY In-PLACE OPT : " + layer.name + " : Top Blob [" + layer.top[
0] + "] => [" + blob_pair[layer.bottom[0]] + "]"
print "MEMORY In-PLACE OPT : " + layer.name + " : Bottom Blob [" + layer.bottom[
0] + "] => [" + blob_pair[layer.bottom[0]] + "]"
net_param.layer[layer_idx].top[0] = blob_pair[
layer.bottom[0]]
net_param.layer[layer_idx].bottom[0] = blob_pair[
layer.bottom[0]]
else:
assert (
1 > 2
), "MEMORY In-PLACE OPT : **ERROR** Should Not Reach Here. ##"
else:
for i in range(0, len(layer.top)):
parameter_blob_name.append(layer.top[i])
for i in range(0, len(layer.bottom)):
if blob_pair.has_key(layer.bottom[i]):
print "MEMORY In-PLACE OPT : " + layer.name + " : Bottom Blob [" + layer.bottom[
i] + "] => [" + blob_pair[layer.bottom[i]] + "]"
net_param.layer[layer_idx].bottom[i] = blob_pair[
layer.bottom[i]]
else:
continue
with open(optimized_prototxt_path, 'wt') as f:
f.write(MessageToString(net_param))
# shutil.copyfile(original_model_path, optimized_model_path)
print "MEMORY In-PLACE OPT : In-place Memory Optimization Done."
print bcolors.OKGREEN + "MEMORY In-PLACE OPT : Model at " + original_model_path + "." + bcolors.ENDC
print bcolors.OKGREEN + "MEMORY In-PLACE OPT : Prototxt at " + optimized_prototxt_path + "." + bcolors.ENDC
def handle_register_acknowledgement(self, message):
log.debug('Processing acknowledgement %s', MessageToString(message))
props_schema = message.properties_schema
out_schema = message.output_schema
pred_schema = message.prediction_schema
self._properties_schema = MessageBuilder().reconstitute(props_schema)
self._output_schema = MessageBuilder().reconstitute(out_schema)
self._prediction_schema = MessageBuilder().reconstitute(pred_schema)
def write_proto_to_file(proto, filename, binary=True):
if binary:
f = open(filename, "wb")
f.write(proto.SerializeToString())
f.close()
else:
f = open(filename, "w")
f.write(MessageToString(proto))
f.close()
def log_state(self):
assert not self._recv_raw, 'EventBus initialized with recv_raw, no state.'
if not self._subscriptions:
logger.warning(
f'{self._name} is not subscribed to any eventbus traffic')
logger.info('\n'.join([
MessageToString(value, as_one_line=True)
for value in self._state.values()
]))
def save(sess, saver, global_step, config, save_folder):
"""Snapshots a model."""
if not os.path.isdir(save_folder):
os.makedirs(save_folder)
config_file = os.path.join(save_folder, 'config.prototxt')
with open(config_file, 'w') as f:
f.write(MessageToString(config))
log.info('Saving to {}'.format(save_folder))
saver.save(sess, os.path.join(save_folder, 'model.ckpt'), global_step=global_step)
async def stream_wav_file(self, filename, volume=50):
""" Plays audio using Vector's speakers.
.. testcode::
import anki_vector
with anki_vector.Robot() as robot:
robot.audio.stream_wav_file('../examples/sounds/vector_alert.wav')
:param filename: the filename/path to the .wav audio file
:param volume: the audio playback level (0-100)
"""
# TODO make this support multiple simultaneous sound playback
if self._is_active_event is None:
self._is_active_event = asyncio.Event()
if self._is_active_event.is_set():
raise VectorExternalAudioPlaybackException(
"Cannot start audio when another sound is playing")
if volume < 0 or volume > 100:
raise VectorExternalAudioPlaybackException(
"Volume must be between 0 and 100")
_file_reader, _file_params = self._open_file(filename)
playback_error = None
self._is_active_event.set()
if self._done_event is None:
self._done_event = asyncio.Event()
try:
async for response in self.grpc_interface.ExternalAudioStreamPlayback(
self._request_handler(_file_reader, _file_params, volume)):
self.logger.info("ExternalAudioStream %s",
MessageToString(response, as_one_line=True))
response_type = response.WhichOneof("audio_response_type")
if response_type == 'audio_stream_playback_complete':
playback_error = None
elif response_type == 'audio_stream_buffer_overrun':
playback_error = response_type
elif response_type == 'audio_stream_playback_failyer':
playback_error = response_type
self._done_event.set()
except asyncio.CancelledError:
self.logger.debug('Audio Stream future was cancelled.')
except futures.CancelledError:
self.logger.debug('Audio Stream handler task was cancelled.')
finally:
self._is_active_event = None
self._done_event = None
if playback_error is not None:
raise VectorExternalAudioPlaybackException(
f"Error reported during audio playback {playback_error}")
def convert(path, dstdir):
name = os.path.basename(path)
TBTypeName = name.split('_')[0]
tb = read_table_from_xlsx(TBTypeName, path)
buff = tb.SerializeToString()
pbinf = '%s/%s.pbin' % (dstdir, TBTypeName)
open(pbinf, "w").write(buff)
ptxtf = '%s/%s.txt' % (dstdir, TBTypeName)
open(ptxtf, "w").write(MessageToString(tb))
logging.debug("convert pbin path:%s -> %s,%s ...", path, pbinf, ptxtf)
def log_protobuf(logger, text, message):
"""Log protobuf message and shorten line length."""
if logger.isEnabledFor(logging.DEBUG):
override_length = int(environ.get("PYATV_PROTOBUF_MAX_LINE", 0))
line_length = override_length or _PROTOBUF_LINE_LENGTH
lines = MessageToString(message, print_unknown_fields=True).splitlines()
msg_str = "\n".join([_shorten(x, line_length) for x in lines])
logger.debug("%s: %s", text, msg_str)
def save_config(config, save_folder):
"""Saves configuration to a file."""
if not os.path.isdir(save_folder):
os.makedirs(save_folder)
config_file = os.path.join(save_folder, "config.prototxt")
with open(config_file, "w") as f:
f.write(MessageToString(config))
cmd_file = os.path.join(save_folder, "cmd-{}.txt".format(time.time()))
if not os.path.exists(cmd_file):
with open(cmd_file, "w") as f:
f.write(' '.join(sys.argv))
def _listen_for_services(self, n_periods):
if self._mc_recv_sock is None:
self._quiet_count += 1
if self._quiet_count == 3:
self._connect_recv_sock()
logger.info('Listening for services.')
self._quiet_count = 0
else:
self._close_recv_sock()
logger.info('Resting for services.')
delete = []
for key, service in self._services.items():
if (time.time() - service.recv_stamp.ToSeconds()) > 10:
logger.info('Dropping service: %s', MessageToString(service, as_one_line=True))
delete.append(key)
else:
logger.info('Active service : %s', MessageToString(service, as_one_line=True))
for key in delete:
del self._services[key]
def printMessage(self, status, msg):
self.printStatus(status)
result = (
MessageToJson(msg)
if self.opts.output_json
else MessageToString(
msg,
pointy_brackets=True,
as_utf8=True,
).rstrip()
)
info(result)
def SendQuoteReplace(submitter, attrs):
global responses, soid
quote_id = GetQuoteId(attrs)
buy_qty = raw_input("Buy Qty=> ")
sell_qty = raw_input("Sell Qty=> ")
buy_prc = raw_input("Buy Price=> ")
sell_prc = raw_input("Sell Price=> ")
quote_side_1=[
{'qty':int(buy_qty),'side':1,'price':int(buy_prc)}, #orginal=32860
{'qty':int(sell_qty),'side':2,'price':int(sell_prc)} #orginal=32870
]
attrs = {
'instrument_id':2057683400770929841, #MCHZ7
'quote_side':quote_side_1,
'ord_type':enums.ORD_TYPE_LIMIT,
'time_in_force':enums.TIME_IN_FORCE_DAY,
'market_id':enums.TT_MARKET_ID_HKEX,
'connection_id': connection_id,
'user_id': user_id,
'account_id': account_id,
'account':account,
'quote_id':quote_id,
'source':enums.SOURCE_PYRATE,
'appl_id':PYRATE_APPLICATION_ID,
'aggressor_indicator':True,
'secondary_order_id': soid
}
# Submit a replace quote
msg = dict_to_protobuf(attrs, QuoteReplace)
print '\nSent QuoteCancel {}\n'.format(quote_id, ) # submitter.send_topic)
# print '\nSent QuoteReplace {} on {}\n'.format(quote_id, ) # submitter.send_topic)
print msg
submitter.send(msg)
# Wait for a response
try:
msgs = submitter.wait_for_response(Header.MSG_QUOTE_RESPONSE, wait_topic)
print '\nQuoteResponse for QuoteReplace received:\n'
responses = responses + submitter.order_responses[quote_id]
soid = submitter.order_responses[quote_id][-1].order_response.secondary_order_id
print MessageToString(submitter.order_responses[quote_id][-1].order_response)
return True
except:
print 'Did not receive a QuoteReplace QuoteResponse from the OC, make sure its up!!'
return False
