def start_grpc(self): channel = implementations.insecure_channel(self.ip, self.port) stub = witness_pb2.beta_create_WitnessService_stub(channel) print "hello" while True: req = witness_pb2.WitnessBatchRequest() Merge( 'Context{\nSessionId:"grpc_test_123"\n Type:1 \n Functions:%s \n}\n' % (self.function), req) for batch in range(0, self.batchsize): message = self.queue.get() image = 'Images {\n Data { \n URI:"%s" \n} \n}' % (message) Merge(image, req) resp = stub.BatchRecognize(req, self.timeout) print resp lock.acquire() global total total = total + self.batchsize log_file = open('./grpc.log', 'a') log_file.write( '%s total send pictures is %d, number of vehicle is %d , status: %s\n' % (time.strftime("%Y-%m-%d %X", time.localtime()), total, len(resp.Results[0].Vehicles), resp.Context.Status)) log_file.close() lock.release() self.queue.task_done()
def test_calibration_heatmap(): calibration = cg.Calibration(_CALIBRATION_DATA) heatmap = calibration.heatmap('t1') figure = mpl.figure.Figure() axes = figure.add_subplot(111) heatmap.plot(axes) assert axes.get_title() == 'T1' heatmap = calibration.heatmap('two_qubit_xeb') figure = mpl.figure.Figure() axes = figure.add_subplot(999) heatmap.plot(axes) assert axes.get_title() == 'Two Qubit Xeb' with pytest.raises(ValueError, match="one or two qubits.*multi_qubit"): multi_qubit_data = Merge( """metrics: [{ name: 'multi_qubit', targets: ['0_0', '0_1', '1_0'], values: [{double_val: 0.999}]}]""", v2.metrics_pb2.MetricsSnapshot(), ) cg.Calibration(multi_qubit_data).heatmap('multi_qubit') with pytest.raises(ValueError, match="single metric values.*multi_value"): multi_qubit_data = Merge( """metrics: [{ name: 'multi_value', targets: ['0_0'], values: [{double_val: 0.999}, {double_val: 0.001}]}]""", v2.metrics_pb2.MetricsSnapshot(), ) cg.Calibration(multi_qubit_data).heatmap('multi_value')
def test_proto(prototxt): import sys sys.path.append('../proto') from berry_pb2 import NetParameter with open(prototxt, 'r') as f: txt = f.read() net = Merge(txt, NetParameter()) print net.ListFields()
def read_prototxt(ptxt_path, message): """Takes a path to a ``.prototxt`` file and a protobuf message and merges the two.""" with open(ptxt_path) as fp: ptxt_contents = fp.read() return Merge(ptxt_contents, message)
def load_prototxt_def(protofile): import caffe.proto.caffe_pb2 as caffe_pb2 from google.protobuf.text_format import Merge net = caffe_pb2.NetParameter() with open(protofile, 'r') as f: Merge(f.read(), net) return net
def _get_config(): """Gets config.""" config_file = FLAGS.config config = ResnetModelConfig() assert config_file is not None, 'Must pass in a configuration file through --config' Merge(open(config_file).read(), config) return config
def convert_prototxt_to_json(path_to_prototxt, path_to_json): net = znet_caffe_pb2.NetParameter() Merge((open(path_to_prototxt, 'r')).read(), net) with open(path_to_json, 'w', encoding="utf8") as f: net_dict = protobuf_to_dict(net) json.dump(net_dict, f, indent=3)
def test_get_calibration(get_calibration): qjob = qtypes.QuantumJob(execution_status=qtypes.ExecutionStatus( calibration_name='projects/a/processors/p/calibrations/123')) calibration = qtypes.QuantumCalibration(data=_to_any( Merge( """ timestamp_ms: 123000, metrics: [{ name: 'xeb', targets: ['0_0', '0_1'], values: [{ double_val: .9999 }] }, { name: 't1', targets: ['0_0'], values: [{ double_val: 321 }] }, { name: 'globalMetric', values: [{ int32_val: 12300 }] }] """, v2.metrics_pb2.MetricsSnapshot()))) get_calibration.return_value = calibration job = cg.EngineJob('a', 'b', 'steve', EngineContext(), _job=qjob) assert list(job.get_calibration()) == ['xeb', 't1', 'globalMetric'] get_calibration.assert_called_once_with('a', 'p', 123)
def get_solver(args): from caffe.proto.caffe_pb2 import SolverParameter from google.protobuf.text_format import Merge solver = SolverParameter() Merge(open(args.solver, 'r').read(), solver) return solver
def main(): args = parse_args() if not args.verbose: os.environ['GLOG_minloglevel'] = '2' import caffe from caffe.proto import caffe_pb2 if args.cpu: caffe.set_mode_cpu() else: caffe.set_mode_gpu() if args.verbose: if args.cpu: print("CPU mode set.") else: print("GPU mode set.") forwardErrorSum = 0.0 diffErrorSum = 0.0 weightDiffErrorSum = 0.0 for i in range(args.tests_number): tmpNetProto = tempfile.NamedTemporaryFile() tmpNetProto.write(createConvolutionsNet(args.params, args.update_weight_diff, args.scale_weight_diff)) tmpNetProto.flush() net = caffe.Net(tmpNetProto.name, caffe.TEST) deploy = caffe_pb2.NetParameter() Merge((open(tmpNetProto.name, 'r').read()), deploy) tmpNetProto.close() sys.stdout.write("{}. ".format(i + 1)) if not args.verbose: sys.stdout.write("Input shape: {},{},{} ".format(net.blobs['convolution_input'].data.shape[2], net.blobs['convolution_input'].data.shape[3], net.blobs['convolution_input'].data.shape[1])) convParams = deploy.layer[2].convolution_param sys.stdout.write("Conv params: {},{},{},{} ".format(convParams.num_output, convParams.kernel_size[0], convParams.stride[0], convParams.pad[0])) sys.stdout.write("Output shape: {},{},{}\n".format(net.blobs['convolution'].data.shape[2], net.blobs['convolution'].data.shape[3], net.blobs['convolution'].data.shape[1])) forwardError, diffError, weightDiffError = testBinaryConvolutionLayer(net, deploy, args) forwardErrorSum += forwardError diffErrorSum += diffError weightDiffErrorSum += weightDiffError meanForwardError = forwardErrorSum / args.tests_number meanDiffError = diffErrorSum / args.tests_number meanWeightDiffError = weightDiffErrorSum / args.tests_number print ("\n#############################################################") print ("Number of tests: {}\n".format(args.tests_number)) print ("Mean forward error: {}".format(meanForwardError)) print ("Mean diff error: {}".format(meanDiffError)) print ("Mean weight diff error: {}".format(meanWeightDiffError)) print ("#############################################################")
def readCaffeModelFile(txt_path='', binary_path=''): net = caffe.Net(txt_path, binary_path, caffe.TEST) result = [] net_proto = caffe_proto.NetParameter() with open(txt_path, 'r') as f: Merge(f.read(), net_proto) dimension = net.blobs['data'].data.shape[1:] proto_layer_counter = 0 scale = 1.0 for i in range(len(net.layers)): l = net.layers[i] type = l.type.upper() if type not in layer_not_in_prototxt: proto_layer_counter += 1 lpt = net_proto.layer[proto_layer_counter] if layer_skipped[type]: if type == DATA: scale = lpt.transform_param.scale continue else: l.ltype = type if type == POOLING: p = lpt.pooling_param fixPooling(l, p) elif type == SOFTMAX_WITHLOSS or type == SOFTMAX: l.ltype = SOFTMAX elif type == CONVOLUTION: if len(lpt.convolution_param.stride._values) == 0: l.stride = 0 else: l.stride = lpt.convolution_param.stride._values[0] if len(lpt.convolution_param.pad._values) == 0: l.pad = 0 else: l.pad = lpt.convolution_param.pad._values[0] # dimension = None l.shape = dimension if net.blobs.has_key( lpt.name ): ## because the shape is its input dimension, which is contained in last layer if len(net.blobs[lpt.name].data.shape) != 0: dimension = net.blobs[lpt.name].data.shape[1:] result.append([l, lpt]) ## use the name in the net_proto.layer to query the input_dimension and set ## if the last input_dimension is not null, set it. ## so even if relu can't search it, it still gets the correct result result.append( [edict({ 'ltype': RETURN_CALLBACK, 'shape': dimension }), None]) dimension = net.blobs['data'].data.shape[1:] return result, dimension, scale
def read_cartridge_template(path): catridge = None with open(path, "r") as f: cartridge = cartridge_pb2.Cartridge() Merge(f.read(), cartridge) return cartridge
def read_proto_text(path, model): """ read protobuf in text form """ with open(path, 'r') as fp: Merge(fp.read(), model) return model
def get_ops(): sdk_path = os.path.dirname(__file__) ops_file = os.path.join(sdk_path, 'mmdnn_ops.pbtxt') ops = op_def_pb2.OpList() with open(ops_file) as fn: Merge(fn.read(), ops) ops_list = [MessageToDict(op) for op in ops.op] return ops_list
def read_proto_from_file(proto, filename, binary=True): if binary: f = open(filename, "rb") proto.ParseFromString(f.read()) f.close() else: f = open(filename, "r") Merge(f.read(), proto) f.close()
def transfer2depthwise(proto_src, proto_dst): net = caffe_pb2.NetParameter() Merge(open(proto_src, 'r').read(), net) for layer in net.layer: if layer.type == "Convolution": if layer.convolution_param.group != 1: layer.type = "DepthwiseConvolution" with open(proto_dst, 'w') as tf: tf.write(str(net))
def test_pack_unpack(self): expected_cartridge = Cartridge() Merge(CARTRIDGE_TEXT, expected_cartridge) crypto = Desx_Crypto() checksum = Crc16_Checksum() manager = Manager(crypto, checksum) unpacked_cartridge = manager.unpack(manager.pack(expected_cartridge)) assert expected_cartridge == unpacked_cartridge
def _get_config(): """Gets config.""" if FLAGS.config is None: config_file = os.path.join( os.path.realpath(cnn.__path__[0]), 'configs/cifar-{}.prototxt'.format(FLAGS.model)) else: config_file = FLAGS.config config = ResnetModelConfig() Merge(open(config_file).read(), config) return config
def start(self): files = glob.glob( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'examples', '*.proto')) for filepath in sorted(files): with open(filepath) as f: feed = gtfs_realtime_pb2.FeedMessage() Merge(f.read(), feed) self.feed_processor.process_feed( self.feed_filter.filterTume(feed))
def _generate_cell_chunks(chunk_text_pbs): from google.protobuf.text_format import Merge from google.cloud.bigtable_v2.proto.bigtable_pb2 import ReadRowsResponse chunks = [] for chunk_text_pb in chunk_text_pbs: chunk = ReadRowsResponse.CellChunk() chunks.append(Merge(chunk_text_pb, chunk)) return chunks
def __init__(self, dataname, txt_path, model_path, config): self.net = caffe.Net(txt_path, model_path, caffe.TEST) self.net_proto = caffe_proto.NetParameter() self.config = {token[0]: token[1] for token in config} self.dataname = dataname with open(txt_path, 'r') as f: Merge(f.read(), self.net_proto) self.layers = [] self.prototxt_dict = {} for i in range(len(self.net_proto.layer)): self.prototxt_dict[ self.net_proto.layer[i].name] = self.net_proto.layer[i] prev_layer = None for i in range(len(self.net.layers)): layer = self.net.layers[i] name = self.net._layer_names[i] model_param, proto_param = None, None try: model_param = self.net.blobs[self.fitDataLayer(name, lowercase=True)] proto_param = self.prototxt_dict[name] except KeyError: pass l = construct( self.fitDataLayer(layer.type), { 'blob': layer.blobs, 'model_param': model_param, 'proto_param': proto_param, 'prev': prev_layer }) if l: if l.param['pad'] > 0: lpadding = construct( PaddingType, { 'blob': None, 'proto_param': { 'pad': l.param['pad'] }, 'model_param': None, 'prev': prev_layer }) self.layers.append(lpadding) prev_layer = l self.layers.append(l) self.layers.append( construct( OutputType, { 'blob': None, 'model_param': None, 'proto_param': None, 'prev': prev_layer }))
def parse_net_proto(prototxt): from caffe.proto import caffe_pb2 from google.protobuf.text_format import Merge from collections import OrderedDict net_param = caffe_pb2.NetParameter() with open(prototxt, 'r') as f: Merge(f.read(), net_param) layer_names = [] all_layer_info = OrderedDict() negative_slope = None is_bn = 0 for layer in net_param.layer: layer_name = layer.name one_layer_info = {} # one_layer_info['ps'] = layer. if layer.type == 'Pooling': param = layer.pooling_param # pooling_param = caffe_pb2.PoolingParameter() one_layer_info['pooling'] = 'max' if param.pool == 0 else 'ave' # default value for Pooling layer # note that Pooling layer don't support ps operation one_layer_info['ps'] = False one_layer_info['bn'] = False one_layer_info['dilation'] = 1 elif layer.type == 'Convolution': param = layer.convolution_param one_layer_info['ps'] = help_parse(param.position_sensetive, 'ps') one_layer_info['dilation'] = help_parse(param.dilation, 'dilation') one_layer_info['bn'] = False elif layer.type == 'ReLU': negative_slope = help_parse( layer.relu_param.negative_slope, 'negative_slope') continue elif layer.type == 'BatchNorm' or layer.type == 'Scale': is_bn += 1 continue else: print 'skip layer {:s}'.format(layer.name) continue if negative_slope is not None: one_layer_info['negative_slope'] = negative_slope negative_slope = None if is_bn == 2: all_layer_info[layer_names[-1]]['bn'] = True is_bn = 0 one_layer_info['kernel_size'] = help_parse(param.kernel_size, None) one_layer_info['pad'] = help_parse(param.pad, 'pad') one_layer_info['stride'] = help_parse(param.stride, 'stride') layer_names.append(layer_name) all_layer_info[layer_name] = one_layer_info all_layer_info[layer_names[-1]]['bn'] = False return all_layer_info
def prototxt_to_forward(prototxt): net = caffe_pb2.NetParameter() with open(prototxt, 'r') as f: Merge(f.read(), net) if net.layer: net_layers = net.layer else: net_layers = net.layers layers = [] for layer in net_layers: layers.append(parse_layer(layer)) return layers
def load_test_message_stream(path): """ Loads a message stream. The message stream is a simple text file of protobuf messages sent and received during the course of a training or prediction run. The file is very simply formatted as a pairs of lines for each message: Line N : SEND|RECV Line N+1: ServerToSimulator or SimulutorToServer protobuf object serialized to text form with google.protobuf.text_format.MessageToString(), or None indicating an empty message. This output file can be easily re-created with any of the gym sample simulators by simply adding a "--messages-out <PATH>" parameter to the command line. :param path: Path to test file to load. :type path: string :return: Array of TestMessage instances representing the back-and-forth communications between the simulator and its BRAIN. """ with open(path, 'r') as infile: line_number = 0 direction = None message_as_text = None message = None messages = [] for line in infile: line_number += 1 line = line.strip() if line_number % 2 == 1: direction = line else: message_as_text = line if message_as_text == 'None': message = None elif direction == 'RECV': message = ServerToSimulator() elif direction == 'SEND': message = SimulatorToServer() else: raise RuntimeError('Error loading file ' 'on line {}'.format(line_number)) if message: Merge(message_as_text, message) tst_msg = TestMessage(direction=direction, message_as_text=message_as_text, message=message) messages.append(tst_msg) return messages
def export(self): pipeline_config = TrainEvalPipelineConfig() Merge(self.config_path.read_text(), pipeline_config) last_ckpt = max(self.training_path.glob("model.ckpt-*.meta"), key=_get_ckpt_number_from_file).with_suffix("") n_steps = last_ckpt.suffix.split("-")[-1] export_inference_graph( input_type="image_tensor", pipeline_config=pipeline_config, trained_checkpoint_prefix=str(last_ckpt), output_directory=str(PIPELINES_DIR / self.task / f"{self.name}__{n_steps}_steps"), )
def create_from_file(cls, config_filename, is_training=True, inp=None, label=None, batch_size=None): config = ResnetModelConfig() Merge(open(config_filename).read(), config) return cls(config, is_training=is_training, inp=inp, label=label, batch_size=batch_size)
def test_pack(self): cartridge = Cartridge() Merge(CARTRIDGE_TEXT, cartridge) expected_packed_eeprom = bytearray( binascii.unhexlify(PACKED_CARTRIDGE_HEX)) crypto = Desx_Crypto() checksum = Crc16_Checksum() manager = Manager(crypto, checksum) packed_eeprom = manager.pack(cartridge) assert expected_packed_eeprom == packed_eeprom
def test_run_delegation(create_job, get_results): dt = datetime.datetime.now(tz=datetime.timezone.utc) create_job.return_value = ( 'steve', quantum.QuantumJob( name='projects/a/programs/b/jobs/steve', execution_status=quantum.ExecutionStatus(state=quantum.ExecutionStatus.State.SUCCESS), update_time=dt, ), ) get_results.return_value = quantum.QuantumResult( result=util.pack_any( Merge( """sweep_results: [{ repetitions: 4, parameterized_results: [{ params: { assignments: { key: 'a' value: 1 } }, measurement_results: { key: 'q' qubit_measurement_results: [{ qubit: { id: '1_1' } results: '\006' }] } }] }] """, v2.result_pb2.Result(), ) ) ) program = cg.EngineProgram('a', 'b', EngineContext()) param_resolver = cirq.ParamResolver({}) results = program.run( job_id='steve', repetitions=10, param_resolver=param_resolver, processor_ids=['mine'] ) assert results == cg.EngineResult( params=cirq.ParamResolver({'a': 1.0}), measurements={'q': np.array([[False], [True], [True], [False]], dtype=bool)}, job_id='steve', job_finished_time=dt, )
def create_header(prototxtFile): # Create the Caffe network parameter object net = caffe_pb2.NetParameter() # merge the given network with the object. This will read the text file and initialize all # parameters of the object with those from the text file. Merge((open(prototxtFile, 'r').read()), net) hfile = open(headerFileName, 'w') sfile = open(sourceFileName, 'w') hfile.write(startString) hfile.write(docString) sfile.write(docString) sfile.write("#include " + '"' + headerFileName + '"' + "\n\n") # find the total number of layers in the network. Input layer is not counted as it is a data layer. # hence -1 NumCnnLayers = len(net.layer) hfile.write("#define NO_DEEP_LAYERS " + str(NumCnnLayers - 1) + "\n") # create header file writer object hw = DnnHeaderCreater(net, hfile, sfile) hw.parse_params() # make sure the structures are arranged in the layer connection format hw.order_structures() hfile.write("#define INPUT_IMG_WIDTH " + str(hw.inputWidth) + "\n") hfile.write("#define INPUT_IMG_HEIGHT " + str(hw.inputHeight) + "\n") hfile.write("#define NO_INPUT_MAPS " + str(hw.noInputMaps) + "\n") # write general struct definition to the header file hw.write_struct_definition(structName) # declare the array of structure hfile.write("extern " + "const " + structName + " " + arrayName + "[NO_DEEP_LAYERS];\n\n") hw.write_struct_array(structName) # write the header end string hfile.write(endString) hfile.write('\n') # print approx no of operations present in the network #hw.compute_no_ops() hfile.close() sfile.close() cprint('Generated source files successfully', 'green') cprint( 'Copy {:s} and {:s} to the main project src and inc directories respectively.' .format(sourceFileName, headerFileName), 'green')
def test_get_engine_device(get_processor): device_spec = _to_any( Merge( """ valid_gate_sets: [{ name: 'test_set', valid_gates: [{ id: 'x', number_of_qubits: 1, gate_duration_picos: 1000, valid_targets: ['1q_targets'] }] }], valid_qubits: ['0_0', '1_1'], valid_targets: [{ name: '1q_targets', target_ordering: SYMMETRIC, targets: [{ ids: ['0_0'] }] }] """, v2.device_pb2.DeviceSpecification(), )) gate_set = cg.SerializableGateSet( gate_set_name='x_gate_set', serializers=[ cg.GateOpSerializer(gate_type=cirq.XPowGate, serialized_gate_id='x', args=[]) ], deserializers=[ cg.GateOpDeserializer(serialized_gate_id='x', gate_constructor=cirq.XPowGate, args=[]) ], ) get_processor.return_value = qtypes.QuantumProcessor( device_spec=device_spec) device = cirq_google.get_engine_device('rainbow', 'project', gatesets=[gate_set]) assert set(device.qubits) == {cirq.GridQubit(0, 0), cirq.GridQubit(1, 1)} device.validate_operation(cirq.X(cirq.GridQubit(0, 0))) with pytest.raises(ValueError): device.validate_operation(cirq.X(cirq.GridQubit(1, 2))) with pytest.raises(ValueError): device.validate_operation(cirq.Y(cirq.GridQubit(0, 0)))