def toStr(self, protoObj): """Used for pretty printing a result from the API.""" return text_format.MessageToString(protoObj)
def _AssertProtoDictEquals(self, expected_dict, actual_dict, verbose=False, update_goldens=False, additional_missing_object_message='', api_version=2): """Diff given dicts of protobufs and report differences a readable way. Args: expected_dict: a dict of TFAPIObject protos constructed from golden files. actual_dict: a ict of TFAPIObject protos constructed by reading from the TF package linked to the test. verbose: Whether to log the full diffs, or simply report which files were different. update_goldens: Whether to update goldens when there are diffs found. additional_missing_object_message: Message to print when a symbol is missing. api_version: TensorFlow API version to test. """ diffs = [] verbose_diffs = [] expected_keys = set(expected_dict.keys()) actual_keys = set(actual_dict.keys()) only_in_expected = expected_keys - actual_keys only_in_actual = actual_keys - expected_keys all_keys = expected_keys | actual_keys # This will be populated below. updated_keys = [] for key in all_keys: diff_message = '' verbose_diff_message = '' # First check if the key is not found in one or the other. if key in only_in_expected: diff_message = 'Object %s expected but not found (removed). %s' % ( key, additional_missing_object_message) verbose_diff_message = diff_message elif key in only_in_actual: diff_message = 'New object %s found (added).' % key verbose_diff_message = diff_message else: # Do not truncate diff self.maxDiff = None # pylint: disable=invalid-name # Now we can run an actual proto diff. try: self.assertProtoEquals(expected_dict[key], actual_dict[key]) except AssertionError as e: updated_keys.append(key) diff_message = 'Change detected in python object: %s.' % key verbose_diff_message = str(e) # All difference cases covered above. If any difference found, add to the # list. if diff_message: diffs.append(diff_message) verbose_diffs.append(verbose_diff_message) # If diffs are found, handle them based on flags. if diffs: diff_count = len(diffs) logging.error(self._test_readme_message) logging.error('%d differences found between API and golden.', diff_count) messages = verbose_diffs if verbose else diffs for i in range(diff_count): print('Issue %d\t: %s' % (i + 1, messages[i]), file=sys.stderr) if update_goldens: # Write files if requested. logging.warning(self._update_golden_warning) # If the keys are only in expected, some objects are deleted. # Remove files. for key in only_in_expected: filepath = _KeyToFilePath(key, api_version) file_io.delete_file(filepath) # If the files are only in actual (current library), these are new # modules. Write them to files. Also record all updates in files. for key in only_in_actual | set(updated_keys): filepath = _KeyToFilePath(key, api_version) file_io.write_string_to_file( filepath, text_format.MessageToString(actual_dict[key])) else: # Fail if we cannot fix the test by updating goldens. self.fail('%d differences found between API and golden.' % diff_count) else: logging.info('No differences found between API and golden.')
def main(): """main """ parser = argparse.ArgumentParser() parser.add_argument('--net', type=str, required=True, help='net prototxt') parser.add_argument('--weight', type=str, required=True, help='net weight') args = parser.parse_args() print(args) net = caffe_pb2.NetParameter() text_format.Merge(open(args.net, 'r').read(), net) weight = caffe_pb2.NetParameter() weight.ParseFromString(open(args.weight, 'rb').read()) # remove useless layers net_layers = [layer.name for layer in net.layer] weight_layers = [layer.name for layer in net.layer] not_used_layers = [ layer for layer in weight_layers if layer not in net_layers ] for layer_name in not_used_layers: weight.layer.remove(get_layer(weight, layer_name)) # search conv-batchnorm-scale pattern remove_list = [] i = 0 while i < len(net_layers): if net.layer[i].type == 'Convolution': conv_param = net.layer[i].convolution_param conv_name = net.layer[i].name conv_layer = get_layer(weight, conv_name) i += 1 if conv_param.group != 1: continue if i < len(net_layers) and net.layer[i].type == 'BatchNorm': batch_norm_param = net.layer[i].batch_norm_param batch_norm_name = net.layer[i].name batch_norm_layer = get_layer(weight, batch_norm_name) i += 1 if net.layer[i].HasField( 'batch_norm_param' ) and not batch_norm_param.use_global_stats: continue if i < len(net_layers) and net.layer[i].type == 'Scale': scale_param = net.layer[i].scale_param scale_name = net.layer[i].name scale_layer = get_layer(weight, scale_name) i += 1 print('fuse (%s, %s, %s)' % (conv_name, batch_norm_name, scale_name)) # weight, bias conv_weight = convert_blob_to_array(conv_layer.blobs[0]) if conv_param.bias_term: conv_bias = convert_blob_to_array(conv_layer.blobs[1]) else: channels = conv_param.num_output conv_bias = np.zeros(channels, dtype=np.float32) # mean, std mean = convert_blob_to_array(batch_norm_layer.blobs[0]) std = convert_blob_to_array(batch_norm_layer.blobs[1]) scale_factor = convert_blob_to_array( batch_norm_layer.blobs[2]) mean /= scale_factor std /= scale_factor # scale, shift scale = convert_blob_to_array(scale_layer.blobs[0]) if scale_param.bias_term: shift = convert_blob_to_array(scale_layer.blobs[1]) else: shift = np.zeros(scale.shape, dtype=np.float32) # eps if batch_norm_param is not None: eps = batch_norm_param.eps else: eps = 1e-5 # fuse conv_weight, conv_bias = fuse(conv_weight, conv_bias, mean, std, scale, shift, eps) conv_layer.blobs.pop() if conv_param.bias_term: conv_layer.blobs.pop() else: conv_param.bias_term = True conv_layer.blobs.extend([ convert_array_to_blob(conv_weight), convert_array_to_blob(conv_bias) ]) # remove batchnorm and scale layer weight.layer.remove(batch_norm_layer) weight.layer.remove(scale_layer) net.layer[i - 3].top[0] = net.layer[i - 1].top[0] remove_list.extend([net.layer[i - 1], net.layer[i - 2]]) else: i += 1 for layer in remove_list: net.layer.remove(layer) # save out_net = '_nobn'.join(os.path.splitext(args.net)) out_weight = '_nobn'.join(os.path.splitext(args.weight)) with open(out_net, 'w') as fout: fout.write(text_format.MessageToString(net)) with open(out_weight, 'wb') as fout: fout.write(weight.SerializeToString())
def do_test(request): response = conformance_pb2.ConformanceResponse() if request.message_type == "conformance.FailureSet": failure_set = conformance_pb2.FailureSet() failures = [] # TODO(gerbens): Remove, this is a hack to detect if the old vs new # parser is used by the cpp code. Relying on a bug in the old parser. hack_proto = test_messages_proto2_pb2.TestAllTypesProto2() old_parser = True try: hack_proto.ParseFromString(b"\322\002\001") except message.DecodeError as e: old_parser = False if old_parser: # the string above is one of the failing conformance test strings of the # old parser. If we succeed the c++ implementation is using the old # parser so we add the list of failing conformance tests. failures = [ "Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE", "Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.BOOL", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.DOUBLE", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.ENUM", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.FIXED32", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.FIXED64", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.FLOAT", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.INT32", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.INT64", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.SFIXED32", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.SFIXED64", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.SINT32", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.SINT64", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.UINT32", "Required.Proto3.ProtobufInput.PrematureEofInPackedField.UINT64", "Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE", "Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.BOOL", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.DOUBLE", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.ENUM", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.FIXED32", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.FIXED64", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.FLOAT", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.INT32", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.INT64", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.SFIXED32", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.SFIXED64", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32", "Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64", ] for x in failures: failure_set.failure.append(x) response.protobuf_payload = failure_set.SerializeToString() return response isProto3 = (request.message_type == "protobuf_test_messages.proto3.TestAllTypesProto3") isJson = (request.WhichOneof('payload') == 'json_payload') isProto2 = (request.message_type == "protobuf_test_messages.proto2.TestAllTypesProto2") if (not isProto3) and (not isJson) and (not isProto2): raise ProtocolError("Protobuf request doesn't have specific payload type") test_message = test_messages_proto2_pb2.TestAllTypesProto2() if isProto2 else \ test_messages_proto3_pb2.TestAllTypesProto3() try: if request.WhichOneof('payload') == 'protobuf_payload': try: test_message.ParseFromString(request.protobuf_payload) except message.DecodeError as e: response.parse_error = str(e) return response elif request.WhichOneof('payload') == 'json_payload': try: ignore_unknown_fields = \ request.test_category == \ conformance_pb2.JSON_IGNORE_UNKNOWN_PARSING_TEST json_format.Parse(request.json_payload, test_message, ignore_unknown_fields) except Exception as e: response.parse_error = str(e) return response elif request.WhichOneof('payload') == 'text_payload': try: text_format.Parse(request.text_payload, test_message) except Exception as e: response.parse_error = str(e) return response else: raise ProtocolError("Request didn't have payload.") if request.requested_output_format == conformance_pb2.UNSPECIFIED: raise ProtocolError("Unspecified output format") elif request.requested_output_format == conformance_pb2.PROTOBUF: response.protobuf_payload = test_message.SerializeToString() elif request.requested_output_format == conformance_pb2.JSON: try: response.json_payload = json_format.MessageToJson(test_message) except Exception as e: response.serialize_error = str(e) return response elif request.requested_output_format == conformance_pb2.TEXT_FORMAT: response.text_payload = text_format.MessageToString(test_message) except Exception as e: response.runtime_error = str(e) return response
def test_standalone(self): """Sample test for run_local_database.py as a standalone process.""" topology = vttest_pb2.VTTestTopology() keyspace = topology.keyspaces.add(name='test_keyspace') keyspace.replica_count = 2 keyspace.rdonly_count = 1 keyspace.shards.add(name='-80') keyspace.shards.add(name='80-') topology.keyspaces.add(name='redirect', served_from='test_keyspace') # launch a backend database based on the provided topology and schema port = environment.reserve_ports(1) args = [ environment.run_local_database, '--port', str(port), '--proto_topo', text_format.MessageToString(topology, as_one_line=True), '--schema_dir', os.path.join(environment.vtroot, 'test', 'vttest_schema'), '--web_dir', environment.vtroot + '/web/vtctld', ] sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) config = json.loads(sp.stdout.readline()) # gather the vars for the vtgate process url = 'http://localhost:%d/debug/vars' % config['port'] f = urllib.urlopen(url) data = f.read() f.close() json_vars = json.loads(data) self.assertIn('vtcombo', json_vars['cmdline'][0]) # build the vtcombo address and protocol protocol = protocols_flavor().vttest_protocol() if protocol == 'grpc': vtgate_addr = 'localhost:%d' % config['grpc_port'] else: vtgate_addr = 'localhost:%d' % config['port'] conn_timeout = 30.0 utils.pause('Paused test after vtcombo was started.\n' 'For manual testing, connect to vtgate at: %s ' 'using protocol: %s.\n' 'Press enter to continue.' % (vtgate_addr, protocol)) # Remember the current timestamp after we sleep for a bit, so we # can use it for UpdateStream later. time.sleep(2) before_insert = long(time.time()) # Connect to vtgate. conn = vtgate_client.connect(protocol, vtgate_addr, conn_timeout) # Insert a row. row_id = 123 keyspace_id = get_keyspace_id(row_id) cursor = conn.cursor(tablet_type='master', keyspace='test_keyspace', keyspace_ids=[pack_kid(keyspace_id)], writable=True) cursor.begin() insert = ('insert into test_table (id, msg, keyspace_id) values (:id, ' ':msg, :keyspace_id)') bind_variables = { 'id': row_id, 'msg': 'test %s' % row_id, 'keyspace_id': keyspace_id, } cursor.execute(insert, bind_variables) cursor.commit() # Read the row back. cursor.execute('select * from test_table where id=:id', {'id': row_id}) result = cursor.fetchall() self.assertEqual(result[0][1], 'test 123') # try to insert again, see if we get the right integrity error exception # (this is meant to test vtcombo properly returns exceptions, and to a # lesser extent that the python client converts it properly) cursor.begin() with self.assertRaises(dbexceptions.IntegrityError): cursor.execute(insert, bind_variables) cursor.rollback() # Insert a bunch of rows with long msg values. bind_variables['msg'] = 'x' * 64 id_start = 1000 rowcount = 500 cursor.begin() for i in xrange(id_start, id_start + rowcount): bind_variables['id'] = i bind_variables['keyspace_id'] = get_keyspace_id(i) cursor.execute(insert, bind_variables) cursor.commit() cursor.close() # Try to fetch a large number of rows, from a rdonly # (more than one streaming result packet). stream_cursor = conn.cursor( tablet_type='rdonly', keyspace='test_keyspace', keyspace_ids=[pack_kid(keyspace_id)], cursorclass=vtgate_cursor.StreamVTGateCursor) stream_cursor.execute('select * from test_table where id >= :id_start', {'id_start': id_start}) self.assertEqual(rowcount, len(list(stream_cursor.fetchall()))) stream_cursor.close() # try to read a row using the redirected keyspace, to a replica this time row_id = 123 keyspace_id = get_keyspace_id(row_id) cursor = conn.cursor(tablet_type='replica', keyspace='redirect', keyspace_ids=[pack_kid(keyspace_id)]) cursor.execute('select * from test_table where id=:id', {'id': row_id}) result = cursor.fetchall() self.assertEqual(result[0][1], 'test 123') cursor.close() # Try to get the update stream from the connection. This makes # sure that part works as well. count = 0 for (event, _) in conn.update_stream('test_keyspace', topodata_pb2.MASTER, timestamp=before_insert, shard='-80'): for statement in event.statements: if statement.table_name == 'test_table': count += 1 if count == rowcount + 1: # We're getting the initial value, plus the 500 updates. break # Insert a sentinel value into the second shard. row_id = 0x8100000000000000 keyspace_id = get_keyspace_id(row_id) cursor = conn.cursor(tablet_type='master', keyspace='test_keyspace', keyspace_ids=[pack_kid(keyspace_id)], writable=True) cursor.begin() bind_variables = { 'id': row_id, 'msg': 'test %s' % row_id, 'keyspace_id': keyspace_id, } cursor.execute(insert, bind_variables) cursor.commit() cursor.close() # Try to connect to an update stream on the other shard. # We may get some random update stream events, but we should not get any # event that's related to the first shard. Only events related to # the Insert we just did. found = False for (event, _) in conn.update_stream('test_keyspace', topodata_pb2.MASTER, timestamp=before_insert, shard='80-'): for statement in event.statements: self.assertEqual(statement.table_name, 'test_table') fields, rows = proto3_encoding.convert_stream_event_statement( statement) self.assertEqual(fields[0], 'id') self.assertEqual(rows[0][0], row_id) found = True if found: break # Clean up the connection conn.close() # Test we can connect to vtcombo for vtctl actions protocol = protocols_flavor().vtctl_python_client_protocol() if protocol == 'grpc': vtgate_addr = 'localhost:%d' % config['grpc_port'] else: vtgate_addr = 'localhost:%d' % config['port'] out, _ = utils.run(environment.binary_args('vtctlclient') + [ '-vtctl_client_protocol', protocol, '-server', vtgate_addr, '-stderrthreshold', '0', 'ListAllTablets', 'test', ], trap_output=True) num_master = 0 num_replica = 0 num_rdonly = 0 num_dash_80 = 0 num_80_dash = 0 for line in out.splitlines(): parts = line.split() self.assertEqual(parts[1], 'test_keyspace', 'invalid keyspace in line: %s' % line) if parts[3] == 'master': num_master += 1 elif parts[3] == 'replica': num_replica += 1 elif parts[3] == 'rdonly': num_rdonly += 1 else: self.fail('invalid tablet type in line: %s' % line) if parts[2] == '-80': num_dash_80 += 1 elif parts[2] == '80-': num_80_dash += 1 else: self.fail('invalid shard name in line: %s' % line) self.assertEqual(num_master, 2) self.assertEqual(num_replica, 2) self.assertEqual(num_rdonly, 2) self.assertEqual(num_dash_80, 3) self.assertEqual(num_80_dash, 3) # and we're done, clean-up process sp.stdin.write('\n') sp.wait()
def build_example(label, param_dict_real, zip_path_label): """Build the model with parameter values set in param_dict_real. Args: label: Label of the model param_dict_real: Parameter dictionary (arguments to the factories make_graph and make_test_inputs) zip_path_label: Filename in the zip Returns: (tflite_model_binary, report) where tflite_model_binary is the serialized flatbuffer as a string and report is a dictionary with keys `tflite_converter_log` (log of conversion), `tf_log` (log of tf conversion), `converter` (a string of success status of the conversion), `tf` (a string success status of the conversion). """ np.random.seed(RANDOM_SEED) report = { "tflite_converter": report_lib.NOTRUN, "tf": report_lib.FAILED } # Build graph report["tf_log"] = "" report["tflite_converter_log"] = "" tf.reset_default_graph() with tf.Graph().as_default(): with tf.device("/cpu:0"): try: inputs, outputs = make_graph(param_dict_real) inputs = [x for x in inputs if x is not None] except (tf.errors.UnimplementedError, tf.errors.InvalidArgumentError, ValueError): report["tf_log"] += traceback.format_exc() return None, report sess = tf.Session() try: baseline_inputs, baseline_outputs = (make_test_inputs( param_dict_real, sess, inputs, outputs)) baseline_inputs = [ x for x in baseline_inputs if x is not None ] # Converts baseline inputs/outputs to maps. The signature input and # output names are set to be the same as the tensor names. input_names = [ _normalize_input_name(x.name) for x in inputs ] output_names = [ _normalize_output_name(x.name) for x in outputs ] baseline_input_map = dict( zip(input_names, baseline_inputs)) baseline_output_map = dict( zip(output_names, baseline_outputs)) except (tf.errors.UnimplementedError, tf.errors.InvalidArgumentError, ValueError): report["tf_log"] += traceback.format_exc() return None, report report["tflite_converter"] = report_lib.FAILED report["tf"] = report_lib.SUCCESS # Builds a saved model with the default signature key. input_names, tensor_info_inputs = _get_tensor_info( inputs, "input_", _normalize_input_name) output_tensors, tensor_info_outputs = _get_tensor_info( outputs, "output_", _normalize_output_name) input_tensors = [(name, t.shape, t.dtype) for name, t in zip(input_names, inputs)] inference_signature = ( tf.saved_model.signature_def_utils.build_signature_def( inputs=tensor_info_inputs, outputs=tensor_info_outputs, method_name="op_test")) saved_model_dir = tempfile.mkdtemp("op_test") saved_model_tags = [tf.saved_model.tag_constants.SERVING] signature_key = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY builder = tf.saved_model.builder.SavedModelBuilder( saved_model_dir) builder.add_meta_graph_and_variables( sess, saved_model_tags, signature_def_map={ signature_key: inference_signature, }, strip_default_attrs=True) builder.save(as_text=False) # pylint: disable=g-long-ternary graph_def = freeze_graph( sess, tf.global_variables() + inputs + outputs) if use_frozen_graph else sess.graph_def if "split_tflite_lstm_inputs" in param_dict_real: extra_convert_options.split_tflite_lstm_inputs = param_dict_real[ "split_tflite_lstm_inputs"] tflite_model_binary, converter_log = options.tflite_convert_function( options, saved_model_dir, input_tensors, output_tensors, extra_convert_options=extra_convert_options, test_params=param_dict_real) report["tflite_converter"] = (report_lib.SUCCESS if tflite_model_binary is not None else report_lib.FAILED) report["tflite_converter_log"] = converter_log if options.save_graphdefs: zipinfo = zipfile.ZipInfo(zip_path_label + ".pbtxt") archive.writestr(zipinfo, text_format.MessageToString(graph_def), zipfile.ZIP_DEFLATED) if tflite_model_binary: if options.make_edgetpu_tests: # Set proper min max values according to input dtype. baseline_input_map, baseline_output_map = generate_inputs_outputs( tflite_model_binary, min_value=0, max_value=255) zipinfo = zipfile.ZipInfo(zip_path_label + ".bin") archive.writestr(zipinfo, tflite_model_binary, zipfile.ZIP_DEFLATED) example = { "inputs": baseline_input_map, "outputs": baseline_output_map } example_fp = StringIO() write_examples(example_fp, [example]) zipinfo = zipfile.ZipInfo(zip_path_label + ".inputs") archive.writestr(zipinfo, example_fp.getvalue(), zipfile.ZIP_DEFLATED) example_fp2 = StringIO() write_test_cases(example_fp2, zip_path_label + ".bin", [example]) zipinfo = zipfile.ZipInfo(zip_path_label + "_tests.txt") archive.writestr(zipinfo, example_fp2.getvalue(), zipfile.ZIP_DEFLATED) zip_manifest_label = zip_path_label + " " + label if zip_path_label == label: zip_manifest_label = zip_path_label zip_manifest.append(zip_manifest_label + "\n") return tflite_model_binary, report
def _test_user_op_graph(test_case, is_cuda): test_case.assertTrue(oneflow.framework.env_util.HasAllMultiClientEnvVars()) x0 = flow.tensor(np.random.rand(20, 30), dtype=flow.float32) weight0 = flow.tensor(np.random.rand(30, 50), dtype=flow.float32) x1 = flow.tensor(np.random.rand(50, 70), dtype=flow.float32) if is_cuda: x0 = x0.to(device=flow.device("cuda")) weight0 = weight0.to(device=flow.device("cuda")) x1 = x1.to(device=flow.device("cuda")) # NOTE(chengcheng): this tiny net is: # x0 * weight0 -> out0 # relu(out0) -> y0 # y0 * x1 -> out1 # relu(out1) -> y1 session = session_ctx.GetDefaultSession() test_case.assertTrue(isinstance(session, MultiClientSession)) session.TryInit() with oneflow._oneflow_internal.lazy_mode.guard(True): oneflow._oneflow_internal.JobBuildAndInferCtx_Open( "cc_test_user_op_expr_job_with_cuda" + str(is_cuda)) job_conf = oneflow.core.job.job_conf_pb2.JobConfigProto() job_conf.job_name = "cc_test_user_op_expr_job_with_cuda" + str(is_cuda) job_conf.predict_conf.SetInParent() c_api_util.CurJobBuildAndInferCtx_SetJobConf(job_conf) x0_conf = oneflow.core.operator.op_conf_pb2.FeedInputOpConf() x0_conf.in_0 = "in_0" x0_conf.out_0 = "out_0" x0_conf_str = text_format.MessageToString(x0_conf) x0_op = oneflow._oneflow_internal.one.FeedInputOpExpr( "cc_Input_0", x0_conf_str, ["in_0"], ["out_0"]) x1_conf = oneflow.core.operator.op_conf_pb2.FeedInputOpConf() x1_conf.in_0 = "in_0" x1_conf.out_0 = "out_0" x1_conf_str = text_format.MessageToString(x1_conf) x1_op = oneflow._oneflow_internal.one.FeedInputOpExpr( "cc_Input_1", x1_conf_str, ["in_0"], ["out_0"]) weight0_conf = oneflow.core.operator.op_conf_pb2.FeedVariableOpConf() weight0_conf.in_0 = "in_0" weight0_conf.out_0 = "out_0" weight0_conf_str = text_format.MessageToString(weight0_conf) weight0_op = oneflow._oneflow_internal.one.FeedVariableOpExpr( "cc_Variable_0", weight0_conf_str, ["in_0"], ["out_0"]) output_conf = oneflow.core.operator.op_conf_pb2.FetchOutputOpConf() output_conf.in_0 = "in_0" output_conf.out_0 = "out_0" output_conf_str = text_format.MessageToString(output_conf) output_op = oneflow._oneflow_internal.one.FetchOutputOpExpr( "cc_Output_0", output_conf_str, ["in_0"], ["out_0"]) x0_lazy_tensor = _C.dispatch_feed_input(x0_op, x0) x1_lazy_tensor = _C.dispatch_feed_input(x1_op, x1) weight0_lazy_tensor = _C.dispatch_feed_input(weight0_op, weight0) test_case.assertEqual(x0_lazy_tensor.shape, (20, 30)) test_case.assertTrue(x0_lazy_tensor.is_lazy) test_case.assertEqual(weight0_lazy_tensor.shape, (30, 50)) test_case.assertTrue(weight0_lazy_tensor.is_lazy) test_case.assertEqual(x1_lazy_tensor.shape, (50, 70)) test_case.assertTrue(x1_lazy_tensor.is_lazy) out0 = flow._C.matmul(x0_lazy_tensor, weight0_lazy_tensor) test_case.assertEqual(out0.shape, (20, 50)) test_case.assertTrue(out0.is_lazy) y0 = flow._C.relu(out0) test_case.assertEqual(y0.shape, (20, 50)) test_case.assertTrue(y0.is_lazy) out1 = flow._C.matmul(y0, x1_lazy_tensor) test_case.assertEqual(out1.shape, (20, 70)) test_case.assertTrue(out1.is_lazy) y1 = flow._C.relu(out1) test_case.assertEqual(y1.shape, (20, 70)) test_case.assertTrue(y1.is_lazy) eager_output = _C.dispatch_fetch_output(output_op, y1) test_case.assertEqual(eager_output.shape, (20, 70)) test_case.assertTrue(not eager_output.is_lazy) if is_cuda: test_case.assertTrue(x0_lazy_tensor.is_cuda) test_case.assertTrue(x1_lazy_tensor.is_cuda) test_case.assertTrue(weight0_lazy_tensor.is_cuda) test_case.assertTrue(out0.is_cuda) test_case.assertTrue(y0.is_cuda) test_case.assertTrue(out1.is_cuda) test_case.assertTrue(y1.is_cuda) oneflow._oneflow_internal.JobBuildAndInferCtx_Close()
start=args.start end=args.end num_attr=end-start net_file=args.prototxt n=caffe_pb2.NetParameter() text_format.Merge(open(net_file).read(), n) n.layer[-6].inner_product_param.num_output=num_attr new_dir=os.path.dirname(net_file)+"/train_net_dir" new_file= new_dir+"/test_net_{}_{}.prototxt".format(start,end) with open(new_file,'w+') as new_prototxt_file: new_prototxt_file.write(unicode(text_format.MessageToString(n))) args.prototxt=new_file # print args.prototxt # print num_attr # time.sleep(10) net = caffe.Net(new_file, args.caffemodel, caffe.TEST) net.name = os.path.splitext(os.path.basename(args.caffemodel))[0] if args.db == 'RAP': """Load RAP database""" from utils.rap_db import RAP db = RAP(os.path.join('data', 'dataset', args.db), args.par_set_id) else: """Load PETA dayanse""" from utils.peta_db import PETA
def CheckAndCompleteUserOpConf(op_conf_proto): serialized_op_conf = str(text_format.MessageToString(op_conf_proto)) new_op_conf = oneflow_api.CheckAndCompleteUserOpConf(serialized_op_conf) return text_format.Parse(new_op_conf, op_conf_util.OperatorConf())
def save_experiment(filename, expt): file_write_safe(filename, text_format.MessageToString(expt))
def RemoteCall(*args, **kwargs): """Dynamically calls a remote API and returns the result value.""" func_msg = self.GetApi(api_name) if not func_msg: raise MirrorObjectError("api %s unknown", func_msg) logging.debug("remote call %s.%s", self._parent_path, api_name) logging.info("remote call %s%s", api_name, args) if args: for arg_msg, value_msg in zip(func_msg.arg, args): logging.debug("arg msg %s", arg_msg) logging.debug("value %s", value_msg) if value_msg is not None: self.ArgToPb(arg_msg, value_msg) logging.debug("final msg %s", func_msg) else: # TODO: use kwargs for arg in func_msg.arg: # TODO: handle other if (arg.type == CompSpecMsg.TYPE_SCALAR and arg.scalar_type == "pointer"): arg.scalar_value.pointer = 0 logging.debug(func_msg) if self._parent_path: func_msg.parent_path = self._parent_path if self._interface_id is not None: func_msg.hidl_interface_id = self._interface_id if isinstance(self._if_spec_msg, CompSpecMsg.ComponentSpecificationMessage): if self._if_spec_msg.component_class: logging.info("component_class %s", self._if_spec_msg.component_class) if self._if_spec_msg.component_class == CompSpecMsg.HAL_CONVENTIONAL_SUBMODULE: submodule_name = self._if_spec_msg.original_data_structure_name if submodule_name.endswith("*"): submodule_name = submodule_name[:-1] func_msg.submodule_name = submodule_name result = self._client.CallApi( text_format.MessageToString(func_msg), self.__caller_uid) logging.debug(result) if (isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], dict) and "coverage" in result[1]): self._last_raw_code_coverage_data = result[1]["coverage"] result = result[0] if (result and isinstance(result, CompSpecMsg.VariableSpecificationMessage) and result.type == CompSpecMsg.TYPE_HIDL_INTERFACE): if result.hidl_interface_id <= -1: return None nested_interface_id = result.hidl_interface_id nested_interface_name = result.predefined_type.split("::")[-1] logging.debug("Nested interface name: %s", nested_interface_name) nested_interface = self.GetHidlNestedInterface( nested_interface_name, nested_interface_id) return nested_interface return result
def to_pbtxt_file(output_path, spec): """Saves a spec encoded as a struct_pb2.StructuredValue in a pbtxt file.""" spec_proto = to_proto(spec) with tf.io.gfile.GFile(output_path, "wb") as f: f.write(text_format.MessageToString(spec_proto))
def commit_portal_manifest(etcd, portal_manifest): etcd_key = os.path.join(portal_manifest.name, 'manifest') etcd.set_data(etcd_key, text_format.MessageToString(portal_manifest))
def commit_data_source(etcd, data_source): etcd_key = os.path.join(data_source.data_source_meta.name, 'master') etcd.set_data(etcd_key, text_format.MessageToString(data_source))
def populate_experiment(run_config, hparams, pipeline_config_path, train_steps=None, eval_steps=None, model_fn_creator=create_model_fn, **kwargs): """Populates an `Experiment` object. Args: run_config: A `RunConfig`. hparams: A `HParams`. pipeline_config_path: A path to a pipeline config file. train_steps: Number of training steps. If None, the number of training steps is set from the `TrainConfig` proto. eval_steps: Number of evaluation steps per evaluation cycle. If None, the number of evaluation steps is set from the `EvalConfig` proto. model_fn_creator: A function that creates a `model_fn` for `Estimator`. Follows the signature: * Args: * `detection_model_fn`: Function that returns `DetectionModel` instance. * `configs`: Dictionary of pipeline config objects. * `hparams`: `HParams` object. * Returns: `model_fn` for `Estimator`. **kwargs: Additional keyword arguments for configuration override. Returns: An `Experiment` that defines all aspects of training, evaluation, and export. """ configs = config_util.get_configs_from_pipeline_file(pipeline_config_path) configs = config_util.merge_external_params_with_configs( configs, hparams, train_steps=train_steps, eval_steps=eval_steps, **kwargs) model_config = configs['model'] train_config = configs['train_config'] train_input_config = configs['train_input_config'] eval_config = configs['eval_config'] eval_input_config = configs['eval_input_config'] if train_steps is None: train_steps = train_config.num_steps if train_config.num_steps else None if eval_steps is None: eval_steps = eval_config.num_examples if eval_config.num_examples else None detection_model_fn = functools.partial( model_builder.build, model_config=model_config) # Create the input functions for TRAIN/EVAL. train_input_fn = inputs.create_train_input_fn( train_config=train_config, train_input_config=train_input_config, model_config=model_config) eval_input_fn = inputs.create_eval_input_fn( eval_config=eval_config, eval_input_config=eval_input_config, model_config=model_config) export_strategies = [ tf.contrib.learn.utils.saved_model_export_utils.make_export_strategy( serving_input_fn=inputs.create_predict_input_fn( model_config=model_config)) ] estimator = tf.estimator.Estimator( model_fn=model_fn_creator(detection_model_fn, configs, hparams), config=run_config) if run_config.is_chief: # Store the final pipeline config for traceability. pipeline_config_final = config_util.create_pipeline_proto_from_configs( configs) pipeline_config_final_path = os.path.join(estimator.model_dir, 'pipeline.config') config_text = text_format.MessageToString(pipeline_config_final) with tf.gfile.Open(pipeline_config_final_path, 'wb') as f: tf.logging.info('Writing as-run pipeline config file to %s', pipeline_config_final_path) f.write(config_text) return tf.contrib.learn.Experiment( estimator=estimator, train_input_fn=train_input_fn, eval_input_fn=eval_input_fn, train_steps=train_steps, eval_steps=eval_steps, export_strategies=export_strategies, eval_delay_secs=120,)
def CurJobBuildAndInferCtx_AddAndInferMirroredOp(op_conf_proto): serialized_op_conf = str(text_format.MessageToString(op_conf_proto)) add_and_infer = oneflow_api.CurJobBuildAndInferCtx_AddAndInferMirroredOp op_attribute_str = add_and_infer(serialized_op_conf) return text_format.Parse(op_attribute_str, op_attribute_pb.OpAttribute())
return mapped data_in = open(args.i, 'r') if args.i else sys.stdin df = pd.read_csv(data_in, sep='\t', names=COLUMNS) for row in (df.loc[i] for i in df.index): datapoint = row_to_dict(row, tokenize) try: serialized = '' if args.o == 'json': serialized = json.dumps(datapoint) + '\n' elif args.o == 'proto' or args.o == 'proto_text': proto = datapoint_to_proto_as_embeddings(datapoint) \ if as_embeddings else \ datapoint_to_proto_as_words(datapoint) serialized = length_prefix_proto(proto) if args.o == 'proto' \ else text_format.MessageToString(proto) elif args.o == 'token_list': serialized = datapoint_to_tokens(datapoint) + ' ' else: print('Unknown output format "%s"' % args.o, file=sys.stderr) sys.exit(2) assert serialized # Note! # sys.stdout.write( ...converted to string with str()) # All errors ignored here. sys.stdout.write(str(serialized, 'utf-8', 'ignore')) except IOError as e: if e.errno == errno.EPIPE: sys.exit(0) raise e
def CurJobBuildAndInferCtx_AddLbiAndDiffWatcherUuidPair(lbi_and_uuid): serialized = str(text_format.MessageToString(lbi_and_uuid)) oneflow_api.CurJobBuildAndInferCtx_AddLbiAndDiffWatcherUuidPair(serialized)
#!/usr/bin/env python3 import socket import sys from google.protobuf import text_format sys.path.append("../src/plugins") from events_pb2 import * HOST, PORT = "localhost", 7777 # Create a socket (SOCK_STREAM means a TCP socket) with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: # Connect to server and send data sock.connect((HOST, PORT)) sock.setblocking(True) while 1: ser = sock.recv(4096) ev = EventMessage() ev.ParseFromString(ser) print(text_format.MessageToString(ev))
def InitEnv(env_proto): assert type(env_proto) is env_pb2.EnvProto env_proto_str = text_format.MessageToString(env_proto) oneflow_api.InitEnv(env_proto_str)
def main(_): if not FLAGS.save_path: print "You must specify --save_path." with tf.name_scope("inference/rnn"): input_data = tf.placeholder(tf.int32, [1, 1], name="inputs") input_data_f = tf.to_float(input_data) initial_state_c_0 = tf.zeros([1, 3], name="zeros") initial_state_h_0 = tf.zeros([1, 3], name="zeros_1") initial_state_c_1 = tf.zeros([1, 3], name="zeros_2") initial_state_h_1 = tf.zeros([1, 3], name="zeros_3") w = tf.constant([[0.2, 0.5, 0.3]]) final_state_c_0 = tf.add( initial_state_c_0 + 0.1, input_data_f, name="RNN/MultiRNNCell/Cell0/BasicLSTMCell/add_2") final_state_h_0 = tf.add( initial_state_h_0 + 0.2, input_data_f, name="RNN/MultiRNNCell/Cell0/BasicLSTMCell/mul_2") final_state_c_1 = tf.add( initial_state_c_1 + 0.3, input_data_f, name="RNN/MultiRNNCell/Cell1/BasicLSTMCell/add_2") final_state_h_1 = tf.add( initial_state_h_1 + 0.4, input_data_f, name="RNN/MultiRNNCell/Cell1/BasicLSTMCell/mul_2") y = final_state_c_0 + final_state_h_0 + final_state_c_1 + final_state_h_1 - ( 3 * input_data_f) predictions = tf.mul(tf.matmul(input_data_f, w), y, name="predictions") with tf.Session() as sess: tf.initialize_all_variables().run() # Save model for use in C++. # -------------------------- # Save meta information about the RNN. rnn_proto = rnn_pb2.RNNProto() rnn_proto.type = 2 rnn_proto.input_tensor_name = input_data.name rnn_proto.logits_tensor_name = predictions.name rnn_proto.predictions_tensor_name = predictions.name rnn_proto.h.add( initial=initial_state_h_0.name, final=final_state_h_0.name, ) rnn_proto.h.add( initial=initial_state_h_1.name, final=final_state_h_1.name, ) rnn_proto.c.add( initial=initial_state_c_0.name, final=final_state_c_0.name, ) rnn_proto.c.add( initial=initial_state_c_1.name, final=final_state_c_1.name, ) with open(os.path.join(FLAGS.save_path, "rnn.pbtxt"), "wb") as f: f.write(text_format.MessageToString(rnn_proto)) # Save the vocabulary. vocab = vocab_pb2.VocabProto() vocab.min_frequency = 1 word_to_id = { "<unk>": 0, "<s>": 1, "the": 2, } for w in word_to_id: item = vocab.item.add() item.id = word_to_id[w] item.word = w with open(os.path.join(FLAGS.save_path, "vocab.pbtxt"), "wb") as f: f.write(text_format.MessageToString(vocab)) # Note: graph_util.convert_variables_to_constants() appends ':0' onto the variable names, which # is why it isn't included in 'inference/lstm/predictions'. graph_def = graph_util.convert_variables_to_constants( sess=sess, input_graph_def=sess.graph.as_graph_def(), output_node_names=[predictions.name.split(':', 1)[0]]) tf.train.write_graph(graph_def, FLAGS.save_path, "graph.pb", as_text=False) tf.train.write_graph(graph_def, FLAGS.save_path, "graph.pbtxt")
def InitLazyGlobalSession(config_proto): assert type(config_proto) is job_set_pb.ConfigProto config_proto_str = text_format.MessageToString(config_proto) oneflow_api.InitLazyGlobalSession(config_proto_str)
if __name__ == '__main__': caffe.set_mode_gpu() p = make_parser() args = p.parse_args() # build and save testable net if not os.path.exists(args.out_dir): os.makedirs(args.out_dir) print "Building BN calc net..." testable_msg = make_testable(args.train_model) BN_calc_path = os.path.join( args.out_dir, '__for_calculating_BN_stats_' + os.path.basename(args.train_model)) with open(BN_calc_path, 'w') as f: f.write(text_format.MessageToString(testable_msg)) # use testable net to calculate BN layer stats print "Calculate BN stats..." train_ims, train_labs = extract_dataset(testable_msg) train_size = len(train_ims) minibatch_size = testable_msg.layer[0].dense_image_data_param.batch_size num_iterations = train_size // minibatch_size + train_size % minibatch_size in_h, in_w = (360, 480) test_net, test_msg = make_test_files(BN_calc_path, args.weights, num_iterations, in_h, in_w) # save deploy prototxt #print "Saving deployment prototext file..." #test_path = os.path.join(args.out_dir, "deploy.prototxt") #with open(test_path, 'w') as f:
def CurJobBuildAndInferCtx_SetTrainConf(train_config_proto): serialized_train_conf = str(text_format.MessageToString(train_config_proto)) oneflow_api.CurJobBuildAndInferCtx_SetTrainConf(serialized_train_conf)
def update_checkpoint_state_internal(save_dir, model_checkpoint_path, all_model_checkpoint_paths=None, latest_filename=None, save_relative_paths=False, all_model_checkpoint_timestamps=None, last_preserved_timestamp=None): """Updates the content of the 'checkpoint' file. This updates the checkpoint file containing a CheckpointState proto. Args: save_dir: Directory where the model was saved. model_checkpoint_path: The checkpoint file. all_model_checkpoint_paths: List of strings. Paths to all not-yet-deleted checkpoints, sorted from oldest to newest. If this is a non-empty list, the last element must be equal to model_checkpoint_path. These paths are also saved in the CheckpointState proto. latest_filename: Optional name of the checkpoint file. Default to 'checkpoint'. save_relative_paths: If `True`, will write relative paths to the checkpoint state file. all_model_checkpoint_timestamps: Optional list of timestamps (floats, seconds since the Epoch) indicating when the checkpoints in `all_model_checkpoint_paths` were created. last_preserved_timestamp: A float, indicating the number of seconds since the Epoch when the last preserved checkpoint was written, e.g. due to a `keep_checkpoint_every_n_hours` parameter (see `tf.contrib.checkpoint.CheckpointManager` for an implementation). Raises: RuntimeError: If any of the model checkpoint paths conflict with the file containing CheckpointSate. """ # Writes the "checkpoint" file for the coordinator for later restoration. coord_checkpoint_filename = _GetCheckpointFilename(save_dir, latest_filename) if save_relative_paths: if os.path.isabs(model_checkpoint_path): rel_model_checkpoint_path = os.path.relpath( model_checkpoint_path, save_dir) else: rel_model_checkpoint_path = model_checkpoint_path rel_all_model_checkpoint_paths = [] for p in all_model_checkpoint_paths: if os.path.isabs(p): rel_all_model_checkpoint_paths.append( os.path.relpath(p, save_dir)) else: rel_all_model_checkpoint_paths.append(p) ckpt = generate_checkpoint_state_proto( save_dir, rel_model_checkpoint_path, all_model_checkpoint_paths=rel_all_model_checkpoint_paths, all_model_checkpoint_timestamps=all_model_checkpoint_timestamps, last_preserved_timestamp=last_preserved_timestamp) else: ckpt = generate_checkpoint_state_proto( save_dir, model_checkpoint_path, all_model_checkpoint_paths=all_model_checkpoint_paths, all_model_checkpoint_timestamps=all_model_checkpoint_timestamps, last_preserved_timestamp=last_preserved_timestamp) if coord_checkpoint_filename == ckpt.model_checkpoint_path: raise RuntimeError( "Save path '%s' conflicts with path used for " "checkpoint state. Please use a different save path." % model_checkpoint_path) # Preventing potential read/write race condition by *atomically* writing to a # file. file_io.atomic_write_string_to_file(coord_checkpoint_filename, text_format.MessageToString(ckpt))
def GetOpParallelSymbolId(op_conf_proto): serialized_op_conf = str(text_format.MessageToString(op_conf_proto)) return oneflow_api.GetOpParallelSymbolId(serialized_op_conf)
def write_to_file(self): prototxt = open(APOLLO_ROOT + self.conf_file, 'w') prototxt.write(text_format.MessageToString(self.proto_root)) prototxt.close()
def build_example(label, param_dict_real, zip_path_label): """Build the model with parameter values set in param_dict_real. Args: label: Label of the model param_dict_real: Parameter dictionary (arguments to the factories make_graph and make_test_inputs) zip_path_label: Filename in the zip Returns: (tflite_model_binary, report) where tflite_model_binary is the serialized flatbuffer as a string and report is a dictionary with keys `toco_log` (log of toco conversion), `tf_log` (log of tf conversion), `toco` (a string of success status of the conversion), `tf` (a string success status of the conversion). """ np.random.seed(RANDOM_SEED) report = {"converter": report_lib.NOTRUN, "tf": report_lib.FAILED} # Build graph report["tf_log"] = "" report["converter_log"] = "" tf.reset_default_graph() with tf.Graph().as_default(): with tf.device("/cpu:0"): try: inputs, outputs = make_graph(param_dict_real) except (tf.errors.UnimplementedError, tf.errors.InvalidArgumentError, ValueError): report["tf_log"] += traceback.format_exc() return None, report sess = tf.Session() try: baseline_inputs, baseline_outputs = ( make_test_inputs(param_dict_real, sess, inputs, outputs)) except (tf.errors.UnimplementedError, tf.errors.InvalidArgumentError, ValueError): report["tf_log"] += traceback.format_exc() return None, report report["converter"] = report_lib.FAILED report["tf"] = report_lib.SUCCESS # Convert graph to toco input_tensors = [(input_tensor.name.split(":")[0], input_tensor.shape, input_tensor.dtype) for input_tensor in inputs] output_tensors = [_normalize_output_name(out.name) for out in outputs] # pylint: disable=g-long-ternary graph_def = freeze_graph( sess, tf.global_variables() + inputs + outputs) if use_frozen_graph else sess.graph_def if "split_tflite_lstm_inputs" in param_dict_real: extra_toco_options.split_tflite_lstm_inputs = param_dict_real[ "split_tflite_lstm_inputs"] tflite_model_binary, toco_log = options.tflite_convert_function( options, graph_def, input_tensors, output_tensors, extra_toco_options=extra_toco_options, test_params=param_dict_real) report["converter"] = ( report_lib.SUCCESS if tflite_model_binary is not None else report_lib.FAILED) report["converter_log"] = toco_log if options.save_graphdefs: zipinfo = zipfile.ZipInfo(zip_path_label + ".pbtxt") archive.writestr(zipinfo, text_format.MessageToString(graph_def), zipfile.ZIP_DEFLATED) if tflite_model_binary: if options.make_edgetpu_tests: # Set proper min max values according to input dtype. baseline_inputs, baseline_outputs = generate_inputs_outputs( tflite_model_binary, min_value=0, max_value=255) zipinfo = zipfile.ZipInfo(zip_path_label + ".bin") archive.writestr(zipinfo, tflite_model_binary, zipfile.ZIP_DEFLATED) example = {"inputs": baseline_inputs, "outputs": baseline_outputs} example_fp = StringIO() write_examples(example_fp, [example]) zipinfo = zipfile.ZipInfo(zip_path_label + ".inputs") archive.writestr(zipinfo, example_fp.getvalue(), zipfile.ZIP_DEFLATED) example_fp2 = StringIO() write_test_cases(example_fp2, zip_path_label + ".bin", [example]) zipinfo = zipfile.ZipInfo(zip_path_label + "_tests.txt") archive.writestr(zipinfo, example_fp2.getvalue(), zipfile.ZIP_DEFLATED) zip_manifest_label = zip_path_label + " " + label if zip_path_label == label: zip_manifest_label = zip_path_label zip_manifest.append(zip_manifest_label + "\n") return tflite_model_binary, report
def test_api(self): logging.getLogger().setLevel(logging.DEBUG) kvstore_type = 'etcd' db_base_dir = 'dp_test' os.environ['ETCD_BASE_DIR'] = db_base_dir data_portal_name = 'test_data_source' kvstore = DBClient(kvstore_type, True) kvstore.delete_prefix(db_base_dir) portal_input_base_dir='./portal_upload_dir' portal_output_base_dir='./portal_output_dir' raw_data_publish_dir = 'raw_data_publish_dir' portal_manifest = dp_pb.DataPortalManifest( name=data_portal_name, data_portal_type=dp_pb.DataPortalType.Streaming, output_partition_num=4, input_file_wildcard="*.done", input_base_dir=portal_input_base_dir, output_base_dir=portal_output_base_dir, raw_data_publish_dir=raw_data_publish_dir, processing_job_id=-1, next_job_id=0 ) kvstore.set_data(common.portal_kvstore_base_dir(data_portal_name), text_format.MessageToString(portal_manifest)) if gfile.Exists(portal_input_base_dir): gfile.DeleteRecursively(portal_input_base_dir) gfile.MakeDirs(portal_input_base_dir) all_fnames = ['1001/{}.done'.format(i) for i in range(100)] all_fnames.append('{}.xx'.format(100)) all_fnames.append('1001/_SUCCESS') for fname in all_fnames: fpath = os.path.join(portal_input_base_dir, fname) gfile.MakeDirs(os.path.dirname(fpath)) with gfile.Open(fpath, "w") as f: f.write('xxx') portal_master_addr = 'localhost:4061' portal_options = dp_pb.DataPotraMasterlOptions( use_mock_etcd=True, long_running=False, check_success_tag=True, ) data_portal_master = DataPortalMasterService( int(portal_master_addr.split(':')[1]), data_portal_name, kvstore_type, portal_options ) data_portal_master.start() channel = make_insecure_channel(portal_master_addr, ChannelType.INTERNAL) portal_master_cli = dp_grpc.DataPortalMasterServiceStub(channel) recv_manifest = portal_master_cli.GetDataPortalManifest(empty_pb2.Empty()) self.assertEqual(recv_manifest.name, portal_manifest.name) self.assertEqual(recv_manifest.data_portal_type, portal_manifest.data_portal_type) self.assertEqual(recv_manifest.output_partition_num, portal_manifest.output_partition_num) self.assertEqual(recv_manifest.input_file_wildcard, portal_manifest.input_file_wildcard) self.assertEqual(recv_manifest.input_base_dir, portal_manifest.input_base_dir) self.assertEqual(recv_manifest.output_base_dir, portal_manifest.output_base_dir) self.assertEqual(recv_manifest.raw_data_publish_dir, portal_manifest.raw_data_publish_dir) self.assertEqual(recv_manifest.next_job_id, 1) self.assertEqual(recv_manifest.processing_job_id, 0) self._check_portal_job(kvstore, all_fnames, portal_manifest, 0) mapped_partition = set() task_0 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=0)) task_0_1 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=0)) self.assertEqual(task_0, task_0_1) self.assertTrue(task_0.HasField('map_task')) mapped_partition.add(task_0.map_task.partition_id) self._check_map_task(task_0.map_task, all_fnames, task_0.map_task.partition_id, portal_manifest) portal_master_cli.FinishTask(dp_pb.FinishTaskRequest( rank_id=0, partition_id=task_0.map_task.partition_id, part_state=dp_pb.PartState.kIdMap) ) task_1 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=0)) self.assertTrue(task_1.HasField('map_task')) mapped_partition.add(task_1.map_task.partition_id) self._check_map_task(task_1.map_task, all_fnames, task_1.map_task.partition_id, portal_manifest) task_2 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=1)) self.assertTrue(task_2.HasField('map_task')) mapped_partition.add(task_2.map_task.partition_id) self._check_map_task(task_2.map_task, all_fnames, task_2.map_task.partition_id, portal_manifest) task_3 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=2)) self.assertTrue(task_3.HasField('map_task')) mapped_partition.add(task_3.map_task.partition_id) self._check_map_task(task_3.map_task, all_fnames, task_3.map_task.partition_id, portal_manifest) self.assertEqual(len(mapped_partition), portal_manifest.output_partition_num) portal_master_cli.FinishTask(dp_pb.FinishTaskRequest( rank_id=0, partition_id=task_1.map_task.partition_id, part_state=dp_pb.PartState.kIdMap) ) pending_1 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=4)) self.assertTrue(pending_1.HasField('pending')) pending_2 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=3)) self.assertTrue(pending_2.HasField('pending')) portal_master_cli.FinishTask(dp_pb.FinishTaskRequest( rank_id=1, partition_id=task_2.map_task.partition_id, part_state=dp_pb.PartState.kIdMap) ) portal_master_cli.FinishTask(dp_pb.FinishTaskRequest( rank_id=2, partition_id=task_3.map_task.partition_id, part_state=dp_pb.PartState.kIdMap) ) reduce_partition = set() task_4 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=0)) task_4_1 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=0)) self.assertEqual(task_4, task_4_1) self.assertTrue(task_4.HasField('reduce_task')) reduce_partition.add(task_4.reduce_task.partition_id) self._check_reduce_task(task_4.reduce_task, task_4.reduce_task.partition_id, portal_manifest) task_5 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=1)) self.assertTrue(task_5.HasField('reduce_task')) reduce_partition.add(task_5.reduce_task.partition_id) self._check_reduce_task(task_5.reduce_task, task_5.reduce_task.partition_id, portal_manifest) task_6 = portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=2)) self.assertTrue(task_6.HasField('reduce_task')) reduce_partition.add(task_6.reduce_task.partition_id) self._check_reduce_task(task_6.reduce_task, task_6.reduce_task.partition_id, portal_manifest) task_7= portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=3)) self.assertTrue(task_7.HasField('reduce_task')) reduce_partition.add(task_7.reduce_task.partition_id) self.assertEqual(len(reduce_partition), 4) self._check_reduce_task(task_7.reduce_task, task_7.reduce_task.partition_id, portal_manifest) task_8= portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=5)) self.assertTrue(task_8.HasField('pending')) portal_master_cli.FinishTask(dp_pb.FinishTaskRequest( rank_id=0, partition_id=task_4.reduce_task.partition_id, part_state=dp_pb.PartState.kEventTimeReduce) ) portal_master_cli.FinishTask(dp_pb.FinishTaskRequest( rank_id=1, partition_id=task_5.reduce_task.partition_id, part_state=dp_pb.PartState.kEventTimeReduce) ) portal_master_cli.FinishTask(dp_pb.FinishTaskRequest( rank_id=2, partition_id=task_6.reduce_task.partition_id, part_state=dp_pb.PartState.kEventTimeReduce) ) portal_master_cli.FinishTask(dp_pb.FinishTaskRequest( rank_id=3, partition_id=task_7.reduce_task.partition_id, part_state=dp_pb.PartState.kEventTimeReduce) ) time.sleep(31) task_9= portal_master_cli.RequestNewTask(dp_pb.NewTaskRequest(rank_id=5)) self.assertTrue(task_9.HasField('finished')) data_portal_master.stop() gfile.DeleteRecursively(portal_input_base_dir)
#get masks for regular convolutions blobmask = get_masks(ref_net, percentile) #get masks for coordinate|confidence convolutions blobmask.update(mboxes) #resize networks sizes = {k: sum(v) for k, v in blobmask.items()} train_par = resize_network(train_par, sizes) test_par = resize_network(test_par, sizes, verbose=False) dep_par = resize_network(dep_par, sizes, verbose=False) #write new parameters with open('models/ssd_face_pruned/face_train.prototxt', 'w') as f: f.write(txtf.MessageToString(train_par)) with open('models/ssd_face_pruned/face_test.prototxt', 'w') as f: f.write(txtf.MessageToString(test_par)) with open('models/ssd_face_pruned/face_deploy.prototxt', 'w') as f: f.write(txtf.MessageToString(dep_par)) #load pruned net with empty parameters new_net = caffe.Net('models/ssd_face_pruned/face_train.prototxt', 'models/empty.caffemodel', caffe.TRAIN) #copy masked parameters to pruned net set_params(ref_net, new_net, train_par, blobmask) #save pruned net parameters new_net.save('models/ssd_face_pruned/face_init.caffemodel')