def _replace_behv_feature_json(params): _replace_behv_feature_proto(Parse(json.dumps(params), ReplaceBehvFeatureMessage()))
def get_execute_pipeline_results(info_str=None): """Ask a TA2 to GetExecutePipelineResults via gRPC""" if info_str is None: info_str = get_test_info_str() if info_str is None: err_msg = 'UI Str for PipelineExecuteResultsRequest is None' return get_failed_precondition_response(err_msg) # -------------------------------- # Is this valid JSON? # -------------------------------- try: info_dict = json.loads(info_str, object_pairs_hook=OrderedDict) except json.decoder.JSONDecodeError as err_obj: err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj) return get_failed_precondition_response(err_msg) # -------------------------------- # convert the JSON string to a gRPC request # -------------------------------- try: req = Parse(info_str, core_pb2.PipelineExecuteResultsRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return get_failed_precondition_response(err_msg) if settings.TA2_STATIC_TEST_MODE: template_info = get_predict_file_info_dict() template_str = get_grpc_test_json( 'test_responses/execute_results_ok.json', template_info) embed_util = FileEmbedUtil(template_str) if embed_util.has_error: return get_failed_precondition_response(embed_util.error_message) return embed_util.get_final_results() #return get_grpc_test_json('test_responses/execute_results_ok.json', # dict()) # -------------------------------- # Get the connection, return an error if there are channel issues # -------------------------------- core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return get_failed_precondition_response(err_msg) #print('req: %s' % req) # -------------------------------- # Send the gRPC request - returns a stream # -------------------------------- try: reply = core_stub.GetExecutePipelineResults(req) except grpc.RpcError as ex: return get_failed_precondition_response(str(ex)) except Exception as ex: return get_failed_precondition_response(str(ex)) #print('reply', reply) """ if reply and str(reply) == VAL_GRPC_STATE_CODE_NONE: err_msg = ('Unknown gRPC state.' ' (Was an ExecutePipeline request sent?)') return get_failed_precondition_response(err_msg) """ try: print(MessageToJson(reply)) except: print('failed unary convert to JSON') #print('reply: %s' % reply) # -------------------------------- # Convert the reply to JSON and send it on # -------------------------------- results = map(MessageToJson, reply) result_str = '[' + ', '.join(results) + ']' embed_util = FileEmbedUtil(result_str) if embed_util.has_error: return get_failed_precondition_response(embed_util.error_message) return embed_util.get_final_results()
def test_get_model_metadata_rest(self, model_version_policy_models, start_server_model_ver_policy, model_name, throw_error): """ <b>Description</b> Execute GetModelMetadata request using REST API interface hosting multiple models <b>input data</b> - directory with 2 models in IR format - docker image <b>fixtures used</b> - model downloader - input data downloader - service launching <b>Expected results</b> - response contains proper response about model metadata for both models set in config file: model resnet_v1_50, pnasnet_large - both served models handles appropriate input formats """ print("Downloaded model files:", model_version_policy_models) print("Getting info about model") versions = [1, 2, 3] expected_outputs_metadata = [{ 'detection_out': { 'dtype': 1, 'shape': [1, 1, 200, 7] } }, { 'detection_out': { 'dtype': 1, 'shape': [1, 1, 200, 7] } }, { 'age': { 'dtype': 1, 'shape': [1, 1, 1, 1] }, 'gender': { 'dtype': 1, 'shape': [1, 2, 1, 1] } }] expected_inputs_metadata = [{ 'data': { 'dtype': 1, 'shape': [1, 3, 300, 300] } }, { 'data': { 'dtype': 1, 'shape': [1, 3, 1024, 1024] } }, { 'new_key': { 'dtype': 1, 'shape': [1, 3, 62, 62] } }] for x in range(len(versions)): print("Getting info about model version:".format(versions[x])) expected_input_metadata = expected_inputs_metadata[x] expected_output_metadata = expected_outputs_metadata[x] rest_url = 'http://localhost:5560/v1/models/{}/' \ 'versions/{}/metadata'.format(model_name, versions[x]) result = requests.get(rest_url) print(result.text) if not throw_error[x]: output_json = result.text metadata_pb = get_model_metadata_pb2. \ GetModelMetadataResponse() response = Parse(output_json, metadata_pb, ignore_unknown_fields=False) input_metadata, output_metadata = model_metadata_response( response=response) print(output_metadata) assert model_name == response.model_spec.name assert expected_input_metadata == input_metadata assert expected_output_metadata == output_metadata else: assert 404 == result.status_code
def parse_proto(json_message): message = KafkaServingMessage() Parse(json_message, message) return message
def parse_replay(replay_player_path, sampled_action_path, reward, race, enemy_race, stat): with open( os.path.join(FLAGS.parsed_replay_path, 'GlobalInfos', replay_player_path)) as f: global_info = json.load(f) feat = SpatialFeatures( Parse(global_info['game_info'], sc_pb.ResponseGameInfo())) states = [ obs for obs in stream.parse( os.path.join(FLAGS.parsed_replay_path, 'SampledObservations', replay_player_path), sc_pb.ResponseObservation) ] # Sampled Actions with open(sampled_action_path) as f: sampled_action = json.load(f) sampled_action_id = [id // FLAGS.step_mul + 1 for id in sampled_action] # Actions with open( os.path.join(FLAGS.parsed_replay_path, 'Actions', replay_player_path)) as f: actions = json.load(f) actions = [ None if len(actions[idx]) == 0 else Parse(actions[idx][0], sc_pb.Action()) for idx in sampled_action_id ] assert len(states) == len(actions) spatial_states_np, global_states_np = [], [] for state, action in zip(states, actions): action_id = -1 if action is not None: try: func_id = feat.reverse_action(action).function func_name = FUNCTIONS[func_id].name if func_name.split('_')[0] in { 'Build', 'Train', 'Research', 'Morph', 'Cancel', 'Halt', 'Stop' }: action_id = func_id except: pass obs = feat.transform_obs(state.observation) spatial_states_np.append( np.concatenate([obs['screen'], obs['minimap']], axis=0)) global_states_np.append( np.hstack([ obs['player'] / (stat['max'] + 1e-5), obs['score'], [reward], [stat['action_id'][action_id]] ])) spatial_states_np = np.asarray(spatial_states_np) global_states_np = np.asarray(global_states_np) spatial_states_np = spatial_states_np.reshape([len(states), -1]) sparse.save_npz( os.path.join(FLAGS.parsed_replay_path, 'SpatialFeatureTensor', replay_player_path + '@S'), sparse.csc_matrix(spatial_states_np)) sparse.save_npz( os.path.join(FLAGS.parsed_replay_path, 'SpatialFeatureTensor', replay_player_path + '@G'), sparse.csc_matrix(global_states_np))
) subprocess.run( f"protoc --plugin=protoc-gen-custom=../plugin.py --custom_out=. {os.path.basename(filename)}", shell=True, ) for filename in json_files: # Reset the internal symbol database so we can import the `Test` message # multiple times. Ugh. sym = symbol_database.Default() sym.pool = DescriptorPool() parts = get_base(filename).split("-") out = filename.replace(".json", ".bin") print(f"Using {parts[0]}_pb2 to generate {os.path.basename(out)}") imported = importlib.import_module(f"{parts[0]}_pb2") input_json = open(filename).read() parsed = Parse(input_json, imported.Test()) serialized = parsed.SerializeToString() preserve = "casing" not in filename serialized_json = MessageToJson(parsed, preserving_proto_field_name=preserve) s_loaded = json.loads(serialized_json) in_loaded = json.loads(input_json) if s_loaded != in_loaded: raise AssertionError("Expected JSON to be equal:", s_loaded, in_loaded) open(out, "wb").write(serialized)
def from_json(json_data): pbdata = xrd_pb2.TokenMetadata() Parse(json_data, pbdata) return TokenMetadata(pbdata)
def _update_malwr_info_json(params): _update_malwr_info_proto(Parse(json.dumps(params), UpdateMalwrInfoMessage()))
def _add_file_type_json(params): _add_file_type_proto(Parse(json.dumps(params), AddFileTypeMessage()))
def _update_hybrid_info_json(params): _update_hybrid_info_proto(Parse(json.dumps(params), UpdateHybridInfoMessage()))
def _update_reverseit_info_json(params): _update_reverseit_info_proto(Parse(json.dumps(params), UpdateReverseItInfoMessage()))
def _update_vt_info_json(params): _update_vt_info_proto(Parse(json.dumps(params), UpdateVtInfoMessage()))
def _clear_behv_feature_json(params): _clear_behv_feature_proto(Parse(json.dumps(params), ClearBehvFeatureMessage()))
def _add_host_os_json(params): _add_host_os_proto(Parse(json.dumps(params), AddHostOsMessage()))
def get_search_solutions_results(raven_json_str, user_obj, websocket_id=None): """ Send a GetSearchSolutionsResultsRequest to the GetSearchSolutionsResults command """ if user_obj is None: return err_resp("The user_obj cannot be None") if not raven_json_str: err_msg = 'No data found for the GetSearchSolutionsResultsRequest' return err_resp(err_msg) # -------------------------------- # Make sure it's valid JSON # -------------------------------- raven_json_info = json_loads(raven_json_str) if not raven_json_info.success: return err_resp(raven_json_info.err_msg) # -------------------------------- # convert the JSON string to a gRPC request # Done for error checking; call repeated in celery task # -------------------------------- try: req = Parse(raven_json_str, core_pb2.GetSearchSolutionsResultsRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return err_resp(err_msg) # -------------------------------- # Save the request to the db # -------------------------------- stored_request = StoredRequest(\ user=user_obj, workspace='(not specified)', request_type='GetSearchSolutionsResults', is_finished=False, request=raven_json_info.result_obj) stored_request.save() # In test mode, return canned response # if settings.TA2_STATIC_TEST_MODE: resp_str = get_grpc_test_json(\ 'test_responses/GetSearchSolutionsResultsResponse_ok.json', dict()) resp_info = json_loads(resp_str) if not resp_info.success: return err_resp(resp_info.err_msg) # Save the stored response # StoredResponse.add_response(\ stored_request.id, response=resp_info.result_obj) StoredRequestUtil.set_finished_ok_status(stored_request.id) # Return the stored **request** (not response) # return ok_resp(stored_request.as_dict()) stream_and_store_results.delay(raven_json_str, stored_request.id, 'core_pb2.GetSearchSolutionsResultsRequest', 'GetSearchSolutionsResults', websocket_id=websocket_id) return ok_resp(stored_request.as_dict())
def _del_file_type_json(params): _del_file_type_proto(Parse(json.dumps(params), DelFileTypeMessage()))
def from_json(json_data): pbdata = qrl_pb2.EncryptedEphemeralMessage() Parse(json_data, pbdata) return EncryptedEphemeralMessage(pbdata)
def _clone_host_os_json(params): _clone_host_os_proto(Parse(params, CloneHostOsMessage()))
def copy_proto(proto): newproto = type(proto)() Parse(json.dumps(proto_to_dict(proto)), newproto) return newproto
def _add_platform_json(params): _add_platform_proto(Parse(json.dumps(params), AddPlatformMessage()))
def start_session(raven_json_str=None): """Start session command This command sends a UserAgent and the protocol version to the TA2 service """ if raven_json_str is None: err_msg = 'No data found. Please send a "user_agent"' return get_failed_precondition_sess_response(err_msg) # Default if the user_agent is not from the UI #raven_dict = dict(user_agent=settings.TA2_GPRC_USER_AGENT) # The UI has sent JSON in string format that contains the user_agent try: raven_dict = json.loads(raven_json_str) except json.decoder.JSONDecodeError as err_obj: err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj) return get_failed_precondition_sess_response(err_msg) # check for a user_agent # if not KEY_USER_AGENT_FROM_UI in raven_dict: return get_failed_precondition_sess_response(ERR_MSG_NO_USER_AGENT) # The protocol version always comes from the latest # version we have in the repo (just copied in for now) # raven_dict['version'] = TA2Connection.get_protocol_version() # -------------------------------- # Convert back to string for TA2 call # -------------------------------- content = json.dumps(raven_dict) # -------------------------------- # convert the JSON string to a gRPC request # -------------------------------- try: req = Parse(content, core_pb2.SessionRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return get_failed_precondition_sess_response(err_msg) # In test mode, check if the incoming JSON is legit (in line above) # -- then return canned response # if settings.TA2_STATIC_TEST_MODE: rnd_session_id = random_info.get_alphanumeric_string(7) info_dict = dict(session_id=rnd_session_id, api_version=TA3TA2Util.get_api_version()) return get_grpc_test_json('test_responses/startsession_ok.json', info_dict) #if random.randint(1,10) == 3: # return get_grpc_test_json('test_responses/startsession_badassertion.json') #else: # return get_grpc_test_json('test_responses/startsession_ok.json', d) # -------------------------------- # Get the connection, return an error if there are channel issues # -------------------------------- core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return get_failed_precondition_sess_response(err_msg) #return dict(status=core_pb2.FAILED_PRECONDITION, # details=err_msg) # -------------------------------- # Send the gRPC request # -------------------------------- try: reply = core_stub.StartSession(req) except Exception as ex: return get_failed_precondition_sess_response(str(ex)) # -------------------------------- # Convert the reply to JSON and send it back # -------------------------------- return MessageToJson(reply)
def _del_platform_json(params): _del_platform_proto(Parse(json.dumps(params), DelPlatformMessage()))
def pipeline_create(info_str=None): """Send the pipeline create request via gRPC""" if info_str is None: info_str = get_test_info_str() if info_str is None: err_msg = 'UI Str for %s is None' % PIPELINE_CREATE_REQUEST return get_failed_precondition_response(err_msg) # -------------------------------- # Convert info string to dict # -------------------------------- try: info_dict = json.loads(info_str, object_pairs_hook=OrderedDict) except json.decoder.JSONDecodeError as err_obj: err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj) return get_failed_precondition_response(err_msg) if KEY_CONTEXT_FROM_UI not in info_dict: return get_failed_precondition_response(ERR_NO_CONTEXT) if KEY_SESSION_ID_FROM_UI not in info_dict[KEY_CONTEXT_FROM_UI]: return get_failed_precondition_response(ERR_NO_SESSION_ID) # -------------------------------- # convert the JSON string to a gRPC request # -------------------------------- try: req = Parse(info_str, core_pb2.PipelineCreateRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return get_failed_precondition_response(err_msg) if settings.TA2_STATIC_TEST_MODE: template_info = get_predict_file_info_dict(info_dict.get('task')) template_str = get_grpc_test_json('test_responses/createpipeline_ok.json', template_info) # These next lines embed file uri content into the JSON embed_util = FileEmbedUtil(template_str) if embed_util.has_error: return get_failed_precondition_response(embed_util.error_message) return embed_util.get_final_results() #return get_grpc_test_json('test_responses/createpipeline_ok.json', # template_info) # -------------------------------- # Get the connection, return an error if there are channel issues # -------------------------------- core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return get_failed_precondition_response(err_msg) # -------------------------------- # Send the gRPC request # -------------------------------- messages = [] try: for reply in core_stub.CreatePipelines(req): user_msg = MessageToJson(reply) print(user_msg) messages.append(user_msg) except Exception as ex: return get_reply_exception_response(str(ex)) print('end of queue. make message list') result_str = '['+', '.join(messages)+']' print('embed file contents') embed_util = FileEmbedUtil(result_str) if embed_util.has_error: print('file embed error') return get_failed_precondition_response(embed_util.error_message) print('return results') return embed_util.get_final_results()
def _add_sample_relationship_type_json(params): _add_sample_relationship_type_proto(Parse(json.dumps(params), AddSampleRelationshipTypeMessage()))
def request_generator(message_bodies: List[dict], request_dataclass: Any): for message_body in message_bodies: yield Parse(json.dumps(message_body), request_dataclass())
def _del_sample_relationship_type_json(params): _del_sample_relationship_type_proto(Parse(json.dumps(params), DelSampleRelationshipTypeMessage()))
def validate(self): route = Route() Parse(json.dumps(self.values), route) return MessageToJson(route, indent=None)
def _add_overall_malicious_type_json(params): _add_overall_malicious_type_proto(Parse(json.dumps(params), AddOverallMaliciousTypeMessage()))
def from_json(json_data): pbdata = qrl_pb2.Block() Parse(json_data, pbdata) return Block(pbdata)
def _append_behv_feature_json(params): _append_behv_feature_proto(Parse(json.dumps(params), AppendBehvFeatureMessage()))