Esempio n. 1
0
def produce_task(websocket_id, model_id, spec):

    model = load_model_helper(websocket_id, model_id)
    if not model:
        return

    if DEBUG_MODE:
        produce_data = model.produce(spec)
    else:
        try:
            produce_data = model.produce(spec)
        except Exception:
            logger.info("caught traceback when running produce:")
            logger.info(traceback.format_exc())

            ws_msg = WebsocketMessage.get_fail_message_with_data( \
                RECEIVE_PRODUCE_MSG,
                'produce failed due to exception',
                data={
                    'model_id': model.model_id,
                    'search_id': model.search_id,
                    'system': model.system
                })
            ws_msg.send_message(websocket_id)
            return

    ws_msg = WebsocketMessage.get_success_message(
        RECEIVE_PRODUCE_MSG,
        "produce successfully completed",
        data=produce_data)
    ws_msg.send_message(websocket_id)
Esempio n. 2
0
    def send_websocket_success_message(self):
        """Send a websocket message with the materialize_result data"""
        LOGGER.info('(5) send_websocket_success_message')
        if self.has_error():
            return

        if not self.websocket_id:
            LOGGER.info('(5a) no websocket_id')
            return

        LOGGER.info('(5b) send the message!')

        ws_string_info = json_dumps(self.new_workspace.to_dict())
        if not ws_string_info.success:
            user_msg = ('Sorry! An error occurred.  (Created workspace but'
                        ' failed JSON conversion.)')
            self.add_err_msg(user_msg)
            return

        ws_msg = WebsocketMessage.get_success_message(\
                    dm_static.DATAMART_AUGMENT_PROCESS,
                    ('The dataset has been augmented '
                     'and a new workspace created'),
                    msg_cnt=99,
                    data={
                        'workspace_json_string': ws_string_info.result_obj,
                        'augment_params': self.augment_params
                    })
        ws_msg.send_message(self.websocket_id)
        LOGGER.info('(5c) sent!')
Esempio n. 3
0
def describe_task(websocket_id, model_id):

    model = load_model_helper(websocket_id, model_id)
    if not model:
        return

    try:
        result = model.describe()
    except Exception:
        logger.info("caught traceback when running describe:")
        logger.info(traceback.format_exc())

        ws_msg = WebsocketMessage.get_fail_message_with_data( \
            RECEIVE_DESCRIBE_MSG,
            'describe failed due to exception',
            data={
                'model_id': model.model_id,
                'search_id': model.search_id,
                'system': model.system
            })
        ws_msg.send_message(websocket_id)
        return

    ws_msg = WebsocketMessage.get_success_message(
        RECEIVE_DESCRIBE_MSG, "describe successfully completed", data=result)
    ws_msg.send_message(websocket_id)
Esempio n. 4
0
def solve_task(websocket_id,
               system_id,
               specification,
               system_params=None,
               search_id=None):

    system_params = system_params or {}

    solver = Solve(system=system_id,
                   specification=specification,
                   callback_found='solve',
                   callback_arguments={
                       'specification': specification,
                       'websocket_id': websocket_id
                   },
                   system_params=system_params,
                   search_id=search_id)

    start_time = time.time()

    if DEBUG_MODE:
        result = solver.run()
    else:
        try:
            result = solver.run()
        except Exception:
            logger.info("caught traceback when running solver:")
            logger.info(traceback.format_exc())
            ws_msg = WebsocketMessage.get_fail_message_with_data( \
                RECEIVE_SOLVE_MSG,
                'solve failed due to exception',
                data={
                    'search_id': solver.search.search_id,
                    'system': solver.system
                })
            ws_msg.send_message(websocket_id)
            return

    stop_time = time.time()
    if KEY_DATA not in result:
        result[KEY_DATA] = {}
    result[KEY_DATA]['elapsed_time'] = stop_time - start_time

    ws_msg = WebsocketMessage.get_success_message(RECEIVE_SOLVE_MSG,
                                                  result.get('message'),
                                                  data=result.get('data'))
    ws_msg.send_message(websocket_id)
Esempio n. 5
0
    def send_websocket_success_message(self):
        """Send a websocket message with the materialize_result data"""
        LOGGER.info('(5) send_websocket_success_message')
        if self.has_error():
            return

        if not self.websocket_id:
            LOGGER.info('(5a) not websocket_id')
            return

        LOGGER.info('(5b) send the message!')
        ws_msg = WebsocketMessage.get_success_message(\
                    DATAMART_MATERIALIZE_PROCESS,
                    'The dataset has been materialized',
                    #data=dict(dog=1))
                    data=self.materialize_result)
        ws_msg.send_message(self.websocket_id)
        LOGGER.info('(5c) sent!')
Esempio n. 6
0
    def run_construct_dataset(self):
        """Go through the steps...."""
        LOGGER.info('>>> run_construct_dataset')

        try:
            self.user = User.objects.get(pk=self.user_id)
        except User.DoesNotExist:
            user_msg = 'No user found for id: %s' % self.user_id
            self.send_websocket_err_msg(user_msg)
            return

        for src_file in self.orig_source_files:
            if not isfile(src_file):
                user_msg = f'File does not exists: {src_file}'
                self.send_websocket_err_msg(user_msg)
                return

        if not isdir(self.writable_output_dir):
            user_msg = f'Directory does not exists: {self.writable_output_dir}'
            self.send_websocket_err_msg(user_msg)
            return

        LOGGER.info('(1) construct_folders')
        if not self.construct_folders():
            return

        LOGGER.info('(2) create dataset doc + move file to .csv')
        if not self.create_files():
            return

        LOGGER.info('(3) create_new_config')
        self.create_new_config()

        # self.send_websocket_err_msg(':( - the augment did not work')

        if not self.has_error() and self.websocket_id:
            ws_msg = WebsocketMessage.get_success_message(\
                        ADD_USER_DATASET_PROCESS,
                        'New user workspace created: %s' % self.new_workspace,
                        msg_cnt=1)
            ws_msg.send_message(self.websocket_id)
Esempio n. 7
0
    def send_websocket_success_message(self):
        """Send a websocket message with the materialize_result data"""
        LOGGER.info('(5) send_websocket_success_message')
        if self.has_error():
            return

        if not self.websocket_id:
            LOGGER.info('(5a) not websocket_id')
            return

        LOGGER.info('(5b) send the message!')

        resp_info = dict(datamart_name=self.datamart_name,
                         search_results=self.search_results)

        ws_msg = WebsocketMessage.get_success_message(\
                    dm_static.DATAMART_SEARCH_BY_DATASET,
                    'The dataset search is complete',
                    data=resp_info)

        ws_msg.send_message(self.websocket_id)
        LOGGER.info('(5c) sent!')
Esempio n. 8
0
    def run_construct_dataset(self):
        """Go through the steps...."""
        LOGGER.info('>>> run_construct_dataset')
        print(self.orig_source_file)
        if not isfile(self.orig_source_file):
            user_msg = 'File does not exists: %s' % self.orig_source_file
            self.send_websocket_err_msg(user_msg)
            return

        LOGGER.info('(1) retrieve workspace')
        if not self.retrieve_workspace():
            return

        LOGGER.info('(2) construct_folders')
        if not self.construct_folders():
            return

        LOGGER.info('(3) move_source_files')
        if not self.move_source_files():
            return

        #LOGGER.info('(4) create problem docs (and dataset doc, if needed)')
        #if not self.create_problem_data_docs():
        #    return

        LOGGER.info('(4) create_new_config')
        self.create_new_config()

        # self.send_websocket_err_msg(':( - the augment did not work')

        if not self.has_error() and self.websocket_id:
            ws_msg = WebsocketMessage.get_success_message(\
                        DATAMART_AUGMENT_PROCESS,
                        'New user workspace created: %s' % self.new_workspace,
                        msg_cnt=1)
            ws_msg.send_message(self.websocket_id)
Esempio n. 9
0
def stream_and_store_results(raven_json_str, stored_request_id,
                             grpc_req_obj_name, grpc_call_name, **kwargs):
    """Make the grpc call which has a streaming response

    grpc_req_obj_name: "core_pb2.GetSearchSolutionsResultsRequest", etc
    grpc_call_name: "GetSearchSolutionsResults", etc
    """
    core_stub, err_msg = TA2Connection.get_grpc_stub()
    if err_msg:
        StoredRequestUtil.set_error_status(stored_request_id, err_msg)
        return

    # optional: used to stream messages back to client via channels
    #
    websocket_id = kwargs.get('websocket_id', None)

    #
    grpc_req_obj = eval(grpc_req_obj_name)

    grpc_rpc_call_function = eval('core_stub.%s' % grpc_call_name)

    # --------------------------------
    # convert the JSON string to a gRPC request
    #  Yes: done for the 2nd time
    # --------------------------------
    try:
        req = Parse(raven_json_str, grpc_req_obj())
    except ParseError as err_obj:
        err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
        StoredRequestUtil.set_error_status(stored_request_id, err_msg)
        return

    # --------------------------------
    # Send the gRPC request
    # --------------------------------
    msg_cnt = 0
    try:
        # -----------------------------------------
        # Iterate through the streaming responses
        # -----------------------------------------
        for reply in grpc_rpc_call_function(\
                req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):

            msg_cnt += 1

            stored_resp = None  # to hold a StoredResponse object

            # -----------------------------------------
            # parse the response
            # -----------------------------------------
            msg_json_str = message_to_json(reply)

            msg_json_info = json_loads(msg_json_str)

            # -----------------------------------------
            # does it look ok?
            # -----------------------------------------
            if not msg_json_info.success:
                print('PROBLEM HERE TO LOG!')

                user_msg = 'failed to store response: %s' % \
                           msg_json_info.err_msg
                ws_msg = WebsocketMessage.get_fail_message(\
                            grpc_call_name, user_msg, msg_cnt=msg_cnt)
                ws_msg.send_message(websocket_id)

                continue

            # -----------------------------------------
            # Looks good, save the response
            # -----------------------------------------
            stored_resp_info = StoredResponse.add_response(\
                            stored_request_id,
                            response=msg_json_info.result_obj)

            # -----------------------------------------
            # Make sure the response was saved (probably won't happen)
            # -----------------------------------------
            if not stored_resp_info.success:
                # Not good but probably won't happen
                # send a message to the user...
                #
                user_msg = 'failed to store response: %s' % \
                            msg_json_info.err_msg
                ws_msg = WebsocketMessage.get_fail_message(\
                        grpc_call_name, user_msg, msg_cnt=msg_cnt)

                ws_msg.send_message(websocket_id)

                # Wait for the next response...
                continue

            # -----------------------------------------------
            # send responses back to any open WebSockets
            # ---------------------------------------------
            if websocket_id:
                stored_resp = stored_resp_info.result_obj

                ws_msg = WebsocketMessage.get_success_message(\
                                    grpc_call_name,
                                    'it worked',
                                    msg_cnt=msg_cnt,
                                    data=stored_resp.as_dict())

                print('ws_msg: %s' % ws_msg)
                #print('ws_msg', ws_msg.as_dict())

                ws_msg.send_message(websocket_id)

                StoredResponse.mark_as_read(stored_resp)
            # -----------------------------------------------

            print('msg received #%d' % msg_cnt)

    except grpc.RpcError as err_obj:
        StoredRequestUtil.set_error_status(\
                        stored_request_id,
                        str(err_obj))
        return

    #except Exception as err_obj:
    #    StoredRequestUtil.set_error_status(\
    #                    stored_request_id,
    #                    str(err_obj))
    #    return

    StoredRequestUtil.set_finished_ok_status(stored_request_id)
    def run_describe_solution(self, pipeline_id, solution_id, msg_cnt=-1):
        """sync: Run a DescribeSolution call for each solution_id"""
        print(f'run_describe_solution 1. pipeline_id: {pipeline_id}')
        # ----------------------------------
        # Create the input
        # ----------------------------------
        req_params = {ta2_static.KEY_SOLUTION_ID: solution_id}
        json_str_info = json_dumps(req_params)
        if not json_str_info.success:
            self.add_err_msg(json_str_info.err_msg)
            return

        json_str_input = json_str_info.result_obj

        # --------------------------------
        # (2) Save request
        # --------------------------------
        stored_request = StoredRequest(\
                        user=self.user_object,
                        search_id=self.search_id,
                        pipeline_id=pipeline_id,
                        workspace='(not specified)',
                        request_type=ta2_static.DESCRIBE_SOLUTION,
                        is_finished=False,
                        request=req_params)
        stored_request.save()

        # --------------------------------
        # (2a) Behavioral logging
        # --------------------------------
        log_data = dict(session_key=self.session_key,
                        feature_id=ta2_static.DESCRIBE_SOLUTION,
                        activity_l1=bl_static.L1_MODEL_SELECTION,
                        activity_l2=bl_static.L2_MODEL_SUMMARIZATION,
                        other=req_params)

        LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)

        print(
            f'run_describe_solution 2. stored_request.pipeline_id: {stored_request.pipeline_id}'
        )

        # ----------------------------------
        # Run Describe Solution
        # ----------------------------------
        describe_info = describe_solution(json_str_input)
        if not describe_info.success:
            self.add_err_msg(describe_info.err_msg)
            StoredResponse.add_err_response(\
                                stored_request,
                                describe_info.err_msg)
            return

        # ----------------------------------
        # Parse the DescribeSolutionResponse
        # ----------------------------------
        describe_data_info = json_loads(describe_info.result_obj)
        if not describe_data_info.success:
            self.add_err_msg(describe_data_info.err_msg)
            StoredResponse.add_err_response(\
                                stored_request,
                                describe_data_info.err_msg)
            return

        # -----------------------------------------------
        # Add the pipline id to the result
        # -----------------------------------------------
        describe_data = describe_data_info.result_obj

        describe_data[ta2_static.KEY_PIPELINE_ID] = pipeline_id
        describe_data[ta2_static.KEY_SEARCH_ID] = self.search_id
        describe_data[ta2_static.KEY_SOLUTION_ID] = solution_id
        describe_data.move_to_end(ta2_static.KEY_PIPELINE_ID, last=False)

        # params = dict()
        # if not stored_request.pipeline_id:
        #    params['pipeline_id'] = describe_data[KEY_PIPELINE_ID]

        stored_info = StoredResponse.add_success_response(\
                                            stored_request,
                                            describe_data,
                                            pipeline_id=pipeline_id)

        if not stored_info.success:
            print('stored info fail!', stored_info.err_msg)

        print(
            f'run_describe_solution 3. stored_info.result_obj.pipeline_id: {stored_info.result_obj.pipeline_id}'
        )

        print(
            f'run_describe_solution 4. stored_request.pipeline_id: {stored_request.pipeline_id}'
        )

        # -----------------------------------------
        # Tracking this in the behavioral log,
        #  e.g. checking time lapse between creation
        #   of solution and if user investigates this model,
        #  later, if at all
        # -----------------------------------------
        log_data = dict(session_key=self.session_key,
                        feature_id=ta2_static.DESCRIBE_SOLUTION_RESPONSE,
                        activity_l1=bl_static.L1_MODEL_SELECTION,
                        activity_l2=bl_static.L2_MODEL_SEARCH,
                        other=describe_data)

        LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)

        # -----------------------------------------------
        # send responses back to WebSocket
        # ---------------------------------------------
        ws_msg = WebsocketMessage.get_success_message(\
                    'DescribeSolution',
                    'it worked',
                    msg_cnt=msg_cnt,
                    data=describe_data)

        print('ws_msg: %s' % ws_msg)
        #print('ws_msg', ws_msg.as_dict())

        ws_msg.send_message(self.websocket_id)
    def run_get_search_solution_results(self):
        """Run SearchSolutions against a TA2"""

        # -----------------------------------
        # (1) make GRPC request object
        # -----------------------------------
        params_dict = dict(searchId=self.search_id)
        params_info = json_dumps(params_dict)
        if not params_info.success:
            self.send_websocket_err_msg(\
                    ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
                    params_info.err_msg)
            return

        try:
            grpc_req = Parse(params_info.result_obj,
                             core_pb2.GetSearchSolutionsResultsRequest())
        except ParseError as err_obj:
            err_msg = ('GetSearchSolutionsResultsRequest: Failed to'
                       ' convert JSON to gRPC: %s') % (err_obj)
            self.send_websocket_err_msg(\
                    ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
                    params_info.err_msg)
            return

        # --------------------------------
        # (2) Save the request to the db
        # --------------------------------
        stored_request = StoredRequest(\
                        user=self.user_object,
                        search_id=self.search_id,
                        workspace='(not specified)',
                        request_type=ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
                        is_finished=False,
                        request=params_dict)
        stored_request.save()

        # --------------------------------
        # (2a) Behavioral logging
        # --------------------------------
        log_data = dict(session_key=self.session_key,
                        feature_id=ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
                        activity_l1=bl_static.L1_MODEL_SELECTION,
                        activity_l2=bl_static.L2_MODEL_SEARCH,
                        other=params_dict)

        LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)

        # --------------------------------
        # (3) Make the gRPC request
        # --------------------------------
        core_stub, err_msg = TA2Connection.get_grpc_stub()
        if err_msg:
            return err_resp(err_msg)

        msg_cnt = 0
        try:
            # -----------------------------------------
            # Iterate through the streaming responses
            # Note: The StoredResponse.id becomes the pipeline id
            # -----------------------------------------
            for reply in core_stub.GetSearchSolutionsResults(\
                    grpc_req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):

                msg_cnt += 1

                # -----------------------------------------------
                # Parse the response into JSON + store response
                # -----------------------------------------------
                msg_json_str = message_to_json(reply)
                msg_json_info = json_loads(msg_json_str)

                if not msg_json_info.success:
                    user_msg = 'Failed to convert response to JSON: %s' % \
                               msg_json_info.err_msg

                    self.send_websocket_err_msg(\
                                    ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
                                    user_msg)

                    StoredResponse.add_stream_err_response(\
                                        stored_response, user_msg)
                    # Wait for next response....
                    continue

                result_json = msg_json_info.result_obj

                # TA2s (specifically NYU) responds once when trying a new pipeline, with a message missing a solutionId
                # the same process responds again once the solution contains a solutionId
                print('results json from TA2')
                print(result_json)

                if not result_json.get('solutionId'):
                    continue

                if ta2_static.KEY_SOLUTION_ID not in result_json:
                    user_msg = '"%s" not found in response to JSON: %s' % \
                               (ta2_static.KEY_SOLUTION_ID, result_json)

                    StoredResponse.add_stream_err_response(\
                                        stored_response, user_msg)

                    self.send_websocket_err_msg(\
                                    ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
                                    user_msg)

                    # Wait for next response....
                    continue

                # Solution id used for DescribeSolution...
                #
                solution_id = result_json[ta2_static.KEY_SOLUTION_ID]

                # -----------------------------------------
                # Looks good, save the response
                # -----------------------------------------
                stored_resp_info = StoredResponse.add_stream_success_response(\
                                    stored_request, result_json)

                # -----------------------------------------
                # Tracking this in the behavioral log,
                #  e.g. checking time lapse between creation
                #   of solution and if user investigates this model,
                #  later, if at all
                # -----------------------------------------
                log_data = dict(session_key=self.session_key,
                                feature_id=ta2_static.
                                GET_SEARCH_SOLUTIONS_RESULTS_RESPONSE,
                                activity_l1=bl_static.L1_MODEL_SELECTION,
                                activity_l2=bl_static.L2_MODEL_SEARCH,
                                other=result_json)

                LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)

                # -----------------------------------------
                # Make sure the response was saved (probably won't happen)
                # -----------------------------------------
                if not stored_resp_info.success:
                    # Not good but probably won't happen
                    # send a message to the user...
                    #
                    user_msg = 'Failed to store response from %s: %s' % \
                                (ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
                                 msg_json_info.err_msg)

                    StoredResponse.add_stream_err_response(\
                                        stored_response, user_msg)

                    self.send_websocket_err_msg(\
                                    ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
                                    user_msg)

                    # Wait for the next response...
                    continue

                # ---------------------------------------------
                # Looks good!  Get the StoredResponse
                # - This id will be used as the pipeline id
                # ---------------------------------------------
                stored_response = stored_resp_info.result_obj
                stored_response.use_id_as_pipeline_id()

                StoredResponse.add_stream_success_response(\
                                    stored_response, stored_response)

                # -----------------------------------------------
                # send responses back to WebSocket
                # ---------------------------------------------
                ws_msg = WebsocketMessage.get_success_message(\
                            ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
                            'it worked',
                            msg_cnt=msg_cnt,
                            data=stored_response.as_dict())

                print('ws_msg: %s' % ws_msg)
                #print('ws_msg', ws_msg.as_dict())

                ws_msg.send_message(self.websocket_id)

                stored_response.mark_as_sent_to_user()
                print('msg received #%d' % msg_cnt)
                # -----------------------------------------------
                # continue the process describe/score/etc
                # -----------------------------------------------

                # DescribeSolution - run sync
                #
                self.run_describe_solution(stored_response.pipeline_id,
                                           solution_id, msg_cnt)

                # FitSolution - run async
                #
                print('PRE run_fit_solution')
                self.run_fit_solution(stored_response.pipeline_id, solution_id)
                print('POST run_fit_solution')

                print('PRE run_score_solution')
                self.run_score_solution(stored_response.pipeline_id,
                                        solution_id)
                print('POST run_score_solution')

            # -----------------------------------------------
            # All results arrived, send message to UI
            # -----------------------------------------------
            ws_msg = WebsocketMessage.get_success_message( \
                ta2_static.ENDGetSearchSolutionsResults,
                {'searchId': self.search_id, 'message': 'it worked'})

            print('ws_msg: %s' % ws_msg)
            ws_msg.send_message(self.websocket_id)

        except grpc.RpcError as err_obj:
            stored_request.set_error_status(str(err_obj))
            return

        except Exception as err_obj:
            stored_request.set_error_status(str(err_obj))
            return

        StoredRequestUtil.set_finished_ok_status(stored_request.id)
Esempio n. 12
0
    def run_get_score_solution_responses(self, request_id):
        """(2) Run GetScoreSolutionResults"""
        if self.has_error():
            return

        if not request_id:
            self.send_websocket_err_msg(ta2_static.GET_SCORE_SOLUTION_RESULTS,
                                        'request_id must be set')
            return

        # -----------------------------------
        # (1) make GRPC request object
        # -----------------------------------
        params_dict = {ta2_static.KEY_REQUEST_ID: request_id}
        params_info = json_dumps(params_dict)

        try:
            grpc_req = Parse(params_info.result_obj,
                             core_pb2.GetScoreSolutionResultsRequest())
        except ParseError as err_obj:
            err_msg = ('Failed to convert JSON to gRPC: %s') % (err_obj)
            self.send_websocket_err_msg(ta2_static.GET_SCORE_SOLUTION_RESULTS,
                                        err_msg)
            return

        # --------------------------------
        # (2) Save the request to the db
        # --------------------------------
        stored_request = StoredRequest(\
                        user=self.user_object,
                        request_type=ta2_static.GET_SCORE_SOLUTION_RESULTS,
                        search_id=self.search_id,
                        pipeline_id=self.pipeline_id,
                        is_finished=False,
                        request=params_dict)
        stored_request.save()

        # --------------------------------
        # (2a) Behavioral logging
        # --------------------------------
        log_data = dict(session_key=self.session_key,
                        feature_id=ta2_static.GET_SCORE_SOLUTION_RESULTS,
                        activity_l1=bl_static.L1_MODEL_SELECTION,
                        activity_l2=bl_static.L2_MODEL_SUMMARIZATION,
                        other=params_dict)

        LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)

        # --------------------------------
        # (3) Make the gRPC request
        # --------------------------------
        core_stub, err_msg = TA2Connection.get_grpc_stub()
        if err_msg:
            return err_resp(err_msg)

        msg_cnt = 0
        try:
            # -----------------------------------------
            # Iterate through the streaming responses
            # Note: The StoredResponse.id becomes the pipeline id
            # -----------------------------------------
            for reply in core_stub.GetScoreSolutionResults(\
                    grpc_req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):

                msg_cnt += 1

                stored_response = None  # to hold a StoredResponse object

                # -----------------------------------------------
                # Parse the response into JSON + store response
                # -----------------------------------------------
                msg_json_str = message_to_json(reply)
                msg_json_info = json_loads(msg_json_str)

                if not msg_json_info.success:
                    err_msg = ('Failed to convert JSON to gRPC: %s') % \
                               (err_obj,)
                    StoredResponse.add_stream_err_response(
                        stored_request, user_msg)

                    self.send_websocket_err_msg(\
                            ta2_static.GET_SCORE_SOLUTION_RESULTS,
                            err_msg)
                    # Wait for next response....
                    continue

                result_json = msg_json_info.result_obj

                # -----------------------------------------
                # Looks good, save the response
                # -----------------------------------------
                stored_resp_info = StoredResponse.add_stream_success_response(\
                                    stored_request, result_json)

                # -----------------------------------------
                # Make sure the response was saved (probably won't happen)
                # -----------------------------------------
                if not stored_resp_info.success:
                    # Not good but probably won't happen
                    # send a message to the user...
                    #
                    self.send_websocket_err_msg(\
                                    ta2_static.GET_SCORE_SOLUTION_RESULTS,
                                    stored_resp_info.err_msg)
                    #
                    StoredResponse.add_stream_err_response(\
                                    stored_request, stored_resp_info.err_msg)
                    #
                    continue

                # ---------------------------------------------
                # Looks good!  Get the StoredResponse
                # - send responses back to WebSocket
                # ---------------------------------------------
                stored_response = stored_resp_info.result_obj
                stored_response.set_pipeline_id(self.pipeline_id)

                # ---------------------------------------------
                # If progress is complete,
                #  send response back to WebSocket
                # ---------------------------------------------
                progress_val = get_dict_value(\
                                result_json,
                                [ta2_static.KEY_PROGRESS,
                                 ta2_static.KEY_PROGRESS_STATE])

                if (not progress_val.success) or \
                   (progress_val.result_obj != ta2_static.KEY_PROGRESS_COMPLETED):
                    user_msg = 'GetScoreSolutionResultsResponse is not yet complete'
                    LOGGER.info(user_msg)
                    # wait for next message...
                    continue


                ws_msg = WebsocketMessage.get_success_message(\
                            ta2_static.GET_SCORE_SOLUTION_RESULTS,
                            'it worked',
                            msg_cnt=msg_cnt,
                            data=stored_response.as_dict())

                LOGGER.info('ws_msg: %s' % ws_msg)
                #print('ws_msg', ws_msg.as_dict())

                ws_msg.send_message(self.websocket_id)
                # stored_response.mark_as_sent_to_user()

        except grpc.RpcError as err_obj:
            stored_request.set_error_status(str(err_obj))
            return

        except Exception as err_obj:
            stored_request.set_error_status(str(err_obj))
            return

        StoredRequestUtil.set_finished_ok_status(stored_request.id)