コード例 #1
0
ファイル: test_util.py プロジェクト: Yelp/mycroft
def test_parse_results(records, res_status, res_lsd):
    status, lsd, _ = parse_results(records)
    assert status == res_status
    assert lsd == res_lsd
コード例 #2
0
ファイル: test_util.py プロジェクト: wlstyy/mycroft
def test_parse_results(records, res_status, res_lsd):
    status, lsd, _ = parse_results(records)
    assert status == res_status
    assert lsd == res_lsd
コード例 #3
0
ファイル: base_worker.py プロジェクト: Yelp/mycroft
    def run(self):
        """Main entry point for the worker. Queries an SQS queue for messages
        and performs the appropriate action on each message received.
        Swallows all exceptions and logs them.
        """
        queue_name = str(self._get_queue_name())
        sqs = self._get_sqs_wrapper(queue_name, JSONMessage)

        scanner_queue_name = str(self._get_scanner_queue_name())
        scanner_sqs = self._get_sqs_wrapper(scanner_queue_name, JSONMessage)
        dummy_message = {"message": "dummy"}  # TODO: make this message meaningful

        while(not self._stop_requested):  # Loop forever while this variable is set.
            try:  # Main try-except
                for msg in sqs.get_messages_from_queue():
                    msg_body = msg.get_body()
                    log({
                        "status": "new message",
                        "queue": queue_name,
                        "msg": msg_body,
                    })

                    results = None
                    final_status = JOBS_ETL_STATUS_ERROR
                    lsd = None
                    try:
                        self._update_scheduled_jobs_on_etl_start(msg_body)
                        # safe to delete message. if worker dies, scanner will resubmit
                        sqs.delete_message_from_queue(msg)

                        try:
                            # Execute etl
                            results, action_dict = self._process_msg(msg)

                            # Parse results
                            final_status, lsd, extra_info = \
                                parse_results(results, msg_body['end_date'])
                            if final_status != JOBS_ETL_STATUS_COMPLETE:
                                if action_dict['delete_requested']:
                                    final_status = JOBS_ETL_STATUS_DELETED
                                elif action_dict['cancel_requested']:
                                    final_status = JOBS_ETL_STATUS_CANCELLED
                                elif action_dict['pause_requested']:
                                    final_status = JOBS_ETL_STATUS_PAUSED

                            log({
                                "status": "processed message OK",
                                "queue": queue_name,
                                "msg": msg_body,
                                "results": results,
                                "job status": final_status,
                                "last OK date": lsd,
                            })
                        except Exception:
                            final_status = JOBS_ETL_STATUS_ERROR
                            log_exception(
                                "Exception in processing msg from queue: " +
                                queue_name + " msg body:" + str(msg_body)
                            )
                        if final_status != JOBS_ETL_STATUS_DELETED:
                            self._update_scheduled_jobs_on_etl_complete(
                                msg_body, final_status, lsd
                            )
                        scanner_sqs.write_message_to_queue(dummy_message)
                        try:
                            self.emailer.mail_result(
                                final_status, msg_body, additional_info=extra_info
                            )
                            log(
                                "Sent emails to:" + str(msg_body['contact_emails'])
                            )
                        except Exception:
                            log_exception(
                                "Exception in sending emails of job:" +
                                str(msg_body)
                            )
                    except Exception:
                        log_exception(
                            "Failed to update scheduled jobs on etl"
                            " start/complete, msg body: " + str(msg_body)
                        )
            except Exception:  # end of main try-except
                log_exception(
                    "Exception in fetching messages from queue:"
                    + queue_name
                )
                # if sqs queue fails, throttle retry
                time.sleep(sqs.get_wait_time())
            if self._run_once:
                break

        self._stop_requested = False
コード例 #4
0
ファイル: base_worker.py プロジェクト: wlstyy/mycroft
    def run(self):
        """Main entry point for the worker. Queries an SQS queue for messages
        and performs the appropriate action on each message received.
        Swallows all exceptions and logs them.
        """
        queue_name = str(self._get_queue_name())
        sqs = self._get_sqs_wrapper(queue_name, JSONMessage)

        scanner_queue_name = str(self._get_scanner_queue_name())
        scanner_sqs = self._get_sqs_wrapper(scanner_queue_name, JSONMessage)
        dummy_message = {
            "message": "dummy"
        }  # TODO: make this message meaningful

        while (not self._stop_requested
               ):  # Loop forever while this variable is set.
            try:  # Main try-except
                for msg in sqs.get_messages_from_queue():
                    msg_body = msg.get_body()
                    log({
                        "status": "new message",
                        "queue": queue_name,
                        "msg": msg_body,
                    })

                    results = None
                    final_status = JOBS_ETL_STATUS_ERROR
                    lsd = None
                    try:
                        self._update_scheduled_jobs_on_etl_start(msg_body)
                        # safe to delete message. if worker dies, scanner will resubmit
                        sqs.delete_message_from_queue(msg)

                        try:
                            # Execute etl
                            results, action_dict = self._process_msg(msg)

                            # Parse results
                            final_status, lsd, extra_info = \
                                parse_results(results, msg_body['end_date'])
                            if final_status != JOBS_ETL_STATUS_COMPLETE:
                                if action_dict['delete_requested']:
                                    final_status = JOBS_ETL_STATUS_DELETED
                                elif action_dict['cancel_requested']:
                                    final_status = JOBS_ETL_STATUS_CANCELLED
                                elif action_dict['pause_requested']:
                                    final_status = JOBS_ETL_STATUS_PAUSED

                            log({
                                "status": "processed message OK",
                                "queue": queue_name,
                                "msg": msg_body,
                                "results": results,
                                "job status": final_status,
                                "last OK date": lsd,
                            })
                        except Exception:
                            final_status = JOBS_ETL_STATUS_ERROR
                            log_exception(
                                "Exception in processing msg from queue: " +
                                queue_name + " msg body:" + str(msg_body))
                        if final_status != JOBS_ETL_STATUS_DELETED:
                            self._update_scheduled_jobs_on_etl_complete(
                                msg_body, final_status, lsd)
                        scanner_sqs.write_message_to_queue(dummy_message)
                        try:
                            self.emailer.mail_result(
                                final_status,
                                msg_body,
                                additional_info=extra_info)
                            log("Sent emails to:" +
                                str(msg_body['contact_emails']))
                        except Exception:
                            log_exception(
                                "Exception in sending emails of job:" +
                                str(msg_body))
                    except Exception:
                        log_exception("Failed to update scheduled jobs on etl"
                                      " start/complete, msg body: " +
                                      str(msg_body))
            except Exception:  # end of main try-except
                log_exception("Exception in fetching messages from queue:" +
                              queue_name)
                # if sqs queue fails, throttle retry
                time.sleep(sqs.get_wait_time())
            if self._run_once:
                break

        self._stop_requested = False