Esempio n. 1
0
    def processRecords(self):
        """
    It consumes all messaged from the MQ (these are failover messages). In case of failure, the messages
    will be inserted to the MQ again.
    """
        result = createConsumer(self.__monitoringType)
        if not result["OK"]:
            gLogger.error("Fail to create Consumer: %s" % result["Message"])
            return S_ERROR("Fail to create Consumer: %s" % result["Message"])
        else:
            mqConsumer = result["Value"]

        result = S_OK()
        while result["OK"]:
            # we consume all messages from the consumer internal queue.
            result = mqConsumer.get()
            mqConsumer.close()  # make sure that we will not proccess any more messages.
            if result["OK"]:
                records = json.loads(result["Value"])
                retVal = monitoringDB.put(list(records), self.__monitoringType)
                if not retVal["OK"]:
                    # the db is not available and we publish again the data to MQ
                    res = self.publishRecords(records)
                    if not res["OK"]:
                        return res

        return S_OK()
Esempio n. 2
0
    def processRecords(self):
        """
    It consumes all messaged from the MQ (these are failover messages). In case of failure, the messages
    will be inserted to the MQ again.
    """
        result = createConsumer(self.__monitoringType)
        if not result['OK']:
            gLogger.error("Fail to create Consumer: %s" % result['Message'])
            return S_ERROR("Fail to create Consumer: %s" % result['Message'])
        else:
            mqConsumer = result['Value']

        result = S_OK()
        while result['OK']:
            # we consume all messages from the consumer internal queue.
            result = mqConsumer.get()
            mqConsumer.close(
            )  # make sure that we will not proccess any more messages.
            if result['OK']:
                records = json.loads(result['Value'])
                retVal = monitoringDB.put(list(records), self.__monitoringType)
                if not retVal['OK']:
                    # the db is not available and we publish again the data to MQ
                    res = self.publishRecords(records)
                    if not res['OK']:
                        return res

        return S_OK()
Esempio n. 3
0
    def commit(self):
        """
    It inserts the accumulated data to the db. In case of failure
    it keeps in memory/MQ
    """
        # before we try to insert the data to the db, we process all the data
        # which are already in the queue
        mqProducer = self.__createProducer(
        )  # we are sure that we can connect to MQ
        if mqProducer:
            result = self.processRecords()
            if not result['OK']:
                gLogger.error("Unable to insert data to the db:",
                              result['Message'])

        self.__documentLock.acquire()
        documents = self.__documents
        self.__documents = []
        self.__documentLock.release()
        recordSent = 0
        try:
            while documents:
                recordsToSend = documents[:self.__maxRecordsInABundle]
                retVal = monitoringDB.put(recordsToSend, self.__monitoringType)
                if retVal['OK']:
                    recordSent += len(recordsToSend)
                    del documents[:self.__maxRecordsInABundle]
                    gLogger.info("%d records inserted to the db" %
                                 (recordSent))
                else:
                    if mqProducer:
                        res = self.publishRecords(recordsToSend, mqProducer)
                        # if we managed to publish the records we can delete from the list
                        if res['OK']:
                            recordSent += len(recordsToSend)
                            del documents[:self.__maxRecordsInABundle]
                        else:
                            return res  # in case of MQ problem
                    else:
                        gLogger.warn("Failed to insert the records:",
                                     retVal['Message'])
        except Exception as e:  # pylint: disable=broad-except
            gLogger.exception("Error committing", lException=e)
            return S_ERROR("Error committing %s" % repr(e).replace(',)', ')'))
        finally:
            if mqProducer:
                mqProducer.close()
            self.__documents.extend(documents)

        return S_OK(recordSent)
Esempio n. 4
0
  def commit(self):
    """
    It inserts the accumulated data to the db. In case of failure
    it keeps in memory/MQ
    """
    # before we try to insert the data to the db, we process all the data
    # which are already in the queue
    mqProducer = self.__createProducer()  # we are sure that we can connect to MQ
    if mqProducer is not None:
      result = self.processRecords()
      if not result['OK']:
        gLogger.error("Unable to insert data to the db:", result['Message'])

    self.__documentLock.acquire()
    documents = self.__documents
    self.__documents = []
    self.__documentLock.release()
    recordSent = 0
    try:
      while documents:
        recordsToSend = documents[:self.__maxRecordsInABundle]
        retVal = monitoringDB.put(recordsToSend, self.__monitoringType)
        if retVal['OK']:
          recordSent += len(recordsToSend)
          del documents[:self.__maxRecordsInABundle]
          gLogger.info("%d records inserted to the db" % (recordSent))
        else:
          if mqProducer is not None:
            res = self.publishRecords(recordsToSend, mqProducer)
            # if we managed to publish the records we can delete from the list
            if res['OK']:
              recordSent += len(recordsToSend)
              del documents[:self.__maxRecordsInABundle]
            else:
              return res  # in case of MQ problem
          else:
            gLogger.warn("Failed to insert the records:", retVal['Message'])
    except Exception as e:  # pylint: disable=broad-except
      gLogger.exception("Error committing", lException=e)
      return S_ERROR("Error committing %s" % repr(e).replace(',)', ')'))
    finally:
      if mqProducer is not None:
        mqProducer.close()
        mqProducer = None
      self.__documents.extend(documents)
    return S_OK(recordSent)
Esempio n. 5
0
    def commit(self):
        """
    It inserts the accumulated data to the db. In case of failure
    it keeps in memory/MQ
    """
        self.__documentLock.acquire()
        documents = self.__documents
        self.__documents = []
        self.__documentLock.release()
        recordSent = 0
        try:
            while documents:
                recordsToSend = documents[: self.__maxRecordsInABundle]
                retVal = monitoringDB.put(recordsToSend, self.__monitoringType)
                if retVal["OK"]:
                    recordSent += len(recordsToSend)
                    del documents[: self.__maxRecordsInABundle]
                    gLogger.info("%d records inserted to the db" % (recordSent))
                else:
                    if self.__mq:
                        res = self.publishRecords(recordsToSend)
                        # if we managed to publish the records we can delete from the list
                        if res["OK"]:
                            recordSent += len(recordsToSend)
                            del documents[: self.__maxRecordsInABundle]
                        else:
                            return res  # in case of MQ problem
                    else:
                        gLogger.warn("Failed to insert the records: %s", retVal["Message"])
        except Exception as e:  # pylint: disable=broad-except
            gLogger.exception("Error committing", lException=e)
            return S_ERROR("Error committing %s" % repr(e).replace(",)", ")"))
        finally:
            self.__documents.extend(documents)

        if self.__mq:
            result = self.processRecords()
            if not result["OK"]:
                gLogger.error("Unable to insert data from the MQ", result["Message"])

        return S_OK(recordSent)
Esempio n. 6
0
    def processRecords(self):
        """
    It consumes all messaged from the MQ (these are failover messages). In case of failure, the messages
    will be inserted to the MQ again.
    """
        retVal = monitoringDB.pingDB(
        )  # if the db is not accessible, the records will be not processed from MQ
        if retVal['OK']:
            if not retVal['Value']:  # false if we can not connect to the db
                return retVal
        else:
            return retVal

        result = createConsumer("Monitoring::Queue::%s" %
                                self.__monitoringType)
        if not result['OK']:
            gLogger.error("Fail to create Consumer: %s" % result['Message'])
            return S_ERROR("Fail to create Consumer: %s" % result['Message'])
        else:
            mqConsumer = result['Value']

        result = S_OK()
        failedToProcess = []
        while result['OK']:
            # we consume all messages from the consumer internal queue.
            result = mqConsumer.get()
            if result['OK']:
                records = json.loads(result['Value'])
                retVal = monitoringDB.put(list(records), self.__monitoringType)
                if not retVal['OK']:
                    failedToProcess.append(records)

        mqConsumer.close(
        )  # make sure that we will not process any more messages.
        # the db is not available and we publish again the data to MQ
        for records in failedToProcess:
            res = self.publishRecords(records)
            if not res['OK']:
                return res
        return S_OK()
Esempio n. 7
0
  def processRecords(self):
    """
    It consumes all messaged from the MQ (these are failover messages). In case of failure, the messages
    will be inserted to the MQ again.
    """
    retVal = monitoringDB.pingDB()  # if the db is not accessible, the records will be not processed from MQ
    if retVal['OK']:
      if not retVal['Value']:  # false if we can not connect to the db
        return retVal
    else:
      return retVal

    result = createConsumer("Monitoring::Queue::%s" % self.__failoverQueueName)
    if not result['OK']:
      gLogger.error("Fail to create Consumer: %s" % result['Message'])
      return S_ERROR("Fail to create Consumer: %s" % result['Message'])
    else:
      mqConsumer = result['Value']

    result = S_OK()
    failedToProcess = []
    while result['OK']:
      # we consume all messages from the consumer internal queue.
      result = mqConsumer.get()
      if result['OK']:
        records = json.loads(result['Value'])
        retVal = monitoringDB.put(list(records), self.__monitoringType)
        if not retVal['OK']:
          failedToProcess.append(records)

    mqConsumer.close()  # make sure that we will not process any more messages.
    # the db is not available and we publish again the data to MQ
    for records in failedToProcess:
      res = self.publishRecords(records)
      if not res['OK']:
        return res
    return S_OK()